General boolean inference improvements

lib/core/common.py

@@ -1582,10 +1582,10 @@ def setPaths(rootPath):
     paths.SQLMAP_XML_PAYLOADS_PATH = os.path.join(paths.SQLMAP_XML_PATH, "payloads")
 
     # sqlmap files
+    paths.CATALOG_IDENTIFIERS = os.path.join(paths.SQLMAP_TXT_PATH, "catalog-identifiers.txt")
     paths.COMMON_COLUMNS = os.path.join(paths.SQLMAP_TXT_PATH, "common-columns.txt")
     paths.COMMON_FILES = os.path.join(paths.SQLMAP_TXT_PATH, "common-files.txt")
     paths.COMMON_TABLES = os.path.join(paths.SQLMAP_TXT_PATH, "common-tables.txt")
-    paths.COMMON_OUTPUTS = os.path.join(paths.SQLMAP_TXT_PATH, 'common-outputs.txt')
     paths.DIGEST_FILE = os.path.join(paths.SQLMAP_TXT_PATH, "sha256sums.txt")
     paths.SQL_KEYWORDS = os.path.join(paths.SQLMAP_TXT_PATH, "keywords.txt")
     paths.SMALL_DICT = os.path.join(paths.SQLMAP_TXT_PATH, "smalldict.txt")
@@ -2605,42 +2605,23 @@ def calculateDeltaSeconds(start):
 
 def initCommonOutputs():
     """
-    Initializes dictionary containing common output values used by "good samaritan" feature
+    Initializes the per-context dictionary of common identifier names used by the
+    predictive-inference feature to shortcut blind table/column name enumeration.
+    Sourced directly from the curated '--common-tables'/'--common-columns' wordlists
+    (real-world, app-focused names); prediction only ever reorders the charset or
+    confirms a whole value, so it never penalizes a miss.
 
-    >>> initCommonOutputs(); "information_schema" in kb.commonOutputs["Databases"]
+    >>> initCommonOutputs(); "users" in kb.commonOutputs["Tables"]
     True
     """
 
     kb.commonOutputs = {}
-    key = None
 
-    with openFile(paths.COMMON_OUTPUTS, 'r') as f:
-        for line in f:
-            if line.find('#') != -1:
-                line = line[:line.find('#')]
-
-            line = line.strip()
-
-            if len(line) > 1:
-                if line.startswith('[') and line.endswith(']'):
-                    key = line[1:-1]
-                elif key:
-                    if key not in kb.commonOutputs:
-                        kb.commonOutputs[key] = set()
-
-                    if line not in kb.commonOutputs[key]:
-                        kb.commonOutputs[key].add(line)
-
-    # The curated '--common-tables'/'--common-columns' brute-force wordlists are far larger and much
-    # more app-focused than the built-in [Tables]/[Columns] prediction sections (which are mostly
-    # system objects), so fold them into the good-samaritan prediction to raise its real-world hit rate.
-    # The mechanism only reorders the charset, so extra coverage never penalizes a miss.
-    for _key, _path in (("Tables", paths.COMMON_TABLES), ("Columns", paths.COMMON_COLUMNS)):
+    for key, path in (("Tables", paths.COMMON_TABLES), ("Columns", paths.COMMON_COLUMNS)):
         try:
-            for _ in getFileItems(_path):
-                kb.commonOutputs.setdefault(_key, set()).add(_)
+            kb.commonOutputs[key] = set(getFileItems(path))
         except SqlmapSystemException:
-            pass
+            kb.commonOutputs[key] = set()
 
 def getFileItems(filename, commentPrefix='#', unicoded=True, lowercase=False, unique=False):
     """
@@ -2684,19 +2665,17 @@ def getFileItems(filename, commentPrefix='#', unicoded=True, lowercase=False, un
 
     return retVal if not unique else list(retVal.keys())
 
-def goGoodSamaritan(prevValue, originalCharset):
+def predictValue(prevValue, originalCharset):
     """
-    Function for retrieving parameters needed for common prediction (good
-    samaritan) feature.
+    Predictive-inference helper: given the value retrieved so far (prefix), consult the
+    per-context common-identifier set (kb.commonOutputs[kb.partRun], from the common-
+    tables/common-columns wordlists) to shortcut blind extraction.
 
     prevValue: retrieved query output so far (e.g. 'i').
 
-    Returns commonValue if there is a complete single match (in kb.partRun
-    of txt/common-outputs.txt under kb.partRun) regarding parameter
-    prevValue. If there is no single value match, but multiple, commonCharset is
-    returned containing more probable characters (retrieved from matched
-    values in txt/common-outputs.txt) together with the rest of charset as
-    otherCharset.
+    Returns commonValue when a single wordlist entry matches the prefix (the whole value
+    can be confirmed in one request); otherwise commonCharset holds the more probable
+    next characters (reordered ahead of otherCharset) so the bisection converges faster.
     """
 
     if kb.commonOutputs is None:
@@ -2755,7 +2734,7 @@ def goGoodSamaritan(prevValue, originalCharset):
 def getPartRun(alias=True):
     """
     Goes through call stack and finds constructs matching
-    conf.dbmsHandler.*. Returns it or its alias used in 'txt/common-outputs.txt'
+    conf.dbmsHandler.*. Returns it or its predictive-inference context alias (e.g. 'Tables'/'Columns')
     """
 
     retVal = None
@@ -3692,8 +3671,8 @@ def setOptimize():
     Sets options turned on by switch '-o'
     """
 
-    # conf.predictOutput = True
-    # Note: persistent (Keep-Alive) connections are now used by default (see _setHTTPHandlers)
+    # Note: persistent (Keep-Alive) connections are now used by default (see _setHTTPHandlers); predictive
+    # inference is now an inherent, always-on part of blind name enumeration (no longer a switch)
     conf.threads = 3 if conf.threads < 3 and cmdLineOptions.threads is None else conf.threads
     conf.nullConnection = not any((conf.data, conf.textOnly, conf.titles, conf.string, conf.notString, conf.regexp, conf.tor))
 

lib/core/option.py

@@ -2243,6 +2243,11 @@ def _setKnowledgeBaseAttributes(flushAll=True):
     kb.disableHuffman = False
     kb.huffmanProbes = 0
     kb.huffmanEscapes = 0
+    kb.lowCardCache = {}
+    kb.dumpCharset = {}
+    kb.dumpCharsetStable = {}
+    kb.litmusCounter = 0
+    kb.reliabilityAlarm = False
     kb.httpErrorCodes = {}
     kb.inferenceMode = False
     kb.ignoreCasted = None
@@ -2256,7 +2261,7 @@ def _setKnowledgeBaseAttributes(flushAll=True):
     kb.lastParserStatus = None
 
     kb.locks = AttribDict()
-    for _ in ("cache", "connError", "count", "handlers", "hint", "identYwaf", "index", "io", "limit", "liveCookies", "log", "socket", "redirect", "request", "value"):
+    for _ in ("cache", "connError", "count", "handlers", "hint", "identYwaf", "index", "io", "limit", "liveCookies", "log", "prediction", "socket", "redirect", "request", "value"):
         kb.locks[_] = threading.Lock()
 
     kb.matchRatio = None
@@ -2914,10 +2919,6 @@ def _basicOptionValidation():
         errMsg = "switch '--dump' is incompatible with switch '--dump-all'"
         raise SqlmapSyntaxException(errMsg)
 
-    if conf.predictOutput and (conf.threads > 1 or conf.optimize):
-        errMsg = "switch '--predict-output' is incompatible with option '--threads' and switch '-o'"
-        raise SqlmapSyntaxException(errMsg)
-
     if conf.threads > MAX_NUMBER_OF_THREADS and not conf.get("skipThreadCheck"):
         errMsg = "maximum number of used threads is %d avoiding potential connection issues" % MAX_NUMBER_OF_THREADS
         raise SqlmapSyntaxException(errMsg)

lib/core/optiondict.py

@@ -79,7 +79,6 @@ optDict = {
 
     "Optimization": {
         "optimize": "boolean",
-        "predictOutput": "boolean",
         "keepAlive": "boolean",
         "noKeepAlive": "boolean",
         "nullConnection": "boolean",

lib/core/settings.py

@@ -20,7 +20,7 @@ from lib.core.enums import OS
 from thirdparty import six
 
 # sqlmap version (<major>.<minor>.<month>.<monthly commit>)
-VERSION = "1.10.7.30"
+VERSION = "1.10.7.31"
 TYPE = "dev" if VERSION.count('.') > 2 and VERSION.split('.')[-1] != '0' else "stable"
 TYPE_COLORS = {"dev": 33, "stable": 90, "pip": 34}
 VERSION_STRING = "sqlmap/%s#%s" % ('.'.join(VERSION.split('.')[:-1]) if VERSION.count('.') > 2 and VERSION.split('.')[-1] == '0' else VERSION, TYPE)
@@ -560,11 +560,52 @@ for _weight, _chars in ((6, " etaoinsrhldcumfgypwbvkxjqz"), (4, "0123456789"), (
     for _char in _chars:
         HUFFMAN_PRIOR_WEIGHTS[ord(_char)] = _weight
 
-# Bounds for feeding extracted values back into the "good samaritan" (--predict-output) common-output
-# pool for their enumeration context, so later same-context items that share structure (e.g.
-# wp_posts / wp_users / wp_options ...) are predicted faster. MAX_LENGTH keeps large data cells from
-# bloating/polluting the pool (identifiers are short); MAX_ITEMS bounds per-context growth so a huge
-# enumeration cannot make the per-character prediction scan costly. Misses always fall back to bisection.
+# Enumeration contexts (kb.partRun) for which predictive inference is active by default: the identifier
+# names retrieved here are drawn from a known, skewed distribution captured by the common-tables/
+# common-columns wordlists, so whole-value prediction / charset reordering pays off. Deliberately NOT
+# applied to arbitrary dumped data (unknown distribution) or one-shot values (banner, current-user).
+NAME_PREDICTION_CONTEXTS = ("Tables", "Columns")
+
+# Order of the character-level Markov model used to seed the Huffman set-membership tree during blind
+# name enumeration: warmed from the shipped identifier corpus so it predicts a name from the first
+# character (identifiers are short, structured and low-entropy). CATALOG_IDENTIFIERS_PRIOR_PEAK is the
+# weight the corpus prior is scaled to (higher -> the predicted next character sits nearer the tree
+# root -> closer to one request per character). Data dumps keep the classic order-0 adaptive model.
+NAME_MARKOV_ORDER = 3
+CATALOG_IDENTIFIERS_PRIOR_PEAK = 20
+
+# Maximum number of distinct values a dumped column may show before it is treated as high-cardinality
+# and whole-value guessing is abandoned for it. At or below this, each new cell is first confirmed by
+# equality against the values already seen for that column (one request on a hit) before per-character
+# extraction. Self-verifying, so it never returns a wrong value; the bound keeps misses cheap.
+LOW_CARDINALITY_THRESHOLD = 32
+
+# Oracle-reliability litmus: during bulk blind extraction (dumps / name enumeration) a known-answer
+# differential is fired every this-many extracted values - one probe that MUST be TRUE (the value we just
+# read equals itself) and one that MUST be FALSE (it equals a deliberately corrupted copy). A healthy
+# oracle always answers T/F; an always-true channel (WAF/200-for-everything, reads-everything-true) or a
+# flaky/degraded one (timing jitter, lease near end-of-life) trips it - converting SILENT data corruption
+# into a one-time "results may be unreliable" warning. The first value is always checked (catch it before
+# a whole garbage dump), then every Nth. Cheap and amortized; set to 0 to disable.
+ORACLE_LITMUS_CHECK_EVERY = 25
+
+# Whole-value guessing only starts once some value has repeated (proof the column is low-cardinality), so
+# an all-unique column - primary key, hash, free text - never wastes a probe. Once armed, at most this
+# many candidates (most-frequent first) are tried per cell, so even a column that trips the threshold with
+# many near-unique values can only ever waste a small, bounded number of probes before falling back.
+LOW_CARDINALITY_MAX_GUESSES = 8
+
+# Number of consecutive dumped rows a column's observed character set must stay unchanged before it is
+# trusted as closed and used to restrict the time-based bisection alphabet. A column whose alphabet keeps
+# growing (e.g. a monotonic primary key or high-entropy text) never reaches this, so it is never charged
+# the speculative restricted-search-then-escalate cost.
+DUMP_CHARSET_STABLE_ROWS = 3
+
+# Bounds for feeding extracted values back into the predictive-inference pool for their enumeration
+# context, so later same-context items that share structure (e.g. wp_posts / wp_users / wp_options ...)
+# are predicted faster. MAX_LENGTH keeps large data cells from bloating/polluting the pool (identifiers
+# are short); MAX_ITEMS bounds per-context growth so a huge enumeration cannot make the per-character
+# prediction scan costly. Only fed single-threaded (never mutated under value-parallel enumeration).
 PREDICTION_FEEDBACK_MAX_LENGTH = 128
 PREDICTION_FEEDBACK_MAX_ITEMS = 10000
 

lib/core/testing.py

@@ -78,7 +78,7 @@ def vulnTest(tests=None, label="vuln"):
         ("-u <base> --flush-session -H \"Foo: Bar\" -H \"Sna: Fu\" --data=\"<root><param name=\\\"id\\\" value=\\\"1*\\\"/></root>\" --union-char=1 --mobile --answers=\"smartphone=3\" --banner --smart -v 5", ("might be injectable", "Payload: <root><param name=\"id\" value=\"1", "Type: boolean-based blind", "Type: time-based blind", "Type: UNION query", "banner: '3.", "Nexus", "Sna: Fu", "Foo: Bar")),
         ("-u <base> --flush-session --technique=BU --method=PUT --data=\"a=1;id=1;b=2\" --param-del=\";\" --skip-static --har=<tmpfile> --dump -T users --start=1 --stop=2", ("might be injectable", "Parameter: id (PUT)", "Type: boolean-based blind", "Type: UNION query", "2 entries")),
         ("-u <url> --flush-session -H \"id: 1*\" --tables -t <tmpfile>", ("might be injectable", "Parameter: id #1* ((custom) HEADER)", "Type: boolean-based blind", "Type: time-based blind", "Type: UNION query", " users ")),
-        ("-u <url> --flush-session --banner --invalid-logical --technique=B --predict-output --titles --test-filter=\"OR boolean\" --tamper=space2dash", ("banner: '3.", " LIKE ")),
+        ("-u <url> --flush-session --banner --invalid-logical --technique=B --titles --test-filter=\"OR boolean\" --tamper=space2dash", ("banner: '3.", " LIKE ")),
         ("-u <url> --flush-session --cookie=\"PHPSESSID=d41d8cd98f00b204e9800998ecf8427e; id=1*; id2=2\" --tables --union-cols=3", ("might be injectable", "Cookie #1* ((custom) HEADER)", "Type: boolean-based blind", "Type: time-based blind", "Type: UNION query", " users ")),
         ("-u <url> --flush-session --null-connection --technique=B --tamper=between,randomcase --banner --count -T users", ("NULL connection is supported with HEAD method", "banner: '3.", "users | 30")),
         ("-u <base> --data=\"aWQ9MQ==\" --flush-session --base64=POST -v 6", ("aWQ9MTtXQUlURk9SIERFTEFZICcwOjA",)),