diff --git a/data/txt/sha256sums.txt b/data/txt/sha256sums.txt index 691b33f21..b72a8e908 100644 --- a/data/txt/sha256sums.txt +++ b/data/txt/sha256sums.txt @@ -189,7 +189,7 @@ ccc4a717e887652b1fcce073d9409d9c59a3b28548c703a9e453d15845f90cd7 lib/core/patch 48797d6c34dd9bb8a53f7f3794c85f4288d82a9a1d6be7fcf317d388cb20d4b3 lib/core/replication.py 0b8c38a01bb01f843d94a6c5f2075ee47520d0c4aa799cecea9c3e2c5a4a23a6 lib/core/revision.py 888daba83fd4a34e9503fe21f01fef4cc730e5cde871b1d40e15d4cbc847d56c lib/core/session.py -f75f15165173becddf439996a85f011262178e1bf5d2d2bf8028455b7ff3ff94 lib/core/settings.py +f79f96c5f073b663cc494c57b9641dc41e7ed13a28d5cec62bb9ca8904110d9c lib/core/settings.py cd5a66deee8963ba8e7e9af3dd36eb5e8127d4d68698811c29e789655f507f82 lib/core/shell.py bcb5d8090d5e3e0ef2a586ba09ba80eef0c6d51feb0f611ed25299fbb254f725 lib/core/subprocessng.py 70ea3768f1b3062b22d20644df41c86238157ec80dd43da40545c620714273c6 lib/core/target.py @@ -241,7 +241,7 @@ f522436fbd14bdab090a1d305fcac0361800cb8e36c8cbcb47933298376a71e0 lib/takeover/r 1966ca704961fb987ab757f0a4afddbf841d1a880631b701487c75cef63d60c3 lib/techniques/__init__.py 1966ca704961fb987ab757f0a4afddbf841d1a880631b701487c75cef63d60c3 lib/techniques/union/__init__.py ceec65f8cb7c3254c4671351c837418c76ac5bc55ccbc40779f67231b54d7085 lib/techniques/union/test.py -3f834b877f0fb684e402d07af1d8a7c7d0cdb4c0a3f9f15fe8488a08d88db4f2 lib/techniques/union/use.py +c65766f71e285fc85cdf58e7448c4c1d015af2a9dbb44fa3b665a9f13362fbcc lib/techniques/union/use.py aeefb42ea0c68f72744bc1bfd7194ec1bc06480d8a7e23f4b8d3d23fbba2b014 lib/utils/api.py 442555ab85277aff7c9e0cf465ea5b0d28395c326f68363449b2d3941f4b6de2 lib/utils/brute.py da5bcbcda3f667582adf5db8c1b5d511b469ac61b55d387cec66de35720ed718 lib/utils/crawler.py diff --git a/lib/core/settings.py b/lib/core/settings.py index 3b7f93206..451a0bc62 100644 --- a/lib/core/settings.py +++ b/lib/core/settings.py @@ -20,7 +20,7 @@ from lib.core.enums import OS from thirdparty import six # sqlmap version (...) -VERSION = "1.10.6.131" +VERSION = "1.10.6.132" TYPE = "dev" if VERSION.count('.') > 2 and VERSION.split('.')[-1] != '0' else "stable" TYPE_COLORS = {"dev": 33, "stable": 90, "pip": 34} VERSION_STRING = "sqlmap/%s#%s" % ('.'.join(VERSION.split('.')[:-1]) if VERSION.count('.') > 2 and VERSION.split('.')[-1] == '0' else VERSION, TYPE) diff --git a/lib/techniques/union/use.py b/lib/techniques/union/use.py index 418cd34ee..dc8517096 100644 --- a/lib/techniques/union/use.py +++ b/lib/techniques/union/use.py @@ -255,15 +255,37 @@ def _chunkedJsonAggUse(expression, expressionFields, expressionFieldsList, count caps. K is halved adaptively if a chunk response still gets truncated. Returns a BigArray of rows, or None to let the caller fall back to the regular per-row UNION path. - NOTE: MySQL only for now (windowed 'LIMIT offset,K' + JSON_ARRAYAGG); other DBMSes return None. + Same DBMS coverage as the single-shot JSON-agg (per-DBMS aggregate + windowing); others -> None. """ - if not Backend.isDbms(DBMS.MYSQL) or not expressionFields or not expressionFieldsList: + dbms = Backend.getIdentifiedDbms() + + if dbms not in (DBMS.MYSQL, DBMS.PGSQL, DBMS.SQLITE, DBMS.H2, DBMS.HSQLDB, DBMS.FIREBIRD) or not expressionFields or not expressionFieldsList: return None + start, stop, delimiter = kb.chars.start, kb.chars.stop, kb.chars.delimiter + # a stable total ordering (all output columns) so the LIMIT/OFFSET windows never overlap or drop rows base = re.sub(r"(?i)\s+ORDER BY\s+.+\Z", "", expression) orderBy = "ORDER BY %s" % ','.join(str(_ + 1) for _ in range(len(expressionFieldsList))) - aggFields = "CONCAT_WS('%s',%s)" % (kb.chars.delimiter, ','.join(agent.nullAndCastField(_) for _ in expressionFieldsList)) + nulled = [agent.nullAndCastField(_) for _ in expressionFieldsList] + + # per-DBMS: aggregate-over-windowed-columns expression (mirrors the single-shot branches) plus + # the "K rows at offset" window clause appended to the inner derived table + if dbms == DBMS.MYSQL: + aggExpr = "CONCAT('%s',JSON_ARRAYAGG(CONCAT_WS('%s',%s)),'%s')" % (start, delimiter, ','.join(nulled), stop) + window = lambda o, k: "%s LIMIT %d,%d" % (orderBy, o, k) + elif dbms == DBMS.PGSQL: + aggExpr = "STRING_AGG('%s'||%s||'%s','')" % (start, ("||'%s'||" % delimiter).join("COALESCE(%s::text,' ')" % _ for _ in expressionFieldsList), stop) + window = lambda o, k: "%s LIMIT %d OFFSET %d" % (orderBy, k, o) + elif dbms == DBMS.SQLITE: + aggExpr = "'%s'||JSON_GROUP_ARRAY(%s)||'%s'" % (start, ("||'%s'||" % delimiter).join("COALESCE(%s,' ')" % _ for _ in expressionFieldsList), stop) + window = lambda o, k: "%s LIMIT %d OFFSET %d" % (orderBy, k, o) + elif dbms in (DBMS.H2, DBMS.HSQLDB): + aggExpr = "GROUP_CONCAT('%s'||%s||'%s' SEPARATOR '')" % (start, ("||'%s'||" % delimiter).join(nulled), stop) + window = lambda o, k: "%s LIMIT %d OFFSET %d" % (orderBy, k, o) + elif dbms == DBMS.FIREBIRD: + aggExpr = "LIST('%s'||%s||'%s','')" % (start, ("||'%s'||" % delimiter).join(nulled), stop) + window = lambda o, k: "%s ROWS %d TO %d" % (orderBy, o + 1, o + k) debugMsg = "single-shot UNION dump output was too large; switching to " debugMsg += "chunked (windowed) JSON aggregation of %d entries" % count @@ -274,8 +296,7 @@ def _chunkedJsonAggUse(expression, expressionFields, expressionFieldsList, count offset = 0 while offset < count: - inner = "%s %s LIMIT %d,%d" % (base, orderBy, offset, chunk) - query = "SELECT CONCAT('%s',JSON_ARRAYAGG(%s),'%s') FROM (%s) AS sqmapx" % (kb.chars.start, aggFields, kb.chars.stop, inner) + query = "SELECT %s FROM (%s %s) sqmapx" % (aggExpr, base, window(offset, chunk)) kb.jsonAggMode = True output = _oneShotUnionUse(query, False) @@ -348,6 +369,18 @@ def unionUse(expression, unpack=True, dump=False): value = parseUnionPage(output) kb.jsonAggMode = False + # If the single-shot aggregate failed (typically too large for the DBMS packet limit / + # response cap) and the table is large, retrieve the rows in bounded windows (chunked + # JSON aggregation) before the slow per-row fallback. Done here (independent of the + # detected UNION where-clause) so it engages for any dumpable FROM-table query. + if value is None and " FROM " in expression.upper() and not re.search(SQL_SCALAR_REGEX, expression, re.I) and not any((kb.forcePartialUnion, conf.forcePartial, conf.disableJson, conf.binaryFields, conf.limitStart, conf.limitStop)): + chunkCountExpr = expression.replace(expressionFields, queries[Backend.getIdentifiedDbms()].count.query % '*', 1) + if " ORDER BY " in chunkCountExpr.upper(): + chunkCountExpr = chunkCountExpr[:chunkCountExpr.upper().rindex(" ORDER BY ")] + chunkCount = unArrayizeValue(parseUnionPage(_oneShotUnionUse(chunkCountExpr, unpack))) + if isNumPosStrValue(chunkCount) and (int(chunkCount) >= JSON_AGG_CHUNK_ROWS or kb.respTruncated): + value = _chunkedJsonAggUse(expression, expressionFields, expressionFieldsList, int(chunkCount)) + # We have to check if the SQL query might return multiple entries # if the technique is partial UNION query and in such case forge the # SQL limiting the query output one entry at a time @@ -398,14 +431,6 @@ def unionUse(expression, unpack=True, dump=False): return value if isNumPosStrValue(count) and int(count) > 1: - # The single-shot full UNION dump failed and the table is large (or its oversized - # response was detected as truncated): retrieve the rows in bounded windows via - # chunked JSON aggregation (K rows/request) instead of the slow per-row path below. - if Backend.isDbms(DBMS.MYSQL) and not any((kb.forcePartialUnion, conf.forcePartial, conf.disableJson, conf.binaryFields, conf.limitStart, conf.limitStop)) and (int(count) >= JSON_AGG_CHUNK_ROWS or kb.respTruncated): - chunked = _chunkedJsonAggUse(expression, expressionFields, expressionFieldsList, int(count)) - if chunked is not None: - return chunked - threadData = getCurrentThreadData() try: