diff --git a/data/txt/sha256sums.txt b/data/txt/sha256sums.txt index 76a24f6bd..25692070e 100644 --- a/data/txt/sha256sums.txt +++ b/data/txt/sha256sums.txt @@ -170,13 +170,13 @@ d69e84f1648cdb907f5d2dd454f03874a4613752b07867510145d51d84b3c56f lib/controller c51c33501cc905586a9aaac93b06f2ac6f71628d032a7dc39fd0ef05d7ee3856 lib/core/bigarray.py d143df718fbaacb617b6046c73cf4e47932e1a25928a4e1ecb87ea77a3b154ed lib/core/common.py 8f1272487e1adfcc8c755a2f56f0c6d21eac5e685a73a9a159482f9dc9142bc5 lib/core/compat.py -a683d0ad9ba543587382c4903d28db610ae20394fcf9045a68b2ab54a39381ae lib/core/convert.py +5301ba2204404d086e9a67271cde00fc10214c63b018a95fc5aa90ff9e0b2ad9 lib/core/convert.py c03dc585f89642cfd81b087ac2723e3e1bb3bfa8c60e6f5fe58ef3b0113ebfe6 lib/core/data.py d9ec034a6d51ab4ddde0b6aa7ed306f9e0b1336557f77d7939ba547600f9b3ae lib/core/datatype.py f8de57606325456928e46ae2896f5f8bbec9ad18b1c644b492a566fa992216f6 lib/core/decorators.py 147823c37596bd6a56d677697781f34b8d1d1671d5a2518fbc9468d623c6d07d lib/core/defaults.py 8e4f4b5ea37a49d445bb0df83bf04b34f61035ec33fd8acf598ebcf371cb19a7 lib/core/dicts.py -854073f899b876ab13b36e93e174b9cfe51408f7343040197a80afd9fc9c65ee lib/core/dump.py +10d8bb671a64cc787fc2fbf2c641560b7797fccd62c4792e55dffe5efab9f544 lib/core/dump.py 6dd47f52082e98dc0cda6969b277b7d81c6f7c68dac4688821f873a1c65c6edf lib/core/enums.py 5387168e5dfedd94ae22af7bb255f27d6baaca50b24179c6b98f4f325f5cc7b4 lib/core/exception.py 1966ca704961fb987ab757f0a4afddbf841d1a880631b701487c75cef63d60c3 lib/core/__init__.py @@ -189,7 +189,7 @@ f8de57606325456928e46ae2896f5f8bbec9ad18b1c644b492a566fa992216f6 lib/core/decor 9bf174058f15d14e24e94f9aaf42df045119d3617c6c54bd2f3af79b462f331d lib/core/replication.py 0b8c38a01bb01f843d94a6c5f2075ee47520d0c4aa799cecea9c3e2c5a4a23a6 lib/core/revision.py 888daba83fd4a34e9503fe21f01fef4cc730e5cde871b1d40e15d4cbc847d56c lib/core/session.py -4d9cc21e2b2a10fd6c06ce6c9b248fd16a4c266511cd01156bbe7643e5327a89 lib/core/settings.py +516d6b40efa04a5a25b0aa317ea49771f6964a57581777761f82d36d1b1b78b0 lib/core/settings.py c7804223319e18eb0b8e2cbf0a8b6896d1cefb7b0b1a2e9f1cf826a8a3b56750 lib/core/shell.py a2e98a94b231432736d6b304fc75525c8b5fdb4768c418387c5b4c1a610dad64 lib/core/subprocessng.py 19f1e3c5e3ba703d28d510cd7a9ab8284d5fbe9df5ce7e77c86e5931571364b7 lib/core/target.py diff --git a/lib/core/convert.py b/lib/core/convert.py index 6588faf1a..31bbf9b8e 100644 --- a/lib/core/convert.py +++ b/lib/core/convert.py @@ -464,6 +464,9 @@ def stdoutEncode(value): return retVal +# str.isascii() is available on Python 3.7+ only (sqlmap still supports 2.7) +_HAS_ISASCII = hasattr(str, "isascii") + def getConsoleLength(value): """ Returns console width of unicode values @@ -475,7 +478,15 @@ def getConsoleLength(value): """ if isinstance(value, six.text_type): - retVal = len(value) + sum(ord(_) >= 0x3000 for _ in value) + # Fast path: ASCII values have no wide (>= U+3000) characters, so their + # console width is simply their length. str.isascii() (Python 3.7+) is a + # C-level scan, far cheaper than the per-character generator below (which + # stays for the rare wide-character case and for Python 2). This runs + # once per dumped cell, so it dominates large table dumps. + if _HAS_ISASCII and value.isascii(): + retVal = len(value) + else: + retVal = len(value) + sum(ord(_) >= 0x3000 for _ in value) else: retVal = len(value) diff --git a/lib/core/dump.py b/lib/core/dump.py index 37264e93e..c81f52519 100644 --- a/lib/core/dump.py +++ b/lib/core/dump.py @@ -627,11 +627,25 @@ class Dump(object): elif conf.dumpFormat == DUMP_FORMAT.SQLITE: rtable.beginTransaction() + # Precompute the per-column layout once. These values are invariant across + # every row, so resolving them per cell (dict lookup, int() conversion and + # identifier normalization) wasted count x ncols work on large dumps. + dumpColumns = [] + for column in columns: + if column != "__infos__": + info = tableValues[column] + dumpColumns.append((unsafeSQLIdentificatorNaming(column), info["values"], int(info["length"]))) + for i in xrange(count): console = (i >= count - TRIM_STDOUT_DUMP_SIZE) field = 1 - values = [] - record = OrderedDict() + + # Only the SQLITE and JSONL paths accumulate a per-row container; the + # others left these unused, wasting an allocation on every single row + if conf.dumpFormat == DUMP_FORMAT.SQLITE: + values = [] + elif conf.dumpFormat == DUMP_FORMAT.JSONL: + record = OrderedDict() if i == 0 and count > TRIM_STDOUT_DUMP_SIZE: self._write(" ...") @@ -639,62 +653,58 @@ class Dump(object): if conf.dumpFormat == DUMP_FORMAT.HTML: dataToDumpFile(dumpFP, "") - for column in columns: - if column != "__infos__": - info = tableValues[column] + for safeColumn, colValues, maxlength in dumpColumns: + if len(colValues) <= i or colValues[i] is None: + value = u'' + else: + value = getUnicode(colValues[i]) + value = DUMP_REPLACEMENTS.get(value, value) - if len(info["values"]) <= i or info["values"][i] is None: - value = u'' + if conf.dumpFormat == DUMP_FORMAT.SQLITE: + # Note: store a real NULL for the NULL sentinel (and the raw value otherwise), + # mirroring the JSONL path below; appending the display-replaced 'NULL'/'' + # text would corrupt the INTEGER/REAL-typed columns inferred above + if len(colValues) <= i or colValues[i] is None or colValues[i] == " ": # NULL + values.append(None) else: - value = getUnicode(info["values"][i]) - value = DUMP_REPLACEMENTS.get(value, value) + values.append(getUnicode(colValues[i])) - if conf.dumpFormat == DUMP_FORMAT.SQLITE: - # Note: store a real NULL for the NULL sentinel (and the raw value otherwise), - # mirroring the JSONL path below; appending the display-replaced 'NULL'/'' - # text would corrupt the INTEGER/REAL-typed columns inferred above - if len(info["values"]) <= i or info["values"][i] is None or info["values"][i] == " ": # NULL - values.append(None) - else: - values.append(getUnicode(info["values"][i])) + blank = " " * (maxlength - getConsoleLength(value)) + self._write("| %s%s" % (value, blank), newline=False, console=console) - maxlength = int(info["length"]) - blank = " " * (maxlength - getConsoleLength(value)) - self._write("| %s%s" % (value, blank), newline=False, console=console) + if len(value) > MIN_BINARY_DISK_DUMP_SIZE and r'\x' in value: + try: + mimetype = getText(magic.from_buffer(getBytes(value), mime=True)) + if any(mimetype.startswith(_) for _ in ("application", "image")): + if not os.path.isdir(dumpDbPath): + os.makedirs(dumpDbPath) - if len(value) > MIN_BINARY_DISK_DUMP_SIZE and r'\x' in value: - try: - mimetype = getText(magic.from_buffer(getBytes(value), mime=True)) - if any(mimetype.startswith(_) for _ in ("application", "image")): - if not os.path.isdir(dumpDbPath): - os.makedirs(dumpDbPath) + _ = re.sub(r"[^\w]", UNSAFE_DUMP_FILEPATH_REPLACEMENT, normalizeUnicode(safeColumn)) + filepath = os.path.join(dumpDbPath, "%s-%d.bin" % (_, randomInt(8))) + warnMsg = "writing binary ('%s') content to file '%s' " % (mimetype, filepath) + logger.warning(warnMsg) - _ = re.sub(r"[^\w]", UNSAFE_DUMP_FILEPATH_REPLACEMENT, normalizeUnicode(unsafeSQLIdentificatorNaming(column))) - filepath = os.path.join(dumpDbPath, "%s-%d.bin" % (_, randomInt(8))) - warnMsg = "writing binary ('%s') content to file '%s' " % (mimetype, filepath) - logger.warning(warnMsg) + with openFile(filepath, "w+b", None) as f: + _ = safechardecode(value, True) + f.write(_) - with openFile(filepath, "w+b", None) as f: - _ = safechardecode(value, True) - f.write(_) + except Exception as ex: + logger.debug(getSafeExString(ex)) - except Exception as ex: - logger.debug(getSafeExString(ex)) + if conf.dumpFormat == DUMP_FORMAT.CSV: + if field == fields: + dataToDumpFile(dumpFP, "%s" % safeCSValue(value)) + else: + dataToDumpFile(dumpFP, "%s%s" % (safeCSValue(value), conf.csvDel)) + elif conf.dumpFormat == DUMP_FORMAT.HTML: + dataToDumpFile(dumpFP, "%s" % getUnicode(htmlEscape(value).encode("ascii", "xmlcharrefreplace"))) + elif conf.dumpFormat == DUMP_FORMAT.JSONL: + if len(colValues) <= i or colValues[i] is None or colValues[i] == " ": # NULL + record[safeColumn] = None + else: + record[safeColumn] = getUnicode(colValues[i]) - if conf.dumpFormat == DUMP_FORMAT.CSV: - if field == fields: - dataToDumpFile(dumpFP, "%s" % safeCSValue(value)) - else: - dataToDumpFile(dumpFP, "%s%s" % (safeCSValue(value), conf.csvDel)) - elif conf.dumpFormat == DUMP_FORMAT.HTML: - dataToDumpFile(dumpFP, "%s" % getUnicode(htmlEscape(value).encode("ascii", "xmlcharrefreplace"))) - elif conf.dumpFormat == DUMP_FORMAT.JSONL: - if len(info["values"]) <= i or info["values"][i] is None or info["values"][i] == " ": # NULL - record[unsafeSQLIdentificatorNaming(column)] = None - else: - record[unsafeSQLIdentificatorNaming(column)] = getUnicode(info["values"][i]) - - field += 1 + field += 1 if conf.dumpFormat == DUMP_FORMAT.SQLITE: try: diff --git a/lib/core/settings.py b/lib/core/settings.py index 17120f469..88b61ddd4 100644 --- a/lib/core/settings.py +++ b/lib/core/settings.py @@ -20,7 +20,7 @@ from lib.core.enums import OS from thirdparty import six # sqlmap version (...) -VERSION = "1.10.6.201" +VERSION = "1.10.7.0" TYPE = "dev" if VERSION.count('.') > 2 and VERSION.split('.')[-1] != '0' else "stable" TYPE_COLORS = {"dev": 33, "stable": 90, "pip": 34} VERSION_STRING = "sqlmap/%s#%s" % ('.'.join(VERSION.split('.')[:-1]) if VERSION.count('.') > 2 and VERSION.split('.')[-1] == '0' else VERSION, TYPE)