diff --git a/data/xml/queries.xml b/data/xml/queries.xml index 36a2c627c..dfaabd38c 100644 --- a/data/xml/queries.xml +++ b/data/xml/queries.xml @@ -83,7 +83,7 @@ - + @@ -119,8 +119,8 @@ - - + + @@ -163,7 +163,7 @@ - + @@ -384,7 +384,7 @@ - + @@ -959,8 +959,8 @@ - - + + @@ -1158,7 +1158,7 @@ - + /> diff --git a/lib/core/agent.py b/lib/core/agent.py index 5db8f450a..520fc9914 100644 --- a/lib/core/agent.py +++ b/lib/core/agent.py @@ -508,10 +508,26 @@ class Agent(object): if field and Backend.getIdentifiedDbms(): rootQuery = queries[Backend.getIdentifiedDbms()] + kb.binaryField = conf.binaryFields and field in conf.binaryFields + hexConvert = conf.hexConvert or kb.binaryField + + # For BINARY fields, wrap the RAW column with the hex function rather than the text-casted one: + # text-casting binary first (e.g. CAST( AS NCHAR) on MySQL) NULLs non-text bytes, so HEX() + # would encode the NULL placeholder and silently lose the value (e.g. binary-stored password + # hashes). This is limited to binary fields: the blanket '--hex' keeps the cast-first path because + # raw-hexing a numeric/temporal column uses the DBMS's numeric HEX semantics (MySQL HEX(255)='FF'), + # which is NOT the hex of its string representation. Every DBMS hex function takes binary directly + # EXCEPT PostgreSQL's CONVERT_TO() (needs text), so PostgreSQL stays on the cast-first path too. + hexRaw = kb.binaryField and not Backend.isDbms(DBMS.PGSQL) + if field.startswith("(CASE") or field.startswith("(IIF") or conf.noCast and not (field.startswith("COUNT(") and Backend.getIdentifiedDbms() == DBMS.MSSQL): nulledCastedField = field + if hexConvert: + nulledCastedField = self.hexConvertField(nulledCastedField) else: - if not (Backend.isDbms(DBMS.SQLITE) and not isDBMSVersionAtLeast('3')): + if hexRaw: + nulledCastedField = self.hexConvertField(field) + elif not (Backend.isDbms(DBMS.SQLITE) and not isDBMSVersionAtLeast('3')): nulledCastedField = rootQuery.cast.query % field if re.search(r"COUNT\(", field) and Backend.getIdentifiedDbms() in (DBMS.RAIMA,): @@ -521,9 +537,8 @@ class Agent(object): else: nulledCastedField = rootQuery.isnull.query % nulledCastedField - kb.binaryField = conf.binaryFields and field in conf.binaryFields - if conf.hexConvert or kb.binaryField: - nulledCastedField = self.hexConvertField(nulledCastedField) + if hexConvert and not hexRaw: + nulledCastedField = self.hexConvertField(nulledCastedField) if suffix: nulledCastedField += suffix diff --git a/lib/core/settings.py b/lib/core/settings.py index 53eb16b54..f4ca3fd09 100644 --- a/lib/core/settings.py +++ b/lib/core/settings.py @@ -20,7 +20,7 @@ from lib.core.enums import OS from thirdparty import six # sqlmap version (...) -VERSION = "1.10.7.75" +VERSION = "1.10.7.76" TYPE = "dev" if VERSION.count('.') > 2 and VERSION.split('.')[-1] != '0' else "stable" TYPE_COLORS = {"dev": 33, "stable": 90, "pip": 34} VERSION_STRING = "sqlmap/%s#%s" % ('.'.join(VERSION.split('.')[:-1]) if VERSION.count('.') > 2 and VERSION.split('.')[-1] == '0' else VERSION, TYPE) @@ -885,6 +885,11 @@ HASH_RECOGNITION_QUIT_THRESHOLD = 1000 # Regular expression used for automatic hex conversion and hash cracking of (RAW) binary column values HASH_BINARY_COLUMNS_REGEX = r"(?i)pass|psw|hash" +# Regular expression matching (declared) binary column types, used to auto-hex their values during dumping +# so raw bytes (e.g. password hashes stored in binary form) are not silently truncated at NUL / mangled by +# the text extraction channel (mirrors a manual '--binary-fields', using the already-fetched column type) +BINARY_FIELDS_TYPE_REGEX = r"(?i)binary|blob|bytea|image|\braw\b" + # Maximum number of redirections to any single URL - this is needed because of the state that cookies introduce MAX_SINGLE_URL_REDIRECTIONS = 4 diff --git a/plugins/generic/entries.py b/plugins/generic/entries.py index 30ee71027..ca4a3d54a 100644 --- a/plugins/generic/entries.py +++ b/plugins/generic/entries.py @@ -40,6 +40,7 @@ from lib.core.exception import SqlmapConnectionException from lib.core.exception import SqlmapMissingMandatoryOptionException from lib.core.exception import SqlmapNoneDataException from lib.core.exception import SqlmapUnsupportedFeatureException +from lib.core.settings import BINARY_FIELDS_TYPE_REGEX from lib.core.settings import CHECK_ZERO_COLUMNS_THRESHOLD from lib.core.settings import CURRENT_DB from lib.core.settings import KEYSET_MIN_ROWS @@ -114,6 +115,8 @@ class Entries(object): for tbl in tblList: tblList[tblList.index(tbl)] = safeSQLIdentificatorNaming(tbl, True) + binaryFields = conf.binaryFields # user-provided '--binary-fields' (auto-detected ones are added per-table) + for tbl in tblList: if kb.dumpKeyboardInterrupt: break @@ -169,6 +172,18 @@ class Entries(object): colNames = colString = ','.join(column for column in colList) rootQuery = queries[Backend.getIdentifiedDbms()].dump_table + # Auto-treat binary-typed columns (blob/varbinary/bytea/image/raw) as '--binary-fields', so their + # raw bytes (e.g. password hashes stored in binary form) are hex-extracted instead of being + # silently truncated at a NUL / mangled by the text channel (issues #8, #582, #2827). The column + # type is already known from the enumeration above, so this costs no extra request. + # (PostgreSQL excluded: its bytea already renders as readable '\xHEX' through the default text + # cast, and its hex path needs text input, so auto-hexing would double-encode.) + autoBinary = [] if Backend.isDbms(DBMS.PGSQL) else [column for column in colList if columns.get(column) and re.search(BINARY_FIELDS_TYPE_REGEX, getUnicode(columns[column]))] + conf.binaryFields = (list(binaryFields) if binaryFields else []) + [_ for _ in autoBinary if not (binaryFields and _ in binaryFields)] + if autoBinary: + debugMsg = "auto-treating binary column(s) '%s' as binary fields" % ', '.join(unsafeSQLIdentificatorNaming(_) for _ in autoBinary) + logger.debug(debugMsg) + infoMsg = "fetching entries" if conf.col: infoMsg += " of column(s) '%s'" % colNames @@ -572,6 +587,7 @@ class Entries(object): finally: kb.dumpColumns = None kb.dumpTable = None + conf.binaryFields = binaryFields # restore user-provided value (drop this table's auto-detected ones) def dumpAll(self): if conf.db is not None and conf.tbl is None: diff --git a/tests/test_dialect.py b/tests/test_dialect.py index 5e8212763..871f16229 100644 --- a/tests/test_dialect.py +++ b/tests/test_dialect.py @@ -68,9 +68,10 @@ class TestNullAndCastField(unittest.TestCase): CASES = { DBMS.MYSQL: "IFNULL(CAST(col AS NCHAR),' ')", - DBMS.MSSQL: "ISNULL(CAST(col AS NVARCHAR(4000)),' ')", + # MSSQL/PGSQL casts are unbounded (NVARCHAR(MAX)/TEXT) so long values are not silently truncated + DBMS.MSSQL: "ISNULL(CAST(col AS NVARCHAR(MAX)),' ')", DBMS.SYBASE: "ISNULL(CONVERT(VARCHAR(4000),col),' ')", - DBMS.PGSQL: "COALESCE(CAST(col AS VARCHAR(10000))::text,' ')", + DBMS.PGSQL: "COALESCE(CAST(col AS TEXT)::text,' ')", DBMS.ORACLE: "NVL(CAST(col AS VARCHAR(4000)),' ')", }