Auto recognize binary fields in blind cases

lib/core/settings.py

@@ -20,7 +20,7 @@ from lib.core.enums import OS
 from thirdparty import six
 
 # sqlmap version (<major>.<minor>.<month>.<monthly commit>)
-VERSION = "1.10.7.77"
+VERSION = "1.10.7.78"
 TYPE = "dev" if VERSION.count('.') > 2 and VERSION.split('.')[-1] != '0' else "stable"
 TYPE_COLORS = {"dev": 33, "stable": 90, "pip": 34}
 VERSION_STRING = "sqlmap/%s#%s" % ('.'.join(VERSION.split('.')[:-1]) if VERSION.count('.') > 2 and VERSION.split('.')[-1] == '0' else VERSION, TYPE)
@@ -890,6 +890,10 @@ HASH_BINARY_COLUMNS_REGEX = r"(?i)pass|psw|hash"
 # the text extraction channel (mirrors a manual '--binary-fields', using the already-fetched column type)
 BINARY_FIELDS_TYPE_REGEX = r"(?i)binary|blob|bytea|image|\braw\b"
 
+# Uppercased keywords of the above, for building an in-SQL "is this column binary-typed?" check when only
+# column names (not types) were fetched - i.e. blind dumping (keep in sync with BINARY_FIELDS_TYPE_REGEX)
+BINARY_FIELDS_TYPE_KEYWORDS = ("BINARY", "BLOB", "BYTEA", "IMAGE", "RAW")
+
 # Maximum number of redirections to any single URL - this is needed because of the state that cookies introduce
 MAX_SINGLE_URL_REDIRECTIONS = 4
 

plugins/generic/databases.py

@@ -46,6 +46,7 @@ from lib.core.enums import PAYLOAD
 from lib.core.exception import SqlmapMissingMandatoryOptionException
 from lib.core.exception import SqlmapNoneDataException
 from lib.core.exception import SqlmapUserQuitException
+from lib.core.settings import BINARY_FIELDS_TYPE_KEYWORDS
 from lib.core.settings import CURRENT_DB
 from lib.core.settings import METADB_SUFFIX
 from lib.core.settings import PLUS_ONE_DBMSES
@@ -931,7 +932,16 @@ class Databases(object):
                                 warnMsg += "possible to get column comments"
                                 singleTimeWarnMessage(warnMsg)
 
-                        if not onlyColNames:
+                        # In dump mode we don't need the exact type, only whether the column is binary (so its raw
+                        # bytes get hex-extracted instead of mangled/truncated - issues #8/#582/#2827). Rather than
+                        # extract the whole type string char-by-char, wrap the type query into a single-bit check and
+                        # extract just that (~10x fewer requests). Skipped where query2 doesn't yield a type name:
+                        # MSSQL (returns the column name), Firebird/Informix (numeric type codes), and PostgreSQL
+                        # (its bytea already renders fine, so it's excluded from auto-hexing anyway).
+                        binaryProbe = onlyColNames and dumpMode and not Backend.getIdentifiedDbms() in (DBMS.MSSQL, DBMS.PGSQL, DBMS.FIREBIRD, DBMS.INFORMIX)
+
+                        if not onlyColNames or binaryProbe:
+                            query = None
                             if Backend.getIdentifiedDbms() in (DBMS.MYSQL, DBMS.PGSQL, DBMS.HSQLDB, DBMS.H2, DBMS.VERTICA, DBMS.PRESTO, DBMS.CRATEDB, DBMS.CACHE, DBMS.FRONTBASE, DBMS.VIRTUOSO, DBMS.CLICKHOUSE):
                                 query = rootQuery.blind.query2 % (unsafeSQLIdentificatorNaming(tbl), column, unsafeSQLIdentificatorNaming(conf.db))
                             elif Backend.getIdentifiedDbms() in (DBMS.ORACLE, DBMS.DB2, DBMS.DERBY, DBMS.ALTIBASE, DBMS.MIMERSQL):
@@ -947,22 +957,35 @@ class Databases(object):
                             elif Backend.isDbms(DBMS.SPANNER):
                                 query = rootQuery.blind.query2 % (unsafeSQLIdentificatorNaming(tbl), column, unsafeSQLIdentificatorNaming(conf.db), unsafeSQLIdentificatorNaming(conf.db))
 
-                            colType = unArrayizeValue(inject.getValue(query, union=False, error=False))
+                            if binaryProbe and query:
-                            key = int(colType) if hasattr(colType, "isdigit") and colType.isdigit() else colType
+                                typeMatch = re.match(r"(?is)\s*SELECT\s+(.+?)\s+FROM\s+(.+)", query)
+                                if typeMatch:
+                                    binaryCondition = " OR ".join("UPPER(%s) LIKE '%%%s%%'" % (typeMatch.group(1), _) for _ in BINARY_FIELDS_TYPE_KEYWORDS)
+                                    query = "SELECT (CASE WHEN %s THEN 1 ELSE 0 END) FROM %s" % (binaryCondition, typeMatch.group(2))
+                                else:
+                                    query = None  # unexpected shape - fall back to leaving the type unknown
 
-                            if Backend.isDbms(DBMS.FIREBIRD):
+                            colType = unArrayizeValue(inject.getValue(query, union=False, error=False)) if query else None
-                                colType = FIREBIRD_TYPES.get(key, colType)
-                            elif Backend.isDbms(DBMS.INFORMIX):
-                                notNull = False
-                                if isinstance(key, int) and key > 255:
-                                    key -= 256
-                                    notNull = True
-                                colType = INFORMIX_TYPES.get(key, colType)
-                                if notNull:
-                                    colType = "%s NOT NULL" % colType
 
-                            column = safeSQLIdentificatorNaming(column)
+                            if binaryProbe:
-                            columns[column] = colType
+                                column = safeSQLIdentificatorNaming(column)
+                                columns[column] = "binary" if colType in ('1', 1) else None  # sentinel matched by BINARY_FIELDS_TYPE_REGEX
+                            else:
+                                key = int(colType) if hasattr(colType, "isdigit") and colType.isdigit() else colType
+
+                                if Backend.isDbms(DBMS.FIREBIRD):
+                                    colType = FIREBIRD_TYPES.get(key, colType)
+                                elif Backend.isDbms(DBMS.INFORMIX):
+                                    notNull = False
+                                    if isinstance(key, int) and key > 255:
+                                        key -= 256
+                                        notNull = True
+                                    colType = INFORMIX_TYPES.get(key, colType)
+                                    if notNull:
+                                        colType = "%s NOT NULL" % colType
+
+                                column = safeSQLIdentificatorNaming(column)
+                                columns[column] = colType
                         else:
                             column = safeSQLIdentificatorNaming(column)
                             columns[column] = None