Compressing common identifiers list

This commit is contained in:
Miroslav Štampar 2026-07-20 11:11:29 +02:00
parent d1eb861d72
commit adc43552d6
5 changed files with 19 additions and 13806 deletions

Binary file not shown.

File diff suppressed because it is too large Load diff

View file

@ -1585,7 +1585,7 @@ def setPaths(rootPath):
paths.SQLMAP_XML_PAYLOADS_PATH = os.path.join(paths.SQLMAP_XML_PATH, "payloads")
# sqlmap files
paths.CATALOG_IDENTIFIERS = os.path.join(paths.SQLMAP_TXT_PATH, "catalog-identifiers.txt")
paths.CATALOG_IDENTIFIERS = os.path.join(paths.SQLMAP_TXT_PATH, "catalog-identifiers.tx_")
paths.COMMON_COLUMNS = os.path.join(paths.SQLMAP_TXT_PATH, "common-columns.txt")
paths.COMMON_FILES = os.path.join(paths.SQLMAP_TXT_PATH, "common-files.txt")
paths.COMMON_TABLES = os.path.join(paths.SQLMAP_TXT_PATH, "common-tables.txt")

View file

@ -20,7 +20,7 @@ from lib.core.enums import OS
from thirdparty import six
# sqlmap version (<major>.<minor>.<month>.<monthly commit>)
VERSION = "1.10.7.145"
VERSION = "1.10.7.146"
TYPE = "dev" if VERSION.count('.') > 2 and VERSION.split('.')[-1] != '0' else "stable"
TYPE_COLORS = {"dev": 33, "stable": 90, "pip": 34}
VERSION_STRING = "sqlmap/%s#%s" % ('.'.join(VERSION.split('.')[:-1]) if VERSION.count('.') > 2 and VERSION.split('.')[-1] == '0' else VERSION, TYPE)

View file

@ -24,7 +24,7 @@ from lib.core.common import getFileItems
from lib.core.common import getPartRun
from lib.core.common import getTechnique
from lib.core.common import getTechniqueData
from lib.core.common import openFile
from lib.core.common import getText
from lib.core.common import predictValue
from lib.core.common import hashDBRetrieve
from lib.core.common import hashDBWrite
@ -44,6 +44,7 @@ from lib.core.enums import DBMS
from lib.core.enums import PAYLOAD
from lib.core.exception import SqlmapThreadException
from lib.core.exception import SqlmapUnsupportedFeatureException
from lib.core.wordlist import Wordlist
from lib.core.settings import CHAR_INFERENCE_MARK
from lib.core.settings import HUFFMAN_PROBE_LIMIT
from lib.core.settings import HUFFMAN_PRIOR_WEIGHTS
@ -109,7 +110,7 @@ def getHuffmanPrior(order, scale, dbms=None):
set-membership tree during blind NAME enumeration (so it predicts from the first character rather
than cold). Trained on the app-identifier wordlists (common-tables/common-columns) plus, when the
back-end is fingerprinted, the system/catalog identifiers harvested for that DBMS (from the matching
[<DBMS>] section of catalog-identifiers.txt - a single global model dilutes across dialects).
[<DBMS>] section of catalog-identifiers.tx_ - a single global model dilutes across dialects).
Per-context counts are scaled to a peak of `scale`. Retrieval is correct regardless of this model.
"""
@ -126,19 +127,23 @@ def getHuffmanPrior(order, scale, dbms=None):
pass
if dbms:
wordlist = None
try:
with openFile(paths.CATALOG_IDENTIFIERS, "r", errors="ignore") as f:
section = None
for line in f:
line = line.strip()
if not line or line.startswith('#'):
continue
if line.startswith('[') and line.endswith(']'):
section = line[1:-1]
elif section == dbms:
names.append(line)
wordlist = Wordlist(paths.CATALOG_IDENTIFIERS) # transparently decompresses the shipped .tx_
section = None
for line in wordlist:
line = getText(line).strip()
if not line or line.startswith('#'):
continue
if line.startswith('[') and line.endswith(']'):
section = line[1:-1]
elif section == dbms:
names.append(line)
except Exception:
pass
finally:
if wordlist is not None:
wordlist.closeFP()
for name in names:
terminated = name + "\x00"