From 06305568267c690e3afdf07d9eab547c3d5f8d0f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Miroslav=20=C5=A0tampar?= Date: Tue, 28 Jul 2026 23:57:47 +0200 Subject: [PATCH] Stripping time outliers from the time stats --- lib/core/common.py | 40 ++++++++++++++++++++++++++++++++++++-- lib/core/settings.py | 8 +++++++- tests/test_error_engine.py | 7 +++++++ 3 files changed, 52 insertions(+), 3 deletions(-) diff --git a/lib/core/common.py b/lib/core/common.py index 8751aa34d..a4be08272 100644 --- a/lib/core/common.py +++ b/lib/core/common.py @@ -183,6 +183,7 @@ from lib.core.settings import STRUCTURAL_ID_REGEX from lib.core.settings import STRUCTURAL_TAG_REGEX from lib.core.settings import SUPPORTED_DBMS from lib.core.settings import TEXT_TAG_REGEX +from lib.core.settings import TIME_OUTLIER_MAD_COEFF from lib.core.settings import TIME_STDEV_COEFF from lib.core.settings import UNICODE_ENCODING from lib.core.settings import UNKNOWN_DBMS_VERSION @@ -2880,6 +2881,38 @@ def wasLastResponseHTTPError(): threadData = getCurrentThreadData() return threadData.lastHTTPError and threadData.lastHTTPError[0] == threadData.lastRequestUID +def stripTimeOutliers(values): + """ + Returns L{values} with high (spike) outliers removed, using a robust median/MAD cutoff. + + A single network spike that lands in the time-response model would otherwise inflate both the + average and the standard deviation, exploding the delay threshold (avg + 7*stdev) so that genuine + time-delays are no longer recognized. MAD is robust to a minority of outliers, so the cutoff is + computed from the clean bulk even when the sample already contains a spike. On a clean model no + value exceeds median + 10*MAD, so it is returned unchanged (identical avg/stdev/threshold). + + >>> len(stripTimeOutliers([0.1, 0.12] * 8 + [9.0])) # a lone 9s spike is dropped (17->16) + 16 + >>> len(stripTimeOutliers([0.1, 0.12] * 8)) # a clean model is left intact (no-op) + 16 + """ + + if not values or len(values) < MIN_TIME_RESPONSES // 2: + return values + + ordered = sorted(values) + median = ordered[len(ordered) // 2] + mad = sorted(abs(_ - median) for _ in values)[len(values) // 2] + + if mad <= 0: # degenerate (near-constant model) - nothing robust to trim on + return values + + cutoff = median + TIME_OUTLIER_MAD_COEFF * 1.4826 * mad + retVal = [_ for _ in values if _ <= cutoff] + + # never trim away the bulk (guards a genuinely wide/bimodal model from being gutted) + return retVal if len(retVal) >= max(MIN_TIME_RESPONSES // 2, len(values) // 2) else values + def wasLastResponseDelayed(): """ Returns True if the last web request resulted in a time-delay @@ -2889,7 +2922,10 @@ def wasLastResponseDelayed(): # response times should be inside +-7*stdev([normal response times]) # Math reference: http://www.answers.com/topic/standard-deviation - deviation = stdev(kb.responseTimes.get(kb.responseTimeMode, [])) + # spike outliers (e.g. a GC pause / retransmit during baseline sampling) are stripped first, so a + # single lagging response can't inflate the model and hide every genuine delay behind it + sample = stripTimeOutliers(kb.responseTimes.get(kb.responseTimeMode, [])) + deviation = stdev(sample) threadData = getCurrentThreadData() if deviation and not conf.direct and not conf.disableStats: @@ -2898,7 +2934,7 @@ def wasLastResponseDelayed(): warnMsg += "with less than %d response times" % MIN_TIME_RESPONSES logger.warning(warnMsg) - lowerStdLimit = average(kb.responseTimes[kb.responseTimeMode]) + TIME_STDEV_COEFF * deviation + lowerStdLimit = average(sample) + TIME_STDEV_COEFF * deviation retVal = (threadData.lastQueryDuration >= max(MIN_VALID_DELAYED_RESPONSE, lowerStdLimit)) if not kb.testMode and retVal: diff --git a/lib/core/settings.py b/lib/core/settings.py index 8af15caef..91ccffb62 100644 --- a/lib/core/settings.py +++ b/lib/core/settings.py @@ -20,7 +20,7 @@ from lib.core.enums import OS from thirdparty import six # sqlmap version (...) -VERSION = "1.10.7.237" +VERSION = "1.10.7.238" TYPE = "dev" if VERSION.count('.') > 2 and VERSION.split('.')[-1] != '0' else "stable" TYPE_COLORS = {"dev": 33, "stable": 90, "pip": 34} VERSION_STRING = "sqlmap/%s#%s" % ('.'.join(VERSION.split('.')[:-1]) if VERSION.count('.') > 2 and VERSION.split('.')[-1] == '0' else VERSION, TYPE) @@ -232,6 +232,12 @@ CONCAT_VALUE_DELIMITER = '|' # Coefficient used for a time-based query delay checking (must be >= 7) TIME_STDEV_COEFF = 7 +# Robust (median/MAD) cutoff for discarding spike outliers from the time-response model before +# computing avg/stdev - a single network spike landing in the baseline would otherwise inflate the +# delay threshold and miss genuine delays. Deliberately wide (~10 robust sigmas) so a clean model is +# left untouched (identical threshold) and only true outliers are dropped. +TIME_OUTLIER_MAD_COEFF = 10 + # Minimum response time that can be even considered as delayed (not a complete requirement) MIN_VALID_DELAYED_RESPONSE = 0.5 diff --git a/tests/test_error_engine.py b/tests/test_error_engine.py index ee960c94c..19f3ab990 100644 --- a/tests/test_error_engine.py +++ b/tests/test_error_engine.py @@ -26,6 +26,7 @@ sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) from _testutils import bootstrap, set_dbms, reset_dbms bootstrap() +from lib.core.common import getCurrentThreadData, setTechnique from lib.core.data import conf, kb from lib.core.datatype import AttribDict from lib.core.enums import PAYLOAD, PLACE @@ -52,6 +53,7 @@ class TestOneShotErrorUse(unittest.TestCase): "conf.paramDict": conf.get("paramDict"), "conf.base64Parameter": conf.get("base64Parameter"), "kb.errorChunkLength": kb.get("errorChunkLength"), "kb.testMode": kb.get("testMode"), "kb.forceWhere": kb.get("forceWhere"), "kb.technique": kb.get("technique"), + "td.technique": getCurrentThreadData().technique, "kb.inj": (kb.injection.place, kb.injection.parameter, kb.injection.data), "qp": Connect.queryPage, } @@ -67,6 +69,7 @@ class TestOneShotErrorUse(unittest.TestCase): kb.injection.place = PLACE.GET kb.injection.parameter = "id" kb.technique = PAYLOAD.TECHNIQUE.ERROR + setTechnique(PAYLOAD.TECHNIQUE.ERROR) # getTechnique() prefers the thread-local; set it so a leaked one can't poison us kb.injection.data = {PAYLOAD.TECHNIQUE.ERROR: _make_vector()} set_dbms("MySQL") @@ -81,6 +84,7 @@ class TestOneShotErrorUse(unittest.TestCase): kb.testMode = self._saved["kb.testMode"] kb.forceWhere = self._saved["kb.forceWhere"] kb.technique = self._saved["kb.technique"] + setTechnique(self._saved["td.technique"]) kb.injection.place, kb.injection.parameter, kb.injection.data = self._saved["kb.inj"] Connect.queryPage = self._saved["qp"] eu.Request.queryPage = self._saved["qp"] @@ -126,6 +130,7 @@ class TestErrorChunkLengthHex(unittest.TestCase): "paramDict": conf.get("paramDict"), "base64Parameter": conf.get("base64Parameter"), "errorChunkLength": kb.get("errorChunkLength"), "testMode": kb.get("testMode"), "forceWhere": kb.get("forceWhere"), "technique": kb.get("technique"), + "td.technique": getCurrentThreadData().technique, "inj": (kb.injection.place, kb.injection.parameter, kb.injection.data), "qp": Connect.queryPage, "dbmsHandler": conf.get("dbmsHandler"), "forceDbms": conf.get("forceDbms"), @@ -145,6 +150,7 @@ class TestErrorChunkLengthHex(unittest.TestCase): kb.injection.place = PLACE.GET kb.injection.parameter = "id" kb.technique = PAYLOAD.TECHNIQUE.ERROR + setTechnique(PAYLOAD.TECHNIQUE.ERROR) # getTechnique() prefers the thread-local; set it so a leaked one can't poison us kb.injection.data = {PAYLOAD.TECHNIQUE.ERROR: _make_vector()} # With testMode=False, getIdentifiedDbms() prefers conf.dbmsHandler._dbms and conf.forceDbms # over the forced DBMS below; a leaked handler/option (e.g. MSSQL) would make the chunk-length @@ -164,6 +170,7 @@ class TestErrorChunkLengthHex(unittest.TestCase): kb.testMode = self._saved["testMode"] kb.forceWhere = self._saved["forceWhere"] kb.technique = self._saved["technique"] + setTechnique(self._saved["td.technique"]) kb.injection.place, kb.injection.parameter, kb.injection.data = self._saved["inj"] Connect.queryPage = self._saved["qp"] eu.Request.queryPage = self._saved["qp"]