Stripping time outliers from the time stats

lib/core/common.py

@@ -183,6 +183,7 @@ from lib.core.settings import STRUCTURAL_ID_REGEX
 from lib.core.settings import STRUCTURAL_TAG_REGEX
 from lib.core.settings import SUPPORTED_DBMS
 from lib.core.settings import TEXT_TAG_REGEX
+from lib.core.settings import TIME_OUTLIER_MAD_COEFF
 from lib.core.settings import TIME_STDEV_COEFF
 from lib.core.settings import UNICODE_ENCODING
 from lib.core.settings import UNKNOWN_DBMS_VERSION
@@ -2880,6 +2881,38 @@ def wasLastResponseHTTPError():
     threadData = getCurrentThreadData()
     return threadData.lastHTTPError and threadData.lastHTTPError[0] == threadData.lastRequestUID
 
+def stripTimeOutliers(values):
+    """
+    Returns L{values} with high (spike) outliers removed, using a robust median/MAD cutoff.
+
+    A single network spike that lands in the time-response model would otherwise inflate both the
+    average and the standard deviation, exploding the delay threshold (avg + 7*stdev) so that genuine
+    time-delays are no longer recognized. MAD is robust to a minority of outliers, so the cutoff is
+    computed from the clean bulk even when the sample already contains a spike. On a clean model no
+    value exceeds median + 10*MAD, so it is returned unchanged (identical avg/stdev/threshold).
+
+    >>> len(stripTimeOutliers([0.1, 0.12] * 8 + [9.0]))   # a lone 9s spike is dropped (17->16)
+    16
+    >>> len(stripTimeOutliers([0.1, 0.12] * 8))           # a clean model is left intact (no-op)
+    16
+    """
+
+    if not values or len(values) < MIN_TIME_RESPONSES // 2:
+        return values
+
+    ordered = sorted(values)
+    median = ordered[len(ordered) // 2]
+    mad = sorted(abs(_ - median) for _ in values)[len(values) // 2]
+
+    if mad <= 0:  # degenerate (near-constant model) - nothing robust to trim on
+        return values
+
+    cutoff = median + TIME_OUTLIER_MAD_COEFF * 1.4826 * mad
+    retVal = [_ for _ in values if _ <= cutoff]
+
+    # never trim away the bulk (guards a genuinely wide/bimodal model from being gutted)
+    return retVal if len(retVal) >= max(MIN_TIME_RESPONSES // 2, len(values) // 2) else values
+
 def wasLastResponseDelayed():
     """
     Returns True if the last web request resulted in a time-delay
@@ -2889,7 +2922,10 @@ def wasLastResponseDelayed():
     # response times should be inside +-7*stdev([normal response times])
     # Math reference: http://www.answers.com/topic/standard-deviation
 
-    deviation = stdev(kb.responseTimes.get(kb.responseTimeMode, []))
+    # spike outliers (e.g. a GC pause / retransmit during baseline sampling) are stripped first, so a
+    # single lagging response can't inflate the model and hide every genuine delay behind it
+    sample = stripTimeOutliers(kb.responseTimes.get(kb.responseTimeMode, []))
+    deviation = stdev(sample)
     threadData = getCurrentThreadData()
 
     if deviation and not conf.direct and not conf.disableStats:
@@ -2898,7 +2934,7 @@ def wasLastResponseDelayed():
             warnMsg += "with less than %d response times" % MIN_TIME_RESPONSES
             logger.warning(warnMsg)
 
-        lowerStdLimit = average(kb.responseTimes[kb.responseTimeMode]) + TIME_STDEV_COEFF * deviation
+        lowerStdLimit = average(sample) + TIME_STDEV_COEFF * deviation
         retVal = (threadData.lastQueryDuration >= max(MIN_VALID_DELAYED_RESPONSE, lowerStdLimit))
 
         if not kb.testMode and retVal:

lib/core/settings.py

@@ -20,7 +20,7 @@ from lib.core.enums import OS
 from thirdparty import six
 
 # sqlmap version (<major>.<minor>.<month>.<monthly commit>)
-VERSION = "1.10.7.237"
+VERSION = "1.10.7.238"
 TYPE = "dev" if VERSION.count('.') > 2 and VERSION.split('.')[-1] != '0' else "stable"
 TYPE_COLORS = {"dev": 33, "stable": 90, "pip": 34}
 VERSION_STRING = "sqlmap/%s#%s" % ('.'.join(VERSION.split('.')[:-1]) if VERSION.count('.') > 2 and VERSION.split('.')[-1] == '0' else VERSION, TYPE)
@@ -232,6 +232,12 @@ CONCAT_VALUE_DELIMITER = '|'
 # Coefficient used for a time-based query delay checking (must be >= 7)
 TIME_STDEV_COEFF = 7
 
+# Robust (median/MAD) cutoff for discarding spike outliers from the time-response model before
+# computing avg/stdev - a single network spike landing in the baseline would otherwise inflate the
+# delay threshold and miss genuine delays. Deliberately wide (~10 robust sigmas) so a clean model is
+# left untouched (identical threshold) and only true outliers are dropped.
+TIME_OUTLIER_MAD_COEFF = 10
+
 # Minimum response time that can be even considered as delayed (not a complete requirement)
 MIN_VALID_DELAYED_RESPONSE = 0.5
 

tests/test_error_engine.py

@@ -26,6 +26,7 @@ sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
 from _testutils import bootstrap, set_dbms, reset_dbms
 bootstrap()
 
+from lib.core.common import getCurrentThreadData, setTechnique
 from lib.core.data import conf, kb
 from lib.core.datatype import AttribDict
 from lib.core.enums import PAYLOAD, PLACE
@@ -52,6 +53,7 @@ class TestOneShotErrorUse(unittest.TestCase):
             "conf.paramDict": conf.get("paramDict"), "conf.base64Parameter": conf.get("base64Parameter"),
             "kb.errorChunkLength": kb.get("errorChunkLength"), "kb.testMode": kb.get("testMode"),
             "kb.forceWhere": kb.get("forceWhere"), "kb.technique": kb.get("technique"),
+            "td.technique": getCurrentThreadData().technique,
             "kb.inj": (kb.injection.place, kb.injection.parameter, kb.injection.data),
             "qp": Connect.queryPage,
         }
@@ -67,6 +69,7 @@ class TestOneShotErrorUse(unittest.TestCase):
         kb.injection.place = PLACE.GET
         kb.injection.parameter = "id"
         kb.technique = PAYLOAD.TECHNIQUE.ERROR
+        setTechnique(PAYLOAD.TECHNIQUE.ERROR)   # getTechnique() prefers the thread-local; set it so a leaked one can't poison us
         kb.injection.data = {PAYLOAD.TECHNIQUE.ERROR: _make_vector()}
         set_dbms("MySQL")
 
@@ -81,6 +84,7 @@ class TestOneShotErrorUse(unittest.TestCase):
         kb.testMode = self._saved["kb.testMode"]
         kb.forceWhere = self._saved["kb.forceWhere"]
         kb.technique = self._saved["kb.technique"]
+        setTechnique(self._saved["td.technique"])
         kb.injection.place, kb.injection.parameter, kb.injection.data = self._saved["kb.inj"]
         Connect.queryPage = self._saved["qp"]
         eu.Request.queryPage = self._saved["qp"]
@@ -126,6 +130,7 @@ class TestErrorChunkLengthHex(unittest.TestCase):
             "paramDict": conf.get("paramDict"), "base64Parameter": conf.get("base64Parameter"),
             "errorChunkLength": kb.get("errorChunkLength"), "testMode": kb.get("testMode"),
             "forceWhere": kb.get("forceWhere"), "technique": kb.get("technique"),
+            "td.technique": getCurrentThreadData().technique,
             "inj": (kb.injection.place, kb.injection.parameter, kb.injection.data),
             "qp": Connect.queryPage,
             "dbmsHandler": conf.get("dbmsHandler"), "forceDbms": conf.get("forceDbms"),
@@ -145,6 +150,7 @@ class TestErrorChunkLengthHex(unittest.TestCase):
         kb.injection.place = PLACE.GET
         kb.injection.parameter = "id"
         kb.technique = PAYLOAD.TECHNIQUE.ERROR
+        setTechnique(PAYLOAD.TECHNIQUE.ERROR)   # getTechnique() prefers the thread-local; set it so a leaked one can't poison us
         kb.injection.data = {PAYLOAD.TECHNIQUE.ERROR: _make_vector()}
         # With testMode=False, getIdentifiedDbms() prefers conf.dbmsHandler._dbms and conf.forceDbms
         # over the forced DBMS below; a leaked handler/option (e.g. MSSQL) would make the chunk-length
@@ -164,6 +170,7 @@ class TestErrorChunkLengthHex(unittest.TestCase):
         kb.testMode = self._saved["testMode"]
         kb.forceWhere = self._saved["forceWhere"]
         kb.technique = self._saved["technique"]
+        setTechnique(self._saved["td.technique"])
         kb.injection.place, kb.injection.parameter, kb.injection.data = self._saved["inj"]
         Connect.queryPage = self._saved["qp"]
         eu.Request.queryPage = self._saved["qp"]