Couple of bug fixes

lib/core/convert.py

@@ -45,6 +45,8 @@ def base64pickle(value):
 
     >>> base64unpickle(base64pickle([1, 2, 3])) == [1, 2, 3]
     True
+    >>> isinstance(base64unpickle(base64pickle(BigArray([1, 2, 3]))), BigArray)
+    True
     """
 
     retVal = None

lib/core/patch.py

@@ -182,15 +182,17 @@ def dirtyPatches():
     import pickle
     if not getattr(pickle, "_patched", False):
         class RestrictedUnpickler(pickle.Unpickler):
+            # Note: allowlist (not blacklist) - a module blacklist is bypassable (e.g. importlib/ctypes/operator), so only
+            # explicitly-safe builtin data types and sqlmap's own (and bundled) classes are permitted to be unpickled
             def find_class(self, module, name):
-                # blacklist for OS-level execution modules
-                if module in ("os", "subprocess", "sys", "posix", "nt", "pty", "commands", "shutil"):
+                # safe builtin data types only (blocks eval/exec/__import__/getattr/etc.)
+                if module in ("builtins", "__builtin__"):
+                    if name not in ("set", "frozenset", "dict", "list", "tuple", "int", "float", "bool", "str", "bytes", "bytearray", "object", "NoneType", "complex"):
+                        raise ValueError("unpickling of '%s.%s' is forbidden" % (module, name))
+                # everything else must be one of sqlmap's own (or bundled) classes (e.g. lib.core.datatype.AttribDict)
+                elif (module or "").split(".")[0] not in ("lib", "plugins", "thirdparty"):
                     raise ValueError("unpickling of module '%s' is forbidden" % module)
 
-                # partial whitelist for builtins to allow safe data types but block eval/exec/__import__
-                if module in ("builtins", "__builtin__") and name not in ("set", "frozenset", "dict", "list", "tuple", "int", "float", "bool", "str", "bytes", "bytearray", "object", "NoneType"):
-                    raise ValueError("unpickling of '%s.%s' is forbidden" % (module, name))
-
                 # Python 2/3 method resolution
                 if hasattr(pickle.Unpickler, "find_class"):
                     return pickle.Unpickler.find_class(self, module, name)

lib/core/settings.py

@@ -20,7 +20,7 @@ from lib.core.enums import OS
 from thirdparty import six
 
 # sqlmap version (<major>.<minor>.<month>.<monthly commit>)
-VERSION = "1.10.6.99"
+VERSION = "1.10.6.100"
 TYPE = "dev" if VERSION.count('.') > 2 and VERSION.split('.')[-1] != '0' else "stable"
 TYPE_COLORS = {"dev": 33, "stable": 90, "pip": 34}
 VERSION_STRING = "sqlmap/%s#%s" % ('.'.join(VERSION.split('.')[:-1]) if VERSION.count('.') > 2 and VERSION.split('.')[-1] == '0' else VERSION, TYPE)
@@ -681,6 +681,9 @@ TRIM_STDOUT_DUMP_SIZE = 256
 # Reference: https://web.archive.org/web/20150407141500/https://support.microsoft.com/en-us/kb/899149
 DUMP_FILE_BUFFER_SIZE = 1024
 
+# Block size used for the in-place secure-overwrite passes of '--purge' (bounds peak memory regardless of file size)
+PURGE_BLOCK_SIZE = 1024 * 1024
+
 # Parse response headers only first couple of times
 PARSE_HEADERS_LIMIT = 3
 

lib/parse/html.py

@@ -27,11 +27,11 @@ class HTMLHandler(ContentHandler):
 
         self._dbms = None
         self._page = (page or "")
+        self._urldecodedPage = urldecode(self._page)
         try:
-            self._lower_page = self._page.lower()
+            self._lowerPage = self._urldecodedPage.lower()  # Note: keyword pre-filter must match the page that re.search() runs on (the URL-decoded one)
         except SystemError:  # https://bugs.python.org/issue18183
-            self._lower_page = None
-        self._urldecoded_page = urldecode(self._page)
+            self._lowerPage = None
 
         self.dbms = None
 
@@ -53,7 +53,7 @@ class HTMLHandler(ContentHandler):
                 keywords = sorted(keywords, key=len)
                 kb.cache.regex[regexp] = keywords[-1].lower()
 
-            if ('|' in regexp or kb.cache.regex[regexp] in (self._lower_page or kb.cache.regex[regexp])) and re.search(regexp, self._urldecoded_page, re.I):
+            if ('|' in regexp or kb.cache.regex[regexp] in (self._lowerPage or kb.cache.regex[regexp])) and re.search(regexp, self._urldecodedPage, re.I):
                 self.dbms = self._dbms
                 self._markAsErrorPage()
                 kb.forkNote = kb.forkNote or attrs.get("fork")

lib/parse/sitemap.py

@@ -41,7 +41,7 @@ def parseSitemap(url, retVal=None, visited=None):
             raise SqlmapSyntaxException(errMsg)
 
         if content:
-            content = re.sub(r"", "", content, flags=re.DOTALL)
+            content = re.sub(r"<!--.*?-->", "", content, flags=re.DOTALL)  # Note: strip (possibly multi-line) XML comments so commented-out <loc> entries aren't harvested
 
             for match in re.finditer(r"<\w*?loc[^>]*>\s*([^<]+)", content, re.I):
                 if abortedFlag:

lib/request/basic.py

@@ -284,6 +284,8 @@ def decodePage(page, contentEncoding, contentType, percentDecode=True):
     '\\t'
     >>> getText(decodePage(b"J", None, "text/html; charset=utf-8"))
     'J'
+    >>> decodePage(b"™", None, "text/html; charset=utf-8") == u"\u2122"
+    True
     """
 
     if not page or (conf.nullConnection and len(page) < 2):
@@ -379,6 +381,16 @@ def decodePage(page, contentEncoding, contentType, percentDecode=True):
                     return retVal
                 page = re.sub(r"&#(\d+);", _, page)
 
+                # e.g. &#x2019;&#x2026;&#x2122; (hex numeric refs >= U+0100; smaller ones already handled at byte-level)
+                def _(match):
+                    retVal = match.group(0)
+                    try:
+                        retVal = _unichr(int(match.group(1), 16))
+                    except (ValueError, OverflowError):
+                        pass
+                    return retVal
+                page = re.sub(r"(?i)&#x([0-9a-f]+);", _, page)
+
             # e.g. &zeta;
             page = re.sub(r"&([^;]+);", lambda _: _unichr(HTML_ENTITIES[_.group(1)]) if HTML_ENTITIES.get(_.group(1), 0) > 255 else _.group(0), page)
         else:

lib/utils/api.py

@@ -327,10 +327,10 @@ def check_authentication():
     except:
         request.environ["PATH_INFO"] = "/error/401"
     else:
-        if creds.count(':') != 1:
+        if ':' not in creds:
             request.environ["PATH_INFO"] = "/error/401"
         else:
-            username, password = creds.split(':')
+            username, password = creds.split(':', 1)
             if username.strip() != (DataStore.username or "") or password.strip() != (DataStore.password or ""):
                 request.environ["PATH_INFO"] = "/error/401"
 

lib/utils/purge.py

@@ -13,11 +13,9 @@ import stat
 import string
 
 from lib.core.common import getSafeExString
-from lib.core.common import openFile
-from lib.core.compat import xrange
 from lib.core.convert import getUnicode
 from lib.core.data import logger
-from thirdparty.six import unichr as _unichr
+from lib.core.settings import PURGE_BLOCK_SIZE
 
 def purge(directory):
     """
@@ -46,12 +44,25 @@ def purge(directory):
         except:
             pass
 
-    logger.debug("writing random data to files")
+    logger.debug("overwriting file contents")
     for filepath in filepaths:
         try:
             filesize = os.path.getsize(filepath)
-            with openFile(filepath, "w+") as f:
-                f.write("".join(_unichr(random.randint(0, 255)) for _ in xrange(filesize)))
+            if filesize:
+                # Note: NIST SP 800-88 ("Clear") / DoD 5220.22-M style multi-pass in-place overwrite
+                # (zeros, ones, random) forcing each pass to disk; performed BEFORE the truncation below
+                # so the original bytes are actually overwritten and not just released to free blocks.
+                # Written in bounded blocks so peak memory stays O(PURGE_BLOCK_SIZE), not O(filesize)
+                with open(filepath, "r+b") as f:
+                    for getBlock in (lambda n: b"\x00" * n, lambda n: b"\xff" * n, lambda n: os.urandom(n)):
+                        f.seek(0)
+                        remaining = filesize
+                        while remaining > 0:
+                            count = min(PURGE_BLOCK_SIZE, remaining)
+                            f.write(getBlock(count))
+                            remaining -= count
+                        f.flush()
+                        os.fsync(f.fileno())
         except:
             pass
 

lib/utils/sqlalchemy.py

@@ -82,7 +82,7 @@ class SQLAlchemy(GenericConnector):
                     engine = _sqlalchemy.create_engine(self.address, connect_args={})
 
                 self.connector = engine.connect()
-            except (TypeError, ValueError):
+            except (TypeError, ValueError) as ex:
                 if "_get_server_version_info" in traceback.format_exc():
                     try:
                         import pymssql
@@ -90,10 +90,14 @@ class SQLAlchemy(GenericConnector):
                             raise SqlmapConnectionException("SQLAlchemy connection issue (obsolete version of pymssql ('%s') is causing problems)" % pymssql.__version__)
                     except ImportError:
                         pass
+                    # Note: surface (as a proper SqlmapConnectionException) instead of silently continuing with self.connector left None
+                    raise SqlmapConnectionException("SQLAlchemy connection issue ('%s')" % getSafeExString(ex))
                 elif "invalid literal for int() with base 10: '0b" in traceback.format_exc():
                     raise SqlmapConnectionException("SQLAlchemy connection issue ('https://bitbucket.org/zzzeek/sqlalchemy/issues/3975')")
                 else:
-                    pass
+                    # Note: raise as SqlmapConnectionException (like the generic handler below) so the caller's native-connector
+                    # fallback engages and no raw TypeError/ValueError can reach sqlmap's top-level handler
+                    raise SqlmapConnectionException("SQLAlchemy connection issue ('%s')" % getSafeExString(ex))
             except SqlmapFilePathException:
                 raise
             except Exception as ex: