Doing some cleaning for web search requests
Some checks are pending
/ build (macos-latest, 3.8) (push) Waiting to run
/ build (ubuntu-latest, pypy-2.7) (push) Waiting to run
/ build (windows-latest, 3.14) (push) Waiting to run

This commit is contained in:
Miroslav Štampar 2026-10-21 12:10:18 +02:00
parent da21091459
commit 5b8a2faab1
5 changed files with 28 additions and 22 deletions

View file

@ -12,7 +12,6 @@ import random
import re
import string
import sys
import time
from lib.core.enums import DBMS
from lib.core.enums import DBMS_DIRECTORY_NAME
@ -20,7 +19,7 @@ from lib.core.enums import OS
from thirdparty import six
# sqlmap version (<major>.<minor>.<month>.<monthly commit>)
VERSION = "1.10.10.16"
VERSION = "1.10.10.17"
TYPE = "dev" if VERSION.count('.') > 2 and VERSION.split('.')[-1] != '0' else "stable"
TYPE_COLORS = {"dev": 33, "stable": 90, "pip": 34}
VERSION_STRING = "sqlmap/%s#%s" % ('.'.join(VERSION.split('.')[:-1]) if VERSION.count('.') > 2 and VERSION.split('.')[-1] == '0' else VERSION, TYPE)
@ -204,12 +203,6 @@ JITTER_JUNK_RESPONSES = (
("", 200),
)
# Regular expression used for extracting results from Google search
GOOGLE_REGEX = r"webcache\.googleusercontent\.com/search\?q=cache:[^:]+:([^+]+)\+&amp;cd=|url\?\w+=((?![^>]+webcache\.googleusercontent\.com)http[^>]+)&(sa=U|rct=j)"
# Google Search consent cookie
GOOGLE_CONSENT_COOKIE = "CONSENT=YES+shp.gws-%s-0-RC1.%s+FX+740" % (time.strftime("%Y%m%d"), "".join(random.sample(string.ascii_lowercase, 2)))
# Regular expression used for extracting results from DuckDuckGo search
DUCKDUCKGO_REGEX = r'<a class="result__url" href="(htt[^"]+)'

View file

@ -138,8 +138,8 @@ def cmdLineParser(argv=None):
target.add_argument("-r", dest="requestFile",
help="Load HTTP request from a file")
target.add_argument("-g", dest="googleDork",
help="Process Google dork results as target URLs")
target.add_argument("-g", "--dork", dest="googleDork", metavar="DORK",
help="Process search engine dork results as target URLs")
target.add_argument("-c", dest="configFile",
help="Load options from a configuration INI file")
@ -736,8 +736,8 @@ def cmdLineParser(argv=None):
general.add_argument("--fresh-queries", dest="freshQueries", action="store_true",
help="Ignore query results stored in session file")
general.add_argument("--gpage", dest="googlePage", type=int,
help="Use Google dork results from specified page number")
general.add_argument("--gpage", dest="googlePage", type=int, metavar="PAGE",
help="Use dork results from specified page number")
general.add_argument("--har", dest="harFile",
help="Log all HTTP traffic into a HAR file")

View file

@ -26,8 +26,6 @@ from lib.core.exception import SqlmapBaseException
from lib.core.settings import BING_REGEX
from lib.core.settings import DUCKDUCKGO_REGEX
from lib.core.settings import DUMMY_SEARCH_USER_AGENT
from lib.core.settings import GOOGLE_CONSENT_COOKIE
from lib.core.settings import GOOGLE_REGEX
from lib.core.settings import HTTP_ACCEPT_ENCODING_HEADER_VALUE
from lib.core.settings import SEARCH_ABUSE_MARKERS
from lib.core.settings import UNICODE_ENCODING
@ -91,7 +89,6 @@ def _search(dork):
requestHeaders = {
HTTP_HEADER.USER_AGENT: dict(conf.httpHeaders).get(HTTP_HEADER.USER_AGENT, DUMMY_SEARCH_USER_AGENT),
HTTP_HEADER.ACCEPT_ENCODING: HTTP_ACCEPT_ENCODING_HEADER_VALUE,
HTTP_HEADER.COOKIE: GOOGLE_CONSENT_COOKIE,
}
gpage = conf.googlePage if conf.googlePage > 1 else 1
@ -99,11 +96,13 @@ def _search(dork):
encoded = urlencode(dork, convall=True)
# Note: (name, url, POST data, regex, regex flags, match->link). Ordered by current scraping reliability; tried in turn until one yields results (DuckDuckGo currently being the only consistently scrapeable one)
# Note: DuckDuckGo's first result page holds 10 entries, while each subsequent one holds 15 (hence the non-linear offset)
ddgOffset = 0 if gpage == 1 else 10 + (gpage - 2) * 15
# Note: (name, url, POST data, regex, regex flags, match->link). Ordered by current scraping reliability; tried in turn until one yields results (Google is not in here as it serves results only to JavaScript capable clients)
engines = (
("DuckDuckGo", "https://html.duckduckgo.com/html/", "q=%s&s=%d" % (encoded, (gpage - 1) * 30), DUCKDUCKGO_REGEX, re.I | re.S, lambda match: match.group(1).replace("&amp;", "&")),
("DuckDuckGo", "https://html.duckduckgo.com/html/", "q=%s&s=%d" % (encoded, ddgOffset), DUCKDUCKGO_REGEX, re.I | re.S, lambda match: match.group(1).replace("&amp;", "&")),
("Bing", "https://www.bing.com/search?q=%s&first=%d" % (encoded, (gpage - 1) * 10 + 1), None, BING_REGEX, re.I | re.S, lambda match: decodeBase64(match.group(1), binary=False) if match.group(1) else match.group(2).replace("&amp;", "&")),
("Google", "https://www.google.com/search?q=%s&num=100&hl=en&complete=0&safe=off&filter=0&btnG=Search&start=%d" % (encoded, (gpage - 1) * 100), None, GOOGLE_REGEX, re.I, lambda match: match.group(1) or match.group(2)),
)
for name, url, data, regex, flags, extract in engines: