mirror of
https://github.com/sqlmapproject/sqlmap.git
synced 2026-11-18 14:09:52 +00:00
Doing some cleaning for web search requests
This commit is contained in:
parent
da21091459
commit
5b8a2faab1
5 changed files with 28 additions and 22 deletions
|
|
@ -12,7 +12,6 @@ import random
|
|||
import re
|
||||
import string
|
||||
import sys
|
||||
import time
|
||||
|
||||
from lib.core.enums import DBMS
|
||||
from lib.core.enums import DBMS_DIRECTORY_NAME
|
||||
|
|
@ -20,7 +19,7 @@ from lib.core.enums import OS
|
|||
from thirdparty import six
|
||||
|
||||
# sqlmap version (<major>.<minor>.<month>.<monthly commit>)
|
||||
VERSION = "1.10.10.16"
|
||||
VERSION = "1.10.10.17"
|
||||
TYPE = "dev" if VERSION.count('.') > 2 and VERSION.split('.')[-1] != '0' else "stable"
|
||||
TYPE_COLORS = {"dev": 33, "stable": 90, "pip": 34}
|
||||
VERSION_STRING = "sqlmap/%s#%s" % ('.'.join(VERSION.split('.')[:-1]) if VERSION.count('.') > 2 and VERSION.split('.')[-1] == '0' else VERSION, TYPE)
|
||||
|
|
@ -204,12 +203,6 @@ JITTER_JUNK_RESPONSES = (
|
|||
("", 200),
|
||||
)
|
||||
|
||||
# Regular expression used for extracting results from Google search
|
||||
GOOGLE_REGEX = r"webcache\.googleusercontent\.com/search\?q=cache:[^:]+:([^+]+)\+&cd=|url\?\w+=((?![^>]+webcache\.googleusercontent\.com)http[^>]+)&(sa=U|rct=j)"
|
||||
|
||||
# Google Search consent cookie
|
||||
GOOGLE_CONSENT_COOKIE = "CONSENT=YES+shp.gws-%s-0-RC1.%s+FX+740" % (time.strftime("%Y%m%d"), "".join(random.sample(string.ascii_lowercase, 2)))
|
||||
|
||||
# Regular expression used for extracting results from DuckDuckGo search
|
||||
DUCKDUCKGO_REGEX = r'<a class="result__url" href="(htt[^"]+)'
|
||||
|
||||
|
|
|
|||
|
|
@ -138,8 +138,8 @@ def cmdLineParser(argv=None):
|
|||
target.add_argument("-r", dest="requestFile",
|
||||
help="Load HTTP request from a file")
|
||||
|
||||
target.add_argument("-g", dest="googleDork",
|
||||
help="Process Google dork results as target URLs")
|
||||
target.add_argument("-g", "--dork", dest="googleDork", metavar="DORK",
|
||||
help="Process search engine dork results as target URLs")
|
||||
|
||||
target.add_argument("-c", dest="configFile",
|
||||
help="Load options from a configuration INI file")
|
||||
|
|
@ -736,8 +736,8 @@ def cmdLineParser(argv=None):
|
|||
general.add_argument("--fresh-queries", dest="freshQueries", action="store_true",
|
||||
help="Ignore query results stored in session file")
|
||||
|
||||
general.add_argument("--gpage", dest="googlePage", type=int,
|
||||
help="Use Google dork results from specified page number")
|
||||
general.add_argument("--gpage", dest="googlePage", type=int, metavar="PAGE",
|
||||
help="Use dork results from specified page number")
|
||||
|
||||
general.add_argument("--har", dest="harFile",
|
||||
help="Log all HTTP traffic into a HAR file")
|
||||
|
|
|
|||
|
|
@ -26,8 +26,6 @@ from lib.core.exception import SqlmapBaseException
|
|||
from lib.core.settings import BING_REGEX
|
||||
from lib.core.settings import DUCKDUCKGO_REGEX
|
||||
from lib.core.settings import DUMMY_SEARCH_USER_AGENT
|
||||
from lib.core.settings import GOOGLE_CONSENT_COOKIE
|
||||
from lib.core.settings import GOOGLE_REGEX
|
||||
from lib.core.settings import HTTP_ACCEPT_ENCODING_HEADER_VALUE
|
||||
from lib.core.settings import SEARCH_ABUSE_MARKERS
|
||||
from lib.core.settings import UNICODE_ENCODING
|
||||
|
|
@ -91,7 +89,6 @@ def _search(dork):
|
|||
requestHeaders = {
|
||||
HTTP_HEADER.USER_AGENT: dict(conf.httpHeaders).get(HTTP_HEADER.USER_AGENT, DUMMY_SEARCH_USER_AGENT),
|
||||
HTTP_HEADER.ACCEPT_ENCODING: HTTP_ACCEPT_ENCODING_HEADER_VALUE,
|
||||
HTTP_HEADER.COOKIE: GOOGLE_CONSENT_COOKIE,
|
||||
}
|
||||
|
||||
gpage = conf.googlePage if conf.googlePage > 1 else 1
|
||||
|
|
@ -99,11 +96,13 @@ def _search(dork):
|
|||
|
||||
encoded = urlencode(dork, convall=True)
|
||||
|
||||
# Note: (name, url, POST data, regex, regex flags, match->link). Ordered by current scraping reliability; tried in turn until one yields results (DuckDuckGo currently being the only consistently scrapeable one)
|
||||
# Note: DuckDuckGo's first result page holds 10 entries, while each subsequent one holds 15 (hence the non-linear offset)
|
||||
ddgOffset = 0 if gpage == 1 else 10 + (gpage - 2) * 15
|
||||
|
||||
# Note: (name, url, POST data, regex, regex flags, match->link). Ordered by current scraping reliability; tried in turn until one yields results (Google is not in here as it serves results only to JavaScript capable clients)
|
||||
engines = (
|
||||
("DuckDuckGo", "https://html.duckduckgo.com/html/", "q=%s&s=%d" % (encoded, (gpage - 1) * 30), DUCKDUCKGO_REGEX, re.I | re.S, lambda match: match.group(1).replace("&", "&")),
|
||||
("DuckDuckGo", "https://html.duckduckgo.com/html/", "q=%s&s=%d" % (encoded, ddgOffset), DUCKDUCKGO_REGEX, re.I | re.S, lambda match: match.group(1).replace("&", "&")),
|
||||
("Bing", "https://www.bing.com/search?q=%s&first=%d" % (encoded, (gpage - 1) * 10 + 1), None, BING_REGEX, re.I | re.S, lambda match: decodeBase64(match.group(1), binary=False) if match.group(1) else match.group(2).replace("&", "&")),
|
||||
("Google", "https://www.google.com/search?q=%s&num=100&hl=en&complete=0&safe=off&filter=0&btnG=Search&start=%d" % (encoded, (gpage - 1) * 100), None, GOOGLE_REGEX, re.I, lambda match: match.group(1) or match.group(2)),
|
||||
)
|
||||
|
||||
for name, url, data, regex, flags, extract in engines:
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue