mirror of
https://github.com/sqlmapproject/sqlmap.git
synced 2026-08-04 14:55:40 +00:00
85 lines
3.3 KiB
Python
85 lines
3.3 KiB
Python
#!/usr/bin/env python
|
|
|
|
"""
|
|
Copyright (c) 2006-2026 sqlmap developers (https://sqlmap.org)
|
|
See the file 'LICENSE' for copying permission
|
|
"""
|
|
|
|
import re
|
|
|
|
from lib.core.common import readInput
|
|
from lib.core.convert import htmlUnescape
|
|
from lib.core.data import kb
|
|
from lib.core.data import logger
|
|
from lib.core.datatype import OrderedSet
|
|
from lib.core.exception import SqlmapSyntaxException
|
|
from lib.core.settings import MAX_SITEMAP_FETCHES
|
|
from lib.core.settings import MAX_SITEMAP_URLS
|
|
from lib.request.connect import Connect as Request
|
|
from thirdparty.six.moves import http_client as _http_client
|
|
|
|
abortedFlag = None
|
|
|
|
def parseSitemap(url, retVal=None, visited=None, urlFilter=None):
|
|
"""Parse a sitemap (recursively following nested sitemap indexes). 'urlFilter' - when given - must
|
|
return True for a URL to be fetched or kept; it is enforced on the initial URL AND every nested
|
|
sitemap fetched recursively, so a hostile sitemap index cannot pull the crawler out of scope."""
|
|
|
|
global abortedFlag
|
|
|
|
if retVal is not None:
|
|
logger.debug("parsing sitemap '%s'" % url)
|
|
|
|
try:
|
|
if retVal is None:
|
|
abortedFlag = False
|
|
retVal = OrderedSet()
|
|
visited = set()
|
|
|
|
if url in visited or (urlFilter is not None and not urlFilter(url)):
|
|
return retVal
|
|
|
|
visited.add(url)
|
|
|
|
if len(visited) > MAX_SITEMAP_FETCHES or len(retVal) >= MAX_SITEMAP_URLS: # bound a hostile sitemap graph
|
|
if not abortedFlag: # warn once on the False->True transition
|
|
logger.warning("sitemap parsing stopped after reaching the fetch/URL limit. sqlmap will use partial list")
|
|
abortedFlag = True
|
|
return retVal
|
|
|
|
try:
|
|
content = Request.getPage(url=url, raise404=True)[0] if not abortedFlag else ""
|
|
except _http_client.InvalidURL:
|
|
errMsg = "invalid URL given for sitemap ('%s')" % url
|
|
raise SqlmapSyntaxException(errMsg)
|
|
|
|
if content:
|
|
content = re.sub(r"<!--.*?-->", "", content, flags=re.DOTALL) # Note: strip (possibly multi-line) XML comments so commented-out <loc> entries aren't harvested
|
|
|
|
for match in re.finditer(r"<\w*?loc[^>]*>\s*([^<]+)", content, re.I):
|
|
if abortedFlag or len(retVal) >= MAX_SITEMAP_URLS:
|
|
break
|
|
|
|
foundUrl = htmlUnescape(match.group(1).strip())
|
|
|
|
# Basic validation to avoid junk
|
|
if not foundUrl.startswith("http"):
|
|
continue
|
|
|
|
if foundUrl.endswith(".xml") and "sitemap" in foundUrl.lower():
|
|
if kb.followSitemapRecursion is None:
|
|
message = "sitemap recursion detected. Do you want to follow? [y/N] "
|
|
kb.followSitemapRecursion = readInput(message, default='N', boolean=True)
|
|
|
|
if kb.followSitemapRecursion:
|
|
parseSitemap(foundUrl, retVal, visited, urlFilter)
|
|
elif urlFilter is None or urlFilter(foundUrl):
|
|
retVal.add(foundUrl)
|
|
|
|
except KeyboardInterrupt:
|
|
abortedFlag = True
|
|
warnMsg = "user aborted during sitemap parsing. sqlmap "
|
|
warnMsg += "will use partial list"
|
|
logger.warning(warnMsg)
|
|
|
|
return retVal
|