diff --git a/scripts/http-grep.nse b/scripts/http-grep.nse
index 3c91792f4..84f50bd63 100644
--- a/scripts/http-grep.nse
+++ b/scripts/http-grep.nse
@@ -1,34 +1,64 @@
+local string = require "string"
local httpspider = require "httpspider"
local shortport = require "shortport"
local stdnse = require "stdnse"
local table = require "table"
+
description = [[
Spiders a website and attempts to match all pages and urls against a given
string. Matches are counted and grouped per url under which they were
discovered.
+
+Features built in patterns like email, ip, ssn, discover, amex and more.
+The script searches for email and ip by default.
+
]]
---
-- @usage
--- nmap -p 80 www.example.com --script http-grep --script-args='http-grep.match="[A-Za-z0-9%.%%%+%-]+@[A-Za-z0-9%.%%%+%-]+%.%w%w%w?%w?",http-grep.breakonmatch'
---
+-- nmap -p 80 www.example.com --script http-grep --script-args='match="[A-Za-z0-9%.%%%+%-]+@[A-Za-z0-9%.%%%+%-]+%.%w%w%w?%w?",breakonmatch'
+-- nmap -p 80 www.example.com --script http-grep --script-args 'http-grep.builtins ={"mastercard", "discover"}, http-grep.url="example.html"'
-- @output
-- PORT STATE SERVICE REASON
-- 80/tcp open http syn-ack
--- | http-grep:
--- | (4) http://example.com/name/
--- | + name@example.com
--- | + name@example.com
--- | + name@example.com
--- | + name@example.com
--- | (4) http://example.com/sales.html
--- | + sales@example.com
--- | + sales@example.com
--- | + sales@example.com
--- |__ + sales@example.com
+-- | http-grep:
+-- | (1) http://nmap.org/book/man-bugs.html:
+-- | (1) email:
+-- | + dev@nmap.org
+-- | (1) http://nmap.org/book/install.html:
+-- | (1) email:
+-- | + fyodor@nmap.org
+-- | (16) http://nmap.org/changelog.html:
+-- | (7) ip:
+-- | + 255.255.255.255
+-- | + 10.99.24.140
+-- | + 74.125.53.103
+-- | + 64.147.188.3
+-- | + 203.65.42.255
+-- | + 192.31.33.7
+-- | + 168.0.40.135
+-- | (9) email:
+-- | + d1n@inbox.com
+-- | + fyodor@insecure.org
+-- | + uce@ftc.gov
+-- | + rhundt@fcc.gov
+-- | + jquello@fcc.gov
+-- | + sness@fcc.gov
+-- | + president@whitehouse.gov
+-- | + haesslich@loyalty.org
+-- | + rchong@fcc.gov
+-- | (6) http://nmap.org/5/#5changes:
+-- | (6) ip:
+-- | + 207.68.200.30
+-- | + 64.13.134.52
+-- | + 4.68.105.6
+-- | + 209.245.176.2
+-- | + 69.63.179.23
+-- |_ + 69.63.180.12
--
--- @args http-grep.match the string to match in urls and page contents
+--
+-- @args http-grep.match the string to match in urls and page contents or list of patterns separated by delimiter
-- @args http-grep.maxdepth the maximum amount of directories beneath
-- the initial url to spider. A negative value disables the limit.
-- (default: 3)
@@ -41,8 +71,55 @@ discovered.
-- @args http-grep.withindomain only spider URLs within the same
-- domain. This widens the scope from withinhost and can
-- not be used in combination. (default: false)
+-- @args http.breakonmatch Returns output if there is a match for a single pattern type.
+-- @args http-grep.builtins supply a single or a list of built in types. supports email, phone, mastercard, discover,
+-- visa, amex, ssn and ip addresses. If you just put in script-args http-grep.builtins then all will be enabled.
+--
+-- @xmloutput
+--
+--
+--
+-- + fyodor@nmap.org
+--
+--
+--
+--
+-- + 255.255.255.255
+-- + 10.99.24.140
+-- + 74.125.53.103
+-- + 64.147.188.3
+-- + 203.65.42.255
+-- + 192.31.33.7
+-- + 168.0.40.135
+--
+--
+-- + d1n@inbox.com
+-- + fyodor@insecure.org
+-- + uce@ftc.gov
+-- + rhundt@fcc.gov
+-- + jquello@fcc.gov
+-- + sness@fcc.gov
+-- + president@whitehouse.gov
+-- + haesslich@loyalty.org
+-- + rchong@fcc.gov
+--
+--
+--
+--
+-- + 207.68.200.30
+-- + 64.13.134.52
+-- + 4.68.105.6
+-- + 209.245.176.2
+-- + 69.63.179.23
+-- + 69.63.180.12
+--
+--
-author = "Patrik Karlsson"
+author = {"Patrik Karlsson", "Gyanendra Mishra"}
license = "Same as Nmap--See http://nmap.org/book/man-legal.html"
categories = {"discovery", "safe"}
@@ -59,18 +136,119 @@ local function shortenMatch(match)
end
end
+-- A function to validate IP addresses.
+local function ip(matched_ip)
+ local oct_1, oct_2, oct_3, oct_4 = matched_ip:match('(%d%d?%d?)%.(%d%d?%d?)%.(%d%d?%d)%.(%d%d?%d?)')
+ oct_1, oct_2, oct_3, oct_4 = tonumber(oct_1), tonumber(oct_2), tonumber(oct_3), tonumber(oct_4)
+ if oct_1 > 255 or oct_2 > 255 or oct_3 > 255 or oct_4 > 255 then
+ return false
+ end
+ return true
+end
+
+-- from rosettacode. A function to validate credit card numbers.
+local function luhn(matched_ccno)
+ matched_ccno = matched_ccno:gsub("-", "")
+ matched_ccno = matched_ccno:gsub(" ", "")
+ local n = string.reverse(matched_ccno)
+ local s1 = 0
+ for i=1, n:len(), 2 do
+ s1 = s1 + tonumber(n:sub(i,i))
+ end
+ local s2 = 0
+ for i=2, n:len(), 2 do
+ local doubled = n:sub(i,i)*2
+ doubled = string.gsub(doubled,'(%d)(%d)',function(a,b)return a+b end)
+ s2 = s2+doubled
+ end
+ local total = s1 + s2
+ if total%10 == 0 then
+ return true
+ end
+ return false
+end
+
+-- A function to validate ssn numbers.
+local function ssn(matched_ssn)
+ local group_1, group_2, group_3 = matched_ssn:match('(%d%d%d%)-(%d%d%)-(%d%d%d%d)')
+ group_1, group_2, group_3 = tonumber(group_1), tonumber(group_2), tonumber(group_3)
+ if group_1*group_2*group_3 == 0 then return false end
+ if group_1 == 666 or (900 <= group_1 and group_1 <= 999) then return false end
+ return true
+end
+
+-- The default function if there is no validator.
+local function default()
+ return true
+end
+
action = function(host, port)
+ -- a list of popular patterns with their validators.
+ local BUILT_IN_PATTERNS = {
+ ['email'] = {'[A-Za-z0-9%.%%%+%-]+@[A-Za-z0-9%.%%%+%-]+%.%w%w%w?%w?'},
+ ['phone'] = {'%f[%d]%d%d%d%-%d%d%d%d%f[^%d]','%f[%d%(]%(%d%d%d%)%s%d%d%d%-%d%d%d%f[^%d]','%f[%d%+]%+%-%d%d%d%-%d%d%d%-%d%d%d%d%f[^%d]','%d%d%d%-%d%d%d%-%d%d%d%d%f[^%d]'},
+ ['mastercard']= {'%f[%d]5%d%d%d%s?%-?%d%d%d%d%s?%-?%d%d%d%d%s?%-?%d%d%d%d%f[^%d]', ['validate'] = luhn},
+ ['visa'] = {'%f[%d]4%d%d%d%s?%-?%d%d%d%d%s?%-?%d%d%d%d%s?%-?%d%d%d%d%f[^%d]', ['validate'] = luhn},
+ ['discover']={'%f[%d]6011%s?%-?%d%d%d%d%s?%-?%d%d%d%d%s?%-?%d%d%d%d%f[^%d]', ['validate'] = luhn},
+ ['amex'] ={'%f[%d]3%d%d%d%s?%-?%d%d%d%d%d%d%s?%-?%d%d%d%d%d%f[^%d]', ['validate'] = luhn},
+ ['ssn'] = {'%f[%d]%d%d%d%-%d%d%-%d%d%d%d%f[^%d]', ['validate'] = ssn},
+ ['ip']={'%f[%d]%d%d?%d?%.%d%d?%d?%.%d%d?%d%.%d%d?%d?%f[^%d]', ['validate'] = ip},
+ }
-- read script specific arguments
- local match = stdnse.get_script_args("http-grep.match")
- local break_on_match = stdnse.get_script_args("http-grep.breakonmatch")
-
- if ( not(match) ) then
- return stdnse.format_output(true, "ERROR: Argument http-grep.match was not set")
- end
+ local match = stdnse.get_script_args(SCRIPT_NAME .. ".match")
+ local break_on_match = stdnse.get_script_args(SCRIPT_NAME .. ".breakonmatch")
+ local builtins = stdnse.get_script_args(SCRIPT_NAME .. ".builtins")
+ local to_be_searched = {}
local crawler = httpspider.Crawler:new(host, port, nil, { scriptname = SCRIPT_NAME } )
- local results = {}
+ local results = stdnse.output_table()
+ local all_match = {} -- a table that stores all matches. used to eliminate duplicates.
+
+ -- check if builtin arugment is a table or a single value
+ if builtins and builtins == 1 then
+ for name, patterns in pairs(BUILT_IN_PATTERNS) do
+ to_be_searched[name] = {}
+ for _, pattern in ipairs(patterns) do
+ table.insert(to_be_searched[name], pattern)
+ end
+ end
+ elseif builtins and type(builtins) ~= 'table' then
+ if BUILT_IN_PATTERNS[builtins] ~= nil then
+ to_be_searched[builtins] = {}
+ for _, pattern in ipairs(BUILT_IN_PATTERNS[builtins]) do
+ table.insert(to_be_searched[builtins], pattern)
+ end
+ end
+ elseif builtins and type(builtins) == 'table' then
+ for _, builtin in ipairs(builtins) do
+ if BUILT_IN_PATTERNS[builtin] ~= nil then
+ to_be_searched[builtin] = {}
+ for _, pattern in ipairs(BUILT_IN_PATTERNS[builtin]) do
+ table.insert(to_be_searched[builtin], pattern)
+ end
+ end
+ end
+ end
+
+ -- check if match arugment is a table or a single value
+ if match and type(match) ~= table then
+ to_be_searched['User Pattern 1'] = {}
+ table.insert(to_be_searched['User Pattern 1'], match)
+ elseif type(match) == 'table' then
+ for i, pattern in pairs(match) do
+ to_be_searched['User Pattern ' .. tostring(i)] = {}
+ table.insert(to_be_searched['User Pattern ' .. tostring(i)], pattern)
+ end
+ end
+
+ -- if nothing is specified then email and ip are checked.
+ if not next(to_be_searched) then
+ to_be_searched['email'] = {}
+ to_be_searched['ip'] = {}
+ table.insert(to_be_searched['email'], BUILT_IN_PATTERNS["email"][1])
+ table.insert(to_be_searched['ip'], BUILT_IN_PATTERNS["ip"][1])
+ end
-- set timeout to 10 seconds
crawler:set_timeout(10000)
@@ -86,26 +264,44 @@ action = function(host, port)
break
end
end
-
- local matches = {}
- local body = r.response.body
- -- try to match the url and body
- if body and ( body:match( match ) or tostring(r.url):match(match) ) then
- local count = select(2, body:gsub(match, match))
- for match in body:gmatch(match) do
- table.insert(matches, "+ " .. shortenMatch(match))
+ local count = 0 -- pattern matches per url
+ local pattern_count = 0 -- number of matches for particual pattern type say 'email'
+ local matches = {} -- a table that stores matches for all pattern types
+ local pattern_type = {} -- a table that resets for every pattern type
+ for pattern_name, pattern_table in pairs(to_be_searched) do
+ pattern_type = {}
+ pattern_count = 0
+ for _, pattern in ipairs(pattern_table) do
+ local body = r.response.body
+ -- try to match the url and body
+ if body and ( body:match( pattern ) or tostring(r.url):match(pattern) ) then
+ pattern_count = select(2, body:gsub(pattern, ""))
+ count = count + pattern_count
+ for match in body:gmatch(pattern) do
+ local validate = BUILT_IN_PATTERNS[pattern_name]and BUILT_IN_PATTERNS[pattern_name]['validate'] or default
+ if validate(match) and not stdnse.contains(all_match, match) then
+ table.insert(pattern_type, "+ " .. shortenMatch(match))
+ table.insert(all_match, match)
+ else
+ count = count - 1
+ pattern_count = pattern_count - 1
+ end
+ end
+ end
end
-
- matches.name = ("(%d) %s"):format(count,tostring(r.url))
- table.insert(results, matches)
-
- -- should we continue to search for matches?
- if ( break_on_match ) then
- crawler:stop()
- break
+ if pattern_count > 0 then
+ matches[("(%d) %s"):format(pattern_count, pattern_name)] = pattern_type
end
end
+ if count > 0 then
+ results[("(%d) %s"):format(count,tostring(r.url))] = matches
+ end
+ -- should we continue to search for matches?
+ if break_on_match and pattern_count > 0 then
+ crawler:stop()
+ break
+ end
end
- table.sort(results, function(a,b) return a.name>b.name end)
- return stdnse.format_output(true, results)
+ if #results > 0 then return results end
end
+