From 4b5ecbccd137b334a92bf0d13aba60fa4b5e002d Mon Sep 17 00:00:00 2001 From: Daniel Black Date: Sun, 22 Sep 2013 13:20:17 +1000 Subject: [PATCH] ENH: debuggex URLs with fail2ban-regex --- ChangeLog | 1 + fail2ban-regex | 45 +++++++++++++++++++++++++++++++----- server/filter.py | 4 ++-- testcases/samplestestcase.py | 2 +- 4 files changed, 43 insertions(+), 9 deletions(-) diff --git a/ChangeLog b/ChangeLog index de8607af..5c5ffc99 100644 --- a/ChangeLog +++ b/ChangeLog @@ -73,6 +73,7 @@ ver. 0.8.11 (2013/XX/XXX) - loves-unittests linux-pam before version 0.99.2.0 (2005) * filter.d/gssftpd - anchored regex at start * filter.d/mysqld-auth.conf - mysql can use syslog + * fail2ban-regex - now generated http://www.debuggex.com urls for debugging Daniel Black & Georgiy Mernov & ftoppi & Мернов Георгий * filter.d/exim.conf -- regex hardening and extra failure examples in sample logs diff --git a/fail2ban-regex b/fail2ban-regex index 4c5a0a11..6ac716ab 100755 --- a/fail2ban-regex +++ b/fail2ban-regex @@ -30,7 +30,7 @@ __author__ = "Cyril Jaquier, Yaroslav Halchenko" __copyright__ = "Copyright (c) 2004-2008 Cyril Jaquier, 2012-2013 Yaroslav Halchenko" __license__ = "GPL" -import getopt, sys, time, logging, os +import getopt, sys, time, logging, os, urllib, re # Inserts our own modules path first in the list # fix for bug #343821 @@ -51,6 +51,12 @@ from testcases.utils import FormatterWithTraceBack # Gets the instance of the logger. logSys = logging.getLogger("fail2ban") +def debuggexURL(sample, regex): + q = urllib.urlencode({ 're': re.sub('', '(?&.ipv4)', regex), + 'str': sample, + 'flavor': 'python' }) + return 'http://www.debuggex.com/?' + q + def shortstr(s, l=53): """Return shortened string """ @@ -94,6 +100,8 @@ IGNOREREGEX: help="Log level for the Fail2Ban logger to use"), Option("-v", "--verbose", action='store_true', help="Be verbose in output"), + Option("-d", "--debuggex", action='store_true', + help="Produce debuggex.com urls for debugging there"), Option("--print-all-missed", action='store_true', help="Either to print all missed lines"), Option("--print-all-ignored", action='store_true', @@ -141,7 +149,9 @@ class LineStats(object): def __init__(self): self.tested = self.matched = 0 self.missed_lines = [] + self.missed_lines_timeextracted = [] self.ignored_lines = [] + self.ignored_lines_timeextracted = [] def __str__(self): return "%(tested)d lines, %(ignored)d ignored, %(matched)d matched, %(missed)d missed" % self @@ -165,6 +175,7 @@ class Fail2banRegex(object): def __init__(self, opts): self._verbose = opts.verbose + self._debuggex = opts.debuggex self._print_all_missed = opts.print_all_missed self._print_all_ignored = opts.print_all_ignored @@ -221,7 +232,7 @@ class Fail2banRegex(object): def testRegex(self, line): try: - ret = self._filter.processLine(line, checkAllRegex=True) + line, ret = self._filter.processLine(line, checkAllRegex=True) for match in ret: # Append True/False flag depending if line was matched by # more than one regex @@ -233,9 +244,9 @@ class Fail2banRegex(object): print e return False except IndexError: - print "Sorry, but no found in regex" + print "Sorry, but no found in regex" return False - return len(ret) > 0 + return line, ret def process(self, test_lines): @@ -245,27 +256,49 @@ class Fail2banRegex(object): # skip comment and empty lines continue is_ignored = fail2banRegex.testIgnoreRegex(line) + line_datetimestripped, ret = fail2banRegex.testRegex(line) + if is_ignored: self._line_stats.ignored_lines.append(line) + self._line_stats.ignored_lines_timeextracted.append(line_datetimestripped) - if fail2banRegex.testRegex(line): + if len(ret) > 0: assert(not is_ignored) self._line_stats.matched += 1 else: if not is_ignored: self._line_stats.missed_lines.append(line) + self._line_stats.missed_lines_timeextracted.append(line_datetimestripped) self._line_stats.tested += 1 if line_no % 10 == 0: self._filter.dateDetector.sortTemplate() + + def printLines(self, ltype): lstats = self._line_stats assert(len(lstats.missed_lines) == lstats.tested - (lstats.matched + lstats.ignored)) l = lstats[ltype + '_lines'] if len(l): header = "%s line(s):" % (ltype.capitalize(),) - if len(l) < 20 or getattr(self, '_print_all_' + ltype): + if self._debuggex: + if ltype == 'missed': + regexlist = self._failregex + else: + regexlist = self._ignoreregex + l = lstats[ltype + '_lines_timeextracted'] + lines = len(l)*len(regexlist) + if lines > 20 or getattr(self, '_print_all_' + ltype): + ans = [[]] + for arg in [l, regexlist]: + ans = [ x + [y] for x in ans for y in arg ] + b = map(lambda a: a[0] + ' | ' + a[1].getFailRegex() + ' | ' + debuggexURL(a[0], a[1].getFailRegex()), ans) + pprint_list([x.rstrip() for x in b], header) + else: + print "%s: too many to print. Use --print-all-%s " \ + "to print all %d lines" % (header, ltype, lines) + elif len(l) < 20 or getattr(self, '_print_all_' + ltype): pprint_list([x.rstrip() for x in l], header) else: print "%s: too many to print. Use --print-all-%s " \ diff --git a/server/filter.py b/server/filter.py index 2f88cbea..80433c01 100644 --- a/server/filter.py +++ b/server/filter.py @@ -306,12 +306,12 @@ class Filter(JailThread): else: timeLine = l logLine = l - return self.findFailure(timeLine, logLine, returnRawHost, checkAllRegex) + return logLine, self.findFailure(timeLine, logLine, returnRawHost, checkAllRegex) def processLineAndAdd(self, line): """Processes the line for failures and populates failManager """ - for element in self.processLine(line): + for element in self.processLine(line)[1]: failregex = element[0] ip = element[1] unixTime = element[2] diff --git a/testcases/samplestestcase.py b/testcases/samplestestcase.py index c7ff0b9a..0b4285ba 100644 --- a/testcases/samplestestcase.py +++ b/testcases/samplestestcase.py @@ -95,7 +95,7 @@ def testSampleRegexsFactory(name): faildata = {} ret = self.filter.processLine( - line, returnRawHost=True, checkAllRegex=True) + line, returnRawHost=True, checkAllRegex=True)[1] if not ret: # Check line is flagged as none match self.assertFalse(faildata.get('match', True),