diff --git a/DEVELOP b/DEVELOP index 2d7a043b..b0f7b47e 100644 --- a/DEVELOP +++ b/DEVELOP @@ -39,8 +39,9 @@ Filters * Include sample logs with 1.2.3.4 used for IP addresses and example.com/example.org used for DNS names -* Ensure ./fail2ban-regex testcases/files/logs/{samplelog} config/filter.d/{filter}.conf - has matches for EVERY regex +* Ensure sample log is provided in testcases/files/logs/ with same name as the + filter. Each log line should include match meta data for time & IP above + every line (see other sample log files for examples) * Ensure regexs start with a ^ and are restrictive as possible. E.g. not .* if \d+ is sufficient * Use the functionality of regexs http://docs.python.org/2/library/re.html diff --git a/fail2ban-regex b/fail2ban-regex index 3326e328..e19b1bc9 100755 --- a/fail2ban-regex +++ b/fail2ban-regex @@ -46,7 +46,6 @@ from client.configparserinc import SafeConfigParserWithIncludes from ConfigParser import NoOptionError, NoSectionError, MissingSectionHeaderError from server.filter import Filter from server.failregex import RegexException -from server.datedetector import DateDetector from testcases.utils import FormatterWithTraceBack # Gets the instance of the logger. @@ -130,7 +129,7 @@ class RegexStat(object): return self._failregex def appendIP(self, value): - self._ipList.extend(value) + self._ipList.append(value) def getIPList(self): return self._ipList @@ -173,8 +172,6 @@ class Fail2banRegex(object): self._ignoreregex = list() self._failregex = list() self._line_stats = LineStats() - self._dateDetector = DateDetector() - self._dateDetector.addDefaultTemplate() def readRegex(self, value, regextype): @@ -204,53 +201,41 @@ class Fail2banRegex(object): regex_values = [RegexStat(value)] setattr(self, "_" + regex, regex_values) + for regex in regex_values: + getattr( + self._filter, + 'add%sRegex' % regextype.title())(regex.getFailRegex()) return True def testIgnoreRegex(self, line): found = False - for regex in self._ignoreregex: - try: - self._filter.addIgnoreRegex(regex.getFailRegex()) - try: - ret = self._filter.ignoreLine(line) - if ret: - found = True - regex.inc() - except RegexException, e: - print e - return False - finally: - self._filter.delIgnoreRegex(0) + try: + ret = self._filter.ignoreLine(line) + if ret is not None: + found = True + regex = self._ignoreregex[ret].inc() + except RegexException, e: + print e + return False return found def testRegex(self, line): - found = False - for regex in self._ignoreregex: - self._filter.addIgnoreRegex(regex.getFailRegex()) - for regex in self._failregex: - try: - self._filter.addFailRegex(regex.getFailRegex()) - try: - ret = self._filter.processLine(line) - if len(ret): - if found == True: - ret[0].append(True) - else: - found = True - ret[0].append(False) - regex.inc() - regex.appendIP(ret) - except RegexException, e: - print e - return False - except IndexError: - print "Sorry, but no found in regex" - return False - finally: - self._filter.delFailRegex(0) - for regex in self._ignoreregex: - self._filter.delIgnoreRegex(0) - return found + try: + ret = self._filter.processLine(line, checkAllRegex=True) + for match in ret: + # Append True/False flag depending if line was matched by + # more than one regex + match.append(len(ret)>1) + regex = self._failregex[match[0]] + regex.inc() + regex.appendIP(match) + except RegexException, e: + print e + return False + except IndexError: + print "Sorry, but no found in regex" + return False + return len(ret) > 0 def process(self, test_lines): @@ -259,9 +244,6 @@ class Fail2banRegex(object): if line.startswith('#') or not line.strip(): # skip comment and empty lines continue - - self._dateDetector.matchTime(line) - is_ignored = fail2banRegex.testIgnoreRegex(line) if is_ignored: self._line_stats.ignored_lines.append(line) @@ -302,10 +284,13 @@ class Fail2banRegex(object): if self._verbose and len(failregex.getIPList()): for ip in failregex.getIPList(): - timeTuple = time.localtime(ip[1]) + timeTuple = time.localtime(ip[2]) timeString = time.strftime("%a %b %d %H:%M:%S %Y", timeTuple) - out.append(" %s %s%s" % ( - ip[0], timeString, ip[2] and " (already matched)" or "")) + out.append( + " %s %s%s" % ( + ip[1], + timeString, + ip[3] and " (multiple regex matched)" or "")) print "\n%s: %d total" % (title, total) pprint_list(out, " #) [# of hits] regular expression") @@ -318,7 +303,7 @@ class Fail2banRegex(object): print "\nDate template hits:" out = [] - for template in self._dateDetector.getTemplates(): + for template in self._filter.dateDetector.getTemplates(): if self._verbose or template.getHits(): out.append("[%d] %s" % (template.getHits(), template.getName())) pprint_list(out, "[# of hits] date format") diff --git a/server/datedetector.py b/server/datedetector.py index 0c8b4df2..0ed9e00a 100644 --- a/server/datedetector.py +++ b/server/datedetector.py @@ -174,6 +174,7 @@ class DateDetector: match = template.matchDate(line) if not match is None: logSys.debug("Matched time template %s" % template.getName()) + template.incHits() return match return None finally: diff --git a/server/datetemplate.py b/server/datetemplate.py index 8c49aa15..86eeee8e 100644 --- a/server/datetemplate.py +++ b/server/datetemplate.py @@ -59,11 +59,12 @@ class DateTemplate: def getHits(self): return self.__hits + + def incHits(self): + self.__hits += 1 def matchDate(self, line): dateMatch = self.__cRegex.search(line) - if not dateMatch is None: - self.__hits += 1 return dateMatch def getDate(self, line): diff --git a/server/filter.py b/server/filter.py index 54efb619..2f88cbea 100644 --- a/server/filter.py +++ b/server/filter.py @@ -284,7 +284,7 @@ class Filter(JailThread): return False - def processLine(self, line, returnRawHost=False): + def processLine(self, line, returnRawHost=False, checkAllRegex=False): """Split the time portion from log msg and return findFailures on them """ try: @@ -306,14 +306,15 @@ class Filter(JailThread): else: timeLine = l logLine = l - return self.findFailure(timeLine, logLine, returnRawHost) + return self.findFailure(timeLine, logLine, returnRawHost, checkAllRegex) def processLineAndAdd(self, line): """Processes the line for failures and populates failManager """ for element in self.processLine(line): - ip = element[0] - unixTime = element[1] + failregex = element[0] + ip = element[1] + unixTime = element[2] logSys.debug("Processing line with time:%s and ip:%s" % (unixTime, ip)) if unixTime < MyTime.time() - self.getFindTime(): @@ -335,11 +336,11 @@ class Filter(JailThread): # @return: a boolean def ignoreLine(self, line): - for ignoreRegex in self.__ignoreRegex: + for ignoreRegexIndex, ignoreRegex in enumerate(self.__ignoreRegex): ignoreRegex.search(line) if ignoreRegex.hasMatched(): - return True - return False + return ignoreRegexIndex + return None ## # Finds the failure in a line given split into time and log parts. @@ -348,21 +349,22 @@ class Filter(JailThread): # to find the logging time. # @return a dict with IP and timestamp. - def findFailure(self, timeLine, logLine, returnRawHost=False): + def findFailure(self, timeLine, logLine, + returnRawHost=False, checkAllRegex=False): logSys.log(5, "Date: %r, message: %r", timeLine, logLine) failList = list() # Checks if we must ignore this line. - if self.ignoreLine(logLine): + if self.ignoreLine(logLine) is not None: # The ignoreregex matched. Return. logSys.log(7, "Matched ignoreregex and was ignored") return failList + date = self.dateDetector.getUnixTime(timeLine) # Iterates over all the regular expressions. - for failRegex in self.__failRegex: + for failRegexIndex, failRegex in enumerate(self.__failRegex): failRegex.search(logLine) if failRegex.hasMatched(): # The failregex matched. logSys.log(7, "Matched %s", failRegex) - date = self.dateDetector.getUnixTime(timeLine) if date is None: logSys.debug("Found a match for %r but no valid date/time " "found for %r. Please file a detailed issue on" @@ -373,14 +375,16 @@ class Filter(JailThread): try: host = failRegex.getHost() if returnRawHost: - failList.append([host, date]) - break - ipMatch = DNSUtils.textToIp(host, self.__useDns) - if ipMatch: - for ip in ipMatch: - failList.append([ip, date]) - # We matched a regex, it is enough to stop. - break + failList.append([failRegexIndex, host, date]) + if not checkAllRegex: + break + else: + ipMatch = DNSUtils.textToIp(host, self.__useDns) + if ipMatch: + for ip in ipMatch: + failList.append([failRegexIndex, ip, date]) + if not checkAllRegex: + break except RegexException, e: # pragma: no cover - unsure if reachable logSys.error(e) return failList diff --git a/testcases/samplestestcase.py b/testcases/samplestestcase.py index 6b0e9ae0..a52873b9 100644 --- a/testcases/samplestestcase.py +++ b/testcases/samplestestcase.py @@ -82,6 +82,7 @@ def testSampleRegexsFactory(name): logFile = fileinput.FileInput( os.path.join(TEST_FILES_DIR, "logs", name)) + regexsUsed = set() for line in logFile: jsonREMatch = re.match("^# ?failJSON:(.+)$", line) if jsonREMatch: @@ -96,7 +97,8 @@ def testSampleRegexsFactory(name): else: faildata = {} - ret = self.filter.processLine(line, returnRawHost=True) + ret = self.filter.processLine( + line, returnRawHost=True, checkAllRegex=True) if not ret: # Check line is flagged as none match self.assertFalse(faildata.get('match', True), @@ -107,15 +109,28 @@ def testSampleRegexsFactory(name): self.assertTrue(faildata.get('match', False), "Line matched when shouldn't have: %s:%i %r" % (logFile.filename(), logFile.filelineno(), line)) - self.assertEqual(len(ret), 1) + self.assertEqual(len(ret), 1, "Multiple regexs matched") # Verify timestamp and host as expected - host, time = ret[0] + failregex, host, time = ret[0] self.assertEqual(host, faildata.get("host", None)) self.assertEqual( datetime.datetime.fromtimestamp(time), datetime.datetime.strptime( faildata.get("time", None), "%Y-%m-%dT%H:%M:%S")) + regexsUsed.add(failregex) + + # TODO: Remove exception handling once all regexs have samples + for failRegexIndex, failRegex in enumerate(self.filter.getFailRegex()): + try: + self.assertTrue( + failRegexIndex in regexsUsed, + "Regex for filter '%s' has no samples: %i: %r" % + (name, failRegexIndex, failRegex)) + except AssertionError: + print "I: Regex for filter '%s' has no samples: %i: %r" % ( + name, failRegexIndex, failRegex) + return testFilter for filter_ in os.listdir(os.path.join(CONFIG_DIR, "filter.d")):