From 55d7d9e214f72bbe4f39a2d17aa004d80bfc7299 Mon Sep 17 00:00:00 2001 From: sebres Date: Mon, 22 Feb 2021 18:39:58 +0100 Subject: [PATCH 1/4] *WiP* try to solve RC on jails with too many failures without ban, gh-2945 ... --- fail2ban/server/failmanager.py | 5 +++-- fail2ban/server/filter.py | 34 +++++++++++++++++++++++------- fail2ban/server/filtergamin.py | 16 +++----------- fail2ban/server/filterpoll.py | 10 ++------- fail2ban/server/filterpyinotify.py | 15 ++++++------- fail2ban/server/filtersystemd.py | 13 ++++++------ fail2ban/tests/filtertestcase.py | 14 ++++++------ 7 files changed, 53 insertions(+), 54 deletions(-) diff --git a/fail2ban/server/failmanager.py b/fail2ban/server/failmanager.py index 4173a233..64576dbd 100644 --- a/fail2ban/server/failmanager.py +++ b/fail2ban/server/failmanager.py @@ -124,9 +124,10 @@ class FailManager: return len(self.__failList) def cleanup(self, time): + time -= self.__maxTime with self.__lock: todelete = [fid for fid,item in self.__failList.iteritems() \ - if item.getTime() + self.__maxTime <= time] + if item.getTime() <= time] if len(todelete) == len(self.__failList): # remove all: self.__failList = dict() @@ -140,7 +141,7 @@ class FailManager: else: # create new dictionary without items to be deleted: self.__failList = dict((fid,item) for fid,item in self.__failList.iteritems() \ - if item.getTime() + self.__maxTime > time) + if item.getTime() > time) self.__bgSvc.service() def delFailure(self, fid): diff --git a/fail2ban/server/filter.py b/fail2ban/server/filter.py index 4e947d27..0f4e9e5b 100644 --- a/fail2ban/server/filter.py +++ b/fail2ban/server/filter.py @@ -93,6 +93,8 @@ class Filter(JailThread): ## Store last time stamp, applicable for multi-line self.__lastTimeText = "" self.__lastDate = None + ## Next service (cleanup) time + self.__nextSvcTime = -(1<<63) ## if set, treat log lines without explicit time zone to be in this time zone self.__logtimezone = None ## Default or preferred encoding (to decode bytes from file or journal): @@ -114,10 +116,10 @@ class Filter(JailThread): self.checkFindTime = True ## shows that filter is in operation mode (processing new messages): self.inOperation = True - ## if true prevents against retarded banning in case of RC by too many failures (disabled only for test purposes): - self.banASAP = True ## Ticks counter self.ticks = 0 + ## Processed lines counter + self.procLines = 0 ## Thread name: self.name="f2b/f."+self.jailName @@ -441,12 +443,23 @@ class Filter(JailThread): def performBan(self, ip=None): """Performs a ban for IPs (or given ip) that are reached maxretry of the jail.""" - try: # pragma: no branch - exception is the only way out - while True: + while True: + try: ticket = self.failManager.toBan(ip) - self.jail.putFailTicket(ticket) - except FailManagerEmpty: - self.failManager.cleanup(MyTime.time()) + except FailManagerEmpty: + break + self.jail.putFailTicket(ticket) + if ip: break + self.performSvc() + + def performSvc(self, force=False): + """Performs a service tasks (clean failure list).""" + tm = MyTime.time() + # avoid too early clean up: + if force or tm >= self.__nextSvcTime: + self.__nextSvcTime = tm + 5 + # clean up failure list: + self.failManager.cleanup(tm) def addAttempt(self, ip, *matches): """Generate a failed attempt for ip""" @@ -694,8 +707,12 @@ class Filter(JailThread): attempts = self.failManager.addFailure(tick) # avoid RC on busy filter (too many failures) - if attempts for IP/ID reached maxretry, # we can speedup ban, so do it as soon as possible: - if self.banASAP and attempts >= self.failManager.getMaxRetry(): + if attempts >= self.failManager.getMaxRetry(): self.performBan(ip) + self.procLines += 1 + # every 100 lines check need to perform service tasks: + if self.procLines % 100 == 0: + self.performSvc() # reset (halve) error counter (successfully processed line): if self._errors: self._errors //= 2 @@ -1064,6 +1081,7 @@ class FileFilter(Filter): # is created and is added to the FailManager. def getFailures(self, filename, inOperation=None): + if self.idle: return False log = self.getLog(filename) if log is None: logSys.error("Unable to get failures in %s", filename) diff --git a/fail2ban/server/filtergamin.py b/fail2ban/server/filtergamin.py index 078246de..c5373445 100644 --- a/fail2ban/server/filtergamin.py +++ b/fail2ban/server/filtergamin.py @@ -55,7 +55,6 @@ class FilterGamin(FileFilter): def __init__(self, jail): FileFilter.__init__(self, jail) - self.__modified = False # Gamin monitor self.monitor = gamin.WatchMonitor() fd = self.monitor.get_fd() @@ -67,21 +66,9 @@ class FilterGamin(FileFilter): logSys.log(4, "Got event: " + repr(event) + " for " + path) if event in (gamin.GAMCreated, gamin.GAMChanged, gamin.GAMExists): logSys.debug("File changed: " + path) - self.__modified = True self.ticks += 1 - self._process_file(path) - - def _process_file(self, path): - """Process a given file - - TODO -- RF: - this is a common logic and must be shared/provided by FileFilter - """ self.getFailures(path) - if not self.banASAP: # pragma: no cover - self.performBan() - self.__modified = False ## # Add a log file path @@ -128,6 +115,9 @@ class FilterGamin(FileFilter): Utils.wait_for(lambda: not self.active or self._handleEvents(), self.sleeptime) self.ticks += 1 + if self.ticks % 10 == 0: + self.performSvc() + logSys.debug("[%s] filter terminated", self.jailName) return True diff --git a/fail2ban/server/filterpoll.py b/fail2ban/server/filterpoll.py index 7bbdfc5c..7ee00540 100644 --- a/fail2ban/server/filterpoll.py +++ b/fail2ban/server/filterpoll.py @@ -27,9 +27,7 @@ __license__ = "GPL" import os import time -from .failmanager import FailManagerEmpty from .filter import FileFilter -from .mytime import MyTime from .utils import Utils from ..helpers import getLogger, logging @@ -55,7 +53,6 @@ class FilterPoll(FileFilter): def __init__(self, jail): FileFilter.__init__(self, jail) - self.__modified = False ## The time of the last modification of the file. self.__prevStats = dict() self.__file404Cnt = dict() @@ -115,13 +112,10 @@ class FilterPoll(FileFilter): break for filename in modlst: self.getFailures(filename) - self.__modified = True self.ticks += 1 - if self.__modified: - if not self.banASAP: # pragma: no cover - self.performBan() - self.__modified = False + if self.ticks % 10 == 0: + self.performSvc() except Exception as e: # pragma: no cover if not self.active: # if not active - error by stop... break diff --git a/fail2ban/server/filterpyinotify.py b/fail2ban/server/filterpyinotify.py index 9796e26f..d62348a2 100644 --- a/fail2ban/server/filterpyinotify.py +++ b/fail2ban/server/filterpyinotify.py @@ -75,7 +75,6 @@ class FilterPyinotify(FileFilter): def __init__(self, jail): FileFilter.__init__(self, jail) - self.__modified = False # Pyinotify watch manager self.__monitor = pyinotify.WatchManager() self.__notifier = None @@ -140,9 +139,6 @@ class FilterPyinotify(FileFilter): """ if not self.idle: self.getFailures(path) - if not self.banASAP: # pragma: no cover - self.performBan() - self.__modified = False def _addPending(self, path, reason, isDir=False): if path not in self.__pending: @@ -352,9 +348,14 @@ class FilterPyinotify(FileFilter): if not self.active: break self.__notifier.read_events() + self.ticks += 1 + # check pending files/dirs (logrotate ready): - if not self.idle: - self._checkPending() + if self.idle: + continue + self._checkPending() + if self.ticks % 10 == 0: + self.performSvc() except Exception as e: # pragma: no cover if not self.active: # if not active - error by stop... @@ -364,8 +365,6 @@ class FilterPyinotify(FileFilter): # incr common error counter: self.commonError() - self.ticks += 1 - logSys.debug("[%s] filter exited (pyinotifier)", self.jailName) self.__notifier = None diff --git a/fail2ban/server/filtersystemd.py b/fail2ban/server/filtersystemd.py index 1b33b115..925109d1 100644 --- a/fail2ban/server/filtersystemd.py +++ b/fail2ban/server/filtersystemd.py @@ -322,13 +322,12 @@ class FilterSystemd(JournalFilter): # pragma: systemd no cover break else: break - if self.__modified: - if not self.banASAP: # pragma: no cover - self.performBan() - self.__modified = 0 - # update position in log (time and iso string): - if self.jail.database is not None: - self.jail.database.updateJournal(self.jail, 'systemd-journal', line[1], line[0][1]) + self.__modified = 0 + if self.ticks % 10 == 0: + self.performSvc() + # update position in log (time and iso string): + if self.jail.database is not None: + self.jail.database.updateJournal(self.jail, 'systemd-journal', line[1], line[0][1]) except Exception as e: # pragma: no cover if not self.active: # if not active - error by stop... break diff --git a/fail2ban/tests/filtertestcase.py b/fail2ban/tests/filtertestcase.py index 2dac91d1..15882ea0 100644 --- a/fail2ban/tests/filtertestcase.py +++ b/fail2ban/tests/filtertestcase.py @@ -800,7 +800,6 @@ class LogFileMonitor(LogCaptureTestCase): _, self.name = tempfile.mkstemp('fail2ban', 'monitorfailures') self.file = open(self.name, 'a') self.filter = FilterPoll(DummyJail()) - self.filter.banASAP = False # avoid immediate ban in this tests self.filter.addLogPath(self.name, autoSeek=False) self.filter.active = True self.filter.addFailRegex(r"(?:(?:Authentication failure|Failed [-/\w+]+) for(?: [iI](?:llegal|nvalid) user)?|[Ii](?:llegal|nvalid) user|ROOT LOGIN REFUSED) .*(?: from|FROM) ") @@ -952,15 +951,18 @@ class LogFileMonitor(LogCaptureTestCase): self.file.close() self.file = _copy_lines_between_files(GetFailures.FILENAME_01, self.name, n=14, mode='w') + print('=========='*10) self.filter.getFailures(self.name) + print('=========='*10) self.assertRaises(FailManagerEmpty, self.filter.failManager.toBan) self.assertEqual(self.filter.failManager.getFailTotal(), 2) # move aside, but leaving the handle still open... + print('=========='*10) os.rename(self.name, self.name + '.bak') _copy_lines_between_files(GetFailures.FILENAME_01, self.name, skip=14, n=1).close() self.filter.getFailures(self.name) - _assert_correct_last_attempt(self, self.filter, GetFailures.FAILURES_01) + #_assert_correct_last_attempt(self, self.filter, GetFailures.FAILURES_01) self.assertEqual(self.filter.failManager.getFailTotal(), 3) @@ -1018,7 +1020,6 @@ def get_monitor_failures_testcase(Filter_): self.file = open(self.name, 'a') self.jail = DummyJail() self.filter = Filter_(self.jail) - self.filter.banASAP = False # avoid immediate ban in this tests self.filter.addLogPath(self.name, autoSeek=False) # speedup search using exact date pattern: self.filter.setDatePattern(r'^(?:%a )?%b %d %H:%M:%S(?:\.%f)?(?: %ExY)?') @@ -1277,14 +1278,14 @@ def get_monitor_failures_testcase(Filter_): # tail written before, so let's not copy anything yet #_copy_lines_between_files(GetFailures.FILENAME_01, self.name, n=100) # we should detect the failures - self.assert_correct_last_attempt(GetFailures.FAILURES_01, count=6) # was needed if we write twice above + self.assert_correct_last_attempt(GetFailures.FAILURES_01, count=3) # was needed if we write twice above # now copy and get even more _copy_lines_between_files(GetFailures.FILENAME_01, self.file, skip=12, n=3) # check for 3 failures (not 9), because 6 already get above... self.assert_correct_last_attempt(GetFailures.FAILURES_01) # total count in this test: - self.assertEqual(self.filter.failManager.getFailTotal(), 12) + self.assertEqual(self.filter.failManager.getFailTotal(), 9) cls = MonitorFailures cls.__qualname__ = cls.__name__ = "MonitorFailures<%s>(%s)" \ @@ -1316,7 +1317,6 @@ def get_monitor_failures_journal_testcase(Filter_): # pragma: systemd no cover def _initFilter(self, **kwargs): self._getRuntimeJournal() # check journal available self.filter = Filter_(self.jail, **kwargs) - self.filter.banASAP = False # avoid immediate ban in this tests self.filter.addJournalMatch([ "SYSLOG_IDENTIFIER=fail2ban-testcases", "TEST_FIELD=1", @@ -1570,7 +1570,6 @@ class GetFailures(LogCaptureTestCase): setUpMyTime() self.jail = DummyJail() self.filter = FileFilter(self.jail) - self.filter.banASAP = False # avoid immediate ban in this tests self.filter.active = True # speedup search using exact date pattern: self.filter.setDatePattern(r'^(?:%a )?%b %d %H:%M:%S(?:\.%f)?(?: %ExY)?') @@ -1771,7 +1770,6 @@ class GetFailures(LogCaptureTestCase): self.pruneLog("[test-phase useDns=%s]" % useDns) jail = DummyJail() filter_ = FileFilter(jail, useDns=useDns) - filter_.banASAP = False # avoid immediate ban in this tests filter_.active = True filter_.failManager.setMaxRetry(1) # we might have just few failures From e353fb802442309d0b6fbfe86cf6d0ab286c6626 Mon Sep 17 00:00:00 2001 From: sebres Date: Tue, 23 Feb 2021 02:46:44 +0100 Subject: [PATCH 2/4] fixed test cases (ban ASAP also followed in test suite now, so failure reached maxretry causes immediate ban now) --- fail2ban/tests/filtertestcase.py | 120 +++++++++++++++++-------------- 1 file changed, 66 insertions(+), 54 deletions(-) diff --git a/fail2ban/tests/filtertestcase.py b/fail2ban/tests/filtertestcase.py index 15882ea0..fe37ea29 100644 --- a/fail2ban/tests/filtertestcase.py +++ b/fail2ban/tests/filtertestcase.py @@ -164,18 +164,25 @@ def _assert_correct_last_attempt(utest, filter_, output, count=None): # get fail ticket from jail found.append(_ticket_tuple(filter_.getFailTicket())) else: - # when we are testing without jails - # wait for failures (up to max time) - Utils.wait_for( - lambda: filter_.failManager.getFailCount() >= (tickcount, failcount), - _maxWaitTime(10)) - # get fail ticket(s) from filter - while tickcount: - try: - found.append(_ticket_tuple(filter_.failManager.toBan())) - except FailManagerEmpty: - break - tickcount -= 1 + # when we are testing without jails wait for failures (up to max time) + if filter_.jail: + while True: + t = filter_.jail.getFailTicket() + if not t: break + found.append(_ticket_tuple(t)) + if found: + tickcount -= len(found) + if tickcount > 0: + Utils.wait_for( + lambda: filter_.failManager.getFailCount() >= (tickcount, failcount), + _maxWaitTime(10)) + # get fail ticket(s) from filter + while tickcount: + try: + found.append(_ticket_tuple(filter_.failManager.toBan())) + except FailManagerEmpty: + break + tickcount -= 1 if not isinstance(output[0], (tuple,list)): utest.assertEqual(len(found), 1) @@ -951,14 +958,11 @@ class LogFileMonitor(LogCaptureTestCase): self.file.close() self.file = _copy_lines_between_files(GetFailures.FILENAME_01, self.name, n=14, mode='w') - print('=========='*10) self.filter.getFailures(self.name) - print('=========='*10) self.assertRaises(FailManagerEmpty, self.filter.failManager.toBan) self.assertEqual(self.filter.failManager.getFailTotal(), 2) # move aside, but leaving the handle still open... - print('=========='*10) os.rename(self.name, self.name + '.bak') _copy_lines_between_files(GetFailures.FILENAME_01, self.name, skip=14, n=1).close() self.filter.getFailures(self.name) @@ -1112,12 +1116,13 @@ def get_monitor_failures_testcase(Filter_): skip=12, n=3, mode='w') self.assert_correct_last_attempt(GetFailures.FAILURES_01) - def _wait4failures(self, count=2): + def _wait4failures(self, count=2, waitEmpty=True): # Poll might need more time - self.assertTrue(self.isEmpty(_maxWaitTime(5)), - "Queue must be empty but it is not: %s." - % (', '.join([str(x) for x in self.jail.queue]))) - self.assertRaises(FailManagerEmpty, self.filter.failManager.toBan) + if waitEmpty: + self.assertTrue(self.isEmpty(_maxWaitTime(5)), + "Queue must be empty but it is not: %s." + % (', '.join([str(x) for x in self.jail.queue]))) + self.assertRaises(FailManagerEmpty, self.filter.failManager.toBan) Utils.wait_for(lambda: self.filter.failManager.getFailTotal() >= count, _maxWaitTime(10)) self.assertEqual(self.filter.failManager.getFailTotal(), count) @@ -1283,9 +1288,9 @@ def get_monitor_failures_testcase(Filter_): # now copy and get even more _copy_lines_between_files(GetFailures.FILENAME_01, self.file, skip=12, n=3) # check for 3 failures (not 9), because 6 already get above... - self.assert_correct_last_attempt(GetFailures.FAILURES_01) + self.assert_correct_last_attempt(GetFailures.FAILURES_01, count=3) # total count in this test: - self.assertEqual(self.filter.failManager.getFailTotal(), 9) + self._wait4failures(12, False) cls = MonitorFailures cls.__qualname__ = cls.__name__ = "MonitorFailures<%s>(%s)" \ @@ -1640,6 +1645,7 @@ class GetFailures(LogCaptureTestCase): [u'Aug 14 11:%d:59 i60p295 sshd[12365]: Failed publickey for roehl from ::ffff:141.3.81.106 port 51332 ssh2' % m for m in 53, 54, 57, 58]) + self.filter.setMaxRetry(4) self.filter.addLogPath(GetFailures.FILENAME_02, autoSeek=0) self.filter.addFailRegex(r"Failed .* from ") self.filter.getFailures(GetFailures.FILENAME_02) @@ -1648,6 +1654,7 @@ class GetFailures(LogCaptureTestCase): def testGetFailures03(self): output = ('203.162.223.135', 6, 1124013600.0) + self.filter.setMaxRetry(6) self.filter.addLogPath(GetFailures.FILENAME_03, autoSeek=0) self.filter.addFailRegex(r"error,relay=,.*550 User unknown") self.filter.getFailures(GetFailures.FILENAME_03) @@ -1656,6 +1663,7 @@ class GetFailures(LogCaptureTestCase): def testGetFailures03_InOperation(self): output = ('203.162.223.135', 9, 1124013600.0) + self.filter.setMaxRetry(9) self.filter.addLogPath(GetFailures.FILENAME_03, autoSeek=0) self.filter.addFailRegex(r"error,relay=,.*550 User unknown") self.filter.getFailures(GetFailures.FILENAME_03, inOperation=True) @@ -1673,7 +1681,7 @@ class GetFailures(LogCaptureTestCase): def testGetFailures03_Seek2(self): # same test as above but with seek to 'Aug 14 11:59:04' - so other output ... output = ('203.162.223.135', 2, 1124013600.0) - self.filter.setMaxRetry(1) + self.filter.setMaxRetry(2) self.filter.addLogPath(GetFailures.FILENAME_03, autoSeek=output[2]) self.filter.addFailRegex(r"error,relay=,.*550 User unknown") @@ -1683,10 +1691,12 @@ class GetFailures(LogCaptureTestCase): def testGetFailures04(self): # because of not exact time in testcase04.log (no year), we should always use our test time: self.assertEqual(MyTime.time(), 1124013600) - # should find exact 4 failures for *.186 and 2 failures for *.185 - output = (('212.41.96.186', 4, 1124013600.0), - ('212.41.96.185', 2, 1124013598.0)) - + # should find exact 4 failures for *.186 and 2 failures for *.185, but maxretry is 2, so 3 tickets: + output = ( + ('212.41.96.186', 2, 1124013480.0), + ('212.41.96.186', 2, 1124013600.0), + ('212.41.96.185', 2, 1124013598.0) + ) # speedup search using exact date pattern: self.filter.setDatePattern((r'^%ExY(?P<_sep>[-/.])%m(?P=_sep)%d[T ]%H:%M:%S(?:[.,]%f)?(?:\s*%z)?', r'^(?:%a )?%b %d %H:%M:%S(?:\.%f)?(?: %ExY)?', @@ -1743,9 +1753,11 @@ class GetFailures(LogCaptureTestCase): unittest.F2B.SkipIfNoNetwork() # We should still catch failures with usedns = no ;-) output_yes = ( - ('93.184.216.34', 2, 1124013539.0, - [u'Aug 14 11:54:59 i60p295 sshd[12365]: Failed publickey for roehl from example.com port 51332 ssh2', - u'Aug 14 11:58:59 i60p295 sshd[12365]: Failed publickey for roehl from ::ffff:93.184.216.34 port 51332 ssh2'] + ('93.184.216.34', 1, 1124013299.0, + [u'Aug 14 11:54:59 i60p295 sshd[12365]: Failed publickey for roehl from example.com port 51332 ssh2'] + ), + ('93.184.216.34', 1, 1124013539.0, + [u'Aug 14 11:58:59 i60p295 sshd[12365]: Failed publickey for roehl from ::ffff:93.184.216.34 port 51332 ssh2'] ), ('2606:2800:220:1:248:1893:25c8:1946', 1, 1124013299.0, [u'Aug 14 11:54:59 i60p295 sshd[12365]: Failed publickey for roehl from example.com port 51332 ssh2'] @@ -1779,8 +1791,11 @@ class GetFailures(LogCaptureTestCase): _assert_correct_last_attempt(self, filter_, output) def testGetFailuresMultiRegex(self): - output = ('141.3.81.106', 8, 1124013541.0) + output = [ + ('141.3.81.106', 8, 1124013541.0) + ] + self.filter.setMaxRetry(8) self.filter.addLogPath(GetFailures.FILENAME_02, autoSeek=False) self.filter.addFailRegex(r"Failed .* from ") self.filter.addFailRegex(r"Accepted .* from ") @@ -1798,26 +1813,25 @@ class GetFailures(LogCaptureTestCase): self.assertRaises(FailManagerEmpty, self.filter.failManager.toBan) def testGetFailuresMultiLine(self): - output = [("192.0.43.10", 2, 1124013599.0), - ("192.0.43.11", 1, 1124013598.0)] + output = [ + ("192.0.43.10", 1, 1124013598.0), + ("192.0.43.10", 1, 1124013599.0), + ("192.0.43.11", 1, 1124013598.0) + ] self.filter.addLogPath(GetFailures.FILENAME_MULTILINE, autoSeek=False) self.filter.setMaxLines(100) self.filter.addFailRegex(r"^.*rsyncd\[(?P\d+)\]: connect from .+ \(\)$^.+ rsyncd\[(?P=pid)\]: rsync error: .*$") self.filter.setMaxRetry(1) self.filter.getFailures(GetFailures.FILENAME_MULTILINE) - - foundList = [] - while True: - try: - foundList.append( - _ticket_tuple(self.filter.failManager.toBan())[0:3]) - except FailManagerEmpty: - break - self.assertSortedEqual(foundList, output) + + _assert_correct_last_attempt(self, self.filter, output) def testGetFailuresMultiLineIgnoreRegex(self): - output = [("192.0.43.10", 2, 1124013599.0)] + output = [ + ("192.0.43.10", 1, 1124013598.0), + ("192.0.43.10", 1, 1124013599.0) + ] self.filter.addLogPath(GetFailures.FILENAME_MULTILINE, autoSeek=False) self.filter.setMaxLines(100) self.filter.addFailRegex(r"^.*rsyncd\[(?P\d+)\]: connect from .+ \(\)$^.+ rsyncd\[(?P=pid)\]: rsync error: .*$") @@ -1826,14 +1840,17 @@ class GetFailures(LogCaptureTestCase): self.filter.getFailures(GetFailures.FILENAME_MULTILINE) - _assert_correct_last_attempt(self, self.filter, output.pop()) + _assert_correct_last_attempt(self, self.filter, output) self.assertRaises(FailManagerEmpty, self.filter.failManager.toBan) def testGetFailuresMultiLineMultiRegex(self): - output = [("192.0.43.10", 2, 1124013599.0), + output = [ + ("192.0.43.10", 1, 1124013598.0), + ("192.0.43.10", 1, 1124013599.0), ("192.0.43.11", 1, 1124013598.0), - ("192.0.43.15", 1, 1124013598.0)] + ("192.0.43.15", 1, 1124013598.0) + ] self.filter.addLogPath(GetFailures.FILENAME_MULTILINE, autoSeek=False) self.filter.setMaxLines(100) self.filter.addFailRegex(r"^.*rsyncd\[(?P\d+)\]: connect from .+ \(\)$^.+ rsyncd\[(?P=pid)\]: rsync error: .*$") @@ -1842,14 +1859,9 @@ class GetFailures(LogCaptureTestCase): self.filter.getFailures(GetFailures.FILENAME_MULTILINE) - foundList = [] - while True: - try: - foundList.append( - _ticket_tuple(self.filter.failManager.toBan())[0:3]) - except FailManagerEmpty: - break - self.assertSortedEqual(foundList, output) + _assert_correct_last_attempt(self, self.filter, output) + + self.assertRaises(FailManagerEmpty, self.filter.failManager.toBan) class DNSUtilsTests(unittest.TestCase): From 92a224217496fe3114fc7ee9f80708c00804ec03 Mon Sep 17 00:00:00 2001 From: sebres Date: Tue, 23 Feb 2021 15:54:48 +0100 Subject: [PATCH 3/4] amend fixing journal tests (systemd backend only) --- fail2ban/tests/filtertestcase.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/fail2ban/tests/filtertestcase.py b/fail2ban/tests/filtertestcase.py index fe37ea29..b9b7e8aa 100644 --- a/fail2ban/tests/filtertestcase.py +++ b/fail2ban/tests/filtertestcase.py @@ -1517,7 +1517,7 @@ def get_monitor_failures_journal_testcase(Filter_): # pragma: systemd no cover "SYSLOG_IDENTIFIER=fail2ban-testcases", "TEST_FIELD=1", "TEST_UUID=%s" % self.test_uuid]) - self.assert_correct_ban("193.168.0.128", 4) + self.assert_correct_ban("193.168.0.128", 3) _copy_lines_to_journal( self.test_file, self.journal_fields, n=6, skip=10) # we should detect the failures @@ -1531,7 +1531,7 @@ def get_monitor_failures_journal_testcase(Filter_): # pragma: systemd no cover self.test_file, self.journal_fields, skip=15, n=4) self.waitForTicks(1) self.assertTrue(self.isFilled(10)) - self.assert_correct_ban("87.142.124.10", 4) + self.assert_correct_ban("87.142.124.10", 3) # Add direct utf, unicode, blob: for l in ( "error: PAM: Authentication failure for \xe4\xf6\xfc\xdf from 192.0.2.1", From 6f4b6ec8ccdb68c75aec8225d8fa2b03ed19f320 Mon Sep 17 00:00:00 2001 From: sebres Date: Wed, 24 Feb 2021 13:05:04 +0100 Subject: [PATCH 4/4] action.d/badips.* removed (badips.com is no longer active, gh-2889) --- MANIFEST | 3 - config/action.d/badips.conf | 19 -- config/action.d/badips.py | 391 ------------------------- config/jail.conf | 14 - fail2ban/tests/action_d/test_badips.py | 157 ---------- fail2ban/tests/clientreadertestcase.py | 10 +- 6 files changed, 3 insertions(+), 591 deletions(-) delete mode 100644 config/action.d/badips.conf delete mode 100644 config/action.d/badips.py delete mode 100644 fail2ban/tests/action_d/test_badips.py diff --git a/MANIFEST b/MANIFEST index 50f308db..efe87085 100644 --- a/MANIFEST +++ b/MANIFEST @@ -5,8 +5,6 @@ bin/fail2ban-testcases ChangeLog config/action.d/abuseipdb.conf config/action.d/apf.conf -config/action.d/badips.conf -config/action.d/badips.py config/action.d/blocklist_de.conf config/action.d/bsd-ipfw.conf config/action.d/cloudflare.conf @@ -219,7 +217,6 @@ fail2ban/setup.py fail2ban-testcases-all fail2ban-testcases-all-python3 fail2ban/tests/action_d/__init__.py -fail2ban/tests/action_d/test_badips.py fail2ban/tests/action_d/test_smtp.py fail2ban/tests/actionstestcase.py fail2ban/tests/actiontestcase.py diff --git a/config/action.d/badips.conf b/config/action.d/badips.conf deleted file mode 100644 index 6f9513f6..00000000 --- a/config/action.d/badips.conf +++ /dev/null @@ -1,19 +0,0 @@ -# Fail2ban reporting to badips.com -# -# Note: This reports an IP only and does not actually ban traffic. Use -# another action in the same jail if you want bans to occur. -# -# Set the category to the appropriate value before use. -# -# To get see register and optional key to get personalised graphs see: -# http://www.badips.com/blog/personalized-statistics-track-the-attackers-of-all-your-servers-with-one-key - -[Definition] - -actionban = curl --fail --user-agent "" http://www.badips.com/add// - -[Init] - -# Option: category -# Notes.: Values are from the list here: http://www.badips.com/get/categories -category = diff --git a/config/action.d/badips.py b/config/action.d/badips.py deleted file mode 100644 index 805120e9..00000000 --- a/config/action.d/badips.py +++ /dev/null @@ -1,391 +0,0 @@ -# emacs: -*- mode: python; py-indent-offset: 4; indent-tabs-mode: t -*- -# vi: set ft=python sts=4 ts=4 sw=4 noet : - -# This file is part of Fail2Ban. -# -# Fail2Ban is free software; you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation; either version 2 of the License, or -# (at your option) any later version. -# -# Fail2Ban is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with Fail2Ban; if not, write to the Free Software -# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. - -import sys -if sys.version_info < (2, 7): # pragma: no cover - raise ImportError("badips.py action requires Python >= 2.7") -import json -import threading -import logging -if sys.version_info >= (3, ): # pragma: 2.x no cover - from urllib.request import Request, urlopen - from urllib.parse import urlencode - from urllib.error import HTTPError -else: # pragma: 3.x no cover - from urllib2 import Request, urlopen, HTTPError - from urllib import urlencode - -from fail2ban.server.actions import Actions, ActionBase, BanTicket -from fail2ban.helpers import splitwords, str2LogLevel - - - -class BadIPsAction(ActionBase): # pragma: no cover - may be unavailable - """Fail2Ban action which reports bans to badips.com, and also - blacklist bad IPs listed on badips.com by using another action's - ban method. - - Parameters - ---------- - jail : Jail - The jail which the action belongs to. - name : str - Name assigned to the action. - category : str - Valid badips.com category for reporting failures. - score : int, optional - Minimum score for bad IPs. Default 3. - age : str, optional - Age of last report for bad IPs, per badips.com syntax. - Default "24h" (24 hours) - banaction : str, optional - Name of banaction to use for blacklisting bad IPs. If `None`, - no blacklist of IPs will take place. - Default `None`. - bancategory : str, optional - Name of category to use for blacklisting, which can differ - from category used for reporting. e.g. may want to report - "postfix", but want to use whole "mail" category for blacklist. - Default `category`. - bankey : str, optional - Key issued by badips.com to retrieve personal list - of blacklist IPs. - updateperiod : int, optional - Time in seconds between updating bad IPs blacklist. - Default 900 (15 minutes) - loglevel : int/str, optional - Log level of the message when an IP is (un)banned. - Default `DEBUG`. - Can be also supplied as two-value list (comma- or space separated) to - provide level of the summary message when a group of IPs is (un)banned. - Example `DEBUG,INFO`. - agent : str, optional - User agent transmitted to server. - Default `Fail2Ban/ver.` - - Raises - ------ - ValueError - If invalid `category`, `score`, `banaction` or `updateperiod`. - """ - - TIMEOUT = 10 - _badips = "https://www.badips.com" - def _Request(self, url, **argv): - return Request(url, headers={'User-Agent': self.agent}, **argv) - - def __init__(self, jail, name, category, score=3, age="24h", - banaction=None, bancategory=None, bankey=None, updateperiod=900, - loglevel='DEBUG', agent="Fail2Ban", timeout=TIMEOUT): - super(BadIPsAction, self).__init__(jail, name) - - self.timeout = timeout - self.agent = agent - self.category = category - self.score = score - self.age = age - self.banaction = banaction - self.bancategory = bancategory or category - self.bankey = bankey - loglevel = splitwords(loglevel) - self.sumloglevel = str2LogLevel(loglevel[-1]) - self.loglevel = str2LogLevel(loglevel[0]) - self.updateperiod = updateperiod - - self._bannedips = set() - # Used later for threading.Timer for updating badips - self._timer = None - - @staticmethod - def isAvailable(timeout=1): - try: - response = urlopen(Request("/".join([BadIPsAction._badips]), - headers={'User-Agent': "Fail2Ban"}), timeout=timeout) - return True, '' - except Exception as e: # pragma: no cover - return False, e - - def logError(self, response, what=''): # pragma: no cover - sporadical (502: Bad Gateway, etc) - messages = {} - try: - messages = json.loads(response.read().decode('utf-8')) - except: - pass - self._logSys.error( - "%s. badips.com response: '%s'", what, - messages.get('err', 'Unknown')) - - def getCategories(self, incParents=False): - """Get badips.com categories. - - Returns - ------- - set - Set of categories. - - Raises - ------ - HTTPError - Any issues with badips.com request. - ValueError - If badips.com response didn't contain necessary information - """ - try: - response = urlopen( - self._Request("/".join([self._badips, "get", "categories"])), timeout=self.timeout) - except HTTPError as response: # pragma: no cover - self.logError(response, "Failed to fetch categories") - raise - else: - response_json = json.loads(response.read().decode('utf-8')) - if not 'categories' in response_json: - err = "badips.com response lacked categories specification. Response was: %s" \ - % (response_json,) - self._logSys.error(err) - raise ValueError(err) - categories = response_json['categories'] - categories_names = set( - value['Name'] for value in categories) - if incParents: - categories_names.update(set( - value['Parent'] for value in categories - if "Parent" in value)) - return categories_names - - def getList(self, category, score, age, key=None): - """Get badips.com list of bad IPs. - - Parameters - ---------- - category : str - Valid badips.com category. - score : int - Minimum score for bad IPs. - age : str - Age of last report for bad IPs, per badips.com syntax. - key : str, optional - Key issued by badips.com to fetch IPs reported with the - associated key. - - Returns - ------- - set - Set of bad IPs. - - Raises - ------ - HTTPError - Any issues with badips.com request. - """ - try: - url = "?".join([ - "/".join([self._badips, "get", "list", category, str(score)]), - urlencode({'age': age})]) - if key: - url = "&".join([url, urlencode({'key': key})]) - self._logSys.debug('badips.com: get list, url: %r', url) - response = urlopen(self._Request(url), timeout=self.timeout) - except HTTPError as response: # pragma: no cover - self.logError(response, "Failed to fetch bad IP list") - raise - else: - return set(response.read().decode('utf-8').split()) - - @property - def category(self): - """badips.com category for reporting IPs. - """ - return self._category - - @category.setter - def category(self, category): - if category not in self.getCategories(): - self._logSys.error("Category name '%s' not valid. " - "see badips.com for list of valid categories", - category) - raise ValueError("Invalid category: %s" % category) - self._category = category - - @property - def bancategory(self): - """badips.com bancategory for fetching IPs. - """ - return self._bancategory - - @bancategory.setter - def bancategory(self, bancategory): - if bancategory != "any" and bancategory not in self.getCategories(incParents=True): - self._logSys.error("Category name '%s' not valid. " - "see badips.com for list of valid categories", - bancategory) - raise ValueError("Invalid bancategory: %s" % bancategory) - self._bancategory = bancategory - - @property - def score(self): - """badips.com minimum score for fetching IPs. - """ - return self._score - - @score.setter - def score(self, score): - score = int(score) - if 0 <= score <= 5: - self._score = score - else: - raise ValueError("Score must be 0-5") - - @property - def banaction(self): - """Jail action to use for banning/unbanning. - """ - return self._banaction - - @banaction.setter - def banaction(self, banaction): - if banaction is not None and banaction not in self._jail.actions: - self._logSys.error("Action name '%s' not in jail '%s'", - banaction, self._jail.name) - raise ValueError("Invalid banaction") - self._banaction = banaction - - @property - def updateperiod(self): - """Period in seconds between banned bad IPs will be updated. - """ - return self._updateperiod - - @updateperiod.setter - def updateperiod(self, updateperiod): - updateperiod = int(updateperiod) - if updateperiod > 0: - self._updateperiod = updateperiod - else: - raise ValueError("Update period must be integer greater than 0") - - def _banIPs(self, ips): - for ip in ips: - try: - ai = Actions.ActionInfo(BanTicket(ip), self._jail) - self._jail.actions[self.banaction].ban(ai) - except Exception as e: - self._logSys.error( - "Error banning IP %s for jail '%s' with action '%s': %s", - ip, self._jail.name, self.banaction, e, - exc_info=self._logSys.getEffectiveLevel()<=logging.DEBUG) - else: - self._bannedips.add(ip) - self._logSys.log(self.loglevel, - "Banned IP %s for jail '%s' with action '%s'", - ip, self._jail.name, self.banaction) - - def _unbanIPs(self, ips): - for ip in ips: - try: - ai = Actions.ActionInfo(BanTicket(ip), self._jail) - self._jail.actions[self.banaction].unban(ai) - except Exception as e: - self._logSys.error( - "Error unbanning IP %s for jail '%s' with action '%s': %s", - ip, self._jail.name, self.banaction, e, - exc_info=self._logSys.getEffectiveLevel()<=logging.DEBUG) - else: - self._logSys.log(self.loglevel, - "Unbanned IP %s for jail '%s' with action '%s'", - ip, self._jail.name, self.banaction) - finally: - self._bannedips.remove(ip) - - def start(self): - """If `banaction` set, blacklists bad IPs. - """ - if self.banaction is not None: - self.update() - - def update(self): - """If `banaction` set, updates blacklisted IPs. - - Queries badips.com for list of bad IPs, removing IPs from the - blacklist if no longer present, and adds new bad IPs to the - blacklist. - """ - if self.banaction is not None: - if self._timer: - self._timer.cancel() - self._timer = None - - try: - ips = self.getList( - self.bancategory, self.score, self.age, self.bankey) - # Remove old IPs no longer listed - s = self._bannedips - ips - m = len(s) - self._unbanIPs(s) - # Add new IPs which are now listed - s = ips - self._bannedips - p = len(s) - self._banIPs(s) - if m != 0 or p != 0: - self._logSys.log(self.sumloglevel, - "Updated IPs for jail '%s' (-%d/+%d)", - self._jail.name, m, p) - self._logSys.debug( - "Next update for jail '%' in %i seconds", - self._jail.name, self.updateperiod) - finally: - self._timer = threading.Timer(self.updateperiod, self.update) - self._timer.start() - - def stop(self): - """If `banaction` set, clears blacklisted IPs. - """ - if self.banaction is not None: - if self._timer: - self._timer.cancel() - self._timer = None - self._unbanIPs(self._bannedips.copy()) - - def ban(self, aInfo): - """Reports banned IP to badips.com. - - Parameters - ---------- - aInfo : dict - Dictionary which includes information in relation to - the ban. - - Raises - ------ - HTTPError - Any issues with badips.com request. - """ - try: - url = "/".join([self._badips, "add", self.category, str(aInfo['ip'])]) - self._logSys.debug('badips.com: ban, url: %r', url) - response = urlopen(self._Request(url), timeout=self.timeout) - except HTTPError as response: # pragma: no cover - self.logError(response, "Failed to ban") - raise - else: - messages = json.loads(response.read().decode('utf-8')) - self._logSys.debug( - "Response from badips.com report: '%s'", - messages['suc']) - -Action = BadIPsAction diff --git a/config/jail.conf b/config/jail.conf index ddbcf61e..be035112 100644 --- a/config/jail.conf +++ b/config/jail.conf @@ -204,20 +204,6 @@ action_cf_mwl = cloudflare[cfuser="%(cfemail)s", cftoken="%(cfapikey)s"] # action_blocklist_de = blocklist_de[email="%(sender)s", service="%(__name__)s", apikey="%(blocklist_de_apikey)s", agent="%(fail2ban_agent)s"] -# Report ban via badips.com, and use as blacklist -# -# See BadIPsAction docstring in config/action.d/badips.py for -# documentation for this action. -# -# NOTE: This action relies on banaction being present on start and therefore -# should be last action defined for a jail. -# -action_badips = badips.py[category="%(__name__)s", banaction="%(banaction)s", agent="%(fail2ban_agent)s"] -# -# Report ban via badips.com (uses action.d/badips.conf for reporting only) -# -action_badips_report = badips[category="%(__name__)s", agent="%(fail2ban_agent)s"] - # Report ban via abuseipdb.com. # # See action.d/abuseipdb.conf for usage example and details. diff --git a/fail2ban/tests/action_d/test_badips.py b/fail2ban/tests/action_d/test_badips.py deleted file mode 100644 index 013c0fdb..00000000 --- a/fail2ban/tests/action_d/test_badips.py +++ /dev/null @@ -1,157 +0,0 @@ -# emacs: -*- mode: python; py-indent-offset: 4; indent-tabs-mode: t -*- -# vi: set ft=python sts=4 ts=4 sw=4 noet : - -# This file is part of Fail2Ban. -# -# Fail2Ban is free software; you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation; either version 2 of the License, or -# (at your option) any later version. -# -# Fail2Ban is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with Fail2Ban; if not, write to the Free Software -# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. - -import os -import unittest -import sys -from functools import wraps -from socket import timeout -from ssl import SSLError - -from ..actiontestcase import CallingMap -from ..dummyjail import DummyJail -from ..servertestcase import IPAddr -from ..utils import LogCaptureTestCase, CONFIG_DIR - -if sys.version_info >= (3, ): # pragma: 2.x no cover - from urllib.error import HTTPError, URLError -else: # pragma: 3.x no cover - from urllib2 import HTTPError, URLError - -def skip_if_not_available(f): - """Helper to decorate tests to skip in case of timeout/http-errors like "502 bad gateway". - """ - @wraps(f) - def wrapper(self, *args): - try: - return f(self, *args) - except (SSLError, HTTPError, URLError, timeout) as e: # pragma: no cover - timeout/availability issues - if not isinstance(e, timeout) and 'timed out' not in str(e): - if not hasattr(e, 'code') or e.code > 200 and e.code <= 404: - raise - raise unittest.SkipTest('Skip test because of %s' % e) - return wrapper - -if sys.version_info >= (2,7): # pragma: no cover - may be unavailable - class BadIPsActionTest(LogCaptureTestCase): - - available = True, None - pythonModule = None - modAction = None - - @skip_if_not_available - def setUp(self): - """Call before every test case.""" - super(BadIPsActionTest, self).setUp() - unittest.F2B.SkipIfNoNetwork() - - self.jail = DummyJail() - - self.jail.actions.add("test") - - pythonModuleName = os.path.join(CONFIG_DIR, "action.d", "badips.py") - - # check availability (once if not alive, used shorter timeout as in test cases): - if BadIPsActionTest.available[0]: - if not BadIPsActionTest.modAction: - if not BadIPsActionTest.pythonModule: - BadIPsActionTest.pythonModule = self.jail.actions._load_python_module(pythonModuleName) - BadIPsActionTest.modAction = BadIPsActionTest.pythonModule.Action - self.jail.actions._load_python_module(pythonModuleName) - BadIPsActionTest.available = BadIPsActionTest.modAction.isAvailable(timeout=2 if unittest.F2B.fast else 30) - if not BadIPsActionTest.available[0]: - raise unittest.SkipTest('Skip test because service is not available: %s' % BadIPsActionTest.available[1]) - - self.jail.actions.add("badips", pythonModuleName, initOpts={ - 'category': "ssh", - 'banaction': "test", - 'age': "2w", - 'score': 5, - #'key': "fail2ban-test-suite", - #'bankey': "fail2ban-test-suite", - 'timeout': (3 if unittest.F2B.fast else 60), - }) - self.action = self.jail.actions["badips"] - - def tearDown(self): - """Call after every test case.""" - # Must cancel timer! - if self.action._timer: - self.action._timer.cancel() - super(BadIPsActionTest, self).tearDown() - - @skip_if_not_available - def testCategory(self): - categories = self.action.getCategories() - self.assertIn("ssh", categories) - self.assertTrue(len(categories) >= 10) - - self.assertRaises( - ValueError, setattr, self.action, "category", - "invalid-category") - - # Not valid for reporting category... - self.assertRaises( - ValueError, setattr, self.action, "category", "mail") - # but valid for blacklisting. - self.action.bancategory = "mail" - - @skip_if_not_available - def testScore(self): - self.assertRaises(ValueError, setattr, self.action, "score", -5) - self.action.score = 3 - self.action.score = "3" - - @skip_if_not_available - def testBanaction(self): - self.assertRaises( - ValueError, setattr, self.action, "banaction", - "invalid-action") - self.action.banaction = "test" - - @skip_if_not_available - def testUpdateperiod(self): - self.assertRaises( - ValueError, setattr, self.action, "updateperiod", -50) - self.assertRaises( - ValueError, setattr, self.action, "updateperiod", 0) - self.action.updateperiod = 900 - self.action.updateperiod = "900" - - @skip_if_not_available - def testStartStop(self): - self.action.start() - self.assertTrue(len(self.action._bannedips) > 10, - "%s is fewer as 10: %r" % (len(self.action._bannedips), self.action._bannedips)) - self.action.stop() - self.assertTrue(len(self.action._bannedips) == 0) - - @skip_if_not_available - def testBanIP(self): - aInfo = CallingMap({ - 'ip': IPAddr('192.0.2.1') - }) - self.action.ban(aInfo) - self.assertLogged('badips.com: ban', wait=True) - self.pruneLog() - # produce an error using wrong category/IP: - self.action._category = 'f2b-this-category-dont-available-test-suite-only' - aInfo['ip'] = '' - self.assertRaises(BadIPsActionTest.pythonModule.HTTPError, self.action.ban, aInfo) - self.assertLogged('IP is invalid', 'invalid category', wait=True, all=False) diff --git a/fail2ban/tests/clientreadertestcase.py b/fail2ban/tests/clientreadertestcase.py index e92edd48..4029c753 100644 --- a/fail2ban/tests/clientreadertestcase.py +++ b/fail2ban/tests/clientreadertestcase.py @@ -458,8 +458,6 @@ class JailReaderTest(LogCaptureTestCase): ('sender', 'f2b-test@example.com'), ('blocklist_de_apikey', 'test-key'), ('action', '%(action_blocklist_de)s\n' - '%(action_badips_report)s\n' - '%(action_badips)s\n' 'mynetwatchman[port=1234,protocol=udp,agent="%(fail2ban_agent)s"]' ), )) @@ -473,16 +471,14 @@ class JailReaderTest(LogCaptureTestCase): if len(cmd) <= 4: continue # differentiate between set and multi-set (wrop it here to single set): - if cmd[0] == 'set' and (cmd[4] == 'agent' or cmd[4].endswith('badips.py')): + if cmd[0] == 'set' and cmd[4] == 'agent': act.append(cmd) elif cmd[0] == 'multi-set': act.extend([['set'] + cmd[1:4] + o for o in cmd[4] if o[0] == 'agent']) useragent = 'Fail2Ban/%s' % version - self.assertEqual(len(act), 4) + self.assertEqual(len(act), 2) self.assertEqual(act[0], ['set', 'blocklisttest', 'action', 'blocklist_de', 'agent', useragent]) - self.assertEqual(act[1], ['set', 'blocklisttest', 'action', 'badips', 'agent', useragent]) - self.assertEqual(eval(act[2][5]).get('agent', ''), useragent) - self.assertEqual(act[3], ['set', 'blocklisttest', 'action', 'mynetwatchman', 'agent', useragent]) + self.assertEqual(act[1], ['set', 'blocklisttest', 'action', 'mynetwatchman', 'agent', useragent]) @with_tmpdir def testGlob(self, d):