diff --git a/.travis.yml b/.travis.yml index 1f218c81..59a50313 100644 --- a/.travis.yml +++ b/.travis.yml @@ -26,6 +26,7 @@ matrix: - python: 3.6 - python: 3.7 - python: 3.8 + - python: 3.9-dev - python: pypy3.5 before_install: - echo "running under $TRAVIS_PYTHON_VERSION" diff --git a/ChangeLog b/ChangeLog index ba64b06b..6223a447 100644 --- a/ChangeLog +++ b/ChangeLog @@ -38,6 +38,9 @@ ver. 0.11.2-dev (20??/??/??) - development edition IPv6-capable now. ### Fixes +* [stability] prevent race condition - no ban if filter (backend) is continuously busy if + too many messages will be found in log, e. g. initial scan of large log-file or journal (gh-2660) +* python 3.9 compatibility (and Travis CI support) * restoring a large number (500+ depending on files ulimit) of current bans when using PyPy fixed * manual ban is written to database, so can be restored by restart (gh-2647) * `filter.d/common.conf`: avoid substitute of default values in related `lt_*` section, `__prefix_line` diff --git a/fail2ban/server/filter.py b/fail2ban/server/filter.py index b1eece17..4e65dd2c 100644 --- a/fail2ban/server/filter.py +++ b/fail2ban/server/filter.py @@ -112,6 +112,8 @@ class Filter(JailThread): self.onIgnoreRegex = None ## if true ignores obsolete failures (failure time < now - findTime): self.checkFindTime = True + ## if true prevents against retarded banning in case of RC by too many failures (disabled only for test purposes): + self.banASAP = True ## Ticks counter self.ticks = 0 ## Thread name: @@ -626,7 +628,11 @@ class Filter(JailThread): logSys.info( "[%s] Found %s - %s", self.jailName, ip, MyTime.time2str(unixTime) ) - self.failManager.addFailure(tick) + attempts = self.failManager.addFailure(tick) + # avoid RC on busy filter (too many failures) - if attempts for IP/ID reached maxretry, + # we can speedup ban, so do it as soon as possible: + if self.banASAP and attempts >= self.failManager.getMaxRetry(): + self.performBan(ip) # report to observer - failure was found, for possibly increasing of it retry counter (asynchronous) if Observers.Main is not None: Observers.Main.add('failureFound', self.failManager, self.jail, tick) diff --git a/fail2ban/server/filtergamin.py b/fail2ban/server/filtergamin.py index 77c81757..078246de 100644 --- a/fail2ban/server/filtergamin.py +++ b/fail2ban/server/filtergamin.py @@ -79,7 +79,8 @@ class FilterGamin(FileFilter): this is a common logic and must be shared/provided by FileFilter """ self.getFailures(path) - self.performBan() + if not self.banASAP: # pragma: no cover + self.performBan() self.__modified = False ## diff --git a/fail2ban/server/filterpoll.py b/fail2ban/server/filterpoll.py index 228a2c8b..b4d8ab14 100644 --- a/fail2ban/server/filterpoll.py +++ b/fail2ban/server/filterpoll.py @@ -117,7 +117,8 @@ class FilterPoll(FileFilter): self.ticks += 1 if self.__modified: - self.performBan() + if not self.banASAP: # pragma: no cover + self.performBan() self.__modified = False except Exception as e: # pragma: no cover if not self.active: # if not active - error by stop... diff --git a/fail2ban/server/filterpyinotify.py b/fail2ban/server/filterpyinotify.py index ca6b253f..185305ca 100644 --- a/fail2ban/server/filterpyinotify.py +++ b/fail2ban/server/filterpyinotify.py @@ -140,7 +140,8 @@ class FilterPyinotify(FileFilter): """ if not self.idle: self.getFailures(path) - self.performBan() + if not self.banASAP: # pragma: no cover + self.performBan() self.__modified = False def _addPending(self, path, reason, isDir=False): diff --git a/fail2ban/server/filtersystemd.py b/fail2ban/server/filtersystemd.py index 870b3058..47fc891e 100644 --- a/fail2ban/server/filtersystemd.py +++ b/fail2ban/server/filtersystemd.py @@ -318,7 +318,8 @@ class FilterSystemd(JournalFilter): # pragma: systemd no cover else: break if self.__modified: - self.performBan() + if not self.banASAP: # pragma: no cover + self.performBan() self.__modified = 0 # update position in log (time and iso string): if self.jail.database is not None: diff --git a/fail2ban/server/jailthread.py b/fail2ban/server/jailthread.py index d0430367..94f34542 100644 --- a/fail2ban/server/jailthread.py +++ b/fail2ban/server/jailthread.py @@ -120,3 +120,6 @@ class JailThread(Thread): ## python 2.x replace binding of private __bootstrap method: if sys.version_info < (3,): # pragma: 3.x no cover JailThread._Thread__bootstrap = JailThread._JailThread__bootstrap +## python 3.9, restore isAlive method: +elif not hasattr(JailThread, 'isAlive'): # pragma: 2.x no cover + JailThread.isAlive = JailThread.is_alive diff --git a/fail2ban/tests/filtertestcase.py b/fail2ban/tests/filtertestcase.py index fabc5488..c3b58408 100644 --- a/fail2ban/tests/filtertestcase.py +++ b/fail2ban/tests/filtertestcase.py @@ -399,6 +399,7 @@ class IgnoreIP(LogCaptureTestCase): self.filter.addFailRegex('^') self.filter.setDatePattern(r'{^LN-BEG}%Y-%m-%d %H:%M:%S(?:\s*%Z)?\s') self.filter.setFindTime(10); # max 10 seconds back + self.filter.setMaxRetry(5); # don't ban here # self.pruneLog('[phase 1] DST time jump') # check local time jump (DST hole): @@ -757,6 +758,7 @@ class LogFileMonitor(LogCaptureTestCase): _, self.name = tempfile.mkstemp('fail2ban', 'monitorfailures') self.file = open(self.name, 'a') self.filter = FilterPoll(DummyJail()) + self.filter.banASAP = False # avoid immediate ban in this tests self.filter.addLogPath(self.name, autoSeek=False) self.filter.active = True self.filter.addFailRegex(r"(?:(?:Authentication failure|Failed [-/\w+]+) for(?: [iI](?:llegal|nvalid) user)?|[Ii](?:llegal|nvalid) user|ROOT LOGIN REFUSED) .*(?: from|FROM) ") @@ -974,6 +976,7 @@ def get_monitor_failures_testcase(Filter_): self.file = open(self.name, 'a') self.jail = DummyJail() self.filter = Filter_(self.jail) + self.filter.banASAP = False # avoid immediate ban in this tests self.filter.addLogPath(self.name, autoSeek=False) # speedup search using exact date pattern: self.filter.setDatePattern(r'^(?:%a )?%b %d %H:%M:%S(?:\.%f)?(?: %ExY)?') @@ -1272,6 +1275,7 @@ def get_monitor_failures_journal_testcase(Filter_): # pragma: systemd no cover def _initFilter(self, **kwargs): self._getRuntimeJournal() # check journal available self.filter = Filter_(self.jail, **kwargs) + self.filter.banASAP = False # avoid immediate ban in this tests self.filter.addJournalMatch([ "SYSLOG_IDENTIFIER=fail2ban-testcases", "TEST_FIELD=1", @@ -1525,6 +1529,7 @@ class GetFailures(LogCaptureTestCase): setUpMyTime() self.jail = DummyJail() self.filter = FileFilter(self.jail) + self.filter.banASAP = False # avoid immediate ban in this tests self.filter.active = True # speedup search using exact date pattern: self.filter.setDatePattern(r'^(?:%a )?%b %d %H:%M:%S(?:\.%f)?(?: %ExY)?') @@ -1714,6 +1719,7 @@ class GetFailures(LogCaptureTestCase): self.pruneLog("[test-phase useDns=%s]" % useDns) jail = DummyJail() filter_ = FileFilter(jail, useDns=useDns) + filter_.banASAP = False # avoid immediate ban in this tests filter_.active = True filter_.failManager.setMaxRetry(1) # we might have just few failures diff --git a/fail2ban/tests/sockettestcase.py b/fail2ban/tests/sockettestcase.py index 69bf8d8b..b107ac6a 100644 --- a/fail2ban/tests/sockettestcase.py +++ b/fail2ban/tests/sockettestcase.py @@ -87,7 +87,7 @@ class Socket(LogCaptureTestCase): def _stopServerThread(self): serverThread = self.serverThread # wait for end of thread : - Utils.wait_for(lambda: not serverThread.isAlive() + Utils.wait_for(lambda: not serverThread.is_alive() or serverThread.join(Utils.DEFAULT_SLEEP_TIME), unittest.F2B.maxWaitTime(10)) self.serverThread = None @@ -98,7 +98,7 @@ class Socket(LogCaptureTestCase): self.server.close() # wait for end of thread : self._stopServerThread() - self.assertFalse(serverThread.isAlive()) + self.assertFalse(serverThread.is_alive()) # clean : self.server.stop() self.assertFalse(self.server.isActive()) @@ -139,7 +139,7 @@ class Socket(LogCaptureTestCase): self.server.stop() # wait for end of thread : self._stopServerThread() - self.assertFalse(serverThread.isAlive()) + self.assertFalse(serverThread.is_alive()) self.assertFalse(self.server.isActive()) self.assertFalse(os.path.exists(self.sock_name)) @@ -180,7 +180,7 @@ class Socket(LogCaptureTestCase): self.server.stop() # wait for end of thread : self._stopServerThread() - self.assertFalse(serverThread.isAlive()) + self.assertFalse(serverThread.is_alive()) def testLoopErrors(self): # replace poll handler to produce error in loop-cycle: @@ -216,7 +216,7 @@ class Socket(LogCaptureTestCase): self.server.stop() # wait for end of thread : self._stopServerThread() - self.assertFalse(serverThread.isAlive()) + self.assertFalse(serverThread.is_alive()) self.assertFalse(self.server.isActive()) self.assertFalse(os.path.exists(self.sock_name))