From 6d72602f6b25aee10a3e0744635d3090a6415518 Mon Sep 17 00:00:00 2001 From: "M. Maraun" Date: Fri, 3 Jul 2015 22:42:22 +0200 Subject: [PATCH 001/126] Set Timeout at urlopen to 3 seconds --- THANKS | 1 + config/action.d/badips.py | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/THANKS b/THANKS index 5ae86a3c..133bd987 100644 --- a/THANKS +++ b/THANKS @@ -71,6 +71,7 @@ kojiro Lars Kneschke Lee Clemens leftyfb (Mike Rushton) +M. Maraun Manuel Arostegui Ramirez Marcel Dopita Mark Edgington diff --git a/config/action.d/badips.py b/config/action.d/badips.py index c2a239f5..58e64990 100644 --- a/config/action.d/badips.py +++ b/config/action.d/badips.py @@ -116,7 +116,7 @@ class BadIPsAction(ActionBase): """ try: response = urlopen( - self._Request("/".join([self._badips, "get", "categories"]))) + self._Request("/".join([self._badips, "get", "categories"])), None, 3) except HTTPError as response: messages = json.loads(response.read().decode('utf-8')) self._logSys.error( From f7af93a677f8b87477c2c6bc4f1b82095be3780b Mon Sep 17 00:00:00 2001 From: weberho Date: Wed, 26 Aug 2015 15:25:59 +0200 Subject: [PATCH 002/126] Added configuration for opensuse path --- config/paths-opensuse.conf | 38 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 38 insertions(+) create mode 100644 config/paths-opensuse.conf diff --git a/config/paths-opensuse.conf b/config/paths-opensuse.conf new file mode 100644 index 00000000..0d6ad522 --- /dev/null +++ b/config/paths-opensuse.conf @@ -0,0 +1,38 @@ +# openSUSE log-file locations + +[INCLUDES] + +before = paths-common.conf + +after = paths-overrides.local + + +[DEFAULT] + +syslog_local0 = /var/log/messages + +syslog_mail = /var/log/mail + +syslog_mail_warn = %(syslog_mail)s + +syslog_authpriv = %(syslog_local0)s + +syslog_user = %(syslog_local0)s + +syslog_ftp = %(syslog_local0)s + +syslog_daemon = %(syslog_local0)s + +apache_error_log = /var/log/apache2/*error_log + +apache_access_log = /var/log/apache2/*access_log + +pureftpd_log = %(syslog_local0)s + +exim_main_log = /var/log/exim/main.log + +mysql_log = /var/log/mysql/mysqld.log + +roundcube_errors_log = /srv/www/roundcubemail/logs/errors + +solidpop3d_log = %(syslog_mail)s From 2d69fd20aea1dedd74996577f54d3b20b4493eaf Mon Sep 17 00:00:00 2001 From: weberho Date: Wed, 26 Aug 2015 15:37:14 +0200 Subject: [PATCH 003/126] Updated ChangeLog regarding openSUSE's path config --- ChangeLog | 1 + 1 file changed, 1 insertion(+) diff --git a/ChangeLog b/ChangeLog index b07f876e..a8922dec 100644 --- a/ChangeLog +++ b/ChangeLog @@ -14,6 +14,7 @@ ver. 0.9.4 (2015/XX/XXX) - wanna-be-released - New Features: - Enhancements: + * Added openSUSE path configuration ver. 0.9.3 (2015/08/01) - lets-all-stay-friends ---------- From 835b3ff483b0b62af6c48fdab3aff23730849d48 Mon Sep 17 00:00:00 2001 From: Edward Beckett Date: Sat, 5 Sep 2015 00:12:28 -0400 Subject: [PATCH 004/126] Update apache-badbots.conf Useragent strings including `+http` need to be escaped to be valid. --- config/filter.d/apache-badbots.conf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config/filter.d/apache-badbots.conf b/config/filter.d/apache-badbots.conf index 75c0de03..48b30666 100644 --- a/config/filter.d/apache-badbots.conf +++ b/config/filter.d/apache-badbots.conf @@ -8,7 +8,7 @@ [Definition] badbotscustom = EmailCollector|WebEMailExtrac|TrackBack/1\.02|sogou music spider -badbots = Atomic_Email_Hunter/4\.0|atSpider/1\.0|autoemailspider|bwh3_user_agent|China Local Browse 2\.6|ContactBot/0\.2|ContentSmartz|DataCha0s/2\.0|DBrowse 1\.4b|DBrowse 1\.4d|Demo Bot DOT 16b|Demo Bot Z 16b|DSurf15a 01|DSurf15a 71|DSurf15a 81|DSurf15a VA|EBrowse 1\.4b|Educate Search VxB|EmailSiphon|EmailSpider|EmailWolf 1\.00|ESurf15a 15|ExtractorPro|Franklin Locator 1\.8|FSurf15a 01|Full Web Bot 0416B|Full Web Bot 0516B|Full Web Bot 2816B|Guestbook Auto Submitter|Industry Program 1\.0\.x|ISC Systems iRc Search 2\.1|IUPUI Research Bot v 1\.9a|LARBIN-EXPERIMENTAL \(efp@gmx\.net\)|LetsCrawl\.com/1\.0 +http\://letscrawl\.com/|Lincoln State Web Browser|LMQueueBot/0\.2|LWP\:\:Simple/5\.803|Mac Finder 1\.0\.xx|MFC Foundation Class Library 4\.0|Microsoft URL Control - 6\.00\.8xxx|Missauga Locate 1\.0\.0|Missigua Locator 1\.9|Missouri College Browse|Mizzu Labs 2\.2|Mo College 1\.9|MVAClient|Mozilla/2\.0 \(compatible; NEWT ActiveX; Win32\)|Mozilla/3\.0 \(compatible; Indy Library\)|Mozilla/3\.0 \(compatible; scan4mail \(advanced version\) http\://www\.peterspages\.net/?scan4mail\)|Mozilla/4\.0 \(compatible; Advanced Email Extractor v2\.xx\)|Mozilla/4\.0 \(compatible; Iplexx Spider/1\.0 http\://www\.iplexx\.at\)|Mozilla/4\.0 \(compatible; MSIE 5\.0; Windows NT; DigExt; DTS Agent|Mozilla/4\.0 efp@gmx\.net|Mozilla/5\.0 \(Version\: xxxx Type\:xx\)|NameOfAgent \(CMS Spider\)|NASA Search 1\.0|Nsauditor/1\.x|PBrowse 1\.4b|PEval 1\.4b|Poirot|Port Huron Labs|Production Bot 0116B|Production Bot 2016B|Production Bot DOT 3016B|Program Shareware 1\.0\.2|PSurf15a 11|PSurf15a 51|PSurf15a VA|psycheclone|RSurf15a 41|RSurf15a 51|RSurf15a 81|searchbot admin@google\.com|ShablastBot 1\.0|snap\.com beta crawler v0|Snapbot/1\.0|Snapbot/1\.0 \(Snap Shots, +http\://www\.snap\.com\)|sogou develop spider|Sogou Orion spider/3\.0\(+http\://www\.sogou\.com/docs/help/webmasters\.htm#07\)|sogou spider|Sogou web spider/3\.0\(+http\://www\.sogou\.com/docs/help/webmasters\.htm#07\)|sohu agent|SSurf15a 11 |TSurf15a 11|Under the Rainbow 2\.2|User-Agent\: Mozilla/4\.0 \(compatible; MSIE 6\.0; Windows NT 5\.1\)|VadixBot|WebVulnCrawl\.unknown/1\.0 libwww-perl/5\.803|Wells Search II|WEP Search 00 +badbots = Atomic_Email_Hunter/4\.0|atSpider/1\.0|autoemailspider|bwh3_user_agent|China Local Browse 2\.6|ContactBot/0\.2|ContentSmartz|DataCha0s/2\.0|DBrowse 1\.4b|DBrowse 1\.4d|Demo Bot DOT 16b|Demo Bot Z 16b|DSurf15a 01|DSurf15a 71|DSurf15a 81|DSurf15a VA|EBrowse 1\.4b|Educate Search VxB|EmailSiphon|EmailSpider|EmailWolf 1\.00|ESurf15a 15|ExtractorPro|Franklin Locator 1\.8|FSurf15a 01|Full Web Bot 0416B|Full Web Bot 0516B|Full Web Bot 2816B|Guestbook Auto Submitter|Industry Program 1\.0\.x|ISC Systems iRc Search 2\.1|IUPUI Research Bot v 1\.9a|LARBIN-EXPERIMENTAL \(efp@gmx\.net\)|LetsCrawl\.com/1\.0 \+http\://letscrawl\.com/|Lincoln State Web Browser|LMQueueBot/0\.2|LWP\:\:Simple/5\.803|Mac Finder 1\.0\.xx|MFC Foundation Class Library 4\.0|Microsoft URL Control - 6\.00\.8xxx|Missauga Locate 1\.0\.0|Missigua Locator 1\.9|Missouri College Browse|Mizzu Labs 2\.2|Mo College 1\.9|MVAClient|Mozilla/2\.0 \(compatible; NEWT ActiveX; Win32\)|Mozilla/3\.0 \(compatible; Indy Library\)|Mozilla/3\.0 \(compatible; scan4mail \(advanced version\) http\://www\.peterspages\.net/?scan4mail\)|Mozilla/4\.0 \(compatible; Advanced Email Extractor v2\.xx\)|Mozilla/4\.0 \(compatible; Iplexx Spider/1\.0 http\://www\.iplexx\.at\)|Mozilla/4\.0 \(compatible; MSIE 5\.0; Windows NT; DigExt; DTS Agent|Mozilla/4\.0 efp@gmx\.net|Mozilla/5\.0 \(Version\: xxxx Type\:xx\)|NameOfAgent \(CMS Spider\)|NASA Search 1\.0|Nsauditor/1\.x|PBrowse 1\.4b|PEval 1\.4b|Poirot|Port Huron Labs|Production Bot 0116B|Production Bot 2016B|Production Bot DOT 3016B|Program Shareware 1\.0\.2|PSurf15a 11|PSurf15a 51|PSurf15a VA|psycheclone|RSurf15a 41|RSurf15a 51|RSurf15a 81|searchbot admin@google\.com|ShablastBot 1\.0|snap\.com beta crawler v0|Snapbot/1\.0|Snapbot/1\.0 \(Snap Shots, \+http\://www\.snap\.com\)|sogou develop spider|Sogou Orion spider/3\.0\(\+http\://www\.sogou\.com/docs/help/webmasters\.htm#07\)|sogou spider|Sogou web spider/3\.0\(\+http\://www\.sogou\.com/docs/help/webmasters\.htm#07\)|sohu agent|SSurf15a 11 |TSurf15a 11|Under the Rainbow 2\.2|User-Agent\: Mozilla/4\.0 \(compatible; MSIE 6\.0; Windows NT 5\.1\)|VadixBot|WebVulnCrawl\.unknown/1\.0 libwww-perl/5\.803|Wells Search II|WEP Search 00 failregex = ^ -.*"(GET|POST|HEAD).*HTTP.*"(?:%(badbots)s|%(badbotscustom)s)"$ From 03460d5ed0be3f6b22fd396710ac71f731e1a830 Mon Sep 17 00:00:00 2001 From: Edward Beckett Date: Sun, 6 Sep 2015 01:05:52 -0400 Subject: [PATCH 005/126] Update gen_badbots Added plus char to the regex for escaping user-agent strings. --- files/gen_badbots | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/files/gen_badbots b/files/gen_badbots index 278058f7..52732317 100755 --- a/files/gen_badbots +++ b/files/gen_badbots @@ -42,7 +42,7 @@ done \ | grep -h -B4 'S '\ | sed -e 's/ //g' \ | awk '/^--/{getline; gsub(" ",""); print $0}' \ -| sed -e 's/\([.\:|()]\)/\\\1/g' \ +| sed -e 's/\([.\:|()+]\)/\\\1/g' \ | uniq \ | tr '\n' '|' \ | sed -e 's/|$//g' From 4bd7991573c266f8d674706b27c816f137676c0d Mon Sep 17 00:00:00 2001 From: Edward Beckett Date: Sun, 6 Sep 2015 01:12:19 -0400 Subject: [PATCH 006/126] Added apache-badbots.conf --- ChangeLog | 2 ++ 1 file changed, 2 insertions(+) diff --git a/ChangeLog b/ChangeLog index 4fe225da..f27e7dfc 100644 --- a/ChangeLog +++ b/ChangeLog @@ -44,6 +44,8 @@ ver. 0.9.3 (2015/08/01) - lets-all-stay-friends * filter.d/roundcube-auth.conf - Updated regex to work with 'errors' log (1.0.5 and 1.1.1) - Added regex to work with 'userlogins' log + * filter.d/apache-badbots.conf + - Updated useragent string regex * action.d/sendmail*.conf - use LC_ALL (superseeding LC_TIME) to override locale on systems with customized LC_ALL * performance fix: minimizes connection overhead, close socket only at From 4cf3b576b9feb0525719aa5d0a614d397bb569ea Mon Sep 17 00:00:00 2001 From: sebres Date: Tue, 8 Sep 2015 11:14:46 +0200 Subject: [PATCH 007/126] Bugfix for dnsToIp resolver for fqdn with large list of IPs; closes #1164 --- ChangeLog | 1 + fail2ban/server/filter.py | 3 ++- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/ChangeLog b/ChangeLog index f27e7dfc..d49ee82a 100644 --- a/ChangeLog +++ b/ChangeLog @@ -11,6 +11,7 @@ ver. 0.9.4 (2015/XX/XXX) - wanna-be-released - Fixes: * roundcube-auth jail typo for logpath + * Fix dnsToIp resolver for fqdn with large list of IPs (gh-1164) - New Features: diff --git a/fail2ban/server/filter.py b/fail2ban/server/filter.py index 18afb135..6fc2cd6c 100644 --- a/fail2ban/server/filter.py +++ b/fail2ban/server/filter.py @@ -855,8 +855,9 @@ class DNSUtils: """ Convert a DNS into an IP address using the Python socket module. Thanks to Kevin Drapel. """ + # retrieve ip (todo: use AF_INET6 for IPv6) try: - return set(socket.gethostbyname_ex(dns)[2]) + return set([i[4][0] for i in socket.getaddrinfo(dns, None, socket.AF_INET, 0, socket.IPPROTO_TCP)]) except socket.error, e: logSys.warning("Unable to find a corresponding IP address for %s: %s" % (dns, e)) From f5b88bd377baca4f08ed93bc7d93468eb1565d69 Mon Sep 17 00:00:00 2001 From: Edward Beckett Date: Fri, 11 Sep 2015 10:12:57 -0400 Subject: [PATCH 008/126] Updated Changelog --- ChangeLog | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ChangeLog b/ChangeLog index d49ee82a..2fec408a 100644 --- a/ChangeLog +++ b/ChangeLog @@ -12,6 +12,8 @@ ver. 0.9.4 (2015/XX/XXX) - wanna-be-released - Fixes: * roundcube-auth jail typo for logpath * Fix dnsToIp resolver for fqdn with large list of IPs (gh-1164) + * filter.d/apache-badbots.conf + - Updated useragent string regex adding escape for `+` - New Features: @@ -45,8 +47,6 @@ ver. 0.9.3 (2015/08/01) - lets-all-stay-friends * filter.d/roundcube-auth.conf - Updated regex to work with 'errors' log (1.0.5 and 1.1.1) - Added regex to work with 'userlogins' log - * filter.d/apache-badbots.conf - - Updated useragent string regex * action.d/sendmail*.conf - use LC_ALL (superseeding LC_TIME) to override locale on systems with customized LC_ALL * performance fix: minimizes connection overhead, close socket only at From 85b298e49cad821d0fccce92670e618088f0aa4b Mon Sep 17 00:00:00 2001 From: Yaroslav Halchenko Date: Sat, 12 Sep 2015 12:56:28 -0400 Subject: [PATCH 009/126] RF: try/except/finally in a single statement (while at it) since we support now python >= 2.6 --- fail2ban/server/action.py | 42 +++++++++++++++++++-------------------- 1 file changed, 21 insertions(+), 21 deletions(-) diff --git a/fail2ban/server/action.py b/fail2ban/server/action.py index c58fde2c..a8e61fb4 100644 --- a/fail2ban/server/action.py +++ b/fail2ban/server/action.py @@ -560,32 +560,32 @@ class CommandAction(ActionBase): return True _cmd_lock.acquire() - try: # Try wrapped within another try needed for python version < 2.5 + try: stdout = tempfile.TemporaryFile(suffix=".stdout", prefix="fai2ban_") stderr = tempfile.TemporaryFile(suffix=".stderr", prefix="fai2ban_") - try: - popen = subprocess.Popen( - realCmd, stdout=stdout, stderr=stderr, shell=True, - preexec_fn=os.setsid # so that killpg does not kill our process - ) - stime = time.time() + + popen = subprocess.Popen( + realCmd, stdout=stdout, stderr=stderr, shell=True, + preexec_fn=os.setsid # so that killpg does not kill our process + ) + stime = time.time() + retcode = popen.poll() + while time.time() - stime <= timeout and retcode is None: + time.sleep(0.1) retcode = popen.poll() - while time.time() - stime <= timeout and retcode is None: + if retcode is None: + logSys.error("%s -- timed out after %i seconds." % + (realCmd, timeout)) + pgid = os.getpgid(popen.pid) + os.killpg(pgid, signal.SIGTERM) # Terminate the process + time.sleep(0.1) + retcode = popen.poll() + if retcode is None: # Still going... + os.killpg(pgid, signal.SIGKILL) # Kill the process time.sleep(0.1) retcode = popen.poll() - if retcode is None: - logSys.error("%s -- timed out after %i seconds." % - (realCmd, timeout)) - pgid = os.getpgid(popen.pid) - os.killpg(pgid, signal.SIGTERM) # Terminate the process - time.sleep(0.1) - retcode = popen.poll() - if retcode is None: # Still going... - os.killpg(pgid, signal.SIGKILL) # Kill the process - time.sleep(0.1) - retcode = popen.poll() - except OSError, e: - logSys.error("%s -- failed with %s" % (realCmd, e)) + except OSError as e: + logSys.error("%s -- failed with %s" % (realCmd, e)) finally: _cmd_lock.release() From 7cbb3980ebaf123eec0013169172f15c2877e711 Mon Sep 17 00:00:00 2001 From: Yaroslav Halchenko Date: Sat, 12 Sep 2015 19:39:44 -0400 Subject: [PATCH 010/126] BF+RF: only return bool status on failed commands execution + mitigate different exit codes between bash/dash Closes #1155 --- fail2ban/server/action.py | 12 +++++++----- fail2ban/tests/actiontestcase.py | 14 ++++++-------- 2 files changed, 13 insertions(+), 13 deletions(-) diff --git a/fail2ban/server/action.py b/fail2ban/server/action.py index a8e61fb4..de0c8efc 100644 --- a/fail2ban/server/action.py +++ b/fail2ban/server/action.py @@ -561,6 +561,7 @@ class CommandAction(ActionBase): _cmd_lock.acquire() try: + retcode = None # to guarantee being defined upon early except stdout = tempfile.TemporaryFile(suffix=".stdout", prefix="fai2ban_") stderr = tempfile.TemporaryFile(suffix=".stderr", prefix="fai2ban_") @@ -603,15 +604,16 @@ class CommandAction(ActionBase): return True elif retcode is None: logSys.error("%s -- unable to kill PID %i" % (realCmd, popen.pid)) - elif retcode < 0: - logSys.error("%s -- killed with %s" % - (realCmd, signame.get(-retcode, "signal %i" % -retcode))) + elif retcode < 0 or retcode > 128: + # dash would return negative while bash 128 + n + sigcode = -retcode if retcode < 0 else retcode - 128 + logSys.error("%s -- killed with %s (return code: %s)" % + (realCmd, signame.get(sigcode, "signal %i" % sigcode), retcode)) else: msg = _RETCODE_HINTS.get(retcode, None) logSys.error("%s -- returned %i" % (realCmd, retcode)) if msg: logSys.info("HINT on %i: %s" % (retcode, msg % locals())) - return False - raise RuntimeError("Command execution failed: %s" % realCmd) + return False diff --git a/fail2ban/tests/actiontestcase.py b/fail2ban/tests/actiontestcase.py index febbc619..9e703bbf 100644 --- a/fail2ban/tests/actiontestcase.py +++ b/fail2ban/tests/actiontestcase.py @@ -196,11 +196,10 @@ class CommandActionTest(LogCaptureTestCase): def testExecuteTimeout(self): stime = time.time() # Should take a minute - self.assertRaises( - RuntimeError, CommandAction.executeCmd, 'sleep 60', timeout=2) + self.assertFalse(CommandAction.executeCmd('sleep 60', timeout=2)) # give a test still 1 second, because system could be too busy self.assertTrue(time.time() >= stime + 2 and time.time() <= stime + 3) - self.assertTrue(self._is_logged('sleep 60 -- timed out after 2 seconds') + self.assertTrue(self._is_logged('sleep 60 -- timed out after 2 seconds') or self._is_logged('sleep 60 -- timed out after 3 seconds')) self.assertTrue(self._is_logged('sleep 60 -- killed with SIGTERM')) @@ -222,17 +221,16 @@ class CommandActionTest(LogCaptureTestCase): return int(f.read()) # First test if can kill the bastard - self.assertRaises( - RuntimeError, CommandAction.executeCmd, 'bash %s' % tmpFilename, timeout=.1) + self.assertFalse(CommandAction.executeCmd( + 'bash %s' % tmpFilename, timeout=.1)) # Verify that the proccess itself got killed self.assertFalse(pid_exists(getnastypid())) # process should have been killed self.assertTrue(self._is_logged('timed out')) self.assertTrue(self._is_logged('killed with SIGTERM')) # A bit evolved case even though, previous test already tests killing children processes - self.assertRaises( - RuntimeError, CommandAction.executeCmd, 'out=`bash %s`; echo ALRIGHT' % tmpFilename, - timeout=.2) + self.assertFalse(CommandAction.executeCmd( + 'out=`bash %s`; echo ALRIGHT' % tmpFilename, timeout=.2)) # Verify that the proccess itself got killed self.assertFalse(pid_exists(getnastypid())) self.assertTrue(self._is_logged('timed out')) From 5ed731d3b3cc99278c21219de70264b1879251cf Mon Sep 17 00:00:00 2001 From: Yaroslav Halchenko Date: Sat, 12 Sep 2015 19:59:55 -0400 Subject: [PATCH 011/126] RF(TST): self.assertTrue(self._is_logged()) -> self.assertLogged and corresponding one for not + support for multiple entries at once, and failure message listing actual log to ease troubleshooting --- fail2ban/tests/actionstestcase.py | 22 ++++++++-------- fail2ban/tests/actiontestcase.py | 36 ++++++++++++++------------ fail2ban/tests/clientreadertestcase.py | 28 ++++++++++---------- fail2ban/tests/databasetestcase.py | 2 +- fail2ban/tests/filtertestcase.py | 19 +++++++------- fail2ban/tests/servertestcase.py | 2 +- fail2ban/tests/utils.py | 36 ++++++++++++++++++++++++++ 7 files changed, 91 insertions(+), 54 deletions(-) diff --git a/fail2ban/tests/actionstestcase.py b/fail2ban/tests/actionstestcase.py index bb295967..0ceb35d5 100644 --- a/fail2ban/tests/actionstestcase.py +++ b/fail2ban/tests/actionstestcase.py @@ -94,15 +94,15 @@ class ExecuteActions(LogCaptureTestCase): "Action", os.path.join(TEST_FILES_DIR, "action.d/action.py"), {'opt1': 'value'}) - self.assertTrue(self._is_logged("TestAction initialised")) + self.assertLogged("TestAction initialised") self.__actions.start() time.sleep(3) - self.assertTrue(self._is_logged("TestAction action start")) + self.assertLogged("TestAction action start") self.__actions.stop() self.__actions.join() - self.assertTrue(self._is_logged("TestAction action stop")) + self.assertLogged("TestAction action stop") self.assertRaises(IOError, self.__actions.add, "Action3", "/does/not/exist.py", {}) @@ -136,10 +136,10 @@ class ExecuteActions(LogCaptureTestCase): {}) self.__actions.start() time.sleep(3) - self.assertTrue(self._is_logged("Failed to start")) + self.assertLogged("Failed to start") self.__actions.stop() self.__actions.join() - self.assertTrue(self._is_logged("Failed to stop")) + self.assertLogged("Failed to stop") def testBanActionsAInfo(self): # Action which deletes IP address from aInfo @@ -155,13 +155,13 @@ class ExecuteActions(LogCaptureTestCase): self.__actions._Actions__checkBan() # Will fail if modification of aInfo from first action propagates # to second action, as both delete same key - self.assertFalse(self._is_logged("Failed to execute ban")) - self.assertTrue(self._is_logged("action1 ban deleted aInfo IP")) - self.assertTrue(self._is_logged("action2 ban deleted aInfo IP")) + self.assertNotLogged("Failed to execute ban") + self.assertLogged("action1 ban deleted aInfo IP") + self.assertLogged("action2 ban deleted aInfo IP") self.__actions._Actions__flushBan() # Will fail if modification of aInfo from first action propagates # to second action, as both delete same key - self.assertFalse(self._is_logged("Failed to execute unban")) - self.assertTrue(self._is_logged("action1 unban deleted aInfo IP")) - self.assertTrue(self._is_logged("action2 unban deleted aInfo IP")) + self.assertNotLogged("Failed to execute unban") + self.assertLogged("action1 unban deleted aInfo IP") + self.assertLogged("action2 unban deleted aInfo IP") diff --git a/fail2ban/tests/actiontestcase.py b/fail2ban/tests/actiontestcase.py index febbc619..a4b453a3 100644 --- a/fail2ban/tests/actiontestcase.py +++ b/fail2ban/tests/actiontestcase.py @@ -143,17 +143,17 @@ class CommandActionTest(LogCaptureTestCase): self.__action.actionunban = "true" self.assertEqual(self.__action.actionunban, 'true') - self.assertFalse(self._is_logged('returned')) + self.assertNotLogged('returned') # no action was actually executed yet self.__action.ban({'ip': None}) - self.assertTrue(self._is_logged('Invariant check failed')) - self.assertTrue(self._is_logged('returned successfully')) + self.assertLogged('Invariant check failed') + self.assertLogged('returned successfully') def testExecuteActionEmptyUnban(self): self.__action.actionunban = "" self.__action.unban({}) - self.assertTrue(self._is_logged('Nothing to do')) + self.assertLogged('Nothing to do') def testExecuteActionStartCtags(self): self.__action.HOST = "192.0.2.0" @@ -168,7 +168,7 @@ class CommandActionTest(LogCaptureTestCase): self.__action.actionban = "rm /tmp/fail2ban.test" self.__action.actioncheck = "[ -e /tmp/fail2ban.test ]" self.assertRaises(RuntimeError, self.__action.ban, {'ip': None}) - self.assertTrue(self._is_logged('Unable to restore environment')) + self.assertLogged('Unable to restore environment') def testExecuteActionChangeCtags(self): self.assertRaises(AttributeError, getattr, self.__action, "ROST") @@ -187,11 +187,11 @@ class CommandActionTest(LogCaptureTestCase): def testExecuteActionStartEmpty(self): self.__action.actionstart = "" self.__action.start() - self.assertTrue(self._is_logged('Nothing to do')) + self.assertLogged('Nothing to do') def testExecuteIncorrectCmd(self): CommandAction.executeCmd('/bin/ls >/dev/null\nbogusXXX now 2>/dev/null') - self.assertTrue(self._is_logged('HINT on 127: "Command not found"')) + self.assertLogged('HINT on 127: "Command not found"') def testExecuteTimeout(self): stime = time.time() @@ -200,9 +200,11 @@ class CommandActionTest(LogCaptureTestCase): RuntimeError, CommandAction.executeCmd, 'sleep 60', timeout=2) # give a test still 1 second, because system could be too busy self.assertTrue(time.time() >= stime + 2 and time.time() <= stime + 3) - self.assertTrue(self._is_logged('sleep 60 -- timed out after 2 seconds') - or self._is_logged('sleep 60 -- timed out after 3 seconds')) - self.assertTrue(self._is_logged('sleep 60 -- killed with SIGTERM')) + self.assertLogged({ + 'sleep 60 -- timed out after 2 seconds', + 'sleep 60 -- timed out after 3 seconds' + }) + self.assertLogged('sleep 60 -- killed with SIGTERM') def testExecuteTimeoutWithNastyChildren(self): # temporary file for a nasty kid shell script @@ -226,8 +228,8 @@ class CommandActionTest(LogCaptureTestCase): RuntimeError, CommandAction.executeCmd, 'bash %s' % tmpFilename, timeout=.1) # Verify that the proccess itself got killed self.assertFalse(pid_exists(getnastypid())) # process should have been killed - self.assertTrue(self._is_logged('timed out')) - self.assertTrue(self._is_logged('killed with SIGTERM')) + self.assertLogged('timed out') + self.assertLogged('killed with SIGTERM') # A bit evolved case even though, previous test already tests killing children processes self.assertRaises( @@ -235,8 +237,8 @@ class CommandActionTest(LogCaptureTestCase): timeout=.2) # Verify that the proccess itself got killed self.assertFalse(pid_exists(getnastypid())) - self.assertTrue(self._is_logged('timed out')) - self.assertTrue(self._is_logged('killed with SIGTERM')) + self.assertLogged('timed out') + self.assertLogged('killed with SIGTERM') os.unlink(tmpFilename) os.unlink(tmpFilename + '.pid') @@ -244,11 +246,11 @@ class CommandActionTest(LogCaptureTestCase): def testCaptureStdOutErr(self): CommandAction.executeCmd('echo "How now brown cow"') - self.assertTrue(self._is_logged("'How now brown cow\\n'")) + self.assertLogged("'How now brown cow\\n'") CommandAction.executeCmd( 'echo "The rain in Spain stays mainly in the plain" 1>&2') - self.assertTrue(self._is_logged( - "'The rain in Spain stays mainly in the plain\\n'")) + self.assertLogged( + "'The rain in Spain stays mainly in the plain\\n'") def testCallingMap(self): mymap = CallingMap(callme=lambda: str(10), error=lambda: int('a'), diff --git a/fail2ban/tests/clientreadertestcase.py b/fail2ban/tests/clientreadertestcase.py index 00128c33..94fe1828 100644 --- a/fail2ban/tests/clientreadertestcase.py +++ b/fail2ban/tests/clientreadertestcase.py @@ -173,16 +173,16 @@ class JailReaderTest(LogCaptureTestCase): self.assertTrue(jail.read()) self.assertTrue(jail.getOptions()) self.assertTrue(jail.isEnabled()) - self.assertTrue(self._is_logged('No filter set for jail emptyaction')) - self.assertTrue(self._is_logged('No actions were defined for emptyaction')) + self.assertLogged('No filter set for jail emptyaction') + self.assertLogged('No actions were defined for emptyaction') def testJailActionFilterMissing(self): jail = JailReader('missingbitsjail', basedir=IMPERFECT_CONFIG, share_config = self.__share_cfg) self.assertTrue(jail.read()) self.assertFalse(jail.getOptions()) self.assertTrue(jail.isEnabled()) - self.assertTrue(self._is_logged("Found no accessible config files for 'filter.d/catchallthebadies' under %s" % IMPERFECT_CONFIG)) - self.assertTrue(self._is_logged('Unable to read the filter')) + self.assertLogged("Found no accessible config files for 'filter.d/catchallthebadies' under %s" % IMPERFECT_CONFIG) + self.assertLogged('Unable to read the filter') def testJailActionBrokenDef(self): jail = JailReader('brokenactiondef', basedir=IMPERFECT_CONFIG, @@ -190,13 +190,13 @@ class JailReaderTest(LogCaptureTestCase): self.assertTrue(jail.read()) self.assertFalse(jail.getOptions()) self.assertTrue(jail.isEnabled()) - self.assertTrue(self._is_logged('Error in action definition joho[foo')) + self.assertLogged('Error in action definition joho[foo') # This unittest has been deactivated for some time... - # self.assertTrue(self._is_logged( - # 'Caught exception: While reading action joho[foo we should have got 1 or 2 groups. Got: 0')) + # self.assertLogged( + # 'Caught exception: While reading action joho[foo we should have got 1 or 2 groups. Got: 0') # let's test for what is actually logged and handle changes in the future - self.assertTrue(self._is_logged( - "Caught exception: 'NoneType' object has no attribute 'endswith'")) + self.assertLogged( + "Caught exception: 'NoneType' object has no attribute 'endswith'") if STOCK: def testStockSSHJail(self): @@ -221,7 +221,7 @@ class JailReaderTest(LogCaptureTestCase): self.assertEqual(('mail--ho_is', {}), JailReader.extractOptions("mail--ho_is['s']")) #self.printLog() - #self.assertTrue(self._is_logged("Invalid argument ['s'] in ''s''")) + #self.assertLogged("Invalid argument ['s'] in ''s''") self.assertEqual(('mail', {'a': ','}), JailReader.extractOptions("mail[a=',']")) @@ -265,7 +265,7 @@ class JailReaderTest(LogCaptureTestCase): self.assertEqual(JailReader._glob(os.path.join(d, '*')), [f1]) # since f2 is dangling -- empty list self.assertEqual(JailReader._glob(f2), []) - self.assertTrue(self._is_logged('File %s is a dangling link, thus cannot be monitored' % f2)) + self.assertLogged('File %s is a dangling link, thus cannot be monitored' % f2) self.assertEqual(JailReader._glob(os.path.join(d, 'nonexisting')), []) os.remove(f1) os.remove(f2) @@ -463,8 +463,8 @@ class JailsReaderTest(LogCaptureTestCase): ['start', 'missinglogfiles'], ['start', 'brokenaction'], ['start', 'parse_to_end_of_jail.conf'],])) - self.assertTrue(self._is_logged("Errors in jail 'missingbitsjail'. Skipping...")) - self.assertTrue(self._is_logged("No file(s) found for glob /weapons/of/mass/destruction")) + self.assertLogged("Errors in jail 'missingbitsjail'. Skipping...") + self.assertLogged("No file(s) found for glob /weapons/of/mass/destruction") if STOCK: def testReadStockActionConf(self): @@ -496,7 +496,7 @@ class JailsReaderTest(LogCaptureTestCase): #old_comm_commands = comm_commands[:] # make a copy #self.assertRaises(ValueError, jails.getOptions, "BOGUS") #self.printLog() - #self.assertTrue(self._is_logged("No section: 'BOGUS'")) + #self.assertLogged("No section: 'BOGUS'") ## and there should be no side-effects #self.assertEqual(jails.convert(), old_comm_commands) diff --git a/fail2ban/tests/databasetestcase.py b/fail2ban/tests/databasetestcase.py index dd813ee6..3d156eda 100644 --- a/fail2ban/tests/databasetestcase.py +++ b/fail2ban/tests/databasetestcase.py @@ -316,7 +316,7 @@ class DatabaseTest(LogCaptureTestCase): ticket.setAttempt(5) self.jail.putFailTicket(ticket) actions._Actions__checkBan() - self.assertTrue(self._is_logged("ban ainfo %s, %s, %s, %s" % (True, True, True, True))) + self.assertLogged("ban ainfo %s, %s, %s, %s" % (True, True, True, True)) def testPurge(self): if Fail2BanDb is None: # pragma: no cover diff --git a/fail2ban/tests/filtertestcase.py b/fail2ban/tests/filtertestcase.py index 891b55a6..acce9625 100644 --- a/fail2ban/tests/filtertestcase.py +++ b/fail2ban/tests/filtertestcase.py @@ -260,14 +260,14 @@ class IgnoreIP(LogCaptureTestCase): self.filter.addIgnoreIP('192.168.1.0/25') self.filter.addFailRegex('') self.filter.processLineAndAdd('1387203300.222 192.168.1.32') - self.assertTrue(self._is_logged('Ignore 192.168.1.32')) + self.assertLogged('Ignore 192.168.1.32') tearDownMyTime() def testIgnoreAddBannedIP(self): self.filter.addIgnoreIP('192.168.1.0/25') self.filter.addBannedIP('192.168.1.32') - self.assertFalse(self._is_logged('Ignore 192.168.1.32')) - self.assertTrue(self._is_logged('Requested to manually ban an ignored IP 192.168.1.32. User knows best. Proceeding to ban it.')) + self.assertNotLogged('Ignore 192.168.1.32') + self.assertLogged('Requested to manually ban an ignored IP 192.168.1.32. User knows best. Proceeding to ban it.') def testIgnoreCommand(self): self.filter.setIgnoreCommand(sys.executable + ' ' + os.path.join(TEST_FILES_DIR, "ignorecommand.py ")) @@ -278,11 +278,11 @@ class IgnoreIP(LogCaptureTestCase): ip = "93.184.216.34" for ignore_source in ["dns", "ip", "command"]: self.filter.logIgnoreIp(ip, True, ignore_source=ignore_source) - self.assertTrue(self._is_logged("[%s] Ignore %s by %s" % (self.jail.name, ip, ignore_source))) + self.assertLogged("[%s] Ignore %s by %s" % (self.jail.name, ip, ignore_source)) def testIgnoreCauseNOK(self): self.filter.logIgnoreIp("example.com", False, ignore_source="NOT_LOGGED") - self.assertFalse(self._is_logged("[%s] Ignore %s by %s" % (self.jail.name, "example.com", "NOT_LOGGED"))) + self.assertNotLogged("[%s] Ignore %s by %s" % (self.jail.name, "example.com", "NOT_LOGGED")) class IgnoreIPDNS(IgnoreIP): @@ -382,18 +382,17 @@ class LogFileMonitor(LogCaptureTestCase): def testNoLogFile(self): _killfile(self.file, self.name) self.filter.getFailures(self.name) - failure_was_logged = self._is_logged('Unable to open %s' % self.name) - self.assertTrue(failure_was_logged) + self.assertLogged('Unable to open %s' % self.name) def testRemovingFailRegex(self): self.filter.delFailRegex(0) - self.assertFalse(self._is_logged('Cannot remove regular expression. Index 0 is not valid')) + self.assertNotLogged('Cannot remove regular expression. Index 0 is not valid') self.filter.delFailRegex(0) - self.assertTrue(self._is_logged('Cannot remove regular expression. Index 0 is not valid')) + self.assertLogged('Cannot remove regular expression. Index 0 is not valid') def testRemovingIgnoreRegex(self): self.filter.delIgnoreRegex(0) - self.assertTrue(self._is_logged('Cannot remove regular expression. Index 0 is not valid')) + self.assertLogged('Cannot remove regular expression. Index 0 is not valid') def testNewChangeViaIsModified(self): # it is a brand new one -- so first we think it is modified diff --git a/fail2ban/tests/servertestcase.py b/fail2ban/tests/servertestcase.py index 57453269..6f946cc0 100644 --- a/fail2ban/tests/servertestcase.py +++ b/fail2ban/tests/servertestcase.py @@ -934,7 +934,7 @@ class LoggingTests(LogCaptureTestCase): badThread = _BadThread() badThread.start() badThread.join() - self.assertTrue(self._is_logged("Unhandled exception")) + self.assertLogged("Unhandled exception") finally: sys.__excepthook__ = prev_exchook self.assertEqual(len(x), 1) diff --git a/fail2ban/tests/utils.py b/fail2ban/tests/utils.py index e7993ead..9d50646a 100644 --- a/fail2ban/tests/utils.py +++ b/fail2ban/tests/utils.py @@ -232,6 +232,42 @@ class LogCaptureTestCase(unittest.TestCase): def _is_logged(self, s): return s in self._log.getvalue() + def assertLogged(self, s): + """Assert that a string was logged + + Preferable to assertTrue(self._is_logged(..))) + since provides message with the actual log. + + Parameters + ---------- + s : string or list/set/tuple of strings + Test should succeed if string (or any of the listed) is present in the log + """ + logged = self._log.getvalue() + + s_iter = s if isinstance(s, (tuple, list, set)) else [s] + for s_ in s_iter: + if s_ in logged: + return + raise AssertionError("%r was not found in the log: %r" % (s, logged)) + + def assertNotLogged(self, s): + """Assert that a string was not logged + + Parameters + ---------- + s : string or list/set/tuple of strings + Test should succeed if the string (or at least one of the listed) is not present in the log + """ + logged = self._log.getvalue() + + s_iter = s if isinstance(s, (tuple, list, set)) else [s] + for s_ in s_iter: + if s_ not in logged: + return + raise AssertionError("%r was found present in the log: %r" % (s, logged)) + + def getLog(self): return self._log.getvalue() From 8a4dcafc8f9455720805054bd027af790bf75673 Mon Sep 17 00:00:00 2001 From: Yaroslav Halchenko Date: Sat, 12 Sep 2015 21:37:40 -0400 Subject: [PATCH 012/126] RF/BF: py26 has no {} sets, so just pass multiple entries as *args --- fail2ban/tests/actiontestcase.py | 4 ++-- fail2ban/tests/utils.py | 23 ++++++++++------------- 2 files changed, 12 insertions(+), 15 deletions(-) diff --git a/fail2ban/tests/actiontestcase.py b/fail2ban/tests/actiontestcase.py index a4b453a3..1cba0bcc 100644 --- a/fail2ban/tests/actiontestcase.py +++ b/fail2ban/tests/actiontestcase.py @@ -200,10 +200,10 @@ class CommandActionTest(LogCaptureTestCase): RuntimeError, CommandAction.executeCmd, 'sleep 60', timeout=2) # give a test still 1 second, because system could be too busy self.assertTrue(time.time() >= stime + 2 and time.time() <= stime + 3) - self.assertLogged({ + self.assertLogged( 'sleep 60 -- timed out after 2 seconds', 'sleep 60 -- timed out after 3 seconds' - }) + ) self.assertLogged('sleep 60 -- killed with SIGTERM') def testExecuteTimeoutWithNastyChildren(self): diff --git a/fail2ban/tests/utils.py b/fail2ban/tests/utils.py index 9d50646a..35fa59fd 100644 --- a/fail2ban/tests/utils.py +++ b/fail2ban/tests/utils.py @@ -232,8 +232,8 @@ class LogCaptureTestCase(unittest.TestCase): def _is_logged(self, s): return s in self._log.getvalue() - def assertLogged(self, s): - """Assert that a string was logged + def assertLogged(self, *s): + """Assert that one of the strings was logged Preferable to assertTrue(self._is_logged(..))) since provides message with the actual log. @@ -244,28 +244,25 @@ class LogCaptureTestCase(unittest.TestCase): Test should succeed if string (or any of the listed) is present in the log """ logged = self._log.getvalue() - - s_iter = s if isinstance(s, (tuple, list, set)) else [s] - for s_ in s_iter: + for s_ in s: if s_ in logged: return - raise AssertionError("%r was not found in the log: %r" % (s, logged)) + raise AssertionError("None among %r was found in the log: %r" % (s, logged)) - def assertNotLogged(self, s): - """Assert that a string was not logged + def assertNotLogged(self, *s): + """Assert that strings were not logged Parameters ---------- s : string or list/set/tuple of strings - Test should succeed if the string (or at least one of the listed) is not present in the log + Test should succeed if the string (or at least one of the listed) is not + present in the log """ logged = self._log.getvalue() - - s_iter = s if isinstance(s, (tuple, list, set)) else [s] - for s_ in s_iter: + for s_ in s: if s_ not in logged: return - raise AssertionError("%r was found present in the log: %r" % (s, logged)) + raise AssertionError("All of the %r were found present in the log: %r" % (s, logged)) def getLog(self): From 63c7ceb81d2a3e46c916a2e3c530fa7ec83f8c56 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ville=20Skytt=C3=A4?= Date: Sun, 13 Sep 2015 10:57:51 +0300 Subject: [PATCH 013/126] logrotate: Remove outdated Fedora comment --- files/fail2ban-logrotate | 3 --- 1 file changed, 3 deletions(-) diff --git a/files/fail2ban-logrotate b/files/fail2ban-logrotate index a09870af..ef4da852 100644 --- a/files/fail2ban-logrotate +++ b/files/fail2ban-logrotate @@ -4,9 +4,6 @@ # # Debian: # https://github.com/fail2ban/fail2ban/blob/debian/debian/fail2ban.logrotate -# -# Fedora view: -# http://pkgs.fedoraproject.org/cgit/fail2ban.git/tree/fail2ban-logrotate /var/log/fail2ban.log { rotate 7 From 67a94733a9b3eec7e325dcdbbf411473770d36bf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ville=20Skytt=C3=A4?= Date: Sun, 13 Sep 2015 11:01:22 +0300 Subject: [PATCH 014/126] logrotate: Do not rotate empty logs As a useful side effect, prevents "Unable to contact server. Is it running?" mails from cron when fail2ban hasn't been (intentionally) running nor thus logging anything either. --- ChangeLog | 1 + files/fail2ban-logrotate | 1 + 2 files changed, 2 insertions(+) diff --git a/ChangeLog b/ChangeLog index 2fec408a..811473e7 100644 --- a/ChangeLog +++ b/ChangeLog @@ -18,6 +18,7 @@ ver. 0.9.4 (2015/XX/XXX) - wanna-be-released - New Features: - Enhancements: + * Do not rotate empty log files ver. 0.9.3 (2015/08/01) - lets-all-stay-friends ---------- diff --git a/files/fail2ban-logrotate b/files/fail2ban-logrotate index ef4da852..8d94a8b3 100644 --- a/files/fail2ban-logrotate +++ b/files/fail2ban-logrotate @@ -8,6 +8,7 @@ /var/log/fail2ban.log { rotate 7 missingok + notifempty compress postrotate /usr/bin/fail2ban-client flushlogs 1>/dev/null || true From 943efdb1a073668096613d6abc4ade4985928b6e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ville=20Skytt=C3=A4?= Date: Sun, 13 Sep 2015 11:08:04 +0300 Subject: [PATCH 015/126] Comment spelling fixes --- fail2ban/tests/actiontestcase.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/fail2ban/tests/actiontestcase.py b/fail2ban/tests/actiontestcase.py index febbc619..0749edc7 100644 --- a/fail2ban/tests/actiontestcase.py +++ b/fail2ban/tests/actiontestcase.py @@ -224,7 +224,7 @@ class CommandActionTest(LogCaptureTestCase): # First test if can kill the bastard self.assertRaises( RuntimeError, CommandAction.executeCmd, 'bash %s' % tmpFilename, timeout=.1) - # Verify that the proccess itself got killed + # Verify that the process itself got killed self.assertFalse(pid_exists(getnastypid())) # process should have been killed self.assertTrue(self._is_logged('timed out')) self.assertTrue(self._is_logged('killed with SIGTERM')) @@ -233,7 +233,7 @@ class CommandActionTest(LogCaptureTestCase): self.assertRaises( RuntimeError, CommandAction.executeCmd, 'out=`bash %s`; echo ALRIGHT' % tmpFilename, timeout=.2) - # Verify that the proccess itself got killed + # Verify that the process itself got killed self.assertFalse(pid_exists(getnastypid())) self.assertTrue(self._is_logged('timed out')) self.assertTrue(self._is_logged('killed with SIGTERM')) From fbdd0b74a10f65336724e78ac9b03b8eb86b84d6 Mon Sep 17 00:00:00 2001 From: Yaroslav Halchenko Date: Sun, 13 Sep 2015 10:45:39 -0400 Subject: [PATCH 016/126] DOC: Changelog entry for this fix --- ChangeLog | 3 +++ 1 file changed, 3 insertions(+) diff --git a/ChangeLog b/ChangeLog index 811473e7..4a95727b 100644 --- a/ChangeLog +++ b/ChangeLog @@ -14,6 +14,9 @@ ver. 0.9.4 (2015/XX/XXX) - wanna-be-released * Fix dnsToIp resolver for fqdn with large list of IPs (gh-1164) * filter.d/apache-badbots.conf - Updated useragent string regex adding escape for `+` + * Treat failed and killed execution of commands identically (only + different log messages), which addresses different behavior on different + exit codes of dash and bash (gh-1155) - New Features: From db1a3f17e18781d5422daf58e45bb3a811b19b47 Mon Sep 17 00:00:00 2001 From: Yaroslav Halchenko Date: Wed, 16 Sep 2015 08:56:46 -0400 Subject: [PATCH 017/126] ENH: new date pattern with year after day (not after entire entry) --- ChangeLog | 2 ++ fail2ban/server/datedetector.py | 4 ++++ fail2ban/tests/datedetectortestcase.py | 1 + 3 files changed, 7 insertions(+) diff --git a/ChangeLog b/ChangeLog index 4a95727b..cf1fcbf4 100644 --- a/ChangeLog +++ b/ChangeLog @@ -22,6 +22,8 @@ ver. 0.9.4 (2015/XX/XXX) - wanna-be-released - Enhancements: * Do not rotate empty log files + * Added new date pattern with year after day (e.g. Sun Jan 23 2005 21:59:59) + http://bugs.debian.org/798923 ver. 0.9.3 (2015/08/01) - lets-all-stay-friends ---------- diff --git a/fail2ban/server/datedetector.py b/fail2ban/server/datedetector.py index 95d368b5..a4839f5c 100644 --- a/fail2ban/server/datedetector.py +++ b/fail2ban/server/datedetector.py @@ -78,6 +78,10 @@ class DateDetector(object): # asctime with optional day, subsecond and/or year: # Sun Jan 23 21:59:59.011 2005 self.appendTemplate("(?:%a )?%b %d %H:%M:%S(?:\.%f)?(?: %Y)?") + # asctime with optional day, subsecond and/or year coming after day + # http://bugs.debian.org/798923 + # Sun Jan 23 2005 21:59:59.011 + self.appendTemplate("(?:%a )?%b %d %Y %H:%M:%S(?:\.%f)?") # simple date, optional subsecond (proftpd): # 2005-01-23 21:59:59 # simple date: 2005/01/23 21:59:59 diff --git a/fail2ban/tests/datedetectortestcase.py b/fail2ban/tests/datedetectortestcase.py index 0d758640..facfda6d 100644 --- a/fail2ban/tests/datedetectortestcase.py +++ b/fail2ban/tests/datedetectortestcase.py @@ -74,6 +74,7 @@ class DateDetectorTest(unittest.TestCase): (False, "Jan 23 21:59:59"), (False, "Sun Jan 23 21:59:59 2005"), (False, "Sun Jan 23 21:59:59"), + (False, "Sun Jan 23 2005 21:59:59"), (False, "2005/01/23 21:59:59"), (False, "2005.01.23 21:59:59"), (False, "23/01/2005 21:59:59"), From 17a42897988f154f8f52c9babbb924fcadca709c Mon Sep 17 00:00:00 2001 From: Yaroslav Halchenko Date: Wed, 23 Sep 2015 08:38:51 -0400 Subject: [PATCH 018/126] BF: relax 1 sec delay testing to 100ms margin (Closes #1195) --- fail2ban/tests/servertestcase.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/fail2ban/tests/servertestcase.py b/fail2ban/tests/servertestcase.py index 6f946cc0..afa6162e 100644 --- a/fail2ban/tests/servertestcase.py +++ b/fail2ban/tests/servertestcase.py @@ -169,7 +169,8 @@ class Transmitter(TransmitterBase): self.assertEqual(self.transm.proceed(["sleep", "1"]), (0, None)) t1 = time.time() # Approx 1 second delay - self.assertAlmostEqual(t1 - t0, 1, places=1) + self.assertGreater(t1 - t0, 0.99) # shouldn't be faster than 1sec + self.assertGreater(1.1, t1 - t0) def testDatabase(self): tmp, tmpFilename = tempfile.mkstemp(".db", "fail2ban_") From 84afcd8b1f57c3115e77f9b75ae86ba9de98b21f Mon Sep 17 00:00:00 2001 From: Yaroslav Halchenko Date: Wed, 23 Sep 2015 09:45:51 -0400 Subject: [PATCH 019/126] BF(PY26): no assertGreater in 2.6 -- use explicit comparison --- fail2ban/tests/servertestcase.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/fail2ban/tests/servertestcase.py b/fail2ban/tests/servertestcase.py index afa6162e..86ffdb46 100644 --- a/fail2ban/tests/servertestcase.py +++ b/fail2ban/tests/servertestcase.py @@ -168,9 +168,9 @@ class Transmitter(TransmitterBase): t0 = time.time() self.assertEqual(self.transm.proceed(["sleep", "1"]), (0, None)) t1 = time.time() - # Approx 1 second delay - self.assertGreater(t1 - t0, 0.99) # shouldn't be faster than 1sec - self.assertGreater(1.1, t1 - t0) + # Approx 1 second delay but not faster + dt = t1 - t0 + self.assertTrue(0.99 < dt < 1.1, msg="Sleep was %g sec" % dt) def testDatabase(self): tmp, tmpFilename = tempfile.mkstemp(".db", "fail2ban_") From 8cf614e2219c999564ce669c5ce0dc29364acb29 Mon Sep 17 00:00:00 2001 From: Yaroslav Halchenko Date: Wed, 23 Sep 2015 12:13:52 -0400 Subject: [PATCH 020/126] ENH: allow to split ignoreip by space and/or comma (Closes #1197) Way too many people ran into this gotcha, so lets just do it --- ChangeLog | 1 + config/jail.conf | 2 +- fail2ban/client/jailreader.py | 7 +++---- fail2ban/helpers.py | 10 ++++++++++ fail2ban/tests/misctestcase.py | 9 +++++++++ 5 files changed, 24 insertions(+), 5 deletions(-) diff --git a/ChangeLog b/ChangeLog index fea070ba..e2dbf69d 100644 --- a/ChangeLog +++ b/ChangeLog @@ -25,6 +25,7 @@ ver. 0.9.4 (2015/XX/XXX) - wanna-be-released * Added new date pattern with year after day (e.g. Sun Jan 23 2005 21:59:59) http://bugs.debian.org/798923 * Added openSUSE path configuration (Thanks Johannes Weberhofer) + * Allow to split ignoreip entries by ',' as well as by ' ' (gh-1197) ver. 0.9.3 (2015/08/01) - lets-all-stay-friends ---------- diff --git a/config/jail.conf b/config/jail.conf index b6f13840..7500f4ff 100644 --- a/config/jail.conf +++ b/config/jail.conf @@ -46,7 +46,7 @@ before = paths-debian.conf # "ignoreip" can be an IP address, a CIDR mask or a DNS host. Fail2ban will not # ban a host which matches an address in this list. Several addresses can be -# defined using space separator. +# defined using space (and/or comma) separator. ignoreip = 127.0.0.1/8 # External command that will take an tagged arguments to ignore, e.g. , diff --git a/fail2ban/client/jailreader.py b/fail2ban/client/jailreader.py index 6d0fddfa..54ac59fa 100644 --- a/fail2ban/client/jailreader.py +++ b/fail2ban/client/jailreader.py @@ -33,6 +33,7 @@ from .configreader import ConfigReaderUnshared, ConfigReader from .filterreader import FilterReader from .actionreader import ActionReader from ..helpers import getLogger +from ..helpers import splitcommaspace # Gets the instance of the logger. logSys = getLogger(__name__) @@ -208,10 +209,8 @@ class JailReader(ConfigReader): elif opt == "maxretry": stream.append(["set", self.__name, "maxretry", self.__opts[opt]]) elif opt == "ignoreip": - for ip in self.__opts[opt].split(): - # Do not send a command if the rule is empty. - if ip != '': - stream.append(["set", self.__name, "addignoreip", ip]) + for ip in splitcommaspace(self.__opts[opt]): + stream.append(["set", self.__name, "addignoreip", ip]) elif opt == "findtime": stream.append(["set", self.__name, "findtime", self.__opts[opt]]) elif opt == "bantime": diff --git a/fail2ban/helpers.py b/fail2ban/helpers.py index f5c3163a..8e1b0e32 100644 --- a/fail2ban/helpers.py +++ b/fail2ban/helpers.py @@ -127,3 +127,13 @@ def excepthook(exctype, value, traceback): getLogger("fail2ban").critical( "Unhandled exception in Fail2Ban:", exc_info=True) return sys.__excepthook__(exctype, value, traceback) + +def splitcommaspace(s): + """Helper to split on any comma or space + + Returns empty list if input is empty (or None) and filters + out empty entries + """ + if not s: + return [] + return filter(bool, re.split('[ ,]', s)) diff --git a/fail2ban/tests/misctestcase.py b/fail2ban/tests/misctestcase.py index c95efa43..e28ce422 100644 --- a/fail2ban/tests/misctestcase.py +++ b/fail2ban/tests/misctestcase.py @@ -33,6 +33,7 @@ from glob import glob from StringIO import StringIO from ..helpers import formatExceptionInfo, mbasename, TraceBack, FormatterWithTraceBack, getLogger +from ..helpers import splitcommaspace from ..server.datetemplate import DatePatternRegex @@ -55,6 +56,14 @@ class HelpersTest(unittest.TestCase): # might be fragile due to ' vs " self.assertEqual(args, "('Very bad', None)") + def testsplitcommaspace(self): + self.assertEqual(splitcommaspace(None), []) + self.assertEqual(splitcommaspace(''), []) + self.assertEqual(splitcommaspace(' '), []) + self.assertEqual(splitcommaspace('1'), ['1']) + self.assertEqual(splitcommaspace(' 1 2 '), ['1', '2']) + self.assertEqual(splitcommaspace(' 1, 2 , '), ['1', '2']) + class SetupTest(unittest.TestCase): From 4744e165394c696144499d95d79ab5b55b556cbd Mon Sep 17 00:00:00 2001 From: Ryan Yoosefi Date: Thu, 24 Sep 2015 06:37:01 -0700 Subject: [PATCH 021/126] README :: Some style/grammar tweaks, and init/service script mention. Re: #1193 --- README.md | 29 ++++++++++++++++++++--------- 1 file changed, 20 insertions(+), 9 deletions(-) diff --git a/README.md b/README.md index d9539a32..a64bb943 100644 --- a/README.md +++ b/README.md @@ -6,13 +6,16 @@ ## Fail2Ban: ban hosts that cause multiple authentication errors -Fail2Ban scans log files like /var/log/pwdfail and bans IP that makes too many -password failures. It updates firewall rules to reject the IP address. These -rules can be defined by the user. Fail2Ban can read multiple log files such as -sshd or Apache web server ones. +Fail2Ban scans log files like `/var/log/auth.log` and bans IP addresses +having too many failed login attempts. +It does this by updating system firewall rules to reject new connections +from those IP addresses, for a configurable amount of time. +Fail2Ban comes out-of-the-box ready to read many standard log files, such as those +for sshd and Apache, and is easy to configure to read any log file you choose, for +any error you choose. -Fail2Ban is able to reduce the rate of incorrect authentications attempts -however it cannot eliminate the risk that weak authentication presents. +Though Fail2Ban is able to reduce the rate of incorrect authentications attempts, +it cannot eliminate the risk that weak authentication presents. Configure services to use only two factor or public/private authentication mechanisms if you really want to protect services. @@ -42,7 +45,7 @@ To install, just do: python setup.py install This will install Fail2Ban into the python library directory. The executable -scripts are placed into /usr/bin, and configuration under /etc/fail2ban. +scripts are placed into `/usr/bin`, and configuration under `/etc/fail2ban`. Fail2Ban should be correctly installed now. Just type: @@ -51,11 +54,19 @@ Fail2Ban should be correctly installed now. Just type: to see if everything is alright. You should always use fail2ban-client and never call fail2ban-server directly. +Please note that the system init/service script is not automatically installed. +To enable fail2ban as an automatic service, simply copy the script for your +distro from the `files` directory to `/etc/init.d`. Example: + + cp files/debian-initd /etc/init.d/fail2ban + update-rc.d fail2ban defaults + service fail2ban start + Configuration: -------------- -You can configure Fail2Ban using the files in /etc/fail2ban. It is possible to -configure the server using commands sent to it by fail2ban-client. The +You can configure Fail2Ban using the files in `/etc/fail2ban`. It is possible to +configure the server using commands sent to it by `fail2ban-client`. The available commands are described in the fail2ban-client(1) manpage. Also see fail2ban(1) and jail.conf(5) manpages for further references. From d618ee3d90faeaacea1096ac43af36450527d604 Mon Sep 17 00:00:00 2001 From: Yaroslav Halchenko Date: Thu, 24 Sep 2015 09:53:55 -0400 Subject: [PATCH 022/126] BF: disable testing on python 3.2 until coverage gets a fix --- .travis.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index f65e4896..adb41e7d 100644 --- a/.travis.yml +++ b/.travis.yml @@ -6,7 +6,8 @@ python: - 2.6 - 2.7 - pypy - - 3.2 + # disabled until coverage module fixes up compatibility issue + # - 3.2 - 3.3 - 3.4 - pypy3 From c1b80a5e1bb9d426e87d4eec3285cf20c405438b Mon Sep 17 00:00:00 2001 From: Ryan Yoosefi Date: Fri, 25 Sep 2015 02:23:08 -0700 Subject: [PATCH 023/126] README :: fitted paragraph style --- README.md | 17 ++++++++--------- 1 file changed, 8 insertions(+), 9 deletions(-) diff --git a/README.md b/README.md index a64bb943..0be4920a 100644 --- a/README.md +++ b/README.md @@ -6,16 +6,15 @@ ## Fail2Ban: ban hosts that cause multiple authentication errors -Fail2Ban scans log files like `/var/log/auth.log` and bans IP addresses -having too many failed login attempts. -It does this by updating system firewall rules to reject new connections -from those IP addresses, for a configurable amount of time. -Fail2Ban comes out-of-the-box ready to read many standard log files, such as those -for sshd and Apache, and is easy to configure to read any log file you choose, for -any error you choose. +Fail2Ban scans log files like `/var/log/auth.log` and bans IP addresses having +too many failed login attempts. It does this by updating system firewall rules +to reject new connections from those IP addresses, for a configurable amount +of time. Fail2Ban comes out-of-the-box ready to read many standard log files, +such as those for sshd and Apache, and is easy to configure to read any log +file you choose, for any error you choose. -Though Fail2Ban is able to reduce the rate of incorrect authentications attempts, -it cannot eliminate the risk that weak authentication presents. +Though Fail2Ban is able to reduce the rate of incorrect authentications +attempts, it cannot eliminate the risk that weak authentication presents. Configure services to use only two factor or public/private authentication mechanisms if you really want to protect services. From 0610791ffecaaade6ed844ad59f99b284e203807 Mon Sep 17 00:00:00 2001 From: Ryan Yoosefi Date: Fri, 25 Sep 2015 02:25:11 -0700 Subject: [PATCH 024/126] README :: init/service example mentions debian based systems as the example --- README.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 0be4920a..cbc075e2 100644 --- a/README.md +++ b/README.md @@ -55,7 +55,8 @@ never call fail2ban-server directly. Please note that the system init/service script is not automatically installed. To enable fail2ban as an automatic service, simply copy the script for your -distro from the `files` directory to `/etc/init.d`. Example: +distro from the `files` directory to `/etc/init.d`. Example (on a Debian-based +system): cp files/debian-initd /etc/init.d/fail2ban update-rc.d fail2ban defaults From 2895d981fa8b91cc6ea2bb74f325453c48dfb31d Mon Sep 17 00:00:00 2001 From: "M. Maraun" Date: Fri, 3 Jul 2015 22:42:22 +0200 Subject: [PATCH 025/126] Set Timeout at urlopen to 3 seconds --- THANKS | 1 + config/action.d/badips.py | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/THANKS b/THANKS index 7bf723c5..68c7af48 100644 --- a/THANKS +++ b/THANKS @@ -71,6 +71,7 @@ kojiro Lars Kneschke Lee Clemens leftyfb (Mike Rushton) +M. Maraun Manuel Arostegui Ramirez Marcel Dopita Mark Edgington diff --git a/config/action.d/badips.py b/config/action.d/badips.py index a1df00a3..99e1866a 100644 --- a/config/action.d/badips.py +++ b/config/action.d/badips.py @@ -117,7 +117,7 @@ class BadIPsAction(ActionBase): """ try: response = urlopen( - self._Request("/".join([self._badips, "get", "categories"]))) + self._Request("/".join([self._badips, "get", "categories"])), None, 3) except HTTPError as response: messages = json.loads(response.read().decode('utf-8')) self._logSys.error( From 6c0f898ec7c5165ca50ee683e8cf371ed9b5a5a4 Mon Sep 17 00:00:00 2001 From: Yaroslav Halchenko Date: Sun, 27 Sep 2015 00:49:57 -0400 Subject: [PATCH 026/126] DOC: changelog for the timeout change --- ChangeLog | 2 ++ 1 file changed, 2 insertions(+) diff --git a/ChangeLog b/ChangeLog index fea070ba..c5fd30aa 100644 --- a/ChangeLog +++ b/ChangeLog @@ -25,6 +25,8 @@ ver. 0.9.4 (2015/XX/XXX) - wanna-be-released * Added new date pattern with year after day (e.g. Sun Jan 23 2005 21:59:59) http://bugs.debian.org/798923 * Added openSUSE path configuration (Thanks Johannes Weberhofer) + * Added a timeout (3 sec) to urlopen within badips.py action + (Thanks M. Maraun) ver. 0.9.3 (2015/08/01) - lets-all-stay-friends ---------- From 0d8968daa99b4a178271ab1db9de407124a7f26e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Viktor=20Sz=C3=A9pe?= Date: Wed, 30 Sep 2015 16:07:45 +0200 Subject: [PATCH 027/126] Added CloudFlare API error codes URL --- config/action.d/cloudflare.conf | 2 ++ 1 file changed, 2 insertions(+) diff --git a/config/action.d/cloudflare.conf b/config/action.d/cloudflare.conf index 4bc90c97..aa87163c 100644 --- a/config/action.d/cloudflare.conf +++ b/config/action.d/cloudflare.conf @@ -9,6 +9,8 @@ # Referenced from http://www.normyee.net/blog/2012/02/02/adding-cloudflare-support-to-fail2ban by NORM YEE # # To get your CloudFlare API Key: https://www.cloudflare.com/a/account/my-account +# +# CloudFlare API error codes: https://www.cloudflare.com/docs/host-api.html#s4.2 [Definition] From 36919d9f976f6140a5bc565cb5aa10068b301a2e Mon Sep 17 00:00:00 2001 From: Kevin Locke Date: Fri, 2 Oct 2015 15:19:42 -0700 Subject: [PATCH 028/126] ssh.conf: Fix disconnect "Auth fail" matching The regex for matching against "Auth fail" disconnect log message does not match against current versions of ssh. OpenSSH 5.9 introduced privilege separation of the pre-auth process, which included [logging through monitor.c](http://cvsweb.openbsd.org/cgi-bin/cvsweb/src/usr.bin/ssh/monitor.c.diff?r1=1.113&r2=1.114) which adds " [preauth]" to the end of each message and causes the log level to be prepended to each message. It also fails to match against clients which send a disconnect message with a description that is either empty or includes a space, since this is the content in the log message after the disconnect code, per [packet.c:1785](http://cvsweb.openbsd.org/cgi-bin/cvsweb/src/usr.bin/ssh/packet.c?annotate=1.215), which was matched by \S+. Although I have not observed this yet, I couldn't find anything which would preclude it in [RFC 4253](https://tools.ietf.org/html/rfc4253#section-11.1) and since the message is attacker-controlled it provides a way to avoid getting banned. This commit fixes both issues. Signed-off-by: Kevin Locke --- config/filter.d/sshd.conf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config/filter.d/sshd.conf b/config/filter.d/sshd.conf index b000cd49..5fad2b32 100644 --- a/config/filter.d/sshd.conf +++ b/config/filter.d/sshd.conf @@ -27,7 +27,7 @@ failregex = ^%(__prefix_line)s(?:error: PAM: )?[aA]uthentication (?:failure|erro ^%(__prefix_line)sUser .+ from not allowed because listed in DenyUsers\s*$ ^%(__prefix_line)sUser .+ from not allowed because not in any group\s*$ ^%(__prefix_line)srefused connect from \S+ \(\)\s*$ - ^%(__prefix_line)sReceived disconnect from : 3: \S+: Auth fail$ + ^%(__prefix_line)s(?:error: )?Received disconnect from : 3: .*: Auth fail(?: \[preauth\])?$ ^%(__prefix_line)sUser .+ from not allowed because a group is listed in DenyGroups\s*$ ^%(__prefix_line)sUser .+ from not allowed because none of user's groups are listed in AllowGroups\s*$ ^(?P<__prefix>%(__prefix_line)s)User .+ not allowed because account is locked(?P=__prefix)(?:error: )?Received disconnect from : 11: .+ \[preauth\]$ From 42b0e9258dc90556f63095abedd2ca72dd3f5670 Mon Sep 17 00:00:00 2001 From: Kevin Locke Date: Fri, 2 Oct 2015 15:56:26 -0700 Subject: [PATCH 029/126] Test cases for ssh.conf disconnect "Auth fail" Add test coverage for the new disconnect "Auth fail" matching added in 36919d9f. Signed-off-by: Kevin Locke --- fail2ban/tests/files/logs/sshd | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/fail2ban/tests/files/logs/sshd b/fail2ban/tests/files/logs/sshd index 2f4400bb..62204339 100644 --- a/fail2ban/tests/files/logs/sshd +++ b/fail2ban/tests/files/logs/sshd @@ -132,6 +132,12 @@ Nov 23 21:50:37 sshd[7148]: Connection closed by 61.0.0.1 [preauth] # failJSON: { "time": "2005-07-13T18:44:28", "match": true , "host": "89.24.13.192", "desc": "from gh-289" } Jul 13 18:44:28 mdop sshd[4931]: Received disconnect from 89.24.13.192: 3: com.jcraft.jsch.JSchException: Auth fail +# failJSON: { "time": "2004-10-01T17:27:44", "match": true , "host": "94.249.236.6", "desc": "newer format per commit 36919d9f" } +Oct 1 17:27:44 localhost sshd[24077]: error: Received disconnect from 94.249.236.6: 3: com.jcraft.jsch.JSchException: Auth fail [preauth] + +# failJSON: { "time": "2004-10-01T17:27:44", "match": true , "host": "94.249.236.6", "desc": "space in disconnect description per commit 36919d9f" } +Oct 1 17:27:44 localhost sshd[24077]: error: Received disconnect from 94.249.236.6: 3: Ha ha, suckers!: Auth fail [preauth] + # failJSON: { "match": false } Feb 12 04:09:18 localhost sshd[26713]: Connection from 115.249.163.77 port 51353 # failJSON: { "time": "2005-02-12T04:09:21", "match": true , "host": "115.249.163.77", "desc": "from gh-457" } From 2a5c93cfb54b1f307d0360c550a7a9477e7cfb0e Mon Sep 17 00:00:00 2001 From: Kevin Locke Date: Mon, 5 Oct 2015 00:31:13 -0700 Subject: [PATCH 030/126] Update ChangeLog and THANKS for "Auth fail" changes Document the changes from 36919d9f in the ChangeLog and add myself to the THANKS file (at @sebres suggestion). Signed-off-by: Kevin Locke --- ChangeLog | 2 ++ THANKS | 1 + 2 files changed, 3 insertions(+) diff --git a/ChangeLog b/ChangeLog index 2c2636a8..3bdd867a 100644 --- a/ChangeLog +++ b/ChangeLog @@ -14,6 +14,8 @@ ver. 0.9.4 (2015/XX/XXX) - wanna-be-released * Fix dnsToIp resolver for fqdn with large list of IPs (gh-1164) * filter.d/apache-badbots.conf - Updated useragent string regex adding escape for `+` + * filter.d/sshd.conf + - Updated "Auth fail" regex for OpenSSH 5.9 and later * Treat failed and killed execution of commands identically (only different log messages), which addresses different behavior on different exit codes of dash and bash (gh-1155) diff --git a/THANKS b/THANKS index 68c7af48..5cea437b 100644 --- a/THANKS +++ b/THANKS @@ -65,6 +65,7 @@ Joël Bertrand JP Espinosa jserrachinha Justin Shore +Kevin Locke Kévin Drapel kjohnsonecl kojiro From 61ac48170308da590d317cac7e0b846461a86352 Mon Sep 17 00:00:00 2001 From: sebres Date: Wed, 7 Oct 2015 13:27:45 +0200 Subject: [PATCH 031/126] IpToName test case fixed ('66.249.66.1' resp. 'crawl-66-249-66-1.googlebot.com' seems to be unresolvable) --- fail2ban/tests/filtertestcase.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/fail2ban/tests/filtertestcase.py b/fail2ban/tests/filtertestcase.py index acce9625..acf15528 100644 --- a/fail2ban/tests/filtertestcase.py +++ b/fail2ban/tests/filtertestcase.py @@ -1025,8 +1025,8 @@ class DNSUtilsTests(unittest.TestCase): self.assertEqual(res, []) def testIpToName(self): - res = DNSUtils.ipToName('66.249.66.1') - self.assertEqual(res, 'crawl-66-249-66-1.googlebot.com') + res = DNSUtils.ipToName('8.8.4.4') + self.assertEqual(res, 'google-public-dns-b.google.com') # invalid ip (TEST-NET-1 according to RFC 5737) res = DNSUtils.ipToName('192.0.2.0') self.assertEqual(res, None) From 2696ede2514261c67123b1e3ba74b6f73b628369 Mon Sep 17 00:00:00 2001 From: sebres Date: Wed, 7 Oct 2015 14:34:13 +0200 Subject: [PATCH 032/126] mysqld-auth: Updated "Access denied ..." regex for MySQL 5.6 and later closes gh-1211 --- ChangeLog | 2 ++ config/filter.d/mysqld-auth.conf | 2 +- fail2ban/tests/files/logs/mysqld-auth | 2 ++ 3 files changed, 5 insertions(+), 1 deletion(-) diff --git a/ChangeLog b/ChangeLog index 3bdd867a..9e145b65 100644 --- a/ChangeLog +++ b/ChangeLog @@ -14,6 +14,8 @@ ver. 0.9.4 (2015/XX/XXX) - wanna-be-released * Fix dnsToIp resolver for fqdn with large list of IPs (gh-1164) * filter.d/apache-badbots.conf - Updated useragent string regex adding escape for `+` + * filter.d/mysqld-auth.conf + - Updated "Access denied ..." regex for MySQL 5.6 and later (gh-1211) * filter.d/sshd.conf - Updated "Auth fail" regex for OpenSSH 5.9 and later * Treat failed and killed execution of commands identically (only diff --git a/config/filter.d/mysqld-auth.conf b/config/filter.d/mysqld-auth.conf index 92dc9a99..f29beffc 100644 --- a/config/filter.d/mysqld-auth.conf +++ b/config/filter.d/mysqld-auth.conf @@ -17,7 +17,7 @@ before = common.conf _daemon = mysqld -failregex = ^%(__prefix_line)s(\d{6} \s?\d{1,2}:\d{2}:\d{2} )?\[Warning\] Access denied for user '\w+'@'' (to database '[^']*'|\(using password: (YES|NO)\))*\s*$ +failregex = ^%(__prefix_line)s(?:\d+ |\d{6} \s?\d{1,2}:\d{2}:\d{2} )?\[Warning\] Access denied for user '\w+'@'' (to database '[^']*'|\(using password: (YES|NO)\))*\s*$ ignoreregex = diff --git a/fail2ban/tests/files/logs/mysqld-auth b/fail2ban/tests/files/logs/mysqld-auth index aa684266..b75ecfc4 100644 --- a/fail2ban/tests/files/logs/mysqld-auth +++ b/fail2ban/tests/files/logs/mysqld-auth @@ -15,3 +15,5 @@ Sep 16 21:30:26 catinthehat mysqld: 130916 21:30:26 [Warning] Access denied for # failJSON: { "time": "2004-09-16T21:30:32", "match": true , "host": "74.207.241.159" } Sep 16 21:30:32 catinthehat mysqld: 130916 21:30:32 [Warning] Access denied for user 'hacker'@'74.207.241.159' (using password: NO) +# failJSON: { "time": "2015-10-07T06:09:42", "match": true , "host": "127.0.0.1", "desc": "mysql 5.6 log format" } +2015-10-07 06:09:42 5907 [Warning] Access denied for user 'root'@'127.0.0.1' (using password: YES) \ No newline at end of file From 078e2048f237fc0929483f78d620c95cad2b5f33 Mon Sep 17 00:00:00 2001 From: Paul Menzel Date: Thu, 8 Oct 2015 16:09:46 +0200 Subject: [PATCH 033/126] files: Strip trailing whitespace from files Run the command `StripWhitespace` from the [Vim Better Whitespace Plugin](https://github.com/ntpeters/vim-better-whitespace). --- files/bash-completion | 2 +- files/gen_badbots | 2 +- files/nagios/README | 10 +++++----- files/nagios/check_fail2ban | 22 +++++++++++----------- files/solaris-svc-fail2ban | 2 +- files/suse-initd | 4 ++-- 6 files changed, 21 insertions(+), 21 deletions(-) diff --git a/files/bash-completion b/files/bash-completion index 57ec15b3..36e0cbba 100644 --- a/files/bash-completion +++ b/files/bash-completion @@ -31,7 +31,7 @@ __fail2ban_jail_action_methods () { _fail2ban () { local cur prev words cword - _init_completion || return + _init_completion || return case $prev in -V|--version|-h|--help) diff --git a/files/gen_badbots b/files/gen_badbots index 52732317..75a6a0d5 100755 --- a/files/gen_badbots +++ b/files/gen_badbots @@ -26,7 +26,7 @@ # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License -# along with this program; if not, write to the +# along with this program; if not, write to the # Free Software Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, # MA 02110-1301, USA. # diff --git a/files/nagios/README b/files/nagios/README index 2b855d83..324260ff 100644 --- a/files/nagios/README +++ b/files/nagios/README @@ -8,8 +8,8 @@ How to use ---------- Just have to run the following command: $ ./check_fail2ban --help - -If you need to use this script with NRPE you just have to do the + +If you need to use this script with NRPE you just have to do the following steps: 1 allow your user to run the script with the sudo rights. Just add @@ -20,7 +20,7 @@ following steps: command[check_fail2ban]=/usr/bin/sudo //check_fail2ban 3 don't forget to restart your NRPE daemon - + /!\ be careful to let no one able to update the check_fail2ban ;) ------------------------------------------------------------------------------ @@ -37,7 +37,7 @@ HELP: 2.) delete the socket if available rm /var/run/fail2ban/fail2ban.sock -3.) start the Service +3.) start the Service /etc/init.d/fail2ban start 4.) check if fail2ban is working @@ -58,7 +58,7 @@ Options: -V, --version Print version information -D, --display=STRING - To modify the output display + To modify the output display default is "CHECK FAIL2BAN ACTIVITY" -P, --path-fail2ban_client=STRING Specify the path to the tw_cli binary diff --git a/files/nagios/check_fail2ban b/files/nagios/check_fail2ban index fc88e654..bbfa63aa 100755 --- a/files/nagios/check_fail2ban +++ b/files/nagios/check_fail2ban @@ -4,12 +4,12 @@ # -=- -=- # ------------------------------------------------------- # -# Description : This plugin checks if the fail2ban server is running +# Description : This plugin checks if the fail2ban server is running # and how many IPs are currently banned. -# +# # # inspired by the work of Sebastian Mueller - http://www.elchtest.eu -# +# # # Version : 0.1 # ------------------------------------------------------- @@ -17,7 +17,7 @@ # - see the How to use section # # Out : -# - only print on the standard output +# - only print on the standard output # # Features : # - perfdata output @@ -51,8 +51,8 @@ # # Just have to run the following command: # $ ./check_fail2ban --help -# -# If you need to use this script with NRPE you just have to do the +# +# If you need to use this script with NRPE you just have to do the # following steps: # # 1 allow your user to run the script with the sudo rights. Just add @@ -64,7 +64,7 @@ # # 3 don't forget to restart your NRPE daemon # -# +# # /!\ be careful to let no one able to update the check_fail2ban ;) # ------------------------------------------------------------------------------ # @@ -251,7 +251,7 @@ Options: -V, --version Print version information -D, --display=STRING - To modify the output display + To modify the output display default is "CHECK FAIL2BAN ACTIVITY" -P, --path-fail2ban_client=STRING Specify the path to the tw_cli binary @@ -269,7 +269,7 @@ Options: If you want to activate the perfdata output -v, --verbose Show details for command-line debugging (Nagios may truncate the output) - + Send email to $a_mail if you have questions regarding use of this software. To submit patches or suggest improvements, send email to $a_mail @@ -315,7 +315,7 @@ sub obtain_jail_list { if ($return_code) { return -1; } - + my @jail_list; foreach (@command_output) { if ($_=~/^.*Jail list:\t+(.*)/) { @@ -323,7 +323,7 @@ sub obtain_jail_list { @jail_list = split(/,/, $1); } } - + return @jail_list; } diff --git a/files/solaris-svc-fail2ban b/files/solaris-svc-fail2ban index e397474b..96415323 100755 --- a/files/solaris-svc-fail2ban +++ b/files/solaris-svc-fail2ban @@ -2,7 +2,7 @@ # # fail2ban This init.d script is used to start fail2ban. # (C) by Hanno Wagner , License is GPL - + #set -x . /lib/svc/share/smf_include.sh diff --git a/files/suse-initd b/files/suse-initd index 09c25687..ddd26ec5 100644 --- a/files/suse-initd +++ b/files/suse-initd @@ -60,12 +60,12 @@ case "$1" in if [ -f $FAIL2BAN_SOCKET ] then - echo "$FAIL2BAN_SOCKET not removed .. removing .." + echo "$FAIL2BAN_SOCKET not removed .. removing .." rm $FAIL2BAN_SOCKET fi if [ -f $FAIL2BAN_PID ] then - echo "$FAIL2BAN_PID not removed .. removing .." + echo "$FAIL2BAN_PID not removed .. removing .." rm $FAIL2BAN_PID fi From 6fb5e3a494d8ec2385e9c44adab7ae3c644f099c Mon Sep 17 00:00:00 2001 From: Yaroslav Halchenko Date: Fri, 9 Oct 2015 14:10:02 -0400 Subject: [PATCH 034/126] removed outdated and "problematic" .pydevproject --- .pydevproject | 12 ------------ 1 file changed, 12 deletions(-) delete mode 100644 .pydevproject diff --git a/.pydevproject b/.pydevproject deleted file mode 100644 index 3c3130d9..00000000 --- a/.pydevproject +++ /dev/null @@ -1,12 +0,0 @@ - - - - -python 2.3 - -/fail2ban-0.8/client -/fail2ban-0.8/server -/fail2ban-0.8/testcases -/fail2ban-0.8 - - From 617302fcc2d0b6dee89593ba6d15b58887bdc832 Mon Sep 17 00:00:00 2001 From: agentmoller001 Date: Fri, 9 Oct 2015 18:16:36 -0700 Subject: [PATCH 035/126] Updated route.conf to clear warnings Does not throw warnings when starting/restarting by adding three lines of code. --- config/action.d/route.conf | 3 +++ 1 file changed, 3 insertions(+) diff --git a/config/action.d/route.conf b/config/action.d/route.conf index 123245e5..9b96a7b5 100644 --- a/config/action.d/route.conf +++ b/config/action.d/route.conf @@ -17,6 +17,9 @@ [Definition] actionban = ip route add actionunban = ip route del +actioncheck = +actionstart = +actionstop = [Init] From a28e6b442e50bf17cefc886b3873ebd9a418a235 Mon Sep 17 00:00:00 2001 From: Pablo Rodriguez Fernandez Date: Tue, 13 Oct 2015 09:55:28 +0200 Subject: [PATCH 036/126] Add check in apache-fakegooglebot to protect against PTR fake record An attacker may return a PTR record which fakes a Googlebot's domain name. This modification resolves the PTR records to verify it. See "Verifying Googlebot": --- ChangeLog | 2 ++ THANKS | 1 + config/filter.d/ignorecommands/apache-fakegooglebot | 5 ++++- 3 files changed, 7 insertions(+), 1 deletion(-) diff --git a/ChangeLog b/ChangeLog index 9e145b65..2f5a158b 100644 --- a/ChangeLog +++ b/ChangeLog @@ -32,6 +32,8 @@ ver. 0.9.4 (2015/XX/XXX) - wanna-be-released * Allow to split ignoreip entries by ',' as well as by ' ' (gh-1197) * Added a timeout (3 sec) to urlopen within badips.py action (Thanks M. Maraun) + * Added check against atacker's Googlebot PTR fake records + (Thanks Pablo Rodriguez Fernandez) ver. 0.9.3 (2015/08/01) - lets-all-stay-friends ---------- diff --git a/THANKS b/THANKS index 5cea437b..45674847 100644 --- a/THANKS +++ b/THANKS @@ -89,6 +89,7 @@ Mika (mkl) Nick Munger onorua Orion Poplawski +Pablo Rodriguez Fernandez Paul Marrapese Paul Traina Noel Butler diff --git a/config/filter.d/ignorecommands/apache-fakegooglebot b/config/filter.d/ignorecommands/apache-fakegooglebot index 47ef51f6..3028d86a 100755 --- a/config/filter.d/ignorecommands/apache-fakegooglebot +++ b/config/filter.d/ignorecommands/apache-fakegooglebot @@ -26,7 +26,10 @@ def is_googlebot(ip): from fail2ban.server.filter import DNSUtils host = DNSUtils.ipToName(ip) - sys.exit(0 if (host and re.match('crawl-.*\.googlebot\.com', host)) else 1) + if not host or not re.match('crawl-.*\.googlebot\.com', host): + sys.exit(1) + host_ips = DNSUtils.dnsToIp(host) + sys.exit(0 if ip in host_ips else 1) if __name__ == '__main__': is_googlebot(process_args(sys.argv)) From 7e6964dd9d0a21cc2219591c628bfd68e8ec0726 Mon Sep 17 00:00:00 2001 From: Pablo Date: Thu, 15 Oct 2015 10:40:56 +0200 Subject: [PATCH 037/126] Fix section jail.conf.5 manpage The section of jail.conf manpage is wrong, should be 5, not 10 --- man/jail.conf.5 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/man/jail.conf.5 b/man/jail.conf.5 index 45eea040..957a04b4 100644 --- a/man/jail.conf.5 +++ b/man/jail.conf.5 @@ -1,4 +1,4 @@ -.TH JAIL.CONF "10" "October 2013" "Fail2Ban" "Fail2Ban Configuration" +.TH JAIL.CONF "5" "October 2013" "Fail2Ban" "Fail2Ban Configuration" .SH NAME jail.conf \- configuration for the fail2ban server .SH SYNOPSIS From 75d33c0f096bda44de2f4bb4cd41703a3e4b86d7 Mon Sep 17 00:00:00 2001 From: Orion Poplawski Date: Wed, 7 Oct 2015 19:50:03 -0600 Subject: [PATCH 038/126] Add *_backend options for services to allow distros to set the default backend per service. Set default to systemd for Fedora as appropriate. --- ChangeLog | 2 ++ config/jail.conf | 28 +++++++++++++++++++++++++++- config/paths-common.conf | 10 ++++++++++ config/paths-fedora.conf | 12 ++++++++++++ 4 files changed, 51 insertions(+), 1 deletion(-) diff --git a/ChangeLog b/ChangeLog index 2f5a158b..08708a08 100644 --- a/ChangeLog +++ b/ChangeLog @@ -34,6 +34,8 @@ ver. 0.9.4 (2015/XX/XXX) - wanna-be-released (Thanks M. Maraun) * Added check against atacker's Googlebot PTR fake records (Thanks Pablo Rodriguez Fernandez) + * Add *_backend options for services to allow distros to set the default + backend per service, set default to systemd for Fedora as appropriate ver. 0.9.3 (2015/08/01) - lets-all-stay-friends ---------- diff --git a/config/jail.conf b/config/jail.conf index 7500f4ff..36fe6bcf 100644 --- a/config/jail.conf +++ b/config/jail.conf @@ -216,6 +216,7 @@ action = %(action_)s port = ssh logpath = %(sshd_log)s +backend = %(sshd_backend)s [sshd-ddos] @@ -224,12 +225,14 @@ logpath = %(sshd_log)s # in the body. port = ssh logpath = %(sshd_log)s +backend = %(sshd_backend)s [dropbear] port = ssh logpath = %(dropbear_log)s +backend = %(dropbear_backend)s [selinux-ssh] @@ -393,6 +396,7 @@ maxretry = 5 port = http,https logpath = %(syslog_daemon)s +backend = %(syslog_backend)s [guacamole] @@ -410,12 +414,14 @@ logpath = /var/log/monit port = 10000 logpath = %(syslog_authpriv)s +backend = %(syslog_backend)s [froxlor-auth] port = http,https logpath = %(syslog_authpriv)s +backend = %(syslog_backend)s # @@ -444,12 +450,14 @@ logpath = /var/log/3proxy.log port = ftp,ftp-data,ftps,ftps-data logpath = %(proftpd_log)s +backend = %(proftpd_backend)s [pure-ftpd] port = ftp,ftp-data,ftps,ftps-data logpath = %(pureftpd_log)s +backend = %(pureftpd_backend)s maxretry = 6 @@ -457,6 +465,7 @@ maxretry = 6 port = ftp,ftp-data,ftps,ftps-data logpath = %(syslog_daemon)s +backend = %(syslog_backend)s maxretry = 6 @@ -464,6 +473,7 @@ maxretry = 6 port = ftp,ftp-data,ftps,ftps-data logpath = %(wuftpd_log)s +backend = %(wuftpd_backend)s maxretry = 6 @@ -491,18 +501,21 @@ logpath = /root/path/to/assp/logs/maillog.txt port = smtp,465,submission logpath = %(syslog_mail)s +backend = %(syslog_backend)s [postfix] port = smtp,465,submission logpath = %(postfix_log)s +backend = %(postfix_backend)s [postfix-rbl] port = smtp,465,submission -logpath = %(syslog_mail)s +logpath = %(postfix_mail)s +backend = %(postfix_backend)s maxretry = 1 @@ -510,12 +523,14 @@ maxretry = 1 port = submission,465,smtp logpath = %(syslog_mail)s +backend = %(syslog_backend)s [sendmail-reject] port = smtp,465,submission logpath = %(syslog_mail)s +backend = %(syslog_backend)s [qmail-rbl] @@ -531,12 +546,14 @@ logpath = /service/qmail/log/main/current port = pop3,pop3s,imap,imaps,submission,465,sieve logpath = %(dovecot_log)s +backend = %(dovecot_backend)s [sieve] port = smtp,465,submission logpath = %(dovecot_log)s +backend = %(dovecot_backend)s [solid-pop3d] @@ -572,6 +589,7 @@ logpath = /opt/kerio/mailserver/store/logs/security.log port = smtp,465,submission,imap3,imaps,pop3,pop3s logpath = %(syslog_mail)s +backend = %(syslog_backend)s [postfix-sasl] @@ -581,12 +599,14 @@ port = smtp,465,submission,imap3,imaps,pop3,pop3s # running postfix since it would provide the same log lines at the # "warn" level but overall at the smaller filesize. logpath = %(postfix_log)s +backend = %(postfix_backend)s [perdition] port = imap3,imaps,pop3,pop3s logpath = %(syslog_mail)s +backend = %(syslog_backend)s [squirrelmail] @@ -599,12 +619,14 @@ logpath = /var/lib/squirrelmail/prefs/squirrelmail_access_log port = imap3,imaps logpath = %(syslog_mail)s +backend = %(syslog_backend)s [uwimap-auth] port = imap3,imaps logpath = %(syslog_mail)s +backend = %(syslog_backend)s # @@ -686,6 +708,7 @@ maxretry = 10 port = 3306 logpath = %(mysql_log)s +backend = %(mysql_backend)s maxretry = 5 @@ -712,12 +735,14 @@ maxretry = 5 # pam-generic filter can be customized to monitor specific subset of 'tty's banaction = iptables-allports logpath = %(syslog_authpriv)s +backend = %(syslog_backend)s [xinetd-fail] banaction = iptables-multiport-log logpath = %(syslog_daemon)s +backend = %(syslog_backend)s maxretry = 2 @@ -748,6 +773,7 @@ action = %(banaction)s[name=%(__name__)s-tcp, port="%(tcpport)s", protocol="tcp enabled = false logpath = %(syslog_daemon)s ; nrpe.cfg may define a different log_facility +backend = %(syslog_backend)s maxretry = 1 diff --git a/config/paths-common.conf b/config/paths-common.conf index bf3cfb6a..7a88e5ce 100644 --- a/config/paths-common.conf +++ b/config/paths-common.conf @@ -8,8 +8,10 @@ after = paths-overrides.local [DEFAULT] sshd_log = %(syslog_authpriv)s +sshd_backend = auto dropbear_log = %(syslog_authpriv)s +dropbear_backend = auto # There is no sensible generic defaults for syslog log targets, thus # leaving them empty here so that no errors while parsing/interpolating configs @@ -18,6 +20,8 @@ syslog_ftp = syslog_local0 = syslog_mail_warn = syslog_user = +# Set the default syslog backend target to auto +syslog_backend = auto # from /etc/audit/auditd.conf auditd_log = /var/log/audit/audit.log @@ -38,14 +42,17 @@ suhosin_log = %(syslog_user)s %(lighttpd_error_log)s # defaults to ftp or local2 if ftp doesn't exist proftpd_log = %(syslog_ftp)s +proftpd_backend = auto # http://svnweb.freebsd.org/ports/head/ftp/proftpd/files/patch-src_proftpd.8.in?view=markup # defaults to ftp but can be overwritten. pureftpd_log = %(syslog_ftp)s +pureftpd_backend = auto # ftp, daemon and then local7 are tried at configure time however it is overwriteable at configure time # wuftpd_log = %(syslog_ftp)s +wuftpd_backend = auto # syslog_enable defaults to no. so it defaults to vsftpd_log_file setting of /var/log/vsftpd.log # No distro seems to set it to syslog by default @@ -54,13 +61,16 @@ vsftpd_log = /var/log/vsftpd.log # Technically syslog_facility in main.cf can overwrite but no-one sane does this. postfix_log = %(syslog_mail_warn)s +postfix_backend = auto dovecot_log = %(syslog_mail_warn)s +dovecot_backend = auto # Seems to be set at compile time only to LOG_LOCAL0 (src/const.h) at Notice level solidpop3d_log = %(syslog_local0)s mysql_log = %(syslog_daemon)s +mysql_backend = auto roundcube_errors_log = /var/log/roundcube/errors diff --git a/config/paths-fedora.conf b/config/paths-fedora.conf index c5601d3c..b3c978ca 100644 --- a/config/paths-fedora.conf +++ b/config/paths-fedora.conf @@ -37,3 +37,15 @@ exim_main_log = /var/log/exim/main.log mysql_log = /var/lib/mysql/mysqld.log roundcube_errors_log = /var/log/roundcubemail/errors + +# These services will log to the journal via syslog, so use the journal by +# default. +syslog_backend = systemd +sshd_backend = systemd +dropbear_backend = systemd +proftpd_backend = systemd +pureftpd_backend = systemd +wuftpd_backend = systemd +postfix_backend = systemd +dovecot_backend = systemd +mysql_backend = systemd From ced7be94b2f60a035355b774e28eb10560e8ee4d Mon Sep 17 00:00:00 2001 From: Orion Poplawski Date: Mon, 19 Oct 2015 19:43:10 -0600 Subject: [PATCH 039/126] Fix postfix_log typo --- config/jail.conf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config/jail.conf b/config/jail.conf index 36fe6bcf..c288a3fd 100644 --- a/config/jail.conf +++ b/config/jail.conf @@ -514,7 +514,7 @@ backend = %(postfix_backend)s [postfix-rbl] port = smtp,465,submission -logpath = %(postfix_mail)s +logpath = %(postfix_log)s backend = %(postfix_backend)s maxretry = 1 From 81a26266a9adc438b4a7347a5bd92c7fb29b35f8 Mon Sep 17 00:00:00 2001 From: Orion Poplawski Date: Mon, 19 Oct 2015 19:46:43 -0600 Subject: [PATCH 040/126] Add changlog entry for postfix-rbl logpath change --- ChangeLog | 1 + 1 file changed, 1 insertion(+) diff --git a/ChangeLog b/ChangeLog index 08708a08..e011be9c 100644 --- a/ChangeLog +++ b/ChangeLog @@ -21,6 +21,7 @@ ver. 0.9.4 (2015/XX/XXX) - wanna-be-released * Treat failed and killed execution of commands identically (only different log messages), which addresses different behavior on different exit codes of dash and bash (gh-1155) + * Use postfix_log logpath for postfix-rbl jail - New Features: From 3a9cf2b3dab47c42b5f4e50b54b31d03b4e93520 Mon Sep 17 00:00:00 2001 From: Orion Poplawski Date: Mon, 19 Oct 2015 19:50:03 -0600 Subject: [PATCH 041/126] Add and use default_backend to set individual backend defaults to auto --- config/paths-common.conf | 22 ++++++++++++---------- 1 file changed, 12 insertions(+), 10 deletions(-) diff --git a/config/paths-common.conf b/config/paths-common.conf index 7a88e5ce..eba6ae4e 100644 --- a/config/paths-common.conf +++ b/config/paths-common.conf @@ -7,11 +7,13 @@ after = paths-overrides.local [DEFAULT] +default_backend = auto + sshd_log = %(syslog_authpriv)s -sshd_backend = auto +sshd_backend = %(default_backend)s dropbear_log = %(syslog_authpriv)s -dropbear_backend = auto +dropbear_backend = %(default_backend)s # There is no sensible generic defaults for syslog log targets, thus # leaving them empty here so that no errors while parsing/interpolating configs @@ -20,8 +22,8 @@ syslog_ftp = syslog_local0 = syslog_mail_warn = syslog_user = -# Set the default syslog backend target to auto -syslog_backend = auto +# Set the default syslog backend target to default_backend +syslog_backend = %(default_backend)s # from /etc/audit/auditd.conf auditd_log = /var/log/audit/audit.log @@ -42,17 +44,17 @@ suhosin_log = %(syslog_user)s %(lighttpd_error_log)s # defaults to ftp or local2 if ftp doesn't exist proftpd_log = %(syslog_ftp)s -proftpd_backend = auto +proftpd_backend = %(default_backend)s # http://svnweb.freebsd.org/ports/head/ftp/proftpd/files/patch-src_proftpd.8.in?view=markup # defaults to ftp but can be overwritten. pureftpd_log = %(syslog_ftp)s -pureftpd_backend = auto +pureftpd_backend = %(default_backend)s # ftp, daemon and then local7 are tried at configure time however it is overwriteable at configure time # wuftpd_log = %(syslog_ftp)s -wuftpd_backend = auto +wuftpd_backend = %(default_backend)s # syslog_enable defaults to no. so it defaults to vsftpd_log_file setting of /var/log/vsftpd.log # No distro seems to set it to syslog by default @@ -61,16 +63,16 @@ vsftpd_log = /var/log/vsftpd.log # Technically syslog_facility in main.cf can overwrite but no-one sane does this. postfix_log = %(syslog_mail_warn)s -postfix_backend = auto +postfix_backend = %(default_backend)s dovecot_log = %(syslog_mail_warn)s -dovecot_backend = auto +dovecot_backend = %(default_backend)s # Seems to be set at compile time only to LOG_LOCAL0 (src/const.h) at Notice level solidpop3d_log = %(syslog_local0)s mysql_log = %(syslog_daemon)s -mysql_backend = auto +mysql_backend = %(default_backend)s roundcube_errors_log = /var/log/roundcube/errors From 74fcb219ab6d17f365befeb10be99e7a9eb05e16 Mon Sep 17 00:00:00 2001 From: Pablo Rodriguez Fernandez Date: Mon, 19 Oct 2015 09:13:05 +0200 Subject: [PATCH 042/126] Enhanced Google domain detection in apache-fakegooglebot Previously, an attacker could fake a domain like crawl-1-1-1-1.googlebot.com.fake.net and get resolved. This change avoids to resolve fake Google domains. --- config/filter.d/ignorecommands/apache-fakegooglebot | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config/filter.d/ignorecommands/apache-fakegooglebot b/config/filter.d/ignorecommands/apache-fakegooglebot index 3028d86a..9e0f0d83 100755 --- a/config/filter.d/ignorecommands/apache-fakegooglebot +++ b/config/filter.d/ignorecommands/apache-fakegooglebot @@ -26,7 +26,7 @@ def is_googlebot(ip): from fail2ban.server.filter import DNSUtils host = DNSUtils.ipToName(ip) - if not host or not re.match('crawl-.*\.googlebot\.com', host): + if not host or not re.match('crawl-.*\.googlebot\.com$', host): sys.exit(1) host_ips = DNSUtils.dnsToIp(host) sys.exit(0 if ip in host_ips else 1) From 2c576c64f8192cad469bd839321c03bcc1f8113a Mon Sep 17 00:00:00 2001 From: Pablo Rodriguez Fernandez Date: Tue, 20 Oct 2015 10:37:07 +0200 Subject: [PATCH 043/126] Change domain filter regex Change domain filter regex since there are other Google crawlers. See "Google crawlers" --- ChangeLog | 3 +++ config/filter.d/ignorecommands/apache-fakegooglebot | 2 +- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/ChangeLog b/ChangeLog index 2f5a158b..9449f2cd 100644 --- a/ChangeLog +++ b/ChangeLog @@ -21,6 +21,7 @@ ver. 0.9.4 (2015/XX/XXX) - wanna-be-released * Treat failed and killed execution of commands identically (only different log messages), which addresses different behavior on different exit codes of dash and bash (gh-1155) + * Fix jail.conf.5 man's section (gh-1226) - New Features: @@ -34,6 +35,8 @@ ver. 0.9.4 (2015/XX/XXX) - wanna-be-released (Thanks M. Maraun) * Added check against atacker's Googlebot PTR fake records (Thanks Pablo Rodriguez Fernandez) + * Enhance filter against atacker's Googlebot PTR fake records + (gh-1226) ver. 0.9.3 (2015/08/01) - lets-all-stay-friends ---------- diff --git a/config/filter.d/ignorecommands/apache-fakegooglebot b/config/filter.d/ignorecommands/apache-fakegooglebot index 9e0f0d83..19fb5107 100755 --- a/config/filter.d/ignorecommands/apache-fakegooglebot +++ b/config/filter.d/ignorecommands/apache-fakegooglebot @@ -26,7 +26,7 @@ def is_googlebot(ip): from fail2ban.server.filter import DNSUtils host = DNSUtils.ipToName(ip) - if not host or not re.match('crawl-.*\.googlebot\.com$', host): + if not host or not re.match('.*\.google(bot)?\.com$', host): sys.exit(1) host_ips = DNSUtils.dnsToIp(host) sys.exit(0 if ip in host_ips else 1) From 2861a957a9ba691a44005673e90883fd1def1890 Mon Sep 17 00:00:00 2001 From: 1technophile Date: Sun, 25 Oct 2015 20:36:40 +0100 Subject: [PATCH 044/126] filter for openhab domotic software authentication failure with the rest api and web interface + test cases; closes gh-1223 --- config/filter.d/openhab.conf | 16 ++++++++++++++++ config/jail.conf | 6 ++++++ fail2ban/tests/files/logs/openhab | 11 +++++++++++ 3 files changed, 33 insertions(+) create mode 100644 config/filter.d/openhab.conf create mode 100644 fail2ban/tests/files/logs/openhab diff --git a/config/filter.d/openhab.conf b/config/filter.d/openhab.conf new file mode 100644 index 00000000..83857c7a --- /dev/null +++ b/config/filter.d/openhab.conf @@ -0,0 +1,16 @@ +# Openhab brute force auth filter: /etc/fail2ban/filter.d/openhab.conf: +# +# Block IPs trying to auth openhab by web or rest api +# +# Matches e.g. +# 12.34.33.22 - - [26/sept./2015:18:04:43 +0200] "GET /openhab.app HTTP/1.1" 401 1382 +# 175.18.15.10 - - [02/sept./2015:00:11:31 +0200] "GET /rest/bindings HTTP/1.1" 401 1384 + +[Definition] +failregex = ^\s+-\s+-\s+\[\]\s+"[A-Z]+ .*" 401 \d+\s*$ + +[Init] +datepattern = %%d/%%b[^/]*/%%Y:%%H:%%M:%%S %%z + + + diff --git a/config/jail.conf b/config/jail.conf index 7500f4ff..fd7f376e 100644 --- a/config/jail.conf +++ b/config/jail.conf @@ -306,6 +306,12 @@ port = http,https logpath = %(apache_error_log)s maxretry = 1 +[openhab-auth] + +filter = openhab +action = iptables-allports[name=NoAuthFailures] +logpath = /opt/openhab/logs/request.log + [nginx-http-auth] port = http,https diff --git a/fail2ban/tests/files/logs/openhab b/fail2ban/tests/files/logs/openhab new file mode 100644 index 00000000..983989a9 --- /dev/null +++ b/fail2ban/tests/files/logs/openhab @@ -0,0 +1,11 @@ +# should match +# failJSON: { "time": "2015-09-02T00:11:31", "match": true , "host": "175.18.15.10" } +175.18.15.10 - - [02/sept./2015:00:11:31 +0200] "GET /openhab.app HTTP/1.1" 401 1382 +# failJSON: { "time": "2015-09-02T00:11:31", "match": true , "host": "175.18.15.10" } +175.18.15.10 - - [02/sept./2015:00:11:31 +0200] "GET /rest/bindings HTTP/1.1" 401 1384 + +# Should not match +# failJSON: { "match": false } +175.18.15.11 - - [17/oct./2015:00:35:12 +0200] "GET /openhab.app?sitemap=default&poll=true&__async=true&__source=waHome HTTP/1.1" 200 92 +# failJSON: { "match": false } +175.18.15.11 - - [16/oct./2015:20:29:38 +0200] "GET /rest/sitemaps/default/maison HTTP/1.1" 200 2837 From eb87638eadbfe215f875675b0f2383222830f424 Mon Sep 17 00:00:00 2001 From: sebres Date: Mon, 26 Oct 2015 15:52:10 +0100 Subject: [PATCH 045/126] ChangeLog entry for OpenHAB home automation filter (gh-1223) --- ChangeLog | 2 ++ THANKS | 1 + 2 files changed, 3 insertions(+) diff --git a/ChangeLog b/ChangeLog index 9449f2cd..391eabc6 100644 --- a/ChangeLog +++ b/ChangeLog @@ -37,6 +37,8 @@ ver. 0.9.4 (2015/XX/XXX) - wanna-be-released (Thanks Pablo Rodriguez Fernandez) * Enhance filter against atacker's Googlebot PTR fake records (gh-1226) + * Added filter for openhab domotic software authentication failure with the + rest api and web interface (gh-1223) ver. 0.9.3 (2015/08/01) - lets-all-stay-friends ---------- diff --git a/THANKS b/THANKS index 45674847..47156d82 100644 --- a/THANKS +++ b/THANKS @@ -40,6 +40,7 @@ Eric Gerbier Enrico Labedzki Eugene Hopkinson (SlowRiot) ftoppi +Florian Robert (1technophile) François Boulogne Frantisek Sumsal Frédéric From 3ec725a2ba50784ee3f17aa16e0e4eff63a89715 Mon Sep 17 00:00:00 2001 From: Simon Brown Date: Mon, 26 Oct 2015 17:35:38 -0700 Subject: [PATCH 046/126] Created file From https://github.com/beezwax/filemaker-fail2ban/blob/master/fail2ban/filter.d/screensharingd.conf --- config/filter.d/screensharingd.conf | 33 +++++++++++++++++++++++++++++ 1 file changed, 33 insertions(+) create mode 100644 config/filter.d/screensharingd.conf diff --git a/config/filter.d/screensharingd.conf b/config/filter.d/screensharingd.conf new file mode 100644 index 00000000..c0b5d32c --- /dev/null +++ b/config/filter.d/screensharingd.conf @@ -0,0 +1,33 @@ +# Fail2Ban configuration file +# +# Author: Simon Brown +# +# $Revision: 1 $ +# +# Filter for Mac OS X Screen Sharing service + +[INCLUDES] + +# Read common prefixes. If any customizations available -- read them from +# common.local +before = common.conf + + +[Definition] + +_daemon = screensharingd + +# Option: failregex +# Notes.: regex to match the password failures messages in the logfile. The +# host must be matched by a group named "host". The tag "" can +# be used for standard IP/hostname matching and is only an alias for +# (?:::f{4,6}:)?(?P[\w\-.^_]+) +# Values: TEXT +# +failregex = ^.+ screensharingd.+: Authentication: FAILED :: User Name: .+ :: Viewer Address: :: .*$ + +# Option: ignoreregex +# Notes.: regex to ignore. If this regex matches, the line is ignored. +# Values: TEXT +# +ignoreregex = From 80546c61642248445b5140bf5f156dca058cd858 Mon Sep 17 00:00:00 2001 From: Simon Brown Date: Mon, 26 Oct 2015 17:50:49 -0700 Subject: [PATCH 047/126] Added in settings for screensharingd filter --- config/jail.conf | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/config/jail.conf b/config/jail.conf index fd7f376e..e056ea3c 100644 --- a/config/jail.conf +++ b/config/jail.conf @@ -786,3 +786,10 @@ returntype = DROP bantime = 3600 maxretry = 1 findtime = 1 + +[screensharing] +# For Mac OS Screen Sharing Service +enabled = true +filter = screensharingd +logpath = %(system_log)s +maxretry = 4 From de14946542f0cbc8eece09329ca03418086c36da Mon Sep 17 00:00:00 2001 From: Simon Brown Date: Mon, 26 Oct 2015 18:02:07 -0700 Subject: [PATCH 048/126] Added new path variable for system.log Logging location for the majority of Mac OS daemons. --- config/paths-osx.conf | 2 ++ 1 file changed, 2 insertions(+) diff --git a/config/paths-osx.conf b/config/paths-osx.conf index d1b99b38..4f4df632 100644 --- a/config/paths-osx.conf +++ b/config/paths-osx.conf @@ -25,3 +25,5 @@ syslog_authpriv = /var/log/secure.log #syslog_local0 = +# Default Mac OS log location for syslog output. +system_log = /var/log/system.log From d17d837b8c855091f7cca3231c8ac05c4963553b Mon Sep 17 00:00:00 2001 From: Simon Brown Date: Tue, 27 Oct 2015 10:28:07 -0700 Subject: [PATCH 049/126] Update jail.conf Added logencoding to screensharing jail to avoid encoding error messages in fail2ban log --- config/jail.conf | 1 + 1 file changed, 1 insertion(+) diff --git a/config/jail.conf b/config/jail.conf index e056ea3c..2b9e9eaf 100644 --- a/config/jail.conf +++ b/config/jail.conf @@ -792,4 +792,5 @@ findtime = 1 enabled = true filter = screensharingd logpath = %(system_log)s +logencoding=utf-8 maxretry = 4 From 4c3f778b826248809273d1287271e599a1f0f64e Mon Sep 17 00:00:00 2001 From: Simon Brown Date: Tue, 27 Oct 2015 10:33:30 -0700 Subject: [PATCH 050/126] Replaced .* with literal Per Serg's suggestions. Possible I'm missing some auth attempt types, but I couldn't find anything where literal wasn't sufficient. --- config/filter.d/screensharingd.conf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config/filter.d/screensharingd.conf b/config/filter.d/screensharingd.conf index c0b5d32c..fc9f6aed 100644 --- a/config/filter.d/screensharingd.conf +++ b/config/filter.d/screensharingd.conf @@ -24,7 +24,7 @@ _daemon = screensharingd # (?:::f{4,6}:)?(?P[\w\-.^_]+) # Values: TEXT # -failregex = ^.+ screensharingd.+: Authentication: FAILED :: User Name: .+ :: Viewer Address: :: .*$ +failregex = ^.+ screensharingd.+: Authentication: FAILED :: User Name: .+ :: Viewer Address: :: Type: DH$ # Option: ignoreregex # Notes.: regex to ignore. If this regex matches, the line is ignored. From b3a18631e214ad970e87d2704e3d8027bb76359d Mon Sep 17 00:00:00 2001 From: Simon Brown Date: Tue, 27 Oct 2015 10:43:43 -0700 Subject: [PATCH 051/126] Sample log for test case --- fail2ban/tests/files/logs/screenshare | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 fail2ban/tests/files/logs/screenshare diff --git a/fail2ban/tests/files/logs/screenshare b/fail2ban/tests/files/logs/screenshare new file mode 100644 index 00000000..d3736f82 --- /dev/null +++ b/fail2ban/tests/files/logs/screenshare @@ -0,0 +1,5 @@ +Oct 27 09:24:46 test1.beezwax.net screensharingd[1170]: Authentication: SUCCEEDED :: User Name: simon :: Viewer Address: 192.168.5.247 :: Type: DH +Oct 27 09:25:41 test1.beezwax.net screensharingd[1170]: Authentication: FAILED :: User Name: sdfsdfs () mro :: Viewer Address: 192.168.5.247 :: Type: DH +Oct 27 09:25:49 test1.beezwax.net screensharingd[1170]: Authentication: FAILED :: User Name: sdfsdfs :: :: Viewer Address: 192.168.5.247 :: Type: DH +Oct 27 09:25:58 test1.beezwax.net screensharingd[1170]: Authentication: FAILED :: User Name: sdfsdfs _+ :: Viewer Address: 192.168.5.247 :: Type: DH +Oct 27 09:26:09 test1.beezwax.net screensharingd[1170]: Authentication: FAILED :: User Name: sdfsdfs @! s:: :: Viewer Address: 192.168.5.247 :: Type: DH From 3e4a77a5687550a47888720856d0b652cf6745d9 Mon Sep 17 00:00:00 2001 From: Simon Brown Date: Tue, 27 Oct 2015 12:31:51 -0700 Subject: [PATCH 052/126] Added json metadata --- fail2ban/tests/files/logs/screenshare | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/fail2ban/tests/files/logs/screenshare b/fail2ban/tests/files/logs/screenshare index d3736f82..73425b43 100644 --- a/fail2ban/tests/files/logs/screenshare +++ b/fail2ban/tests/files/logs/screenshare @@ -1,5 +1,14 @@ +# failJSON: { "time": "Oct 27 09:24:46", "match": false , "host": "192.168.5.247" } Oct 27 09:24:46 test1.beezwax.net screensharingd[1170]: Authentication: SUCCEEDED :: User Name: simon :: Viewer Address: 192.168.5.247 :: Type: DH +# failJSON: { "time": "Oct 27 09:25:41", "match": true , "host": "192.168.5.247" } Oct 27 09:25:41 test1.beezwax.net screensharingd[1170]: Authentication: FAILED :: User Name: sdfsdfs () mro :: Viewer Address: 192.168.5.247 :: Type: DH +# failJSON: { "time": "Oct 27 09:25:49", "match": true , "host": "192.168.5.247" } Oct 27 09:25:49 test1.beezwax.net screensharingd[1170]: Authentication: FAILED :: User Name: sdfsdfs :: :: Viewer Address: 192.168.5.247 :: Type: DH -Oct 27 09:25:58 test1.beezwax.net screensharingd[1170]: Authentication: FAILED :: User Name: sdfsdfs _+ :: Viewer Address: 192.168.5.247 :: Type: DH +# failJSON: { "time": "Oct 27 09:26:09", "match": true , "host": "192.168.5.247" } Oct 27 09:26:09 test1.beezwax.net screensharingd[1170]: Authentication: FAILED :: User Name: sdfsdfs @! s:: :: Viewer Address: 192.168.5.247 :: Type: DH +# failJSON: { "time": "Oct 27 09:23:20", "match": false , "host": "192.168.5.247" } +Oct 27 09:23:20 fm100 com.apple.xpc.launchd[1] (com.apple.xpc.launchd.domain.pid.SecurityAgent.1156): Path not allowed in target domain: type = uid, path = /System/Library/Frameworks/AppKit.framework/Versions/C/XPCServices/SandboxedServiceRunner.xpc/Contents/MacOS/SandboxedServiceRunner error = 1: Operation not permitted, origin = /System/Library/Frameworks/Security.framework/Versions/A/XPCServices/SecurityAgent.xpc +# failJSON: { "time": "Oct 27 09:23:20", "match": false , "host": "192.168.5.247" } +Oct 27 09:23:20 fm100.beezwax.net ManagedClient[1155]: MCXCCacheMCXRecordAndGraph(): vproc_swap_integer(NULL, VPROC_GSK_PERUSER_SUSPEND, &(uid=1027), NULL) failed +# failJSON: { "time": "Oct 27 12:26:44", "match": false , "host": "192.168.5.247" } +Oct 27 12:26:44 fm100.beezwax.net digest-service[3828]: digest-request: kdc failed with 36150275 proto=unknown From 6a5f10ee72eed02fb8aa66af6beb4c7e57fc6c97 Mon Sep 17 00:00:00 2001 From: Simon Brown Date: Tue, 27 Oct 2015 16:27:14 -0700 Subject: [PATCH 053/126] name change & new sample data changed name to match daemon, log samples with year --- fail2ban/tests/files/logs/screenshare | 14 -------------- fail2ban/tests/files/logs/screenshared | 18 ++++++++++++++++++ 2 files changed, 18 insertions(+), 14 deletions(-) delete mode 100644 fail2ban/tests/files/logs/screenshare create mode 100644 fail2ban/tests/files/logs/screenshared diff --git a/fail2ban/tests/files/logs/screenshare b/fail2ban/tests/files/logs/screenshare deleted file mode 100644 index 73425b43..00000000 --- a/fail2ban/tests/files/logs/screenshare +++ /dev/null @@ -1,14 +0,0 @@ -# failJSON: { "time": "Oct 27 09:24:46", "match": false , "host": "192.168.5.247" } -Oct 27 09:24:46 test1.beezwax.net screensharingd[1170]: Authentication: SUCCEEDED :: User Name: simon :: Viewer Address: 192.168.5.247 :: Type: DH -# failJSON: { "time": "Oct 27 09:25:41", "match": true , "host": "192.168.5.247" } -Oct 27 09:25:41 test1.beezwax.net screensharingd[1170]: Authentication: FAILED :: User Name: sdfsdfs () mro :: Viewer Address: 192.168.5.247 :: Type: DH -# failJSON: { "time": "Oct 27 09:25:49", "match": true , "host": "192.168.5.247" } -Oct 27 09:25:49 test1.beezwax.net screensharingd[1170]: Authentication: FAILED :: User Name: sdfsdfs :: :: Viewer Address: 192.168.5.247 :: Type: DH -# failJSON: { "time": "Oct 27 09:26:09", "match": true , "host": "192.168.5.247" } -Oct 27 09:26:09 test1.beezwax.net screensharingd[1170]: Authentication: FAILED :: User Name: sdfsdfs @! s:: :: Viewer Address: 192.168.5.247 :: Type: DH -# failJSON: { "time": "Oct 27 09:23:20", "match": false , "host": "192.168.5.247" } -Oct 27 09:23:20 fm100 com.apple.xpc.launchd[1] (com.apple.xpc.launchd.domain.pid.SecurityAgent.1156): Path not allowed in target domain: type = uid, path = /System/Library/Frameworks/AppKit.framework/Versions/C/XPCServices/SandboxedServiceRunner.xpc/Contents/MacOS/SandboxedServiceRunner error = 1: Operation not permitted, origin = /System/Library/Frameworks/Security.framework/Versions/A/XPCServices/SecurityAgent.xpc -# failJSON: { "time": "Oct 27 09:23:20", "match": false , "host": "192.168.5.247" } -Oct 27 09:23:20 fm100.beezwax.net ManagedClient[1155]: MCXCCacheMCXRecordAndGraph(): vproc_swap_integer(NULL, VPROC_GSK_PERUSER_SUSPEND, &(uid=1027), NULL) failed -# failJSON: { "time": "Oct 27 12:26:44", "match": false , "host": "192.168.5.247" } -Oct 27 12:26:44 fm100.beezwax.net digest-service[3828]: digest-request: kdc failed with 36150275 proto=unknown diff --git a/fail2ban/tests/files/logs/screenshared b/fail2ban/tests/files/logs/screenshared new file mode 100644 index 00000000..63e1dfb4 --- /dev/null +++ b/fail2ban/tests/files/logs/screenshared @@ -0,0 +1,18 @@ +# NOTE: dates here include years -- this is NOT the typical configuration for the system.log +# file on Mac OS. However, matches will not pass unless year is included. +# +# failJSON: { "match": false } +Oct 27 2015 09:24:46 test1.beezwax.net screensharingd[1170]: Authentication: SUCCEEDED :: User Name: simon :: Viewer Address: 192.168.5.247 :: Type: DH +# failJSON: { "match": false } +Oct 27 2015 09:23:20 test1 com.apple.xpc.launchd[1] (com.apple.xpc.launchd.domain.pid.SecurityAgent.1156): Path not allowed in target domain: type = uid, path = /System/Library/Frameworks/AppKit.framework/Versions/C/XPCServices/SandboxedServiceRunner.xpc/Contents/MacOS/SandboxedServiceRunner error = 1: Operation not permitted, origin = /System/Library/Frameworks/Security.framework/Versions/A/XPCServices/SecurityAgent.xpc +# failJSON: { "match": false } +Oct 27 2015 09:23:20 test1.beezwax.net ManagedClient[1155]: MCXCCacheMCXRecordAndGraph(): vproc_swap_integer(NULL, VPROC_GSK_PERUSER_SUSPEND, &(uid=1027), NULL) failed +# failJSON: { "match": false } +Oct 27 2015 12:26:44 test1.beezwax.net digest-service[3828]: digest-request: kdc failed with 36150275 proto=unknown +# +# failJSON: { "time": "2015-10-27T12:35:40", "match": true , "host": "192.168.5.247" } +Oct 27 2015 12:35:40 test1.beezwax.net screensharingd[1170]: Authentication: FAILED :: User Name: sdfsdfs () mro :: Viewer Address: 192.168.5.247 :: Type: DH +# failJSON: { "time": "2015-10-27T12:35:50", "match": true , "host": "192.168.5.247" } +Oct 27 2015 12:35:50 test1.beezwax.net screensharingd[1170]: Authentication: FAILED :: User Name: brown_s :: :: Viewer Address: 192.168.5.247 :: Type: DH +# failJSON: { "time": "2015-10-27T12:26:01", "match": true , "host": "192.168.5.247" } +Oct 27 2015 12:26:01 test1.beezwax.net screensharingd[1170]: Authentication: FAILED :: User Name: brown @! s:: :: Viewer Address: 192.168.5.247 :: Type: DH From 3dd1c305ce536d42fba397ac516abc3dd43d5f35 Mon Sep 17 00:00:00 2001 From: Simon Brown Date: Tue, 27 Oct 2015 21:20:12 -0700 Subject: [PATCH 054/126] added entry for new screensharingd filter --- ChangeLog | 2 ++ 1 file changed, 2 insertions(+) diff --git a/ChangeLog b/ChangeLog index 391eabc6..a6e53a02 100644 --- a/ChangeLog +++ b/ChangeLog @@ -39,6 +39,8 @@ ver. 0.9.4 (2015/XX/XXX) - wanna-be-released (gh-1226) * Added filter for openhab domotic software authentication failure with the rest api and web interface (gh-1223) + * Added filter for Mac OS screen sharing (VNC) daemon + ver. 0.9.3 (2015/08/01) - lets-all-stay-friends ---------- From 4b4d5a95b7af7aaca0c8bc9858bd78c6d3b75320 Mon Sep 17 00:00:00 2001 From: Simon Brown Date: Tue, 27 Oct 2015 21:30:20 -0700 Subject: [PATCH 055/126] Changed regex prequel Use standard prefix macro instead of literal daemon name. --- config/filter.d/screensharingd.conf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config/filter.d/screensharingd.conf b/config/filter.d/screensharingd.conf index fc9f6aed..fbaff542 100644 --- a/config/filter.d/screensharingd.conf +++ b/config/filter.d/screensharingd.conf @@ -24,7 +24,7 @@ _daemon = screensharingd # (?:::f{4,6}:)?(?P[\w\-.^_]+) # Values: TEXT # -failregex = ^.+ screensharingd.+: Authentication: FAILED :: User Name: .+ :: Viewer Address: :: Type: DH$ +failregex = ^%(__prefix_line)sAuthentication: Authentication: FAILED :: User Name: .+ :: Viewer Address: :: Type: DH$ # Option: ignoreregex # Notes.: regex to ignore. If this regex matches, the line is ignored. From acee68a9ee94a084576e91f74f226efb41b231d7 Mon Sep 17 00:00:00 2001 From: Simon Brown Date: Wed, 28 Oct 2015 15:11:11 -0700 Subject: [PATCH 056/126] Made screensharing jail off by default Also added note about requiring paths-osx.conf. --- config/jail.conf | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/config/jail.conf b/config/jail.conf index 2b9e9eaf..65ffdef1 100644 --- a/config/jail.conf +++ b/config/jail.conf @@ -788,8 +788,9 @@ maxretry = 1 findtime = 1 [screensharing] -# For Mac OS Screen Sharing Service -enabled = true +# For Mac OS Screen Sharing Service (VNC) +# Requires the 'before' statement in the [INCLUDE] section to include paths-osx.conf +enabled = false filter = screensharingd logpath = %(system_log)s logencoding=utf-8 From c936d19805aae62fa71fb8087313b889713901be Mon Sep 17 00:00:00 2001 From: Simon Brown Date: Wed, 28 Oct 2015 15:30:31 -0700 Subject: [PATCH 057/126] Fixed name (again?) --- fail2ban/tests/files/logs/{screenshared => screensharingd} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename fail2ban/tests/files/logs/{screenshared => screensharingd} (100%) diff --git a/fail2ban/tests/files/logs/screenshared b/fail2ban/tests/files/logs/screensharingd similarity index 100% rename from fail2ban/tests/files/logs/screenshared rename to fail2ban/tests/files/logs/screensharingd From bed28eaa62b6c1737c2030ce2ced56dc5ef20316 Mon Sep 17 00:00:00 2001 From: Simon Brown Date: Wed, 28 Oct 2015 15:32:58 -0700 Subject: [PATCH 058/126] clarified comments on sample log format --- fail2ban/tests/files/logs/screensharingd | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/fail2ban/tests/files/logs/screensharingd b/fail2ban/tests/files/logs/screensharingd index 63e1dfb4..c43a5c41 100644 --- a/fail2ban/tests/files/logs/screensharingd +++ b/fail2ban/tests/files/logs/screensharingd @@ -1,5 +1,5 @@ -# NOTE: dates here include years -- this is NOT the typical configuration for the system.log -# file on Mac OS. However, matches will not pass unless year is included. +# NOTE: dates here include years -- this is not the typical configuration for the system.log +# file on Mac OS. However, without it the test routines will use 2004 as the year and matches will not pass. # # failJSON: { "match": false } Oct 27 2015 09:24:46 test1.beezwax.net screensharingd[1170]: Authentication: SUCCEEDED :: User Name: simon :: Viewer Address: 192.168.5.247 :: Type: DH From cabd46f069d7fc585b02c0ac88e7069af202e419 Mon Sep 17 00:00:00 2001 From: Simon Brown Date: Wed, 28 Oct 2015 20:58:25 -0700 Subject: [PATCH 059/126] Fixed blatant typo in regex However, still failing test, even though ```PYTHONPATH=. fail2ban-regex -v fail2ban/tests/files/logs/screensharingd /etc/fail2ban/filter.d/screensharingd.conf``` gives desired result --- config/filter.d/screensharingd.conf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config/filter.d/screensharingd.conf b/config/filter.d/screensharingd.conf index fbaff542..c5e3e858 100644 --- a/config/filter.d/screensharingd.conf +++ b/config/filter.d/screensharingd.conf @@ -24,7 +24,7 @@ _daemon = screensharingd # (?:::f{4,6}:)?(?P[\w\-.^_]+) # Values: TEXT # -failregex = ^%(__prefix_line)sAuthentication: Authentication: FAILED :: User Name: .+ :: Viewer Address: :: Type: DH$ +failregex = ^%(__prefix_line)sAuthentication: FAILED :: User Name: .+ :: Viewer Address: :: Type: DH$ # Option: ignoreregex # Notes.: regex to ignore. If this regex matches, the line is ignored. From 65bc5cf6ba08f64fc13c03f5c44d7911a7f0179f Mon Sep 17 00:00:00 2001 From: Simon Brown Date: Thu, 29 Oct 2015 09:03:01 -0700 Subject: [PATCH 060/126] Now using a literal logpath for screensharing jail --- config/jail.conf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config/jail.conf b/config/jail.conf index 65ffdef1..7470039c 100644 --- a/config/jail.conf +++ b/config/jail.conf @@ -792,6 +792,6 @@ findtime = 1 # Requires the 'before' statement in the [INCLUDE] section to include paths-osx.conf enabled = false filter = screensharingd -logpath = %(system_log)s +logpath = /var/log/system.log logencoding=utf-8 maxretry = 4 From 6884593ab825068cdecc55658a9213513ca6ed16 Mon Sep 17 00:00:00 2001 From: sebres Date: Thu, 29 Oct 2015 23:15:20 +0100 Subject: [PATCH 061/126] New filter `nginx-limit-req` ban hosts, that were failed through nginx by limit request processing rate (ngx_http_limit_req_module) --- ChangeLog | 7 ++-- config/filter.d/nginx-limit-req.conf | 39 +++++++++++++++++++++++ config/jail.conf | 8 +++++ fail2ban/tests/files/logs/nginx-limit-req | 6 ++++ 4 files changed, 58 insertions(+), 2 deletions(-) create mode 100644 config/filter.d/nginx-limit-req.conf create mode 100644 fail2ban/tests/files/logs/nginx-limit-req diff --git a/ChangeLog b/ChangeLog index 391eabc6..cd1ce7bd 100644 --- a/ChangeLog +++ b/ChangeLog @@ -24,6 +24,11 @@ ver. 0.9.4 (2015/XX/XXX) - wanna-be-released * Fix jail.conf.5 man's section (gh-1226) - New Features: + * New filters: + - openhab - domotic software authentication failure with the + rest api and web interface (gh-1223) + - nginx-limit-req - ban hosts, that were failed through nginx by limit + request processing rate (ngx_http_limit_req_module) - Enhancements: * Do not rotate empty log files @@ -37,8 +42,6 @@ ver. 0.9.4 (2015/XX/XXX) - wanna-be-released (Thanks Pablo Rodriguez Fernandez) * Enhance filter against atacker's Googlebot PTR fake records (gh-1226) - * Added filter for openhab domotic software authentication failure with the - rest api and web interface (gh-1223) ver. 0.9.3 (2015/08/01) - lets-all-stay-friends ---------- diff --git a/config/filter.d/nginx-limit-req.conf b/config/filter.d/nginx-limit-req.conf new file mode 100644 index 00000000..eb804798 --- /dev/null +++ b/config/filter.d/nginx-limit-req.conf @@ -0,0 +1,39 @@ +# Fail2ban filter configuration for nginx :: limit_req +# used to ban hosts, that were failed through nginx by limit request processing rate +# +# Author: Serg G. Brester (sebres) +# +# To use 'nginx-limit-req' filter you should have `ngx_http_limit_req_module` +# and define `limit_req` and `limit_req_zone` as described in nginx documentation +# http://nginx.org/en/docs/http/ngx_http_limit_req_module.html +# +# Example: +# +# http { +# ... +# limit_req_zone $binary_remote_addr zone=lr_zone:10m rate=1r/s; +# ... +# # http, server, or location: +# location ... { +# limit_req zone=lr_zone burst=1 nodelay; +# ... +# } +# ... +# } +# ... +# + +[Definition] + +# Specify following expression to define exact zones, if you want to ban IPs limited +# from specified zones only. +# Example: +# +# ngx_limit_req_zones = lr_zone|lr_zone2 +# +ngx_limit_req_zones = [^"]+ + +failregex = ^\s*\[error\] \d+#\d+: \*\d+ limiting requests, excess: [\d\.]+ by zone "(?:%(ngx_limit_req_zones)s)", client: , server: \S*, request: "\S+ \S+ HTTP/\d+\.\d+", host: "\S+"(, referrer: "\S+")?\s*$ + +ignoreregex = + diff --git a/config/jail.conf b/config/jail.conf index fd7f376e..fc175550 100644 --- a/config/jail.conf +++ b/config/jail.conf @@ -317,6 +317,14 @@ logpath = /opt/openhab/logs/request.log port = http,https logpath = %(nginx_error_log)s +# To use 'nginx-limit-req' jail you should have `ngx_http_limit_req_module` +# and define `limit_req` and `limit_req_zone` as described in nginx documentation +# http://nginx.org/en/docs/http/ngx_http_limit_req_module.html +# or for example see in 'config/filter.d/nginx-limit-req.conf' +[nginx-limit-req] +port = http,https +logpath = %(nginx_error_log)s + [nginx-botsearch] port = http,https diff --git a/fail2ban/tests/files/logs/nginx-limit-req b/fail2ban/tests/files/logs/nginx-limit-req new file mode 100644 index 00000000..68f1b239 --- /dev/null +++ b/fail2ban/tests/files/logs/nginx-limit-req @@ -0,0 +1,6 @@ + +# failJSON: { "time": "2015-10-29T20:01:02", "match": true , "host": "1.2.3.4" } +2015/10/29 20:01:02 [error] 256554#0: *99927 limiting requests, excess: 1.852 by zone "one", client: 1.2.3.4, server: example.com, request: "POST /index.htm HTTP/1.0", host: "exmaple.com" + +# failJSON: { "time": "2015-10-29T19:24:05", "match": true , "host": "192.0.2.0" } +2015/10/29 19:24:05 [error] 12684#12684: *22174 limiting requests, excess: 1.495 by zone "one", client: 192.0.2.0, server: example.com, request: "GET /index.php HTTP/1.1", host: "example.com", referrer: "https://example.com" From 53b39162a1b2d026bf0e9eb14f3ffa61f5cc1f8a Mon Sep 17 00:00:00 2001 From: sebres Date: Thu, 29 Oct 2015 23:55:23 +0100 Subject: [PATCH 062/126] Shortly, much faster and stable version of regexp (possible because expression is start-anchored and does not contains closely to catch-all sub expressions) --- config/filter.d/nginx-limit-req.conf | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/config/filter.d/nginx-limit-req.conf b/config/filter.d/nginx-limit-req.conf index eb804798..589d3d78 100644 --- a/config/filter.d/nginx-limit-req.conf +++ b/config/filter.d/nginx-limit-req.conf @@ -33,7 +33,13 @@ # ngx_limit_req_zones = [^"]+ -failregex = ^\s*\[error\] \d+#\d+: \*\d+ limiting requests, excess: [\d\.]+ by zone "(?:%(ngx_limit_req_zones)s)", client: , server: \S*, request: "\S+ \S+ HTTP/\d+\.\d+", host: "\S+"(, referrer: "\S+")?\s*$ +# Use following full expression if you should range limit request to specified +# servers, requests, referrers etc. only : +# +# failregex = ^\s*\[error\] \d+#\d+: \*\d+ limiting requests, excess: [\d\.]+ by zone "(?:%(ngx_limit_req_zones)s)", client: , server: \S*, request: "\S+ \S+ HTTP/\d+\.\d+", host: "\S+"(, referrer: "\S+")?\s*$ + +# Shortly, much faster and stable version of regexp: +failregex = ^\s*\[error\] \d+#\d+: \*\d+ limiting requests, excess: [\d\.]+ by zone "(?:%(ngx_limit_req_zones)s)", client: ignoreregex = From 5839a3bd80c73a5ff6a0f725d4453963505eb68a Mon Sep 17 00:00:00 2001 From: Simon Brown Date: Thu, 29 Oct 2015 16:07:54 -0700 Subject: [PATCH 063/126] Removed includes comment for screensharing jail --- config/jail.conf | 1 - 1 file changed, 1 deletion(-) diff --git a/config/jail.conf b/config/jail.conf index 7470039c..2aaea7ea 100644 --- a/config/jail.conf +++ b/config/jail.conf @@ -789,7 +789,6 @@ findtime = 1 [screensharing] # For Mac OS Screen Sharing Service (VNC) -# Requires the 'before' statement in the [INCLUDE] section to include paths-osx.conf enabled = false filter = screensharingd logpath = /var/log/system.log From f359ed8c367a97060afcc54164263bfad9ed92d9 Mon Sep 17 00:00:00 2001 From: sebres Date: Fri, 30 Oct 2015 15:36:18 +0100 Subject: [PATCH 064/126] Fixed directly defined banaction for allports jails like pam-generic, recidive, etc with new default variable `banaction_allports` (+ man entries for both variables added); closes gh-1216 --- ChangeLog | 2 ++ config/jail.conf | 7 ++++--- man/jail.conf.5 | 6 ++++++ 3 files changed, 12 insertions(+), 3 deletions(-) diff --git a/ChangeLog b/ChangeLog index cd1ce7bd..beaba762 100644 --- a/ChangeLog +++ b/ChangeLog @@ -22,6 +22,8 @@ ver. 0.9.4 (2015/XX/XXX) - wanna-be-released different log messages), which addresses different behavior on different exit codes of dash and bash (gh-1155) * Fix jail.conf.5 man's section (gh-1226) + * Fixed default banaction for allports jails like pam-generic, recidive, etc + with new default variable `banaction_allports` (gh-1216) - New Features: * New filters: diff --git a/config/jail.conf b/config/jail.conf index fc175550..69fef818 100644 --- a/config/jail.conf +++ b/config/jail.conf @@ -154,6 +154,7 @@ port = 0:65535 # action_* variables. Can be overridden globally or per # section within jail.local file banaction = iptables-multiport +banaction_allports = iptables-allports # The simplest action to take: ban only action_ = %(banaction)s[name=%(__name__)s, bantime="%(bantime)s", port="%(port)s", protocol="%(protocol)s", chain="%(chain)s"] @@ -713,7 +714,7 @@ maxretry = 5 [recidive] logpath = /var/log/fail2ban.log -banaction = iptables-allports +banaction = %(banaction_allports)s bantime = 604800 ; 1 week findtime = 86400 ; 1 day maxretry = 5 @@ -724,7 +725,7 @@ maxretry = 5 [pam-generic] # pam-generic filter can be customized to monitor specific subset of 'tty's -banaction = iptables-allports +banaction = %(banaction_allports)s logpath = %(syslog_authpriv)s @@ -770,7 +771,7 @@ maxretry = 1 enabled = false logpath = /opt/sun/comms/messaging64/log/mail.log_current maxretry = 6 -banaction = iptables-allports +banaction = %(banaction_allports)s [directadmin] enabled = false diff --git a/man/jail.conf.5 b/man/jail.conf.5 index 957a04b4..5dc64a4b 100644 --- a/man/jail.conf.5 +++ b/man/jail.conf.5 @@ -146,6 +146,12 @@ Ensure syslog or the program that generates the log file isn't configured to com .B logencoding encoding of log files used for decoding. Default value of "auto" uses current system locale. .TP +.B banaction +default banning action (iptables-multiport) for all jails specified in the \fI[DEFAULT]\fR section. +.TP +.B banaction_allports +default allports banning action (iptables-allports) for some jails like "pam-generic" or "recidive", specified in the \fI[DEFAULT]\fR section. +.TP .B action action(s) from \fI/etc/fail2ban/action.d/\fR without the \fI.conf\fR/\fI.local\fR extension. Arguments can be passed to actions to override the default values from the [Init] section in the action file. Arguments are specified by: .RS From e825e977ccb897ddeadd573854e2666ee8963038 Mon Sep 17 00:00:00 2001 From: sebres Date: Fri, 30 Oct 2015 17:51:30 +0100 Subject: [PATCH 065/126] Nginx log paths extended (prefixed with "*" wildcard) closes gh-1237 --- ChangeLog | 1 + config/paths-common.conf | 4 ++-- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/ChangeLog b/ChangeLog index cd1ce7bd..66805c36 100644 --- a/ChangeLog +++ b/ChangeLog @@ -42,6 +42,7 @@ ver. 0.9.4 (2015/XX/XXX) - wanna-be-released (Thanks Pablo Rodriguez Fernandez) * Enhance filter against atacker's Googlebot PTR fake records (gh-1226) + * Nginx log paths extended (prefixed with "*" wildcard) (gh-1237) ver. 0.9.3 (2015/08/01) - lets-all-stay-friends ---------- diff --git a/config/paths-common.conf b/config/paths-common.conf index bf3cfb6a..1aac027b 100644 --- a/config/paths-common.conf +++ b/config/paths-common.conf @@ -24,9 +24,9 @@ auditd_log = /var/log/audit/audit.log exim_main_log = /var/log/exim/mainlog -nginx_error_log = /var/log/nginx/error.log +nginx_error_log = /var/log/nginx/*error.log -nginx_access_log = /var/log/nginx/access.log +nginx_access_log = /var/log/nginx/*access.log lighttpd_error_log = /var/log/lighttpd/error.log From fcf03790f4377f4083ca114fffc8aea5e67685ee Mon Sep 17 00:00:00 2001 From: sebres Date: Sun, 1 Nov 2015 16:55:45 +0100 Subject: [PATCH 066/126] fixed misleading documentation of `banaction` --- man/jail.conf.5 | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/man/jail.conf.5 b/man/jail.conf.5 index 5dc64a4b..51ea7097 100644 --- a/man/jail.conf.5 +++ b/man/jail.conf.5 @@ -147,10 +147,10 @@ Ensure syslog or the program that generates the log file isn't configured to com encoding of log files used for decoding. Default value of "auto" uses current system locale. .TP .B banaction -default banning action (iptables-multiport) for all jails specified in the \fI[DEFAULT]\fR section. +banning action (default iptables-multiport) typically specified in the \fI[DEFAULT]\fR section for all jails. This parameter will be used by the standard substitution of \fIaction\fR and can be redefined central in the \fI[DEFAULT]\fR section inside \fIjail.local\fR (to apply it to all jails at once) or separately in each jail, where this substitution will be used. .TP .B banaction_allports -default allports banning action (iptables-allports) for some jails like "pam-generic" or "recidive", specified in the \fI[DEFAULT]\fR section. +the same as \fIbanaction\fR but for some "allports" jails like "pam-generic" or "recidive" (default iptables-allports). .TP .B action action(s) from \fI/etc/fail2ban/action.d/\fR without the \fI.conf\fR/\fI.local\fR extension. Arguments can be passed to actions to override the default values from the [Init] section in the action file. Arguments are specified by: From b40c6cbd9a0c1b51e0cc0f0c4629b84d60c2c129 Mon Sep 17 00:00:00 2001 From: Yaroslav Halchenko Date: Sun, 1 Nov 2015 11:28:58 -0500 Subject: [PATCH 067/126] ENH: .mailmap file to bring some names together for git shortlog -sn --- .mailmap | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 .mailmap diff --git a/.mailmap b/.mailmap new file mode 100644 index 00000000..9b7cce3a --- /dev/null +++ b/.mailmap @@ -0,0 +1,5 @@ +Lee Clemens +Serg G. Brester +Serg G. Brester +Serg G. Brester +Viktor Szépe From d16ad805975afdade337dc052f88a198866efe3e Mon Sep 17 00:00:00 2001 From: Simon Brown Date: Mon, 2 Nov 2015 09:06:32 -0800 Subject: [PATCH 068/126] removed false matches For non-screensharingd related messages --- fail2ban/tests/files/logs/screensharingd | 6 ------ 1 file changed, 6 deletions(-) diff --git a/fail2ban/tests/files/logs/screensharingd b/fail2ban/tests/files/logs/screensharingd index c43a5c41..0ec0ebd6 100644 --- a/fail2ban/tests/files/logs/screensharingd +++ b/fail2ban/tests/files/logs/screensharingd @@ -3,12 +3,6 @@ # # failJSON: { "match": false } Oct 27 2015 09:24:46 test1.beezwax.net screensharingd[1170]: Authentication: SUCCEEDED :: User Name: simon :: Viewer Address: 192.168.5.247 :: Type: DH -# failJSON: { "match": false } -Oct 27 2015 09:23:20 test1 com.apple.xpc.launchd[1] (com.apple.xpc.launchd.domain.pid.SecurityAgent.1156): Path not allowed in target domain: type = uid, path = /System/Library/Frameworks/AppKit.framework/Versions/C/XPCServices/SandboxedServiceRunner.xpc/Contents/MacOS/SandboxedServiceRunner error = 1: Operation not permitted, origin = /System/Library/Frameworks/Security.framework/Versions/A/XPCServices/SecurityAgent.xpc -# failJSON: { "match": false } -Oct 27 2015 09:23:20 test1.beezwax.net ManagedClient[1155]: MCXCCacheMCXRecordAndGraph(): vproc_swap_integer(NULL, VPROC_GSK_PERUSER_SUSPEND, &(uid=1027), NULL) failed -# failJSON: { "match": false } -Oct 27 2015 12:26:44 test1.beezwax.net digest-service[3828]: digest-request: kdc failed with 36150275 proto=unknown # # failJSON: { "time": "2015-10-27T12:35:40", "match": true , "host": "192.168.5.247" } Oct 27 2015 12:35:40 test1.beezwax.net screensharingd[1170]: Authentication: FAILED :: User Name: sdfsdfs () mro :: Viewer Address: 192.168.5.247 :: Type: DH From 3e16f33dbe840d507b687517699072a5e100101c Mon Sep 17 00:00:00 2001 From: Simon Brown Date: Mon, 2 Nov 2015 09:08:47 -0800 Subject: [PATCH 069/126] Removed old svn revision comment --- config/filter.d/screensharingd.conf | 2 -- 1 file changed, 2 deletions(-) diff --git a/config/filter.d/screensharingd.conf b/config/filter.d/screensharingd.conf index c5e3e858..4cd76465 100644 --- a/config/filter.d/screensharingd.conf +++ b/config/filter.d/screensharingd.conf @@ -2,8 +2,6 @@ # # Author: Simon Brown # -# $Revision: 1 $ -# # Filter for Mac OS X Screen Sharing service [INCLUDES] From 69bb532db0c9c34bf7c68ba0ed7adb769206f58d Mon Sep 17 00:00:00 2001 From: Simon Brown Date: Mon, 2 Nov 2015 09:26:45 -0800 Subject: [PATCH 070/126] removed system.log --- config/paths-osx.conf | 3 --- 1 file changed, 3 deletions(-) diff --git a/config/paths-osx.conf b/config/paths-osx.conf index 4f4df632..180923c0 100644 --- a/config/paths-osx.conf +++ b/config/paths-osx.conf @@ -24,6 +24,3 @@ syslog_authpriv = /var/log/secure.log #syslog_daemon = #syslog_local0 = - -# Default Mac OS log location for syslog output. -system_log = /var/log/system.log From 94cffece12dc2ff0efd6a8b6761b079eb9491c68 Mon Sep 17 00:00:00 2001 From: sebres Date: Mon, 2 Nov 2015 21:19:15 +0100 Subject: [PATCH 071/126] New interpolation feature for definition config readers - ``, as extension to interpolation `%(known/parameter)s`, that does not works for filter and action init parameters; --- ChangeLog | 7 ++++ fail2ban/client/configreader.py | 4 +- fail2ban/tests/clientreadertestcase.py | 57 ++++++++++++++++++-------- man/jail.conf.5 | 30 ++++++++++++-- 4 files changed, 77 insertions(+), 21 deletions(-) diff --git a/ChangeLog b/ChangeLog index 7e1d84cb..151b5a9c 100644 --- a/ChangeLog +++ b/ChangeLog @@ -26,6 +26,13 @@ ver. 0.9.4 (2015/XX/XXX) - wanna-be-released with new default variable `banaction_allports` (gh-1216) - New Features: + * New interpolation feature for definition config readers - `` + (means last known init definition of filters or actions with name `parameter`). + This interpolation makes possible to extend a parameters of stock filter or + action directly in jail inside jail.local file, without creating a separately + filter.d/*.local file. + As extension to interpolation `%(known/parameter)s`, that does not works for + filter and action init parameters * New filters: - openhab - domotic software authentication failure with the rest api and web interface (gh-1223) diff --git a/fail2ban/client/configreader.py b/fail2ban/client/configreader.py index d5675cc8..c6dd1b60 100644 --- a/fail2ban/client/configreader.py +++ b/fail2ban/client/configreader.py @@ -285,8 +285,10 @@ class DefinitionInitConfigReader(ConfigReader): if self.has_section("Init"): for opt in self.options("Init"): + v = self.get("Init", opt) + self._initOpts['known/'+opt] = v if not opt in self._initOpts: - self._initOpts[opt] = self.get("Init", opt) + self._initOpts[opt] = v def convert(self): raise NotImplementedError diff --git a/fail2ban/tests/clientreadertestcase.py b/fail2ban/tests/clientreadertestcase.py index 94fe1828..d19090be 100644 --- a/fail2ban/tests/clientreadertestcase.py +++ b/fail2ban/tests/clientreadertestcase.py @@ -165,11 +165,11 @@ class JailReaderTest(LogCaptureTestCase): self.__share_cfg = {} def testIncorrectJail(self): - jail = JailReader('XXXABSENTXXX', basedir=CONFIG_DIR, share_config = self.__share_cfg) + jail = JailReader('XXXABSENTXXX', basedir=CONFIG_DIR, share_config=self.__share_cfg) self.assertRaises(ValueError, jail.read) def testJailActionEmpty(self): - jail = JailReader('emptyaction', basedir=IMPERFECT_CONFIG, share_config = self.__share_cfg) + jail = JailReader('emptyaction', basedir=IMPERFECT_CONFIG, share_config=self.__share_cfg) self.assertTrue(jail.read()) self.assertTrue(jail.getOptions()) self.assertTrue(jail.isEnabled()) @@ -177,7 +177,7 @@ class JailReaderTest(LogCaptureTestCase): self.assertLogged('No actions were defined for emptyaction') def testJailActionFilterMissing(self): - jail = JailReader('missingbitsjail', basedir=IMPERFECT_CONFIG, share_config = self.__share_cfg) + jail = JailReader('missingbitsjail', basedir=IMPERFECT_CONFIG, share_config=self.__share_cfg) self.assertTrue(jail.read()) self.assertFalse(jail.getOptions()) self.assertTrue(jail.isEnabled()) @@ -200,7 +200,7 @@ class JailReaderTest(LogCaptureTestCase): if STOCK: def testStockSSHJail(self): - jail = JailReader('sshd', basedir=CONFIG_DIR, share_config = self.__share_cfg) # we are running tests from root project dir atm + jail = JailReader('sshd', basedir=CONFIG_DIR, share_config=self.__share_cfg) # we are running tests from root project dir atm self.assertTrue(jail.read()) self.assertTrue(jail.getOptions()) self.assertFalse(jail.isEnabled()) @@ -274,6 +274,10 @@ class JailReaderTest(LogCaptureTestCase): class FilterReaderTest(unittest.TestCase): + def __init__(self, *args, **kwargs): + super(FilterReaderTest, self).__init__(*args, **kwargs) + self.__share_cfg = {} + def testConvert(self): output = [['set', 'testcase01', 'addfailregex', "^\\s*(?:\\S+ )?(?:kernel: \\[\\d+\\.\\d+\\] )?(?:@vserver_\\S+ )" @@ -311,9 +315,8 @@ class FilterReaderTest(unittest.TestCase): # is unreliable self.assertEqual(sorted(filterReader.convert()), sorted(output)) - filterReader = FilterReader( - "testcase01", "testcase01", {'maxlines': "5"}) - filterReader.setBaseDir(TEST_FILES_DIR) + filterReader = FilterReader("testcase01", "testcase01", {'maxlines': "5"}, + share_config=self.__share_cfg, basedir=TEST_FILES_DIR) filterReader.read() #filterReader.getOptions(["failregex", "ignoreregex"]) filterReader.getOptions(None) @@ -322,8 +325,8 @@ class FilterReaderTest(unittest.TestCase): def testFilterReaderSubstitionDefault(self): output = [['set', 'jailname', 'addfailregex', 'to=sweet@example.com fromip=']] - filterReader = FilterReader('substition', "jailname", {}) - filterReader.setBaseDir(TEST_FILES_DIR) + filterReader = FilterReader('substition', "jailname", {}, + share_config=self.__share_cfg, basedir=TEST_FILES_DIR) filterReader.read() filterReader.getOptions(None) c = filterReader.convert() @@ -331,16 +334,34 @@ class FilterReaderTest(unittest.TestCase): def testFilterReaderSubstitionSet(self): output = [['set', 'jailname', 'addfailregex', 'to=sour@example.com fromip=']] - filterReader = FilterReader('substition', "jailname", {'honeypot': 'sour@example.com'}) - filterReader.setBaseDir(TEST_FILES_DIR) + filterReader = FilterReader('substition', "jailname", {'honeypot': 'sour@example.com'}, + share_config=self.__share_cfg, basedir=TEST_FILES_DIR) + filterReader.read() + filterReader.getOptions(None) + c = filterReader.convert() + self.assertEqual(sorted(c), sorted(output)) + + def testFilterReaderSubstitionKnown(self): + output = [['set', 'jailname', 'addfailregex', 'to=test,sweet@example.com,test2,sweet@example.com fromip=']] + filterName, filterOpt = JailReader.extractOptions( + 'substition[honeypot=",", sweet="test,,test2"]') + filterReader = FilterReader('substition', "jailname", filterOpt, + share_config=self.__share_cfg, basedir=TEST_FILES_DIR) filterReader.read() filterReader.getOptions(None) c = filterReader.convert() self.assertEqual(sorted(c), sorted(output)) def testFilterReaderSubstitionFail(self): - filterReader = FilterReader('substition', "jailname", {'honeypot': '', 'sweet': ''}) - filterReader.setBaseDir(TEST_FILES_DIR) + # directly subst the same var : + filterReader = FilterReader('substition', "jailname", {'honeypot': ''}, + share_config=self.__share_cfg, basedir=TEST_FILES_DIR) + filterReader.read() + filterReader.getOptions(None) + self.assertRaises(ValueError, FilterReader.convert, filterReader) + # cross subst the same var : + filterReader = FilterReader('substition', "jailname", {'honeypot': '', 'sweet': ''}, + share_config=self.__share_cfg, basedir=TEST_FILES_DIR) filterReader.read() filterReader.getOptions(None) self.assertRaises(ValueError, FilterReader.convert, filterReader) @@ -508,12 +529,13 @@ class JailsReaderTest(LogCaptureTestCase): if jail == 'INCLUDES': continue filterName = jails.get(jail, 'filter') + filterName, filterOpt = JailReader.extractOptions(filterName) allFilters.add(filterName) self.assertTrue(len(filterName)) # moreover we must have a file for it # and it must be readable as a Filter - filterReader = FilterReader(filterName, jail, {}) - filterReader.setBaseDir(CONFIG_DIR) + filterReader = FilterReader(filterName, jail, filterOpt, + share_config=self.__share_cfg, basedir=CONFIG_DIR) self.assertTrue(filterReader.read(),"Failed to read filter:" + filterName) # opens fine filterReader.getOptions({}) # reads fine @@ -551,7 +573,10 @@ class JailsReaderTest(LogCaptureTestCase): filters = set(os.path.splitext(os.path.split(a)[1])[0] for a in glob.glob(os.path.join('config', 'filter.d', '*.conf')) if not a.endswith('common.conf')) - filters_jail = set(jail.options['filter'] for jail in jails.jails) + # get filters of all jails (filter names without options inside filter[...]) + filters_jail = set( + JailReader.extractOptions(jail.options['filter'])[0] for jail in jails.jails + ) self.maxDiff = None self.assertTrue(filters.issubset(filters_jail), "More filters exists than are referenced in stock jail.conf %r" % filters.difference(filters_jail)) diff --git a/man/jail.conf.5 b/man/jail.conf.5 index 51ea7097..7a31e8b1 100644 --- a/man/jail.conf.5 +++ b/man/jail.conf.5 @@ -89,13 +89,33 @@ indicates that the specified file is to be parsed before the current file. indicates that the specified file is to be parsed after the current file. .RE -Using Python "string interpolation" mechanisms, other definitions are allowed and can later be used within other definitions as %(name)s. For example. +Using Python "string interpolation" mechanisms, other definitions are allowed and can later be used within other definitions as %(name)s. +Additionaly fail2ban has an extended interpolation feature named \fB%(known/parameter)s\fR (means last known option with name \fBparameter\fR). This interpolation makes possible to extend a stock filter or jail regexp in .local file (opposite to simply set failregex/ignoreregex that overwrites it). For example. .RS +.nf baduseragents = IE|wget +failregex = %(known/failregex)s + useragent=%(baduseragents)s +.fi .RE + +Additionally to interpolation \fB%(known/parameter)s\fR, that does not works for filter/action init parameters, an interpolation tag \fB\fR can be used (means last known init definition of filters or actions with name \fBparameter\fR). This interpolation makes possible to extend a parameters of stock filter or action directly in jail inside \fIjail.conf/jail.local\fR file without creating a separately filter.d/*.local file. For example. + .RS -failregex = useragent=%(baduseragents)s +# filter.d/test.conf: +.nf +[Init] +test.method = GET +baduseragents = IE|wget +[Definition] +failregex = ^%(__prefix_line)\\s+""\\s+test\\s+regexp\\s+-\\s+useragent=(?:) + +# jail.local: +[test] +# use filter "test", overwrite method to "POST" and extend known bad agents with "badagent": +filter = test[test.method=POST, baduseragents="badagent|"] +.fi .RE Comments: use '#' for comment lines and '; ' (space is important) for inline comments. When using Python2.X '; ' can only be used on the first line due to an Python library bug. @@ -253,7 +273,7 @@ The maximum period of time in seconds that a command can executed, before being Commands specified in the [Definition] section are executed through a system shell so shell redirection and process control is allowed. The commands should return 0, otherwise error would be logged. Moreover if \fBactioncheck\fR exits with non-0 status, it is taken as indication that firewall status has changed and fail2ban needs to reinitialize itself (i.e. issue \fBactionstop\fR and \fBactionstart\fR commands). Tags are enclosed in <>. All the elements of [Init] are tags that are replaced in all action commands. Tags can be added by the -\fBfail2ban-client\fR using the "set action " command. \fB
\fR is a tag that is always a new line (\\n). +\fBfail2ban-client\fR using the "set action " command. \fB
\fR is a tag that is always a new line (\\n). More than a single command is allowed to be specified. Each command needs to be on a separate line and indented with whitespace(s) without blank lines. The following example defines two commands to be executed. @@ -312,7 +332,7 @@ is the regex to identify log entries that should be ignored by Fail2Ban, even if .PP -Similar to actions, filters have an [Init] section which can be overridden in \fIjail.conf/jail.local\fR. The filter [Init] section is limited to the following options: +Similar to actions, filters have an [Init] section which can be overridden in \fIjail.conf/jail.local\fR. Besides the filter-specific settings, the filter [Init] section can be used to set following standard options: .TP \fBmaxlines\fR specifies the maximum number of lines to buffer to match multi-line regexs. For some log formats this will not required to be changed. Other logs may require to increase this value if a particular log file is frequently written to. @@ -327,6 +347,8 @@ Also, special values of \fIEpoch\fR (UNIX Timestamp), \fITAI64N\fR and \fIISO860 \fBjournalmatch\fR specifies the systemd journal match used to filter the journal entries. See \fBjournalctl(1)\fR and \fBsystemd.journal-fields(7)\fR for matches syntax and more details on special journal fields. This option is only valid for the \fIsystemd\fR backend. .PP +Similar to actions [Init] section enables filter-specific settings. All parameters specified in [Init] section can be redefined or extended in \fIjail.conf/jail.local\fR. + Filters can also have a section called [INCLUDES]. This is used to read other configuration files. .TP From ba76f4ca2fa058ca187a2990d3ff0b66b6211940 Mon Sep 17 00:00:00 2001 From: Orion Poplawski Date: Mon, 2 Nov 2015 15:21:14 -0700 Subject: [PATCH 072/126] Fix typo --- config/jail.conf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config/jail.conf b/config/jail.conf index 69fef818..3bb17e0a 100644 --- a/config/jail.conf +++ b/config/jail.conf @@ -80,7 +80,7 @@ maxretry = 5 # auto: will try to use the following backends, in order: # pyinotify, gamin, polling. # -# Note: if systemd backend is choses as the default but you enable a jail +# Note: if systemd backend is chosen as the default but you enable a jail # for which logs are present only in its own log files, specify some other # backend for that jail (e.g. polling) and provide empty value for # journalmatch. See https://github.com/fail2ban/fail2ban/issues/959#issuecomment-74901200 From a42aa726ab22d07980ba811bf09151c1e49ce2a4 Mon Sep 17 00:00:00 2001 From: sebres Date: Mon, 9 Nov 2015 20:13:03 +0100 Subject: [PATCH 073/126] fixed fail2ban-regex reads invalid character (in sense of given encoding); continuing to process line ignoring invalid characters (still has no test cases). filter test cases added for same issue inside fail2ban-server / fail2ban-testcases; closes gh-1248 --- bin/fail2ban-regex | 10 +++++-- fail2ban/tests/filtertestcase.py | 49 +++++++++++++++++++++++++++++++- 2 files changed, 56 insertions(+), 3 deletions(-) diff --git a/bin/fail2ban-regex b/bin/fail2ban-regex index b7b0579f..fdbfcb7a 100755 --- a/bin/fail2ban-regex +++ b/bin/fail2ban-regex @@ -84,8 +84,14 @@ def file_lines_gen(hdlr): try: line = line.decode(fail2banRegex.encoding, 'strict') except UnicodeDecodeError: - if sys.version_info >= (3,): # Python 3 must be decoded - line = line.decode(fail2banRegex.encoding, 'ignore') + logSys.warning( + "Error decoding line from '%s' with '%s'." + " Consider setting logencoding=utf-8 (or another appropriate" + " encoding) for this jail. Continuing" + " to process line ignoring invalid characters: %r" % + ('', fail2banRegex.encoding, line)) + # decode with replacing error chars: + line = line.decode(fail2banRegex.encoding, 'replace') yield line def journal_lines_gen(myjournal): diff --git a/fail2ban/tests/filtertestcase.py b/fail2ban/tests/filtertestcase.py index acf15528..b15494f5 100644 --- a/fail2ban/tests/filtertestcase.py +++ b/fail2ban/tests/filtertestcase.py @@ -90,7 +90,11 @@ def _assert_equal_entries(utest, found, output, count=None): found_time, output_time = \ MyTime.localtime(found[2]),\ MyTime.localtime(output[2]) - utest.assertEqual(found_time, output_time) + try: + utest.assertEqual(found_time, output_time) + except AssertionError as e: + # assert more structured: + utest.assertEqual((float(found[2]), found_time), (float(output[2]), output_time)) if len(output) > 3 and count is None: # match matches # do not check if custom count (e.g. going through them twice) if os.linesep != '\n' or sys.platform.startswith('cygwin'): @@ -216,6 +220,14 @@ class BasicFilter(unittest.TestCase): ("^%Y-%m-%d-%H%M%S.%f %z", "^Year-Month-Day-24hourMinuteSecond.Microseconds Zone offset")) + def testAssertWrongTime(self): + self.assertRaises(AssertionError, + lambda: _assert_equal_entries(self, + ('1.1.1.1', 1, 1421262060.0), + ('1.1.1.1', 1, 1421262059.0), + 1) + ) + class IgnoreIP(LogCaptureTestCase): @@ -900,6 +912,41 @@ class GetFailures(unittest.TestCase): except FailManagerEmpty: pass + def testGetFailuresWrongChar(self): + # write wrong utf-8 char: + fname = tempfile.mktemp(prefix='tmp_fail2ban', suffix='crlf') + fout = fopen(fname, 'wb') + try: + # write: + for l in ( + b'2015-01-14 20:00:58 user \"test\xf1ing\" from \"192.0.2.0\"\n', # wrong utf-8 char + b'2015-01-14 20:00:59 user \"\xd1\xe2\xe5\xf2\xe0\" from \"192.0.2.0\"\n', # wrong utf-8 chars + b'2015-01-14 20:01:00 user \"testing\" from \"192.0.2.0\"\n' # correct utf-8 chars + ): + fout.write(l) + fout.close() + # + output = ('192.0.2.0', 3, 1421262060.0) + failregex = "^\s*user \"[^\"]*\" from \"\"\s*$" + + # encoding - auto + self.filter.addLogPath(fname) + self.filter.addFailRegex(failregex) + self.filter.getFailures(fname) + _assert_correct_last_attempt(self, self.filter, output) + + # test direct set of encoding: + for enc in ('utf-8', 'ascii'): + self.tearDown();self.setUp(); + self.filter.setLogEncoding('utf-8'); + self.filter.addLogPath(fname) + self.filter.addFailRegex(failregex) + self.filter.getFailures(fname) + _assert_correct_last_attempt(self, self.filter, output) + + finally: + _killfile(fout, fname) + def testGetFailuresUseDNS(self): # We should still catch failures with usedns = no ;-) output_yes = ('93.184.216.34', 2, 1124013539.0, From 46b116e86abb37428220972dfb064d0a8d101a67 Mon Sep 17 00:00:00 2001 From: sebres Date: Mon, 9 Nov 2015 21:52:06 +0100 Subject: [PATCH 074/126] filter test cases improved + log captured inside such tests + python 3.x compatibility; changelog entry; --- ChangeLog | 2 ++ fail2ban/tests/filtertestcase.py | 24 +++++++++++++----------- 2 files changed, 15 insertions(+), 11 deletions(-) diff --git a/ChangeLog b/ChangeLog index 7e1d84cb..862f46f5 100644 --- a/ChangeLog +++ b/ChangeLog @@ -24,6 +24,8 @@ ver. 0.9.4 (2015/XX/XXX) - wanna-be-released * Fix jail.conf.5 man's section (gh-1226) * Fixed default banaction for allports jails like pam-generic, recidive, etc with new default variable `banaction_allports` (gh-1216) + * Fixed `fail2ban-regex` stops working on invalid (wrong encoded) character + for python version < 3.x (gh-1248) - New Features: * New filters: diff --git a/fail2ban/tests/filtertestcase.py b/fail2ban/tests/filtertestcase.py index b15494f5..3674a574 100644 --- a/fail2ban/tests/filtertestcase.py +++ b/fail2ban/tests/filtertestcase.py @@ -822,7 +822,7 @@ def get_monitor_failures_journal_testcase(Filter_): # pragma: systemd no cover return MonitorJournalFailures -class GetFailures(unittest.TestCase): +class GetFailures(LogCaptureTestCase): FILENAME_01 = os.path.join(TEST_FILES_DIR, "testcase01.log") FILENAME_02 = os.path.join(TEST_FILES_DIR, "testcase02.log") @@ -837,6 +837,7 @@ class GetFailures(unittest.TestCase): def setUp(self): """Call before every test case.""" + LogCaptureTestCase.setUp(self) setUpMyTime() self.jail = DummyJail() self.filter = FileFilter(self.jail) @@ -848,6 +849,7 @@ class GetFailures(unittest.TestCase): def tearDown(self): """Call after every test case.""" tearDownMyTime() + LogCaptureTestCase.tearDown(self) def testTail(self): self.filter.addLogPath(GetFailures.FILENAME_01, tail=True) @@ -929,20 +931,20 @@ class GetFailures(unittest.TestCase): output = ('192.0.2.0', 3, 1421262060.0) failregex = "^\s*user \"[^\"]*\" from \"\"\s*$" - # encoding - auto - self.filter.addLogPath(fname) - self.filter.addFailRegex(failregex) - self.filter.getFailures(fname) - _assert_correct_last_attempt(self, self.filter, output) - - # test direct set of encoding: - for enc in ('utf-8', 'ascii'): - self.tearDown();self.setUp(); - self.filter.setLogEncoding('utf-8'); + # test encoding auto or direct set of encoding: + for enc in (None, 'utf-8', 'ascii'): + if enc is not None: + self.tearDown();self.setUp(); + self.filter.setLogEncoding(enc); + self.assertNotLogged('Error decoding line'); self.filter.addLogPath(fname) self.filter.addFailRegex(failregex) self.filter.getFailures(fname) _assert_correct_last_attempt(self, self.filter, output) + + self.assertLogged('Error decoding line'); + self.assertLogged('Continuing to process line ignoring invalid characters:', '2015-01-14 20:00:58 user '); + self.assertLogged('Continuing to process line ignoring invalid characters:', '2015-01-14 20:00:59 user '); finally: _killfile(fout, fname) From 0877d662287bb74cb403c0159841dbbafda28c08 Mon Sep 17 00:00:00 2001 From: sebres Date: Tue, 10 Nov 2015 11:46:19 +0100 Subject: [PATCH 075/126] fail2ban-regex moved to the client + test cases for initial coverage added --- .../client/fail2banregex.py | 203 +++++++++--------- fail2ban/server/filter.py | 18 +- fail2ban/tests/fail2banregextestcase.py | 155 +++++++++++++ fail2ban/tests/files/testcase-wrong-char.log | 4 + fail2ban/tests/utils.py | 4 + 5 files changed, 275 insertions(+), 109 deletions(-) rename bin/fail2ban-regex => fail2ban/client/fail2banregex.py (81%) create mode 100644 fail2ban/tests/fail2banregextestcase.py create mode 100644 fail2ban/tests/files/testcase-wrong-char.log diff --git a/bin/fail2ban-regex b/fail2ban/client/fail2banregex.py similarity index 81% rename from bin/fail2ban-regex rename to fail2ban/client/fail2banregex.py index fdbfcb7a..12dde7f1 100755 --- a/bin/fail2ban-regex +++ b/fail2ban/client/fail2banregex.py @@ -44,16 +44,16 @@ from ConfigParser import NoOptionError, NoSectionError, MissingSectionHeaderErro try: from systemd import journal - from fail2ban.server.filtersystemd import FilterSystemd + from ..server.filtersystemd import FilterSystemd except ImportError: journal = None -from fail2ban.version import version -from fail2ban.client.filterreader import FilterReader -from fail2ban.server.filter import Filter -from fail2ban.server.failregex import RegexException +from ..version import version +from .filterreader import FilterReader +from ..server.filter import Filter, FileContainer +from ..server.failregex import RegexException -from fail2ban.helpers import FormatterWithTraceBack, getLogger +from ..helpers import FormatterWithTraceBack, getLogger # Gets the instance of the logger. logSys = getLogger("fail2ban") @@ -63,6 +63,9 @@ def debuggexURL(sample, regex): 'flavor': 'python' }) return 'http://www.debuggex.com/?' + q +def output(args): + print(args) + def shortstr(s, l=53): """Return shortened string """ @@ -77,22 +80,7 @@ def pprint_list(l, header=None): s = "|- %s\n" % header else: s = '' - print s + "| " + "\n| ".join(l) + '\n`-' - -def file_lines_gen(hdlr): - for line in hdlr: - try: - line = line.decode(fail2banRegex.encoding, 'strict') - except UnicodeDecodeError: - logSys.warning( - "Error decoding line from '%s' with '%s'." - " Consider setting logencoding=utf-8 (or another appropriate" - " encoding) for this jail. Continuing" - " to process line ignoring invalid characters: %r" % - ('', fail2banRegex.encoding, line)) - # decode with replacing error chars: - line = line.decode(fail2banRegex.encoding, 'replace') - yield line + output( s + "| " + "\n| ".join(l) + '\n`-' ) def journal_lines_gen(myjournal): while True: @@ -259,14 +247,14 @@ class Fail2banRegex(object): self._filter.setDatePattern(pattern) self._datepattern_set = True if pattern is not None: - print "Use datepattern : %s" % ( - self._filter.getDatePattern()[1], ) + output( "Use datepattern : %s" % ( + self._filter.getDatePattern()[1], ) ) def setMaxLines(self, v): if not self._maxlines_set: self._filter.setMaxLines(int(v)) self._maxlines_set = True - print "Use maxlines : %d" % self._filter.getMaxLines() + output( "Use maxlines : %d" % self._filter.getMaxLines() ) def setJournalMatch(self, v): if self._journalmatch is None: @@ -280,18 +268,18 @@ class Fail2banRegex(object): ## within filter.d folder - use standard loading algorithm to load filter completely (with .local etc.): basedir = os.path.dirname(os.path.dirname(value)) value = os.path.splitext(os.path.basename(value))[0] - print "Use %11s filter file : %s, basedir: %s" % (regex, value, basedir) + output( "Use %11s filter file : %s, basedir: %s" % (regex, value, basedir) ) reader = FilterReader(value, 'fail2ban-regex-jail', {}, share_config=self.share_config, basedir=basedir) if not reader.read(): - print "ERROR: failed to load filter %s" % value + output( "ERROR: failed to load filter %s" % value ) return False else: ## foreign file - readexplicit this file and includes if possible: - print "Use %11s file : %s" % (regex, value) + output( "Use %11s file : %s" % (regex, value) ) reader = FilterReader(value, 'fail2ban-regex-jail', {}, share_config=self.share_config) reader.setBaseDir(None) if not reader.readexplicit(): - print "ERROR: failed to read %s" % value + output( "ERROR: failed to read %s" % value ) return False reader.getOptions(None) readercommands = reader.convert() @@ -307,8 +295,8 @@ class Fail2banRegex(object): try: self.setMaxLines(maxlines) except ValueError: - print "ERROR: Invalid value for maxlines (%(maxlines)r) " \ - "read from %(value)s" % locals() + output( "ERROR: Invalid value for maxlines (%(maxlines)r) " \ + "read from %(value)s" % locals() ) return False elif command[2] == 'addjournalmatch': journalmatch = command[3:] @@ -317,7 +305,7 @@ class Fail2banRegex(object): datepattern = command[3] self.setDatePattern(datepattern) else: - print "Use %11s line : %s" % (regex, shortstr(value)) + output( "Use %11s line : %s" % (regex, shortstr(value)) ) regex_values = [RegexStat(value)] setattr(self, "_" + regex, regex_values) @@ -335,7 +323,7 @@ class Fail2banRegex(object): found = True regex = self._ignoreregex[ret].inc() except RegexException, e: - print e + output( e ) return False return found @@ -352,10 +340,10 @@ class Fail2banRegex(object): regex.inc() regex.appendIP(match) except RegexException, e: - print e + output( e ) return False except IndexError: - print "Sorry, but no found in regex" + output( "Sorry, but no found in regex" ) return False for bufLine in orgLineBuffer[int(fullBuffer):]: if bufLine not in self._filter._Filter__lineBuffer: @@ -376,7 +364,7 @@ class Fail2banRegex(object): t0 = time.time() for line_no, line in enumerate(test_lines): if isinstance(line, tuple): - line_datetimestripped, ret = fail2banRegex.testRegex( + line_datetimestripped, ret = self.testRegex( line[0], line[1]) line = "".join(line[0]) else: @@ -384,8 +372,8 @@ class Fail2banRegex(object): if line.startswith('#') or not line: # skip comment and empty lines continue - line_datetimestripped, ret = fail2banRegex.testRegex(line) - is_ignored = fail2banRegex.testIgnoreRegex(line_datetimestripped) + line_datetimestripped, ret = self.testRegex(line) + is_ignored = self.testIgnoreRegex(line_datetimestripped) if is_ignored: self._line_stats.ignored += 1 @@ -432,18 +420,18 @@ class Fail2banRegex(object): b = map(lambda a: a[0] + ' | ' + a[1].getFailRegex() + ' | ' + debuggexURL(a[0], a[1].getFailRegex()), ans) pprint_list([x.rstrip() for x in b], header) else: - print "%s too many to print. Use --print-all-%s " \ - "to print all %d lines" % (header, ltype, lines) + output( "%s too many to print. Use --print-all-%s " \ + "to print all %d lines" % (header, ltype, lines) ) elif lines < self._maxlines or getattr(self, '_print_all_' + ltype): pprint_list([x.rstrip() for x in l], header) else: - print "%s too many to print. Use --print-all-%s " \ - "to print all %d lines" % (header, ltype, lines) + output( "%s too many to print. Use --print-all-%s " \ + "to print all %d lines" % (header, ltype, lines) ) def printStats(self): - print - print "Results" - print "=======" + output( "" ) + output( "Results" ) + output( "=======" ) def print_failregexes(title, failregexes): # Print title @@ -464,7 +452,7 @@ class Fail2banRegex(object): timeString, ip[-1] and " (multiple regex matched)" or "")) - print "\n%s: %d total" % (title, total) + output( "\n%s: %d total" % (title, total) ) pprint_list(out, " #) [# of hits] regular expression") return total @@ -474,7 +462,7 @@ class Fail2banRegex(object): if self._filter.dateDetector is not None: - print "\nDate template hits:" + output( "\nDate template hits:" ) out = [] for template in self._filter.dateDetector.templates: if self._verbose or template.hits: @@ -482,10 +470,10 @@ class Fail2banRegex(object): template.hits, template.name)) pprint_list(out, "[# of hits] date format") - print "\nLines: %s" % self._line_stats, + output( "\nLines: %s" % self._line_stats, ) if self._time_elapsed is not None: - print "[processed in %.2f sec]" % self._time_elapsed, - print + output( "[processed in %.2f sec]" % self._time_elapsed, ) + output( "" ) if self._print_all_matched: self.printLines('matched') @@ -496,9 +484,62 @@ class Fail2banRegex(object): return True + def file_lines_gen(self, hdlr): + for line in hdlr: + yield FileContainer.decode_line('', self.encoding, line) -if __name__ == "__main__": + def start(self, opts, args): + cmd_log, cmd_regex = args[:2] + + if not self.readRegex(cmd_regex, 'fail'): + return False + + if len(args) == 3 and not self.readRegex(args[2], 'ignore'): + return False + + if os.path.isfile(cmd_log): + try: + hdlr = open(cmd_log, 'rb') + output( "Use log file : %s" % cmd_log ) + output( "Use encoding : %s" % self.encoding ) + test_lines = self.file_lines_gen(hdlr) + except IOError, e: + output( e ) + return False + elif cmd_log == "systemd-journal": + if not journal: + output( "Error: systemd library not found. Exiting..." ) + return False + myjournal = journal.Reader(converters={'__CURSOR': lambda x: x}) + journalmatch = self._journalmatch + self.setDatePattern(None) + if journalmatch: + try: + for element in journalmatch: + if element == "+": + myjournal.add_disjunction() + else: + myjournal.add_match(element) + except ValueError: + output( "Error: Invalid journalmatch: %s" % shortstr(" ".join(journalmatch)) ) + return False + output( "Use journal match : %s" % " ".join(journalmatch) ) + test_lines = journal_lines_gen(myjournal) + else: + output( "Use single line : %s" % shortstr(cmd_log) ) + test_lines = [ cmd_log ] + output( "" ) + + self.process(test_lines) + + if not self.printStats(): + return False + + return True + + +def exec_command_line(): # pragma: no cover parser = get_opt_parser() (opts, args) = parser.parse_args() if opts.print_no_missed and opts.print_all_missed: @@ -510,18 +551,16 @@ if __name__ == "__main__": parser.print_help() sys.exit(-1) - print - print "Running tests" - print "=============" - print - - fail2banRegex = Fail2banRegex(opts) - # We need 2 or 3 parameters if not len(args) in (2, 3): sys.stderr.write("ERROR: provide both and .\n\n") parser.print_help() - sys.exit(-1) + return False + + output( "" ) + output( "Running tests" ) + output( "=============" ) + output( "" ) # TODO: taken from -testcases -- move common functionality somewhere if opts.log_level is not None: # pragma: no cover @@ -552,46 +591,6 @@ if __name__ == "__main__": stdout.setFormatter(Formatter(fmt)) logSys.addHandler(stdout) - cmd_log, cmd_regex = args[:2] - - fail2banRegex.readRegex(cmd_regex, 'fail') or sys.exit(-1) - - if len(args) == 3: - fail2banRegex.readRegex(args[2], 'ignore') or sys.exit(-1) - - if os.path.isfile(cmd_log): - try: - hdlr = open(cmd_log, 'rb') - print "Use log file : %s" % cmd_log - print "Use encoding : %s" % fail2banRegex.encoding - test_lines = file_lines_gen(hdlr) - except IOError, e: - print e - sys.exit(-1) - elif cmd_log == "systemd-journal": - if not journal: - print "Error: systemd library not found. Exiting..." - sys.exit(-1) - myjournal = journal.Reader(converters={'__CURSOR': lambda x: x}) - journalmatch = fail2banRegex._journalmatch - fail2banRegex.setDatePattern(None) - if journalmatch: - try: - for element in journalmatch: - if element == "+": - myjournal.add_disjunction() - else: - myjournal.add_match(element) - except ValueError: - print "Error: Invalid journalmatch: %s" % shortstr(" ".join(journalmatch)) - sys.exit(-1) - print "Use journal match : %s" % " ".join(journalmatch) - test_lines = journal_lines_gen(myjournal) - else: - print "Use single line : %s" % shortstr(cmd_log) - test_lines = [ cmd_log ] - print - - fail2banRegex.process(test_lines) - - fail2banRegex.printStats() or sys.exit(-1) + fail2banRegex = Fail2banRegex(opts) + if not fail2banRegex.start(opts, args): + sys.exit(-1) diff --git a/fail2ban/server/filter.py b/fail2ban/server/filter.py index 6fc2cd6c..65be0467 100644 --- a/fail2ban/server/filter.py +++ b/fail2ban/server/filter.py @@ -792,23 +792,27 @@ class FileContainer: self.__handler.seek(self.__pos) return True - def readline(self): - if self.__handler is None: - return "" - line = self.__handler.readline() + @staticmethod + def decode_line(filename, enc, line): try: - line = line.decode(self.getEncoding(), 'strict') + line = line.decode(enc, 'strict') except UnicodeDecodeError: logSys.warning( "Error decoding line from '%s' with '%s'." " Consider setting logencoding=utf-8 (or another appropriate" " encoding) for this jail. Continuing" " to process line ignoring invalid characters: %r" % - (self.getFileName(), self.getEncoding(), line)) + (filename, enc, line)) # decode with replacing error chars: - line = line.decode(self.getEncoding(), 'replace') + line = line.decode(enc, 'replace') return line + def readline(self): + if self.__handler is None: + return "" + return FileContainer.decode_line( + self.getFileName(), self.getEncoding(), self.__handler.readline()) + def close(self): if not self.__handler is None: # Saves the last position. diff --git a/fail2ban/tests/fail2banregextestcase.py b/fail2ban/tests/fail2banregextestcase.py new file mode 100644 index 00000000..ee10128e --- /dev/null +++ b/fail2ban/tests/fail2banregextestcase.py @@ -0,0 +1,155 @@ +# emacs: -*- mode: python; py-indent-offset: 4; indent-tabs-mode: t -*- +# vi: set ft=python sts=4 ts=4 sw=4 noet : + +# This file is part of Fail2Ban. +# +# Fail2Ban is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. +# +# Fail2Ban is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with Fail2Ban; if not, write to the Free Software +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + +# Fail2Ban developers + +__author__ = "Serg Brester" +__copyright__ = "Copyright (c) 2015 Serg G. Brester (sebres), 2008- Fail2Ban Contributors" +__license__ = "GPL" + +from __builtin__ import open as fopen +import unittest +import getpass +import os +import sys +import time +import tempfile +import uuid + +try: + from systemd import journal +except ImportError: + journal = None + +from ..client import fail2banregex +from ..client.fail2banregex import Fail2banRegex, get_opt_parser, output +from .utils import LogCaptureTestCase, logSys + + +fail2banregex.logSys = logSys +def _test_output(*args): + logSys.info(args[0]) + +fail2banregex.output = _test_output + +CONF_FILES_DIR = os.path.abspath( + os.path.join(os.path.dirname(__file__),"..", "..", "config")) +TEST_FILES_DIR = os.path.join(os.path.dirname(__file__), "files") + + +def _Fail2banRegex(*args): + parser = get_opt_parser() + (opts, args) = parser.parse_args(list(args)) + return (opts, args, Fail2banRegex(opts)) + +class Fail2banRegexTest(LogCaptureTestCase): + + FILENAME_01 = os.path.join(TEST_FILES_DIR, "testcase01.log") + FILENAME_02 = os.path.join(TEST_FILES_DIR, "testcase02.log") + FILENAME_WRONGCHAR = os.path.join(TEST_FILES_DIR, "testcase-wrong-char.log") + + FILTER_SSHD = os.path.join(CONF_FILES_DIR, 'filter.d', 'sshd.conf') + + def setUp(self): + """Call before every test case.""" + LogCaptureTestCase.setUp(self) + + def tearDown(self): + """Call after every test case.""" + LogCaptureTestCase.tearDown(self) + + def testWrongRE(self): + (opts, args, fail2banRegex) = _Fail2banRegex( + "test", r".** from $" + ) + self.assertRaises(Exception, lambda: fail2banRegex.start(opts, args)) + self.assertLogged("Unable to compile regular expression") + + def testWrongIngnoreRE(self): + (opts, args, fail2banRegex) = _Fail2banRegex( + "test", r".*? from $", r".**" + ) + self.assertRaises(Exception, lambda: fail2banRegex.start(opts, args)) + self.assertLogged("Unable to compile regular expression") + + def testDirectFound(self): + (opts, args, fail2banRegex) = _Fail2banRegex( + "--print-all-matched", "--print-no-missed", + "Dec 31 11:59:59 [sshd] error: PAM: Authentication failure for kevin from 192.0.2.0", + r"Authentication failure for .*? from $" + ) + self.assertTrue(fail2banRegex.start(opts, args)) + self.assertLogged('Lines: 1 lines, 0 ignored, 1 matched, 0 missed') + + def testDirectNotFound(self): + (opts, args, fail2banRegex) = _Fail2banRegex( + "--print-all-missed", + "Dec 31 11:59:59 [sshd] error: PAM: Authentication failure for kevin from 192.0.2.0", + r"XYZ from $" + ) + self.assertTrue(fail2banRegex.start(opts, args)) + self.assertLogged('Lines: 1 lines, 0 ignored, 0 matched, 1 missed') + + def testDirectIgnored(self): + (opts, args, fail2banRegex) = _Fail2banRegex( + "--print-all-ignored", + "Dec 31 11:59:59 [sshd] error: PAM: Authentication failure for kevin from 192.0.2.0", + r"Authentication failure for .*? from $", + r"kevin from 192.0.2.0$" + ) + self.assertTrue(fail2banRegex.start(opts, args)) + self.assertLogged('Lines: 1 lines, 1 ignored, 0 matched, 0 missed') + + def testDirectRE_1(self): + (opts, args, fail2banRegex) = _Fail2banRegex( + "--print-all-matched", + Fail2banRegexTest.FILENAME_01, + r"(?:(?:Authentication failure|Failed [-/\w+]+) for(?: [iI](?:llegal|nvalid) user)?|[Ii](?:llegal|nvalid) user|ROOT LOGIN REFUSED) .*(?: from|FROM) " + ) + self.assertTrue(fail2banRegex.start(opts, args)) + self.assertLogged('Lines: 19 lines, 0 ignored, 13 matched, 6 missed') + + self.assertLogged('Error decoding line'); + self.assertLogged('Continuing to process line ignoring invalid characters') + + self.assertLogged('Dez 31 11:59:59 [sshd] error: PAM: Authentication failure for kevin from 193.168.0.128') + self.assertLogged('Dec 31 11:59:59 [sshd] error: PAM: Authentication failure for kevin from 87.142.124.10') + + def testDirectRE_2(self): + (opts, args, fail2banRegex) = _Fail2banRegex( + "--print-all-matched", + Fail2banRegexTest.FILENAME_02, + r"(?:(?:Authentication failure|Failed [-/\w+]+) for(?: [iI](?:llegal|nvalid) user)?|[Ii](?:llegal|nvalid) user|ROOT LOGIN REFUSED) .*(?: from|FROM) " + ) + self.assertTrue(fail2banRegex.start(opts, args)) + self.assertLogged('Lines: 13 lines, 0 ignored, 5 matched, 8 missed') + + def testWronChar(self): + (opts, args, fail2banRegex) = _Fail2banRegex( + Fail2banRegexTest.FILENAME_WRONGCHAR, Fail2banRegexTest.FILTER_SSHD + ) + self.assertTrue(fail2banRegex.start(opts, args)) + self.assertLogged('Lines: 4 lines, 0 ignored, 2 matched, 2 missed') + + self.assertLogged('Error decoding line'); + self.assertLogged('Continuing to process line ignoring invalid characters:', '2015-01-14 20:00:58 user '); + self.assertLogged('Continuing to process line ignoring invalid characters:', '2015-01-14 20:00:59 user '); + + self.assertLogged('Nov 8 00:16:12 main sshd[32548]: input_userauth_request: invalid user llinco') + self.assertLogged('Nov 8 00:16:12 main sshd[32547]: pam_succeed_if(sshd:auth): error retrieving information about user llinco') diff --git a/fail2ban/tests/files/testcase-wrong-char.log b/fail2ban/tests/files/testcase-wrong-char.log new file mode 100644 index 00000000..9736020e --- /dev/null +++ b/fail2ban/tests/files/testcase-wrong-char.log @@ -0,0 +1,4 @@ +Nov 8 00:16:12 main sshd[32547]: Invalid user llinco\361ir from 192.0.2.0 +Nov 8 00:16:12 main sshd[32548]: input_userauth_request: invalid user llinco\361ir +Nov 8 00:16:12 main sshd[32547]: pam_succeed_if(sshd:auth): error retrieving information about user llincoir +Nov 8 00:16:14 main sshd[32547]: Failed password for invalid user llinco\361ir from 192.0.2.0 port 57025 ssh2 diff --git a/fail2ban/tests/utils.py b/fail2ban/tests/utils.py index 35fa59fd..8172e7ec 100644 --- a/fail2ban/tests/utils.py +++ b/fail2ban/tests/utils.py @@ -85,6 +85,7 @@ def gatherTests(regexps=None, no_network=False): from . import misctestcase from . import databasetestcase from . import samplestestcase + from . import fail2banregextestcase if not regexps: # pragma: no cover tests = unittest.TestSuite() @@ -152,6 +153,9 @@ def gatherTests(regexps=None, no_network=False): # Filter Regex tests with sample logs tests.addTest(unittest.makeSuite(samplestestcase.FilterSamplesRegex)) + # bin/fail2ban-regex + tests.addTest(unittest.makeSuite(fail2banregextestcase.Fail2banRegexTest)) + # # Python action testcases # From 38f09b417ad514254b81edb210322d6658c5a9e9 Mon Sep 17 00:00:00 2001 From: sebres Date: Tue, 10 Nov 2015 11:48:37 +0100 Subject: [PATCH 076/126] fail2ban-regex command line (after fail2ban-regex functionality moved to the client) --- bin/fail2ban-regex | 34 ++++++++++++++++++++++++++++++++++ 1 file changed, 34 insertions(+) create mode 100755 bin/fail2ban-regex diff --git a/bin/fail2ban-regex b/bin/fail2ban-regex new file mode 100755 index 00000000..584c1ea7 --- /dev/null +++ b/bin/fail2ban-regex @@ -0,0 +1,34 @@ +#!/usr/bin/python +# emacs: -*- mode: python; py-indent-offset: 4; indent-tabs-mode: t -*- +# vi: set ft=python sts=4 ts=4 sw=4 noet : +# +# This file is part of Fail2Ban. +# +# Fail2Ban is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. +# +# Fail2Ban is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with Fail2Ban; if not, write to the Free Software +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +""" +Fail2Ban reads log file that contains password failure report +and bans the corresponding IP addresses using firewall rules. + +This tools can test regular expressions for "fail2ban". + +""" + +__author__ = "Fail2Ban Developers" +__copyright__ = "Copyright (c) 2004-2008 Cyril Jaquier, 2012-2014 Yaroslav Halchenko" +__license__ = "GPL" + +from fail2ban.client.fail2banregex import exec_command_line + +exec_command_line() From 689dfa1e6a27aa49e0c06895d7cb049e6352174c Mon Sep 17 00:00:00 2001 From: sebres Date: Tue, 10 Nov 2015 12:19:46 +0100 Subject: [PATCH 077/126] debuggexURL fixed for wrong encoded character; test cases extended; --- fail2ban/client/fail2banregex.py | 15 ++++++++----- fail2ban/tests/fail2banregextestcase.py | 30 +++++++++++++++++++++++-- 2 files changed, 37 insertions(+), 8 deletions(-) diff --git a/fail2ban/client/fail2banregex.py b/fail2ban/client/fail2banregex.py index 12dde7f1..92ad4a91 100755 --- a/fail2ban/client/fail2banregex.py +++ b/fail2ban/client/fail2banregex.py @@ -204,7 +204,7 @@ class LineStats(object): # just for convenient str def __getitem__(self, key): - return getattr(self, key) + return getattr(self, key) if hasattr(self, key) else '' class Fail2banRegex(object): @@ -240,7 +240,11 @@ class Fail2banRegex(object): else: self.encoding = locale.getpreferredencoding() + def decode_line(self, line): + return FileContainer.decode_line('', self.encoding, line) + def encode_line(self, line): + return line.encode(self.encoding, 'ignore') def setDatePattern(self, pattern): if not self._datepattern_set: @@ -398,8 +402,6 @@ class Fail2banRegex(object): self._filter.dateDetector.sortTemplate() self._time_elapsed = time.time() - t0 - - def printLines(self, ltype): lstats = self._line_stats assert(self._line_stats.missed == lstats.tested - (lstats.matched + lstats.ignored)) @@ -417,7 +419,8 @@ class Fail2banRegex(object): ans = [[]] for arg in [l, regexlist]: ans = [ x + [y] for x in ans for y in arg ] - b = map(lambda a: a[0] + ' | ' + a[1].getFailRegex() + ' | ' + debuggexURL(a[0], a[1].getFailRegex()), ans) + b = map(lambda a: a[0] + ' | ' + a[1].getFailRegex() + ' | ' + + debuggexURL(self.encode_line(a[0]), a[1].getFailRegex()), ans) pprint_list([x.rstrip() for x in b], header) else: output( "%s too many to print. Use --print-all-%s " \ @@ -486,7 +489,7 @@ class Fail2banRegex(object): def file_lines_gen(self, hdlr): for line in hdlr: - yield FileContainer.decode_line('', self.encoding, line) + yield self.decode_line(line) def start(self, opts, args): @@ -507,7 +510,7 @@ class Fail2banRegex(object): except IOError, e: output( e ) return False - elif cmd_log == "systemd-journal": + elif cmd_log == "systemd-journal": # pragma: no cover if not journal: output( "Error: systemd library not found. Exiting..." ) return False diff --git a/fail2ban/tests/fail2banregextestcase.py b/fail2ban/tests/fail2banregextestcase.py index ee10128e..2fd362c7 100644 --- a/fail2ban/tests/fail2banregextestcase.py +++ b/fail2ban/tests/fail2banregextestcase.py @@ -60,6 +60,8 @@ def _Fail2banRegex(*args): class Fail2banRegexTest(LogCaptureTestCase): + RE_00 = r"(?:(?:Authentication failure|Failed [-/\w+]+) for(?: [iI](?:llegal|nvalid) user)?|[Ii](?:llegal|nvalid) user|ROOT LOGIN REFUSED) .*(?: from|FROM) " + FILENAME_01 = os.path.join(TEST_FILES_DIR, "testcase01.log") FILENAME_02 = os.path.join(TEST_FILES_DIR, "testcase02.log") FILENAME_WRONGCHAR = os.path.join(TEST_FILES_DIR, "testcase-wrong-char.log") @@ -120,7 +122,7 @@ class Fail2banRegexTest(LogCaptureTestCase): (opts, args, fail2banRegex) = _Fail2banRegex( "--print-all-matched", Fail2banRegexTest.FILENAME_01, - r"(?:(?:Authentication failure|Failed [-/\w+]+) for(?: [iI](?:llegal|nvalid) user)?|[Ii](?:llegal|nvalid) user|ROOT LOGIN REFUSED) .*(?: from|FROM) " + Fail2banRegexTest.RE_00 ) self.assertTrue(fail2banRegex.start(opts, args)) self.assertLogged('Lines: 19 lines, 0 ignored, 13 matched, 6 missed') @@ -135,11 +137,23 @@ class Fail2banRegexTest(LogCaptureTestCase): (opts, args, fail2banRegex) = _Fail2banRegex( "--print-all-matched", Fail2banRegexTest.FILENAME_02, - r"(?:(?:Authentication failure|Failed [-/\w+]+) for(?: [iI](?:llegal|nvalid) user)?|[Ii](?:llegal|nvalid) user|ROOT LOGIN REFUSED) .*(?: from|FROM) " + Fail2banRegexTest.RE_00 ) self.assertTrue(fail2banRegex.start(opts, args)) self.assertLogged('Lines: 13 lines, 0 ignored, 5 matched, 8 missed') + def testVerbose(self): + (opts, args, fail2banRegex) = _Fail2banRegex( + "--verbose", "--print-no-missed", + Fail2banRegexTest.FILENAME_02, + Fail2banRegexTest.RE_00 + ) + self.assertTrue(fail2banRegex.start(opts, args)) + self.assertLogged('Lines: 13 lines, 0 ignored, 5 matched, 8 missed') + + self.assertLogged('141.3.81.106 Fri Aug 14 11:53:59 2015') + self.assertLogged('141.3.81.106 Fri Aug 14 11:54:59 2015') + def testWronChar(self): (opts, args, fail2banRegex) = _Fail2banRegex( Fail2banRegexTest.FILENAME_WRONGCHAR, Fail2banRegexTest.FILTER_SSHD @@ -153,3 +167,15 @@ class Fail2banRegexTest(LogCaptureTestCase): self.assertLogged('Nov 8 00:16:12 main sshd[32548]: input_userauth_request: invalid user llinco') self.assertLogged('Nov 8 00:16:12 main sshd[32547]: pam_succeed_if(sshd:auth): error retrieving information about user llinco') + + def testWronCharDebuggex(self): + (opts, args, fail2banRegex) = _Fail2banRegex( + "--debuggex", "--print-all-matched", + Fail2banRegexTest.FILENAME_WRONGCHAR, Fail2banRegexTest.FILTER_SSHD + ) + self.assertTrue(fail2banRegex.start(opts, args)) + self.assertLogged('Lines: 4 lines, 0 ignored, 2 matched, 2 missed') + + self.assertLogged('http://') + + From 441dffbe2a539a3910c5ae4fc4e1341d9a103640 Mon Sep 17 00:00:00 2001 From: Yaroslav Halchenko Date: Tue, 10 Nov 2015 08:31:56 -0500 Subject: [PATCH 078/126] ENH: Pruned some "pragma: no cover"s in fail2banregex This code should and can be unit-tested, so no reason to keep it with no cover --- fail2ban/client/fail2banregex.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/fail2ban/client/fail2banregex.py b/fail2ban/client/fail2banregex.py index 92ad4a91..d0cdab84 100755 --- a/fail2ban/client/fail2banregex.py +++ b/fail2ban/client/fail2banregex.py @@ -542,7 +542,7 @@ class Fail2banRegex(object): return True -def exec_command_line(): # pragma: no cover +def exec_command_line(): parser = get_opt_parser() (opts, args) = parser.parse_args() if opts.print_no_missed and opts.print_all_missed: @@ -566,10 +566,10 @@ def exec_command_line(): # pragma: no cover output( "" ) # TODO: taken from -testcases -- move common functionality somewhere - if opts.log_level is not None: # pragma: no cover + if opts.log_level is not None: # so we had explicit settings logSys.setLevel(getattr(logging, opts.log_level.upper())) - else: # pragma: no cover + else: # suppress the logging but it would leave unittests' progress dots # ticking, unless like with '-l critical' which would be silent # unless error occurs @@ -587,9 +587,9 @@ def exec_command_line(): # pragma: no cover Formatter = logging.Formatter # Custom log format for the verbose tests runs - if opts.verbose: # pragma: no cover + if opts.verbose: stdout.setFormatter(Formatter(' %(asctime)-15s %(thread)s' + fmt)) - else: # pragma: no cover + else: # just prefix with the space stdout.setFormatter(Formatter(fmt)) logSys.addHandler(stdout) From b3ed19b36adb17bfbd550bd0e32aac34e9ab8243 Mon Sep 17 00:00:00 2001 From: Yaroslav Halchenko Date: Tue, 10 Nov 2015 08:47:13 -0500 Subject: [PATCH 079/126] DOC: tune up to jail.conf.5 - some line breaks, typos etc --- man/jail.conf.5 | 50 +++++++++++++++++++++++++++++++++---------------- 1 file changed, 34 insertions(+), 16 deletions(-) diff --git a/man/jail.conf.5 b/man/jail.conf.5 index 7a31e8b1..865c689e 100644 --- a/man/jail.conf.5 +++ b/man/jail.conf.5 @@ -1,4 +1,4 @@ -.TH JAIL.CONF "5" "October 2013" "Fail2Ban" "Fail2Ban Configuration" +.TH JAIL.CONF "5" "November 2015" "Fail2Ban" "Fail2Ban Configuration" .SH NAME jail.conf \- configuration for the fail2ban server .SH SYNOPSIS @@ -89,8 +89,8 @@ indicates that the specified file is to be parsed before the current file. indicates that the specified file is to be parsed after the current file. .RE -Using Python "string interpolation" mechanisms, other definitions are allowed and can later be used within other definitions as %(name)s. -Additionaly fail2ban has an extended interpolation feature named \fB%(known/parameter)s\fR (means last known option with name \fBparameter\fR). This interpolation makes possible to extend a stock filter or jail regexp in .local file (opposite to simply set failregex/ignoreregex that overwrites it). For example. +Using Python "string interpolation" mechanisms, other definitions are allowed and can later be used within other definitions as %(name)s. +Additionally fail2ban has an extended interpolation feature named \fB%(known/parameter)s\fR (means last known option with name \fBparameter\fR). This interpolation makes possible to extend a stock filter or jail regexp in .local file (opposite to simply set failregex/ignoreregex that overwrites it), e.g. .RS .nf @@ -100,7 +100,7 @@ failregex = %(known/failregex)s .fi .RE -Additionally to interpolation \fB%(known/parameter)s\fR, that does not works for filter/action init parameters, an interpolation tag \fB\fR can be used (means last known init definition of filters or actions with name \fBparameter\fR). This interpolation makes possible to extend a parameters of stock filter or action directly in jail inside \fIjail.conf/jail.local\fR file without creating a separately filter.d/*.local file. For example. +Additionally to interpolation \fB%(known/parameter)s\fR, that does not works for filter/action init parameters, an interpolation tag \fB\fR can be used (means last known init definition of filters or actions with name \fBparameter\fR). This interpolation makes possible to extend a parameters of stock filter or action directly in jail inside \fIjail.conf/jail.local\fR file without creating a separately filter.d/*.local file, e.g. .RS # filter.d/test.conf: @@ -118,7 +118,7 @@ filter = test[test.method=POST, baduseragents="badagent|"] .fi .RE -Comments: use '#' for comment lines and '; ' (space is important) for inline comments. When using Python2.X '; ' can only be used on the first line due to an Python library bug. +Comments: use '#' for comment lines and '; ' (space is important) for inline comments. When using Python2.X, '; ' can only be used on the first line due to an Python library bug. .SH "FAIL2BAN CONFIGURATION FILE(S) (\fIfail2ban.conf\fB)" @@ -130,34 +130,44 @@ The items that can be set are: verbosity level of log output: CRITICAL, ERROR, WARNING, NOTICE, INFO, DEBUG. Default: ERROR .TP .B logtarget -log target: filename, SYSLOG, STDERR or STDOUT. Default: STDERR . Only a single log target can be specified. +log target: filename, SYSLOG, STDERR or STDOUT. Default: STDERR +.br +Only a single log target can be specified. If you change logtarget from the default value and you are using logrotate -- also adjust or disable rotation in the corresponding configuration file (e.g. /etc/logrotate.d/fail2ban on Debian systems). .TP .B socket -socket filename. Default: /var/run/fail2ban/fail2ban.sock . +socket filename. Default: /var/run/fail2ban/fail2ban.sock +.br This is used for communication with the fail2ban server daemon. Do not remove this file when Fail2ban is running. It will not be possible to communicate with the server afterwards. .TP .B pidfile -PID filename. Default: /var/run/fail2ban/fail2ban.pid. +PID filename. Default: /var/run/fail2ban/fail2ban.pid +.br This is used to store the process ID of the fail2ban server. .TP .B dbfile Database filename. Default: /var/lib/fail2ban/fail2ban.sqlite3 +.br This defines where the persistent data for fail2ban is stored. This persistent data allows bans to be reinstated and continue reading log files from the last read position when fail2ban is restarted. A value of \fINone\fR disables this feature. .TP .B dbpurgeage Database purge age in seconds. Default: 86400 (24hours) +.br This sets the age at which bans should be purged from the database. .SH "JAIL CONFIGURATION FILE(S) (\fIjail.conf\fB)" The following options are applicable to any jail. They appear in a section specifying the jail name or in the \fI[DEFAULT]\fR section which defines default values to be used if not specified in the individual section. .TP .B filter -name of the filter -- filename of the filter in /etc/fail2ban/filter.d/ without the .conf/.local extension. Only one filter can be specified. +name of the filter -- filename of the filter in /etc/fail2ban/filter.d/ without the .conf/.local extension. +.br +Only one filter can be specified. .TP .B logpath -filename(s) of the log files to be monitored, separated by new lines. Globs -- paths containing * and ? or [0-9] -- can be used however only the files that exist at start up matching this glob pattern will be considered. +filename(s) of the log files to be monitored, separated by new lines. +.br +Globs -- paths containing * and ? or [0-9] -- can be used however only the files that exist at start up matching this glob pattern will be considered. Optional space separated option 'tail' can be added to the end of the path to cause the log file to be read from the end, else default 'head' option reads file from the beginning @@ -167,13 +177,17 @@ Ensure syslog or the program that generates the log file isn't configured to com encoding of log files used for decoding. Default value of "auto" uses current system locale. .TP .B banaction -banning action (default iptables-multiport) typically specified in the \fI[DEFAULT]\fR section for all jails. This parameter will be used by the standard substitution of \fIaction\fR and can be redefined central in the \fI[DEFAULT]\fR section inside \fIjail.local\fR (to apply it to all jails at once) or separately in each jail, where this substitution will be used. +banning action (default iptables-multiport) typically specified in the \fI[DEFAULT]\fR section for all jails. +.br +This parameter will be used by the standard substitution of \fIaction\fR and can be redefined central in the \fI[DEFAULT]\fR section inside \fIjail.local\fR (to apply it to all jails at once) or separately in each jail, where this substitution will be used. .TP .B banaction_allports the same as \fIbanaction\fR but for some "allports" jails like "pam-generic" or "recidive" (default iptables-allports). .TP .B action -action(s) from \fI/etc/fail2ban/action.d/\fR without the \fI.conf\fR/\fI.local\fR extension. Arguments can be passed to actions to override the default values from the [Init] section in the action file. Arguments are specified by: +action(s) from \fI/etc/fail2ban/action.d/\fR without the \fI.conf\fR/\fI.local\fR extension. +.br +Arguments can be passed to actions to override the default values from the [Init] section in the action file. Arguments are specified by: .RS .RS @@ -187,7 +201,9 @@ Values can also be quoted (required when value includes a ","). More that one ac list of IPs not to ban. They can include a CIDR mask too. .TP .B ignorecommand -command that is executed to determine if the current candidate IP for banning should not be banned. IP will not be banned if command returns successfully (exit code 0). +command that is executed to determine if the current candidate IP for banning should not be banned. +.br +IP will not be banned if command returns successfully (exit code 0). Like ACTION FILES, tags like are can be included in the ignorecommand value and will be substituted before execution. Currently only is supported however more will be added later. .TP .B bantime @@ -200,7 +216,9 @@ time interval (in seconds) before the current time where failures will count tow number of failures that have to occur in the last \fBfindtime\fR seconds to ban then IP. .TP .B backend -backend to be used to detect changes in the logpath. It defaults to "auto" which will try "pyinotify", "gamin", "systemd" before "polling". Any of these can be specified. "pyinotify" is only valid on Linux systems with the "pyinotify" Python libraries. "gamin" requires the "gamin" libraries. +backend to be used to detect changes in the logpath. +.br +It defaults to "auto" which will try "pyinotify", "gamin", "systemd" before "polling". Any of these can be specified. "pyinotify" is only valid on Linux systems with the "pyinotify" Python libraries. "gamin" requires the "gamin" libraries. .TP .B usedns use DNS to resolve HOST names that appear in the logs. By default it is "warn" which will resolve hostnames to IPs however it will also log a warning. If you are using DNS here you could be blocking the wrong IPs due to the asymmetric nature of reverse DNS (that the application used to write the domain name to log) compared to forward DNS that fail2ban uses to resolve this back to an IP (but not necessarily the same one). Ideally you should configure your applications to log a real IP. This can be set to "yes" to prevent warnings in the log or "no" to disable DNS resolution altogether (thus ignoring entries where hostname, not an IP is logged).. @@ -271,9 +289,9 @@ The maximum period of time in seconds that a command can executed, before being .RE Commands specified in the [Definition] section are executed through a system shell so shell redirection and process control is allowed. The commands should -return 0, otherwise error would be logged. Moreover if \fBactioncheck\fR exits with non-0 status, it is taken as indication that firewall status has changed and fail2ban needs to reinitialize itself (i.e. issue \fBactionstop\fR and \fBactionstart\fR commands). +return 0, otherwise error would be logged. Moreover if \fBactioncheck\fR exits with non-0 status, it is taken as indication that firewall status has changed and fail2ban needs to reinitialize itself (i.e. issue \fBactionstop\fR and \fBactionstart\fR commands). Tags are enclosed in <>. All the elements of [Init] are tags that are replaced in all action commands. Tags can be added by the -\fBfail2ban-client\fR using the "set action " command. \fB
\fR is a tag that is always a new line (\\n). +\fBfail2ban-client\fR using the "set action " command. \fB
\fR is a tag that is always a new line (\\n). More than a single command is allowed to be specified. Each command needs to be on a separate line and indented with whitespace(s) without blank lines. The following example defines two commands to be executed. From 9905396eb8bf0563a67ea9dc1d09d31a8af7e077 Mon Sep 17 00:00:00 2001 From: James Hartig Date: Wed, 11 Nov 2015 23:10:35 -0500 Subject: [PATCH 080/126] Added PartOf to service file so f2b restarts when deps do --- files/fail2ban.service | 1 + 1 file changed, 1 insertion(+) diff --git a/files/fail2ban.service b/files/fail2ban.service index 6ebbacc0..e79faed1 100644 --- a/files/fail2ban.service +++ b/files/fail2ban.service @@ -2,6 +2,7 @@ Description=Fail2Ban Service Documentation=man:fail2ban(1) After=network.target iptables.service firewalld.service +PartOf=iptables.service firewalld.service [Service] Type=forking From fa59a6850ff5b093ed2c5928e5d86172bedc1aa9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Aleksandrs=20=C4=BBedovskis?= Date: Sun, 22 Nov 2015 12:01:15 +0200 Subject: [PATCH 081/126] Add 'Sender address rejected: Domain not found' Postfix failregex MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Aleksandrs Ļedovskis --- ChangeLog | 1 + config/filter.d/postfix.conf | 1 + fail2ban/tests/files/logs/postfix | 3 +++ 3 files changed, 5 insertions(+) diff --git a/ChangeLog b/ChangeLog index 20b708a8..15a3129d 100644 --- a/ChangeLog +++ b/ChangeLog @@ -27,6 +27,7 @@ ver. 0.9.4 (2015/XX/XXX) - wanna-be-released * Fixed `fail2ban-regex` stops working on invalid (wrong encoded) character for python version < 3.x (gh-1248) * Use postfix_log logpath for postfix-rbl jail + * filters.d/postfix.conf - add 'Sender address rejected: Domain not found' failregex - New Features: * New interpolation feature for definition config readers - `` diff --git a/config/filter.d/postfix.conf b/config/filter.d/postfix.conf index a994d772..f6a8578b 100644 --- a/config/filter.d/postfix.conf +++ b/config/filter.d/postfix.conf @@ -16,6 +16,7 @@ failregex = ^%(__prefix_line)sNOQUEUE: reject: RCPT from \S+\[\]: 554 5\.7 ^%(__prefix_line)sNOQUEUE: reject: RCPT from \S+\[\]: 450 4\.7\.1 Client host rejected: cannot find your hostname, (\[\S*\]); from=<\S*> to=<\S+> proto=ESMTP helo=<\S*>$ ^%(__prefix_line)sNOQUEUE: reject: RCPT from \S+\[\]: 450 4\.7\.1 : Helo command rejected: Host not found; from=<> to=<> proto=ESMTP helo= *$ ^%(__prefix_line)sNOQUEUE: reject: VRFY from \S+\[\]: 550 5\.1\.1 .*$ + ^%(__prefix_line)sNOQUEUE: reject: RCPT from \S+\[\]: 450 4\.1\.8 <\S*>: Sender address rejected: Domain not found; from=<\S*> to=<\S+> proto=ESMTP helo=<\S*>$ ^%(__prefix_line)simproper command pipelining after \S+ from [^[]*\[\]:?$ ignoreregex = diff --git a/fail2ban/tests/files/logs/postfix b/fail2ban/tests/files/logs/postfix index ee8720f8..4934a29e 100644 --- a/fail2ban/tests/files/logs/postfix +++ b/fail2ban/tests/files/logs/postfix @@ -23,3 +23,6 @@ Dec 18 02:05:46 platypus postfix/smtpd[16349]: improper command pipelining after # failJSON: { "time": "2004-12-21T21:17:29", "match": true , "host": "93.184.216.34" } Dec 21 21:17:29 xxx postfix/smtpd[7150]: NOQUEUE: reject: RCPT from badserver.example.com[93.184.216.34]: 450 4.7.1 Client host rejected: cannot find your hostname, [93.184.216.34]; from= to= proto=ESMTP helo= + +# failJSON: { "time": "2004-11-22T22:33:44", "match": true , "host": "1.2.3.4" } +Nov 22 22:33:44 xxx postfix/smtpd[11111]: NOQUEUE: reject: RCPT from 1-2-3-4.example.com[1.2.3.4]: 450 4.1.8 : Sender address rejected: Domain not found; from= to= proto=ESMTP helo=<1-2-3-4.example.com> From 5b88a84fe8f86817653ce1f4b5c0e48b80696878 Mon Sep 17 00:00:00 2001 From: sarneaud Date: Sat, 28 Nov 2015 15:03:09 +1100 Subject: [PATCH 082/126] Small fixes for Gentoo initd script These fixes are pretty pedantic, but they do simplify the script a little. * Checking the existence of a file/directory before creating/deleting it adds complexity and raciness. There are better options. * mkdir -p does the job of making sure a directory exists. (It only fails if there's a filesystem error or something.) * Likewise, rm -f doesn't fail if the file doesn't exist. * rm -r isn't neccessary because the socket shouldn't be a directory. (If it is for some reason, that should be an error.) --- files/gentoo-initd | 12 ++++-------- 1 file changed, 4 insertions(+), 8 deletions(-) diff --git a/files/gentoo-initd b/files/gentoo-initd index b56d4bdb..98c5edf9 100755 --- a/files/gentoo-initd +++ b/files/gentoo-initd @@ -30,14 +30,10 @@ depend() { start() { ebegin "Starting fail2ban" - if [ ! -d /var/run/fail2ban ]; then - mkdir /var/run/fail2ban || return 1 - fi - if [ -e /var/run/fail2ban/fail2ban.sock ]; then - # remove stalled sock file after system crash - # bug 347477 - rm -rf /var/run/fail2ban/fail2ban.sock || return 1 - fi + mkdir -p /var/run/fail2ban || return 1 + # remove stalled sock file after system crash + # bug 347477 + rm -f /var/run/fail2ban/fail2ban.sock || return 1 ${FAIL2BAN} start &> /dev/null eend $? "Failed to start fail2ban" } From 106c3eab9ab62828e6562ee1f0aa421e5af0312f Mon Sep 17 00:00:00 2001 From: Ross Brown Date: Sun, 29 Nov 2015 15:56:56 +0000 Subject: [PATCH 083/126] Added filter and jail for murmur/mumble-server. --- config/filter.d/murmur.conf | 21 +++++++++++++++++++++ config/jail.conf | 9 +++++++++ 2 files changed, 30 insertions(+) create mode 100644 config/filter.d/murmur.conf diff --git a/config/filter.d/murmur.conf b/config/filter.d/murmur.conf new file mode 100644 index 00000000..cc47f022 --- /dev/null +++ b/config/filter.d/murmur.conf @@ -0,0 +1,21 @@ +# Fail2Ban filter for murmur/mumble-server +# + +[INCLUDES] + +before = common.conf + + +[Definition] + +_daemon = murmurd + +failregex = Rejected connection from :\d+: Invalid server password$ + Rejected connection from :\d+: Wrong certificate or password for existing user$ + +ignoreregex = + + +# DEV Notes: +# +# Author: Ross Brown diff --git a/config/jail.conf b/config/jail.conf index c98392ba..c8dc6d9c 100644 --- a/config/jail.conf +++ b/config/jail.conf @@ -821,3 +821,12 @@ returntype = DROP bantime = 3600 maxretry = 1 findtime = 1 + + +[murmur] +# AKA mumble-server +port = 64738 +filter = murmur +action = %(banaction)s[name=%(__name__)s-tcp, port="%(port)s", protocol=tcp, chain="%(chain)s", actname=%(banaction)s-tcp] + %(banaction)s[name=%(__name__)s-udp, port="%(port)s", protocol=udp, chain="%(chain)s", actname=%(banaction)s-udp] +logpath = /var/log/mumble-server/mumble-server.log From 4c837f033323fbbfd8bccd98a462e2139b3183e6 Mon Sep 17 00:00:00 2001 From: Ross Brown Date: Sun, 29 Nov 2015 16:28:47 +0000 Subject: [PATCH 084/126] Added sample log file for 'murmur' filter. --- fail2ban/tests/files/logs/murmur | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 fail2ban/tests/files/logs/murmur diff --git a/fail2ban/tests/files/logs/murmur b/fail2ban/tests/files/logs/murmur new file mode 100644 index 00000000..0b738853 --- /dev/null +++ b/fail2ban/tests/files/logs/murmur @@ -0,0 +1,5 @@ +# failJSON: { "time": "2015-11-29T16:38:01", "match": true , "host": "192.168.0.1" } +2015-11-29 16:38:01.818 1 => <4:test(-1)> Rejected connection from 192.168.0.1:29530: Invalid server password + +# failJSON: { "time": "2015-11-29T17:18:20", "match": true , "host": "192.168.1.2" } +2015-11-29 17:18:20.962 1 => <8:test(-1)> Rejected connection from 192.168.1.2:29761: Wrong certificate or password for existing user From 3a179ec5d7121003683c16db77096247b0fe63f7 Mon Sep 17 00:00:00 2001 From: sebres Date: Wed, 2 Dec 2015 19:51:29 +0100 Subject: [PATCH 085/126] small code review: (much pretty) handling of filename as key - FileFilter contains (ordered) dict of files (not list), as discussed in gh-1265 --- ChangeLog | 2 ++ fail2ban/server/filter.py | 46 +++++++++++++++++++-------------------- 2 files changed, 25 insertions(+), 23 deletions(-) diff --git a/ChangeLog b/ChangeLog index 20b708a8..225a8a8b 100644 --- a/ChangeLog +++ b/ChangeLog @@ -59,6 +59,8 @@ ver. 0.9.4 (2015/XX/XXX) - wanna-be-released rest api and web interface (gh-1223) * Add *_backend options for services to allow distros to set the default backend per service, set default to systemd for Fedora as appropriate + * small improvement for better handling of many log files (gh-1265) + Thanks @kshetragia ver. 0.9.3 (2015/08/01) - lets-all-stay-friends ---------- diff --git a/fail2ban/server/filter.py b/fail2ban/server/filter.py index 65be0467..e0579c6e 100644 --- a/fail2ban/server/filter.py +++ b/fail2ban/server/filter.py @@ -38,6 +38,11 @@ from .failregex import FailRegex, Regex, RegexException from .action import CommandAction from ..helpers import getLogger +try: + from collections import OrderedDict +except ImportError: + OrderedDict = dict + # Gets the instance of the logger. logSys = getLogger(__name__) @@ -552,7 +557,7 @@ class FileFilter(Filter): def __init__(self, jail, **kwargs): Filter.__init__(self, jail, **kwargs) ## The log file path. - self.__logPath = [] + self.__logs = OrderedDict() self.setLogEncoding("auto") ## @@ -561,7 +566,7 @@ class FileFilter(Filter): # @param path log file path def addLogPath(self, path, tail = False): - if self.containsLogPath(path): + if path in self.__logs: logSys.error(path + " already exists") else: container = FileContainer(path, self.getLogEncoding(), tail) @@ -570,7 +575,7 @@ class FileFilter(Filter): lastpos = db.addLog(self.jail, container) if lastpos and not tail: container.setPos(lastpos) - self.__logPath.append(container) + self.__logs[path] = container logSys.info("Added logfile = %s" % path) self._addLogPath(path) # backend specific @@ -585,15 +590,16 @@ class FileFilter(Filter): # @param path the log file to delete def delLogPath(self, path): - for log in self.__logPath: - if log.getFileName() == path: - self.__logPath.remove(log) - db = self.jail.database - if db is not None: - db.updateLog(self.jail, log) - logSys.info("Removed logfile = %s" % path) - self._delLogPath(path) - return + try: + log = self.__logs.pop(path) + except KeyError: + return + db = self.jail.database + if db is not None: + db.updateLog(self.jail, log) + logSys.info("Removed logfile = %s" % path) + self._delLogPath(path) + return def _delLogPath(self, path): # pragma: no cover - overwritten function # nothing to do by default @@ -606,7 +612,7 @@ class FileFilter(Filter): # @return log file path def getLogPath(self): - return self.__logPath + return self.__logs.values() ## # Check whether path is already monitored. @@ -615,10 +621,7 @@ class FileFilter(Filter): # @return True if the path is already monitored else False def containsLogPath(self, path): - for log in self.__logPath: - if log.getFileName() == path: - return True - return False + return path in self.__logs ## # Set the log file encoding @@ -629,7 +632,7 @@ class FileFilter(Filter): if encoding.lower() == "auto": encoding = locale.getpreferredencoding() codecs.lookup(encoding) # Raise LookupError if invalid codec - for log in self.getLogPath(): + for log in self.__logs.itervalues(): log.setEncoding(encoding) self.__encoding = encoding logSys.info("Set jail log file encoding to %s" % encoding) @@ -643,10 +646,7 @@ class FileFilter(Filter): return self.__encoding def getFileContainer(self, path): - for log in self.__logPath: - if log.getFileName() == path: - return log - return None + return self.__logs.get(path, None) ## # Gets all the failure in the log file. @@ -698,7 +698,7 @@ class FileFilter(Filter): """Status of Filter plus files being monitored. """ ret = super(FileFilter, self).status(flavor=flavor) - path = [m.getFileName() for m in self.getLogPath()] + path = self.__logs.keys() ret.append(("File list", path)) return ret From 6ce7522d3cebbc6a3adcd066d3189f7cff40536c Mon Sep 17 00:00:00 2001 From: sebres Date: Wed, 2 Dec 2015 21:06:32 +0100 Subject: [PATCH 086/126] unordered (python 2.6) compatibility fix and coverage extended; --- fail2ban/tests/filtertestcase.py | 10 ++++++++++ fail2ban/tests/servertestcase.py | 20 ++++++++------------ 2 files changed, 18 insertions(+), 12 deletions(-) diff --git a/fail2ban/tests/filtertestcase.py b/fail2ban/tests/filtertestcase.py index 3674a574..69b6e5b5 100644 --- a/fail2ban/tests/filtertestcase.py +++ b/fail2ban/tests/filtertestcase.py @@ -859,6 +859,16 @@ class GetFailures(LogCaptureTestCase): self.filter.delLogPath(GetFailures.FILENAME_01) self.assertEqual(self.filter.getLogPath(),[]) + def testNoLogAdded(self): + self.filter.addLogPath(GetFailures.FILENAME_01, tail=True) + self.assertTrue(self.filter.containsLogPath(GetFailures.FILENAME_01)) + self.filter.delLogPath(GetFailures.FILENAME_01) + self.assertFalse(self.filter.containsLogPath(GetFailures.FILENAME_01)) + # and unknown (safety and cover) + self.assertFalse(self.filter.containsLogPath('unknown.log')) + self.filter.delLogPath('unknown.log') + + def testGetFailures01(self, filename=None, failures=None): filename = filename or GetFailures.FILENAME_01 failures = failures or GetFailures.FAILURES_01 diff --git a/fail2ban/tests/servertestcase.py b/fail2ban/tests/servertestcase.py index 86ffdb46..07e10c7d 100644 --- a/fail2ban/tests/servertestcase.py +++ b/fail2ban/tests/servertestcase.py @@ -113,19 +113,15 @@ class TransmitterBase(unittest.TestCase): self.assertEqual( self.transm.proceed(["get", jail, cmd]), (0, [])) for n, value in enumerate(values): - self.assertEqual( - self.transm.proceed(["set", jail, cmdAdd, value]), - (0, values[:n+1])) - self.assertEqual( - self.transm.proceed(["get", jail, cmd]), - (0, values[:n+1])) + ret = self.transm.proceed(["set", jail, cmdAdd, value]) + self.assertEqual((ret[0], sorted(ret[1])), (0, sorted(values[:n+1]))) + ret = self.transm.proceed(["get", jail, cmd]) + self.assertEqual((ret[0], sorted(ret[1])), (0, sorted(values[:n+1]))) for n, value in enumerate(values): - self.assertEqual( - self.transm.proceed(["set", jail, cmdDel, value]), - (0, values[n+1:])) - self.assertEqual( - self.transm.proceed(["get", jail, cmd]), - (0, values[n+1:])) + ret = self.transm.proceed(["set", jail, cmdDel, value]) + self.assertEqual((ret[0], sorted(ret[1])), (0, sorted(values[n+1:]))) + ret = self.transm.proceed(["get", jail, cmd]) + self.assertEqual((ret[0], sorted(ret[1])), (0, sorted(values[n+1:]))) def jailAddDelRegexTest(self, cmd, inValues, outValues, jail): cmdAdd = "add" + cmd From dd9d1912e88d09910ae8173a70d76c58afd9d3f1 Mon Sep 17 00:00:00 2001 From: Yaroslav Halchenko Date: Wed, 2 Dec 2015 22:49:47 -0500 Subject: [PATCH 087/126] RF: Filter.getLogPaths -> getLogs Since it returns log containers not paths per se --- fail2ban/server/filter.py | 8 ++++---- fail2ban/server/filtergamin.py | 4 ++-- fail2ban/server/filterpoll.py | 4 ++-- fail2ban/server/server.py | 2 +- fail2ban/tests/filtertestcase.py | 8 ++++---- 5 files changed, 13 insertions(+), 13 deletions(-) diff --git a/fail2ban/server/filter.py b/fail2ban/server/filter.py index e0579c6e..ab4e4d5b 100644 --- a/fail2ban/server/filter.py +++ b/fail2ban/server/filter.py @@ -565,7 +565,7 @@ class FileFilter(Filter): # # @param path log file path - def addLogPath(self, path, tail = False): + def addLogPath(self, path, tail=False): if path in self.__logs: logSys.error(path + " already exists") else: @@ -607,11 +607,11 @@ class FileFilter(Filter): pass ## - # Get the log file path + # Get the log containers # - # @return log file path + # @return log containers - def getLogPath(self): + def getLogs(self): return self.__logs.values() ## diff --git a/fail2ban/server/filtergamin.py b/fail2ban/server/filtergamin.py index 1f51744b..e731a8e9 100644 --- a/fail2ban/server/filtergamin.py +++ b/fail2ban/server/filtergamin.py @@ -129,6 +129,6 @@ class FilterGamin(FileFilter): # Desallocates the resources used by Gamin. def __cleanup(self): - for path in self.getLogPath(): - self.monitor.stop_watch(path.getFileName()) + for log in self.getLogs(): + self.monitor.stop_watch(log.getFileName()) del self.monitor diff --git a/fail2ban/server/filterpoll.py b/fail2ban/server/filterpoll.py index 25c3e119..d0b37775 100644 --- a/fail2ban/server/filterpoll.py +++ b/fail2ban/server/filterpoll.py @@ -88,10 +88,10 @@ class FilterPoll(FileFilter): while self.active: if logSys.getEffectiveLevel() <= 6: logSys.log(6, "Woke up idle=%s with %d files monitored", - self.idle, len(self.getLogPath())) + self.idle, len(self.getLogs())) if not self.idle: # Get file modification - for container in self.getLogPath(): + for container in self.getLogs(): filename = container.getFileName() if self.isModified(filename): self.getFailures(filename) diff --git a/fail2ban/server/server.py b/fail2ban/server/server.py index 6d19544d..3e371945 100644 --- a/fail2ban/server/server.py +++ b/fail2ban/server/server.py @@ -212,7 +212,7 @@ class Server: filter_ = self.__jails[name].filter if isinstance(filter_, FileFilter): return [m.getFileName() - for m in filter_.getLogPath()] + for m in filter_.getLogs()] else: # pragma: systemd no cover logSys.info("Jail %s is not a FileFilter instance" % name) return [] diff --git a/fail2ban/tests/filtertestcase.py b/fail2ban/tests/filtertestcase.py index 69b6e5b5..39a1b352 100644 --- a/fail2ban/tests/filtertestcase.py +++ b/fail2ban/tests/filtertestcase.py @@ -853,11 +853,11 @@ class GetFailures(LogCaptureTestCase): def testTail(self): self.filter.addLogPath(GetFailures.FILENAME_01, tail=True) - self.assertEqual(self.filter.getLogPath()[-1].getPos(), 1653) - self.filter.getLogPath()[-1].close() - self.assertEqual(self.filter.getLogPath()[-1].readline(), "") + self.assertEqual(self.filter.getLogs()[-1].getPos(), 1653) + self.filter.getLogs()[-1].close() + self.assertEqual(self.filter.getLogs()[-1].readline(), "") self.filter.delLogPath(GetFailures.FILENAME_01) - self.assertEqual(self.filter.getLogPath(),[]) + self.assertEqual(self.filter.getLogs(), []) def testNoLogAdded(self): self.filter.addLogPath(GetFailures.FILENAME_01, tail=True) From 59da27b9f659badc2d28ada465eb575ebf4a651c Mon Sep 17 00:00:00 2001 From: Yaroslav Halchenko Date: Wed, 2 Dec 2015 22:53:28 -0500 Subject: [PATCH 088/126] ENH: add a check to testTail to assure correct test logic below it --- fail2ban/tests/filtertestcase.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/fail2ban/tests/filtertestcase.py b/fail2ban/tests/filtertestcase.py index 39a1b352..40879b66 100644 --- a/fail2ban/tests/filtertestcase.py +++ b/fail2ban/tests/filtertestcase.py @@ -852,6 +852,8 @@ class GetFailures(LogCaptureTestCase): LogCaptureTestCase.tearDown(self) def testTail(self): + # There must be no containters registered, otherwise [-1] indexing would be wrong + self.assertEqual(self.filter.getLogs(), []) self.filter.addLogPath(GetFailures.FILENAME_01, tail=True) self.assertEqual(self.filter.getLogs()[-1].getPos(), 1653) self.filter.getLogs()[-1].close() From 48202f998d3ad672f8b56ae5e9d1854131d0a31c Mon Sep 17 00:00:00 2001 From: Yaroslav Halchenko Date: Wed, 2 Dec 2015 22:57:40 -0500 Subject: [PATCH 089/126] RF: prefer log over container in getLog and local variables Even though I have left FileContainer class name intact --- fail2ban/server/filter.py | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/fail2ban/server/filter.py b/fail2ban/server/filter.py index ab4e4d5b..8f4f602a 100644 --- a/fail2ban/server/filter.py +++ b/fail2ban/server/filter.py @@ -569,13 +569,13 @@ class FileFilter(Filter): if path in self.__logs: logSys.error(path + " already exists") else: - container = FileContainer(path, self.getLogEncoding(), tail) + log = FileContainer(path, self.getLogEncoding(), tail) db = self.jail.database if db is not None: - lastpos = db.addLog(self.jail, container) + lastpos = db.addLog(self.jail, log) if lastpos and not tail: - container.setPos(lastpos) - self.__logs[path] = container + log.setPos(lastpos) + self.__logs[path] = log logSys.info("Added logfile = %s" % path) self._addLogPath(path) # backend specific @@ -645,7 +645,7 @@ class FileFilter(Filter): def getLogEncoding(self): return self.__encoding - def getFileContainer(self, path): + def getLog(self, path): return self.__logs.get(path, None) ## @@ -656,13 +656,13 @@ class FileFilter(Filter): # is created and is added to the FailManager. def getFailures(self, filename): - container = self.getFileContainer(filename) - if container is None: + log = self.getLog(filename) + if log is None: logSys.error("Unable to get failures in " + filename) return False # Try to open log file. try: - has_content = container.open() + has_content = log.open() # see http://python.org/dev/peps/pep-3151/ except IOError, e: logSys.error("Unable to open %s" % filename) @@ -683,15 +683,15 @@ class FileFilter(Filter): # start reading tested to be empty container -- race condition # might occur leading at least to tests failures. while has_content: - line = container.readline() + line = log.readline() if not line or not self.active: # The jail reached the bottom or has been stopped break self.processLineAndAdd(line) - container.close() + log.close() db = self.jail.database if db is not None: - db.updateLog(self.jail, container) + db.updateLog(self.jail, log) return True def status(self, flavor="basic"): From 6d984717b5f743d836840d851b1d0814f3da842a Mon Sep 17 00:00:00 2001 From: sebres Date: Sat, 12 Dec 2015 15:46:45 +0100 Subject: [PATCH 090/126] ordered dict replaced with dict + change log entry fix # Conflicts: # fail2ban/server/filter.py --- ChangeLog | 5 +++-- fail2ban/server/filter.py | 7 +------ 2 files changed, 4 insertions(+), 8 deletions(-) diff --git a/ChangeLog b/ChangeLog index 225a8a8b..42fc0b38 100644 --- a/ChangeLog +++ b/ChangeLog @@ -59,8 +59,9 @@ ver. 0.9.4 (2015/XX/XXX) - wanna-be-released rest api and web interface (gh-1223) * Add *_backend options for services to allow distros to set the default backend per service, set default to systemd for Fedora as appropriate - * small improvement for better handling of many log files (gh-1265) - Thanks @kshetragia + * Performance improvements while monitoring large number of files (gh-1265). + Use associative array (dict) for monitored log files to speed up lookup + operations. Thanks @kshetragia ver. 0.9.3 (2015/08/01) - lets-all-stay-friends ---------- diff --git a/fail2ban/server/filter.py b/fail2ban/server/filter.py index 8f4f602a..2b354f7a 100644 --- a/fail2ban/server/filter.py +++ b/fail2ban/server/filter.py @@ -38,11 +38,6 @@ from .failregex import FailRegex, Regex, RegexException from .action import CommandAction from ..helpers import getLogger -try: - from collections import OrderedDict -except ImportError: - OrderedDict = dict - # Gets the instance of the logger. logSys = getLogger(__name__) @@ -557,7 +552,7 @@ class FileFilter(Filter): def __init__(self, jail, **kwargs): Filter.__init__(self, jail, **kwargs) ## The log file path. - self.__logs = OrderedDict() + self.__logs = dict() self.setLogEncoding("auto") ## From 9ee08fed48388a8c81d2fb1f5c1fdf6c33b9b096 Mon Sep 17 00:00:00 2001 From: Yaroslav Halchenko Date: Sun, 13 Dec 2015 23:05:06 -0500 Subject: [PATCH 091/126] ENH: log at heavydebug level what actually we are matching for failregex --- fail2ban/server/filter.py | 1 + 1 file changed, 1 insertion(+) diff --git a/fail2ban/server/filter.py b/fail2ban/server/filter.py index 2b354f7a..dae5c527 100644 --- a/fail2ban/server/filter.py +++ b/fail2ban/server/filter.py @@ -491,6 +491,7 @@ class Filter(JailThread): self.__lineBuffer = ( self.__lineBuffer + [tupleLine])[-self.__lineBufferSize:] + logSys.log(5, "Looking for failregex match of %r" % self.__lineBuffer) # Iterates over all the regular expressions. for failRegexIndex, failRegex in enumerate(self.__failRegex): From 5d6cead99694efae8710ab12cd68a924a83d7396 Mon Sep 17 00:00:00 2001 From: Yaroslav Halchenko Date: Sun, 13 Dec 2015 23:21:04 -0500 Subject: [PATCH 092/126] ENH: sshd filter -- match new "maximum auth attempts exceeded" (Closes #1269) --- ChangeLog | 2 ++ config/filter.d/sshd.conf | 1 + fail2ban/tests/files/logs/sshd | 3 +++ 3 files changed, 6 insertions(+) diff --git a/ChangeLog b/ChangeLog index 42fc0b38..90cdae59 100644 --- a/ChangeLog +++ b/ChangeLog @@ -41,6 +41,8 @@ ver. 0.9.4 (2015/XX/XXX) - wanna-be-released rest api and web interface (gh-1223) - nginx-limit-req - ban hosts, that were failed through nginx by limit request processing rate (ngx_http_limit_req_module) + * sshd filter got new failregex to match "maximum authentication + attempts exceeded" (introduced in openssh 6.8) - Enhancements: * Do not rotate empty log files diff --git a/config/filter.d/sshd.conf b/config/filter.d/sshd.conf index 5fad2b32..180ac52a 100644 --- a/config/filter.d/sshd.conf +++ b/config/filter.d/sshd.conf @@ -33,6 +33,7 @@ failregex = ^%(__prefix_line)s(?:error: PAM: )?[aA]uthentication (?:failure|erro ^(?P<__prefix>%(__prefix_line)s)User .+ not allowed because account is locked(?P=__prefix)(?:error: )?Received disconnect from : 11: .+ \[preauth\]$ ^(?P<__prefix>%(__prefix_line)s)Disconnecting: Too many authentication failures for .+? \[preauth\](?P=__prefix)(?:error: )?Connection closed by \[preauth\]$ ^(?P<__prefix>%(__prefix_line)s)Connection from port \d+(?: on \S+ port \d+)?(?P=__prefix)Disconnecting: Too many authentication failures for .+? \[preauth\]$ + ^%(__prefix_line)s(error: )?maximum authentication attempts exceeded for .* from (?: port \d*)?(?: ssh\d*)? \[preauth\]$ ^%(__prefix_line)spam_unix\(sshd:auth\):\s+authentication failure;\s*logname=\S*\s*uid=\d*\s*euid=\d*\s*tty=\S*\s*ruser=\S*\s*rhost=\s.*$ ignoreregex = diff --git a/fail2ban/tests/files/logs/sshd b/fail2ban/tests/files/logs/sshd index 62204339..7baf4be7 100644 --- a/fail2ban/tests/files/logs/sshd +++ b/fail2ban/tests/files/logs/sshd @@ -148,6 +148,9 @@ Feb 12 04:09:18 localhost sshd[26713]: Connection from 115.249.163.77 port 51353 # failJSON: { "time": "2005-02-12T04:09:21", "match": true , "host": "115.249.163.77", "desc": "Multiline match with interface address" } Feb 12 04:09:21 localhost sshd[26713]: Disconnecting: Too many authentication failures for root [preauth] +# failJSON: { "time": "2004-11-23T21:50:37", "match": true , "host": "61.0.0.1", "desc": "New logline format as openssh 6.8 to replace prev multiline version" } +Nov 23 21:50:37 myhost sshd[21810]: error: maximum authentication attempts exceeded for root from 61.0.0.1 port 49940 ssh2 [preauth] + # failJSON: { "match": false } Apr 27 13:02:04 host sshd[29116]: User root not allowed because account is locked # failJSON: { "match": false } From ba535826a83136032c9e6f993804356f2890068e Mon Sep 17 00:00:00 2001 From: Ross Brown Date: Tue, 15 Dec 2015 21:46:35 +0000 Subject: [PATCH 093/126] Updated ChangeLog to include new murmur filter. --- ChangeLog | 2 ++ 1 file changed, 2 insertions(+) diff --git a/ChangeLog b/ChangeLog index 20b708a8..03bed0cd 100644 --- a/ChangeLog +++ b/ChangeLog @@ -41,6 +41,8 @@ ver. 0.9.4 (2015/XX/XXX) - wanna-be-released rest api and web interface (gh-1223) - nginx-limit-req - ban hosts, that were failed through nginx by limit request processing rate (ngx_http_limit_req_module) + - murmur - ban hosts that repeatedly attempt to connect to + murmur/mumble-server with an invalid server password or certificate. - Enhancements: * Do not rotate empty log files From fd36b058cee39dac4524cef32a66931de017c6dd Mon Sep 17 00:00:00 2001 From: Ross Brown Date: Tue, 15 Dec 2015 21:54:41 +0000 Subject: [PATCH 094/126] Changed usernames in sample log file for 'murmur' filter. --- fail2ban/tests/files/logs/murmur | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/fail2ban/tests/files/logs/murmur b/fail2ban/tests/files/logs/murmur index 0b738853..bc18b7ea 100644 --- a/fail2ban/tests/files/logs/murmur +++ b/fail2ban/tests/files/logs/murmur @@ -1,5 +1,5 @@ # failJSON: { "time": "2015-11-29T16:38:01", "match": true , "host": "192.168.0.1" } -2015-11-29 16:38:01.818 1 => <4:test(-1)> Rejected connection from 192.168.0.1:29530: Invalid server password +2015-11-29 16:38:01.818 1 => <4:testUsernameOne(-1)> Rejected connection from 192.168.0.1:29530: Invalid server password # failJSON: { "time": "2015-11-29T17:18:20", "match": true , "host": "192.168.1.2" } -2015-11-29 17:18:20.962 1 => <8:test(-1)> Rejected connection from 192.168.1.2:29761: Wrong certificate or password for existing user +2015-11-29 17:18:20.962 1 => <8:testUsernameTwo(-1)> Rejected connection from 192.168.1.2:29761: Wrong certificate or password for existing user From ead2d509dc0101807f0cad53e896963a5498e47b Mon Sep 17 00:00:00 2001 From: Ross Brown Date: Thu, 17 Dec 2015 17:45:24 +0000 Subject: [PATCH 095/126] Updated 'murmur' filter to use new double-anchored regex based on @yarikoptic's suggestions. --- config/filter.d/murmur.conf | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/config/filter.d/murmur.conf b/config/filter.d/murmur.conf index cc47f022..3775a9d2 100644 --- a/config/filter.d/murmur.conf +++ b/config/filter.d/murmur.conf @@ -10,8 +10,15 @@ before = common.conf _daemon = murmurd -failregex = Rejected connection from :\d+: Invalid server password$ - Rejected connection from :\d+: Wrong certificate or password for existing user$ +# N.B. If you allow users to have usernames that include the '>' character you +# should change this to match the regex assigned to the 'username' +# variable in your server config file (murmur.ini / mumble-server.ini). +_usernameregex = [^>]+ + +_prefix = [\n\s]*(\.\d{3})?\s+\d+ => <\d+:%(_usernameregex)s\(-1\)> Rejected connection from :\d+: + +failregex = ^%(_prefix)s Invalid server password$ + ^%(_prefix)s Wrong certificate or password for existing user$ ignoreregex = From 16aa2fa13e387b8eb539349192879fb9af9170b3 Mon Sep 17 00:00:00 2001 From: Ross Brown Date: Thu, 17 Dec 2015 17:57:45 +0000 Subject: [PATCH 096/126] Updated ChangeLog to include new murmur jail. --- ChangeLog | 2 ++ 1 file changed, 2 insertions(+) diff --git a/ChangeLog b/ChangeLog index 03bed0cd..20ca6def 100644 --- a/ChangeLog +++ b/ChangeLog @@ -43,6 +43,8 @@ ver. 0.9.4 (2015/XX/XXX) - wanna-be-released request processing rate (ngx_http_limit_req_module) - murmur - ban hosts that repeatedly attempt to connect to murmur/mumble-server with an invalid server password or certificate. + * New jails: + - murmur - bans TCP and UDP from the bad host on the default murmur port. - Enhancements: * Do not rotate empty log files From dfaf82d68a39662e475b3a0403a64de04fda7e76 Mon Sep 17 00:00:00 2001 From: Yaroslav Halchenko Date: Fri, 18 Dec 2015 09:23:12 -0500 Subject: [PATCH 097/126] Changelog entry for PartOf in .service fix --- ChangeLog | 3 +++ 1 file changed, 3 insertions(+) diff --git a/ChangeLog b/ChangeLog index 90bcfcb3..74a88115 100644 --- a/ChangeLog +++ b/ChangeLog @@ -68,6 +68,9 @@ ver. 0.9.4 (2015/XX/XXX) - wanna-be-released * Performance improvements while monitoring large number of files (gh-1265). Use associative array (dict) for monitored log files to speed up lookup operations. Thanks @kshetragia + * Specified that fail2ban is PartOf iptables.service firewalld.service in + .service file -- would reload fail2ban if those services are restarted + ver. 0.9.3 (2015/08/01) - lets-all-stay-friends ---------- From 28c98322933f36d5ba4ea12ea638fb573b53dbc6 Mon Sep 17 00:00:00 2001 From: Yaroslav Halchenko Date: Tue, 29 Dec 2015 19:43:52 -0500 Subject: [PATCH 098/126] RF: harmonize jail.conf (no explicit enabled=false in jails, match filter name for screesharingd, etc) --- config/jail.conf | 26 ++++++-------------------- 1 file changed, 6 insertions(+), 20 deletions(-) diff --git a/config/jail.conf b/config/jail.conf index 02543705..41a43e56 100644 --- a/config/jail.conf +++ b/config/jail.conf @@ -240,7 +240,6 @@ backend = %(dropbear_backend)s port = ssh logpath = %(auditd_log)s -maxretry = 5 # @@ -266,7 +265,6 @@ maxretry = 1 port = http,https logpath = %(apache_error_log)s -maxretry = 6 [apache-overflows] @@ -304,18 +302,21 @@ port = http,https logpath = %(apache_error_log)s maxretry = 2 + [apache-shellshock] port = http,https logpath = %(apache_error_log)s maxretry = 1 + [openhab-auth] filter = openhab action = iptables-allports[name=NoAuthFailures] logpath = /opt/openhab/logs/request.log + [nginx-http-auth] port = http,https @@ -335,6 +336,7 @@ port = http,https logpath = %(nginx_error_log)s maxretry = 2 + # Ban attackers that try to use PHP's URL-fopen() functionality # through GET/POST variables. - Experimental, with more than a year # of usage in production environments. @@ -399,7 +401,6 @@ logpath = /var/log/sogo/sogo.log logpath = /var/log/tine20/tine20.log port = http,https -maxretry = 5 # @@ -420,7 +421,6 @@ logpath = /var/log/tomcat*/catalina.out [monit] #Ban clients brute-forcing the monit gui login -filter = monit port = 2812 logpath = /var/log/monit @@ -473,7 +473,6 @@ backend = %(proftpd_backend)s port = ftp,ftp-data,ftps,ftps-data logpath = %(pureftpd_log)s backend = %(pureftpd_backend)s -maxretry = 6 [gssftpd] @@ -481,7 +480,6 @@ maxretry = 6 port = ftp,ftp-data,ftps,ftps-data logpath = %(syslog_daemon)s backend = %(syslog_backend)s -maxretry = 6 [wuftpd] @@ -489,7 +487,6 @@ maxretry = 6 port = ftp,ftp-data,ftps,ftps-data logpath = %(wuftpd_log)s backend = %(wuftpd_backend)s -maxretry = 6 [vsftpd] @@ -724,7 +721,6 @@ maxretry = 10 port = 3306 logpath = %(mysql_log)s backend = %(mysql_backend)s -maxretry = 5 # Jail for more extended banning of persistent abusers @@ -740,7 +736,6 @@ logpath = /var/log/fail2ban.log banaction = %(banaction_allports)s bantime = 604800 ; 1 week findtime = 86400 ; 1 day -maxretry = 5 # Generic filter for PAM. Has to be used with action which bans all @@ -786,7 +781,6 @@ action = %(banaction)s[name=%(__name__)s-tcp, port="%(tcpport)s", protocol="tcp # nobody except your own Nagios server should ever probe nrpe [nagios] -enabled = false logpath = %(syslog_daemon)s ; nrpe.cfg may define a different log_facility backend = %(syslog_backend)s maxretry = 1 @@ -794,18 +788,14 @@ maxretry = 1 [oracleims] # see "oracleims" filter file for configuration requirement for Oracle IMS v6 and above -enabled = false logpath = /opt/sun/comms/messaging64/log/mail.log_current -maxretry = 6 banaction = %(banaction_allports)s [directadmin] -enabled = false logpath = /var/log/directadmin/login.log port = 2222 [portsentry] -enabled = false logpath = /var/lib/portsentry/portsentry.history maxretry = 1 @@ -826,16 +816,12 @@ findtime = 1 [murmur] # AKA mumble-server port = 64738 -filter = murmur action = %(banaction)s[name=%(__name__)s-tcp, port="%(port)s", protocol=tcp, chain="%(chain)s", actname=%(banaction)s-tcp] %(banaction)s[name=%(__name__)s-udp, port="%(port)s", protocol=udp, chain="%(chain)s", actname=%(banaction)s-udp] logpath = /var/log/mumble-server/mumble-server.log -[screensharing] +[screensharingd] # For Mac OS Screen Sharing Service (VNC) -enabled = false -filter = screensharingd logpath = /var/log/system.log -logencoding=utf-8 -maxretry = 4 +logencoding = utf-8 From b76aede40d4202bdf5b5c8908162429b30370e33 Mon Sep 17 00:00:00 2001 From: Yaroslav Halchenko Date: Tue, 29 Dec 2015 19:56:45 -0500 Subject: [PATCH 099/126] ENH(TST): verify that passed bantime is non-0 and int --- fail2ban/tests/clientreadertestcase.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/fail2ban/tests/clientreadertestcase.py b/fail2ban/tests/clientreadertestcase.py index d19090be..e6860a47 100644 --- a/fail2ban/tests/clientreadertestcase.py +++ b/fail2ban/tests/clientreadertestcase.py @@ -594,6 +594,12 @@ class JailsReaderTest(LogCaptureTestCase): # by default we have lots of jails ;) self.assertTrue(len(comm_commands)) + # some common sanity checks for commands + for command in comm_commands: + if len(command) >= 3 and [command[0], command[2]] == ['set', 'bantime']: + self.assertTrue(isinstance(command[3], int)) + self.assertTrue(command[3] > 0) + # and we know even some of them by heart for j in ['sshd', 'recidive']: # by default we have 'auto' backend ATM From cf334421bdd287e4b5fe81344d50386f1f2d8f4a Mon Sep 17 00:00:00 2001 From: sebres Date: Wed, 30 Dec 2015 20:17:12 +0100 Subject: [PATCH 100/126] Provides fail2ban version to jail (as interpolation variable during parse of jail.conf); BF: use `fail2ban_agent` as user-agent in actions badips, blocklist_de, etc. (closes #1271, closes #1272) --- config/action.d/badips.conf | 2 +- config/action.d/blocklist_de.conf | 2 +- config/action.d/mynetwatchman.conf | 8 +++++-- config/jail.conf | 11 +++++++-- fail2ban/client/jailreader.py | 5 ++++ fail2ban/tests/clientreadertestcase.py | 32 +++++++++++++++++++++++++- 6 files changed, 53 insertions(+), 7 deletions(-) diff --git a/config/action.d/badips.conf b/config/action.d/badips.conf index 4a5c0f97..70b46546 100644 --- a/config/action.d/badips.conf +++ b/config/action.d/badips.conf @@ -10,7 +10,7 @@ [Definition] -actionban = curl --fail --user-agent "fail2ban v0.8.12" http://www.badips.com/add// +actionban = curl --fail --user-agent "" http://www.badips.com/add// [Init] diff --git a/config/action.d/blocklist_de.conf b/config/action.d/blocklist_de.conf index 6d520694..2f31d8b9 100644 --- a/config/action.d/blocklist_de.conf +++ b/config/action.d/blocklist_de.conf @@ -54,7 +54,7 @@ actioncheck = # Tags: See jail.conf(5) man page # Values: CMD # -actionban = curl --fail --data-urlencode 'server=' --data 'apikey=' --data 'service=' --data 'ip=' --data-urlencode 'logs=' --data 'format=text' --user-agent "fail2ban v0.8.12" "https://www.blocklist.de/en/httpreports.html" +actionban = curl --fail --data-urlencode 'server=' --data 'apikey=' --data 'service=' --data 'ip=' --data-urlencode 'logs=' --data 'format=text' --user-agent "" "https://www.blocklist.de/en/httpreports.html" # Option: actionunban # Notes.: command executed when unbanning an IP. Take care that the diff --git a/config/action.d/mynetwatchman.conf b/config/action.d/mynetwatchman.conf index 5245a4e3..8f3edf9e 100644 --- a/config/action.d/mynetwatchman.conf +++ b/config/action.d/mynetwatchman.conf @@ -111,13 +111,17 @@ myip = `ip -4 addr show dev eth0 | grep inet | head -n 1 | sed -r 's/.*inet ([0- # protocol = tcp +# Option: agent +# Default: Fail2ban +agent = Fail2ban + # Option: getcmd # Notes.: A command to fetch a URL. Should output page to STDOUT # Values: CMD Default: wget # -getcmd = wget --no-verbose --tries=3 --waitretry=10 --connect-timeout=10 --read-timeout=60 --retry-connrefused --output-document=- --user-agent=Fail2Ban +getcmd = wget --no-verbose --tries=3 --waitretry=10 --connect-timeout=10 --read-timeout=60 --retry-connrefused --output-document=- --user-agent= # Alternative value: -# getcmd = curl --silent --show-error --retry 3 --connect-timeout 10 --max-time 60 --user-agent Fail2Ban +# getcmd = curl --silent --show-error --retry 3 --connect-timeout 10 --max-time 60 --user-agent # Option: srcport # Notes.: The source port of the attack. You're unlikely to have this info, so diff --git a/config/jail.conf b/config/jail.conf index 02543705..1d560288 100644 --- a/config/jail.conf +++ b/config/jail.conf @@ -146,6 +146,9 @@ chain = INPUT # Usually should be overridden in a particular jail port = 0:65535 +# Format of user-agent https://tools.ietf.org/html/rfc7231#section-5.5.3 +fail2ban_agent = Fail2Ban/%(fail2ban_version)s + # # Action shortcuts. To be used to define action parameter @@ -187,7 +190,7 @@ action_cf_mwl = cloudflare[cfuser="%(cfemail)s", cftoken="%(cfapikey)s"] # [Init] # blocklist_de_apikey = {api key from registration] # -action_blocklist_de = blocklist_de[email="%(sender)s", service=%(filter)s, apikey="%(blocklist_de_apikey)s"] +action_blocklist_de = blocklist_de[email="%(sender)s", service=%(filter)s, apikey="%(blocklist_de_apikey)s", agent="%(fail2ban_agent)s"] # Report ban via badips.com, and use as blacklist # @@ -197,7 +200,11 @@ action_blocklist_de = blocklist_de[email="%(sender)s", service=%(filter)s, apik # NOTE: This action relies on banaction being present on start and therefore # should be last action defined for a jail. # -action_badips = badips.py[category="%(name)s", banaction="%(banaction)s"] +action_badips = badips.py[category="%(__name__)s", banaction="%(banaction)s", agent="%(fail2ban_agent)s"] +# +# Report ban via badips.com (uses action.d/badips.conf for reporting only) +# +action_badips_report = badips[category="%(__name__)s", agent="%(fail2ban_agent)s"] # Choose default action. To change, just override value of 'action' with the # interpolation to the chosen action shortcut (e.g. action_mw, action_mwl, etc) in jail.local diff --git a/fail2ban/client/jailreader.py b/fail2ban/client/jailreader.py index 54ac59fa..56b8889c 100644 --- a/fail2ban/client/jailreader.py +++ b/fail2ban/client/jailreader.py @@ -32,6 +32,7 @@ import re from .configreader import ConfigReaderUnshared, ConfigReader from .filterreader import FilterReader from .actionreader import ActionReader +from ..version import version from ..helpers import getLogger from ..helpers import splitcommaspace @@ -108,6 +109,10 @@ class JailReader(ConfigReader): ["string", "filter", ""], ["string", "action", ""]] + # Before interpolation (substitution) add static options always available as default: + defsec = self._cfg.get_defaults() + defsec["fail2ban_version"] = version + # Read first options only needed for merge defaults ('known/...' from filter): self.__opts = ConfigReader.getOptions(self, self.__name, opts1st) if not self.__opts: diff --git a/fail2ban/tests/clientreadertestcase.py b/fail2ban/tests/clientreadertestcase.py index d19090be..fcabc727 100644 --- a/fail2ban/tests/clientreadertestcase.py +++ b/fail2ban/tests/clientreadertestcase.py @@ -28,18 +28,20 @@ import re import shutil import tempfile import unittest -from ..client.configreader import ConfigReaderUnshared +from ..client.configreader import ConfigReader, ConfigReaderUnshared from ..client import configparserinc from ..client.jailreader import JailReader from ..client.filterreader import FilterReader from ..client.jailsreader import JailsReader from ..client.actionreader import ActionReader from ..client.configurator import Configurator +from ..version import version from .utils import LogCaptureTestCase TEST_FILES_DIR = os.path.join(os.path.dirname(__file__), "files") from .utils import CONFIG_DIR +CONFIG_DIR_TESTSHARE_CFG = {} STOCK = os.path.exists(os.path.join('config','fail2ban.conf')) @@ -251,6 +253,34 @@ class JailReaderTest(LogCaptureTestCase): result = JailReader.extractOptions(option) self.assertEqual(expected, result) + def testVersionAgent(self): + jail = JailReader('blocklisttest', force_enable=True, basedir=CONFIG_DIR) + # emulate jail.read(), because such jail not exists: + ConfigReader.read(jail, "jail"); + sections = jail._cfg.get_sections() + sections['blocklisttest'] = dict((('__name__', 'blocklisttest'), + ('filter', ''), ('failregex', '^test $'), + ('sender', 'f2b-test@example.com'), ('blocklist_de_apikey', 'test-key'), + ('action', + '%(action_blocklist_de)s\n' + '%(action_badips_report)s\n' + '%(action_badips)s\n' + 'mynetwatchman[port=1234,protocol=udp,agent="%(fail2ban_agent)s"]' + ), + )) + # get options: + self.assertTrue(jail.getOptions()) + # convert and get stream + stream = jail.convert() + # get action and retrieve agent from it, compare with agent saved in version: + act = [o for o in stream if len(o) > 4 and (o[4] == 'agent' or o[4].endswith('badips.py'))] + useragent = 'Fail2Ban/%s' % version + self.assertEqual(len(act), 4) + self.assertEqual(act[0], ['set', 'blocklisttest', 'action', 'blocklist_de', 'agent', useragent]) + self.assertEqual(act[1], ['set', 'blocklisttest', 'action', 'badips', 'agent', useragent]) + self.assertEqual(eval(act[2][5]).get('agent', ''), useragent) + self.assertEqual(act[3], ['set', 'blocklisttest', 'action', 'mynetwatchman', 'agent', useragent]) + def testGlob(self): d = tempfile.mkdtemp(prefix="f2b-temp") # Generate few files From e133762a282e8059ad89b5130a3123ea6e0f526f Mon Sep 17 00:00:00 2001 From: Jordan Moeser Date: Thu, 31 Dec 2015 11:16:23 +1000 Subject: [PATCH 101/126] Added HAProxy HTTP Auth filter --- ChangeLog | 2 ++ config/filter.d/haproxy-http-auth.conf | 37 +++++++++++++++++++++ config/jail.conf | 9 +++++ fail2ban/tests/files/logs/haproxy-http-auth | 4 +++ 4 files changed, 52 insertions(+) create mode 100644 config/filter.d/haproxy-http-auth.conf create mode 100644 fail2ban/tests/files/logs/haproxy-http-auth diff --git a/ChangeLog b/ChangeLog index 8b5344be..81575f62 100644 --- a/ChangeLog +++ b/ChangeLog @@ -44,6 +44,8 @@ ver. 0.9.4 (2015/XX/XXX) - wanna-be-released request processing rate (ngx_http_limit_req_module) - murmur - ban hosts that repeatedly attempt to connect to murmur/mumble-server with an invalid server password or certificate. + - haproxy-http-auth - filter to match failed HTTP Authentications against a + HAProxy server * New jails: - murmur - bans TCP and UDP from the bad host on the default murmur port. * sshd filter got new failregex to match "maximum authentication diff --git a/config/filter.d/haproxy-http-auth.conf b/config/filter.d/haproxy-http-auth.conf new file mode 100644 index 00000000..7c371dde --- /dev/null +++ b/config/filter.d/haproxy-http-auth.conf @@ -0,0 +1,37 @@ +# Fail2Ban filter configuration file to match failed login attempts to +# HAProxy HTTP Authentication protected servers. +# +# PLEASE NOTE - When a user first hits the HTTP Auth a 401 is returned by the server +# which prompts their browser to ask for login details. +# This initial 401 is logged by HAProxy. +# In other words, even successful logins will have at least 1 fail regex match. +# Please keep this in mind when setting findtime and maxretry for jails. +# +# Author: Jordan Moeser +# + +[INCLUDES] + +# Read common prefixes. If any customizations available -- read them from +# common.local +before = common.conf + + +[Definition] + +_daemon = haproxy + +# Option: failregex +# Notes.: regex to match the password failures messages in the logfile. The +# host must be matched by a group named "host". The tag "" can +# be used for standard IP/hostname matching and is only an alias for +# (?:::f{4,6}:)?(?P[\w\-.^_]+) +# Values: TEXT +# +failregex = ^%(__prefix_line)s.*NOSRV.*401 + +# Option: ignoreregex +# Notes.: regex to ignore. If this regex matches, the line is ignored. +# Values: TEXT +# +ignoreregex = diff --git a/config/jail.conf b/config/jail.conf index 02543705..9788bd25 100644 --- a/config/jail.conf +++ b/config/jail.conf @@ -839,3 +839,12 @@ filter = screensharingd logpath = /var/log/system.log logencoding=utf-8 maxretry = 4 + +[haproxy-http-auth] +# HAProxy by default doesn't log to file you'll need to set it up to forward +# logs to a syslog server which would then write them to disk. +# See "haproxy-http-auth" filter for a brief cautionary note when setting +# maxretry and findtime. +enabled = false +filter = haproxy-http-auth +logpath = /var/log/haproxy.log diff --git a/fail2ban/tests/files/logs/haproxy-http-auth b/fail2ban/tests/files/logs/haproxy-http-auth new file mode 100644 index 00000000..298f1972 --- /dev/null +++ b/fail2ban/tests/files/logs/haproxy-http-auth @@ -0,0 +1,4 @@ +# failJSON: { "match": false } +Nov 14 22:45:27 test haproxy[760]: 192.168.33.1:58444 [14/Nov/2015:22:45:25.439] main app/app1 1939/0/1/0/1940 403 5168 - - ---- 3/3/0/0/0 0/0 "GET / HTTP/1.1" +# failJSON: { "time": "2004-11-14T22:45:11", "match": true , "host": "192.168.33.1" } +Nov 14 22:45:11 test haproxy[760]: 192.168.33.1:58430 [14/Nov/2015:22:45:11.608] main main/ -1/-1/-1/-1/0 401 248 - - PR-- 0/0/0/0/0 0/0 "GET / HTTP/1.1" From ac311214329c3e8100ba9b37091de01475dd522f Mon Sep 17 00:00:00 2001 From: sebres Date: Thu, 31 Dec 2015 02:30:25 +0100 Subject: [PATCH 102/126] amend to fix fail2ban-version: correct user-agent for badips.py "Fail2Ban/ver", changeable within jail/config now; --- config/action.d/badips.py | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/config/action.d/badips.py b/config/action.d/badips.py index 99e1866a..5248e994 100644 --- a/config/action.d/badips.py +++ b/config/action.d/badips.py @@ -21,7 +21,6 @@ import sys if sys.version_info < (2, 7): raise ImportError("badips.py action requires Python >= 2.7") import json -from functools import partial import threading import logging if sys.version_info >= (3, ): @@ -72,6 +71,9 @@ class BadIPsAction(ActionBase): updateperiod : int, optional Time in seconds between updating bad IPs blacklist. Default 900 (15 minutes) + agent : str, optional + User agent transmitted to server. + Default `Fail2Ban/ver.` Raises ------ @@ -80,13 +82,14 @@ class BadIPsAction(ActionBase): """ _badips = "http://www.badips.com" - _Request = partial( - Request, headers={'User-Agent': "Fail2Ban %s" % f2bVersion}) + def _Request(self, url, **argv): + return Request(url, headers={'User-Agent': self.agent}, **argv) def __init__(self, jail, name, category, score=3, age="24h", key=None, - banaction=None, bancategory=None, bankey=None, updateperiod=900): + banaction=None, bancategory=None, bankey=None, updateperiod=900, agent=None): super(BadIPsAction, self).__init__(jail, name) + self.agent = agent if agent is not None else ("Fail2Ban/%s" % f2bVersion) self.category = category self.score = score self.age = age From 618e97bce8a1ad359e191d84835f3400152acbed Mon Sep 17 00:00:00 2001 From: Alexander Belykh Date: Mon, 4 Jan 2016 01:36:28 +0600 Subject: [PATCH 103/126] Add nftables actions --- config/action.d/nftables-allports.conf | 52 +++++++++++++++++++++++++ config/action.d/nftables-common.conf | 50 ++++++++++++++++++++++++ config/action.d/nftables-multiport.conf | 51 ++++++++++++++++++++++++ 3 files changed, 153 insertions(+) create mode 100644 config/action.d/nftables-allports.conf create mode 100644 config/action.d/nftables-common.conf create mode 100644 config/action.d/nftables-multiport.conf diff --git a/config/action.d/nftables-allports.conf b/config/action.d/nftables-allports.conf new file mode 100644 index 00000000..34622f65 --- /dev/null +++ b/config/action.d/nftables-allports.conf @@ -0,0 +1,52 @@ +# Fail2Ban configuration file +# +# Author: Cyril Jaquier +# Modified: Yaroslav O. Halchenko +# made active on all ports from original iptables.conf +# Modified: Alexander Belykh +# adapted for nftables +# + +[INCLUDES] + +before = nftables-common.conf + +[Definition] + +# Option: actionstart +# Notes.: command executed once at the start of Fail2Ban. +# Values: CMD +# +actionstart = add set filter f2b- { type ipv4_addr\; } + insert rule filter ip protocol ip saddr @f2b- + +# Option: actionstop +# Notes.: command executed once at the end of Fail2Ban +# Values: CMD +# +actionstop = HANDLE_ID=$( --handle --numeric list chain filter | grep -m1 'ip saddr @f2b- # handle' | grep -oe ' handle [0-9]*'); delete rule filter $HANDLE_ID + delete set filter f2b- +# Option: actioncheck +# Notes.: command executed once before each actionban command +# Values: CMD +# +actioncheck = list chain filter | grep -q '@f2b-[ \t]' + +# Option: actionban +# Notes.: command executed when banning an IP. Take care that the +# command is executed with Fail2Ban user rights. +# Tags: See jail.conf(5) man page +# Values: CMD +# +actionban = add element filter f2b- { } + +# Option: actionunban +# Notes.: command executed when unbanning an IP. Take care that the +# command is executed with Fail2Ban user rights. +# Tags: See jail.conf(5) man page +# Values: CMD +# +actionunban = delete element filter f2b- { } + +[Init] + diff --git a/config/action.d/nftables-common.conf b/config/action.d/nftables-common.conf new file mode 100644 index 00000000..a0534d35 --- /dev/null +++ b/config/action.d/nftables-common.conf @@ -0,0 +1,50 @@ +# Fail2Ban configuration file +# +# Author: Daniel Black +# Modified: Alexander Belykh +# adapted for nftables +# +# This is a included configuration file and includes the definitions for the nftables +# used in all nftables based actions by default. +# +# The user can override the defaults in nftables-common.local + +[INCLUDES] + +after = nftables-common.local + +[Init] + +# Option: chain +# Notes specifies the nftables chain to which the Fail2Ban rules should be +# added +# Values: STRING Default: input +chain = input + +# Default name of the filtering set +# +name = default + +# Option: port +# Notes.: specifies port to monitor +# Values: [ NUM | STRING ] Default: +# +port = ssh + +# Option: protocol +# Notes.: internally used by config reader for interpolations. +# Values: [ tcp | udp ] Default: tcp +# +protocol = tcp + +# Option: blocktype +# Note: This is what the action does with rules. This can be any jump target +# as per the nftables man page (section 8). Common values are drop +# reject, reject with icmp type host-unreachable +# Values: STRING +blocktype = reject + +# Option: nftables +# Notes.: Actual command to be executed, including common to all calls options +# Values: STRING +nftables = nft diff --git a/config/action.d/nftables-multiport.conf b/config/action.d/nftables-multiport.conf new file mode 100644 index 00000000..ad61bf63 --- /dev/null +++ b/config/action.d/nftables-multiport.conf @@ -0,0 +1,51 @@ +# Fail2Ban configuration file +# +# Author: Cyril Jaquier +# Modified by Yaroslav Halchenko for multiport banning +# Modified: Alexander Belykh +# adapted for nftables +# + +[INCLUDES] + +before = nftables-common.conf + +[Definition] + +# Option: actionstart +# Notes.: command executed once at the start of Fail2Ban. +# Values: CMD +# +actionstart = add set filter f2b- { type ipv4_addr\; } + insert rule filter dport { } ip saddr @f2b- + +# Option: actionstop +# Notes.: command executed once at the end of Fail2Ban +# Values: CMD +# +actionstop = HANDLE_ID=$( --handle --numeric list chain filter | grep -m1 'ip saddr @f2b- # handle' | grep -oe ' handle [0-9]*'); delete rule filter $HANDLE_ID + delete set filter f2b- +# Option: actioncheck +# Notes.: command executed once before each actionban command +# Values: CMD +# +actioncheck = list chain filter | grep -q '@f2b-[ \t]' + +# Option: actionban +# Notes.: command executed when banning an IP. Take care that the +# command is executed with Fail2Ban user rights. +# Tags: See jail.conf(5) man page +# Values: CMD +# +actionban = add element filter f2b- { } + +# Option: actionunban +# Notes.: command executed when unbanning an IP. Take care that the +# command is executed with Fail2Ban user rights. +# Tags: See jail.conf(5) man page +# Values: CMD +# +actionunban = delete element filter f2b- { } + +[Init] + From 69f5623f83180dda4e9461f553fd676f029b3ba9 Mon Sep 17 00:00:00 2001 From: sebres Date: Mon, 4 Jan 2016 09:30:32 +0100 Subject: [PATCH 104/126] code simplifying (remove duplication): agent will be always supplied as parameter from jail.conf --- config/action.d/badips.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/config/action.d/badips.py b/config/action.d/badips.py index 5248e994..025289ca 100644 --- a/config/action.d/badips.py +++ b/config/action.d/badips.py @@ -32,7 +32,6 @@ else: from urllib import urlencode from fail2ban.server.actions import ActionBase -from fail2ban.version import version as f2bVersion class BadIPsAction(ActionBase): @@ -86,10 +85,10 @@ class BadIPsAction(ActionBase): return Request(url, headers={'User-Agent': self.agent}, **argv) def __init__(self, jail, name, category, score=3, age="24h", key=None, - banaction=None, bancategory=None, bankey=None, updateperiod=900, agent=None): + banaction=None, bancategory=None, bankey=None, updateperiod=900, agent="Fail2Ban"): super(BadIPsAction, self).__init__(jail, name) - self.agent = agent if agent is not None else ("Fail2Ban/%s" % f2bVersion) + self.agent = agent self.category = category self.score = score self.age = age From 25a09352e48a303d66d1e207e2e3b51b3d2e9793 Mon Sep 17 00:00:00 2001 From: sebres Date: Mon, 4 Jan 2016 14:46:43 +0100 Subject: [PATCH 105/126] + ChangeLog entry --- ChangeLog | 3 +++ 1 file changed, 3 insertions(+) diff --git a/ChangeLog b/ChangeLog index 8b5344be..36c727fe 100644 --- a/ChangeLog +++ b/ChangeLog @@ -28,6 +28,7 @@ ver. 0.9.4 (2015/XX/XXX) - wanna-be-released for python version < 3.x (gh-1248) * Use postfix_log logpath for postfix-rbl jail * filters.d/postfix.conf - add 'Sender address rejected: Domain not found' failregex + * use `fail2ban_agent` as user-agent in actions badips, blocklist_de, etc (gh-1271) - New Features: * New interpolation feature for definition config readers - `` @@ -72,6 +73,8 @@ ver. 0.9.4 (2015/XX/XXX) - wanna-be-released operations. Thanks @kshetragia * Specified that fail2ban is PartOf iptables.service firewalld.service in .service file -- would reload fail2ban if those services are restarted + * Provides new default `fail2ban_version` and interpolation variable + `fail2ban_agent` in jail.conf ver. 0.9.3 (2015/08/01) - lets-all-stay-friends From f7f91a8bd45d83fa106c2c827e91d53d1e9b75dc Mon Sep 17 00:00:00 2001 From: Alexander Belykh Date: Tue, 5 Jan 2016 19:03:47 +0600 Subject: [PATCH 106/126] Refactor common code out of nftables-multiport/allports.conf --- config/action.d/nftables-allports.conf | 38 +++----------------- config/action.d/nftables-common.conf | 46 +++++++++++++++++++++++++ config/action.d/nftables-multiport.conf | 41 ++++------------------ 3 files changed, 56 insertions(+), 69 deletions(-) diff --git a/config/action.d/nftables-allports.conf b/config/action.d/nftables-allports.conf index 34622f65..afd0ca84 100644 --- a/config/action.d/nftables-allports.conf +++ b/config/action.d/nftables-allports.conf @@ -13,40 +13,10 @@ before = nftables-common.conf [Definition] -# Option: actionstart -# Notes.: command executed once at the start of Fail2Ban. -# Values: CMD +# Option: nftables_mode +# Notes.: additional expressions for nftables filter rule +# Values: nftables expressions # -actionstart = add set filter f2b- { type ipv4_addr\; } - insert rule filter ip protocol ip saddr @f2b- - -# Option: actionstop -# Notes.: command executed once at the end of Fail2Ban -# Values: CMD -# -actionstop = HANDLE_ID=$( --handle --numeric list chain filter | grep -m1 'ip saddr @f2b- # handle' | grep -oe ' handle [0-9]*'); delete rule filter $HANDLE_ID - delete set filter f2b- -# Option: actioncheck -# Notes.: command executed once before each actionban command -# Values: CMD -# -actioncheck = list chain filter | grep -q '@f2b-[ \t]' - -# Option: actionban -# Notes.: command executed when banning an IP. Take care that the -# command is executed with Fail2Ban user rights. -# Tags: See jail.conf(5) man page -# Values: CMD -# -actionban = add element filter f2b- { } - -# Option: actionunban -# Notes.: command executed when unbanning an IP. Take care that the -# command is executed with Fail2Ban user rights. -# Tags: See jail.conf(5) man page -# Values: CMD -# -actionunban = delete element filter f2b- { } +nftables_mode = ip protocol [Init] - diff --git a/config/action.d/nftables-common.conf b/config/action.d/nftables-common.conf index a0534d35..e65618ef 100644 --- a/config/action.d/nftables-common.conf +++ b/config/action.d/nftables-common.conf @@ -1,6 +1,9 @@ # Fail2Ban configuration file # # Author: Daniel Black +# Author: Cyril Jaquier +# Modified: Yaroslav O. Halchenko +# made active on all ports from original iptables.conf # Modified: Alexander Belykh # adapted for nftables # @@ -13,6 +16,49 @@ after = nftables-common.local +[Definition] + +# Option: nftables_mode +# Notes.: additional expressions for nftables filter rule +# Values: nftables expressions +# +nftables_mode = dport { } + +# Option: actionstart +# Notes.: command executed once at the start of Fail2Ban. +# Values: CMD +# +actionstart = add set filter f2b- { type ipv4_addr\; } + insert rule filter %(nftables_mode)s ip saddr @f2b- + +# Option: actionstop +# Notes.: command executed once at the end of Fail2Ban +# Values: CMD +# +actionstop = HANDLE_ID=$( --handle --numeric list chain filter | grep -m1 'ip saddr @f2b- # handle' | grep -oe ' handle [0-9]*'); delete rule filter $HANDLE_ID + delete set filter f2b- +# Option: actioncheck +# Notes.: command executed once before each actionban command +# Values: CMD +# +actioncheck = list chain filter | grep -q '@f2b-[ \t]' + +# Option: actionban +# Notes.: command executed when banning an IP. Take care that the +# command is executed with Fail2Ban user rights. +# Tags: See jail.conf(5) man page +# Values: CMD +# +actionban = add element filter f2b- { } + +# Option: actionunban +# Notes.: command executed when unbanning an IP. Take care that the +# command is executed with Fail2Ban user rights. +# Tags: See jail.conf(5) man page +# Values: CMD +# +actionunban = delete element filter f2b- { } + [Init] # Option: chain diff --git a/config/action.d/nftables-multiport.conf b/config/action.d/nftables-multiport.conf index ad61bf63..3c6455e2 100644 --- a/config/action.d/nftables-multiport.conf +++ b/config/action.d/nftables-multiport.conf @@ -1,7 +1,8 @@ # Fail2Ban configuration file # # Author: Cyril Jaquier -# Modified by Yaroslav Halchenko for multiport banning +# Modified: Yaroslav O. Halchenko +# made active on all ports from original iptables.conf # Modified: Alexander Belykh # adapted for nftables # @@ -12,40 +13,10 @@ before = nftables-common.conf [Definition] -# Option: actionstart -# Notes.: command executed once at the start of Fail2Ban. -# Values: CMD +# Option: nftables_mode +# Notes.: additional expressions for nftables filter rule +# Values: nftables expressions # -actionstart = add set filter f2b- { type ipv4_addr\; } - insert rule filter dport { } ip saddr @f2b- - -# Option: actionstop -# Notes.: command executed once at the end of Fail2Ban -# Values: CMD -# -actionstop = HANDLE_ID=$( --handle --numeric list chain filter | grep -m1 'ip saddr @f2b- # handle' | grep -oe ' handle [0-9]*'); delete rule filter $HANDLE_ID - delete set filter f2b- -# Option: actioncheck -# Notes.: command executed once before each actionban command -# Values: CMD -# -actioncheck = list chain filter | grep -q '@f2b-[ \t]' - -# Option: actionban -# Notes.: command executed when banning an IP. Take care that the -# command is executed with Fail2Ban user rights. -# Tags: See jail.conf(5) man page -# Values: CMD -# -actionban = add element filter f2b- { } - -# Option: actionunban -# Notes.: command executed when unbanning an IP. Take care that the -# command is executed with Fail2Ban user rights. -# Tags: See jail.conf(5) man page -# Values: CMD -# -actionunban = delete element filter f2b- { } +nftables_mode = dport { } [Init] - From cb2d70d7a8a47a8f42e6c2a443b27cc150d669ba Mon Sep 17 00:00:00 2001 From: Alexander Belykh Date: Tue, 5 Jan 2016 19:04:44 +0600 Subject: [PATCH 107/126] Add ChangeLog entry for new nftables actions --- ChangeLog | 3 +++ 1 file changed, 3 insertions(+) diff --git a/ChangeLog b/ChangeLog index 8b5344be..6d42a837 100644 --- a/ChangeLog +++ b/ChangeLog @@ -37,6 +37,9 @@ ver. 0.9.4 (2015/XX/XXX) - wanna-be-released filter.d/*.local file. As extension to interpolation `%(known/parameter)s`, that does not works for filter and action init parameters + * New actions: + - nftables-multiport and nftables-allports - filtering using nftables + framework. Note: it requires a pre-existing chain for the filtering rule. * New filters: - openhab - domotic software authentication failure with the rest api and web interface (gh-1223) From 1983e155808f30eb0eb775061252e3dad1bd2b19 Mon Sep 17 00:00:00 2001 From: Alexander Belykh Date: Wed, 6 Jan 2016 16:55:29 +0600 Subject: [PATCH 108/126] Add empty line between parameters in nftables-common.conf --- config/action.d/nftables-common.conf | 1 + 1 file changed, 1 insertion(+) diff --git a/config/action.d/nftables-common.conf b/config/action.d/nftables-common.conf index e65618ef..534eec2f 100644 --- a/config/action.d/nftables-common.conf +++ b/config/action.d/nftables-common.conf @@ -37,6 +37,7 @@ actionstart = add set filter f2b- { type ipv4_addr\; } # actionstop = HANDLE_ID=$( --handle --numeric list chain filter | grep -m1 'ip saddr @f2b- # handle' | grep -oe ' handle [0-9]*'); delete rule filter $HANDLE_ID delete set filter f2b- + # Option: actioncheck # Notes.: command executed once before each actionban command # Values: CMD From 260c30535d71c5ee1d07f843536bd4e9f6163358 Mon Sep 17 00:00:00 2001 From: Alexander Belykh Date: Wed, 6 Jan 2016 17:13:30 +0600 Subject: [PATCH 109/126] Escape curly braces in nftables actions --- config/action.d/nftables-common.conf | 8 ++++---- config/action.d/nftables-multiport.conf | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/config/action.d/nftables-common.conf b/config/action.d/nftables-common.conf index 534eec2f..157f3c77 100644 --- a/config/action.d/nftables-common.conf +++ b/config/action.d/nftables-common.conf @@ -22,13 +22,13 @@ after = nftables-common.local # Notes.: additional expressions for nftables filter rule # Values: nftables expressions # -nftables_mode = dport { } +nftables_mode = dport \{ \} # Option: actionstart # Notes.: command executed once at the start of Fail2Ban. # Values: CMD # -actionstart = add set filter f2b- { type ipv4_addr\; } +actionstart = add set filter f2b- \{ type ipv4_addr\; \} insert rule filter %(nftables_mode)s ip saddr @f2b- # Option: actionstop @@ -50,7 +50,7 @@ actioncheck = list chain filter | grep -q '@f2b-[ \t]' # Tags: See jail.conf(5) man page # Values: CMD # -actionban = add element filter f2b- { } +actionban = add element filter f2b- \{ \} # Option: actionunban # Notes.: command executed when unbanning an IP. Take care that the @@ -58,7 +58,7 @@ actionban = add element filter f2b- { } # Tags: See jail.conf(5) man page # Values: CMD # -actionunban = delete element filter f2b- { } +actionunban = delete element filter f2b- \{ \} [Init] diff --git a/config/action.d/nftables-multiport.conf b/config/action.d/nftables-multiport.conf index 3c6455e2..d1afafb3 100644 --- a/config/action.d/nftables-multiport.conf +++ b/config/action.d/nftables-multiport.conf @@ -17,6 +17,6 @@ before = nftables-common.conf # Notes.: additional expressions for nftables filter rule # Values: nftables expressions # -nftables_mode = dport { } +nftables_mode = dport \{ \} [Init] From 9779eeb986e323608207f9790556d7b9ce6816f1 Mon Sep 17 00:00:00 2001 From: Alexander Belykh Date: Wed, 6 Jan 2016 17:33:14 +0600 Subject: [PATCH 110/126] Add nftables_type/family/table parameters --- config/action.d/nftables-common.conf | 32 ++++++++++++++++++++++------ 1 file changed, 25 insertions(+), 7 deletions(-) diff --git a/config/action.d/nftables-common.conf b/config/action.d/nftables-common.conf index 157f3c77..26e35892 100644 --- a/config/action.d/nftables-common.conf +++ b/config/action.d/nftables-common.conf @@ -28,21 +28,21 @@ nftables_mode = dport \{ \} # Notes.: command executed once at the start of Fail2Ban. # Values: CMD # -actionstart = add set filter f2b- \{ type ipv4_addr\; \} - insert rule filter %(nftables_mode)s ip saddr @f2b- +actionstart = add set f2b- \{ type \; \} + insert rule %(nftables_mode)s ip saddr @f2b- # Option: actionstop # Notes.: command executed once at the end of Fail2Ban # Values: CMD # -actionstop = HANDLE_ID=$( --handle --numeric list chain filter | grep -m1 'ip saddr @f2b- # handle' | grep -oe ' handle [0-9]*'); delete rule filter $HANDLE_ID - delete set filter f2b- +actionstop = HANDLE_ID=$( --handle --numeric list chain | grep -m1 'ip saddr @f2b- # handle' | grep -oe ' handle [0-9]*'); delete rule $HANDLE_ID + delete set f2b- # Option: actioncheck # Notes.: command executed once before each actionban command # Values: CMD # -actioncheck = list chain filter | grep -q '@f2b-[ \t]' +actioncheck = list chain | grep -q '@f2b-[ \t]' # Option: actionban # Notes.: command executed when banning an IP. Take care that the @@ -50,7 +50,7 @@ actioncheck = list chain filter | grep -q '@f2b-[ \t]' # Tags: See jail.conf(5) man page # Values: CMD # -actionban = add element filter f2b- \{ \} +actionban = add element f2b- \{ \} # Option: actionunban # Notes.: command executed when unbanning an IP. Take care that the @@ -58,10 +58,28 @@ actionban = add element filter f2b- \{ \} # Tags: See jail.conf(5) man page # Values: CMD # -actionunban = delete element filter f2b- \{ \} +actionunban = delete element f2b- \{ \} [Init] +# Option: nftables_type +# Notes.: address type to work with +# Values: [ipv4_addr | ipv6_addr] Default: ipv4_addr +# +nftables_type = ipv4_addr + +# Option: nftables_family +# Notes.: address family to work in +# Values: [ip | ip6 | inet] Default: inet +# +nftables_family = inet + +# Option: nftables_table +# Notes.: table in the address family to work in +# Values: STRING Default: filter +# +nftables_table = filter + # Option: chain # Notes specifies the nftables chain to which the Fail2Ban rules should be # added From 985e8938a4a7aa4181851ad962cfe0c6a3a8fba3 Mon Sep 17 00:00:00 2001 From: Alexander Belykh Date: Wed, 6 Jan 2016 17:39:54 +0600 Subject: [PATCH 111/126] Refactor nftables actionstop into smaller parts --- config/action.d/nftables-common.conf | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/config/action.d/nftables-common.conf b/config/action.d/nftables-common.conf index 26e35892..80657c5c 100644 --- a/config/action.d/nftables-common.conf +++ b/config/action.d/nftables-common.conf @@ -31,11 +31,15 @@ nftables_mode = dport \{ \} actionstart = add set f2b- \{ type \; \} insert rule %(nftables_mode)s ip saddr @f2b- +_nft_list = --handle --numeric list chain +_nft_get_handle_id = grep -m1 'ip saddr @f2b- # handle' | grep -oe ' handle [0-9]*' + # Option: actionstop # Notes.: command executed once at the end of Fail2Ban # Values: CMD # -actionstop = HANDLE_ID=$( --handle --numeric list chain | grep -m1 'ip saddr @f2b- # handle' | grep -oe ' handle [0-9]*'); delete rule $HANDLE_ID +actionstop = HANDLE_ID=$(%(_nft_list)s | %(_nft_get_handle_id)s) + delete rule $HANDLE_ID delete set f2b- # Option: actioncheck From 40c0bed82c649ad7731bbe33ba773a25e55a6163 Mon Sep 17 00:00:00 2001 From: local Date: Sun, 10 Jan 2016 00:05:03 +0100 Subject: [PATCH 112/126] action_mw, action_mwl, action_cf_mwl ignore the "sender" option when sending a notification email. This commit adds "sender="%(sender)s"" to the three actions to correct this issue. --- config/jail.conf | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/config/jail.conf b/config/jail.conf index 7560f582..2bf28f6d 100644 --- a/config/jail.conf +++ b/config/jail.conf @@ -164,12 +164,12 @@ action_ = %(banaction)s[name=%(__name__)s, bantime="%(bantime)s", port="%(port)s # ban & send an e-mail with whois report to the destemail. action_mw = %(banaction)s[name=%(__name__)s, bantime="%(bantime)s", port="%(port)s", protocol="%(protocol)s", chain="%(chain)s"] - %(mta)s-whois[name=%(__name__)s, dest="%(destemail)s", protocol="%(protocol)s", chain="%(chain)s"] + %(mta)s-whois[name=%(__name__)s, sender="%(sender)s", dest="%(destemail)s", protocol="%(protocol)s", chain="%(chain)s"] # ban & send an e-mail with whois report and relevant log lines # to the destemail. action_mwl = %(banaction)s[name=%(__name__)s, bantime="%(bantime)s", port="%(port)s", protocol="%(protocol)s", chain="%(chain)s"] - %(mta)s-whois-lines[name=%(__name__)s, dest="%(destemail)s", logpath=%(logpath)s, chain="%(chain)s"] + %(mta)s-whois-lines[name=%(__name__)s, sender="%(sender)s", dest="%(destemail)s", logpath=%(logpath)s, chain="%(chain)s"] # See the IMPORTANT note in action.d/xarf-login-attack for when to use this action # @@ -181,7 +181,7 @@ action_xarf = %(banaction)s[name=%(__name__)s, bantime="%(bantime)s", port="%(po # ban IP on CloudFlare & send an e-mail with whois report and relevant log lines # to the destemail. action_cf_mwl = cloudflare[cfuser="%(cfemail)s", cftoken="%(cfapikey)s"] - %(mta)s-whois-lines[name=%(__name__)s, dest="%(destemail)s", logpath=%(logpath)s, chain="%(chain)s"] + %(mta)s-whois-lines[name=%(__name__)s, sender="%(sender)s", dest="%(destemail)s", logpath=%(logpath)s, chain="%(chain)s"] # Report block via blocklist.de fail2ban reporting service API # From 58a8736e0f2e35eba89b3b236fbfafae84194821 Mon Sep 17 00:00:00 2001 From: local Date: Sun, 10 Jan 2016 00:10:05 +0100 Subject: [PATCH 113/126] Updating changelog. --- ChangeLog | 1 + 1 file changed, 1 insertion(+) diff --git a/ChangeLog b/ChangeLog index 36c727fe..2220097e 100644 --- a/ChangeLog +++ b/ChangeLog @@ -29,6 +29,7 @@ ver. 0.9.4 (2015/XX/XXX) - wanna-be-released * Use postfix_log logpath for postfix-rbl jail * filters.d/postfix.conf - add 'Sender address rejected: Domain not found' failregex * use `fail2ban_agent` as user-agent in actions badips, blocklist_de, etc (gh-1271) + * Fix ignoring the sender option by action_mw, action_mwl and action_c_mwl - New Features: * New interpolation feature for definition config readers - `` From d7b46509d86cbf7fce447a7439bcfcfc91801cc8 Mon Sep 17 00:00:00 2001 From: Jordan Moeser Date: Tue, 12 Jan 2016 08:37:33 +1000 Subject: [PATCH 114/126] Update haproxy-http-auth.conf Updated failregex to be more strict --- config/filter.d/haproxy-http-auth.conf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config/filter.d/haproxy-http-auth.conf b/config/filter.d/haproxy-http-auth.conf index 7c371dde..298ca292 100644 --- a/config/filter.d/haproxy-http-auth.conf +++ b/config/filter.d/haproxy-http-auth.conf @@ -28,7 +28,7 @@ _daemon = haproxy # (?:::f{4,6}:)?(?P[\w\-.^_]+) # Values: TEXT # -failregex = ^%(__prefix_line)s.*NOSRV.*401 +failregex = ^%(__prefix_line)s.* -1/-1/-1/-1/\+*\d* 401 # Option: ignoreregex # Notes.: regex to ignore. If this regex matches, the line is ignored. From 294a7790a9a32f98448e1e2a2d7a5cfdf741c35f Mon Sep 17 00:00:00 2001 From: Louis Sautier Date: Thu, 28 Jan 2016 23:40:34 +0100 Subject: [PATCH 115/126] gentoo-initd: do not hide useful output Gentoo applies a patch for this: https://bugs.gentoo.org/show_bug.cgi?id=536320 --- files/gentoo-initd | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/files/gentoo-initd b/files/gentoo-initd index 98c5edf9..e939b987 100755 --- a/files/gentoo-initd +++ b/files/gentoo-initd @@ -34,19 +34,19 @@ start() { # remove stalled sock file after system crash # bug 347477 rm -f /var/run/fail2ban/fail2ban.sock || return 1 - ${FAIL2BAN} start &> /dev/null + ${FAIL2BAN} start eend $? "Failed to start fail2ban" } stop() { ebegin "Stopping fail2ban" - ${FAIL2BAN} stop &> /dev/null + ${FAIL2BAN} stop eend $? "Failed to stop fail2ban" } reload() { ebegin "Reloading fail2ban" - ${FAIL2BAN} reload > /dev/null + ${FAIL2BAN} reload eend $? "Failed to reload fail2ban" } From 869d99dd377ff45efa5796bb3be2500e41f32dc3 Mon Sep 17 00:00:00 2001 From: Louis Sautier Date: Thu, 28 Jan 2016 23:52:02 +0100 Subject: [PATCH 116/126] Remove compression and count from logrotate Initially reported at https://bugs.gentoo.org/show_bug.cgi?id=549856 --- ChangeLog | 2 ++ files/fail2ban-logrotate | 4 +--- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/ChangeLog b/ChangeLog index 4460f94c..69245dc9 100644 --- a/ChangeLog +++ b/ChangeLog @@ -30,6 +30,8 @@ ver. 0.9.4 (2015/XX/XXX) - wanna-be-released * filters.d/postfix.conf - add 'Sender address rejected: Domain not found' failregex * use `fail2ban_agent` as user-agent in actions badips, blocklist_de, etc (gh-1271) * Fix ignoring the sender option by action_mw, action_mwl and action_c_mwl + * Remove compression and rotation count from logrotate (inherit them from + the global logrotate config) - New Features: * New interpolation feature for definition config readers - `` diff --git a/files/fail2ban-logrotate b/files/fail2ban-logrotate index 8d94a8b3..13a94537 100644 --- a/files/fail2ban-logrotate +++ b/files/fail2ban-logrotate @@ -6,11 +6,9 @@ # https://github.com/fail2ban/fail2ban/blob/debian/debian/fail2ban.logrotate /var/log/fail2ban.log { - rotate 7 missingok notifempty - compress postrotate - /usr/bin/fail2ban-client flushlogs 1>/dev/null || true + /usr/bin/fail2ban-client flushlogs >/dev/null || true endscript } From b5a07741c82acdb9448584a0d39d3df09d28e162 Mon Sep 17 00:00:00 2001 From: Pierre GINDRAUD Date: Mon, 8 Feb 2016 11:08:10 +0100 Subject: [PATCH 117/126] Add new regex into postfix filter. The new regexp is able to detect bad formatted SMTP EHLO command --- ChangeLog | 1 + config/filter.d/postfix.conf | 1 + fail2ban/tests/files/logs/postfix | 3 +++ 3 files changed, 5 insertions(+) diff --git a/ChangeLog b/ChangeLog index 4460f94c..644a6d16 100644 --- a/ChangeLog +++ b/ChangeLog @@ -79,6 +79,7 @@ ver. 0.9.4 (2015/XX/XXX) - wanna-be-released .service file -- would reload fail2ban if those services are restarted * Provides new default `fail2ban_version` and interpolation variable `fail2ban_agent` in jail.conf + * Enhance filter 'postfix' to ban incoming SMTP client with no fqdn hostname ver. 0.9.3 (2015/08/01) - lets-all-stay-friends diff --git a/config/filter.d/postfix.conf b/config/filter.d/postfix.conf index f6a8578b..25141863 100644 --- a/config/filter.d/postfix.conf +++ b/config/filter.d/postfix.conf @@ -15,6 +15,7 @@ _daemon = postfix/(submission/)?smtp(d|s) failregex = ^%(__prefix_line)sNOQUEUE: reject: RCPT from \S+\[\]: 554 5\.7\.1 .*$ ^%(__prefix_line)sNOQUEUE: reject: RCPT from \S+\[\]: 450 4\.7\.1 Client host rejected: cannot find your hostname, (\[\S*\]); from=<\S*> to=<\S+> proto=ESMTP helo=<\S*>$ ^%(__prefix_line)sNOQUEUE: reject: RCPT from \S+\[\]: 450 4\.7\.1 : Helo command rejected: Host not found; from=<> to=<> proto=ESMTP helo= *$ + ^%(__prefix_line)sNOQUEUE: reject: EHLO from \S+\[\]: 504 5\.5\.2 <\S+>: Helo command rejected: need fully-qualified hostname; ^%(__prefix_line)sNOQUEUE: reject: VRFY from \S+\[\]: 550 5\.1\.1 .*$ ^%(__prefix_line)sNOQUEUE: reject: RCPT from \S+\[\]: 450 4\.1\.8 <\S*>: Sender address rejected: Domain not found; from=<\S*> to=<\S+> proto=ESMTP helo=<\S*>$ ^%(__prefix_line)simproper command pipelining after \S+ from [^[]*\[\]:?$ diff --git a/fail2ban/tests/files/logs/postfix b/fail2ban/tests/files/logs/postfix index 4934a29e..800c7f0c 100644 --- a/fail2ban/tests/files/logs/postfix +++ b/fail2ban/tests/files/logs/postfix @@ -26,3 +26,6 @@ Dec 21 21:17:29 xxx postfix/smtpd[7150]: NOQUEUE: reject: RCPT from badserver.ex # failJSON: { "time": "2004-11-22T22:33:44", "match": true , "host": "1.2.3.4" } Nov 22 22:33:44 xxx postfix/smtpd[11111]: NOQUEUE: reject: RCPT from 1-2-3-4.example.com[1.2.3.4]: 450 4.1.8 : Sender address rejected: Domain not found; from= to= proto=ESMTP helo=<1-2-3-4.example.com> + +# failJSON: { "time": "2005-01-31T13:55:24", "match": true , "host": "78.107.251.238" } +Jan 31 13:55:24 xxx postfix/smtpd[3462]: NOQUEUE: reject: EHLO from s271272.static.corbina.ru[78.107.251.238]: 504 5.5.2 : Helo command rejected: need fully-qualified hostname; proto=SMTP helo= From 257b7049d859c45f514fd1d9c83eed85cdc48384 Mon Sep 17 00:00:00 2001 From: 3eBoP Date: Thu, 28 Jan 2016 14:41:10 +0100 Subject: [PATCH 118/126] Update asterisk filter: changed regex for "Call from ...". Sometimes extension can have a plus symbol (+) because they can be phone number. Closes #1309 --- ChangeLog | 1 + config/filter.d/asterisk.conf | 2 +- fail2ban/tests/files/logs/asterisk | 8 ++++++++ 3 files changed, 10 insertions(+), 1 deletion(-) diff --git a/ChangeLog b/ChangeLog index 644a6d16..e128a91a 100644 --- a/ChangeLog +++ b/ChangeLog @@ -30,6 +30,7 @@ ver. 0.9.4 (2015/XX/XXX) - wanna-be-released * filters.d/postfix.conf - add 'Sender address rejected: Domain not found' failregex * use `fail2ban_agent` as user-agent in actions badips, blocklist_de, etc (gh-1271) * Fix ignoring the sender option by action_mw, action_mwl and action_c_mwl + * Changed filter.d/asterisk regex for "Call from ..." (few vulnerable now) - New Features: * New interpolation feature for definition config readers - `` diff --git a/config/filter.d/asterisk.conf b/config/filter.d/asterisk.conf index b446c44e..6ce65c4f 100644 --- a/config/filter.d/asterisk.conf +++ b/config/filter.d/asterisk.conf @@ -19,7 +19,7 @@ iso8601 = \d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\.\d+[+-]\d{4} log_prefix= (?:NOTICE|SECURITY)%(__pid_re)s:?(?:\[C-[\da-f]*\])? \S+:\d*( in \w+:)? failregex = ^(%(__prefix_line)s|\[\]\s*)%(log_prefix)s Registration from '[^']*' failed for '(:\d+)?' - (Wrong password|Username/auth name mismatch|No matching peer found|Not a local domain|Device does not match ACL|Peer is not supposed to register|ACL error \(permit/deny\)|Not a local domain)$ - ^(%(__prefix_line)s|\[\]\s*)%(log_prefix)s Call from '[^']*' \(:\d+\) to extension '\d+' rejected because extension not found in context 'default'\.$ + ^(%(__prefix_line)s|\[\]\s*)%(log_prefix)s Call from '[^']*' \(:\d+\) to extension '[\d+]+' rejected because extension not found in context 'default'\.$ ^(%(__prefix_line)s|\[\]\s*)%(log_prefix)s Host failed to authenticate as '[^']*'$ ^(%(__prefix_line)s|\[\]\s*)%(log_prefix)s No registration for peer '[^']*' \(from \)$ ^(%(__prefix_line)s|\[\]\s*)%(log_prefix)s Host failed MD5 authentication for '[^']*' \([^)]+\)$ diff --git a/fail2ban/tests/files/logs/asterisk b/fail2ban/tests/files/logs/asterisk index ab018ba9..57b53a02 100644 --- a/fail2ban/tests/files/logs/asterisk +++ b/fail2ban/tests/files/logs/asterisk @@ -59,3 +59,11 @@ Nov 4 18:30:40 localhost asterisk[32229]: NOTICE[32257]: chan_sip.c:23417 in han # match UTF-8 in SessionID # failJSON: { "time": "2015-05-25T07:52:36", "match": true, "host": "10.250.251.252" } [2015-05-25 07:52:36] SECURITY[6988] res_security_log.c: SecurityEvent="InvalidAccountID",EventTV="2015-05-25T07:52:36.888+0300",Severity="Error",Service="PJSIP",EventVersion="1",AccountID="70000180",SessionID="Негодяй",LocalAddress="IPV4/UDP/1.2.3.4/5060",RemoteAddress="IPV4/UDP/10.250.251.252/5061" + +# match phone numbers with + symbol +# failJSON: { "time": "2016-01-28T10:22:27", "match": true , "host": "1.2.3.4" } +[2016-01-28 10:22:27] NOTICE[3477][C-000003bb] chan_sip.c: Call from '' (1.2.3.4:10836) to extension '++441772285411' rejected because extension not found in context 'default'. +# failJSON: { "time": "2016-01-28T10:23:57", "match": true , "host": "1.2.3.4" } +[2016-01-28 10:23:57] NOTICE[3477][C-000003bc] chan_sip.c: Call from '' (1.2.3.4:10836) to extension '+441772285403' rejected because extension not found in context 'default'. +# failJSON: { "time": "2016-01-28T10:34:31", "match": true , "host": "1.2.3.4" } +[2016-01-28 10:34:31] NOTICE[3477][C-000003c3] chan_sip.c: Call from '' (1.2.3.4:10836) to extension '0+441772285407' rejected because extension not found in context 'default'. From d8e81eb417ae0e91b077c5c8b2fad26a9ff6de87 Mon Sep 17 00:00:00 2001 From: sebres Date: Mon, 8 Feb 2016 11:47:21 +0100 Subject: [PATCH 119/126] regexp rewritten (few vulnerable as previous) + test case added --- config/filter.d/asterisk.conf | 2 +- fail2ban/tests/files/logs/asterisk | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/config/filter.d/asterisk.conf b/config/filter.d/asterisk.conf index 6ce65c4f..3975fb29 100644 --- a/config/filter.d/asterisk.conf +++ b/config/filter.d/asterisk.conf @@ -19,7 +19,7 @@ iso8601 = \d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\.\d+[+-]\d{4} log_prefix= (?:NOTICE|SECURITY)%(__pid_re)s:?(?:\[C-[\da-f]*\])? \S+:\d*( in \w+:)? failregex = ^(%(__prefix_line)s|\[\]\s*)%(log_prefix)s Registration from '[^']*' failed for '(:\d+)?' - (Wrong password|Username/auth name mismatch|No matching peer found|Not a local domain|Device does not match ACL|Peer is not supposed to register|ACL error \(permit/deny\)|Not a local domain)$ - ^(%(__prefix_line)s|\[\]\s*)%(log_prefix)s Call from '[^']*' \(:\d+\) to extension '[\d+]+' rejected because extension not found in context 'default'\.$ + ^(%(__prefix_line)s|\[\]\s*)%(log_prefix)s Call from '[^']*' \(:\d+\) to extension '[^']*' rejected because extension not found in context ^(%(__prefix_line)s|\[\]\s*)%(log_prefix)s Host failed to authenticate as '[^']*'$ ^(%(__prefix_line)s|\[\]\s*)%(log_prefix)s No registration for peer '[^']*' \(from \)$ ^(%(__prefix_line)s|\[\]\s*)%(log_prefix)s Host failed MD5 authentication for '[^']*' \([^)]+\)$ diff --git a/fail2ban/tests/files/logs/asterisk b/fail2ban/tests/files/logs/asterisk index 57b53a02..aa32a290 100644 --- a/fail2ban/tests/files/logs/asterisk +++ b/fail2ban/tests/files/logs/asterisk @@ -60,10 +60,10 @@ Nov 4 18:30:40 localhost asterisk[32229]: NOTICE[32257]: chan_sip.c:23417 in han # failJSON: { "time": "2015-05-25T07:52:36", "match": true, "host": "10.250.251.252" } [2015-05-25 07:52:36] SECURITY[6988] res_security_log.c: SecurityEvent="InvalidAccountID",EventTV="2015-05-25T07:52:36.888+0300",Severity="Error",Service="PJSIP",EventVersion="1",AccountID="70000180",SessionID="Негодяй",LocalAddress="IPV4/UDP/1.2.3.4/5060",RemoteAddress="IPV4/UDP/10.250.251.252/5061" -# match phone numbers with + symbol +# match phone numbers with + symbol (and without number, or other context) # failJSON: { "time": "2016-01-28T10:22:27", "match": true , "host": "1.2.3.4" } [2016-01-28 10:22:27] NOTICE[3477][C-000003bb] chan_sip.c: Call from '' (1.2.3.4:10836) to extension '++441772285411' rejected because extension not found in context 'default'. -# failJSON: { "time": "2016-01-28T10:23:57", "match": true , "host": "1.2.3.4" } -[2016-01-28 10:23:57] NOTICE[3477][C-000003bc] chan_sip.c: Call from '' (1.2.3.4:10836) to extension '+441772285403' rejected because extension not found in context 'default'. # failJSON: { "time": "2016-01-28T10:34:31", "match": true , "host": "1.2.3.4" } [2016-01-28 10:34:31] NOTICE[3477][C-000003c3] chan_sip.c: Call from '' (1.2.3.4:10836) to extension '0+441772285407' rejected because extension not found in context 'default'. +# failJSON: { "time": "2016-01-28T10:34:33", "match": true , "host": "1.2.3.4" } +[2016-01-28 10:34:33] NOTICE[3477][C-000003c3] chan_sip.c: Call from '' (1.2.3.4:10836) to extension '' rejected because extension not found in context 'my-context'. From 72638975a9b1837d4fedb899e2611506074eb47e Mon Sep 17 00:00:00 2001 From: Yaroslav Halchenko Date: Wed, 17 Feb 2016 20:43:50 -0500 Subject: [PATCH 120/126] ENH: add codecov support to travis.yml and bandge to README.md --- .travis.yml | 3 ++- README.md | 2 ++ 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index adb41e7d..50894a94 100644 --- a/.travis.yml +++ b/.travis.yml @@ -23,7 +23,7 @@ install: # coverage - travis_retry pip install coverage # coveralls - - travis_retry pip install coveralls + - travis_retry pip install coveralls codecov # dnspython or dnspython3 - if [[ "$F2B_PY_2" ]]; then travis_retry pip install dnspython; fi - if [[ "$F2B_PY_3" ]]; then travis_retry pip install dnspython3; fi @@ -43,6 +43,7 @@ script: - sudo $VENV_BIN/pip install . after_success: - coveralls + - codecov matrix: fast_finish: true # Might be worth looking into diff --git a/README.md b/README.md index cbc075e2..a05b24c6 100644 --- a/README.md +++ b/README.md @@ -77,6 +77,8 @@ Code status: * [![Coverage Status](https://coveralls.io/repos/fail2ban/fail2ban/badge.png?branch=master)](https://coveralls.io/r/fail2ban/fail2ban) +* [![codecov.io](https://codecov.io/github/fail2ban/fail2ban/coverage.svg?branch=master)](https://codecov.io/github/fail2ban/fail2ban?branch=master) + Contact: -------- From 9667c4cb4251fb884cbc59232d5e9aeca2af73db Mon Sep 17 00:00:00 2001 From: Yaroslav Halchenko Date: Wed, 17 Feb 2016 21:20:51 -0500 Subject: [PATCH 121/126] ENH: github templates for issues and PRs --- .github/ISSUE_TEMPLATE.md | 40 ++++++++++++++++++++++++++++++++ .github/PULL_REQUEST_TEMPLATE.md | 9 +++++++ 2 files changed, 49 insertions(+) create mode 100644 .github/ISSUE_TEMPLATE.md create mode 100644 .github/PULL_REQUEST_TEMPLATE.md diff --git a/.github/ISSUE_TEMPLATE.md b/.github/ISSUE_TEMPLATE.md new file mode 100644 index 00000000..092a7854 --- /dev/null +++ b/.github/ISSUE_TEMPLATE.md @@ -0,0 +1,40 @@ +_Please fill out following information while also paying attention to comments. +Depending on your issue not all fields might apply -- just remove those which +do not apply, and remove the comments. Thanks!_ + +### Environment: + +#### Your OS (including release name/version): +#### Fail2Ban version: + +_If your Fail2Ban version is outdated, and you can't verify that the issue persists in +the recent release, better seek support from the distribution you obtained Fail2Ban from_ + +### The issue: + +_Summary here_ + +#### Steps to reproduce + +#### Expected behavior + +#### Observed behavior + +#### Any additional information + +### Supplementals + +#### Any customizations done to /etc/fail2ban/ configuration +``` +``` + +#### Relevant parts of /var/log/fail2ban.log file: +_preferably obtained while running fail2ban with `loglevel = 4`_ + +``` +``` + +#### Relevant lines from monitored log files in question: + +``` +``` \ No newline at end of file diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md new file mode 100644 index 00000000..a9fff350 --- /dev/null +++ b/.github/PULL_REQUEST_TEMPLATE.md @@ -0,0 +1,9 @@ +Before submitting your PR, please review the following checklist: + +- [ ] **CONSIDER adding a unit test** if your PR resolves an issue +- [ ] **LIST ISSUES** this PR resolves +- [ ] **MAKE SURE** this PR doesn't break existing tests +- [ ] **KEEP PR small** so it could be easily reviewed. +- [ ] **AVOID** making unnecessary stylistic changes in unrelated code +- [ ] **ACCOMPANY** each new `failregex` for filter `X` with sample log lines + within `fail2ban/tests/files/logs/X` file \ No newline at end of file From 705b91e6a75c6b83cd054f7cffe02b0fcc2ca819 Mon Sep 17 00:00:00 2001 From: Yaroslav Halchenko Date: Sat, 20 Feb 2016 11:20:33 -0500 Subject: [PATCH 122/126] DOC: adjusted ISSUE_TEMPLATE.md picking on @sebres's version --- .github/ISSUE_TEMPLATE.md | 25 +++++++++++++++++-------- 1 file changed, 17 insertions(+), 8 deletions(-) diff --git a/.github/ISSUE_TEMPLATE.md b/.github/ISSUE_TEMPLATE.md index 092a7854..cb4b4bc6 100644 --- a/.github/ISSUE_TEMPLATE.md +++ b/.github/ISSUE_TEMPLATE.md @@ -1,14 +1,23 @@ -_Please fill out following information while also paying attention to comments. -Depending on your issue not all fields might apply -- just remove those which -do not apply, and remove the comments. Thanks!_ +_We will be very grateful, if your problem was described as completely as possible, +enclosing excerpts from logs (if possible within DEBUG mode, if no errors evident +within INFO mode), and configuration in particular of effected relevant settings +(e.g., with ` fail2ban-client -d | grep 'affected-jail-name' ` for a particular +jail troubleshooting). +Thank you in advance for the details, because such issues like "It does not work" +alone could not help to resolve anything! +Thanks! (remove this paragraph and other comments upon reading)_ ### Environment: -#### Your OS (including release name/version): -#### Fail2Ban version: +_Fill out and check (`[x]`) the boxes which apply. If your Fail2Ban version is outdated, +and you can't verify that the issue persists in the recent release, better seek support +from the distribution you obtained Fail2Ban from_ -_If your Fail2Ban version is outdated, and you can't verify that the issue persists in -the recent release, better seek support from the distribution you obtained Fail2Ban from_ +- Fail2Ban version (including any possible distribution suffixes): +- OS, including release name/version: +- [ ] Fail2Ban installed via OS/distribution mechanisms +- [ ] You have not applied any additional foreign patches to the codebase +- [ ] Some customizations were done to the configuration (provide details below is so) ### The issue: @@ -22,7 +31,7 @@ _Summary here_ #### Any additional information -### Supplementals +### Configuration, dump and another helpful excerpts #### Any customizations done to /etc/fail2ban/ configuration ``` From bd822d02a47d72f416f01d411a8e4c1c232821bc Mon Sep 17 00:00:00 2001 From: Yaroslav Halchenko Date: Mon, 22 Feb 2016 09:13:30 -0500 Subject: [PATCH 123/126] DOC: removed Nick from listed as FreeBSD maintainer --- RELEASE | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/RELEASE b/RELEASE index ae6738a1..d4a4f971 100644 --- a/RELEASE +++ b/RELEASE @@ -116,7 +116,7 @@ Pre Release * http://packages.qa.debian.org/f/fail2ban.html - * FreeBSD: Christoph Theis theis@gmx.at>, Nick Hilliard + * FreeBSD: Christoph Theis theis@gmx.at> * http://svnweb.freebsd.org/ports/head/security/py-fail2ban/Makefile?view=markup * http://www.freebsd.org/cgi/query-pr-summary.cgi?text=fail2ban From 6c606cf98feb7d99ff90f2a5c8ee201848ac25da Mon Sep 17 00:00:00 2001 From: Tom Hendrikx Date: Tue, 23 Feb 2016 20:23:04 +0100 Subject: [PATCH 124/126] Add support for matching postfix multi-instance daemon names by default --- config/filter.d/postfix-rbl.conf | 2 +- config/filter.d/postfix-sasl.conf | 2 +- config/filter.d/postfix.conf | 2 +- fail2ban/tests/files/logs/postfix | 3 +++ fail2ban/tests/files/logs/postfix-rbl | 3 +++ fail2ban/tests/files/logs/postfix-sasl | 2 ++ 6 files changed, 11 insertions(+), 3 deletions(-) diff --git a/config/filter.d/postfix-rbl.conf b/config/filter.d/postfix-rbl.conf index 05a8bbc7..c3f8c332 100644 --- a/config/filter.d/postfix-rbl.conf +++ b/config/filter.d/postfix-rbl.conf @@ -10,7 +10,7 @@ before = common.conf [Definition] -_daemon = postfix/smtpd +_daemon = postfix(-\w+)?/smtpd failregex = ^%(__prefix_line)sNOQUEUE: reject: RCPT from \S+\[\]: 454 4\.7\.1 Service unavailable; Client host \[\S+\] blocked using .* from=<\S*> to=<\S+> proto=ESMTP helo=<\S*>$ diff --git a/config/filter.d/postfix-sasl.conf b/config/filter.d/postfix-sasl.conf index 199e29bf..7ff995c9 100644 --- a/config/filter.d/postfix-sasl.conf +++ b/config/filter.d/postfix-sasl.conf @@ -7,7 +7,7 @@ before = common.conf [Definition] -_daemon = postfix/(submission/)?smtp(d|s) +_daemon = postfix(-\w+)?/(submission/)?smtp(d|s) failregex = ^%(__prefix_line)swarning: [-._\w]+\[\]: SASL ((?i)LOGIN|PLAIN|(?:CRAM|DIGEST)-MD5) authentication failed(: [ A-Za-z0-9+/:]*={0,2})?\s*$ diff --git a/config/filter.d/postfix.conf b/config/filter.d/postfix.conf index 25141863..002b02b2 100644 --- a/config/filter.d/postfix.conf +++ b/config/filter.d/postfix.conf @@ -10,7 +10,7 @@ before = common.conf [Definition] -_daemon = postfix/(submission/)?smtp(d|s) +_daemon = postfix(-\w+)?/(submission/)?smtp(d|s) failregex = ^%(__prefix_line)sNOQUEUE: reject: RCPT from \S+\[\]: 554 5\.7\.1 .*$ ^%(__prefix_line)sNOQUEUE: reject: RCPT from \S+\[\]: 450 4\.7\.1 Client host rejected: cannot find your hostname, (\[\S*\]); from=<\S*> to=<\S+> proto=ESMTP helo=<\S*>$ diff --git a/fail2ban/tests/files/logs/postfix b/fail2ban/tests/files/logs/postfix index 800c7f0c..3ec2886a 100644 --- a/fail2ban/tests/files/logs/postfix +++ b/fail2ban/tests/files/logs/postfix @@ -29,3 +29,6 @@ Nov 22 22:33:44 xxx postfix/smtpd[11111]: NOQUEUE: reject: RCPT from 1-2-3-4.exa # failJSON: { "time": "2005-01-31T13:55:24", "match": true , "host": "78.107.251.238" } Jan 31 13:55:24 xxx postfix/smtpd[3462]: NOQUEUE: reject: EHLO from s271272.static.corbina.ru[78.107.251.238]: 504 5.5.2 : Helo command rejected: need fully-qualified hostname; proto=SMTP helo= + +# failJSON: { "time": "2005-01-31T13:55:24", "match": true , "host": "78.107.251.238" } +Jan 31 13:55:24 xxx postfix-incoming/smtpd[3462]: NOQUEUE: reject: EHLO from s271272.static.corbina.ru[78.107.251.238]: 504 5.5.2 : Helo command rejected: need fully-qualified hostname; proto=SMTP helo= diff --git a/fail2ban/tests/files/logs/postfix-rbl b/fail2ban/tests/files/logs/postfix-rbl index fd420fa7..eff01bf9 100644 --- a/fail2ban/tests/files/logs/postfix-rbl +++ b/fail2ban/tests/files/logs/postfix-rbl @@ -1,2 +1,5 @@ # failJSON: { "time": "2004-12-30T18:19:15", "match": true , "host": "93.184.216.34" } Dec 30 18:19:15 xxx postfix/smtpd[1574]: NOQUEUE: reject: RCPT from badguy.example.com[93.184.216.34]: 454 4.7.1 Service unavailable; Client host [93.184.216.34] blocked using rbl.example.com; http://www.example.com/query?ip=93.184.216.34; from= to= proto=ESMTP helo= + +# failJSON: { "time": "2004-12-30T18:19:15", "match": true , "host": "93.184.216.34" } +Dec 30 18:19:15 xxx postfix-incoming/smtpd[1574]: NOQUEUE: reject: RCPT from badguy.example.com[93.184.216.34]: 454 4.7.1 Service unavailable; Client host [93.184.216.34] blocked using rbl.example.com; http://www.example.com/query?ip=93.184.216.34; from= to= proto=ESMTP helo= diff --git a/fail2ban/tests/files/logs/postfix-sasl b/fail2ban/tests/files/logs/postfix-sasl index beb8b11d..586f9584 100644 --- a/fail2ban/tests/files/logs/postfix-sasl +++ b/fail2ban/tests/files/logs/postfix-sasl @@ -21,3 +21,5 @@ Jan 29 08:11:45 mail postfix/smtpd[10752]: warning: unknown[1.1.1.1]: SASL LOGIN # failJSON: { "time": "2005-02-03T08:29:28", "match": false , "host": "1.1.1.1" } Feb 3 08:29:28 mail postfix/smtpd[21022]: warning: unknown[1.1.1.1]: SASL LOGIN authentication failed: Connection lost to authentication server +# failJSON: { "time": "2005-01-29T08:11:45", "match": true , "host": "1.1.1.1" } +Jan 29 08:11:45 mail postfix-incoming/smtpd[10752]: warning: unknown[1.1.1.1]: SASL LOGIN authentication failed: Password: From 667785b608aab5e8e172aa7a7d8db9309aed4f0b Mon Sep 17 00:00:00 2001 From: sebres Date: Wed, 24 Feb 2016 17:11:09 +0100 Subject: [PATCH 125/126] mysqld: failregex fixed (accepts different log level, more secure expression now); closes #1332 --- ChangeLog | 2 +- config/filter.d/mysqld-auth.conf | 2 +- fail2ban/tests/files/logs/mysqld-auth | 4 +++- 3 files changed, 5 insertions(+), 3 deletions(-) diff --git a/ChangeLog b/ChangeLog index ab1106c9..7760a7c0 100644 --- a/ChangeLog +++ b/ChangeLog @@ -15,7 +15,7 @@ ver. 0.9.4 (2015/XX/XXX) - wanna-be-released * filter.d/apache-badbots.conf - Updated useragent string regex adding escape for `+` * filter.d/mysqld-auth.conf - - Updated "Access denied ..." regex for MySQL 5.6 and later (gh-1211) + - Updated "Access denied ..." regex for MySQL 5.6 and later (gh-1211, gh-1332) * filter.d/sshd.conf - Updated "Auth fail" regex for OpenSSH 5.9 and later * Treat failed and killed execution of commands identically (only diff --git a/config/filter.d/mysqld-auth.conf b/config/filter.d/mysqld-auth.conf index f29beffc..a18f60b0 100644 --- a/config/filter.d/mysqld-auth.conf +++ b/config/filter.d/mysqld-auth.conf @@ -17,7 +17,7 @@ before = common.conf _daemon = mysqld -failregex = ^%(__prefix_line)s(?:\d+ |\d{6} \s?\d{1,2}:\d{2}:\d{2} )?\[Warning\] Access denied for user '\w+'@'' (to database '[^']*'|\(using password: (YES|NO)\))*\s*$ +failregex = ^%(__prefix_line)s(?:\d+ |\d{6} \s?\d{1,2}:\d{2}:\d{2} )?\[\w+\] Access denied for user '[^']+'@'' ignoreregex = diff --git a/fail2ban/tests/files/logs/mysqld-auth b/fail2ban/tests/files/logs/mysqld-auth index b75ecfc4..2739ba07 100644 --- a/fail2ban/tests/files/logs/mysqld-auth +++ b/fail2ban/tests/files/logs/mysqld-auth @@ -16,4 +16,6 @@ Sep 16 21:30:26 catinthehat mysqld: 130916 21:30:26 [Warning] Access denied for Sep 16 21:30:32 catinthehat mysqld: 130916 21:30:32 [Warning] Access denied for user 'hacker'@'74.207.241.159' (using password: NO) # failJSON: { "time": "2015-10-07T06:09:42", "match": true , "host": "127.0.0.1", "desc": "mysql 5.6 log format" } -2015-10-07 06:09:42 5907 [Warning] Access denied for user 'root'@'127.0.0.1' (using password: YES) \ No newline at end of file +2015-10-07 06:09:42 5907 [Warning] Access denied for user 'root'@'127.0.0.1' (using password: YES) +# failJSON: { "time": "2016-02-24T15:26:18", "match": true , "host": "localhost", "desc": "mysql 5.6 log format, Note instead of Warning" } +2016-02-24T15:26:18.237955 6 [Note] Access denied for user 'root'@'localhost' (using password: YES) \ No newline at end of file From 2adf5855ac3df2d7cd780930ef6a341e9a421165 Mon Sep 17 00:00:00 2001 From: Yaroslav Halchenko Date: Sun, 28 Feb 2016 12:02:37 -0500 Subject: [PATCH 126/126] Changelog for the recent PR and added Tom to THANKS --- ChangeLog | 4 +++- THANKS | 1 + 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/ChangeLog b/ChangeLog index 7760a7c0..fb45721b 100644 --- a/ChangeLog +++ b/ChangeLog @@ -84,7 +84,9 @@ ver. 0.9.4 (2015/XX/XXX) - wanna-be-released .service file -- would reload fail2ban if those services are restarted * Provides new default `fail2ban_version` and interpolation variable `fail2ban_agent` in jail.conf - * Enhance filter 'postfix' to ban incoming SMTP client with no fqdn hostname + * Enhance filter 'postfix' to ban incoming SMTP client with no fqdn hostname, + and to support multiple instances of postfix having varying suffix (gh-1331) + (Thanks Tom Hendrikx) ver. 0.9.3 (2015/08/01) - lets-all-stay-friends diff --git a/THANKS b/THANKS index 47156d82..0b45c019 100644 --- a/THANKS +++ b/THANKS @@ -115,6 +115,7 @@ Steven Hiscocks TESTOVIK Thomas Mayer Tom Pike +Tom Hendrikx Tomas Pihl Tony Lawrence Tomasz Ciolek