From 0426a24719c13e795d3cf8b68eaea20d2723b41f Mon Sep 17 00:00:00 2001 From: sebres Date: Tue, 14 May 2019 15:27:20 +0200 Subject: [PATCH 1/5] filter.d/postfix.conf: (closes gh-2426) filter extended to catch "5.1.1" (Recipient address rejected: User unknown in local recipient table) with RCPT (and some session-id instead of "NOQUEUE") --- config/filter.d/postfix.conf | 4 ++-- fail2ban/tests/files/logs/postfix | 3 ++- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/config/filter.d/postfix.conf b/config/filter.d/postfix.conf index d1505e32..dfc68b5a 100644 --- a/config/filter.d/postfix.conf +++ b/config/filter.d/postfix.conf @@ -15,12 +15,12 @@ _port = (?::\d+)? prefregex = ^%(__prefix_line)s> .+$ -mdpr-normal = (?:NOQUEUE: reject:|improper command pipelining after \S+) +mdpr-normal = (?:(?:\w+): reject:|improper command pipelining after \S+) mdre-normal=^RCPT from [^[]*\[\]%(_port)s: 55[04] 5\.7\.1\s ^RCPT from [^[]*\[\]%(_port)s: 45[04] 4\.7\.1 (?:Service unavailable\b|Client host rejected: cannot find your (reverse )?hostname\b) ^RCPT from [^[]*\[\]%(_port)s: 450 4\.7\.1 (<[^>]*>)?: Helo command rejected: Host not found\b ^EHLO from [^[]*\[\]%(_port)s: 504 5\.5\.2 (<[^>]*>)?: Helo command rejected: need fully-qualified hostname\b - ^VRFY from [^[]*\[\]%(_port)s: 550 5\.1\.1\s + ^(RCPT|VRFY) from [^[]*\[\]%(_port)s: 550 5\.1\.1\s ^RCPT from [^[]*\[\]%(_port)s: 450 4\.1\.8 (<[^>]*>)?: Sender address rejected: Domain not found\b ^from [^[]*\[\]%(_port)s:? diff --git a/fail2ban/tests/files/logs/postfix b/fail2ban/tests/files/logs/postfix index b56619a4..51e26c8c 100644 --- a/fail2ban/tests/files/logs/postfix +++ b/fail2ban/tests/files/logs/postfix @@ -12,7 +12,8 @@ Jul 18 23:12:56 xxx postfix/smtpd[8738]: NOQUEUE: reject: RCPT from foo[192.51.1 Jul 18 23:12:56 xxx postfix/smtpd[8738]: NOQUEUE: reject: RCPT from foo[192.51.100.43]: 554 5.7.1 : Sender address rejected: match bad.domain; from= to= proto=SMTP helo=<192.51.100.43> # failJSON: { "time": "2005-08-10T10:55:38", "match": true , "host": "72.53.132.234" } Aug 10 10:55:38 f-vanier-bourgeois postfix/smtpd[2162]: NOQUEUE: reject: VRFY from 72-53-132-234.cpe.distributel.net[72.53.132.234]: 550 5.1.1 : Recipient address rejected: User unknown in local recipient tab - +# failJSON: { "time": "2005-08-13T15:45:46", "match": true , "host": "192.0.2.1" } +Aug 13 15:45:46 server postfix/smtpd[13844]: 00ADB3C0899: reject: RCPT from example.com[192.0.2.1]: 550 5.1.1 : Recipient address rejected: User unknown in local recipient table; from= to= proto=ESMTP helo= # failJSON: { "time": "2005-01-12T11:07:49", "match": true , "host": "181.21.131.88" } Jan 12 11:07:49 emf1pt2-2-35-70 postfix/smtpd[13767]: improper command pipelining after DATA from unknown[181.21.131.88]: From 3036ed18893b6aae6619e53201aa53deb701b94f Mon Sep 17 00:00:00 2001 From: sebres Date: Tue, 14 May 2019 15:44:20 +0200 Subject: [PATCH 2/5] resolve deprecated syntax (eliminate "invalid sequence" warnings) --- fail2ban/server/datedetector.py | 34 ++++++------ fail2ban/server/datetemplate.py | 2 +- fail2ban/tests/actiontestcase.py | 6 +- fail2ban/tests/datedetectortestcase.py | 4 +- fail2ban/tests/fail2banclienttestcase.py | 6 +- fail2ban/tests/fail2banregextestcase.py | 16 +++--- fail2ban/tests/filtertestcase.py | 70 ++++++++++++------------ fail2ban/tests/servertestcase.py | 22 ++++---- 8 files changed, 80 insertions(+), 80 deletions(-) diff --git a/fail2ban/server/datedetector.py b/fail2ban/server/datedetector.py index 2e85b940..5942e3e0 100644 --- a/fail2ban/server/datedetector.py +++ b/fail2ban/server/datedetector.py @@ -128,52 +128,52 @@ class DateDetectorCache(object): # 2005-01-23T21:59:59.981746, 2005-01-23 21:59:59, 2005-01-23 8:59:59 # simple date: 2005/01/23 21:59:59 # custom for syslog-ng 2006.12.21 06:43:20 - "%ExY(?P<_sep>[-/.])%m(?P=_sep)%d(?:T| ?)%H:%M:%S(?:[.,]%f)?(?:\s*%z)?", + r"%ExY(?P<_sep>[-/.])%m(?P=_sep)%d(?:T| ?)%H:%M:%S(?:[.,]%f)?(?:\s*%z)?", # asctime with optional day, subsecond and/or year: # Sun Jan 23 21:59:59.011 2005 - "(?:%a )?%b %d %k:%M:%S(?:\.%f)?(?: %ExY)?", + r"(?:%a )?%b %d %k:%M:%S(?:\.%f)?(?: %ExY)?", # asctime with optional day, subsecond and/or year coming after day # http://bugs.debian.org/798923 # Sun Jan 23 2005 21:59:59.011 - "(?:%a )?%b %d %ExY %k:%M:%S(?:\.%f)?", + r"(?:%a )?%b %d %ExY %k:%M:%S(?:\.%f)?", # simple date too (from x11vnc): 23/01/2005 21:59:59 # and with optional year given by 2 digits: 23/01/05 21:59:59 # (See http://bugs.debian.org/537610) # 17-07-2008 17:23:25 - "%d(?P<_sep>[-/])%m(?P=_sep)(?:%ExY|%Exy) %k:%M:%S", + r"%d(?P<_sep>[-/])%m(?P=_sep)(?:%ExY|%Exy) %k:%M:%S", # Apache format optional time zone: # [31/Oct/2006:09:22:55 -0000] # 26-Jul-2007 15:20:52 # named 26-Jul-2007 15:20:52.252 # roundcube 26-Jul-2007 15:20:52 +0200 - "%d(?P<_sep>[-/])%b(?P=_sep)%ExY[ :]?%H:%M:%S(?:\.%f)?(?: %z)?", + r"%d(?P<_sep>[-/])%b(?P=_sep)%ExY[ :]?%H:%M:%S(?:\.%f)?(?: %z)?", # CPanel 05/20/2008:01:57:39 - "%m/%d/%ExY:%H:%M:%S", + r"%m/%d/%ExY:%H:%M:%S", # 01-27-2012 16:22:44.252 # subseconds explicit to avoid possible %m<->%d confusion # with previous ("%d-%m-%ExY %k:%M:%S" by "%d(?P<_sep>[-/])%m(?P=_sep)(?:%ExY|%Exy) %k:%M:%S") - "%m-%d-%ExY %k:%M:%S(?:\.%f)?", + r"%m-%d-%ExY %k:%M:%S(?:\.%f)?", # Epoch - "EPOCH", + r"EPOCH", # Only time information in the log - "{^LN-BEG}%H:%M:%S", + r"{^LN-BEG}%H:%M:%S", # <09/16/08@05:03:30> - "^<%m/%d/%Exy@%H:%M:%S>", + r"^<%m/%d/%Exy@%H:%M:%S>", # MySQL: 130322 11:46:11 - "%Exy%Exm%Exd ?%H:%M:%S", + r"%Exy%Exm%Exd ?%H:%M:%S", # Apache Tomcat - "%b %d, %ExY %I:%M:%S %p", + r"%b %d, %ExY %I:%M:%S %p", # ASSP: Apr-27-13 02:33:06 - "^%b-%d-%Exy %k:%M:%S", + r"^%b-%d-%Exy %k:%M:%S", # 20050123T215959, 20050123 215959, 20050123 85959 - "%ExY%Exm%Exd(?:T| ?)%ExH%ExM%ExS(?:[.,]%f)?(?:\s*%z)?", + r"%ExY%Exm%Exd(?:T| ?)%ExH%ExM%ExS(?:[.,]%f)?(?:\s*%z)?", # prefixed with optional named time zone (monit): # PDT Apr 16 21:05:29 - "(?:%Z )?(?:%a )?%b %d %k:%M:%S(?:\.%f)?(?: %ExY)?", + r"(?:%Z )?(?:%a )?%b %d %k:%M:%S(?:\.%f)?(?: %ExY)?", # +00:00 Jan 23 21:59:59.011 2005 - "(?:%z )?(?:%a )?%b %d %k:%M:%S(?:\.%f)?(?: %ExY)?", + r"(?:%z )?(?:%a )?%b %d %k:%M:%S(?:\.%f)?(?: %ExY)?", # TAI64N - "TAI64N", + r"TAI64N", ] @property diff --git a/fail2ban/server/datetemplate.py b/fail2ban/server/datetemplate.py index d827cd1a..973a8a51 100644 --- a/fail2ban/server/datetemplate.py +++ b/fail2ban/server/datetemplate.py @@ -82,7 +82,7 @@ class DateTemplate(object): return self._regex def setRegex(self, regex, wordBegin=True, wordEnd=True): - """Sets regex to use for searching for date in log line. + r"""Sets regex to use for searching for date in log line. Parameters ---------- diff --git a/fail2ban/tests/actiontestcase.py b/fail2ban/tests/actiontestcase.py index 6df54e13..1c5a0807 100644 --- a/fail2ban/tests/actiontestcase.py +++ b/fail2ban/tests/actiontestcase.py @@ -206,15 +206,15 @@ class CommandActionTest(LogCaptureTestCase): "Text 890 text 123 ABC") self.assertEqual( self.__action.replaceTag("", - {'matches': "some >char< should \< be[ escap}ed&\n"}), + {'matches': "some >char< should \\< be[ escap}ed&\n"}), "some \\>char\\< should \\\\\\< be\\[ escap\\}ed\\&\n") self.assertEqual( self.__action.replaceTag("", - {'ipmatches': "some >char< should \< be[ escap}ed&\n"}), + {'ipmatches': "some >char< should \\< be[ escap}ed&\n"}), "some \\>char\\< should \\\\\\< be\\[ escap\\}ed\\&\n") self.assertEqual( self.__action.replaceTag("", - {'ipjailmatches': "some >char< should \< be[ escap}ed&\n"}), + {'ipjailmatches': "some >char< should \\< be[ escap}ed&\n"}), "some \\>char\\< should \\\\\\< be\\[ escap\\}ed\\&\n") # Recursive diff --git a/fail2ban/tests/datedetectortestcase.py b/fail2ban/tests/datedetectortestcase.py index 36471489..458f76ef 100644 --- a/fail2ban/tests/datedetectortestcase.py +++ b/fail2ban/tests/datedetectortestcase.py @@ -103,7 +103,7 @@ class DateDetectorTest(LogCaptureTestCase): def testGetEpochPattern(self): self.__datedetector = DateDetector() - self.__datedetector.appendTemplate('(?<=\|\s){LEPOCH}(?=\s\|)') + self.__datedetector.appendTemplate(r'(?<=\|\s){LEPOCH}(?=\s\|)') # correct short/long epoch time, using all variants: for fact in (1, 1000, 1000000): for dateUnix in (1138049999, 32535244799): @@ -385,7 +385,7 @@ class DateDetectorTest(LogCaptureTestCase): self.assertRaises(Exception, t.getDate, 'no date line') -iso8601 = DatePatternRegex("%Y-%m-%d[T ]%H:%M:%S(?:\.%f)?%z") +iso8601 = DatePatternRegex(r"%Y-%m-%d[T ]%H:%M:%S(?:\.%f)?%z") class CustomDateFormatsTest(unittest.TestCase): diff --git a/fail2ban/tests/fail2banclienttestcase.py b/fail2ban/tests/fail2banclienttestcase.py index 12b092a8..2542f314 100644 --- a/fail2ban/tests/fail2banclienttestcase.py +++ b/fail2ban/tests/fail2banclienttestcase.py @@ -835,7 +835,7 @@ class Fail2banServerTest(Fail2banClientServerBase): "usedns = no", "maxretry = 3", "findtime = 10m", - "failregex = ^\s*failure 401|403 from ", + r"failregex = ^\s*failure 401|403 from ", "datepattern = {^LN-BEG}EPOCH", "ignoreip = 127.0.0.1/8 ::1", # just to cover ignoreip in jailreader/transmitter "", @@ -851,8 +851,8 @@ class Fail2banServerTest(Fail2banClientServerBase): "logpath = " + test1log, " " + test2log if 2 in enabled else "", " " + test3log if 2 in enabled else "", - "failregex = ^\s*failure 401|403 from ", - " ^\s*error 401|403 from " \ + r"failregex = ^\s*failure 401|403 from ", + r" ^\s*error 401|403 from " \ if 2 in enabled else "", "enabled = true" if 1 in enabled else "", "", diff --git a/fail2ban/tests/fail2banregextestcase.py b/fail2ban/tests/fail2banregextestcase.py index 8d6df537..c751b13d 100644 --- a/fail2ban/tests/fail2banregextestcase.py +++ b/fail2ban/tests/fail2banregextestcase.py @@ -120,7 +120,7 @@ class Fail2banRegexTest(LogCaptureTestCase): def testDirectFound(self): (opts, args, fail2banRegex) = _Fail2banRegex( - "--datepattern", "^(?:%a )?%b %d %H:%M:%S(?:\.%f)?(?: %ExY)?", + "--datepattern", r"^(?:%a )?%b %d %H:%M:%S(?:\.%f)?(?: %ExY)?", "--print-all-matched", "--print-no-missed", "Dec 31 11:59:59 [sshd] error: PAM: Authentication failure for kevin from 192.0.2.0", r"Authentication failure for .*? from $" @@ -149,7 +149,7 @@ class Fail2banRegexTest(LogCaptureTestCase): def testDirectRE_1(self): (opts, args, fail2banRegex) = _Fail2banRegex( - "--datepattern", "^(?:%a )?%b %d %H:%M:%S(?:\.%f)?(?: %ExY)?", + "--datepattern", r"^(?:%a )?%b %d %H:%M:%S(?:\.%f)?(?: %ExY)?", "--print-all-matched", Fail2banRegexTest.FILENAME_01, Fail2banRegexTest.RE_00 @@ -165,7 +165,7 @@ class Fail2banRegexTest(LogCaptureTestCase): def testDirectRE_1raw(self): (opts, args, fail2banRegex) = _Fail2banRegex( - "--datepattern", "^(?:%a )?%b %d %H:%M:%S(?:\.%f)?(?: %ExY)?", + "--datepattern", r"^(?:%a )?%b %d %H:%M:%S(?:\.%f)?(?: %ExY)?", "--print-all-matched", "--raw", Fail2banRegexTest.FILENAME_01, Fail2banRegexTest.RE_00 @@ -175,7 +175,7 @@ class Fail2banRegexTest(LogCaptureTestCase): def testDirectRE_1raw_noDns(self): (opts, args, fail2banRegex) = _Fail2banRegex( - "--datepattern", "^(?:%a )?%b %d %H:%M:%S(?:\.%f)?(?: %ExY)?", + "--datepattern", r"^(?:%a )?%b %d %H:%M:%S(?:\.%f)?(?: %ExY)?", "--print-all-matched", "--raw", "--usedns=no", Fail2banRegexTest.FILENAME_01, Fail2banRegexTest.RE_00 @@ -185,7 +185,7 @@ class Fail2banRegexTest(LogCaptureTestCase): def testDirectRE_2(self): (opts, args, fail2banRegex) = _Fail2banRegex( - "--datepattern", "^(?:%a )?%b %d %H:%M:%S(?:\.%f)?(?: %ExY)?", + "--datepattern", r"^(?:%a )?%b %d %H:%M:%S(?:\.%f)?(?: %ExY)?", "--print-all-matched", Fail2banRegexTest.FILENAME_02, Fail2banRegexTest.RE_00 @@ -195,7 +195,7 @@ class Fail2banRegexTest(LogCaptureTestCase): def testVerbose(self): (opts, args, fail2banRegex) = _Fail2banRegex( - "--datepattern", "^(?:%a )?%b %d %H:%M:%S(?:\.%f)?(?: %ExY)?", + "--datepattern", r"^(?:%a )?%b %d %H:%M:%S(?:\.%f)?(?: %ExY)?", "--timezone", "UTC+0200", "--verbose", "--verbose-date", "--print-no-missed", Fail2banRegexTest.FILENAME_02, @@ -332,7 +332,7 @@ class Fail2banRegexTest(LogCaptureTestCase): self._reset() (opts, args, fail2banRegex) = _Fail2banRegex( "-l", "notice", # put down log-level, because of too many debug-messages - "--datepattern", "^(?:%a )?%b %d %H:%M:%S(?:\.%f)?(?: %ExY)?", + "--datepattern", r"^(?:%a )?%b %d %H:%M:%S(?:\.%f)?(?: %ExY)?", Fail2banRegexTest.FILENAME_WRONGCHAR, Fail2banRegexTest.FILTER_SSHD ) self.assertTrue(fail2banRegex.start(args)) @@ -349,7 +349,7 @@ class Fail2banRegexTest(LogCaptureTestCase): self._reset() (opts, args, fail2banRegex) = _Fail2banRegex( "-l", "notice", # put down log-level, because of too many debug-messages - "--datepattern", "^(?:%a )?%b %d %H:%M:%S(?:\.%f)?(?: %ExY)?", + "--datepattern", r"^(?:%a )?%b %d %H:%M:%S(?:\.%f)?(?: %ExY)?", "--debuggex", "--print-all-matched", Fail2banRegexTest.FILENAME_WRONGCHAR, Fail2banRegexTest.FILTER_SSHD, r"llinco[^\\]" diff --git a/fail2ban/tests/filtertestcase.py b/fail2ban/tests/filtertestcase.py index fb499413..6a96080b 100644 --- a/fail2ban/tests/filtertestcase.py +++ b/fail2ban/tests/filtertestcase.py @@ -279,10 +279,10 @@ class BasicFilter(unittest.TestCase): def testGetSetDatePattern(self): self.assertEqual(self.filter.getDatePattern(), (None, "Default Detectors")) - self.filter.setDatePattern("^%Y-%m-%d-%H%M%S.%f %z **") + self.filter.setDatePattern(r"^%Y-%m-%d-%H%M%S\.%f %z **") self.assertEqual(self.filter.getDatePattern(), - ("^%Y-%m-%d-%H%M%S.%f %z **", - "^Year-Month-Day-24hourMinuteSecond.Microseconds Zone offset **")) + (r"^%Y-%m-%d-%H%M%S\.%f %z **", + r"^Year-Month-Day-24hourMinuteSecond\.Microseconds Zone offset **")) def testGetSetLogTimeZone(self): self.assertEqual(self.filter.getLogTimeZone(), None) @@ -389,7 +389,7 @@ class IgnoreIP(LogCaptureTestCase): setUpMyTime() self.filter.addIgnoreIP('192.168.1.0/25') self.filter.addFailRegex('') - self.filter.setDatePattern('{^LN-BEG}EPOCH') + self.filter.setDatePattern(r'{^LN-BEG}EPOCH') self.filter.processLineAndAdd('1387203300.222 192.168.1.32') self.assertLogged('Ignore 192.168.1.32') tearDownMyTime() @@ -580,7 +580,7 @@ class LogFileFilterPoll(unittest.TestCase): def testSeekToTimeSmallFile(self): # speedup search using exact date pattern: - self.filter.setDatePattern('^%ExY-%Exm-%Exd %ExH:%ExM:%ExS') + self.filter.setDatePattern(r'^%ExY-%Exm-%Exd %ExH:%ExM:%ExS') fname = tempfile.mktemp(prefix='tmp_fail2ban', suffix='.log') time = 1417512352 f = open(fname, 'w') @@ -666,7 +666,7 @@ class LogFileFilterPoll(unittest.TestCase): def testSeekToTimeLargeFile(self): # speedup search using exact date pattern: - self.filter.setDatePattern('^%ExY-%Exm-%Exd %ExH:%ExM:%ExS') + self.filter.setDatePattern(r'^%ExY-%Exm-%Exd %ExH:%ExM:%ExS') fname = tempfile.mktemp(prefix='tmp_fail2ban', suffix='.log') time = 1417512352 f = open(fname, 'w') @@ -723,7 +723,7 @@ class LogFileMonitor(LogCaptureTestCase): self.filter = FilterPoll(DummyJail()) self.filter.addLogPath(self.name, autoSeek=False) self.filter.active = True - self.filter.addFailRegex("(?:(?:Authentication failure|Failed [-/\w+]+) for(?: [iI](?:llegal|nvalid) user)?|[Ii](?:llegal|nvalid) user|ROOT LOGIN REFUSED) .*(?: from|FROM) ") + self.filter.addFailRegex(r"(?:(?:Authentication failure|Failed [-/\w+]+) for(?: [iI](?:llegal|nvalid) user)?|[Ii](?:llegal|nvalid) user|ROOT LOGIN REFUSED) .*(?: from|FROM) ") def tearDown(self): tearDownMyTime() @@ -765,7 +765,7 @@ class LogFileMonitor(LogCaptureTestCase): def testErrorProcessLine(self): # speedup search using exact date pattern: - self.filter.setDatePattern('^%ExY-%Exm-%Exd %ExH:%ExM:%ExS') + self.filter.setDatePattern(r'^%ExY-%Exm-%Exd %ExH:%ExM:%ExS') self.filter.sleeptime /= 1000.0 ## produce error with not callable processLine: _org_processLine = self.filter.processLine @@ -829,7 +829,7 @@ class LogFileMonitor(LogCaptureTestCase): def testNewChangeViaGetFailures_simple(self): # speedup search using exact date pattern: - self.filter.setDatePattern('^(?:%a )?%b %d %H:%M:%S(?:\.%f)?(?: %ExY)?') + self.filter.setDatePattern(r'^(?:%a )?%b %d %H:%M:%S(?:\.%f)?(?: %ExY)?') # suck in lines from this sample log file self.filter.getFailures(self.name) self.assertRaises(FailManagerEmpty, self.filter.failManager.toBan) @@ -846,7 +846,7 @@ class LogFileMonitor(LogCaptureTestCase): def testNewChangeViaGetFailures_rewrite(self): # speedup search using exact date pattern: - self.filter.setDatePattern('^(?:%a )?%b %d %H:%M:%S(?:\.%f)?(?: %ExY)?') + self.filter.setDatePattern(r'^(?:%a )?%b %d %H:%M:%S(?:\.%f)?(?: %ExY)?') # # if we rewrite the file at once self.file.close() @@ -866,7 +866,7 @@ class LogFileMonitor(LogCaptureTestCase): def testNewChangeViaGetFailures_move(self): # speedup search using exact date pattern: - self.filter.setDatePattern('^(?:%a )?%b %d %H:%M:%S(?:\.%f)?(?: %ExY)?') + self.filter.setDatePattern(r'^(?:%a )?%b %d %H:%M:%S(?:\.%f)?(?: %ExY)?') # # if we move file into a new location while it has been open already self.file.close() @@ -940,9 +940,9 @@ def get_monitor_failures_testcase(Filter_): self.filter = Filter_(self.jail) self.filter.addLogPath(self.name, autoSeek=False) # speedup search using exact date pattern: - self.filter.setDatePattern('^(?:%a )?%b %d %H:%M:%S(?:\.%f)?(?: %ExY)?') + self.filter.setDatePattern(r'^(?:%a )?%b %d %H:%M:%S(?:\.%f)?(?: %ExY)?') self.filter.active = True - self.filter.addFailRegex("(?:(?:Authentication failure|Failed [-/\w+]+) for(?: [iI](?:llegal|nvalid) user)?|[Ii](?:llegal|nvalid) user|ROOT LOGIN REFUSED) .*(?: from|FROM) ") + self.filter.addFailRegex(r"(?:(?:Authentication failure|Failed [-/\w+]+) for(?: [iI](?:llegal|nvalid) user)?|[Ii](?:llegal|nvalid) user|ROOT LOGIN REFUSED) .*(?: from|FROM) ") self.filter.start() # If filter is polling it would sleep a bit to guarantee that # we have initial time-stamp difference to trigger "actions" @@ -1244,7 +1244,7 @@ def get_monitor_failures_journal_testcase(Filter_): # pragma: systemd no cover "SYSLOG_IDENTIFIER=fail2ban-testcases", "TEST_FIELD=2", "TEST_UUID=%s" % self.test_uuid]) - self.filter.addFailRegex("(?:(?:Authentication failure|Failed [-/\w+]+) for(?: [iI](?:llegal|nvalid) user)?|[Ii](?:llegal|nvalid) user|ROOT LOGIN REFUSED) .*(?: from|FROM) ") + self.filter.addFailRegex(r"(?:(?:Authentication failure|Failed [-/\w+]+) for(?: [iI](?:llegal|nvalid) user)?|[Ii](?:llegal|nvalid) user|ROOT LOGIN REFUSED) .*(?: from|FROM) ") def tearDown(self): if self.filter and self.filter.active: @@ -1440,7 +1440,7 @@ class GetFailures(LogCaptureTestCase): self.filter = FileFilter(self.jail) self.filter.active = True # speedup search using exact date pattern: - self.filter.setDatePattern('^(?:%a )?%b %d %H:%M:%S(?:\.%f)?(?: %ExY)?') + self.filter.setDatePattern(r'^(?:%a )?%b %d %H:%M:%S(?:\.%f)?(?: %ExY)?') # TODO Test this #self.filter.setTimeRegex("\S{3}\s{1,2}\d{1,2} \d{2}:\d{2}:\d{2}") #self.filter.setTimePattern("%b %d %H:%M:%S") @@ -1485,7 +1485,7 @@ class GetFailures(LogCaptureTestCase): failures = failures or GetFailures.FAILURES_01 self.filter.addLogPath(filename, autoSeek=0) - self.filter.addFailRegex("(?:(?:Authentication failure|Failed [-/\w+]+) for(?: [iI](?:llegal|nvalid) user)?|[Ii](?:llegal|nvalid) user|ROOT LOGIN REFUSED) .*(?: from|FROM) $") + self.filter.addFailRegex(r"(?:(?:Authentication failure|Failed [-/\w+]+) for(?: [iI](?:llegal|nvalid) user)?|[Ii](?:llegal|nvalid) user|ROOT LOGIN REFUSED) .*(?: from|FROM) $") self.filter.getFailures(filename) _assert_correct_last_attempt(self, self.filter, failures) @@ -1509,7 +1509,7 @@ class GetFailures(LogCaptureTestCase): % m for m in 53, 54, 57, 58]) self.filter.addLogPath(GetFailures.FILENAME_02, autoSeek=0) - self.filter.addFailRegex("Failed .* from ") + self.filter.addFailRegex(r"Failed .* from ") self.filter.getFailures(GetFailures.FILENAME_02) _assert_correct_last_attempt(self, self.filter, output) @@ -1517,7 +1517,7 @@ class GetFailures(LogCaptureTestCase): output = ('203.162.223.135', 7, 1124013544.0) self.filter.addLogPath(GetFailures.FILENAME_03, autoSeek=0) - self.filter.addFailRegex("error,relay=,.*550 User unknown") + self.filter.addFailRegex(r"error,relay=,.*550 User unknown") self.filter.getFailures(GetFailures.FILENAME_03) _assert_correct_last_attempt(self, self.filter, output) @@ -1526,7 +1526,7 @@ class GetFailures(LogCaptureTestCase): output = ('203.162.223.135', 5, 1124013544.0) self.filter.addLogPath(GetFailures.FILENAME_03, autoSeek=output[2] - 4*60) - self.filter.addFailRegex("error,relay=,.*550 User unknown") + self.filter.addFailRegex(r"error,relay=,.*550 User unknown") self.filter.getFailures(GetFailures.FILENAME_03) _assert_correct_last_attempt(self, self.filter, output) @@ -1536,7 +1536,7 @@ class GetFailures(LogCaptureTestCase): self.filter.setMaxRetry(1) self.filter.addLogPath(GetFailures.FILENAME_03, autoSeek=output[2]) - self.filter.addFailRegex("error,relay=,.*550 User unknown") + self.filter.addFailRegex(r"error,relay=,.*550 User unknown") self.filter.getFailures(GetFailures.FILENAME_03) _assert_correct_last_attempt(self, self.filter, output) @@ -1548,13 +1548,13 @@ class GetFailures(LogCaptureTestCase): ('212.41.96.185', 2, 1124013598.0)) # speedup search using exact date pattern: - self.filter.setDatePattern(('^%ExY(?P<_sep>[-/.])%m(?P=_sep)%d[T ]%H:%M:%S(?:[.,]%f)?(?:\s*%z)?', - '^(?:%a )?%b %d %H:%M:%S(?:\.%f)?(?: %ExY)?', - '^EPOCH' + self.filter.setDatePattern((r'^%ExY(?P<_sep>[-/.])%m(?P=_sep)%d[T ]%H:%M:%S(?:[.,]%f)?(?:\s*%z)?', + r'^(?:%a )?%b %d %H:%M:%S(?:\.%f)?(?: %ExY)?', + r'^EPOCH' )) self.filter.setMaxRetry(2) self.filter.addLogPath(GetFailures.FILENAME_04, autoSeek=0) - self.filter.addFailRegex("Invalid user .* ") + self.filter.addFailRegex(r"Invalid user .* ") self.filter.getFailures(GetFailures.FILENAME_04) _assert_correct_last_attempt(self, self.filter, output) @@ -1574,7 +1574,7 @@ class GetFailures(LogCaptureTestCase): fout.close() # output = ('192.0.2.0', 3, 1421262060.0) - failregex = "^\s*user \"[^\"]*\" from \"\"\s*$" + failregex = r"^\s*user \"[^\"]*\" from \"\"\s*$" # test encoding auto or direct set of encoding: for enc in (None, 'utf-8', 'ascii'): @@ -1582,7 +1582,7 @@ class GetFailures(LogCaptureTestCase): self.tearDown();self.setUp(); self.filter.setLogEncoding(enc); # speedup search using exact date pattern: - self.filter.setDatePattern('^%ExY-%Exm-%Exd %ExH:%ExM:%ExS') + self.filter.setDatePattern(r'^%ExY-%Exm-%Exd %ExH:%ExM:%ExS') self.assertNotLogged('Error decoding line'); self.filter.addLogPath(fname) self.filter.addFailRegex(failregex) @@ -1631,7 +1631,7 @@ class GetFailures(LogCaptureTestCase): filter_.failManager.setMaxRetry(1) # we might have just few failures filter_.addLogPath(GetFailures.FILENAME_USEDNS, autoSeek=False) - filter_.addFailRegex("Failed .* from ") + filter_.addFailRegex(r"Failed .* from ") filter_.getFailures(GetFailures.FILENAME_USEDNS) _assert_correct_last_attempt(self, filter_, output) @@ -1639,15 +1639,15 @@ class GetFailures(LogCaptureTestCase): output = ('141.3.81.106', 8, 1124013541.0) self.filter.addLogPath(GetFailures.FILENAME_02, autoSeek=False) - self.filter.addFailRegex("Failed .* from ") - self.filter.addFailRegex("Accepted .* from ") + self.filter.addFailRegex(r"Failed .* from ") + self.filter.addFailRegex(r"Accepted .* from ") self.filter.getFailures(GetFailures.FILENAME_02) _assert_correct_last_attempt(self, self.filter, output) def testGetFailuresIgnoreRegex(self): self.filter.addLogPath(GetFailures.FILENAME_02, autoSeek=False) - self.filter.addFailRegex("Failed .* from ") - self.filter.addFailRegex("Accepted .* from ") + self.filter.addFailRegex(r"Failed .* from ") + self.filter.addFailRegex(r"Accepted .* from ") self.filter.addIgnoreRegex("for roehl") self.filter.getFailures(GetFailures.FILENAME_02) @@ -1659,7 +1659,7 @@ class GetFailures(LogCaptureTestCase): ("192.0.43.11", 1, 1124013598.0)] self.filter.addLogPath(GetFailures.FILENAME_MULTILINE, autoSeek=False) self.filter.setMaxLines(100) - self.filter.addFailRegex("^.*rsyncd\[(?P\d+)\]: connect from .+ \(\)$^.+ rsyncd\[(?P=pid)\]: rsync error: .*$") + self.filter.addFailRegex(r"^.*rsyncd\[(?P\d+)\]: connect from .+ \(\)$^.+ rsyncd\[(?P=pid)\]: rsync error: .*$") self.filter.setMaxRetry(1) self.filter.getFailures(GetFailures.FILENAME_MULTILINE) @@ -1677,7 +1677,7 @@ class GetFailures(LogCaptureTestCase): output = [("192.0.43.10", 2, 1124013599.0)] self.filter.addLogPath(GetFailures.FILENAME_MULTILINE, autoSeek=False) self.filter.setMaxLines(100) - self.filter.addFailRegex("^.*rsyncd\[(?P\d+)\]: connect from .+ \(\)$^.+ rsyncd\[(?P=pid)\]: rsync error: .*$") + self.filter.addFailRegex(r"^.*rsyncd\[(?P\d+)\]: connect from .+ \(\)$^.+ rsyncd\[(?P=pid)\]: rsync error: .*$") self.filter.addIgnoreRegex("rsync error: Received SIGINT") self.filter.setMaxRetry(1) @@ -1693,8 +1693,8 @@ class GetFailures(LogCaptureTestCase): ("192.0.43.15", 1, 1124013598.0)] self.filter.addLogPath(GetFailures.FILENAME_MULTILINE, autoSeek=False) self.filter.setMaxLines(100) - self.filter.addFailRegex("^.*rsyncd\[(?P\d+)\]: connect from .+ \(\)$^.+ rsyncd\[(?P=pid)\]: rsync error: .*$") - self.filter.addFailRegex("^.* sendmail\[.*, msgid=<(?P[^>]+).*relay=\[\].*$^.+ spamd: result: Y \d+ .*,mid=<(?P=msgid)>(,bayes=[.\d]+)?(,autolearn=\S+)?\s*$") + self.filter.addFailRegex(r"^.*rsyncd\[(?P\d+)\]: connect from .+ \(\)$^.+ rsyncd\[(?P=pid)\]: rsync error: .*$") + self.filter.addFailRegex(r"^.* sendmail\[.*, msgid=<(?P[^>]+).*relay=\[\].*$^.+ spamd: result: Y \d+ .*,mid=<(?P=msgid)>(,bayes=[.\d]+)?(,autolearn=\S+)?\s*$") self.filter.setMaxRetry(1) self.filter.getFailures(GetFailures.FILENAME_MULTILINE) diff --git a/fail2ban/tests/servertestcase.py b/fail2ban/tests/servertestcase.py index c1c5cc10..305105a4 100644 --- a/fail2ban/tests/servertestcase.py +++ b/fail2ban/tests/servertestcase.py @@ -1017,28 +1017,28 @@ class RegexTests(unittest.TestCase): def testHost(self): self.assertRaises(RegexException, FailRegex, '') self.assertRaises(RegexException, FailRegex, '^test no group$') - self.assertTrue(FailRegex('^test group$')) - self.assertTrue(FailRegex('^test group$')) - self.assertTrue(FailRegex('^test group$')) - self.assertTrue(FailRegex('^test group$')) - self.assertTrue(FailRegex('^test id group: ip:port = (?::)?$')) - self.assertTrue(FailRegex('^test id group: user:\([^\)]+\)$')) - self.assertTrue(FailRegex('^test id group: anything = $')) + self.assertTrue(FailRegex(r'^test group$')) + self.assertTrue(FailRegex(r'^test group$')) + self.assertTrue(FailRegex(r'^test group$')) + self.assertTrue(FailRegex(r'^test group$')) + self.assertTrue(FailRegex(r'^test id group: ip:port = (?::)?$')) + self.assertTrue(FailRegex(r'^test id group: user:\([^\)]+\)$')) + self.assertTrue(FailRegex(r'^test id group: anything = $')) # Testing obscure case when host group might be missing in the matched pattern, # e.g. if we made it optional. - fr = FailRegex('%%?') + fr = FailRegex(r'%%?') self.assertFalse(fr.hasMatched()) fr.search([('%%',"","")]) self.assertTrue(fr.hasMatched()) self.assertRaises(RegexException, fr.getHost) # The same as above but using separated IPv4/IPv6 expressions - fr = FailRegex('%%inet(?:=|inet6=)?') + fr = FailRegex(r'%%inet(?:=|inet6=)?') self.assertFalse(fr.hasMatched()) fr.search([('%%inet=test',"","")]) self.assertTrue(fr.hasMatched()) self.assertRaises(RegexException, fr.getHost) # Success case: using separated IPv4/IPv6 expressions (no HOST) - fr = FailRegex('%%(?:inet(?:=|6=)?|dns=?)') + fr = FailRegex(r'%%(?:inet(?:=|6=)?|dns=?)') self.assertFalse(fr.hasMatched()) fr.search([('%%inet=192.0.2.1',"","")]) self.assertTrue(fr.hasMatched()) @@ -1050,7 +1050,7 @@ class RegexTests(unittest.TestCase): self.assertTrue(fr.hasMatched()) self.assertEqual(fr.getHost(), 'example.com') # Success case: using user as failure-id - fr = FailRegex('^test id group: user:\([^\)]+\)$') + fr = FailRegex(r'^test id group: user:\([^\)]+\)$') self.assertFalse(fr.hasMatched()) fr.search([('test id group: user:(test login name)',"","")]) self.assertTrue(fr.hasMatched()) From d310c4992ff30b266b6bcddd912527d681e71d52 Mon Sep 17 00:00:00 2001 From: sebres Date: Tue, 14 May 2019 18:24:45 +0200 Subject: [PATCH 3/5] .travis.yml: coverage for python 3.8 (test with newest python/pypy versions) --- .travis.yml | 30 +++++++++++++++++------------- 1 file changed, 17 insertions(+), 13 deletions(-) diff --git a/.travis.yml b/.travis.yml index bd095598..0dfaa5cb 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,21 +1,20 @@ # vim ft=yaml # travis-ci.org definition for Fail2Ban build # https://travis-ci.org/fail2ban/fail2ban/ +dist: xenial # required for Python >= 3.7 language: python python: - - 2.6 + #- 2.6 - 2.7 - pypy - # disabled until coverage module fixes up compatibility issue # - 3.2 - - 3.3 + # - 3.3 - 3.4 - 3.5 - 3.6 - - 3.7-dev - # disabled since setuptools dropped support for Python 3.0 - 3.2 - # - pypy3 - - pypy3.3-5.5-alpha + - 3.7 + - 3.8-dev + - pypy3.5 before_install: - echo "running under $TRAVIS_PYTHON_VERSION" - if [[ $TRAVIS_PYTHON_VERSION == 2* || $TRAVIS_PYTHON_VERSION == pypy* && $TRAVIS_PYTHON_VERSION != pypy3* ]]; then export F2B_PY=2; fi @@ -35,17 +34,22 @@ install: # codecov: - travis_retry pip install codecov # dnspython or dnspython3 - - if [[ "$F2B_PY" = 2 ]]; then travis_retry pip install dnspython; fi - - if [[ "$F2B_PY" = 3 ]]; then travis_retry pip install dnspython3; fi + - if [[ "$F2B_PY" = 2 ]]; then travis_retry pip install dnspython || echo 'not installed'; fi + - if [[ "$F2B_PY" = 3 ]]; then travis_retry pip install dnspython3 || echo 'not installed'; fi + # python systemd bindings: + - if [[ "$F2B_PY" = 2 ]]; then travis_retry sudo apt-get python-systemd || echo 'not installed'; fi + - if [[ "$F2B_PY" = 3 ]]; then travis_retry sudo apt-get python3-systemd || echo 'not installed'; fi # gamin - install manually (not in PyPI) - travis-ci system Python is 2.7 - - if [[ $TRAVIS_PYTHON_VERSION == 2.7 ]]; then travis_retry sudo apt-get install -qq python-gamin && cp /usr/share/pyshared/gamin.py /usr/lib/pyshared/python2.7/_gamin.so $VIRTUAL_ENV/lib/python2.7/site-packages/; fi + - if [[ $TRAVIS_PYTHON_VERSION == 2.7 ]]; then (travis_retry sudo apt-get install -qq python-gamin && cp /usr/share/pyshared/gamin.py /usr/lib/pyshared/python2.7/_gamin.so $VIRTUAL_ENV/lib/python2.7/site-packages/) || echo 'not installed'; fi # pyinotify - - travis_retry pip install pyinotify + - travis_retry pip install pyinotify || echo 'not installed' # Install helper tools - sudo apt-get install shellcheck before_script: # Manually execute 2to3 for now - if [[ "$F2B_PY" = 3 ]]; then ./fail2ban-2to3; fi + # (debug) output current preferred encoding: + - python -c 'import locale, sys; from fail2ban.helpers import PREFER_ENC; print(PREFER_ENC, locale.getpreferredencoding(), (sys.stdout and sys.stdout.encoding))' script: # Keep the legacy setup.py test approach of checking coverage for python2 - if [[ "$F2B_PY" = 2 ]]; then coverage run setup.py test; fi @@ -53,8 +57,8 @@ script: - if [[ "$F2B_PY" = 3 ]]; then coverage run bin/fail2ban-testcases --verbosity=2; fi # Use $VENV_BIN (not python) or else sudo will always run the system's python (2.7) - sudo $VENV_BIN/pip install . - # Doc files should get installed on Travis under Linux - - test -e /usr/share/doc/fail2ban/FILTERS + # Doc files should get installed on Travis under Linux (python >= 3.8 seem to use another path segment) + - if [[ $TRAVIS_PYTHON_VERSION < 3.8 ]]; then test -e /usr/share/doc/fail2ban/FILTERS; fi # Test initd script - shellcheck -s bash -e SC1090,SC1091 files/debian-initd after_success: From 1cca374d0491b50483c0a797680d764730c817eb Mon Sep 17 00:00:00 2001 From: sebres Date: Tue, 14 May 2019 19:07:47 +0200 Subject: [PATCH 4/5] .travis.yml: several distributions in matrix (trusty & xenial together) --- .travis.yml | 53 +++++++++++++++++++++++++++++++++++++---------------- 1 file changed, 37 insertions(+), 16 deletions(-) diff --git a/.travis.yml b/.travis.yml index 0dfaa5cb..811ff6f1 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,20 +1,42 @@ # vim ft=yaml # travis-ci.org definition for Fail2Ban build # https://travis-ci.org/fail2ban/fail2ban/ -dist: xenial # required for Python >= 3.7 -language: python -python: - #- 2.6 - - 2.7 - - pypy - # - 3.2 - # - 3.3 - - 3.4 - - 3.5 - - 3.6 - - 3.7 - - 3.8-dev - - pypy3.5 + +#os: linux + +matrix: + fast_finish: true + include: + - dist: trusty # required for Python 2.6 + language: python + python: 2.6 + - dist: trusty + language: python + python: pypy + - dist: xenial + language: python + python: 2.7 + - dist: trusty + language: python + python: 3.3 + - dist: xenial + language: python + python: 3.4 + - dist: xenial + language: python + python: 3.5 + - dist: xenial + language: python + python: 3.6 + - dist: xenial + language: python + python: 3.7 + - dist: xenial + language: python + python: 3.8-dev + - dist: xenial + language: python + python: pypy3.5 before_install: - echo "running under $TRAVIS_PYTHON_VERSION" - if [[ $TRAVIS_PYTHON_VERSION == 2* || $TRAVIS_PYTHON_VERSION == pypy* && $TRAVIS_PYTHON_VERSION != pypy3* ]]; then export F2B_PY=2; fi @@ -64,8 +86,7 @@ script: after_success: - if [[ "$F2B_COV" = 1 ]]; then coveralls; fi - codecov -matrix: - fast_finish: true + # Might be worth looking into #notifications: # email: true From 08d261502002c44f4c517af693a6d314b6ef23b8 Mon Sep 17 00:00:00 2001 From: sebres Date: Tue, 14 May 2019 19:57:01 +0200 Subject: [PATCH 5/5] small amend: review, simplification, etc --- .travis.yml | 54 ++++++++++++++++++++++------------------------------- 1 file changed, 22 insertions(+), 32 deletions(-) diff --git a/.travis.yml b/.travis.yml index 811ff6f1..158cff99 100644 --- a/.travis.yml +++ b/.travis.yml @@ -4,39 +4,29 @@ #os: linux +language: python +dist: xenial + matrix: fast_finish: true include: - - dist: trusty # required for Python 2.6 - language: python - python: 2.6 - - dist: trusty - language: python - python: pypy - - dist: xenial - language: python - python: 2.7 - - dist: trusty - language: python - python: 3.3 - - dist: xenial - language: python - python: 3.4 - - dist: xenial - language: python - python: 3.5 - - dist: xenial - language: python - python: 3.6 - - dist: xenial - language: python - python: 3.7 - - dist: xenial - language: python - python: 3.8-dev - - dist: xenial - language: python - python: pypy3.5 + - python: 2.6 + dist: trusty # required for Python 2.6 + - python: 2.7 + dist: trusty # required for packages like gamin + name: 2.7 (trusty) + - python: 2.7 + name: 2.7 (xenial) + - python: pypy + dist: trusty + - python: 3.3 + dist: trusty + - python: 3.4 + - python: 3.5 + - python: 3.6 + - python: 3.7 + - python: 3.8-dev + - python: pypy3.5 before_install: - echo "running under $TRAVIS_PYTHON_VERSION" - if [[ $TRAVIS_PYTHON_VERSION == 2* || $TRAVIS_PYTHON_VERSION == pypy* && $TRAVIS_PYTHON_VERSION != pypy3* ]]; then export F2B_PY=2; fi @@ -59,8 +49,8 @@ install: - if [[ "$F2B_PY" = 2 ]]; then travis_retry pip install dnspython || echo 'not installed'; fi - if [[ "$F2B_PY" = 3 ]]; then travis_retry pip install dnspython3 || echo 'not installed'; fi # python systemd bindings: - - if [[ "$F2B_PY" = 2 ]]; then travis_retry sudo apt-get python-systemd || echo 'not installed'; fi - - if [[ "$F2B_PY" = 3 ]]; then travis_retry sudo apt-get python3-systemd || echo 'not installed'; fi + - if [[ "$F2B_PY" = 2 ]]; then travis_retry sudo apt-get install -qq python-systemd || echo 'not installed'; fi + - if [[ "$F2B_PY" = 3 ]]; then travis_retry sudo apt-get install -qq python3-systemd || echo 'not installed'; fi # gamin - install manually (not in PyPI) - travis-ci system Python is 2.7 - if [[ $TRAVIS_PYTHON_VERSION == 2.7 ]]; then (travis_retry sudo apt-get install -qq python-gamin && cp /usr/share/pyshared/gamin.py /usr/lib/pyshared/python2.7/_gamin.so $VIRTUAL_ENV/lib/python2.7/site-packages/) || echo 'not installed'; fi # pyinotify