diff --git a/ChangeLog b/ChangeLog index e30ea8c6..13b4501c 100644 --- a/ChangeLog +++ b/ChangeLog @@ -72,11 +72,26 @@ ver. 0.9.5 (2016/XX/XXX) - wanna-be-released - failregex of previous monit version merged as single expression. * filter.d/postfix.conf, filter.d/postfix-sasl.conf - extended failregex daemon part, matching also `postfix/smtps/smtpd` now (gh-1391) + * fixed a grave bug within tags substitutions because of incorrect detection of recursion + in case of multiple inline substitutions of the same tag (affected actions: `bsd-ipfw`, etc). + Now tracks the actual list of the already substituted tags (per tag instead of single list) + * filter.d/common.conf + - unexpected extra regex-space in generic `__prefix_line` (gh-1405) + - all optional spaces normalized in `common.conf`, test covered now + - generic `__prefix_line` extended with optional brackets for the date ambit (gh-1421), + added new parameter `__date_ambit` + * gentoo-initd fixed --pidfile bug: `--pidfile` is option of start-stop-daemon, + not argument of fail2ban (see gh-1434) - New Features: * New Actions: - action.d/firewallcmd-rich-rules and action.d/firewallcmd-rich-logging (gh-1367) - Enhancements: + * Extreme speedup of all sqlite database operations (gh-1436), + by using of following sqlite options: + - (synchronous = OFF) write data through OS without syncing + - (journal_mode = MEMORY) use memory for the transaction logging + - (temp_store = MEMORY) temporary tables and indices are kept in memory * journald journalmatch for pure-ftpd (gh-1362) * Add additional regex filter for dovecot ldap authentication failures (gh-1370) * added additional regex filters for exim (gh-1371) diff --git a/config/filter.d/asterisk.conf b/config/filter.d/asterisk.conf index 3975fb29..01063efa 100644 --- a/config/filter.d/asterisk.conf +++ b/config/filter.d/asterisk.conf @@ -16,17 +16,17 @@ __pid_re = (?:\[\d+\]) iso8601 = \d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\.\d+[+-]\d{4} # All Asterisk log messages begin like this: -log_prefix= (?:NOTICE|SECURITY)%(__pid_re)s:?(?:\[C-[\da-f]*\])? \S+:\d*( in \w+:)? +log_prefix= (?:NOTICE|SECURITY|WARNING)%(__pid_re)s:?(?:\[C-[\da-f]*\])? [^:]+:\d*( in \w+:)? -failregex = ^(%(__prefix_line)s|\[\]\s*)%(log_prefix)s Registration from '[^']*' failed for '(:\d+)?' - (Wrong password|Username/auth name mismatch|No matching peer found|Not a local domain|Device does not match ACL|Peer is not supposed to register|ACL error \(permit/deny\)|Not a local domain)$ - ^(%(__prefix_line)s|\[\]\s*)%(log_prefix)s Call from '[^']*' \(:\d+\) to extension '[^']*' rejected because extension not found in context - ^(%(__prefix_line)s|\[\]\s*)%(log_prefix)s Host failed to authenticate as '[^']*'$ - ^(%(__prefix_line)s|\[\]\s*)%(log_prefix)s No registration for peer '[^']*' \(from \)$ - ^(%(__prefix_line)s|\[\]\s*)%(log_prefix)s Host failed MD5 authentication for '[^']*' \([^)]+\)$ - ^(%(__prefix_line)s|\[\]\s*)%(log_prefix)s Failed to authenticate (user|device) [^@]+@\S*$ - ^(%(__prefix_line)s|\[\]\s*)%(log_prefix)s hacking attempt detected ''$ - ^(%(__prefix_line)s|\[\]\s*)%(log_prefix)s SecurityEvent="(FailedACL|InvalidAccountID|ChallengeResponseFailed|InvalidPassword)",EventTV="([\d-]+|%(iso8601)s)",Severity="[\w]+",Service="[\w]+",EventVersion="\d+",AccountID="(\d*|)",SessionID=".+",LocalAddress="IPV[46]/(UDP|TCP|WS)/[\da-fA-F:.]+/\d+",RemoteAddress="IPV[46]/(UDP|TCP|WS)//\d+"(,Challenge="[\w/]+")?(,ReceivedChallenge="\w+")?(,Response="\w+",ExpectedResponse="\w*")?(,ReceivedHash="[\da-f]+")?(,ACLName="\w+")?$ - ^(%(__prefix_line)s|\[\]\s*WARNING%(__pid_re)s:?(?:\[C-[\da-f]*\])? )Ext\. s: "Rejecting unknown SIP connection from "$ +failregex = ^%(__prefix_line)s%(log_prefix)s Registration from '[^']*' failed for '(:\d+)?' - (Wrong password|Username/auth name mismatch|No matching peer found|Not a local domain|Device does not match ACL|Peer is not supposed to register|ACL error \(permit/deny\)|Not a local domain)$ + ^%(__prefix_line)s%(log_prefix)s Call from '[^']*' \(:\d+\) to extension '[^']*' rejected because extension not found in context + ^%(__prefix_line)s%(log_prefix)s Host failed to authenticate as '[^']*'$ + ^%(__prefix_line)s%(log_prefix)s No registration for peer '[^']*' \(from \)$ + ^%(__prefix_line)s%(log_prefix)s Host failed MD5 authentication for '[^']*' \([^)]+\)$ + ^%(__prefix_line)s%(log_prefix)s Failed to authenticate (user|device) [^@]+@\S*$ + ^%(__prefix_line)s%(log_prefix)s hacking attempt detected ''$ + ^%(__prefix_line)s%(log_prefix)s SecurityEvent="(FailedACL|InvalidAccountID|ChallengeResponseFailed|InvalidPassword)",EventTV="([\d-]+|%(iso8601)s)",Severity="[\w]+",Service="[\w]+",EventVersion="\d+",AccountID="(\d*|)",SessionID=".+",LocalAddress="IPV[46]/(UDP|TCP|WS)/[\da-fA-F:.]+/\d+",RemoteAddress="IPV[46]/(UDP|TCP|WS)//\d+"(,Challenge="[\w/]+")?(,ReceivedChallenge="\w+")?(,Response="\w+",ExpectedResponse="\w*")?(,ReceivedHash="[\da-f]+")?(,ACLName="\w+")?$ + ^%(__prefix_line)s%(log_prefix)s "Rejecting unknown SIP connection from "$ ignoreregex = diff --git a/config/filter.d/common.conf b/config/filter.d/common.conf index 3e35f1d8..586f428a 100644 --- a/config/filter.d/common.conf +++ b/config/filter.d/common.conf @@ -26,7 +26,7 @@ __daemon_re = [\[\(]?%(_daemon)s(?:\(\S+\))?[\]\)]?:? # extra daemon info # EXAMPLE: [ID 800047 auth.info] -__daemon_extra_re = (?:\[ID \d+ \S+\]) +__daemon_extra_re = \[ID \d+ \S+\] # Combinations of daemon name and PID # EXAMPLES: sshd[31607], pop(pam_unix)[4920] @@ -44,14 +44,18 @@ __md5hex = (?:[\da-f]{2}:){15}[\da-f]{2} # bsdverbose is where syslogd is started with -v or -vv and results in <4.3> or # appearing before the host as per testcases/files/logs/bsd/*. -__bsd_syslog_verbose = (<[^.]+\.[^.]+>) +__bsd_syslog_verbose = <[^.]+\.[^.]+> + +__vserver = @vserver_\S+ + +__date_ambit = (?:\[\]) # Common line prefixes (beginnings) which could be used in filters # # [bsdverbose]? [hostname] [vserver tag] daemon_id spaces # # This can be optional (for instance if we match named native log files) -__prefix_line = \s*%(__bsd_syslog_verbose)s?\s*(?:%(__hostname)s )?(?:%(__kernel_prefix)s )?(?:@vserver_\S+ )?%(__daemon_combs_re)s?\s%(__daemon_extra_re)s?\s* +__prefix_line = %(__date_ambit)s?\s*(?:%(__bsd_syslog_verbose)s\s+)?(?:%(__hostname)s\s+)?(?:%(__kernel_prefix)s\s+)?(?:%(__vserver)s\s+)?(?:%(__daemon_combs_re)s\s+)?(?:%(__daemon_extra_re)s\s+)? # PAM authentication mechanism check for failures, e.g.: pam_unix, pam_sss, # pam_ldap diff --git a/config/filter.d/nsd.conf b/config/filter.d/nsd.conf index 70b41ca4..8f32f7be 100644 --- a/config/filter.d/nsd.conf +++ b/config/filter.d/nsd.conf @@ -22,7 +22,7 @@ _daemon = nsd # (?:::f{4,6}:)?(?P[\w\-.^_]+) # Values: TEXT -failregex = ^\[\]%(__prefix_line)sinfo: ratelimit block .* query TYPE255$ - ^\[\]%(__prefix_line)sinfo: .* refused, no acl matches\.$ +failregex = ^%(__prefix_line)sinfo: ratelimit block .* query TYPE255$ + ^%(__prefix_line)sinfo: .* refused, no acl matches\.$ ignoreregex = diff --git a/fail2ban/server/database.py b/fail2ban/server/database.py index a6b3feb1..46d40051 100644 --- a/fail2ban/server/database.py +++ b/fail2ban/server/database.py @@ -182,8 +182,23 @@ class Fail2BanDb(object): filename, e.args[0]) raise + # differentiate pypy: switch journal mode later (save it during the upgrade), + # to prevent errors like "database table is locked": + try: + import __pypy__ + pypy = True + except ImportError: + pypy = False + cur = self._db.cursor() - cur.execute("PRAGMA foreign_keys = ON;") + cur.execute("PRAGMA foreign_keys = ON") + # speedup: write data through OS without syncing (no wait): + cur.execute("PRAGMA synchronous = OFF") + # speedup: transaction log in memory, alternate using OFF (disable, rollback will be impossible): + if not pypy: + cur.execute("PRAGMA journal_mode = MEMORY") + # speedup: temporary tables and indices are kept in memory: + cur.execute("PRAGMA temp_store = MEMORY") try: cur.execute("SELECT version FROM fail2banDb LIMIT 1") @@ -203,6 +218,9 @@ class Fail2BanDb(object): Fail2BanDb.__version__, version, newversion) raise RuntimeError('Failed to fully update') finally: + # pypy: set journal mode after possible upgrade db: + if pypy: + cur.execute("PRAGMA journal_mode = MEMORY") cur.close() @property @@ -245,13 +263,14 @@ class Fail2BanDb(object): A timestamped backup is also created prior to attempting the update. """ - self._dbBackupFilename = self.filename + '.' + time.strftime('%Y%m%d-%H%M%S', MyTime.gmtime()) - shutil.copyfile(self.filename, self._dbBackupFilename) - logSys.info("Database backup created: %s", self._dbBackupFilename) if version > Fail2BanDb.__version__: raise NotImplementedError( "Attempt to travel to future version of database ...how did you get here??") + self._dbBackupFilename = self.filename + '.' + time.strftime('%Y%m%d-%H%M%S', MyTime.gmtime()) + shutil.copyfile(self.filename, self._dbBackupFilename) + logSys.info("Database backup created: %s", self._dbBackupFilename) + if version < 2: cur.executescript("BEGIN TRANSACTION;" "CREATE TEMPORARY TABLE logs_temp AS SELECT * FROM logs;" diff --git a/fail2ban/tests/actiontestcase.py b/fail2ban/tests/actiontestcase.py index 00a4d6be..ac59a400 100644 --- a/fail2ban/tests/actiontestcase.py +++ b/fail2ban/tests/actiontestcase.py @@ -67,6 +67,12 @@ class CommandActionTest(LogCaptureTestCase): self.assertRaises(ValueError, lambda: CommandAction.substituteRecursiveTags({'failregex': 'to= fromip=', 'sweet': '', 'honeypot': '', 'ignoreregex': ''})) # No-recursion, just multiple replacement of tag , should be successful + if OrderedDict: # we need here an ordered, because the sequence of iteration is very important for this test + self.assertEqual(CommandAction.substituteRecursiveTags( + OrderedDict((('X', 'x=x'), ('T', '1'), ('Z', ' '), ('Y', 'y=y'))) + ), {'X': 'x=x1', 'T': '1', 'Y': 'y=y1', 'Z': 'x=x1 1 y=y1'} + ) + # No-recursion, just multiple replacement of tag , should be successful if OrderedDict: # we need here an ordered, because the sequence of iteration is very important for this test self.assertEqual(CommandAction.substituteRecursiveTags( OrderedDict((('X', 'x=x'), ('T', '1'), ('Z', ' '), ('Y', 'y=y'))) diff --git a/fail2ban/tests/config/filter.d/zzz-generic-example.conf b/fail2ban/tests/config/filter.d/zzz-generic-example.conf new file mode 100644 index 00000000..a59ccb1e --- /dev/null +++ b/fail2ban/tests/config/filter.d/zzz-generic-example.conf @@ -0,0 +1,17 @@ +# Fail2Ban generic example resp. test filter +# +# Author: Serg G. Brester (sebres) +# + +[INCLUDES] + +# Read common prefixes. If any customizations available -- read them from +# common.local +before = ../../../../config/filter.d/common.conf + +[Definition] + +_daemon = test-demo + +failregex = ^%(__prefix_line)sF2B: failure from $ +ignoreregex = diff --git a/fail2ban/tests/files/logs/zzz-generic-example b/fail2ban/tests/files/logs/zzz-generic-example new file mode 100644 index 00000000..e3480b63 --- /dev/null +++ b/fail2ban/tests/files/logs/zzz-generic-example @@ -0,0 +1,31 @@ +# -- _daemon with __pid_re, without __hostname -- +# failJSON: { "time": "2005-06-21T16:47:46", "match": true , "host": "192.0.2.1" } +Jun 21 16:47:46 machine test-demo[13709]: F2B: failure from 192.0.2.1 +# -- _daemon with __pid_re -- +# failJSON: { "time": "2005-06-21T16:47:48", "match": true , "host": "192.0.2.1" } +Jun 21 16:47:48 test-demo[13709]: F2B: failure from 192.0.2.1 + +# -- __kernel_prefix -- +# failJSON: { "time": "2005-06-21T16:47:50", "match": true , "host": "192.0.2.2" } +Jun 21 16:47:50 machine kernel: [ 970.699396] F2B: failure from 192.0.2.2 + +# -- _daemon_re with and without __pid_re -- +# failJSON: { "time": "2005-06-21T16:47:52", "match": true , "host": "192.0.2.3" } +Jun 21 16:47:52 machine [test-demo] F2B: failure from 192.0.2.3 +# failJSON: { "time": "2005-06-21T16:47:53", "match": true , "host": "192.0.2.3" } +Jun 21 16:47:53 machine [test-demo][13709] F2B: failure from 192.0.2.3 +# failJSON: { "time": "2005-06-21T16:50:00", "match": true , "host": "192.0.2.3" } +Jun 21 16:50:00 machine test-demo(pam_unix) F2B: failure from 192.0.2.3 +# failJSON: { "time": "2005-06-21T16:50:02", "match": true , "host": "192.0.2.3" } +Jun 21 16:50:02 machine test-demo(pam_unix)[13709] F2B: failure from 192.0.2.3 + + +# -- all common definitions together (bsdverbose hostname kernel_prefix vserver tag daemon_id space) -- +# failJSON: { "time": "2005-06-21T16:55:01", "match": true , "host": "192.0.2.3" } +Jun 21 16:55:01 machine kernel: [ 970.699396] @vserver_demo test-demo(pam_unix)[13709] [ID 255 test] F2B: failure from 192.0.2.3 +# -- the same as above with additional spaces around -- +# failJSON: { "time": "2005-06-21T16:55:02", "match": true , "host": "192.0.2.3" } +Jun 21 16:55:02 machine kernel: [ 970.699396] @vserver_demo test-demo(pam_unix)[13709] [ID 255 test] F2B: failure from 192.0.2.3 +# -- the same as above with brackets as date ambit -- +# failJSON: { "time": "2005-06-21T16:55:03", "match": true , "host": "192.0.2.3" } +[Jun 21 16:55:03] machine kernel: [ 970.699396] @vserver_demo test-demo(pam_unix)[13709] [ID 255 test] F2B: failure from 192.0.2.3 diff --git a/fail2ban/tests/samplestestcase.py b/fail2ban/tests/samplestestcase.py index 074ba24c..326d09f7 100644 --- a/fail2ban/tests/samplestestcase.py +++ b/fail2ban/tests/samplestestcase.py @@ -35,6 +35,7 @@ from ..server.filter import Filter from ..client.filterreader import FilterReader from .utils import setUpMyTime, tearDownMyTime, CONFIG_DIR +TEST_CONFIG_DIR = os.path.join(os.path.dirname(__file__), "config") TEST_FILES_DIR = os.path.join(os.path.dirname(__file__), "files") @@ -60,12 +61,12 @@ class FilterSamplesRegex(unittest.TestCase): "Expected more FilterSampleRegexs tests") -def testSampleRegexsFactory(name): +def testSampleRegexsFactory(name, basedir): def testFilter(self): # Check filter exists filterConf = FilterReader(name, "jail", {}, - basedir=CONFIG_DIR, share_config=unittest.F2B.share_config) + basedir=basedir, share_config=unittest.F2B.share_config) self.assertEqual(filterConf.getFile(), name) self.assertEqual(filterConf.getJailName(), "jail") filterConf.read() @@ -155,11 +156,15 @@ def testSampleRegexsFactory(name): return testFilter -for filter_ in filter(lambda x: not x.endswith('common.conf') and x.endswith('.conf'), - os.listdir(os.path.join(CONFIG_DIR, "filter.d"))): - filterName = filter_.rpartition(".")[0] - if not filterName.startswith('.'): - setattr( - FilterSamplesRegex, - "testSampleRegexs%s" % filterName.upper(), - testSampleRegexsFactory(filterName)) +for basedir_, filter_ in ( + (CONFIG_DIR, lambda x: not x.endswith('common.conf') and x.endswith('.conf')), + (TEST_CONFIG_DIR, lambda x: x.startswith('zzz-') and x.endswith('.conf')), +): + for filter_ in filter(filter_, + os.listdir(os.path.join(basedir_, "filter.d"))): + filterName = filter_.rpartition(".")[0] + if not filterName.startswith('.'): + setattr( + FilterSamplesRegex, + "testSampleRegexs%s" % filterName.upper(), + testSampleRegexsFactory(filterName, basedir_)) diff --git a/files/gentoo-initd b/files/gentoo-initd index b7e1067b..c5b5b702 100755 --- a/files/gentoo-initd +++ b/files/gentoo-initd @@ -34,15 +34,15 @@ start() { # remove stalled sock file after system crash # bug 347477 rm -f /var/run/fail2ban/fail2ban.sock || return 1 - start-stop-daemon --start --exec ${FAIL2BAN} start \ - --pidfile /var/run/fail2ban/fail2ban.pid + start-stop-daemon --start --pidfile /var/run/fail2ban/fail2ban.pid \ + -- ${FAIL2BAN} start eend $? "Failed to start fail2ban" } stop() { ebegin "Stopping fail2ban" - start-stop-daemon --stop --exec ${FAIL2BAN} stop \ - --pidfile /var/run/fail2ban/fail2ban.pid + start-stop-daemon --stop --pidfile /var/run/fail2ban/fail2ban.pid \ + -- ${FAIL2BAN} stop eend $? "Failed to stop fail2ban" }