diff --git a/.travis.yml b/.travis.yml index 59a50313..064b678b 100644 --- a/.travis.yml +++ b/.travis.yml @@ -18,7 +18,6 @@ matrix: - python: 2.7 name: 2.7 (xenial) - python: pypy - dist: trusty - python: 3.3 dist: trusty - python: 3.4 @@ -70,8 +69,8 @@ script: - if [[ "$F2B_PY" = 3 ]]; then coverage run bin/fail2ban-testcases --verbosity=2; fi # Use $VENV_BIN (not python) or else sudo will always run the system's python (2.7) - sudo $VENV_BIN/pip install . - # Doc files should get installed on Travis under Linux (python >= 3.8 seem to use another path segment) - - if [[ $TRAVIS_PYTHON_VERSION < 3.8 ]]; then test -e /usr/share/doc/fail2ban/FILTERS; fi + # Doc files should get installed on Travis under Linux (some builds/python's seem to use another path segment) + - test -e /usr/share/doc/fail2ban/FILTERS && echo 'found' || echo 'not found' # Test initd script - shellcheck -s bash -e SC1090,SC1091 files/debian-initd after_success: diff --git a/ChangeLog b/ChangeLog index 8a3faf1a..5b2d7ef9 100644 --- a/ChangeLog +++ b/ChangeLog @@ -32,6 +32,7 @@ ver. 1.0.1-dev-1 (20??/??/??) - development nightly edition between ipset and fail2ban (removal from ipset will be managed by fail2ban only, gh-2703) * `action.d/cloudflare.conf`: fixed `actionunban` (considering new-line chars and optionally real json-parsing with `jq`, gh-2140, gh-2656) +* `action.d/nftables.conf` (type=multiport only): fixed port range selector, replacing `:` with `-` (gh-2763) * `filter.d/common.conf`: avoid substitute of default values in related `lt_*` section, `__prefix_line` should be interpolated in definition section (inside the filter-config, gh-2650) * `filter.d/courier-smtp.conf`: prefregex extended to consider port in log-message (gh-2697) @@ -49,6 +50,8 @@ ver. 1.0.1-dev-1 (20??/??/??) - development nightly edition * introduced new prefix `{UNB}` for `datepattern` to disable word boundaries in regex; * datetemplate: improved anchor detection for capturing groups `(^...)`; * performance optimization of `datepattern` (better search algorithm in datedetector, especially for single template); +* extended capturing of alternate tags in filter, allowing combine of multiple groups to single tuple token with new tag + prefix `` with all value of `` tags (gh-2755) * `actioncheck` behavior is changed now (gh-488), so invariant check as well as restore or repair of sane environment (in case of recognized unsane state) would only occur on action errors (e. g. if ban or unban operations are exiting with other code as 0) diff --git a/config/action.d/firewallcmd-ipset.conf b/config/action.d/firewallcmd-ipset.conf index ab7cdac7..99541910 100644 --- a/config/action.d/firewallcmd-ipset.conf +++ b/config/action.d/firewallcmd-ipset.conf @@ -18,7 +18,7 @@ before = firewallcmd-common.conf [Definition] -actionstart = ipset create hash:ip timeout +actionstart = ipset create hash:ip timeout firewall-cmd --direct --add-rule filter 0 -m set --match-set src -j actionflush = ipset flush @@ -27,7 +27,7 @@ actionstop = firewall-cmd --direct --remove-rule filter 0 ipset destroy -actionban = ipset add timeout -exist +actionban = ipset add timeout -exist # actionprolong = %(actionban)s @@ -42,18 +42,18 @@ actionunban = ipset del -exist # chain = INPUT_direct -# Option: default-timeout +# Option: default-ipsettime # Notes: specifies default timeout in seconds (handled default ipset timeout only) # Values: [ NUM ] Default: 0 (no timeout, managed by fail2ban by unban) -default-timeout = 0 +default-ipsettime = 0 -# Option: timeout +# Option: ipsettime # Notes: specifies ticket timeout (handled ipset timeout only) # Values: [ NUM ] Default: 0 (managed by fail2ban by unban) -timeout = 0 +ipsettime = 0 # expresion to caclulate timeout from bantime, example: -# banaction = %(known/banaction)s[timeout=''] +# banaction = %(known/banaction)s[ipsettime=''] timeout-bantime = $([ "" -le 2147483 ] && echo "" || echo 0) # Option: actiontype diff --git a/config/action.d/iptables-ipset-proto6-allports.conf b/config/action.d/iptables-ipset-proto6-allports.conf index bf299259..67d7947b 100644 --- a/config/action.d/iptables-ipset-proto6-allports.conf +++ b/config/action.d/iptables-ipset-proto6-allports.conf @@ -26,7 +26,7 @@ before = iptables-common.conf # Notes.: command executed on demand at the first ban (or at the start of Fail2Ban if actionstart_on_demand is set to false). # Values: CMD # -actionstart = ipset create hash:ip timeout +actionstart = ipset create hash:ip timeout -I -m set --match-set src -j # Option: actionflush @@ -49,7 +49,7 @@ actionstop = -D -m set --match-set src -j timeout -exist +actionban = ipset add timeout -exist # actionprolong = %(actionban)s @@ -63,18 +63,18 @@ actionunban = ipset del -exist [Init] -# Option: default-timeout +# Option: default-ipsettime # Notes: specifies default timeout in seconds (handled default ipset timeout only) # Values: [ NUM ] Default: 0 (no timeout, managed by fail2ban by unban) -default-timeout = 0 +default-ipsettime = 0 -# Option: timeout +# Option: ipsettime # Notes: specifies ticket timeout (handled ipset timeout only) # Values: [ NUM ] Default: 0 (managed by fail2ban by unban) -timeout = 0 +ipsettime = 0 # expresion to caclulate timeout from bantime, example: -# banaction = %(known/banaction)s[timeout=''] +# banaction = %(known/banaction)s[ipsettime=''] timeout-bantime = $([ "" -le 2147483 ] && echo "" || echo 0) ipmset = f2b- diff --git a/config/action.d/iptables-ipset-proto6.conf b/config/action.d/iptables-ipset-proto6.conf index e94751fa..87601027 100644 --- a/config/action.d/iptables-ipset-proto6.conf +++ b/config/action.d/iptables-ipset-proto6.conf @@ -26,7 +26,7 @@ before = iptables-common.conf # Notes.: command executed on demand at the first ban (or at the start of Fail2Ban if actionstart_on_demand is set to false). # Values: CMD # -actionstart = ipset create hash:ip timeout +actionstart = ipset create hash:ip timeout -I -p -m multiport --dports -m set --match-set src -j # Option: actionflush @@ -49,7 +49,7 @@ actionstop = -D -p -m multiport --dports -m # Tags: See jail.conf(5) man page # Values: CMD # -actionban = ipset add timeout -exist +actionban = ipset add timeout -exist # actionprolong = %(actionban)s @@ -63,18 +63,18 @@ actionunban = ipset del -exist [Init] -# Option: default-timeout +# Option: default-ipsettime # Notes: specifies default timeout in seconds (handled default ipset timeout only) # Values: [ NUM ] Default: 0 (no timeout, managed by fail2ban by unban) -default-timeout = 0 +default-ipsettime = 0 -# Option: timeout +# Option: ipsettime # Notes: specifies ticket timeout (handled ipset timeout only) # Values: [ NUM ] Default: 0 (managed by fail2ban by unban) -timeout = 0 +ipsettime = 0 # expresion to caclulate timeout from bantime, example: -# banaction = %(known/banaction)s[timeout=''] +# banaction = %(known/banaction)s[ipsettime=''] timeout-bantime = $([ "" -le 2147483 ] && echo "" || echo 0) ipmset = f2b- diff --git a/config/action.d/nftables.conf b/config/action.d/nftables.conf index c1fb8550..77cf3661 100644 --- a/config/action.d/nftables.conf +++ b/config/action.d/nftables.conf @@ -34,7 +34,7 @@ type = multiport rule_match-custom = rule_match-allports = meta l4proto \{ \} -rule_match-multiport = $proto dport \{ \} +rule_match-multiport = $proto dport \{ $(echo '' | sed s/:/-/g) \} match = > # Option: rule_stat diff --git a/config/action.d/shorewall-ipset-proto6.conf b/config/action.d/shorewall-ipset-proto6.conf index 8cbbd347..eacb53d9 100644 --- a/config/action.d/shorewall-ipset-proto6.conf +++ b/config/action.d/shorewall-ipset-proto6.conf @@ -51,7 +51,7 @@ # Values: CMD # actionstart = if ! ipset -quiet -name list f2b- >/dev/null; - then ipset -quiet -exist create f2b- hash:ip timeout ; + then ipset -quiet -exist create f2b- hash:ip timeout ; fi # Option: actionstop @@ -66,7 +66,7 @@ actionstop = ipset flush f2b- # Tags: See jail.conf(5) man page # Values: CMD # -actionban = ipset add f2b- timeout -exist +actionban = ipset add f2b- timeout -exist # actionprolong = %(actionban)s @@ -78,16 +78,16 @@ actionban = ipset add f2b- timeout -exist # actionunban = ipset del f2b- -exist -# Option: default-timeout +# Option: default-ipsettime # Notes: specifies default timeout in seconds (handled default ipset timeout only) # Values: [ NUM ] Default: 0 (no timeout, managed by fail2ban by unban) -default-timeout = 0 +default-ipsettime = 0 -# Option: timeout +# Option: ipsettime # Notes: specifies ticket timeout (handled ipset timeout only) # Values: [ NUM ] Default: 0 (managed by fail2ban by unban) -timeout = 0 +ipsettime = 0 # expresion to caclulate timeout from bantime, example: -# banaction = %(known/banaction)s[timeout=''] +# banaction = %(known/banaction)s[ipsettime=''] timeout-bantime = $([ "" -le 2147483 ] && echo "" || echo 0) diff --git a/config/fail2ban.conf b/config/fail2ban.conf index ba0e9204..f3867839 100644 --- a/config/fail2ban.conf +++ b/config/fail2ban.conf @@ -19,7 +19,7 @@ # NOTICE # INFO # DEBUG -# Values: [ LEVEL ] Default: ERROR +# Values: [ LEVEL ] Default: INFO # loglevel = INFO diff --git a/fail2ban/server/action.py b/fail2ban/server/action.py index d5ac1679..be9c48fd 100644 --- a/fail2ban/server/action.py +++ b/fail2ban/server/action.py @@ -868,7 +868,7 @@ class CommandAction(ActionBase): tickData = aInfo.get("F-*") if not tickData: tickData = {} def substTag(m): - tag = mapTag2Opt(m.groups()[0]) + tag = mapTag2Opt(m.group(1)) try: value = uni_string(tickData[tag]) except KeyError: diff --git a/fail2ban/server/failregex.py b/fail2ban/server/failregex.py index 0ae9acc5..4032ecdb 100644 --- a/fail2ban/server/failregex.py +++ b/fail2ban/server/failregex.py @@ -87,20 +87,24 @@ RH4TAG = { # default failure groups map for customizable expressions (with different group-id): R_MAP = { - "ID": "fid", - "PORT": "fport", + "id": "fid", + "port": "fport", } def mapTag2Opt(tag): - try: # if should be mapped: - return R_MAP[tag] - except KeyError: - return tag.lower() + tag = tag.lower() + return R_MAP.get(tag, tag) -# alternate names to be merged, e. g. alt_user_1 -> user ... +# complex names: +# ALT_ - alternate names to be merged, e. g. alt_user_1 -> user ... ALTNAME_PRE = 'alt_' -ALTNAME_CRE = re.compile(r'^' + ALTNAME_PRE + r'(.*)(?:_\d+)?$') +# TUPLE_ - names of parts to be combined to single value as tuple +TUPNAME_PRE = 'tuple_' + +COMPLNAME_PRE = (ALTNAME_PRE, TUPNAME_PRE) +COMPLNAME_CRE = re.compile(r'^(' + '|'.join(COMPLNAME_PRE) + r')(.*?)(?:_\d+)?$') + ## # Regular expression class. @@ -127,19 +131,27 @@ class Regex: try: self._regexObj = re.compile(regex, re.MULTILINE if multiline else 0) self._regex = regex - self._altValues = {} + self._altValues = [] + self._tupleValues = [] for k in filter( - lambda k: len(k) > len(ALTNAME_PRE) and k.startswith(ALTNAME_PRE), - self._regexObj.groupindex + lambda k: len(k) > len(COMPLNAME_PRE[0]), self._regexObj.groupindex ): - n = ALTNAME_CRE.match(k).group(1) - self._altValues[k] = n - self._altValues = list(self._altValues.items()) if len(self._altValues) else None + n = COMPLNAME_CRE.match(k) + if n: + g, n = n.group(1), mapTag2Opt(n.group(2)) + if g == ALTNAME_PRE: + self._altValues.append((k,n)) + else: + self._tupleValues.append((k,n)) + self._altValues.sort() + self._tupleValues.sort() + self._altValues = self._altValues if len(self._altValues) else None + self._tupleValues = self._tupleValues if len(self._tupleValues) else None except sre_constants.error: raise RegexException("Unable to compile regular expression '%s'" % regex) - # set fetch handler depending on presence of alternate tags: - self.getGroups = self._getGroupsWithAlt if self._altValues else self._getGroups + # set fetch handler depending on presence of alternate (or tuple) tags: + self.getGroups = self._getGroupsWithAlt if (self._altValues or self._tupleValues) else self._getGroups def __str__(self): return "%s(%r)" % (self.__class__.__name__, self._regex) @@ -284,12 +296,23 @@ class Regex: def _getGroupsWithAlt(self): fail = self._matchCache.groupdict() - # merge alternate values (e. g. 'alt_user_1' -> 'user' or 'alt_host' -> 'host'): #fail = fail.copy() - for k,n in self._altValues: - v = fail.get(k) - if v and not fail.get(n): - fail[n] = v + # merge alternate values (e. g. 'alt_user_1' -> 'user' or 'alt_host' -> 'host'): + if self._altValues: + for k,n in self._altValues: + v = fail.get(k) + if v and not fail.get(n): + fail[n] = v + # combine tuple values (e. g. 'id', 'tuple_id' ... 'tuple_id_N' -> 'id'): + if self._tupleValues: + for k,n in self._tupleValues: + v = fail.get(k) + t = fail.get(n) + if isinstance(t, tuple): + t += (v,) + else: + t = (t,v,) + fail[n] = t return fail def getGroups(self): # pragma: no cover - abstract function (replaced in __init__) diff --git a/fail2ban/server/ipdns.py b/fail2ban/server/ipdns.py index 335fc473..18e1bd02 100644 --- a/fail2ban/server/ipdns.py +++ b/fail2ban/server/ipdns.py @@ -337,7 +337,7 @@ class IPAddr(object): return repr(self.ntoa) def __str__(self): - return self.ntoa + return self.ntoa if isinstance(self.ntoa, basestring) else str(self.ntoa) def __reduce__(self): """IPAddr pickle-handler, that simply wraps IPAddr to the str diff --git a/fail2ban/tests/fail2banregextestcase.py b/fail2ban/tests/fail2banregextestcase.py index 8c6a0e47..4ce43096 100644 --- a/fail2ban/tests/fail2banregextestcase.py +++ b/fail2ban/tests/fail2banregextestcase.py @@ -340,6 +340,23 @@ class Fail2banRegexTest(LogCaptureTestCase): self.assertTrue(_test_exec('-o', 'id', STR_00, RE_00_ID)) self.assertLogged('kevin') self.pruneLog() + # multiple id combined to a tuple (id, tuple_id): + self.assertTrue(_test_exec('-o', 'id', + '1591983743.667 192.0.2.1 192.0.2.2', + r'^\s* \S+')) + self.assertLogged(str(('192.0.2.1', '192.0.2.2'))) + self.pruneLog() + # multiple id combined to a tuple, id first - (id, tuple_id_1, tuple_id_2): + self.assertTrue(_test_exec('-o', 'id', + '1591983743.667 left 192.0.2.3 right', + r'^\s*\S+ \S+')) + self.pruneLog() + # id had higher precedence as ip-address: + self.assertTrue(_test_exec('-o', 'id', + '1591983743.667 left [192.0.2.4]:12345 right', + r'^\s*\S+ : \S+')) + self.assertLogged(str(('[192.0.2.4]:12345', 'left', 'right'))) + self.pruneLog() # row with id : self.assertTrue(_test_exec('-o', 'row', STR_00, RE_00_ID)) self.assertLogged("['kevin'", "'ip4': '192.0.2.0'", "'fid': 'kevin'", all=True) @@ -485,7 +502,7 @@ class Fail2banRegexTest(LogCaptureTestCase): def testLogtypeSystemdJournal(self): # pragma: no cover if not fail2banregex.FilterSystemd: - raise unittest.SkipTest('Skip test because no systemd backand available') + raise unittest.SkipTest('Skip test because no systemd backend available') self.assertTrue(_test_exec( "systemd-journal", FILTER_ZZZ_GEN +'[journalmatch="SYSLOG_IDENTIFIER=\x01\x02dummy\x02\x01",' diff --git a/fail2ban/tests/misctestcase.py b/fail2ban/tests/misctestcase.py index c1a6a345..458e9a23 100644 --- a/fail2ban/tests/misctestcase.py +++ b/fail2ban/tests/misctestcase.py @@ -201,7 +201,8 @@ class TestsUtilsTest(LogCaptureTestCase): uni_decode((b'test\xcf' if sys.version_info >= (3,) else u'test\xcf')) uni_string(b'test\xcf') uni_string('test\xcf') - uni_string(u'test\xcf') + if sys.version_info < (3,) and 'PyPy' not in sys.version: + uni_string(u'test\xcf') def testSafeLogging(self): # logging should be exception-safe, to avoid possible errors (concat, str. conversion, representation failures, etc) diff --git a/fail2ban/tests/servertestcase.py b/fail2ban/tests/servertestcase.py index 9bc90853..0c15509a 100644 --- a/fail2ban/tests/servertestcase.py +++ b/fail2ban/tests/servertestcase.py @@ -1361,11 +1361,11 @@ class ServerConfigReaderTests(LogCaptureTestCase): ), 'ip4-start': ( r"`nft add set inet f2b-table addr-set-j-w-nft-mp \{ type ipv4_addr\; \}`", - r"`nft add rule inet f2b-table f2b-chain $proto dport \{ http,https \} ip saddr @addr-set-j-w-nft-mp reject`", + r"`nft add rule inet f2b-table f2b-chain $proto dport \{ $(echo 'http,https' | sed s/:/-/g) \} ip saddr @addr-set-j-w-nft-mp reject`", ), 'ip6-start': ( r"`nft add set inet f2b-table addr6-set-j-w-nft-mp \{ type ipv6_addr\; \}`", - r"`nft add rule inet f2b-table f2b-chain $proto dport \{ http,https \} ip6 saddr @addr6-set-j-w-nft-mp reject`", + r"`nft add rule inet f2b-table f2b-chain $proto dport \{ $(echo 'http,https' | sed s/:/-/g) \} ip6 saddr @addr6-set-j-w-nft-mp reject`", ), 'flush': ( "`{ nft flush set inet f2b-table addr-set-j-w-nft-mp 2> /dev/null; } || ", diff --git a/man/jail.conf.5 b/man/jail.conf.5 index 662b3f48..4d01b6a1 100644 --- a/man/jail.conf.5 +++ b/man/jail.conf.5 @@ -130,7 +130,7 @@ Comments: use '#' for comment lines and '; ' (space is important) for inline com The items that can be set in section [Definition] are: .TP .B loglevel -verbosity level of log output: CRITICAL, ERROR, WARNING, NOTICE, INFO, DEBUG, TRACEDEBUG, HEAVYDEBUG or corresponding numeric value (50-5). Default: ERROR (equal 40) +verbosity level of log output: CRITICAL, ERROR, WARNING, NOTICE, INFO, DEBUG, TRACEDEBUG, HEAVYDEBUG or corresponding numeric value (50-5). Default: INFO (equal 20) .TP .B logtarget log target: filename, SYSLOG, STDERR or STDOUT. Default: STDOUT if not set in fail2ban.conf/fail2ban.local