From 8614ca8c41a0d868f0d54d219666374216fb70d4 Mon Sep 17 00:00:00 2001 From: Shane Forsythe <2287983+shaneforsythe@users.noreply.github.com> Date: Tue, 2 Oct 2018 17:24:33 -0400 Subject: [PATCH 01/27] Update proftpd.conf proftpd 1.3.5e can leave inconsistent error message if ftp or mod_sftp is used Oct 2 15:45:31 ftp01 proftpd[5516]: 10.10.2.13 (10.10.2.189[10.10.2.189]) - SECURITY VIOLATION: Root login attempted Oct 2 15:45:44 ftp01 proftpd[5517]: 10.10.2.13 (10.10.2.189[10.10.2.189]) - SECURITY VIOLATION: Root login attempted. Fix regex to make trailing period optional, otherwise brute force attacks against root account using ftp are not blocked correctly. --- config/filter.d/proftpd.conf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config/filter.d/proftpd.conf b/config/filter.d/proftpd.conf index 303be5e5..feb59f11 100644 --- a/config/filter.d/proftpd.conf +++ b/config/filter.d/proftpd.conf @@ -18,7 +18,7 @@ __suffix_failed_login = (User not authorized for login|No such user found|Incorr failregex = ^%(__prefix_line)s%(__hostname)s \(\S+\[\]\)[: -]+ USER .*: no such user found from \S+ \[\S+\] to \S+:\S+ *$ ^%(__prefix_line)s%(__hostname)s \(\S+\[\]\)[: -]+ USER .* \(Login failed\): %(__suffix_failed_login)s\s*$ - ^%(__prefix_line)s%(__hostname)s \(\S+\[\]\)[: -]+ SECURITY VIOLATION: .* login attempted\. *$ + ^%(__prefix_line)s%(__hostname)s \(\S+\[\]\)[: -]+ SECURITY VIOLATION: .* login attempted\.? *$ ^%(__prefix_line)s%(__hostname)s \(\S+\[\]\)[: -]+ Maximum login attempts \(\d+\) exceeded *$ ignoreregex = From 606bf110c99c0b491b10f336b67675311f279f1a Mon Sep 17 00:00:00 2001 From: sebres Date: Mon, 16 Mar 2020 17:29:09 +0100 Subject: [PATCH 02/27] filter.d/sshd.conf (mode `ddos`): fixed "connection reset" regex (seems to have same syntax now as closed), so both regex's combined now to single RE (closes gh-2662) --- config/filter.d/sshd.conf | 3 +-- .../tests/config/filter.d/zzz-sshd-obsolete-multiline.conf | 3 +-- fail2ban/tests/files/logs/sshd | 3 +++ 3 files changed, 5 insertions(+), 4 deletions(-) diff --git a/config/filter.d/sshd.conf b/config/filter.d/sshd.conf index 12631cb3..7a7f5e48 100644 --- a/config/filter.d/sshd.conf +++ b/config/filter.d/sshd.conf @@ -73,11 +73,10 @@ mdre-normal-other = ^(Connection closed|Disconnected) ^kex_exchange_identification: client sent invalid protocol identifier ^Bad protocol version identification '.*' from - ^Connection reset by ^SSH: Server;Ltype: (?:Authname|Version|Kex);Remote: -\d+;[A-Z]\w+: ^Read from socket failed: Connection reset by peer # same as mdre-normal-other, but as failure (without ) and [preauth] only: -mdre-ddos-other = ^(Connection closed|Disconnected) (?:by|from)%(__authng_user)s %(__on_port_opt)s\s+\[preauth\]\s*$ +mdre-ddos-other = ^(Connection (?:closed|reset)|Disconnected) (?:by|from)%(__authng_user)s %(__on_port_opt)s\s+\[preauth\]\s*$ mdre-extra = ^Received disconnect from %(__on_port_opt)s:\s*14: No supported authentication methods available ^Unable to negotiate with %(__on_port_opt)s: no matching <__alg_match> found. diff --git a/fail2ban/tests/config/filter.d/zzz-sshd-obsolete-multiline.conf b/fail2ban/tests/config/filter.d/zzz-sshd-obsolete-multiline.conf index d61a6520..4ff4ac68 100644 --- a/fail2ban/tests/config/filter.d/zzz-sshd-obsolete-multiline.conf +++ b/fail2ban/tests/config/filter.d/zzz-sshd-obsolete-multiline.conf @@ -57,8 +57,7 @@ mdre-normal = mdre-ddos = ^%(__prefix_line_sl)sDid not receive identification string from ^%(__prefix_line_sl)sBad protocol version identification '.*' from - ^%(__prefix_line_sl)sConnection closed by%(__authng_user)s %(__on_port_opt)s\s+\[preauth\]\s*$ - ^%(__prefix_line_sl)sConnection reset by + ^%(__prefix_line_sl)sConnection (?:closed|reset) by%(__authng_user)s %(__on_port_opt)s\s+\[preauth\]\s*$ ^%(__prefix_line_ml1)sSSH: Server;Ltype: (?:Authname|Version|Kex);Remote: -\d+;[A-Z]\w+:.*%(__prefix_line_ml2)sRead from socket failed: Connection reset by peer%(__suff)s$ mdre-extra = ^%(__prefix_line_sl)sReceived disconnect from %(__on_port_opt)s:\s*14: No supported authentication methods available diff --git a/fail2ban/tests/files/logs/sshd b/fail2ban/tests/files/logs/sshd index 0385f38c..e45ca90d 100644 --- a/fail2ban/tests/files/logs/sshd +++ b/fail2ban/tests/files/logs/sshd @@ -296,6 +296,9 @@ Nov 24 23:46:43 host sshd[32686]: fatal: Read from socket failed: Connection res # failJSON: { "time": "2005-03-15T09:20:57", "match": true , "host": "192.0.2.39", "desc": "Singleline for connection reset by" } Mar 15 09:20:57 host sshd[28972]: Connection reset by 192.0.2.39 port 14282 [preauth] +# failJSON: { "time": "2005-03-16T09:29:50", "match": true , "host": "192.0.2.20", "desc": "connection reset by user (gh-2662)" } +Mar 16 09:29:50 host sshd[19131]: Connection reset by authenticating user root 192.0.2.20 port 1558 [preauth] + # failJSON: { "time": "2005-07-17T23:03:05", "match": true , "host": "192.0.2.10", "user": "root", "desc": "user name additionally, gh-2185" } Jul 17 23:03:05 srv sshd[1296]: Connection closed by authenticating user root 192.0.2.10 port 46038 [preauth] # failJSON: { "time": "2005-07-17T23:04:00", "match": true , "host": "192.0.2.11", "user": "test 127.0.0.1", "desc": "check inject on username, gh-2185" } From 343ec1cdd296530f331637c725bd2bb0549e01e6 Mon Sep 17 00:00:00 2001 From: sebres Date: Wed, 18 Mar 2020 20:37:25 +0100 Subject: [PATCH 03/27] test-causes: avoid host-depending issue (mistakenly ignoring IP 127.0.0.2 as own address) - replace loop-back addr with test sub-net addr (and disable ignoreself) --- fail2ban/tests/observertestcase.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/fail2ban/tests/observertestcase.py b/fail2ban/tests/observertestcase.py index 8e944454..e379ccd1 100644 --- a/fail2ban/tests/observertestcase.py +++ b/fail2ban/tests/observertestcase.py @@ -36,7 +36,6 @@ from ..server.failmanager import FailManager from ..server.observer import Observers, ObserverThread from ..server.utils import Utils from .utils import LogCaptureTestCase -from ..server.filter import Filter from .dummyjail import DummyJail from .databasetestcase import getFail2BanDb, Fail2BanDb @@ -224,7 +223,7 @@ class BanTimeIncrDB(LogCaptureTestCase): jail.actions.setBanTime(10) jail.setBanTimeExtra('increment', 'true') jail.setBanTimeExtra('multipliers', '1 2 4 8 16 32 64 128 256 512 1024 2048') - ip = "127.0.0.2" + ip = "192.0.2.1" # used as start and fromtime (like now but time independence, cause test case can run slow): stime = int(MyTime.time()) ticket = FailTicket(ip, stime, []) @@ -385,10 +384,12 @@ class BanTimeIncrDB(LogCaptureTestCase): # two separate jails : jail1 = DummyJail(backend='polling') + jail1.filter.ignoreSelf = False jail1.setBanTimeExtra('increment', 'true') jail1.database = self.db self.db.addJail(jail1) jail2 = DummyJail(name='DummyJail-2', backend='polling') + jail2.filter.ignoreSelf = False jail2.database = self.db self.db.addJail(jail2) ticket1 = FailTicket(ip, stime, []) @@ -477,7 +478,7 @@ class BanTimeIncrDB(LogCaptureTestCase): self.assertEqual(tickets, []) # add failure: - ip = "127.0.0.2" + ip = "192.0.2.1" ticket = FailTicket(ip, stime-120, []) failManager = FailManager() failManager.setMaxRetry(3) From fc175fa78a2c0c91f6f90745c12a56e67605a279 Mon Sep 17 00:00:00 2001 From: sebres Date: Mon, 6 Apr 2020 12:10:32 +0200 Subject: [PATCH 04/27] performance: optimize simplest case whether the ignoreip is a single IP (not subnet/dns) - uses a set instead of list (holds single IPs and subnets/dns in different lists); decrease log level for ignored duplicates (warning is too heavy here) --- fail2ban/server/filter.py | 24 ++++++++++++++++++------ fail2ban/server/ipdns.py | 6 ++++++ 2 files changed, 24 insertions(+), 6 deletions(-) diff --git a/fail2ban/server/filter.py b/fail2ban/server/filter.py index e7f3e01d..a92acb8b 100644 --- a/fail2ban/server/filter.py +++ b/fail2ban/server/filter.py @@ -80,6 +80,7 @@ class Filter(JailThread): ## Ignore own IPs flag: self.__ignoreSelf = True ## The ignore IP list. + self.__ignoreIpSet = set() self.__ignoreIpList = [] ## External command self.__ignoreCommand = False @@ -489,28 +490,36 @@ class Filter(JailThread): # Create IP address object ip = IPAddr(ipstr) # Avoid exact duplicates - if ip in self.__ignoreIpList: - logSys.warn(" Ignore duplicate %r (%r), already in ignore list", ip, ipstr) + if ip in self.__ignoreIpSet or ip in self.__ignoreIpList: + logSys.log(logging.MSG, " Ignore duplicate %r (%r), already in ignore list", ip, ipstr) return # log and append to ignore list logSys.debug(" Add %r to ignore list (%r)", ip, ipstr) - self.__ignoreIpList.append(ip) + # if single IP (not DNS or a subnet) add to set, otherwise to list: + if ip.isSingle: + self.__ignoreIpSet.add(ip) + else: + self.__ignoreIpList.append(ip) def delIgnoreIP(self, ip=None): # clear all: if ip is None: + self.__ignoreIpSet.clear() del self.__ignoreIpList[:] return # delete by ip: logSys.debug(" Remove %r from ignore list", ip) - self.__ignoreIpList.remove(ip) + if ip in self.__ignoreIpSet: + self.__ignoreIpSet.remove(ip) + else: + self.__ignoreIpList.remove(ip) def logIgnoreIp(self, ip, log_ignore, ignore_source="unknown source"): if log_ignore: logSys.info("[%s] Ignore %s by %s", self.jailName, ip, ignore_source) def getIgnoreIP(self): - return self.__ignoreIpList + return self.__ignoreIpList + list(self.__ignoreIpSet) ## # Check if IP address/DNS is in the ignore list. @@ -550,8 +559,11 @@ class Filter(JailThread): if self.__ignoreCache: c.set(key, True) return True + # check if the IP is covered by ignore IP (in set or in subnet/dns): + if ip in self.__ignoreIpSet: + self.logIgnoreIp(ip, log_ignore, ignore_source="ip") + return True for net in self.__ignoreIpList: - # check if the IP is covered by ignore IP if ip.isInNet(net): self.logIgnoreIp(ip, log_ignore, ignore_source=("ip" if net.isValid else "dns")) if self.__ignoreCache: c.set(key, True) diff --git a/fail2ban/server/ipdns.py b/fail2ban/server/ipdns.py index 6648dac6..335fc473 100644 --- a/fail2ban/server/ipdns.py +++ b/fail2ban/server/ipdns.py @@ -379,6 +379,12 @@ class IPAddr(object): """ return self._family != socket.AF_UNSPEC + @property + def isSingle(self): + """Returns whether the object is a single IP address (not DNS and subnet) + """ + return self._plen == {socket.AF_INET: 32, socket.AF_INET6: 128}.get(self._family, -1000) + def __eq__(self, other): if self._family == IPAddr.CIDR_RAW and not isinstance(other, IPAddr): return self._raw == other From d21a24de8e1bb998fe8a54908b650250ba524fd0 Mon Sep 17 00:00:00 2001 From: sebres Date: Mon, 6 Apr 2020 12:39:36 +0200 Subject: [PATCH 05/27] more test cases for IP/DNS (and use dummies if no-network set by testing) --- fail2ban/tests/filtertestcase.py | 16 ++++++++++++---- fail2ban/tests/utils.py | 12 ++++++++++-- 2 files changed, 22 insertions(+), 6 deletions(-) diff --git a/fail2ban/tests/filtertestcase.py b/fail2ban/tests/filtertestcase.py index 202f3fbb..a511b5d0 100644 --- a/fail2ban/tests/filtertestcase.py +++ b/fail2ban/tests/filtertestcase.py @@ -1899,7 +1899,9 @@ class DNSUtilsNetworkTests(unittest.TestCase): ip4 = IPAddr('192.0.2.1') ip6 = IPAddr('2001:DB8::') self.assertTrue(ip4.isIPv4) + self.assertTrue(ip4.isSingle) self.assertTrue(ip6.isIPv6) + self.assertTrue(ip6.isSingle) self.assertTrue(asip('192.0.2.1').isIPv4) self.assertTrue(id(asip(ip4)) == id(ip4)) @@ -1908,6 +1910,7 @@ class DNSUtilsNetworkTests(unittest.TestCase): r = IPAddr('xxx', IPAddr.CIDR_RAW) self.assertFalse(r.isIPv4) self.assertFalse(r.isIPv6) + self.assertFalse(r.isSingle) self.assertTrue(r.isValid) self.assertEqual(r, 'xxx') self.assertEqual('xxx', str(r)) @@ -1916,6 +1919,7 @@ class DNSUtilsNetworkTests(unittest.TestCase): r = IPAddr('1:2', IPAddr.CIDR_RAW) self.assertFalse(r.isIPv4) self.assertFalse(r.isIPv6) + self.assertFalse(r.isSingle) self.assertTrue(r.isValid) self.assertEqual(r, '1:2') self.assertEqual('1:2', str(r)) @@ -1938,7 +1942,7 @@ class DNSUtilsNetworkTests(unittest.TestCase): def testUseDns(self): res = DNSUtils.textToIp('www.example.com', 'no') self.assertSortedEqual(res, []) - unittest.F2B.SkipIfNoNetwork() + #unittest.F2B.SkipIfNoNetwork() res = DNSUtils.textToIp('www.example.com', 'warn') # sort ipaddr, IPv4 is always smaller as IPv6 self.assertSortedEqual(res, ['93.184.216.34', '2606:2800:220:1:248:1893:25c8:1946']) @@ -1947,7 +1951,7 @@ class DNSUtilsNetworkTests(unittest.TestCase): self.assertSortedEqual(res, ['93.184.216.34', '2606:2800:220:1:248:1893:25c8:1946']) def testTextToIp(self): - unittest.F2B.SkipIfNoNetwork() + #unittest.F2B.SkipIfNoNetwork() # Test hostnames hostnames = [ 'www.example.com', @@ -1971,7 +1975,7 @@ class DNSUtilsNetworkTests(unittest.TestCase): self.assertTrue(isinstance(ip, IPAddr)) def testIpToName(self): - unittest.F2B.SkipIfNoNetwork() + #unittest.F2B.SkipIfNoNetwork() res = DNSUtils.ipToName('8.8.4.4') self.assertTrue(res.endswith(('.google', '.google.com'))) # same as above, but with IPAddr: @@ -1993,8 +1997,10 @@ class DNSUtilsNetworkTests(unittest.TestCase): self.assertEqual(res.addr, 167772160L) res = IPAddr('10.0.0.1', cidr=32L) self.assertEqual(res.addr, 167772161L) + self.assertTrue(res.isSingle) res = IPAddr('10.0.0.1', cidr=31L) self.assertEqual(res.addr, 167772160L) + self.assertFalse(res.isSingle) self.assertEqual(IPAddr('10.0.0.0').hexdump, '0a000000') self.assertEqual(IPAddr('1::2').hexdump, '00010000000000000000000000000002') @@ -2019,6 +2025,8 @@ class DNSUtilsNetworkTests(unittest.TestCase): def testIPAddr_InInet(self): ip4net = IPAddr('93.184.0.1/24') ip6net = IPAddr('2606:2800:220:1:248:1893:25c8:0/120') + self.assertFalse(ip4net.isSingle) + self.assertFalse(ip6net.isSingle) # ip4: self.assertTrue(IPAddr('93.184.0.1').isInNet(ip4net)) self.assertTrue(IPAddr('93.184.0.255').isInNet(ip4net)) @@ -2114,7 +2122,7 @@ class DNSUtilsNetworkTests(unittest.TestCase): ) def testIPAddr_CompareDNS(self): - unittest.F2B.SkipIfNoNetwork() + #unittest.F2B.SkipIfNoNetwork() ips = IPAddr('example.com') self.assertTrue(IPAddr("93.184.216.34").isInNet(ips)) self.assertTrue(IPAddr("2606:2800:220:1:248:1893:25c8:1946").isInNet(ips)) diff --git a/fail2ban/tests/utils.py b/fail2ban/tests/utils.py index f5ffb978..dc12a5be 100644 --- a/fail2ban/tests/utils.py +++ b/fail2ban/tests/utils.py @@ -39,7 +39,7 @@ from cStringIO import StringIO from functools import wraps from ..helpers import getLogger, str2LogLevel, getVerbosityFormat, uni_decode -from ..server.ipdns import DNSUtils +from ..server.ipdns import IPAddr, DNSUtils from ..server.mytime import MyTime from ..server.utils import Utils # for action_d.test_smtp : @@ -331,13 +331,21 @@ def initTests(opts): c.set('2001:db8::ffff', 'test-other') c.set('87.142.124.10', 'test-host') if unittest.F2B.no_network: # pragma: no cover - # precache all wrong dns to ip's used in test cases: + # precache all ip to dns used in test cases: + c.set('192.0.2.888', None) + c.set('8.8.4.4', 'dns.google') + c.set('8.8.4.4', 'dns.google') + # precache all dns to ip's used in test cases: c = DNSUtils.CACHE_nameToIp for i in ( ('999.999.999.999', set()), ('abcdef.abcdef', set()), ('192.168.0.', set()), ('failed.dns.ch', set()), + ('doh1.2.3.4.buga.xxxxx.yyy.invalid', set()), + ('1.2.3.4.buga.xxxxx.yyy.invalid', set()), + ('example.com', set([IPAddr('2606:2800:220:1:248:1893:25c8:1946'), IPAddr('93.184.216.34')])), + ('www.example.com', set([IPAddr('2606:2800:220:1:248:1893:25c8:1946'), IPAddr('93.184.216.34')])), ): c.set(*i) # if fast - precache all host names as localhost addresses (speed-up getSelfIPs/ignoreself): From 136781d627aa70ab88f3fac3b6df398c21dc7387 Mon Sep 17 00:00:00 2001 From: sebres Date: Wed, 8 Apr 2020 12:17:59 +0200 Subject: [PATCH 06/27] filter.d/sshd.conf: fixed regex for mode `extra` - "No authentication methods available" (supported seems to be optional now, gh-2682) --- config/filter.d/sshd.conf | 2 +- fail2ban/tests/config/filter.d/zzz-sshd-obsolete-multiline.conf | 2 +- fail2ban/tests/files/logs/sshd | 2 ++ 3 files changed, 4 insertions(+), 2 deletions(-) diff --git a/config/filter.d/sshd.conf b/config/filter.d/sshd.conf index 7a7f5e48..31e61b96 100644 --- a/config/filter.d/sshd.conf +++ b/config/filter.d/sshd.conf @@ -78,7 +78,7 @@ mdre-ddos = ^Did not receive identification string from # same as mdre-normal-other, but as failure (without ) and [preauth] only: mdre-ddos-other = ^(Connection (?:closed|reset)|Disconnected) (?:by|from)%(__authng_user)s %(__on_port_opt)s\s+\[preauth\]\s*$ -mdre-extra = ^Received disconnect from %(__on_port_opt)s:\s*14: No supported authentication methods available +mdre-extra = ^Received disconnect from %(__on_port_opt)s:\s*14: No(?: supported)? authentication methods available ^Unable to negotiate with %(__on_port_opt)s: no matching <__alg_match> found. ^Unable to negotiate a <__alg_match> ^no matching <__alg_match> found: diff --git a/fail2ban/tests/config/filter.d/zzz-sshd-obsolete-multiline.conf b/fail2ban/tests/config/filter.d/zzz-sshd-obsolete-multiline.conf index 4ff4ac68..ad8adeb6 100644 --- a/fail2ban/tests/config/filter.d/zzz-sshd-obsolete-multiline.conf +++ b/fail2ban/tests/config/filter.d/zzz-sshd-obsolete-multiline.conf @@ -60,7 +60,7 @@ mdre-ddos = ^%(__prefix_line_sl)sDid not receive identification string from %(__on_port_opt)s\s+\[preauth\]\s*$ ^%(__prefix_line_ml1)sSSH: Server;Ltype: (?:Authname|Version|Kex);Remote: -\d+;[A-Z]\w+:.*%(__prefix_line_ml2)sRead from socket failed: Connection reset by peer%(__suff)s$ -mdre-extra = ^%(__prefix_line_sl)sReceived disconnect from %(__on_port_opt)s:\s*14: No supported authentication methods available +mdre-extra = ^%(__prefix_line_sl)sReceived disconnect from %(__on_port_opt)s:\s*14: No(?: supported)? authentication methods available ^%(__prefix_line_sl)sUnable to negotiate with %(__on_port_opt)s: no matching <__alg_match> found. ^%(__prefix_line_ml1)sConnection from %(__on_port_opt)s%(__prefix_line_ml2)sUnable to negotiate a <__alg_match> ^%(__prefix_line_ml1)sConnection from %(__on_port_opt)s%(__prefix_line_ml2)sno matching <__alg_match> found: diff --git a/fail2ban/tests/files/logs/sshd b/fail2ban/tests/files/logs/sshd index e45ca90d..1bf9d913 100644 --- a/fail2ban/tests/files/logs/sshd +++ b/fail2ban/tests/files/logs/sshd @@ -330,6 +330,8 @@ Nov 25 01:34:12 srv sshd[123]: Received disconnect from 127.0.0.1: 14: No suppor Nov 25 01:35:13 srv sshd[123]: error: Received disconnect from 127.0.0.1: 14: No supported authentication methods available [preauth] # failJSON: { "time": "2004-11-25T01:35:14", "match": true , "host": "192.168.2.92", "desc": "Optional space after port" } Nov 25 01:35:14 srv sshd[3625]: error: Received disconnect from 192.168.2.92 port 1684:14: No supported authentication methods available [preauth] +# failJSON: { "time": "2004-11-25T01:35:15", "match": true , "host": "192.168.2.93", "desc": "No authentication methods available (supported is optional, gh-2682)" } +Nov 25 01:35:15 srv sshd[3626]: error: Received disconnect from 192.168.2.93 port 1883:14: No authentication methods available [preauth] # gh-1545: # failJSON: { "time": "2004-11-26T13:03:29", "match": true , "host": "192.0.2.1", "desc": "No matching cipher" } From 2912bc640b3335bffe20d03afa74d84c7a3c4f56 Mon Sep 17 00:00:00 2001 From: benrubson <6764151+benrubson@users.noreply.github.com> Date: Thu, 9 Apr 2020 16:42:08 +0200 Subject: [PATCH 07/27] New Gitlab jail --- config/filter.d/gitlab.conf | 6 ++++++ config/jail.conf | 4 ++++ fail2ban/tests/files/logs/gitlab | 5 +++++ 3 files changed, 15 insertions(+) create mode 100644 config/filter.d/gitlab.conf create mode 100644 fail2ban/tests/files/logs/gitlab diff --git a/config/filter.d/gitlab.conf b/config/filter.d/gitlab.conf new file mode 100644 index 00000000..0c614ae5 --- /dev/null +++ b/config/filter.d/gitlab.conf @@ -0,0 +1,6 @@ +# Fail2Ban filter for Gitlab +# Detecting unauthorized access to the Gitlab Web portal +# typically logged in /var/log/gitlab/gitlab-rails/application.log + +[Definition] +failregex = ^: Failed Login: username=.+ ip=$ diff --git a/config/jail.conf b/config/jail.conf index f7c84fac..e5d16656 100644 --- a/config/jail.conf +++ b/config/jail.conf @@ -821,6 +821,10 @@ udpport = 1200,27000,27001,27002,27003,27004,27005,27006,27007,27008,27009,27010 action = %(banaction)s[name=%(__name__)s-tcp, port="%(tcpport)s", protocol="tcp", chain="%(chain)s", actname=%(banaction)s-tcp] %(banaction)s[name=%(__name__)s-udp, port="%(udpport)s", protocol="udp", chain="%(chain)s", actname=%(banaction)s-udp] +[gitlab] +port = http,https +logpath = /var/log/gitlab/gitlab-rails/application.log + [bitwarden] port = http,https logpath = /home/*/bwdata/logs/identity/Identity/log.txt diff --git a/fail2ban/tests/files/logs/gitlab b/fail2ban/tests/files/logs/gitlab new file mode 100644 index 00000000..222df642 --- /dev/null +++ b/fail2ban/tests/files/logs/gitlab @@ -0,0 +1,5 @@ +# Access of unauthorized host in /var/log/gitlab/gitlab-rails/application.log +# failJSON: { "time": "2020-04-09T14:04:00", "match": true , "host": "80.10.11.12" } +2020-04-09T14:04:00.667Z: Failed Login: username=admin ip=80.10.11.12 +# failJSON: { "time": "2020-04-09T14:15:09", "match": true , "host": "80.10.11.12" } +2020-04-09T14:15:09.344Z: Failed Login: username=user name ip=80.10.11.12 From 78651de7e504bad84e73040dfd7ea530591d0385 Mon Sep 17 00:00:00 2001 From: "Sergey G. Brester" Date: Tue, 14 Apr 2020 12:25:18 +0200 Subject: [PATCH 08/27] Update ChangeLog --- ChangeLog | 1 + 1 file changed, 1 insertion(+) diff --git a/ChangeLog b/ChangeLog index 3781f467..c722db2b 100644 --- a/ChangeLog +++ b/ChangeLog @@ -42,6 +42,7 @@ ver. 0.10.6-dev (20??/??/??) - development edition should be interpolated in definition section (inside the filter-config, gh-2650) ### New Features +* new filter and jail for GitLab recognizing failed application logins (gh-2689) ### Enhancements * introduced new prefix `{UNB}` for `datepattern` to disable word boundaries in regex; From 7e3061e7ace0e973378a17de04a2692142e1a6c1 Mon Sep 17 00:00:00 2001 From: sebres Date: Wed, 15 Apr 2020 17:35:04 +0200 Subject: [PATCH 09/27] fail2ban.service systemd unit template: don't add user site directory to python system path (avoids accessing of `/root/.local` directory, prevents SE linux audit warning at daemon startup, gh-2688) --- files/fail2ban.service.in | 1 + 1 file changed, 1 insertion(+) diff --git a/files/fail2ban.service.in b/files/fail2ban.service.in index 5e540545..9a245c61 100644 --- a/files/fail2ban.service.in +++ b/files/fail2ban.service.in @@ -6,6 +6,7 @@ PartOf=iptables.service firewalld.service ip6tables.service ipset.service nftabl [Service] Type=simple +Environment="PYTHONNOUSERSITE=1" ExecStartPre=/bin/mkdir -p /run/fail2ban ExecStart=@BINDIR@/fail2ban-server -xf start # if should be logged in systemd journal, use following line or set logtarget to sysout in fail2ban.local From 06b46e92eb3fc11c29974fea1753a96edaf8ec20 Mon Sep 17 00:00:00 2001 From: sebres Date: Tue, 14 Apr 2020 21:34:30 +0200 Subject: [PATCH 10/27] jail.conf: don't specify `action` directly in jails (use `action_` or `banaction` instead); no mails-action added per default anymore (e. g. to allow that `action = %(action_mw)s` should be specified per jail or in default section in jail.local), closes gh-2357; ensure we've unique action name per jail (also if parameter `actname` is not set but name deviates from standard name, gh-2686); don't use %(banaction)s interpolation because it can be complex value (containing `[...]`), so would bother the action interpolation. --- ChangeLog | 6 ++++++ config/jail.conf | 30 ++++++++++++++---------------- fail2ban/client/actionreader.py | 10 +++++++--- 3 files changed, 27 insertions(+), 19 deletions(-) diff --git a/ChangeLog b/ChangeLog index 3781f467..6329d39a 100644 --- a/ChangeLog +++ b/ChangeLog @@ -38,6 +38,12 @@ ver. 0.10.6-dev (20??/??/??) - development edition * python 3.9 compatibility (and Travis CI support) * restoring a large number (500+ depending on files ulimit) of current bans when using PyPy fixed * manual ban is written to database, so can be restored by restart (gh-2647) +* `jail.conf`: don't specify `action` directly in jails (use `action_` or `banaction` instead) +* no mails-action added per default anymore (e. g. to allow that `action = %(action_mw)s` should be specified + per jail or in default section in jail.local), closes gh-2357 +* ensure we've unique action name per jail (also if parameter `actname` is not set but name deviates from standard name, gh-2686) +* don't use `%(banaction)s` interpolation because it can be complex value (containing `[...]` and/or quotes), + so would bother the action interpolation * `filter.d/common.conf`: avoid substitute of default values in related `lt_*` section, `__prefix_line` should be interpolated in definition section (inside the filter-config, gh-2650) diff --git a/config/jail.conf b/config/jail.conf index f7c84fac..bbf8740f 100644 --- a/config/jail.conf +++ b/config/jail.conf @@ -174,19 +174,19 @@ banaction_allports = iptables-allports action_ = %(banaction)s[name=%(__name__)s, bantime="%(bantime)s", port="%(port)s", protocol="%(protocol)s", chain="%(chain)s"] # ban & send an e-mail with whois report to the destemail. -action_mw = %(banaction)s[name=%(__name__)s, bantime="%(bantime)s", port="%(port)s", protocol="%(protocol)s", chain="%(chain)s"] +action_mw = %(action_)s %(mta)s-whois[name=%(__name__)s, sender="%(sender)s", dest="%(destemail)s", protocol="%(protocol)s", chain="%(chain)s"] # ban & send an e-mail with whois report and relevant log lines # to the destemail. -action_mwl = %(banaction)s[name=%(__name__)s, bantime="%(bantime)s", port="%(port)s", protocol="%(protocol)s", chain="%(chain)s"] +action_mwl = %(action_)s %(mta)s-whois-lines[name=%(__name__)s, sender="%(sender)s", dest="%(destemail)s", logpath="%(logpath)s", chain="%(chain)s"] # See the IMPORTANT note in action.d/xarf-login-attack for when to use this action # # ban & send a xarf e-mail to abuse contact of IP address and include relevant log lines # to the destemail. -action_xarf = %(banaction)s[name=%(__name__)s, bantime="%(bantime)s", port="%(port)s", protocol="%(protocol)s", chain="%(chain)s"] +action_xarf = %(action_)s xarf-login-attack[service=%(__name__)s, sender="%(sender)s", logpath="%(logpath)s", port="%(port)s"] # ban IP on CloudFlare & send an e-mail with whois report and relevant log lines @@ -333,7 +333,7 @@ maxretry = 1 [openhab-auth] filter = openhab -action = iptables-allports[name=NoAuthFailures] +banaction = %(banaction_allports)s logpath = /opt/openhab/logs/request.log @@ -706,8 +706,8 @@ logpath = /var/log/named/security.log [nsd] port = 53 -action = %(banaction)s[name=%(__name__)s-tcp, port="%(port)s", protocol="tcp", chain="%(chain)s", actname=%(banaction)s-tcp] - %(banaction)s[name=%(__name__)s-udp, port="%(port)s", protocol="udp", chain="%(chain)s", actname=%(banaction)s-udp] +action_ = %(default/action_)s[name=%(__name__)s-tcp, protocol="tcp"] + %(default/action_)s[name=%(__name__)s-udp, protocol="udp"] logpath = /var/log/nsd.log @@ -718,9 +718,8 @@ logpath = /var/log/nsd.log [asterisk] port = 5060,5061 -action = %(banaction)s[name=%(__name__)s-tcp, port="%(port)s", protocol="tcp", chain="%(chain)s", actname=%(banaction)s-tcp] - %(banaction)s[name=%(__name__)s-udp, port="%(port)s", protocol="udp", chain="%(chain)s", actname=%(banaction)s-udp] - %(mta)s-whois[name=%(__name__)s, dest="%(destemail)s"] +action_ = %(default/action_)s[name=%(__name__)s-tcp, protocol="tcp"] + %(default/action_)s[name=%(__name__)s-udp, protocol="udp"] logpath = /var/log/asterisk/messages maxretry = 10 @@ -728,9 +727,8 @@ maxretry = 10 [freeswitch] port = 5060,5061 -action = %(banaction)s[name=%(__name__)s-tcp, port="%(port)s", protocol="tcp", chain="%(chain)s", actname=%(banaction)s-tcp] - %(banaction)s[name=%(__name__)s-udp, port="%(port)s", protocol="udp", chain="%(chain)s", actname=%(banaction)s-udp] - %(mta)s-whois[name=%(__name__)s, dest="%(destemail)s"] +action_ = %(default/action_)s[name=%(__name__)s-tcp, protocol="tcp"] + %(default/action_)s[name=%(__name__)s-udp, protocol="udp"] logpath = /var/log/freeswitch.log maxretry = 10 @@ -818,8 +816,8 @@ logpath = /opt/cstrike/logs/L[0-9]*.log # Firewall: http://www.cstrike-planet.com/faq/6 tcpport = 27030,27031,27032,27033,27034,27035,27036,27037,27038,27039 udpport = 1200,27000,27001,27002,27003,27004,27005,27006,27007,27008,27009,27010,27011,27012,27013,27014,27015 -action = %(banaction)s[name=%(__name__)s-tcp, port="%(tcpport)s", protocol="tcp", chain="%(chain)s", actname=%(banaction)s-tcp] - %(banaction)s[name=%(__name__)s-udp, port="%(udpport)s", protocol="udp", chain="%(chain)s", actname=%(banaction)s-udp] +action_ = %(default/action_)s[name=%(__name__)s-tcp, port="%(tcpport)s", protocol="tcp"] + %(default/action_)s[name=%(__name__)s-udp, port="%(udpport)s", protocol="udp"] [bitwarden] port = http,https @@ -871,8 +869,8 @@ findtime = 1 [murmur] # AKA mumble-server port = 64738 -action = %(banaction)s[name=%(__name__)s-tcp, port="%(port)s", protocol=tcp, chain="%(chain)s", actname=%(banaction)s-tcp] - %(banaction)s[name=%(__name__)s-udp, port="%(port)s", protocol=udp, chain="%(chain)s", actname=%(banaction)s-udp] +action_ = %(default/action_)s[name=%(__name__)s-tcp, protocol="tcp"] + %(default/action_)s[name=%(__name__)s-udp, protocol="udp"] logpath = /var/log/mumble-server/mumble-server.log diff --git a/fail2ban/client/actionreader.py b/fail2ban/client/actionreader.py index e5bee154..131e37cb 100644 --- a/fail2ban/client/actionreader.py +++ b/fail2ban/client/actionreader.py @@ -52,13 +52,17 @@ class ActionReader(DefinitionInitConfigReader): } def __init__(self, file_, jailName, initOpts, **kwargs): + # always supply jail name as name parameter if not specified in options: + n = initOpts.get("name") + if n is None: + initOpts["name"] = n = jailName actname = initOpts.get("actname") if actname is None: actname = file_ + # ensure we've unique action name per jail: + if n != jailName: + actname += n[len(jailName):] if n.startswith(jailName) else '-' + n initOpts["actname"] = actname - # always supply jail name as name parameter if not specified in options: - if initOpts.get("name") is None: - initOpts["name"] = jailName self._name = actname DefinitionInitConfigReader.__init__( self, file_, jailName, initOpts, **kwargs) From affd9cef5f2ddb5c596e1aa3789ba18b5c987ba1 Mon Sep 17 00:00:00 2001 From: sebres Date: Tue, 21 Apr 2020 13:32:17 +0200 Subject: [PATCH 11/27] filter.d/courier-smtp.conf: prefregex extended to consider port in log-message (closes gh-2697) --- ChangeLog | 1 + config/filter.d/courier-smtp.conf | 2 +- fail2ban/tests/files/logs/courier-smtp | 2 ++ 3 files changed, 4 insertions(+), 1 deletion(-) diff --git a/ChangeLog b/ChangeLog index 6329d39a..b001ae58 100644 --- a/ChangeLog +++ b/ChangeLog @@ -46,6 +46,7 @@ ver. 0.10.6-dev (20??/??/??) - development edition so would bother the action interpolation * `filter.d/common.conf`: avoid substitute of default values in related `lt_*` section, `__prefix_line` should be interpolated in definition section (inside the filter-config, gh-2650) +* `filter.d/courier-smtp.conf`: prefregex extended to consider port in log-message (gh-2697) ### New Features diff --git a/config/filter.d/courier-smtp.conf b/config/filter.d/courier-smtp.conf index 888753c4..4b2b8d87 100644 --- a/config/filter.d/courier-smtp.conf +++ b/config/filter.d/courier-smtp.conf @@ -12,7 +12,7 @@ before = common.conf _daemon = courieresmtpd -prefregex = ^%(__prefix_line)serror,relay=,.+$ +prefregex = ^%(__prefix_line)serror,relay=,(?:port=\d+,)?.+$ failregex = ^[^:]*: 550 User (<.*> )?unknown\.?$ ^msg="535 Authentication failed\.",cmd:( AUTH \S+)?( [0-9a-zA-Z\+/=]+)?(?: \S+)$ diff --git a/fail2ban/tests/files/logs/courier-smtp b/fail2ban/tests/files/logs/courier-smtp index ab99d322..cea73073 100644 --- a/fail2ban/tests/files/logs/courier-smtp +++ b/fail2ban/tests/files/logs/courier-smtp @@ -12,3 +12,5 @@ Nov 21 23:16:17 server courieresmtpd: error,relay=::ffff:1.2.3.4,from=<>,to=<>: Aug 14 12:51:04 HOSTNAME courieresmtpd: error,relay=::ffff:1.2.3.4,from=,to=: 550 User unknown. # failJSON: { "time": "2004-08-14T12:51:04", "match": true , "host": "1.2.3.4" } Aug 14 12:51:04 mail.server courieresmtpd[26762]: error,relay=::ffff:1.2.3.4,msg="535 Authentication failed.",cmd: AUTH PLAIN AAAAABBBBCCCCWxlZA== admin +# failJSON: { "time": "2004-08-14T12:51:05", "match": true , "host": "192.0.2.3" } +Aug 14 12:51:05 mail.server courieresmtpd[425070]: error,relay=::ffff:192.0.2.3,port=43632,msg="535 Authentication failed.",cmd: AUTH LOGIN PlcmSpIp@example.com From 6b90ca820f5244fd88e8b407419da359d5d068b9 Mon Sep 17 00:00:00 2001 From: sebres Date: Thu, 23 Apr 2020 13:08:24 +0200 Subject: [PATCH 12/27] filter.d/traefik-auth.conf: filter extended with parameter mode (`normal`, `ddos`, `aggressive`) to handle the match of username differently: - `normal`: matches 401 with supplied username only - `ddos`: matches 401 without supplied username only - `aggressive`: matches 401 and any variant (with and without username) closes gh-2693 --- ChangeLog | 5 +++++ config/filter.d/traefik-auth.conf | 22 +++++++++++++++++++++- fail2ban/tests/files/logs/traefik-auth | 17 +++++++++++++++++ 3 files changed, 43 insertions(+), 1 deletion(-) diff --git a/ChangeLog b/ChangeLog index b001ae58..f5d3dd6d 100644 --- a/ChangeLog +++ b/ChangeLog @@ -47,6 +47,11 @@ ver. 0.10.6-dev (20??/??/??) - development edition * `filter.d/common.conf`: avoid substitute of default values in related `lt_*` section, `__prefix_line` should be interpolated in definition section (inside the filter-config, gh-2650) * `filter.d/courier-smtp.conf`: prefregex extended to consider port in log-message (gh-2697) +* `filter.d/traefik-auth.conf`: filter extended with parameter mode (`normal`, `ddos`, `aggressive`) to handle + the match of username differently (gh-2693): + - `normal`: matches 401 with supplied username only + - `ddos`: matches 401 without supplied username only + - `aggressive`: matches 401 and any variant (with and without username) ### New Features diff --git a/config/filter.d/traefik-auth.conf b/config/filter.d/traefik-auth.conf index 8321a138..8022fee1 100644 --- a/config/filter.d/traefik-auth.conf +++ b/config/filter.d/traefik-auth.conf @@ -51,6 +51,26 @@ [Definition] -failregex = ^ \- (?!- )\S+ \[\] \"(GET|POST|HEAD) [^\"]+\" 401\b +# Parameter "method" can be used to specifiy request method +req-method = \S+ +# Usage example (for jail.local): +# filter = traefik-auth[req-method="GET|POST|HEAD"] + +failregex = ^ \- > \[\] \"(?:) [^\"]+\" 401\b ignoreregex = + +# Parameter "mode": normal (default), ddos or aggressive +# Usage example (for jail.local): +# [traefik-auth] +# mode = aggressive +# # or another jail (rewrite filter parameters of jail): +# [traefik-auth-ddos] +# filter = traefik-auth[mode=ddos] +# +mode = normal + +# part of failregex matches user name (must be available in normal mode, must be empty in ddos mode, and both for aggressive mode): +usrre-normal = (?!- )\S+ +usrre-ddos = - +usrre-aggressive = \S+ \ No newline at end of file diff --git a/fail2ban/tests/files/logs/traefik-auth b/fail2ban/tests/files/logs/traefik-auth index 3e7a8987..edfe7306 100644 --- a/fail2ban/tests/files/logs/traefik-auth +++ b/fail2ban/tests/files/logs/traefik-auth @@ -1,6 +1,23 @@ +# filterOptions: [{"mode": "normal"}] + # failJSON: { "match": false } 10.0.0.2 - - [18/Nov/2018:21:34:30 +0000] "GET /dashboard/ HTTP/2.0" 401 17 "-" "Mozilla/5.0 (X11; Linux x86_64; rv:57.0) Gecko/20100101 Firefox/57.0" 72 "Auth for frontend-Host-traefik-0" "/dashboard/" 0ms + +# filterOptions: [{"mode": "ddos"}] + +# failJSON: { "match": false } +10.0.0.2 - username [18/Nov/2018:21:34:30 +0000] "GET /dashboard/ HTTP/2.0" 401 17 "-" "Mozilla/5.0 (X11; Linux x86_64; rv:57.0) Gecko/20100101 Firefox/57.0" 72 "Auth for frontend-Host-traefik-0" "/dashboard/" 0ms + +# filterOptions: [{"mode": "normal"}, {"mode": "aggressive"}] + # failJSON: { "time": "2018-11-18T22:34:34", "match": true , "host": "10.0.0.2" } 10.0.0.2 - username [18/Nov/2018:21:34:34 +0000] "GET /dashboard/ HTTP/2.0" 401 17 "-" "Mozilla/5.0 (X11; Linux x86_64; rv:57.0) Gecko/20100101 Firefox/57.0" 72 "Auth for frontend-Host-traefik-0" "/dashboard/" 0ms +# failJSON: { "time": "2018-11-18T22:34:34", "match": true , "host": "10.0.0.2", "desc": "other request method" } +10.0.0.2 - username [18/Nov/2018:21:34:34 +0000] "TRACE /dashboard/ HTTP/2.0" 401 17 "-" "Mozilla/5.0 (X11; Linux x86_64; rv:57.0) Gecko/20100101 Firefox/57.0" 72 "Auth for frontend-Host-traefik-0" "/dashboard/" 0ms # failJSON: { "match": false } 10.0.0.2 - username [27/Nov/2018:23:33:31 +0000] "GET /dashboard/ HTTP/2.0" 200 716 "-" "Mozilla/5.0 (X11; Linux x86_64; rv:57.0) Gecko/20100101 Firefox/57.0" 118 "Host-traefik-0" "/dashboard/" 4ms + +# filterOptions: [{"mode": "ddos"}, {"mode": "aggressive"}] + +# failJSON: { "time": "2018-11-18T22:34:30", "match": true , "host": "10.0.0.2" } +10.0.0.2 - - [18/Nov/2018:21:34:30 +0000] "GET /dashboard/ HTTP/2.0" 401 17 "-" "Mozilla/5.0 (X11; Linux x86_64; rv:57.0) Gecko/20100101 Firefox/57.0" 72 "Auth for frontend-Host-traefik-0" "/dashboard/" 0ms From 87a1a2f1a112ce5bfef8357758435266c4f6b43d Mon Sep 17 00:00:00 2001 From: sebres Date: Sat, 25 Apr 2020 14:52:38 +0200 Subject: [PATCH 13/27] action.d/*-ipset*.conf: several ipset actions fixed (no timeout per default anymore), so no discrepancy between ipset and fail2ban (removal from ipset will be managed by fail2ban only) --- config/action.d/firewallcmd-ipset.conf | 22 +++++++++++------ .../iptables-ipset-proto6-allports.conf | 24 ++++++++++++------- config/action.d/iptables-ipset-proto6.conf | 24 ++++++++++++------- config/action.d/shorewall-ipset-proto6.conf | 22 ++++++++++------- 4 files changed, 61 insertions(+), 31 deletions(-) diff --git a/config/action.d/firewallcmd-ipset.conf b/config/action.d/firewallcmd-ipset.conf index dcf20375..9dd9fbb2 100644 --- a/config/action.d/firewallcmd-ipset.conf +++ b/config/action.d/firewallcmd-ipset.conf @@ -18,7 +18,7 @@ before = firewallcmd-common.conf [Definition] -actionstart = ipset create hash:ip timeout +actionstart = ipset create hash:ip timeout firewall-cmd --direct --add-rule filter 0 -m set --match-set src -j actionflush = ipset flush @@ -27,7 +27,7 @@ actionstop = firewall-cmd --direct --remove-rule filter 0 ipset destroy -actionban = ipset add timeout -exist +actionban = ipset add timeout -exist actionunban = ipset del -exist @@ -40,11 +40,19 @@ actionunban = ipset del -exist # chain = INPUT_direct -# Option: bantime -# Notes: specifies the bantime in seconds (handled internally rather than by fail2ban) -# Values: [ NUM ] Default: 600 +# Option: default-timeout +# Notes: specifies default timeout in seconds (handled default ipset timeout only) +# Values: [ NUM ] Default: 0 (no timeout, managed by fail2ban by unban) +default-timeout = 0 -bantime = 600 +# Option: timeout +# Notes: specifies ticket timeout (handled ipset timeout only) +# Values: [ NUM ] Default: 0 (managed by fail2ban by unban) +timeout = 0 + +# expresion to caclulate timeout from bantime, example: +# banaction = %(known/banaction)s[timeout=''] +timeout-bantime = $([ "" -le 2147483 ] && echo "" || echo 0) # Option: actiontype # Notes.: defines additions to the blocking rule @@ -69,7 +77,7 @@ familyopt = [Init?family=inet6] ipmset = f2b-6 -familyopt = family inet6 +familyopt = family inet6 # DEV NOTES: diff --git a/config/action.d/iptables-ipset-proto6-allports.conf b/config/action.d/iptables-ipset-proto6-allports.conf index dc7d63a7..4f200db0 100644 --- a/config/action.d/iptables-ipset-proto6-allports.conf +++ b/config/action.d/iptables-ipset-proto6-allports.conf @@ -26,7 +26,7 @@ before = iptables-common.conf # Notes.: command executed on demand at the first ban (or at the start of Fail2Ban if actionstart_on_demand is set to false). # Values: CMD # -actionstart = ipset create hash:ip timeout +actionstart = ipset create hash:ip timeout -I -m set --match-set src -j # Option: actionflush @@ -49,7 +49,7 @@ actionstop = -D -m set --match-set src -j timeout -exist +actionban = ipset add timeout -exist # Option: actionunban # Notes.: command executed when unbanning an IP. Take care that the @@ -61,11 +61,19 @@ actionunban = ipset del -exist [Init] -# Option: bantime -# Notes: specifies the bantime in seconds (handled internally rather than by fail2ban) -# Values: [ NUM ] Default: 600 -# -bantime = 600 +# Option: default-timeout +# Notes: specifies default timeout in seconds (handled default ipset timeout only) +# Values: [ NUM ] Default: 0 (no timeout, managed by fail2ban by unban) +default-timeout = 0 + +# Option: timeout +# Notes: specifies ticket timeout (handled ipset timeout only) +# Values: [ NUM ] Default: 0 (managed by fail2ban by unban) +timeout = 0 + +# expresion to caclulate timeout from bantime, example: +# banaction = %(known/banaction)s[timeout=''] +timeout-bantime = $([ "" -le 2147483 ] && echo "" || echo 0) ipmset = f2b- familyopt = @@ -74,4 +82,4 @@ familyopt = [Init?family=inet6] ipmset = f2b-6 -familyopt = family inet6 +familyopt = family inet6 diff --git a/config/action.d/iptables-ipset-proto6.conf b/config/action.d/iptables-ipset-proto6.conf index f88777b8..8956ec6a 100644 --- a/config/action.d/iptables-ipset-proto6.conf +++ b/config/action.d/iptables-ipset-proto6.conf @@ -26,7 +26,7 @@ before = iptables-common.conf # Notes.: command executed on demand at the first ban (or at the start of Fail2Ban if actionstart_on_demand is set to false). # Values: CMD # -actionstart = ipset create hash:ip timeout +actionstart = ipset create hash:ip timeout -I -p -m multiport --dports -m set --match-set src -j # Option: actionflush @@ -49,7 +49,7 @@ actionstop = -D -p -m multiport --dports -m # Tags: See jail.conf(5) man page # Values: CMD # -actionban = ipset add timeout -exist +actionban = ipset add timeout -exist # Option: actionunban # Notes.: command executed when unbanning an IP. Take care that the @@ -61,11 +61,19 @@ actionunban = ipset del -exist [Init] -# Option: bantime -# Notes: specifies the bantime in seconds (handled internally rather than by fail2ban) -# Values: [ NUM ] Default: 600 -# -bantime = 600 +# Option: default-timeout +# Notes: specifies default timeout in seconds (handled default ipset timeout only) +# Values: [ NUM ] Default: 0 (no timeout, managed by fail2ban by unban) +default-timeout = 0 + +# Option: timeout +# Notes: specifies ticket timeout (handled ipset timeout only) +# Values: [ NUM ] Default: 0 (managed by fail2ban by unban) +timeout = 0 + +# expresion to caclulate timeout from bantime, example: +# banaction = %(known/banaction)s[timeout=''] +timeout-bantime = $([ "" -le 2147483 ] && echo "" || echo 0) ipmset = f2b- familyopt = @@ -74,4 +82,4 @@ familyopt = [Init?family=inet6] ipmset = f2b-6 -familyopt = family inet6 +familyopt = family inet6 diff --git a/config/action.d/shorewall-ipset-proto6.conf b/config/action.d/shorewall-ipset-proto6.conf index fc7dd24e..cbcc5524 100644 --- a/config/action.d/shorewall-ipset-proto6.conf +++ b/config/action.d/shorewall-ipset-proto6.conf @@ -51,7 +51,7 @@ # Values: CMD # actionstart = if ! ipset -quiet -name list f2b- >/dev/null; - then ipset -quiet -exist create f2b- hash:ip timeout ; + then ipset -quiet -exist create f2b- hash:ip timeout ; fi # Option: actionstop @@ -66,7 +66,7 @@ actionstop = ipset flush f2b- # Tags: See jail.conf(5) man page # Values: CMD # -actionban = ipset add f2b- timeout -exist +actionban = ipset add f2b- timeout -exist # Option: actionunban # Notes.: command executed when unbanning an IP. Take care that the @@ -76,10 +76,16 @@ actionban = ipset add f2b- timeout -exist # actionunban = ipset del f2b- -exist -[Init] +# Option: default-timeout +# Notes: specifies default timeout in seconds (handled default ipset timeout only) +# Values: [ NUM ] Default: 0 (no timeout, managed by fail2ban by unban) +default-timeout = 0 -# Option: bantime -# Notes: specifies the bantime in seconds (handled internally rather than by fail2ban) -# Values: [ NUM ] Default: 600 -# -bantime = 600 +# Option: timeout +# Notes: specifies ticket timeout (handled ipset timeout only) +# Values: [ NUM ] Default: 0 (managed by fail2ban by unban) +timeout = 0 + +# expresion to caclulate timeout from bantime, example: +# banaction = %(known/banaction)s[timeout=''] +timeout-bantime = $([ "" -le 2147483 ] && echo "" || echo 0) From 12be3ed77d9cbe0ef6fed4ac26f9859527f9c448 Mon Sep 17 00:00:00 2001 From: sebres Date: Sat, 25 Apr 2020 15:17:42 +0200 Subject: [PATCH 14/27] test cases fixed --- fail2ban/tests/servertestcase.py | 40 ++++++++++++++++---------------- 1 file changed, 20 insertions(+), 20 deletions(-) diff --git a/fail2ban/tests/servertestcase.py b/fail2ban/tests/servertestcase.py index a51c4f85..b771ab50 100644 --- a/fail2ban/tests/servertestcase.py +++ b/fail2ban/tests/servertestcase.py @@ -1509,14 +1509,14 @@ class ServerConfigReaderTests(LogCaptureTestCase): ), }), # iptables-ipset-proto6 -- - ('j-w-iptables-ipset', 'iptables-ipset-proto6[name=%(__name__)s, bantime="10m", port="http", protocol="tcp", chain=""]', { + ('j-w-iptables-ipset', 'iptables-ipset-proto6[name=%(__name__)s, port="http", protocol="tcp", chain=""]', { 'ip4': (' f2b-j-w-iptables-ipset ',), 'ip6': (' f2b-j-w-iptables-ipset6 ',), 'ip4-start': ( - "`ipset create f2b-j-w-iptables-ipset hash:ip timeout 600`", + "`ipset create f2b-j-w-iptables-ipset hash:ip timeout 0 `", "`iptables -w -I INPUT -p tcp -m multiport --dports http -m set --match-set f2b-j-w-iptables-ipset src -j REJECT --reject-with icmp-port-unreachable`", ), 'ip6-start': ( - "`ipset create f2b-j-w-iptables-ipset6 hash:ip timeout 600 family inet6`", + "`ipset create f2b-j-w-iptables-ipset6 hash:ip timeout 0 family inet6`", "`ip6tables -w -I INPUT -p tcp -m multiport --dports http -m set --match-set f2b-j-w-iptables-ipset6 src -j REJECT --reject-with icmp6-port-unreachable`", ), 'flush': ( @@ -1532,27 +1532,27 @@ class ServerConfigReaderTests(LogCaptureTestCase): "`ipset destroy f2b-j-w-iptables-ipset6`", ), 'ip4-ban': ( - r"`ipset add f2b-j-w-iptables-ipset 192.0.2.1 timeout 600 -exist`", + r"`ipset add f2b-j-w-iptables-ipset 192.0.2.1 timeout 0 -exist`", ), 'ip4-unban': ( r"`ipset del f2b-j-w-iptables-ipset 192.0.2.1 -exist`", ), 'ip6-ban': ( - r"`ipset add f2b-j-w-iptables-ipset6 2001:db8:: timeout 600 -exist`", + r"`ipset add f2b-j-w-iptables-ipset6 2001:db8:: timeout 0 -exist`", ), 'ip6-unban': ( r"`ipset del f2b-j-w-iptables-ipset6 2001:db8:: -exist`", ), }), # iptables-ipset-proto6-allports -- - ('j-w-iptables-ipset-ap', 'iptables-ipset-proto6-allports[name=%(__name__)s, bantime="10m", chain=""]', { + ('j-w-iptables-ipset-ap', 'iptables-ipset-proto6-allports[name=%(__name__)s, chain=""]', { 'ip4': (' f2b-j-w-iptables-ipset-ap ',), 'ip6': (' f2b-j-w-iptables-ipset-ap6 ',), 'ip4-start': ( - "`ipset create f2b-j-w-iptables-ipset-ap hash:ip timeout 600`", + "`ipset create f2b-j-w-iptables-ipset-ap hash:ip timeout 0 `", "`iptables -w -I INPUT -m set --match-set f2b-j-w-iptables-ipset-ap src -j REJECT --reject-with icmp-port-unreachable`", ), 'ip6-start': ( - "`ipset create f2b-j-w-iptables-ipset-ap6 hash:ip timeout 600 family inet6`", + "`ipset create f2b-j-w-iptables-ipset-ap6 hash:ip timeout 0 family inet6`", "`ip6tables -w -I INPUT -m set --match-set f2b-j-w-iptables-ipset-ap6 src -j REJECT --reject-with icmp6-port-unreachable`", ), 'flush': ( @@ -1568,13 +1568,13 @@ class ServerConfigReaderTests(LogCaptureTestCase): "`ipset destroy f2b-j-w-iptables-ipset-ap6`", ), 'ip4-ban': ( - r"`ipset add f2b-j-w-iptables-ipset-ap 192.0.2.1 timeout 600 -exist`", + r"`ipset add f2b-j-w-iptables-ipset-ap 192.0.2.1 timeout 0 -exist`", ), 'ip4-unban': ( r"`ipset del f2b-j-w-iptables-ipset-ap 192.0.2.1 -exist`", ), 'ip6-ban': ( - r"`ipset add f2b-j-w-iptables-ipset-ap6 2001:db8:: timeout 600 -exist`", + r"`ipset add f2b-j-w-iptables-ipset-ap6 2001:db8:: timeout 0 -exist`", ), 'ip6-unban': ( r"`ipset del f2b-j-w-iptables-ipset-ap6 2001:db8:: -exist`", @@ -1852,14 +1852,14 @@ class ServerConfigReaderTests(LogCaptureTestCase): ), }), # firewallcmd-ipset (multiport) -- - ('j-w-fwcmd-ipset', 'firewallcmd-ipset[name=%(__name__)s, bantime="10m", port="http", protocol="tcp", chain=""]', { + ('j-w-fwcmd-ipset', 'firewallcmd-ipset[name=%(__name__)s, port="http", protocol="tcp", chain=""]', { 'ip4': (' f2b-j-w-fwcmd-ipset ',), 'ip6': (' f2b-j-w-fwcmd-ipset6 ',), 'ip4-start': ( - "`ipset create f2b-j-w-fwcmd-ipset hash:ip timeout 600`", + "`ipset create f2b-j-w-fwcmd-ipset hash:ip timeout 0 `", "`firewall-cmd --direct --add-rule ipv4 filter INPUT_direct 0 -p tcp -m multiport --dports http -m set --match-set f2b-j-w-fwcmd-ipset src -j REJECT --reject-with icmp-port-unreachable`", ), 'ip6-start': ( - "`ipset create f2b-j-w-fwcmd-ipset6 hash:ip timeout 600 family inet6`", + "`ipset create f2b-j-w-fwcmd-ipset6 hash:ip timeout 0 family inet6`", "`firewall-cmd --direct --add-rule ipv6 filter INPUT_direct 0 -p tcp -m multiport --dports http -m set --match-set f2b-j-w-fwcmd-ipset6 src -j REJECT --reject-with icmp6-port-unreachable`", ), 'flush': ( @@ -1875,27 +1875,27 @@ class ServerConfigReaderTests(LogCaptureTestCase): "`ipset destroy f2b-j-w-fwcmd-ipset6`", ), 'ip4-ban': ( - r"`ipset add f2b-j-w-fwcmd-ipset 192.0.2.1 timeout 600 -exist`", + r"`ipset add f2b-j-w-fwcmd-ipset 192.0.2.1 timeout 0 -exist`", ), 'ip4-unban': ( r"`ipset del f2b-j-w-fwcmd-ipset 192.0.2.1 -exist`", ), 'ip6-ban': ( - r"`ipset add f2b-j-w-fwcmd-ipset6 2001:db8:: timeout 600 -exist`", + r"`ipset add f2b-j-w-fwcmd-ipset6 2001:db8:: timeout 0 -exist`", ), 'ip6-unban': ( r"`ipset del f2b-j-w-fwcmd-ipset6 2001:db8:: -exist`", ), }), # firewallcmd-ipset (allports) -- - ('j-w-fwcmd-ipset-ap', 'firewallcmd-ipset[name=%(__name__)s, bantime="10m", actiontype=, protocol="tcp", chain=""]', { + ('j-w-fwcmd-ipset-ap', 'firewallcmd-ipset[name=%(__name__)s, actiontype=, protocol="tcp", chain=""]', { 'ip4': (' f2b-j-w-fwcmd-ipset-ap ',), 'ip6': (' f2b-j-w-fwcmd-ipset-ap6 ',), 'ip4-start': ( - "`ipset create f2b-j-w-fwcmd-ipset-ap hash:ip timeout 600`", + "`ipset create f2b-j-w-fwcmd-ipset-ap hash:ip timeout 0 `", "`firewall-cmd --direct --add-rule ipv4 filter INPUT_direct 0 -p tcp -m set --match-set f2b-j-w-fwcmd-ipset-ap src -j REJECT --reject-with icmp-port-unreachable`", ), 'ip6-start': ( - "`ipset create f2b-j-w-fwcmd-ipset-ap6 hash:ip timeout 600 family inet6`", + "`ipset create f2b-j-w-fwcmd-ipset-ap6 hash:ip timeout 0 family inet6`", "`firewall-cmd --direct --add-rule ipv6 filter INPUT_direct 0 -p tcp -m set --match-set f2b-j-w-fwcmd-ipset-ap6 src -j REJECT --reject-with icmp6-port-unreachable`", ), 'flush': ( @@ -1911,13 +1911,13 @@ class ServerConfigReaderTests(LogCaptureTestCase): "`ipset destroy f2b-j-w-fwcmd-ipset-ap6`", ), 'ip4-ban': ( - r"`ipset add f2b-j-w-fwcmd-ipset-ap 192.0.2.1 timeout 600 -exist`", + r"`ipset add f2b-j-w-fwcmd-ipset-ap 192.0.2.1 timeout 0 -exist`", ), 'ip4-unban': ( r"`ipset del f2b-j-w-fwcmd-ipset-ap 192.0.2.1 -exist`", ), 'ip6-ban': ( - r"`ipset add f2b-j-w-fwcmd-ipset-ap6 2001:db8:: timeout 600 -exist`", + r"`ipset add f2b-j-w-fwcmd-ipset-ap6 2001:db8:: timeout 0 -exist`", ), 'ip6-unban': ( r"`ipset del f2b-j-w-fwcmd-ipset-ap6 2001:db8:: -exist`", From da1652d0d71074a2af8ae69512fbca24fafdf385 Mon Sep 17 00:00:00 2001 From: "Sergey G. Brester" Date: Sun, 26 Apr 2020 12:26:55 +0200 Subject: [PATCH 15/27] Update ChangeLog --- ChangeLog | 2 ++ 1 file changed, 2 insertions(+) diff --git a/ChangeLog b/ChangeLog index f5d3dd6d..2ae56d99 100644 --- a/ChangeLog +++ b/ChangeLog @@ -44,6 +44,8 @@ ver. 0.10.6-dev (20??/??/??) - development edition * ensure we've unique action name per jail (also if parameter `actname` is not set but name deviates from standard name, gh-2686) * don't use `%(banaction)s` interpolation because it can be complex value (containing `[...]` and/or quotes), so would bother the action interpolation +* `action.d/*-ipset*.conf`: several ipset actions fixed (no timeout per default anymore), so no discrepancy + between ipset and fail2ban (removal from ipset will be managed by fail2ban only, gh-2703) * `filter.d/common.conf`: avoid substitute of default values in related `lt_*` section, `__prefix_line` should be interpolated in definition section (inside the filter-config, gh-2650) * `filter.d/courier-smtp.conf`: prefregex extended to consider port in log-message (gh-2697) From 5da2422f616f6245982f38693df2e61ccf24dbb4 Mon Sep 17 00:00:00 2001 From: Ilya Date: Wed, 11 Mar 2020 14:43:45 +0300 Subject: [PATCH 16/27] Fix actionunban Add command to remove new line character. Needed for working removing rule from cloudflare firewall. --- config/action.d/cloudflare.conf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config/action.d/cloudflare.conf b/config/action.d/cloudflare.conf index 1c48a37f..70e5ee3f 100644 --- a/config/action.d/cloudflare.conf +++ b/config/action.d/cloudflare.conf @@ -60,7 +60,7 @@ actionban = curl -s -o /dev/null -X POST -H 'X-Auth-Email: ' -H 'X-Auth- # API v4 actionunban = curl -s -o /dev/null -X DELETE -H 'X-Auth-Email: ' -H 'X-Auth-Key: ' \ https://api.cloudflare.com/client/v4/user/firewall/access_rules/rules/$(curl -s -X GET -H 'X-Auth-Email: ' -H 'X-Auth-Key: ' \ - 'https://api.cloudflare.com/client/v4/user/firewall/access_rules/rules?mode=block&configuration_target=ip&configuration_value=&page=1&per_page=1' | cut -d'"' -f6) + 'https://api.cloudflare.com/client/v4/user/firewall/access_rules/rules?mode=block&configuration_target=ip&configuration_value=&page=1&per_page=1' | tr -d '\n' | cut -d'"' -f6) [Init] From 8b3b9addd10dec9fec47b8f7fd3a971c326fe430 Mon Sep 17 00:00:00 2001 From: Ilya Date: Fri, 20 Mar 2020 13:52:17 +0300 Subject: [PATCH 17/27] Change tool from 'cut' to 'sed' Sed regex was tested - it works. --- config/action.d/cloudflare.conf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config/action.d/cloudflare.conf b/config/action.d/cloudflare.conf index 70e5ee3f..d00db98b 100644 --- a/config/action.d/cloudflare.conf +++ b/config/action.d/cloudflare.conf @@ -60,7 +60,7 @@ actionban = curl -s -o /dev/null -X POST -H 'X-Auth-Email: ' -H 'X-Auth- # API v4 actionunban = curl -s -o /dev/null -X DELETE -H 'X-Auth-Email: ' -H 'X-Auth-Key: ' \ https://api.cloudflare.com/client/v4/user/firewall/access_rules/rules/$(curl -s -X GET -H 'X-Auth-Email: ' -H 'X-Auth-Key: ' \ - 'https://api.cloudflare.com/client/v4/user/firewall/access_rules/rules?mode=block&configuration_target=ip&configuration_value=&page=1&per_page=1' | tr -d '\n' | cut -d'"' -f6) + 'https://api.cloudflare.com/client/v4/user/firewall/access_rules/rules?mode=block&configuration_target=ip&configuration_value=&page=1&per_page=1' | tr -d '\n' | sed -nE 's/^.*"result"\s*:\s*\[\s*\{\s*"id"\s*:\s*"([^"]+)".*$/\1/p' ) [Init] From 852670bc99be2f30a64dd5d096d5ff375b6a2e3a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Viktor=20Sz=C3=A9pe?= Date: Sun, 27 May 2018 08:15:33 +0200 Subject: [PATCH 18/27] CloudFlare started to indent their API responses We need to use https://github.com/stedolan/jq to parse it. --- config/action.d/cloudflare.conf | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/config/action.d/cloudflare.conf b/config/action.d/cloudflare.conf index d00db98b..27a0b6b5 100644 --- a/config/action.d/cloudflare.conf +++ b/config/action.d/cloudflare.conf @@ -5,7 +5,7 @@ # # Please set jail.local's permission to 640 because it contains your CF API key. # -# This action depends on curl. +# This action depends on curl and jq. # Referenced from http://www.normyee.net/blog/2012/02/02/adding-cloudflare-support-to-fail2ban by NORM YEE # # To get your CloudFlare API Key: https://www.cloudflare.com/a/account/my-account @@ -60,7 +60,7 @@ actionban = curl -s -o /dev/null -X POST -H 'X-Auth-Email: ' -H 'X-Auth- # API v4 actionunban = curl -s -o /dev/null -X DELETE -H 'X-Auth-Email: ' -H 'X-Auth-Key: ' \ https://api.cloudflare.com/client/v4/user/firewall/access_rules/rules/$(curl -s -X GET -H 'X-Auth-Email: ' -H 'X-Auth-Key: ' \ - 'https://api.cloudflare.com/client/v4/user/firewall/access_rules/rules?mode=block&configuration_target=ip&configuration_value=&page=1&per_page=1' | tr -d '\n' | sed -nE 's/^.*"result"\s*:\s*\[\s*\{\s*"id"\s*:\s*"([^"]+)".*$/\1/p' ) + 'https://api.cloudflare.com/client/v4/user/firewall/access_rules/rules?mode=block&configuration_target=ip&configuration_value=&page=1&per_page=1' | jq -r '.result[0].configuration.value') [Init] From 5b8fc3b51a203a7428dd66f07fb8480ec894860d Mon Sep 17 00:00:00 2001 From: "Sergey G. Brester" Date: Thu, 6 Dec 2018 14:35:17 +0100 Subject: [PATCH 19/27] cloudflare: fixes ip to id conversion by unban using jq normalized URIs and parameters, notes gets a jail-name (should be possible to differentiate the same IP across several jails) --- config/action.d/cloudflare.conf | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) diff --git a/config/action.d/cloudflare.conf b/config/action.d/cloudflare.conf index 27a0b6b5..5125777c 100644 --- a/config/action.d/cloudflare.conf +++ b/config/action.d/cloudflare.conf @@ -43,9 +43,9 @@ actioncheck = # API v1 #actionban = curl -s -o /dev/null https://www.cloudflare.com/api_json.html -d 'a=ban' -d 'tkn=' -d 'email=' -d 'key=' # API v4 -actionban = curl -s -o /dev/null -X POST -H 'X-Auth-Email: ' -H 'X-Auth-Key: ' \ - -H 'Content-Type: application/json' -d '{ "mode": "block", "configuration": { "target": "ip", "value": "" } }' \ - https://api.cloudflare.com/client/v4/user/firewall/access_rules/rules +actionban = curl -s -o /dev/null -X POST <_cf_api_prms> \ + -d '{"mode":"block","configuration":{"target":"ip","value":""},"notes":"Fail2Ban "}' \ + <_cf_api_url> # Option: actionunban # Notes.: command executed when unbanning an IP. Take care that the @@ -58,9 +58,14 @@ actionban = curl -s -o /dev/null -X POST -H 'X-Auth-Email: ' -H 'X-Auth- # API v1 #actionunban = curl -s -o /dev/null https://www.cloudflare.com/api_json.html -d 'a=nul' -d 'tkn=' -d 'email=' -d 'key=' # API v4 -actionunban = curl -s -o /dev/null -X DELETE -H 'X-Auth-Email: ' -H 'X-Auth-Key: ' \ - https://api.cloudflare.com/client/v4/user/firewall/access_rules/rules/$(curl -s -X GET -H 'X-Auth-Email: ' -H 'X-Auth-Key: ' \ - 'https://api.cloudflare.com/client/v4/user/firewall/access_rules/rules?mode=block&configuration_target=ip&configuration_value=&page=1&per_page=1' | jq -r '.result[0].configuration.value') +actionunban = id=$(curl -s -X GET <_cf_api_prms> \ + "<_cf_api_url>?mode=block&configuration_target=ip&configuration_value=&page=1&per_page=1¬es=Fail2Ban%20" \ + | jq -r '.result[0].id') + if [ -z "$id" ]; then echo ": id for cannot be found"; exit 0; fi; + curl -s -o /dev/null -X DELETE <_cf_api_prms> "<_cf_api_url>/$id" + +_cf_api_url = https://api.cloudflare.com/client/v4/user/firewall/access_rules/rules +_cf_api_prms = -H 'X-Auth-Email: ' -H 'X-Auth-Key: ' -H 'Content-Type: application/json' [Init] From 1c1b671c745dbe0e1f9a096fd1953d0257e8b958 Mon Sep 17 00:00:00 2001 From: "Sergey G. Brester" Date: Mon, 10 Dec 2018 11:27:53 +0100 Subject: [PATCH 20/27] Update cloudflare.conf --- config/action.d/cloudflare.conf | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/config/action.d/cloudflare.conf b/config/action.d/cloudflare.conf index 5125777c..4c0a3810 100644 --- a/config/action.d/cloudflare.conf +++ b/config/action.d/cloudflare.conf @@ -59,13 +59,13 @@ actionban = curl -s -o /dev/null -X POST <_cf_api_prms> \ #actionunban = curl -s -o /dev/null https://www.cloudflare.com/api_json.html -d 'a=nul' -d 'tkn=' -d 'email=' -d 'key=' # API v4 actionunban = id=$(curl -s -X GET <_cf_api_prms> \ - "<_cf_api_url>?mode=block&configuration_target=ip&configuration_value=&page=1&per_page=1¬es=Fail2Ban%20" \ + "<_cf_api_url>?mode=block&configuration_target=ip&configuration_value=&page=1&per_page=1¬es=Fail2Ban%%20" \ | jq -r '.result[0].id') if [ -z "$id" ]; then echo ": id for cannot be found"; exit 0; fi; curl -s -o /dev/null -X DELETE <_cf_api_prms> "<_cf_api_url>/$id" _cf_api_url = https://api.cloudflare.com/client/v4/user/firewall/access_rules/rules -_cf_api_prms = -H 'X-Auth-Email: ' -H 'X-Auth-Key: ' -H 'Content-Type: application/json' +_cf_api_prms = -H 'X-Auth-Email: ' -H 'X-Auth-Key: ' -H 'Content-Type: application/json' [Init] From 01e92ce4a617ff8cf95d47b67cdd77481a3efdd7 Mon Sep 17 00:00:00 2001 From: "Sergey G. Brester" Date: Mon, 16 Mar 2020 18:28:45 +0100 Subject: [PATCH 21/27] added fallback using tr and sed (jq is optional now) --- config/action.d/cloudflare.conf | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/config/action.d/cloudflare.conf b/config/action.d/cloudflare.conf index 4c0a3810..361cb177 100644 --- a/config/action.d/cloudflare.conf +++ b/config/action.d/cloudflare.conf @@ -5,7 +5,7 @@ # # Please set jail.local's permission to 640 because it contains your CF API key. # -# This action depends on curl and jq. +# This action depends on curl (and optionally jq). # Referenced from http://www.normyee.net/blog/2012/02/02/adding-cloudflare-support-to-fail2ban by NORM YEE # # To get your CloudFlare API Key: https://www.cloudflare.com/a/account/my-account @@ -60,7 +60,7 @@ actionban = curl -s -o /dev/null -X POST <_cf_api_prms> \ # API v4 actionunban = id=$(curl -s -X GET <_cf_api_prms> \ "<_cf_api_url>?mode=block&configuration_target=ip&configuration_value=&page=1&per_page=1¬es=Fail2Ban%%20" \ - | jq -r '.result[0].id') + | { jq -r '.result[0].id' 2>/dev/null || tr -d '\n' | sed -nE 's/^.*"result"\s*:\s*\[\s*\{\s*"id"\s*:\s*"([^"]+)".*$/\1/p'; }) if [ -z "$id" ]; then echo ": id for cannot be found"; exit 0; fi; curl -s -o /dev/null -X DELETE <_cf_api_prms> "<_cf_api_url>/$id" From 42aef09d695f98794066118015db7b72442c6116 Mon Sep 17 00:00:00 2001 From: "Sergey G. Brester" Date: Mon, 27 Apr 2020 19:38:48 +0200 Subject: [PATCH 22/27] Update ChangeLog --- ChangeLog | 2 ++ 1 file changed, 2 insertions(+) diff --git a/ChangeLog b/ChangeLog index 2ae56d99..d744dba0 100644 --- a/ChangeLog +++ b/ChangeLog @@ -46,6 +46,8 @@ ver. 0.10.6-dev (20??/??/??) - development edition so would bother the action interpolation * `action.d/*-ipset*.conf`: several ipset actions fixed (no timeout per default anymore), so no discrepancy between ipset and fail2ban (removal from ipset will be managed by fail2ban only, gh-2703) +* `action.d/cloudflare.conf`: fixed `actionunban` (considering new-line chars and optionally real json-parsing + with `jq`, gh-2140, gh-2656) * `filter.d/common.conf`: avoid substitute of default values in related `lt_*` section, `__prefix_line` should be interpolated in definition section (inside the filter-config, gh-2650) * `filter.d/courier-smtp.conf`: prefregex extended to consider port in log-message (gh-2697) From 43f699b872ab75132a2188be29706d602830254e Mon Sep 17 00:00:00 2001 From: "Sergey G. Brester" Date: Wed, 6 May 2020 17:32:13 +0200 Subject: [PATCH 23/27] grammar / typos --- config/jail.conf | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/config/jail.conf b/config/jail.conf index c7177f13..4d236b34 100644 --- a/config/jail.conf +++ b/config/jail.conf @@ -52,7 +52,7 @@ before = paths-debian.conf # to prevent "clever" botnets calculate exact time IP can be unbanned again: #bantime.rndtime = -# "bantime.maxtime" is the max number of seconds using the ban time can reach (don't grows further) +# "bantime.maxtime" is the max number of seconds using the ban time can reach (doesn't grow further) #bantime.maxtime = # "bantime.factor" is a coefficient to calculate exponent growing of the formula or common multiplier, @@ -60,7 +60,7 @@ before = paths-debian.conf # grows by 1, 2, 4, 8, 16 ... #bantime.factor = 1 -# "bantime.formula" used by default to calculate next value of ban time, default value bellow, +# "bantime.formula" used by default to calculate next value of ban time, default value below, # the same ban time growing will be reached by multipliers 1, 2, 4, 8, 16, 32... #bantime.formula = ban.Time * (1<<(ban.Count if ban.Count<20 else 20)) * banFactor # From afb7a931632cded1312facf2f82d2a33caf7ffd2 Mon Sep 17 00:00:00 2001 From: sebres Date: Wed, 20 May 2020 15:27:48 +0200 Subject: [PATCH 24/27] amend to 368aa9e77570519b37fb57c9dbc5112d4c4b7382: fix time in gitlab test (GMT in log due to TZ-suffix `Z`, CEST in test-suite) --- fail2ban/tests/files/logs/gitlab | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/fail2ban/tests/files/logs/gitlab b/fail2ban/tests/files/logs/gitlab index 222df642..70ddc0e8 100644 --- a/fail2ban/tests/files/logs/gitlab +++ b/fail2ban/tests/files/logs/gitlab @@ -1,5 +1,5 @@ # Access of unauthorized host in /var/log/gitlab/gitlab-rails/application.log -# failJSON: { "time": "2020-04-09T14:04:00", "match": true , "host": "80.10.11.12" } +# failJSON: { "time": "2020-04-09T16:04:00", "match": true , "host": "80.10.11.12" } 2020-04-09T14:04:00.667Z: Failed Login: username=admin ip=80.10.11.12 -# failJSON: { "time": "2020-04-09T14:15:09", "match": true , "host": "80.10.11.12" } +# failJSON: { "time": "2020-04-09T16:15:09", "match": true , "host": "80.10.11.12" } 2020-04-09T14:15:09.344Z: Failed Login: username=user name ip=80.10.11.12 From 0ae2ef68be57f056038adaf064570e6725c6f20f Mon Sep 17 00:00:00 2001 From: sebres Date: Wed, 20 May 2020 15:36:06 +0200 Subject: [PATCH 25/27] ensure iterator is safe (traverse over the list in snapshot created within a lock), avoids getting modified state as well as "dictionary changed size during iteration" errors --- fail2ban/server/banmanager.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/fail2ban/server/banmanager.py b/fail2ban/server/banmanager.py index 8c0a6965..479ba26f 100644 --- a/fail2ban/server/banmanager.py +++ b/fail2ban/server/banmanager.py @@ -104,7 +104,7 @@ class BanManager: def getBanList(self): with self.__lock: - return self.__banList.keys() + return list(self.__banList.keys()) ## # Returns a iterator to ban list (used in reload, so idle). @@ -112,8 +112,9 @@ class BanManager: # @return ban list iterator def __iter__(self): + # ensure iterator is safe (traverse over the list in snapshot created within lock): with self.__lock: - return self.__banList.itervalues() + return iter(list(self.__banList.values())) ## # Returns normalized value From 54b2208690e3c2fff00fbd9b197984d880e29a02 Mon Sep 17 00:00:00 2001 From: sebres Date: Wed, 20 May 2020 16:31:54 +0200 Subject: [PATCH 26/27] extends protocol/client with banned status (retrieve information whether an IP is banned and/or in which jails), implements FR gh-2725 --- fail2ban/protocol.py | 4 +++ fail2ban/server/actions.py | 8 ++++++ fail2ban/server/server.py | 26 ++++++++++++++++++ fail2ban/server/transmitter.py | 8 +++++- fail2ban/tests/fail2banclienttestcase.py | 34 +++++++++++++++++++++++- man/fail2ban-client.1 | 16 +++++++++++ 6 files changed, 94 insertions(+), 2 deletions(-) diff --git a/fail2ban/protocol.py b/fail2ban/protocol.py index 92b0dcc0..18b901e8 100644 --- a/fail2ban/protocol.py +++ b/fail2ban/protocol.py @@ -55,6 +55,8 @@ protocol = [ ["stop", "stops all jails and terminate the server"], ["unban --all", "unbans all IP addresses (in all jails and database)"], ["unban ... ", "unbans (in all jails and database)"], +["banned", "return jails with banned IPs as dictionary"], +["banned ... ]", "return list(s) of jails where given IP(s) are banned"], ["status", "gets the current status of the server"], ["ping", "tests if the server is alive"], ["echo", "for internal usage, returns back and outputs a given string"], @@ -120,6 +122,8 @@ protocol = [ ["set action ", "sets the of for the action for "], ["set action [ ]", "calls the with for the action for "], ['', "JAIL INFORMATION", ""], +["get banned", "return banned IPs of "], +["get banned ... ]", "return 1 if IP is banned in otherwise 0, or a list of 1/0 for multiple IPs"], ["get logpath", "gets the list of the monitored files for "], ["get logencoding", "gets the encoding of the log files for "], ["get journalmatch", "gets the journal filter match for "], diff --git a/fail2ban/server/actions.py b/fail2ban/server/actions.py index 35b027ae..6123605d 100644 --- a/fail2ban/server/actions.py +++ b/fail2ban/server/actions.py @@ -210,6 +210,14 @@ class Actions(JailThread, Mapping): def getBanTime(self): return self.__banManager.getBanTime() + def getBanned(self, ids): + lst = self.__banManager.getBanList() + if not ids: + return lst + if len(ids) == 1: + return 1 if ids[0] in lst else 0 + return map(lambda ip: 1 if ip in lst else 0, ids) + def addBannedIP(self, ip): """Ban an IP or list of IPs.""" unixTime = MyTime.time() diff --git a/fail2ban/server/server.py b/fail2ban/server/server.py index abb312da..b12c8f9f 100644 --- a/fail2ban/server/server.py +++ b/fail2ban/server/server.py @@ -521,6 +521,32 @@ class Server: cnt += jail.actions.removeBannedIP(value, ifexists=ifexists) return cnt + def banned(self, name=None, ids=None): + if name is not None: + # single jail: + jails = [self.__jails[name]] + else: + # in all jails: + jails = self.__jails.values() + # check banned ids: + res = [] + if name is None and ids: + for ip in ids: + ret = [] + for jail in jails: + if jail.actions.getBanned([ip]): + ret.append(jail.name) + res.append(ret) + else: + for jail in jails: + ret = jail.actions.getBanned(ids) + if name is not None: + return ret + res.append(ret) + else: + res.append({jail.name: ret}) + return res + def getBanTime(self, name): return self.__jails[name].actions.getBanTime() diff --git a/fail2ban/server/transmitter.py b/fail2ban/server/transmitter.py index de80f624..aff9071c 100644 --- a/fail2ban/server/transmitter.py +++ b/fail2ban/server/transmitter.py @@ -118,6 +118,9 @@ class Transmitter: if len(value) == 1 and value[0] == "--all": return self.__server.setUnbanIP() return self.__server.setUnbanIP(None, value) + elif name == "banned": + # check IP is banned in all jails: + return self.__server.banned(None, command[1:]) elif name == "echo": return command[1:] elif name == "server-status": @@ -424,7 +427,10 @@ class Transmitter: return None else: return db.purgeage - # Filter + # Jail, Filter + elif command[1] == "banned": + # check IP is banned in all jails: + return self.__server.banned(name, command[2:]) elif command[1] == "logpath": return self.__server.getLogPath(name) elif command[1] == "logencoding": diff --git a/fail2ban/tests/fail2banclienttestcase.py b/fail2ban/tests/fail2banclienttestcase.py index 6c79800e..a334b568 100644 --- a/fail2ban/tests/fail2banclienttestcase.py +++ b/fail2ban/tests/fail2banclienttestcase.py @@ -37,7 +37,7 @@ from threading import Thread from ..client import fail2banclient, fail2banserver, fail2bancmdline from ..client.fail2bancmdline import Fail2banCmdLine -from ..client.fail2banclient import exec_command_line as _exec_client, VisualWait +from ..client.fail2banclient import exec_command_line as _exec_client, CSocket, VisualWait from ..client.fail2banserver import Fail2banServer, exec_command_line as _exec_server from .. import protocol from ..server import server @@ -421,6 +421,14 @@ class Fail2banClientServerBase(LogCaptureTestCase): self.assertRaises(exitType, self.exec_command_line[0], (self.exec_command_line[1:] + startparams + args)) + def execCmdDirect(self, startparams, *args): + sock = startparams[startparams.index('-s')+1] + s = CSocket(sock) + try: + return s.send(args) + finally: + s.close() + # # Common tests # @@ -1007,6 +1015,30 @@ class Fail2banServerTest(Fail2banClientServerBase): "[test-jail2] Found 192.0.2.3", "[test-jail2] Ban 192.0.2.3", all=True) + # test banned command: + self.assertSortedEqual(self.execCmdDirect(startparams, + 'banned'), (0, [ + {'test-jail1': ['192.0.2.4', '192.0.2.1', '192.0.2.8', '192.0.2.3', '192.0.2.2']}, + {'test-jail2': ['192.0.2.4', '192.0.2.9', '192.0.2.8']} + ] + )) + self.assertSortedEqual(self.execCmdDirect(startparams, + 'banned', '192.0.2.1', '192.0.2.4', '192.0.2.222'), (0, [ + ['test-jail1'], ['test-jail1', 'test-jail2'], [] + ] + )) + self.assertSortedEqual(self.execCmdDirect(startparams, + 'get', 'test-jail1', 'banned')[1], [ + '192.0.2.4', '192.0.2.1', '192.0.2.8', '192.0.2.3', '192.0.2.2']) + self.assertSortedEqual(self.execCmdDirect(startparams, + 'get', 'test-jail2', 'banned')[1], [ + '192.0.2.4', '192.0.2.9', '192.0.2.8']) + self.assertEqual(self.execCmdDirect(startparams, + 'get', 'test-jail1', 'banned', '192.0.2.3')[1], 1) + self.assertEqual(self.execCmdDirect(startparams, + 'get', 'test-jail1', 'banned', '192.0.2.9')[1], 0) + self.assertEqual(self.execCmdDirect(startparams, + 'get', 'test-jail1', 'banned', '192.0.2.3', '192.0.2.9')[1], [1, 0]) # rotate logs: _write_file(test1log, "w+") diff --git a/man/fail2ban-client.1 b/man/fail2ban-client.1 index 84f846f2..6a3247e1 100644 --- a/man/fail2ban-client.1 +++ b/man/fail2ban-client.1 @@ -111,6 +111,14 @@ jails and database) unbans (in all jails and database) .TP +\fBbanned\fR +return jails with banned IPs as +dictionary +.TP +\fBbanned ... ]\fR +return list(s) of jails where +given IP(s) are banned +.TP \fBstatus\fR gets the current status of the server @@ -356,6 +364,14 @@ for .IP JAIL INFORMATION .TP +\fBget banned\fR +return banned IPs of +.TP +\fBget banned ... ]\fR +return 1 if IP is banned in +otherwise 0, or a list of 1/0 for +multiple IPs +.TP \fBget logpath\fR gets the list of the monitored files for From fa1ff4c5d8757bf8e07bd43794d27290293192e5 Mon Sep 17 00:00:00 2001 From: sebres Date: Mon, 25 May 2020 13:36:09 +0200 Subject: [PATCH 27/27] assertSortedEqual: fixed sort of nested lists, switch default of nestedOnly to False (comparison of unsorted lists is rarely needed) --- fail2ban/tests/misctestcase.py | 10 +++++++++- fail2ban/tests/utils.py | 13 ++++++++++--- 2 files changed, 19 insertions(+), 4 deletions(-) diff --git a/fail2ban/tests/misctestcase.py b/fail2ban/tests/misctestcase.py index 9b986f53..43d76802 100644 --- a/fail2ban/tests/misctestcase.py +++ b/fail2ban/tests/misctestcase.py @@ -390,7 +390,15 @@ class TestsUtilsTest(LogCaptureTestCase): self.assertSortedEqual(['Z', {'A': ['B', 'C'], 'B': ['E', 'F']}], [{'B': ['F', 'E'], 'A': ['C', 'B']}, 'Z'], level=-1) self.assertRaises(AssertionError, lambda: self.assertSortedEqual( - ['Z', {'A': ['B', 'C'], 'B': ['E', 'F']}], [{'B': ['F', 'E'], 'A': ['C', 'B']}, 'Z'])) + ['Z', {'A': ['B', 'C'], 'B': ['E', 'F']}], [{'B': ['F', 'E'], 'A': ['C', 'B']}, 'Z'], + nestedOnly=True)) + self.assertSortedEqual( + (0, [['A1'], ['A2', 'A1'], []]), + (0, [['A1'], ['A1', 'A2'], []]), + ) + self.assertSortedEqual(list('ABC'), list('CBA')) + self.assertRaises(AssertionError, self.assertSortedEqual, ['ABC'], ['CBA']) + self.assertRaises(AssertionError, self.assertSortedEqual, [['ABC']], [['CBA']]) self._testAssertionErrorRE(r"\['A'\] != \['C', 'B'\]", self.assertSortedEqual, ['A'], ['C', 'B']) self._testAssertionErrorRE(r"\['A', 'B'\] != \['B', 'C'\]", diff --git a/fail2ban/tests/utils.py b/fail2ban/tests/utils.py index dc12a5be..47e5b909 100644 --- a/fail2ban/tests/utils.py +++ b/fail2ban/tests/utils.py @@ -556,7 +556,7 @@ if not hasattr(unittest.TestCase, 'assertDictEqual'): self.fail(msg) unittest.TestCase.assertDictEqual = assertDictEqual -def assertSortedEqual(self, a, b, level=1, nestedOnly=True, key=repr, msg=None): +def assertSortedEqual(self, a, b, level=1, nestedOnly=False, key=repr, msg=None): """Compare complex elements (like dict, list or tuple) in sorted order until level 0 not reached (initial level = -1 meant all levels), or if nestedOnly set to True and some of the objects still contains nested lists or dicts. @@ -566,6 +566,13 @@ def assertSortedEqual(self, a, b, level=1, nestedOnly=True, key=repr, msg=None): if isinstance(v, dict): return any(isinstance(v, (dict, list, tuple)) for v in v.itervalues()) return any(isinstance(v, (dict, list, tuple)) for v in v) + if nestedOnly: + _nest_sorted = sorted + else: + def _nest_sorted(v, key=key): + if isinstance(v, (set, list, tuple)): + return sorted(list(_nest_sorted(v, key) for v in v), key=key) + return v # level comparison routine: def _assertSortedEqual(a, b, level, nestedOnly, key): # first the lengths: @@ -584,8 +591,8 @@ def assertSortedEqual(self, a, b, level=1, nestedOnly=True, key=repr, msg=None): elif v1 != v2: raise ValueError('%r != %r' % (a, b)) else: # list, tuple, something iterable: - a = sorted(a, key=key) - b = sorted(b, key=key) + a = _nest_sorted(a, key=key) + b = _nest_sorted(b, key=key) for v1, v2 in zip(a, b): if isinstance(v1, (dict, list, tuple)) and isinstance(v2, (dict, list, tuple)): _assertSortedEqual(v1, v2, level-1 if level != 0 else 0, nestedOnly, key)