From bdae15b5220634f7ffc19d3d0bbf6ec8cfcfc842 Mon Sep 17 00:00:00 2001 From: sebres Date: Mon, 3 Mar 2025 18:40:15 +0100 Subject: [PATCH 01/24] ipdns.py: implemented FileIPAddrSet supporting file with IP-set, what may contain IP, subnet, or dns, with lazy load and dynamically reloaded by changes (with small latency to avoid expensive stats check on every compare) --- fail2ban/server/ipdns.py | 112 +++++++++++++++++++++++++++++++++++++-- 1 file changed, 108 insertions(+), 4 deletions(-) diff --git a/fail2ban/server/ipdns.py b/fail2ban/server/ipdns.py index 7ca1e432..cff8d60d 100644 --- a/fail2ban/server/ipdns.py +++ b/fail2ban/server/ipdns.py @@ -23,10 +23,11 @@ __license__ = "GPL" import socket import struct +import os import re from .utils import Utils -from ..helpers import getLogger +from ..helpers import getLogger, MyTime, splitwords # Gets the instance of the logger. logSys = getLogger(__name__) @@ -79,6 +80,8 @@ class DNSUtils: # todo: make configurable the expired time and max count of cache entries: CACHE_nameToIp = Utils.Cache(maxCount=1000, maxTime=5*60) CACHE_ipToName = Utils.Cache(maxCount=1000, maxTime=5*60) + # static cache used to hold sets read from files: + CACHE_fileToIp = Utils.Cache(maxCount=100, maxTime=5*60) @staticmethod def dnsToIp(dns): @@ -229,6 +232,20 @@ class DNSUtils: DNSUtils.CACHE_nameToIp.set(DNSUtils._getSelfIPs_key, ips) return ips + @staticmethod + def getIPsFromFile(fileName, noError=True): + """Get set of IP addresses or subnets from file""" + # to find cached IPs: + ips = DNSUtils.CACHE_fileToIp.get(fileName) + if ips is not None: + return ips + # try to obtain set from file: + ips = FileIPAddrSet(fileName) + #ips.load() - load on demand + # cache and return : + DNSUtils.CACHE_fileToIp.set(fileName, ips) + return ips + _IPv6IsAllowed = None @staticmethod @@ -457,6 +474,10 @@ class IPAddr(object): def familyStr(self): return IPAddr.FAM2STR.get(self._family) + @property + def instanceType(self): + return "ip" if self.isValid else "dns" + @property def plen(self): return self._plen @@ -598,6 +619,9 @@ class IPAddr(object): def isInNet(self, net): """Return either the IP object is in the provided network """ + # if addr-set: + if isinstance(net, IPAddrSet): + return self in net # if it isn't a valid IP address, try DNS resolution if not net.isValid and net.raw != "": # Check if IP in DNS @@ -675,15 +699,32 @@ IPAddr.IP6_4COMPAT = IPAddr("::ffff:0:0", 96) class IPAddrSet(set): - hasSubNet = False + hasSubNet = 0 def __init__(self, ips=[]): + ips, subnet = IPAddrSet._list2set(ips) + set.__init__(self, ips) + self.hasSubNet = subnet + + @staticmethod + def _list2set(ips): ips2 = set() + subnet = 0 for ip in ips: if not isinstance(ip, IPAddr): ip = IPAddr(ip) ips2.add(ip) - self.hasSubNet |= not ip.isSingle - set.__init__(self, ips2) + subnet += not ip.isSingle + return ips2, subnet + + @property + def instanceType(self): + return "ip-set" + + def set(self, ips): + ips, subnet = IPAddrSet._list2set(ips) + self.clear() + self.update(ips) + self.hasSubNet = subnet def add(self, ip): if not isinstance(ip, IPAddr): ip = IPAddr(ip) @@ -696,6 +737,69 @@ class IPAddrSet(set): return set.__contains__(self, ip) or (self.hasSubNet and any(n.contains(ip) for n in self)) +class FileIPAddrSet(IPAddrSet): + + # RE matching file://... + RE_FILE_IGN_IP = re.compile(r'^file:/{0,2}(.*)$') + + fileName = '' + _shortRepr = None + maxUpdateLatency = 1 # latency in seconds to update by changes + _nextCheck = 0 + _fileStats = () + + def __init__(self, fileName=''): + self.fileName = fileName + # self.load() - lazy load on demand by first check (in, __contains__ etc) + + @property + def instanceType(self): + return repr(self) + + def __eq__(self, other): + if id(self) == id(other): return 1 + # to allow remove file-set from list (delIgnoreIP) by its name: + if isinstance(other, FileIPAddrSet): + return self.fileName == other.fileName + m = FileIPAddrSet.RE_FILE_IGN_IP.match(other) + if m: + return self.fileName == m.group(1) + + def load(self, ifNeeded=True, noError=True): + try: + if ifNeeded: + tm = MyTime.time() + if tm > self._nextCheck: + self._nextCheck = tm + self.maxUpdateLatency + stats = os.stat(self.fileName) + stats = stats.st_mtime, stats.st_ino, stats.st_size + if self._fileStats == stats: + return + self._fileStats = stats + with open(self.fileName, 'r') as f: + ips = f.read() + ips = splitwords(ips) + self.set(ips) + except Exception as e: # pragma: no cover + if not noError: raise e + logSys.warning("Retrieving IPs set from %r failed: %s", self.fileName, e) + + def __repr__(self): + if not self._shortRepr: + shortfn = os.path.basename(self.fileName) + if shortfn != self.fileName: + shortfn = '.../' + shortfn + self._shortRepr = 'file:' + shortfn + ')' + return self._shortRepr + + def __contains__(self, ip): + # check it is uptodate (not often than maxUpdateLatency): + if self.fileName: + self.load(ifNeeded=True) + # inherited contains: + return IPAddrSet.__contains__(self, ip) + + def _NetworkInterfacesAddrs(withMask=False): # Closure implementing lazy load modules and libc and define _NetworkInterfacesAddrs on demand: From d684339edd02646b71deb786349723f5bb47abdf Mon Sep 17 00:00:00 2001 From: sebres Date: Mon, 3 Mar 2025 19:00:09 +0100 Subject: [PATCH 02/24] allow comments in file with ip-set: text followed # or ; chars after space or newline would be ignored --- fail2ban/helpers.py | 17 +++++++++++++++-- fail2ban/server/ipdns.py | 2 +- fail2ban/tests/misctestcase.py | 12 +++++++++++- 3 files changed, 27 insertions(+), 4 deletions(-) diff --git a/fail2ban/helpers.py b/fail2ban/helpers.py index 0d7d8ba9..220753a7 100644 --- a/fail2ban/helpers.py +++ b/fail2ban/helpers.py @@ -282,7 +282,18 @@ def excepthook(exctype, value, traceback): "Unhandled exception in Fail2Ban:", exc_info=True) return sys.__excepthook__(exctype, value, traceback) -def splitwords(s): +RE_REM_COMMENTS = re.compile(r'(?m)(?:^|\s)[\#;].*') +def removeComments(s): + """Helper to remove comments: + # comment ... + ; comment ... + no comment # comment ... + no comment ; comment ... + """ + return RE_REM_COMMENTS.sub('', s) + +RE_SPLT_WORDS = re.compile(r'[\s,]+') +def splitwords(s, ignoreComments=False): """Helper to split words on any comma, space, or a new line Returns empty list if input is empty (or None) and filters @@ -290,7 +301,9 @@ def splitwords(s): """ if not s: return [] - return list(filter(bool, [v.strip() for v in re.split(r'[\s,]+', s)])) + if ignoreComments: + s = removeComments(s) + return list(filter(bool, [v.strip() for v in RE_SPLT_WORDS.split(s)])) def _merge_dicts(x, y): """Helper to merge dicts. diff --git a/fail2ban/server/ipdns.py b/fail2ban/server/ipdns.py index cff8d60d..665bddc6 100644 --- a/fail2ban/server/ipdns.py +++ b/fail2ban/server/ipdns.py @@ -778,7 +778,7 @@ class FileIPAddrSet(IPAddrSet): self._fileStats = stats with open(self.fileName, 'r') as f: ips = f.read() - ips = splitwords(ips) + ips = splitwords(ips, ignoreComments=True) self.set(ips) except Exception as e: # pragma: no cover if not noError: raise e diff --git a/fail2ban/tests/misctestcase.py b/fail2ban/tests/misctestcase.py index bfce434f..6de4ff09 100644 --- a/fail2ban/tests/misctestcase.py +++ b/fail2ban/tests/misctestcase.py @@ -34,7 +34,7 @@ from io import StringIO from .utils import LogCaptureTestCase, logSys as DefLogSys from ..helpers import formatExceptionInfo, mbasename, TraceBack, FormatterWithTraceBack, getLogger, \ - getVerbosityFormat, splitwords, uni_decode, uni_string + getVerbosityFormat, removeComments, splitwords, uni_decode, uni_string from ..server.mytime import MyTime @@ -68,6 +68,16 @@ class HelpersTest(unittest.TestCase): self.assertEqual(splitwords(' 1\n 2, 3'), ['1', '2', '3']) self.assertEqual(splitwords('\t1\t 2,\r\n 3\n'), ['1', '2', '3']); # other spaces + def testSplitNoComments(self): + s = ''' + # comment ... + ; comment ... + line1 A # comment ... + line2 B ; comment ... + ''' + self.assertEqual(splitwords(s, ignoreComments=True), ['line1', 'A', 'line2', 'B']) + self.assertEqual(splitwords(removeComments(s)), ['line1', 'A', 'line2', 'B']) + def _sh_call(cmd): import subprocess From 81a5b1596b026a7c11746a896a6d3c42b38c0519 Mon Sep 17 00:00:00 2001 From: sebres Date: Mon, 3 Mar 2025 19:03:48 +0100 Subject: [PATCH 03/24] filter and configuration `ignoreip` extended with file:... to ignore IPs from file-ip-set (containing IP, subnet, dns/fqdn or raw strings); the file would be read lazy on demand, by first ban (and automatically reloaded by update after small latency) --- fail2ban/server/filter.py | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/fail2ban/server/filter.py b/fail2ban/server/filter.py index a33303d6..d132ce6a 100644 --- a/fail2ban/server/filter.py +++ b/fail2ban/server/filter.py @@ -32,7 +32,7 @@ import time from .actions import Actions from .failmanager import FailManagerEmpty, FailManager -from .ipdns import DNSUtils, IPAddr +from .ipdns import DNSUtils, IPAddr, FileIPAddrSet from .observer import Observers from .ticket import FailTicket from .jailthread import JailThread @@ -510,6 +510,12 @@ class Filter(JailThread): # An empty string is always false if ipstr == "": return + # File? + ip = FileIPAddrSet.RE_FILE_IGN_IP.match(ipstr) + if ip: + ip = DNSUtils.getIPsFromFile(ip.group(1)) # FileIPAddrSet + self.__ignoreIpList.append(ip) + return # Create IP address object ip = IPAddr(ipstr) # Avoid exact duplicates @@ -532,6 +538,11 @@ class Filter(JailThread): return # delete by ip: logSys.debug(" Remove %r from ignore list", ip) + # File? + if FileIPAddrSet.RE_FILE_IGN_IP.match(ip): + self.__ignoreIpList.remove(ip) + return + # IP / DNS if ip in self.__ignoreIpSet: self.__ignoreIpSet.remove(ip) else: @@ -588,7 +599,7 @@ class Filter(JailThread): return True for net in self.__ignoreIpList: if ip.isInNet(net): - self.logIgnoreIp(ip, log_ignore, ignore_source=("ip" if net.isValid else "dns")) + self.logIgnoreIp(ip, log_ignore, ignore_source=(net.instanceType)) if self.__ignoreCache: c.set(key, True) return True From fe37047061012803491c0ddbd070a352429c2046 Mon Sep 17 00:00:00 2001 From: sebres Date: Mon, 3 Mar 2025 19:06:08 +0100 Subject: [PATCH 04/24] test coverage for FileIPAddrSet and ignoreip for file://... --- fail2ban/tests/files/test-ign-ips-file | 4 + fail2ban/tests/filtertestcase.py | 123 +++++++++++++++++++++++++ fail2ban/tests/utils.py | 8 +- 3 files changed, 134 insertions(+), 1 deletion(-) create mode 100644 fail2ban/tests/files/test-ign-ips-file diff --git a/fail2ban/tests/files/test-ign-ips-file b/fail2ban/tests/files/test-ign-ips-file new file mode 100644 index 00000000..61af63d5 --- /dev/null +++ b/fail2ban/tests/files/test-ign-ips-file @@ -0,0 +1,4 @@ +test-local-net +test-subnet-a, test-subnet-b +192.0.2.200, 2001:0db8::00c8 +192.0.2.216/29, 2001:db8::d8/125 \ No newline at end of file diff --git a/fail2ban/tests/filtertestcase.py b/fail2ban/tests/filtertestcase.py index 26961a1b..ece75201 100644 --- a/fail2ban/tests/filtertestcase.py +++ b/fail2ban/tests/filtertestcase.py @@ -399,6 +399,82 @@ class IgnoreIP(LogCaptureTestCase): self.filter.addIgnoreIP('192.168.1.0/255.255.0.0') self.assertRaises(ValueError, self.filter.addIgnoreIP, '192.168.1.0/255.255.0.128') + def testIgnoreIPDNS(self): + # test subnets are pre-cached (as IPAddrSet), so it shall work even without network: + for dns in ("test-subnet-a", "test-subnet-b"): + self.filter.addIgnoreIP(dns) + self.assertTrue(self.filter.inIgnoreIPList(IPAddr('192.0.2.1'))) + self.assertTrue(self.filter.inIgnoreIPList(IPAddr('192.0.2.7'))) + self.assertTrue(self.filter.inIgnoreIPList(IPAddr('192.0.2.16'))) + self.assertTrue(self.filter.inIgnoreIPList(IPAddr('192.0.2.23'))) + self.assertFalse(self.filter.inIgnoreIPList(IPAddr('192.0.2.8'))) + self.assertFalse(self.filter.inIgnoreIPList(IPAddr('192.0.2.15'))) + self.assertTrue(self.filter.inIgnoreIPList(IPAddr('2001:db8::00'))) + self.assertTrue(self.filter.inIgnoreIPList(IPAddr('2001:db8::07'))) + self.assertTrue(self.filter.inIgnoreIPList(IPAddr('2001:0db8:0000:0000:0000:0000:0000:0000'))) + self.assertTrue(self.filter.inIgnoreIPList(IPAddr('2001:0db8:0000:0000:0000:0000:0000:0007'))) + self.assertTrue(self.filter.inIgnoreIPList(IPAddr('2001:db8::10'))) + self.assertTrue(self.filter.inIgnoreIPList(IPAddr('2001:db8::17'))) + self.assertTrue(self.filter.inIgnoreIPList(IPAddr('2001:0db8:0000:0000:0000:0000:0000:0010'))) + self.assertTrue(self.filter.inIgnoreIPList(IPAddr('2001:0db8:0000:0000:0000:0000:0000:0017'))) + self.assertFalse(self.filter.inIgnoreIPList(IPAddr('2001:db8::08'))) + self.assertFalse(self.filter.inIgnoreIPList(IPAddr('2001:db8::0f'))) + self.assertFalse(self.filter.inIgnoreIPList(IPAddr('2001:0db8:0000:0000:0000:0000:0000:0008'))) + self.assertFalse(self.filter.inIgnoreIPList(IPAddr('2001:0db8:0000:0000:0000:0000:0000:000f'))) + + # to test several IPs in ip-set from file "files/test-ign-ips-file": + TEST_IPS_IGN_FILE = { + '127.0.0.1': True, + '127.255.255.255': True, + '127.0.0.1/8': True, + '192.0.2.1': True, + '192.0.2.7': True, + '192.0.2.0/29': True, + '192.0.2.16': True, + '192.0.2.23': True, + '192.0.2.200': True, + '192.0.2.216': True, + '192.0.2.223': True, + '192.0.2.216/29': True, + '192.0.2.8': False, + '192.0.2.15': False, + '192.0.2.100': False, + '192.0.2.224': False, + '::1': True, + '2001:db8::00': True, + '2001:db8::07': True, + '2001:db8::0/125': True, + '2001:0db8:0000:0000:0000:0000:0000:0000': True, + '2001:0db8:0000:0000:0000:0000:0000:0007': True, + '2001:db8::10': True, + '2001:db8::17': True, + '2001:0db8:0000:0000:0000:0000:0000:0010': True, + '2001:0db8:0000:0000:0000:0000:0000:0017': True, + '2001:db8::c8': True, + '2001:db8::d8': True, + '2001:db8::df': True, + '2001:db8::d8/125': True, + '2001:0db8:0000:0000:0000:0000:0000:00d8': True, + '2001:0db8:0000:0000:0000:0000:0000:00df': True, + '2001:db8::08': False, + '2001:db8::0f': False, + '2001:0db8:0000:0000:0000:0000:0000:0008': False, + '2001:0db8:0000:0000:0000:0000:0000:000f': False, + '2001:db8::e0': False, + '2001:0db8:0000:0000:0000:0000:0000:00e0': False, + } + + def testIgnoreIPFileIPAddr(self): + fname = 'file://' + os.path.join(TEST_FILES_DIR, "test-ign-ips-file") + self.filter.ignoreSelf = False + self.filter.addIgnoreIP(fname) + for ip, v in IgnoreIP.TEST_IPS_IGN_FILE.items(): + self.assertEqual(self.filter.inIgnoreIPList(IPAddr(ip)), v, ("for %r in ignoreip, file://test-ign-ips-file)" % (ip,))) + # now remove it: + self.filter.delIgnoreIP(fname) + for ip in IgnoreIP.TEST_IPS_IGN_FILE.keys(): + self.assertEqual(self.filter.inIgnoreIPList(IPAddr(ip)), False, ("for %r ignoreip, without file://test-ign-ips-file)" % (ip,))) + def testIgnoreInProcessLine(self): setUpMyTime() try: @@ -2427,6 +2503,53 @@ class DNSUtilsNetworkTests(unittest.TestCase): DNSUtils.CACHE_nameToIp.unset(DNSUtils._getSelfIPs_key) DNSUtils.CACHE_nameToIp.unset(DNSUtils._getNetIntrfIPs_key) + def test_FileIPAddrSet(self): + fname = os.path.join(TEST_FILES_DIR, "test-ign-ips-file") + ips = DNSUtils.getIPsFromFile(fname) + for ip, v in IgnoreIP.TEST_IPS_IGN_FILE.items(): + self.assertEqual(IPAddr(ip) in ips, v, ("for %r in test-ign-ips-file\n containing %s)" % (ip, set(ips)))) + + def test_FileIPAddrSet_Update(self): + fname = tempfile.mktemp(prefix='tmp_fail2ban', suffix='.ips') + f = open(fname, 'wb') + try: + f.write(b"192.0.2.200, 192.0.2.201\n") + f.flush() + ips = DNSUtils.getIPsFromFile(fname) + self.assertTrue(IPAddr('192.0.2.200') in ips) + self.assertTrue(IPAddr('192.0.2.201') in ips) + self.assertFalse(IPAddr('192.0.2.202') in ips) + # +1m, jump to next minute to force next check for update: + MyTime.setTime(MyTime.time() + 60) + # add .202, some comment and check all 3 IPs are there: + f.write(b"""192.0.2.202\n + # 2001:db8::ca/127 ; IPv6 commented yet + """) + f.flush() + self.assertTrue(IPAddr('192.0.2.200') in ips) + self.assertTrue(IPAddr('192.0.2.201') in ips) + self.assertTrue(IPAddr('192.0.2.202') in ips) + self.assertFalse(IPAddr('2001:db8::ca') in ips) + self.assertFalse(IPAddr('2001:db8::cb') in ips) + # +1m, jump to next minute to force next check for update: + MyTime.setTime(MyTime.time() + 60) + # remove .200, add IPv6-subnet and check all new IPs are there: + f.seek(0); f.truncate() + f.write(b""" + # 192.0.2.200 ; commented + 192.0.2.201, 192.0.2.202 # no .200 anymore + 2001:db8::ca/127 ; but 2 new IPv6 + """) + f.flush() + self.assertFalse(IPAddr('192.0.2.200') in ips) + self.assertTrue(IPAddr('192.0.2.201') in ips) + self.assertTrue(IPAddr('192.0.2.202') in ips) + self.assertTrue(IPAddr('2001:db8::ca') in ips) + self.assertTrue(IPAddr('2001:db8::cb') in ips) + finally: + tearDownMyTime() + _killfile(f, fname) + def testFQDN(self): unittest.F2B.SkipIfNoNetwork() sname = DNSUtils.getHostname(fqdn=False) diff --git a/fail2ban/tests/utils.py b/fail2ban/tests/utils.py index e6ef54f3..bedd9b12 100644 --- a/fail2ban/tests/utils.py +++ b/fail2ban/tests/utils.py @@ -39,7 +39,7 @@ from io import StringIO from functools import wraps from ..helpers import getLogger, str2LogLevel, getVerbosityFormat, uni_decode -from ..server.ipdns import IPAddr, DNSUtils +from ..server.ipdns import IPAddr, IPAddrSet, DNSUtils from ..server.mytime import MyTime from ..server.utils import Utils # for action_d.test_smtp : @@ -335,6 +335,12 @@ def initTests(opts): ips = set([IPAddr('127.0.0.1'), IPAddr('::1')]); # DNSUtils.dnsToIp('localhost') for i in DNSUtils.getSelfNames(): c.set(i, ips) + # some test subnets (although normally they are not resolved to addr/cidr, + # we'll use IPAddrSet here to seek through the resolved subnet in tests): + c = DNSUtils.CACHE_nameToIp + c.set('test-local-net', IPAddrSet([IPAddr('127.0.0.1/8'), IPAddr('::1')])) + c.set('test-subnet-a', IPAddrSet([IPAddr('192.0.2.0/29'), IPAddr('2001:db8::0/125')])); # 192.0.2.0 .. 192.0.2.7, 2001:db8::00 .. 2001:db8::07 + c.set('test-subnet-b', IPAddrSet([IPAddr('192.0.2.16/29'), IPAddr('2001:db8::10/125')])); # 192.0.2.16 .. 192.0.2.23, 2001:db8::10 .. 2001:db8::17 def mtimesleep(): From 6efa3a3144a96c52aa52fc0ec0a29f0fd25a0adb Mon Sep 17 00:00:00 2001 From: sebres Date: Mon, 3 Mar 2025 19:19:21 +0100 Subject: [PATCH 05/24] man extended (`ignoreip` supports file://path/file-with-ip-set) --- man/jail.conf.5 | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/man/jail.conf.5 b/man/jail.conf.5 index 6a34cbf8..d9b2debf 100644 --- a/man/jail.conf.5 +++ b/man/jail.conf.5 @@ -247,7 +247,8 @@ Values can also be quoted (required when value includes a ","). More that one ac boolean value (default true) indicates the banning of own IP addresses should be prevented .TP .B ignoreip -list of IPs not to ban. They can include a DNS resp. CIDR mask too. The option affects additionally to \fBignoreself\fR (if true) and don't need to contain own DNS resp. IPs of the running host. +list of IPs not to ban. They can also include CIDR mask or can be DNS (FQDN), or even raw string (if jail banning IDs instead of IPs). The option affects additionally to \fBignoreself\fR (if true) and don't need to contain own DNS resp. IPs of the running host. +This can also contain a filename (prefixed with "file:") with entries to ignore, which will be lazy loaded to the runtime on demand by first ban and automatically reloaded by update after small latency. .TP .B ignorecommand command that is executed to determine if the current candidate IP for banning (or failure-ID for raw IDs) should not be banned. This option operates alongside the \fBignoreself\fR and \fBignoreip\fR options. It is executed first, only if neither \fBignoreself\fR nor \fBignoreip\fR match the criteria. From 5bea1c87f135989a6458f37c3214aa463a052a2b Mon Sep 17 00:00:00 2001 From: sebres Date: Mon, 3 Mar 2025 19:52:23 +0100 Subject: [PATCH 06/24] add few comments to test-ign-ips-file for the sake of completeness and coverage --- fail2ban/tests/files/test-ign-ips-file | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/fail2ban/tests/files/test-ign-ips-file b/fail2ban/tests/files/test-ign-ips-file index 61af63d5..b3419147 100644 --- a/fail2ban/tests/files/test-ign-ips-file +++ b/fail2ban/tests/files/test-ign-ips-file @@ -1,4 +1,4 @@ -test-local-net -test-subnet-a, test-subnet-b -192.0.2.200, 2001:0db8::00c8 -192.0.2.216/29, 2001:db8::d8/125 \ No newline at end of file +test-local-net ; test local subnet +test-subnet-a, test-subnet-b ; further test subnets +192.0.2.200, 2001:0db8::00c8 # 2 IPs +192.0.2.216/29, 2001:db8::d8/125 # 2 subnets by IP/CIDR notation From 7233edd0bff030eb886541905ce9ce07714e4dbe Mon Sep 17 00:00:00 2001 From: sebres Date: Mon, 3 Mar 2025 20:05:11 +0100 Subject: [PATCH 07/24] amend ChangeLog updated: `ignoreip` extended with `file:...` syntax to ignore IPs from file-ip-set; + silence codespell --- ChangeLog | 5 +++++ fail2ban/server/ipdns.py | 2 +- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/ChangeLog b/ChangeLog index f087fbf2..b43aac80 100644 --- a/ChangeLog +++ b/ChangeLog @@ -40,6 +40,11 @@ ver. 1.1.1-dev-1 (20??/??/??) - development nightly edition ### New Features and Enhancements * new jail option `skip_if_nologs` to ignore jail if no `logpath` matches found, fail2ban continue to start with warnings/errors, thus other jails become running (gh-2756) +* configuration `ignoreip` and fail2ban-client commands `addignoreip`/`delignoreip` extended with `file:...` syntax + to ignore IPs from file-ip-set (containing IP, subnet, dns/fqdn or raw strings); the file would be read lazy on demand, + by first ban (and automatically reloaded by update after small latency to avoid expensive stats check on every compare); + the entries inside the file can be separated by comma, space or new line with optional comments (text following chars + `#` or `;` after space or newline would be ignored up to next newline) * `action.d/*-ipset.conf`: - parameter `ipsettype` to set type of ipset, e. g. hash:ip, hash:net, etc (gh-3760) * `action.d/firewallcmd-rich-*.conf` - fixed incorrect quoting, disabling port variable expansion diff --git a/fail2ban/server/ipdns.py b/fail2ban/server/ipdns.py index 665bddc6..a485a1b2 100644 --- a/fail2ban/server/ipdns.py +++ b/fail2ban/server/ipdns.py @@ -793,7 +793,7 @@ class FileIPAddrSet(IPAddrSet): return self._shortRepr def __contains__(self, ip): - # check it is uptodate (not often than maxUpdateLatency): + # check it is up-to-date (not often than maxUpdateLatency): if self.fileName: self.load(ifNeeded=True) # inherited contains: From 9145db8de3872011e090fc6d22c33d705af95850 Mon Sep 17 00:00:00 2001 From: sebres Date: Mon, 3 Mar 2025 23:59:36 +0100 Subject: [PATCH 08/24] small code review of FileIPAddrSet: encapsulate check for changed logic to _isModified and slightly increase coverage for it (latency, changed, unchanged) --- fail2ban/server/ipdns.py | 44 ++++++++++++++++++++------------ fail2ban/tests/filtertestcase.py | 10 ++++++++ 2 files changed, 38 insertions(+), 16 deletions(-) diff --git a/fail2ban/server/ipdns.py b/fail2ban/server/ipdns.py index a485a1b2..1264479e 100644 --- a/fail2ban/server/ipdns.py +++ b/fail2ban/server/ipdns.py @@ -765,21 +765,33 @@ class FileIPAddrSet(IPAddrSet): if m: return self.fileName == m.group(1) - def load(self, ifNeeded=True, noError=True): + def _isModified(self): + """Check whether the file is modified (file stats changed) + + Side effect: if modified, _fileStats will be updated to last known stats of file + """ + tm = MyTime.time() + # avoid to check it always (not often than maxUpdateLatency): + if tm <= self._nextCheck: + return None; # no check needed + self._nextCheck = tm + self.maxUpdateLatency + stats = os.stat(self.fileName) + stats = stats.st_mtime, stats.st_ino, stats.st_size + if self._fileStats != stats: + self._fileStats = stats + return True; # modified, needs to be reloaded + return False; # unmodified + + def load(self, forceReload=False, noError=True): + """Load set from file (on demand if needed or by forceReload) + """ try: - if ifNeeded: - tm = MyTime.time() - if tm > self._nextCheck: - self._nextCheck = tm + self.maxUpdateLatency - stats = os.stat(self.fileName) - stats = stats.st_mtime, stats.st_ino, stats.st_size - if self._fileStats == stats: - return - self._fileStats = stats - with open(self.fileName, 'r') as f: - ips = f.read() - ips = splitwords(ips, ignoreComments=True) - self.set(ips) + # load only if needed and modified (or first time load on demand) + if self._isModified() or forceReload: + with open(self.fileName, 'r') as f: + ips = f.read() + ips = splitwords(ips, ignoreComments=True) + self.set(ips) except Exception as e: # pragma: no cover if not noError: raise e logSys.warning("Retrieving IPs set from %r failed: %s", self.fileName, e) @@ -793,9 +805,9 @@ class FileIPAddrSet(IPAddrSet): return self._shortRepr def __contains__(self, ip): - # check it is up-to-date (not often than maxUpdateLatency): + # load if needed: if self.fileName: - self.load(ifNeeded=True) + self.load() # inherited contains: return IPAddrSet.__contains__(self, ip) diff --git a/fail2ban/tests/filtertestcase.py b/fail2ban/tests/filtertestcase.py index ece75201..d12c93bc 100644 --- a/fail2ban/tests/filtertestcase.py +++ b/fail2ban/tests/filtertestcase.py @@ -2546,6 +2546,16 @@ class DNSUtilsNetworkTests(unittest.TestCase): self.assertTrue(IPAddr('192.0.2.202') in ips) self.assertTrue(IPAddr('2001:db8::ca') in ips) self.assertTrue(IPAddr('2001:db8::cb') in ips) + # +1m, jump to next minute to force next check for update: + MyTime.setTime(MyTime.time() + 60) + self.assertFalse(ips._isModified()); # must be unchanged + self.assertEqual(ips._isModified(), None); # not checked by latency (same time) + f.write(b"""#END of file\n""") + f.flush() + # +1m, jump to next minute to force next check for update: + MyTime.setTime(MyTime.time() + 60) + self.assertTrue(ips._isModified()); # must be modified + self.assertEqual(ips._isModified(), None); # not checked by latency (same time) finally: tearDownMyTime() _killfile(f, fname) From e3ab969047cd340333dba9fc27b253797efa1dfc Mon Sep 17 00:00:00 2001 From: sebres Date: Tue, 4 Mar 2025 00:07:31 +0100 Subject: [PATCH 09/24] increase interval for up-to-date check (to 1 minute) after error, to avoid continuous flood in log on further possible errors --- fail2ban/server/ipdns.py | 1 + 1 file changed, 1 insertion(+) diff --git a/fail2ban/server/ipdns.py b/fail2ban/server/ipdns.py index 1264479e..d188d5c8 100644 --- a/fail2ban/server/ipdns.py +++ b/fail2ban/server/ipdns.py @@ -793,6 +793,7 @@ class FileIPAddrSet(IPAddrSet): ips = splitwords(ips, ignoreComments=True) self.set(ips) except Exception as e: # pragma: no cover + self._nextCheck += 60; # increase interval to check (to 1 minute, to avoid log flood on errors) if not noError: raise e logSys.warning("Retrieving IPs set from %r failed: %s", self.fileName, e) From 65d473fc8e3edea0419e061d88dac5586d9c223e Mon Sep 17 00:00:00 2001 From: Lucian Maly Date: Tue, 4 Mar 2025 11:43:38 +1100 Subject: [PATCH 10/24] Added regex for systemd-journal matches of vsftpd --- config/filter.d/vsftpd.conf | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/config/filter.d/vsftpd.conf b/config/filter.d/vsftpd.conf index 2ecc44d3..53b1f4b3 100644 --- a/config/filter.d/vsftpd.conf +++ b/config/filter.d/vsftpd.conf @@ -15,8 +15,9 @@ _daemon = vsftpd failregex = ^%(__prefix_line)s%(__pam_re)s\s+authentication failure; logname=\S* uid=\S* euid=\S* tty=(ftp)? ruser=\S* rhost=(?:\s+user=.*)?\s*$ ^ \[pid \d+\] \[[^\]]+\] FAIL LOGIN: Client ""(?:\s*$|,) + ^(([a-zA-Z0-9]|[a-zA-Z0-9][a-zA-Z0-9\-]*[a-zA-Z0-9])\.)*([A-Za-z0-9]|[A-Za-z0-9][A-Za-z0-9\-]*[A-Za-z0-9]) vsftpd\[\d+\]: \[[^\]]+\] FAIL LOGIN: Client ""(?:\s*$|,) -ignoreregex = +ignoreregex = # Author: Cyril Jaquier # Documentation from fail2ban wiki From bd4cb606e59e612a6dac124d296389ff28b45f0f Mon Sep 17 00:00:00 2001 From: Lucian Maly Date: Tue, 4 Mar 2025 11:47:49 +1100 Subject: [PATCH 11/24] Added sample log line --- fail2ban/tests/files/logs/vsftpd | 3 +++ 1 file changed, 3 insertions(+) diff --git a/fail2ban/tests/files/logs/vsftpd b/fail2ban/tests/files/logs/vsftpd index 3205fac3..747cb6e1 100644 --- a/fail2ban/tests/files/logs/vsftpd +++ b/fail2ban/tests/files/logs/vsftpd @@ -15,3 +15,6 @@ Oct 23 21:15:42 vps vsftpd: pam_unix(vsftpd:auth): authentication failure; logna # failJSON: { "time": "2016-09-08T00:39:49", "match": true , "host": "192.0.2.1" } Thu Sep 8 00:39:49 2016 [pid 15019] [guest] FAIL LOGIN: Client "::ffff:192.0.2.1", "User is not in the allow user list." + +# systemd-journal +2025-03-04T01:06:36.645577 ip-172-31-3-150.ap-southeast-2.compute.internal vsftpd[1658]: [username] FAIL LOGIN: Client "121.251.18.222" From fd1d0d25a828321dc705d05006216e26d399fdea Mon Sep 17 00:00:00 2001 From: Lucian Maly Date: Tue, 4 Mar 2025 12:20:24 +1100 Subject: [PATCH 12/24] Added regex for systemd-journal matches of lighttpd-auth --- config/filter.d/lighttpd-auth.conf | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/config/filter.d/lighttpd-auth.conf b/config/filter.d/lighttpd-auth.conf index dcf19d3e..f6b5893e 100644 --- a/config/filter.d/lighttpd-auth.conf +++ b/config/filter.d/lighttpd-auth.conf @@ -4,7 +4,8 @@ [Definition] failregex = ^\s*(?:: )?\(?(?:http|mod)_auth\.c\.\d+\) (?:password doesn\'t match for (?:\S+|.*?) username:\s+(?:\S+|.*?)\s*|digest: auth failed(?: for\s+(?:\S+|.*?)\s*)?: (?:wrong password|uri mismatch \([^\)]*\))|get_password failed),? IP: \s*$ + ^(([a-zA-Z0-9]|[a-zA-Z0-9][a-zA-Z0-9\-]*[a-zA-Z0-9])\.)*([A-Za-z0-9]|[A-Za-z0-9][A-Za-z0-9\-]*[A-Za-z0-9]) lighttpd\[\d+\]: \s*(?:: )?\(?(?:http|mod)_auth\.c\.\d+\) (?:password doesn\'t match for (?:\S+|.*?) username:\s+(?:\S+|.*?)\s*|digest: auth failed(?: for\s+(?:\S+|.*?)\s*)?: (?:wrong password|uri mismatch \([^\)]*\))|get_password failed),? IP: \s*$ -ignoreregex = +ignoreregex = # Author: Francois Boulogne From f5ba525cd2c665037ac88b2d7380f1e028b1c163 Mon Sep 17 00:00:00 2001 From: Lucian Maly Date: Tue, 4 Mar 2025 12:22:35 +1100 Subject: [PATCH 13/24] Added sample log line --- fail2ban/tests/files/logs/lighttpd-auth | 3 +++ 1 file changed, 3 insertions(+) diff --git a/fail2ban/tests/files/logs/lighttpd-auth b/fail2ban/tests/files/logs/lighttpd-auth index c8a922b5..4619b2a8 100644 --- a/fail2ban/tests/files/logs/lighttpd-auth +++ b/fail2ban/tests/files/logs/lighttpd-auth @@ -12,3 +12,6 @@ 2021-09-30 17:44:37: (mod_auth.c.791) digest: auth failed for tester : wrong password, IP: 192.0.2.3 # failJSON: { "time": "2021-09-30T17:44:37", "match": true , "host": "192.0.2.4", "desc": "gh-3116" } 2021-09-30 17:44:37: (mod_auth.c.791) digest: auth failed: uri mismatch (/uri1 != /uri2), IP: 192.0.2.4 + +# systemd-journal +2025-03-04T02:11:57.602061 ip-172-31-3-150.ap-southeast-2.compute.internal lighttpd[764]: (mod_auth.c.853) password doesn't match for / username: user1 IP: 122.251.111.211 From 9d7646e6c0b0f03db7f9a1dd3a9aef5178e4da8d Mon Sep 17 00:00:00 2001 From: Lucian Maly Date: Tue, 4 Mar 2025 12:25:27 +1100 Subject: [PATCH 14/24] Added author --- config/filter.d/lighttpd-auth.conf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config/filter.d/lighttpd-auth.conf b/config/filter.d/lighttpd-auth.conf index f6b5893e..56fc4cae 100644 --- a/config/filter.d/lighttpd-auth.conf +++ b/config/filter.d/lighttpd-auth.conf @@ -8,4 +8,4 @@ failregex = ^\s*(?:: )?\(?(?:http|mod)_auth\.c\.\d+\) (?:password doesn\'t match ignoreregex = -# Author: Francois Boulogne +# Authors: Francois Boulogne , Lucian Maly From 6e3bfd800c1cd7e8a769e754edc6db5e0e6f11de Mon Sep 17 00:00:00 2001 From: Lucian Maly Date: Tue, 4 Mar 2025 12:26:14 +1100 Subject: [PATCH 15/24] Added author --- config/filter.d/vsftpd.conf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config/filter.d/vsftpd.conf b/config/filter.d/vsftpd.conf index 53b1f4b3..44646086 100644 --- a/config/filter.d/vsftpd.conf +++ b/config/filter.d/vsftpd.conf @@ -19,5 +19,5 @@ failregex = ^%(__prefix_line)s%(__pam_re)s\s+authentication failure; logname=\S* ignoreregex = -# Author: Cyril Jaquier +# Authors: Cyril Jaquier, Lucian Maly # Documentation from fail2ban wiki From 13a74feaad10f52b6b54fc60f54334f089e2e8cf Mon Sep 17 00:00:00 2001 From: "Sergey G. Brester" Date: Tue, 4 Mar 2025 13:02:50 +0100 Subject: [PATCH 16/24] 2nd RE unneeded, fix single RE - bypass everything before open parenthesis --- config/filter.d/lighttpd-auth.conf | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/config/filter.d/lighttpd-auth.conf b/config/filter.d/lighttpd-auth.conf index 56fc4cae..7e8be0f9 100644 --- a/config/filter.d/lighttpd-auth.conf +++ b/config/filter.d/lighttpd-auth.conf @@ -3,8 +3,7 @@ [Definition] -failregex = ^\s*(?:: )?\(?(?:http|mod)_auth\.c\.\d+\) (?:password doesn\'t match for (?:\S+|.*?) username:\s+(?:\S+|.*?)\s*|digest: auth failed(?: for\s+(?:\S+|.*?)\s*)?: (?:wrong password|uri mismatch \([^\)]*\))|get_password failed),? IP: \s*$ - ^(([a-zA-Z0-9]|[a-zA-Z0-9][a-zA-Z0-9\-]*[a-zA-Z0-9])\.)*([A-Za-z0-9]|[A-Za-z0-9][A-Za-z0-9\-]*[A-Za-z0-9]) lighttpd\[\d+\]: \s*(?:: )?\(?(?:http|mod)_auth\.c\.\d+\) (?:password doesn\'t match for (?:\S+|.*?) username:\s+(?:\S+|.*?)\s*|digest: auth failed(?: for\s+(?:\S+|.*?)\s*)?: (?:wrong password|uri mismatch \([^\)]*\))|get_password failed),? IP: \s*$ +failregex = ^[^\)]*\(?(?:http|mod)_auth\.c\.\d+\) (?:password doesn\'t match for (?:\S+|.*?) username:\s+(?:\S+|.*?)\s*|digest: auth failed(?: for\s+(?:\S+|.*?)\s*)?: (?:wrong password|uri mismatch \([^\)]*\))|get_password failed),? IP: \s*$ ignoreregex = From 95cdf553f551a23dec37fcfef996d6b156cc594e Mon Sep 17 00:00:00 2001 From: "Sergey G. Brester" Date: Tue, 4 Mar 2025 13:09:21 +0100 Subject: [PATCH 17/24] fixes test in lighttpd-auth: added failJSON to match the line --- fail2ban/tests/files/logs/lighttpd-auth | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/fail2ban/tests/files/logs/lighttpd-auth b/fail2ban/tests/files/logs/lighttpd-auth index 4619b2a8..7c48eeb6 100644 --- a/fail2ban/tests/files/logs/lighttpd-auth +++ b/fail2ban/tests/files/logs/lighttpd-auth @@ -14,4 +14,5 @@ 2021-09-30 17:44:37: (mod_auth.c.791) digest: auth failed: uri mismatch (/uri1 != /uri2), IP: 192.0.2.4 # systemd-journal -2025-03-04T02:11:57.602061 ip-172-31-3-150.ap-southeast-2.compute.internal lighttpd[764]: (mod_auth.c.853) password doesn't match for / username: user1 IP: 122.251.111.211 +# failJSON: { "time": "2025-03-04T02:11:57", "match": true , "host": "192.0.2.211", "desc": "gh-3955" } +2025-03-04T02:11:57.602061 ip-172-31-3-150.ap-southeast-2.compute.internal lighttpd[764]: (mod_auth.c.853) password doesn't match for / username: user1 IP: 192.0.2.211 From 3e9a4b4a4865d56488a007722a4da96d0f5a9bea Mon Sep 17 00:00:00 2001 From: "Sergey G. Brester" Date: Tue, 4 Mar 2025 13:20:54 +0100 Subject: [PATCH 18/24] Update ChangeLog --- ChangeLog | 1 + 1 file changed, 1 insertion(+) diff --git a/ChangeLog b/ChangeLog index b43aac80..d579a40e 100644 --- a/ChangeLog +++ b/ChangeLog @@ -26,6 +26,7 @@ ver. 1.1.1-dev-1 (20??/??/??) - development nightly edition * `filter.d/exim.conf` - mode `aggressive` extended to catch dropped by ACL failures, e.g. "ACL: Country is banned" * `filter.d/freeswitch.conf` - bypass some new info in prefix before [WARNING] (changed default `_pref_line`), FreeSWITCH log line prefix has changed in newer versions (gh-3143) +* `filter.d/lighttpd-auth.conf` - fixed regex (if failures generated by systemd-journal), bypass several prefixes now (gh-3955) * `filter.d/postfix.conf` - consider CONNECT and other rejected commands as a valid `_pref` (gh-3800) * `filter.d/dropbear.conf`: - recognizes extra pid/timestamp if logged into stdout/journal, added `journalmatch` (gh-3597) From 1e06ab68b4eba5391b78c5da8fbd5d062a965376 Mon Sep 17 00:00:00 2001 From: sebres Date: Tue, 4 Mar 2025 13:47:59 +0100 Subject: [PATCH 19/24] fixed filter (new regex is unneeded), tests format of failures produced by system journal --- config/filter.d/vsftpd.conf | 3 +-- fail2ban/tests/files/logs/vsftpd | 6 ++++-- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/config/filter.d/vsftpd.conf b/config/filter.d/vsftpd.conf index 44646086..859a67c3 100644 --- a/config/filter.d/vsftpd.conf +++ b/config/filter.d/vsftpd.conf @@ -14,8 +14,7 @@ __pam_re=\(?%(__pam_auth)s(?:\(\S+\))?\)?:? _daemon = vsftpd failregex = ^%(__prefix_line)s%(__pam_re)s\s+authentication failure; logname=\S* uid=\S* euid=\S* tty=(ftp)? ruser=\S* rhost=(?:\s+user=.*)?\s*$ - ^ \[pid \d+\] \[[^\]]+\] FAIL LOGIN: Client ""(?:\s*$|,) - ^(([a-zA-Z0-9]|[a-zA-Z0-9][a-zA-Z0-9\-]*[a-zA-Z0-9])\.)*([A-Za-z0-9]|[A-Za-z0-9][A-Za-z0-9\-]*[A-Za-z0-9]) vsftpd\[\d+\]: \[[^\]]+\] FAIL LOGIN: Client ""(?:\s*$|,) + ^(?:\s*\[pid \d+\] |%(__prefix_line)s)\[[^\]]+\] FAIL LOGIN: Client ""(?:\s*$|,) ignoreregex = diff --git a/fail2ban/tests/files/logs/vsftpd b/fail2ban/tests/files/logs/vsftpd index 747cb6e1..ab51fd75 100644 --- a/fail2ban/tests/files/logs/vsftpd +++ b/fail2ban/tests/files/logs/vsftpd @@ -16,5 +16,7 @@ Oct 23 21:15:42 vps vsftpd: pam_unix(vsftpd:auth): authentication failure; logna # failJSON: { "time": "2016-09-08T00:39:49", "match": true , "host": "192.0.2.1" } Thu Sep 8 00:39:49 2016 [pid 15019] [guest] FAIL LOGIN: Client "::ffff:192.0.2.1", "User is not in the allow user list." -# systemd-journal -2025-03-04T01:06:36.645577 ip-172-31-3-150.ap-southeast-2.compute.internal vsftpd[1658]: [username] FAIL LOGIN: Client "121.251.18.222" +# fileOptions: {"logtype": "journal"} + +# failJSON: { "match": true , "host": "192.0.2.222" } +2025-03-04T01:06:36.645577 ip-172-31-3-150.ap-southeast-2.compute.internal vsftpd[1658]: [username] FAIL LOGIN: Client "192.0.2.222" From 94fe9cf4a8ce86c11af71be87f0e682bae210edd Mon Sep 17 00:00:00 2001 From: sebres Date: Tue, 4 Mar 2025 14:13:07 +0100 Subject: [PATCH 20/24] more fixes, capture user names, more tests... since line 7 matches successfully now (it was disabled in gh-358 because of obsolete format), it is marked as match:true (line can be removed later if unneeded) --- config/filter.d/vsftpd.conf | 4 ++-- fail2ban/tests/files/logs/vsftpd | 11 ++++++++--- 2 files changed, 10 insertions(+), 5 deletions(-) diff --git a/config/filter.d/vsftpd.conf b/config/filter.d/vsftpd.conf index 859a67c3..8b3047ca 100644 --- a/config/filter.d/vsftpd.conf +++ b/config/filter.d/vsftpd.conf @@ -10,10 +10,10 @@ before = common.conf [Definition] -__pam_re=\(?%(__pam_auth)s(?:\(\S+\))?\)?:? +__pam_re=(?:\(?%(__pam_auth)s(?:\(\S+\))?\)?:?\s+)? _daemon = vsftpd -failregex = ^%(__prefix_line)s%(__pam_re)s\s+authentication failure; logname=\S* uid=\S* euid=\S* tty=(ftp)? ruser=\S* rhost=(?:\s+user=.*)?\s*$ +failregex = ^%(__prefix_line)s%(__pam_re)sauthentication failure; logname=\S* uid=\S* euid=\S* tty=(?:ftp)? ruser=\S* rhost=(?:\s+user=\S*)?\s*$ ^(?:\s*\[pid \d+\] |%(__prefix_line)s)\[[^\]]+\] FAIL LOGIN: Client ""(?:\s*$|,) ignoreregex = diff --git a/fail2ban/tests/files/logs/vsftpd b/fail2ban/tests/files/logs/vsftpd index ab51fd75..18f3879c 100644 --- a/fail2ban/tests/files/logs/vsftpd +++ b/fail2ban/tests/files/logs/vsftpd @@ -2,8 +2,8 @@ # failJSON: { "time": "2004-10-11T01:06:47", "match": true , "host": "209.67.1.67" } Oct 11 01:06:47 ServerJV vsftpd: (pam_unix) authentication failure; logname= uid=0 euid=0 tty= ruser= rhost=209.67.1.67 -# Pam pre 0.99.2.0 - https://github.com/fail2ban/fail2ban/pull/358 -# failJSON: { "time": "2005-02-06T12:02:29", "match": false , "host": "64.168.103.1" } +# Pam pre 0.99.2.0 - https://github.com/fail2ban/fail2ban/pull/358 (format is obsolete, can be removed, but still match right now) +# failJSON: { "time": "2005-02-06T12:02:29", "match": true , "host": "64.168.103.1", "desc": "obsolete, can be removed, but still match right now" } Feb 6 12:02:29 server vsftpd(pam_unix)[15522]: authentication failure; logname= uid=0 euid=0 tty= ruser= rhost=64.168.103.1 user=user1 #2 Internal @@ -18,5 +18,10 @@ Thu Sep 8 00:39:49 2016 [pid 15019] [guest] FAIL LOGIN: Client "::ffff:192.0.2. # fileOptions: {"logtype": "journal"} -# failJSON: { "match": true , "host": "192.0.2.222" } +# failJSON: { "match": true , "host": "192.0.2.222", "desc": "gh-3954" } 2025-03-04T01:06:36.645577 ip-172-31-3-150.ap-southeast-2.compute.internal vsftpd[1658]: [username] FAIL LOGIN: Client "192.0.2.222" + +# failJSON: { "match": true , "host": "192.0.2.223", "desc": "gh-3954, more tests, without part `pam_unix(vsftpd:auth): ` (unknown if it is needed)" } +2025-03-04T01:06:37.123456 ip-172-31-3-150.ap-southeast-2.compute.internal vsftpd[1659]: authentication failure; logname= uid=0 euid=0 tty= ruser= rhost=192.0.2.223 user=tester +# failJSON: { "match": true , "host": "192.0.2.224", "desc": "gh-3954, more tests, with part `pam_unix(vsftpd:auth): ` (unknown if it is needed, but it matches)" } +2025-03-04T01:06:38.123456 ip-172-31-3-150.ap-southeast-2.compute.internal vsftpd[1660]: pam_unix(vsftpd:auth): authentication failure; logname= uid=0 euid=0 tty= ruser= rhost=192.0.2.224 user=tester From 79346e4f2c8a1f43be0ea02657a4318eda7cec37 Mon Sep 17 00:00:00 2001 From: sebres Date: Tue, 4 Mar 2025 14:15:14 +0100 Subject: [PATCH 21/24] updated ChangeLog --- ChangeLog | 1 + 1 file changed, 1 insertion(+) diff --git a/ChangeLog b/ChangeLog index b43aac80..bc9094d1 100644 --- a/ChangeLog +++ b/ChangeLog @@ -36,6 +36,7 @@ ver. 1.1.1-dev-1 (20??/??/??) - development nightly edition - adapted to conform possible new daemon name sshd-session, since OpenSSH 9.8 several log messages will be tagged with as originating from a process named "sshd-session" rather than "sshd" (gh-3782) - `ddos` and `aggressive` modes: regex extended for timeout before authentication (optional connection from part, gh-3907) +* `filter.d/vsftpd.conf` - fixed regex (if failures generated by systemd-journal, gh-3954) ### New Features and Enhancements * new jail option `skip_if_nologs` to ignore jail if no `logpath` matches found, fail2ban continue to start with warnings/errors, From cf9c8f1e9b3a712525e46d5e6a894aab90e80ca5 Mon Sep 17 00:00:00 2001 From: sebres Date: Tue, 4 Mar 2025 14:27:21 +0100 Subject: [PATCH 22/24] test-suite: fixed sample regexs factory counting of line number (if it errors, the line number showing in error line was incorrect, because of missing increment) --- fail2ban/tests/samplestestcase.py | 1 + 1 file changed, 1 insertion(+) diff --git a/fail2ban/tests/samplestestcase.py b/fail2ban/tests/samplestestcase.py index 15cdf646..40c9a8ae 100644 --- a/fail2ban/tests/samplestestcase.py +++ b/fail2ban/tests/samplestestcase.py @@ -205,6 +205,7 @@ def testSampleRegexsFactory(name, basedir): raise ValueError("%s: %s:%i" % (e, logFile.getFileName(), lnnum)) line = next(logFile) + lnnum += 1 elif ignoreBlock or line.startswith("#") or not line.strip(): continue else: # pragma: no cover - normally unreachable From 4bb1fd519db09c44a76b35963cc0152a703c90f4 Mon Sep 17 00:00:00 2001 From: sebres Date: Tue, 4 Mar 2025 14:39:24 +0100 Subject: [PATCH 23/24] test-suite: if failed, sample regexs factory would show responsible header line (failJSON) together with the error line --- fail2ban/tests/samplestestcase.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/fail2ban/tests/samplestestcase.py b/fail2ban/tests/samplestestcase.py index 40c9a8ae..fe3676fd 100644 --- a/fail2ban/tests/samplestestcase.py +++ b/fail2ban/tests/samplestestcase.py @@ -163,6 +163,7 @@ def testSampleRegexsFactory(name, basedir): ignoreBlock = False lnnum = 0 for line in logFile: + jsonline = '' lnnum += 1 jsonREMatch = re.match("^#+ ?(failJSON|(?:file|filter)Options|addFILE):(.+)$", line) if jsonREMatch: @@ -204,6 +205,7 @@ def testSampleRegexsFactory(name, basedir): except ValueError as e: # pragma: no cover - we've valid json's raise ValueError("%s: %s:%i" % (e, logFile.getFileName(), lnnum)) + jsonline = line line = next(logFile) lnnum += 1 elif ignoreBlock or line.startswith("#") or not line.strip(): @@ -300,8 +302,9 @@ def testSampleRegexsFactory(name, basedir): import pprint raise AssertionError("%s: %s on: %s:%i, line:\n %s\nregex (%s):\n %s\n" "faildata: %s\nfail: %s" % ( - fltName, e, logFile.getFileName(), lnnum, - line, failregex, regexList[failregex] if failregex != -1 else None, + fltName, e, logFile.getFileName(), lnnum, + (("%s\n\u25ba %s" % (jsonline, line)) if jsonline else line), + failregex, regexList[failregex] if failregex != -1 else None, '\n'.join(pprint.pformat(faildata).splitlines()), '\n'.join(pprint.pformat(fail).splitlines()))) From 505d51fd5df151a093c6423d1762785cff435fb2 Mon Sep 17 00:00:00 2001 From: "Sergey G. Brester" Date: Tue, 4 Mar 2025 19:19:57 +0100 Subject: [PATCH 24/24] Update PULL_REQUEST_TEMPLATE.md --- .github/PULL_REQUEST_TEMPLATE.md | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md index 350d6ee2..b53003d5 100644 --- a/.github/PULL_REQUEST_TEMPLATE.md +++ b/.github/PULL_REQUEST_TEMPLATE.md @@ -1,12 +1,10 @@ Before submitting your PR, please review the following checklist: -- [ ] **CHOOSE CORRECT BRANCH**: if filing a bugfix/enhancement - against certain release version, choose `0.9`, `0.10` or `0.11` branch, - for dev-edition use `master` branch - [ ] **CONSIDER adding a unit test** if your PR resolves an issue -- [ ] **LIST ISSUES** this PR resolves +- [ ] **LIST ISSUES** this PR resolves or describe the approach in detail - [ ] **MAKE SURE** this PR doesn't break existing tests -- [ ] **KEEP PR small** so it could be easily reviewed. +- [ ] **KEEP PR small** so it could be easily reviewed - [ ] **AVOID** making unnecessary stylistic changes in unrelated code - [ ] **ACCOMPANY** each new `failregex` for filter `X` with sample log lines - within `fail2ban/tests/files/logs/X` file + (and `# failJSON`) within `fail2ban/tests/files/logs/X` file +- [ ] **PROVIDE ChangeLog** entry describing the pull request