Merge pull request #2560 from sebres/gh-927-subnet

subnet implementation (filter, parsing tags)

ChangeLog

@@ -79,8 +79,13 @@ ver. 0.10.5-dev-1 (20??/??/??) - development edition
     - `prefregex` extended, more selective now (denied/NOTAUTH suffix moved from failregex, so no catch-all there anymore)
 * `filter.d/sendmail-auth.conf`, `filter.d/sendmail-reject.conf` :
   - ID in prefix can be longer as 14 characters (gh-2563);
+* all filters would accept square brackets around IPv4 addresses also (e. g. monit-filter, gh-2494)
 
 ### New Features
+* new replacement tags for failregex to match subnets in form of IP-addresses with CIDR mask (gh-2559):
+  - `<CIDR>` - helper regex to match CIDR (simple integer form of net-mask);
+  - `<SUBNET>` - regex to match sub-net adresses (in form of IP/CIDR, also single IP is matched, so part /CIDR is optional);
+* grouped tags (`<ADDR>`, `<HOST>`, `<SUBNET>`) recognize IP addresses enclosed in square brackets
 * new failregex-flag tag `<F-MLFGAINED>` for failregex, signaled that the access to service was gained
   (ATM used similar to tag `<F-NOFAIL>`, but it does not add the log-line to matches, gh-2279)
 * filters: introduced new configuration parameter `logtype` (default `file` for file-backends, and 

fail2ban/server/failregex.py

@@ -37,25 +37,28 @@ R_HOST = [
 		r"""(?:::f{4,6}:)?(?P<ip4>%s)""" % (IPAddr.IP_4_RE,),
 		# separated ipv6:
 		r"""(?P<ip6>%s)""" % (IPAddr.IP_6_RE,),
-		# place-holder for ipv6 enclosed in optional [] (used in addr-, host-regex)
-		"",
 		# separated dns:
 		r"""(?P<dns>[\w\-.^_]*\w)""",
 		# place-holder for ADDR tag-replacement (joined):
 		"",
 		# place-holder for HOST tag replacement (joined):
-		""
+		"",
+		# CIDR in simplest integer form:
+		r"(?P<cidr>\d+)",
+		# place-holder for SUBNET tag-replacement
+		"",
 ]
 RI_IPV4 =		0
 RI_IPV6 =		1
-RI_IPV6BR =	2
+RI_DNS =		2
-RI_DNS =		3
+RI_ADDR =		3
-RI_ADDR =		4
+RI_HOST =		4
-RI_HOST =		5
+RI_CIDR =		5
+RI_SUBNET =	6
 
-R_HOST[RI_IPV6BR] =	r"""\[?%s\]?""" % (R_HOST[RI_IPV6],)
+R_HOST[RI_ADDR] =		r"\[?(?:%s|%s)\]?" % (R_HOST[RI_IPV4], R_HOST[RI_IPV6],)
-R_HOST[RI_ADDR] =		"(?:%s)" % ("|".join((R_HOST[RI_IPV4], R_HOST[RI_IPV6BR])),)
+R_HOST[RI_HOST] =		r"(?:%s|%s)" % (R_HOST[RI_ADDR], R_HOST[RI_DNS],)
-R_HOST[RI_HOST] =		"(?:%s)" % ("|".join((R_HOST[RI_IPV4], R_HOST[RI_IPV6BR], R_HOST[RI_DNS])),)
+R_HOST[RI_SUBNET] =	r"\[?(?:%s|%s)(?:/%s)?\]?" % (R_HOST[RI_IPV4], R_HOST[RI_IPV6], R_HOST[RI_CIDR],)
 
 RH4TAG = {
 	# separated ipv4 (self closed, closed):
@@ -68,6 +71,11 @@ RH4TAG = {
 	# for separate usage of 2 address groups only (regardless of `usedns`), `ip4` and `ip6` together
 	"ADDR":			R_HOST[RI_ADDR],
 	"F-ADDR/":	R_HOST[RI_ADDR],
+	# subnet tags for usage as `<ADDR>/<CIDR>` or `<SUBNET>`:
+	"CIDR":			R_HOST[RI_CIDR],
+	"F-CIDR/":	R_HOST[RI_CIDR],
+	"SUBNET":		R_HOST[RI_SUBNET],
+	"F-SUBNET/":R_HOST[RI_SUBNET],
 	# separated dns (self closed, closed):
 	"DNS":			R_HOST[RI_DNS],
 	"F-DNS/":		R_HOST[RI_DNS],
@@ -416,3 +424,7 @@ class FailRegex(Regex):
 	
 	def getHost(self):
 		return self.getFailID(("ip4", "ip6", "dns"))
+
+	def getIP(self):
+		fail = self.getGroups()
+		return IPAddr(self.getFailID(("ip4", "ip6")), int(fail.get("cidr") or IPAddr.CIDR_UNSPEC))

fail2ban/server/filter.py

@@ -865,12 +865,12 @@ class Filter(JailThread):
 				# ip-address or host:
 				host = fail.get('ip4')
 				if host is not None:
-					cidr = IPAddr.FAM_IPv4
+					cidr = int(fail.get('cidr') or IPAddr.FAM_IPv4)
 					raw = True
 				else:
 					host = fail.get('ip6')
 					if host is not None:
-						cidr = IPAddr.FAM_IPv6
+						cidr = int(fail.get('cidr') or IPAddr.FAM_IPv6)
 						raw = True
 				if host is None:
 					host = fail.get('dns')

fail2ban/tests/fail2banregextestcase.py

@@ -315,6 +315,19 @@ class Fail2banRegexTest(LogCaptureTestCase):
 		self.assertTrue(fail2banRegex.start(args))
 		self.assertLogged('Lines: 4 lines, 0 ignored, 4 matched, 0 missed')
 
+	def testRegexSubnet(self):
+		(opts, args, fail2banRegex) = _Fail2banRegex(
+			"-vv", "-d", r"^\[{LEPOCH}\]\s+", "--maxlines", "5",
+			"[1516469849] 192.0.2.1 FAIL: failure\n"
+			"[1516469849] 192.0.2.1/24 FAIL: failure\n"
+			"[1516469849] 2001:DB8:FF:FF::1 FAIL: failure\n"
+			"[1516469849] 2001:DB8:FF:FF::1/60 FAIL: failure\n",
+			r"^<SUBNET> FAIL\b"
+		)
+		self.assertTrue(fail2banRegex.start(args))
+		self.assertLogged('Lines: 4 lines, 0 ignored, 4 matched, 0 missed')
+		self.assertLogged('192.0.2.0/24', '2001:db8:ff:f0::/60', all=True)
+
 	def testWrongFilterFile(self):
 		# use test log as filter file to cover eror cases...
 		(opts, args, fail2banRegex) = _Fail2banRegex(

fail2ban/tests/files/logs/monit

@@ -19,3 +19,6 @@ Mar  9 09:18:32 hostname monit[5731]: HttpRequest: access denied -- client 1.2.3
 Mar  9 09:18:33 hostname monit[5731]: HttpRequest: access denied -- client 1.2.3.4: unknown user 'test1'
 # failJSON: { "time": "2005-03-09T09:18:34", "match": true, "host": "1.2.3.4", "desc": "wrong password try" }
 Mar  9 09:18:34 hostname monit[5731]: HttpRequest: access denied -- client 1.2.3.4: wrong password for user 'test2'
+
+# failJSON: { "time": "2005-08-06T10:14:52", "match": true, "host": "192.168.1.85", "desc": "IP in brackets, gh-2494" }
+[CEST Aug  6 10:14:52] error    : HttpRequest: access denied -- client [192.168.1.85]: wrong password for user 'root'

fail2ban/tests/servertestcase.py

@@ -1055,6 +1055,34 @@ class RegexTests(unittest.TestCase):
 		fr.search([('test id group: user:(test login name)',"","")])
 		self.assertTrue(fr.hasMatched())
 		self.assertEqual(fr.getFailID(), 'test login name')
+		# Success case: subnet with IPAddr (IP and subnet) conversion:
+		fr = FailRegex(r'%%net=<SUBNET>')
+		fr.search([('%%net=192.0.2.1',"","")])
+		ip = fr.getIP()
+		self.assertEqual((ip, ip.familyStr), ('192.0.2.1', 'inet4'))
+		fr.search([('%%net=192.0.2.1/24',"","")])
+		ip = fr.getIP()
+		self.assertEqual((ip, ip.familyStr), ('192.0.2.0/24', 'inet4'))
+		fr.search([('%%net=2001:DB8:FF:FF::1',"","")])
+		ip = fr.getIP()
+		self.assertEqual((ip, ip.familyStr), ('2001:db8:ff:ff::1', 'inet6'))
+		fr.search([('%%net=2001:DB8:FF:FF::1/60',"","")])
+		ip = fr.getIP()
+		self.assertEqual((ip, ip.familyStr), ('2001:db8:ff:f0::/60', 'inet6'))
+		# CIDR:
+		fr = FailRegex(r'%%ip="<ADDR>", mask="<CIDR>?"')
+		fr.search([('%%ip="192.0.2.2", mask=""',"","")])
+		ip = fr.getIP()
+		self.assertEqual((ip, ip.familyStr), ('192.0.2.2', 'inet4'))
+		fr.search([('%%ip="192.0.2.2", mask="24"',"","")])
+		ip = fr.getIP()
+		self.assertEqual((ip, ip.familyStr), ('192.0.2.0/24', 'inet4'))
+		fr.search([('%%ip="2001:DB8:2FF:FF::1", mask=""',"","")])
+		ip = fr.getIP()
+		self.assertEqual((ip, ip.familyStr), ('2001:db8:2ff:ff::1', 'inet6'))
+		fr.search([('%%ip="2001:DB8:2FF:FF::1", mask="60"',"","")])
+		ip = fr.getIP()
+		self.assertEqual((ip, ip.familyStr), ('2001:db8:2ff:f0::/60', 'inet6'))
 
 
 class _BadThread(JailThread):

man/jail.conf.5

@@ -277,7 +277,7 @@ It defaults to "auto" which will try "pyinotify", "gamin", "systemd" before "pol
 use DNS to resolve HOST names that appear in the logs. By default it is "warn" which will resolve hostnames to IPs however it will also log a warning. If you are using DNS here you could be blocking the wrong IPs due to the asymmetric nature of reverse DNS (that the application used to write the domain name to log) compared to forward DNS that fail2ban uses to resolve this back to an IP (but not necessarily the same one). Ideally you should configure your applications to log a real IP. This can be set to "yes" to prevent warnings in the log or "no" to disable DNS resolution altogether (thus ignoring entries where hostname, not an IP is logged)..
 .TP
 .B failregex
-regex (Python \fBreg\fRular \fBex\fRpression) to be added to the filter's failregexes. If this is useful for others using your application please share you regular expression with the fail2ban developers by reporting an issue (see REPORTING BUGS below).
+regex (Python \fBreg\fRular \fBex\fRpression) to be added to the filter's failregexes (see \fBfailregex\fR in section FILTER FILES for details). If this is useful for others using your application please share you regular expression with the fail2ban developers by reporting an issue (see REPORTING BUGS below).
 .TP
 .B ignoreregex
 regex which, if the log line matches, would cause Fail2Ban not consider that line.  This line will be ignored even if it matches a failregex of the jail or any of its filters.
@@ -428,8 +428,24 @@ Like action files, filter files are ini files. The main section is the [Definiti
 There are two filter definitions used in the [Definition] section:
 .TP
 .B failregex
-is the regex (\fBreg\fRular \fBex\fRpression) that will match failed attempts. The tag \fI<HOST>\fR is used as part of the regex and is itself a regex
+is the regex (\fBreg\fRular \fBex\fRpression) that will match failed attempts. The standard replacement tags can be used as part of the regex:
-for IPv4 addresses (and hostnames if \fBusedns\fR). Fail2Ban will work out which one of these it actually is.
+.RS
+.IP
+\fI<HOST>\fR - common regex for IP addresses and hostnames (if \fBusedns\fR is enabled). Fail2Ban will work out which one of these it actually is.
+.IP
+\fI<ADDR>\fR - regex for IP addresses (both families).
+.IP
+\fI<IP4>\fR - regex for IPv4 addresses.
+.IP
+\fI<IP6>\fR - regex for IPv6 addresses (also IP enclosed in brackets).
+.IP
+\fI<DNS>\fR - regex to match hostnames.
+.IP
+\fI<CIDR>\fR - helper regex to match CIDR (simple integer form of net-mask).
+.IP
+\fI<SUBNET>\fR - regex to match sub-net adresses (in form of IP/CIDR, also single IP is matched, so part /CIDR is optional).
+.RE
+.TP
 For multiline regexs the tag \fI<SKIPLINES>\fR should be used to separate lines. This allows lines between the matched lines to continue to be searched for other failures. The tag can be used multiple times.
 
 .TP