Merge remote-tracking branch 'remotes/upstream/master' into sebres:ban-time-incr

Conflicts:
	fail2ban/server/actions.py
	fail2ban/server/database.py
	fail2ban/tests/databasetestcase.py
	fail2ban/tests/servertestcase.py

ChangeLog

@@ -10,15 +10,23 @@ Fail2Ban (version 0.9.2.dev)                                          2014/xx/xx
 ver. 0.9.2 (2014/xx/xx) - increment ban time
 ----------
 
-- New features:
+- Fixes:
-   - increment ban time (+ observer) functionality introduced.
+   * $ typo in jail.conf. Thanks Skibbi. Debian bug #767255
+   * grep'ing for IP in *mail-whois-lines.conf should now match also
+     at the begginning and EOL.  Thanks Dean Lee
+   * jail.conf
+     - php-url-fopen: separate logpath entries by newline
+   * purge database will be executed now (within observer).
+   * database functionality extended with bad ips.
+   * restoring currently banned ip after service restart fixed 
+     (now < timeofban + bantime), ignore old log failures (already banned)
+
+- New Features:
+   * increment ban time (+ observer) functionality introduced.
      Thanks Serg G. Brester (sebres)
 
-- Fixes:
+- Enhancements:
-   - purge database will be executed now (within observer).
+   * Enable multiport for firewallcmd-new action.  Closes gh-834
-   - database functionality extended with bad ips.
-   - restoring currently banned ip after service restart fixed 
-     (now < timeofban + bantime), ignore old log failures (already banned)
 
 ver. 0.9.1 (2014/10/29) - better, faster, stronger
 ----------

config/action.d/firewallcmd-new.conf

@@ -10,9 +10,9 @@ before = iptables-common.conf
 
 actionstart = firewall-cmd --direct --add-chain ipv4 filter f2b-<name>
               firewall-cmd --direct --add-rule ipv4 filter f2b-<name> 1000 -j RETURN
-              firewall-cmd --direct --add-rule ipv4 filter <chain> 0 -m state --state NEW -p <protocol> --dport <port> -j f2b-<name>
+              firewall-cmd --direct --add-rule ipv4 filter <chain> 0 -m state --state NEW -p <protocol> -m multiport --dports <port> -j f2b-<name>
 
-actionstop = firewall-cmd --direct --remove-rule ipv4 filter <chain> 0 -m state --state NEW -p <protocol> --dport <port> -j f2b-<name>
+actionstop = firewall-cmd --direct --remove-rule ipv4 filter <chain> 0 -m state --state NEW -p <protocol> -m multiport --dports <port> -j f2b-<name>
              firewall-cmd --direct --remove-rules ipv4 filter f2b-<name>
              firewall-cmd --direct --remove-chain ipv4 filter f2b-<name>
 
@@ -43,7 +43,7 @@ chain = INPUT_direct
 # success
 # $ firewall-cmd --direct --add-rule ipv4 filter fail2ban-name 1000 -j RETURN
 # success
-# $ sudo firewall-cmd --direct --add-rule ipv4 filter INPUT_direct 0 -m state --state NEW -p tcp --dport 22 -j fail2ban-name
+# $ sudo firewall-cmd --direct --add-rule ipv4 filter INPUT_direct 0 -m state --state NEW -p tcp -m multiport --dports 22 -j fail2ban-name
 # success
 # $ firewall-cmd --direct --get-chains ipv4 filter
 # fail2ban-name

config/action.d/mail-whois-lines.conf

@@ -42,7 +42,7 @@ actionban = printf %%b "Hi,\n
             Here is more information about <ip>:\n
             `whois <ip> || echo missing whois program`\n\n
             Lines containing IP:<ip> in <logpath>\n
-            `grep '[^0-9]<ip>[^0-9]' <logpath>`\n\n
+            `grep -E '(^|[^0-9])<ip>([^0-9]|$)' <logpath>`\n\n
             Regards,\n
             Fail2Ban"|mail -s "[Fail2Ban] <name>: banned <ip> from  `uname -n`" <dest>
 

config/action.d/sendmail-whois-lines.conf

@@ -26,7 +26,7 @@ actionban = printf %%b "Subject: [Fail2Ban] <name>: banned <ip> from `uname -n`
             Here is more information about <ip>:\n
             `/usr/bin/whois <ip> || echo missing whois program`\n\n
             Lines containing IP:<ip> in <logpath>\n
-            `grep '[^0-9]<ip>[^0-9]' <logpath>`\n\n
+            `grep -E '(^|[^0-9])<ip>([^0-9]|$)' <logpath>`\n\n
             Regards,\n
             Fail2Ban" | /usr/sbin/sendmail -f <sender> <dest>
 

config/filter.d/named-refused.conf

@@ -38,6 +38,8 @@ failregex = ^%(__line_prefix)s( error:)?\s*client <HOST>#\S+( \([\S.]+\))?: (vie
             ^%(__line_prefix)s( error:)?\s*client <HOST>#\S+( \([\S.]+\))?: zone transfer '\S+/AXFR/\w+' denied\s*$
             ^%(__line_prefix)s( error:)?\s*client <HOST>#\S+( \([\S.]+\))?: bad zone transfer request: '\S+/IN': non-authoritative zone \(NOTAUTH\)\s*$
 
+ignoreregex =
+
 # DEV Notes:
 # Trying to generalize the
 #          structure which is general to capture general patterns in log

config/filter.d/recidive.conf

@@ -29,6 +29,8 @@ _jailname = recidive
 
 failregex = ^(%(__prefix_line)s| %(_daemon)s%(__pid_re)s?:\s+)NOTICE\s+\[(?!%(_jailname)s\])(?:.*)\]\s+Ban\s+<HOST>\s*$
 
+ignoreregex = 
+
 [Init]
 
 journalmatch = _SYSTEMD_UNIT=fail2ban.service PRIORITY=5

config/jail.conf

@@ -324,7 +324,7 @@ maxretry = 2
 [apache-shellshock]
 
 port    = http,https
-logpath = $(apache_error_log)s
+logpath = %(apache_error_log)s
 maxretry = 1
 
 [nginx-http-auth]
@@ -340,7 +340,8 @@ logpath = %(nginx_error_log)s
 [php-url-fopen]
 
 port    = http,https
-logpath = %(nginx_access_log)s %(apache_access_log)s
+logpath = %(nginx_access_log)s
+          %(apache_access_log)s
 
 
 [suhosin]

fail2ban/server/actions.py

@@ -246,8 +246,8 @@ class Actions(JailThread, Mapping):
 		logSys.debug(self._jail.name + ": action terminated")
 		return True
 
-	def __getBansMerged(self, mi, idx):
+	def __getBansMerged(self, mi, overalljails=False):
-		"""Helper for lamda to get bans merged once
+		"""Gets bans merged once, a helper for lambda(s), prevents stop of executing action by any exception inside.
 
 		This function never returns None for ainfo lambdas - always a ticket (merged or single one)
 		and prevents any errors through merging (to guarantee ban actions will be executed).
@@ -256,26 +256,27 @@ class Actions(JailThread, Mapping):
 		Parameters
 		----------
 		mi : dict
-			initial for lambda should contains {ip, ticket}
+			merge info, initial for lambda should contains {ip, ticket}
-		idx : str
+		overalljails : bool
-			key to get a merged bans :
+			switch to get a merged bans :
-			'all' - bans merged for all jails
+			False - (default) bans merged for current jail only
-			'jail' - bans merged for current jail only
+			True - bans merged for all jails of current ip address
 
 		Returns
 		-------
 		BanTicket 
 			merged or self ticket only
 		"""
+		idx = 'all' if overalljails else 'jail'
 		if idx in mi:
 			return mi[idx] if mi[idx] is not None else mi['ticket']
 		try:
 			jail=self._jail
 			ip=mi['ip']
 			mi[idx] = None
-			if idx == 'all':
+			if overalljails:
 				mi[idx] = jail.database.getBansMerged(ip=ip)
-			elif idx == 'jail':
+			else:
 				mi[idx] = jail.database.getBansMerged(ip=ip, jail=jail)
 		except Exception as e:
 			logSys.error(
@@ -312,11 +313,12 @@ class Actions(JailThread, Mapping):
 			btime = bTicket.getBanTime(self.__banManager.getBanTime())
 			# retarded merge info via twice lambdas : once for merge, once for matches/failures:
 			if self._jail.database is not None:
-				mi4ip = lambda idx, self=self, mi={'ip':ip, 'ticket':bTicket}: self.__getBansMerged(mi, idx)
+				mi4ip = lambda overalljails=False, self=self, \
-				aInfo["ipmatches"]      = lambda: "\n".join(mi4ip('all').getMatches())
+					mi={'ip':ip, 'ticket':bTicket}: self.__getBansMerged(mi, overalljails)
-				aInfo["ipjailmatches"]  = lambda: "\n".join(mi4ip('jail').getMatches())
+				aInfo["ipmatches"]      = lambda: "\n".join(mi4ip(True).getMatches())
-				aInfo["ipfailures"]     = lambda: mi4ip('all').getAttempt()
+				aInfo["ipjailmatches"]  = lambda: "\n".join(mi4ip().getMatches())
-				aInfo["ipjailfailures"] = lambda: mi4ip('jail').getAttempt()
+				aInfo["ipfailures"]     = lambda: mi4ip(True).getAttempt()
+				aInfo["ipjailfailures"] = lambda: mi4ip().getAttempt()
 
 			if btime != -1:
 				bendtime = aInfo["time"] + btime

fail2ban/server/filtersystemd.py

@@ -167,9 +167,10 @@ class FilterSystemd(JournalFilter): # pragma: systemd no cover
 			logelements.append(logentry['_HOSTNAME'])
 		if logentry.get('SYSLOG_IDENTIFIER'):
 			logelements.append(logentry['SYSLOG_IDENTIFIER'])
-			if logentry.get('SYSLOG_PID') or logentry.get('_PID'):
+			if logentry.get('SYSLOG_PID'):
-				logelements[-1] += ("[%i]" % logentry.get(
+				logelements[-1] += ("[%i]" % logentry['SYSLOG_PID'])
-					'SYSLOG_PID', logentry['_PID']))
+			elif logentry.get('_PID'):
+				logelements[-1] += ("[%i]" % logentry['_PID'])
 			logelements[-1] += ":"
 		elif logentry.get('_COMM'):
 			logelements.append(logentry['_COMM'])

fail2ban/tests/databasetestcase.py

@@ -323,5 +323,3 @@ class DatabaseTest(LogCaptureTestCase):
 		self.db.purge() # Should leave jail as ban present
 		self.assertEqual(len(self.db.getJailNames()), 1)
 		self.assertEqual(len(self.db.getBans(jail=self.jail)), 1)
-
-

fail2ban/tests/servertestcase.py

@@ -816,14 +816,6 @@ class _BadThread(JailThread):
 
 class LoggingTests(LogCaptureTestCase):
 
-	def setUp(self):
-		"""Call before every test case."""
-		LogCaptureTestCase.setUp(self)
-
-	def tearDown(self):
-		"""Call after every test case."""
-		LogCaptureTestCase.tearDown(self)
-
 	def testGetF2BLogger(self):
 		testLogSys = getLogger("fail2ban.some.string.with.name")
 		self.assertEqual(testLogSys.parent.name, "fail2ban")
@@ -833,10 +825,12 @@ class LoggingTests(LogCaptureTestCase):
 		prev_exchook = sys.__excepthook__
 		x = []
 		sys.__excepthook__ = lambda *args: x.append(args)
-		badThread = _BadThread()
+		try:
-		badThread.start()
+			badThread = _BadThread()
-		badThread.join()
+			badThread.start()
-		self.assertTrue(self._is_logged("Unhandled exception"))
+			badThread.join()
-		sys.__excepthook__ = prev_exchook
+			self.assertTrue(self._is_logged("Unhandled exception"))
+		finally:
+			sys.__excepthook__ = prev_exchook
 		self.assertEqual(len(x), 1)
 		self.assertEqual(x[0][0], RuntimeError)