Merge branch '0.10' into fix-sshd-filter-suff

# Conflicts resolved:
#	fail2ban/server/filter.py

ChangeLog

@@ -52,6 +52,7 @@ ver. 0.10.3-dev-1 (20??/??/??) - development edition
 ### New Features
 
 ### Enhancements
+* `filter.d/apache-noscript.conf`: extend failregex to match "Primary script unknown", e. g. from php-fpm (gh-2073);
 * date-detector extended with long epoch (`LEPOCH`) to parse milliseconds/microseconds posix-dates (gh-2029);
 * possibility to specify own regex-pattern to match epoch date-time, e. g. `^\[{EPOCH}\]` or `^\[{LEPOCH}\]` (gh-2038);
   the epoch-pattern similar to `{DATE}` patterns does the capture and cuts out the match of whole pattern from the log-line,

README.md

@@ -6,50 +6,51 @@
 
 ## Fail2Ban: ban hosts that cause multiple authentication errors
 
-Fail2Ban scans log files like `/var/log/auth.log` and bans IP addresses having
+Fail2Ban scans log files like `/var/log/auth.log` and bans IP addresses conducting
 too many failed login attempts. It does this by updating system firewall rules
 to reject new connections from those IP addresses, for a configurable amount
 of time. Fail2Ban comes out-of-the-box ready to read many standard log files,
-such as those for sshd and Apache, and is easy to configure to read any log
+such as those for sshd and Apache, and is easily configured to read any log
-file you choose, for any error you choose.
+file of your choosing, for any error you wish.
 
-Though Fail2Ban is able to reduce the rate of incorrect authentications
+Though Fail2Ban is able to reduce the rate of incorrect authentication
-attempts, it cannot eliminate the risk that weak authentication presents.
+attempts, it cannot eliminate the risk presented by weak authentication.
-Configure services to use only two factor or public/private authentication
+Set up services to use only two factor, or public/private authentication
 mechanisms if you really want to protect services.
      
 <img src="http://www.worldipv6launch.org/wp-content/themes/ipv6/downloads/World_IPv6_launch_logo.svg" height="52pt"/> | Since v0.10 fail2ban supports the matching of the IPv6 addresses.
 ------|------
 
-This README is a quick introduction to Fail2ban. More documentation, FAQ, HOWTOs
+This README is a quick introduction to Fail2Ban. More documentation, FAQ, and HOWTOs
-are available in fail2ban(1) manpage, [Wiki](https://github.com/fail2ban/fail2ban/wiki)
+to be found on fail2ban(1) manpage, [Wiki](https://github.com/fail2ban/fail2ban/wiki)
-and on the website http://www.fail2ban.org
+and the website: https://www.fail2ban.org
 
 Installation:
 -------------
 
-**It is possible that Fail2ban is already packaged for your distribution.  In
+**It is possible that Fail2Ban is already packaged for your distribution.  In
-this case, you should use it instead.**
+this case, you should use that instead.**
 
 Required:
-- [Python2 >= 2.6 or Python >= 3.2](http://www.python.org) or [PyPy](http://pypy.org)
+- [Python2 >= 2.6 or Python >= 3.2](https://www.python.org) or [PyPy](https://pypy.org)
 
 Optional:
-- [pyinotify >= 0.8.3](https://github.com/seb-m/pyinotify)
+- [pyinotify >= 0.8.3](https://github.com/seb-m/pyinotify), may require:
-  - Linux >= 2.6.13
+  * Linux >= 2.6.13
 - [gamin >= 0.0.21](http://www.gnome.org/~veillard/gamin)
 - [systemd >= 204](http://www.freedesktop.org/wiki/Software/systemd) and python bindings:
-  - [python-systemd package](https://www.freedesktop.org/software/systemd/python-systemd/index.html)
+  * [python-systemd package](https://www.freedesktop.org/software/systemd/python-systemd/index.html)
 - [dnspython](http://www.dnspython.org/)
 
-To install, just do:
+
+To install:
 
     tar xvfj fail2ban-0.10.3.tar.bz2
     cd fail2ban-0.10.3
     python setup.py install
 
 This will install Fail2Ban into the python library directory. The executable
-scripts are placed into `/usr/bin`, and configuration under `/etc/fail2ban`.
+scripts are placed into `/usr/bin`, and configuration in `/etc/fail2ban`.
 
 Fail2Ban should be correctly installed now. Just type:
 
@@ -91,7 +92,7 @@ Contact:
 See [CONTRIBUTING.md](https://github.com/fail2ban/fail2ban/blob/master/CONTRIBUTING.md)
 
 ### You just appreciate this program:
-send kudos to the original author ([Cyril Jaquier](mailto:cyril.jaquier@fail2ban.org))
+Send kudos to the original author ([Cyril Jaquier](mailto:cyril.jaquier@fail2ban.org))
 or *better* to the [mailing list](https://lists.sourceforge.net/lists/listinfo/fail2ban-users)
 since Fail2Ban is "community-driven" for years now.
 

config/filter.d/apache-noscript.conf

@@ -17,8 +17,13 @@ before = apache-common.conf
 
 [Definition]
 
-failregex = ^%(_apache_error_client)s ((AH001(28|30): )?File does not exist|(AH01264: )?script not found or unable to stat): /\S*(php([45]|[.-]cgi)?|\.asp|\.exe|\.pl)(, referer: \S+)?\s*$
+script = /\S*(?:php(?:[45]|[.-]cgi)?|\.asp|\.exe|\.pl)
-            ^%(_apache_error_client)s script '/\S*(php([45]|[.-]cgi)?|\.asp|\.exe|\.pl)\S*' not found or unable to stat(, referer: \S+)?\s*$
+
+prefregex = ^%(_apache_error_client)s (?:AH0(?:01(?:28|30)|1(?:264|071)): )?(?:(?:[Ff]ile|script|[Gg]ot) )<F-CONTENT>.+</F-CONTENT>$
+
+failregex = ^(?:does not exist|not found or unable to stat): <script>\b
+            ^'<script>\S*' not found or unable to stat
+            ^error '[Pp]rimary script unknown\\n'
 
 ignoreregex = 
 

fail2ban/server/datetemplate.py

@@ -207,12 +207,12 @@ class DateEpoch(DateTemplate):
 
 	def __init__(self, lineBeginOnly=False, pattern=None, longFrm=False):
 		DateTemplate.__init__(self)
-		self.name = "Epoch"
+		self.name = "Epoch" if not pattern else pattern
 		self._longFrm = longFrm;
 		self._grpIdx = 1
 		epochRE = r"\d{10,11}\b(?:\.\d{3,6})?"
 		if longFrm:
-			self.name = "LongEpoch";
+			self.name = "LongEpoch" if not pattern else pattern
 			epochRE = r"\d{10,11}(?:\d{3}(?:\.\d{1,6}|\d{3})?)?"
 		if pattern:
 			# pattern should capture/cut out the whole match:

fail2ban/server/failregex.py

@@ -198,6 +198,13 @@ class Regex:
 	def getRegex(self):
 		return self._regex
 	
+	##
+	# Returns string buffer using join of the tupleLines.
+	#
+	@staticmethod
+	def _tupleLinesBuf(tupleLines):
+		return "\n".join(map(lambda v: "".join(v[::2]), tupleLines)) + "\n"
+
 	##
 	# Searches the regular expression.
 	#
@@ -207,8 +214,10 @@ class Regex:
 	# @param a list of tupples. The tupples are ( prematch, datematch, postdatematch )
 	
 	def search(self, tupleLines, orgLines=None):
-		self._matchCache = self._regexObj.search(
+		buf = tupleLines
-			"\n".join("".join(value[::2]) for value in tupleLines) + "\n")
+		if not isinstance(tupleLines, basestring):
+			buf = Regex._tupleLinesBuf(tupleLines)
+		self._matchCache = self._regexObj.search(buf)
 		if self._matchCache:
 			if orgLines is None: orgLines = tupleLines
 			# if single-line:

fail2ban/server/filter.py

@@ -582,8 +582,9 @@ class Filter(JailThread):
 	# @return: a boolean
 
 	def ignoreLine(self, tupleLines):
+		buf = Regex._tupleLinesBuf(tupleLines)
 		for ignoreRegexIndex, ignoreRegex in enumerate(self.__ignoreRegex):
-			ignoreRegex.search(tupleLines)
+			ignoreRegex.search(buf, tupleLines)
 			if ignoreRegex.hasMatched():
 				return ignoreRegexIndex
 		return None
@@ -681,6 +682,7 @@ class Filter(JailThread):
 	def findFailure(self, tupleLine, date=None):
 		failList = list()
 
+		ll = logSys.getEffectiveLevel()
 		returnRawHost = self.returnRawHost
 		cidr = IPAddr.CIDR_UNSPEC
 		if self.__useDns == "raw":
@@ -690,7 +692,7 @@ class Filter(JailThread):
 		# Checks if we mut ignore this line.
 		if self.ignoreLine([tupleLine[::2]]) is not None:
 			# The ignoreregex matched. Return.
-			logSys.log(7, "Matched ignoreregex and was \"%s\" ignored",
+			if ll <= 7: logSys.log(7, "Matched ignoreregex and was \"%s\" ignored",
 				"".join(tupleLine[::2]))
 			return failList
 
@@ -717,7 +719,7 @@ class Filter(JailThread):
 			date = self.__lastDate
 
 		if self.checkFindTime and date is not None and date < MyTime.time() - self.getFindTime():
-			logSys.log(5, "Ignore line since time %s < %s - %s", 
+			if ll <= 5: logSys.log(5, "Ignore line since time %s < %s - %s", 
 				date, MyTime.time(), self.getFindTime())
 			return failList
 
@@ -726,44 +728,45 @@ class Filter(JailThread):
 				self.__lineBuffer + [tupleLine[:3]])[-self.__lineBufferSize:]
 		else:
 			orgBuffer = self.__lineBuffer = [tupleLine[:3]]
-		logSys.log(5, "Looking for match of %r", self.__lineBuffer)
+		if ll <= 5: logSys.log(5, "Looking for match of %r", self.__lineBuffer)
+		buf = Regex._tupleLinesBuf(self.__lineBuffer)
 
 		# Pre-filter fail regex (if available):
 		preGroups = {}
 		if self.__prefRegex:
-			if logSys.getEffectiveLevel() <= logging.HEAVYDEBUG: # pragma: no cover
+			if ll <= 5: logSys.log(5, "  Looking for prefregex %r", self.__prefRegex.getRegex())
-				logSys.log(5, "  Looking for prefregex %r", self.__prefRegex.getRegex())
+			self.__prefRegex.search(buf, self.__lineBuffer)
-			self.__prefRegex.search(self.__lineBuffer)
 			if not self.__prefRegex.hasMatched():
-				logSys.log(5, "  Prefregex not matched")
+				if ll <= 5: logSys.log(5, "  Prefregex not matched")
 				return failList
 			preGroups = self.__prefRegex.getGroups()
-			logSys.log(7, "  Pre-filter matched %s", preGroups)
+			if ll <= 7: logSys.log(7, "  Pre-filter matched %s", preGroups)
 			repl = preGroups.get('content')
 			# Content replacement:
 			if repl:
 				del preGroups['content']
-				self.__lineBuffer = [('', '', repl)]
+				self.__lineBuffer, buf = [('', '', repl)], None
 
 		# Iterates over all the regular expressions.
 		for failRegexIndex, failRegex in enumerate(self.__failRegex):
-			# retrieve failure-id, host, etc from failure match:
 			try:
-				if logSys.getEffectiveLevel() <= logging.HEAVYDEBUG: # pragma: no cover
+				# buffer from tuples if changed: 
-					logSys.log(5, "  Looking for failregex %d - %r", failRegexIndex, failRegex.getRegex())
+				if buf is None:
-				failRegex.search(self.__lineBuffer, orgBuffer)
+					buf = Regex._tupleLinesBuf(self.__lineBuffer)
+				if ll <= 5: logSys.log(5, "  Looking for failregex %d - %r", failRegexIndex, failRegex.getRegex())
+				failRegex.search(buf, orgBuffer)
 				if not failRegex.hasMatched():
 					continue
 				# current failure data (matched group dict):
 				fail = failRegex.getGroups()
 				# The failregex matched.
-				logSys.log(7, "  Matched failregex %d: %s", failRegexIndex, fail)
+				if ll <= 7: logSys.log(7, "  Matched failregex %d: %s", failRegexIndex, fail)
 				# Checks if we must ignore this match.
 				if self.ignoreLine(failRegex.getMatchedTupleLines()) \
 						is not None:
 					# The ignoreregex matched. Remove ignored match.
-					self.__lineBuffer = failRegex.getUnmatchedTupleLines()
+					self.__lineBuffer, buf = failRegex.getUnmatchedTupleLines(), None
-					logSys.log(7, "  Matched ignoreregex and was ignored")
+					if ll <= 7: logSys.log(7, "  Matched ignoreregex and was ignored")
 					if not self.checkAllRegex:
 						break
 					else:
@@ -781,7 +784,7 @@ class Filter(JailThread):
 					continue
 				# we should check all regex (bypass on multi-line, otherwise too complex):
 				if not self.checkAllRegex or self.getMaxLines() > 1:
-					self.__lineBuffer = failRegex.getUnmatchedTupleLines()
+					self.__lineBuffer, buf = failRegex.getUnmatchedTupleLines(), None
 				# merge data if multi-line failure:
 				raw = returnRawHost
 				if preGroups:
@@ -793,8 +796,7 @@ class Filter(JailThread):
 					fail = self._mergeFailure(mlfid, fail, failRegex)
 					# bypass if no-failure case:
 					if fail.get('nofail'):
-						# if not users or len(users) <= 1:
+						if ll <= 7: logSys.log(7, "Nofail by mlfid %r in regex %s: %s",
-						logSys.log(7, "Nofail by mlfid %r in regex %s: %s",
 							mlfid, failRegexIndex, fail.get('mlfforget', "waiting for failure"))
 						if not self.checkAllRegex: return failList
 				else:
@@ -823,7 +825,7 @@ class Filter(JailThread):
 						cidr = IPAddr.CIDR_RAW
 				# if mlfid case (not failure):
 				if host is None:
-					logSys.log(7, "No failure-id by mlfid %r in regex %s: %s",
+					if ll <= 7: logSys.log(7, "No failure-id by mlfid %r in regex %s: %s",
 						mlfid, failRegexIndex, fail.get('mlfforget', "waiting for identifier"))
 					if not self.checkAllRegex: return failList
 					ips = [None]

fail2ban/tests/action_d/test_badips.py

@@ -58,6 +58,7 @@ if sys.version_info >= (2,7): # pragma: no cover - may be unavailable
 			self.jail.actions.add("badips", pythonModuleName, initOpts={
 				'category': "ssh",
 				'banaction': "test",
+				'age': "2w",
 				'score': 5,
 				'key': "fail2ban-test-suite",
 				#'bankey': "fail2ban-test-suite",

fail2ban/tests/files/logs/apache-noscript

@@ -16,3 +16,5 @@
 # apache 2.4
 # failJSON: { "time": "2013-12-23T07:49:01", "match": true , "host": "204.232.202.107" }
 [Mon Dec 23 07:49:01.981912 2013] [:error] [pid 3790] [client 204.232.202.107:46301] script '/var/www/timthumb.php' not found or unable to stat
+# failJSON: { "time": "2018-03-11T08:56:20", "match": true , "host": "192.0.2.106", "desc": "php-fpm error" }
+[Sun Mar 11 08:56:20.913548 2018] [proxy_fcgi:error] [pid 742:tid 140142593419008] [client 192.0.2.106:50900] AH01071: Got error 'Primary script unknown\n'

fail2ban/tests/filtertestcase.py

@@ -1145,6 +1145,7 @@ def get_monitor_failures_journal_testcase(Filter_): # pragma: systemd no cover
 		def setUp(self):
 			"""Call before every test case."""
 			super(MonitorJournalFailures, self).setUp()
+			self._runtimeJournal = None
 			self.test_file = os.path.join(TEST_FILES_DIR, "testcase-journal.log")
 			self.jail = DummyJail()
 			self.filter = None
@@ -1156,6 +1157,7 @@ def get_monitor_failures_journal_testcase(Filter_): # pragma: systemd no cover
 				'TEST_FIELD': "1", 'TEST_UUID': self.test_uuid}
 
 		def _initFilter(self, **kwargs):
+			self._getRuntimeJournal() # check journal available
 			self.filter = Filter_(self.jail, **kwargs)
 			self.filter.addJournalMatch([
 				"SYSLOG_IDENTIFIER=fail2ban-testcases",
@@ -1176,21 +1178,26 @@ def get_monitor_failures_journal_testcase(Filter_): # pragma: systemd no cover
 		def _getRuntimeJournal(self):
 			"""Retrieve current system journal path
 
-			If none found, None will be returned
+			If not found, SkipTest exception will be raised.
 			"""
-			# Depending on the system, it could be found under /run or /var/log (e.g. Debian)
+			# we can cache it:
-			# which are pointed by different systemd-path variables.  We will
+			if self._runtimeJournal is None:
-			# check one at at time until the first hit
+				# Depending on the system, it could be found under /run or /var/log (e.g. Debian)
-			for systemd_var in 'system-runtime-logs', 'system-state-logs':
+				# which are pointed by different systemd-path variables.  We will
-				tmp = Utils.executeCmd(
+				# check one at at time until the first hit
-					'find "$(systemd-path %s)" -name system.journal' % systemd_var,
+				for systemd_var in 'system-runtime-logs', 'system-state-logs':
-					timeout=10, shell=True, output=True
+					tmp = Utils.executeCmd(
-				)
+						'find "$(systemd-path %s)" -name system.journal' % systemd_var,
-				self.assertTrue(tmp)
+						timeout=10, shell=True, output=True
-				out = str(tmp[1].decode('utf-8')).split('\n')[0]
+					)
-				if out:
+					self.assertTrue(tmp)
-					return out
+					out = str(tmp[1].decode('utf-8')).split('\n')[0]
-
+					if out: break
+				self._runtimeJournal = out
+			if self._runtimeJournal:
+				return self._runtimeJournal
+			raise unittest.SkipTest('systemd journal seems to be not available (e. g. no rights to read)')
+		
 		def testJournalFilesArg(self):
 			# retrieve current system journal path
 			jrnlfile = self._getRuntimeJournal()

fail2ban/tests/utils.py

@@ -680,7 +680,7 @@ class LogCaptureTestCase(unittest.TestCase):
 				return self._val
 			# try to lock, if not possible - return cached/empty (max 5 times):
 			lck = self._lock.acquire(False)
-			if not lck: # pargma: no cover (may be too sporadic on slow systems)
+			if not lck: # pragma: no cover (may be too sporadic on slow systems)
 				self._nolckCntr += 1
 				if self._nolckCntr <= 5:
 					return self._val if self._val is not None else ''