TST: more test of filters

.gitignore

@@ -7,3 +7,4 @@ htmlcov
 *.orig
 *.rej
 *.bak
+__pycache__

ChangeLog

@@ -77,7 +77,7 @@ ver. 0.8.12 (2013/12/XX) - things-can-only-get-better
   - allow for ",milliseconds" in the custom date format of proftpd.log
   - allow for ", referer ..." in apache-* filter for apache error logs.
   - allow for spaces at the beginning of kernel messages. Closes gh-448
-  - recidive jail to block all protocols. Closes gh-440. Thanks Ioan Indreias
+  - recidive jail to block all protocols. Closes gh-440. Thanksg Ioan Indreias
   - smtps not a IANA standard and has been removed from Arch. Replaced with
     465. Thanks Stefan. Closes gh-447
   - mysqld-syslog-iptables rule was too long. Part of gh-447.
@@ -86,6 +86,8 @@ ver. 0.8.12 (2013/12/XX) - things-can-only-get-better
   - complain action - ensure where not matching other IPs in log sample.
     Closes gh-467
   - Fix firewall-cmd actioncheck - patch from Adam Tkac. Redhat Bug #979622
+  - Fix apache-common for apache-2.4 log file format. Thanks Mark White.
+    Closes gh-516
 
 - Enhancements:
   - long names on jails documented based on iptables limit of 30 less
@@ -93,11 +95,16 @@ ver. 0.8.12 (2013/12/XX) - things-can-only-get-better
   - remove indentation of name and loglevel while logging to SYSLOG to
     resolve syslog(-ng) parsing problems. Closes Debian bug #730202.
   - added squid filter. Thanks Roman Gelfand.
+  - updated check_fail2ban to return performance data for all jails.
+  - filter apache-noscript now includes php cgi scripts.
+    Thanks dani. Closes gh-503
 
 - New Features:
 
   Daniel Black
    * filter.d/solid-pop3d -- added thanks to Jacques Lav!gnotte on mailinglist.
+  Bas van den Dikkenberg & Steven Hiscocks
+   * filter.d/nsd.conf -- also amended Unix date template to match nsd format
 
 - Enhancements:
   - loglines now also report "[PID]" after the name portion

MANIFEST

@@ -5,13 +5,14 @@ TODO
 THANKS
 COPYING
 DEVELOP
-doc/run-rootless.txt
 fail2ban-2to3
+fail2ban-testcases-all
 fail2ban-testcases-all-python3
 bin/fail2ban-client
 bin/fail2ban-server
 bin/fail2ban-testcases
 bin/fail2ban-regex
+doc/run-rootless.txt
 fail2ban/client/configreader.py
 fail2ban/client/configparserinc.py
 fail2ban/client/jailreader.py
@@ -61,23 +62,26 @@ fail2ban/tests/sockettestcase.py
 fail2ban/tests/utils.py
 fail2ban/tests/misctestcase.py
 fail2ban/tests/databasetestcase.py
-fail2ban/tests/config/apache-auth/digest/.htaccess
+fail2ban/tests/files/config/apache-auth/digest/.htaccess
-fail2ban/tests/config/apache-auth/digest/.htpasswd
+fail2ban/tests/files/config/apache-auth/digest/.htpasswd
-fail2ban/tests/config/apache-auth/digest_time/.htaccess
+fail2ban/tests/files/config/apache-auth/digest_time/.htaccess
-fail2ban/tests/config/apache-auth/digest_time/.htpasswd
+fail2ban/tests/files/config/apache-auth/digest_time/.htpasswd
-fail2ban/tests/config/apache-auth/basic/authz_owner/.htaccess
+fail2ban/tests/files/config/apache-auth/basic/authz_owner/.htaccess
-fail2ban/tests/config/apache-auth/basic/authz_owner/cant_get_me.html
+fail2ban/tests/files/config/apache-auth/basic/authz_owner/cant_get_me.html
-fail2ban/tests/config/apache-auth/basic/authz_owner/.htpasswd
+fail2ban/tests/files/config/apache-auth/basic/authz_owner/.htpasswd
-fail2ban/tests/config/apache-auth/basic/file/.htaccess
+fail2ban/tests/files/config/apache-auth/basic/file/.htaccess
-fail2ban/tests/config/apache-auth/basic/file/.htpasswd
+fail2ban/tests/files/config/apache-auth/basic/file/.htpasswd
-fail2ban/tests/config/apache-auth/digest.py
+fail2ban/tests/files/config/apache-auth/digest.py
-fail2ban/tests/config/apache-auth/digest_wrongrelm/.htaccess
+fail2ban/tests/files/config/apache-auth/digest_wrongrelm/.htaccess
-fail2ban/tests/config/apache-auth/digest_wrongrelm/.htpasswd
+fail2ban/tests/files/config/apache-auth/digest_wrongrelm/.htpasswd
-fail2ban/tests/config/apache-auth/digest_anon/.htaccess
+fail2ban/tests/files/config/apache-auth/digest_anon/.htaccess
-fail2ban/tests/config/apache-auth/digest_anon/.htpasswd
+fail2ban/tests/files/config/apache-auth/digest_anon/.htpasswd
-fail2ban/tests/config/apache-auth/README
+fail2ban/tests/files/config/apache-auth/README
-fail2ban/tests/config/apache-auth/noentry/.htaccess
+fail2ban/tests/files/config/apache-auth/noentry/.htaccess
 fail2ban/tests/files/database_v1.db
+fail2ban/tests/files/ignorecommand.py
+fail2ban/tests/files/filter.d/testcase-common.conf
+fail2ban/tests/files/filter.d/testcase01.conf
 fail2ban/tests/files/testcase01.log
 fail2ban/tests/files/testcase02.log
 fail2ban/tests/files/testcase03.log
@@ -88,54 +92,58 @@ fail2ban/tests/files/testcase-multiline.log
 fail2ban/tests/files/logs/bsd/syslog-plain.txt
 fail2ban/tests/files/logs/bsd/syslog-v.txt
 fail2ban/tests/files/logs/bsd/syslog-vv.txt
+fail2ban/tests/files/logs/3proxy
+fail2ban/tests/files/logs/apache-auth
+fail2ban/tests/files/logs/apache-badbots
+fail2ban/tests/files/logs/apache-modsecurity
+fail2ban/tests/files/logs/apache-nohome
+fail2ban/tests/files/logs/apache-noscript
 fail2ban/tests/files/logs/apache-overflows
 fail2ban/tests/files/logs/assp
 fail2ban/tests/files/logs/asterisk
+fail2ban/tests/files/logs/courier-auth
+fail2ban/tests/files/logs/courier-smtp
+fail2ban/tests/files/logs/cyrus-imap
 fail2ban/tests/files/logs/dovecot
+fail2ban/tests/files/logs/dropbear
+fail2ban/tests/files/logs/ejabberd-auth
 fail2ban/tests/files/logs/exim
-fail2ban/tests/files/logs/nginx-http-auth
+fail2ban/tests/files/logs/exim-spam
+fail2ban/tests/files/logs/gssftpd
+fail2ban/tests/files/logs/guacamole
 fail2ban/tests/files/logs/lighttpd-auth
+fail2ban/tests/files/logs/mysqld-auth
+fail2ban/tests/files/logs/nsd
+fail2ban/tests/files/logs/perdition
+fail2ban/tests/files/logs/php-url-fopen
+fail2ban/tests/files/logs/postfix-sasl
 fail2ban/tests/files/logs/named-refused
+fail2ban/tests/files/logs/nginx-http-auth
 fail2ban/tests/files/logs/pam-generic
 fail2ban/tests/files/logs/postfix
 fail2ban/tests/files/logs/proftpd
 fail2ban/tests/files/logs/pure-ftpd
+fail2ban/tests/files/logs/qmail
+fail2ban/tests/files/logs/recidive
 fail2ban/tests/files/logs/roundcube-auth
+fail2ban/tests/files/logs/selinux-ssh
+fail2ban/tests/files/logs/sendmail-spam
+fail2ban/tests/files/logs/sieve
+fail2ban/tests/files/logs/squid
+fail2ban/tests/files/logs/suhosin
 fail2ban/tests/files/logs/sogo-auth
 fail2ban/tests/files/logs/solid-pop3d
 fail2ban/tests/files/logs/sshd
 fail2ban/tests/files/logs/sshd-ddos
 fail2ban/tests/files/logs/vsftpd
 fail2ban/tests/files/logs/webmin-auth
-fail2ban/tests/files/logs/3proxy
-fail2ban/tests/files/logs/apache-auth
-fail2ban/tests/files/logs/apache-badbots
-fail2ban/tests/files/logs/apache-nohome
-fail2ban/tests/files/logs/apache-noscript
-fail2ban/tests/files/logs/courier-auth
-fail2ban/tests/files/logs/courier-smtp
-fail2ban/tests/files/logs/cyrus-imap
-fail2ban/tests/files/logs/dropbear
-fail2ban/tests/files/logs/ejabberd-auth
-fail2ban/tests/files/logs/exim-spam
-fail2ban/tests/files/logs/gssftpd
-fail2ban/tests/files/logs/guacamole
-fail2ban/tests/files/logs/lighttpd-auth
-fail2ban/tests/files/logs/mysqld-auth
-fail2ban/tests/files/logs/perdition
-fail2ban/tests/files/logs/php-url-fopen
-fail2ban/tests/files/logs/postfix-sasl
-fail2ban/tests/files/logs/qmail
-fail2ban/tests/files/logs/recidive
-fail2ban/tests/files/logs/selinux-ssh
-fail2ban/tests/files/logs/sendmail-spam
-fail2ban/tests/files/logs/sieve
-fail2ban/tests/files/logs/suhosin
-fail2ban/tests/files/logs/uwimap-auth
 fail2ban/tests/files/logs/wuftpd
+fail2ban/tests/files/logs/uwimap-auth
 fail2ban/tests/files/logs/xinetd-fail
-fail2ban/tests/files/filter.d/testcase-common.conf
+fail2ban/tests/config/jail.conf
-fail2ban/tests/files/filter.d/testcase01.conf
+fail2ban/tests/config/fail2ban.conf
+fail2ban/tests/config/filter.d/simple.conf
+fail2ban/tests/config/action.d/brokenaction.conf
 setup.py
 setup.cfg
 fail2ban/__init__.py
@@ -262,7 +270,3 @@ files/fail2ban-tmpfiles.conf
 files/fail2ban.service
 files/ipmasq-ZZZzzz_fail2ban.rul
 files/gen_badbots
-testcases/config/jail.conf
-testcases/config/fail2ban.conf
-testcases/config/filter.d/simple.conf
-testcases/config/action.d/brokenaction.conf

THANKS

@@ -10,11 +10,13 @@ Adam Tkac
 Adrien Clerc
 ache
 ag4ve (Shawn)
+Alasdair D. Campbell
 Amir Caspi
 Andrey G. Grozin
 Andy Fragen
 Arturo 'Buanzo' Busleiman
 Axel Thimm
+Bas van den Dikkenberg
 Beau Raines
 Bill Heaton
 Carlos Alberto Lopez Perez
@@ -24,6 +26,7 @@ Christoph Haas
 Christos Psonis
 Cyril Jaquier
 Daniel B. Cid
+Daniel B.
 Daniel Black
 David Nutter
 Eric Gerbier
@@ -47,10 +50,12 @@ Justin Shore
 Kévin Drapel
 kjohnsonecl
 kojiro
+Lee Clemens
 Manuel Arostegui Ramirez
 Marcel Dopita
 Mark Edgington
 Mark McKinstry
+Mark White
 Markus Hoffmann
 Marvin Rouge
 mEDI

config/filter.d/apache-common.conf

@@ -8,12 +8,13 @@ after = apache-common.local
 
 [DEFAULT]
 
-_apache_error_client = \[\] \[(error|\S+:\S+)\]( \[pid \d+:\S+ \d+\])? \[client <HOST>(:\d{1,5})?\]
+_apache_error_client = \[\] \[(:?error|\S+:\S+)\]( \[pid \d+(:\S+ \d+)?\])? \[client <HOST>(:\d{1,5})?\]
 
 # Common prefix for [error] apache messages which also would include <HOST>
 # Depending on the version it could be
 # 2.2: [Sat Jun 01 11:23:08 2013] [error] [client 1.2.3.4]
 # 2.4: [Thu Jun 27 11:55:44.569531 2013] [core:info] [pid 4101:tid 2992634688] [client 1.2.3.4:46652]
+# 2.4 (perfork): [Mon Dec 23 07:49:01.981912 2013] [:error] [pid 3790] [client 204.232.202.107:46301] script '/var/www/timthumb.php' not found or unable to 
 #
 # Reference: https://github.com/fail2ban/fail2ban/issues/268
 #

config/filter.d/apache-noscript.conf

@@ -9,8 +9,8 @@ before = apache-common.conf
 
 [Definition]
 
-failregex = ^%(_apache_error_client)s ((AH001(28|30): )?File does not exist|(AH01264: )?script not found or unable to stat): /\S*(\.php|\.asp|\.exe|\.pl)(, referer: \S+)?\s*$
+failregex = ^%(_apache_error_client)s ((AH001(28|30): )?File does not exist|(AH01264: )?script not found or unable to stat): /\S*(php([45]|[.-]cgi)?|\.asp|\.exe|\.pl)(, referer: \S+)?\s*$
-            ^%(_apache_error_client)s script '/\S*(\.php|\.asp|\.exe|\.pl)\S*' not found or unable to stat(, referer: \S+)?\s*$
+            ^%(_apache_error_client)s script '/\S*(php([45]|[.-]cgi)?|\.asp|\.exe|\.pl)\S*' not found or unable to stat(, referer: \S+)?\s*$
 
 ignoreregex = 
 

config/filter.d/nsd.conf

@@ -0,0 +1,26 @@
+# Fail2Ban configuration file
+#
+# Author: Bas van den Dikkenberg
+#
+#
+
+[INCLUDES]
+
+# Read common prefixes. If any customizations available -- read them from
+# common.local
+before = common.conf
+
+
+[Definition]
+
+_daemon = nsd
+
+# Option:  failregex
+# Notes.:  regex to match the password failures messages in the logfile. The
+#          host must be matched by a group named "host". The tag "<HOST>" can
+#          be used for standard IP/hostname matching and is only an alias for
+#          (?:::f{4,6}:)?(?P<host>[\w\-.^_]+)
+# Values:  TEXT
+
+failregex =  ^\[\]%(__prefix_line)sinfo: ratelimit block .* query <HOST> TYPE255$
+              ^\[\]%(__prefix_line)sinfo: .* <HOST> refused, no acl matches\.$

config/jail.conf

@@ -44,6 +44,12 @@
 # defined using space separator.
 ignoreip = 127.0.0.1/8
 
+# External command that will take an tagged arguments to ignore, e.g. <ip>,
+# and return true if the IP is to be ignored. False otherwise.
+#
+# ignorecommand = /path/to/command <ip>
+ignorecommand =
+
 # "bantime" is the number of seconds that a host is banned.
 bantime  = 600
 

fail2ban/client/configreader.py

@@ -138,12 +138,13 @@ class DefinitionInitConfigReader(ConfigReader):
 	
 	def __init__(self, file_, jailName, initOpts, **kwargs):
 		ConfigReader.__init__(self, **kwargs)
-		self._file = file_
+		self.setFile(file_)
-		self._jailName = jailName
+		self.setJailName(jailName)
 		self._initOpts = initOpts
 	
 	def setFile(self, fileName):
 		self._file = fileName
+		self._initOpts = {}
 	
 	def getFile(self):
 		return self._file

fail2ban/client/filterreader.py

@@ -35,6 +35,7 @@ class FilterReader(DefinitionInitConfigReader):
 	_configOpts = [
 		["string", "ignoreregex", ""],
 		["string", "failregex", ""],
+        ["string", "ignorecommand", ""],
 	]
 
 	def read(self):

fail2ban/client/jailreader.py

@@ -89,6 +89,7 @@ class JailReader(ConfigReader):
 				["string", "usedns", "warn"],
 				["string", "failregex", None],
 				["string", "ignoreregex", None],
+				["string", "ignorecommand", None],
 				["string", "ignoreip", None],
 				["string", "filter", ""],
 				["string", "action", ""]]
@@ -179,6 +180,8 @@ class JailReader(ConfigReader):
 				stream.append(["set", self.__name, "usedns", self.__opts[opt]])
 			elif opt == "failregex":
 				stream.append(["set", self.__name, "addfailregex", self.__opts[opt]])
+			elif opt == "ignorecommand":
+				stream.append(["set", self.__name, "ignorecommand", self.__opts[opt]])
 			elif opt == "ignoreregex":
 				for regex in self.__opts[opt].split('\n'):
 					# Do not send a command if the rule is empty.

fail2ban/protocol.py

@@ -65,6 +65,7 @@ protocol = [
 ["set <JAIL> deljournalmatch <MATCH>", "removes <MATCH> from the journal filter of <JAIL>"],
 ["set <JAIL> addfailregex <REGEX>", "adds the regular expression <REGEX> which must match failures for <JAIL>"], 
 ["set <JAIL> delfailregex <INDEX>", "removes the regular expression at <INDEX> for failregex"], 
+["set <JAIL> ignorecommand <VALUE>", "sets ignorecommand of <JAIL>"],
 ["set <JAIL> addignoreregex <REGEX>", "adds the regular expression <REGEX> which should match pattern to exclude for <JAIL>"],
 ["set <JAIL> delignoreregex <INDEX>", "removes the regular expression at <INDEX> for ignoreregex"], 
 ["set <JAIL> findtime <TIME>", "sets the number of seconds <TIME> for which the filter will look back for <JAIL>"], 
@@ -90,6 +91,7 @@ protocol = [
 ["get <JAIL> logencoding <ENCODING>", "gets the <ENCODING> of the log files for <JAIL>"],
 ["get <JAIL> journalmatch", "gets the journal filter match for <JAIL>"],
 ["get <JAIL> ignoreip", "gets the list of ignored IP addresses for <JAIL>"],
+["get <JAIL> ignorecommand", "gets ignorecommand of <JAIL>"],
 ["get <JAIL> failregex", "gets the list of regular expressions which matches the failures for <JAIL>"],
 ["get <JAIL> ignoreregex", "gets the list of regular expressions which matches patterns to ignore for <JAIL>"],
 ["get <JAIL> findtime", "gets the time for which the filter will look back for failures for <JAIL>"],

fail2ban/server/datedetector.py

@@ -21,7 +21,7 @@ __author__ = "Cyril Jaquier and Fail2Ban Contributors"
 __copyright__ = "Copyright (c) 2004 Cyril Jaquier"
 __license__ = "GPL"
 
-import time, logging
+import sys, time, logging
 
 from datetemplate import DatePatternRegex, DateTai64n, DateEpoch, DateISO8601
 from threading import Lock
@@ -53,16 +53,18 @@ class DateDetector:
 	def addDefaultTemplate(self):
 		self.__lock.acquire()
 		try:
-			# asctime with subsecond: Sun Jan 23 21:59:59.011 2005 
+			if sys.version_info >= (2, 5): # because of '%.f'
-			self.appendTemplate("%a %b %d %H:%M:%S.%f %Y")
+				# asctime with subsecond: Sun Jan 23 21:59:59.011 2005 
+				self.appendTemplate("%a %b %d %H:%M:%S.%f %Y")
 			# asctime: Sun Jan 23 21:59:59 2005 
 			self.appendTemplate("%a %b %d %H:%M:%S %Y")
 			# asctime without year: Sun Jan 23 21:59:59 
 			self.appendTemplate("%a %b %d %H:%M:%S")
 			# standard: Jan 23 21:59:59 
 			self.appendTemplate("%b %d %H:%M:%S")
-			# proftpd date: 2005-01-23 21:59:59,333
+			if sys.version_info >= (2, 5): # because of '%.f'
-			self.appendTemplate("%Y-%m-%d %H:%M:%S,%f")
+				# proftpd date: 2005-01-23 21:59:59,333
+				self.appendTemplate("%Y-%m-%d %H:%M:%S,%f")
 			# simple date: 2005-01-23 21:59:59 
 			self.appendTemplate("%Y-%m-%d %H:%M:%S")
 			# simple date: 2005/01/23 21:59:59 
@@ -80,8 +82,9 @@ class DateDetector:
 			self.appendTemplate("%m/%d/%Y:%H:%M:%S")
 			# custom for syslog-ng 2006.12.21 06:43:20
 			self.appendTemplate("%Y.%m.%d %H:%M:%S")
-			# named 26-Jul-2007 15:20:52.252 
+			if sys.version_info >= (2, 5): # because of '%.f'
-			self.appendTemplate("%d-%b-%Y %H:%M:%S.%f")
+				# named 26-Jul-2007 15:20:52.252 
+				self.appendTemplate("%d-%b-%Y %H:%M:%S.%f")
 			# roundcube 26-Jul-2007 15:20:52 +0200
 			self.appendTemplate("%d-%b-%Y %H:%M:%S %z")
 			# 26-Jul-2007 15:20:52

fail2ban/server/datetemplate.py

@@ -82,7 +82,7 @@ class DateEpoch(DateTemplate):
 	
 	def __init__(self):
 		DateTemplate.__init__(self)
-		self.setRegex("(?:^|(?P<selinux>(?<=audit\()))\d{10}(?:\.\d{3,6})?(?(selinux)(?=:\d+\)))")
+		self.setRegex("(?:^|(?P<square>(?<=^\[))|(?P<selinux>(?<=audit\()))\d{10}(?:\.\d{3,6})?(?(selinux)(?=:\d+\))(?(square)(?=\])))")
 	
 	def getDate(self, line):
 		dateMatch = self.matchDate(line)

fail2ban/server/filter.py

@@ -21,8 +21,6 @@ __author__ = "Cyril Jaquier and Fail2Ban Contributors"
 __copyright__ = "Copyright (c) 2004 Cyril Jaquier, 2011-2013 Yaroslav Halchenko"
 __license__ = "GPL"
 
-import sys
-
 from failmanager import FailManagerEmpty
 from failmanager import FailManager
 from ticket import FailTicket
@@ -31,6 +29,7 @@ from datedetector import DateDetector
 from datetemplate import DatePatternRegex, DateISO8601, DateEpoch, DateTai64n
 from mytime import MyTime
 from failregex import FailRegex, Regex, RegexException
+from action import Action
 
 import logging, re, os, fcntl, time, sys, locale, codecs
 
@@ -75,6 +74,8 @@ class Filter(JailThread):
 		## Store last time stamp, applicable for multi-line
 		self.__lastTimeText = ""
 		self.__lastDate = None
+		## External command
+		self.__ignoreCommand = False
 
 		self.dateDetector = DateDetector()
 		self.dateDetector.addDefaultTemplate()
@@ -289,6 +290,20 @@ class Filter(JailThread):
 	def run(self): # pragma: no cover
 		raise Exception("run() is abstract")
 
+	##
+	# Set external command, for ignoredips
+	#
+
+	def setIgnoreCommand(self, command):
+		self.__ignoreCommand = command
+
+	##
+	# Get external command, for ignoredips
+	#
+
+	def getIgnoreCommand(self):
+		return self.__ignoreCommand
+
 	##
 	# Ban an IP - http://blogs.buanzo.com.ar/2009/04/fail2ban-patch-ban-ip-address-manually.html
 	# Arturo 'Buanzo' Busleiman <buanzo@buanzo.com.ar>
@@ -361,6 +376,12 @@ class Filter(JailThread):
 					continue
 			if a == b:
 				return True
+
+		if self.__ignoreCommand:
+			command = Action.replaceTag(self.__ignoreCommand, { 'ip': ip } )
+			logSys.debug('ignore command: ' + command)
+			return Action.executeCmd(command)
+
 		return False
 
 

fail2ban/server/filterpyinotify.py

@@ -115,9 +115,6 @@ class FilterPyinotify(FileFilter):
 		wd = self.__monitor.add_watch(path, pyinotify.IN_MODIFY)
 		self.__watches.update(wd)
 		logSys.debug("Added file watcher for %s", path)
-		# process the file since we did get even
-		self._process_file(path)
-
 
 	def _delFileWatcher(self, path):
 		wdInt = self.__watches[path]
@@ -143,6 +140,7 @@ class FilterPyinotify(FileFilter):
 			logSys.debug("Added monitor for the parent directory %s", path_dir)
 
 		self._addFileWatcher(path)
+		self._process_file(path)
 
 
     ##

fail2ban/server/server.py

@@ -234,6 +234,12 @@ class Server:
 	def getDatePattern(self, name):
 		return self.__jails.getFilter(name).getDatePattern()
 
+	def setIgnoreCommand(self, name, value):
+		self.__jails.getFilter(name).setIgnoreCommand(value)
+
+	def getIgnoreCommand(self, name):
+		return self.__jails.getFilter(name).getIgnoreCommand()
+
 	def addFailRegex(self, name, value):
 		self.__jails.getFilter(name).addFailRegex(value)
 	

fail2ban/server/transmitter.py

@@ -148,6 +148,10 @@ class Transmitter:
 			value = command[2]
 			self.__server.delIgnoreIP(name, value)
 			return self.__server.getIgnoreIP(name)
+		elif command[1] == "ignorecommand":
+			value = command[2]
+			self.__server.setIgnoreCommand(name, value)
+			return self.__server.getIgnoreCommand(name)
 		elif command[1] == "addlogpath":
 			value = command[2:]
 			for path in value:
@@ -296,6 +300,8 @@ class Transmitter:
 			return self.__server.getJournalMatch(name)
 		elif command[1] == "ignoreip":
 			return self.__server.getIgnoreIP(name)
+		elif command[1] == "ignorecommand":
+			return self.__server.getIgnoreCommand(name)
 		elif command[1] == "failregex":
 			return self.__server.getFailRegex(name)
 		elif command[1] == "ignoreregex":

fail2ban/tests/actionstestcase.py

@@ -0,0 +1,79 @@
+# emacs: -*- mode: python; py-indent-offset: 4; indent-tabs-mode: t -*-
+# vi: set ft=python sts=4 ts=4 sw=4 noet :
+
+# This file is part of Fail2Ban.
+#
+# Fail2Ban is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# Fail2Ban is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with Fail2Ban; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
+
+# Author: Daniel Black
+# 
+
+__author__ = "Daniel Black"
+__copyright__ = "Copyright (c) 2013 Daniel Black"
+__license__ = "GPL"
+
+import unittest, time
+import sys, os, tempfile
+from fail2ban.server.actions import Actions
+from dummyjail import DummyJail
+
+class ExecuteActions(unittest.TestCase):
+
+	def setUp(self):
+		"""Call before every test case."""
+		self.__jail = DummyJail()
+		self.__actions = Actions(self.__jail)
+		self.__tmpfile, self.__tmpfilename  = tempfile.mkstemp()
+
+	def tearDown(self):
+		os.remove(self.__tmpfilename)
+
+	def defaultActions(self):
+		self.__actions.addAction('ip')
+		self.__ip = self.__actions.getAction('ip')
+		self.__ip.setActionStart('echo ip start 64 >> "%s"' % self.__tmpfilename )
+		self.__ip.setActionBan('echo ip ban <ip> >> "%s"' % self.__tmpfilename )
+		self.__ip.setActionUnban('echo ip unban <ip> >> "%s"' % self.__tmpfilename )
+		self.__ip.setActionCheck('echo ip check <ip> >> "%s"' % self.__tmpfilename )
+		self.__ip.setActionStop('echo ip stop >> "%s"' % self.__tmpfilename )
+
+	def testActionsManipulation(self):
+		self.__actions.addAction('test')
+		self.assertTrue(self.__actions.getAction('test'))
+		self.assertTrue(self.__actions.getLastAction())
+		self.assertRaises(KeyError,self.__actions.getAction,*['nonexistant action'])
+		self.__actions.addAction('test1')
+		self.__actions.delAction('test')
+		self.__actions.delAction('test1')
+		self.assertRaises(KeyError, self.__actions.getAction, *['test'])
+		self.assertRaises(IndexError,self.__actions.getLastAction)
+
+		self.__actions.setBanTime(127)
+		self.assertEqual(self.__actions.getBanTime(),127)
+		self.assertRaises(ValueError, self.__actions.removeBannedIP, '127.0.0.1')
+
+
+	def testActionsOutput(self):
+		self.defaultActions()
+		self.__actions.start()
+		with open(self.__tmpfilename) as f:
+			time.sleep(3)
+			self.assertEqual(f.read(),"ip start 64\n")
+
+		self.__actions.stop()
+		self.__actions.join()
+		self.assertEqual(self.__actions.status(),[("Currently banned", 0 ),
+               ("Total banned", 0 ), ("IP list", [] )])
+

fail2ban/tests/files/ignorecommand.py

@@ -0,0 +1,5 @@
+#!/usr/bin/python
+import sys
+if sys.argv[1] == "10.0.0.1":
+	exit(0)
+exit(1)

fail2ban/tests/files/logs/apache-noscript

@@ -2,3 +2,17 @@
 [Sun Jun 09 07:57:47 2013] [error] [client 192.0.43.10] script '/usr/lib/cgi-bin/gitweb.cgiwp-login.php' not found or unable to stat
 # failJSON: { "time": "2008-07-22T06:48:30", "match": true , "host": "198.51.100.86" }
 [Tue Jul 22 06:48:30 2008] [error] [client 198.51.100.86] File does not exist: /home/southern/public_html/azenv.php
+
+# failJSON: { "time": "2008-07-22T06:48:30", "match": true , "host": "198.51.100.86" }
+[Tue Jul 22 06:48:30 2008] [error] [client 198.51.100.86] script not found or unable to stat: /home/e-smith/files/ibays/Primary/cgi-bin/php
+# failJSON: { "time": "2008-07-22T06:48:30", "match": true , "host": "198.51.100.86" }
+[Tue Jul 22 06:48:30 2008] [error] [client 198.51.100.86] script not found or unable to stat: /home/e-smith/files/ibays/Primary/cgi-bin/php5
+# failJSON: { "time": "2008-07-22T06:48:30", "match": true , "host": "198.51.100.86" }
+[Tue Jul 22 06:48:30 2008] [error] [client 198.51.100.86] script not found or unable to stat: /home/e-smith/files/ibays/Primary/cgi-bin/php-cgi
+# failJSON: { "time": "2008-07-22T06:48:30", "match": true , "host": "198.51.100.86" }
+[Tue Jul 22 06:48:30 2008] [error] [client 198.51.100.86] script not found or unable to stat: /home/e-smith/files/ibays/Primary/cgi-bin/php.cgi
+# failJSON: { "time": "2008-07-22T06:48:30", "match": true , "host": "198.51.100.86" }
+[Tue Jul 22 06:48:30 2008] [error] [client 198.51.100.86] script not found or unable to stat: /home/e-smith/files/ibays/Primary/cgi-bin/php4
+# apache 2.4
+# failJSON: { "time": "2013-12-23T07:49:01", "match": true , "host": "204.232.202.107" }
+[Mon Dec 23 07:49:01.981912 2013] [:error] [pid 3790] [client 204.232.202.107:46301] script '/var/www/timthumb.php' not found or unable to stat

fail2ban/tests/files/logs/nsd

@@ -0,0 +1,4 @@
+# failJSON: { "time": "2013-12-17T14:58:14", "match": true , "host": "192.0.2.105" }
+[1387288694] nsd[7745]: info: ratelimit block example.com. type any target 192.0.2.0/24 query 192.0.2.105 TYPE255
+# failJSON: { "time": "2013-12-18T07:42:15", "match": true , "host": "192.0.2.115" }
+[1387348935] nsd[23600]: info: axfr for zone domain.nl. from client 192.0.2.115 refused, no acl matches.

fail2ban/tests/filtertestcase.py

@@ -222,7 +222,6 @@ class IgnoreIP(LogCaptureTestCase):
 		ipList = "127.0.0.1", "192.168.0.1", "255.255.255.255", "99.99.99.99"
 		for ip in ipList:
 			self.filter.addIgnoreIP(ip)
-
 			self.assertTrue(self.filter.inIgnoreIPList(ip))
 
 	def testIgnoreIPNOK(self):
@@ -254,6 +253,11 @@ class IgnoreIP(LogCaptureTestCase):
 		self.assertFalse(self._is_logged('Ignore 192.168.1.32'))
 		self.assertTrue(self._is_logged('Requested to manually ban an ignored IP 192.168.1.32. User knows best. Proceeding to ban it.'))
 
+	def testIgnoreCommand(self):
+		self.filter.setIgnoreCommand("fail2ban/tests/files/ignorecommand.py <ip>")
+		self.assertTrue(self.filter.inIgnoreIPList("10.0.0.1"))
+		self.assertFalse(self.filter.inIgnoreIPList("10.0.0.0"))
+
 
 class IgnoreIPDNS(IgnoreIP):
 
@@ -268,15 +272,28 @@ class IgnoreIPDNS(IgnoreIP):
 		self.assertFalse(self.filter.inIgnoreIPList("128.178.50.11"))
 		self.assertFalse(self.filter.inIgnoreIPList("128.178.50.13"))
 
+class LogFile(LogCaptureTestCase):
 
-class LogFile(unittest.TestCase):
+	MISSING = 'testcases/missingLogFile'
+
+	def setUp(self):
+		LogCaptureTestCase.setUp(self)
+
+	def tearDown(self):
+		LogCaptureTestCase.tearDown(self)
+
+	def testMissingLogFiles(self):
+		self.filter = FilterPoll(None)
+		self.assertRaises(IOError, self.filter.addLogPath, LogFile.MISSING)
+
+class LogFileFilterPoll(unittest.TestCase):
 
 	FILENAME = os.path.join(TEST_FILES_DIR, "testcase01.log")
 
 	def setUp(self):
 		"""Call before every test case."""
 		self.filter = FilterPoll(DummyJail())
-		self.filter.addLogPath(LogFile.FILENAME)
+		self.filter.addLogPath(LogFileFilterPoll.FILENAME)
 
 	def tearDown(self):
 		"""Call after every test case."""
@@ -286,7 +303,8 @@ class LogFile(unittest.TestCase):
 	#	self.filter.openLogFile(LogFile.FILENAME)
 
 	def testIsModified(self):
-		self.assertTrue(self.filter.isModified(LogFile.FILENAME))
+		self.assertTrue(self.filter.isModified(LogFileFilterPoll.FILENAME))
+		self.assertFalse(self.filter.isModified(LogFileFilterPoll.FILENAME))
 
 
 class LogFileMonitor(LogCaptureTestCase):
@@ -790,11 +808,11 @@ class GetFailures(unittest.TestCase):
 		tearDownMyTime()
 
 	def testTail(self):
-		self.filter.addLogPath(LogFile.FILENAME, tail=True)
+		self.filter.addLogPath(GetFailures.FILENAME_01, tail=True)
 		self.assertEqual(self.filter.getLogPath()[-1].getPos(), 1653)
 		self.filter.getLogPath()[-1].close()
 		self.assertEqual(self.filter.getLogPath()[-1].readline(), "")
-		self.filter.delLogPath(LogFile.FILENAME)
+		self.filter.delLogPath(GetFailures.FILENAME_01)
 		self.assertEqual(self.filter.getLogPath(),[])
 
 	def testGetFailures01(self, filename=None, failures=None):

fail2ban/tests/samplestestcase.py

@@ -66,6 +66,7 @@ def testSampleRegexsFactory(name):
 
 		# Check filter exists
 		filterConf = FilterReader(name, "jail", {}, basedir=CONFIG_DIR)
+		self.assertEqual(filterConf.getFile(), name)
 		filterConf.read()
 		filterConf.getOptions({})
 
@@ -77,10 +78,6 @@ def testSampleRegexsFactory(name):
 			elif opt[2] == "addignoreregex":
 				self.filter.addIgnoreRegex(opt[3])
 
-		if not self.filter.getFailRegex():
-			# No fail regexs set: likely just common file for includes.
-			return
-
 		self.assertTrue(
 			os.path.isfile(os.path.join(TEST_FILES_DIR, "logs", name)),
 			"No sample log file available for '%s' filter" % name)

fail2ban/tests/servertestcase.py

@@ -26,7 +26,8 @@ __license__ = "GPL"
 
 import unittest, socket, time, tempfile, os, locale, sys, logging
 
-from fail2ban.server.server import Server
+from fail2ban.server.failregex import Regex, FailRegex, RegexException
+from fail2ban.server.server import Server, logSys
 from fail2ban.server.jail import Jail
 from fail2ban.exceptions import UnknownJailException
 from fail2ban.tests.utils import LogCaptureTestCase
@@ -403,6 +404,9 @@ class Transmitter(TransmitterBase):
 			self.transm.proceed(["set", self.jailName, "delignoreip", value]),
 			(0, [value]))
 
+	def testJailIgnoreCommand(self):
+		self.setGetTest("ignorecommand", "bin ", jail=self.jailName)
+
 	def testJailRegex(self):
 		self.jailAddDelRegexTest("failregex",
 			[
@@ -709,6 +713,7 @@ class TransmitterLogging(TransmitterBase):
 						line1 = f.next()
 					self.assertTrue(line1.endswith("After flushlogs\n"))
 					self.assertRaises(StopIteration, f.next)
+					f.close()
 			finally:
 				os.remove(fn2)
 		finally:
@@ -727,3 +732,30 @@ class JailTests(unittest.TestCase):
 		longname = "veryveryverylongname"
 		jail = Jail(longname)
 		self.assertEqual(jail.getName(), longname)
+
+class RegexTests(unittest.TestCase):
+
+	def testInit(self):
+		# Should raise an Exception upon empty regex
+		self.assertRaises(RegexException, Regex, '')
+		self.assertRaises(RegexException, Regex, ' ')
+		self.assertRaises(RegexException, Regex, '\t')
+
+	def testStr(self):
+		# .replace just to guarantee uniform use of ' or " in the %r
+		self.assertEqual(str(Regex('a')).replace('"', "'"), "Regex('a')")
+		# Class name should be proper
+		self.assertTrue(str(FailRegex('<HOST>')).startswith("FailRegex("))
+
+	def testHost(self):
+		self.assertRaises(RegexException, FailRegex, '')
+		# Testing obscure case when host group might be missing in the matched pattern,
+		# e.g. if we made it optional.
+		fr = FailRegex('%%<HOST>?')
+		self.assertFalse(fr.hasMatched())
+		fr.search(("%%",))
+		self.assertTrue(fr.hasMatched())
+		self.assertRaises(RegexException, fr.getHost)
+
+
+

fail2ban/tests/utils.py

@@ -145,6 +145,7 @@ def gatherTests(regexps=None, no_network=False):
 	from fail2ban.tests import servertestcase
 	from fail2ban.tests import datedetectortestcase
 	from fail2ban.tests import actiontestcase
+	from fail2ban.tests import actionstestcase
 	from fail2ban.tests import sockettestcase
 	from fail2ban.tests import misctestcase
 	from fail2ban.tests import databasetestcase
@@ -170,7 +171,9 @@ def gatherTests(regexps=None, no_network=False):
 	#tests.addTest(unittest.makeSuite(servertestcase.StartStop))
 	tests.addTest(unittest.makeSuite(servertestcase.Transmitter))
 	tests.addTest(unittest.makeSuite(servertestcase.JailTests))
+	tests.addTest(unittest.makeSuite(servertestcase.RegexTests))
 	tests.addTest(unittest.makeSuite(actiontestcase.ExecuteAction))
+	tests.addTest(unittest.makeSuite(actionstestcase.ExecuteActions))
 	# FailManager
 	tests.addTest(unittest.makeSuite(failmanagertestcase.AddFailure))
 	# BanManager
@@ -195,6 +198,7 @@ def gatherTests(regexps=None, no_network=False):
 	tests.addTest(unittest.makeSuite(filtertestcase.BasicFilter))
 	tests.addTest(unittest.makeSuite(filtertestcase.LogFile))
 	tests.addTest(unittest.makeSuite(filtertestcase.LogFileMonitor))
+	tests.addTest(unittest.makeSuite(filtertestcase.LogFileFilterPoll))
 	if not no_network:
 		tests.addTest(unittest.makeSuite(filtertestcase.IgnoreIPDNS))
 		tests.addTest(unittest.makeSuite(filtertestcase.GetFailures))

files/nagios/check_fail2ban

@@ -165,7 +165,7 @@ if (($critical < 0) or ($warning < 0) or ($critical < $warning)) {
 
 # Core script
 # -----------
-my ($how_many_jail,$how_many_banned,$return_print,$plugstate) = (0,0,"","OK");
+my ($how_many_jail,$how_many_banned,$return_print,$perf_print,$plugstate) = (0,0,"","","OK");
 
 
 ### Test the connection to the fail2ban server
@@ -190,6 +190,7 @@ if ($jail_specific) {
     else {
         $how_many_banned = int($current_ban_number);
         $return_print = $how_many_banned.' current banned IP(s) for the specific jail '.$jail_specific;
+        $perf_print .= "$jail_name.currentBannedIP=$current_ban_number " if ($perfdata_value);
     }
 }
 ### To analyze all the jail
@@ -214,6 +215,7 @@ else {
         else {
             print "DEBUG : the jail $jail_name has currently $current_ban_number banned IPs\n" if ($verbose_value);
             $how_many_banned += int($current_ban_number);
+            $perf_print .= "$jail_name.currentBannedIP=$current_ban_number " if ($perfdata_value);
         }
     }
     $return_print = $how_many_jail.' detected jails with '.$how_many_banned.' current banned IP(s)';
@@ -224,7 +226,7 @@ $plugstate = "CRITICAL" if ($how_many_banned >= $critical);
 $plugstate = "WARNING" if (($how_many_banned >= $warning) && ($how_many_banned < $critical));
 
 $return_print = $display." - ".$plugstate." - ".$return_print;
-$return_print .= " | currentBannedIP=$how_many_banned" if ($perfdata_value);
+$return_print .= " | $perf_print" if ($perfdata_value);
 
 print $return_print;
 exit $ERRORS{"$plugstate"};

man/fail2ban-client.1

@@ -128,6 +128,9 @@ for <JAIL>
 removes the regular expression at
 <INDEX> for failregex
 .TP
+\fBset <JAIL> ignorecommand <VALUE>\fR
+sets ignorecommand of <JAIL>
+.TP
 \fBset <JAIL> addignoreregex <REGEX>\fR
 adds the regular expression
 <REGEX> which should match pattern
@@ -206,6 +209,9 @@ files for <JAIL>
 gets the list of ignored IP
 addresses for <JAIL>
 .TP
+\fBget <JAIL> ignorecommand\fR
+gets ignorecommand of <JAIL>
+.TP
 \fBget <JAIL> failregex\fR
 gets the list of regular
 expressions which matches the

man/jail.conf.5

@@ -71,6 +71,11 @@ The following options are applicable to all jails. Their meaning is described in
 \fBaction\fR 
 .TP
 \fBignoreip\fR 
+A space separated list of IPs not to ban.
+.TP
+\fBignorecommand\fR
+A command that is executed to determine if the current ban's actionban is to be executed. This command will return true if the current ban should be ignored. A false return value will result in the ban's actionban executed.
+Like ACTION FILES, tags like <ip> are can be included in the ignore command value and will be substitued before execution. Currently only <ip> is supported however more will be added later.
 .TP
 \fBbantime\fR
 .TP