Merge branch '0.10' into wc/debian

.coveragerc

@@ -7,5 +7,6 @@ source =
 
 [report]
 exclude_lines =
-    pragma: no cover
+    pragma: ?no ?cover
-    pragma: systemd no cover
+    pragma: ?${F2B_PY}.x no ?cover
+    pragma: ?systemd no ?cover

.travis.yml

@@ -18,8 +18,9 @@ python:
   - pypy3.3-5.5-alpha
 before_install:
   - echo "running under $TRAVIS_PYTHON_VERSION"
-  - if [[ $TRAVIS_PYTHON_VERSION == 2* || $TRAVIS_PYTHON_VERSION == pypy* && $TRAVIS_PYTHON_VERSION != pypy3* ]]; then export F2B_PY_2=true && echo "Set F2B_PY_2"; fi
+  - if [[ $TRAVIS_PYTHON_VERSION == 2* || $TRAVIS_PYTHON_VERSION == pypy* && $TRAVIS_PYTHON_VERSION != pypy3* ]]; then export F2B_PY=2; fi
-  - if [[ $TRAVIS_PYTHON_VERSION == 3* || $TRAVIS_PYTHON_VERSION == pypy3* ]]; then export F2B_PY_3=true && echo "Set F2B_PY_3"; fi
+  - if [[ $TRAVIS_PYTHON_VERSION == 3* || $TRAVIS_PYTHON_VERSION == pypy3* ]]; then export F2B_PY=3; fi
+  - echo "Set F2B_PY=$F2B_PY"
   - travis_retry sudo apt-get update -qq
   # Set this so sudo executes the correct python binary
   #   Anything not using sudo will already have the correct environment
@@ -28,29 +29,32 @@ install:
   # Install Python packages / dependencies
   #   coverage
   - travis_retry pip install coverage
-  #   coveralls
+  #   coveralls (note coveralls doesn't support 2.6 now):
-  - travis_retry pip install coveralls codecov
+  - if [[ $TRAVIS_PYTHON_VERSION != 2.6* ]]; then F2B_COV=1; else F2B_COV=0; fi
+  - if [[ "$F2B_COV" = 1 ]]; then travis_retry pip install coveralls; fi
+  #   codecov:
+  - travis_retry pip install codecov
   #   dnspython or dnspython3
-  - if [[ "$F2B_PY_2" ]]; then travis_retry pip install dnspython; fi
+  - if [[ "$F2B_PY" = 2 ]]; then travis_retry pip install dnspython; fi
-  - if [[ "$F2B_PY_3" ]]; then travis_retry pip install dnspython3; fi
+  - if [[ "$F2B_PY" = 3 ]]; then travis_retry pip install dnspython3; fi
   #   gamin - install manually (not in PyPI) - travis-ci system Python is 2.7
   - if [[ $TRAVIS_PYTHON_VERSION == 2.7 ]]; then travis_retry sudo apt-get install -qq python-gamin && cp /usr/share/pyshared/gamin.py /usr/lib/pyshared/python2.7/_gamin.so $VIRTUAL_ENV/lib/python2.7/site-packages/; fi
   #   pyinotify
   - travis_retry pip install pyinotify
 before_script:
   # Manually execute 2to3 for now
-  - if [[ "$F2B_PY_3" ]]; then ./fail2ban-2to3; fi
+  - if [[ "$F2B_PY" = 3 ]]; then ./fail2ban-2to3; fi
 script:
   # Keep the legacy setup.py test approach of checking coverage for python2
-  - if [[ "$F2B_PY_2" ]]; then coverage run setup.py test; fi
+  - if [[ "$F2B_PY" = 2 ]]; then coverage run setup.py test; fi
   # Coverage doesn't pick up setup.py test with python3, so run it directly (with same verbosity as from setup)
-  - if [[ "$F2B_PY_3" ]]; then coverage run bin/fail2ban-testcases --verbosity=2; fi
+  - if [[ "$F2B_PY" = 3 ]]; then coverage run bin/fail2ban-testcases --verbosity=2; fi
   # Use $VENV_BIN (not python) or else sudo will always run the system's python (2.7)
   - sudo $VENV_BIN/pip install .
   # Doc files should get installed on Travis under Linux
   - test -e /usr/share/doc/fail2ban/FILTERS
 after_success:
-  - coveralls
+  - if [[ "$F2B_COV" = 1 ]]; then coveralls; fi
   - codecov
 matrix:
   fast_finish: true

ChangeLog

@@ -27,10 +27,117 @@ Incompatibility list (compared to v.0.9):
 * v.0.10 uses more precise date template handling, that can be theoretically incompatible to some
   user configurations resp. `datepattern`.
 
-* Since v0.10 fail2ban supports the matching of the IPv6 addresses, but not all ban actions are
+* Since v0.10 fail2ban supports the matching of IPv6 addresses, but not all ban actions are
   IPv6-capable now.
 
 
+ver. 0.10.5-dev-1 (20??/??/??) - development edition
+-----------
+
+### Fixes
+* fixed read of included config-files (`.local` overwrites options of `.conf` for config-files 
+  included with before/after)
+* `filter.d/sshd.conf`:
+  - captures `Disconnecting ...: Change of username or service not allowed` (gh-2239, gh-2279)
+  - captures `Disconnected from ... [preauth]` (`extra`/`aggressive` mode and preauth phase only, gh-2239, gh-2279)
+* `filter.d/mysqld-auth.conf`: 
+  - MYSQL 8.0.13 compatibility (log-error-verbosity = 3), log-format contains few additional words
+    enclosed in brackets after "[Note]" (gh-2314)
+* `files/fail2ban.service.in`: fixed systemd-unit template - missing nftables dependency (gh-2313)
+
+### New Features
+* new failregex-flag tag `<F-MLFGAINED>` for failregex, signaled that the access to service was gained
+  (ATM used similar to tag `<F-NOFAIL>`, but it does not add the log-line to matches, gh-2279)
+
+### Enhancements
+
+
+ver. 0.10.4 (2018/10/04) - ten-four-on-due-date-ten-four
+-----------
+
+### Fixes
+* `filter.d/dovecot.conf`: 
+  - failregex enhancement to catch sql password mismatch errors (gh-2153);
+  - disconnected with "proxy dest auth failed" (gh-2184);
+* `filter.d/freeswitch.conf`:
+  - provide compatibility for log-format from gh-2193:
+    * extended with new default date-pattern `^(?:%%Y-)?%%m-%%d[ T]%%H:%%M:%%S(?:\.%%f)?` to cover
+      `YYYY-mm-dd HH:MM::SS.ms` as well as `mm-dd HH:MM::SS.ms` (so year is optional);
+    * more optional arguments in log-line (so accept [WARN] as well as [WARNING] and optional [SOFIA] hereafter);
+  - extended with mode parameter, allows to avoid matching of messages like `auth challenge (REGISTER)`
+    (see gh-2163) (currently `extra` as default to be backwards-compatible), see comments in filter
+    how to set it to mode `normal`.
+* `filter.d/domino-smtp.conf`:
+  - recognizes failures logged using another format (something like session-id, IP enclosed in square brackets);
+  - failregex extended to catch connections rejected for policy reasons (gh-2228);
+* `action.d/hostsdeny.conf`: fix parameter in config (dynamic parameters stating with '_' are protected 
+  and don't allowed in command-actions), see gh-2114;
+* decoding stability fix by wrong encoded characters like utf-8 surrogate pairs, etc (gh-2171):
+  - fail2ban running in the preferred encoding now (as default encoding also within python 2.x), mostly
+    `UTF-8` in opposite to `ascii` previously, so minimizes influence of implicit conversions errors;
+  - actions: avoid possible conversion errors on wrong-chars by replace tags;
+  - database: improve adapter/converter handlers working on invalid characters in sense of json and/or sqlite-database;
+    additionally both are exception-safe now, so avoid possible locking of database (closes gh-2137);
+  - logging in fail2ban is process-wide exception-safe now.
+* repaired start-time of initial seek to time (as well as other log-parsing related data), 
+  if parameter `logpath` specified before `findtime`, `backend`, `datepattern`, etc (gh-2173)
+* systemd: fixed type error on option `journalflags`: an integer is required (gh-2125);
+
+### New Features
+* new option `ignorecache` to improve performance of ignore failure check (using caching of `ignoreip`, 
+  `ignoreself` and `ignorecommand`), see `man jail.conf` for syntax-example;
+* `ignorecommand` extended to use actions-similar replacement (capable to interpolate 
+  all possible tags like `<ip-host>`, `<family>`, `<fid>`, `F-USER` etc.)
+
+### Enhancements
+* `filter.d/dovecot.conf`: extended with tags F-USER (and alternatives) to collect user-logins (gh-2168)
+* since v.0.10.4, fail2ban-client, fail2ban-server and fail2ban-regex will return version without logo info,
+  additionally option `-V` can be used to get version in normalized machine-readable short format.
+
+
+ver. 0.10.3 (2018/04/04) - the-time-is-always-right-to-do-what-is-right
+-----------
+
+### ver. 0.10.3.1:
+* fixed JSON serialization for the set-object within dump into database (gh-2103).
+
+### Fixes
+* `filter.d/asterisk.conf`: fixed failregex prefix by log over remote syslog server (gh-2060);
+* `filter.d/exim.conf`: failregex extended - SMTP call dropped: too many syntax or protocol errors (gh-2048);
+* `filter.d/recidive.conf`: fixed if logging into systemd-journal (SYSLOG) with daemon name in prefix, gh-2069;
+* `filter.d/sendmail-auth.conf`, `filter.d/sendmail-reject.conf` :
+  - fixed failregex, sendmail uses prefix 'IPv6:' logging of IPv6 addresses (gh-2064);
+* `filter.d/sshd.conf`:
+  - failregex got an optional space in order to match new log-format (see gh-2061);
+  - fixed ddos-mode regex to match refactored message (some versions can contain port now, see gh-2062);
+  - fixed root login refused regex (optional port before preauth, gh-2080);
+  - avoid banning of legitimate users when pam_unix used in combination with other password method, so
+    bypass pam_unix failures if accepted available for this user gh-2070;
+  - amend to gh-1263 with better handling of multiple attempts (failures for different user-names recognized immediatelly);
+  - mode `ddos` (and `aggressive`) extended to catch `Connection closed by ... [preauth]`, so in DDOS mode
+    it counts failure on closing connection within preauth-stage (gh-2085);
+* `action.d/abuseipdb.conf`: fixed curl cypher errors and comment quote-issue (gh-2044, gh-2101);
+* `action.d/badips.py`: implicit convert IPAddr to str, solves an issue "expected string, IPAddr found" (gh-2059);
+* `action.d/hostsdeny.conf`: fixed IPv6 syntax (enclosed in square brackets, gh-2066);
+* (Free)BSD ipfw actionban fixed to allow same rule added several times (gh-2054);
+
+### New Features
+* several stability and performance optimizations, more effective filter parsing, etc;
+* stable runnable within python versions 3.6 (as well as within 3.7-dev);
+
+### Enhancements
+* `filter.d/apache-auth.conf`: detection of Apache SNI errors resp. misredirect attempts (gh-2017, gh-2097);
+* `filter.d/apache-noscript.conf`: extend failregex to match "Primary script unknown", e. g. from php-fpm (gh-2073);
+* date-detector extended with long epoch (`LEPOCH`) to parse milliseconds/microseconds posix-dates (gh-2029);
+* possibility to specify own regex-pattern to match epoch date-time, e. g. `^\[{EPOCH}\]` or `^\[{LEPOCH}\]` (gh-2038);
+  the epoch-pattern similar to `{DATE}` patterns does the capture and cuts out the match of whole pattern from the log-line,
+  e. g. date-pattern `^\[{LEPOCH}\]\s+:` will match and cut out `[1516469849551000] :` from begin of the log-line.
+* badips.py now uses https instead of plain http when requesting badips.com (gh-2057);
+* add support for "any" badips.py bancategory, to be able to retrieve IPs from all categories with a desired score (gh-2056);
+* Introduced new parameter `padding` for logging within fail2ban-server (default on, excepting SYSLOG):
+  Usage `logtarget = target[padding=on|off]`
+
+
 ver. 0.10.2 (2018/01/18) - nothing-burns-like-the-cold
 -----------
 
@@ -93,7 +200,7 @@ ver. 0.10.1 (2017/10/12) - succeeded-before-friday-the-13th
 * avoid using "ANSI_X3.4-1968" as preferred encoding (if missing environment variables 
   'LANGUAGE', 'LC_ALL', 'LC_CTYPE', and 'LANG', see gh-1587).
 * action.d/pf.conf: several fixes for pf-action like anchoring, etc. (see gh-1866, gh-1867);
-* fixed ignorself issue "Retrieving own IPs of localhost failed: inet_pton() argument 2 must be string, not int" (see gh-1865);
+* fixed ignoreself issue "Retrieving own IPs of localhost failed: inet_pton() argument 2 must be string, not int" (see gh-1865);
 * fixed tags `<fq-hostname>` and `<sh-hostname>`, could be used without ticket (a. g. in `actionstart` etc., gh-1859).
 
 * setup.py: fixed several setup facilities (gh-1874):

DEVELOP

@@ -262,12 +262,16 @@ FileContainer
     Keeps the position pointer
 
 
-dnsutils.py
+ipdns.py
-~~~~~~~~~~~
+~~~~~~~~
 
 DNSUtils
 
-  Utility class for DNS and IP handling
+  Utility class for DNS handling
+
+IPAddr
+
+  Object-class for IP address handling
 
 
 filter*.py

MANIFEST

@@ -391,6 +391,8 @@ kill-server
 man/fail2ban.1
 man/fail2ban-client.1
 man/fail2ban-client.h2m
+man/fail2ban-python.1
+man/fail2ban-python.h2m
 man/fail2ban-regex.1
 man/fail2ban-regex.h2m
 man/fail2ban-server.1

README.md

@@ -2,53 +2,55 @@
                         / _|__ _(_) |_  ) |__  __ _ _ _  
                        |  _/ _` | | |/ /| '_ \/ _` | ' \ 
                        |_| \__,_|_|_/___|_.__/\__,_|_||_|
-                       v0.10.2                 2018/01/18
+                       v0.10.3.dev1            20??/??/??
 
 ## Fail2Ban: ban hosts that cause multiple authentication errors
 
-Fail2Ban scans log files like `/var/log/auth.log` and bans IP addresses having
+Fail2Ban scans log files like `/var/log/auth.log` and bans IP addresses conducting
 too many failed login attempts. It does this by updating system firewall rules
 to reject new connections from those IP addresses, for a configurable amount
 of time. Fail2Ban comes out-of-the-box ready to read many standard log files,
-such as those for sshd and Apache, and is easy to configure to read any log
+such as those for sshd and Apache, and is easily configured to read any log
-file you choose, for any error you choose.
+file of your choosing, for any error you wish.
 
-Though Fail2Ban is able to reduce the rate of incorrect authentications
+Though Fail2Ban is able to reduce the rate of incorrect authentication
-attempts, it cannot eliminate the risk that weak authentication presents.
+attempts, it cannot eliminate the risk presented by weak authentication.
-Configure services to use only two factor or public/private authentication
+Set up services to use only two factor, or public/private authentication
 mechanisms if you really want to protect services.
      
 <img src="http://www.worldipv6launch.org/wp-content/themes/ipv6/downloads/World_IPv6_launch_logo.svg" height="52pt"/> | Since v0.10 fail2ban supports the matching of the IPv6 addresses.
 ------|------
 
-This README is a quick introduction to Fail2ban. More documentation, FAQ, HOWTOs
+This README is a quick introduction to Fail2Ban. More documentation, FAQ, and HOWTOs
-are available in fail2ban(1) manpage, [Wiki](https://github.com/fail2ban/fail2ban/wiki)
+to be found on fail2ban(1) manpage, [Wiki](https://github.com/fail2ban/fail2ban/wiki)
-and on the website http://www.fail2ban.org
+and the website: https://www.fail2ban.org
 
 Installation:
 -------------
 
-**It is possible that Fail2ban is already packaged for your distribution.  In
+**It is possible that Fail2Ban is already packaged for your distribution.  In
-this case, you should use it instead.**
+this case, you should use that instead.**
 
 Required:
-- [Python2 >= 2.6 or Python >= 3.2](http://www.python.org) or [PyPy](http://pypy.org)
+- [Python2 >= 2.6 or Python >= 3.2](https://www.python.org) or [PyPy](https://pypy.org)
 
 Optional:
-- [pyinotify >= 0.8.3](https://github.com/seb-m/pyinotify)
+- [pyinotify >= 0.8.3](https://github.com/seb-m/pyinotify), may require:
-  - Linux >= 2.6.13
+  * Linux >= 2.6.13
 - [gamin >= 0.0.21](http://www.gnome.org/~veillard/gamin)
-- [systemd >= 204](http://www.freedesktop.org/wiki/Software/systemd)
+- [systemd >= 204](http://www.freedesktop.org/wiki/Software/systemd) and python bindings:
+  * [python-systemd package](https://www.freedesktop.org/software/systemd/python-systemd/index.html)
 - [dnspython](http://www.dnspython.org/)
 
-To install, just do:
 
-    tar xvfj fail2ban-0.10.2.tar.bz2
+To install:
-    cd fail2ban-0.10.2
+
+    tar xvfj fail2ban-0.10.3.tar.bz2
+    cd fail2ban-0.10.3
     python setup.py install
 
 This will install Fail2Ban into the python library directory. The executable
-scripts are placed into `/usr/bin`, and configuration under `/etc/fail2ban`.
+scripts are placed into `/usr/bin`, and configuration in `/etc/fail2ban`.
 
 Fail2Ban should be correctly installed now. Just type:
 
@@ -90,7 +92,7 @@ Contact:
 See [CONTRIBUTING.md](https://github.com/fail2ban/fail2ban/blob/master/CONTRIBUTING.md)
 
 ### You just appreciate this program:
-send kudos to the original author ([Cyril Jaquier](mailto:cyril.jaquier@fail2ban.org))
+Send kudos to the original author ([Cyril Jaquier](mailto:cyril.jaquier@fail2ban.org))
 or *better* to the [mailing list](https://lists.sourceforge.net/lists/listinfo/fail2ban-users)
 since Fail2Ban is "community-driven" for years now.
 

bin/fail2ban-server

@@ -22,7 +22,7 @@
 Fail2Ban reads log file that contains password failure report
 and bans the corresponding IP addresses using firewall rules.
 
-This tools starts/stops fail2ban server or does client/server communication,
+This tool starts/stops fail2ban server or does client/server communication
 to change/read parameters of the server or jails.
 
 """

bin/fail2ban-testcases

@@ -31,7 +31,7 @@ import time
 import unittest
 
 # Check if local fail2ban module exists, and use if it exists by
-# modifying the path. This is such that tests can be used in dev
+# modifying the path. This is done so that tests can be used in dev
 # environment.
 if os.path.exists("fail2ban/__init__.py"):
 	sys.path.insert(0, ".")

config/action.d/abuseipdb.conf

@@ -48,13 +48,13 @@
 [Definition]
 
 # Option:  actionstart
-# Notes.:  command executed once at the start of Fail2Ban.
+# Notes.:  command executed on demand at the first ban (or at the start of Fail2Ban if actionstart_on_demand is set to false).
 # Values:  CMD
 #
 actionstart =
 
 # Option:  actionstop
-# Notes.:  command executed once at the end of Fail2Ban
+# Notes.:  command executed at the stop of jail (or at the end of Fail2Ban)
 # Values:  CMD
 #
 actionstop =
@@ -86,7 +86,7 @@ actioncheck =
 # Tags:    See jail.conf(5) man page
 # Values:  CMD
 #
-actionban = curl --fail --ciphers ecdhe_ecdsa_aes_256_sha --data 'key=<abuseipdb_apikey>' --data-urlencode 'comment=<matches>' --data 'ip=<ip>' --data 'category=<abuseipdb_category>' "https://www.abuseipdb.com/report/json"
+actionban = lgm=$(printf '%%s\n...' "<matches>"); curl --fail --tlsv1.1 --data "key=<abuseipdb_apikey>" --data-urlencode "comment=$lgm" --data "ip=<ip>" --data "category=<abuseipdb_category>" "https://www.abuseipdb.com/report/json"
 
 # Option:  actionunban
 # Notes.:  command executed when unbanning an IP. Take care that the

config/action.d/badips.py

@@ -18,20 +18,22 @@
 # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
 
 import sys
-if sys.version_info < (2, 7):
+if sys.version_info < (2, 7): # pragma: no cover
 	raise ImportError("badips.py action requires Python >= 2.7")
 import json
 import threading
 import logging
-if sys.version_info >= (3, ):
+if sys.version_info >= (3, ): # pragma: 2.x no cover
 	from urllib.request import Request, urlopen
 	from urllib.parse import urlencode
 	from urllib.error import HTTPError
-else:
+else: # pragma: 3.x no cover
 	from urllib2 import Request, urlopen, HTTPError
 	from urllib import urlencode
 
 from fail2ban.server.actions import ActionBase
+from fail2ban.helpers import str2LogLevel
+
 
 
 class BadIPsAction(ActionBase): # pragma: no cover - may be unavailable
@@ -70,6 +72,9 @@ class BadIPsAction(ActionBase): # pragma: no cover - may be unavailable
 	updateperiod : int, optional
 		Time in seconds between updating bad IPs blacklist.
 		Default 900 (15 minutes)
+	loglevel : int/str, optional
+		Log level of the message when an IP is (un)banned.
+		Default `DEBUG`.
 	agent : str, optional
 		User agent transmitted to server.
 		Default `Fail2Ban/ver.`
@@ -81,12 +86,12 @@ class BadIPsAction(ActionBase): # pragma: no cover - may be unavailable
 	"""
 
 	TIMEOUT = 10
-	_badips = "http://www.badips.com"
+	_badips = "https://www.badips.com"
 	def _Request(self, url, **argv):
 		return Request(url, headers={'User-Agent': self.agent}, **argv)
 
 	def __init__(self, jail, name, category, score=3, age="24h", key=None,
-		banaction=None, bancategory=None, bankey=None, updateperiod=900, agent="Fail2Ban", 
+		banaction=None, bancategory=None, bankey=None, updateperiod=900, loglevel='DEBUG', agent="Fail2Ban", 
 		timeout=TIMEOUT):
 		super(BadIPsAction, self).__init__(jail, name)
 
@@ -99,6 +104,7 @@ class BadIPsAction(ActionBase): # pragma: no cover - may be unavailable
 		self.banaction = banaction
 		self.bancategory = bancategory or category
 		self.bankey = bankey
+		self.loglevel = str2LogLevel(loglevel)
 		self.updateperiod = updateperiod
 
 		self._bannedips = set()
@@ -114,6 +120,15 @@ class BadIPsAction(ActionBase): # pragma: no cover - may be unavailable
 		except Exception as e: # pragma: no cover
 			return False, e
 
+	def logError(self, response, what=''): # pragma: no cover - sporadical (502: Bad Gateway, etc)
+		messages = {}
+		try:
+			messages = json.loads(response.read().decode('utf-8'))
+		except:
+			pass
+		self._logSys.error(
+			"%s. badips.com response: '%s'", what,
+				messages.get('err', 'Unknown'))
 
 	def getCategories(self, incParents=False):
 		"""Get badips.com categories.
@@ -133,11 +148,8 @@ class BadIPsAction(ActionBase): # pragma: no cover - may be unavailable
 		try:
 			response = urlopen(
 				self._Request("/".join([self._badips, "get", "categories"])), timeout=self.timeout)
-		except HTTPError as response:
+		except HTTPError as response: # pragma: no cover
-			messages = json.loads(response.read().decode('utf-8'))
+			self.logError(response, "Failed to fetch categories")
-			self._logSys.error(
-				"Failed to fetch categories. badips.com response: '%s'",
-				messages['err'])
 			raise
 		else:
 			response_json = json.loads(response.read().decode('utf-8'))
@@ -186,12 +198,10 @@ class BadIPsAction(ActionBase): # pragma: no cover - may be unavailable
 				urlencode({'age': age})])
 			if key:
 				url = "&".join([url, urlencode({'key': key})])
+			self._logSys.debug('badips.com: get list, url: %r', url)
 			response = urlopen(self._Request(url), timeout=self.timeout)
-		except HTTPError as response:
+		except HTTPError as response: # pragma: no cover
-			messages = json.loads(response.read().decode('utf-8'))
+			self.logError(response, "Failed to fetch bad IP list")
-			self._logSys.error(
-				"Failed to fetch bad IP list. badips.com response: '%s'",
-				messages['err'])
 			raise
 		else:
 			return set(response.read().decode('utf-8').split())
@@ -219,7 +229,7 @@ class BadIPsAction(ActionBase): # pragma: no cover - may be unavailable
 
 	@bancategory.setter
 	def bancategory(self, bancategory):
-		if bancategory not in self.getCategories(incParents=True):
+		if bancategory != "any" and bancategory not in self.getCategories(incParents=True):
 			self._logSys.error("Category name '%s' not valid. "
 				"see badips.com for list of valid categories",
 				bancategory)
@@ -285,7 +295,7 @@ class BadIPsAction(ActionBase): # pragma: no cover - may be unavailable
 					exc_info=self._logSys.getEffectiveLevel()<=logging.DEBUG)
 			else:
 				self._bannedips.add(ip)
-				self._logSys.info(
+				self._logSys.log(self.loglevel,
 					"Banned IP %s for jail '%s' with action '%s'",
 					ip, self._jail.name, self.banaction)
 
@@ -300,12 +310,12 @@ class BadIPsAction(ActionBase): # pragma: no cover - may be unavailable
 					'ipjailmatches': "",
 				})
 			except Exception as e:
-				self._logSys.info(
+				self._logSys.error(
 					"Error unbanning IP %s for jail '%s' with action '%s': %s",
 					ip, self._jail.name, self.banaction, e,
 					exc_info=self._logSys.getEffectiveLevel()<=logging.DEBUG)
 			else:
-				self._logSys.info(
+				self._logSys.log(self.loglevel,
 					"Unbanned IP %s for jail '%s' with action '%s'",
 					ip, self._jail.name, self.banaction)
 			finally:
@@ -333,13 +343,16 @@ class BadIPsAction(ActionBase): # pragma: no cover - may be unavailable
 				ips = self.getList(
 					self.bancategory, self.score, self.age, self.bankey)
 				# Remove old IPs no longer listed
-				self._unbanIPs(self._bannedips - ips)
+				s = self._bannedips - ips
+				m = len(s)
+				self._unbanIPs(s)
 				# Add new IPs which are now listed
-				self._banIPs(ips - self._bannedips)
+				s = ips - self._bannedips
-
+				p = len(s)
-				self._logSys.info(
+				self._banIPs(s)
-					"Updated IPs for jail '%s'. Update again in %i seconds",
+				self._logSys.log(self.loglevel,
-					self._jail.name, self.updateperiod)
+					"Updated IPs for jail '%s' (-%d/+%d). Update again in %i seconds",
+					self._jail.name, m, p, self.updateperiod)
 			finally:
 				self._timer = threading.Timer(self.updateperiod, self.update)
 				self._timer.start()
@@ -368,19 +381,17 @@ class BadIPsAction(ActionBase): # pragma: no cover - may be unavailable
 			Any issues with badips.com request.
 		"""
 		try:
-			url = "/".join([self._badips, "add", self.category, aInfo['ip']])
+			url = "/".join([self._badips, "add", self.category, str(aInfo['ip'])])
 			if self.key:
 				url = "?".join([url, urlencode({'key': self.key})])
+			self._logSys.debug('badips.com: ban, url: %r', url)
 			response = urlopen(self._Request(url), timeout=self.timeout)
-		except HTTPError as response:
+		except HTTPError as response: # pragma: no cover
-			messages = json.loads(response.read().decode('utf-8'))
+			self.logError(response, "Failed to ban")
-			self._logSys.error(
-				"Response from badips.com report: '%s'",
-				messages['err'])
 			raise
 		else:
 			messages = json.loads(response.read().decode('utf-8'))
-			self._logSys.info(
+			self._logSys.debug(
 				"Response from badips.com report: '%s'",
 				messages['suc'])
 

config/action.d/blocklist_de.conf

@@ -31,13 +31,13 @@
 [Definition]
 
 # Option:  actionstart
-# Notes.:  command executed once at the start of Fail2Ban.
+# Notes.:  command executed on demand at the first ban (or at the start of Fail2Ban if actionstart_on_demand is set to false).
 # Values:  CMD
 #
 actionstart = 
 
 # Option:  actionstop
-# Notes.:  command executed once at the end of Fail2Ban
+# Notes.:  command executed at the stop of jail (or at the end of Fail2Ban)
 # Values:  CMD
 #
 actionstop =
@@ -54,7 +54,7 @@ actioncheck =
 # Tags:    See jail.conf(5) man page
 # Values:  CMD
 #
-actionban = curl --fail --data-urlencode 'server=<email>' --data 'apikey=<apikey>' --data 'service=<service>' --data 'ip=<ip>' --data-urlencode 'logs=<matches>' --data 'format=text' --user-agent "<agent>" "https://www.blocklist.de/en/httpreports.html"
+actionban = curl --fail --data-urlencode "server=<email>" --data "apikey=<apikey>" --data "service=<service>" --data "ip=<ip>" --data-urlencode "logs=<matches><br>" --data 'format=text' --user-agent "<agent>" "https://www.blocklist.de/en/httpreports.html"
 
 # Option:  actionunban
 # Notes.:  command executed when unbanning an IP. Take care that the
@@ -64,10 +64,8 @@ actionban = curl --fail --data-urlencode 'server=<email>' --data 'apikey=<apikey
 #
 actionunban =
 
-[Init]
-
 # Option:  email
-# Notes    server email address, as per blocklise.de account
+# Notes    server email address, as per blocklist.de account
 # Values:  STRING  Default: None
 #
 #email =

config/action.d/bsd-ipfw.conf

@@ -11,14 +11,14 @@
 [Definition]
 
 # Option:  actionstart
-# Notes.:  command executed once at the start of Fail2Ban.
+# Notes.:  command executed on demand at the first ban (or at the start of Fail2Ban if actionstart_on_demand is set to false).
 # Values:  CMD
 #
 actionstart = ipfw show | fgrep -c -m 1 -s 'table(<table>)' > /dev/null 2>&1 || ( ipfw show | awk 'BEGIN { b = <lowest_rule_num> } { if ($1 < b) {} else if ($1 == b) { b = $1 + 1 } else { e = b } } END { if (e) exit e <br> else exit b }'; num=$?; ipfw -q add $num <blocktype> <block> from table\(<table>\) to me <port>; echo $num > "<startstatefile>" )
 
 
 # Option:  actionstop
-# Notes.:  command executed once at the end of Fail2Ban
+# Notes.:  command executed at the stop of jail (or at the end of Fail2Ban)
 # Values:  CMD
 #
 actionstop =  [ ! -f <startstatefile> ] || ( read num < "<startstatefile>" <br> ipfw -q delete $num <br> rm "<startstatefile>" )
@@ -38,7 +38,7 @@ actioncheck =
 # Values:  CMD
 #
 # requires an ipfw rule like "deny ip from table(1) to me"
-actionban = e=`ipfw table <table> add <ip> 2>&1`; x=$?; [ $x -eq 0 -o "$e" = 'ipfw: setsockopt(IP_FW_TABLE_XADD): File exists' ] || { echo "$e" 1>&2; exit $x; }
+actionban = e=`ipfw table <table> add <ip> 2>&1`; x=$?; [ $x -eq 0 -o "$e" = 'ipfw: setsockopt(IP_FW_TABLE_XADD): File exists' ] || echo "$e" | grep -q "record already exists" || { echo "$e" 1>&2; exit $x; }
 
 
 # Option:  actionunban
@@ -47,7 +47,7 @@ actionban = e=`ipfw table <table> add <ip> 2>&1`; x=$?; [ $x -eq 0 -o "$e" = 'ip
 # Tags:    See jail.conf(5) man page
 # Values:  CMD
 #
-actionunban = e=`ipfw table <table> delete <ip> 2>&1`; x=$?; [ $x -eq 0 -o "$e" = 'ipfw: setsockopt(IP_FW_TABLE_XDEL): No such process' ] || { echo "$e" 1>&2; exit $x; }
+actionunban = e=`ipfw table <table> delete <ip> 2>&1`; x=$?; [ $x -eq 0 -o "$e" = 'ipfw: setsockopt(IP_FW_TABLE_XDEL): No such process' ] || echo "$e" | grep -q "record not found" || { echo "$e" 1>&2; exit $x; }
 
 [Init]
 # Option:  table

config/action.d/cloudflare.conf

@@ -15,13 +15,13 @@
 [Definition]
 
 # Option:  actionstart
-# Notes.:  command executed once at the start of Fail2Ban.
+# Notes.:  command executed on demand at the first ban (or at the start of Fail2Ban if actionstart_on_demand is set to false).
 # Values:  CMD
 #
 actionstart =
 
 # Option:  actionstop
-# Notes.:  command executed once at the end of Fail2Ban
+# Notes.:  command executed at the stop of jail (or at the end of Fail2Ban)
 # Values:  CMD
 #
 actionstop =

config/action.d/complain.conf

@@ -41,13 +41,13 @@ debug = 0
 norestored = 1
 
 # Option:  actionstart
-# Notes.:  command executed once at the start of Fail2Ban.
+# Notes.:  command executed on demand at the first ban (or at the start of Fail2Ban if actionstart_on_demand is set to false).
 # Values:  CMD
 #
 actionstart =
 
 # Option:  actionstop
-# Notes.:  command executed once at the end of Fail2Ban
+# Notes.:  command executed at the stop of jail (or at the end of Fail2Ban)
 # Values:  CMD
 #
 actionstop =

config/action.d/dshield.conf

@@ -32,13 +32,13 @@
 norestored = 1
 
 # Option:  actionstart
-# Notes.:  command executed once at the start of Fail2Ban.
+# Notes.:  command executed on demand at the first ban (or at the start of Fail2Ban if actionstart_on_demand is set to false).
 # Values:  CMD
 #
 actionstart =
 
 # Option:  actionstop
-# Notes.:  command executed once at the end of Fail2Ban
+# Notes.:  command executed at the stop of jail (or at the end of Fail2Ban)
 # Values:  CMD
 #
 actionstop = if [ -f <tmpfile>.buffer ]; then

config/action.d/dummy.conf

@@ -7,7 +7,7 @@
 [Definition]
 
 # Option:  actionstart
-# Notes.:  command executed once at the start of Fail2Ban.
+# Notes.:  command executed on demand at the first ban (or at the start of Fail2Ban if actionstart_on_demand is set to false).
 # Values:  CMD
 #
 actionstart = if [ ! -z '<target>' ]; then touch <target>; fi;
@@ -22,7 +22,7 @@ actionflush = printf %%b "-*\n" <to_target>
               echo "%(debug)s clear all"
 
 # Option:  actionstop
-# Notes.:  command executed once at the end of Fail2Ban
+# Notes.:  command executed at the stop of jail (or at the end of Fail2Ban)
 # Values:  CMD
 #
 actionstop = if [ ! -z '<target>' ]; then rm -f <target>; fi;

config/action.d/helpers-common.conf

@@ -1,16 +1,16 @@
-[DEFAULT]
+[DEFAULT]
-
+
-# Usage:
+# Usage:
-#   _grep_logs_args = 'test'
+#   _grep_logs_args = 'test'
-#   (printf %%b "Log-excerpt contains 'test':\n"; %(_grep_logs)s; printf %%b "Log-excerpt contains 'test':\n") | mail ...
+#   (printf %%b "Log-excerpt contains 'test':\n"; %(_grep_logs)s; printf %%b "Log-excerpt contains 'test':\n") | mail ...
-#
+#
-_grep_logs = logpath="<logpath>"; grep <grepopts> -E %(_grep_logs_args)s $logpath | <greplimit>
+_grep_logs = logpath="<logpath>"; grep <grepopts> -E %(_grep_logs_args)s $logpath | <greplimit>
-_grep_logs_args = "(^|[^0-9a-fA-F:])$(echo '<ip>' | sed 's/\./\\./g')([^0-9a-fA-F:]|$)"
+_grep_logs_args = "(^|[^0-9a-fA-F:])$(echo '<ip>' | sed 's/\./\\./g')([^0-9a-fA-F:]|$)"
-
+
-# Used for actions, that should not by executed if ticket was restored:
+# Used for actions, that should not by executed if ticket was restored:
-_bypass_if_restored = if [ '<restored>' = '1' ]; then exit 0; fi;
+_bypass_if_restored = if [ '<restored>' = '1' ]; then exit 0; fi;
-
+
-[Init]
+[Init]
-greplimit = tail -n <grepmax>
+greplimit = tail -n <grepmax>
-grepmax = 1000
+grepmax = 1000
-grepopts = -m <grepmax>
+grepopts = -m <grepmax>

config/action.d/hostsdeny.conf

@@ -8,13 +8,13 @@
 [Definition]
 
 # Option:  actionstart
-# Notes.:  command executed once at the start of Fail2Ban.
+# Notes.:  command executed on demand at the first ban (or at the start of Fail2Ban if actionstart_on_demand is set to false).
 # Values:  CMD
 #
 actionstart = 
 
 # Option:  actionstop
-# Notes.:  command executed once at the end of Fail2Ban
+# Notes.:  command executed at the stop of jail (or at the end of Fail2Ban)
 # Values:  CMD
 #
 actionstop = 
@@ -31,7 +31,7 @@ actioncheck =
 # Tags:    See jail.conf(5) man page
 # Values:  CMD
 #
-actionban = IP=<ip> && printf %%b "<daemon_list>: $IP\n" >> <file>
+actionban = printf %%b "<daemon_list>: <ip_value>\n" >> <file>
 
 # Option:  actionunban
 # Notes.:  command executed when unbanning an IP. Take care that the
@@ -39,7 +39,7 @@ actionban = IP=<ip> && printf %%b "<daemon_list>: $IP\n" >> <file>
 # Tags:    See jail.conf(5) man page
 # Values:  CMD
 #
-actionunban = IP=$(echo <ip> | sed 's/\./\\./g') && sed -i "/^<daemon_list>: $IP$/d" <file>
+actionunban = IP=$(echo "<ip_value>" | sed 's/[][\.]/\\\0/g') && sed -i "/^<daemon_list>: $IP$/d" <file>
 
 [Init]
 
@@ -54,3 +54,9 @@ file = /etc/hosts.deny
 #          for hosts.deny/hosts_access. Default is all services.
 # Values:  STR  Default: ALL
 daemon_list = ALL
+
+# internal variable IP (to differentiate the IPv4 and IPv6 syntax, where it is enclosed in brackets):
+ip_value = <ip>
+
+[Init?family=inet6]
+ip_value = [<ip>]

config/action.d/ipfilter.conf

@@ -9,7 +9,7 @@
 [Definition]
 
 # Option:  actionstart
-# Notes.:  command executed once at the start of Fail2Ban.
+# Notes.:  command executed on demand at the first ban (or at the start of Fail2Ban if actionstart_on_demand is set to false).
 # Values:  CMD
 #
 # enable IPF if not already enabled
@@ -17,7 +17,7 @@ actionstart = /sbin/ipf -E
 
 
 # Option:  actionstop
-# Notes.:  command executed once at the end of Fail2Ban
+# Notes.:  command executed at the stop of jail (or at the end of Fail2Ban)
 # Values:  CMD
 #
 # don't disable IPF with "/sbin/ipf -D", there may be other filters in use

config/action.d/ipfw.conf

@@ -8,14 +8,14 @@
 [Definition]
 
 # Option:  actionstart
-# Notes.:  command executed once at the start of Fail2Ban.
+# Notes.:  command executed on demand at the first ban (or at the start of Fail2Ban if actionstart_on_demand is set to false).
 # Values:  CMD
 #
 actionstart = 
 
 
 # Option:  actionstop
-# Notes.:  command executed once at the end of Fail2Ban
+# Notes.:  command executed at the stop of jail (or at the end of Fail2Ban)
 # Values:  CMD
 #
 actionstop = 

config/action.d/iptables-allports.conf

@@ -14,7 +14,7 @@ before = iptables-common.conf
 [Definition]
 
 # Option:  actionstart
-# Notes.:  command executed once at the start of Fail2Ban.
+# Notes.:  command executed on demand at the first ban (or at the start of Fail2Ban if actionstart_on_demand is set to false).
 # Values:  CMD
 #
 actionstart = <iptables> -N f2b-<name>
@@ -22,7 +22,7 @@ actionstart = <iptables> -N f2b-<name>
               <iptables> -I <chain> -p <protocol> -j f2b-<name>
 
 # Option:  actionstop
-# Notes.:  command executed once at the end of Fail2Ban
+# Notes.:  command executed at the stop of jail (or at the end of Fail2Ban)
 # Values:  CMD
 #
 actionstop = <iptables> -D <chain> -p <protocol> -j f2b-<name>

config/action.d/iptables-ipset-proto4.conf

@@ -24,7 +24,7 @@ before = iptables-common.conf
 [Definition]
 
 # Option:  actionstart
-# Notes.:  command executed once at the start of Fail2Ban.
+# Notes.:  command executed on demand at the first ban (or at the start of Fail2Ban if actionstart_on_demand is set to false).
 # Values:  CMD
 #
 actionstart = ipset --create f2b-<name> iphash
@@ -38,7 +38,7 @@ actionstart = ipset --create f2b-<name> iphash
 actionflush = ipset --flush f2b-<name>
 
 # Option:  actionstop
-# Notes.:  command executed once at the end of Fail2Ban
+# Notes.:  command executed at the stop of jail (or at the end of Fail2Ban)
 # Values:  CMD
 #
 actionstop = <iptables> -D <chain> -p <protocol> -m multiport --dports <port> -m set --match-set f2b-<name> src -j <blocktype>

config/action.d/iptables-ipset-proto6-allports.conf

@@ -23,7 +23,7 @@ before = iptables-common.conf
 [Definition]
 
 # Option:  actionstart
-# Notes.:  command executed once at the start of Fail2Ban.
+# Notes.:  command executed on demand at the first ban (or at the start of Fail2Ban if actionstart_on_demand is set to false).
 # Values:  CMD
 #
 actionstart = ipset create <ipmset> hash:ip timeout <bantime><familyopt>
@@ -36,7 +36,7 @@ actionstart = ipset create <ipmset> hash:ip timeout <bantime><familyopt>
 actionflush = ipset flush <ipmset>
 
 # Option:  actionstop
-# Notes.:  command executed once at the end of Fail2Ban
+# Notes.:  command executed at the stop of jail (or at the end of Fail2Ban)
 # Values:  CMD
 #
 actionstop = <iptables> -D <chain> -m set --match-set <ipmset> src -j <blocktype>

config/action.d/iptables-ipset-proto6.conf

@@ -23,7 +23,7 @@ before = iptables-common.conf
 [Definition]
 
 # Option:  actionstart
-# Notes.:  command executed once at the start of Fail2Ban.
+# Notes.:  command executed on demand at the first ban (or at the start of Fail2Ban if actionstart_on_demand is set to false).
 # Values:  CMD
 #
 actionstart = ipset create <ipmset> hash:ip timeout <bantime><familyopt>
@@ -36,7 +36,7 @@ actionstart = ipset create <ipmset> hash:ip timeout <bantime><familyopt>
 actionflush = ipset flush <ipmset>
 
 # Option:  actionstop
-# Notes.:  command executed once at the end of Fail2Ban
+# Notes.:  command executed at the stop of jail (or at the end of Fail2Ban)
 # Values:  CMD
 #
 actionstop = <iptables> -D <chain> -p <protocol> -m multiport --dports <port> -m set --match-set <ipmset> src -j <blocktype>

config/action.d/iptables-multiport-log.conf

@@ -16,7 +16,7 @@ before = iptables-common.conf
 [Definition]
 
 # Option:  actionstart
-# Notes.:  command executed once at the start of Fail2Ban.
+# Notes.:  command executed on demand at the first ban (or at the start of Fail2Ban if actionstart_on_demand is set to false).
 # Values:  CMD
 #
 actionstart = <iptables> -N f2b-<name>
@@ -34,7 +34,7 @@ actionflush = <iptables> -F f2b-<name>
               <iptables> -F f2b-<name>-log
 
 # Option:  actionstop
-# Notes.:  command executed once at the end of Fail2Ban
+# Notes.:  command executed at the stop of jail (or at the end of Fail2Ban)
 # Values:  CMD
 #
 actionstop = <iptables> -D <chain> -p <protocol> -m multiport --dports <port> -j f2b-<name>

config/action.d/iptables-multiport.conf

@@ -11,7 +11,7 @@ before = iptables-common.conf
 [Definition]
 
 # Option:  actionstart
-# Notes.:  command executed once at the start of Fail2Ban.
+# Notes.:  command executed on demand at the first ban (or at the start of Fail2Ban if actionstart_on_demand is set to false).
 # Values:  CMD
 #
 actionstart = <iptables> -N f2b-<name>
@@ -19,7 +19,7 @@ actionstart = <iptables> -N f2b-<name>
               <iptables> -I <chain> -p <protocol> -m multiport --dports <port> -j f2b-<name>
 
 # Option:  actionstop
-# Notes.:  command executed once at the end of Fail2Ban
+# Notes.:  command executed at the stop of jail (or at the end of Fail2Ban)
 # Values:  CMD
 #
 actionstop = <iptables> -D <chain> -p <protocol> -m multiport --dports <port> -j f2b-<name>

config/action.d/iptables-new.conf

@@ -13,7 +13,7 @@ before = iptables-common.conf
 [Definition]
 
 # Option:  actionstart
-# Notes.:  command executed once at the start of Fail2Ban.
+# Notes.:  command executed on demand at the first ban (or at the start of Fail2Ban if actionstart_on_demand is set to false).
 # Values:  CMD
 #
 actionstart = <iptables> -N f2b-<name>
@@ -21,7 +21,7 @@ actionstart = <iptables> -N f2b-<name>
               <iptables> -I <chain> -m state --state NEW -p <protocol> --dport <port> -j f2b-<name>
 
 # Option:  actionstop
-# Notes.:  command executed once at the end of Fail2Ban
+# Notes.:  command executed at the stop of jail (or at the end of Fail2Ban)
 # Values:  CMD
 #
 actionstop = <iptables> -D <chain> -m state --state NEW -p <protocol> --dport <port> -j f2b-<name>

config/action.d/iptables-xt_recent-echo.conf

@@ -12,7 +12,7 @@ before = iptables-common.conf
 [Definition]
 
 # Option:  actionstart
-# Notes.:  command executed once at the start of Fail2Ban.
+# Notes.:  command executed on demand at the first ban (or at the start of Fail2Ban if actionstart_on_demand is set to false).
 # Values:  CMD
 #
 # Changing iptables rules requires root privileges. If fail2ban is
@@ -42,7 +42,7 @@ actionstart = if [ `id -u` -eq 0 ];then <iptables> -I <chain> -m recent --update
 actionflush = 
 
 # Option:  actionstop
-# Notes.:  command executed once at the end of Fail2Ban
+# Notes.:  command executed at the stop of jail (or at the end of Fail2Ban)
 # Values:  CMD
 #
 actionstop = echo / > /proc/net/xt_recent/<iptname>

config/action.d/iptables.conf

@@ -11,7 +11,7 @@ before = iptables-common.conf
 [Definition]
 
 # Option:  actionstart
-# Notes.:  command executed once at the start of Fail2Ban.
+# Notes.:  command executed on demand at the first ban (or at the start of Fail2Ban if actionstart_on_demand is set to false).
 # Values:  CMD
 #
 actionstart = <iptables> -N f2b-<name>
@@ -19,7 +19,7 @@ actionstart = <iptables> -N f2b-<name>
               <iptables> -I <chain> -p <protocol> --dport <port> -j f2b-<name>
 
 # Option:  actionstop
-# Notes.:  command executed once at the end of Fail2Ban
+# Notes.:  command executed at the stop of jail (or at the end of Fail2Ban)
 # Values:  CMD
 #
 actionstop = <iptables> -D <chain> -p <protocol> --dport <port> -j f2b-<name>

config/action.d/mail-buffered.conf

@@ -10,7 +10,7 @@
 norestored = 1
 
 # Option:  actionstart
-# Notes.:  command executed once at the start of Fail2Ban.
+# Notes.:  command executed on demand at the first ban (or at the start of Fail2Ban if actionstart_on_demand is set to false).
 # Values:  CMD
 #
 actionstart = printf %%b "Hi,\n
@@ -20,7 +20,7 @@ actionstart = printf %%b "Hi,\n
               Fail2Ban"|mail -s "[Fail2Ban] <name>: started on <fq-hostname>" <dest>
 
 # Option:  actionstop
-# Notes.:  command executed once at the end of Fail2Ban
+# Notes.:  command executed at the stop of jail (or at the end of Fail2Ban)
 # Values:  CMD
 #
 actionstop = if [ -f <tmpfile> ]; then

config/action.d/mail-whois-lines.conf

@@ -15,7 +15,7 @@ before = mail-whois-common.conf
 norestored = 1
 
 # Option:  actionstart
-# Notes.:  command executed once at the start of Fail2Ban.
+# Notes.:  command executed on demand at the first ban (or at the start of Fail2Ban if actionstart_on_demand is set to false).
 # Values:  CMD
 #
 actionstart = printf %%b "Hi,\n
@@ -24,7 +24,7 @@ actionstart = printf %%b "Hi,\n
               Fail2Ban" | <mailcmd> "[Fail2Ban] <name>: started on <fq-hostname>" <dest>
 
 # Option:  actionstop
-# Notes.:  command executed once at the end of Fail2Ban
+# Notes.:  command executed at the stop of jail (or at the end of Fail2Ban)
 # Values:  CMD
 #
 actionstop = printf %%b "Hi,\n

config/action.d/mail-whois.conf

@@ -14,7 +14,7 @@ before = mail-whois-common.conf
 norestored = 1
 
 # Option:  actionstart
-# Notes.:  command executed once at the start of Fail2Ban.
+# Notes.:  command executed on demand at the first ban (or at the start of Fail2Ban if actionstart_on_demand is set to false).
 # Values:  CMD
 #
 actionstart = printf %%b "Hi,\n
@@ -23,7 +23,7 @@ actionstart = printf %%b "Hi,\n
               Fail2Ban"|mail -s "[Fail2Ban] <name>: started on <fq-hostname>" <dest>
 
 # Option:  actionstop
-# Notes.:  command executed once at the end of Fail2Ban
+# Notes.:  command executed at the stop of jail (or at the end of Fail2Ban)
 # Values:  CMD
 #
 actionstop = printf %%b "Hi,\n

config/action.d/mail.conf

@@ -10,7 +10,7 @@
 norestored = 1
 
 # Option:  actionstart
-# Notes.:  command executed once at the start of Fail2Ban.
+# Notes.:  command executed on demand at the first ban (or at the start of Fail2Ban if actionstart_on_demand is set to false).
 # Values:  CMD
 #
 actionstart = printf %%b "Hi,\n
@@ -19,7 +19,7 @@ actionstart = printf %%b "Hi,\n
               Fail2Ban"|mail -s "[Fail2Ban] <name>: started  on <fq-hostname>" <dest>
 
 # Option:  actionstop
-# Notes.:  command executed once at the end of Fail2Ban
+# Notes.:  command executed at the stop of jail (or at the end of Fail2Ban)
 # Values:  CMD
 #
 actionstop = printf %%b "Hi,\n

config/action.d/mynetwatchman.conf

@@ -28,13 +28,13 @@
 [Definition]
 
 # Option:  actionstart
-# Notes.:  command executed once at the start of Fail2Ban.
+# Notes.:  command executed on demand at the first ban (or at the start of Fail2Ban if actionstart_on_demand is set to false).
 # Values:  CMD
 #
 actionstart =
 
 # Option:  actionstop
-# Notes.:  command executed once at the end of Fail2Ban
+# Notes.:  command executed at the stop of jail (or at the end of Fail2Ban)
 # Values:  CMD
 #
 actionstop =

config/action.d/nftables-common.conf

@@ -25,7 +25,7 @@ after = nftables-common.local
 nftables_mode = <protocol> dport \{ <port> \}
 
 # Option:  actionstart
-# Notes.:  command executed once at the start of Fail2Ban.
+# Notes.:  command executed on demand at the first ban (or at the start of Fail2Ban if actionstart_on_demand is set to false).
 # Values:  CMD
 #
 actionstart = <nftables> add set <nftables_family> <nftables_table> <set_name> \{ type <nftables_type>\; \}
@@ -35,7 +35,7 @@ _nft_list = <nftables> --handle --numeric list chain <nftables_family> <nftables
 _nft_get_handle_id = grep -m1 '<address_family> saddr @<set_name> <blocktype> # handle' | grep -oe ' handle [0-9]*'
 
 # Option:  actionstop
-# Notes.:  command executed once at the end of Fail2Ban
+# Notes.:  command executed at the stop of jail (or at the end of Fail2Ban)
 # Values:  CMD
 #
 actionstop = HANDLE_ID=$(%(_nft_list)s | %(_nft_get_handle_id)s)

config/action.d/npf.conf

@@ -9,7 +9,7 @@
 [Definition]
 
 # Option:  actionstart
-# Notes.:  command executed once at the start of Fail2Ban.
+# Notes.:  command executed on demand at the first ban (or at the start of Fail2Ban if actionstart_on_demand is set to false).
 # Values:  CMD
 #
 # we don't enable NPF automatically, as it will be enabled elsewhere
@@ -17,7 +17,7 @@ actionstart =
 
 
 # Option:  actionstop
-# Notes.:  command executed once at the end of Fail2Ban
+# Notes.:  command executed at the stop of jail (or at the end of Fail2Ban)
 # Values:  CMD
 #
 # we don't disable NPF automatically either

config/action.d/nsupdate.conf

@@ -42,14 +42,14 @@
 [Definition]
 
 # Option:  actionstart
-# Notes.:  command executed once at the start of Fail2Ban.
+# Notes.:  command executed on demand at the first ban (or at the start of Fail2Ban if actionstart_on_demand is set to false).
 # Values:  CMD
 #
 actionstart =
 
 
 # Option:  actionstop
-# Notes.:  command executed once at the end of Fail2Ban
+# Notes.:  command executed at the stop of jail (or at the end of Fail2Ban)
 # Values:  CMD
 #
 actionstop =

config/action.d/osx-ipfw.conf

@@ -9,14 +9,14 @@
 [Definition]
 
 # Option:  actionstart
-# Notes.:  command executed once at the start of Fail2Ban.
+# Notes.:  command executed on demand at the first ban (or at the start of Fail2Ban if actionstart_on_demand is set to false).
 # Values:  CMD
 #
 actionstart = 
 
 
 # Option:  actionstop
-# Notes.:  command executed once at the end of Fail2Ban
+# Notes.:  command executed at the stop of jail (or at the end of Fail2Ban)
 # Values:  CMD
 #
 actionstop = 

config/action.d/pf.conf

@@ -10,7 +10,7 @@
 [Definition]
 
 # Option:  actionstart
-# Notes.:  command executed once at the start of Fail2Ban.
+# Notes.:  command executed on demand at the first ban (or at the start of Fail2Ban if actionstart_on_demand is set to false).
 # Values:  CMD
 #
 # we don't enable PF automatically; to enable run pfctl -e 
@@ -35,7 +35,7 @@ actionstart = echo "table <<tablename>-<name>> persist counters" | <pfctl> -f-
 actionstart_on_demand = false
 
 # Option:  actionstop
-# Notes.:  command executed once at the end of Fail2Ban
+# Notes.:  command executed at the stop of jail (or at the end of Fail2Ban)
 # Values:  CMD
 #
 # we only disable PF rules we've installed prior

config/action.d/sendmail-buffered.conf

@@ -14,7 +14,7 @@ before = sendmail-common.conf
 norestored = 1
 
 # Option:  actionstart
-# Notes.:  command executed once at the start of Fail2Ban.
+# Notes.:  command executed on demand at the first ban (or at the start of Fail2Ban if actionstart_on_demand is set to false).
 # Values:  CMD
 #
 actionstart = printf %%b "Subject: [Fail2Ban] <name>: started on <fq-hostname>
@@ -27,7 +27,7 @@ actionstart = printf %%b "Subject: [Fail2Ban] <name>: started on <fq-hostname>
               Fail2Ban" | /usr/sbin/sendmail -f <sender> <dest>
 
 # Option:  actionstop
-# Notes.:  command executed once at the end of Fail2Ban
+# Notes.:  command executed at the stop of jail (or at the end of Fail2Ban)
 # Values:  CMD
 #
 actionstop = if [ -f <tmpfile> ]; then

config/action.d/sendmail-common.conf

@@ -11,7 +11,7 @@ after = sendmail-common.local
 [Definition]
 
 # Option:  actionstart
-# Notes.:  command executed once at the start of Fail2Ban.
+# Notes.:  command executed on demand at the first ban (or at the start of Fail2Ban if actionstart_on_demand is set to false).
 # Values:  CMD
 #
 actionstart = printf %%b "Subject: [Fail2Ban] <name>: started on <fq-hostname>
@@ -24,7 +24,7 @@ actionstart = printf %%b "Subject: [Fail2Ban] <name>: started on <fq-hostname>
               Fail2Ban" | /usr/sbin/sendmail -f <sender> <dest>
 
 # Option:  actionstop
-# Notes.:  command executed once at the end of Fail2Ban
+# Notes.:  command executed at the stop of jail (or at the end of Fail2Ban)
 # Values:  CMD
 #
 actionstop = printf %%b "Subject: [Fail2Ban] <name>: stopped on <fq-hostname>

config/action.d/shorewall-ipset-proto6.conf

@@ -47,7 +47,7 @@
 [Definition]
 
 # Option:  actionstart
-# Notes.:  command executed once at the start of Fail2Ban.
+# Notes.:  command executed on demand at the first ban (or at the start of Fail2Ban if actionstart_on_demand is set to false).
 # Values:  CMD
 #
 actionstart = if ! ipset -quiet -name list f2b-<name> >/dev/null;
@@ -55,7 +55,7 @@ actionstart = if ! ipset -quiet -name list f2b-<name> >/dev/null;
               fi
 
 # Option:  actionstop
-# Notes.:  command executed once at the end of Fail2Ban
+# Notes.:  command executed at the stop of jail (or at the end of Fail2Ban)
 # Values:  CMD
 #
 actionstop = ipset flush f2b-<name>

config/action.d/shorewall.conf

@@ -9,7 +9,7 @@
 # connections. So if the attempter goes on trying using the same connection
 # he could even log in. In order to get the same behavior of the iptable
 # action (so that the ban is immediate) the /etc/shorewall/shorewall.conf
-# file should me modified with "BLACKLISTNEWONLY=No". Note that as of
+# file should be modified with "BLACKLISTNEWONLY=No". Note that as of
 # Shorewall 4.5.13 BLACKLISTNEWONLY is deprecated; however the equivalent
 # of BLACKLISTNEWONLY=No can now be achieved by setting BLACKLIST="ALL".
 # 
@@ -17,13 +17,13 @@
 [Definition]
 
 # Option:  actionstart
-# Notes.:  command executed once at the start of Fail2Ban.
+# Notes.:  command executed on demand at the first ban (or at the start of Fail2Ban if actionstart_on_demand is set to false).
 # Values:  CMD
 #
 actionstart = 
 
 # Option:  actionstop
-# Notes.:  command executed once at the end of Fail2Ban
+# Notes.:  command executed at the stop of jail (or at the end of Fail2Ban)
 # Values:  CMD
 #
 actionstop = 

config/action.d/symbiosis-blacklist-allports.conf

@@ -10,13 +10,13 @@ before = iptables-common.conf
 [Definition]
 
 # Option:  actionstart
-# Notes.:  command executed once at the start of Fail2Ban.
+# Notes.:  command executed on demand at the first ban (or at the start of Fail2Ban if actionstart_on_demand is set to false).
 # Values:  CMD
 #
 actionstart =
 
 # Option:  actionstop
-# Notes.:  command executed once at the end of Fail2Ban
+# Notes.:  command executed at the stop of jail (or at the end of Fail2Ban)
 # Values:  CMD
 #
 actionstop =

config/filter.d/apache-auth.conf

@@ -15,15 +15,16 @@ prefregex = ^%(_apache_error_client)s (?:AH\d+: )?<F-CONTENT>.+</F-CONTENT>$
 auth_type = ([A-Z]\w+: )?
 
 failregex = ^client (?:denied by server configuration|used wrong authentication scheme)\b
-            ^user <F-USER>(?:\S*|.*?)</F-USER> (?:auth(?:oriz|entic)ation failure|not found|denied by provider)\b
+            ^user (?!`)<F-USER>(?:\S*|.*?)</F-USER> (?:auth(?:oriz|entic)ation failure|not found|denied by provider)\b
             ^Authorization of user <F-USER>(?:\S*|.*?)</F-USER> to access .*? failed\b
             ^%(auth_type)suser <F-USER>(?:\S*|.*?)</F-USER>: password mismatch\b
-            ^%(auth_type)suser `<F-USER>(?:[^']*|.*?)</F-USER>' in realm `.+' (not found|denied by provider)\b
+            ^%(auth_type)suser `<F-USER>(?:[^']*|.*?)</F-USER>' in realm `.+' (auth(?:oriz|entic)ation failure|not found|denied by provider)\b
             ^%(auth_type)sinvalid nonce .* received - length is not\b
             ^%(auth_type)srealm mismatch - got `(?:[^']*|.*?)' but expected\b
             ^%(auth_type)sunknown algorithm `(?:[^']*|.*?)' received\b
             ^invalid qop `(?:[^']*|.*?)' received\b
             ^%(auth_type)sinvalid nonce .*? received - user attempted time travel\b
+            ^(?:No h|H)ostname \S+ provided via SNI(?:, but no hostname provided| and hostname \S+ provided| for a name based virtual host)\b
 
 ignoreregex = 
 

config/filter.d/apache-noscript.conf

@@ -17,8 +17,13 @@ before = apache-common.conf
 
 [Definition]
 
-failregex = ^%(_apache_error_client)s ((AH001(28|30): )?File does not exist|(AH01264: )?script not found or unable to stat): /\S*(php([45]|[.-]cgi)?|\.asp|\.exe|\.pl)(, referer: \S+)?\s*$
+script = /\S*(?:php(?:[45]|[.-]cgi)?|\.asp|\.exe|\.pl)
-            ^%(_apache_error_client)s script '/\S*(php([45]|[.-]cgi)?|\.asp|\.exe|\.pl)\S*' not found or unable to stat(, referer: \S+)?\s*$
+
+prefregex = ^%(_apache_error_client)s (?:AH0(?:01(?:28|30)|1(?:264|071)): )?(?:(?:[Ff]ile|script|[Gg]ot) )<F-CONTENT>.+</F-CONTENT>$
+
+failregex = ^(?:does not exist|not found or unable to stat): <script>\b
+            ^'<script>\S*' not found or unable to stat
+            ^error '[Pp]rimary script unknown\\n'
 
 ignoreregex = 
 

config/filter.d/asterisk.conf

@@ -16,7 +16,7 @@ __pid_re = (?:\s*\[\d+\])
 iso8601 = \d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\.\d+[+-]\d{4}
 
 # All Asterisk log messages begin like this:
-log_prefix= (?:NOTICE|SECURITY|WARNING)%(__pid_re)s:?(?:\[C-[\da-f]*\])? [^:]+:\d*(?:(?: in)? \w+:)?
+log_prefix= (?:NOTICE|SECURITY|WARNING)%(__pid_re)s:?(?:\[C-[\da-f]*\])?:? [^:]+:\d*(?:(?: in)? [^:]+:)?
 
 prefregex = ^%(__prefix_line)s%(log_prefix)s <F-CONTENT>.+</F-CONTENT>$
 

config/filter.d/domino-smtp.conf

@@ -35,9 +35,12 @@
 # 08-09-2014 06:14:27   smtp: postmaster [1.2.3.4] authentication failure using internet password
 # 08-09-2014 06:14:27   SMTP Server: Authentication failed for user postmaster ; connecting host 1.2.3.4
 
-__prefix = (?:\[[^\]]+\])?\s+
+__prefix = (?:\[[^\]]+\])?\s*
-failregex = ^%(__prefix)sSMTP Server: Authentication failed for user .*? \; connecting host <HOST>$
+__opt_data = (?::|\s+\[[^\]]+\])
-            ^%(__prefix)ssmtp: (?:[^\[]+ )*\[<HOST>\] authentication failure using internet password\s*$
+failregex = ^%(__prefix)sSMTP Server%(__opt_data)s Authentication failed for user .*? \; connecting host \[?<HOST>\]?$
+            ^%(__prefix)ssmtp: (?:[^\[]+ )*\[?<HOST>\]? authentication failure using internet password\s*$
+            ^%(__prefix)sSMTP Server%(__opt_data)s Connection from \[?<HOST>\]? rejected for policy reasons\.
+
 # Option:  ignoreregex
 # Notes.:  regex to ignore. If this regex matches, the line is ignored.
 # Values:  TEXT

config/filter.d/dovecot.conf

@@ -12,10 +12,10 @@ _daemon = (?:dovecot(?:-auth)?|auth)
 
 prefregex = ^%(__prefix_line)s(?:%(_auth_worker)s(?:\([^\)]+\))?: )?(?:%(__pam_auth)s(?:\(dovecot:auth\))?: |(?:pop3|imap)-login: )?(?:Info: )?<F-CONTENT>.+</F-CONTENT>$
 
-failregex = ^authentication failure; logname=\S* uid=\S* euid=\S* tty=dovecot ruser=\S* rhost=<HOST>(?:\s+user=\S*)?\s*$
+failregex = ^authentication failure; logname=<F-ALT_USER1>\S*</F-ALT_USER1> uid=\S* euid=\S* tty=dovecot ruser=<F-USER>\S*</F-USER> rhost=<HOST>(?:\s+user=<F-ALT_USER>\S*</F-ALT_USER>)?\s*$
-            ^(?:Aborted login|Disconnected)(?::(?: [^ \(]+)+)? \((?:auth failed, \d+ attempts(?: in \d+ secs)?|tried to use (?:disabled|disallowed) \S+ auth)\):(?: user=<[^>]*>,)?(?: method=\S+,)? rip=<HOST>(?:[^>]*(?:, session=<\S+>)?)\s*$
+            ^(?:Aborted login|Disconnected)(?::(?: [^ \(]+)+)? \((?:auth failed, \d+ attempts(?: in \d+ secs)?|tried to use (?:disabled|disallowed) \S+ auth|proxy dest auth failed)\):(?: user=<<F-USER>[^>]*</F-USER>>,)?(?: method=\S+,)? rip=<HOST>(?:[^>]*(?:, session=<\S+>)?)\s*$
             ^pam\(\S+,<HOST>(?:,\S*)?\): pam_authenticate\(\) failed: (?:User not known to the underlying authentication module: \d+ Time\(s\)|Authentication failure \(password mismatch\?\)|Permission denied)\s*$
-            ^[a-z\-]{3,15}\(\S*,<HOST>(?:,\S*)?\): (?:unknown user|invalid credentials)\s*$
+            ^[a-z\-]{3,15}\(\S*,<HOST>(?:,\S*)?\): (?:unknown user|invalid credentials|Password mismatch)\s*$
             <mdre-<mode>>
 
 mdre-aggressive = ^(?:Aborted login|Disconnected)(?::(?: [^ \(]+)+)? \((?:no auth attempts|disconnected before auth was ready,|client didn't finish \S+ auth,)(?: (?:in|waited) \d+ secs)?\):(?: user=<[^>]*>,)?(?: method=\S+,)? rip=<HOST>(?:[^>]*(?:, session=<\S+>)?)\s*$

config/filter.d/exim.conf

@@ -20,7 +20,7 @@ failregex = ^%(pid)s %(host_info)ssender verify fail for <\S+>: (?:Unknown user|
             ^%(pid)s \w+ authenticator failed for (?:[^\[\( ]* )?(?:\(\S*\) )?\[<HOST>\](?::\d+)?(?: I=\[\S+\](:\d+)?)?: 535 Incorrect authentication data( \(set_id=.*\)|: \d+ Time\(s\))?\s*$
             ^%(pid)s %(host_info)srejected RCPT [^@]+@\S+: (?:relay not permitted|Sender verify failed|Unknown user|Unrouteable address)\s*$
             ^%(pid)s SMTP protocol synchronization error \([^)]*\): rejected (?:connection from|"\S+") %(host_info)s(?:next )?input=".*"\s*$
-            ^%(pid)s SMTP call from \S+ %(host_info)sdropped: too many nonmail commands \(last was "\S+"\)\s*$
+            ^%(pid)s SMTP call from (?:[^\[\( ]* )?%(host_info)sdropped: too many (?:nonmail commands|syntax or protocol errors) \(last (?:command )?was "[^"]*"\)\s*$
             ^%(pid)s SMTP protocol error in "[^"]+(?:"+[^"]*(?="))*?" %(host_info)sAUTH command used when not advertised\s*$
             ^%(pid)s no MAIL in SMTP connection from (?:[^\[\( ]* )?(?:\(\S*\) )?%(host_info)sD=\d\S*s(?: C=\S*)?\s*$
             ^%(pid)s (?:[\w\-]+ )?SMTP connection from (?:[^\[\( ]* )?(?:\(\S*\) )?%(host_info)sclosed by DROP in ACL\s*$

config/filter.d/freeswitch.conf

@@ -18,17 +18,39 @@ before = common.conf
 
 _daemon = freeswitch
 
+# Parameter "mode": normal, ddos or extra (default, combines all)
+# Usage example (for jail.local):
+#   [freeswitch]
+#   mode = normal
+#   # or with rewrite filter parameters of jail:
+#   [freeswitch-ddos]
+#   filter = freeswitch[mode=ddos]
+#
+mode = extra
+              
 # Prefix contains common prefix line (server, daemon, etc.) and 2 datetimes if used systemd backend
-_pref_line = ^%(__prefix_line)s(?:\d+-\d+-\d+ \d+:\d+:\d+\.\d+)?
+_pref_line = ^%(__prefix_line)s(?:(?:\d+-)?\d+-\d+ \d+:\d+:\d+\.\d+)?
 
-failregex = %(_pref_line)s \[WARNING\] sofia_reg\.c:\d+ SIP auth (failure|challenge) \((REGISTER|INVITE)\) on sofia profile \'[^']+\' for \[[^\]]*\] from ip <HOST>$
+prefregex = ^%(_pref_line)s \[WARN(?:ING)?\](?: \[SOFIA\])? \[?sofia_reg\.c:\d+\]? <F-CONTENT>.+</F-CONTENT>$
-            %(_pref_line)s \[WARNING\] sofia_reg\.c:\d+ Can't find user \[[^@]+@[^\]]+\] from <HOST>$
+
+cmnfailre = ^Can't find user \[[^@]+@[^\]]+\] from <HOST>$
+
+mdre-normal = %(cmnfailre)s
+              ^SIP auth failure \((REGISTER|INVITE)\) on sofia profile \'[^']+\' for \[[^\]]*\] from ip <HOST>$
+
+mdre-ddos   = ^SIP auth (?:failure|challenge) \((REGISTER|INVITE)\) on sofia profile \'[^']+\' for \[[^\]]*\] from ip <HOST>$
+
+mdre-extra  = %(cmnfailre)s
+              <mdre-ddos>
+
+failregex = <mdre-<mode>>
 
 ignoreregex =
 
-datepattern = {^LN-BEG}
+datepattern = ^(?:%%Y-)?%%m-%%d[ T]%%H:%%M:%%S(?:\.%%f)?
+              {^LN-BEG}
 
-# Author: Rupa SChomaker, soapee01, Daniel Black
+# Author: Rupa SChomaker, soapee01, Daniel Black, Sergey Brester aka sebres
 # https://freeswitch.org/confluence/display/FREESWITCH/Fail2Ban
 # Thanks to Jim on mailing list of samples and guidance
 #

config/filter.d/murmur.conf

@@ -1,11 +1,6 @@
 # Fail2Ban filter for murmur/mumble-server
 #
 
-[INCLUDES]
-
-before = common.conf
-
-
 [Definition]
 
 _daemon = murmurd
@@ -15,7 +10,13 @@ _daemon = murmurd
 #      variable in your server config file (murmur.ini / mumble-server.ini).
 _usernameregex = [^>]+
 
-_prefix = \s+\d+ => <\d+:%(_usernameregex)s\(-1\)> Rejected connection from <HOST>:\d+:
+# Prefix for systemd-journal (with second date-pattern as optional match):
+#
+__prefix_journal = (?:\S+\s+%(_daemon)s\[\d+\]:(?:\s+\<W\>[\d\-]+ [\d:]+.\d+)?)
+
+__prefix_line = %(__prefix_journal)s?
+
+_prefix = %(__prefix_line)s\s+\d+ => <\d+:%(_usernameregex)s\(-1\)> Rejected connection from <HOST>:\d+:
 
 prefregex = ^%(_prefix)s <F-CONTENT>.+</F-CONTENT>$
 
@@ -26,6 +27,8 @@ ignoreregex =
 
 datepattern = ^<W>{DATE}
 
+journalmatch = _SYSTEMD_UNIT=murmurd.service + _COMM=murmurd
+
 # DEV Notes:
 #
 # Author: Ross Brown

config/filter.d/mysqld-auth.conf

@@ -17,7 +17,7 @@ before = common.conf
 
 _daemon = mysqld
 
-failregex = ^%(__prefix_line)s(?:\d+ |\d{6} \s?\d{1,2}:\d{2}:\d{2} )?\[\w+\] Access denied for user '[^']+'@'<HOST>' (to database '[^']*'|\(using password: (YES|NO)\))*\s*$
+failregex = ^%(__prefix_line)s(?:\d+ |\d{6} \s?\d{1,2}:\d{2}:\d{2} )?\[\w+\] (?:\[[^\]]+\] )*Access denied for user '[^']+'@'<HOST>' (to database '[^']*'|\(using password: (YES|NO)\))*\s*$
 
 ignoreregex = 
 

config/filter.d/pam-generic.conf

@@ -16,15 +16,14 @@ _ttys_re=\S*
 __pam_re=\(?%(__pam_auth)s(?:\(\S+\))?\)?:?
 _daemon = \S+
 
-prefregex = ^%(__prefix_line)s%(__pam_re)s\s+authentication failure; logname=\S* uid=\S* euid=\S* tty=%(_ttys_re)s <F-CONTENT>.+</F-CONTENT>$
+prefregex = ^%(__prefix_line)s%(__pam_re)s\s+authentication failure;(?:\s+(?:(?:logname|e?uid)=\S*)){0,3} tty=%(_ttys_re)s <F-CONTENT>.+</F-CONTENT>$
 
-failregex = ^ruser=<F-USER>\S*</F-USER> rhost=<HOST>\s*$
+failregex = ^ruser=<F-ALT_USER>(?:\S*|.*?)</F-ALT_USER> rhost=<HOST>(?:\s+user=<F-USER>(?:\S*|.*?)</F-USER>)?\s*$
-            ^ruser= rhost=<HOST>\s+user=<F-USER>\S*</F-USER>\s*$
-            ^ruser= rhost=<HOST>\s+user=<F-USER>.*?</F-USER>\s*$
-            ^ruser=<F-USER>.*?</F-USER> rhost=<HOST>\s*$
 
 ignoreregex = 
 
+datepattern = {^LN-BEG}
+
 # DEV Notes:
 #
 # for linux-pam before 0.99.2.0 (late 2005) (removed before 0.8.11 release)

config/filter.d/recidive.conf

@@ -21,18 +21,18 @@ before = common.conf
 
 [Definition]
 
-_daemon = fail2ban\.actions\s*
+_daemon = (?:fail2ban(?:-server|\.actions)\s*)
 
-# The name of the jail that this filter is used for. In jail.conf, name the 
+# The name of the jail that this filter is used for. In jail.conf, name the jail using
-# jail using this filter 'recidive', or change this line!
+# this filter 'recidive', or supply another name with `filter = recidive[_jailname="jail"]`
 _jailname = recidive
 
-failregex = ^(%(__prefix_line)s| %(_daemon)s%(__pid_re)s?:\s+)NOTICE\s+\[(?!%(_jailname)s\])(?:.*)\]\s+Ban\s+<HOST>\s*$
+failregex = ^%(__prefix_line)s(?:\s*fail2ban\.actions\s*%(__pid_re)s?:\s+)?NOTICE\s+\[(?!%(_jailname)s\])(?:.*)\]\s+Ban\s+<HOST>\s*$
+
+datepattern = ^{DATE}
 
 ignoreregex = 
 
-[Init]
-
 journalmatch = _SYSTEMD_UNIT=fail2ban.service PRIORITY=5
 
 # Author: Tom Hendrikx, modifications by Amir Caspi 

config/filter.d/sendmail-auth.conf

@@ -9,7 +9,7 @@ before = common.conf
 
 _daemon = (?:sendmail|sm-(?:mta|acceptingconnections))
 
-failregex = ^%(__prefix_line)s\w{14}: (\S+ )?\[<HOST>\]( \(may be forged\))?: possible SMTP attack: command=AUTH, count=\d+$
+failregex = ^%(__prefix_line)s\w{14}: (\S+ )?\[(?:IPv6:<IP6>|<IP4>)\]( \(may be forged\))?: possible SMTP attack: command=AUTH, count=\d+$
 
 ignoreregex =
 

config/filter.d/sendmail-reject.conf

@@ -23,16 +23,16 @@ _daemon = (?:(sm-(mta|acceptingconnections)|sendmail))
 
 prefregex = ^<F-MLFID>%(__prefix_line)s(?:\w{14}: )?</F-MLFID><F-CONTENT>.+</F-CONTENT>$
 
-cmnfailre = ^ruleset=check_rcpt, arg1=(?P<email><\S+@\S+>), relay=(\S+ )?\[<HOST>\](?: \(may be forged\))?, reject=(550 5\.7\.1 (?P=email)\.\.\. Relaying denied\. (IP name possibly forged \[(\d+\.){3}\d+\]|Proper authentication required\.|IP name lookup failed \[(\d+\.){3}\d+\])|553 5\.1\.8 (?P=email)\.\.\. Domain of sender address \S+ does not exist|550 5\.[71]\.1 (?P=email)\.\.\. (Rejected: .*|User unknown))$
+cmnfailre = ^ruleset=check_rcpt, arg1=(?P<email><\S+@\S+>), relay=(\S+ )?\[(?:IPv6:<IP6>|<IP4>)\](?: \(may be forged\))?, reject=(550 5\.7\.1 (?P=email)\.\.\. Relaying denied\. (IP name possibly forged \[(\d+\.){3}\d+\]|Proper authentication required\.|IP name lookup failed \[(\d+\.){3}\d+\])|553 5\.1\.8 (?P=email)\.\.\. Domain of sender address \S+ does not exist|550 5\.[71]\.1 (?P=email)\.\.\. (Rejected: .*|User unknown))$
-            ^ruleset=check_relay, arg1=(?P<dom>\S+), arg2=<HOST>, relay=((?P=dom) )?\[(\d+\.){3}\d+\](?: \(may be forged\))?, reject=421 4\.3\.2 (Connection rate limit exceeded\.|Too many open connections\.)$
+            ^ruleset=check_relay, arg1=(?P<dom>\S+), arg2=(?:IPv6:<IP6>|<IP4>), relay=((?P=dom) )?\[(\d+\.){3}\d+\](?: \(may be forged\))?, reject=421 4\.3\.2 (Connection rate limit exceeded\.|Too many open connections\.)$
-            ^rejecting commands from (\S* )?\[<HOST>\] due to pre-greeting traffic after \d+ seconds$
+            ^rejecting commands from (\S* )?\[(?:IPv6:<IP6>|<IP4>)\] due to pre-greeting traffic after \d+ seconds$
-            ^(?:\S+ )?\[<HOST>\]: (?:(?i)expn|vrfy) \S+ \[rejected\]$
+            ^(?:\S+ )?\[(?:IPv6:<IP6>|<IP4>)\]: (?:(?i)expn|vrfy) \S+ \[rejected\]$
             ^<[^@]+@[^>]+>\.\.\. No such user here$
-            ^<F-NOFAIL>from=<[^@]+@[^>]+></F-NOFAIL>, size=\d+, class=\d+, nrcpts=\d+, bodytype=\w+, proto=E?SMTP, daemon=MTA, relay=\S+ \[<HOST>\]$
+            ^<F-NOFAIL>from=<[^@]+@[^>]+></F-NOFAIL>, size=\d+, class=\d+, nrcpts=\d+, bodytype=\w+, proto=E?SMTP, daemon=MTA, relay=\S+ \[(?:IPv6:<IP6>|<IP4>)\]$
 
 mdre-normal =
 
-mdre-extra = ^(?:\S+ )?\[<HOST>\](?: \(may be forged\))? did not issue (?:[A-Z]{4}[/ ]?)+during connection to M(?:TA|SP)(?:-\w+)?$
+mdre-extra = ^(?:\S+ )?\[(?:IPv6:<IP6>|<IP4>)\](?: \(may be forged\))? did not issue (?:[A-Z]{4}[/ ]?)+during connection to M(?:TA|SP)(?:-\w+)?$
 
 mdre-aggressive = %(mdre-extra)s
 

config/filter.d/sshd.conf

@@ -21,52 +21,66 @@ _daemon = sshd
 # optional prefix (logged from several ssh versions) like "error: ", "error: PAM: " or "fatal: "
 __pref = (?:(?:error|fatal): (?:PAM: )?)?
 # optional suffix (logged from several ssh versions) like " [preauth]"
-__suff = (?: \[preauth\])?\s*
+#__suff = (?: port \d+)?(?: \[preauth\])?\s*
-__on_port_opt = (?: port \d+)?(?: on \S+(?: port \d+)?)?
+__suff = (?: (?:port \d+|on \S+|\[preauth\])){0,3}\s*
+__on_port_opt = (?: (?:port \d+|on \S+)){0,2}
+# close by authenticating user:
+__authng_user = (?: authenticating user <F-USER>\S+|.+?</F-USER>)?
 
 # for all possible (also future) forms of "no matching (cipher|mac|MAC|compression method|key exchange method|host key type) found",
 # see ssherr.c for all possible SSH_ERR_..._ALG_MATCH errors.
 __alg_match = (?:(?:\w+ (?!found\b)){0,2}\w+)
 
+# PAM authentication mechanism, can be overridden, e. g. `filter = sshd[__pam_auth='pam_ldap']`:
+__pam_auth = pam_[a-z]+
+
 [Definition]
 
 prefregex = ^<F-MLFID>%(__prefix_line)s</F-MLFID>%(__pref)s<F-CONTENT>.+</F-CONTENT>$
 
-cmnfailre = ^[aA]uthentication (?:failure|error|failed) for <F-USER>.*</F-USER> from <HOST>( via \S+)?\s*%(__suff)s$
+cmnfailre = ^[aA]uthentication (?:failure|error|failed) for <F-USER>.*</F-USER> from <HOST>( via \S+)?%(__suff)s$
-            ^User not known to the underlying authentication module for <F-USER>.*</F-USER> from <HOST>\s*%(__suff)s$
+            ^User not known to the underlying authentication module for <F-USER>.*</F-USER> from <HOST>%(__suff)s$
-            ^Failed \S+ for invalid user <F-USER>(?P<cond_user>\S+)|(?:(?! from ).)*?</F-USER> from <HOST>%(__on_port_opt)s(?: ssh\d*)?(?(cond_user): |(?:(?:(?! from ).)*)$)
+            ^Failed publickey for invalid user <F-USER>(?P<cond_user>\S+)|(?:(?! from ).)*?</F-USER> from <HOST>%(__on_port_opt)s(?: ssh\d*)?(?(cond_user): |(?:(?:(?! from ).)*)$)
             ^Failed \b(?!publickey)\S+ for (?P<cond_inv>invalid user )?<F-USER>(?P<cond_user>\S+)|(?(cond_inv)(?:(?! from ).)*?|[^:]+)</F-USER> from <HOST>%(__on_port_opt)s(?: ssh\d*)?(?(cond_user): |(?:(?:(?! from ).)*)$)
-            ^<F-USER>ROOT</F-USER> LOGIN REFUSED.* FROM <HOST>\s*%(__suff)s$
+            ^<F-USER>ROOT</F-USER> LOGIN REFUSED FROM <HOST>
-            ^[iI](?:llegal|nvalid) user <F-USER>.*?</F-USER> from <HOST>%(__on_port_opt)s\s*$
+            ^[iI](?:llegal|nvalid) user <F-USER>.*?</F-USER> from <HOST>%(__suff)s$
-            ^User <F-USER>.+</F-USER> from <HOST> not allowed because not listed in AllowUsers\s*%(__suff)s$
+            ^User <F-USER>.+</F-USER> from <HOST> not allowed because not listed in AllowUsers%(__suff)s$
-            ^User <F-USER>.+</F-USER> from <HOST> not allowed because listed in DenyUsers\s*%(__suff)s$
+            ^User <F-USER>.+</F-USER> from <HOST> not allowed because listed in DenyUsers%(__suff)s$
-            ^User <F-USER>.+</F-USER> from <HOST> not allowed because not in any group\s*%(__suff)s$
+            ^User <F-USER>.+</F-USER> from <HOST> not allowed because not in any group%(__suff)s$
-            ^refused connect from \S+ \(<HOST>\)\s*%(__suff)s$
+            ^refused connect from \S+ \(<HOST>\)
             ^Received <F-MLFFORGET>disconnect</F-MLFFORGET> from <HOST>%(__on_port_opt)s:\s*3: .*: Auth fail%(__suff)s$
-            ^User <F-USER>.+</F-USER> from <HOST> not allowed because a group is listed in DenyGroups\s*%(__suff)s$
+            ^User <F-USER>.+</F-USER> from <HOST> not allowed because a group is listed in DenyGroups%(__suff)s$
-            ^User <F-USER>.+</F-USER> from <HOST> not allowed because none of user's groups are listed in AllowGroups\s*%(__suff)s$
+            ^User <F-USER>.+</F-USER> from <HOST> not allowed because none of user's groups are listed in AllowGroups%(__suff)s$
-            ^pam_unix\(sshd:auth\):\s+authentication failure;\s*logname=\S*\s*uid=\d*\s*euid=\d*\s*tty=\S*\s*ruser=<F-USER>\S*</F-USER>\s*rhost=<HOST>\s.*%(__suff)s$
+            ^<F-NOFAIL>%(__pam_auth)s\(sshd:auth\):\s+authentication failure;</F-NOFAIL>(?:\s+(?:(?:logname|e?uid|tty)=\S*)){0,4}\s+ruser=<F-ALT_USER>\S*</F-ALT_USER>\s+rhost=<HOST>(?:\s+user=<F-USER>\S*</F-USER>)?%(__suff)s$
             ^(error: )?maximum authentication attempts exceeded for <F-USER>.*</F-USER> from <HOST>%(__on_port_opt)s(?: ssh\d*)?%(__suff)s$
             ^User <F-USER>.+</F-USER> not allowed because account is locked%(__suff)s
-            ^<F-MLFFORGET>Disconnecting</F-MLFFORGET>: Too many authentication failures(?: for <F-USER>.+?</F-USER>)?%(__suff)s
+            ^<F-MLFFORGET>Disconnecting</F-MLFFORGET>(?: from)?(?: (?:invalid|authenticating)) user <F-USER>\S+</F-USER> <HOST>%(__on_port_opt)s:\s*Change of username or service not allowed:\s*.*\[preauth\]\s*$
-            ^<F-NOFAIL>Received <F-MLFFORGET>disconnect</F-MLFFORGET></F-NOFAIL> from <HOST>: 11:
+            ^<F-MLFFORGET>Disconnecting</F-MLFFORGET>: Too many authentication failures(?: for <F-USER>.+?</F-USER>)?%(__suff)s$
-            ^<F-NOFAIL>Connection <F-MLFFORGET>closed</F-MLFFORGET></F-NOFAIL> by <HOST>%(__suff)s$
+            ^<F-NOFAIL>Received <F-MLFFORGET>disconnect</F-MLFFORGET></F-NOFAIL> from <HOST>%(__on_port_opt)s:\s*11:
-            ^<F-MLFFORGET><F-NOFAIL>Accepted publickey</F-NOFAIL></F-MLFFORGET> for \S+ from <HOST>(?:\s|$)
+            ^<F-NOFAIL>Connection <F-MLFFORGET>closed</F-MLFFORGET></F-NOFAIL> by%(__authng_user)s <HOST><mdrp-<mode>-suff-onclosed>
+            ^<F-MLFFORGET><F-MLFGAINED>Accepted \w+</F-MLFGAINED></F-MLFFORGET> for <F-USER>\S+</F-USER> from <HOST>(?:\s|$)
 
 mdre-normal =
+# used to differentiate "connection closed" with and without `[preauth]` (fail/nofail cases in ddos mode)
+mdrp-normal-suff-onclosed = (?:%(__suff)s|\s*)$
 
-mdre-ddos = ^Did not receive identification string from <HOST>%(__suff)s$
+mdre-ddos = ^Did not receive identification string from <HOST>
-            ^Connection <F-MLFFORGET>reset</F-MLFFORGET> by <HOST>%(__on_port_opt)s%(__suff)s
+            ^Connection <F-MLFFORGET>reset</F-MLFFORGET> by <HOST>
+            ^Connection <F-MLFFORGET>closed</F-MLFFORGET> by%(__authng_user)s <HOST>%(__on_port_opt)s\s+\[preauth\]\s*$
             ^<F-NOFAIL>SSH: Server;Ltype:</F-NOFAIL> (?:Authname|Version|Kex);Remote: <HOST>-\d+;[A-Z]\w+:
-            ^Read from socket failed: Connection <F-MLFFORGET>reset</F-MLFFORGET> by peer%(__suff)s
+            ^Read from socket failed: Connection <F-MLFFORGET>reset</F-MLFFORGET> by peer
+mdrp-ddos-suff-onclosed = %(__on_port_opt)s\s*$
 
-mdre-extra = ^Received <F-MLFFORGET>disconnect</F-MLFFORGET> from <HOST>%(__on_port_opt)s:\s*14: No supported authentication methods available%(__suff)s$
+mdre-extra = ^Received <F-MLFFORGET>disconnect</F-MLFFORGET> from <HOST>%(__on_port_opt)s:\s*14: No supported authentication methods available
             ^Unable to negotiate with <HOST>%(__on_port_opt)s: no matching <__alg_match> found.
-            ^Unable to negotiate a <__alg_match>%(__suff)s$
+            ^Unable to negotiate a <__alg_match>
             ^no matching <__alg_match> found:
+            ^<F-MLFFORGET>Disconnected</F-MLFFORGET>(?: from)?(?: (?:invalid|authenticating)) user <F-USER>\S+</F-USER> <HOST>%(__on_port_opt)s \[preauth\]\s*$
+mdrp-extra-suff-onclosed = %(mdrp-normal-suff-onclosed)s
 
 mdre-aggressive = %(mdre-ddos)s
                   %(mdre-extra)s
+mdrp-aggressive-suff-onclosed = %(mdrp-ddos-suff-onclosed)s
 
 cfooterre = ^<F-NOFAIL>Connection from</F-NOFAIL> <HOST>
 

config/jail.conf

@@ -44,9 +44,9 @@ before = paths-debian.conf
 # MISCELLANEOUS OPTIONS
 #
 
-# "ignorself" specifies whether the local resp. own IP addresses should be ignored
+# "ignoreself" specifies whether the local resp. own IP addresses should be ignored
 # (default is true). Fail2ban will not ban a host which matches such addresses.
-#ignorself = true
+#ignoreself = true
 
 # "ignoreip" can be a list of IP addresses, CIDR masks or DNS hosts. Fail2ban
 # will not ban a host which matches an address in this list. Several addresses

debian/patches/saucy-dsc-patch

@@ -1 +0,0 @@
-neurodebian_use_python2

debian/patches/saucy-dsc-patch

@@ -0,0 +1,62 @@
+--- a/debian/control
++++ b/debian/control
+@@ -5,9 +5,10 @@ Maintainer: Yaroslav Halchenko <debian@o
+ Build-Depends:
+  debhelper (>= 9)
+  , debhelper (>= 9.20160709) | dh-systemd
+- , python3
+- , python3-pyinotify
++ , python
++ , python-pyinotify
+  , sqlite3
++ , dh-python
+ Homepage: http://www.fail2ban.org
+ Vcs-Git: git://github.com/fail2ban/fail2ban.git -b debian
+ Vcs-Browser: http://github.com/fail2ban/fail2ban
+@@ -16,8 +17,8 @@ Standards-Version: 4.1.3
+ 
+ Package: fail2ban
+ Architecture: all
+-Depends: ${python3:Depends}, ${misc:Depends}, lsb-base (>=2.0-7)
+-Recommends: python, iptables, whois, python3-pyinotify, python3-systemd
++Depends: ${python:Depends}, ${misc:Depends}, lsb-base (>=2.0-7)
++Recommends: python, iptables, whois, python-pyinotify
+ Suggests: mailx, system-log-daemon, monit, sqlite3
+ Description: ban hosts that cause multiple authentication errors
+  Fail2ban monitors log files (e.g. /var/log/auth.log,
+@@ -39,5 +40,5 @@ Description: ban hosts that cause multip
+   - whois -- used by a number of *mail-whois* actions to send notification
+     emails with whois information about attacker hosts. Unless you will use
+     those you don't need whois
+-  - python3-pyinotify -- unless you monitor services logs via systemd, you
++  - python-pyinotify -- unless you monitor services logs via systemd, you
+     need pyinotify for efficient monitoring for log files changes
+--- a/debian/rules
++++ b/debian/rules
+@@ -9,13 +9,13 @@
+ # Uncomment this to turn on verbose mode.
+ export DH_VERBOSE=1
+ 
+-export PYBUILD_DISABLE_python2=1
++export PYBUILD_DISABLE_python3=1
+ 
+ %:
+-	dh $@ --with python3,systemd --buildsystem pybuild
++	dh $@ --with python2 --buildsystem pybuild
+ 
+ DESTDIR=$(CURDIR)/debian/fail2ban
+-PYVERSION=$(shell py3versions -dv)
++PYVERSION=$(shell pyversions -dv)
+ 
+ override_dh_clean:
+ 	rm -rf fail2ban.egg-info
+@@ -40,7 +40,8 @@ override_dh_install:
+ 	: # Install bash completion
+ 	install -d $(DESTDIR)/etc/bash_completion.d
+ 	install -m 644 files/bash-completion $(DESTDIR)/etc/bash_completion.d/fail2ban
+-	: # Install systemd files
++	: # Install systemd files, even in backport version just in case even though
++	: # other systemd preparation activities are not carried out
+ 	install -d $(DESTDIR)/lib/systemd/system
+ 	install -d $(DESTDIR)/usr/lib/tmpfiles.d
+ 	install -m 644 build/fail2ban.service $(DESTDIR)/lib/systemd/system

debian/patches/trusty-dsc-patch

@@ -1 +0,0 @@
-neurodebian_use_python2

debian/patches/trusty-dsc-patch

@@ -0,0 +1,62 @@
+--- a/debian/control
++++ b/debian/control
+@@ -5,9 +5,10 @@ Maintainer: Yaroslav Halchenko <debian@o
+ Build-Depends:
+  debhelper (>= 9)
+  , debhelper (>= 9.20160709) | dh-systemd
+- , python3
+- , python3-pyinotify
++ , python
++ , python-pyinotify
+  , sqlite3
++ , dh-python
+ Homepage: http://www.fail2ban.org
+ Vcs-Git: git://github.com/fail2ban/fail2ban.git -b debian
+ Vcs-Browser: http://github.com/fail2ban/fail2ban
+@@ -16,8 +17,8 @@ Standards-Version: 4.1.3
+ 
+ Package: fail2ban
+ Architecture: all
+-Depends: ${python3:Depends}, ${misc:Depends}, lsb-base (>=2.0-7)
+-Recommends: python, iptables, whois, python3-pyinotify, python3-systemd
++Depends: ${python:Depends}, ${misc:Depends}, lsb-base (>=2.0-7)
++Recommends: python, iptables, whois, python-pyinotify
+ Suggests: mailx, system-log-daemon, monit, sqlite3
+ Description: ban hosts that cause multiple authentication errors
+  Fail2ban monitors log files (e.g. /var/log/auth.log,
+@@ -39,5 +40,5 @@ Description: ban hosts that cause multip
+   - whois -- used by a number of *mail-whois* actions to send notification
+     emails with whois information about attacker hosts. Unless you will use
+     those you don't need whois
+-  - python3-pyinotify -- unless you monitor services logs via systemd, you
++  - python-pyinotify -- unless you monitor services logs via systemd, you
+     need pyinotify for efficient monitoring for log files changes
+--- a/debian/rules
++++ b/debian/rules
+@@ -9,13 +9,13 @@
+ # Uncomment this to turn on verbose mode.
+ export DH_VERBOSE=1
+ 
+-export PYBUILD_DISABLE_python2=1
++export PYBUILD_DISABLE_python3=1
+ 
+ %:
+-	dh $@ --with python3,systemd --buildsystem pybuild
++	dh $@ --with python2 --buildsystem pybuild
+ 
+ DESTDIR=$(CURDIR)/debian/fail2ban
+-PYVERSION=$(shell py3versions -dv)
++PYVERSION=$(shell pyversions -dv)
+ 
+ override_dh_clean:
+ 	rm -rf fail2ban.egg-info
+@@ -40,7 +40,8 @@ override_dh_install:
+ 	: # Install bash completion
+ 	install -d $(DESTDIR)/etc/bash_completion.d
+ 	install -m 644 files/bash-completion $(DESTDIR)/etc/bash_completion.d/fail2ban
+-	: # Install systemd files
++	: # Install systemd files, even in backport version just in case even though
++	: # other systemd preparation activities are not carried out
+ 	install -d $(DESTDIR)/lib/systemd/system
+ 	install -d $(DESTDIR)/usr/lib/tmpfiles.d
+ 	install -m 644 build/fail2ban.service $(DESTDIR)/lib/systemd/system

debian/patches/utopic-dsc-patch

@@ -1 +0,0 @@
-neurodebian_use_python2

debian/patches/utopic-dsc-patch

@@ -0,0 +1,62 @@
+--- a/debian/control
++++ b/debian/control
+@@ -5,9 +5,10 @@ Maintainer: Yaroslav Halchenko <debian@o
+ Build-Depends:
+  debhelper (>= 9)
+  , debhelper (>= 9.20160709) | dh-systemd
+- , python3
+- , python3-pyinotify
++ , python
++ , python-pyinotify
+  , sqlite3
++ , dh-python
+ Homepage: http://www.fail2ban.org
+ Vcs-Git: git://github.com/fail2ban/fail2ban.git -b debian
+ Vcs-Browser: http://github.com/fail2ban/fail2ban
+@@ -16,8 +17,8 @@ Standards-Version: 4.1.3
+ 
+ Package: fail2ban
+ Architecture: all
+-Depends: ${python3:Depends}, ${misc:Depends}, lsb-base (>=2.0-7)
+-Recommends: python, iptables, whois, python3-pyinotify, python3-systemd
++Depends: ${python:Depends}, ${misc:Depends}, lsb-base (>=2.0-7)
++Recommends: python, iptables, whois, python-pyinotify
+ Suggests: mailx, system-log-daemon, monit, sqlite3
+ Description: ban hosts that cause multiple authentication errors
+  Fail2ban monitors log files (e.g. /var/log/auth.log,
+@@ -39,5 +40,5 @@ Description: ban hosts that cause multip
+   - whois -- used by a number of *mail-whois* actions to send notification
+     emails with whois information about attacker hosts. Unless you will use
+     those you don't need whois
+-  - python3-pyinotify -- unless you monitor services logs via systemd, you
++  - python-pyinotify -- unless you monitor services logs via systemd, you
+     need pyinotify for efficient monitoring for log files changes
+--- a/debian/rules
++++ b/debian/rules
+@@ -9,13 +9,13 @@
+ # Uncomment this to turn on verbose mode.
+ export DH_VERBOSE=1
+ 
+-export PYBUILD_DISABLE_python2=1
++export PYBUILD_DISABLE_python3=1
+ 
+ %:
+-	dh $@ --with python3,systemd --buildsystem pybuild
++	dh $@ --with python2 --buildsystem pybuild
+ 
+ DESTDIR=$(CURDIR)/debian/fail2ban
+-PYVERSION=$(shell py3versions -dv)
++PYVERSION=$(shell pyversions -dv)
+ 
+ override_dh_clean:
+ 	rm -rf fail2ban.egg-info
+@@ -40,7 +40,8 @@ override_dh_install:
+ 	: # Install bash completion
+ 	install -d $(DESTDIR)/etc/bash_completion.d
+ 	install -m 644 files/bash-completion $(DESTDIR)/etc/bash_completion.d/fail2ban
+-	: # Install systemd files
++	: # Install systemd files, even in backport version just in case even though
++	: # other systemd preparation activities are not carried out
+ 	install -d $(DESTDIR)/lib/systemd/system
+ 	install -d $(DESTDIR)/usr/lib/tmpfiles.d
+ 	install -m 644 build/fail2ban.service $(DESTDIR)/lib/systemd/system

debian/patches/wheezy-dsc-patch

@@ -1 +0,0 @@
-neurodebian_use_python2

debian/patches/wheezy-dsc-patch

@@ -0,0 +1,62 @@
+--- a/debian/control
++++ b/debian/control
+@@ -5,9 +5,10 @@ Maintainer: Yaroslav Halchenko <debian@o
+ Build-Depends:
+  debhelper (>= 9)
+  , debhelper (>= 9.20160709) | dh-systemd
+- , python3
+- , python3-pyinotify
++ , python
++ , python-pyinotify
+  , sqlite3
++ , dh-python
+ Homepage: http://www.fail2ban.org
+ Vcs-Git: git://github.com/fail2ban/fail2ban.git -b debian
+ Vcs-Browser: http://github.com/fail2ban/fail2ban
+@@ -16,8 +17,8 @@ Standards-Version: 4.1.3
+ 
+ Package: fail2ban
+ Architecture: all
+-Depends: ${python3:Depends}, ${misc:Depends}, lsb-base (>=2.0-7)
+-Recommends: python, iptables, whois, python3-pyinotify, python3-systemd
++Depends: ${python:Depends}, ${misc:Depends}, lsb-base (>=2.0-7)
++Recommends: python, iptables, whois, python-pyinotify
+ Suggests: mailx, system-log-daemon, monit, sqlite3
+ Description: ban hosts that cause multiple authentication errors
+  Fail2ban monitors log files (e.g. /var/log/auth.log,
+@@ -39,5 +40,5 @@ Description: ban hosts that cause multip
+   - whois -- used by a number of *mail-whois* actions to send notification
+     emails with whois information about attacker hosts. Unless you will use
+     those you don't need whois
+-  - python3-pyinotify -- unless you monitor services logs via systemd, you
++  - python-pyinotify -- unless you monitor services logs via systemd, you
+     need pyinotify for efficient monitoring for log files changes
+--- a/debian/rules
++++ b/debian/rules
+@@ -9,13 +9,13 @@
+ # Uncomment this to turn on verbose mode.
+ export DH_VERBOSE=1
+ 
+-export PYBUILD_DISABLE_python2=1
++export PYBUILD_DISABLE_python3=1
+ 
+ %:
+-	dh $@ --with python3,systemd --buildsystem pybuild
++	dh $@ --with python2 --buildsystem pybuild
+ 
+ DESTDIR=$(CURDIR)/debian/fail2ban
+-PYVERSION=$(shell py3versions -dv)
++PYVERSION=$(shell pyversions -dv)
+ 
+ override_dh_clean:
+ 	rm -rf fail2ban.egg-info
+@@ -40,7 +40,8 @@ override_dh_install:
+ 	: # Install bash completion
+ 	install -d $(DESTDIR)/etc/bash_completion.d
+ 	install -m 644 files/bash-completion $(DESTDIR)/etc/bash_completion.d/fail2ban
+-	: # Install systemd files
++	: # Install systemd files, even in backport version just in case even though
++	: # other systemd preparation activities are not carried out
+ 	install -d $(DESTDIR)/lib/systemd/system
+ 	install -d $(DESTDIR)/usr/lib/tmpfiles.d
+ 	install -m 644 build/fail2ban.service $(DESTDIR)/lib/systemd/system

fail2ban/__init__.py

@@ -79,3 +79,10 @@ logging.handlers.SysLogHandler.priority_map['NOTICE'] = 'notice'
 from time import strptime
 # strptime thread safety hack-around - http://bugs.python.org/issue7980
 strptime("2012", "%Y")
+
+# short names for pure numeric log-level ("Level 25" could be truncated by short formats):
+def _init():
+	for i in range(50):
+		if logging.getLevelName(i).startswith('Level'):
+			logging.addLevelName(i, '#%02d-Lev.' % i)
+_init()

fail2ban/client/configparserinc.py

@@ -20,7 +20,7 @@
 # Author: Yaroslav Halchenko
 # Modified: Cyril Jaquier
 
-__author__ = 'Yaroslav Halhenko, Serg G. Brester (aka sebres)'
+__author__ = 'Yaroslav Halchenko, Serg G. Brester (aka sebres)'
 __copyright__ = 'Copyright (c) 2007 Yaroslav Halchenko, 2015 Serg G. Brester (aka sebres)'
 __license__ = 'GPL'
 
@@ -33,7 +33,7 @@ if sys.version_info >= (3,2):
 
 	# SafeConfigParser deprecated from Python 3.2 (renamed to ConfigParser)
 	from configparser import ConfigParser as SafeConfigParser, BasicInterpolation, \
-		InterpolationMissingOptionError, NoSectionError
+		InterpolationMissingOptionError, NoOptionError, NoSectionError
 
 	# And interpolation of __name__ was simply removed, thus we need to
 	# decorate default interpolator to handle it
@@ -63,7 +63,7 @@ if sys.version_info >= (3,2):
 
 else: # pragma: no cover
 	from ConfigParser import SafeConfigParser, \
-		InterpolationMissingOptionError, NoSectionError
+		InterpolationMissingOptionError, NoOptionError, NoSectionError
 
 	# Interpolate missing known/option as option from default section
 	SafeConfigParser._cp_interpolate_some = SafeConfigParser._interpolate_some
@@ -73,6 +73,17 @@ else: # pragma: no cover
 		return self._cp_interpolate_some(option, accum, rest, section, map, *args, **kwargs)
 	SafeConfigParser._interpolate_some = _interpolate_some
 
+def _expandConfFilesWithLocal(filenames):
+	"""Expands config files with local extension.
+	"""
+	newFilenames = []
+	for filename in filenames:
+		newFilenames.append(filename)
+		localname = os.path.splitext(filename)[0] + '.local'
+		if localname not in filenames and os.path.isfile(localname):
+			newFilenames.append(localname)
+	return newFilenames
+
 # Gets the instance of the logger.
 logSys = getLogger(__name__)
 logLevel = 7
@@ -112,6 +123,8 @@ after = 1.conf
 
 	SECTION_NAME = "INCLUDES"
 
+	SECTION_OPTNAME_CRE = re.compile(r'^([\w\-]+)/([^\s>]+)$')
+
 	SECTION_OPTSUBST_CRE = re.compile(r'%\(([\w\-]+/([^\)]+))\)s')
 
 	CONDITIONAL_RE = re.compile(r"^(\w+)(\?.+)$")
@@ -131,7 +144,36 @@ after = 1.conf
 			SafeConfigParser.__init__(self, *args, **kwargs)
 			self._cfg_share = share_config
 
-	def _map_section_options(self, section, option, rest, map):
+	def get_ex(self, section, option, raw=False, vars={}):
+		"""Get an option value for a given section.
+		
+		In opposite to `get`, it differentiate session-related option name like `sec/opt`.
+		"""
+		sopt = None
+		# if option name contains section:
+		if '/' in option:
+			sopt = SafeConfigParserWithIncludes.SECTION_OPTNAME_CRE.search(option)
+		# try get value from named section/option:
+		if sopt:
+			sec = sopt.group(1)
+			opt = sopt.group(2)
+			seclwr = sec.lower()
+			if seclwr == 'known':
+				# try get value firstly from known options, hereafter from current section:
+				sopt = ('KNOWN/'+section, section)
+			else:
+				sopt = (sec,) if seclwr != 'default' else ("DEFAULT",)
+			for sec in sopt:
+				try:
+					v = self.get(sec, opt, raw=raw)
+					return v
+				except (NoSectionError, NoOptionError) as e:
+					pass
+		# get value of section/option using given section and vars (fallback):
+		v = self.get(section, option, raw=raw, vars=vars)
+		return v
+
+	def _map_section_options(self, section, option, rest, defaults):
 		"""
 		Interpolates values of the section options (name syntax `%(section/option)s`).
 
@@ -139,37 +181,54 @@ after = 1.conf
 		"""
 		if '/' not in rest or '%(' not in rest: # pragma: no cover
 			return 0
+		rplcmnt = 0
 		soptrep = SafeConfigParserWithIncludes.SECTION_OPTSUBST_CRE.findall(rest)
 		if not soptrep: # pragma: no cover
 			return 0
 		for sopt, opt in soptrep:
-			if sopt not in map:
+			if sopt not in defaults:
 				sec = sopt[:~len(opt)]
 				seclwr = sec.lower()
 				if seclwr != 'default':
+					usedef = 0
 					if seclwr == 'known':
 						# try get raw value from known options:
 						try:
 							v = self._sections['KNOWN/'+section][opt]
 						except KeyError:
 							# fallback to default:
-							try:
+							usedef = 1
-								v = self._defaults[opt]
-							except KeyError: # pragma: no cover
-								continue
 					else:
 						# get raw value of opt in section:
-						v = self.get(sec, opt, raw=True)
+						try:
+							# if section not found - ignore:
+							try:
+								sec = self._sections[sec]
+							except KeyError: # pragma: no cover
+								continue
+							v = sec[opt]
+						except KeyError: # pragma: no cover
+							# fallback to default:
+							usedef = 1
 				else:
+					usedef = 1
+				if usedef:
 					try:
 						v = self._defaults[opt]
 					except KeyError: # pragma: no cover
 						continue
-				self._defaults[sopt] = v
+				# replacement found:
-				try: # for some python versions need to duplicate it in map-vars also:
+				rplcmnt = 1
-					map[sopt] = v
+				try: # set it in map-vars (consider different python versions):
-				except: pass
+					defaults[sopt] = v
-		return 1
+				except:
+					# try to set in first default map (corresponding vars):
+					try:
+						defaults._maps[0][sopt] = v
+					except: # pragma: no cover
+						# no way to update vars chain map - overwrite defaults:
+						self._defaults[sopt] = v
+		return rplcmnt
 
 	@property
 	def share_config(self):
@@ -197,6 +256,7 @@ after = 1.conf
 	def _getIncludes(self, filenames, seen=[]):
 		if not isinstance(filenames, list):
 			filenames = [ filenames ]
+		filenames = _expandConfFilesWithLocal(filenames)
 		# retrieve or cache include paths:
 		if self._cfg_share:
 			# cache/share include list:

fail2ban/client/configreader.py

@@ -29,7 +29,7 @@ import os
 from ConfigParser import NoOptionError, NoSectionError
 
 from .configparserinc import sys, SafeConfigParserWithIncludes, logLevel
-from ..helpers import getLogger, _merge_dicts, substituteRecursiveTags
+from ..helpers import getLogger, _as_bool, _merge_dicts, substituteRecursiveTags
 
 # Gets the instance of the logger.
 logSys = getLogger(__name__)
@@ -339,7 +339,7 @@ class DefinitionInitConfigReader(ConfigReader):
 
 
 	def _convert_to_boolean(self, value):
-		return value.lower() in ("1", "yes", "true", "on")
+		return _as_bool(value)
 	
 	def getCombOption(self, optname):
 		"""Get combined definition option (as string) using pre-set and init
@@ -351,7 +351,7 @@ class DefinitionInitConfigReader(ConfigReader):
 			return self._defCache[optname]
 		except KeyError:
 			try:
-				v = self.get("Definition", optname, vars=self._pOpts)
+				v = self._cfg.get_ex("Definition", optname, vars=self._pOpts)
 			except (NoSectionError, NoOptionError, ValueError):
 				v = None
 			self._defCache[optname] = v

fail2ban/client/csocket.py

@@ -43,33 +43,47 @@ class CSocket:
 		self.__csock.connect(sock)
 
 	def __del__(self):
-		self.close(False)
+		self.close()
 	
-	def send(self, msg):
+	def send(self, msg, nonblocking=False, timeout=None):
 		# Convert every list member to string
-		obj = dumps(map(
+		obj = dumps(map(CSocket.convert, msg), HIGHEST_PROTOCOL)
-			lambda m: str(m) if not isinstance(m, (list, dict, set)) else m, msg),
-		  HIGHEST_PROTOCOL)
 		self.__csock.send(obj + CSPROTO.END)
-		return self.receive(self.__csock)
+		return self.receive(self.__csock, nonblocking, timeout)
 
 	def settimeout(self, timeout):
 		self.__csock.settimeout(timeout if timeout != -1 else self.__deftout)
 
-	def close(self, sendEnd=True):
+	def close(self):
 		if not self.__csock:
 			return
-		if sendEnd:
+		try:
 			self.__csock.sendall(CSPROTO.CLOSE + CSPROTO.END)
-		self.__csock.close()
+			self.__csock.shutdown(socket.SHUT_RDWR)
+		except socket.error: # pragma: no cover - normally unreachable
+			pass
+		try:
+			self.__csock.close()
+		except socket.error: # pragma: no cover - normally unreachable
+			pass
 		self.__csock = None
 	
 	@staticmethod
-	def receive(sock):
+	def convert(m):
+		"""Convert every "unexpected" member of message to string"""
+		if isinstance(m, (basestring, bool, int, float, list, dict, set)):
+			return m
+		else: # pragma: no cover
+			return str(m)
+
+	@staticmethod
+	def receive(sock, nonblocking=False, timeout=None):
 		msg = CSPROTO.EMPTY
+		if nonblocking: sock.setblocking(0)
+		if timeout: sock.settimeout(timeout)
 		while msg.rfind(CSPROTO.END) == -1:
-			chunk = sock.recv(6)
+			chunk = sock.recv(512)
-			if chunk == '':
+			if chunk in ('', b''): # python 3.x may return b'' instead of ''
 				raise RuntimeError("socket connection broken")
 			msg = msg + chunk
 		return loads(msg)

fail2ban/client/fail2banclient.py

@@ -69,7 +69,7 @@ class Fail2banClient(Fail2banCmdLine, Thread):
 		# Print a new line because we probably come from wait
 		output("")
 		logSys.warning("Caught signal %d. Exiting" % signum)
-		exit(-1)
+		exit(255)
 
 	def __ping(self, timeout=0.1):
 		return self.__processCmd([["ping"] + ([timeout] if timeout != -1 else [])],
@@ -99,7 +99,7 @@ class Fail2banClient(Fail2banCmdLine, Thread):
 					ret = client.send(c)
 					if ret[0] == 0:
 						logSys.log(5, "OK : %r", ret[1])
-						if showRet or c[0] == 'echo':
+						if showRet or c[0] in ('echo', 'server-status'):
 							output(beautifier.beautify(ret[1]))
 					else:
 						logSys.error("NOK: %r", ret[1].args)
@@ -128,7 +128,7 @@ class Fail2banClient(Fail2banCmdLine, Thread):
 				except Exception as e: # pragma: no cover
 					if showRet or self._conf["verbose"] > 1:
 						logSys.debug(e)
-			if showRet or c[0] == 'echo':
+			if showRet or c[0] in ('echo', 'server-status'):
 				sys.stdout.flush()
 		return streamRet
 
@@ -186,7 +186,7 @@ class Fail2banClient(Fail2banCmdLine, Thread):
 			logSys.error("Fail2ban seems to be in unexpected state (not running but the socket exists)")
 			return None
 
-		stream.append(['echo', 'Server ready'])
+		stream.append(['server-status'])
 		return stream
 
 	##
@@ -228,9 +228,9 @@ class Fail2banClient(Fail2banCmdLine, Thread):
 		return True
 
 	##
-	def configureServer(self, async=True, phase=None):
+	def configureServer(self, nonsync=True, phase=None):
-		# if asynchron start this operation in the new thread:
+		# if asynchronous start this operation in the new thread:
-		if async:
+		if nonsync:
 			th = Thread(target=Fail2banClient.configureServer, args=(self, False, phase))
 			th.daemon = True
 			return th.start()
@@ -500,5 +500,5 @@ def exec_command_line(argv):
 	if client.start(argv):
 		exit(0)
 	else:
-		exit(-1)
+		exit(255)
 

fail2ban/client/fail2bancmdline.py

@@ -25,7 +25,7 @@ import logging
 import os
 import sys
 
-from ..version import version
+from ..version import version, normVersion
 from ..protocol import printFormatted
 from ..helpers import getLogger, str2LogLevel, getVerbosityFormat
 
@@ -78,12 +78,11 @@ class Fail2banCmdLine():
 		for o in obj.__dict__:
 			self.__dict__[o] = obj.__dict__[o]
 
-	def dispVersion(self):
+	def dispVersion(self, short=False):
-		output("Fail2Ban v" + version)
+		if not short:
-		output("")
+			output("Fail2Ban v" + version)
-		output("Copyright (c) 2004-2008 Cyril Jaquier, 2008- Fail2Ban Contributors")
+		else:
-		output("Copyright of modifications held by their respective authors.")
+			output(normVersion())
-		output("Licensed under the GNU General Public License v2 (GPL).")
 
 	def dispUsage(self):
 		""" Prints Fail2Ban command line options and exits
@@ -114,7 +113,7 @@ class Fail2banCmdLine():
 		output("    --timeout               timeout to wait for the server (for internal usage only, don't read configuration)")
 		output("    --str2sec <STRING>      convert time abbreviation format to seconds")
 		output("    -h, --help              display this help message")
-		output("    -V, --version           print the version")
+		output("    -V, --version           print the version (-V returns machine-readable short format)")
 
 		if not caller.endswith('server'):
 			output("")
@@ -168,7 +167,7 @@ class Fail2banCmdLine():
 				self.dispUsage()
 				return True
 			elif o in ["-V", "--version"]:
-				self.dispVersion()
+				self.dispVersion(o == "-V")
 				return True
 		return None
 

fail2ban/client/fail2banregex.py

@@ -45,7 +45,7 @@ try: # pragma: no cover
 except ImportError:
 	FilterSystemd = None
 
-from ..version import version
+from ..version import version, normVersion
 from .filterreader import FilterReader
 from ..server.filter import Filter, FileContainer
 from ..server.failregex import Regex, RegexException
@@ -93,6 +93,10 @@ def journal_lines_gen(flt, myjournal): # pragma: no cover
 			break
 		yield flt.formatJournalEntry(entry)
 
+def dumpNormVersion(*args):
+	output(normVersion())
+	sys.exit(0)
+
 def get_opt_parser():
 	# use module docstring for help output
 	p = OptionParser(
@@ -145,6 +149,8 @@ Report bugs to https://github.com/fail2ban/fail2ban/issues
 			   dest="log_level",
 			   default='critical',
 			   help="Log level for the Fail2Ban logger to use"),
+		Option('-V', action="callback", callback=dumpNormVersion,
+			   help="get version in machine-readable short format"),
 		Option('-v', '--verbose', action="count", dest="verbose",
 			   default=0,
 			   help="Increase verbosity"),
@@ -226,7 +232,7 @@ class LineStats(object):
 class Fail2banRegex(object):
 
 	def __init__(self, opts):
-		# set local protected memebers from given options:
+		# set local protected members from given options:
 		self.__dict__.update(dict(('_'+o,v) for o,v in opts.__dict__.iteritems()))
 		self._maxlines_set = False		  # so we allow to override maxlines in cmdline
 		self._datepattern_set = False
@@ -405,17 +411,23 @@ class Fail2banRegex(object):
 	def testRegex(self, line, date=None):
 		orgLineBuffer = self._filter._Filter__lineBuffer
 		fullBuffer = len(orgLineBuffer) >= self._filter.getMaxLines()
+		is_ignored = False
 		try:
-			ret = self._filter.processLine(line, date)
+			found = self._filter.processLine(line, date)
 			lines = []
 			line = self._filter.processedLine()
-			for match in ret:
+			ret = []
+			for match in found:
 				# Append True/False flag depending if line was matched by
 				# more than one regex
 				match.append(len(ret)>1)
 				regex = self._failregex[match[0]]
 				regex.inc()
 				regex.appendIP(match)
+				if not match[3].get('nofail'):
+					ret.append(match)
+				else:
+					is_ignored = True
 		except RegexException as e: # pragma: no cover
 			output( 'ERROR: %s' % e )
 			return False
@@ -441,13 +453,13 @@ class Fail2banRegex(object):
 		if lines: # pre-lines parsed in multiline mode (buffering)
 			lines.append(line)
 			line = "\n".join(lines)
-		return line, ret
+		return line, ret, is_ignored
 
 	def process(self, test_lines):
 		t0 = time.time()
 		for line in test_lines:
 			if isinstance(line, tuple):
-				line_datetimestripped, ret = self.testRegex(
+				line_datetimestripped, ret, is_ignored = self.testRegex(
 					line[0], line[1])
 				line = "".join(line[0])
 			else:
@@ -455,8 +467,9 @@ class Fail2banRegex(object):
 				if line.startswith('#') or not line:
 					# skip comment and empty lines
 					continue
-				line_datetimestripped, ret = self.testRegex(line)
+				line_datetimestripped, ret, is_ignored = self.testRegex(line)
-			is_ignored = self.testIgnoreRegex(line_datetimestripped)
+			if not is_ignored:
+				is_ignored = self.testIgnoreRegex(line_datetimestripped)
 
 			if is_ignored:
 				self._line_stats.ignored += 1
@@ -614,7 +627,7 @@ class Fail2banRegex(object):
 			self.setDatePattern(None)
 			if journalmatch:
 				flt.addJournalMatch(journalmatch)
-			output( "Use    journal match : %s" % " ".join(journalmatch) )
+				output( "Use    journal match : %s" % " ".join(journalmatch) )
 			test_lines = journal_lines_gen(flt, myjournal)
 		else:
 			# if single line parsing (without buffering)
@@ -655,7 +668,7 @@ def exec_command_line(*args):
 	if errors:
 		sys.stderr.write("\n".join(errors) + "\n\n")
 		parser.print_help()
-		sys.exit(-1)
+		sys.exit(255)
 
 	output( "" )
 	output( "Running tests" )
@@ -683,4 +696,4 @@ def exec_command_line(*args):
 
 	fail2banRegex = Fail2banRegex(opts)
 	if not fail2banRegex.start(args):
-		sys.exit(-1)
+		sys.exit(255)

fail2ban/client/fail2banserver.py

@@ -128,10 +128,10 @@ class Fail2banServer(Fail2banCmdLine):
 	def getServerPath():
 		startdir = sys.path[0]
 		exe = os.path.abspath(os.path.join(startdir, SERVER))
-		if not os.path.isfile(exe): # may be uresolved in test-cases, so get relative starter (client):
+		if not os.path.isfile(exe): # may be unresolved in test-cases, so get relative starter (client):
 			startdir = os.path.dirname(sys.argv[0])
 			exe = os.path.abspath(os.path.join(startdir, SERVER))
-			if not os.path.isfile(exe): # may be uresolved in test-cases, so try to get relative bin-directory:
+			if not os.path.isfile(exe): # may be unresolved in test-cases, so try to get relative bin-directory:
 				startdir = os.path.dirname(os.path.abspath(__file__))
 				startdir = os.path.join(os.path.dirname(os.path.dirname(startdir)), "bin")
 				exe = os.path.abspath(os.path.join(startdir, SERVER))
@@ -164,21 +164,24 @@ class Fail2banServer(Fail2banCmdLine):
 					cli = self._Fail2banClient()
 					return cli.start(argv)
 
-			# Start the server:
+			# Start the server, corresponding options:
-			from ..server.utils import Utils
+			#   background = True, if should be new process running in background, otherwise start in
-			# background = True, if should be new process running in background, otherwise start in foreground
+			#     foreground process will be forked in daemonize, inside of Server module.
-			# process will be forked in daemonize, inside of Server module.
+			#   nonsync = True, normally internal call only, if started from client, so configures
-			# async = True, if started from client, should...
+			#     the server via asynchronous thread.
 			background = self._conf["background"]
-			async = self._conf.get("async", False)
+			nonsync = self._conf.get("async", False)
+
 			# If was started not from the client:
-			if not async:
+			if not nonsync:
+				# Load requirements on demand (we need utils only when asynchronous handling):
+				from ..server.utils import Utils
 				# Start new thread with client to read configuration and
 				# transfer it to the server:
 				cli = self._Fail2banClient()
 				phase = dict()
 				logSys.debug('Configure via async client thread')
-				cli.configureServer(async=True, phase=phase)
+				cli.configureServer(phase=phase)
 				# wait, do not continue if configuration is not 100% valid:
 				Utils.wait_for(lambda: phase.get('ready', None) is not None, self._conf["timeout"], 0.001)
 				logSys.log(5, '  server phase %s', phase)
@@ -195,7 +198,7 @@ class Fail2banServer(Fail2banCmdLine):
 			pid = os.getpid()
 			server = Fail2banServer.startServerDirect(self._conf, background)
 			# notify waiting thread server ready resp. done (background execution, error case, etc):
-			if not async:
+			if not nonsync:
 				_server_ready()
 			# If forked - just exit other processes
 			if pid != os.getpid(): # pragma: no cover
@@ -204,12 +207,12 @@ class Fail2banServer(Fail2banCmdLine):
 				cli._server = server
 
 			# wait for client answer "done":
-			if not async and cli:
+			if not nonsync and cli:
 				Utils.wait_for(lambda: phase.get('done', None) is not None, self._conf["timeout"], 0.001)
 				if not phase.get('done', False):
 					if server: # pragma: no cover
 						server.quit()
-					exit(-1)
+					exit(255)
 				if background:
 					logSys.debug('Starting server done')
 
@@ -220,7 +223,7 @@ class Fail2banServer(Fail2banCmdLine):
 				logSys.error(e)
 			if server: # pragma: no cover
 				server.quit()
-			exit(-1)
+			exit(255)
 
 		return True
 
@@ -235,4 +238,4 @@ def exec_command_line(argv):
 	if server.start(argv):
 		exit(0)
 	else:
-		exit(-1)
+		exit(255)

fail2ban/client/jailreader.py

@@ -90,9 +90,6 @@ class JailReader(ConfigReader):
 		opts1st = [["bool", "enabled", False],
 				["string", "filter", ""]]
 		opts = [["bool", "enabled", False],
-				["string", "logpath", None],
-				["string", "logtimezone", None],
-				["string", "logencoding", None],
 				["string", "backend", "auto"],
 				["int",    "maxretry", None],
 				["string", "findtime", None],
@@ -103,8 +100,12 @@ class JailReader(ConfigReader):
 				["string", "ignorecommand", None],
 				["bool",   "ignoreself", None],
 				["string", "ignoreip", None],
+				["string", "ignorecache", None],
 				["string", "filter", ""],
 				["string", "datepattern", None],
+				["string", "logtimezone", None],
+				["string", "logencoding", None],
+				["string", "logpath", None], # logpath after all log-related data (backend, date-pattern, etc)
 				["string", "action", ""]]
 
 		# Before interpolation (substitution) add static options always available as default:
@@ -220,7 +221,7 @@ class JailReader(ConfigReader):
 					path, tail = path if len(path) > 1 else (path[0], "head")
 					pathList = JailReader._glob(path)
 					if len(pathList) == 0:
-						logSys.error("No file(s) found for glob %s" % path)
+						logSys.notice("No file(s) found for glob %s" % path)
 					for p in pathList:
 						found_files += 1
 						stream.append(

fail2ban/helpers.py

@@ -32,18 +32,93 @@ from threading import Lock
 
 from .server.mytime import MyTime
 
-
 PREFER_ENC = locale.getpreferredencoding()
-# correct prefered encoding if lang not set in environment:
+# correct preferred encoding if lang not set in environment:
 if PREFER_ENC.startswith('ANSI_'): # pragma: no cover
-	if all((os.getenv(v) in (None, "") for v in ('LANGUAGE', 'LC_ALL', 'LC_CTYPE', 'LANG'))):
+	if sys.stdout and sys.stdout.encoding is not None and not sys.stdout.encoding.startswith('ANSI_'):
+		PREFER_ENC = sys.stdout.encoding
+	elif all((os.getenv(v) in (None, "") for v in ('LANGUAGE', 'LC_ALL', 'LC_CTYPE', 'LANG'))):
 		PREFER_ENC = 'UTF-8';
 
+# py-2.x: try to minimize influence of sporadic conversion errors on python 2.x,
+# caused by implicit converting of string/unicode (e. g. `str(u"\uFFFD")` produces an error
+# if default encoding is 'ascii');
+if sys.version_info < (3,): # pragma: 3.x no cover
+	# correct default (global system) encoding (mostly UTF-8):
+	def __resetDefaultEncoding(encoding):
+		global PREFER_ENC
+		ode = sys.getdefaultencoding().upper()
+		if ode == 'ASCII' and ode != PREFER_ENC.upper():
+			# setdefaultencoding is normally deleted after site initialized, so hack-in using load of sys-module:
+			_sys = sys
+			if not hasattr(_sys, "setdefaultencoding"):
+				try:
+					from imp import load_dynamic as __ldm
+					_sys = __ldm('_sys', 'sys')
+				except ImportError: # pragma: no cover - only if load_dynamic fails
+					reload(sys)
+					_sys = sys
+			if hasattr(_sys, "setdefaultencoding"):
+				_sys.setdefaultencoding(encoding)
+	# override to PREFER_ENC:
+	__resetDefaultEncoding(PREFER_ENC)
+	del __resetDefaultEncoding
+
+# todo: rewrite explicit (and implicit) str-conversions via encode/decode with IO-encoding (sys.stdout.encoding),
+# e. g. inside tags-replacement by command-actions, etc.
+
+#
+# Following "uni_decode", "uni_string" functions unified python independent any 
+# to string converting.
+#
+# Typical example resp. work-case for understanding the coding/decoding issues:
+#
+#   [isinstance('', str), isinstance(b'', str), isinstance(u'', str)]
+#   [True, True, False]; # -- python2
+#	  [True, False, True]; # -- python3
+#
+if sys.version_info >= (3,): # pragma: 2.x no cover
+	def uni_decode(x, enc=PREFER_ENC, errors='strict'):
+		try:
+			if isinstance(x, bytes):
+				return x.decode(enc, errors)
+			return x
+		except (UnicodeDecodeError, UnicodeEncodeError): # pragma: no cover - unsure if reachable
+			if errors != 'strict': 
+				raise
+			return x.decode(enc, 'replace')
+	def uni_string(x):
+		if not isinstance(x, bytes):
+			return str(x)
+		return x.decode(PREFER_ENC, 'replace')
+else: # pragma: 3.x no cover
+	def uni_decode(x, enc=PREFER_ENC, errors='strict'):
+		try:
+			if isinstance(x, unicode):
+				return x.encode(enc, errors)
+			return x
+		except (UnicodeDecodeError, UnicodeEncodeError): # pragma: no cover - unsure if reachable
+			if errors != 'strict':
+				raise
+			return x.encode(enc, 'replace')
+	if sys.getdefaultencoding().upper() != 'UTF-8': # pragma: no cover - utf-8 is default encoding now
+		def uni_string(x):
+			if not isinstance(x, unicode):
+				return str(x)
+			return x.encode(PREFER_ENC, 'replace')
+	else:
+		uni_string = str
+
+
+def _as_bool(val):
+	return bool(val) if not isinstance(val, basestring) \
+		else val.lower() in ('1', 'on', 'true', 'yes')
+
 
 def formatExceptionInfo():
 	""" Consistently format exception information """
 	cla, exc = sys.exc_info()[:2]
-	return (cla.__name__, str(exc))
+	return (cla.__name__, uni_string(exc))
 
 
 #
@@ -126,6 +201,35 @@ class FormatterWithTraceBack(logging.Formatter):
 		return logging.Formatter.format(self, record)
 
 
+__origLog = logging.Logger._log
+def __safeLog(self, level, msg, args, **kwargs):
+	"""Safe log inject to avoid possible errors by unsafe log-handlers, 
+	concat, str. conversion, representation fails, etc.
+
+	Used to intrude exception-safe _log-method instead of _log-method 
+	of Logger class to be always safe by logging and to get more-info about.
+
+	See testSafeLogging test-case for more information. At least the errors
+	covered in phase 3 seems to affected in all known pypy/python versions 
+	until now.
+	"""
+	try:
+		# if isEnabledFor(level) already called...
+		__origLog(self, level, msg, args, **kwargs)
+	except Exception as e: # pragma: no cover - unreachable if log-handler safe in this python-version
+		try:
+			for args in (
+				("logging failed: %r on %s", (e, uni_string(msg))),
+				("  args: %r", ([uni_string(a) for a in args],))
+			):
+				try:
+					__origLog(self, level, *args)
+				except: # pragma: no cover
+					pass
+		except: # pragma: no cover
+			pass
+logging.Logger._log = __safeLog
+
 def getLogger(name):
 	"""Get logging.Logger instance with Fail2Ban logger name convention
 	"""
@@ -143,7 +247,7 @@ def str2LogLevel(value):
 		raise ValueError("Invalid log level %r" % value)
 	return ll
 
-def getVerbosityFormat(verbosity, fmt=' %(message)s', addtime=True):
+def getVerbosityFormat(verbosity, fmt=' %(message)s', addtime=True, padding=True):
 	"""Custom log format for the verbose runs
 	"""
 	if verbosity > 1: # pragma: no cover
@@ -155,6 +259,13 @@ def getVerbosityFormat(verbosity, fmt=' %(message)s', addtime=True):
 			fmt = ' %(thread)X %(levelname)-5.5s' + fmt
 			if addtime:
 				fmt = ' %(asctime)-15s' + fmt
+	else: # default (not verbose):
+		fmt = "%(name)-23.23s [%(process)d]: %(levelname)-7s" + fmt
+		if addtime:
+			fmt = "%(asctime)s " + fmt
+	# remove padding if not needed:
+	if not padding:
+		fmt = re.sub(r'(?<=\))-?\d+(?:\.\d+)?s', lambda m: 's', fmt)
 	return fmt
 
 
@@ -206,37 +317,6 @@ else:
 			r.update(y)
 		return r
 
-#
-# Following "uni_decode" function unified python independent any to string converting
-#
-# Typical example resp. work-case for understanding the coding/decoding issues:
-#
-#   [isinstance('', str), isinstance(b'', str), isinstance(u'', str)]
-#   [True, True, False]; # -- python2
-#	  [True, False, True]; # -- python3
-#
-if sys.version_info >= (3,):
-	def uni_decode(x, enc=PREFER_ENC, errors='strict'):
-		try:
-			if isinstance(x, bytes):
-				return x.decode(enc, errors)
-			return x
-		except (UnicodeDecodeError, UnicodeEncodeError): # pragma: no cover - unsure if reachable
-			if errors != 'strict': 
-				raise
-			return uni_decode(x, enc, 'replace')
-else:
-	def uni_decode(x, enc=PREFER_ENC, errors='strict'):
-		try:
-			if isinstance(x, unicode):
-				return x.encode(enc, errors)
-			return x
-		except (UnicodeDecodeError, UnicodeEncodeError): # pragma: no cover - unsure if reachable
-			if errors != 'strict':
-				raise
-			return uni_decode(x, enc, 'replace')
-
-
 #
 # Following function used for parse options from parameter (e.g. `name[p1=0, p2="..."][p3='...']`).
 #
@@ -314,7 +394,7 @@ def substituteRecursiveTags(inptags, conditional='',
 			if tag in ignore or tag in done: continue
 			# ignore replacing callable items from calling map - should be converted on demand only (by get):
 			if noRecRepl and callable(tags.getRawItem(tag)): continue
-			value = orgval = str(tags[tag])
+			value = orgval = uni_string(tags[tag])
 			# search and replace all tags within value, that can be interpolated using other tags:
 			m = tre_search(value)
 			refCounts = {}
@@ -349,7 +429,7 @@ def substituteRecursiveTags(inptags, conditional='',
 					m = tre_search(value, m.end())
 					continue
 				# if calling map - be sure we've string:
-				if noRecRepl: repl = str(repl)
+				if not isinstance(repl, basestring): repl = uni_string(repl)
 				value = value.replace('<%s>' % rtag, repl)
 				#logSys.log(5, 'value now: %s' % value)
 				# increment reference count:
@@ -393,7 +473,9 @@ class BgService(object):
 		self.__count = self.__threshold;
 		if hasattr(gc, 'set_threshold'):
 			gc.set_threshold(0)
-		gc.disable()
+		# don't disable auto garbage, because of non-reference-counting python's (like pypy),
+		# otherwise it may leak there on objects like unix-socket, etc.
+		#gc.disable()
 
 	def service(self, force=False, wait=False):
 		self.__count -= 1

fail2ban/protocol.py

@@ -84,6 +84,8 @@ protocol = [
 ["set <JAIL> ignoreself true|false", "allows the ignoring of own IP addresses"], 
 ["set <JAIL> addignoreip <IP>", "adds <IP> to the ignore list of <JAIL>"], 
 ["set <JAIL> delignoreip <IP>", "removes <IP> from the ignore list of <JAIL>"], 
+["set <JAIL> ignorecommand <VALUE>", "sets ignorecommand of <JAIL>"],
+["set <JAIL> ignorecache <VALUE>", "sets ignorecache of <JAIL>"],
 ["set <JAIL> addlogpath <FILE> ['tail']", "adds <FILE> to the monitoring list of <JAIL>, optionally starting at the 'tail' of the file (default 'head')."], 
 ["set <JAIL> dellogpath <FILE>", "removes <FILE> from the monitoring list of <JAIL>"],
 ["set <JAIL> logencoding <ENCODING>", "sets the <ENCODING> of the log files for <JAIL>"],
@@ -91,7 +93,6 @@ protocol = [
 ["set <JAIL> deljournalmatch <MATCH>", "removes <MATCH> from the journal filter of <JAIL>"],
 ["set <JAIL> addfailregex <REGEX>", "adds the regular expression <REGEX> which must match failures for <JAIL>"], 
 ["set <JAIL> delfailregex <INDEX>", "removes the regular expression at <INDEX> for failregex"], 
-["set <JAIL> ignorecommand <VALUE>", "sets ignorecommand of <JAIL>"],
 ["set <JAIL> addignoreregex <REGEX>", "adds the regular expression <REGEX> which should match pattern to exclude for <JAIL>"],
 ["set <JAIL> delignoreregex <INDEX>", "removes the regular expression at <INDEX> for ignoreregex"], 
 ["set <JAIL> findtime <TIME>", "sets the number of seconds <TIME> for which the filter will look back for <JAIL>"], 

fail2ban/server/action.py

@@ -36,7 +36,7 @@ from .failregex import mapTag2Opt
 from .ipdns import asip, DNSUtils
 from .mytime import MyTime
 from .utils import Utils
-from ..helpers import getLogger, _merge_copy_dicts, substituteRecursiveTags, TAG_CRE, MAX_TAG_REPLACE_COUNT
+from ..helpers import getLogger, _merge_copy_dicts, uni_string, substituteRecursiveTags, TAG_CRE, MAX_TAG_REPLACE_COUNT
 
 # Gets the instance of the logger.
 logSys = getLogger(__name__)
@@ -83,6 +83,8 @@ class CallingMap(MutableMapping, object):
 		The dictionary data which can be accessed to obtain items uncalled
 	"""
 
+	CM_REPR_ITEMS = ()
+
 	# immutable=True saves content between actions, without interim copying (save original on demand, recoverable via reset)
 	__slots__ = ('data', 'storage', 'immutable', '__org_data')
 	def __init__(self, *args, **kwargs):
@@ -98,14 +100,29 @@ class CallingMap(MutableMapping, object):
 			pass
 		self.immutable = immutable
 
-	def __repr__(self):
+	def _asrepr(self, calculated=False):
-		return "%s(%r)" % (self.__class__.__name__, self._asdict())
+		# be sure it is suitable as string, so use str as checker:
+		return "%s(%r)" % (self.__class__.__name__, self._asdict(calculated, str))
 
-	def _asdict(self):
+	__repr__ = _asrepr
-		try:
+
-			return dict(self)
+	def _asdict(self, calculated=False, checker=None):
-		except:
+		d = dict(self.data, **self.storage)
-			return dict(self.data, **self.storage)
+		if not calculated:
+			return dict((n,v) for n,v in d.iteritems() \
+				if not callable(v) or n in self.CM_REPR_ITEMS)
+		for n,v in d.items():
+			if callable(v):
+				try:
+					# calculate:
+					v = self.__getitem__(n)
+					# convert if needed:
+					if checker: checker(v)
+					# store calculated:
+					d[n] = v
+				except: # can't calculate - just ignore it
+					pass
+		return d
 
 	def getRawItem(self, key):
 		try:
@@ -156,7 +173,7 @@ class CallingMap(MutableMapping, object):
 	def __len__(self):
 		return len(self.data)
 
-	def copy(self): # pargma: no cover
+	def copy(self): # pragma: no cover
 		return self.__class__(_merge_copy_dicts(self.data, self.storage))
 
 
@@ -312,9 +329,9 @@ class CommandAction(ActionBase):
 
 	def __setattr__(self, name, value):
 		if not name.startswith('_') and not self.__init and not callable(value):
-			# special case for some pasrameters:
+			# special case for some parameters:
 			if name in ('timeout', 'bantime'):
-				value = str(MyTime.str2seconds(value))
+				value = MyTime.str2seconds(value)
 			# parameters changed - clear properties and substitution cache:
 			self.__properties = None
 			self.__substCache.clear()
@@ -337,7 +354,7 @@ class CommandAction(ActionBase):
 	def _properties(self):
 		"""A dictionary of the actions properties.
 
-		This is used to subsitute "tags" in the commands.
+		This is used to substitute "tags" in the commands.
 		"""
 		# if we have a properties - return it:
 		if self.__properties is not None:
@@ -383,7 +400,7 @@ class CommandAction(ActionBase):
 		except ValueError as e:
 			raise RuntimeError("Error %s action %s/%s: %r" % (operation, self._jail, self._name, e))
 
-	COND_FAMILIES = {'inet4':1, 'inet6':1}
+	COND_FAMILIES = ('inet4', 'inet6')
 
 	@property
 	def _startOnDemand(self):
@@ -460,13 +477,13 @@ class CommandAction(ActionBase):
 		"""Executes the "actionflush" command.
 		
 		Command executed in order to flush all bans at once (e. g. by stop/shutdown 
-		the system), instead of unbunning of each single ticket.
+		the system), instead of unbanning of each single ticket.
 
 		Replaces the tags in the action command with actions properties
 		and executes the resulting command.
 		"""
 		family = []
-		# cumulate started families, if started on demand (conditional):
+		# collect started families, if started on demand (conditional):
 		if self._startOnDemand:
 			for f in CommandAction.COND_FAMILIES:
 				if self.__started.get(f) == 1: # only real started:
@@ -482,7 +499,7 @@ class CommandAction(ActionBase):
 		and executes the resulting command.
 		"""
 		family = []
-		# cumulate started families, if started on demand (conditional):
+		# collect started families, if started on demand (conditional):
 		if self._startOnDemand:
 			for f in CommandAction.COND_FAMILIES:
 				if self.__started.get(f) == 1: # only real started:
@@ -591,7 +608,7 @@ class CommandAction(ActionBase):
 				if value is None:
 					# fallback (no or default replacement)
 					return ADD_REPL_TAGS_CM.get(tag, m.group())
-			value = str(value)		# assure string
+			value = uni_string(value)		# assure string
 			if tag in cls._escapedTags:
 				# That one needs to be escaped since its content is
 				# out of our control
@@ -670,7 +687,7 @@ class CommandAction(ActionBase):
 			except KeyError:
 				# fallback (no or default replacement)
 				return ADD_REPL_TAGS_CM.get(tag, m.group())
-			value = str(value)		# assure string
+			value = uni_string(value)		# assure string
 			# replacement for tag:
 			return escapeVal(tag, value)
 		
@@ -684,7 +701,7 @@ class CommandAction(ActionBase):
 			def substTag(m):
 				tag = mapTag2Opt(m.groups()[0])
 				try:
-					value = str(tickData[tag])
+					value = uni_string(tickData[tag])
 				except KeyError:
 					return ""
 				return escapeVal("F_"+tag, value)

fail2ban/server/actions.py

@@ -290,6 +290,8 @@ class Actions(JailThread, Mapping):
 
 	class ActionInfo(CallingMap):
 
+		CM_REPR_ITEMS = ("fid", "raw-ticket")
+
 		AI_DICT = {
 			"ip":				lambda self: self.__ticket.getIP(),
 			"family":   lambda self: self['ip'].familyStr,
@@ -307,7 +309,9 @@ class Actions(JailThread, Mapping):
 			"ipmatches":			lambda self: "\n".join(self._mi4ip(True).getMatches()),
 			"ipjailmatches":	lambda self: "\n".join(self._mi4ip().getMatches()),
 			"ipfailures":			lambda self: self._mi4ip(True).getAttempt(),
-			"ipjailfailures":	lambda self: self._mi4ip().getAttempt()
+			"ipjailfailures":	lambda self: self._mi4ip().getAttempt(),
+			# raw ticket info:
+			"raw-ticket":			lambda self: repr(self.__ticket)
 		}
 
 		__slots__ = CallingMap.__slots__ + ('__ticket', '__jail', '__mi4ip')
@@ -319,7 +323,7 @@ class Actions(JailThread, Mapping):
 			self.immutable = immutable
 			self.data = data
 		
-		def copy(self): # pargma: no cover
+		def copy(self): # pragma: no cover
 			return self.__class__(self.__ticket, self.__jail, self.immutable, self.data.copy())
 
 		def _mi4ip(self, overalljails=False):
@@ -415,7 +419,7 @@ class Actions(JailThread, Mapping):
 					diftm = ticket.getTime() - bTicket.getTime()
 					# log already banned with following level:
 					#   DEBUG   - before 3 seconds - certain interval for it, because of possible latency by recognizing in backends, etc.
-					#   NOTICE  - before 60 seconds - may still occurre if action are slow, or very high load in backend,
+					#   NOTICE  - before 60 seconds - may still occur if action is slow, or very high load in backend,
 					#   WARNING - after 60 seconds - very long time, something may be wrong
 					ll = logging.DEBUG   if diftm < 3 \
 					else logging.NOTICE  if diftm < 60 \

fail2ban/server/asyncserver.py

@@ -42,25 +42,44 @@ from ..helpers import logging, getLogger, formatExceptionInfo
 # Gets the instance of the logger.
 logSys = getLogger(__name__)
 
+
 ##
 # Request handler class.
 #
 # This class extends asynchat in order to provide a request handler for
 # incoming query.
-
 class RequestHandler(asynchat.async_chat):
 	
 	def __init__(self, conn, transmitter):
 		asynchat.async_chat.__init__(self, conn)
+		self.__conn = conn
 		self.__transmitter = transmitter
 		self.__buffer = []
 		# Sets the terminator.
 		self.set_terminator(CSPROTO.END)
 
+	def __close(self):
+		if self.__conn:
+			conn = self.__conn
+			self.__conn = None
+			try:
+				conn.shutdown(socket.SHUT_RDWR)
+				conn.close()
+			except socket.error: # pragma: no cover - normally unreachable
+				pass
+
+	def handle_close(self):
+		self.__close()
+		asynchat.async_chat.handle_close(self)
+
 	def collect_incoming_data(self, data):
 		#logSys.debug("Received raw data: " + str(data))
 		self.__buffer.append(data)
 
+	# exception identifies deserialization errors (exception by load in pickle):
+	class LoadError(Exception):
+		pass
+
 	##
 	# Handles a new request.
 	#
@@ -78,7 +97,12 @@ class RequestHandler(asynchat.async_chat):
 				self.close_when_done()
 				return
 			# Deserialize
-			message = loads(message)
+			try:
+				message = loads(message)
+			except Exception as e:
+				logSys.error('PROTO-error: load message failed: %s', e,
+					exc_info=logSys.getEffectiveLevel()<logging.DEBUG)
+				raise RequestHandler.LoadError(e)
 			# Gives the message to the transmitter.
 			if self.__transmitter:
 				message = self.__transmitter.proceed(message)
@@ -89,8 +113,9 @@ class RequestHandler(asynchat.async_chat):
 			# Sends the response to the client.
 			self.push(message + CSPROTO.END)
 		except Exception as e:
-			logSys.error("Caught unhandled exception: %r", e,
+			if not isinstance(e, RequestHandler.LoadError): # pragma: no cover - normally unreachable
-				exc_info=logSys.getEffectiveLevel()<=logging.DEBUG)
+				logSys.error("Caught unhandled exception: %r", e,
+					exc_info=logSys.getEffectiveLevel()<=logging.DEBUG)
 			# Sends the response to the client.
 			message = dumps("ERROR: %s" % e, HIGHEST_PROTOCOL)
 			self.push(message + CSPROTO.END)
@@ -111,14 +136,15 @@ class RequestHandler(asynchat.async_chat):
 		self.close_when_done()
 
 
-def loop(active, timeout=None, use_poll=False):
+def loop(active, timeout=None, use_poll=False, err_count=None):
 	"""Custom event loop implementation
 
 	Uses poll instead of loop to respect `active` flag,
 	to avoid loop timeout mistake: different in poll and poll2 (sec vs ms),
 	and to prevent sporadic errors like EBADF 'Bad file descriptor' etc. (see gh-161)
 	"""
-	errCount = 0
+	if not err_count: err_count={}
+	err_count['listen'] = 0
 	if timeout is None:
 		timeout = Utils.DEFAULT_SLEEP_TIME
 	poll = asyncore.poll
@@ -133,22 +159,29 @@ def loop(active, timeout=None, use_poll=False):
 	while active():
 		try:
 			poll(timeout)
-			if errCount:
+			if err_count['listen']:
-				errCount -= 1
+				err_count['listen'] -= 1
 		except Exception as e:
 			if not active():
 				break
-			errCount += 1
+			err_count['listen'] += 1
-			if errCount < 20:
+			if err_count['listen'] < 20:
 				# errno.ENOTCONN - 'Socket is not connected'
 				# errno.EBADF - 'Bad file descriptor'
 				if e.args[0] in (errno.ENOTCONN, errno.EBADF): # pragma: no cover (too sporadic)
 					logSys.info('Server connection was closed: %s', str(e))
 				else:
 					logSys.error('Server connection was closed: %s', str(e))
-			elif errCount == 20:
+			elif err_count['listen'] == 20:
 				logSys.exception(e)
 				logSys.error('Too many errors - stop logging connection errors')
+			elif err_count['listen'] > 100: # pragma: no cover - normally unreachable
+				if (
+					   e.args[0] == errno.EMFILE # [Errno 24] Too many open files
+					or sum(err_count.itervalues()) > 1000
+				):
+					logSys.critical("Too many errors - critical count reached %r", err_count)
+					break
 
 
 ##
@@ -165,6 +198,7 @@ class AsyncServer(asyncore.dispatcher):
 		self.__sock = "/var/run/fail2ban/fail2ban.sock"
 		self.__init = False
 		self.__active = False
+		self.__errCount = {'accept': 0, 'listen': 0}
 		self.onstart = None
 
 	##
@@ -176,12 +210,23 @@ class AsyncServer(asyncore.dispatcher):
 	def handle_accept(self):
 		try:
 			conn, addr = self.accept()
-		except socket.error: # pragma: no cover
+		except Exception as e: # pragma: no cover
-			logSys.warning("Socket error")
+			self.__errCount['accept'] += 1
-			return
+			if self.__errCount['accept'] < 20:
-		except TypeError: # pragma: no cover
+				logSys.warning("Accept socket error: %s", e,
-			logSys.warning("Type error")
+					exc_info=(self.__errCount['accept'] <= 1))
+			elif self.__errCount['accept'] == 20:
+				logSys.error("Too many acceptor errors - stop logging errors")
+			elif self.__errCount['accept'] > 100:
+				if (
+					  (isinstance(e, socket.error) and e.args[0] == errno.EMFILE) # [Errno 24] Too many open files
+					or sum(self.__errCount.itervalues()) > 1000
+				):
+					logSys.critical("Too many errors - critical count reached %r", self.__errCount)
+					self.stop()
 			return
+		if self.__errCount['accept']:
+			self.__errCount['accept'] -= 1;
 		AsyncServer.__markCloseOnExec(conn)
 		# Creates an instance of the handler class to handle the
 		# request/response on the incoming connection.
@@ -219,7 +264,7 @@ class AsyncServer(asyncore.dispatcher):
 		if self.onstart:
 			self.onstart()
 		# Event loop as long as active:
-		loop(lambda: self.__loop, timeout=timeout, use_poll=use_poll)
+		loop(lambda: self.__loop, timeout=timeout, use_poll=use_poll, err_count=self.__errCount)
 		self.__active = False
 		# Cleanup all
 		self.stop()
@@ -228,6 +273,13 @@ class AsyncServer(asyncore.dispatcher):
 		stopflg = False
 		if self.__active:
 			self.__loop = False
+			# shutdown socket here:
+			if self.socket:
+				try:
+					self.socket.shutdown(socket.SHUT_RDWR)
+				except socket.error: # pragma: no cover - normally unreachable
+					pass
+			# close connection:
 			asyncore.dispatcher.close(self)
 			# If not the loop thread (stops self in handler), wait (a little bit) 
 			# for the server leaves loop, before remove socket
@@ -246,13 +298,15 @@ class AsyncServer(asyncore.dispatcher):
 	# Stops the communication server.
 	
 	def stop_communication(self):
-		logSys.debug("Stop communication")
+		if self.__transmitter:
-		self.__transmitter = None
+			logSys.debug("Stop communication, shutdown")
+			self.__transmitter = None
 
 	##
 	# Stops the server.
 	
 	def stop(self):
+		self.stop_communication()
 		self.close()
 
 	# better remains a method (not a property) since used as a callable for wait_for

fail2ban/server/banmanager.py

@@ -156,7 +156,7 @@ class BanManager:
 		# get cymru info:
 		try:
 			for ip in banIPs:
-				# Reference: http://www.team-cymru.org/Services/ip-to-asn.html#dns
+				# Reference: https://www.team-cymru.com/IP-ASN-mapping.html#dns
 				question = ip.getPTR(
 					"origin.asn.cymru.com" if ip.isIPv4
 					else "origin6.asn.cymru.com"
@@ -166,15 +166,21 @@ class BanManager:
 					answers = resolver.query(question, "TXT")
 					if not answers:
 						raise ValueError("No data retrieved")
+					asns = set()
+					countries = set()
+					rirs = set()
 					for rdata in answers:
 						asn, net, country, rir, changed =\
 							[answer.strip("'\" ") for answer in rdata.to_text().split("|")]
 						asn = self.handleBlankResult(asn)
 						country = self.handleBlankResult(country)
 						rir = self.handleBlankResult(rir)
-						return_dict["asn"].append(self.handleBlankResult(asn))
+						asns.add(self.handleBlankResult(asn))
-						return_dict["country"].append(self.handleBlankResult(country))
+						countries.add(self.handleBlankResult(country))
-						return_dict["rir"].append(self.handleBlankResult(rir))
+						rirs.add(self.handleBlankResult(rir))
+					return_dict["asn"].append(', '.join(sorted(asns)))
+					return_dict["country"].append(', '.join(sorted(countries)))
+					return_dict["rir"].append(', '.join(sorted(rirs)))
 				except dns.resolver.NXDOMAIN:
 					return_dict["asn"].append("nxdomain")
 					return_dict["country"].append("nxdomain")

fail2ban/server/database.py

@@ -33,55 +33,65 @@ from threading import RLock
 from .mytime import MyTime
 from .ticket import FailTicket
 from .utils import Utils
-from ..helpers import getLogger, PREFER_ENC
+from ..helpers import getLogger, uni_string, PREFER_ENC
 
 # Gets the instance of the logger.
 logSys = getLogger(__name__)
 
-if sys.version_info >= (3,):
+
+def _json_default(x):
+	"""Avoid errors on types unknown in json-adapters."""
+	if isinstance(x, set):
+		x = list(x)
+	return uni_string(x)
+
+if sys.version_info >= (3,): # pragma: 2.x no cover
 	def _json_dumps_safe(x):
 		try:
-			x = json.dumps(x, ensure_ascii=False).encode(
+			x = json.dumps(x, ensure_ascii=False, default=_json_default).encode(
 				PREFER_ENC, 'replace')
-		except Exception as e: # pragma: no cover
+		except Exception as e:
-			logSys.error('json dumps failed: %s', e)
+			# adapter handler should be exception-safe
+			logSys.error('json dumps failed: %r', e, exc_info=logSys.getEffectiveLevel() <= 4)
 			x = '{}'
 		return x
 
 	def _json_loads_safe(x):
 		try:
-			x = json.loads(x.decode(
+			x = json.loads(x.decode(PREFER_ENC, 'replace'))
-				PREFER_ENC, 'replace'))
+		except Exception as e:
-		except Exception as e: # pragma: no cover
+			# converter handler should be exception-safe
-			logSys.error('json loads failed: %s', e)
+			logSys.error('json loads failed: %r', e, exc_info=logSys.getEffectiveLevel() <= 4)
 			x = {}
 		return x
-else:
+else: # pragma: 3.x no cover
 	def _normalize(x):
 		if isinstance(x, dict):
 			return dict((_normalize(k), _normalize(v)) for k, v in x.iteritems())
-		elif isinstance(x, list):
+		elif isinstance(x, (list, set)):
 			return [_normalize(element) for element in x]
 		elif isinstance(x, unicode):
-			return x.encode(PREFER_ENC)
+			# in 2.x default text_factory is unicode - so return proper unicode here:
-		else:
+			return x.encode(PREFER_ENC, 'replace').decode(PREFER_ENC)
-			return x
+		elif isinstance(x, basestring):
+			return x.decode(PREFER_ENC, 'replace')
+		return x
 
 	def _json_dumps_safe(x):
 		try:
-			x = json.dumps(_normalize(x), ensure_ascii=False).decode(
+			x = json.dumps(_normalize(x), ensure_ascii=False, default=_json_default)
-				PREFER_ENC, 'replace')
+		except Exception as e:
-		except Exception as e: # pragma: no cover
+			# adapter handler should be exception-safe
-			logSys.error('json dumps failed: %s', e)
+			logSys.error('json dumps failed: %r', e, exc_info=logSys.getEffectiveLevel() <= 4)
 			x = '{}'
 		return x
 
 	def _json_loads_safe(x):
 		try:
-			x = _normalize(json.loads(x.decode(
+			x = json.loads(x.decode(PREFER_ENC, 'replace'))
-				PREFER_ENC, 'replace')))
+		except Exception as e:
-		except Exception as e: # pragma: no cover
+			# converter handler should be exception-safe
-			logSys.error('json loads failed: %s', e)
+			logSys.error('json loads failed: %r', e, exc_info=logSys.getEffectiveLevel() <= 4)
 			x = {}
 		return x
 
@@ -179,6 +189,8 @@ class Fail2BanDb(object):
 			self._db = sqlite3.connect(
 				filename, check_same_thread=False,
 				detect_types=sqlite3.PARSE_DECLTYPES)
+			# # to allow use multi-byte utf-8
+			# self._db.text_factory = str
 
 			self._bansMergedCache = {}
 
@@ -527,10 +539,13 @@ class Fail2BanDb(object):
 		except KeyError:
 			pass
 		#TODO: Implement data parts once arbitrary match keys completed
+		data = ticket.getData()
+		matches = data.get('matches')
+		if matches and len(matches) > self.maxEntries:
+			data['matches'] = matches[-self.maxEntries:]
 		cur.execute(
 			"INSERT INTO bans(jail, ip, timeofban, data) VALUES(?, ?, ?, ?)",
-			(jail.name, ip, int(round(ticket.getTime())),
+			(jail.name, ip, int(round(ticket.getTime())), data))
-				ticket.getData()))
 
 	@commitandrollback
 	def delBan(self, cur, jail, *args):
@@ -659,11 +674,11 @@ class Fail2BanDb(object):
 						else:
 							matches = m[-maxadd:] + matches
 					failures += data.get('failures', 1)
-					tickdata.update(data.get('data', {}))
+					data['failures'] = failures
+					data['matches'] = matches
+					tickdata.update(data)
 					prev_timeofban = timeofban
-				ticket = FailTicket(banip, prev_timeofban, matches)
+				ticket = FailTicket(banip, prev_timeofban, data=tickdata)
-				ticket.setAttempt(failures)
-				ticket.setData(**tickdata)
 				tickets.append(ticket)
 
 			if cacheKey:

fail2ban/server/datedetector.py

@@ -26,7 +26,8 @@ import time
 
 from threading import Lock
 
-from .datetemplate import re, DateTemplate, DatePatternRegex, DateTai64n, DateEpoch
+from .datetemplate import re, DateTemplate, DatePatternRegex, DateTai64n, DateEpoch, \
+	RE_EPOCH_PATTERN
 from .strptime import validateTimeZone
 from .utils import Utils
 from ..helpers import getLogger
@@ -36,7 +37,7 @@ logSys = getLogger(__name__)
 
 logLevel = 6
 
-RE_DATE_PREMATCH = re.compile("\{DATE\}", re.IGNORECASE)
+RE_DATE_PREMATCH = re.compile(r"(?<!\\)\{DATE\}", re.IGNORECASE)
 DD_patternCache = Utils.Cache(maxCount=1000, maxTime=60*60)
 
 
@@ -48,12 +49,18 @@ def _getPatternTemplate(pattern, key=None):
 	template = DD_patternCache.get(key)
 
 	if not template:
-		if key in ("EPOCH", "{^LN-BEG}EPOCH", "^EPOCH"):
+		if "EPOCH" in key:
-			template = DateEpoch(lineBeginOnly=(key != "EPOCH"))
+			if RE_EPOCH_PATTERN.search(pattern):
-		elif key in ("TAI64N", "{^LN-BEG}TAI64N", "^TAI64N"):
+				template = DateEpoch(pattern=pattern, longFrm="LEPOCH" in key)
-			template = DateTai64n(wordBegin=('start' if key != "TAI64N" else False))
+			elif key in ("EPOCH", "{^LN-BEG}EPOCH", "^EPOCH"):
-		else:
+				template = DateEpoch(lineBeginOnly=(key != "EPOCH"))
-			template = DatePatternRegex(pattern)
+			elif key in ("LEPOCH", "{^LN-BEG}LEPOCH", "^LEPOCH"):
+				template = DateEpoch(lineBeginOnly=(key != "LEPOCH"), longFrm=True)
+		if template is None:
+			if key in ("TAI64N", "{^LN-BEG}TAI64N", "^TAI64N"):
+				template = DateTai64n(wordBegin=('start' if key != "TAI64N" else False))
+			else:
+				template = DatePatternRegex(pattern)
 
 	DD_patternCache.set(key, template)
 	return template
@@ -102,9 +109,6 @@ class DateDetectorCache(object):
 		"""Cache Fail2Ban's default template.
 
 		"""
-		if isinstance(template, str):
-			# exact given template with word begin-end boundary:
-			template = _getPatternTemplate(template)
 		# if not already line-begin anchored, additional template, that prefers datetime 
 		# at start of a line (safety+performance feature):
 		name = template.name
@@ -119,60 +123,74 @@ class DateDetectorCache(object):
 		# add template:
 		self.__tmpcache[1].append(template)
 
-	def _addDefaultTemplate(self):
+	DEFAULT_TEMPLATES = [
-		"""Add resp. cache Fail2Ban's default set of date templates.
-		"""
-		self.__tmpcache = [], []
 		# ISO 8601, simple date, optional subsecond and timezone:
 		# 2005-01-23T21:59:59.981746, 2005-01-23 21:59:59, 2005-01-23  8:59:59
 		# simple date: 2005/01/23 21:59:59 
 		# custom for syslog-ng 2006.12.21 06:43:20
-		self._cacheTemplate("%ExY(?P<_sep>[-/.])%m(?P=_sep)%d(?:T|  ?)%H:%M:%S(?:[.,]%f)?(?:\s*%z)?")
+		"%ExY(?P<_sep>[-/.])%m(?P=_sep)%d(?:T|  ?)%H:%M:%S(?:[.,]%f)?(?:\s*%z)?",
 		# asctime with optional day, subsecond and/or year:
 		# Sun Jan 23 21:59:59.011 2005 
-		self._cacheTemplate("(?:%a )?%b %d %k:%M:%S(?:\.%f)?(?: %ExY)?")
+		"(?:%a )?%b %d %k:%M:%S(?:\.%f)?(?: %ExY)?",
 		# asctime with optional day, subsecond and/or year coming after day
 		# http://bugs.debian.org/798923
 		# Sun Jan 23 2005 21:59:59.011
-		self._cacheTemplate("(?:%a )?%b %d %ExY %k:%M:%S(?:\.%f)?")
+		"(?:%a )?%b %d %ExY %k:%M:%S(?:\.%f)?",
 		# simple date too (from x11vnc): 23/01/2005 21:59:59 
 		# and with optional year given by 2 digits: 23/01/05 21:59:59 
 		# (See http://bugs.debian.org/537610)
 		# 17-07-2008 17:23:25
-		self._cacheTemplate("%d(?P<_sep>[-/])%m(?P=_sep)(?:%ExY|%Exy) %k:%M:%S")
+		"%d(?P<_sep>[-/])%m(?P=_sep)(?:%ExY|%Exy) %k:%M:%S",
 		# Apache format optional time zone:
 		# [31/Oct/2006:09:22:55 -0000]
 		# 26-Jul-2007 15:20:52
 		# named 26-Jul-2007 15:20:52.252
 		# roundcube 26-Jul-2007 15:20:52 +0200
-		self._cacheTemplate("%d(?P<_sep>[-/])%b(?P=_sep)%ExY[ :]?%H:%M:%S(?:\.%f)?(?: %z)?")
+		"%d(?P<_sep>[-/])%b(?P=_sep)%ExY[ :]?%H:%M:%S(?:\.%f)?(?: %z)?",
 		# CPanel 05/20/2008:01:57:39
-		self._cacheTemplate("%m/%d/%ExY:%H:%M:%S")
+		"%m/%d/%ExY:%H:%M:%S",
 		# 01-27-2012 16:22:44.252
 		# subseconds explicit to avoid possible %m<->%d confusion
 		# with previous ("%d-%m-%ExY %k:%M:%S" by "%d(?P<_sep>[-/])%m(?P=_sep)(?:%ExY|%Exy) %k:%M:%S")
-		self._cacheTemplate("%m-%d-%ExY %k:%M:%S(?:\.%f)?")
+		"%m-%d-%ExY %k:%M:%S(?:\.%f)?",
 		# Epoch
-		self._cacheTemplate('EPOCH')
+		"EPOCH",
 		# Only time information in the log
-		self._cacheTemplate("{^LN-BEG}%H:%M:%S")
+		"{^LN-BEG}%H:%M:%S",
 		# <09/16/08@05:03:30>
-		self._cacheTemplate("^<%m/%d/%Exy@%H:%M:%S>")
+		"^<%m/%d/%Exy@%H:%M:%S>",
 		# MySQL: 130322 11:46:11
-		self._cacheTemplate("%Exy%Exm%Exd  ?%H:%M:%S")
+		"%Exy%Exm%Exd  ?%H:%M:%S",
 		# Apache Tomcat
-		self._cacheTemplate("%b %d, %ExY %I:%M:%S %p")
+		"%b %d, %ExY %I:%M:%S %p",
 		# ASSP: Apr-27-13 02:33:06
-		self._cacheTemplate("^%b-%d-%Exy %k:%M:%S")
+		"^%b-%d-%Exy %k:%M:%S",
 		# 20050123T215959, 20050123 215959, 20050123  85959
-		self._cacheTemplate("%ExY%Exm%Exd(?:T|  ?)%ExH%ExM%ExS(?:[.,]%f)?(?:\s*%z)?")
+		"%ExY%Exm%Exd(?:T|  ?)%ExH%ExM%ExS(?:[.,]%f)?(?:\s*%z)?",
 		# prefixed with optional named time zone (monit):
 		# PDT Apr 16 21:05:29
-		self._cacheTemplate("(?:%Z )?(?:%a )?%b %d %k:%M:%S(?:\.%f)?(?: %ExY)?")
+		"(?:%Z )?(?:%a )?%b %d %k:%M:%S(?:\.%f)?(?: %ExY)?",
 		# +00:00 Jan 23 21:59:59.011 2005 
-		self._cacheTemplate("(?:%z )?(?:%a )?%b %d %k:%M:%S(?:\.%f)?(?: %ExY)?")
+		"(?:%z )?(?:%a )?%b %d %k:%M:%S(?:\.%f)?(?: %ExY)?",
 		# TAI64N
-		self._cacheTemplate("TAI64N")
+		"TAI64N",
+	]
+
+	@property
+	def defaultTemplates(self):
+		if isinstance(DateDetectorCache.DEFAULT_TEMPLATES[0], str):
+			for i, dt in enumerate(DateDetectorCache.DEFAULT_TEMPLATES):
+				dt = _getPatternTemplate(dt)
+				DateDetectorCache.DEFAULT_TEMPLATES[i] = dt
+		return DateDetectorCache.DEFAULT_TEMPLATES
+
+	def _addDefaultTemplate(self):
+		"""Add resp. cache Fail2Ban's default set of date templates.
+		"""
+		self.__tmpcache = [], []
+		# cache default templates:
+		for dt in self.defaultTemplates:
+			self._cacheTemplate(dt)
 		#
 		self.__templates = self.__tmpcache[0] + self.__tmpcache[1]
 		del self.__tmpcache
@@ -262,8 +280,7 @@ class DateDetector(object):
 					self.addDefaultTemplate(flt)
 					return
 				elif "{DATE}" in key:
-					self.addDefaultTemplate(
+					self.addDefaultTemplate(preMatch=pattern, allDefaults=False)
-						lambda template: not template.flags & DateTemplate.LINE_BEGIN, pattern)
 					return
 				else:
 					template = _getPatternTemplate(pattern, key)
@@ -276,18 +293,20 @@ class DateDetector(object):
 		logSys.debug("  date pattern regex for %r: %s",
 			getattr(template, 'pattern', ''), template.regex)
 
-	def addDefaultTemplate(self, filterTemplate=None, preMatch=None):
+	def addDefaultTemplate(self, filterTemplate=None, preMatch=None, allDefaults=True):
 		"""Add Fail2Ban's default set of date templates.
 		"""
 		ignoreDup = len(self.__templates) > 0
-		for template in DateDetector._defCache.templates:
+		for template in (
+			DateDetector._defCache.templates if allDefaults else DateDetector._defCache.defaultTemplates
+		):
 			# filter if specified:
 			if filterTemplate is not None and not filterTemplate(template): continue
 			# if exact pattern available - create copy of template, contains replaced {DATE} with default regex:
 			if preMatch is not None:
 				# get cached or create a copy with modified name/pattern, using preMatch replacement for {DATE}:
 				template = _getAnchoredTemplate(template,
-					wrap=lambda s: RE_DATE_PREMATCH.sub(lambda m: s, preMatch))
+					wrap=lambda s: RE_DATE_PREMATCH.sub(lambda m: DateTemplate.unboundPattern(s), preMatch))
 			# append date detector template (ignore duplicate if some was added before default):
 			self._appendTemplate(template, ignoreDup=ignoreDup)
 

fail2ban/server/datetemplate.py

@@ -37,8 +37,10 @@ RE_GROUPED = re.compile(r'(?<!(?:\(\?))(?<!\\)\((?!\?)')
 RE_GROUP = ( re.compile(r'^((?:\(\?\w+\))?\^?(?:\(\?\w+\))?)(.*?)(\$?)$'), r"\1(\2)\3" )
 
 RE_EXLINE_BOUND_BEG = re.compile(r'^\{\^LN-BEG\}')
+RE_EXSANC_BOUND_BEG = re.compile(r'^\(\?:\^\|\\b\|\\W\)')
+RE_EXEANC_BOUND_BEG = re.compile(r'\(\?=\\b\|\\W\|\$\)$')
 RE_NO_WRD_BOUND_BEG = re.compile(r'^\(*(?:\(\?\w+\))?(?:\^|\(*\*\*|\(\?:\^)')
-RE_NO_WRD_BOUND_END = re.compile(r'(?<!\\)(?:\$\)?|\*\*\)*)$')
+RE_NO_WRD_BOUND_END = re.compile(r'(?<!\\)(?:\$\)?|\\b|\\s|\*\*\)*)$')
 RE_DEL_WRD_BOUNDS = ( re.compile(r'^\(*(?:\(\?\w+\))?\(*\*\*|(?<!\\)\*\*\)*$'), 
 	                    lambda m: m.group().replace('**', '') )
 
@@ -47,6 +49,9 @@ RE_LINE_BOUND_END = re.compile(r'(?<![\\\|])(?:\$\)?)$')
 
 RE_ALPHA_PATTERN = re.compile(r'(?<!\%)\%[aAbBpc]')
 
+RE_EPOCH_PATTERN = re.compile(r"(?<!\\)\{L?EPOCH\}", re.IGNORECASE)
+
+
 class DateTemplate(object):
 	"""A template which searches for and returns a date from a log line.
 
@@ -128,7 +133,7 @@ class DateTemplate(object):
 		# remove possible special pattern "**" in front and end of regex:
 		regex = RE_DEL_WRD_BOUNDS[0].sub(RE_DEL_WRD_BOUNDS[1], regex)
 		self._regex = regex
-		logSys.debug('  constructed regex %s', regex)
+		logSys.log(7, '  constructed regex %s', regex)
 		self._cRegex = None
 
 	regex = property(getRegex, setRegex, doc=
@@ -179,6 +184,14 @@ class DateTemplate(object):
 		"""
 		raise NotImplementedError("getDate() is abstract")
 
+	@staticmethod
+	def unboundPattern(pattern):
+		return RE_EXEANC_BOUND_BEG.sub('',
+			RE_EXSANC_BOUND_BEG.sub('',
+				RE_EXLINE_BOUND_BEG.sub('', pattern)
+			)
+		)
+
 
 class DateEpoch(DateTemplate):
 	"""A date template which searches for Unix timestamps.
@@ -192,14 +205,25 @@ class DateEpoch(DateTemplate):
 	regex
 	"""
 
-	def __init__(self, lineBeginOnly=False):
+	def __init__(self, lineBeginOnly=False, pattern=None, longFrm=False):
 		DateTemplate.__init__(self)
-		self.name = "Epoch"
+		self.name = "Epoch" if not pattern else pattern
-		if not lineBeginOnly:
+		self._longFrm = longFrm;
-			regex = r"((?:^|(?P<square>(?<=^\[))|(?P<selinux>(?<=\baudit\()))\d{10,11}\b(?:\.\d{3,6})?)(?:(?(selinux)(?=:\d+\)))|(?(square)(?=\])))"
+		self._grpIdx = 1
+		epochRE = r"\d{10,11}\b(?:\.\d{3,6})?"
+		if longFrm:
+			self.name = "LongEpoch" if not pattern else pattern
+			epochRE = r"\d{10,11}(?:\d{3}(?:\.\d{1,6}|\d{3})?)?"
+		if pattern:
+			# pattern should capture/cut out the whole match:
+			regex = "(" + RE_EPOCH_PATTERN.sub(lambda v: "(%s)" % epochRE, pattern) + ")"
+			self._grpIdx = 2
+			self.setRegex(regex)
+		elif not lineBeginOnly:
+			regex = r"((?:^|(?P<square>(?<=^\[))|(?P<selinux>(?<=\baudit\()))%s)(?:(?(selinux)(?=:\d+\)))|(?(square)(?=\])))" % epochRE
 			self.setRegex(regex, wordBegin=False) ;# already line begin resp. word begin anchored
 		else:
-			regex = r"((?P<square>(?<=^\[))?\d{10,11}\b(?:\.\d{3,6})?)(?(square)(?=\]))"
+			regex = r"((?P<square>(?<=^\[))?%s)(?(square)(?=\]))" % epochRE
 			self.setRegex(regex, wordBegin='start', wordEnd=True)
 
 	def getDate(self, line, dateMatch=None, default_tz=None):
@@ -220,8 +244,14 @@ class DateEpoch(DateTemplate):
 		if not dateMatch:
 			dateMatch = self.matchDate(line)
 		if dateMatch:
+			v = dateMatch.group(self._grpIdx)
 			# extract part of format which represents seconds since epoch
-			return (float(dateMatch.group(1)), dateMatch)
+			if self._longFrm and len(v) >= 13:
+				if len(v) >= 16 and '.' not in v:
+					v = float(v) / 1000000
+				else:
+					v = float(v) / 1000
+			return (float(v), dateMatch)
 
 
 class DatePatternRegex(DateTemplate):

fail2ban/server/failregex.py

@@ -89,6 +89,11 @@ def mapTag2Opt(tag):
 	except KeyError:
 		return tag.lower()
 
+
+# alternate names to be merged, e. g. alt_user_1 -> user ...
+ALTNAME_PRE = 'alt_'
+ALTNAME_CRE = re.compile(r'^' + ALTNAME_PRE + r'(.*)(?:_\d+)?$')
+
 ##
 # Regular expression class.
 #
@@ -114,6 +119,14 @@ class Regex:
 		try:
 			self._regexObj = re.compile(regex, re.MULTILINE if multiline else 0)
 			self._regex = regex
+			self._altValues = {}
+			for k in filter(
+				lambda k: len(k) > len(ALTNAME_PRE) and k.startswith(ALTNAME_PRE),
+				self._regexObj.groupindex
+			):
+				n = ALTNAME_CRE.match(k).group(1)
+				self._altValues[k] = n
+			self._altValues = list(self._altValues.items()) if len(self._altValues) else None
 		except sre_constants.error:
 			raise RegexException("Unable to compile regular expression '%s'" %
 								 regex)
@@ -185,6 +198,13 @@ class Regex:
 	def getRegex(self):
 		return self._regex
 	
+	##
+	# Returns string buffer using join of the tupleLines.
+	#
+	@staticmethod
+	def _tupleLinesBuf(tupleLines):
+		return "\n".join(map(lambda v: "".join(v[::2]), tupleLines)) + "\n"
+
 	##
 	# Searches the regular expression.
 	#
@@ -194,8 +214,10 @@ class Regex:
 	# @param a list of tupples. The tupples are ( prematch, datematch, postdatematch )
 	
 	def search(self, tupleLines, orgLines=None):
-		self._matchCache = self._regexObj.search(
+		buf = tupleLines
-			"\n".join("".join(value[::2]) for value in tupleLines) + "\n")
+		if not isinstance(tupleLines, basestring):
+			buf = Regex._tupleLinesBuf(tupleLines)
+		self._matchCache = self._regexObj.search(buf)
 		if self._matchCache:
 			if orgLines is None: orgLines = tupleLines
 			# if single-line:
@@ -248,7 +270,16 @@ class Regex:
 	#
 
 	def getGroups(self):
-		return self._matchCache.groupdict()
+		if not self._altValues:
+			return self._matchCache.groupdict()
+		# merge alternate values (e. g. 'alt_user_1' -> 'user' or 'alt_host' -> 'host'):
+		fail = self._matchCache.groupdict()
+		#fail = fail.copy()
+		for k,n in self._altValues:
+			v = fail.get(k)
+			if v and not fail.get(n):
+				fail[n] = v
+		return fail
 
 	##
 	# Returns skipped lines.

fail2ban/server/filter.py

@@ -30,6 +30,7 @@ import re
 import sys
 import time
 
+from .actions import Actions
 from .failmanager import FailManagerEmpty, FailManager
 from .ipdns import DNSUtils, IPAddr
 from .ticket import FailTicket
@@ -80,6 +81,10 @@ class Filter(JailThread):
 		self.__ignoreSelf = True
 		## The ignore IP list.
 		self.__ignoreIpList = []
+		## External command
+		self.__ignoreCommand = False
+		## Cache for ignoreip:
+		self.__ignoreCache = None
 		## Size of line buffer
 		self.__lineBufferSize = 1
 		## Line buffer
@@ -89,8 +94,6 @@ class Filter(JailThread):
 		self.__lastDate = None
 		## if set, treat log lines without explicit time zone to be in this time zone
 		self.__logtimezone = None
-		## External command
-		self.__ignoreCommand = False
 		## Default or preferred encoding (to decode bytes from file or journal):
 		self.__encoding = PREFER_ENC
 		## Cache temporary holds failures info (used by multi-line for wrapping e. g. conn-id to host):
@@ -396,19 +399,34 @@ class Filter(JailThread):
 		raise Exception("run() is abstract")
 
 	##
-	# Set external command, for ignoredips
+	# External command, for ignoredips
 	#
 
-	def setIgnoreCommand(self, command):
+	@property
+	def ignoreCommand(self):
+		return self.__ignoreCommand
+
+	@ignoreCommand.setter
+	def ignoreCommand(self, command):
 		self.__ignoreCommand = command
 
 	##
-	# Get external command, for ignoredips
+	# Cache parameters for ignoredips
 	#
 
-	def getIgnoreCommand(self):
+	@property
-		return self.__ignoreCommand
+	def ignoreCache(self):
+		return [self.__ignoreCache[0], self.__ignoreCache[1].maxCount, self.__ignoreCache[1].maxTime] \
+			if self.__ignoreCache else None
 
+	@ignoreCache.setter
+	def ignoreCache(self, command):
+		if command:
+			self.__ignoreCache = command['key'], Utils.Cache(
+				maxCount=int(command.get('max-count', 100)), maxTime=MyTime.str2seconds(command.get('max-time', 5*60))
+			)
+		else:
+			self.__ignoreCache = None
 	##
 	# Ban an IP - http://blogs.buanzo.com.ar/2009/04/fail2ban-patch-ban-ip-address-manually.html
 	# Arturo 'Buanzo' Busleiman <buanzo@buanzo.com.ar>
@@ -418,11 +436,12 @@ class Filter(JailThread):
 	def addBannedIP(self, ip):
 		if not isinstance(ip, IPAddr):
 			ip = IPAddr(ip)
-		if self.inIgnoreIPList(ip):
-			logSys.warning('Requested to manually ban an ignored IP %s. User knows best. Proceeding to ban it.', ip)
 
 		unixTime = MyTime.time()
-		self.failManager.addFailure(FailTicket(ip, unixTime), self.failManager.getMaxRetry())
+		ticket = FailTicket(ip, unixTime)
+		if self._inIgnoreIPList(ip, ticket, log_ignore=False):
+			logSys.warning('Requested to manually ban an ignored IP %s. User knows best. Proceeding to ban it.', ip)
+		self.failManager.addFailure(ticket, self.failManager.getMaxRetry())
 
 		# Perform the banning of the IP now.
 		try: # pragma: no branch - exception is the only way out
@@ -487,31 +506,61 @@ class Filter(JailThread):
 	#
 	# Check if the given IP address matches an IP address/DNS or a CIDR
 	# mask in the ignore list.
-	# @param ip IP address object
+	# @param ip IP address object or ticket
 	# @return True if IP address is in ignore list
 
-	def inIgnoreIPList(self, ip, log_ignore=False):
+	def inIgnoreIPList(self, ip, log_ignore=True):
-		if not isinstance(ip, IPAddr):
+		ticket = None
+		if isinstance(ip, FailTicket):
+			ticket = ip
+			ip = ticket.getIP()
+		elif not isinstance(ip, IPAddr):
 			ip = IPAddr(ip)
+		return self._inIgnoreIPList(ip, ticket, log_ignore)
+
+	def _inIgnoreIPList(self, ip, ticket, log_ignore=True):
+		aInfo = None
+		# cached ?
+		if self.__ignoreCache:
+			key, c = self.__ignoreCache
+			if ticket:
+				aInfo = Actions.ActionInfo(ticket, self.jail)
+				key = CommandAction.replaceDynamicTags(key, aInfo)
+			else:
+				aInfo = { 'ip': ip }
+				key = CommandAction.replaceTag(key, aInfo)
+			v = c.get(key)
+			if v is not None:
+				return v
 
 		# check own IPs should be ignored and 'ip' is self IP:
 		if self.__ignoreSelf and ip in DNSUtils.getSelfIPs():
+			self.logIgnoreIp(ip, log_ignore, ignore_source="ignoreself rule")
+			if self.__ignoreCache: c.set(key, True)
 			return True
 
 		for net in self.__ignoreIpList:
 			# check if the IP is covered by ignore IP
 			if ip.isInNet(net):
 				self.logIgnoreIp(ip, log_ignore, ignore_source=("ip" if net.isValid else "dns"))
+				if self.__ignoreCache: c.set(key, True)
 				return True
 
 		if self.__ignoreCommand:
-			command = CommandAction.replaceTag(self.__ignoreCommand, { 'ip': ip } )
+			if ticket:
+				if not aInfo: aInfo = Actions.ActionInfo(ticket, self.jail)
+				command = CommandAction.replaceDynamicTags(self.__ignoreCommand, aInfo)
+			else:
+				if not aInfo: aInfo = { 'ip': ip }
+				command = CommandAction.replaceTag(self.__ignoreCommand, aInfo)
 			logSys.debug('ignore command: %s', command)
 			ret, ret_ignore = CommandAction.executeCmd(command, success_codes=(0, 1))
 			ret_ignore = ret and ret_ignore == 0
 			self.logIgnoreIp(ip, log_ignore and ret_ignore, ignore_source="command")
+			if self.__ignoreCache: c.set(key, ret_ignore)
 			return ret_ignore
 
+		if self.__ignoreCache: c.set(key, False)
 		return False
 
 	def processLine(self, line, date=None):
@@ -548,12 +597,12 @@ class Filter(JailThread):
 				fail = element[3]
 				logSys.debug("Processing line with time:%s and ip:%s", 
 						unixTime, ip)
-				if self.inIgnoreIPList(ip, log_ignore=True):
+				tick = FailTicket(ip, unixTime, data=fail)
+				if self._inIgnoreIPList(ip, tick):
 					continue
 				logSys.info(
 					"[%s] Found %s - %s", self.jailName, ip, datetime.datetime.fromtimestamp(unixTime).strftime("%Y-%m-%d %H:%M:%S")
 				)
-				tick = FailTicket(ip, unixTime, data=fail)
 				self.failManager.addFailure(tick)
 			# reset (halve) error counter (successfully processed line):
 			if self._errors:
@@ -582,37 +631,99 @@ class Filter(JailThread):
 	# @return: a boolean
 
 	def ignoreLine(self, tupleLines):
+		buf = Regex._tupleLinesBuf(tupleLines)
 		for ignoreRegexIndex, ignoreRegex in enumerate(self.__ignoreRegex):
-			ignoreRegex.search(tupleLines)
+			ignoreRegex.search(buf, tupleLines)
 			if ignoreRegex.hasMatched():
 				return ignoreRegexIndex
 		return None
 
+	def _updateUsers(self, fail, user=()):
+		users = fail.get('users')
+		# only for regex contains user:
+		if user:
+			if not users:
+				fail['users'] = users = set()
+			users.add(user)
+			return users
+		return None
+
+	# # ATM incremental (non-empty only) merge deactivated ...
+	# @staticmethod
+	# def _updateFailure(self, mlfidGroups, fail):
+	# 	# reset old failure-ids when new types of id available in this failure:
+	# 	fids = set()
+	# 	for k in ('fid', 'ip4', 'ip6', 'dns'):
+	# 		if fail.get(k):
+	# 			fids.add(k)
+	# 	if fids:
+	# 		for k in ('fid', 'ip4', 'ip6', 'dns'):
+	# 			if k not in fids:
+	# 				try:
+	# 					del mlfidGroups[k]
+	# 				except:
+	# 					pass
+	# 	# update not empty values:
+	# 	mlfidGroups.update(((k,v) for k,v in fail.iteritems() if v))
+
 	def _mergeFailure(self, mlfid, fail, failRegex):
 		mlfidFail = self.mlfidCache.get(mlfid) if self.__mlfidCache else None
+		users = None
+		nfflgs = 0
+		if fail.get("mlfgained"):
+			nfflgs |= 9
+			if not fail.get('nofail'):
+				fail['nofail'] = fail["mlfgained"]
+		elif fail.get('nofail'): nfflgs |= 1
+		if fail.get('mlfforget'): nfflgs |= 2
 		# if multi-line failure id (connection id) known:
 		if mlfidFail:
 			mlfidGroups = mlfidFail[1]
-			# update - if not forget (disconnect/reset):
+			# update users set (hold all users of connect):
-			if not fail.get('mlfforget'):
+			users = self._updateUsers(mlfidGroups, fail.get('user'))
-				mlfidGroups.update(fail)
+			# be sure we've correct current state ('nofail' and 'mlfgained' only from last failure)
-			else:
+			try:
-				self.mlfidCache.unset(mlfid) # remove cached entry
+				del mlfidGroups['nofail']
-			# merge with previous info:
+				del mlfidGroups['mlfgained']
-			fail2 = mlfidGroups.copy()
+			except KeyError:
-			fail2.update(fail)
+				pass
-			if not fail.get('nofail'): # be sure we've correct current state
+			# # ATM incremental (non-empty only) merge deactivated (for future version only),
-				try:
+			# # it can be simulated using alternate value tags, like <F-ALT_VAL>...</F-ALT_VAL>,
-					del fail2['nofail']
+			# # so previous value 'val' will be overwritten only if 'alt_val' is not empty...		
-				except KeyError:
+			# _updateFailure(mlfidGroups, fail)
-					pass
+			#
-			fail2["matches"] = fail.get("matches", []) + failRegex.getMatchedTupleLines()
+			# overwrite multi-line failure with all values, available in fail:
-			fail = fail2
+			mlfidGroups.update(fail)
-		elif not fail.get('mlfforget'):
+			# new merged failure data:
+			fail = mlfidGroups
+			# if forget (disconnect/reset) - remove cached entry:
+			if nfflgs & 2:
+				self.mlfidCache.unset(mlfid)
+		elif not (nfflgs & 2): # not mlfforget
+			users = self._updateUsers(fail, fail.get('user'))
 			mlfidFail = [self.__lastDate, fail]
 			self.mlfidCache.set(mlfid, mlfidFail)
-			if fail.get('nofail'):
+		# check users in order to avoid reset failure by multiple logon-attempts:
-				fail["matches"] = failRegex.getMatchedTupleLines()
+		if users and len(users) > 1:
+			# we've new user, reset 'nofail' because of multiple users attempts:
+			try:
+				del fail['nofail']
+				nfflgs &= ~1 # reset nofail
+			except KeyError:
+				pass
+		# merge matches:
+		if not (nfflgs & 1): # current nofail state (corresponding users)
+			try:
+				m = fail.pop("nofail-matches")
+				m += fail.get("matches", [])
+			except KeyError:
+				m = fail.get("matches", [])
+			if not (nfflgs & 8): # no gain signaled
+				m += failRegex.getMatchedTupleLines()
+			fail["matches"] = m
+		elif not (nfflgs & 2) and (nfflgs & 1): # not mlfforget and nofail:
+			fail["nofail-matches"] = fail.get("nofail-matches", []) + failRegex.getMatchedTupleLines()
+		# return merged:
 		return fail
 
 
@@ -626,6 +737,7 @@ class Filter(JailThread):
 	def findFailure(self, tupleLine, date=None):
 		failList = list()
 
+		ll = logSys.getEffectiveLevel()
 		returnRawHost = self.returnRawHost
 		cidr = IPAddr.CIDR_UNSPEC
 		if self.__useDns == "raw":
@@ -635,7 +747,7 @@ class Filter(JailThread):
 		# Checks if we mut ignore this line.
 		if self.ignoreLine([tupleLine[::2]]) is not None:
 			# The ignoreregex matched. Return.
-			logSys.log(7, "Matched ignoreregex and was \"%s\" ignored",
+			if ll <= 7: logSys.log(7, "Matched ignoreregex and was \"%s\" ignored",
 				"".join(tupleLine[::2]))
 			return failList
 
@@ -662,7 +774,7 @@ class Filter(JailThread):
 			date = self.__lastDate
 
 		if self.checkFindTime and date is not None and date < MyTime.time() - self.getFindTime():
-			logSys.log(5, "Ignore line since time %s < %s - %s", 
+			if ll <= 5: logSys.log(5, "Ignore line since time %s < %s - %s", 
 				date, MyTime.time(), self.getFindTime())
 			return failList
 
@@ -671,71 +783,75 @@ class Filter(JailThread):
 				self.__lineBuffer + [tupleLine[:3]])[-self.__lineBufferSize:]
 		else:
 			orgBuffer = self.__lineBuffer = [tupleLine[:3]]
-		logSys.log(5, "Looking for match of %r", self.__lineBuffer)
+		if ll <= 5: logSys.log(5, "Looking for match of %r", self.__lineBuffer)
+		buf = Regex._tupleLinesBuf(self.__lineBuffer)
 
 		# Pre-filter fail regex (if available):
 		preGroups = {}
 		if self.__prefRegex:
-			if logSys.getEffectiveLevel() <= logging.HEAVYDEBUG: # pragma: no cover
+			if ll <= 5: logSys.log(5, "  Looking for prefregex %r", self.__prefRegex.getRegex())
-				logSys.log(5, "  Looking for prefregex %r", self.__prefRegex.getRegex())
+			self.__prefRegex.search(buf, self.__lineBuffer)
-			self.__prefRegex.search(self.__lineBuffer)
 			if not self.__prefRegex.hasMatched():
-				logSys.log(5, "  Prefregex not matched")
+				if ll <= 5: logSys.log(5, "  Prefregex not matched")
 				return failList
 			preGroups = self.__prefRegex.getGroups()
-			logSys.log(7, "  Pre-filter matched %s", preGroups)
+			if ll <= 7: logSys.log(7, "  Pre-filter matched %s", preGroups)
 			repl = preGroups.get('content')
 			# Content replacement:
 			if repl:
 				del preGroups['content']
-				self.__lineBuffer = [('', '', repl)]
+				self.__lineBuffer, buf = [('', '', repl)], None
 
 		# Iterates over all the regular expressions.
 		for failRegexIndex, failRegex in enumerate(self.__failRegex):
-			if logSys.getEffectiveLevel() <= logging.HEAVYDEBUG: # pragma: no cover
-				logSys.log(5, "  Looking for failregex %r", failRegex.getRegex())
-			failRegex.search(self.__lineBuffer, orgBuffer)
-			if not failRegex.hasMatched():
-				continue
-			# The failregex matched.
-			logSys.log(7, "  Matched %s", failRegex)
-			# Checks if we must ignore this match.
-			if self.ignoreLine(failRegex.getMatchedTupleLines()) \
-					is not None:
-				# The ignoreregex matched. Remove ignored match.
-				self.__lineBuffer = failRegex.getUnmatchedTupleLines()
-				logSys.log(7, "  Matched ignoreregex and was ignored")
-				if not self.checkAllRegex:
-					break
-				else:
-					continue
-			if date is None:
-				logSys.warning(
-					"Found a match for %r but no valid date/time "
-					"found for %r. Please try setting a custom "
-					"date pattern (see man page jail.conf(5)). "
-					"If format is complex, please "
-					"file a detailed issue on"
-					" https://github.com/fail2ban/fail2ban/issues "
-					"in order to get support for this format.",
-					 "\n".join(failRegex.getMatchedLines()), timeText)
-				continue
-			self.__lineBuffer = failRegex.getUnmatchedTupleLines()
-			# retrieve failure-id, host, etc from failure match:
 			try:
+				# buffer from tuples if changed: 
+				if buf is None:
+					buf = Regex._tupleLinesBuf(self.__lineBuffer)
+				if ll <= 5: logSys.log(5, "  Looking for failregex %d - %r", failRegexIndex, failRegex.getRegex())
+				failRegex.search(buf, orgBuffer)
+				if not failRegex.hasMatched():
+					continue
+				# current failure data (matched group dict):
+				fail = failRegex.getGroups()
+				# The failregex matched.
+				if ll <= 7: logSys.log(7, "  Matched failregex %d: %s", failRegexIndex, fail)
+				# Checks if we must ignore this match.
+				if self.ignoreLine(failRegex.getMatchedTupleLines()) \
+						is not None:
+					# The ignoreregex matched. Remove ignored match.
+					self.__lineBuffer, buf = failRegex.getUnmatchedTupleLines(), None
+					if ll <= 7: logSys.log(7, "  Matched ignoreregex and was ignored")
+					if not self.checkAllRegex:
+						break
+					else:
+						continue
+				if date is None:
+					logSys.warning(
+						"Found a match for %r but no valid date/time "
+						"found for %r. Please try setting a custom "
+						"date pattern (see man page jail.conf(5)). "
+						"If format is complex, please "
+						"file a detailed issue on"
+						" https://github.com/fail2ban/fail2ban/issues "
+						"in order to get support for this format.",
+						 "\n".join(failRegex.getMatchedLines()), timeText)
+					continue
+				# we should check all regex (bypass on multi-line, otherwise too complex):
+				if not self.checkAllRegex or self.getMaxLines() > 1:
+					self.__lineBuffer, buf = failRegex.getUnmatchedTupleLines(), None
+				# merge data if multi-line failure:
 				raw = returnRawHost
 				if preGroups:
-					fail = preGroups.copy()
+					currFail, fail = fail, preGroups.copy()
-					fail.update(failRegex.getGroups())
+					fail.update(currFail)
-				else:
-					fail = failRegex.getGroups()
 				# first try to check we have mlfid case (caching of connection id by multi-line):
 				mlfid = fail.get('mlfid')
 				if mlfid is not None:
 					fail = self._mergeFailure(mlfid, fail, failRegex)
 					# bypass if no-failure case:
 					if fail.get('nofail'):
-						logSys.log(7, "Nofail by mlfid %r in regex %s: %s",
+						if ll <= 7: logSys.log(7, "Nofail by mlfid %r in regex %s: %s",
 							mlfid, failRegexIndex, fail.get('mlfforget', "waiting for failure"))
 						if not self.checkAllRegex: return failList
 				else:
@@ -764,7 +880,7 @@ class Filter(JailThread):
 						cidr = IPAddr.CIDR_RAW
 				# if mlfid case (not failure):
 				if host is None:
-					logSys.log(7, "No failure-id by mlfid %r in regex %s: %s",
+					if ll <= 7: logSys.log(7, "No failure-id by mlfid %r in regex %s: %s",
 						mlfid, failRegexIndex, fail.get('mlfforget', "waiting for identifier"))
 					if not self.checkAllRegex: return failList
 					ips = [None]
@@ -825,9 +941,6 @@ class FileFilter(Filter):
 			self.__logs[path] = log
 			logSys.info("Added logfile: %r (pos = %s, hash = %s)" , path, log.getPos(), log.getHash())
 			if autoSeek:
-				# if default, seek to "current time" - "find time":
-				if isinstance(autoSeek, bool):
-					autoSeek = MyTime.time() - self.getFindTime()
 				self.__autoSeek[path] = autoSeek
 			self._addLogPath(path)			# backend specific
 
@@ -927,28 +1040,31 @@ class FileFilter(Filter):
 				if e.errno != 2: # errno.ENOENT
 					logSys.exception(e)
 				return False
-			except OSError as e: # pragma: no cover - requires race condition to tigger this
+			except OSError as e: # pragma: no cover - requires race condition to trigger this
 				logSys.error("Error opening %s", filename)
 				logSys.exception(e)
 				return False
-			except Exception as e: # pragma: no cover - Requires implemention error in FileContainer to generate
+			except Exception as e: # pragma: no cover - Requires implementation error in FileContainer to generate
 				logSys.error("Internal error in FileContainer open method - please report as a bug to https://github.com/fail2ban/fail2ban/issues")
 				logSys.exception(e)
 				return False
 
 			# seek to find time for first usage only (prevent performance decline with polling of big files)
-			if self.__autoSeek.get(filename):
+			if self.__autoSeek:
-				startTime = self.__autoSeek[filename]
+				startTime = self.__autoSeek.pop(filename, None)
-				del self.__autoSeek[filename]
+				if startTime:
-				# prevent completely read of big files first time (after start of service), 
+					# if default, seek to "current time" - "find time":
-				# initial seek to start time using half-interval search algorithm:
+					if isinstance(startTime, bool):
-				try:
+						startTime = MyTime.time() - self.getFindTime()
-					self.seekToTime(log, startTime)
+					# prevent completely read of big files first time (after start of service), 
-				except Exception as e: # pragma: no cover
+					# initial seek to start time using half-interval search algorithm:
-					logSys.error("Error during seek to start time in \"%s\"", filename)
+					try:
-					raise
+						self.seekToTime(log, startTime)
-					logSys.exception(e)
+					except Exception as e: # pragma: no cover
-					return False
+						logSys.error("Error during seek to start time in \"%s\"", filename)
+						raise
+						logSys.exception(e)
+						return False
 
 			if has_content:
 				while not self.idle:
@@ -1039,7 +1155,7 @@ class FileFilter(Filter):
 				movecntr -= 1
 				if movecntr <= 0:
 		 			break
-				# we have found large area without any date mached 
+				# we have found large area without any date matched 
 				# or end of search - try min position (because can be end of previous line):
 				if minp != lastPos:
 					lastPos = tryPos = minp

fail2ban/server/filterpoll.py

@@ -158,7 +158,7 @@ class FilterPoll(FileFilter):
 			self.__prevStats[filename] = stats
 			return True
 		except Exception as e:
-			# stil alive (may be deleted because multi-threaded):
+			# still alive (may be deleted because multi-threaded):
 			if not self.getLog(filename) or self.__prevStats.get(filename) is None:
 				logSys.warning("Log %r seems to be down: %s", filename, e)
 				return False

fail2ban/server/filterpyinotify.py

@@ -374,8 +374,11 @@ class FilterPyinotify(FileFilter):
 	def stop(self):
 		# stop filter thread:
 		super(FilterPyinotify, self).stop()
-		if self.__notifier: # stop the notifier
+		try:
-			self.__notifier.stop()
+			if self.__notifier: # stop the notifier
+				self.__notifier.stop()
+		except AttributeError: # pragma: no cover
+			if self.__notifier: raise
 
 	##
 	# Wait for exit with cleanup.

fail2ban/server/filtersystemd.py

@@ -87,7 +87,7 @@ class FilterSystemd(JournalFilter): # pragma: systemd no cover
 			args['files'] = list(set(files))
 
 		try:
-			args['flags'] = kwargs.pop('journalflags')
+			args['flags'] = int(kwargs.pop('journalflags'))
 		except KeyError:
 			pass
 

fail2ban/server/ipdns.py

@@ -64,7 +64,7 @@ class DNSUtils:
 		if ips is not None: 
 			return ips
 		# retrieve ips
-		ips = list()
+		ips = set()
 		saveerr = None
 		for fam, ipfam in ((socket.AF_INET, IPAddr.FAM_IPv4), (socket.AF_INET6, IPAddr.FAM_IPv6)):
 			try:
@@ -75,7 +75,7 @@ class DNSUtils:
 					# (some python-versions resp. host configurations causes returning of integer there):
 					ip = IPAddr(str(result[4][0]), ipfam)
 					if ip.isValid:
-						ips.append(ip)
+						ips.add(ip)
 			except Exception as e:
 				saveerr = e
 		if not ips and saveerr:
@@ -103,19 +103,19 @@ class DNSUtils:
 	def textToIp(text, useDns):
 		""" Return the IP of DNS found in a given text.
 		"""
-		ipList = list()
+		ipList = set()
 		# Search for plain IP
 		plainIP = IPAddr.searchIP(text)
 		if plainIP is not None:
 			ip = IPAddr(plainIP)
 			if ip.isValid:
-				ipList.append(ip)
+				ipList.add(ip)
 
 		# If we are allowed to resolve -- give it a try if nothing was found
 		if useDns in ("yes", "warn") and not ipList:
 			# Try to get IP from possible DNS
 			ip = DNSUtils.dnsToIp(text)
-			ipList.extend(ip)
+			ipList.update(ip)
 			if ip and useDns == "warn":
 				logSys.warning("Determined IP using DNS Lookup: %s = %s",
 					text, ipList)

fail2ban/server/server.py

@@ -37,7 +37,8 @@ from .filter import FileFilter, JournalFilter
 from .transmitter import Transmitter
 from .asyncserver import AsyncServer, AsyncServerException
 from .. import version
-from ..helpers import getLogger, extractOptions, str2LogLevel, getVerbosityFormat, excepthook
+from ..helpers import getLogger, _as_bool, extractOptions, str2LogLevel, \
+	getVerbosityFormat, excepthook
 
 # Gets the instance of the logger.
 logSys = getLogger(__name__)
@@ -313,7 +314,7 @@ class Server:
 	
 	# Filter
 	def setIgnoreSelf(self, name, value):
-		self.__jails[name].filter.ignoreSelf = value
+		self.__jails[name].filter.ignoreSelf = _as_bool(value)
 	
 	def getIgnoreSelf(self, name):
 		return self.__jails[name].filter.ignoreSelf
@@ -390,10 +391,17 @@ class Server:
 		return self.__jails[name].filter.getLogTimeZone()
 
 	def setIgnoreCommand(self, name, value):
-		self.__jails[name].filter.setIgnoreCommand(value)
+		self.__jails[name].filter.ignoreCommand = value
 
 	def getIgnoreCommand(self, name):
-		return self.__jails[name].filter.getIgnoreCommand()
+		return self.__jails[name].filter.ignoreCommand
+
+	def setIgnoreCache(self, name, value):
+		value, options = extractOptions("cache["+value+"]")
+		self.__jails[name].filter.ignoreCache = options
+
+	def getIgnoreCache(self, name):
+		return self.__jails[name].filter.ignoreCache
 
 	def setPrefRegex(self, name, value):
 		flt = self.__jails[name].filter
@@ -565,10 +573,12 @@ class Server:
 			if systarget == "INHERITED":
 				self.__logTarget = target
 				return True
+			padding = logOptions.get('padding')
 			# set a format which is simpler for console use
-			fmt = "%(name)-24s[%(process)d]: %(levelname)-7s %(message)s"
 			if systarget == "SYSLOG":
 				facility = logOptions.get('facility', 'DAEMON').upper()
+				# backwards compatibility - default no padding for syslog handler:
+				if padding is None: padding = '0'
 				try:
 					facility = getattr(logging.handlers.SysLogHandler, 'LOG_' + facility)
 				except AttributeError: # pragma: no cover
@@ -626,18 +636,22 @@ class Server:
 			# If handler don't already add date to the message:
 			addtime = logOptions.get('datetime')
 			if addtime is not None:
-				addtime = addtime in ('1', 'on', 'true', 'yes')
+				addtime = _as_bool(addtime)
 			else:
 				addtime = systarget not in ("SYSLOG", "SYSOUT")
+			if padding is not None:
+				padding = _as_bool(padding) 
+			else:
+				padding = True
 			# If log-format is redefined in options:
 			if logOptions.get('format', '') != '':
 				fmt = logOptions.get('format')
-			# verbose log-format:
+			else:
-			elif self.__verbose is not None and self.__verbose > 2: # pragma: no cover
+				# verbose log-format:
-				fmt = getVerbosityFormat(self.__verbose-1,
+				verbose = 0
-					addtime=addtime)
+				if self.__verbose is not None and self.__verbose > 2: # pragma: no cover
-			elif addtime:
+					verbose = self.__verbose-1
-				fmt = "%(asctime)s " + fmt
+				fmt = getVerbosityFormat(verbose, addtime=addtime, padding=padding)
 			# tell the handler to use this format
 			hdlr.setFormatter(logging.Formatter(fmt))
 			logger.addHandler(hdlr)

fail2ban/server/ticket.py

@@ -138,7 +138,7 @@ class Ticket(object):
 		self._data['matches'] = matches or []
 
 	def getMatches(self):
-		return [(line if isinstance(line, basestring) else "".join(line)) \
+		return [(line if not isinstance(line, (list, tuple)) else "".join(line)) \
 			for line in self._data.get('matches', ())]
 
 	@property

fail2ban/server/transmitter.py

@@ -115,6 +115,9 @@ class Transmitter:
 			return cnt
 		elif command[0] == "echo":
 			return command[1:]
+		elif command[0] == "server-status":
+			logSys.debug("Status: ready")
+			return "Server ready"
 		elif command[0] == "sleep":
 			value = command[1]
 			time.sleep(float(value))
@@ -197,6 +200,10 @@ class Transmitter:
 			value = command[2]
 			self.__server.setIgnoreCommand(name, value)
 			return self.__server.getIgnoreCommand(name)
+		elif command[1] == "ignorecache":
+			value = command[2]
+			self.__server.setIgnoreCache(name, value)
+			return self.__server.getIgnoreCache(name)
 		elif command[1] == "addlogpath":
 			value = command[2]
 			tail = False
@@ -355,6 +362,8 @@ class Transmitter:
 			return self.__server.getIgnoreIP(name)
 		elif command[1] == "ignorecommand":
 			return self.__server.getIgnoreCommand(name)
+		elif command[1] == "ignorecache":
+			return self.__server.getIgnoreCache(name)
 		elif command[1] == "prefregex":
 			return self.__server.getPrefRegex(name)
 		elif command[1] == "failregex":

fail2ban/server/utils.py

@@ -27,9 +27,15 @@ import os
 import signal
 import subprocess
 import sys
+from	 threading import Lock
 import time
 from ..helpers import getLogger, _merge_dicts, uni_decode
 
+try:
+	from collections import OrderedDict
+except ImportError: # pragma: 3.x no cover
+	OrderedDict = dict
+
 if sys.version_info >= (3, 3):
 	import importlib.machinery
 else:
@@ -69,7 +75,8 @@ class Utils():
 
 		def __init__(self, *args, **kwargs):
 			self.setOptions(*args, **kwargs)
-			self._cache = {}
+			self._cache = OrderedDict()
+			self.__lock = Lock()
 
 		def setOptions(self, maxCount=1000, maxTime=60):
 			self.maxCount = maxCount
@@ -83,7 +90,7 @@ class Utils():
 			if v: 
 				if v[1] > time.time():
 					return v[0]
-				del self._cache[k]
+				self.unset(k)
 			return defv
 			
 		def set(self, k, v):
@@ -91,18 +98,27 @@ class Utils():
 			cache = self._cache  # for shorter local access
 			# clean cache if max count reached:
 			if len(cache) >= self.maxCount:
-				for (ck, cv) in cache.items():
+				# avoid multiple modification of list multi-threaded:
-					if cv[1] < t:
+				with self.__lock:
-						del cache[ck]
+					if len(cache) >= self.maxCount:
-				# if still max count - remove any one:
+						for (ck, cv) in cache.items():
-				if len(cache) >= self.maxCount:
+							# if expired:
-					cache.popitem()
+							if cv[1] <= t:
+								self.unset(ck)
+							elif OrderedDict is not dict:
+								break
+						# if still max count - remove any one:
+						if len(cache) >= self.maxCount:
+							if OrderedDict is not dict: # first (older):
+								cache.popitem(False)
+							else: # pragma: 3.x no cover
+								cache.popitem()
 			cache[k] = (v, t + self.maxTime)
 
 		def unset(self, k):
 			try:
 				del self._cache[k]
-			except KeyError: # pragme: no cover
+			except KeyError:
 				pass
 
 

fail2ban/tests/action_d/test_badips.py

@@ -20,16 +20,41 @@
 import os
 import unittest
 import sys
+from functools import wraps
+from socket import timeout
+from ssl import SSLError
 
+from ..actiontestcase import CallingMap
 from ..dummyjail import DummyJail
-from ..utils import CONFIG_DIR
+from ..servertestcase import IPAddr
+from ..utils import LogCaptureTestCase, CONFIG_DIR
+
+if sys.version_info >= (3, ): # pragma: 2.x no cover
+	from urllib.error import HTTPError, URLError
+else: # pragma: 3.x no cover
+	from urllib2 import HTTPError, URLError
+
+def skip_if_not_available(f):
+	"""Helper to decorate tests to skip in case of timeout/http-errors like "502 bad gateway".
+	"""
+	@wraps(f)
+	def wrapper(self, *args):
+		try:
+			return f(self, *args)
+		except (SSLError, HTTPError, URLError, timeout) as e: # pragma: no cover - timeout/availability issues
+			if not isinstance(e, timeout) and 'timed out' not in str(e):
+				if not hasattr(e, 'code') or e.code > 200 and e.code <= 404:
+					raise
+			raise unittest.SkipTest('Skip test because of %s' % e)
+	return wrapper
 
 if sys.version_info >= (2,7): # pragma: no cover - may be unavailable
-	class BadIPsActionTest(unittest.TestCase):
+	class BadIPsActionTest(LogCaptureTestCase):
 
 		available = True, None
+		pythonModule = None
 		modAction = None
-		
+
 		def setUp(self):
 			"""Call before every test case."""
 			super(BadIPsActionTest, self).setUp()
@@ -39,20 +64,27 @@ if sys.version_info >= (2,7): # pragma: no cover - may be unavailable
 
 			self.jail.actions.add("test")
 
-			pythonModule = os.path.join(CONFIG_DIR, "action.d", "badips.py")
+			pythonModuleName = os.path.join(CONFIG_DIR, "action.d", "badips.py")
 
 			# check availability (once if not alive, used shorter timeout as in test cases):
 			if BadIPsActionTest.available[0]:
 				if not BadIPsActionTest.modAction:
-					BadIPsActionTest.modAction = self.jail.actions._load_python_module(pythonModule).Action
+					if not BadIPsActionTest.pythonModule:
-				BadIPsActionTest.available = BadIPsActionTest.modAction.isAvailable(timeout=2 if unittest.F2B.fast else 10)
+						BadIPsActionTest.pythonModule = self.jail.actions._load_python_module(pythonModuleName)
+					BadIPsActionTest.modAction = BadIPsActionTest.pythonModule.Action
+					self.jail.actions._load_python_module(pythonModuleName)
+				BadIPsActionTest.available = BadIPsActionTest.modAction.isAvailable(timeout=2 if unittest.F2B.fast else 30)
 			if not BadIPsActionTest.available[0]:
 				raise unittest.SkipTest('Skip test because service is not available: %s' % BadIPsActionTest.available[1])
 
-			self.jail.actions.add("badips", pythonModule, initOpts={
+			self.jail.actions.add("badips", pythonModuleName, initOpts={
 				'category': "ssh",
 				'banaction': "test",
-				'timeout': (3 if unittest.F2B.fast else 30),
+				'age': "2w",
+				'score': 5,
+				'key': "fail2ban-test-suite",
+				#'bankey': "fail2ban-test-suite",
+				'timeout': (3 if unittest.F2B.fast else 60),
 				})
 			self.action = self.jail.actions["badips"]
 
@@ -63,6 +95,7 @@ if sys.version_info >= (2,7): # pragma: no cover - may be unavailable
 				self.action._timer.cancel()
 			super(BadIPsActionTest, self).tearDown()
 
+		@skip_if_not_available
 		def testCategory(self):
 			categories = self.action.getCategories()
 			self.assertIn("ssh", categories)
@@ -78,17 +111,20 @@ if sys.version_info >= (2,7): # pragma: no cover - may be unavailable
 			# but valid for blacklisting.
 			self.action.bancategory = "mail"
 
+		@skip_if_not_available
 		def testScore(self):
 			self.assertRaises(ValueError, setattr, self.action, "score", -5)
-			self.action.score = 5
+			self.action.score = 3
-			self.action.score = "5"
+			self.action.score = "3"
 
+		@skip_if_not_available
 		def testBanaction(self):
 			self.assertRaises(
 				ValueError, setattr, self.action, "banaction",
 				"invalid-action")
 			self.action.banaction = "test"
 
+		@skip_if_not_available
 		def testUpdateperiod(self):
 			self.assertRaises(
 				ValueError, setattr, self.action, "updateperiod", -50)
@@ -97,11 +133,24 @@ if sys.version_info >= (2,7): # pragma: no cover - may be unavailable
 			self.action.updateperiod = 900
 			self.action.updateperiod = "900"
 
-		def testStart(self):
+		@skip_if_not_available
+		def testStartStop(self):
 			self.action.start()
-			self.assertTrue(len(self.action._bannedips) > 10)
+			self.assertTrue(len(self.action._bannedips) > 10,
-
+				"%s is fewer as 10: %r" % (len(self.action._bannedips), self.action._bannedips))
-		def testStop(self):
-			self.testStart()
 			self.action.stop()
 			self.assertTrue(len(self.action._bannedips) == 0)
+
+		@skip_if_not_available
+		def testBanIP(self):
+			aInfo = CallingMap({
+				'ip': IPAddr('192.0.2.1')
+			})
+			self.action.ban(aInfo)
+			self.assertLogged('badips.com: ban', wait=True)
+			self.pruneLog()
+			# produce an error using wrong category/IP:
+			self.action._category = 'f2b-this-category-dont-available-test-suite-only'
+			aInfo['ip'] = ''
+			self.assertRaises(BadIPsActionTest.pythonModule.HTTPError, self.action.ban, aInfo)
+			self.assertLogged('IP is invalid', 'invalid category', wait=True, all=False)

fail2ban/tests/action_d/test_smtp.py

@@ -52,6 +52,7 @@ class SMTPActionTest(unittest.TestCase):
 
 	def setUp(self):
 		"""Call before every test case."""
+		unittest.F2B.SkipIfCfgMissing(action='smtp.py')
 		super(SMTPActionTest, self).setUp()
 		self.jail = DummyJail()
 		pythonModule = os.path.join(CONFIG_DIR, "action.d", "smtp.py")

fail2ban/tests/actiontestcase.py

@@ -567,13 +567,18 @@ class CommandActionTest(LogCaptureTestCase):
 			'b': lambda self: self['a'] + 6,
 			'c': ''
 		})
-		s = repr(m)
+		s = repr(m); # only stored values (no calculated)
+		self.assertNotIn("'a': ", s)
+		self.assertNotIn("'b': ", s)
+		self.assertIn("'c': ''", s)
+
+		s = m._asrepr(True) # all values (including calculated)
 		self.assertIn("'a': 5", s)
 		self.assertIn("'b': 11", s)
 		self.assertIn("'c': ''", s)
 		
 		m['c'] = lambda self: self['xxx'] + 7; # unresolvable
-		s = repr(m)
+		s = m._asrepr(True)
 		self.assertIn("'a': 5", s)
 		self.assertIn("'b': 11", s)
 		self.assertIn("'c': ", s) # presents as callable

fail2ban/tests/banmanagertestcase.py

@@ -156,7 +156,7 @@ class StatusExtendedCymruInfo(unittest.TestCase):
 		if tc.available[0]:
 			cymru_info = self.__banManager.getBanListExtendedCymruInfo(
 				timeout=(2 if unittest.F2B.fast else 20))
-		else:
+		else: # pragma: no cover - availability (once after error case only)
 			cymru_info = tc.available[1]
 		if cymru_info.get("error"): # pragma: no cover - availability
 			tc.available = False, cymru_info

fail2ban/tests/clientreadertestcase.py

@@ -28,7 +28,8 @@ import re
 import shutil
 import tempfile
 import unittest
-from ..client.configreader import ConfigReader, ConfigReaderUnshared, NoSectionError
+from ..client.configreader import ConfigReader, ConfigReaderUnshared, \
+	DefinitionInitConfigReader, NoSectionError
 from ..client import configparserinc
 from ..client.jailreader import JailReader, extractOptions
 from ..client.filterreader import FilterReader
@@ -45,8 +46,6 @@ TEST_FILES_DIR_SHARE_CFG = {}
 from .utils import CONFIG_DIR
 CONFIG_DIR_SHARE_CFG = unittest.F2B.share_config
 
-STOCK = os.path.exists(os.path.join('config', 'fail2ban.conf'))
-
 IMPERFECT_CONFIG = os.path.join(os.path.dirname(__file__), 'config')
 IMPERFECT_CONFIG_SHARE_CFG = {}
 
@@ -127,6 +126,54 @@ option = %s
 		self._remove("c.d/90.conf")
 		self.assertEqual(self._getoption(), 2)
 
+	def testLocalInIncludes(self):
+		self._write("c.conf", value=None, content="""
+[INCLUDES]
+before = ib.conf
+after  = ia.conf
+[Definition]
+test = %(default/test)s
+""")
+		self._write("ib.conf", value=None, content="""
+[DEFAULT]
+test = A
+[Definition]
+option = 1
+""")
+		self._write("ib.local", value=None, content="""
+[DEFAULT]
+test = B
+[Definition]
+option = 2
+""")
+		self._write("ia.conf", value=None, content="""
+[DEFAULT]
+test = C
+[Definition]
+oafter = 3
+""")
+		self._write("ia.local", value=None, content="""
+[DEFAULT]
+test = D
+[Definition]
+oafter = 4
+""")
+		class TestDefConfReader(DefinitionInitConfigReader):
+			_configOpts = {
+			  "option": ["int", None],
+			  "oafter": ["int", None],
+				"test":   ["string", None],
+			}
+		self.c = TestDefConfReader('c', 'option', {})
+		self.c.setBaseDir(self.d)
+		self.assertTrue(self.c.read())
+		self.c.getOptions({}, all=True)
+		o = self.c.getCombined()
+		# test local wins (overwrite all options):
+		self.assertEqual(o.get('option'), 2)
+		self.assertEqual(o.get('oafter'), 4)
+		self.assertEqual(o.get('test'), 'D')
+
 	def testInterpolations(self):
 		self.assertFalse(self.c.read('i'))	# nothing is there yet
 		self._write("i.conf", value=None, content="""
@@ -190,7 +237,7 @@ y = %(jail/y)s
 		self.assertEqual(self.c.get('jail', 'c'), 'def-c,b:"jail-b-test-b-def-b,a:`jail-a-test-a-def-a`"')
 		self.assertEqual(self.c.get('jail', 'd'), 'def-d-b:"def-b,a:`jail-a-test-a-def-a`"')
 		self.assertEqual(self.c.get('test', 'c'), 'def-c,b:"test-b-def-b,a:`test-a-def-a`"')
-		self.assertEqual(self.c.get('test', 'd'),  'def-d-b:"def-b,a:`test-a-def-a`"')
+		self.assertEqual(self.c.get('test', 'd'), 'def-d-b:"def-b,a:`test-a-def-a`"')
 		self.assertEqual(self.c.get('DEFAULT', 'c'), 'def-c,b:"def-b,a:`def-a`"')
 		self.assertEqual(self.c.get('DEFAULT', 'd'), 'def-d-b:"def-b,a:`def-a`"')
 		self.assertRaises(Exception, self.c.get, 'test', 'x')
@@ -246,15 +293,15 @@ class JailReaderTest(LogCaptureTestCase):
 		self.assertTrue(jail.isEnabled())
 		self.assertLogged("Invalid filter definition 'flt[test'")
 
-	if STOCK:
+	def testStockSSHJail(self):
-		def testStockSSHJail(self):
+		unittest.F2B.SkipIfCfgMissing(stock=True)
-			jail = JailReader('sshd', basedir=CONFIG_DIR, share_config=CONFIG_DIR_SHARE_CFG) # we are running tests from root project dir atm
+		jail = JailReader('sshd', basedir=CONFIG_DIR, share_config=CONFIG_DIR_SHARE_CFG) # we are running tests from root project dir atm
-			self.assertTrue(jail.read())
+		self.assertTrue(jail.read())
-			self.assertTrue(jail.getOptions())
+		self.assertTrue(jail.getOptions())
-			self.assertFalse(jail.isEnabled())
+		self.assertFalse(jail.isEnabled())
-			self.assertEqual(jail.getName(), 'sshd')
+		self.assertEqual(jail.getName(), 'sshd')
-			jail.setName('ssh-funky-blocker')
+		jail.setName('ssh-funky-blocker')
-			self.assertEqual(jail.getName(), 'ssh-funky-blocker')
+		self.assertEqual(jail.getName(), 'ssh-funky-blocker')
 		
 	def testSplitOption(self):
 		# Simple example
@@ -268,7 +315,7 @@ class JailReaderTest(LogCaptureTestCase):
 		self.assertEqual(('mail--ho_is', {}), extractOptions("mail--ho_is"))
 
 		self.assertEqual(('mail--ho_is', {}), extractOptions("mail--ho_is['s']"))
-		#self.printLog()
+		#print(self.getLog())
 		#self.assertLogged("Invalid argument ['s'] in ''s''")
 
 		self.assertEqual(('mail', {'a': ','}), extractOptions("mail[a=',']"))
@@ -307,6 +354,7 @@ class JailReaderTest(LogCaptureTestCase):
 		self.assertEqual(expected2, result)
 
 	def testVersionAgent(self):
+		unittest.F2B.SkipIfCfgMissing(stock=True)
 		jail = JailReader('blocklisttest', force_enable=True, basedir=CONFIG_DIR)
 		# emulate jail.read(), because such jail not exists:
 		ConfigReader.read(jail, "jail"); 
@@ -438,9 +486,20 @@ class FilterReaderTest(unittest.TestCase):
 		self.assertSortedEqual(c, output)
 
 	def testFilterReaderSubstitionKnown(self):
-		output = [['set', 'jailname', 'addfailregex', 'to=test,sweet@example.com,test2,sweet@example.com fromip=<IP>']]
+		output = [['set', 'jailname', 'addfailregex', '^to=test,sweet@example.com,test2,sweet@example.com fromip=<IP>$']]
 		filterName, filterOpt = extractOptions(
-			'substition[honeypot="<sweet>,<known/honeypot>", sweet="test,<known/honeypot>,test2"]')
+			'substition[failregex="^<known/failregex>$", honeypot="<sweet>,<known/honeypot>", sweet="test,<known/honeypot>,test2"]')
+		filterReader = FilterReader('substition', "jailname", filterOpt,
+		  share_config=TEST_FILES_DIR_SHARE_CFG, basedir=TEST_FILES_DIR)
+		filterReader.read()
+		filterReader.getOptions(None)
+		c = filterReader.convert()
+		self.assertSortedEqual(c, output)
+
+	def testFilterReaderSubstitionSection(self):
+		output = [['set', 'jailname', 'addfailregex', '^\\s*to=fail2ban@localhost fromip=<IP>\\s*$']]
+		filterName, filterOpt = extractOptions(
+			'substition[failregex="^\\s*<Definition/failregex>\\s*$", honeypot="<default/honeypot>"]')
 		filterReader = FilterReader('substition', "jailname", filterOpt,
 		  share_config=TEST_FILES_DIR_SHARE_CFG, basedir=TEST_FILES_DIR)
 		filterReader.read()
@@ -518,7 +577,7 @@ class JailsReaderTestCache(LogCaptureTestCase):
 			# how many times jail.local was read:
 			cnt = self._getLoggedReadCount('jail.local')
 			# if cnt > 1:
-			# 	self.printLog()
+			# 	print(self.getLog())
 			self.assertTrue(cnt == 1, "Unexpected count by reading of jail files, cnt = %s" % cnt)
 
 			# read whole configuration like a file2ban-client, again ...
@@ -597,222 +656,226 @@ class JailsReaderTest(LogCaptureTestCase):
 		self.assertNotLogged("Skipping...")
 		self.assertLogged("No file(s) found for glob /weapons/of/mass/destruction")
 
-	if STOCK:
+	def testReadStockActionConf(self):
-		def testReadStockActionConf(self):
+		unittest.F2B.SkipIfCfgMissing(stock=True)
-			for actionConfig in glob.glob(os.path.join(CONFIG_DIR, 'action.d', '*.conf')):
+		for actionConfig in glob.glob(os.path.join(CONFIG_DIR, 'action.d', '*.conf')):
-				actionName = os.path.basename(actionConfig).replace('.conf', '')
+			actionName = os.path.basename(actionConfig).replace('.conf', '')
-				actionReader = ActionReader(actionName, "TEST", {}, basedir=CONFIG_DIR)
+			actionReader = ActionReader(actionName, "TEST", {}, basedir=CONFIG_DIR)
-				self.assertTrue(actionReader.read())
+			self.assertTrue(actionReader.read())
-				try:
+			try:
-					actionReader.getOptions({})	  # populate _opts
+				actionReader.getOptions({})	  # populate _opts
-				except Exception as e: # pragma: no cover
+			except Exception as e: # pragma: no cover
-					self.fail("action %r\n%s: %s" % (actionName, type(e).__name__, e))
+				self.fail("action %r\n%s: %s" % (actionName, type(e).__name__, e))
-				if not actionName.endswith('-common'):
+			if not actionName.endswith('-common'):
-					self.assertIn('Definition', actionReader.sections(),
+				self.assertIn('Definition', actionReader.sections(),
-						msg="Action file %r is lacking [Definition] section" % actionConfig)
+					msg="Action file %r is lacking [Definition] section" % actionConfig)
-					# all must have some actionban defined
+				# all must have some actionban defined
-					self.assertTrue(actionReader._opts.get('actionban', '').strip(),
+				self.assertTrue(actionReader._opts.get('actionban', '').strip(),
-						msg="Action file %r is lacking actionban" % actionConfig)
+					msg="Action file %r is lacking actionban" % actionConfig)
-					# test name of jail is set in options (also if not supplied within parameters):
+				# test name of jail is set in options (also if not supplied within parameters):
-					opts = actionReader.getCombined(
+				opts = actionReader.getCombined(
-						ignore=CommandAction._escapedTags | set(('timeout', 'bantime')))
+					ignore=CommandAction._escapedTags | set(('timeout', 'bantime')))
-					self.assertEqual(opts.get('name'), 'TEST',
+				self.assertEqual(opts.get('name'), 'TEST',
-						msg="Action file %r does not contains jail-name 'f2b-TEST'" % actionConfig)
+					msg="Action file %r does not contains jail-name 'f2b-TEST'" % actionConfig)
-					# and the name is substituted (test several actions surely contains name-interpolation):
+				# and the name is substituted (test several actions surely contains name-interpolation):
-					if actionName in ('pf', 'iptables-allports', 'iptables-multiport'):
+				if actionName in ('pf', 'iptables-allports', 'iptables-multiport'):
-						#print('****', actionName, opts.get('actionstart', ''))
+					#print('****', actionName, opts.get('actionstart', ''))
-						self.assertIn('f2b-TEST', opts.get('actionstart', ''),
+					self.assertIn('f2b-TEST', opts.get('actionstart', ''),
-							msg="Action file %r: interpolation of actionstart does not contains jail-name 'f2b-TEST'" % actionConfig)
+						msg="Action file %r: interpolation of actionstart does not contains jail-name 'f2b-TEST'" % actionConfig)
 
-		def testReadStockJailConf(self):
+	def testReadStockJailConf(self):
-			jails = JailsReader(basedir=CONFIG_DIR, share_config=CONFIG_DIR_SHARE_CFG) # we are running tests from root project dir atm
+		unittest.F2B.SkipIfCfgMissing(stock=True)
-			self.assertTrue(jails.read())		  # opens fine
+		jails = JailsReader(basedir=CONFIG_DIR, share_config=CONFIG_DIR_SHARE_CFG) # we are running tests from root project dir atm
-			self.assertTrue(jails.getOptions())	  # reads fine
+		self.assertTrue(jails.read())		  # opens fine
-			comm_commands = jails.convert()
+		self.assertTrue(jails.getOptions())	  # reads fine
-			# by default None of the jails is enabled and we get no
+		comm_commands = jails.convert()
-			# commands to communicate to the server
+		# by default None of the jails is enabled and we get no
-			self.assertEqual(comm_commands, [])
+		# commands to communicate to the server
+		self.assertEqual(comm_commands, [])
 
-			# TODO: make sure this is handled well
+		# TODO: make sure this is handled well
-			## We should not "read" some bogus jail
+		## We should not "read" some bogus jail
-			#old_comm_commands = comm_commands[:]   # make a copy
+		#old_comm_commands = comm_commands[:]   # make a copy
-			#self.assertRaises(ValueError, jails.getOptions, "BOGUS")
+		#self.assertRaises(ValueError, jails.getOptions, "BOGUS")
-			#self.printLog()
+		#print(self.getLog())
-			#self.assertLogged("No section: 'BOGUS'")
+		#self.assertLogged("No section: 'BOGUS'")
-			## and there should be no side-effects
+		## and there should be no side-effects
-			#self.assertEqual(jails.convert(), old_comm_commands)
+		#self.assertEqual(jails.convert(), old_comm_commands)
 
-			allFilters = set()
+		allFilters = set()
 
-			# All jails must have filter and action set
+		# All jails must have filter and action set
-			# TODO: evolve into a parametric test
+		# TODO: evolve into a parametric test
-			for jail in jails.sections():
+		for jail in jails.sections():
-				if jail == 'INCLUDES':
+			if jail == 'INCLUDES':
-					continue
+				continue
-				filterName = jails.get(jail, 'filter')
+			filterName = jails.get(jail, 'filter')
-				filterName, filterOpt = extractOptions(filterName)
+			filterName, filterOpt = extractOptions(filterName)
-				allFilters.add(filterName)
+			allFilters.add(filterName)
-				self.assertTrue(len(filterName))
+			self.assertTrue(len(filterName))
-				# moreover we must have a file for it
+			# moreover we must have a file for it
-				# and it must be readable as a Filter
+			# and it must be readable as a Filter
-				filterReader = FilterReader(filterName, jail, filterOpt, 
+			filterReader = FilterReader(filterName, jail, filterOpt, 
+				share_config=CONFIG_DIR_SHARE_CFG, basedir=CONFIG_DIR)
+			self.assertTrue(filterReader.read(),"Failed to read filter:" + filterName)		  # opens fine
+			filterReader.getOptions({})	  # reads fine
+
+			#  test if filter has failregex set
+			self.assertTrue(filterReader._opts.get('failregex', '').strip())
+
+			actions = jails.get(jail, 'action')
+			self.assertTrue(len(actions.strip()))
+
+			# somewhat duplicating here what is done in JailsReader if
+			# the jail is enabled
+			for act in actions.split('\n'):
+				actName, actOpt = extractOptions(act)
+				self.assertTrue(len(actName))
+				self.assertTrue(isinstance(actOpt, dict))
+				if actName == 'iptables-multiport':
+					self.assertIn('port', actOpt)
+
+				actionReader = ActionReader(actName, jail, {}, 
 					share_config=CONFIG_DIR_SHARE_CFG, basedir=CONFIG_DIR)
-				self.assertTrue(filterReader.read(),"Failed to read filter:" + filterName)		  # opens fine
+				self.assertTrue(actionReader.read())
-				filterReader.getOptions({})	  # reads fine
+				actionReader.getOptions({})	  # populate _opts
+				cmds = actionReader.convert()
+				self.assertTrue(len(cmds))
 
-				#  test if filter has failregex set
+				# all must have some actionban
-				self.assertTrue(filterReader._opts.get('failregex', '').strip())
+				self.assertTrue(actionReader._opts.get('actionban', '').strip())
 
-				actions = jails.get(jail, 'action')
+	# Verify that all filters found under config/ have a jail
-				self.assertTrue(len(actions.strip()))
+	def testReadStockJailFilterComplete(self):
+		unittest.F2B.SkipIfCfgMissing(stock=True)
+		jails = JailsReader(basedir=CONFIG_DIR, force_enable=True, share_config=CONFIG_DIR_SHARE_CFG)
+		self.assertTrue(jails.read())             # opens fine
+		self.assertTrue(jails.getOptions())       # reads fine
+		# grab all filter names
+		filters = set(os.path.splitext(os.path.split(a)[1])[0]
+			for a in glob.glob(os.path.join('config', 'filter.d', '*.conf'))
+				if not (a.endswith('common.conf') or a.endswith('-aggressive.conf')))
+		# get filters of all jails (filter names without options inside filter[...])
+		filters_jail = set(
+			extractOptions(jail.options['filter'])[0] for jail in jails.jails
+		)
+		self.maxDiff = None
+		self.assertTrue(filters.issubset(filters_jail),
+				"More filters exists than are referenced in stock jail.conf %r" % filters.difference(filters_jail))
+		self.assertTrue(filters_jail.issubset(filters),
+				"Stock jail.conf references non-existent filters %r" % filters_jail.difference(filters))
 
-				# somewhat duplicating here what is done in JailsReader if
+	def testReadStockJailConfForceEnabled(self):
-				# the jail is enabled
+		unittest.F2B.SkipIfCfgMissing(stock=True)
-				for act in actions.split('\n'):
+		# more of a smoke test to make sure that no obvious surprises
-					actName, actOpt = extractOptions(act)
+		# on users' systems when enabling shipped jails
-					self.assertTrue(len(actName))
+		jails = JailsReader(basedir=CONFIG_DIR, force_enable=True, share_config=CONFIG_DIR_SHARE_CFG) # we are running tests from root project dir atm
-					self.assertTrue(isinstance(actOpt, dict))
+		self.assertTrue(jails.read())		  # opens fine
-					if actName == 'iptables-multiport':
+		self.assertTrue(jails.getOptions())	  # reads fine
-						self.assertIn('port', actOpt)
+		comm_commands = jails.convert(allow_no_files=True)
 
-					actionReader = ActionReader(actName, jail, {}, 
+		# by default we have lots of jails ;)
-						share_config=CONFIG_DIR_SHARE_CFG, basedir=CONFIG_DIR)
+		self.assertTrue(len(comm_commands))
-					self.assertTrue(actionReader.read())
-					actionReader.getOptions({})	  # populate _opts
-					cmds = actionReader.convert()
-					self.assertTrue(len(cmds))
 
-					# all must have some actionban
+		# some common sanity checks for commands
-					self.assertTrue(actionReader._opts.get('actionban', '').strip())
+		for command in comm_commands:
+			if len(command) >= 3 and [command[0], command[2]] == ['set', 'bantime']:
+				self.assertTrue(MyTime.str2seconds(command[3]) > 0)
+				
 
-		# Verify that all filters found under config/ have a jail
+		# and we know even some of them by heart
-		def testReadStockJailFilterComplete(self):
+		for j in ['sshd', 'recidive']:
-			jails = JailsReader(basedir=CONFIG_DIR, force_enable=True, share_config=CONFIG_DIR_SHARE_CFG)
+			# by default we have 'auto' backend ATM, but some distributions can overwrite it, 
-			self.assertTrue(jails.read())             # opens fine
+			# (e.g. fedora default is 'systemd') therefore let check it without backend...
-			self.assertTrue(jails.getOptions())       # reads fine
+			self.assertIn(['add', j], 
-			# grab all filter names
+				(cmd[:2] for cmd in comm_commands if len(cmd) == 3 and cmd[0] == 'add'))
-			filters = set(os.path.splitext(os.path.split(a)[1])[0]
+			# and warn on useDNS
-				for a in glob.glob(os.path.join('config', 'filter.d', '*.conf'))
+			self.assertIn(['set', j, 'usedns', 'warn'], comm_commands)
-					if not (a.endswith('common.conf') or a.endswith('-aggressive.conf')))
+			self.assertIn(['start', j], comm_commands)
-			# get filters of all jails (filter names without options inside filter[...])
-			filters_jail = set(
-				extractOptions(jail.options['filter'])[0] for jail in jails.jails
-			)
-			self.maxDiff = None
-			self.assertTrue(filters.issubset(filters_jail),
-					"More filters exists than are referenced in stock jail.conf %r" % filters.difference(filters_jail))
-			self.assertTrue(filters_jail.issubset(filters),
-					"Stock jail.conf references non-existent filters %r" % filters_jail.difference(filters))
 
-		def testReadStockJailConfForceEnabled(self):
+		# last commands should be the 'start' commands
-			# more of a smoke test to make sure that no obvious surprises
+		self.assertEqual(comm_commands[-1][0], 'start')
-			# on users' systems when enabling shipped jails
-			jails = JailsReader(basedir=CONFIG_DIR, force_enable=True, share_config=CONFIG_DIR_SHARE_CFG) # we are running tests from root project dir atm
-			self.assertTrue(jails.read())		  # opens fine
-			self.assertTrue(jails.getOptions())	  # reads fine
-			comm_commands = jails.convert(allow_no_files=True)
 
-			# by default we have lots of jails ;)
+		for j in  jails._JailsReader__jails:
-			self.assertTrue(len(comm_commands))
+			actions = j._JailReader__actions
+			jail_name = j.getName()
+			# make sure that all of the jails have actions assigned,
+			# otherwise it makes little to no sense
+			self.assertTrue(len(actions),
+							msg="No actions found for jail %s" % jail_name)
 
-			# some common sanity checks for commands
+			# Test for presence of blocktype (in relation to gh-232)
-			for command in comm_commands:
+			for action in actions:
-				if len(command) >= 3 and [command[0], command[2]] == ['set', 'bantime']:
+				commands = action.convert()
-					self.assertTrue(MyTime.str2seconds(command[3]) > 0)
+				action_name = action.getName()
-					
+				if '<blocktype>' in str(commands):
+					# Verify that it is among cInfo
+					self.assertIn('blocktype', action._initOpts)
+					# Verify that we have a call to set it up
+					blocktype_present = False
+					target_command = [jail_name, 'action', action_name]
+					for command in commands:
+						if (len(command) > 4 and command[0] == 'multi-set' and
+							command[1:4] == target_command):
+								blocktype_present = ('blocktype' in [cmd[0] for cmd in command[4]])
+						elif (len(command) > 5 and command[0] == 'set' and
+							command[1:4] == target_command and command[4] == 'blocktype'): # pragma: no cover - because of multi-set
+								blocktype_present = True
+						if blocktype_present:
+							break
+					self.assertTrue(
+						blocktype_present,
+						msg="Found no %s command among %s"
+							% (target_command, str(commands)) )
 
-			# and we know even some of them by heart
+	def testStockConfigurator(self):
-			for j in ['sshd', 'recidive']:
+		unittest.F2B.SkipIfCfgMissing(stock=True)
-				# by default we have 'auto' backend ATM, but some distributions can overwrite it, 
+		configurator = Configurator()
-				# (e.g. fedora default is 'systemd') therefore let check it without backend...
+		configurator.setBaseDir(CONFIG_DIR)
-				self.assertIn(['add', j], 
+		self.assertEqual(configurator.getBaseDir(), CONFIG_DIR)
-					(cmd[:2] for cmd in comm_commands if len(cmd) == 3 and cmd[0] == 'add'))
-				# and warn on useDNS
-				self.assertIn(['set', j, 'usedns', 'warn'], comm_commands)
-				self.assertIn(['start', j], comm_commands)
 
-			# last commands should be the 'start' commands
+		configurator.readEarly()
-			self.assertEqual(comm_commands[-1][0], 'start')
+		opts = configurator.getEarlyOptions()
+		# our current default settings
+		self.assertEqual(opts['socket'], '/var/run/fail2ban/fail2ban.sock')
+		self.assertEqual(opts['pidfile'], '/var/run/fail2ban/fail2ban.pid')
 
-			for j in  jails._JailsReader__jails:
+		configurator.readAll()
-				actions = j._JailReader__actions
+		configurator.getOptions()
-				jail_name = j.getName()
+		configurator.convertToProtocol()
-				# make sure that all of the jails have actions assigned,
+		commands = configurator.getConfigStream()
-				# otherwise it makes little to no sense
-				self.assertTrue(len(actions),
-								msg="No actions found for jail %s" % jail_name)
 
-				# Test for presence of blocktype (in relation to gh-232)
+		# verify that dbfile comes before dbpurgeage
-				for action in actions:
+		def find_set(option):
-					commands = action.convert()
+			for i, e in enumerate(commands):
-					action_name = action.getName()
+				if e[0] == 'set' and e[1] == option:
-					if '<blocktype>' in str(commands):
+					return i
-						# Verify that it is among cInfo
+			raise ValueError("Did not find command 'set %s' among commands %s"
-						self.assertIn('blocktype', action._initOpts)
+							 % (option, commands))
-						# Verify that we have a call to set it up
-						blocktype_present = False
-						target_command = [jail_name, 'action', action_name]
-						for command in commands:
-							if (len(command) > 4 and command[0] == 'multi-set' and
-								command[1:4] == target_command):
-									blocktype_present = ('blocktype' in [cmd[0] for cmd in command[4]])
-							elif (len(command) > 5 and command[0] == 'set' and
-								command[1:4] == target_command and command[4] == 'blocktype'): # pragma: no cover - because of multi-set
-									blocktype_present = True
-							if blocktype_present:
-								break
-						self.assertTrue(
-							blocktype_present,
-							msg="Found no %s command among %s"
-								% (target_command, str(commands)) )
 
-		def testStockConfigurator(self):
+		# Set up of logging should come first
-			configurator = Configurator()
+		self.assertEqual(find_set('syslogsocket'), 0)
-			configurator.setBaseDir(CONFIG_DIR)
+		self.assertEqual(find_set('loglevel'), 1)
-			self.assertEqual(configurator.getBaseDir(), CONFIG_DIR)
+		self.assertEqual(find_set('logtarget'), 2)
+		# then dbfile should be before dbpurgeage
+		self.assertTrue(find_set('dbpurgeage') > find_set('dbfile'))
 
-			configurator.readEarly()
+		# and there is logging information left to be passed into the
-			opts = configurator.getEarlyOptions()
+		# server
-			# our current default settings
+		self.assertSortedEqual(commands,
-			self.assertEqual(opts['socket'], '/var/run/fail2ban/fail2ban.sock')
+						 [['set', 'dbfile',
-			self.assertEqual(opts['pidfile'], '/var/run/fail2ban/fail2ban.pid')
+							'/var/lib/fail2ban/fail2ban.sqlite3'],
+						  ['set', 'dbpurgeage', '1d'],
+						  ['set', 'loglevel', "INFO"],
+						  ['set', 'logtarget', '/var/log/fail2ban.log'],
+						  ['set', 'syslogsocket', 'auto']])
 
-			configurator.readAll()
+		# and if we force change configurator's fail2ban's baseDir
-			configurator.getOptions()
+		# there should be an error message (test visually ;) --
-			configurator.convertToProtocol()
+		# otherwise just a code smoke test)
-			commands = configurator.getConfigStream()
+		configurator._Configurator__jails.setBaseDir('/tmp')
-
+		self.assertEqual(configurator._Configurator__jails.getBaseDir(), '/tmp')
-			# verify that dbfile comes before dbpurgeage
+		self.assertEqual(configurator.getBaseDir(), CONFIG_DIR)
-			def find_set(option):
-				for i, e in enumerate(commands):
-					if e[0] == 'set' and e[1] == option:
-						return i
-				raise ValueError("Did not find command 'set %s' among commands %s"
-								 % (option, commands))
-
-			# Set up of logging should come first
-			self.assertEqual(find_set('syslogsocket'), 0)
-			self.assertEqual(find_set('loglevel'), 1)
-			self.assertEqual(find_set('logtarget'), 2)
-			# then dbfile should be before dbpurgeage
-			self.assertTrue(find_set('dbpurgeage') > find_set('dbfile'))
-
-			# and there is logging information left to be passed into the
-			# server
-			self.assertSortedEqual(commands,
-							 [['set', 'dbfile',
-								'/var/lib/fail2ban/fail2ban.sqlite3'],
-							  ['set', 'dbpurgeage', '1d'],
-							  ['set', 'loglevel', "INFO"],
-							  ['set', 'logtarget', '/var/log/fail2ban.log'],
-							  ['set', 'syslogsocket', 'auto']])
-
-			# and if we force change configurator's fail2ban's baseDir
-			# there should be an error message (test visually ;) --
-			# otherwise just a code smoke test)
-			configurator._Configurator__jails.setBaseDir('/tmp')
-			self.assertEqual(configurator._Configurator__jails.getBaseDir(), '/tmp')
-			self.assertEqual(configurator.getBaseDir(), CONFIG_DIR)
 
 	@with_tmpdir
 	def testMultipleSameAction(self, basedir):