Merge commit '0.8.8-160-g74e76e0' into 0.9

* commit '0.8.8-160-g74e76e0': (65 commits)
  TST+BF: Use separate coveragerc for Travis CI
  RF+TST: bring inBanList back from private to protected and enabled its rudimentary unittests
  TST: coverage ignore Travis CI python virtual environments
  ENH: increase waiting to 4 sec for gamin/pyinotify
  TST+BF: Fix incorrect commands for coveralls support
  TST: Add support for coveralls for python 2.6 and python 2.7
  ENH: deleted trailing spaces in fail2ban- cmdline tools
  DOC: minor change -- refer to the fail2ban manpage
  TST: be more aggressive in cleanup of temp files + use mktemp instead of mkstemp
  ENH(BF?): overload open() (for buffering) within filtertestcase to guarantee atomic writing
  BF: delay check for the existence of config directory until read()
  DOC: minor fix ups of manpages. fixes #159
  non-static (get|set)BaseDir for Configurator. fixes #160
  ENH: Slight tune ups for fresh SOGo filter + comment into the sample log file
  ENH: postfix filter -- react also on (450 4.7.1) with empty from/to. fixes #126
  TST: basic testing of reading the shipped jail.conf (forcing all jails to be enabled)
  ENH: allow to force enable all jails (for testing), do not crash for jails without actions (just warn)
  ENH: minor -- add default value into the warning if option had none provided
  ENH: _copy_lines_between_files -- read all needed, and only then write/flush at once
  ENH: move pyinotify callback debug message into callback + delay string interpolations
  ...

Conflicts:
	fail2ban-testcases
	testcases/clientreadertestcase.py -- fix for setBaseDir will follow

.gitignore

@@ -4,3 +4,5 @@ dist
 *.pyc
 htmlcov
 .coverage
+*.orig
+*.rej

.travis.yml

@@ -6,6 +6,9 @@ python:
   - "2.6"
   - "2.7"
 install:
-  - "pip install pyinotify"
+  - pip install pyinotify
+  - if [[ $TRAVIS_PYTHON_VERSION == 2.[6-7] ]]; then pip install -q coveralls; fi
 script:
-    - python ./fail2ban-testcases
+  - if [[ $TRAVIS_PYTHON_VERSION == 2.[6-7] ]]; then coverage run --rcfile=.travis_coveragerc fail2ban-testcases; else python ./fail2ban-testcases; fi
+after_script:
+  - if [[ $TRAVIS_PYTHON_VERSION == 2.[6-7] ]]; then coveralls; fi

.travis_coveragerc

@@ -0,0 +1,7 @@
+
+[run]
+branch = True
+omit =
+    /usr/*
+    /home/travis/virtualenv/*
+    server/filtergamin.py

DEVELOP

@@ -44,7 +44,7 @@ coverage html
 
 Then look at htmlcov/index.html and see how much coverage your test cases
 exert over the codebase. Full coverage is a good thing however it may not be
-complete. Try to ensure tests cover as many independant paths through the
+complete. Try to ensure tests cover as many independent paths through the
 code.
 
 Manual Execution. To run in a development environment do:
@@ -67,7 +67,7 @@ status test
 
 
 Coding Standards
-================ 
+================
 
 Style
 -----
@@ -106,9 +106,12 @@ Git
 
 Use the following tags in your commit messages:
 
-'ENH:' for enhancements
 'BF:' for bug fixes
 'DOC:' for documentation fixes
+'ENH:' for enhancements
+'TST:' for commits concerning tests only (thus not touching the main code-base)
+
+Multiple tags could be joined with +, e.g. "BF+TST:".
 
 Adding Actions
 --------------
@@ -246,24 +249,30 @@ Takes care about executing start/check/ban/unban/stop commands
 Releasing
 =========
 
-Ensure the version is correct in ./common/version.py
+# Ensure the version is correct in ./common/version.py
 
-Add/finalize the corresponding entry in the ChangeLog
+# Add/finalize the corresponding entry in the ChangeLog
 
-# update man pages
+# Update man pages
-(cd man ; ./generate-man )
 
-git commit -m 'update man pages for release' man/*
+  (cd man ; ./generate-man )
+  git commit -m 'update man pages for release' man/*
 
-python setup.py check
+# Make sure the tests pass
-python setup.py sdist
-python setup.py bdist_rpm
-python setup.py upload
 
-Run the following and update the wiki with output:
+  ./fail2ban-testcases-all
 
-python -c 'import common.protocol; common.protocol.printWiki()'
+# Prepare/upload source and rpm binary distributions
 
-email users and development list of release
+  python setup.py check
+  python setup.py sdist
+  python setup.py bdist_rpm
+  python setup.py upload
+
+# Run the following and update the wiki with output:
+
+  python -c 'import common.protocol; common.protocol.printWiki()'
+
+# Email users and development list of release
 
 TODO notifying distributors etc.

MANIFEST

@@ -42,6 +42,7 @@ server/banmanager.py
 server/datetemplate.py
 server/mytime.py
 server/failregex.py
+testcases/files/testcase-usedns.log
 testcases/banmanagertestcase.py
 testcases/failmanagertestcase.py
 testcases/clientreadertestcase.py
@@ -50,6 +51,7 @@ testcases/__init__.py
 testcases/datedetectortestcase.py
 testcases/actiontestcase.py
 testcases/servertestcase.py
+testcases/sockettestcase.py
 testcases/files/testcase01.log
 testcases/files/testcase02.log
 testcases/files/testcase03.log
@@ -57,6 +59,7 @@ testcases/files/testcase04.log
 setup.py
 setup.cfg
 common/__init__.py
+common/exceptions.py
 common/helpers.py
 common/version.py
 common/protocol.py
@@ -88,6 +91,17 @@ config/filter.d/vsftpd.conf
 config/filter.d/webmin-auth.conf
 config/filter.d/wuftpd.conf
 config/filter.d/xinetd-fail.conf
+config/filter.d/asterisk.conf
+config/filter.d/dovecot.conf
+config/filter.d/dropbear.conf
+config/filter.d/lighttpd-auth.conf
+config/filter.d/recidive.conf
+config/filter.d/roundcube-auth.conf
+config/action.d/dummy.conf
+config/action.d/iptables-ipset-proto4.conf
+config/action.d/iptables-ipset-proto6.conf
+config/action.d/iptables-xt_recent-echo.conf
+config/action.d/route.conf
 config/action.d/complain.conf
 config/action.d/dshield.conf
 config/action.d/hostsdeny.conf
@@ -110,6 +124,8 @@ config/action.d/sendmail-whois-lines.conf
 config/action.d/shorewall.conf
 config/fail2ban.conf
 man/fail2ban-client.1
+man/fail2ban.1
+man/jail.conf.5
 man/fail2ban-client.h2m
 man/fail2ban-server.1
 man/fail2ban-server.h2m

README

@@ -51,10 +51,12 @@ call fail2ban-server directly.
 Configuration:
 --------------
 
-You can configure Fail2ban using the files in /etc/fail2ban. It is possible to
+You can configure Fail2Ban using the files in /etc/fail2ban. It is
-configure the server using commands sent to it by fail2ban-client. The available
+possible to configure the server using commands sent to it by
-commands are described in the man page of fail2ban-client. Please refer to it or
+fail2ban-client. The available commands are described in the
-to the website: http://www.fail2ban.org
+fail2ban-client(1) manpage.  Also see fail2ban(1) manpage for further
+references and find even more documentation on the website:
+http://www.fail2ban.org
 
 Contact:
 --------

THANKS

@@ -13,6 +13,7 @@ Christian Rauch
 Christoph Haas
 Christos Psonis
 Daniel B. Cid
+Daniel Black
 David Nutter
 Eric Gerbier
 Guillaume Delvit
@@ -38,6 +39,7 @@ Robert Edeker
 Russell Odom
 Sireyessire
 Stephen Gildea
+Steven Hiscocks
 Tom Pike
 Tyler
 Vaclav Misek

TODO

@@ -16,9 +16,9 @@ Legend:
 - Run tests though all filters/examples files - (see sshd example file) as unit
   test
 
-- Removed relative imports
+* Removed relative imports
 
-- Cleanup fail2ban-client and fail2ban-server. Move code to server/ and client/
+* Cleanup fail2ban-client and fail2ban-server. Move code to server/ and client/
 
 - Add timeout to external commands (signal alarm, watchdog thread, etc)
 

client/actionreader.py

@@ -35,8 +35,8 @@ logSys = logging.getLogger("fail2ban.client.config")
 
 class ActionReader(ConfigReader):
 	
-	def __init__(self, action, name):
+	def __init__(self, action, name, **kwargs):
-		ConfigReader.__init__(self)
+		ConfigReader.__init__(self, **kwargs)
 		self.__file = action[0]
 		self.__cInfo = action[1]
 		self.__name = name

client/configreader.py

@@ -27,7 +27,7 @@ __date__ = "$Date$"
 __copyright__ = "Copyright (c) 2004 Cyril Jaquier"
 __license__ = "GPL"
 
-import logging, os
+import glob, logging, os
 from configparserinc import SafeConfigParserWithIncludes
 from ConfigParser import NoOptionError, NoSectionError
 
@@ -35,36 +35,64 @@ from ConfigParser import NoOptionError, NoSectionError
 logSys = logging.getLogger("fail2ban.client.config")
 
 class ConfigReader(SafeConfigParserWithIncludes):
+
+	DEFAULT_BASEDIR = '/etc/fail2ban'
 	
-	BASE_DIRECTORY = "/etc/fail2ban/"
+	def __init__(self, basedir=None):
-	
-	def __init__(self):
 		SafeConfigParserWithIncludes.__init__(self)
+		self.setBaseDir(basedir)
 		self.__opts = None
 	
-	#@staticmethod
+	def setBaseDir(self, basedir):
-	def setBaseDir(folderName):
+		if basedir is None:
-		path = folderName.rstrip('/')
+			basedir = ConfigReader.DEFAULT_BASEDIR	# stock system location
-		ConfigReader.BASE_DIRECTORY = path + '/'
+		self._basedir = basedir.rstrip('/')
-	setBaseDir = staticmethod(setBaseDir)
+	
-		
+	def getBaseDir(self):
-	#@staticmethod
+		return self._basedir
-	def getBaseDir():
-		return ConfigReader.BASE_DIRECTORY
-	getBaseDir = staticmethod(getBaseDir)
 	
 	def read(self, filename):
-		basename = ConfigReader.BASE_DIRECTORY + filename
+		if not (os.path.exists(self._basedir) and os.access(self._basedir, os.R_OK | os.X_OK)):
-		logSys.debug("Reading " + basename)
+			raise ValueError("Base configuration directory %s either does not exist "
-		bConf = basename + ".conf"
+							 "or is not accessible" % self._basedir)
-		bLocal = basename + ".local"
+		basename = os.path.join(self._basedir, filename)
-		if os.path.exists(bConf) or os.path.exists(bLocal):
+		logSys.debug("Reading configs for %s under %s "  % (basename, self._basedir))
-			SafeConfigParserWithIncludes.read(self, [bConf, bLocal])
+		config_files = [ basename + ".conf",
+						 basename + ".local" ]
+
+		# choose only existing ones
+		config_files = filter(os.path.exists, config_files)
+
+		# possible further customizations under a .conf.d directory
+		config_dir = basename + '.d'
+		if os.path.exists(config_dir):
+			if os.path.isdir(config_dir) and os.access(config_dir, os.X_OK | os.R_OK):
+				# files must carry .conf suffix as well
+				config_files += sorted(glob.glob('%s/*.conf' % config_dir))
+			else:
+				logSys.warn("%s exists but not a directory or not accessible"
+							 % config_dir)
+
+		# check if files are accessible, warn if any is not accessible
+		# and remove it from the list
+		config_files_accessible = []
+		for f in config_files:
+			if os.access(f, os.R_OK):
+				config_files_accessible.append(f)
+			else:
+				logSys.warn("%s exists but not accessible - skipping" % f)
+
+		if len(config_files_accessible):
+			# at least one config exists and accessible
+			SafeConfigParserWithIncludes.read(self, config_files_accessible)
 			return True
 		else:
-			logSys.error(bConf + " and " + bLocal + " do not exist")
+			logSys.error("Found no accessible config files for %r " % filename
+						 + (["under %s" % self.getBaseDir(),
+							 "among existing ones: " + ', '.join(config_files)][bool(len(config_files))]))
+
 			return False
-	
+
 	##
 	# Read the options.
 	#
@@ -94,8 +122,8 @@ class ConfigReader(SafeConfigParserWithIncludes):
 				values[option[1]] = option[2]
 			except NoOptionError:
 				if not option[2] == None:
-					logSys.warn("'%s' not defined in '%s'. Using default value"
+					logSys.warn("'%s' not defined in '%s'. Using default one: %r"
-								% (option[1], sec))
+								% (option[1], sec, option[2]))
 					values[option[1]] = option[2]
 			except ValueError:
 				logSys.warn("Wrong value for '" + option[1] + "' in '" + sec +

client/configurator.py

@@ -43,15 +43,19 @@ class Configurator:
 		self.__fail2ban = Fail2banReader()
 		self.__jails = JailsReader()
 	
-	#@staticmethod
+	def setBaseDir(self, folderName):
-	def setBaseDir(folderName):
+		self.__fail2ban.setBaseDir(folderName)
-		ConfigReader.setBaseDir(folderName)
+		self.__jails.setBaseDir(folderName)
-	setBaseDir = staticmethod(setBaseDir)
 	
-	#@staticmethod
+	def getBaseDir(self):
-	def getBaseDir():
+		fail2ban_basedir = self.__fail2ban.getBaseDir()
-		return ConfigReader.getBaseDir()
+		jails_basedir = self.__jails.getBaseDir()
-	getBaseDir = staticmethod(getBaseDir)
+		if fail2ban_basedir != jails_basedir:
+			logSys.error("fail2ban.conf and jails.conf readers have differing "
+						 "basedirs: %r and %r. "
+						 "Returning the one for fail2ban.conf"
+						 % (fail2ban_basedir, jails_basedir))
+		return fail2ban_basedir
 	
 	def readEarly(self):
 		self.__fail2ban.read()

client/fail2banreader.py

@@ -35,8 +35,8 @@ logSys = logging.getLogger("fail2ban.client.config")
 
 class Fail2banReader(ConfigReader):
 	
-	def __init__(self):
+	def __init__(self, **kwargs):
-		ConfigReader.__init__(self)
+		ConfigReader.__init__(self, **kwargs)
 	
 	def read(self):
 		ConfigReader.read(self, "fail2ban")

client/filterreader.py

@@ -35,8 +35,8 @@ logSys = logging.getLogger("fail2ban.client.config")
 
 class FilterReader(ConfigReader):
 	
-	def __init__(self, fileName, name):
+	def __init__(self, fileName, name, **kwargs):
-		ConfigReader.__init__(self)
+		ConfigReader.__init__(self, **kwargs)
 		self.__file = fileName
 		self.__name = name
 	

client/jailreader.py

@@ -40,10 +40,11 @@ class JailReader(ConfigReader):
 	
 	actionCRE = re.compile("^((?:\w|-|_|\.)+)(?:\[(.*)\])?$")
 	
-	def __init__(self, name):
+	def __init__(self, name, force_enable=False, **kwargs):
-		ConfigReader.__init__(self)
+		ConfigReader.__init__(self, **kwargs)
 		self.__name = name
 		self.__filter = None
+		self.__force_enable = force_enable
 		self.__actions = list()
 	
 	def setName(self, value):
@@ -53,10 +54,10 @@ class JailReader(ConfigReader):
 		return self.__name
 	
 	def read(self):
-		ConfigReader.read(self, "jail")
+		return ConfigReader.read(self, "jail")
 	
 	def isEnabled(self):
-		return self.__opts["enabled"]
+		return self.__force_enable or self.__opts["enabled"]
 	
 	def getOptions(self):
 		opts = [["bool", "enabled", "false"],
@@ -76,7 +77,8 @@ class JailReader(ConfigReader):
 		
 		if self.isEnabled():
 			# Read filter
-			self.__filter = FilterReader(self.__opts["filter"], self.__name)
+			self.__filter = FilterReader(self.__opts["filter"], self.__name,
+										 basedir=self.getBaseDir())
 			ret = self.__filter.read()
 			if ret:
 				self.__filter.getOptions(self.__opts)
@@ -87,8 +89,10 @@ class JailReader(ConfigReader):
 			# Read action
 			for act in self.__opts["action"].split('\n'):
 				try:
+					if not act:			  # skip empty actions
+						continue
 					splitAct = JailReader.splitAction(act)
-					action = ActionReader(splitAct, self.__name)
+					action = ActionReader(splitAct, self.__name, basedir=self.getBaseDir())
 					ret = action.read()
 					if ret:
 						action.getOptions(self.__opts)
@@ -97,8 +101,10 @@ class JailReader(ConfigReader):
 						raise AttributeError("Unable to read action")
 				except Exception, e:
 					logSys.error("Error in action definition " + act)
-					logSys.debug(e)
+					logSys.debug("Caught exception: %s" % (e,))
 					return False
+			if not len(self.__actions):
+				logSys.warn("No actions were defined for %s" % self.__name)
 		return True
 	
 	def convert(self):
@@ -145,12 +151,20 @@ class JailReader(ConfigReader):
 	def splitAction(action):
 		m = JailReader.actionCRE.match(action)
 		d = dict()
-		if not m.group(2) == None:
+		mgroups = m.groups()
+		if len(mgroups) == 2:
+			action_name, action_opts = mgroups
+		elif len(mgroups) == 1:
+			action_name, action_opts = mgroups[0], None
+		else:
+			raise ValueError("While reading action %s we should have got up to "
+							 "2 groups. Got: %r" % (action, mgroups))
+		if not action_opts is None:
 			# Huge bad hack :( This method really sucks. TODO Reimplement it.
 			actions = ""
 			escapeChar = None
 			allowComma = False
-			for c in m.group(2):
+			for c in action_opts:
 				if c in ('"', "'") and not allowComma:
 					# Start
 					escapeChar = c
@@ -175,6 +189,6 @@ class JailReader(ConfigReader):
 				try:
 					d[p[0].strip()] = p[1].strip()
 				except IndexError:
-					logSys.error("Invalid argument %s in '%s'" % (p, m.group(2)))
+					logSys.error("Invalid argument %s in '%s'" % (p, action_opts))
-		return [m.group(1), d]
+		return [action_name, d]
 	splitAction = staticmethod(splitAction)

client/jailsreader.py

@@ -36,12 +36,20 @@ logSys = logging.getLogger("fail2ban.client.config")
 
 class JailsReader(ConfigReader):
 	
-	def __init__(self):
+	def __init__(self, force_enable=False, **kwargs):
-		ConfigReader.__init__(self)
+		"""
+		Parameters
+		----------
+		force_enable : bool, optional
+		  Passed to JailReader to force enable the jails.
+		  It is for internal use
+		"""
+		ConfigReader.__init__(self, **kwargs)
 		self.__jails = list()
+		self.__force_enable = force_enable
 	
 	def read(self):
-		ConfigReader.read(self, "jail")
+		return ConfigReader.read(self, "jail")
 	
 	def getOptions(self, section = None):
 		opts = []
@@ -49,7 +57,7 @@ class JailsReader(ConfigReader):
 
 		if section:
 			# Get the options of a specific jail.
-			jail = JailReader(section)
+			jail = JailReader(section, basedir=self.getBaseDir(), force_enable=self.__force_enable)
 			jail.read()
 			ret = jail.getOptions()
 			if ret:
@@ -62,7 +70,7 @@ class JailsReader(ConfigReader):
 		else:
 			# Get the options of all jails.
 			for sec in self.sections():
-				jail = JailReader(sec)
+				jail = JailReader(sec, basedir=self.getBaseDir(), force_enable=self.__force_enable)
 				jail.read()
 				ret = jail.getOptions()
 				if ret:

config/action.d/complain.conf

@@ -52,10 +52,7 @@ actioncheck =
 # Option:  actionban
 # Notes.:  command executed when banning an IP. Take care that the
 #          command is executed with Fail2Ban user rights.
-# Tags:    <ip>  IP address
+# Tags:    See jail.conf(5) man page
-#          <failures>  number of failures
-#          <failtime>  unix timestamp of the last failure
-#          <bantime>  unix timestamp of the ban time
 # Values:  CMD
 #
 actionban = ADDRESSES=`whois <ip> | perl -e 'while (<STDIN>) { next if /^changed|@(ripe|apnic)\.net/io; $m += (/abuse|trouble:|report|spam|security/io?3:0); if (/([a-z0-9_\-\.+]+@[a-z0-9\-]+(\.[[a-z0-9\-]+)+)/io) { while (s/([a-z0-9_\-\.+]+@[a-z0-9\-]+(\.[[a-z0-9\-]+)+)//io) { if ($m) { $a{lc($1)}=$m } else { $b{lc($1)}=$m } } $m=0 } else { $m && --$m } } if (%%a) {print join(",",keys(%%a))} else {print join(",",keys(%%b))}'`
@@ -67,9 +64,7 @@ actionban = ADDRESSES=`whois <ip> | perl -e 'while (<STDIN>) { next if /^changed
 # Option:  actionunban
 # Notes.:  command executed when unbanning an IP. Take care that the
 #          command is executed with Fail2Ban user rights.
-# Tags:    <ip>  IP address
+# Tags:    See jail.conf(5) man page
-#          <bantime>  unix timestamp of the ban time
-#          <unbantime>  unix timestamp of the unban time
 # Values:  CMD
 #
 actionunban =

config/action.d/dshield.conf

@@ -54,9 +54,7 @@ actioncheck =
 # Option:  actionban
 # Notes.:  command executed when banning an IP. Take care that the
 #          command is executed with Fail2Ban user rights.
-# Tags:    <ip>  IP address
+# Tags:    See jail.conf(5) man page
-#          <failures>  number of failures
-#          <time>  unix timestamp of the ban time
 # Values:  CMD
 #
 # See http://www.dshield.org/specs.html for more on report format/notes
@@ -91,9 +89,7 @@ actionban = TZONE=`date +%%z | sed 's/\([+-]..\)\(..\)/\1:\2/'`
 # Option:  actionunban
 # Notes.:  command executed when unbanning an IP. Take care that the
 #          command is executed with Fail2Ban user rights.
-# Tags:    <ip>  IP address
+# Tags:    See jail.conf(5) man page
-#          <failures>  number of failures
-#          <time>  unix timestamp of the ban time
 # Values:  CMD
 #
 actionunban = if [ -f <tmpfile>.first ]; then
@@ -159,7 +155,6 @@ minreportinterval = 3600
 #          submit the batch, even if we haven't reached <lines> yet. Note that
 #          this is only checked on each ban/unban, and that we always send
 #          anything in the buffer on shutdown. Must be greater than
-#          <minreportinterval>.
 # Values:  [ NUM ]  Default: 21600 (6 hours)
 #
 maxbufferage = 21600

config/action.d/dummy.conf

@@ -29,9 +29,7 @@ actioncheck =
 # Option:  actionban
 # Notes.:  command executed when banning an IP. Take care that the
 #          command is executed with Fail2Ban user rights.
-# Tags:    <ip>  IP address
+# Tags:    See jail.conf(5) man page
-#          <failures>  number of failures
-#          <time>  unix timestamp of the ban time
 # Values:  CMD
 #
 actionban = printf %%b "+<ip>\n" >> /tmp/fail2ban.dummy
@@ -39,9 +37,7 @@ actionban = printf %%b "+<ip>\n" >> /tmp/fail2ban.dummy
 # Option:  actionunban
 # Notes.:  command executed when unbanning an IP. Take care that the
 #          command is executed with Fail2Ban user rights.
-# Tags:    <ip>  IP address
+# Tags:    See jail.conf(5) man page
-#          <failures>  number of failures
-#          <time>  unix timestamp of the ban time
 # Values:  CMD
 #
 actionunban = printf %%b "-<ip>\n" >> /tmp/fail2ban.dummy

config/action.d/hostsdeny.conf

@@ -28,9 +28,7 @@ actioncheck =
 # Option:  actionban
 # Notes.:  command executed when banning an IP. Take care that the
 #          command is executed with Fail2Ban user rights.
-# Tags:    <ip>  IP address
+# Tags:    See jail.conf(5) man page
-#          <failures>  number of failures
-#          <time>  unix timestamp of the ban time
 # Values:  CMD
 #
 actionban = IP=<ip> &&
@@ -39,9 +37,7 @@ actionban = IP=<ip> &&
 # Option:  actionunban
 # Notes.:  command executed when unbanning an IP. Take care that the
 #          command is executed with Fail2Ban user rights.
-# Tags:    <ip>  IP address
+# Tags:    See jail.conf(5) man page
-#          <failures>  number of failures
-#          <time>  unix timestamp of the ban time
 # Values:  CMD
 #
 actionunban = IP=<ip> && sed -i.old /ALL:\ $IP/d <file>

config/action.d/ipfilter.conf

@@ -34,9 +34,7 @@ actioncheck =
 # Option:  actionban
 # Notes.:  command executed when banning an IP. Take care that the
 #          command is executed with Fail2Ban user rights.
-# Tags:    <ip>  IP address
+# Tags:    See jail.conf(5) man page
-#          <failures>  number of failures
-#          <time>  unix timestamp of the ban time
 # Values:  CMD
 #
 actionban = echo block in quick from <ip>/32 | /sbin/ipf -f -
@@ -45,9 +43,7 @@ actionban = echo block in quick from <ip>/32 | /sbin/ipf -f -
 # Option:  actionunban
 # Notes.:  command executed when unbanning an IP. Take care that the
 #          command is executed with Fail2Ban user rights.
-# Tags:    <ip>  IP address
+# Tags:    See jail.conf(5) man page
-#          <failures>  number of failures
-#          <time>  unix timestamp of the ban time
 # Values:  CMD
 #
 # note -r option used to remove matching rule

config/action.d/ipfw.conf

@@ -32,9 +32,7 @@ actioncheck =
 # Option:  actionban
 # Notes.:  command executed when banning an IP. Take care that the
 #          command is executed with Fail2Ban user rights.
-# Tags:    <ip>  IP address
+# Tags:    See jail.conf(5) man page
-#          <failures>  number of failures
-#          <time>  unix timestamp of the ban time
 # Values:  CMD
 #
 actionban = ipfw add deny tcp from <ip> to <localhost> <port>
@@ -43,9 +41,7 @@ actionban = ipfw add deny tcp from <ip> to <localhost> <port>
 # Option:  actionunban
 # Notes.:  command executed when unbanning an IP. Take care that the
 #          command is executed with Fail2Ban user rights.
-# Tags:    <ip>  IP address
+# Tags:    See jail.conf(5) man page
-#          <failures>  number of failures
-#          <time>  unix timestamp of the ban time
 # Values:  CMD
 #
 actionunban = ipfw delete `ipfw list | grep -i <ip> | awk '{print $1;}'`

config/action.d/iptables-allports.conf

@@ -34,9 +34,7 @@ actioncheck = iptables -n -L <chain> | grep -q 'fail2ban-<name>[ \t]'
 # Option:  actionban
 # Notes.:  command executed when banning an IP. Take care that the
 #          command is executed with Fail2Ban user rights.
-# Tags:    <ip>  IP address
+# Tags:    See jail.conf(5) man page
-#          <failures>  number of failures
-#          <time>  unix timestamp of the ban time
 # Values:  CMD
 #
 actionban = iptables -I fail2ban-<name> 1 -s <ip> -j DROP
@@ -44,9 +42,7 @@ actionban = iptables -I fail2ban-<name> 1 -s <ip> -j DROP
 # Option:  actionunban
 # Notes.:  command executed when unbanning an IP. Take care that the
 #          command is executed with Fail2Ban user rights.
-# Tags:    <ip>  IP address
+# Tags:    See jail.conf(5) man page
-#          <failures>  number of failures
-#          <time>  unix timestamp of the ban time
 # Values:  CMD
 #
 actionunban = iptables -D fail2ban-<name> -s <ip> -j DROP

config/action.d/iptables-ipset-proto4.conf

@@ -38,7 +38,7 @@ actionstop = iptables -D INPUT -p <protocol> -m multiport --dports <port> -m set
 # Option:  actionban
 # Notes.:  command executed when banning an IP. Take care that the
 #          command is executed with Fail2Ban user rights.
-# Tags:    <ip>  IP address
+# Tags:    See jail.conf(5) man page
 # Values:  CMD
 #
 actionban = ipset --test fail2ban-<name> <ip> ||  ipset --add fail2ban-<name> <ip>
@@ -46,7 +46,7 @@ actionban = ipset --test fail2ban-<name> <ip> ||  ipset --add fail2ban-<name> <i
 # Option:  actionunban
 # Notes.:  command executed when unbanning an IP. Take care that the
 #          command is executed with Fail2Ban user rights.
-# Tags:    <ip>  IP address
+# Tags:    See jail.conf(5) man page
 # Values:  CMD
 #
 actionunban = ipset --test fail2ban-<name> <ip> && ipset --del fail2ban-<name> <ip>

config/action.d/iptables-ipset-proto6.conf

@@ -38,7 +38,7 @@ actionstop = iptables -D INPUT -p <protocol> -m multiport --dports <port> -m set
 # Option:  actionban
 # Notes.:  command executed when banning an IP. Take care that the
 #          command is executed with Fail2Ban user rights.
-# Tags:    <ip>  IP address
+# Tags:    See jail.conf(5) man page
 # Values:  CMD
 #
 actionban = ipset add fail2ban-<name> <ip> timeout <bantime> -exist
@@ -46,7 +46,7 @@ actionban = ipset add fail2ban-<name> <ip> timeout <bantime> -exist
 # Option:  actionunban
 # Notes.:  command executed when unbanning an IP. Take care that the
 #          command is executed with Fail2Ban user rights.
-# Tags:    <ip>  IP address
+# Tags:    See jail.conf(5) man page
 # Values:  CMD
 #
 actionunban = ipset del fail2ban-<name> <ip> -exist

config/action.d/iptables-multiport-log.conf

@@ -42,9 +42,7 @@ actioncheck = iptables -n -L fail2ban-<name>-log >/dev/null
 # Option:  actionban
 # Notes.:  command executed when banning an IP. Take care that the
 #          command is executed with Fail2Ban user rights.
-# Tags:    <ip>  IP address
+# Tags:    See jail.conf(5) man page
-#          <failures>  number of failures
-#          <time>  unix timestamp of the ban time
 # Values:  CMD
 #
 actionban = iptables -I fail2ban-<name> 1 -s <ip> -j fail2ban-<name>-log
@@ -52,9 +50,7 @@ actionban = iptables -I fail2ban-<name> 1 -s <ip> -j fail2ban-<name>-log
 # Option:  actionunban
 # Notes.:  command executed when unbanning an IP. Take care that the
 #          command is executed with Fail2Ban user rights.
-# Tags:    <ip>  IP address
+# Tags:    See jail.conf(5) man page
-#          <failures>  number of failures
-#          <time>  unix timestamp of the ban time
 # Values:  CMD
 #
 actionunban = iptables -D fail2ban-<name> -s <ip> -j fail2ban-<name>-log

config/action.d/iptables-multiport.conf

@@ -32,9 +32,7 @@ actioncheck = iptables -n -L <chain> | grep -q 'fail2ban-<name>[ \t]'
 # Option:  actionban
 # Notes.:  command executed when banning an IP. Take care that the
 #          command is executed with Fail2Ban user rights.
-# Tags:    <ip>  IP address
+# Tags:    See jail.conf(5) man page
-#          <failures>  number of failures
-#          <time>  unix timestamp of the ban time
 # Values:  CMD
 #
 actionban = iptables -I fail2ban-<name> 1 -s <ip> -j DROP
@@ -42,9 +40,7 @@ actionban = iptables -I fail2ban-<name> 1 -s <ip> -j DROP
 # Option:  actionunban
 # Notes.:  command executed when unbanning an IP. Take care that the
 #          command is executed with Fail2Ban user rights.
-# Tags:    <ip>  IP address
+# Tags:    See jail.conf(5) man page
-#          <failures>  number of failures
-#          <time>  unix timestamp of the ban time
 # Values:  CMD
 #
 actionunban = iptables -D fail2ban-<name> -s <ip> -j DROP

config/action.d/iptables-new.conf

@@ -34,9 +34,7 @@ actioncheck = iptables -n -L <chain> | grep -q 'fail2ban-<name>[ \t]'
 # Option:  actionban
 # Notes.:  command executed when banning an IP. Take care that the
 #          command is executed with Fail2Ban user rights.
-# Tags:    <ip>  IP address
+# Tags:    See jail.conf(5) man page
-#          <failures>  number of failures
-#          <time>  unix timestamp of the ban time
 # Values:  CMD
 #
 actionban = iptables -I fail2ban-<name> 1 -s <ip> -j DROP
@@ -44,9 +42,7 @@ actionban = iptables -I fail2ban-<name> 1 -s <ip> -j DROP
 # Option:  actionunban
 # Notes.:  command executed when unbanning an IP. Take care that the
 #          command is executed with Fail2Ban user rights.
-# Tags:    <ip>  IP address
+# Tags:    See jail.conf(5) man page
-#          <failures>  number of failures
-#          <time>  unix timestamp of the ban time
 # Values:  CMD
 #
 actionunban = iptables -D fail2ban-<name> -s <ip> -j DROP

config/action.d/iptables-xt_recent-echo.conf

@@ -46,9 +46,7 @@ actioncheck = test -e /proc/net/xt_recent/fail2ban-<name>
 # Option:  actionban
 # Notes.:  command executed when banning an IP. Take care that the
 #          command is executed with Fail2Ban user rights.
-# Tags:    <ip>  IP address
+# Tags:    See jail.conf(5) man page
-#          <failures>  number of failures
-#          <time>  unix timestamp of the ban time
 # Values:  CMD
 #
 actionban = echo +<ip> > /proc/net/xt_recent/fail2ban-<name>
@@ -56,9 +54,7 @@ actionban = echo +<ip> > /proc/net/xt_recent/fail2ban-<name>
 # Option:  actionunban
 # Notes.:  command executed when unbanning an IP. Take care that the
 #          command is executed with Fail2Ban user rights.
-# Tags:    <ip>  IP address
+# Tags:    See jail.conf(5) man page
-#          <failures>  number of failures
-#          <time>  unix timestamp of the ban time
 # Values:  CMD
 #
 actionunban = echo -<ip> > /proc/net/xt_recent/fail2ban-<name>

config/action.d/iptables.conf

@@ -32,9 +32,7 @@ actioncheck = iptables -n -L <chain> | grep -q 'fail2ban-<name>[ \t]'
 # Option:  actionban
 # Notes.:  command executed when banning an IP. Take care that the
 #          command is executed with Fail2Ban user rights.
-# Tags:    <ip>  IP address
+# Tags:    See jail.conf(5) man page
-#          <failures>  number of failures
-#          <time>  unix timestamp of the ban time
 # Values:  CMD
 #
 actionban = iptables -I fail2ban-<name> 1 -s <ip> -j DROP
@@ -42,9 +40,7 @@ actionban = iptables -I fail2ban-<name> 1 -s <ip> -j DROP
 # Option:  actionunban
 # Notes.:  command executed when unbanning an IP. Take care that the
 #          command is executed with Fail2Ban user rights.
-# Tags:    <ip>  IP address
+# Tags:    See jail.conf(5) man page
-#          <failures>  number of failures
-#          <time>  unix timestamp of the ban time
 # Values:  CMD
 #
 actionunban = iptables -D fail2ban-<name> -s <ip> -j DROP

config/action.d/mail-buffered.conf

@@ -43,9 +43,7 @@ actioncheck =
 # Option:  actionban
 # Notes.:  command executed when banning an IP. Take care that the
 #          command is executed with Fail2Ban user rights.
-# Tags:    <ip>  IP address
+# Tags:    See jail.conf(5) man page
-#          <failures>  number of failures
-#          <time>  unix timestamp of the ban time
 # Values:  CMD
 #
 actionban = printf %%b "`date`: <ip> (<failures> failures)\n" >> <tmpfile>
@@ -62,9 +60,7 @@ actionban = printf %%b "`date`: <ip> (<failures> failures)\n" >> <tmpfile>
 # Option:  actionunban
 # Notes.:  command executed when unbanning an IP. Take care that the
 #          command is executed with Fail2Ban user rights.
-# Tags:    <ip>  IP address
+# Tags:    See jail.conf(5) man page
-#          <failures>  number of failures
-#          <time>  unix timestamp of the ban time
 # Values:  CMD
 #
 actionunban = 

config/action.d/mail-whois-lines.conf

@@ -34,10 +34,7 @@ actioncheck =
 # Option:  actionban
 # Notes.:  command executed when banning an IP. Take care that the
 #          command is executed with Fail2Ban user rights.
-# Tags:    <ip>  IP address
+# Tags:    See jail.conf(5) man page
-#          <failures>  number of failures
-#          <failtime>  unix timestamp of the last failure
-#          <bantime>  unix timestamp of the ban time
 # Values:  CMD
 #
 actionban = printf %%b "Hi,\n
@@ -53,9 +50,7 @@ actionban = printf %%b "Hi,\n
 # Option:  actionunban
 # Notes.:  command executed when unbanning an IP. Take care that the
 #          command is executed with Fail2Ban user rights.
-# Tags:    <ip>  IP address
+# Tags:    See jail.conf(5) man page
-#          <bantime>  unix timestamp of the ban time
-#          <unbantime>  unix timestamp of the unban time
 # Values:  CMD
 #
 actionunban = 

config/action.d/mail-whois.conf

@@ -34,9 +34,7 @@ actioncheck =
 # Option:  actionban
 # Notes.:  command executed when banning an IP. Take care that the
 #          command is executed with Fail2Ban user rights.
-# Tags:    <ip>  IP address
+# Tags:    See jail.conf(5) man page
-#          <failures>  number of failures
-#          <time>  unix timestamp of the ban time
 # Values:  CMD
 #
 actionban = printf %%b "Hi,\n
@@ -50,9 +48,7 @@ actionban = printf %%b "Hi,\n
 # Option:  actionunban
 # Notes.:  command executed when unbanning an IP. Take care that the
 #          command is executed with Fail2Ban user rights.
-# Tags:    <ip>  IP address
+# Tags:    See jail.conf(5) man page
-#          <failures>  number of failures
-#          <time>  unix timestamp of the ban time
 # Values:  CMD
 #
 actionunban = 

config/action.d/mail.conf

@@ -34,9 +34,7 @@ actioncheck =
 # Option:  actionban
 # Notes.:  command executed when banning an IP. Take care that the
 #          command is executed with Fail2Ban user rights.
-# Tags:    <ip>  IP address
+# Tags:    See jail.conf(5) man page
-#          <failures>  number of failures
-#          <time>  unix timestamp of the ban time
 # Values:  CMD
 #
 actionban = printf %%b "Hi,\n
@@ -48,9 +46,7 @@ actionban = printf %%b "Hi,\n
 # Option:  actionunban
 # Notes.:  command executed when unbanning an IP. Take care that the
 #          command is executed with Fail2Ban user rights.
-# Tags:    <ip>  IP address
+# Tags:    See jail.conf(5) man page
-#          <failures>  number of failures
-#          <time>  unix timestamp of the ban time
 # Values:  CMD
 #
 actionunban = 

config/action.d/mynetwatchman.conf

@@ -49,9 +49,7 @@ actioncheck =
 # Option:  actionban
 # Notes.:  command executed when banning an IP. Take care that the
 #          command is executed with Fail2Ban user rights.
-# Tags:    <ip>  IP address
+# Tags:    See jail.conf(5) man page
-#          <failures>  number of failures
-#          <time>  unix timestamp of the ban time
 # Values:  CMD
 #
 #
@@ -71,9 +69,7 @@ actionban = MNWLOGIN=`perl -e '$s=shift;$s=~s/([\W])/"%%".uc(sprintf("%%2.2x",or
 # Option:  actionunban
 # Notes.:  command executed when unbanning an IP. Take care that the
 #          command is executed with Fail2Ban user rights.
-# Tags:    <ip>  IP address
+# Tags:    See jail.conf(5) man page
-#          <failures>  number of failures
-#          <time>  unix timestamp of the ban time
 # Values:  CMD
 #
 actionunban =

config/action.d/sendmail-buffered.conf

@@ -52,9 +52,7 @@ actioncheck =
 # Option:  actionban
 # Notes.:  command executed when banning an IP. Take care that the
 #          command is executed with Fail2Ban user rights.
-# Tags:    <ip>  IP address
+# Tags:    See jail.conf(5) man page
-#          <failures>  number of failures
-#          <time>  unix timestamp of the ban time
 # Values:  CMD
 #
 actionban = printf %%b "`date`: <ip> (<failures> failures)\n" >> <tmpfile>
@@ -74,9 +72,7 @@ actionban = printf %%b "`date`: <ip> (<failures> failures)\n" >> <tmpfile>
 # Option:  actionunban
 # Notes.:  command executed when unbanning an IP. Take care that the
 #          command is executed with Fail2Ban user rights.
-# Tags:    <ip>  IP address
+# Tags:    See jail.conf(5) man page
-#          <failures>  number of failures
-#          <time>  unix timestamp of the ban time
 # Values:  CMD
 #
 actionunban = 

config/action.d/sendmail-whois-lines.conf

@@ -42,9 +42,7 @@ actioncheck =
 # Option:  actionban
 # Notes.:  command executed when banning an IP. Take care that the
 #          command is executed with Fail2Ban user rights.
-# Tags:    <ip>  IP address
+# Tags:    See jail.conf(5) man page
-#          <failures>  number of failures
-#          <time>  unix timestamp of the ban time
 # Values:  CMD
 #
 actionban = printf %%b "Subject: [Fail2Ban] <name>: banned <ip>
@@ -64,9 +62,7 @@ actionban = printf %%b "Subject: [Fail2Ban] <name>: banned <ip>
 # Option:  actionunban
 # Notes.:  command executed when unbanning an IP. Take care that the
 #          command is executed with Fail2Ban user rights.
-# Tags:    <ip>  IP address
+# Tags:    See jail.conf(5) man page
-#          <failures>  number of failures
-#          <time>  unix timestamp of the ban time
 # Values:  CMD
 #
 actionunban = 

config/action.d/sendmail-whois.conf

@@ -42,9 +42,7 @@ actioncheck =
 # Option:  actionban
 # Notes.:  command executed when banning an IP. Take care that the
 #          command is executed with Fail2Ban user rights.
-# Tags:    <ip>  IP address
+# Tags:    See jail.conf(5) man page
-#          <failures>  number of failures
-#          <time>  unix timestamp of the ban time
 # Values:  CMD
 #
 actionban = printf %%b "Subject: [Fail2Ban] <name>: banned <ip>
@@ -62,9 +60,7 @@ actionban = printf %%b "Subject: [Fail2Ban] <name>: banned <ip>
 # Option:  actionunban
 # Notes.:  command executed when unbanning an IP. Take care that the
 #          command is executed with Fail2Ban user rights.
-# Tags:    <ip>  IP address
+# Tags:    See jail.conf(5) man page
-#          <failures>  number of failures
-#          <time>  unix timestamp of the ban time
 # Values:  CMD
 #
 actionunban = 

config/action.d/sendmail.conf

@@ -42,9 +42,7 @@ actioncheck =
 # Option:  actionban
 # Notes.:  command executed when banning an IP. Take care that the
 #          command is executed with Fail2Ban user rights.
-# Tags:    <ip>  IP address
+# Tags:    See jail.conf(5) man page
-#          <failures>  number of failures
-#          <time>  unix timestamp of the ban time
 # Values:  CMD
 #
 actionban = printf %%b "Subject: [Fail2Ban] <name>: banned <ip>
@@ -60,9 +58,7 @@ actionban = printf %%b "Subject: [Fail2Ban] <name>: banned <ip>
 # Option:  actionunban
 # Notes.:  command executed when unbanning an IP. Take care that the
 #          command is executed with Fail2Ban user rights.
-# Tags:    <ip>  IP address
+# Tags:    See jail.conf(5) man page
-#          <failures>  number of failures
-#          <time>  unix timestamp of the ban time
 # Values:  CMD
 #
 actionunban = 

config/action.d/shorewall.conf

@@ -36,9 +36,7 @@ actioncheck =
 # Option:  actionban
 # Notes.:  command executed when banning an IP. Take care that the
 #          command is executed with Fail2Ban user rights.
-# Tags:    <ip>  IP address
+# Tags:    See jail.conf(5) man page
-#          <failures>  number of failures
-#          <time>  unix timestamp of the ban time
 # Values:  CMD
 #
 actionban = shorewall drop <ip>
@@ -46,9 +44,7 @@ actionban = shorewall drop <ip>
 # Option:  actionunban
 # Notes.:  command executed when unbanning an IP. Take care that the
 #          command is executed with Fail2Ban user rights.
-# Tags:    <ip>  IP address
+# Tags:    See jail.conf(5) man page
-#          <failures>  number of failures
-#          <time>  unix timestamp of the ban time
 # Values:  CMD
 #
 actionunban = shorewall allow <ip>

config/filter.d/postfix.conf

@@ -15,6 +15,7 @@
 # Values:  TEXT
 #
 failregex = reject: RCPT from (.*)\[<HOST>\]: 554
+            reject: RCPT from (.*)\[<HOST>\]: 450 4\.7\.1 : Helo command rejected: Host not found; from=<> to=<> proto=ESMTP helo= *$
 
 # Option:  ignoreregex
 # Notes.:  regex to ignore. If this regex matches, the line is ignored.

config/filter.d/sogo-auth.conf

@@ -0,0 +1,20 @@
+# /etc/fail2ban/filter.d/sogo-auth.conf
+#
+# Fail2Ban configuration file
+# By Arnd Brandes
+# SOGo
+#
+
+[Definition]
+# Option: failregex
+# Filter Ban in /var/log/sogo/sogo.log
+# Note: the error log may contain multiple hosts, whereas the first one 
+# is the client and all others are poxys. We match the first one, only
+
+failregex = Login from '<HOST>' for user '.*' might not have worked( - password policy: \d*  grace: -?\d*  expire: -?\d*  bound: -?\d*)?\s*$
+
+# Option: ignoreregex
+# Notes.: regex to ignore. If this regex matches, the line is ignored.
+# Values: TEXT
+#
+ignoreregex = 

config/jail.conf

@@ -212,6 +212,20 @@ filter   = roundcube-auth
 action   = iptables[name=RoundCube, port="http,https"]
 logpath  = /var/log/roundcube/userlogins
 
+
+# Monitor SOGo groupware server
+
+[sogo-iptables]
+
+enabled  = false
+filter   = sogo-auth
+port     = http, https
+# without proxy this would be:
+# port    = 20000
+
+action   = iptables[name=SOGo, port="http,https"]
+logpath  = /var/log/sogo/sogo.log
+
 # Ban attackers that try to use PHP's URL-fopen() functionality
 # through GET/POST variables. - Experimental, with more than a year
 # of usage in production environments.

fail2ban-client

@@ -63,7 +63,7 @@ class Fail2banClient:
 		self.__conf["interactive"] = False
 		self.__conf["socket"] = None
 		self.__conf["pidfile"] = None
-		
+
 	def dispVersion(self):
 		print "Fail2Ban v" + version
 		print
@@ -73,7 +73,7 @@ class Fail2banClient:
 		print
 		print "Written by Cyril Jaquier <cyril.jaquier@fail2ban.org>."
 		print "Many contributions by Yaroslav O. Halchenko <debian@onerussian.com>."
-	
+
 	def dispUsage(self):
 		""" Prints Fail2Ban command line options and exits
 		"""
@@ -95,17 +95,17 @@ class Fail2banClient:
 		print "    -V, --version           print the version"
 		print
 		print "Command:"
-		
+
 		# Prints the protocol
 		printFormatted()
-		
+
 		print
 		print "Report bugs to https://github.com/fail2ban/fail2ban/issues"
-	
+
 	def dispInteractive(self):
 		print "Fail2Ban v" + version + " reads log file that contains password failure report"
 		print "and bans the corresponding IP addresses using firewall rules."
-		print 
+		print
 
 	def __sigTERMhandler(self, signum, frame):
 		# Print a new line because we probably come from wait
@@ -139,10 +139,10 @@ class Fail2banClient:
 			elif opt[0] in ["-V", "--version"]:
 				self.dispVersion()
 				sys.exit(0)
-	
+
 	def __ping(self):
 		return self.__processCmd([["ping"]], False)
-	
+
 	def __processCmd(self, cmd, showRet = True):
 		beautifier = Beautifier()
 		for c in cmd:
@@ -167,7 +167,7 @@ class Fail2banClient:
 					logSys.error(e)
 				return False
 		return True
-		
+
 	##
 	# Process a command line.
 	#
@@ -241,13 +241,13 @@ class Fail2banClient:
 				return False
 		else:
 			return self.__processCmd([cmd])
-	
+
-	
+
 	##
 	# Start Fail2Ban server.
 	#
 	# Start the Fail2ban server in daemon mode.
-	
+
 	def __startServerAsync(self, socket, pidfile, force = False):
 		# Forks the current process.
 		pid = os.fork()
@@ -278,7 +278,7 @@ class Fail2banClient:
 				except OSError:
 					logSys.error("Could not start %s" % self.SERVER)
 					os.exit(-1)
-	
+
 	def __waitOnServer(self):
 		# Wait for the server to start
 		cnt = 0
@@ -306,16 +306,16 @@ class Fail2banClient:
 			cnt += 1
 		if self.__conf["verbose"] > 1:
 			sys.stdout.write('\n')
-			
+
-	
+
 	def start(self, argv):
 		# Command line options
 		self.__argv = argv
-		
+
 		# Install signal handlers
 		signal.signal(signal.SIGTERM, self.__sigTERMhandler)
 		signal.signal(signal.SIGINT, self.__sigTERMhandler)
-		
+
 		# Reads the command line options.
 		try:
 			cmdOpts = 'hc:s:p:xdviqV'
@@ -324,9 +324,9 @@ class Fail2banClient:
 		except getopt.GetoptError:
 			self.dispUsage()
 			return False
-			
+
 		self.__getCmdLineOptions(optList)
-		
+
 		verbose = self.__conf["verbose"]
 		if verbose <= 0:
 			logSys.setLevel(logging.ERROR)
@@ -346,7 +346,7 @@ class Fail2banClient:
 
 		# Set the configuration path
 		self.__configurator.setBaseDir(self.__conf["conf"])
-		
+
 		# Set socket path
 		self.__configurator.readEarly()
 		conf = self.__configurator.getEarlyOptions()
@@ -360,7 +360,7 @@ class Fail2banClient:
 			ret = self.__readConfig()
 			self.dumpConfig(self.__stream)
 			return ret
-		
+
 		# Interactive mode
 		if self.__conf["interactive"]:
 			try:
@@ -401,14 +401,14 @@ class Fail2banClient:
 		self.__configurator.convertToProtocol()
 		self.__stream = self.__configurator.getConfigStream()
 		return ret
-	
+
 	def __readJailConfig(self, jail):
 		self.__configurator.readAll()
 		ret = self.__configurator.getOptions(jail)
 		self.__configurator.convertToProtocol()
 		self.__stream = self.__configurator.getConfigStream()
 		return ret
-	
+
 	#@staticmethod
 	def dumpConfig(cmd):
 		for c in cmd:

fail2ban-regex

@@ -50,24 +50,24 @@ class RegexStat:
 	def __str__(self):
 		return "%s(%r) %d failed: %s" \
 		  % (self.__class__, self.__failregex, self.__stats, self.__ipList)
-	
+
 	def inc(self):
 		self.__stats += 1
-	
+
 	def getStats(self):
 		return self.__stats
 
 	def getFailRegex(self):
 		return self.__failregex
-	
+
 	def appendIP(self, value):
 		self.__ipList.extend(value)
-	
+
 	def getIPList(self):
 		return self.__ipList
 
 class Fail2banRegex:
-	
+
 	test = None
 
 	CONFIG_DEFAULTS = {'configpath' : "/etc/fail2ban/"}
@@ -87,7 +87,7 @@ class Fail2banRegex:
 		self.__logging_level = self.__verbose and logging.DEBUG or logging.WARN
 		logging.getLogger("fail2ban").addHandler(self.__hdlr)
 		logging.getLogger("fail2ban").setLevel(logging.ERROR)
-	
+
 	#@staticmethod
 	def dispVersion():
 		print "Fail2Ban v" + version
@@ -99,7 +99,7 @@ class Fail2banRegex:
 		print "Written by Cyril Jaquier <cyril.jaquier@fail2ban.org>."
 		print "Many contributions by Yaroslav O. Halchenko <debian@onerussian.com>."
 	dispVersion = staticmethod(dispVersion)
-	
+
 	#@staticmethod
 	def dispUsage():
 		print "Usage: "+sys.argv[0]+" [OPTIONS] <LOG> <REGEX> [IGNOREREGEX]"
@@ -129,7 +129,7 @@ class Fail2banRegex:
 		print
 		print "Report bugs to https://github.com/fail2ban/fail2ban/issues"
 	dispUsage = staticmethod(dispUsage)
-	
+
 	def getCmdLineOptions(self, optList):
 		""" Gets the command line options
 		"""
@@ -213,7 +213,7 @@ class Fail2banRegex:
 			print "Use regex line : " + stripReg
 			self.__failregex = [RegexStat(value)]
 		return True
-	
+
 	def testIgnoreRegex(self, line):
 		found = False
 		for regex in self.__ignoreregex:
@@ -230,7 +230,7 @@ class Fail2banRegex:
 			finally:
 				self.__filter.delIgnoreRegex(0)
 				logging.getLogger("fail2ban").setLevel(self.__logging_level)
-	
+
 	def testRegex(self, line):
 		found = False
 		for regex in self.__ignoreregex:
@@ -260,7 +260,7 @@ class Fail2banRegex:
 				logging.getLogger("fail2ban").setLevel(logging.CRITICAL)
 		for regex in self.__ignoreregex:
 					self.__filter.delIgnoreRegex(0)
-	
+
 	def printStats(self):
 		print
 		print "Results"
@@ -309,20 +309,20 @@ class Fail2banRegex:
 						print "    %s (%s)%s" % (
 							ip[0], timeString, ip[2] and " (already matched)" or "")
 			print
-			
+
 			print "Date template hits:"
 			for template in self.__filter.dateDetector.getTemplates():
 				if self.__verbose or template.getHits():
 					print `template.getHits()` + " hit(s): " + template.getName()
 			print
-			
+
 			print "Success, the total number of match is " + str(total)
 			print
 			print "However, look at the above section 'Running tests' which could contain important"
 			print "information."
 			return True
 
-			
+
 if __name__ == "__main__":
 	fail2banRegex = Fail2banRegex()
 	# Reads the command line options.

fail2ban-server

@@ -46,7 +46,7 @@ logSys = logging.getLogger("fail2ban")
 # Its first goal was to protect a SSH server.
 
 class Fail2banServer:
-	
+
 	def __init__(self):
 		self.__server = None
 		self.__argv = None
@@ -55,7 +55,7 @@ class Fail2banServer:
 		self.__conf["force"] = False
 		self.__conf["socket"] = "/var/run/fail2ban/fail2ban.sock"
 		self.__conf["pidfile"] = "/var/run/fail2ban/fail2ban.pid"
-	
+
 	def dispVersion(self):
 		print "Fail2Ban v" + version
 		print
@@ -65,7 +65,7 @@ class Fail2banServer:
 		print
 		print "Written by Cyril Jaquier <cyril.jaquier@fail2ban.org>."
 		print "Many contributions by Yaroslav O. Halchenko <debian@onerussian.com>."
-	
+
 	def dispUsage(self):
 		""" Prints Fail2Ban command line options and exits
 		"""
@@ -88,7 +88,7 @@ class Fail2banServer:
 		print "    -V, --version        print the version"
 		print
 		print "Report bugs to https://github.com/fail2ban/fail2ban/issues"
-	
+
 	def __getCmdLineOptions(self, optList):
 		""" Gets the command line options
 		"""
@@ -109,11 +109,11 @@ class Fail2banServer:
 			if opt[0] in ["-V", "--version"]:
 	 			self.dispVersion()
 				sys.exit(0)
-		
+
 	def start(self, argv):
 		# Command line options
 		self.__argv = argv
-		
+
 		# Reads the command line options.
 		try:
 			cmdOpts = 'bfs:p:xhV'
@@ -122,9 +122,9 @@ class Fail2banServer:
 		except getopt.GetoptError:
 			self.dispUsage()
 			sys.exit(-1)
-			
+
 		self.__getCmdLineOptions(optList)
-		
+
 		try:
 			self.__server = Server(self.__conf["background"])
 			self.__server.start(self.__conf["socket"],
@@ -135,7 +135,7 @@ class Fail2banServer:
 			logSys.exception(e)
 			self.__server.quit()
 			return False
-		
+
 if __name__ == "__main__":
 	server = Fail2banServer()
 	if server.start(sys.argv):

fail2ban-testcases

@@ -35,6 +35,9 @@ from testcases import filtertestcase
 from testcases import servertestcase
 from testcases import datedetectortestcase
 from testcases import actiontestcase
+from testcases import sockettestcase
+
+from testcases.utils import FormatterWithTraceBack
 from server.mytime import MyTime
 
 from optparse import OptionParser, Option
@@ -51,12 +54,14 @@ def get_opt_parser():
 			   choices=('debug', 'info', 'warn', 'error', 'fatal'),
 			   default=None,
 			   help="Log level for the logger to use during running tests"),
-		])
-
-	p.add_options([
 		Option('-n', "--no-network", action="store_true",
 			   dest="no_network",
 			   help="Do not run tests that require the network"),
+		Option("-t", "--log-traceback", action='store_true',
+			   help="Enrich log-messages with compressed tracebacks"),
+		Option("--full-traceback", action='store_true',
+			   help="Either to make the tracebacks full, not compressed (as by default)"),
+
 		])
 
 	return p
@@ -88,12 +93,21 @@ else: # pragma: no cover
 
 # Add the default logging handler
 stdout = logging.StreamHandler(sys.stdout)
+
+fmt = ' %(message)s'
+
+if opts.log_traceback:
+	Formatter = FormatterWithTraceBack
+	fmt = (opts.full_traceback and ' %(tb)s' or ' %(tbc)s') + fmt
+else:
+	Formatter = logging.Formatter
+
 # Custom log format for the verbose tests runs
 if verbosity > 1: # pragma: no cover
-	stdout.setFormatter(logging.Formatter(' %(asctime)-15s %(thread)s %(message)s'))
+	stdout.setFormatter(Formatter(' %(asctime)-15s %(thread)s' + fmt))
 else: # pragma: no cover
 	# just prefix with the space
-	stdout.setFormatter(logging.Formatter(' %(message)s'))
+	stdout.setFormatter(Formatter(fmt))
 logSys.addHandler(stdout)
 
 #
@@ -130,9 +144,13 @@ tests.addTest(unittest.makeSuite(actiontestcase.ExecuteAction))
 tests.addTest(unittest.makeSuite(failmanagertestcase.AddFailure))
 # BanManager
 tests.addTest(unittest.makeSuite(banmanagertestcase.AddFailure))
-# ClientReader
+# ClientReaders
+tests.addTest(unittest.makeSuite(clientreadertestcase.ConfigReaderTest))
 tests.addTest(unittest.makeSuite(clientreadertestcase.JailReaderTest))
 tests.addTest(unittest.makeSuite(clientreadertestcase.FilterReaderTest))
+tests.addTest(unittest.makeSuite(clientreadertestcase.JailsReaderTest))
+# CSocket and AsyncServer
+tests.addTest(unittest.makeSuite(sockettestcase.Socket))
 
 # Filter
 if not opts.no_network:
@@ -173,6 +191,9 @@ for Filter_ in filters:
 	tests.addTest(unittest.makeSuite(
 		filtertestcase.get_monitor_failures_testcase(Filter_)))
 
+# Server test for logging elements which break logging used to support
+# testcases analysis
+tests.addTest(unittest.makeSuite(servertestcase.TransmitterLogging))
 
 #
 # Run the tests

man/fail2ban.1

@@ -0,0 +1,44 @@
+.TH FAIL2BAN "1" "March 2013" "Fail2Ban"
+.SH NAME
+fail2ban \- a set of server and client programs to limit brute force authentication attempts.
+.SH DESCRIPTION
+Fail2Ban consists of a client, server and configuration files to limit
+brute force authentication attempts.
+
+The server program \fBfail2ban-server\fR is responsible for monitoring
+log files and issuing ban/unban commands.  It gets configured through
+a simple protocol by \fBfail2ban-client\fR, which can also read
+configuration files and issue corresponding configuration commands to
+the server.
+
+For details on the configuration of fail2ban see the jail.conf(5)
+manual page.  A jail (as specified in jail.conf) couples filters and
+actions definitions for any given list of files to get monitored.
+
+For details on the command-line options of fail2ban-server see the
+fail2ban-server(1) manual page.
+
+For details on the command-line options and commands for configuring
+the server via fail2ban-client see the fail2ban-client(1) manual page.
+
+For testing regular expressions specified in a filter using the
+fail2ban-regex program may be of use and its manual page is
+fail2ban-regex(1).
+
+.SH FILES
+\fI/etc/fail2ban/*\fR
+.SH AUTHOR
+Manual page written by Daniel Black and Yaroslav Halchenko
+.SH "REPORTING BUGS"
+Report bugs to https://github.com/fail2ban/fail2ban/issues
+.SH COPYRIGHT
+Copyright \(co 2013
+.br
+Copyright of modifications held by their respective authors.
+Licensed under the GNU General Public License v2 (GPL).
+.SH "SEE ALSO"
+.br 
+fail2ban-server(1)
+fail2ban-client(1)
+fail2ban-regex(1)
+jail.conf(5)

man/jail.conf.5

@@ -0,0 +1,168 @@
+.TH JAIL.CONF "5" "March 2013" "Fail2Ban" "Fail2Ban Configuration"
+.SH NAME
+jail.conf \- configuration for the fail2ban server
+.SH SYNOPSIS
+.I jail.conf / jail.local
+
+.I action.d/*.conf action.d/*.local
+
+.I filter.d/*.conf filter.d/*.local
+.SH DESCRIPTION
+Fail2ban has three configuration file types. Action files are the commands for banning and unbanning of IP address,
+Filter files tell fail2ban how to detect authentication failures, and Jail configurations combine filters with actions into jails.
+
+There are *.conf files that are distributed by fail2ban and *.local file that contain user customizations.
+It is recommended that *.conf files should remain unchanged.  If needed, customizations should be provided in *.local files.
+For instance, if you would like to customize the [ssh-iptables-ipset] jail, create a jail.local to extend jail.conf
+(the configuration for the fail2ban server).  The jail.local file will be the following if you only need to enable
+it:
+
+.TP
+\fIjail.local\fR
+[ssh-iptables-ipset]
+
+enabled = true
+
+.PP
+Override only the settings you need to change and the rest of the configuration will come from the corresponding
+*.conf file.
+
+\fI*.d/\fR
+.RS
+In addition to .local, for any .conf file there can be a corresponding
+\fI.d/\fR directory to contain additional .conf files that will be read after the
+appropriate .local file.  Last parsed file will take precidence over
+identical entries, parsed alphabetically, e.g.
+
+.RS
+\fIjail.d/01_enable.conf\fR - to enable a specific jail
+.RE
+.RS
+\fIjail.d/02_custom_port.conf\fR - containing specific configuration entry to change the port of the jail specified in the configuration
+.RE
+.RS
+\fIfail2ban.d/01_custom_log.conf\fR - containing specific configuration entry to use a different log path.
+.RE
+.RE
+
+.SH DEFAULT
+The following options are applicable to all jails. Their meaning is described in the default \fIjail.conf\fR file.
+.TP
+\fBignoreip\fR 
+.TP
+\fBbantime\fR
+.TP
+\fBfindtime\fR
+.TP
+\fBmaxretry\fR
+.TP
+\fBbackend\fR
+.TP
+\fBusedns\fR
+
+
+.SH "ACTION FILES"
+Action files specify which commands are executed to ban and unban an IP address. They are located under \fI/etc/fail2ban/action.d\fR.
+
+Like with jail.conf files, if you desire local changes create an \fI[actionname].local\fR file in the \fI/etc/fail2ban/action.d\fR directory
+and override the required settings.
+
+Action files are ini files that have two sections, \fBDefinition\fR and \fBInit\fR . 
+
+The [Init] section allows for action-specific settings. In \fIjail.conf/jail.local\fR these can be overwritten for a particular jail as options to the jail.
+
+The following commands can be present in the [Definition] section.
+.TP
+\fBactionstart\fR
+command(s) executed when the jail starts.
+.TP
+\fBactionstop\fR
+command(s) executed when the jail stops.
+.TP
+\fBactioncheck\fR
+the command ran before any other action. It aims to verify if the environment is still ok.
+.TP
+\fBactionban\fR
+command(s) that bans the IP address after \fBmaxretry\fR log lines matches within last \fBfindtime\fR seconds.
+.TP
+\fBactionunban\fR
+command(s) that unbans the IP address after \fBbantime\fR.
+
+Commands specified in the [Definition] section are executed through a system shell so shell redirection and process control is allowed. The commands should
+return 0, otherwise error would be logged.  Moreover if \fBactioncheck\fR exits with non-0 status, it is taken as indication that firewall status has changed and fail2ban needs to reinitialize itself (i.e. issue \fBactionstop\fR and \fBactionstart\fR commands).
+
+Tags are enclosed in <>.  All the elements of [Init] are tags that are replaced in all action commands.  Tags can be added by the
+\fBfail2ban-client\fR using the setctag command.
+
+More than a single command is allowed to be specified. Each command needs to be on a separate line and indented with whitespaces without blank lines. The following example defines
+two commands to be executed.
+
+ actionban = iptables -I fail2ban-<name> --source <ip> -j DROP
+             echo ip=<ip>, match=<match>, time=<time> >> /var/log/fail2ban.log
+
+.SS "Action Tags"
+The following tags are substituted in the actionban, actionunban and actioncheck (when called before actionban/actionunban) commands.
+.TP
+\fBip\fR
+An IPv4 ip address to be banned. e.g. 192.168.0.2
+.TP
+\fBfailures\fR
+The number of times the failure occurred in the log file. e.g. 3
+.TP
+\fBtime\fR
+The unix time of the ban. e.g. 1357508484
+.TP
+\fBmatches\fR
+The concatenated string of the log file lines of the matches that generated the ban. Many characters interpreted by shell get escaped.
+
+.SH FILTER FILES
+
+Filter definitions are those in \fI/etc/fail2ban/filter.d/*.conf\fR and \fIfilter.d/*.local\fR.
+
+These are used to identify failed authentication attempts in logs and to extract the host IP address (or hostname if \fBusedns\fR is \fBtrue\fR).
+
+Like action files, filter files are ini files. The main section is the [Definition] section.
+
+There are two filter definitions used in the [Definition] section:
+
+.TP
+\fBfailregex\fR
+is the regex (\fBreg\fRular \fBex\fRpression) that will match failed attempts. The tag <HOST> is used as part of the regex and is itself a regex
+for IPv4 addresses and hostnames. fail2ban will work out which one of these it actually is.
+
+.TP
+\fBignoreregex\fR
+is the regex to identify log entries that should be ignored by fail2ban, even if they match failregex.
+
+
+Using Python "string interpolation" mechanisms, other definitions are allowed and can later be used within other definitions as %(defnname)s. For example.
+
+ baduseragents = IE|wget
+ failregex = useragent=%(baduseragents)s
+
+.PP
+Filters can also have a section called [INCLUDES]. This is used to read other configuration files.
+
+.TP
+\fBbefore\fR
+indicates that this file is read before the [Definition] section.
+
+.TP
+\fBafter\fR
+indicates that this file is read after the [Definition] section.
+
+.SH AUTHOR
+Fail2ban was originally written by Cyril Jaquier <cyril.jaquier@fail2ban.org>.
+At the moment it is maintained and further developed by Yaroslav O. Halchenko <debian@onerussian.com> and a number of contributors.  See \fBTHANKS\fR file shipped with Fail2Ban for a full list.
+.
+Manual page written by Daniel Black and Yaroslav Halchenko.
+.SH "REPORTING BUGS"
+Report bugs to https://github.com/fail2ban/fail2ban/issues
+.SH COPYRIGHT
+Copyright \(co 2013 Daniel Black
+.br
+Copyright of modifications held by their respective authors.
+Licensed under the GNU General Public License v2 (GPL).
+.SH "SEE ALSO"
+.br
+fail2ban-server(1)

server/banmanager.py

@@ -148,7 +148,7 @@ class BanManager:
 	def addBanTicket(self, ticket):
 		try:
 			self.__lock.acquire()
-			if not self.__inBanList(ticket):
+			if not self._inBanList(ticket):
 				self.__banList.append(ticket)
 				self.__banTotal += 1
 				return True
@@ -177,7 +177,7 @@ class BanManager:
 	# @param ticket the ticket
 	# @return True if a ticket already exists
 	
-	def __inBanList(self, ticket):
+	def _inBanList(self, ticket):
 		for i in self.__banList:
 			if ticket.getIP() == i.getIP():
 				return True

server/datetemplate.py

@@ -50,8 +50,11 @@ class DateTemplate:
 	def getName(self):
 		return self.__name
 	
-	def setRegex(self, regex):
+	def setRegex(self, regex, wordBegin=True):
-		self.__regex = regex.strip()
+		regex = regex.strip()
+		if (wordBegin and not re.search(r'^\^', regex)):
+			regex = r'\b' + regex
+		self.__regex = regex
 		self.__cRegex = re.compile(regex)
 		
 	def getRegex(self):
@@ -158,6 +161,7 @@ class DateStrptime(DateTemplate):
 							"pattern" % (opattern, e))
 			if date[0] < 2000:
 				# There is probably no year field in the logs
+				# NOTE: Possibly makes week/year day incorrect
 				date[0] = MyTime.gmtime()[0]
 				# Bug fix for #1241756
 				# If the date is greater than the current time, we suppose
@@ -166,10 +170,12 @@ class DateStrptime(DateTemplate):
 					logSys.debug(
 						u"Correcting deduced year from %d to %d since %f > %f" %
 						(date[0], date[0]-1, time.mktime(date), MyTime.time()))
+					# NOTE: Possibly makes week/year day incorrect
 					date[0] -= 1
 				elif date[1] == 1 and date[2] == 1:
 					# If it is Jan 1st, it is either really Jan 1st or there
 					# is neither month nor day in the log.
+					# NOTE: Possibly makes week/year day incorrect
 					date[1] = MyTime.gmtime()[1]
 					date[2] = MyTime.gmtime()[2]
 		return date
@@ -180,7 +186,8 @@ class DateTai64n(DateTemplate):
 	def __init__(self):
 		DateTemplate.__init__(self)
 		# We already know the format for TAI64N
-		self.setRegex("@[0-9a-f]{24}")
+		# yoh: we should not add an additional front anchor
+		self.setRegex("@[0-9a-f]{24}", wordBegin=False)
 	
 	def getDate(self, line):
 		date = None

server/failmanager.py

@@ -105,9 +105,17 @@ class FailManager:
 				fData.setLastReset(unixTime)
 				fData.setLastTime(unixTime)
 				self.__failList[ip] = fData
-			logSys.debug("Currently have failures from %d IPs: %s"
+
-						 % (len(self.__failList), self.__failList.keys()))
 			self.__failTotal += 1
+
+			if logSys.getEffectiveLevel() <= logging.DEBUG:
+				# yoh: Since composing this list might be somewhat time consuming
+				# in case of having many active failures, it should be ran only
+				# if debug level is "low" enough
+				failures_summary = ', '.join(['%s:%d' % (k, v.getRetry())
+											  for k,v in  self.__failList.iteritems()])
+				logSys.debug("Total # of detected failures: %d. Current failures from %d IPs (IP:count): %s"
+							 % (self.__failTotal, len(self.__failList), failures_summary))
 		finally:
 			self.__lock.release()
 	

server/filterpyinotify.py

@@ -63,16 +63,17 @@ class FilterPyinotify(FileFilter):
 		logSys.debug("Created FilterPyinotify")
 
 
-	def callback(self, event):
+	def callback(self, event, origin=''):
+		logSys.debug("%sCallback for Event: %s", origin, event)
 		path = event.pathname
 		if event.mask & pyinotify.IN_CREATE:
 			# skip directories altogether
 			if event.mask & pyinotify.IN_ISDIR:
-				logSys.debug("Ignoring creation of directory %s" % path)
+				logSys.debug("Ignoring creation of directory %s", path)
 				return
 			# check if that is a file we care about
 			if not path in self.__watches:
-				logSys.debug("Ignoring creation of %s we do not monitor" % path)
+				logSys.debug("Ignoring creation of %s we do not monitor", path)
 				return
 			else:
 				# we need to substitute the watcher with a new one, so first
@@ -104,8 +105,8 @@ class FilterPyinotify(FileFilter):
 	def _addFileWatcher(self, path):
 		wd = self.__monitor.add_watch(path, pyinotify.IN_MODIFY)
 		self.__watches.update(wd)
-		logSys.debug("Added file watcher for %s" % path)
+		logSys.debug("Added file watcher for %s", path)
-		# process the file since we did get even 
+		# process the file since we did get even
 		self._process_file(path)
 
 
@@ -114,7 +115,7 @@ class FilterPyinotify(FileFilter):
 		wd = self.__monitor.rm_watch(wdInt)
 		if wd[wdInt]:
 			del self.__watches[path]
-			logSys.debug("Removed file watcher for %s" % path)
+			logSys.debug("Removed file watcher for %s", path)
 			return True
 		else:
 			return False
@@ -130,7 +131,7 @@ class FilterPyinotify(FileFilter):
 			# we need to watch also  the directory for IN_CREATE
 			self.__watches.update(
 				self.__monitor.add_watch(path_dir, pyinotify.IN_CREATE))
-			logSys.debug("Added monitor for the parent directory %s" % path_dir)
+			logSys.debug("Added monitor for the parent directory %s", path_dir)
 
 		self._addFileWatcher(path)
 
@@ -151,7 +152,7 @@ class FilterPyinotify(FileFilter):
 			# since there is no other monitored file under this directory
 			wdInt = self.__watches.pop(path_dir)
 			_ = self.__monitor.rm_watch(wdInt)
-			logSys.debug("Removed monitor for the parent directory %s" % path_dir)
+			logSys.debug("Removed monitor for the parent directory %s", path_dir)
 
 
 	##
@@ -165,7 +166,7 @@ class FilterPyinotify(FileFilter):
 		self.__notifier = pyinotify.ThreadedNotifier(self.__monitor,
 			ProcessPyinotify(self))
 		self.__notifier.start()
-		logSys.debug("pyinotifier started for %s." % self.jail.getName())
+		logSys.debug("pyinotifier started for %s.", self.jail.getName())
 		# TODO: verify that there is nothing really to be done for
 		#       idle jails
 		return True
@@ -201,5 +202,4 @@ class ProcessPyinotify(pyinotify.ProcessEvent):
 
 	# just need default, since using mask on watch to limit events
 	def process_default(self, event):
-		logSys.debug("Callback for Event: %s" % event)
+		self.__FileFilter.callback(event, origin='Default ')
-		self.__FileFilter.callback(event)

server/server.py

@@ -385,7 +385,7 @@ class Server:
 				try:
 					handler.flush()
 					handler.close()
-				except ValueError:
+				except (ValueError, KeyError):
 					if sys.version_info >= (2,6):
 						raise
 					# is known to be thrown after logging was shutdown once

setup.cfg

@@ -3,3 +3,11 @@ install-purelib=/usr/share/fail2ban
 
 [sdist]
 formats=bztar
+
+[bdist_rpm]
+release = 1
+packager = Yaroslav Halchenko <debian@onerussian.com>, Daniel Black <grooverdan@users.sourceforge.net>
+doc_files = DEVELOP
+            README
+            THANKS
+            doc/run-rootless.txt

setup.py

@@ -67,6 +67,9 @@ setup(
 						),
 						('/var/run/fail2ban',
 							''
+						),
+						('/usr/share/doc/fail2ban',
+							['README', 'DEVELOP', 'doc/run-rootless.txt']
 						)
 					]
 )
@@ -82,13 +85,7 @@ elements =	{
 				"/usr/bin/":
 					[
 						"fail2ban.py"
-					],
+					], 
-				"/usr/lib/fail2ban/firewall/":
-					[
-						"iptables.py",
-						"ipfwadm.py",
-						"ipfw.py"
-					],
 				"/usr/lib/fail2ban/":
 					[
 						"version.py",

testcases/banmanagertestcase.py

@@ -49,11 +49,11 @@ class AddFailure(unittest.TestCase):
 		self.assertFalse(self.__banManager.addBanTicket(self.__ticket))
 		self.assertEqual(self.__banManager.size(), 1)
 		
-	def _testInListOK(self):
+	def testInListOK(self):
 		ticket = BanTicket('193.168.0.128', 1167605999.0)
-		self.assertTrue(self.__banManager.inBanList(ticket))
+		self.assertTrue(self.__banManager._inBanList(ticket))
 	
-	def _testInListNOK(self):
+	def testInListNOK(self):
 		ticket = BanTicket('111.111.1.111', 1167605999.0)
-		self.assertFalse(self.__banManager.inBanList(ticket))
+		self.assertFalse(self.__banManager._inBanList(ticket))
 		

testcases/clientreadertestcase.py

@@ -17,28 +17,93 @@
 # along with Fail2Ban; if not, write to the Free Software
 # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
 
-# Author: Cyril Jaquier
+__author__ = "Cyril Jaquier, Yaroslav Halchenko"
-# 
+__copyright__ = "Copyright (c) 2004 Cyril Jaquier, 2011-2013 Yaroslav Halchenko"
-# $Revision$
-
-__author__ = "Cyril Jaquier"
-__version__ = "$Revision$"
-__date__ = "$Date$"
-__copyright__ = "Copyright (c) 2004 Cyril Jaquier"
 __license__ = "GPL"
 
-import unittest
+import os, shutil, tempfile, unittest
-from client.jailreader import JailReader
 from client.configreader import ConfigReader
+from client.jailreader import JailReader
 from client.filterreader import FilterReader
+from client.jailsreader import JailsReader
+from client.configurator import Configurator
 
-class JailReaderTest(unittest.TestCase):
+class ConfigReaderTest(unittest.TestCase):
 
 	def setUp(self):
 		"""Call before every test case."""
+		self.d = tempfile.mkdtemp(prefix="f2b-temp")
+		self.c = ConfigReader(basedir=self.d)
 
 	def tearDown(self):
 		"""Call after every test case."""
+		shutil.rmtree(self.d)
+
+	def _write(self, fname, value):
+		# verify if we don't need to create .d directory
+		if os.path.sep in fname:
+			d = os.path.dirname(fname)
+			d_ = os.path.join(self.d, d)
+			if not os.path.exists(d_):
+				os.makedirs(d_)
+		open("%s/%s" % (self.d, fname), "w").write("""
+[section]
+option = %s
+""" % value)
+
+	def _remove(self, fname):
+		os.unlink("%s/%s" % (self.d, fname))
+		self.assertTrue(self.c.read('c'))	# we still should have some
+
+
+	def _getoption(self, f='c'):
+		self.assertTrue(self.c.read(f))	# we got some now
+		return self.c.getOptions('section', [("int", 'option')])['option']
+
+
+	def testInaccessibleFile(self):
+		f = os.path.join(self.d, "d.conf")  # inaccessible file
+		self._write('d.conf', 0)
+		self.assertEqual(self._getoption('d'), 0)
+		os.chmod(f, 0)
+		self.assertFalse(self.c.read('d'))	# should not be readable BUT present
+
+
+	def testOptionalDotDDir(self):
+		self.assertFalse(self.c.read('c'))	# nothing is there yet
+		self._write("c.conf", "1")
+		self.assertEqual(self._getoption(), 1)
+		self._write("c.conf", "2")		# overwrite
+		self.assertEqual(self._getoption(), 2)
+		self._write("c.local", "3")		# add override in .local
+		self.assertEqual(self._getoption(), 3)
+		self._write("c.d/98.conf", "998") # add 1st override in .d/
+		self.assertEqual(self._getoption(), 998)
+		self._write("c.d/90.conf", "990") # add previously sorted override in .d/
+		self.assertEqual(self._getoption(), 998) #  should stay the same
+		self._write("c.d/99.conf", "999") # now override in a way without sorting we possibly get a failure
+		self.assertEqual(self._getoption(), 999)
+		self._remove("c.d/99.conf")
+		self.assertEqual(self._getoption(), 998)
+		self._remove("c.d/98.conf")
+		self.assertEqual(self._getoption(), 990)
+		self._remove("c.d/90.conf")
+		self.assertEqual(self._getoption(), 3)
+		self._remove("c.conf")			#  we allow to stay without .conf
+		self.assertEqual(self._getoption(), 3)
+		self._write("c.conf", "1")
+		self._remove("c.local")
+		self.assertEqual(self._getoption(), 1)
+
+
+class JailReaderTest(unittest.TestCase):
+
+	def testStockSSHJail(self):
+		jail = JailReader('ssh-iptables', basedir='config') # we are running tests from root project dir atm
+		self.assertTrue(jail.read())
+		self.assertTrue(jail.getOptions())
+		self.assertFalse(jail.isEnabled())
+		self.assertEqual(jail.getName(), 'ssh-iptables')
 
 	def testSplitAction(self):
 		action = "mail-whois[name=SSH]"
@@ -81,3 +146,60 @@ class FilterReaderTest(unittest.TestCase):
 		filterReader.getOptions(None)
 
 		self.assertEquals(filterReader.convert(), output)
+
+class JailsReaderTest(unittest.TestCase):
+
+	def testProvidingBadBasedir(self):
+		if not os.path.exists('/XXX'):
+			reader = JailsReader(basedir='/XXX')
+			self.assertRaises(ValueError, reader.read)
+
+	def testReadStockJailConf(self):
+		jails = JailsReader(basedir='config') # we are running tests from root project dir atm
+		self.assertTrue(jails.read())		  # opens fine
+		self.assertTrue(jails.getOptions())	  # reads fine
+		comm_commands = jails.convert()
+		# by default None of the jails is enabled and we get no
+		# commands to communicate to the server
+		self.assertEqual(comm_commands, [])
+
+	def testReadStockJailConfForceEnabled(self):
+		# more of a smoke test to make sure that no obvious surprises
+		# on users' systems when enabling shipped jails
+		jails = JailsReader(basedir='config', force_enable=True) # we are running tests from root project dir atm
+		self.assertTrue(jails.read())		  # opens fine
+		self.assertTrue(jails.getOptions())	  # reads fine
+		comm_commands = jails.convert()
+
+		# by default we have lots of jails ;)
+		self.assertTrue(len(comm_commands))
+
+		# and we know even some of them by heart
+		for j in ['ssh-iptables', 'recidive']:
+			# by default we have 'auto' backend ATM
+			self.assertTrue(['add', j, 'auto'] in comm_commands)
+			# and warn on useDNS
+			self.assertTrue(['set', j, 'usedns', 'warn'] in comm_commands)
+			self.assertTrue(['start', j] in comm_commands)
+		# last commands should be the 'start' commands
+		self.assertEqual(comm_commands[-1][0], 'start')
+		# TODO: make sure that all of the jails have actions assigned,
+		#       otherwise it makes little to no sense
+
+	def testConfigurator(self):
+		configurator = Configurator()
+		configurator.setBaseDir('config')
+		self.assertEqual(configurator.getBaseDir(), 'config')
+
+		configurator.readEarly()
+		opts = configurator.getEarlyOptions()
+		# our current default settings
+		self.assertEqual(opts['socket'], '/var/run/fail2ban/fail2ban.sock')
+		self.assertEqual(opts['pidfile'], '/var/run/fail2ban/fail2ban.pid')
+
+		# and if we force change configurator's fail2ban's baseDir
+		# there should be an error message (test visually ;) --
+		# otherwise just a code smoke test)
+		configurator._Configurator__jails.setBaseDir('/tmp')
+		self.assertEqual(configurator._Configurator__jails.getBaseDir(), '/tmp')
+		self.assertEqual(configurator.getBaseDir(), 'config')

testcases/datedetectortestcase.py

@@ -45,36 +45,50 @@ class DateDetectorTest(unittest.TestCase):
 		log = "1138049999 [sshd] error: PAM: Authentication failure"
 		date = [2006, 1, 23, 21, 59, 59, 0, 23, 0]
 		dateUnix = 1138049999.0
-		
+
 		self.assertEqual(self.__datedetector.getTime(log), date)
 		self.assertEqual(self.__datedetector.getUnixTime(log), dateUnix)
 	
 	def testGetTime(self):
 		log = "Jan 23 21:59:59 [sshd] error: PAM: Authentication failure"
-		date = [2005, 1, 23, 21, 59, 59, 1, 23, -1]
+		date = [2005, 1, 23, 21, 59, 59, 6, 23, -1]
 		dateUnix = 1106513999.0
-	
+		# yoh: testing only up to 6 elements, since the day of the week
-		self.assertEqual(self.__datedetector.getTime(log), date)
+		#      is not correctly determined atm, since year is not present
+		#      in the log entry.  Since this doesn't effect the operation
+		#      of fail2ban -- we just ignore incorrect day of the week
+		self.assertEqual(self.__datedetector.getTime(log)[:6], date[:6])
 		self.assertEqual(self.__datedetector.getUnixTime(log), dateUnix)
 
 	def testVariousTimes(self):
 		"""Test detection of various common date/time formats f2b should understand
 		"""
-		date = [2005, 1, 23, 21, 59, 59, 1, 23, -1]
+		date = [2005, 1, 23, 21, 59, 59, 6, 23, -1]
 		dateUnix = 1106513999.0
 
 		for sdate in (
 			"Jan 23 21:59:59",
+			"Sun Jan 23 21:59:59 2005",
+			"Sun Jan 23 21:59:59",
+			"2005/01/23 21:59:59",
 			"2005.01.23 21:59:59",
 			"23/01/2005 21:59:59",
+			"23/01/05 21:59:59",
+			"23/Jan/2005:21:59:59",
+			"01/23/2005:21:59:59",
+			"2005-01-23 21:59:59",
+			"23-Jan-2005 21:59:59",
+			"23-01-2005 21:59:59",
 			"01-23-2005 21:59:59.252", # reported on f2b, causes Feb29 fix to break
+			"@4000000041f4104f00000000", # TAI64N
+			"2005-01-23T21:59:59.252Z", #ISO 8601
+			"2005-01-23T21:59:59-05:00Z", #ISO 8601 with TZ
+			"<01/23/05@21:59:59>",
 			):
 			log = sdate + "[sshd] error: PAM: Authentication failure"
 			# exclude
 
-			# TODO (Yarik is confused): figure out why for above it is
+			# yoh: on [:6] see in above test
-			#      "1" as day of the week which would be Tue, although it
-			#      was Sun
 			self.assertEqual(self.__datedetector.getTime(log)[:6], date[:6])
 			self.assertEqual(self.__datedetector.getUnixTime(log), dateUnix)
 
@@ -89,6 +103,27 @@ class DateDetectorTest(unittest.TestCase):
 		self.assertRaises(ValueError, self.__datedetector._appendTemplate,
 						  self.__datedetector.getTemplates()[0])
 
+	def testFullYearMatch_gh130(self):
+		# see https://github.com/fail2ban/fail2ban/pull/130
+		# yoh: unfortunately this test is not really effective to reproduce the
+		#      situation but left in place to assure consistent behavior
+		m1 = [2012, 10, 11, 2, 37, 17]
+		self.assertEqual(
+			self.__datedetector.getTime('2012/10/11 02:37:17 [error] 18434#0')[:6],
+			m1)
+		self.__datedetector.sortTemplate()
+		# confuse it with year being at the end
+		for i in xrange(10):
+			self.assertEqual(
+				self.__datedetector.getTime('11/10/2012 02:37:17 [error] 18434#0')[:6],
+				m1)
+		self.__datedetector.sortTemplate()
+		# and now back to the original
+		self.assertEqual(
+			self.__datedetector.getTime('2012/10/11 02:37:17 [error] 18434#0')[:6],
+			m1)
+
+
 #	def testDefaultTempate(self):
 #		self.__datedetector.setDefaultRegex("^\S{3}\s{1,2}\d{1,2} \d{2}:\d{2}:\d{2}")
 #		self.__datedetector.setDefaultPattern("%b %d %H:%M:%S")

testcases/files/logs/postfix

@@ -0,0 +1,3 @@
+# per https://github.com/fail2ban/fail2ban/issues/125
+# and https://github.com/fail2ban/fail2ban/issues/126
+Feb 21 09:21:54 xxx postfix/smtpd[14398]: NOQUEUE: reject: RCPT from example.com[192.0.43.10]: 450 4.7.1 : Helo command rejected: Host not found; from=<> to=<> proto=ESMTP helo=

testcases/files/logs/sogo-auth

@@ -0,0 +1,17 @@
+# yoh: Kept original apache log lines as well, just in case they might come useful
+#      for (testing) multiline regular expressions which would further constraint
+#      SOGo log lines
+Mar 24 08:58:32 sogod [26818]: <0x0xb8537990[LDAPSource]> <NSException: 0xb87c3008> NAME:LDAPException REASON:operation bind failed: Invalid credentials (0x31) INFO:{login = "uid=hack0r,ou=users,dc=mail,dc=example,dc=org"; }
+Mar 24 08:58:32 sogod [26818]: SOGoRootPage Login from '173.194.44.31' for user 'hack0r' might not have worked - password policy: 65535  grace: -1  expire: -1  bound: 0
+173.194.44.31 - - [24/Mar/2013:08:58:32 GMT] "POST /SOGo/connect HTTP/1.1" 403 34/38 0.311 - - 2M
+Mar 24 08:58:40 sogod [26818]: <0x0xb8537990[LDAPSource]> <NSException: 0xb87bb8d8> NAME:LDAPException REASON:operation bind failed: Invalid credentials (0x31) INFO:{login = "uid=kiddy,ou=users,dc=mail,dc=example,dc=org"; }
+Mar 24 08:58:40 sogod [26818]: SOGoRootPage Login from '173.194.44.31' for user 'kiddy' might not have worked - password policy: 65535  grace: -1  expire: -1  bound: 0
+173.194.44.31 - - [24/Mar/2013:08:58:40 GMT] "POST /SOGo/connect HTTP/1.1" 403 34/37 0.007 - - 32K
+Mar 24 08:58:50 sogod [26818]: <0x0xb8537990[LDAPSource]> <NSException: 0xb87c27f8> NAME:LDAPException REASON:operation bind failed: Invalid credentials (0x31) INFO:{login = "uid=plsBanMe,ou=users,dc=mail,dc=example,dc=org"; }
+Mar 24 08:58:50 sogod [26818]: SOGoRootPage Login from '173.194.44.31' for user 'plsBanMe' might not have worked - password policy: 65535  grace: -1  expire: -1  bound: 0
+173.194.44.31 - - [24/Mar/2013:08:58:50 GMT] "POST /SOGo/connect HTTP/1.1" 403 34/40 0.008 - - 0
+Mar 24 08:58:59 sogod [26818]: <0x0xb8537990[LDAPSource]> <NSException: 0xb87be830> NAME:LDAPException REASON:operation bind failed: Invalid credentials (0x31) INFO:{login = "uid=root,ou=users,dc=mail,dc=example,dc=org"; }
+Mar 24 08:58:59 sogod [26818]: SOGoRootPage Login from '173.194.44.31' for user 'root' might not have worked - password policy: 65535  grace: -1  expire: -1  bound: 0
+173.194.44.31 - - [24/Mar/2013:08:58:59 GMT] "POST /SOGo/connect HTTP/1.1" 403 34/36 0.007 - - 0
+Mar 24 08:59:04 sogod [26818]: <0x0xb8537990[LDAPSource]> <NSException: 0xb87bc088> NAME:LDAPException REASON:operation bind failed: Invalid credentials (0x31) INFO:{login = "uid=admin,ou=users,dc=mail,dc=example,dc=org"; }
+Mar 24 08:59:04 sogod [26818]: SOGoRootPage Login from '173.194.44.31' for user 'admin' might not have worked - password policy: 65535  grace: -1  expire: -1  bound: 0

testcases/filtertestcase.py

@@ -22,6 +22,7 @@
 __copyright__ = "Copyright (c) 2004 Cyril Jaquier; 2012 Yaroslav Halchenko"
 __license__ = "GPL"
 
+from __builtin__ import open as fopen
 import unittest
 import os
 import sys
@@ -38,6 +39,20 @@ from server.failmanager import FailManagerEmpty
 # Useful helpers
 #
 
+# yoh: per Steven Hiscocks's insight while troubleshooting
+# https://github.com/fail2ban/fail2ban/issues/103#issuecomment-15542836
+# adding a sufficiently large buffer might help to guarantee that
+# writes happen atomically.
+def open(*args):
+	"""Overload built in open so we could assure sufficiently large buffer
+
+	Explicit .flush would be needed to assure that changes leave the buffer
+	"""
+	if len(args) == 2:
+		# ~50kB buffer should be sufficient for all tests here.
+		args = args + (50000,)
+	return fopen(*args)
+
 def _killfile(f, name):
 	try:
 		f.close()
@@ -48,6 +63,11 @@ def _killfile(f, name):
 	except:
 		pass
 
+	# there might as well be the .bak file
+	if os.path.exists(name + '.bak'):
+		_killfile(None, name + '.bak')
+
+
 def _sleep_4_poll():
 	"""PollFilter relies on file timestamps - so we might need to
 	sleep to guarantee that they differ
@@ -105,20 +125,23 @@ def _copy_lines_between_files(fin, fout, n=None, skip=0, mode='a', terminal_line
 		time.sleep(1)
 	if isinstance(fin, str): # pragma: no branch - only used with str in test cases
 		fin = open(fin, 'r')
-	if isinstance(fout, str):
-		fout = open(fout, mode)
 	# Skip
 	for i in xrange(skip):
 		_ = fin.readline()
-	# Read/Write
+	# Read
 	i = 0
+	lines = []
 	while n is None or i < n:
 		l = fin.readline()
 		if terminal_line is not None and l == terminal_line:
 			break
-		fout.write(l)
+		lines.append(l)
-		fout.flush()
 		i += 1
+	# Write: all at once and flush
+	if isinstance(fout, str):
+		fout = open(fout, mode)
+	fout.write('\n'.join(lines))
+	fout.flush()
 	# to give other threads possibly some time to crunch
 	time.sleep(0.1)
 	return fout
@@ -324,11 +347,17 @@ def get_monitor_failures_testcase(Filter_):
 	"""Generator of TestCase's for different filters/backends
 	"""
 
+	# add Filter_'s name so we could easily identify bad cows
+	testclass_name = tempfile.mktemp(
+		'fail2ban', 'monitorfailures_%s' % (Filter_.__name__,))
+
 	class MonitorFailures(unittest.TestCase):
+		count = 0
 		def setUp(self):
 			"""Call before every test case."""
 			self.filter = self.name = 'NA'
-			_, self.name = tempfile.mkstemp('fail2ban', 'monitorfailures')
+			self.name = '%s-%d' % (testclass_name, self.count)
+			MonitorFailures.count += 1 # so we have unique filenames across tests
 			self.file = open(self.name, 'a')
 			self.jail = DummyJail()
 			self.filter = Filter_(self.jail)
@@ -351,12 +380,9 @@ def get_monitor_failures_testcase(Filter_):
 			self.filter.join()		  # wait for the thread to terminate
 			#print "D: KILLING THE FILE"
 			_killfile(self.file, self.name)
+			#time.sleep(0.2)			  # Give FS time to ack the removal
 			pass
 
-		def __str__(self): # pragma: no cover - will only show up if unexpected exception is thrown
-			return "MonitorFailures%s(%s)" \
-			  % (Filter_, hasattr(self, 'name') and self.name or 'tempfile')
-
 		def isFilled(self, delay=2.):
 			"""Wait up to `delay` sec to assure that it was modified or not
 			"""
@@ -379,7 +405,7 @@ def get_monitor_failures_testcase(Filter_):
 			return not self.isFilled(delay)
 
 		def assert_correct_last_attempt(self, failures, count=None):
-			self.assertTrue(self.isFilled(10)) # give Filter a chance to react
+			self.assertTrue(self.isFilled(20)) # give Filter a chance to react
 			_assert_correct_last_attempt(self, self.jail, failures, count=count)
 
 
@@ -431,9 +457,10 @@ def get_monitor_failures_testcase(Filter_):
 			# if we move file into a new location while it has been open already
 			self.file = _copy_lines_between_files(GetFailures.FILENAME_01, self.name,
 												  n=14, mode='w')
-			self.assertTrue(self.isEmpty(2))
+			# Poll might need more time
+			self.assertTrue(self.isEmpty(4 + int(isinstance(self.filter, FilterPoll))*2))
 			self.assertRaises(FailManagerEmpty, self.filter.failManager.toBan)
-			self.assertEqual(self.filter.failManager.getFailTotal(), 2) # Fails with Poll from time to time
+			self.assertEqual(self.filter.failManager.getFailTotal(), 2)
 
 			# move aside, but leaving the handle still open...
 			os.rename(self.name, self.name + '.bak')
@@ -488,7 +515,8 @@ def get_monitor_failures_testcase(Filter_):
 			# yoh: not sure why count here is not 9... TODO
 			self.assert_correct_last_attempt(GetFailures.FAILURES_01)#, count=9)
 
-
+	MonitorFailures.__name__ = "MonitorFailures<%s>(%s)" \
+			  % (Filter_.__name__, testclass_name) # 'tempfile')
 	return MonitorFailures
 
 

testcases/servertestcase.py

@@ -50,27 +50,29 @@ class StartStop(unittest.TestCase):
 		time.sleep(1)
 		self.__server.stopJail(name)
 
+class TestServer(Server):
+	def setLogLevel(self, *args, **kwargs):
+		pass
+	def setLogTarget(self, *args, **kwargs):
+		pass
 
-class Transmitter(unittest.TestCase):
+class TransmitterBase(unittest.TestCase):
 	
 	def setUp(self):
 		"""Call before every test case."""
-		self.__server = Server()
+		self.transm = self.server._Server__transm
-		self.__transm = self.__server._Server__transm
-		self.__server.setLogTarget("/dev/null")
-		self.__server.setLogLevel(0)
 		sock_fd, sock_name = tempfile.mkstemp('fail2ban.sock', 'transmitter')
 		os.close(sock_fd)
 		pidfile_fd, pidfile_name = tempfile.mkstemp(
 			'fail2ban.pid', 'transmitter')
 		os.close(pidfile_fd)
-		self.__server.start(sock_name, pidfile_name, force=False)
+		self.server.start(sock_name, pidfile_name, force=False)
 		self.jailName = "TestJail1"
-		self.__server.addJail(self.jailName, "auto")
+		self.server.addJail(self.jailName, "auto")
 
 	def tearDown(self):
 		"""Call after every test case."""
-		self.__server.quit()
+		self.server.quit()
 
 	def setGetTest(self, cmd, inValue, outValue=None, jail=None):
 		setCmd = ["set", cmd, inValue]
@@ -81,8 +83,8 @@ class Transmitter(unittest.TestCase):
 		if outValue is None:
 			outValue = inValue
 
-		self.assertEqual(self.__transm.proceed(setCmd), (0, outValue))
+		self.assertEqual(self.transm.proceed(setCmd), (0, outValue))
-		self.assertEqual(self.__transm.proceed(getCmd), (0, outValue))
+		self.assertEqual(self.transm.proceed(getCmd), (0, outValue))
 
 	def setGetTestNOK(self, cmd, inValue, jail=None):
 		setCmd = ["set", cmd, inValue]
@@ -92,30 +94,30 @@ class Transmitter(unittest.TestCase):
 			getCmd.insert(1, jail)
 
 		# Get initial value before trying invalid value
-		initValue = self.__transm.proceed(getCmd)[1]
+		initValue = self.transm.proceed(getCmd)[1]
-		self.assertEqual(self.__transm.proceed(setCmd)[0], 1)
+		self.assertEqual(self.transm.proceed(setCmd)[0], 1)
 		# Check after failed set that value is same as previous
-		self.assertEqual(self.__transm.proceed(getCmd), (0, initValue))
+		self.assertEqual(self.transm.proceed(getCmd), (0, initValue))
 
 	def jailAddDelTest(self, cmd, values, jail):
 		cmdAdd = "add" + cmd
 		cmdDel = "del" + cmd
 
 		self.assertEqual(
-			self.__transm.proceed(["get", jail, cmd]), (0, []))
+			self.transm.proceed(["get", jail, cmd]), (0, []))
 		for n, value in enumerate(values):
 			self.assertEqual(
-				self.__transm.proceed(["set", jail, cmdAdd, value]),
+				self.transm.proceed(["set", jail, cmdAdd, value]),
 				(0, values[:n+1]))
 			self.assertEqual(
-				self.__transm.proceed(["get", jail, cmd]),
+				self.transm.proceed(["get", jail, cmd]),
 				(0, values[:n+1]))
 		for n, value in enumerate(values):
 			self.assertEqual(
-				self.__transm.proceed(["set", jail, cmdDel, value]),
+				self.transm.proceed(["set", jail, cmdDel, value]),
 				(0, values[n+1:]))
 			self.assertEqual(
-				self.__transm.proceed(["get", jail, cmd]),
+				self.transm.proceed(["get", jail, cmd]),
 				(0, values[n+1:]))
 
 	def jailAddDelRegexTest(self, cmd, inValues, outValues, jail):
@@ -126,107 +128,94 @@ class Transmitter(unittest.TestCase):
 			outValues = inValues
 
 		self.assertEqual(
-			self.__transm.proceed(["get", jail, cmd]), (0, []))
+			self.transm.proceed(["get", jail, cmd]), (0, []))
 		for n, value in enumerate(inValues):
 			self.assertEqual(
-				self.__transm.proceed(["set", jail, cmdAdd, value]),
+				self.transm.proceed(["set", jail, cmdAdd, value]),
 				(0, outValues[:n+1]))
 			self.assertEqual(
-				self.__transm.proceed(["get", jail, cmd]),
+				self.transm.proceed(["get", jail, cmd]),
 				(0, outValues[:n+1]))
 		for n, value in enumerate(inValues):
 			self.assertEqual(
-				self.__transm.proceed(["set", jail, cmdDel, 0]), # First item
+				self.transm.proceed(["set", jail, cmdDel, 0]), # First item
 				(0, outValues[n+1:]))
 			self.assertEqual(
-				self.__transm.proceed(["get", jail, cmd]),
+				self.transm.proceed(["get", jail, cmd]),
 				(0, outValues[n+1:]))
 
+class Transmitter(TransmitterBase):
+
+	def setUp(self):
+		self.server = TestServer()
+		super(Transmitter, self).setUp()
+
 	def testStopServer(self):
-		self.assertEqual(self.__transm.proceed(["stop"]), (0, None))
+		self.assertEqual(self.transm.proceed(["stop"]), (0, None))
 
 	def testPing(self):
-		self.assertEqual(self.__transm.proceed(["ping"]), (0, "pong"))
+		self.assertEqual(self.transm.proceed(["ping"]), (0, "pong"))
 
 	def testSleep(self):
 		t0 = time.time()
-		self.assertEqual(self.__transm.proceed(["sleep", "1"]), (0, None))
+		self.assertEqual(self.transm.proceed(["sleep", "1"]), (0, None))
 		t1 = time.time()
 		# Approx 1 second delay
 		self.assertAlmostEqual(t1 - t0, 1, places=2)
 
-	def testLogTarget(self):
-		logTargets = []
-		for _ in xrange(3):
-			tmpFile = tempfile.mkstemp("fail2ban", "transmitter")
-			logTargets.append(tmpFile[1])
-			os.close(tmpFile[0])
-		for logTarget in logTargets:
-			self.setGetTest("logtarget", logTarget)
-
-		# If path is invalid, do not change logtarget
-		value = "/this/path/should/not/exist"
-		self.setGetTestNOK("logtarget", value)
-
-		self.__transm.proceed(["set", "/dev/null"])
-		for logTarget in logTargets:
-			os.remove(logTarget)
-
-	def testLogLevel(self):
-		self.setGetTest("loglevel", "4", 4)
-		self.setGetTest("loglevel", "2", 2)
-		self.setGetTest("loglevel", "-1", -1)
-		self.setGetTestNOK("loglevel", "Bird")
-
 	def testAddJail(self):
 		jail2 = "TestJail2"
 		jail3 = "TestJail3"
 		jail4 = "TestJail4"
 		self.assertEqual(
-			self.__transm.proceed(["add", jail2, "polling"]), (0, jail2))
+			self.transm.proceed(["add", jail2, "polling"]), (0, jail2))
-		self.assertEqual(self.__transm.proceed(["add", jail3]), (0, jail3))
+		self.assertEqual(self.transm.proceed(["add", jail3]), (0, jail3))
 		self.assertEqual(
-			self.__transm.proceed(["add", jail4, "invalid backend"])[0], 1)
+			self.transm.proceed(["add", jail4, "invalid backend"])[0], 1)
 		self.assertEqual(
-			self.__transm.proceed(["add", jail4, "auto"]), (0, jail4))
+			self.transm.proceed(["add", jail4, "auto"]), (0, jail4))
 		# Duplicate Jail
 		self.assertEqual(
-			self.__transm.proceed(["add", self.jailName, "polling"])[0], 1)
+			self.transm.proceed(["add", self.jailName, "polling"])[0], 1)
 		# All name is reserved
 		self.assertEqual(
-			self.__transm.proceed(["add", "all", "polling"])[0], 1)
+			self.transm.proceed(["add", "all", "polling"])[0], 1)
 
 	def testStartStopJail(self):
 		self.assertEqual(
-			self.__transm.proceed(["start", self.jailName]), (0, None))
+			self.transm.proceed(["start", self.jailName]), (0, None))
 		time.sleep(1)
 		self.assertEqual(
-			self.__transm.proceed(["stop", self.jailName]), (0, None))
+			self.transm.proceed(["stop", self.jailName]), (0, None))
 		self.assertRaises(
-			UnknownJailException, self.__server.isAlive, self.jailName)
+			UnknownJailException, self.server.isAlive, self.jailName)
 
 	def testStartStopAllJail(self):
-		self.__server.addJail("TestJail2", "auto")
+		self.server.addJail("TestJail2", "auto")
 		self.assertEqual(
-			self.__transm.proceed(["start", self.jailName]), (0, None))
+			self.transm.proceed(["start", self.jailName]), (0, None))
 		self.assertEqual(
-			self.__transm.proceed(["start", "TestJail2"]), (0, None))
+			self.transm.proceed(["start", "TestJail2"]), (0, None))
-		self.assertEqual(self.__transm.proceed(["stop", "all"]), (0, None))
+		# yoh: workaround for gh-146.  I still think that there is some
+		#      race condition and missing locking somewhere, but for now
+		#      giving it a small delay reliably helps to proceed with tests
+		time.sleep(0.1)
+		self.assertEqual(self.transm.proceed(["stop", "all"]), (0, None))
 		time.sleep(1)
 		self.assertRaises(
-			UnknownJailException, self.__server.isAlive, self.jailName)
+			UnknownJailException, self.server.isAlive, self.jailName)
 		self.assertRaises(
-			UnknownJailException, self.__server.isAlive, "TestJail2")
+			UnknownJailException, self.server.isAlive, "TestJail2")
 
 	def testJailIdle(self):
 		self.assertEqual(
-			self.__transm.proceed(["set", self.jailName, "idle", "on"]),
+			self.transm.proceed(["set", self.jailName, "idle", "on"]),
 			(0, True))
 		self.assertEqual(
-			self.__transm.proceed(["set", self.jailName, "idle", "off"]),
+			self.transm.proceed(["set", self.jailName, "idle", "off"]),
 			(0, False))
 		self.assertEqual(
-			self.__transm.proceed(["set", self.jailName, "idle", "CAT"])[0],
+			self.transm.proceed(["set", self.jailName, "idle", "CAT"])[0],
 			1)
 
 	def testJailFindTime(self):
@@ -249,28 +238,28 @@ class Transmitter(unittest.TestCase):
 		# Safe default should be "no"
 		value = "Fish"
 		self.assertEqual(
-			self.__transm.proceed(["set", self.jailName, "usedns", value]),
+			self.transm.proceed(["set", self.jailName, "usedns", value]),
 			(0, "no"))
 
 	def testJailBanIP(self):
-		self.__server.startJail(self.jailName) # Jail must be started
+		self.server.startJail(self.jailName) # Jail must be started
 
 		self.assertEqual(
-			self.__transm.proceed(["set", self.jailName, "banip", "127.0.0.1"]),
+			self.transm.proceed(["set", self.jailName, "banip", "127.0.0.1"]),
 			(0, "127.0.0.1"))
 		time.sleep(1) # Give chance to ban
 		self.assertEqual(
-			self.__transm.proceed(["set", self.jailName, "banip", "Badger"]),
+			self.transm.proceed(["set", self.jailName, "banip", "Badger"]),
 			(0, "Badger")) #NOTE: Is IP address validated? Is DNS Lookup done?
 		time.sleep(1) # Give chance to ban
 		# Unban IP
 		self.assertEqual(
-			self.__transm.proceed(
+			self.transm.proceed(
 				["set", self.jailName, "unbanip", "127.0.0.1"]),
 			(0, "127.0.0.1"))
 		# Unban IP which isn't banned
 		self.assertEqual(
-			self.__transm.proceed(
+			self.transm.proceed(
 				["set", self.jailName, "unbanip", "192.168.1.1"])[0],1)
 
 	def testJailMaxRetry(self):
@@ -292,22 +281,22 @@ class Transmitter(unittest.TestCase):
 		# Try duplicates
 		value = "testcases/files/testcase04.log"
 		self.assertEqual(
-			self.__transm.proceed(["set", self.jailName, "addlogpath", value]),
+			self.transm.proceed(["set", self.jailName, "addlogpath", value]),
 			(0, [value]))
 		# Will silently ignore duplicate
 		self.assertEqual(
-			self.__transm.proceed(["set", self.jailName, "addlogpath", value]),
+			self.transm.proceed(["set", self.jailName, "addlogpath", value]),
 			(0, [value]))
 		self.assertEqual(
-			self.__transm.proceed(["get", self.jailName, "logpath"]),
+			self.transm.proceed(["get", self.jailName, "logpath"]),
 			(0, [value]))
 		self.assertEqual(
-			self.__transm.proceed(["set", self.jailName, "dellogpath", value]),
+			self.transm.proceed(["set", self.jailName, "dellogpath", value]),
 			(0, []))
 
 		# Invalid file
 		value = "this_file_shouldn't_exist"
-		result = self.__transm.proceed(
+		result = self.transm.proceed(
 			["set", self.jailName, "addlogpath", value])
 		self.assertTrue(isinstance(result[1], IOError))
 
@@ -325,18 +314,18 @@ class Transmitter(unittest.TestCase):
 		# Try duplicates
 		value = "127.0.0.1"
 		self.assertEqual(
-			self.__transm.proceed(["set", self.jailName, "addignoreip", value]),
+			self.transm.proceed(["set", self.jailName, "addignoreip", value]),
 			(0, [value]))
 		# Will allow duplicate
 		#NOTE: Should duplicates be allowed, or silent ignore like logpath?
 		self.assertEqual(
-			self.__transm.proceed(["set", self.jailName, "addignoreip", value]),
+			self.transm.proceed(["set", self.jailName, "addignoreip", value]),
 			(0, [value, value]))
 		self.assertEqual(
-			self.__transm.proceed(["get", self.jailName, "ignoreip"]),
+			self.transm.proceed(["get", self.jailName, "ignoreip"]),
 			(0, [value, value]))
 		self.assertEqual(
-			self.__transm.proceed(["set", self.jailName, "delignoreip", value]),
+			self.transm.proceed(["set", self.jailName, "delignoreip", value]),
 			(0, [value]))
 
 	def testJailRegex(self):
@@ -355,11 +344,11 @@ class Transmitter(unittest.TestCase):
 		)
 
 		self.assertEqual(
-			self.__transm.proceed(
+			self.transm.proceed(
 				["set", self.jailName, "addfailregex", "No host regex"])[0],
 			1)
 		self.assertEqual(
-			self.__transm.proceed(
+			self.transm.proceed(
 				["set", self.jailName, "addfailregex", 654])[0],
 			1)
 
@@ -379,25 +368,25 @@ class Transmitter(unittest.TestCase):
 		)
 
 		self.assertEqual(
-			self.__transm.proceed(
+			self.transm.proceed(
 				["set", self.jailName, "addignoreregex", "Invalid [regex"])[0],
 			1)
 		self.assertEqual(
-			self.__transm.proceed(
+			self.transm.proceed(
 				["set", self.jailName, "addignoreregex", 50])[0],
 			1)
 
 	def testStatus(self):
 		jails = [self.jailName]
-		self.assertEqual(self.__transm.proceed(["status"]),
+		self.assertEqual(self.transm.proceed(["status"]),
 			(0, [('Number of jail', len(jails)), ('Jail list', ", ".join(jails))]))
-		self.__server.addJail("TestJail2", "auto")
+		self.server.addJail("TestJail2", "auto")
 		jails.append("TestJail2")
-		self.assertEqual(self.__transm.proceed(["status"]),
+		self.assertEqual(self.transm.proceed(["status"]),
 			(0, [('Number of jail', len(jails)), ('Jail list', ", ".join(jails))]))
 
 	def testJailStatus(self):
-		self.assertEqual(self.__transm.proceed(["status", self.jailName]),
+		self.assertEqual(self.transm.proceed(["status", self.jailName]),
 			(0,
 				[
 					('filter', [
@@ -432,54 +421,90 @@ class Transmitter(unittest.TestCase):
 		]
 
 		self.assertEqual(
-			self.__transm.proceed(["set", self.jailName, "addaction", action]),
+			self.transm.proceed(["set", self.jailName, "addaction", action]),
 			(0, action))
 		self.assertEqual(
-			self.__transm.proceed(["get", self.jailName, "addaction", action]),
+			self.transm.proceed(["get", self.jailName, "addaction", action]),
 			(0, action))
 		for cmd, value in zip(cmdList, cmdValueList):
 			self.assertEqual(
-				self.__transm.proceed(
+				self.transm.proceed(
 					["set", self.jailName, cmd, action, value]),
 				(0, value))
 		for cmd, value in zip(cmdList, cmdValueList):
 			self.assertEqual(
-				self.__transm.proceed(["get", self.jailName, cmd, action]),
+				self.transm.proceed(["get", self.jailName, cmd, action]),
 				(0, value))
 		self.assertEqual(
-			self.__transm.proceed(
+			self.transm.proceed(
 				["set", self.jailName, "setcinfo", action, "KEY", "VALUE"]),
 			(0, "VALUE"))
 		self.assertEqual(
-			self.__transm.proceed(
+			self.transm.proceed(
 				["get", self.jailName, "cinfo", action, "KEY"]),
 			(0, "VALUE"))
 		self.assertEqual(
-			self.__transm.proceed(
+			self.transm.proceed(
 				["get", self.jailName, "cinfo", action, "InvalidKey"])[0],
 			1)
 		self.assertEqual(
-			self.__transm.proceed(
+			self.transm.proceed(
 				["set", self.jailName, "delcinfo", action, "KEY"]),
 			(0, None))
 		self.assertEqual(
-			self.__transm.proceed(["set", self.jailName, "delaction", action]),
+			self.transm.proceed(["set", self.jailName, "delaction", action]),
 			(0, None))
 		self.assertEqual(
-			self.__transm.proceed(
+			self.transm.proceed(
 				["set", self.jailName, "delaction", "Doesn't exist"])[0],1)
 
 	def testNOK(self):
-		self.assertEqual(self.__transm.proceed(["INVALID", "COMMAND"])[0],1)
+		self.assertEqual(self.transm.proceed(["INVALID", "COMMAND"])[0],1)
 
 	def testSetNOK(self):
 		self.assertEqual(
-			self.__transm.proceed(["set", "INVALID", "COMMAND"])[0],1)
+			self.transm.proceed(["set", "INVALID", "COMMAND"])[0],1)
 
 	def testGetNOK(self):
 		self.assertEqual(
-			self.__transm.proceed(["get", "INVALID", "COMMAND"])[0],1)
+			self.transm.proceed(["get", "INVALID", "COMMAND"])[0],1)
 
 	def testStatusNOK(self):
 		self.assertEqual(
-			self.__transm.proceed(["status", "INVALID", "COMMAND"])[0],1)
+			self.transm.proceed(["status", "INVALID", "COMMAND"])[0],1)
+
+class TransmitterLogging(TransmitterBase):
+
+	def setUp(self):
+		self.server = Server()
+		self.server.setLogTarget("/dev/null")
+		self.server.setLogLevel(0)
+		super(TransmitterLogging, self).setUp()
+
+	def testLogTarget(self):
+		logTargets = []
+		for _ in xrange(3):
+			tmpFile = tempfile.mkstemp("fail2ban", "transmitter")
+			logTargets.append(tmpFile[1])
+			os.close(tmpFile[0])
+		for logTarget in logTargets:
+			self.setGetTest("logtarget", logTarget)
+
+		# If path is invalid, do not change logtarget
+		value = "/this/path/should/not/exist"
+		self.setGetTestNOK("logtarget", value)
+
+		self.transm.proceed(["set", "/dev/null"])
+		for logTarget in logTargets:
+			os.remove(logTarget)
+
+		self.setGetTest("logtarget", "STDOUT")
+		self.setGetTest("logtarget", "STDERR")
+		self.setGetTest("logtarget", "SYSLOG")
+
+	def testLogLevel(self):
+		self.setGetTest("loglevel", "4", 4)
+		self.setGetTest("loglevel", "2", 2)
+		self.setGetTest("loglevel", "-1", -1)
+		self.setGetTest("loglevel", "0", 0)
+		self.setGetTestNOK("loglevel", "Bird")

testcases/sockettestcase.py

@@ -0,0 +1,82 @@
+# emacs: -*- mode: python; py-indent-offset: 4; indent-tabs-mode: t -*-
+# vi: set ft=python sts=4 ts=4 sw=4 noet :
+
+# This file is part of Fail2Ban.
+#
+# Fail2Ban is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# Fail2Ban is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with Fail2Ban; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
+
+# Author: Steven Hiscocks
+# 
+# $Revision$
+
+__author__ = "Steven Hiscocks"
+__version__ = "$Revision$"
+__date__ = "$Date$"
+__copyright__ = "Copyright (c) 2013 Steven Hiscocks"
+__license__ = "GPL"
+
+import unittest, time, tempfile, os, threading
+from server.asyncserver import AsyncServer, AsyncServerException
+from client.csocket import CSocket
+
+class Socket(unittest.TestCase):
+
+	def setUp(self):
+		"""Call before every test case."""
+		self.server = AsyncServer(self)
+		sock_fd, sock_name = tempfile.mkstemp('fail2ban.sock', 'socket')
+		os.close(sock_fd)
+		os.remove(sock_name)
+		self.sock_name = sock_name
+
+	def tearDown(self):
+		"""Call after every test case."""
+
+	@staticmethod
+	def proceed(message):
+		"""Test transmitter proceed method which just returns first arg"""
+		return message
+
+	def testSocket(self):
+		serverThread = threading.Thread(
+			target=self.server.start, args=(self.sock_name, False))
+		serverThread.daemon = True
+		serverThread.start()
+		time.sleep(1)
+
+		client = CSocket(self.sock_name)
+		testMessage = ["A", "test", "message"]
+		self.assertEqual(client.send(testMessage), testMessage)
+
+		self.server.stop()
+		serverThread.join(1)
+		self.assertFalse(os.path.exists(self.sock_name))
+
+	def testSocketForce(self):
+		open(self.sock_name, 'w').close() # Create sock file
+		# Try to start without force
+		self.assertRaises(
+			AsyncServerException, self.server.start, self.sock_name, False)
+
+		# Try agin with force set
+		serverThread = threading.Thread(
+			target=self.server.start, args=(self.sock_name, True))
+		serverThread.daemon = True
+		serverThread.start()
+		time.sleep(1)
+
+		self.server.stop()
+		serverThread.join(1)
+		self.assertFalse(os.path.exists(self.sock_name))

testcases/utils.py

@@ -0,0 +1,101 @@
+# emacs: -*- mode: python; py-indent-offset: 4; indent-tabs-mode: t -*-
+# vi: set ft=python sts=4 ts=4 sw=4 noet :
+
+# This file is part of Fail2Ban.
+#
+# Fail2Ban is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# Fail2Ban is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with Fail2Ban; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
+
+
+__author__ = "Yaroslav Halchenko"
+__copyright__ = "Copyright (c) 2013 Yaroslav Halchenko"
+__license__ = "GPL"
+
+import logging, os, re, traceback
+from os.path import basename, dirname
+
+#
+# Following "traceback" functions are adopted from PyMVPA distributed
+# under MIT/Expat and copyright by PyMVPA developers (i.e. me and
+# Michael).  Hereby I re-license derivative work on these pieces under GPL
+# to stay in line with the main Fail2Ban license
+#
+def mbasename(s):
+	"""Custom function to include directory name if filename is too common
+
+	Also strip .py at the end
+	"""
+	base = basename(s)
+	if base.endswith('.py'):
+		base = base[:-3]
+	if base in set(['base', '__init__']):
+		base = basename(dirname(s)) + '.' + base
+	return base
+
+class TraceBack(object):
+	"""Customized traceback to be included in debug messages
+	"""
+
+	def __init__(self, compress=False):
+		"""Initialize TrackBack metric
+
+		Parameters
+		----------
+		compress : bool
+		  if True then prefix common with previous invocation gets
+		  replaced with ...
+		"""
+		self.__prev = ""
+		self.__compress = compress
+
+	def __call__(self):
+		ftb = traceback.extract_stack(limit=100)[:-2]
+		entries = [[mbasename(x[0]), str(x[1])] for x in ftb]
+		entries = [ e for e in entries
+					if not e[0] in ['unittest', 'logging.__init__' ]]
+
+		# lets make it more consize
+		entries_out = [entries[0]]
+		for entry in entries[1:]:
+			if entry[0] == entries_out[-1][0]:
+				entries_out[-1][1] += ',%s' % entry[1]
+			else:
+				entries_out.append(entry)
+		sftb = '>'.join(['%s:%s' % (mbasename(x[0]),
+									x[1]) for x in entries_out])
+		if self.__compress:
+			# lets remove part which is common with previous invocation
+			prev_next = sftb
+			common_prefix = os.path.commonprefix((self.__prev, sftb))
+			common_prefix2 = re.sub('>[^>]*$', '', common_prefix)
+
+			if common_prefix2 != "":
+				sftb = '...' + sftb[len(common_prefix2):]
+			self.__prev = prev_next
+
+		return sftb
+
+class FormatterWithTraceBack(logging.Formatter):
+	"""Custom formatter which expands %(tb) and %(tbc) with tracebacks
+
+	TODO: might need locking in case of compressed tracebacks
+	"""
+	def __init__(self, fmt, *args, **kwargs):
+		logging.Formatter.__init__(self, fmt=fmt, *args, **kwargs)
+		compress = '%(tbc)s' in fmt
+		self._tb = TraceBack(compress=compress)
+
+	def format(self, record):
+		record.tbc = record.tb = self._tb()
+		return logging.Formatter.format(self, record)