Merge branch 'master' into 0.9

Conflicts:
	fail2ban/client/jailreader.py
	fail2ban/tests/clientreadertestcase.py
	fail2ban/tests/files/logs/sshd

ChangeLog

@@ -45,7 +45,8 @@ ver. 0.8.11 (2013/XX/XXX) - wanna-be-released
 -----------
 
 - Fixes:
-  
+  Yaroslav Halchenko
+   * filter.d/common.conf -- make colon after [daemon] optional. Closes gh-267
 - New Features:
   Daniel Black & ykimon
    * filter.d/3proxy.conf -- filter added
@@ -53,12 +54,15 @@ ver. 0.8.11 (2013/XX/XXX) - wanna-be-released
   Daniel Black
    * filter.d/{asterisk,assp,dovecot,proftpd}.conf -- regex hardening
      and extra failure examples in sample logs
-  Daniel Black & Georgiy Mernov
+  Daniel Black & Georgiy Mernov & ftoppi
    * filter.d/exim.conf -- regex hardening and extra failure examples in
      sample logs
   Yaroslav Halchenko
    * fail2ban-regex -- refactored to provide more details (missing and
-     ignored lines, control over logging, etc) while maintaining look&feel.
+     ignored lines, control over logging, etc) while maintaining look&feel
+   * fail2ban-client -- log to standard error. Closes gh-264
+   * Fail to configure if not a single log file was found for an
+     enabled jail. Closes gh-63
 
 ver. 0.8.10 (2013/06/12) - wanna-be-secure
 -----------

THANKS

@@ -18,6 +18,7 @@ Daniel Black
 David Nutter
 Eric Gerbier
 Enrico Labedzki
+ftoppi
 Georgiy Mernov
 Guillaume Delvit
 Hanno 'Rince' Wagner

bin/fail2ban-client

@@ -221,7 +221,7 @@ class Fail2banClient:
 		elif len(cmd) == 2 and cmd[0] == "reload":
 			if self.__ping():
 				jail = cmd[1]
-				ret = self.__readJailConfig(jail)
+				ret = self.__readConfig(jail)
 				# Do not continue if configuration is not 100% valid
 				if not ret:
 					return False
@@ -328,13 +328,13 @@ class Fail2banClient:
 			logSys.setLevel(logging.INFO)
 		else:
 			logSys.setLevel(logging.DEBUG)
-		# Add the default logging handler
+		# Add the default logging handler to dump to stderr
-		stdout = logging.StreamHandler(sys.stdout)
+		logout = logging.StreamHandler(sys.stderr)
 		# set a format which is simpler for console use
 		formatter = logging.Formatter('%(levelname)-6s %(message)s')
 		# tell the handler to use this format
-		stdout.setFormatter(formatter)
+		logout.setFormatter(formatter)
-		logSys.addHandler(stdout)
+		logSys.addHandler(logout)
 
 		# Set the configuration path
 		self.__configurator.setBaseDir(self.__conf["conf"])
@@ -386,19 +386,18 @@ class Fail2banClient:
 				return False
 			return self.__processCommand(args)
 
-	def __readConfig(self):
+	def __readConfig(self, jail=None):
 		# Read the configuration
-		self.__configurator.readAll()
+		# TODO: get away from stew of return codes and exception
-		ret = self.__configurator.getOptions()
+		# handling -- handle via exceptions
-		self.__configurator.convertToProtocol()
+		try:
-		self.__stream = self.__configurator.getConfigStream()
+			self.__configurator.readAll()
-		return ret
+			ret = self.__configurator.getOptions(jail)
-
+			self.__configurator.convertToProtocol()
-	def __readJailConfig(self, jail):
+			self.__stream = self.__configurator.getConfigStream()
-		self.__configurator.readAll()
+		except Exception, e:
-		ret = self.__configurator.getOptions(jail)
+			logSys.error("Failed during configuration: %s" % e)
-		self.__configurator.convertToProtocol()
+			ret = False
-		self.__stream = self.__configurator.getConfigStream()
 		return ret
 
 	#@staticmethod

bin/fail2ban-regex

@@ -213,8 +213,10 @@ class Fail2banRegex(object):
 				reader.read(value)
 				print "Use %11s file : %s" % (regex, value)
 				# TODO: reuse functionality in client
-				regex_values = [RegexStat(m)
+				regex_values = [
-							    for m in reader.get("Definition", regex).split('\n')]
+					RegexStat(m)
+					for m in reader.get("Definition", regex).split('\n')
+					if m != ""]
 			except NoSectionError:
 				print "No [Definition] section in %s" % value
 				return False

config/filter.d/common.conf

@@ -33,7 +33,7 @@ __daemon_extra_re = (?:\[ID \d+ \S+\])
 
 # Combinations of daemon name and PID
 # EXAMPLES: sshd[31607], pop(pam_unix)[4920]
-__daemon_combs_re = (?:%(__pid_re)s?:\s+%(__daemon_re)s|%(__daemon_re)s%(__pid_re)s?:)
+__daemon_combs_re = (?:%(__pid_re)s?:\s+%(__daemon_re)s|%(__daemon_re)s%(__pid_re)s?:?)
 
 # Some messages have a kernel prefix with a timestamp
 # EXAMPLES: kernel: [769570.846956]

config/filter.d/exim.conf

@@ -1,7 +1,7 @@
 # Fail2Ban configuration file
 #
 # Author: Cyril Jaquier
-#
+#         Daniel Black (rewrote with strong regexs)
 #
 
 [Definition]
@@ -13,8 +13,18 @@
 #          (?:::f{4,6}:)?(?P<host>[\w\-.^_]+)
 # Values:  TEXT
 #
-failregex = ^ H=\S+ \(\S+\) \[<HOST>\] sender verify fail for <\S+>: (?:rejected by local_scan|Unrouteable address)\s*$
+
-             ^ login authenticator failed for (\S+ )?\(\S+\) \[<HOST>\]: 535 Incorrect authentication data( \(set_id=.*\)|: \d+ Time\(s\))?\s*$
+# From exim source code: ./src/receive.c:add_host_info_for_log
+host_info = H=([\w.-]+ )?(\(\S+\) )?\[<HOST>\](:\d+)? (?:I=\[\S+\]:\d+ )?(?:U=\S+ )?(P=e?smtp )?
+pid = ( \[\d+\])?
+
+failregex = ^%(pid)s %(host_info)ssender verify fail for <\S+>: Unrouteable address\s*$
+             ^%(pid)s \S+ F=(?:<>|\S+@\S+) %(host_info)s(?:temporarily )?rejected by local_scan\(\): .{0,256}$
+             ^%(pid)s login authenticator failed for (\S+ )?\(\S+\) \[<HOST>\]: 535 Incorrect authentication data( \(set_id=.*\)|: \d+ Time\(s\))?\s*$
+             ^%(pid)s %(host_info)sF=(?:<>|[^@]+@\S+) rejected RCPT [^@]+@\S+: (rejected found in dnsbl \S+|relay not permitted)\s*$
+             ^%(pid)s \S+ %(host_info)sF=(?:<>|[^@]+@\S+) rejected after DATA: This message contains a virus \(\S+\)\.\s*$
+             ^%(pid)s SMTP protocol synchronization error \(.*\): rejected (connection from|"\S+") %(host_info)s(next )?input=".*"\s*$
+             ^%(pid)s SMTP call from \S+ \[<HOST>\](:\d+)? (I=\[\S+\]:\d+ )?dropped: too many nonmail commands \(last was "\S+"\)\s*$
 
 # Option:  ignoreregex
 # Notes.:  regex to ignore. If this regex matches, the line is ignored.

fail2ban/client/jailreader.py

@@ -115,16 +115,30 @@ class JailReader(ConfigReader):
 				logSys.warning("No actions were defined for %s" % self.__name)
 		return True
 	
-	def convert(self):
+	def convert(self, allow_no_files=False):
+		"""Convert read before __opts to the commands stream
+
+		Parameters
+		----------
+		allow_missing : bool
+		  Either to allow log files to be missing entirely.  Primarily is
+		  used for testing
+		 """
+
 		stream = []
 		for opt in self.__opts:
 			if opt == "logpath":
+				found_files = 0
 				for path in self.__opts[opt].split("\n"):
 					pathList = glob.glob(path)
 					if len(pathList) == 0:
-						logSys.error("No file found for " + path)
+						logSys.error("No file(s) found for glob %s" % path)
 					for p in pathList:
+						found_files += 1
 						stream.append(["set", self.__name, "addlogpath", p])
+				if not (found_files or allow_no_files):
+					raise ValueError(
+						"Have not found any log file for %s jail" % self.__name)
 			elif opt == "logencoding":
 				stream.append(["set", self.__name, "logencoding", self.__opts[opt]])
 			elif opt == "backend":

fail2ban/client/jailsreader.py

@@ -79,14 +79,23 @@ class JailsReader(ConfigReader):
 					return False
 		return True
 	
-	def convert(self):
+	def convert(self, allow_no_files=False):
+		"""Convert read before __opts and jails to the commands stream
+
+		Parameters
+		----------
+		allow_missing : bool
+		  Either to allow log files to be missing entirely.  Primarily is
+		  used for testing
+		"""
+
 		stream = list()
 		for opt in self.__opts:
 			if opt == "":
 				stream.append([])
 		# Convert jails
 		for jail in self.__jails:
-			stream.extend(jail.convert())
+			stream.extend(jail.convert(allow_no_files=allow_no_files))
 		# Start jails
 		for jail in self.__jails:
 			stream.append(["start", jail.getName()])

fail2ban/tests/actiontestcase.py

@@ -70,11 +70,11 @@ class ExecuteAction(unittest.TestCase):
 		self.assertFalse(Action.substituteRecursiveTags({'A': '<B>', 'B': '<A>'}))
 		self.assertFalse(Action.substituteRecursiveTags({'A': '<B>', 'B': '<C>', 'C': '<A>'}))
 		# missing tags are ok
-		self.assertEquals(Action.substituteRecursiveTags({'A': '<C>'}), {'A': '<C>'})
+		self.assertEqual(Action.substituteRecursiveTags({'A': '<C>'}), {'A': '<C>'})
-		self.assertEquals(Action.substituteRecursiveTags({'A': '<C> <D> <X>','X':'fun'}), {'A': '<C> <D> fun', 'X':'fun'})
+		self.assertEqual(Action.substituteRecursiveTags({'A': '<C> <D> <X>','X':'fun'}), {'A': '<C> <D> fun', 'X':'fun'})
-		self.assertEquals(Action.substituteRecursiveTags({'A': '<C> <B>', 'B': 'cool'}), {'A': '<C> cool', 'B': 'cool'})
+		self.assertEqual(Action.substituteRecursiveTags({'A': '<C> <B>', 'B': 'cool'}), {'A': '<C> cool', 'B': 'cool'})
 		# rest is just cool
-		self.assertEquals(Action.substituteRecursiveTags(aInfo),
+		self.assertEqual(Action.substituteRecursiveTags(aInfo),
 								{ 'HOST': "192.0.2.0",
 									'ABC': '123 192.0.2.0',
 									'xyz': '890 123 192.0.2.0',

fail2ban/tests/clientreadertestcase.py

@@ -220,7 +220,7 @@ class FilterReaderTest(unittest.TestCase):
 		#filterReader.getOptions(["failregex", "ignoreregex"])
 		filterReader.getOptions(None)
 		output[-1][-1] = "5"
-		self.assertEquals(sorted(filterReader.convert()), sorted(output))
+		self.assertEqual(sorted(filterReader.convert()), sorted(output))
 
 class JailsReaderTest(unittest.TestCase):
 
@@ -297,7 +297,7 @@ class JailsReaderTest(unittest.TestCase):
 		jails = JailsReader(basedir=CONFIG_DIR, force_enable=True) # we are running tests from root project dir atm
 		self.assertTrue(jails.read())		  # opens fine
 		self.assertTrue(jails.getOptions())	  # reads fine
-		comm_commands = jails.convert()
+		comm_commands = jails.convert(allow_no_files=True)
 
 		# by default we have lots of jails ;)
 		self.assertTrue(len(comm_commands))

fail2ban/tests/files/logs/exim

@@ -4,3 +4,17 @@
 2013-06-12 03:57:58 login authenticator failed for (ylmf-pc) [120.196.140.45]: 535 Incorrect authentication data: 1 Time(s)
 2013-06-12 13:18:11 login authenticator failed for (USER-KVI9FGS9KP) [101.66.165.86]: 535 Incorrect authentication data
 2013-06-10 10:10:59 H=ufficioestampa.it (srv.ufficioestampa.it) [193.169.56.211] sender verify fail for <user@example.com>: Unrouteable address
+# http://forum.lissyara.su/viewtopic.php?f=20&t=2985
+2010-11-24 21:48:41 1PLKOW-00046U-EW F=wvhluo@droolindog.com H=93-143-146-237.adsl.net.t-com.hr (droolindog.com) [93.143.146.237] I=[10.10.10.32]:25 P=esmtp temporarily rejected by local_scan(): Temporary local problem
+# http://us.generation-nt.com/answer/exim-spamassassin-2010-0-x64-help-204020461.html
+2011-07-07 15:44:16 1QexIu-0006dj-PX F=XXXXXX@XXXXXXXXXXXX H=localhost (saf.bio.caltech.edu) [127.0.0.1] P=esmtp temporarily rejected by local_scan(): Local configuration error - local_scan() library failure/usr/lib/exim/sa-exim.so: cannot open shared object file: No such file or directory
+# http://www.clues.ltd.uk/howto/debian-sa-fprot-HOWTO.html
+2004-01-18 07:15:35 1Ai79e-0000Dq-8i F=uzwltcmwto24@melissacam.biz H=lsanca1-ar3-4-47-028-040.lsanca1.elnk.dsl.genuity.net [4.47.28.40] P=smtp rejected by local_scan(): Rejected: hits=7.5 required=5.0 trigger=5.0
+# https://github.com/fail2ban/fail2ban/pull/251#issuecomment-19493875
+2013-06-15 11:19:33 [2249] H=([2.181.148.95]) [2.181.148.95]:52391 I=[1.2.3.4]:25 F=fantasizesg4@google.com rejected RCPT some@email.com: rejected found in dnsbl zen.spamhaus.org
+2013-06-10 18:33:32 [10099] H=(yakult.com.tw) [202.132.70.178]:3755 I=[1.2.3.4]:25 F=menacedsj04@listserv.eurasia.org rejected RCPT dir@ml3.ru: relay not permitted
+2013-06-09 10:21:28 [14127] 1UlasQ-0003fr-45 F=mcorporation4@aol.com H=(mail38.fssprus.ru) [46.254.240.82]:43671 I=[1.2.3.4]:25 P=esmtp rejected by local_scan(): Rejected
+2013-06-15 11:20:36 [2516] 1Unmew-0000ea-SE H=egeftech.static.otenet.gr [83.235.177.148]:32706 I=[1.2.3.4]:25 F=auguriesvbd40@google.com rejected after DATA: This message contains a virus (Sanesecurity.Junk.39934.UNOFFICIAL).
+2013-06-02 06:54:20 [13314] SMTP protocol synchronization error (input sent without waiting for greeting): rejected connection from H=[211.148.195.192]:25936 I=[1.2.3.4]:25 input="GET / HTTP/1.1\r\n\r\n"
+2013-06-02 09:05:48 [18505] SMTP protocol synchronization error (next input sent too soon: pipelining was not advertised): rejected "RSET" H=ba77.mx83.fr [82.96.160.77]:58302 I=[1.2.3.4]:25 next input="QUIT\r\n"
+2013-06-02 09:22:05 [19591] SMTP call from pc012-6201.spo.scu.edu.tw [163.14.21.161]:3767 I=[1.2.3.4]:25 dropped: too many nonmail commands (last was "RSET")

fail2ban/tests/files/logs/sshd

@@ -47,7 +47,10 @@ Apr 29 16:53:38 Jamess-iMac.local sshd[47831]: error: PAM: authentication error
 Apr 29 17:53:38 Jamess-iMac.local sshd[47831]: error: PAM: authentication error for james from 205.186.180.102
 Apr 29 18:53:38 Jamess-iMac.local sshd[47831]: error: PAM: authentication error for james from 205.186.180.103
 
-#11
+#11 https://github.com/fail2ban/fail2ban/issues/267  There might be no colon after [daemon]
+Jun 25 23:53:34 [sshd] User root from 1.2.3.4 not allowed because not listed in AllowUsers
+
+#12
 Apr 24 01:39:19 host sshd[3719]: User root not allowed because account is locked
 Apr 24 01:39:19 host sshd[3719]: input_userauth_request: invalid user root [preauth]
 Apr 24 01:39:19 host sshd[3719]: error: Received disconnect from 198.51.100.34: 11: Bye Bye [preauth]