From 4ce240ed4055b37b870027a41ba8e432a69dc531 Mon Sep 17 00:00:00 2001 From: sebres Date: Tue, 9 Feb 2016 14:23:40 +0100 Subject: [PATCH 01/13] try to start server in foreground # Conflicts: # fail2ban/server/server.py --- bin/fail2ban-client | 117 ++++++++++++++++++++++-------- bin/fail2ban-server | 3 +- fail2ban/client/fail2banreader.py | 9 ++- fail2ban/server/server.py | 18 ++--- 4 files changed, 106 insertions(+), 41 deletions(-) diff --git a/bin/fail2ban-client b/bin/fail2ban-client index bc0c0be8..631afb31 100755 --- a/bin/fail2ban-client +++ b/bin/fail2ban-client @@ -54,6 +54,7 @@ class Fail2banClient: PROMPT = "fail2ban> " def __init__(self): + self.__server = None self.__argv = None self.__stream = None self.__configurator = Configurator() @@ -89,13 +90,16 @@ class Fail2banClient: print " -c configuration directory" print " -s socket path" print " -p pidfile path" + print " --loglevel logging level" + print " --logtarget |STDOUT|STDERR|SYSLOG" + print " --syslogsocket auto|file" print " -d dump configuration. For debugging" print " -i interactive mode" print " -v increase verbosity" print " -q decrease verbosity" print " -x force execution of the server (remove socket file)" print " -b start server in background (default)" - print " -f start server in foreground (note that the client forks once itself)" + print " -f start server in foreground" print " -h, --help display this help message" print " -V, --version print the version" print @@ -128,6 +132,8 @@ class Fail2banClient: self.__conf["socket"] = opt[1] elif opt[0] == "-p": self.__conf["pidfile"] = opt[1] + elif opt[0].startswith("--log") or opt[0].startswith("--sys"): + self.__conf[ opt[0][2:] ] = opt[1] elif opt[0] == "-d": self.__conf["dump"] = True elif opt[0] == "-v": @@ -234,24 +240,32 @@ class Fail2banClient: "Directory %s exists but not accessible for writing" % (socket_dir,)) return False - # Start the server - self.__startServerAsync(self.__conf["socket"], - self.__conf["pidfile"], - self.__conf["force"], - self.__conf["background"]) - try: - # Wait for the server to start - self.__waitOnServer() - # Configure the server - self.__processCmd(self.__stream, False) - return True - except ServerExecutionException: - logSys.error("Could not start server. Maybe an old " - "socket file is still present. Try to " - "remove " + self.__conf["socket"] + ". If " - "you used fail2ban-client to start the " - "server, adding the -x option will do it") + + # Check already running + if not self.__conf["force"] and os.path.exists(self.__conf["socket"]): + logSys.error("Fail2ban seems to be in unexpected state (not running but socket exists)") return False + + # Start the server + t = None + if self.__conf["background"]: + # Start server daemon as fork of client process: + self.__startServerAsync() + # Send config stream to server: + return self.__processStartStreamAfterWait() + else: + # In foreground mode we should start server/client communication in other thread: + from threading import Thread + t = Thread(target=Fail2banClient.__processStartStreamAfterWait, args=(self,)) + t.start() + # Start server direct here in main thread: + try: + self.__startServerDirect() + except KeyboardInterrupt: + None + + return True + elif len(cmd) == 1 and cmd[0] == "reload": if self.__ping(): ret = self.__readConfig() @@ -281,12 +295,50 @@ class Fail2banClient: return self.__processCmd([cmd]) + def __processStartStreamAfterWait(self): + try: + # Wait for the server to start + self.__waitOnServer() + # Configure the server + self.__processCmd(self.__stream, False) + except ServerExecutionException: + logSys.error("Could not start server. Maybe an old " + "socket file is still present. Try to " + "remove " + self.__conf["socket"] + ". If " + "you used fail2ban-client to start the " + "server, adding the -x option will do it") + if not self.__conf["background"]: + self.__server.quit() + sys.exit(-1) + return False + return True + + + ## + # Start Fail2Ban server in main thread without fork (foreground). + # + # Start the Fail2ban server in foreground (daemon mode or not). + + def __startServerDirect(self): + from fail2ban.server.server import Server + try: + self.__server = Server(False) + self.__server.start(self.__conf["socket"], + self.__conf["pidfile"], self.__conf["force"], + conf=self.__conf) + except Exception, e: + logSys.exception(e) + if self.__server: + self.__server.quit() + sys.exit(-1) + + ## # Start Fail2Ban server. # # Start the Fail2ban server in daemon mode. - def __startServerAsync(self, socket, pidfile, force = False, background = True): + def __startServerAsync(self): # Forks the current process. pid = os.fork() if pid == 0: @@ -294,18 +346,15 @@ class Fail2banClient: args.append(self.SERVER) # Set the socket path. args.append("-s") - args.append(socket) + args.append(self.__conf["socket"]) # Set the pidfile args.append("-p") - args.append(pidfile) + args.append(self.__conf["pidfile"]) # Force the execution if needed. - if force: + if self.__conf["force"]: args.append("-x") - # Start in foreground mode if requested. - if background: - args.append("-b") - else: - args.append("-f") + # Start in background as requested. + args.append("-b") try: # Use the current directory. @@ -361,7 +410,7 @@ class Fail2banClient: # Reads the command line options. try: cmdOpts = 'hc:s:p:xfbdviqV' - cmdLongOpts = ['help', 'version'] + cmdLongOpts = ['loglevel', 'logtarget', 'syslogsocket', 'help', 'version'] optList, args = getopt.getopt(self.__argv[1:], cmdOpts, cmdLongOpts) except getopt.GetoptError: self.dispUsage() @@ -398,7 +447,17 @@ class Fail2banClient: self.__conf["socket"] = conf["socket"] if self.__conf["pidfile"] is None: self.__conf["pidfile"] = conf["pidfile"] - logSys.info("Using socket file " + self.__conf["socket"]) + if self.__conf.get("logtarget", None) is None: + self.__conf["logtarget"] = conf["logtarget"] + if self.__conf.get("loglevel", None) is None: + self.__conf["loglevel"] = conf["loglevel"] + if self.__conf.get("syslogsocket", None) is None: + self.__conf["syslogsocket"] = conf["syslogsocket"] + + logSys.info("Using socket file %s", self.__conf["socket"]) + + logSys.info("Using pid file %s, [%s] logging to %s", + self.__conf["pidfile"], self.__conf["loglevel"], self.__conf["logtarget"]) if self.__conf["dump"]: ret = self.__readConfig() diff --git a/bin/fail2ban-server b/bin/fail2ban-server index f522f418..0b8b6418 100755 --- a/bin/fail2ban-server +++ b/bin/fail2ban-server @@ -129,7 +129,8 @@ class Fail2banServer: return True except Exception, e: logSys.exception(e) - self.__server.quit() + if self.__server: + self.__server.quit() return False if __name__ == "__main__": diff --git a/fail2ban/client/fail2banreader.py b/fail2ban/client/fail2banreader.py index c55f65ea..b3012c9b 100644 --- a/fail2ban/client/fail2banreader.py +++ b/fail2ban/client/fail2banreader.py @@ -40,8 +40,13 @@ class Fail2banReader(ConfigReader): ConfigReader.read(self, "fail2ban") def getEarlyOptions(self): - opts = [["string", "socket", "/var/run/fail2ban/fail2ban.sock"], - ["string", "pidfile", "/var/run/fail2ban/fail2ban.pid"]] + opts = [ + ["string", "socket", "/var/run/fail2ban/fail2ban.sock"], + ["string", "pidfile", "/var/run/fail2ban/fail2ban.pid"], + ["string", "loglevel", "INFO"], + ["string", "logtarget", "/var/log/fail2ban.log"], + ["string", "syslogsocket", "auto"] + ] return ConfigReader.getOptions(self, "Definition", opts) def getOptions(self): diff --git a/fail2ban/server/server.py b/fail2ban/server/server.py index 7f75c347..1d3e89d3 100644 --- a/fail2ban/server/server.py +++ b/fail2ban/server/server.py @@ -67,10 +67,6 @@ class Server: 'FreeBSD': '/var/run/log', 'Linux': '/dev/log', } - self.setSyslogSocket("auto") - # Set logging level - self.setLogLevel("INFO") - self.setLogTarget("STDOUT") def __sigTERMhandler(self, signum, frame): logSys.debug("Caught signal %d. Exiting" % signum) @@ -80,7 +76,12 @@ class Server: logSys.debug("Caught signal %d. Flushing logs" % signum) self.flushLogs() - def start(self, sock, pidfile, force = False): + def start(self, sock, pidfile, force=False, conf={}): + # First set all logging parameters: + self.setSyslogSocket(conf.get("syslogsocket", "auto")) + self.setLogLevel(conf.get("loglevel", "INFO")) + self.setLogTarget(conf.get("logtarget", "STDOUT")) + logSys.info("Starting Fail2ban v%s", version.version) # Install signal handlers @@ -402,8 +403,9 @@ class Server: # @param target the logging target def setLogTarget(self, target): - try: - self.__loggingLock.acquire() + with self.__loggingLock: + if self.__logTarget == target: + return True # set a format which is simpler for console use formatter = logging.Formatter("%(asctime)s %(name)-24s[%(process)d]: %(levelname)-7s %(message)s") if target == "SYSLOG": @@ -471,8 +473,6 @@ class Server: # Sets the logging target. self.__logTarget = target return True - finally: - self.__loggingLock.release() ## # Sets the syslog socket. From 556ddaabd7a0b1f1ac94d014680fa83d980f770f Mon Sep 17 00:00:00 2001 From: sebres Date: Wed, 10 Feb 2016 21:00:00 +0100 Subject: [PATCH 02/13] temporary commit (move client/server from bin) --- bin/fail2ban-client => fail2ban/client/fail2banclient.py | 0 bin/fail2ban-server => fail2ban/client/fail2banserver.py | 0 2 files changed, 0 insertions(+), 0 deletions(-) rename bin/fail2ban-client => fail2ban/client/fail2banclient.py (100%) rename bin/fail2ban-server => fail2ban/client/fail2banserver.py (100%) diff --git a/bin/fail2ban-client b/fail2ban/client/fail2banclient.py similarity index 100% rename from bin/fail2ban-client rename to fail2ban/client/fail2banclient.py diff --git a/bin/fail2ban-server b/fail2ban/client/fail2banserver.py similarity index 100% rename from bin/fail2ban-server rename to fail2ban/client/fail2banserver.py From 5a053f4b743ca482bc720c5e851c5a793cad0089 Mon Sep 17 00:00:00 2001 From: sebres Date: Thu, 11 Feb 2016 08:56:12 +0100 Subject: [PATCH 03/13] starting of the server (and client/server communication behavior during start and daemonize) completely rewritten: - client/server functionality moved away from bin and using now the common interface (introduced in fail2bancmdline); - start in foreground fixed; - server can act as client corresponding command line; - command "restart" added: in opposite to "reload" in reality restarts the server (new process); - several client/server bugs during starting process fixed. --- MANIFEST | 3 + bin/fail2ban-client | 37 ++ bin/fail2ban-server | 37 ++ fail2ban/client/beautifier.py | 2 + fail2ban/client/fail2banclient.py | 580 +++++++++++------------------ fail2ban/client/fail2bancmdline.py | 247 ++++++++++++ fail2ban/client/fail2banserver.py | 234 +++++++----- fail2ban/protocol.py | 6 +- fail2ban/server/server.py | 95 ++--- fail2ban/server/transmitter.py | 2 + 10 files changed, 745 insertions(+), 498 deletions(-) create mode 100755 bin/fail2ban-client create mode 100755 bin/fail2ban-server create mode 100644 fail2ban/client/fail2bancmdline.py mode change 100755 => 100644 fail2ban/client/fail2banserver.py diff --git a/MANIFEST b/MANIFEST index 11f635c3..713b64c3 100644 --- a/MANIFEST +++ b/MANIFEST @@ -156,8 +156,11 @@ fail2ban/client/configparserinc.py fail2ban/client/configreader.py fail2ban/client/configurator.py fail2ban/client/csocket.py +fail2ban/client/fail2banclient.py +fail2ban/client/fail2bancmdline.py fail2ban/client/fail2banreader.py fail2ban/client/fail2banregex.py +fail2ban/client/fail2banserver.py fail2ban/client/filterreader.py fail2ban/client/__init__.py fail2ban/client/jailreader.py diff --git a/bin/fail2ban-client b/bin/fail2ban-client new file mode 100755 index 00000000..19e76a98 --- /dev/null +++ b/bin/fail2ban-client @@ -0,0 +1,37 @@ +#!/usr/bin/python +# emacs: -*- mode: python; py-indent-offset: 4; indent-tabs-mode: t -*- +# vi: set ft=python sts=4 ts=4 sw=4 noet : + +# This file is part of Fail2Ban. +# +# Fail2Ban is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. +# +# Fail2Ban is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with Fail2Ban; if not, write to the Free Software +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + +""" +Fail2Ban reads log file that contains password failure report +and bans the corresponding IP addresses using firewall rules. + +This tools starts/stops fail2ban server or does client/server communication, +to change/read parameters of the server or jails. + +""" + +__author__ = "Fail2Ban Developers" +__copyright__ = "Copyright (c) 2004-2008 Cyril Jaquier, 2012-2014 Yaroslav Halchenko, 2014-2016 Serg G. Brester" +__license__ = "GPL" + +from fail2ban.client.fail2banclient import exec_command_line + +if __name__ == "__main__": + exec_command_line() diff --git a/bin/fail2ban-server b/bin/fail2ban-server new file mode 100755 index 00000000..8e64d865 --- /dev/null +++ b/bin/fail2ban-server @@ -0,0 +1,37 @@ +#!/usr/bin/python +# emacs: -*- mode: python; py-indent-offset: 4; indent-tabs-mode: t -*- +# vi: set ft=python sts=4 ts=4 sw=4 noet : + +# This file is part of Fail2Ban. +# +# Fail2Ban is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. +# +# Fail2Ban is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with Fail2Ban; if not, write to the Free Software +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + +""" +Fail2Ban reads log file that contains password failure report +and bans the corresponding IP addresses using firewall rules. + +This tools starts/stops fail2ban server or does client/server communication, +to change/read parameters of the server or jails. + +""" + +__author__ = "Fail2Ban Developers" +__copyright__ = "Copyright (c) 2004-2008 Cyril Jaquier, 2012-2014 Yaroslav Halchenko, 2014-2016 Serg G. Brester" +__license__ = "GPL" + +from fail2ban.client.fail2banserver import exec_command_line + +if __name__ == "__main__": + exec_command_line() diff --git a/fail2ban/client/beautifier.py b/fail2ban/client/beautifier.py index 7b27e92f..bd803a6a 100644 --- a/fail2ban/client/beautifier.py +++ b/fail2ban/client/beautifier.py @@ -68,6 +68,8 @@ class Beautifier: msg = "Added jail " + response elif inC[0] == "flushlogs": msg = "logs: " + response + elif inC[0] == "echo": + msg = ' '.join(msg) elif inC[0:1] == ['status']: if len(inC) > 1: # Display information diff --git a/fail2ban/client/fail2banclient.py b/fail2ban/client/fail2banclient.py index 631afb31..7adbab95 100755 --- a/fail2ban/client/fail2banclient.py +++ b/fail2ban/client/fail2banclient.py @@ -1,7 +1,7 @@ #!/usr/bin/python # emacs: -*- mode: python; py-indent-offset: 4; indent-tabs-mode: t -*- # vi: set ft=python sts=4 ts=4 sw=4 noet : - +# # This file is part of Fail2Ban. # # Fail2Ban is free software; you can redistribute it and/or modify @@ -17,99 +17,38 @@ # You should have received a copy of the GNU General Public License # along with Fail2Ban; if not, write to the Free Software # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. - -__author__ = "Cyril Jaquier" -__copyright__ = "Copyright (c) 2004 Cyril Jaquier" +__author__ = "Fail2Ban Developers" +__copyright__ = "Copyright (c) 2004-2008 Cyril Jaquier, 2012-2014 Yaroslav Halchenko, 2014-2016 Serg G. Brester" __license__ = "GPL" -import getopt -import logging import os -import pickle -import re import shlex import signal import socket -import string import sys import time -from fail2ban.version import version -from fail2ban.protocol import printFormatted -from fail2ban.client.csocket import CSocket -from fail2ban.client.configurator import Configurator -from fail2ban.client.beautifier import Beautifier -from fail2ban.helpers import getLogger +from threading import Thread -# Gets the instance of the logger. -logSys = getLogger("fail2ban") +from ..version import version +from .csocket import CSocket +from .beautifier import Beautifier +from .fail2bancmdline import Fail2banCmdLine, logSys, exit ## # # @todo This class needs cleanup. -class Fail2banClient: +class Fail2banClient(Fail2banCmdLine, Thread): - SERVER = "fail2ban-server" PROMPT = "fail2ban> " def __init__(self): - self.__server = None - self.__argv = None - self.__stream = None - self.__configurator = Configurator() - self.__conf = dict() - self.__conf["conf"] = "/etc/fail2ban" - self.__conf["dump"] = False - self.__conf["force"] = False - self.__conf["background"] = True - self.__conf["verbose"] = 1 - self.__conf["interactive"] = False - self.__conf["socket"] = None - self.__conf["pidfile"] = None - - def dispVersion(self): - print "Fail2Ban v" + version - print - print "Copyright (c) 2004-2008 Cyril Jaquier, 2008- Fail2Ban Contributors" - print "Copyright of modifications held by their respective authors." - print "Licensed under the GNU General Public License v2 (GPL)." - print - print "Written by Cyril Jaquier ." - print "Many contributions by Yaroslav O. Halchenko ." - - def dispUsage(self): - """ Prints Fail2Ban command line options and exits - """ - print "Usage: "+self.__argv[0]+" [OPTIONS] " - print - print "Fail2Ban v" + version + " reads log file that contains password failure report" - print "and bans the corresponding IP addresses using firewall rules." - print - print "Options:" - print " -c configuration directory" - print " -s socket path" - print " -p pidfile path" - print " --loglevel logging level" - print " --logtarget |STDOUT|STDERR|SYSLOG" - print " --syslogsocket auto|file" - print " -d dump configuration. For debugging" - print " -i interactive mode" - print " -v increase verbosity" - print " -q decrease verbosity" - print " -x force execution of the server (remove socket file)" - print " -b start server in background (default)" - print " -f start server in foreground" - print " -h, --help display this help message" - print " -V, --version print the version" - print - print "Command:" - - # Prints the protocol - printFormatted() - - print - print "Report bugs to https://github.com/fail2ban/fail2ban/issues" + Fail2banCmdLine.__init__(self) + Thread.__init__(self) + self._alive = True + self._server = None + self._beautifier = None def dispInteractive(self): print "Fail2Ban v" + version + " reads log file that contains password failure report" @@ -120,58 +59,32 @@ class Fail2banClient: # Print a new line because we probably come from wait print logSys.warning("Caught signal %d. Exiting" % signum) - sys.exit(-1) - - def __getCmdLineOptions(self, optList): - """ Gets the command line options - """ - for opt in optList: - if opt[0] == "-c": - self.__conf["conf"] = opt[1] - elif opt[0] == "-s": - self.__conf["socket"] = opt[1] - elif opt[0] == "-p": - self.__conf["pidfile"] = opt[1] - elif opt[0].startswith("--log") or opt[0].startswith("--sys"): - self.__conf[ opt[0][2:] ] = opt[1] - elif opt[0] == "-d": - self.__conf["dump"] = True - elif opt[0] == "-v": - self.__conf["verbose"] = self.__conf["verbose"] + 1 - elif opt[0] == "-q": - self.__conf["verbose"] = self.__conf["verbose"] - 1 - elif opt[0] == "-x": - self.__conf["force"] = True - elif opt[0] == "-i": - self.__conf["interactive"] = True - elif opt[0] == "-b": - self.__conf["background"] = True - elif opt[0] == "-f": - self.__conf["background"] = False - elif opt[0] in ["-h", "--help"]: - self.dispUsage() - sys.exit(0) - elif opt[0] in ["-V", "--version"]: - self.dispVersion() - sys.exit(0) + exit(-1) def __ping(self): return self.__processCmd([["ping"]], False) - def __processCmd(self, cmd, showRet = True): + @property + def beautifier(self): + if self._beautifier: + return self._beautifier + self._beautifier = Beautifier() + return self._beautifier + + def __processCmd(self, cmd, showRet=True): client = None try: - beautifier = Beautifier() + beautifier = self.beautifier streamRet = True for c in cmd: beautifier.setInputCmd(c) try: if not client: - client = CSocket(self.__conf["socket"]) + client = CSocket(self._conf["socket"]) ret = client.send(c) if ret[0] == 0: logSys.debug("OK : " + `ret[1]`) - if showRet: + if showRet or c[0] == 'echo': print beautifier.beautify(ret[1]) else: logSys.error("NOK: " + `ret[1].args`) @@ -179,38 +92,126 @@ class Fail2banClient: print beautifier.beautifyError(ret[1]) streamRet = False except socket.error: - if showRet: + if showRet or self._conf["verbose"] > 1: self.__logSocketError() return False except Exception, e: - if showRet: + if showRet or self._conf["verbose"] > 1: logSys.error(e) return False finally: if client: client.close() + if showRet or c[0] == 'echo': + sys.stdout.flush() return streamRet def __logSocketError(self): try: - if os.access(self.__conf["socket"], os.F_OK): + if os.access(self._conf["socket"], os.F_OK): # This doesn't check if path is a socket, # but socket.error should be raised - if os.access(self.__conf["socket"], os.W_OK): + if os.access(self._conf["socket"], os.W_OK): # Permissions look good, but socket.error was raised logSys.error("Unable to contact server. Is it running?") else: logSys.error("Permission denied to socket: %s," - " (you must be root)", self.__conf["socket"]) + " (you must be root)", self._conf["socket"]) else: logSys.error("Failed to access socket path: %s." " Is fail2ban running?", - self.__conf["socket"]) + self._conf["socket"]) except Exception as e: logSys.error("Exception while checking socket access: %s", - self.__conf["socket"]) + self._conf["socket"]) logSys.error(e) + ## + def __prepareStartServer(self): + if self.__ping(): + logSys.error("Server already running") + return None + + # Read the config + ret, stream = self.readConfig() + # Do not continue if configuration is not 100% valid + if not ret: + return None + + # verify that directory for the socket file exists + socket_dir = os.path.dirname(self._conf["socket"]) + if not os.path.exists(socket_dir): + logSys.error( + "There is no directory %s to contain the socket file %s." + % (socket_dir, self._conf["socket"])) + return None + if not os.access(socket_dir, os.W_OK | os.X_OK): + logSys.error( + "Directory %s exists but not accessible for writing" + % (socket_dir,)) + return None + + # Check already running + if not self._conf["force"] and os.path.exists(self._conf["socket"]): + logSys.error("Fail2ban seems to be in unexpected state (not running but the socket exists)") + return None + + stream.append(['echo', 'Server ready']) + return stream + + ## + def __startServer(self, background=True): + from .fail2banserver import Fail2banServer + stream = self.__prepareStartServer() + self._alive = True + if not stream: + return False + # Start the server or just initialize started one: + try: + if background: + # Start server daemon as fork of client process: + Fail2banServer.startServerAsync(self._conf) + # Send config stream to server: + if not self.__processStartStreamAfterWait(stream, False): + return False + else: + # In foreground mode we should make server/client communication in different threads: + Thread(target=Fail2banClient.__processStartStreamAfterWait, args=(self, stream, False)).start() + # Mark current (main) thread as daemon: + self.setDaemon(True) + # Start server direct here in main thread (not fork): + self._server = Fail2banServer.startServerDirect(self._conf, False) + + except Exception as e: + print + logSys.error("Exception while starting server foreground") + logSys.error(e) + finally: + self._alive = False + + return True + + ## + def configureServer(self, async=True, phase=None): + # if asynchron start this operation in the new thread: + if async: + return Thread(target=Fail2banClient.configureServer, args=(self, False, phase)).start() + # prepare: read config, check configuration is valid, etc.: + if phase is not None: + phase['start'] = True + logSys.debug('-- client phase %s', phase) + stream = self.__prepareStartServer() + if phase is not None: + phase['ready'] = phase['start'] = (True if stream else False) + logSys.debug('-- client phase %s', phase) + if not stream: + return False + # configure server with config stream: + ret = self.__processStartStreamAfterWait(stream, False) + if phase is not None: + phase['done'] = ret + return ret + ## # Process a command line. # @@ -219,253 +220,101 @@ class Fail2banClient: def __processCommand(self, cmd): if len(cmd) == 1 and cmd[0] == "start": - if self.__ping(): - logSys.error("Server already running") + + ret = self.__startServer(self._conf["background"]) + if not ret: return False - else: - # Read the config - ret = self.__readConfig() - # Do not continue if configuration is not 100% valid - if not ret: - return False - # verify that directory for the socket file exists - socket_dir = os.path.dirname(self.__conf["socket"]) - if not os.path.exists(socket_dir): - logSys.error( - "There is no directory %s to contain the socket file %s." - % (socket_dir, self.__conf["socket"])) - return False - if not os.access(socket_dir, os.W_OK | os.X_OK): - logSys.error( - "Directory %s exists but not accessible for writing" - % (socket_dir,)) - return False + return ret - # Check already running - if not self.__conf["force"] and os.path.exists(self.__conf["socket"]): - logSys.error("Fail2ban seems to be in unexpected state (not running but socket exists)") - return False + elif len(cmd) == 1 and cmd[0] == "restart": - # Start the server - t = None - if self.__conf["background"]: - # Start server daemon as fork of client process: - self.__startServerAsync() - # Send config stream to server: - return self.__processStartStreamAfterWait() + if self._conf.get("interactive", False): + print(' ## stop ... ') + self.__processCommand(['stop']) + self.__waitOnServer(False) + # in interactive mode reset config, to make full-reload if there something changed: + if self._conf.get("interactive", False): + print(' ## load configuration ... ') + self.resetConf() + ret = self.initCmdLine(self._argv) + if ret is not None: + return ret + if self._conf.get("interactive", False): + print(' ## start ... ') + return self.__processCommand(['start']) + + elif len(cmd) >= 1 and cmd[0] == "reload": + if self.__ping(): + if len(cmd) == 1: + jail = 'all' + ret, stream = self.readConfig() else: - # In foreground mode we should start server/client communication in other thread: - from threading import Thread - t = Thread(target=Fail2banClient.__processStartStreamAfterWait, args=(self,)) - t.start() - # Start server direct here in main thread: - try: - self.__startServerDirect() - except KeyboardInterrupt: - None - - return True - - elif len(cmd) == 1 and cmd[0] == "reload": - if self.__ping(): - ret = self.__readConfig() - # Do not continue if configuration is not 100% valid - if not ret: - return False - self.__processCmd([['stop', 'all']], False) - # Configure the server - return self.__processCmd(self.__stream, False) - else: - logSys.error("Could not find server") - return False - elif len(cmd) == 2 and cmd[0] == "reload": - if self.__ping(): - jail = cmd[1] - ret = self.__readConfig(jail) + jail = cmd[1] + ret, stream = self.readConfig(jail) # Do not continue if configuration is not 100% valid if not ret: return False self.__processCmd([['stop', jail]], False) # Configure the server - return self.__processCmd(self.__stream, False) + return self.__processCmd(stream, True) else: logSys.error("Could not find server") return False + else: return self.__processCmd([cmd]) - def __processStartStreamAfterWait(self): + def __processStartStreamAfterWait(self, *args): try: # Wait for the server to start self.__waitOnServer() - # Configure the server - self.__processCmd(self.__stream, False) + # Configure the server + self.__processCmd(*args) except ServerExecutionException: logSys.error("Could not start server. Maybe an old " "socket file is still present. Try to " - "remove " + self.__conf["socket"] + ". If " + "remove " + self._conf["socket"] + ". If " "you used fail2ban-client to start the " "server, adding the -x option will do it") - if not self.__conf["background"]: - self.__server.quit() - sys.exit(-1) + if self._server: + self._server.quit() + exit(-1) return False return True - - ## - # Start Fail2Ban server in main thread without fork (foreground). - # - # Start the Fail2ban server in foreground (daemon mode or not). - - def __startServerDirect(self): - from fail2ban.server.server import Server - try: - self.__server = Server(False) - self.__server.start(self.__conf["socket"], - self.__conf["pidfile"], self.__conf["force"], - conf=self.__conf) - except Exception, e: - logSys.exception(e) - if self.__server: - self.__server.quit() - sys.exit(-1) - - - ## - # Start Fail2Ban server. - # - # Start the Fail2ban server in daemon mode. - - def __startServerAsync(self): - # Forks the current process. - pid = os.fork() - if pid == 0: - args = list() - args.append(self.SERVER) - # Set the socket path. - args.append("-s") - args.append(self.__conf["socket"]) - # Set the pidfile - args.append("-p") - args.append(self.__conf["pidfile"]) - # Force the execution if needed. - if self.__conf["force"]: - args.append("-x") - # Start in background as requested. - args.append("-b") - - try: - # Use the current directory. - exe = os.path.abspath(os.path.join(sys.path[0], self.SERVER)) - logSys.debug("Starting %r with args %r" % (exe, args)) - os.execv(exe, args) - except OSError: - try: - # Use the PATH env. - logSys.warning("Initial start attempt failed. Starting %r with the same args" % (self.SERVER,)) - os.execvp(self.SERVER, args) - except OSError: - logSys.error("Could not start %s" % self.SERVER) - os.exit(-1) - - def __waitOnServer(self): - # Wait for the server to start - cnt = 0 - if self.__conf["verbose"] > 1: - pos = 0 - delta = 1 - mask = "[ ]" - while not self.__ping(): - # Wonderful visual :) - if self.__conf["verbose"] > 1: - pos += delta - sys.stdout.write("\rINFO " + mask[:pos] + '#' + mask[pos+1:] + - " Waiting on the server...") - sys.stdout.flush() - if pos > len(mask)-3: - delta = -1 - elif pos < 2: - delta = 1 - # The server has 30 seconds to start. - if cnt >= 300: - if self.__conf["verbose"] > 1: - sys.stdout.write('\n') - raise ServerExecutionException("Failed to start server") - time.sleep(0.1) - cnt += 1 - if self.__conf["verbose"] > 1: - sys.stdout.write('\n') - + def __waitOnServer(self, alive=True, maxtime=30): + # Wait for the server to start (the server has 30 seconds to answer ping) + starttime = time.time() + with VisualWait(self._conf["verbose"]) as vis: + while self._alive and not self.__ping() == alive or ( + not alive and os.path.exists(self._conf["socket"]) + ): + now = time.time() + # Wonderful visual :) + if now > starttime + 1: + vis.heartbeat() + # f end time reached: + if now - starttime >= maxtime: + raise ServerExecutionException("Failed to start server") + time.sleep(0.1) def start(self, argv): - # Command line options - self.__argv = argv - # Install signal handlers signal.signal(signal.SIGTERM, self.__sigTERMhandler) signal.signal(signal.SIGINT, self.__sigTERMhandler) - # Reads the command line options. - try: - cmdOpts = 'hc:s:p:xfbdviqV' - cmdLongOpts = ['loglevel', 'logtarget', 'syslogsocket', 'help', 'version'] - optList, args = getopt.getopt(self.__argv[1:], cmdOpts, cmdLongOpts) - except getopt.GetoptError: - self.dispUsage() - return False + # Command line options + if self._argv is None: + ret = self.initCmdLine(argv) + if ret is not None: + return ret - self.__getCmdLineOptions(optList) - - verbose = self.__conf["verbose"] - if verbose <= 0: - logSys.setLevel(logging.ERROR) - elif verbose == 1: - logSys.setLevel(logging.WARNING) - elif verbose == 2: - logSys.setLevel(logging.INFO) - elif verbose == 3: - logSys.setLevel(logging.DEBUG) - else: - logSys.setLevel(logging.HEAVYDEBUG) - # Add the default logging handler to dump to stderr - logout = logging.StreamHandler(sys.stderr) - # set a format which is simpler for console use - formatter = logging.Formatter('%(levelname)-6s %(message)s') - # tell the handler to use this format - logout.setFormatter(formatter) - logSys.addHandler(logout) - - # Set the configuration path - self.__configurator.setBaseDir(self.__conf["conf"]) - - # Set socket path - self.__configurator.readEarly() - conf = self.__configurator.getEarlyOptions() - if self.__conf["socket"] is None: - self.__conf["socket"] = conf["socket"] - if self.__conf["pidfile"] is None: - self.__conf["pidfile"] = conf["pidfile"] - if self.__conf.get("logtarget", None) is None: - self.__conf["logtarget"] = conf["logtarget"] - if self.__conf.get("loglevel", None) is None: - self.__conf["loglevel"] = conf["loglevel"] - if self.__conf.get("syslogsocket", None) is None: - self.__conf["syslogsocket"] = conf["syslogsocket"] - - logSys.info("Using socket file %s", self.__conf["socket"]) - - logSys.info("Using pid file %s, [%s] logging to %s", - self.__conf["pidfile"], self.__conf["loglevel"], self.__conf["logtarget"]) - - if self.__conf["dump"]: - ret = self.__readConfig() - self.dumpConfig(self.__stream) - return ret + # Commands + args = self._args # Interactive mode - if self.__conf["interactive"]: + if self._conf.get("interactive", False): try: import readline except ImportError: @@ -500,35 +349,56 @@ class Fail2banClient: return False return self.__processCommand(args) - def __readConfig(self, jail=None): - # Read the configuration - # TODO: get away from stew of return codes and exception - # handling -- handle via exceptions - try: - self.__configurator.Reload() - self.__configurator.readAll() - ret = self.__configurator.getOptions(jail) - self.__configurator.convertToProtocol() - self.__stream = self.__configurator.getConfigStream() - except Exception, e: - logSys.error("Failed during configuration: %s" % e) - ret = False - return ret - - @staticmethod - def dumpConfig(cmd): - for c in cmd: - print c - return True - class ServerExecutionException(Exception): pass -if __name__ == "__main__": # pragma: no cover - can't test main + +## +# Wonderful visual :) +# + +class _VisualWait: + pos = 0 + delta = 1 + maxpos = 10 + def __enter__(self): + return self + def __exit__(self, *args): + if self.pos: + sys.stdout.write('\r'+(' '*(35+self.maxpos))+'\r') + sys.stdout.flush() + def heartbeat(self): + if not self.pos: + sys.stdout.write("\nINFO [#" + (' '*self.maxpos) + "] Waiting on the server...\r\x1b[8C") + self.pos += self.delta + if self.delta > 0: + s = " #\x1b[1D" if self.pos > 1 else "# \x1b[2D" + else: + s = "\x1b[1D# \x1b[2D" + sys.stdout.write(s) + sys.stdout.flush() + if self.pos > self.maxpos: + self.delta = -1 + elif self.pos < 2: + self.delta = 1 +class _NotVisualWait: + def __enter__(self): + return self + def __exit__(self, *args): + pass + def heartbeat(self): + pass + +def VisualWait(verbose): + return _VisualWait() if verbose > 1 else _NotVisualWait() + + +def exec_command_line(): # pragma: no cover - can't test main client = Fail2banClient() # Exit with correct return value if client.start(sys.argv): - sys.exit(0) + exit(0) else: - sys.exit(-1) + exit(-1) + diff --git a/fail2ban/client/fail2bancmdline.py b/fail2ban/client/fail2bancmdline.py new file mode 100644 index 00000000..e23f9b19 --- /dev/null +++ b/fail2ban/client/fail2bancmdline.py @@ -0,0 +1,247 @@ +#!/usr/bin/python +# emacs: -*- mode: python; py-indent-offset: 4; indent-tabs-mode: t -*- +# vi: set ft=python sts=4 ts=4 sw=4 noet : +# +# This file is part of Fail2Ban. +# +# Fail2Ban is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. +# +# Fail2Ban is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with Fail2Ban; if not, write to the Free Software +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +__author__ = "Fail2Ban Developers" +__copyright__ = "Copyright (c) 2004-2008 Cyril Jaquier, 2012-2014 Yaroslav Halchenko, 2014-2016 Serg G. Brester" +__license__ = "GPL" + +import getopt +import logging +import os +import sys + +from ..version import version +from ..protocol import printFormatted +from ..helpers import getLogger + +# Gets the instance of the logger. +logSys = getLogger("fail2ban") + +CONFIG_PARAMS = ("socket", "pidfile", "logtarget", "loglevel", "syslogsocket",) + + +class Fail2banCmdLine(): + + def __init__(self): + self._argv = self._args = None + self._configurator = None + self.resetConf() + + def resetConf(self): + self._conf = { + "async": False, + "conf": "/etc/fail2ban", + "force": False, + "background": True, + "verbose": 1, + "socket": None, + "pidfile": None + } + + @property + def configurator(self): + if self._configurator: + return self._configurator + # New configurator + from .configurator import Configurator + self._configurator = Configurator() + # Set the configuration path + self._configurator.setBaseDir(self._conf["conf"]) + return self._configurator + + + def applyMembers(self, obj): + for o in obj.__dict__: + self.__dict__[o] = obj.__dict__[o] + + def dispVersion(self): + print "Fail2Ban v" + version + print + print "Copyright (c) 2004-2008 Cyril Jaquier, 2008- Fail2Ban Contributors" + print "Copyright of modifications held by their respective authors." + print "Licensed under the GNU General Public License v2 (GPL)." + print + print "Written by Cyril Jaquier ." + print "Many contributions by Yaroslav O. Halchenko ." + + def dispUsage(self): + """ Prints Fail2Ban command line options and exits + """ + caller = os.path.basename(self._argv[0]) + print "Usage: "+caller+" [OPTIONS]" + (" " if not caller.endswith('server') else "") + print + print "Fail2Ban v" + version + " reads log file that contains password failure report" + print "and bans the corresponding IP addresses using firewall rules." + print + print "Options:" + print " -c configuration directory" + print " -s socket path" + print " -p pidfile path" + print " --loglevel logging level" + print " --logtarget |STDOUT|STDERR|SYSLOG" + print " --syslogsocket auto|" + print " -d dump configuration. For debugging" + print " -i interactive mode" + print " -v increase verbosity" + print " -q decrease verbosity" + print " -x force execution of the server (remove socket file)" + print " -b start server in background (default)" + print " -f start server in foreground" + print " --async start server in async mode (for internal usage only, don't read configuration)" + print " -h, --help display this help message" + print " -V, --version print the version" + + if not caller.endswith('server'): + print + print "Command:" + # Prints the protocol + printFormatted() + + print + print "Report bugs to https://github.com/fail2ban/fail2ban/issues" + + def __getCmdLineOptions(self, optList): + """ Gets the command line options + """ + for opt in optList: + o = opt[0] + if o == "--async": + self._conf["async"] = True + if o == "-c": + self._conf["conf"] = opt[1] + elif o == "-s": + self._conf["socket"] = opt[1] + elif o == "-p": + self._conf["pidfile"] = opt[1] + elif o.startswith("--log") or o.startswith("--sys"): + self._conf[ o[2:] ] = opt[1] + elif o == "-d": + self._conf["dump"] = True + elif o == "-v": + self._conf["verbose"] += 1 + elif o == "-q": + self._conf["verbose"] -= 1 + elif o == "-x": + self._conf["force"] = True + elif o == "-i": + self._conf["interactive"] = True + elif o == "-b": + self._conf["background"] = True + elif o == "-f": + self._conf["background"] = False + elif o in ["-h", "--help"]: + self.dispUsage() + exit(0) + elif o in ["-V", "--version"]: + self.dispVersion() + exit(0) + + def initCmdLine(self, argv): + # First time? + initial = (self._argv is None) + + # Command line options + self._argv = argv + + # Reads the command line options. + try: + cmdOpts = 'hc:s:p:xfbdviqV' + cmdLongOpts = ['loglevel=', 'logtarget=', 'syslogsocket=', 'async', 'help', 'version'] + optList, self._args = getopt.getopt(self._argv[1:], cmdOpts, cmdLongOpts) + except getopt.GetoptError: + self.dispUsage() + exit(-1) + + self.__getCmdLineOptions(optList) + + if initial: + verbose = self._conf["verbose"] + if verbose <= 0: + logSys.setLevel(logging.ERROR) + elif verbose == 1: + logSys.setLevel(logging.WARNING) + elif verbose == 2: + logSys.setLevel(logging.INFO) + elif verbose == 3: + logSys.setLevel(logging.DEBUG) + else: + logSys.setLevel(logging.HEAVYDEBUG) + # Add the default logging handler to dump to stderr + logout = logging.StreamHandler(sys.stderr) + # set a format which is simpler for console use + formatter = logging.Formatter('%(levelname)-6s %(message)s') + # tell the handler to use this format + logout.setFormatter(formatter) + logSys.addHandler(logout) + + # Set expected parameters (like socket, pidfile, etc) from configuration, + # if those not yet specified, in which read configuration only if needed here: + conf = None + for o in CONFIG_PARAMS: + if self._conf.get(o, None) is None: + if not conf: + self.configurator.readEarly() + conf = self.configurator.getEarlyOptions() + self._conf[o] = conf[o] + + logSys.info("Using socket file %s", self._conf["socket"]) + + logSys.info("Using pid file %s, [%s] logging to %s", + self._conf["pidfile"], self._conf["loglevel"], self._conf["logtarget"]) + + if self._conf.get("dump", False): + ret, stream = self.readConfig() + self.dumpConfig(stream) + return ret + + # Nothing to do here, process in client/server + return None + + def readConfig(self, jail=None): + # Read the configuration + # TODO: get away from stew of return codes and exception + # handling -- handle via exceptions + stream = None + try: + self.configurator.Reload() + self.configurator.readAll() + ret = self.configurator.getOptions(jail) + self.configurator.convertToProtocol() + stream = self.configurator.getConfigStream() + except Exception, e: + logSys.error("Failed during configuration: %s" % e) + ret = False + return ret, stream + + @staticmethod + def dumpConfig(cmd): + for c in cmd: + print c + return True + + @staticmethod + def exit(code=0): + logSys.debug("Exit with code %s", code) + if os._exit: + os._exit(code) + else: + sys.exit(code) + +# global exit handler: +exit = Fail2banCmdLine.exit \ No newline at end of file diff --git a/fail2ban/client/fail2banserver.py b/fail2ban/client/fail2banserver.py old mode 100755 new mode 100644 index 0b8b6418..6c1dd694 --- a/fail2ban/client/fail2banserver.py +++ b/fail2ban/client/fail2banserver.py @@ -1,7 +1,7 @@ #!/usr/bin/python # emacs: -*- mode: python; py-indent-offset: 4; indent-tabs-mode: t -*- # vi: set ft=python sts=4 ts=4 sw=4 noet : - +# # This file is part of Fail2Ban. # # Fail2Ban is free software; you can redistribute it and/or modify @@ -17,125 +17,171 @@ # You should have received a copy of the GNU General Public License # along with Fail2Ban; if not, write to the Free Software # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. - -__author__ = "Cyril Jaquier" -__copyright__ = "Copyright (c) 2004 Cyril Jaquier" +__author__ = "Fail2Ban Developers" +__copyright__ = "Copyright (c) 2004-2008 Cyril Jaquier, 2012-2014 Yaroslav Halchenko, 2014-2016 Serg G. Brester" __license__ = "GPL" -import getopt import os import sys -from fail2ban.version import version -from fail2ban.server.server import Server -from fail2ban.helpers import getLogger - -# Gets the instance of the logger. -logSys = getLogger("fail2ban") +from ..version import version +from ..server.server import Server, ServerDaemonize +from ..server.utils import Utils +from .fail2bancmdline import Fail2banCmdLine, logSys, exit +SERVER = "fail2ban-server" ## # \mainpage Fail2Ban # # \section Introduction # -# Fail2ban is designed to protect your server against brute force attacks. -# Its first goal was to protect a SSH server. +class Fail2banServer(Fail2banCmdLine): -class Fail2banServer: + # def __init__(self): + # Fail2banCmdLine.__init__(self) - def __init__(self): - self.__server = None - self.__argv = None - self.__conf = dict() - self.__conf["background"] = True - self.__conf["force"] = False - self.__conf["socket"] = "/var/run/fail2ban/fail2ban.sock" - self.__conf["pidfile"] = "/var/run/fail2ban/fail2ban.pid" + ## + # Start Fail2Ban server in main thread without fork (foreground). + # + # Start the Fail2ban server in foreground (daemon mode or not). - def dispVersion(self): - print "Fail2Ban v" + version - print - print "Copyright (c) 2004-2008 Cyril Jaquier, 2008- Fail2Ban Contributors" - print "Copyright of modifications held by their respective authors." - print "Licensed under the GNU General Public License v2 (GPL)." - print - print "Written by Cyril Jaquier ." - print "Many contributions by Yaroslav O. Halchenko ." + @staticmethod + def startServerDirect(conf, daemon=True): + server = None + try: + # Start it in foreground (current thread, not new process), + # server object will internally fork self if daemon is True + server = Server(daemon) + server.start(conf["socket"], + conf["pidfile"], conf["force"], + conf=conf) + except ServerDaemonize: + pass + except Exception, e: + logSys.exception(e) + if server: + server.quit() + exit(-1) - def dispUsage(self): - """ Prints Fail2Ban command line options and exits - """ - print "Usage: "+self.__argv[0]+" [OPTIONS]" - print - print "Fail2Ban v" + version + " reads log file that contains password failure report" - print "and bans the corresponding IP addresses using firewall rules." - print - print "Only use this command for debugging purpose. Start the server with" - print "fail2ban-client instead. The default behaviour is to start the server" - print "in background." - print - print "Options:" - print " -b start in background" - print " -f start in foreground" - print " -s socket path" - print " -p pidfile path" - print " -x force execution of the server (remove socket file)" - print " -h, --help display this help message" - print " -V, --version print the version" - print - print "Report bugs to https://github.com/fail2ban/fail2ban/issues" + return server - def __getCmdLineOptions(self, optList): - """ Gets the command line options - """ - for opt in optList: - if opt[0] == "-b": - self.__conf["background"] = True - if opt[0] == "-f": - self.__conf["background"] = False - if opt[0] == "-s": - self.__conf["socket"] = opt[1] - if opt[0] == "-p": - self.__conf["pidfile"] = opt[1] - if opt[0] == "-x": - self.__conf["force"] = True - if opt[0] in ["-h", "--help"]: - self.dispUsage() - sys.exit(0) - if opt[0] in ["-V", "--version"]: - self.dispVersion() - sys.exit(0) + ## + # Start Fail2Ban server. + # + # Start the Fail2ban server in daemon mode (background, start from client). + + @staticmethod + def startServerAsync(conf): + # Forks the current process. + pid = os.fork() + if pid == 0: + args = list() + args.append(SERVER) + # Start async (don't read config) and in background as requested. + args.append("--async") + args.append("-b") + # Set the socket path. + args.append("-s") + args.append(conf["socket"]) + # Set the pidfile + args.append("-p") + args.append(conf["pidfile"]) + # Force the execution if needed. + if conf["force"]: + args.append("-x") + # Logging parameters: + for o in ('loglevel', 'logtarget', 'syslogsocket'): + args.append("--"+o) + args.append(conf[o]) + + try: + # Use the current directory. + exe = os.path.abspath(os.path.join(sys.path[0], SERVER)) + logSys.debug("Starting %r with args %r", exe, args) + os.execv(exe, args) + except OSError: + try: + # Use the PATH env. + logSys.warning("Initial start attempt failed. Starting %r with the same args", SERVER) + os.execvp(SERVER, args) + except OSError: + exit(-1) + + def _Fail2banClient(self): + from .fail2banclient import Fail2banClient + cli = Fail2banClient() + cli.applyMembers(self) + return cli def start(self, argv): # Command line options - self.__argv = argv + ret = self.initCmdLine(argv) + if ret is not None: + return ret - # Reads the command line options. + # Commands + args = self._args + + cli = None + # If client mode - whole processing over client: + if len(args) or self._conf.get("interactive", False): + cli = self._Fail2banClient() + return cli.start(argv) + + # Start the server: + server = None try: - cmdOpts = 'bfs:p:xhV' - cmdLongOpts = ['help', 'version'] - optList, args = getopt.getopt(self.__argv[1:], cmdOpts, cmdLongOpts) - except getopt.GetoptError: - self.dispUsage() - sys.exit(-1) + # async = True, if started from client, should fork, daemonize, etc... + # background = True, if should start in new process, otherwise start in foreground + async = self._conf.get("async", False) + background = self._conf["background"] + # If was started not from the client: + if not async: + # Start new thread with client to read configuration and + # transfer it to the server: + cli = self._Fail2banClient() + phase = dict() + logSys.debug('Configure via async client thread') + cli.configureServer(async=True, phase=phase) + # wait up to 30 seconds, do not continue if configuration is not 100% valid: + Utils.wait_for(lambda: phase.get('ready', None) is not None, 30) + if not phase.get('start', False): + return False - self.__getCmdLineOptions(optList) + # Start server, daemonize it, etc. + if async or not background: + server = Fail2banServer.startServerDirect(self._conf, background) + else: + Fail2banServer.startServerAsync(self._conf) + if cli: + cli._server = server + + # wait for client answer "done": + if not async and cli: + Utils.wait_for(lambda: phase.get('done', None) is not None, 30) + if not phase.get('done', False): + if server: + server.quit() + exit(-1) + logSys.debug('Starting server done') - try: - self.__server = Server(self.__conf["background"]) - self.__server.start(self.__conf["socket"], - self.__conf["pidfile"], - self.__conf["force"]) - return True except Exception, e: logSys.exception(e) - if self.__server: - self.__server.quit() - return False + if server: + server.quit() + exit(-1) -if __name__ == "__main__": + return True + + @staticmethod + def exit(code=0): # pragma: no cover + if code != 0: + logSys.error("Could not start %s", SERVER) + exit(code) + +def exec_command_line(): # pragma: no cover - can't test main server = Fail2banServer() if server.start(sys.argv): - sys.exit(0) + exit(0) else: - sys.exit(-1) + exit(-1) diff --git a/fail2ban/protocol.py b/fail2ban/protocol.py index 5d9fdd65..857d5fa6 100644 --- a/fail2ban/protocol.py +++ b/fail2ban/protocol.py @@ -42,11 +42,13 @@ CSPROTO = dotdict({ protocol = [ ['', "BASIC", ""], ["start", "starts the server and the jails"], -["reload", "reloads the configuration"], +["restart", "restarts the server"], +["reload", "reloads the configuration without restart"], ["reload ", "reloads the jail "], ["stop", "stops all jails and terminate the server"], ["status", "gets the current status of the server"], -["ping", "tests if the server is alive"], +["ping", "tests if the server is alive"], +["echo", "for internal usage, returns back and outputs a given string"], ["help", "return this output"], ["version", "return the server version"], ['', "LOGGING", ""], diff --git a/fail2ban/server/server.py b/fail2ban/server/server.py index 1d3e89d3..d7f212c8 100644 --- a/fail2ban/server/server.py +++ b/fail2ban/server/server.py @@ -67,6 +67,11 @@ class Server: 'FreeBSD': '/var/run/log', 'Linux': '/dev/log', } + # todo: remove that, if test cases are fixed + self.setSyslogSocket("auto") + # Set logging level + self.setLogLevel("INFO") + self.setLogTarget("STDOUT") def __sigTERMhandler(self, signum, frame): logSys.debug("Caught signal %d. Exiting" % signum) @@ -77,13 +82,27 @@ class Server: self.flushLogs() def start(self, sock, pidfile, force=False, conf={}): - # First set all logging parameters: - self.setSyslogSocket(conf.get("syslogsocket", "auto")) - self.setLogLevel(conf.get("loglevel", "INFO")) - self.setLogTarget(conf.get("logtarget", "STDOUT")) + # First set the mask to only allow access to owner + os.umask(0077) + # Second daemonize before logging etc, because it will close all handles: + if self.__daemon: # pragma: no cover + logSys.info("Starting in daemon mode") + ret = self.__createDaemon() + if not ret: + logSys.error("Could not create daemon") + raise ServerInitializationError("Could not create daemon") + + # Set all logging parameters (or use default if not specified): + self.setSyslogSocket(conf.get("syslogsocket", self.__syslogSocket)) + self.setLogLevel(conf.get("loglevel", self.__logLevel)) + self.setLogTarget(conf.get("logtarget", self.__logTarget)) + logSys.info("-"*50) logSys.info("Starting Fail2ban v%s", version.version) + if self.__daemon: # pragma: no cover + logSys.info("Daemon started") + # Install signal handlers signal.signal(signal.SIGTERM, self.__sigTERMhandler) signal.signal(signal.SIGINT, self.__sigTERMhandler) @@ -92,17 +111,6 @@ class Server: # Ensure unhandled exceptions are logged sys.excepthook = excepthook - # First set the mask to only allow access to owner - os.umask(0077) - if self.__daemon: # pragma: no cover - logSys.info("Starting in daemon mode") - ret = self.__createDaemon() - if ret: - logSys.info("Daemon started") - else: - logSys.error("Could not create daemon") - raise ServerInitializationError("Could not create daemon") - # Creates a PID file. try: logSys.debug("Creating PID file %s" % pidfile) @@ -139,11 +147,8 @@ class Server: self.stopAllJail() # Only now shutdown the logging. - try: - self.__loggingLock.acquire() + with self.__loggingLock: logging.shutdown() - finally: - self.__loggingLock.release() def addJail(self, name, backend): self.__jails.add(name, backend, self.__db) @@ -372,16 +377,15 @@ class Server: # @param value the level def setLogLevel(self, value): - try: - self.__loggingLock.acquire() - getLogger("fail2ban").setLevel( - getattr(logging, value.upper())) - except AttributeError: - raise ValueError("Invalid log level") - else: - self.__logLevel = value.upper() - finally: - self.__loggingLock.release() + value = value.upper() + with self.__loggingLock: + if self.__logLevel == value: + return + try: + getLogger("fail2ban").setLevel(getattr(logging, value)) + self.__logLevel = value + except AttributeError: + raise ValueError("Invalid log level") ## # Get the logging level. @@ -390,11 +394,8 @@ class Server: # @return the log level def getLogLevel(self): - try: - self.__loggingLock.acquire() + with self.__loggingLock: return self.__logLevel - finally: - self.__loggingLock.release() ## # Sets the logging target. @@ -480,24 +481,21 @@ class Server: # syslogsocket is the full path to the syslog socket # @param syslogsocket the syslog socket path def setSyslogSocket(self, syslogsocket): - self.__syslogSocket = syslogsocket - # Conditionally reload, logtarget depends on socket path when SYSLOG - return self.__logTarget != "SYSLOG"\ - or self.setLogTarget(self.__logTarget) + with self.__loggingLock: + if self.__syslogSocket == syslogsocket: + return True + self.__syslogSocket = syslogsocket + # Conditionally reload, logtarget depends on socket path when SYSLOG + return self.__logTarget != "SYSLOG"\ + or self.setLogTarget(self.__logTarget) def getLogTarget(self): - try: - self.__loggingLock.acquire() + with self.__loggingLock: return self.__logTarget - finally: - self.__loggingLock.release() def getSyslogSocket(self): - try: - self.__loggingLock.acquire() + with self.__loggingLock: return self.__syslogSocket - finally: - self.__loggingLock.release() def flushLogs(self): if self.__logTarget not in ['STDERR', 'STDOUT', 'SYSLOG']: @@ -552,7 +550,7 @@ class Server: # child process, and this makes sure that it is effect even if the parent # terminates quickly. signal.signal(signal.SIGHUP, signal.SIG_IGN) - + try: # Fork a child process so the parent can exit. This will return control # to the command line or shell. This is required so that the new process @@ -593,7 +591,7 @@ class Server: os._exit(0) # Exit parent (the first child) of the second child. else: os._exit(0) # Exit parent of the first child. - + # Close all open files. Try the system configuration variable, SC_OPEN_MAX, # for the maximum number of open files to close. If it doesn't exist, use # the default value (configurable). @@ -625,3 +623,6 @@ class Server: class ServerInitializationError(Exception): pass + +class ServerDaemonize(Exception): + pass \ No newline at end of file diff --git a/fail2ban/server/transmitter.py b/fail2ban/server/transmitter.py index 29d6d189..2d6f0d77 100644 --- a/fail2ban/server/transmitter.py +++ b/fail2ban/server/transmitter.py @@ -93,6 +93,8 @@ class Transmitter: name = command[1] self.__server.stopJail(name) return None + elif command[0] == "echo": + return command[1:] elif command[0] == "sleep": value = command[1] time.sleep(float(value)) From afa1cdc3ae658e8595e7ca788cda1566c65bbadb Mon Sep 17 00:00:00 2001 From: sebres Date: Thu, 11 Feb 2016 17:57:23 +0100 Subject: [PATCH 04/13] client/server (bin) test cases introduced, ultimate closes #1121, closes #1139 small code review and fixing of some bugs during client-server communication process (in the test cases); --- bin/fail2ban-client | 4 +- bin/fail2ban-server | 4 +- fail2ban/client/fail2banclient.py | 146 ++++---- fail2ban/client/fail2bancmdline.py | 194 ++++++----- fail2ban/client/fail2banserver.py | 41 ++- fail2ban/protocol.py | 29 +- fail2ban/server/server.py | 46 ++- fail2ban/tests/fail2banclienttestcase.py | 411 +++++++++++++++++++++++ fail2ban/tests/fail2banregextestcase.py | 12 - fail2ban/tests/utils.py | 25 ++ 10 files changed, 707 insertions(+), 205 deletions(-) create mode 100644 fail2ban/tests/fail2banclienttestcase.py diff --git a/bin/fail2ban-client b/bin/fail2ban-client index 19e76a98..f5ae7946 100755 --- a/bin/fail2ban-client +++ b/bin/fail2ban-client @@ -31,7 +31,7 @@ __author__ = "Fail2Ban Developers" __copyright__ = "Copyright (c) 2004-2008 Cyril Jaquier, 2012-2014 Yaroslav Halchenko, 2014-2016 Serg G. Brester" __license__ = "GPL" -from fail2ban.client.fail2banclient import exec_command_line +from fail2ban.client.fail2banclient import exec_command_line, sys if __name__ == "__main__": - exec_command_line() + exec_command_line(sys.argv) diff --git a/bin/fail2ban-server b/bin/fail2ban-server index 8e64d865..ffafabe2 100755 --- a/bin/fail2ban-server +++ b/bin/fail2ban-server @@ -31,7 +31,7 @@ __author__ = "Fail2Ban Developers" __copyright__ = "Copyright (c) 2004-2008 Cyril Jaquier, 2012-2014 Yaroslav Halchenko, 2014-2016 Serg G. Brester" __license__ = "GPL" -from fail2ban.client.fail2banserver import exec_command_line +from fail2ban.client.fail2banserver import exec_command_line, sys if __name__ == "__main__": - exec_command_line() + exec_command_line(sys.argv) diff --git a/fail2ban/client/fail2banclient.py b/fail2ban/client/fail2banclient.py index 7adbab95..736f8fd2 100755 --- a/fail2ban/client/fail2banclient.py +++ b/fail2ban/client/fail2banclient.py @@ -28,12 +28,20 @@ import socket import sys import time +import threading from threading import Thread from ..version import version from .csocket import CSocket from .beautifier import Beautifier -from .fail2bancmdline import Fail2banCmdLine, logSys, exit +from .fail2bancmdline import Fail2banCmdLine, ExitException, logSys, exit, output + +MAX_WAITTIME = 30 + + +def _thread_name(): + return threading.current_thread().__class__.__name__ + ## # @@ -51,13 +59,13 @@ class Fail2banClient(Fail2banCmdLine, Thread): self._beautifier = None def dispInteractive(self): - print "Fail2Ban v" + version + " reads log file that contains password failure report" - print "and bans the corresponding IP addresses using firewall rules." - print + output("Fail2Ban v" + version + " reads log file that contains password failure report") + output("and bans the corresponding IP addresses using firewall rules.") + output("") def __sigTERMhandler(self, signum, frame): # Print a new line because we probably come from wait - print + output("") logSys.warning("Caught signal %d. Exiting" % signum) exit(-1) @@ -85,11 +93,11 @@ class Fail2banClient(Fail2banCmdLine, Thread): if ret[0] == 0: logSys.debug("OK : " + `ret[1]`) if showRet or c[0] == 'echo': - print beautifier.beautify(ret[1]) + output(beautifier.beautify(ret[1])) else: logSys.error("NOK: " + `ret[1].args`) if showRet: - print beautifier.beautifyError(ret[1]) + output(beautifier.beautifyError(ret[1])) streamRet = False except socket.error: if showRet or self._conf["verbose"] > 1: @@ -182,10 +190,13 @@ class Fail2banClient(Fail2banCmdLine, Thread): # Start server direct here in main thread (not fork): self._server = Fail2banServer.startServerDirect(self._conf, False) + except ExitException: + pass except Exception as e: - print - logSys.error("Exception while starting server foreground") + output("") + logSys.error("Exception while starting server " + ("background" if background else "foreground")) logSys.error(e) + return False finally: self._alive = False @@ -229,18 +240,18 @@ class Fail2banClient(Fail2banCmdLine, Thread): elif len(cmd) == 1 and cmd[0] == "restart": if self._conf.get("interactive", False): - print(' ## stop ... ') + output(' ## stop ... ') self.__processCommand(['stop']) self.__waitOnServer(False) # in interactive mode reset config, to make full-reload if there something changed: if self._conf.get("interactive", False): - print(' ## load configuration ... ') + output(' ## load configuration ... ') self.resetConf() ret = self.initCmdLine(self._argv) if ret is not None: return ret if self._conf.get("interactive", False): - print(' ## start ... ') + output(' ## start ... ') return self.__processCommand(['start']) elif len(cmd) >= 1 and cmd[0] == "reload": @@ -283,7 +294,9 @@ class Fail2banClient(Fail2banCmdLine, Thread): return False return True - def __waitOnServer(self, alive=True, maxtime=30): + def __waitOnServer(self, alive=True, maxtime=None): + if maxtime is None: + maxtime = MAX_WAITTIME # Wait for the server to start (the server has 30 seconds to answer ping) starttime = time.time() with VisualWait(self._conf["verbose"]) as vis: @@ -301,53 +314,59 @@ class Fail2banClient(Fail2banCmdLine, Thread): def start(self, argv): # Install signal handlers - signal.signal(signal.SIGTERM, self.__sigTERMhandler) - signal.signal(signal.SIGINT, self.__sigTERMhandler) + _prev_signals = {} + if _thread_name() == '_MainThread': + for s in (signal.SIGTERM, signal.SIGINT): + _prev_signals[s] = signal.getsignal(s) + signal.signal(s, self.__sigTERMhandler) + try: + # Command line options + if self._argv is None: + ret = self.initCmdLine(argv) + if ret is not None: + return ret - # Command line options - if self._argv is None: - ret = self.initCmdLine(argv) - if ret is not None: - return ret + # Commands + args = self._args - # Commands - args = self._args - - # Interactive mode - if self._conf.get("interactive", False): - try: - import readline - except ImportError: - logSys.error("Readline not available") - return False - try: - ret = True - if len(args) > 0: - ret = self.__processCommand(args) - if ret: - readline.parse_and_bind("tab: complete") - self.dispInteractive() - while True: - cmd = raw_input(self.PROMPT) - if cmd == "exit" or cmd == "quit": - # Exit - return True - if cmd == "help": - self.dispUsage() - elif not cmd == "": - try: - self.__processCommand(shlex.split(cmd)) - except Exception, e: - logSys.error(e) - except (EOFError, KeyboardInterrupt): - print - return True - # Single command mode - else: - if len(args) < 1: - self.dispUsage() - return False - return self.__processCommand(args) + # Interactive mode + if self._conf.get("interactive", False): + try: + import readline + except ImportError: + logSys.error("Readline not available") + return False + try: + ret = True + if len(args) > 0: + ret = self.__processCommand(args) + if ret: + readline.parse_and_bind("tab: complete") + self.dispInteractive() + while True: + cmd = raw_input(self.PROMPT) + if cmd == "exit" or cmd == "quit": + # Exit + return True + if cmd == "help": + self.dispUsage() + elif not cmd == "": + try: + self.__processCommand(shlex.split(cmd)) + except Exception, e: + logSys.error(e) + except (EOFError, KeyboardInterrupt): + output("") + return True + # Single command mode + else: + if len(args) < 1: + self.dispUsage() + return False + return self.__processCommand(args) + finally: + for s, sh in _prev_signals.iteritems(): + signal.signal(s, sh) class ServerExecutionException(Exception): @@ -361,7 +380,8 @@ class ServerExecutionException(Exception): class _VisualWait: pos = 0 delta = 1 - maxpos = 10 + def __init__(self, maxpos=10): + self.maxpos = maxpos def __enter__(self): return self def __exit__(self, *args): @@ -390,14 +410,14 @@ class _NotVisualWait: def heartbeat(self): pass -def VisualWait(verbose): - return _VisualWait() if verbose > 1 else _NotVisualWait() +def VisualWait(verbose, *args, **kwargs): + return _VisualWait(*args, **kwargs) if verbose > 1 else _NotVisualWait() -def exec_command_line(): # pragma: no cover - can't test main +def exec_command_line(argv): client = Fail2banClient() # Exit with correct return value - if client.start(sys.argv): + if client.start(argv): exit(0) else: exit(-1) diff --git a/fail2ban/client/fail2bancmdline.py b/fail2ban/client/fail2bancmdline.py index e23f9b19..2ed6a499 100644 --- a/fail2ban/client/fail2bancmdline.py +++ b/fail2ban/client/fail2bancmdline.py @@ -33,7 +33,12 @@ from ..helpers import getLogger # Gets the instance of the logger. logSys = getLogger("fail2ban") +def output(s): + print(s) + CONFIG_PARAMS = ("socket", "pidfile", "logtarget", "loglevel", "syslogsocket",) +# Used to signal - we are in test cases (ex: prevents change logging params, log capturing, etc) +PRODUCTION = True class Fail2banCmdLine(): @@ -71,50 +76,50 @@ class Fail2banCmdLine(): self.__dict__[o] = obj.__dict__[o] def dispVersion(self): - print "Fail2Ban v" + version - print - print "Copyright (c) 2004-2008 Cyril Jaquier, 2008- Fail2Ban Contributors" - print "Copyright of modifications held by their respective authors." - print "Licensed under the GNU General Public License v2 (GPL)." - print - print "Written by Cyril Jaquier ." - print "Many contributions by Yaroslav O. Halchenko ." + output("Fail2Ban v" + version) + output("") + output("Copyright (c) 2004-2008 Cyril Jaquier, 2008- Fail2Ban Contributors") + output("Copyright of modifications held by their respective authors.") + output("Licensed under the GNU General Public License v2 (GPL).") + output("") + output("Written by Cyril Jaquier .") + output("Many contributions by Yaroslav O. Halchenko .") def dispUsage(self): """ Prints Fail2Ban command line options and exits """ caller = os.path.basename(self._argv[0]) - print "Usage: "+caller+" [OPTIONS]" + (" " if not caller.endswith('server') else "") - print - print "Fail2Ban v" + version + " reads log file that contains password failure report" - print "and bans the corresponding IP addresses using firewall rules." - print - print "Options:" - print " -c configuration directory" - print " -s socket path" - print " -p pidfile path" - print " --loglevel logging level" - print " --logtarget |STDOUT|STDERR|SYSLOG" - print " --syslogsocket auto|" - print " -d dump configuration. For debugging" - print " -i interactive mode" - print " -v increase verbosity" - print " -q decrease verbosity" - print " -x force execution of the server (remove socket file)" - print " -b start server in background (default)" - print " -f start server in foreground" - print " --async start server in async mode (for internal usage only, don't read configuration)" - print " -h, --help display this help message" - print " -V, --version print the version" + output("Usage: "+caller+" [OPTIONS]" + (" " if not caller.endswith('server') else "")) + output("") + output("Fail2Ban v" + version + " reads log file that contains password failure report") + output("and bans the corresponding IP addresses using firewall rules.") + output("") + output("Options:") + output(" -c configuration directory") + output(" -s socket path") + output(" -p pidfile path") + output(" --loglevel logging level") + output(" --logtarget |STDOUT|STDERR|SYSLOG") + output(" --syslogsocket auto|") + output(" -d dump configuration. For debugging") + output(" -i interactive mode") + output(" -v increase verbosity") + output(" -q decrease verbosity") + output(" -x force execution of the server (remove socket file)") + output(" -b start server in background (default)") + output(" -f start server in foreground") + output(" --async start server in async mode (for internal usage only, don't read configuration)") + output(" -h, --help display this help message") + output(" -V, --version print the version") if not caller.endswith('server'): - print - print "Command:" + output("") + output("Command:") # Prints the protocol printFormatted() - print - print "Report bugs to https://github.com/fail2ban/fail2ban/issues" + output("") + output("Report bugs to https://github.com/fail2ban/fail2ban/issues") def __getCmdLineOptions(self, optList): """ Gets the command line options @@ -147,71 +152,80 @@ class Fail2banCmdLine(): self._conf["background"] = False elif o in ["-h", "--help"]: self.dispUsage() - exit(0) + return True elif o in ["-V", "--version"]: self.dispVersion() - exit(0) + return True + return None def initCmdLine(self, argv): - # First time? - initial = (self._argv is None) - - # Command line options - self._argv = argv - - # Reads the command line options. try: - cmdOpts = 'hc:s:p:xfbdviqV' - cmdLongOpts = ['loglevel=', 'logtarget=', 'syslogsocket=', 'async', 'help', 'version'] - optList, self._args = getopt.getopt(self._argv[1:], cmdOpts, cmdLongOpts) - except getopt.GetoptError: - self.dispUsage() - exit(-1) + # First time? + initial = (self._argv is None) - self.__getCmdLineOptions(optList) + # Command line options + self._argv = argv + logSys.info("Using start params %s", argv[1:]) - if initial: - verbose = self._conf["verbose"] - if verbose <= 0: - logSys.setLevel(logging.ERROR) - elif verbose == 1: - logSys.setLevel(logging.WARNING) - elif verbose == 2: - logSys.setLevel(logging.INFO) - elif verbose == 3: - logSys.setLevel(logging.DEBUG) - else: - logSys.setLevel(logging.HEAVYDEBUG) - # Add the default logging handler to dump to stderr - logout = logging.StreamHandler(sys.stderr) - # set a format which is simpler for console use - formatter = logging.Formatter('%(levelname)-6s %(message)s') - # tell the handler to use this format - logout.setFormatter(formatter) - logSys.addHandler(logout) + # Reads the command line options. + try: + cmdOpts = 'hc:s:p:xfbdviqV' + cmdLongOpts = ['loglevel=', 'logtarget=', 'syslogsocket=', 'async', 'help', 'version'] + optList, self._args = getopt.getopt(self._argv[1:], cmdOpts, cmdLongOpts) + except getopt.GetoptError: + self.dispUsage() + return False - # Set expected parameters (like socket, pidfile, etc) from configuration, - # if those not yet specified, in which read configuration only if needed here: - conf = None - for o in CONFIG_PARAMS: - if self._conf.get(o, None) is None: - if not conf: - self.configurator.readEarly() - conf = self.configurator.getEarlyOptions() - self._conf[o] = conf[o] + ret = self.__getCmdLineOptions(optList) + if ret is not None: + return ret - logSys.info("Using socket file %s", self._conf["socket"]) + if initial and PRODUCTION: # pragma: no cover - can't test + verbose = self._conf["verbose"] + if verbose <= 0: + logSys.setLevel(logging.ERROR) + elif verbose == 1: + logSys.setLevel(logging.WARNING) + elif verbose == 2: + logSys.setLevel(logging.INFO) + elif verbose == 3: + logSys.setLevel(logging.DEBUG) + else: + logSys.setLevel(logging.HEAVYDEBUG) + # Add the default logging handler to dump to stderr + logout = logging.StreamHandler(sys.stderr) + # set a format which is simpler for console use + formatter = logging.Formatter('%(levelname)-6s %(message)s') + # tell the handler to use this format + logout.setFormatter(formatter) + logSys.addHandler(logout) - logSys.info("Using pid file %s, [%s] logging to %s", - self._conf["pidfile"], self._conf["loglevel"], self._conf["logtarget"]) + # Set expected parameters (like socket, pidfile, etc) from configuration, + # if those not yet specified, in which read configuration only if needed here: + conf = None + for o in CONFIG_PARAMS: + if self._conf.get(o, None) is None: + if not conf: + self.configurator.readEarly() + conf = self.configurator.getEarlyOptions() + self._conf[o] = conf[o] - if self._conf.get("dump", False): - ret, stream = self.readConfig() - self.dumpConfig(stream) - return ret + logSys.info("Using socket file %s", self._conf["socket"]) - # Nothing to do here, process in client/server - return None + logSys.info("Using pid file %s, [%s] logging to %s", + self._conf["pidfile"], self._conf["loglevel"], self._conf["logtarget"]) + + if self._conf.get("dump", False): + ret, stream = self.readConfig() + self.dumpConfig(stream) + return ret + + # Nothing to do here, process in client/server + return None + except Exception as e: + output("ERROR: %s" % (e,)) + #logSys.exception(e) + return False def readConfig(self, jail=None): # Read the configuration @@ -244,4 +258,8 @@ class Fail2banCmdLine(): sys.exit(code) # global exit handler: -exit = Fail2banCmdLine.exit \ No newline at end of file +exit = Fail2banCmdLine.exit + + +class ExitException: + pass \ No newline at end of file diff --git a/fail2ban/client/fail2banserver.py b/fail2ban/client/fail2banserver.py index 6c1dd694..da8e57b8 100644 --- a/fail2ban/client/fail2banserver.py +++ b/fail2ban/client/fail2banserver.py @@ -29,7 +29,11 @@ from ..server.server import Server, ServerDaemonize from ..server.utils import Utils from .fail2bancmdline import Fail2banCmdLine, logSys, exit +MAX_WAITTIME = 30 + SERVER = "fail2ban-server" + + ## # \mainpage Fail2Ban # @@ -72,8 +76,15 @@ class Fail2banServer(Fail2banCmdLine): @staticmethod def startServerAsync(conf): - # Forks the current process. - pid = os.fork() + # Directory of client (to try the first start from the same directory as client): + startdir = sys.path[0] + if startdir in ("", "."): # may be uresolved in test-cases, so get bin-directory: + startdir = os.path.dirname(sys.argv[0]) + # Forks the current process, don't fork if async specified (ex: test cases) + pid = 0 + frk = not conf["async"] + if frk: + pid = os.fork() if pid == 0: args = list() args.append(SERVER) @@ -96,14 +107,20 @@ class Fail2banServer(Fail2banCmdLine): try: # Use the current directory. - exe = os.path.abspath(os.path.join(sys.path[0], SERVER)) + exe = os.path.abspath(os.path.join(startdir, SERVER)) logSys.debug("Starting %r with args %r", exe, args) - os.execv(exe, args) - except OSError: + if frk: + os.execv(exe, args) + else: + os.spawnv(os.P_NOWAITO, exe, args) + except OSError as e: try: # Use the PATH env. - logSys.warning("Initial start attempt failed. Starting %r with the same args", SERVER) - os.execvp(SERVER, args) + logSys.warning("Initial start attempt failed (%s). Starting %r with the same args", e, SERVER) + if frk: + os.execvp(SERVER, args) + else: + os.spawnvp(os.P_NOWAITO, SERVER, args) except OSError: exit(-1) @@ -143,8 +160,8 @@ class Fail2banServer(Fail2banCmdLine): phase = dict() logSys.debug('Configure via async client thread') cli.configureServer(async=True, phase=phase) - # wait up to 30 seconds, do not continue if configuration is not 100% valid: - Utils.wait_for(lambda: phase.get('ready', None) is not None, 30) + # wait up to MAX_WAITTIME, do not continue if configuration is not 100% valid: + Utils.wait_for(lambda: phase.get('ready', None) is not None, MAX_WAITTIME) if not phase.get('start', False): return False @@ -158,7 +175,7 @@ class Fail2banServer(Fail2banCmdLine): # wait for client answer "done": if not async and cli: - Utils.wait_for(lambda: phase.get('done', None) is not None, 30) + Utils.wait_for(lambda: phase.get('done', None) is not None, MAX_WAITTIME) if not phase.get('done', False): if server: server.quit() @@ -179,9 +196,9 @@ class Fail2banServer(Fail2banCmdLine): logSys.error("Could not start %s", SERVER) exit(code) -def exec_command_line(): # pragma: no cover - can't test main +def exec_command_line(argv): server = Fail2banServer() - if server.start(sys.argv): + if server.start(argv): exit(0) else: exit(-1) diff --git a/fail2ban/protocol.py b/fail2ban/protocol.py index 857d5fa6..648666a1 100644 --- a/fail2ban/protocol.py +++ b/fail2ban/protocol.py @@ -26,6 +26,9 @@ __license__ = "GPL" import textwrap +def output(s): + print(s) + ## # Describes the protocol used to communicate with the server. @@ -143,7 +146,7 @@ def printFormatted(): firstHeading = False for m in protocol: if m[0] == '' and firstHeading: - print + output("") firstHeading = True first = True if len(m[0]) >= MARGIN: @@ -154,7 +157,7 @@ def printFormatted(): first = False else: line = ' ' * (INDENT + MARGIN) + n.strip() - print line + output(line) ## @@ -165,20 +168,20 @@ def printWiki(): for m in protocol: if m[0] == '': if firstHeading: - print "|}" + output("|}") __printWikiHeader(m[1], m[2]) firstHeading = True else: - print "|-" - print "| " + m[0] + " || || " + m[1] - print "|}" + output("|-") + output("| " + m[0] + " || || " + m[1]) + output("|}") def __printWikiHeader(section, desc): - print - print "=== " + section + " ===" - print - print desc - print - print "{|" - print "| '''Command''' || || '''Description'''" + output("") + output("=== " + section + " ===") + output("") + output(desc) + output("") + output("{|") + output("| '''Command''' || || '''Description'''") diff --git a/fail2ban/server/server.py b/fail2ban/server/server.py index d7f212c8..d4786f29 100644 --- a/fail2ban/server/server.py +++ b/fail2ban/server/server.py @@ -24,6 +24,7 @@ __author__ = "Cyril Jaquier" __copyright__ = "Copyright (c) 2004 Cyril Jaquier" __license__ = "GPL" +import threading from threading import Lock, RLock import logging import logging.handlers @@ -42,6 +43,10 @@ from ..helpers import getLogger, excepthook # Gets the instance of the logger. logSys = getLogger(__name__) +DEF_SYSLOGSOCKET = "auto" +DEF_LOGLEVEL = "INFO" +DEF_LOGTARGET = "STDOUT" + try: from .database import Fail2BanDb except ImportError: # pragma: no cover @@ -49,6 +54,10 @@ except ImportError: # pragma: no cover Fail2BanDb = None +def _thread_name(): + return threading.current_thread().__class__.__name__ + + class Server: def __init__(self, daemon = False): @@ -67,11 +76,7 @@ class Server: 'FreeBSD': '/var/run/log', 'Linux': '/dev/log', } - # todo: remove that, if test cases are fixed - self.setSyslogSocket("auto") - # Set logging level - self.setLogLevel("INFO") - self.setLogTarget("STDOUT") + self.__prev_signals = {} def __sigTERMhandler(self, signum, frame): logSys.debug("Caught signal %d. Exiting" % signum) @@ -93,9 +98,12 @@ class Server: raise ServerInitializationError("Could not create daemon") # Set all logging parameters (or use default if not specified): - self.setSyslogSocket(conf.get("syslogsocket", self.__syslogSocket)) - self.setLogLevel(conf.get("loglevel", self.__logLevel)) - self.setLogTarget(conf.get("logtarget", self.__logTarget)) + self.setSyslogSocket(conf.get("syslogsocket", + self.__syslogSocket if self.__syslogSocket is not None else DEF_SYSLOGSOCKET)) + self.setLogLevel(conf.get("loglevel", + self.__logLevel if self.__logLevel is not None else DEF_LOGLEVEL)) + self.setLogTarget(conf.get("logtarget", + self.__logTarget if self.__logTarget is not None else DEF_LOGTARGET)) logSys.info("-"*50) logSys.info("Starting Fail2ban v%s", version.version) @@ -104,10 +112,10 @@ class Server: logSys.info("Daemon started") # Install signal handlers - signal.signal(signal.SIGTERM, self.__sigTERMhandler) - signal.signal(signal.SIGINT, self.__sigTERMhandler) - signal.signal(signal.SIGUSR1, self.__sigUSR1handler) - + if _thread_name() == '_MainThread': + for s in (signal.SIGTERM, signal.SIGINT, signal.SIGUSR1): + self.__prev_signals[s] = signal.getsignal(s) + signal.signal(s, self.__sigTERMhandler if s != signal.SIGUSR1 else self.__sigUSR1handler) # Ensure unhandled exceptions are logged sys.excepthook = excepthook @@ -150,6 +158,10 @@ class Server: with self.__loggingLock: logging.shutdown() + # Restore default signal handlers: + for s, sh in self.__prev_signals.iteritems(): + signal.signal(s, sh) + def addJail(self, name, backend): self.__jails.add(name, backend, self.__db) if self.__db is not None: @@ -405,8 +417,13 @@ class Server: def setLogTarget(self, target): with self.__loggingLock: + # don't set new handlers if already the same + # or if "INHERITED" (foreground worker of the test cases, to prevent stop logging): if self.__logTarget == target: return True + if target == "INHERITED": + self.__logTarget = target + return True # set a format which is simpler for console use formatter = logging.Formatter("%(asctime)s %(name)-24s[%(process)d]: %(levelname)-7s %(message)s") if target == "SYSLOG": @@ -549,7 +566,10 @@ class Server: # We need to set this in the parent process, so it gets inherited by the # child process, and this makes sure that it is effect even if the parent # terminates quickly. - signal.signal(signal.SIGHUP, signal.SIG_IGN) + if _thread_name() == '_MainThread': + for s in (signal.SIGHUP,): + self.__prev_signals[s] = signal.getsignal(s) + signal.signal(s, signal.SIG_IGN) try: # Fork a child process so the parent can exit. This will return control diff --git a/fail2ban/tests/fail2banclienttestcase.py b/fail2ban/tests/fail2banclienttestcase.py new file mode 100644 index 00000000..7db48fe8 --- /dev/null +++ b/fail2ban/tests/fail2banclienttestcase.py @@ -0,0 +1,411 @@ +# emacs: -*- mode: python; py-indent-offset: 4; indent-tabs-mode: t -*- +# vi: set ft=python sts=4 ts=4 sw=4 noet : + +# This file is part of Fail2Ban. +# +# Fail2Ban is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. +# +# Fail2Ban is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with Fail2Ban; if not, write to the Free Software +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + +# Fail2Ban developers + +__author__ = "Serg Brester" +__copyright__ = "Copyright (c) 2014- Serg G. Brester (sebres), 2008- Fail2Ban Contributors" +__license__ = "GPL" + +import fileinput +import os +import re +import time +import unittest + +from threading import Thread + +from ..client import fail2banclient, fail2banserver, fail2bancmdline +from ..client.fail2banclient import Fail2banClient, exec_command_line as _exec_client, VisualWait +from ..client.fail2banserver import Fail2banServer, exec_command_line as _exec_server +from .. import protocol +from ..server import server +from ..server.utils import Utils +from .utils import LogCaptureTestCase, logSys, withtmpdir, shutil, logging + + +STOCK_CONF_DIR = "config" +STOCK = os.path.exists(os.path.join(STOCK_CONF_DIR,'fail2ban.conf')) +TEST_CONF_DIR = os.path.join(os.path.dirname(__file__), "config") +if STOCK: + CONF_DIR = STOCK_CONF_DIR +else: + CONF_DIR = TEST_CONF_DIR + +CLIENT = "fail2ban-client" +SERVER = "fail2ban-server" +BIN = os.path.join(os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))), "bin") + +MAX_WAITTIME = 10 +MAX_WAITTIME = unittest.F2B.maxWaitTime(MAX_WAITTIME) + +## +# Several wrappers and settings for proper testing: +# + +fail2banclient.MAX_WAITTIME = \ +fail2banserver.MAX_WAITTIME = MAX_WAITTIME + + +fail2bancmdline.logSys = \ +fail2banclient.logSys = \ +fail2banserver.logSys = logSys + +LOG_LEVEL = logSys.level + +server.DEF_LOGTARGET = "/dev/null" + +def _test_output(*args): + logSys.info(args[0]) +fail2bancmdline.output = \ +fail2banclient.output = \ +fail2banserver.output = \ +protocol.output = _test_output + +def _test_exit(code=0): + logSys.debug("Exit with code %s", code) + if code == 0: + raise ExitException() + else: + raise FailExitException() +fail2bancmdline.exit = \ +fail2banclient.exit = \ +fail2banserver.exit = _test_exit + +INTERACT = [] +def _test_raw_input(*args): + if len(INTERACT): + #print('--- interact command: ', INTERACT[0]) + return INTERACT.pop(0) + else: + return "exit" +fail2banclient.raw_input = _test_raw_input + +# prevents change logging params, log capturing, etc: +fail2bancmdline.PRODUCTION = False + + +class ExitException(fail2bancmdline.ExitException): + pass +class FailExitException(fail2bancmdline.ExitException): + pass + + +def _out_file(fn): # pragma: no cover + logSys.debug('---- ' + fn + ' ----') + for line in fileinput.input(fn): + line = line.rstrip('\n') + logSys.debug(line) + logSys.debug('-'*30) + +def _start_params(tmp, use_stock=False, logtarget="/dev/null"): + cfg = tmp+"/config" + if use_stock and STOCK: + # copy config: + def ig_dirs(dir, files): + return [f for f in files if not os.path.isfile(os.path.join(dir, f))] + shutil.copytree(STOCK_CONF_DIR, cfg, ignore=ig_dirs) + os.symlink(STOCK_CONF_DIR+"/action.d", cfg+"/action.d") + os.symlink(STOCK_CONF_DIR+"/filter.d", cfg+"/filter.d") + # replace fail2ban params (database with memory): + r = re.compile(r'^dbfile\s*=') + for line in fileinput.input(cfg+"/fail2ban.conf", inplace=True): + line = line.rstrip('\n') + if r.match(line): + line = "dbfile = :memory:" + print(line) + # replace jail params (polling as backend to be fast in initialize): + r = re.compile(r'^backend\s*=') + for line in fileinput.input(cfg+"/jail.conf", inplace=True): + line = line.rstrip('\n') + if r.match(line): + line = "backend = polling" + print(line) + else: + # just empty config directory without anything (only fail2ban.conf/jail.conf): + os.mkdir(cfg) + f = open(cfg+"/fail2ban.conf", "wb") + f.write('\n'.join(( + "[Definition]", + "loglevel = INFO", + "logtarget = " + logtarget, + "syslogsocket = auto", + "socket = "+tmp+"/f2b.sock", + "pidfile = "+tmp+"/f2b.pid", + "backend = polling", + "dbfile = :memory:", + "dbpurgeage = 1d", + "", + ))) + f.close() + f = open(cfg+"/jail.conf", "wb") + f.write('\n'.join(( + "[INCLUDES]", "", + "[DEFAULT]", "", + "", + ))) + f.close() + if LOG_LEVEL < logging.DEBUG: # if HEAVYDEBUG + _out_file(cfg+"/fail2ban.conf") + _out_file(cfg+"/jail.conf") + # parameters: + return ("-c", cfg, + "--logtarget", logtarget, "--loglevel", "DEBUG", "--syslogsocket", "auto", + "-s", tmp+"/f2b.sock", "-p", tmp+"/f2b.pid") + + +class Fail2banClientTest(LogCaptureTestCase): + + def setUp(self): + """Call before every test case.""" + LogCaptureTestCase.setUp(self) + + def tearDown(self): + """Call after every test case.""" + LogCaptureTestCase.tearDown(self) + + def testClientUsage(self): + self.assertRaises(ExitException, _exec_client, + (CLIENT, "-h",)) + self.assertLogged("Usage: " + CLIENT) + self.assertLogged("Report bugs to ") + + @withtmpdir + def testClientStartBackgroundInside(self, tmp): + # always add "--async" by start inside, should don't fork by async (not replace client with server, just start in new process) + # (we can't fork the test cases process): + startparams = _start_params(tmp, True) + # start: + self.assertRaises(ExitException, _exec_client, + (CLIENT, "--async", "-b") + startparams + ("start",)) + self.assertLogged("Server ready") + self.assertLogged("Exit with code 0") + try: + self.assertRaises(ExitException, _exec_client, + (CLIENT,) + startparams + ("echo", "TEST-ECHO",)) + self.assertRaises(FailExitException, _exec_client, + (CLIENT,) + startparams + ("~~unknown~cmd~failed~~",)) + finally: + self.pruneLog() + # stop: + self.assertRaises(ExitException, _exec_client, + (CLIENT,) + startparams + ("stop",)) + self.assertLogged("Shutdown successful") + self.assertLogged("Exit with code 0") + + @withtmpdir + def testClientStartBackgroundCall(self, tmp): + global INTERACT + startparams = _start_params(tmp) + # start (without async in new process): + cmd = os.path.join(os.path.join(BIN), CLIENT) + logSys.debug('Start %s ...', cmd) + Utils.executeCmd((cmd,) + startparams + ("start",), + timeout=MAX_WAITTIME, shell=False, output=False) + self.pruneLog() + try: + # echo from client (inside): + self.assertRaises(ExitException, _exec_client, + (CLIENT,) + startparams + ("echo", "TEST-ECHO",)) + self.assertLogged("TEST-ECHO") + self.assertLogged("Exit with code 0") + self.pruneLog() + # interactive client chat with started server: + INTERACT += [ + "echo INTERACT-ECHO", + "status", + "exit" + ] + self.assertRaises(ExitException, _exec_client, + (CLIENT,) + startparams + ("-i",)) + self.assertLogged("INTERACT-ECHO") + self.assertLogged("Status", "Number of jail:") + self.assertLogged("Exit with code 0") + self.pruneLog() + # test reload and restart over interactive client: + INTERACT += [ + "reload", + "restart", + "exit" + ] + self.assertRaises(ExitException, _exec_client, + (CLIENT,) + startparams + ("-i",)) + self.assertLogged("Reading config files:") + self.assertLogged("Shutdown successful") + self.assertLogged("Server ready") + self.assertLogged("Exit with code 0") + self.pruneLog() + finally: + self.pruneLog() + # stop: + self.assertRaises(ExitException, _exec_client, + (CLIENT,) + startparams + ("stop",)) + self.assertLogged("Shutdown successful") + self.assertLogged("Exit with code 0") + + def _testClientStartForeground(self, tmp, startparams, phase): + # start and wait to end (foreground): + phase['start'] = True + self.assertRaises(ExitException, _exec_client, + (CLIENT, "-f") + startparams + ("start",)) + # end : + phase['end'] = True + + @withtmpdir + def testClientStartForeground(self, tmp): + # started directly here, so prevent overwrite test cases logger with "INHERITED" + startparams = _start_params(tmp, logtarget="INHERITED") + # because foreground block execution - start it in thread: + phase = dict() + Thread(name="_TestCaseWorker", + target=Fail2banClientTest._testClientStartForeground, args=(self, tmp, startparams, phase)).start() + try: + # wait for start thread: + Utils.wait_for(lambda: phase.get('start', None) is not None, MAX_WAITTIME) + self.assertTrue(phase.get('start', None)) + # wait for server (socket): + Utils.wait_for(lambda: os.path.exists(tmp+"/f2b.sock"), MAX_WAITTIME) + self.assertLogged("Starting communication") + self.assertRaises(ExitException, _exec_client, + (CLIENT,) + startparams + ("ping",)) + self.assertRaises(FailExitException, _exec_client, + (CLIENT,) + startparams + ("~~unknown~cmd~failed~~",)) + self.assertRaises(ExitException, _exec_client, + (CLIENT,) + startparams + ("echo", "TEST-ECHO",)) + finally: + self.pruneLog() + # stop: + self.assertRaises(ExitException, _exec_client, + (CLIENT,) + startparams + ("stop",)) + # wait for end: + Utils.wait_for(lambda: phase.get('end', None) is not None, MAX_WAITTIME) + self.assertTrue(phase.get('end', None)) + self.assertLogged("Shutdown successful", "Exiting Fail2ban") + self.assertLogged("Exit with code 0") + + @withtmpdir + def testClientFailStart(self, tmp): + self.assertRaises(FailExitException, _exec_client, + (CLIENT, "--async", "-c", tmp+"/miss", "start",)) + self.assertLogged("Base configuration directory " + tmp+"/miss" + " does not exist") + + self.assertRaises(FailExitException, _exec_client, + (CLIENT, "--async", "-c", CONF_DIR, "-s", tmp+"/miss/f2b.sock", "start",)) + self.assertLogged("There is no directory " + tmp+"/miss" + " to contain the socket file") + + def testVisualWait(self): + sleeptime = 0.035 + for verbose in (2, 0): + cntr = 15 + with VisualWait(verbose, 5) as vis: + while cntr: + vis.heartbeat() + if verbose and not unittest.F2B.fast: + time.sleep(sleeptime) + cntr -= 1 + + +class Fail2banServerTest(LogCaptureTestCase): + + def setUp(self): + """Call before every test case.""" + LogCaptureTestCase.setUp(self) + + def tearDown(self): + """Call after every test case.""" + LogCaptureTestCase.tearDown(self) + + def testServerUsage(self): + self.assertRaises(ExitException, _exec_server, + (SERVER, "-h",)) + self.assertLogged("Usage: " + SERVER) + self.assertLogged("Report bugs to ") + + @withtmpdir + def testServerStartBackground(self, tmp): + # don't add "--async" by start, because if will fork current process by daemonize + # (we can't fork the test cases process), + # because server started internal communication in new thread use INHERITED as logtarget here: + startparams = _start_params(tmp, logtarget="INHERITED") + # start: + self.assertRaises(ExitException, _exec_server, + (SERVER, "-b") + startparams) + self.assertLogged("Server ready") + self.assertLogged("Exit with code 0") + try: + self.assertRaises(ExitException, _exec_server, + (SERVER,) + startparams + ("echo", "TEST-ECHO",)) + self.assertRaises(FailExitException, _exec_server, + (SERVER,) + startparams + ("~~unknown~cmd~failed~~",)) + finally: + self.pruneLog() + # stop: + self.assertRaises(ExitException, _exec_server, + (SERVER,) + startparams + ("stop",)) + self.assertLogged("Shutdown successful") + self.assertLogged("Exit with code 0") + + def _testServerStartForeground(self, tmp, startparams, phase): + # start and wait to end (foreground): + phase['start'] = True + self.assertRaises(ExitException, _exec_server, + (SERVER, "-f") + startparams + ("start",)) + # end : + phase['end'] = True + @withtmpdir + def testServerStartForeground(self, tmp): + # started directly here, so prevent overwrite test cases logger with "INHERITED" + startparams = _start_params(tmp, logtarget="INHERITED") + # because foreground block execution - start it in thread: + phase = dict() + Thread(name="_TestCaseWorker", + target=Fail2banServerTest._testServerStartForeground, args=(self, tmp, startparams, phase)).start() + try: + # wait for start thread: + Utils.wait_for(lambda: phase.get('start', None) is not None, MAX_WAITTIME) + self.assertTrue(phase.get('start', None)) + # wait for server (socket): + Utils.wait_for(lambda: os.path.exists(tmp+"/f2b.sock"), MAX_WAITTIME) + self.assertLogged("Starting communication") + self.assertRaises(ExitException, _exec_server, + (SERVER,) + startparams + ("ping",)) + self.assertRaises(FailExitException, _exec_server, + (SERVER,) + startparams + ("~~unknown~cmd~failed~~",)) + self.assertRaises(ExitException, _exec_server, + (SERVER,) + startparams + ("echo", "TEST-ECHO",)) + finally: + self.pruneLog() + # stop: + self.assertRaises(ExitException, _exec_server, + (SERVER,) + startparams + ("stop",)) + # wait for end: + Utils.wait_for(lambda: phase.get('end', None) is not None, MAX_WAITTIME) + self.assertTrue(phase.get('end', None)) + self.assertLogged("Shutdown successful", "Exiting Fail2ban") + self.assertLogged("Exit with code 0") + + @withtmpdir + def testServerFailStart(self, tmp): + self.assertRaises(FailExitException, _exec_server, + (SERVER, "-c", tmp+"/miss",)) + self.assertLogged("Base configuration directory " + tmp+"/miss" + " does not exist") + + self.assertRaises(FailExitException, _exec_server, + (SERVER, "-c", CONF_DIR, "-s", tmp+"/miss/f2b.sock",)) + self.assertLogged("There is no directory " + tmp+"/miss" + " to contain the socket file") diff --git a/fail2ban/tests/fail2banregextestcase.py b/fail2ban/tests/fail2banregextestcase.py index 49d6a3a6..d9f4081f 100644 --- a/fail2ban/tests/fail2banregextestcase.py +++ b/fail2ban/tests/fail2banregextestcase.py @@ -23,19 +23,7 @@ __author__ = "Serg Brester" __copyright__ = "Copyright (c) 2015 Serg G. Brester (sebres), 2008- Fail2Ban Contributors" __license__ = "GPL" -from __builtin__ import open as fopen -import unittest -import getpass import os -import sys -import time -import tempfile -import uuid - -try: - from systemd import journal -except ImportError: - journal = None from ..client import fail2banregex from ..client.fail2banregex import Fail2banRegex, get_opt_parser, output diff --git a/fail2ban/tests/utils.py b/fail2ban/tests/utils.py index a0036979..e373aa5f 100644 --- a/fail2ban/tests/utils.py +++ b/fail2ban/tests/utils.py @@ -26,10 +26,14 @@ import logging import optparse import os import re +import tempfile +import shutil import sys import time import unittest + from StringIO import StringIO +from functools import wraps from ..helpers import getLogger from ..server.ipdns import DNSUtils @@ -71,6 +75,17 @@ class F2B(optparse.Values): return wtime +def withtmpdir(f): + @wraps(f) + def wrapper(self, *args, **kwargs): + tmp = tempfile.mkdtemp(prefix="f2b-temp") + try: + return f(self, tmp, *args, **kwargs) + finally: + # clean up + shutil.rmtree(tmp) + return wrapper + def initTests(opts): unittest.F2B = F2B(opts) # --fast : @@ -156,6 +171,7 @@ def gatherTests(regexps=None, opts=None): from . import misctestcase from . import databasetestcase from . import samplestestcase + from . import fail2banclienttestcase from . import fail2banregextestcase if not regexps: # pragma: no cover @@ -239,6 +255,9 @@ def gatherTests(regexps=None, opts=None): # Filter Regex tests with sample logs tests.addTest(unittest.makeSuite(samplestestcase.FilterSamplesRegex)) + # bin/fail2ban-client, bin/fail2ban-server + tests.addTest(unittest.makeSuite(fail2banclienttestcase.Fail2banClientTest)) + tests.addTest(unittest.makeSuite(fail2banclienttestcase.Fail2banServerTest)) # bin/fail2ban-regex tests.addTest(unittest.makeSuite(fail2banregextestcase.Fail2banRegexTest)) @@ -321,8 +340,11 @@ class LogCaptureTestCase(unittest.TestCase): # Let's log everything into a string self._log = StringIO() logSys.handlers = [logging.StreamHandler(self._log)] + if self._old_level <= logging.DEBUG: + print("") if self._old_level < logging.DEBUG: # so if HEAVYDEBUG etc -- show them! logSys.handlers += self._old_handlers + logSys.debug('--'*40) logSys.setLevel(getattr(logging, 'DEBUG')) def tearDown(self): @@ -386,6 +408,9 @@ class LogCaptureTestCase(unittest.TestCase): def pruneLog(self): self._log.truncate(0) + def pruneLog(self): + self._log.truncate(0) + def getLog(self): return self._log.getvalue() From 2fcb6358ff1958fd0029dd8c9280f4d87a1aa909 Mon Sep 17 00:00:00 2001 From: sebres Date: Thu, 11 Feb 2016 21:15:03 +0100 Subject: [PATCH 05/13] several bug fixed: fork in client-server test cases prohibited, all worker threads daemonized (to prevent hanging on exit). --- fail2ban/client/fail2banclient.py | 27 +- fail2ban/client/fail2banserver.py | 11 +- fail2ban/server/jailthread.py | 2 + fail2ban/server/server.py | 6 +- fail2ban/tests/action_d/test_smtp.py | 1 + fail2ban/tests/fail2banclienttestcase.py | 373 ++++++++++++++--------- fail2ban/tests/utils.py | 3 +- 7 files changed, 255 insertions(+), 168 deletions(-) diff --git a/fail2ban/client/fail2banclient.py b/fail2ban/client/fail2banclient.py index 736f8fd2..4f4caf79 100755 --- a/fail2ban/client/fail2banclient.py +++ b/fail2ban/client/fail2banclient.py @@ -34,7 +34,7 @@ from threading import Thread from ..version import version from .csocket import CSocket from .beautifier import Beautifier -from .fail2bancmdline import Fail2banCmdLine, ExitException, logSys, exit, output +from .fail2bancmdline import Fail2banCmdLine, ExitException, PRODUCTION, logSys, exit, output MAX_WAITTIME = 30 @@ -108,8 +108,13 @@ class Fail2banClient(Fail2banCmdLine, Thread): logSys.error(e) return False finally: + # prevent errors by close during shutdown (on exit command): if client: - client.close() + try : + client.close() + except Exception as e: + if showRet or self._conf["verbose"] > 1: + logSys.debug(e) if showRet or c[0] == 'echo': sys.stdout.flush() return streamRet @@ -184,7 +189,9 @@ class Fail2banClient(Fail2banCmdLine, Thread): return False else: # In foreground mode we should make server/client communication in different threads: - Thread(target=Fail2banClient.__processStartStreamAfterWait, args=(self, stream, False)).start() + th = Thread(target=Fail2banClient.__processStartStreamAfterWait, args=(self, stream, False)) + th.daemon = True + th.start() # Mark current (main) thread as daemon: self.setDaemon(True) # Start server direct here in main thread (not fork): @@ -197,8 +204,6 @@ class Fail2banClient(Fail2banCmdLine, Thread): logSys.error("Exception while starting server " + ("background" if background else "foreground")) logSys.error(e) return False - finally: - self._alive = False return True @@ -206,7 +211,9 @@ class Fail2banClient(Fail2banCmdLine, Thread): def configureServer(self, async=True, phase=None): # if asynchron start this operation in the new thread: if async: - return Thread(target=Fail2banClient.configureServer, args=(self, False, phase)).start() + th = Thread(target=Fail2banClient.configureServer, args=(self, False, phase)) + th.daemon = True + return th.start() # prepare: read config, check configuration is valid, etc.: if phase is not None: phase['start'] = True @@ -290,7 +297,6 @@ class Fail2banClient(Fail2banCmdLine, Thread): "server, adding the -x option will do it") if self._server: self._server.quit() - exit(-1) return False return True @@ -299,10 +305,12 @@ class Fail2banClient(Fail2banCmdLine, Thread): maxtime = MAX_WAITTIME # Wait for the server to start (the server has 30 seconds to answer ping) starttime = time.time() + logSys.debug("__waitOnServer: %r", (alive, maxtime)) with VisualWait(self._conf["verbose"]) as vis: - while self._alive and not self.__ping() == alive or ( + while self._alive and ( + not self.__ping() == alive or ( not alive and os.path.exists(self._conf["socket"]) - ): + )): now = time.time() # Wonderful visual :) if now > starttime + 1: @@ -365,6 +373,7 @@ class Fail2banClient(Fail2banCmdLine, Thread): return False return self.__processCommand(args) finally: + self._alive = False for s, sh in _prev_signals.iteritems(): signal.signal(s, sh) diff --git a/fail2ban/client/fail2banserver.py b/fail2ban/client/fail2banserver.py index da8e57b8..ac927251 100644 --- a/fail2ban/client/fail2banserver.py +++ b/fail2ban/client/fail2banserver.py @@ -25,15 +25,14 @@ import os import sys from ..version import version -from ..server.server import Server, ServerDaemonize +from ..server.server import Server from ..server.utils import Utils -from .fail2bancmdline import Fail2banCmdLine, logSys, exit +from .fail2bancmdline import Fail2banCmdLine, logSys, PRODUCTION, exit MAX_WAITTIME = 30 SERVER = "fail2ban-server" - ## # \mainpage Fail2Ban # @@ -51,6 +50,7 @@ class Fail2banServer(Fail2banCmdLine): @staticmethod def startServerDirect(conf, daemon=True): + logSys.debug("-- direct starting of server in %s, deamon: %s", os.getpid(), daemon) server = None try: # Start it in foreground (current thread, not new process), @@ -59,8 +59,6 @@ class Fail2banServer(Fail2banCmdLine): server.start(conf["socket"], conf["pidfile"], conf["force"], conf=conf) - except ServerDaemonize: - pass except Exception, e: logSys.exception(e) if server: @@ -82,9 +80,10 @@ class Fail2banServer(Fail2banCmdLine): startdir = os.path.dirname(sys.argv[0]) # Forks the current process, don't fork if async specified (ex: test cases) pid = 0 - frk = not conf["async"] + frk = not conf["async"] and PRODUCTION if frk: pid = os.fork() + logSys.debug("-- async starting of server in %s, fork: %s - %s", os.getpid(), frk, pid) if pid == 0: args = list() args.append(SERVER) diff --git a/fail2ban/server/jailthread.py b/fail2ban/server/jailthread.py index eb43e453..39a86c2b 100644 --- a/fail2ban/server/jailthread.py +++ b/fail2ban/server/jailthread.py @@ -51,6 +51,8 @@ class JailThread(Thread): def __init__(self, name=None): super(JailThread, self).__init__(name=name) + ## Should going with main thread also: + self.daemon = True ## Control the state of the thread. self.active = False ## Control the idle state of the thread. diff --git a/fail2ban/server/server.py b/fail2ban/server/server.py index d4786f29..9c970b49 100644 --- a/fail2ban/server/server.py +++ b/fail2ban/server/server.py @@ -351,6 +351,9 @@ class Server: def getBanTime(self, name): return self.__jails[name].actions.getBanTime() + def isStarted(self): + self.__asyncServer.isActive() + def isAlive(self, jailnum=None): if jailnum is not None and len(self.__jails) != jailnum: return 0 @@ -643,6 +646,3 @@ class Server: class ServerInitializationError(Exception): pass - -class ServerDaemonize(Exception): - pass \ No newline at end of file diff --git a/fail2ban/tests/action_d/test_smtp.py b/fail2ban/tests/action_d/test_smtp.py index 37fe0138..1385fe82 100644 --- a/fail2ban/tests/action_d/test_smtp.py +++ b/fail2ban/tests/action_d/test_smtp.py @@ -65,6 +65,7 @@ class SMTPActionTest(unittest.TestCase): self._active = True self._loop_thread = threading.Thread( target=asyncserver.loop, kwargs={'active': lambda: self._active}) + self._loop_thread.daemon = True self._loop_thread.start() def tearDown(self): diff --git a/fail2ban/tests/fail2banclienttestcase.py b/fail2ban/tests/fail2banclienttestcase.py index 7db48fe8..fd8a074b 100644 --- a/fail2ban/tests/fail2banclienttestcase.py +++ b/fail2ban/tests/fail2banclienttestcase.py @@ -98,7 +98,9 @@ def _test_raw_input(*args): fail2banclient.raw_input = _test_raw_input # prevents change logging params, log capturing, etc: -fail2bancmdline.PRODUCTION = False +fail2bancmdline.PRODUCTION = \ +fail2banclient.PRODUCTION = \ +fail2banserver.PRODUCTION = False class ExitException(fail2bancmdline.ExitException): @@ -117,9 +119,9 @@ def _out_file(fn): # pragma: no cover def _start_params(tmp, use_stock=False, logtarget="/dev/null"): cfg = tmp+"/config" if use_stock and STOCK: - # copy config: + # copy config (sub-directories as alias): def ig_dirs(dir, files): - return [f for f in files if not os.path.isfile(os.path.join(dir, f))] + return [f for f in files if os.path.isdir(os.path.join(dir, f))] shutil.copytree(STOCK_CONF_DIR, cfg, ignore=ig_dirs) os.symlink(STOCK_CONF_DIR+"/action.d", cfg+"/action.d") os.symlink(STOCK_CONF_DIR+"/filter.d", cfg+"/filter.d") @@ -169,6 +171,47 @@ def _start_params(tmp, use_stock=False, logtarget="/dev/null"): "--logtarget", logtarget, "--loglevel", "DEBUG", "--syslogsocket", "auto", "-s", tmp+"/f2b.sock", "-p", tmp+"/f2b.pid") +def _kill_srv(pidfile): # pragma: no cover + def _pid_exists(pid): + try: + os.kill(pid, 0) + return True + except OSError: + return False + logSys.debug("-- cleanup: %r", (pidfile, os.path.isdir(pidfile))) + if os.path.isdir(pidfile): + piddir = pidfile + pidfile = piddir + "/f2b.pid" + if not os.path.isfile(pidfile): + pidfile = piddir + "/fail2ban.pid" + if not os.path.isfile(pidfile): + logSys.debug("--- cleanup: no pidfile for %r", piddir) + return True + f = pid = None + try: + logSys.debug("--- cleanup pidfile: %r", pidfile) + f = open(pidfile) + pid = f.read().split()[1] + pid = int(pid) + logSys.debug("--- cleanup pid: %r", pid) + if pid <= 0: + raise ValueError('pid %s of %s is invalid' % (pid, pidfile)) + if not _pid_exists(pid): + return True + ## try to preper stop (have signal handler): + os.kill(pid, signal.SIGTERM) + ## check still exists after small timeout: + if not Utils.wait_for(lambda: not _pid_exists(pid), MAX_WAITTIME / 3): + ## try to kill hereafter: + os.kill(pid, signal.SIGKILL) + return not _pid_exists(pid) + except Exception as e: + sysLog.debug(e) + finally: + if f is not None: + f.close() + return True + class Fail2banClientTest(LogCaptureTestCase): @@ -188,126 +231,144 @@ class Fail2banClientTest(LogCaptureTestCase): @withtmpdir def testClientStartBackgroundInside(self, tmp): - # always add "--async" by start inside, should don't fork by async (not replace client with server, just start in new process) - # (we can't fork the test cases process): - startparams = _start_params(tmp, True) - # start: - self.assertRaises(ExitException, _exec_client, - (CLIENT, "--async", "-b") + startparams + ("start",)) - self.assertLogged("Server ready") - self.assertLogged("Exit with code 0") try: + # always add "--async" by start inside, should don't fork by async (not replace client with server, just start in new process) + # (we can't fork the test cases process): + startparams = _start_params(tmp, True) + # start: self.assertRaises(ExitException, _exec_client, - (CLIENT,) + startparams + ("echo", "TEST-ECHO",)) - self.assertRaises(FailExitException, _exec_client, - (CLIENT,) + startparams + ("~~unknown~cmd~failed~~",)) - finally: - self.pruneLog() - # stop: - self.assertRaises(ExitException, _exec_client, - (CLIENT,) + startparams + ("stop",)) - self.assertLogged("Shutdown successful") + (CLIENT, "--async", "-b") + startparams + ("start",)) + self.assertLogged("Server ready") self.assertLogged("Exit with code 0") + try: + self.assertRaises(ExitException, _exec_client, + (CLIENT,) + startparams + ("echo", "TEST-ECHO",)) + self.assertRaises(FailExitException, _exec_client, + (CLIENT,) + startparams + ("~~unknown~cmd~failed~~",)) + finally: + self.pruneLog() + # stop: + self.assertRaises(ExitException, _exec_client, + (CLIENT,) + startparams + ("stop",)) + self.assertLogged("Shutdown successful") + self.assertLogged("Exit with code 0") + finally: + _kill_srv(tmp) @withtmpdir def testClientStartBackgroundCall(self, tmp): - global INTERACT - startparams = _start_params(tmp) - # start (without async in new process): - cmd = os.path.join(os.path.join(BIN), CLIENT) - logSys.debug('Start %s ...', cmd) - Utils.executeCmd((cmd,) + startparams + ("start",), - timeout=MAX_WAITTIME, shell=False, output=False) - self.pruneLog() try: - # echo from client (inside): - self.assertRaises(ExitException, _exec_client, - (CLIENT,) + startparams + ("echo", "TEST-ECHO",)) - self.assertLogged("TEST-ECHO") - self.assertLogged("Exit with code 0") - self.pruneLog() - # interactive client chat with started server: - INTERACT += [ - "echo INTERACT-ECHO", - "status", - "exit" - ] - self.assertRaises(ExitException, _exec_client, - (CLIENT,) + startparams + ("-i",)) - self.assertLogged("INTERACT-ECHO") - self.assertLogged("Status", "Number of jail:") - self.assertLogged("Exit with code 0") - self.pruneLog() - # test reload and restart over interactive client: - INTERACT += [ - "reload", - "restart", - "exit" - ] - self.assertRaises(ExitException, _exec_client, - (CLIENT,) + startparams + ("-i",)) - self.assertLogged("Reading config files:") - self.assertLogged("Shutdown successful") - self.assertLogged("Server ready") - self.assertLogged("Exit with code 0") + global INTERACT + startparams = _start_params(tmp) + # start (without async in new process): + cmd = os.path.join(os.path.join(BIN), CLIENT) + logSys.debug('Start %s ...', cmd) + Utils.executeCmd((cmd,) + startparams + ("start",), + timeout=MAX_WAITTIME, shell=False, output=False) self.pruneLog() + try: + # echo from client (inside): + self.assertRaises(ExitException, _exec_client, + (CLIENT,) + startparams + ("echo", "TEST-ECHO",)) + self.assertLogged("TEST-ECHO") + self.assertLogged("Exit with code 0") + self.pruneLog() + # interactive client chat with started server: + INTERACT += [ + "echo INTERACT-ECHO", + "status", + "exit" + ] + self.assertRaises(ExitException, _exec_client, + (CLIENT,) + startparams + ("-i",)) + self.assertLogged("INTERACT-ECHO") + self.assertLogged("Status", "Number of jail:") + self.assertLogged("Exit with code 0") + self.pruneLog() + # test reload and restart over interactive client: + INTERACT += [ + "reload", + "restart", + "exit" + ] + self.assertRaises(ExitException, _exec_client, + (CLIENT,) + startparams + ("-i",)) + self.assertLogged("Reading config files:") + self.assertLogged("Shutdown successful") + self.assertLogged("Server ready") + self.assertLogged("Exit with code 0") + self.pruneLog() + finally: + self.pruneLog() + # stop: + self.assertRaises(ExitException, _exec_client, + (CLIENT,) + startparams + ("stop",)) + self.assertLogged("Shutdown successful") + self.assertLogged("Exit with code 0") finally: - self.pruneLog() - # stop: - self.assertRaises(ExitException, _exec_client, - (CLIENT,) + startparams + ("stop",)) - self.assertLogged("Shutdown successful") - self.assertLogged("Exit with code 0") + _kill_srv(tmp) def _testClientStartForeground(self, tmp, startparams, phase): # start and wait to end (foreground): + logSys.debug("-- start of test worker") phase['start'] = True self.assertRaises(ExitException, _exec_client, (CLIENT, "-f") + startparams + ("start",)) # end : phase['end'] = True + logSys.debug("-- end of test worker") @withtmpdir def testClientStartForeground(self, tmp): - # started directly here, so prevent overwrite test cases logger with "INHERITED" - startparams = _start_params(tmp, logtarget="INHERITED") - # because foreground block execution - start it in thread: - phase = dict() - Thread(name="_TestCaseWorker", - target=Fail2banClientTest._testClientStartForeground, args=(self, tmp, startparams, phase)).start() + th = None try: - # wait for start thread: - Utils.wait_for(lambda: phase.get('start', None) is not None, MAX_WAITTIME) - self.assertTrue(phase.get('start', None)) - # wait for server (socket): - Utils.wait_for(lambda: os.path.exists(tmp+"/f2b.sock"), MAX_WAITTIME) - self.assertLogged("Starting communication") - self.assertRaises(ExitException, _exec_client, - (CLIENT,) + startparams + ("ping",)) - self.assertRaises(FailExitException, _exec_client, - (CLIENT,) + startparams + ("~~unknown~cmd~failed~~",)) - self.assertRaises(ExitException, _exec_client, - (CLIENT,) + startparams + ("echo", "TEST-ECHO",)) + # started directly here, so prevent overwrite test cases logger with "INHERITED" + startparams = _start_params(tmp, logtarget="INHERITED") + # because foreground block execution - start it in thread: + phase = dict() + th = Thread(name="_TestCaseWorker", + target=Fail2banClientTest._testClientStartForeground, args=(self, tmp, startparams, phase)) + th.daemon = True + th.start() + try: + # wait for start thread: + Utils.wait_for(lambda: phase.get('start', None) is not None, MAX_WAITTIME) + self.assertTrue(phase.get('start', None)) + # wait for server (socket): + Utils.wait_for(lambda: os.path.exists(tmp+"/f2b.sock"), MAX_WAITTIME) + self.assertLogged("Starting communication") + self.assertRaises(ExitException, _exec_client, + (CLIENT,) + startparams + ("ping",)) + self.assertRaises(FailExitException, _exec_client, + (CLIENT,) + startparams + ("~~unknown~cmd~failed~~",)) + self.assertRaises(ExitException, _exec_client, + (CLIENT,) + startparams + ("echo", "TEST-ECHO",)) + finally: + self.pruneLog() + # stop: + self.assertRaises(ExitException, _exec_client, + (CLIENT,) + startparams + ("stop",)) + # wait for end: + Utils.wait_for(lambda: phase.get('end', None) is not None, MAX_WAITTIME) + self.assertTrue(phase.get('end', None)) + self.assertLogged("Shutdown successful", "Exiting Fail2ban") finally: - self.pruneLog() - # stop: - self.assertRaises(ExitException, _exec_client, - (CLIENT,) + startparams + ("stop",)) - # wait for end: - Utils.wait_for(lambda: phase.get('end', None) is not None, MAX_WAITTIME) - self.assertTrue(phase.get('end', None)) - self.assertLogged("Shutdown successful", "Exiting Fail2ban") - self.assertLogged("Exit with code 0") + _kill_srv(tmp) + if th: + th.join() @withtmpdir def testClientFailStart(self, tmp): - self.assertRaises(FailExitException, _exec_client, - (CLIENT, "--async", "-c", tmp+"/miss", "start",)) - self.assertLogged("Base configuration directory " + tmp+"/miss" + " does not exist") + try: + self.assertRaises(FailExitException, _exec_client, + (CLIENT, "--async", "-c", tmp+"/miss", "start",)) + self.assertLogged("Base configuration directory " + tmp+"/miss" + " does not exist") - self.assertRaises(FailExitException, _exec_client, - (CLIENT, "--async", "-c", CONF_DIR, "-s", tmp+"/miss/f2b.sock", "start",)) - self.assertLogged("There is no directory " + tmp+"/miss" + " to contain the socket file") + self.assertRaises(FailExitException, _exec_client, + (CLIENT, "--async", "-c", CONF_DIR, "-s", tmp+"/miss/f2b.sock", "start",)) + self.assertLogged("There is no directory " + tmp+"/miss" + " to contain the socket file") + finally: + _kill_srv(tmp) def testVisualWait(self): sleeptime = 0.035 @@ -339,73 +400,89 @@ class Fail2banServerTest(LogCaptureTestCase): @withtmpdir def testServerStartBackground(self, tmp): - # don't add "--async" by start, because if will fork current process by daemonize - # (we can't fork the test cases process), - # because server started internal communication in new thread use INHERITED as logtarget here: - startparams = _start_params(tmp, logtarget="INHERITED") - # start: - self.assertRaises(ExitException, _exec_server, - (SERVER, "-b") + startparams) - self.assertLogged("Server ready") - self.assertLogged("Exit with code 0") try: + # don't add "--async" by start, because if will fork current process by daemonize + # (we can't fork the test cases process), + # because server started internal communication in new thread use INHERITED as logtarget here: + startparams = _start_params(tmp, logtarget="INHERITED") + # start: self.assertRaises(ExitException, _exec_server, - (SERVER,) + startparams + ("echo", "TEST-ECHO",)) - self.assertRaises(FailExitException, _exec_server, - (SERVER,) + startparams + ("~~unknown~cmd~failed~~",)) - finally: - self.pruneLog() - # stop: - self.assertRaises(ExitException, _exec_server, - (SERVER,) + startparams + ("stop",)) - self.assertLogged("Shutdown successful") + (SERVER, "-b") + startparams) + self.assertLogged("Server ready") self.assertLogged("Exit with code 0") + try: + self.assertRaises(ExitException, _exec_server, + (SERVER,) + startparams + ("echo", "TEST-ECHO",)) + self.assertRaises(FailExitException, _exec_server, + (SERVER,) + startparams + ("~~unknown~cmd~failed~~",)) + finally: + self.pruneLog() + # stop: + self.assertRaises(ExitException, _exec_server, + (SERVER,) + startparams + ("stop",)) + self.assertLogged("Shutdown successful") + self.assertLogged("Exit with code 0") + finally: + _kill_srv(tmp) def _testServerStartForeground(self, tmp, startparams, phase): # start and wait to end (foreground): + logSys.debug("-- start of test worker") phase['start'] = True self.assertRaises(ExitException, _exec_server, (SERVER, "-f") + startparams + ("start",)) # end : phase['end'] = True + logSys.debug("-- end of test worker") + @withtmpdir def testServerStartForeground(self, tmp): - # started directly here, so prevent overwrite test cases logger with "INHERITED" - startparams = _start_params(tmp, logtarget="INHERITED") - # because foreground block execution - start it in thread: - phase = dict() - Thread(name="_TestCaseWorker", - target=Fail2banServerTest._testServerStartForeground, args=(self, tmp, startparams, phase)).start() + th = None try: - # wait for start thread: - Utils.wait_for(lambda: phase.get('start', None) is not None, MAX_WAITTIME) - self.assertTrue(phase.get('start', None)) - # wait for server (socket): - Utils.wait_for(lambda: os.path.exists(tmp+"/f2b.sock"), MAX_WAITTIME) - self.assertLogged("Starting communication") - self.assertRaises(ExitException, _exec_server, - (SERVER,) + startparams + ("ping",)) - self.assertRaises(FailExitException, _exec_server, - (SERVER,) + startparams + ("~~unknown~cmd~failed~~",)) - self.assertRaises(ExitException, _exec_server, - (SERVER,) + startparams + ("echo", "TEST-ECHO",)) + # started directly here, so prevent overwrite test cases logger with "INHERITED" + startparams = _start_params(tmp, logtarget="INHERITED") + # because foreground block execution - start it in thread: + phase = dict() + th = Thread(name="_TestCaseWorker", + target=Fail2banServerTest._testServerStartForeground, args=(self, tmp, startparams, phase)) + th.daemon = True + th.start() + try: + # wait for start thread: + Utils.wait_for(lambda: phase.get('start', None) is not None, MAX_WAITTIME) + self.assertTrue(phase.get('start', None)) + # wait for server (socket): + Utils.wait_for(lambda: os.path.exists(tmp+"/f2b.sock"), MAX_WAITTIME) + self.assertLogged("Starting communication") + self.assertRaises(ExitException, _exec_server, + (SERVER,) + startparams + ("ping",)) + self.assertRaises(FailExitException, _exec_server, + (SERVER,) + startparams + ("~~unknown~cmd~failed~~",)) + self.assertRaises(ExitException, _exec_server, + (SERVER,) + startparams + ("echo", "TEST-ECHO",)) + finally: + self.pruneLog() + # stop: + self.assertRaises(ExitException, _exec_server, + (SERVER,) + startparams + ("stop",)) + # wait for end: + Utils.wait_for(lambda: phase.get('end', None) is not None, MAX_WAITTIME) + self.assertTrue(phase.get('end', None)) + self.assertLogged("Shutdown successful", "Exiting Fail2ban") finally: - self.pruneLog() - # stop: - self.assertRaises(ExitException, _exec_server, - (SERVER,) + startparams + ("stop",)) - # wait for end: - Utils.wait_for(lambda: phase.get('end', None) is not None, MAX_WAITTIME) - self.assertTrue(phase.get('end', None)) - self.assertLogged("Shutdown successful", "Exiting Fail2ban") - self.assertLogged("Exit with code 0") + _kill_srv(tmp) + if th: + th.join() @withtmpdir def testServerFailStart(self, tmp): - self.assertRaises(FailExitException, _exec_server, - (SERVER, "-c", tmp+"/miss",)) - self.assertLogged("Base configuration directory " + tmp+"/miss" + " does not exist") + try: + self.assertRaises(FailExitException, _exec_server, + (SERVER, "-c", tmp+"/miss",)) + self.assertLogged("Base configuration directory " + tmp+"/miss" + " does not exist") - self.assertRaises(FailExitException, _exec_server, - (SERVER, "-c", CONF_DIR, "-s", tmp+"/miss/f2b.sock",)) - self.assertLogged("There is no directory " + tmp+"/miss" + " to contain the socket file") + self.assertRaises(FailExitException, _exec_server, + (SERVER, "-c", CONF_DIR, "-s", tmp+"/miss/f2b.sock",)) + self.assertLogged("There is no directory " + tmp+"/miss" + " to contain the socket file") + finally: + _kill_srv(tmp) diff --git a/fail2ban/tests/utils.py b/fail2ban/tests/utils.py index e373aa5f..61bd8abf 100644 --- a/fail2ban/tests/utils.py +++ b/fail2ban/tests/utils.py @@ -340,9 +340,8 @@ class LogCaptureTestCase(unittest.TestCase): # Let's log everything into a string self._log = StringIO() logSys.handlers = [logging.StreamHandler(self._log)] - if self._old_level <= logging.DEBUG: + if self._old_level <= logging.DEBUG: # so if DEBUG etc -- show them (and log it in travis)! print("") - if self._old_level < logging.DEBUG: # so if HEAVYDEBUG etc -- show them! logSys.handlers += self._old_handlers logSys.debug('--'*40) logSys.setLevel(getattr(logging, 'DEBUG')) From 0b4143730de2ef252984e4a7bc1310db8331b618 Mon Sep 17 00:00:00 2001 From: sebres Date: Fri, 12 Feb 2016 21:51:32 +0100 Subject: [PATCH 06/13] some compatibility fixes (prevent forking of testcase-process, code review), wait 4 server ready, test cases fixed (py2/py3) --- fail2ban/client/fail2banclient.py | 80 ++++++++++------ fail2ban/client/fail2bancmdline.py | 8 +- fail2ban/client/fail2banserver.py | 108 +++++++++++++-------- fail2ban/server/server.py | 36 ++++--- fail2ban/tests/fail2banclienttestcase.py | 116 +++++++++++++---------- fail2ban/tests/utils.py | 6 +- 6 files changed, 219 insertions(+), 135 deletions(-) diff --git a/fail2ban/client/fail2banclient.py b/fail2ban/client/fail2banclient.py index 4f4caf79..ad5cc57e 100755 --- a/fail2ban/client/fail2banclient.py +++ b/fail2ban/client/fail2banclient.py @@ -34,14 +34,18 @@ from threading import Thread from ..version import version from .csocket import CSocket from .beautifier import Beautifier -from .fail2bancmdline import Fail2banCmdLine, ExitException, PRODUCTION, logSys, exit, output +from .fail2bancmdline import Fail2banCmdLine, ServerExecutionException, ExitException, \ + logSys, PRODUCTION, exit, output MAX_WAITTIME = 30 +PROMPT = "fail2ban> " def _thread_name(): return threading.current_thread().__class__.__name__ +def input_command(): + return raw_input(PROMPT) ## # @@ -49,8 +53,6 @@ def _thread_name(): class Fail2banClient(Fail2banCmdLine, Thread): - PROMPT = "fail2ban> " - def __init__(self): Fail2banCmdLine.__init__(self) Thread.__init__(self) @@ -91,11 +93,11 @@ class Fail2banClient(Fail2banCmdLine, Thread): client = CSocket(self._conf["socket"]) ret = client.send(c) if ret[0] == 0: - logSys.debug("OK : " + `ret[1]`) + logSys.debug("OK : %r", ret[1]) if showRet or c[0] == 'echo': output(beautifier.beautify(ret[1])) else: - logSys.error("NOK: " + `ret[1].args`) + logSys.error("NOK: %r", ret[1].args) if showRet: output(beautifier.beautifyError(ret[1])) streamRet = False @@ -202,7 +204,10 @@ class Fail2banClient(Fail2banCmdLine, Thread): except Exception as e: output("") logSys.error("Exception while starting server " + ("background" if background else "foreground")) - logSys.error(e) + if self._conf["verbose"] > 1: + logSys.exception(e) + else: + logSys.error(e) return False return True @@ -249,7 +254,9 @@ class Fail2banClient(Fail2banCmdLine, Thread): if self._conf.get("interactive", False): output(' ## stop ... ') self.__processCommand(['stop']) - self.__waitOnServer(False) + if not self.__waitOnServer(False): + logSys.error("Could not stop server") + return False # in interactive mode reset config, to make full-reload if there something changed: if self._conf.get("interactive", False): output(' ## load configuration ... ') @@ -286,10 +293,14 @@ class Fail2banClient(Fail2banCmdLine, Thread): def __processStartStreamAfterWait(self, *args): try: # Wait for the server to start - self.__waitOnServer() + if not self.__waitOnServer(): + logSys.error("Could not find server, waiting failed") + return False # Configure the server self.__processCmd(*args) - except ServerExecutionException: + except ServerExecutionException as e: + if self._conf["verbose"] > 1: + logSys.exception(e) logSys.error("Could not start server. Maybe an old " "socket file is still present. Try to " "remove " + self._conf["socket"] + ". If " @@ -306,11 +317,13 @@ class Fail2banClient(Fail2banCmdLine, Thread): # Wait for the server to start (the server has 30 seconds to answer ping) starttime = time.time() logSys.debug("__waitOnServer: %r", (alive, maxtime)) + test = lambda: os.path.exists(self._conf["socket"]) and self.__ping() with VisualWait(self._conf["verbose"]) as vis: - while self._alive and ( - not self.__ping() == alive or ( - not alive and os.path.exists(self._conf["socket"]) - )): + sltime = 0.0125 / 2 + while self._alive: + runf = test() + if runf == alive: + return True now = time.time() # Wonderful visual :) if now > starttime + 1: @@ -318,7 +331,9 @@ class Fail2banClient(Fail2banCmdLine, Thread): # f end time reached: if now - starttime >= maxtime: raise ServerExecutionException("Failed to start server") - time.sleep(0.1) + sltime = min(sltime * 2, 0.5) + time.sleep(sltime) + return False def start(self, argv): # Install signal handlers @@ -332,27 +347,31 @@ class Fail2banClient(Fail2banCmdLine, Thread): if self._argv is None: ret = self.initCmdLine(argv) if ret is not None: - return ret + if ret: + return True + raise ServerExecutionException("Init of command line failed") # Commands args = self._args # Interactive mode if self._conf.get("interactive", False): - try: - import readline - except ImportError: - logSys.error("Readline not available") - return False + # no readline in test: + if PRODUCTION: # pragma: no cover + try: + import readline + except ImportError: + raise ServerExecutionException("Readline not available") try: ret = True if len(args) > 0: ret = self.__processCommand(args) if ret: - readline.parse_and_bind("tab: complete") + if PRODUCTION: # pragma: no cover + readline.parse_and_bind("tab: complete") self.dispInteractive() while True: - cmd = raw_input(self.PROMPT) + cmd = input_command() if cmd == "exit" or cmd == "quit": # Exit return True @@ -362,26 +381,31 @@ class Fail2banClient(Fail2banCmdLine, Thread): try: self.__processCommand(shlex.split(cmd)) except Exception, e: - logSys.error(e) + if self._conf["verbose"] > 1: + logSys.exception(e) + else: + logSys.error(e) except (EOFError, KeyboardInterrupt): output("") - return True + raise # Single command mode else: if len(args) < 1: self.dispUsage() return False return self.__processCommand(args) + except Exception as e: + if self._conf["verbose"] > 1: + logSys.exception(e) + else: + logSys.error(e) + return False finally: self._alive = False for s, sh in _prev_signals.iteritems(): signal.signal(s, sh) -class ServerExecutionException(Exception): - pass - - ## # Wonderful visual :) # diff --git a/fail2ban/client/fail2bancmdline.py b/fail2ban/client/fail2bancmdline.py index 2ed6a499..781d55fc 100644 --- a/fail2ban/client/fail2bancmdline.py +++ b/fail2ban/client/fail2bancmdline.py @@ -261,5 +261,9 @@ class Fail2banCmdLine(): exit = Fail2banCmdLine.exit -class ExitException: - pass \ No newline at end of file +class ExitException(Exception): + pass + + +class ServerExecutionException(Exception): + pass diff --git a/fail2ban/client/fail2banserver.py b/fail2ban/client/fail2banserver.py index ac927251..73e528ca 100644 --- a/fail2ban/client/fail2banserver.py +++ b/fail2ban/client/fail2banserver.py @@ -24,10 +24,8 @@ __license__ = "GPL" import os import sys -from ..version import version -from ..server.server import Server -from ..server.utils import Utils -from .fail2bancmdline import Fail2banCmdLine, logSys, PRODUCTION, exit +from .fail2bancmdline import Fail2banCmdLine, ServerExecutionException, \ + logSys, PRODUCTION, exit MAX_WAITTIME = 30 @@ -44,13 +42,14 @@ class Fail2banServer(Fail2banCmdLine): # Fail2banCmdLine.__init__(self) ## - # Start Fail2Ban server in main thread without fork (foreground). + # Start Fail2Ban server in main thread without fork (direct, it can fork itself in Server if daemon=True). # - # Start the Fail2ban server in foreground (daemon mode or not). + # Start the Fail2ban server in background/foreground (daemon mode or not). @staticmethod def startServerDirect(conf, daemon=True): logSys.debug("-- direct starting of server in %s, deamon: %s", os.getpid(), daemon) + from ..server.server import Server server = None try: # Start it in foreground (current thread, not new process), @@ -59,11 +58,14 @@ class Fail2banServer(Fail2banCmdLine): server.start(conf["socket"], conf["pidfile"], conf["force"], conf=conf) - except Exception, e: - logSys.exception(e) - if server: - server.quit() - exit(-1) + except Exception as e: + try: + if server: + server.quit() + except Exception as e2: + if conf["verbose"] > 1: + logSys.exception(e2) + raise return server @@ -74,10 +76,6 @@ class Fail2banServer(Fail2banCmdLine): @staticmethod def startServerAsync(conf): - # Directory of client (to try the first start from the same directory as client): - startdir = sys.path[0] - if startdir in ("", "."): # may be uresolved in test-cases, so get bin-directory: - startdir = os.path.dirname(sys.argv[0]) # Forks the current process, don't fork if async specified (ex: test cases) pid = 0 frk = not conf["async"] and PRODUCTION @@ -103,25 +101,43 @@ class Fail2banServer(Fail2banCmdLine): for o in ('loglevel', 'logtarget', 'syslogsocket'): args.append("--"+o) args.append(conf[o]) - try: - # Use the current directory. - exe = os.path.abspath(os.path.join(startdir, SERVER)) + # Directory of client (to try the first start from current or the same directory as client, and from relative bin): + exe = Fail2banServer.getServerPath() + if not frk: + # Wrapr args to use the same python version in client/server (important for multi-python systems): + args[0] = exe + exe = sys.executable + args[0:0] = [exe] logSys.debug("Starting %r with args %r", exe, args) if frk: - os.execv(exe, args) + return os.execv(exe, args) else: - os.spawnv(os.P_NOWAITO, exe, args) + # use P_WAIT instead of P_NOWAIT (to prevent defunct-zomby process), it startet as daemon, so parent exit fast after fork): + return os.spawnv(os.P_WAIT, exe, args) except OSError as e: - try: - # Use the PATH env. - logSys.warning("Initial start attempt failed (%s). Starting %r with the same args", e, SERVER) - if frk: - os.execvp(SERVER, args) - else: - os.spawnvp(os.P_NOWAITO, SERVER, args) - except OSError: - exit(-1) + # Use the PATH env. + logSys.warning("Initial start attempt failed (%s). Starting %r with the same args", e, SERVER) + if frk: + return os.execvp(SERVER, args) + else: + del args[0] + args[0] = SERVER + return os.spawnvp(os.P_WAIT, SERVER, args) + return pid + + @staticmethod + def getServerPath(): + startdir = sys.path[0] + exe = os.path.abspath(os.path.join(startdir, SERVER)) + if not os.path.isfile(exe): # may be uresolved in test-cases, so get relative starter (client): + startdir = os.path.dirname(sys.argv[0]) + exe = os.path.abspath(os.path.join(startdir, SERVER)) + if not os.path.isfile(exe): # may be uresolved in test-cases, so try to get relative bin-directory: + startdir = os.path.dirname(os.path.abspath(__file__)) + startdir = os.path.join(os.path.dirname(os.path.dirname(startdir)), "bin") + exe = os.path.abspath(os.path.join(startdir, SERVER)) + return exe def _Fail2banClient(self): from .fail2banclient import Fail2banClient @@ -139,18 +155,24 @@ class Fail2banServer(Fail2banCmdLine): args = self._args cli = None - # If client mode - whole processing over client: - if len(args) or self._conf.get("interactive", False): - cli = self._Fail2banClient() - return cli.start(argv) + # Just start: + if len(args) == 1 and args[0] == 'start' and not self._conf.get("interactive", False): + pass + else: + # If client mode - whole processing over client: + if len(args) or self._conf.get("interactive", False): + cli = self._Fail2banClient() + return cli.start(argv) # Start the server: server = None try: - # async = True, if started from client, should fork, daemonize, etc... - # background = True, if should start in new process, otherwise start in foreground - async = self._conf.get("async", False) + from ..server.utils import Utils + # background = True, if should be new process running in background, otherwise start in foreground + # process will be forked in daemonize, inside of Server module. + # async = True, if started from client, should... background = self._conf["background"] + async = self._conf.get("async", False) # If was started not from the client: if not async: # Start new thread with client to read configuration and @@ -162,13 +184,14 @@ class Fail2banServer(Fail2banCmdLine): # wait up to MAX_WAITTIME, do not continue if configuration is not 100% valid: Utils.wait_for(lambda: phase.get('ready', None) is not None, MAX_WAITTIME) if not phase.get('start', False): - return False + raise ServerExecutionException('Async configuration of server failed') # Start server, daemonize it, etc. - if async or not background: - server = Fail2banServer.startServerDirect(self._conf, background) - else: - Fail2banServer.startServerAsync(self._conf) + pid = os.getpid() + server = Fail2banServer.startServerDirect(self._conf, background) + # If forked - just exit other processes + if pid != os.getpid(): + os._exit(0) if cli: cli._server = server @@ -182,7 +205,8 @@ class Fail2banServer(Fail2banCmdLine): logSys.debug('Starting server done') except Exception, e: - logSys.exception(e) + if self._conf["verbose"] > 1: + logSys.exception(e) if server: server.quit() exit(-1) diff --git a/fail2ban/server/server.py b/fail2ban/server/server.py index 9c970b49..fa6ca43d 100644 --- a/fail2ban/server/server.py +++ b/fail2ban/server/server.py @@ -93,9 +93,15 @@ class Server: if self.__daemon: # pragma: no cover logSys.info("Starting in daemon mode") ret = self.__createDaemon() - if not ret: - logSys.error("Could not create daemon") - raise ServerInitializationError("Could not create daemon") + # If forked parent - return here (parent process will configure server later): + if ret is None: + return False + # If error: + if not ret[0]: + err = "Could not create daemon %s", ret[1:] + logSys.error(err) + raise ServerInitializationError(err) + # We are daemon. # Set all logging parameters (or use default if not specified): self.setSyslogSocket(conf.get("syslogsocket", @@ -159,8 +165,12 @@ class Server: logging.shutdown() # Restore default signal handlers: - for s, sh in self.__prev_signals.iteritems(): - signal.signal(s, sh) + if _thread_name() == '_MainThread': + for s, sh in self.__prev_signals.iteritems(): + signal.signal(s, sh) + + # Prevent to call quit twice: + self.quit = lambda: False def addJail(self, name, backend): self.__jails.add(name, backend, self.__db) @@ -569,10 +579,9 @@ class Server: # We need to set this in the parent process, so it gets inherited by the # child process, and this makes sure that it is effect even if the parent # terminates quickly. - if _thread_name() == '_MainThread': - for s in (signal.SIGHUP,): - self.__prev_signals[s] = signal.getsignal(s) - signal.signal(s, signal.SIG_IGN) + for s in (signal.SIGHUP,): + self.__prev_signals[s] = signal.getsignal(s) + signal.signal(s, signal.SIG_IGN) try: # Fork a child process so the parent can exit. This will return control @@ -583,7 +592,7 @@ class Server: # PGID. pid = os.fork() except OSError, e: - return((e.errno, e.strerror)) # ERROR (return a tuple) + return (False, (e.errno, e.strerror)) # ERROR (return a tuple) if pid == 0: # The first child. @@ -604,7 +613,7 @@ class Server: # preventing the daemon from ever acquiring a controlling terminal. pid = os.fork() # Fork a second child. except OSError, e: - return((e.errno, e.strerror)) # ERROR (return a tuple) + return (False, (e.errno, e.strerror)) # ERROR (return a tuple) if (pid == 0): # The second child. # Ensure that the daemon doesn't keep any directory in use. Failure @@ -613,7 +622,8 @@ class Server: else: os._exit(0) # Exit parent (the first child) of the second child. else: - os._exit(0) # Exit parent of the first child. + # Signal to exit, parent of the first child. + return None # Close all open files. Try the system configuration variable, SC_OPEN_MAX, # for the maximum number of open files to close. If it doesn't exist, use @@ -641,7 +651,7 @@ class Server: os.open("/dev/null", os.O_RDONLY) # standard input (0) os.open("/dev/null", os.O_RDWR) # standard output (1) os.open("/dev/null", os.O_RDWR) # standard error (2) - return True + return (True,) class ServerInitializationError(Exception): diff --git a/fail2ban/tests/fail2banclienttestcase.py b/fail2ban/tests/fail2banclienttestcase.py index fd8a074b..6fee732b 100644 --- a/fail2ban/tests/fail2banclienttestcase.py +++ b/fail2ban/tests/fail2banclienttestcase.py @@ -26,6 +26,7 @@ __license__ = "GPL" import fileinput import os import re +import sys import time import unittest @@ -50,10 +51,9 @@ else: CLIENT = "fail2ban-client" SERVER = "fail2ban-server" -BIN = os.path.join(os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))), "bin") +BIN = os.path.dirname(Fail2banServer.getServerPath()) -MAX_WAITTIME = 10 -MAX_WAITTIME = unittest.F2B.maxWaitTime(MAX_WAITTIME) +MAX_WAITTIME = 30 if not unittest.F2B.fast else 5 ## # Several wrappers and settings for proper testing: @@ -67,8 +67,6 @@ fail2bancmdline.logSys = \ fail2banclient.logSys = \ fail2banserver.logSys = logSys -LOG_LEVEL = logSys.level - server.DEF_LOGTARGET = "/dev/null" def _test_output(*args): @@ -89,13 +87,13 @@ fail2banclient.exit = \ fail2banserver.exit = _test_exit INTERACT = [] -def _test_raw_input(*args): +def _test_input_command(*args): if len(INTERACT): - #print('--- interact command: ', INTERACT[0]) + #logSys.debug('--- interact command: %r', INTERACT[0]) return INTERACT.pop(0) else: return "exit" -fail2banclient.raw_input = _test_raw_input +fail2banclient.input_command = _test_input_command # prevents change logging params, log capturing, etc: fail2bancmdline.PRODUCTION = \ @@ -142,7 +140,7 @@ def _start_params(tmp, use_stock=False, logtarget="/dev/null"): else: # just empty config directory without anything (only fail2ban.conf/jail.conf): os.mkdir(cfg) - f = open(cfg+"/fail2ban.conf", "wb") + f = open(cfg+"/fail2ban.conf", "w") f.write('\n'.join(( "[Definition]", "loglevel = INFO", @@ -156,20 +154,20 @@ def _start_params(tmp, use_stock=False, logtarget="/dev/null"): "", ))) f.close() - f = open(cfg+"/jail.conf", "wb") + f = open(cfg+"/jail.conf", "w") f.write('\n'.join(( "[INCLUDES]", "", "[DEFAULT]", "", "", ))) f.close() - if LOG_LEVEL < logging.DEBUG: # if HEAVYDEBUG + if logSys.level < logging.DEBUG: # if HEAVYDEBUG _out_file(cfg+"/fail2ban.conf") _out_file(cfg+"/jail.conf") - # parameters: - return ("-c", cfg, - "--logtarget", logtarget, "--loglevel", "DEBUG", "--syslogsocket", "auto", - "-s", tmp+"/f2b.sock", "-p", tmp+"/f2b.pid") + # parameters (sock/pid and config, increase verbosity, set log, etc.): + return ("-c", cfg, "-s", tmp+"/f2b.sock", "-p", tmp+"/f2b.pid", + "-vv", "--logtarget", logtarget, "--loglevel", "DEBUG", "--syslogsocket", "auto", + ) def _kill_srv(pidfile): # pragma: no cover def _pid_exists(pid): @@ -206,14 +204,14 @@ def _kill_srv(pidfile): # pragma: no cover os.kill(pid, signal.SIGKILL) return not _pid_exists(pid) except Exception as e: - sysLog.debug(e) + logSys.debug(e) finally: if f is not None: f.close() return True -class Fail2banClientTest(LogCaptureTestCase): +class Fail2banClientServerBase(LogCaptureTestCase): def setUp(self): """Call before every test case.""" @@ -223,6 +221,21 @@ class Fail2banClientTest(LogCaptureTestCase): """Call after every test case.""" LogCaptureTestCase.tearDown(self) + def _wait_for_srv(self, tmp, ready=True, startparams=None): + sock = tmp+"/f2b.sock" + # wait for server (socket): + ret = Utils.wait_for(lambda: os.path.exists(sock), MAX_WAITTIME) + if not ret: + raise Exception('Unexpected: Socket file does not exists.\nStart failed: %r' % (startparams,)) + if ready: + # wait for communication with worker ready: + ret = Utils.wait_for(lambda: "Server ready" in self.getLog(), MAX_WAITTIME) + if not ret: + raise Exception('Unexpected: Server ready was not found.\nStart failed: %r' % (startparams,)) + + +class Fail2banClientTest(Fail2banClientServerBase): + def testClientUsage(self): self.assertRaises(ExitException, _exec_client, (CLIENT, "-h",)) @@ -232,12 +245,13 @@ class Fail2banClientTest(LogCaptureTestCase): @withtmpdir def testClientStartBackgroundInside(self, tmp): try: - # always add "--async" by start inside, should don't fork by async (not replace client with server, just start in new process) - # (we can't fork the test cases process): + # use once the stock configuration (to test starting also) startparams = _start_params(tmp, True) # start: self.assertRaises(ExitException, _exec_client, - (CLIENT, "--async", "-b") + startparams + ("start",)) + (CLIENT, "-b") + startparams + ("start",)) + # wait for server (socket and ready): + self._wait_for_srv(tmp, True, startparams=startparams) self.assertLogged("Server ready") self.assertLogged("Exit with code 0") try: @@ -245,6 +259,11 @@ class Fail2banClientTest(LogCaptureTestCase): (CLIENT,) + startparams + ("echo", "TEST-ECHO",)) self.assertRaises(FailExitException, _exec_client, (CLIENT,) + startparams + ("~~unknown~cmd~failed~~",)) + self.pruneLog() + # start again (should fail): + self.assertRaises(FailExitException, _exec_client, + (CLIENT, "-b") + startparams + ("start",)) + self.assertLogged("Server already running") finally: self.pruneLog() # stop: @@ -260,11 +279,14 @@ class Fail2banClientTest(LogCaptureTestCase): try: global INTERACT startparams = _start_params(tmp) - # start (without async in new process): - cmd = os.path.join(os.path.join(BIN), CLIENT) + # start (in new process, using the same python version): + cmd = (sys.executable, os.path.join(os.path.join(BIN), CLIENT)) logSys.debug('Start %s ...', cmd) - Utils.executeCmd((cmd,) + startparams + ("start",), - timeout=MAX_WAITTIME, shell=False, output=False) + cmd = cmd + startparams + ("start",) + Utils.executeCmd(cmd, timeout=MAX_WAITTIME, shell=False, output=True) + # wait for server (socket and ready): + self._wait_for_srv(tmp, True, startparams=cmd) + self.assertLogged("Server ready") self.pruneLog() try: # echo from client (inside): @@ -312,7 +334,7 @@ class Fail2banClientTest(LogCaptureTestCase): # start and wait to end (foreground): logSys.debug("-- start of test worker") phase['start'] = True - self.assertRaises(ExitException, _exec_client, + self.assertRaises(fail2bancmdline.ExitException, _exec_client, (CLIENT, "-f") + startparams + ("start",)) # end : phase['end'] = True @@ -334,9 +356,10 @@ class Fail2banClientTest(LogCaptureTestCase): # wait for start thread: Utils.wait_for(lambda: phase.get('start', None) is not None, MAX_WAITTIME) self.assertTrue(phase.get('start', None)) - # wait for server (socket): - Utils.wait_for(lambda: os.path.exists(tmp+"/f2b.sock"), MAX_WAITTIME) - self.assertLogged("Starting communication") + # wait for server (socket and ready): + self._wait_for_srv(tmp, True, startparams=startparams) + self.pruneLog() + # several commands to server: self.assertRaises(ExitException, _exec_client, (CLIENT,) + startparams + ("ping",)) self.assertRaises(FailExitException, _exec_client, @@ -382,15 +405,7 @@ class Fail2banClientTest(LogCaptureTestCase): cntr -= 1 -class Fail2banServerTest(LogCaptureTestCase): - - def setUp(self): - """Call before every test case.""" - LogCaptureTestCase.setUp(self) - - def tearDown(self): - """Call after every test case.""" - LogCaptureTestCase.tearDown(self) +class Fail2banServerTest(Fail2banClientServerBase): def testServerUsage(self): self.assertRaises(ExitException, _exec_server, @@ -401,15 +416,17 @@ class Fail2banServerTest(LogCaptureTestCase): @withtmpdir def testServerStartBackground(self, tmp): try: - # don't add "--async" by start, because if will fork current process by daemonize - # (we can't fork the test cases process), - # because server started internal communication in new thread use INHERITED as logtarget here: - startparams = _start_params(tmp, logtarget="INHERITED") - # start: - self.assertRaises(ExitException, _exec_server, - (SERVER, "-b") + startparams) + # to prevent fork of test-cases process, start server in background via command: + startparams = _start_params(tmp) + # start (in new process, using the same python version): + cmd = (sys.executable, os.path.join(os.path.join(BIN), SERVER)) + logSys.debug('Start %s ...', cmd) + cmd = cmd + startparams + ("-b",) + Utils.executeCmd(cmd, timeout=MAX_WAITTIME, shell=False, output=True) + # wait for server (socket and ready): + self._wait_for_srv(tmp, True, startparams=cmd) self.assertLogged("Server ready") - self.assertLogged("Exit with code 0") + self.pruneLog() try: self.assertRaises(ExitException, _exec_server, (SERVER,) + startparams + ("echo", "TEST-ECHO",)) @@ -429,7 +446,7 @@ class Fail2banServerTest(LogCaptureTestCase): # start and wait to end (foreground): logSys.debug("-- start of test worker") phase['start'] = True - self.assertRaises(ExitException, _exec_server, + self.assertRaises(fail2bancmdline.ExitException, _exec_server, (SERVER, "-f") + startparams + ("start",)) # end : phase['end'] = True @@ -451,9 +468,10 @@ class Fail2banServerTest(LogCaptureTestCase): # wait for start thread: Utils.wait_for(lambda: phase.get('start', None) is not None, MAX_WAITTIME) self.assertTrue(phase.get('start', None)) - # wait for server (socket): - Utils.wait_for(lambda: os.path.exists(tmp+"/f2b.sock"), MAX_WAITTIME) - self.assertLogged("Starting communication") + # wait for server (socket and ready): + self._wait_for_srv(tmp, True, startparams=startparams) + self.pruneLog() + # several commands to server: self.assertRaises(ExitException, _exec_server, (SERVER,) + startparams + ("ping",)) self.assertRaises(FailExitException, _exec_server, diff --git a/fail2ban/tests/utils.py b/fail2ban/tests/utils.py index 61bd8abf..2d096276 100644 --- a/fail2ban/tests/utils.py +++ b/fail2ban/tests/utils.py @@ -54,6 +54,10 @@ if not CONFIG_DIR: else: CONFIG_DIR = '/etc/fail2ban' +# In not installed env (setup, test-cases) use fail2ban modules from main directory: +if 1 or os.environ.get('PYTHONPATH', None) is None: + os.putenv('PYTHONPATH', os.path.dirname(os.path.dirname(os.path.dirname( + os.path.abspath(__file__))))) class F2B(optparse.Values): def __init__(self, opts={}): @@ -343,7 +347,7 @@ class LogCaptureTestCase(unittest.TestCase): if self._old_level <= logging.DEBUG: # so if DEBUG etc -- show them (and log it in travis)! print("") logSys.handlers += self._old_handlers - logSys.debug('--'*40) + logSys.debug('='*10 + ' %s ' + '='*20, self.id()) logSys.setLevel(getattr(logging, 'DEBUG')) def tearDown(self): From 22576d715089839da0720eedefcb4b49673ecf8f Mon Sep 17 00:00:00 2001 From: sebres Date: Mon, 15 Feb 2016 19:19:31 +0100 Subject: [PATCH 07/13] code review, timeout fix, better tracing (and test coverage) by start of server/client (with or without fork) --- fail2ban/client/fail2banclient.py | 21 ++-- fail2ban/client/fail2bancmdline.py | 22 +++-- fail2ban/client/fail2banserver.py | 21 ++-- fail2ban/tests/fail2banclienttestcase.py | 116 ++++++++++++++++++----- 4 files changed, 129 insertions(+), 51 deletions(-) diff --git a/fail2ban/client/fail2banclient.py b/fail2ban/client/fail2banclient.py index ad5cc57e..23d31dc5 100755 --- a/fail2ban/client/fail2banclient.py +++ b/fail2ban/client/fail2banclient.py @@ -37,14 +37,13 @@ from .beautifier import Beautifier from .fail2bancmdline import Fail2banCmdLine, ServerExecutionException, ExitException, \ logSys, PRODUCTION, exit, output -MAX_WAITTIME = 30 PROMPT = "fail2ban> " def _thread_name(): return threading.current_thread().__class__.__name__ -def input_command(): +def input_command(): # pragma: no cover return raw_input(PROMPT) ## @@ -101,13 +100,19 @@ class Fail2banClient(Fail2banCmdLine, Thread): if showRet: output(beautifier.beautifyError(ret[1])) streamRet = False - except socket.error: + except socket.error as e: if showRet or self._conf["verbose"] > 1: - self.__logSocketError() + if showRet or c != ["ping"]: + self.__logSocketError() + else: + logSys.debug(" -- ping failed -- %r", e) return False - except Exception, e: + except Exception as e: # pragma: no cover if showRet or self._conf["verbose"] > 1: - logSys.error(e) + if self._conf["verbose"] > 1: + logSys.exception(e) + else: + logSys.error(e) return False finally: # prevent errors by close during shutdown (on exit command): @@ -123,7 +128,7 @@ class Fail2banClient(Fail2banCmdLine, Thread): def __logSocketError(self): try: - if os.access(self._conf["socket"], os.F_OK): + if os.access(self._conf["socket"], os.F_OK): # pragma: no cover # This doesn't check if path is a socket, # but socket.error should be raised if os.access(self._conf["socket"], os.W_OK): @@ -313,7 +318,7 @@ class Fail2banClient(Fail2banCmdLine, Thread): def __waitOnServer(self, alive=True, maxtime=None): if maxtime is None: - maxtime = MAX_WAITTIME + maxtime = self._conf["timeout"] # Wait for the server to start (the server has 30 seconds to answer ping) starttime = time.time() logSys.debug("__waitOnServer: %r", (alive, maxtime)) diff --git a/fail2ban/client/fail2bancmdline.py b/fail2ban/client/fail2bancmdline.py index 781d55fc..d49f0e3d 100644 --- a/fail2ban/client/fail2bancmdline.py +++ b/fail2ban/client/fail2bancmdline.py @@ -33,13 +33,14 @@ from ..helpers import getLogger # Gets the instance of the logger. logSys = getLogger("fail2ban") -def output(s): +def output(s): # pragma: no cover print(s) CONFIG_PARAMS = ("socket", "pidfile", "logtarget", "loglevel", "syslogsocket",) # Used to signal - we are in test cases (ex: prevents change logging params, log capturing, etc) PRODUCTION = True +MAX_WAITTIME = 30 class Fail2banCmdLine(): @@ -56,7 +57,8 @@ class Fail2banCmdLine(): "background": True, "verbose": 1, "socket": None, - "pidfile": None + "pidfile": None, + "timeout": MAX_WAITTIME } @property @@ -109,6 +111,7 @@ class Fail2banCmdLine(): output(" -b start server in background (default)") output(" -f start server in foreground") output(" --async start server in async mode (for internal usage only, don't read configuration)") + output(" --timeout timeout to wait for the server (for internal usage only, don't read configuration)") output(" -h, --help display this help message") output(" -V, --version print the version") @@ -126,8 +129,6 @@ class Fail2banCmdLine(): """ for opt in optList: o = opt[0] - if o == "--async": - self._conf["async"] = True if o == "-c": self._conf["conf"] = opt[1] elif o == "-s": @@ -150,6 +151,11 @@ class Fail2banCmdLine(): self._conf["background"] = True elif o == "-f": self._conf["background"] = False + elif o == "--async": + self._conf["async"] = True + elif o == "-timeout": + from ..mytime import MyTime + self._conf["timeout"] = MyTime.str2seconds(opt[1]) elif o in ["-h", "--help"]: self.dispUsage() return True @@ -170,7 +176,7 @@ class Fail2banCmdLine(): # Reads the command line options. try: cmdOpts = 'hc:s:p:xfbdviqV' - cmdLongOpts = ['loglevel=', 'logtarget=', 'syslogsocket=', 'async', 'help', 'version'] + cmdLongOpts = ['loglevel=', 'logtarget=', 'syslogsocket=', 'async', 'timeout=', 'help', 'version'] optList, self._args = getopt.getopt(self._argv[1:], cmdOpts, cmdLongOpts) except getopt.GetoptError: self.dispUsage() @@ -180,6 +186,8 @@ class Fail2banCmdLine(): if ret is not None: return ret + logSys.debug("-- conf: %r, args: %r", self._conf, self._args) + if initial and PRODUCTION: # pragma: no cover - can't test verbose = self._conf["verbose"] if verbose <= 0: @@ -246,11 +254,11 @@ class Fail2banCmdLine(): @staticmethod def dumpConfig(cmd): for c in cmd: - print c + output(c) return True @staticmethod - def exit(code=0): + def exit(code=0): # pragma: no cover - can't test logSys.debug("Exit with code %s", code) if os._exit: os._exit(code) diff --git a/fail2ban/client/fail2banserver.py b/fail2ban/client/fail2banserver.py index 73e528ca..a6c8a7c5 100644 --- a/fail2ban/client/fail2banserver.py +++ b/fail2ban/client/fail2banserver.py @@ -27,8 +27,6 @@ import sys from .fail2bancmdline import Fail2banCmdLine, ServerExecutionException, \ logSys, PRODUCTION, exit -MAX_WAITTIME = 30 - SERVER = "fail2ban-server" ## @@ -114,16 +112,17 @@ class Fail2banServer(Fail2banCmdLine): return os.execv(exe, args) else: # use P_WAIT instead of P_NOWAIT (to prevent defunct-zomby process), it startet as daemon, so parent exit fast after fork): - return os.spawnv(os.P_WAIT, exe, args) - except OSError as e: + ret = os.spawnv(os.P_WAIT, exe, args) + if ret != 0: + raise OSError(ret, "Unknown error by executing server %r with %r" % (args[1], exe)) + return 0 + except OSError as e: # pragma: no cover + if not frk: #not PRODUCTION: + raise # Use the PATH env. logSys.warning("Initial start attempt failed (%s). Starting %r with the same args", e, SERVER) if frk: return os.execvp(SERVER, args) - else: - del args[0] - args[0] = SERVER - return os.spawnvp(os.P_WAIT, SERVER, args) return pid @staticmethod @@ -181,8 +180,8 @@ class Fail2banServer(Fail2banCmdLine): phase = dict() logSys.debug('Configure via async client thread') cli.configureServer(async=True, phase=phase) - # wait up to MAX_WAITTIME, do not continue if configuration is not 100% valid: - Utils.wait_for(lambda: phase.get('ready', None) is not None, MAX_WAITTIME) + # wait, do not continue if configuration is not 100% valid: + Utils.wait_for(lambda: phase.get('ready', None) is not None, self._conf["timeout"]) if not phase.get('start', False): raise ServerExecutionException('Async configuration of server failed') @@ -197,7 +196,7 @@ class Fail2banServer(Fail2banCmdLine): # wait for client answer "done": if not async and cli: - Utils.wait_for(lambda: phase.get('done', None) is not None, MAX_WAITTIME) + Utils.wait_for(lambda: phase.get('done', None) is not None, self._conf["timeout"]) if not phase.get('done', False): if server: server.quit() diff --git a/fail2ban/tests/fail2banclienttestcase.py b/fail2ban/tests/fail2banclienttestcase.py index 6fee732b..aa507fe6 100644 --- a/fail2ban/tests/fail2banclienttestcase.py +++ b/fail2ban/tests/fail2banclienttestcase.py @@ -43,11 +43,6 @@ from .utils import LogCaptureTestCase, logSys, withtmpdir, shutil, logging STOCK_CONF_DIR = "config" STOCK = os.path.exists(os.path.join(STOCK_CONF_DIR,'fail2ban.conf')) -TEST_CONF_DIR = os.path.join(os.path.dirname(__file__), "config") -if STOCK: - CONF_DIR = STOCK_CONF_DIR -else: - CONF_DIR = TEST_CONF_DIR CLIENT = "fail2ban-client" SERVER = "fail2ban-server" @@ -59,9 +54,7 @@ MAX_WAITTIME = 30 if not unittest.F2B.fast else 5 # Several wrappers and settings for proper testing: # -fail2banclient.MAX_WAITTIME = \ -fail2banserver.MAX_WAITTIME = MAX_WAITTIME - +fail2bancmdline.MAX_WAITTIME = MAX_WAITTIME-1 fail2bancmdline.logSys = \ fail2banclient.logSys = \ @@ -167,6 +160,7 @@ def _start_params(tmp, use_stock=False, logtarget="/dev/null"): # parameters (sock/pid and config, increase verbosity, set log, etc.): return ("-c", cfg, "-s", tmp+"/f2b.sock", "-p", tmp+"/f2b.pid", "-vv", "--logtarget", logtarget, "--loglevel", "DEBUG", "--syslogsocket", "auto", + "--timeout", str(fail2bancmdline.MAX_WAITTIME), ) def _kill_srv(pidfile): # pragma: no cover @@ -199,7 +193,7 @@ def _kill_srv(pidfile): # pragma: no cover ## try to preper stop (have signal handler): os.kill(pid, signal.SIGTERM) ## check still exists after small timeout: - if not Utils.wait_for(lambda: not _pid_exists(pid), MAX_WAITTIME / 3): + if not Utils.wait_for(lambda: not _pid_exists(pid), 1): ## try to kill hereafter: os.kill(pid, signal.SIGKILL) return not _pid_exists(pid) @@ -222,25 +216,50 @@ class Fail2banClientServerBase(LogCaptureTestCase): LogCaptureTestCase.tearDown(self) def _wait_for_srv(self, tmp, ready=True, startparams=None): - sock = tmp+"/f2b.sock" - # wait for server (socket): - ret = Utils.wait_for(lambda: os.path.exists(sock), MAX_WAITTIME) - if not ret: - raise Exception('Unexpected: Socket file does not exists.\nStart failed: %r' % (startparams,)) - if ready: - # wait for communication with worker ready: - ret = Utils.wait_for(lambda: "Server ready" in self.getLog(), MAX_WAITTIME) + try: + sock = tmp+"/f2b.sock" + # wait for server (socket): + ret = Utils.wait_for(lambda: os.path.exists(sock), MAX_WAITTIME) if not ret: - raise Exception('Unexpected: Server ready was not found.\nStart failed: %r' % (startparams,)) + raise Exception('Unexpected: Socket file does not exists.\nStart failed: %r' % (startparams,)) + if ready: + # wait for communication with worker ready: + ret = Utils.wait_for(lambda: "Server ready" in self.getLog(), MAX_WAITTIME) + if not ret: + raise Exception('Unexpected: Server ready was not found.\nStart failed: %r' % (startparams,)) + except: # pragma: no cover + log = tmp+"/f2b.log" + if os.path.isfile(log): + _out_file(log) + else: + logSys.debug("No log file %s to examine details of error", log) + raise class Fail2banClientTest(Fail2banClientServerBase): + def testConsistency(self): + self.assertTrue(os.path.isfile(os.path.join(os.path.join(BIN), CLIENT))) + self.assertTrue(os.path.isfile(os.path.join(os.path.join(BIN), SERVER))) + def testClientUsage(self): self.assertRaises(ExitException, _exec_client, (CLIENT, "-h",)) self.assertLogged("Usage: " + CLIENT) self.assertLogged("Report bugs to ") + self.pruneLog() + self.assertRaises(ExitException, _exec_client, + (CLIENT, "-vq", "-V",)) + self.assertLogged("Fail2Ban v" + fail2bancmdline.version) + + @withtmpdir + def testClientDump(self, tmp): + # use here the stock configuration (if possible) + startparams = _start_params(tmp, True) + self.assertRaises(ExitException, _exec_client, + ((CLIENT,) + startparams + ("-vvd",))) + self.assertLogged("Loading files") + self.assertLogged("logtarget") @withtmpdir def testClientStartBackgroundInside(self, tmp): @@ -271,6 +290,13 @@ class Fail2banClientTest(Fail2banClientServerBase): (CLIENT,) + startparams + ("stop",)) self.assertLogged("Shutdown successful") self.assertLogged("Exit with code 0") + + self.pruneLog() + # stop again (should fail): + self.assertRaises(FailExitException, _exec_client, + (CLIENT,) + startparams + ("stop",)) + self.assertLogged("Failed to access socket path") + self.assertLogged("Is fail2ban running?") finally: _kill_srv(tmp) @@ -278,12 +304,13 @@ class Fail2banClientTest(Fail2banClientServerBase): def testClientStartBackgroundCall(self, tmp): try: global INTERACT - startparams = _start_params(tmp) + startparams = _start_params(tmp, logtarget=tmp+"/f2b.log") # start (in new process, using the same python version): cmd = (sys.executable, os.path.join(os.path.join(BIN), CLIENT)) logSys.debug('Start %s ...', cmd) - cmd = cmd + startparams + ("start",) - Utils.executeCmd(cmd, timeout=MAX_WAITTIME, shell=False, output=True) + cmd = cmd + startparams + ("--async", "start",) + ret = Utils.executeCmd(cmd, timeout=MAX_WAITTIME, shell=False, output=True) + self.assertTrue(len(ret) and ret[0]) # wait for server (socket and ready): self._wait_for_srv(tmp, True, startparams=cmd) self.assertLogged("Server ready") @@ -383,13 +410,35 @@ class Fail2banClientTest(Fail2banClientServerBase): @withtmpdir def testClientFailStart(self, tmp): try: + # started directly here, so prevent overwrite test cases logger with "INHERITED" + startparams = _start_params(tmp, logtarget="INHERITED") + + ## wrong config directory self.assertRaises(FailExitException, _exec_client, (CLIENT, "--async", "-c", tmp+"/miss", "start",)) self.assertLogged("Base configuration directory " + tmp+"/miss" + " does not exist") + self.pruneLog() + ## wrong socket self.assertRaises(FailExitException, _exec_client, - (CLIENT, "--async", "-c", CONF_DIR, "-s", tmp+"/miss/f2b.sock", "start",)) + (CLIENT, "--async", "-c", tmp+"/config", "-s", tmp+"/miss/f2b.sock", "start",)) self.assertLogged("There is no directory " + tmp+"/miss" + " to contain the socket file") + self.pruneLog() + + ## already exists: + open(tmp+"/f2b.sock", 'a').close() + self.assertRaises(FailExitException, _exec_client, + (CLIENT, "--async", "-c", tmp+"/config", "-s", tmp+"/f2b.sock", "start",)) + self.assertLogged("Fail2ban seems to be in unexpected state (not running but the socket exists)") + self.pruneLog() + os.remove(tmp+"/f2b.sock") + + ## wrong option: + self.assertRaises(FailExitException, _exec_client, + (CLIENT, "-s",)) + self.assertLogged("Usage: ") + self.pruneLog() + finally: _kill_srv(tmp) @@ -417,12 +466,13 @@ class Fail2banServerTest(Fail2banClientServerBase): def testServerStartBackground(self, tmp): try: # to prevent fork of test-cases process, start server in background via command: - startparams = _start_params(tmp) + startparams = _start_params(tmp, logtarget=tmp+"/f2b.log") # start (in new process, using the same python version): cmd = (sys.executable, os.path.join(os.path.join(BIN), SERVER)) logSys.debug('Start %s ...', cmd) cmd = cmd + startparams + ("-b",) - Utils.executeCmd(cmd, timeout=MAX_WAITTIME, shell=False, output=True) + ret = Utils.executeCmd(cmd, timeout=MAX_WAITTIME, shell=False, output=True) + self.assertTrue(len(ret) and ret[0]) # wait for server (socket and ready): self._wait_for_srv(tmp, True, startparams=cmd) self.assertLogged("Server ready") @@ -495,12 +545,28 @@ class Fail2banServerTest(Fail2banClientServerBase): @withtmpdir def testServerFailStart(self, tmp): try: + # started directly here, so prevent overwrite test cases logger with "INHERITED" + startparams = _start_params(tmp, logtarget="INHERITED") + + ## wrong config directory self.assertRaises(FailExitException, _exec_server, (SERVER, "-c", tmp+"/miss",)) self.assertLogged("Base configuration directory " + tmp+"/miss" + " does not exist") + self.pruneLog() + ## wrong socket self.assertRaises(FailExitException, _exec_server, - (SERVER, "-c", CONF_DIR, "-s", tmp+"/miss/f2b.sock",)) + (SERVER, "-c", tmp+"/config", "-x", "-s", tmp+"/miss/f2b.sock",)) self.assertLogged("There is no directory " + tmp+"/miss" + " to contain the socket file") + self.pruneLog() + + ## already exists: + open(tmp+"/f2b.sock", 'a').close() + self.assertRaises(FailExitException, _exec_server, + (SERVER, "-c", tmp+"/config", "-s", tmp+"/f2b.sock",)) + self.assertLogged("Fail2ban seems to be in unexpected state (not running but the socket exists)") + self.pruneLog() + os.remove(tmp+"/f2b.sock") + finally: _kill_srv(tmp) From 060ea085f4b63c04518ec446abfd75896bc673d1 Mon Sep 17 00:00:00 2001 From: sebres Date: Mon, 15 Feb 2016 20:41:20 +0100 Subject: [PATCH 08/13] reader bug fix: prevent to silent "load" of not existing jail; coverage of test cases increased; --- fail2ban/client/configreader.py | 4 +++- fail2ban/client/fail2banclient.py | 20 ++++++++++---------- fail2ban/client/fail2banserver.py | 24 ++++++++++++------------ fail2ban/client/jailreader.py | 2 +- fail2ban/tests/fail2banclienttestcase.py | 21 +++++++++++++++++++++ 5 files changed, 47 insertions(+), 24 deletions(-) diff --git a/fail2ban/client/configreader.py b/fail2ban/client/configreader.py index 37e66249..bcad59c3 100644 --- a/fail2ban/client/configreader.py +++ b/fail2ban/client/configreader.py @@ -208,7 +208,7 @@ class ConfigReaderUnshared(SafeConfigParserWithIncludes): # Or it is a dict: # {name: [type, default], ...} - def getOptions(self, sec, options, pOptions=None): + def getOptions(self, sec, options, pOptions=None, shouldExist=False): values = dict() for optname in options: if isinstance(options, (list,tuple)): @@ -229,6 +229,8 @@ class ConfigReaderUnshared(SafeConfigParserWithIncludes): continue values[optname] = v except NoSectionError, e: + if shouldExist: + raise # No "Definition" section or wrong basedir logSys.error(e) values[optname] = optvalue diff --git a/fail2ban/client/fail2banclient.py b/fail2ban/client/fail2banclient.py index 23d31dc5..a8e0a331 100755 --- a/fail2ban/client/fail2banclient.py +++ b/fail2ban/client/fail2banclient.py @@ -64,7 +64,7 @@ class Fail2banClient(Fail2banCmdLine, Thread): output("and bans the corresponding IP addresses using firewall rules.") output("") - def __sigTERMhandler(self, signum, frame): + def __sigTERMhandler(self, signum, frame): # pragma: no cover # Print a new line because we probably come from wait output("") logSys.warning("Caught signal %d. Exiting" % signum) @@ -141,7 +141,7 @@ class Fail2banClient(Fail2banCmdLine, Thread): logSys.error("Failed to access socket path: %s." " Is fail2ban running?", self._conf["socket"]) - except Exception as e: + except Exception as e: # pragma: no cover logSys.error("Exception while checking socket access: %s", self._conf["socket"]) logSys.error(e) @@ -165,7 +165,7 @@ class Fail2banClient(Fail2banCmdLine, Thread): "There is no directory %s to contain the socket file %s." % (socket_dir, self._conf["socket"])) return None - if not os.access(socket_dir, os.W_OK | os.X_OK): + if not os.access(socket_dir, os.W_OK | os.X_OK): # pragma: no cover logSys.error( "Directory %s exists but not accessible for writing" % (socket_dir,)) @@ -204,9 +204,9 @@ class Fail2banClient(Fail2banCmdLine, Thread): # Start server direct here in main thread (not fork): self._server = Fail2banServer.startServerDirect(self._conf, False) - except ExitException: + except ExitException: # pragma: no cover pass - except Exception as e: + except Exception as e: # pragma: no cover output("") logSys.error("Exception while starting server " + ("background" if background else "foreground")) if self._conf["verbose"] > 1: @@ -259,7 +259,7 @@ class Fail2banClient(Fail2banCmdLine, Thread): if self._conf.get("interactive", False): output(' ## stop ... ') self.__processCommand(['stop']) - if not self.__waitOnServer(False): + if not self.__waitOnServer(False): # pragma: no cover logSys.error("Could not stop server") return False # in interactive mode reset config, to make full-reload if there something changed: @@ -298,12 +298,12 @@ class Fail2banClient(Fail2banCmdLine, Thread): def __processStartStreamAfterWait(self, *args): try: # Wait for the server to start - if not self.__waitOnServer(): + if not self.__waitOnServer(): # pragma: no cover logSys.error("Could not find server, waiting failed") return False # Configure the server self.__processCmd(*args) - except ServerExecutionException as e: + except ServerExecutionException as e: # pragma: no cover if self._conf["verbose"] > 1: logSys.exception(e) logSys.error("Could not start server. Maybe an old " @@ -385,12 +385,12 @@ class Fail2banClient(Fail2banCmdLine, Thread): elif not cmd == "": try: self.__processCommand(shlex.split(cmd)) - except Exception, e: + except Exception, e: # pragma: no cover if self._conf["verbose"] > 1: logSys.exception(e) else: logSys.error(e) - except (EOFError, KeyboardInterrupt): + except (EOFError, KeyboardInterrupt): # pragma: no cover output("") raise # Single command mode diff --git a/fail2ban/client/fail2banserver.py b/fail2ban/client/fail2banserver.py index a6c8a7c5..a511e017 100644 --- a/fail2ban/client/fail2banserver.py +++ b/fail2ban/client/fail2banserver.py @@ -56,7 +56,7 @@ class Fail2banServer(Fail2banCmdLine): server.start(conf["socket"], conf["pidfile"], conf["force"], conf=conf) - except Exception as e: + except Exception as e: # pragma: no cover try: if server: server.quit() @@ -77,7 +77,7 @@ class Fail2banServer(Fail2banCmdLine): # Forks the current process, don't fork if async specified (ex: test cases) pid = 0 frk = not conf["async"] and PRODUCTION - if frk: + if frk: # pragma: no cover pid = os.fork() logSys.debug("-- async starting of server in %s, fork: %s - %s", os.getpid(), frk, pid) if pid == 0: @@ -108,22 +108,20 @@ class Fail2banServer(Fail2banCmdLine): exe = sys.executable args[0:0] = [exe] logSys.debug("Starting %r with args %r", exe, args) - if frk: - return os.execv(exe, args) + if frk: # pragma: no cover + os.execv(exe, args) else: # use P_WAIT instead of P_NOWAIT (to prevent defunct-zomby process), it startet as daemon, so parent exit fast after fork): ret = os.spawnv(os.P_WAIT, exe, args) - if ret != 0: + if ret != 0: # pragma: no cover raise OSError(ret, "Unknown error by executing server %r with %r" % (args[1], exe)) - return 0 except OSError as e: # pragma: no cover if not frk: #not PRODUCTION: raise # Use the PATH env. logSys.warning("Initial start attempt failed (%s). Starting %r with the same args", e, SERVER) - if frk: - return os.execvp(SERVER, args) - return pid + if frk: # pragma: no cover + os.execvp(SERVER, args) @staticmethod def getServerPath(): @@ -189,7 +187,7 @@ class Fail2banServer(Fail2banCmdLine): pid = os.getpid() server = Fail2banServer.startServerDirect(self._conf, background) # If forked - just exit other processes - if pid != os.getpid(): + if pid != os.getpid(): # pragma: no cover os._exit(0) if cli: cli._server = server @@ -198,7 +196,7 @@ class Fail2banServer(Fail2banCmdLine): if not async and cli: Utils.wait_for(lambda: phase.get('done', None) is not None, self._conf["timeout"]) if not phase.get('done', False): - if server: + if server: # pragma: no cover server.quit() exit(-1) logSys.debug('Starting server done') @@ -206,7 +204,9 @@ class Fail2banServer(Fail2banCmdLine): except Exception, e: if self._conf["verbose"] > 1: logSys.exception(e) - if server: + else: + logSys.error(e) + if server: # pragma: no cover server.quit() exit(-1) diff --git a/fail2ban/client/jailreader.py b/fail2ban/client/jailreader.py index fda5d40c..f0cf293a 100644 --- a/fail2ban/client/jailreader.py +++ b/fail2ban/client/jailreader.py @@ -114,7 +114,7 @@ class JailReader(ConfigReader): defsec["fail2ban_version"] = version # Read first options only needed for merge defaults ('known/...' from filter): - self.__opts = ConfigReader.getOptions(self, self.__name, opts1st) + self.__opts = ConfigReader.getOptions(self, self.__name, opts1st, shouldExist=True) if not self.__opts: return False diff --git a/fail2ban/tests/fail2banclienttestcase.py b/fail2ban/tests/fail2banclienttestcase.py index aa507fe6..82ffe9b3 100644 --- a/fail2ban/tests/fail2banclienttestcase.py +++ b/fail2ban/tests/fail2banclienttestcase.py @@ -347,6 +347,21 @@ class Fail2banClientTest(Fail2banClientServerBase): self.assertLogged("Server ready") self.assertLogged("Exit with code 0") self.pruneLog() + # test reload missing jail (interactive): + INTERACT += [ + "reload ~~unknown~jail~fail~~", + "exit" + ] + self.assertRaises(ExitException, _exec_client, + (CLIENT,) + startparams + ("-i",)) + self.assertLogged("Failed during configuration: No section: '~~unknown~jail~fail~~'") + self.pruneLog() + # test reload missing jail (direct): + self.assertRaises(FailExitException, _exec_client, + (CLIENT,) + startparams + ("reload", "~~unknown~jail~fail~~")) + self.assertLogged("Failed during configuration: No section: '~~unknown~jail~fail~~'") + self.assertLogged("Exit with code -1") + self.pruneLog() finally: self.pruneLog() # stop: @@ -425,6 +440,12 @@ class Fail2banClientTest(Fail2banClientServerBase): self.assertLogged("There is no directory " + tmp+"/miss" + " to contain the socket file") self.pruneLog() + ## not running + self.assertRaises(FailExitException, _exec_client, + (CLIENT, "-c", tmp+"/config", "-s", tmp+"/f2b.sock", "reload",)) + self.assertLogged("Could not find server") + self.pruneLog() + ## already exists: open(tmp+"/f2b.sock", 'a').close() self.assertRaises(FailExitException, _exec_client, From 53956501dae839638ee2359ece32190a9a2ba59c Mon Sep 17 00:00:00 2001 From: sebres Date: Fri, 12 Feb 2016 21:57:12 +0100 Subject: [PATCH 09/13] increase readability and details level by increased verbosity --- bin/fail2ban-testcases | 12 +++++++---- fail2ban/server/server.py | 13 +++++++---- fail2ban/tests/actiontestcase.py | 2 +- fail2ban/tests/servertestcase.py | 37 +++++++++++++++++++++----------- fail2ban/tests/utils.py | 1 + 5 files changed, 43 insertions(+), 22 deletions(-) diff --git a/bin/fail2ban-testcases b/bin/fail2ban-testcases index 606b0b06..3b18b7c2 100755 --- a/bin/fail2ban-testcases +++ b/bin/fail2ban-testcases @@ -119,10 +119,14 @@ else: # Custom log format for the verbose tests runs if verbosity > 1: # pragma: no cover - stdout.setFormatter(Formatter(' %(asctime)-15s %(thread)s' + fmt)) -else: # pragma: no cover - # just prefix with the space - stdout.setFormatter(Formatter(fmt)) + if verbosity > 3: + fmt = ' | %(module)15.15s-%(levelno)-2d: %(funcName)-20.20s |' + fmt + if verbosity > 2: + fmt = ' +%(relativeCreated)5d %(thread)X %(levelname)-5.5s' + fmt + else: + fmt = ' %(asctime)-15s %(thread)X %(levelname)-5.5s' + fmt +# +stdout.setFormatter(Formatter(fmt)) logSys.addHandler(stdout) # diff --git a/fail2ban/server/server.py b/fail2ban/server/server.py index fa6ca43d..9377ad5b 100644 --- a/fail2ban/server/server.py +++ b/fail2ban/server/server.py @@ -67,7 +67,8 @@ class Server: self.__db = None self.__daemon = daemon self.__transm = Transmitter(self) - self.__asyncServer = AsyncServer(self.__transm) + #self.__asyncServer = AsyncServer(self.__transm) + self.__asyncServer = None self.__logLevel = None self.__logTarget = None self.__syslogSocket = None @@ -137,6 +138,7 @@ class Server: # Start the communication logSys.debug("Starting communication") try: + self.__asyncServer = AsyncServer(self.__transm) self.__asyncServer.start(sock, force) except AsyncServerException, e: logSys.error("Could not start server: %s", e) @@ -155,14 +157,17 @@ class Server: # communications first (which should be ok anyways since we # are exiting) # See https://github.com/fail2ban/fail2ban/issues/7 - self.__asyncServer.stop() + if self.__asyncServer is not None: + self.__asyncServer.stop() + self.__asyncServer = None # Now stop all the jails self.stopAllJail() # Only now shutdown the logging. - with self.__loggingLock: - logging.shutdown() + if self.__logTarget is not None: + with self.__loggingLock: + logging.shutdown() # Restore default signal handlers: if _thread_name() == '_MainThread': diff --git a/fail2ban/tests/actiontestcase.py b/fail2ban/tests/actiontestcase.py index f3778022..cffabcf0 100644 --- a/fail2ban/tests/actiontestcase.py +++ b/fail2ban/tests/actiontestcase.py @@ -44,8 +44,8 @@ class CommandActionTest(LogCaptureTestCase): def tearDown(self): """Call after every test case.""" - LogCaptureTestCase.tearDown(self) self.__action.stop() + LogCaptureTestCase.tearDown(self) def testSubstituteRecursiveTags(self): aInfo = { diff --git a/fail2ban/tests/servertestcase.py b/fail2ban/tests/servertestcase.py index 90f7aaf2..e253ae84 100644 --- a/fail2ban/tests/servertestcase.py +++ b/fail2ban/tests/servertestcase.py @@ -66,25 +66,18 @@ class TransmitterBase(unittest.TestCase): def setUp(self): """Call before every test case.""" + #super(TransmitterBase, self).setUp() self.transm = self.server._Server__transm - self.tmp_files = [] - sock_fd, sock_name = tempfile.mkstemp('fail2ban.sock', 'transmitter') - os.close(sock_fd) - self.tmp_files.append(sock_name) - pidfile_fd, pidfile_name = tempfile.mkstemp( - 'fail2ban.pid', 'transmitter') - os.close(pidfile_fd) - self.tmp_files.append(pidfile_name) - self.server.start(sock_name, pidfile_name, force=False) + # To test thransmitter we don't need to start server... + #self.server.start('/dev/null', '/dev/null', force=False) self.jailName = "TestJail1" self.server.addJail(self.jailName, FAST_BACKEND) def tearDown(self): """Call after every test case.""" + # stop jails, etc. self.server.quit() - for f in self.tmp_files: - if os.path.exists(f): - os.remove(f) + #super(TransmitterBase, self).tearDown() def setGetTest(self, cmd, inValue, outValue=(None,), outCode=0, jail=None, repr_=False): """Process set/get commands and compare both return values @@ -796,10 +789,10 @@ class TransmitterLogging(TransmitterBase): def setUp(self): self.server = Server() + super(TransmitterLogging, self).setUp() self.server.setLogTarget("/dev/null") self.server.setLogLevel("CRITICAL") self.server.setSyslogSocket("auto") - super(TransmitterLogging, self).setUp() def testLogTarget(self): logTargets = [] @@ -968,6 +961,24 @@ class LoggingTests(LogCaptureTestCase): self.assertEqual(len(x), 1) self.assertEqual(x[0][0], RuntimeError) + def testStartFailedSockExists(self): + tmp_files = [] + sock_fd, sock_name = tempfile.mkstemp('fail2ban.sock', 'f2b-test') + os.close(sock_fd) + tmp_files.append(sock_name) + pidfile_fd, pidfile_name = tempfile.mkstemp('fail2ban.pid', 'f2b-test') + os.close(pidfile_fd) + tmp_files.append(pidfile_name) + server = TestServer() + try: + server.start(sock_name, pidfile_name, force=False) + self.assertLogged("Server already running") + finally: + server.quit() + for f in tmp_files: + if os.path.exists(f): + os.remove(f) + from clientreadertestcase import ActionReader, JailReader, JailsReader, CONFIG_DIR, STOCK diff --git a/fail2ban/tests/utils.py b/fail2ban/tests/utils.py index 2d096276..bfbef4de 100644 --- a/fail2ban/tests/utils.py +++ b/fail2ban/tests/utils.py @@ -353,6 +353,7 @@ class LogCaptureTestCase(unittest.TestCase): def tearDown(self): """Call after every test case.""" # print "O: >>%s<<" % self._log.getvalue() + self.pruneLog() logSys = getLogger("fail2ban") logSys.handlers = self._old_handlers logSys.level = self._old_level From 6353de8b0f0d2f130255564e92ed470d5b41822d Mon Sep 17 00:00:00 2001 From: sebres Date: Wed, 30 Mar 2016 21:50:58 +0200 Subject: [PATCH 10/13] possibility to increase verbosity up to heavy debug (command line parameter `-vvv`) --- fail2ban/client/fail2bancmdline.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/fail2ban/client/fail2bancmdline.py b/fail2ban/client/fail2bancmdline.py index d49f0e3d..83379a9a 100644 --- a/fail2ban/client/fail2bancmdline.py +++ b/fail2ban/client/fail2bancmdline.py @@ -165,6 +165,7 @@ class Fail2banCmdLine(): return None def initCmdLine(self, argv): + verbose = 1 try: # First time? initial = (self._argv is None) @@ -232,7 +233,8 @@ class Fail2banCmdLine(): return None except Exception as e: output("ERROR: %s" % (e,)) - #logSys.exception(e) + if verbose > 2: + logSys.exception(e) return False def readConfig(self, jail=None): From e57321ab1e47aa12fbe4d73273477196c80584cd Mon Sep 17 00:00:00 2001 From: Yaroslav Halchenko Date: Thu, 12 May 2016 09:37:30 -0400 Subject: [PATCH 11/13] BF+ENH: import signal used in the code, withtmpdir -> with_tmpdir (more readable) case shows how evil except Exception can be ;) --- fail2ban/tests/fail2banclienttestcase.py | 19 ++++++++++--------- fail2ban/tests/utils.py | 8 +++++++- 2 files changed, 17 insertions(+), 10 deletions(-) diff --git a/fail2ban/tests/fail2banclienttestcase.py b/fail2ban/tests/fail2banclienttestcase.py index 82ffe9b3..53bc41d9 100644 --- a/fail2ban/tests/fail2banclienttestcase.py +++ b/fail2ban/tests/fail2banclienttestcase.py @@ -28,6 +28,7 @@ import os import re import sys import time +import signal import unittest from threading import Thread @@ -38,7 +39,7 @@ from ..client.fail2banserver import Fail2banServer, exec_command_line as _exec_s from .. import protocol from ..server import server from ..server.utils import Utils -from .utils import LogCaptureTestCase, logSys, withtmpdir, shutil, logging +from .utils import LogCaptureTestCase, logSys, with_tmpdir, shutil, logging STOCK_CONF_DIR = "config" @@ -252,7 +253,7 @@ class Fail2banClientTest(Fail2banClientServerBase): (CLIENT, "-vq", "-V",)) self.assertLogged("Fail2Ban v" + fail2bancmdline.version) - @withtmpdir + @with_tmpdir def testClientDump(self, tmp): # use here the stock configuration (if possible) startparams = _start_params(tmp, True) @@ -261,7 +262,7 @@ class Fail2banClientTest(Fail2banClientServerBase): self.assertLogged("Loading files") self.assertLogged("logtarget") - @withtmpdir + @with_tmpdir def testClientStartBackgroundInside(self, tmp): try: # use once the stock configuration (to test starting also) @@ -300,7 +301,7 @@ class Fail2banClientTest(Fail2banClientServerBase): finally: _kill_srv(tmp) - @withtmpdir + @with_tmpdir def testClientStartBackgroundCall(self, tmp): try: global INTERACT @@ -382,7 +383,7 @@ class Fail2banClientTest(Fail2banClientServerBase): phase['end'] = True logSys.debug("-- end of test worker") - @withtmpdir + @with_tmpdir def testClientStartForeground(self, tmp): th = None try: @@ -422,7 +423,7 @@ class Fail2banClientTest(Fail2banClientServerBase): if th: th.join() - @withtmpdir + @with_tmpdir def testClientFailStart(self, tmp): try: # started directly here, so prevent overwrite test cases logger with "INHERITED" @@ -483,7 +484,7 @@ class Fail2banServerTest(Fail2banClientServerBase): self.assertLogged("Usage: " + SERVER) self.assertLogged("Report bugs to ") - @withtmpdir + @with_tmpdir def testServerStartBackground(self, tmp): try: # to prevent fork of test-cases process, start server in background via command: @@ -523,7 +524,7 @@ class Fail2banServerTest(Fail2banClientServerBase): phase['end'] = True logSys.debug("-- end of test worker") - @withtmpdir + @with_tmpdir def testServerStartForeground(self, tmp): th = None try: @@ -563,7 +564,7 @@ class Fail2banServerTest(Fail2banClientServerBase): if th: th.join() - @withtmpdir + @with_tmpdir def testServerFailStart(self, tmp): try: # started directly here, so prevent overwrite test cases logger with "INHERITED" diff --git a/fail2ban/tests/utils.py b/fail2ban/tests/utils.py index bfbef4de..be8ce015 100644 --- a/fail2ban/tests/utils.py +++ b/fail2ban/tests/utils.py @@ -79,7 +79,12 @@ class F2B(optparse.Values): return wtime -def withtmpdir(f): +def with_tmpdir(f): + """Helper decorator to create a temporary directory + + Directory gets removed after function returns, regardless + if exception was thrown of not + """ @wraps(f) def wrapper(self, *args, **kwargs): tmp = tempfile.mkdtemp(prefix="f2b-temp") @@ -90,6 +95,7 @@ def withtmpdir(f): shutil.rmtree(tmp) return wrapper + def initTests(opts): unittest.F2B = F2B(opts) # --fast : From d7ff7d18cdcd94877b7f17fc5634f0331f62232b Mon Sep 17 00:00:00 2001 From: Yaroslav Halchenko Date: Thu, 12 May 2016 09:47:01 -0400 Subject: [PATCH 12/13] RF+ENH: @with_kill_srv fixture to kill_srv in the tests --- fail2ban/tests/fail2banclienttestcase.py | 362 ++++++++++++----------- 1 file changed, 183 insertions(+), 179 deletions(-) diff --git a/fail2ban/tests/fail2banclienttestcase.py b/fail2ban/tests/fail2banclienttestcase.py index 53bc41d9..1a0880c7 100644 --- a/fail2ban/tests/fail2banclienttestcase.py +++ b/fail2ban/tests/fail2banclienttestcase.py @@ -31,8 +31,10 @@ import time import signal import unittest +from functools import wraps from threading import Thread + from ..client import fail2banclient, fail2banserver, fail2bancmdline from ..client.fail2banclient import Fail2banClient, exec_command_line as _exec_client, VisualWait from ..client.fail2banserver import Fail2banServer, exec_command_line as _exec_server @@ -205,6 +207,20 @@ def _kill_srv(pidfile): # pragma: no cover f.close() return True +def with_kill_srv(f): + """Helper to decorate tests which receive in the last argument tmpdir to pass to kill_srv + + To be used in tandem with @with_tmpdir + """ + @wraps(f) + def wrapper(self, *args): + pidfile = args[-1] + try: + return f(self, *args) + finally: + _kill_srv(pidfile) + return wrapper + class Fail2banClientServerBase(LogCaptureTestCase): @@ -263,115 +279,111 @@ class Fail2banClientTest(Fail2banClientServerBase): self.assertLogged("logtarget") @with_tmpdir + @with_kill_srv def testClientStartBackgroundInside(self, tmp): + # use once the stock configuration (to test starting also) + startparams = _start_params(tmp, True) + # start: + self.assertRaises(ExitException, _exec_client, + (CLIENT, "-b") + startparams + ("start",)) + # wait for server (socket and ready): + self._wait_for_srv(tmp, True, startparams=startparams) + self.assertLogged("Server ready") + self.assertLogged("Exit with code 0") try: - # use once the stock configuration (to test starting also) - startparams = _start_params(tmp, True) - # start: self.assertRaises(ExitException, _exec_client, - (CLIENT, "-b") + startparams + ("start",)) - # wait for server (socket and ready): - self._wait_for_srv(tmp, True, startparams=startparams) - self.assertLogged("Server ready") - self.assertLogged("Exit with code 0") - try: - self.assertRaises(ExitException, _exec_client, - (CLIENT,) + startparams + ("echo", "TEST-ECHO",)) - self.assertRaises(FailExitException, _exec_client, - (CLIENT,) + startparams + ("~~unknown~cmd~failed~~",)) - self.pruneLog() - # start again (should fail): - self.assertRaises(FailExitException, _exec_client, - (CLIENT, "-b") + startparams + ("start",)) - self.assertLogged("Server already running") - finally: - self.pruneLog() - # stop: - self.assertRaises(ExitException, _exec_client, - (CLIENT,) + startparams + ("stop",)) - self.assertLogged("Shutdown successful") - self.assertLogged("Exit with code 0") - - self.pruneLog() - # stop again (should fail): + (CLIENT,) + startparams + ("echo", "TEST-ECHO",)) self.assertRaises(FailExitException, _exec_client, - (CLIENT,) + startparams + ("stop",)) - self.assertLogged("Failed to access socket path") - self.assertLogged("Is fail2ban running?") + (CLIENT,) + startparams + ("~~unknown~cmd~failed~~",)) + self.pruneLog() + # start again (should fail): + self.assertRaises(FailExitException, _exec_client, + (CLIENT, "-b") + startparams + ("start",)) + self.assertLogged("Server already running") finally: - _kill_srv(tmp) + self.pruneLog() + # stop: + self.assertRaises(ExitException, _exec_client, + (CLIENT,) + startparams + ("stop",)) + self.assertLogged("Shutdown successful") + self.assertLogged("Exit with code 0") + + self.pruneLog() + # stop again (should fail): + self.assertRaises(FailExitException, _exec_client, + (CLIENT,) + startparams + ("stop",)) + self.assertLogged("Failed to access socket path") + self.assertLogged("Is fail2ban running?") @with_tmpdir + @with_kill_srv def testClientStartBackgroundCall(self, tmp): + global INTERACT + startparams = _start_params(tmp, logtarget=tmp+"/f2b.log") + # start (in new process, using the same python version): + cmd = (sys.executable, os.path.join(os.path.join(BIN), CLIENT)) + logSys.debug('Start %s ...', cmd) + cmd = cmd + startparams + ("--async", "start",) + ret = Utils.executeCmd(cmd, timeout=MAX_WAITTIME, shell=False, output=True) + self.assertTrue(len(ret) and ret[0]) + # wait for server (socket and ready): + self._wait_for_srv(tmp, True, startparams=cmd) + self.assertLogged("Server ready") + self.pruneLog() try: - global INTERACT - startparams = _start_params(tmp, logtarget=tmp+"/f2b.log") - # start (in new process, using the same python version): - cmd = (sys.executable, os.path.join(os.path.join(BIN), CLIENT)) - logSys.debug('Start %s ...', cmd) - cmd = cmd + startparams + ("--async", "start",) - ret = Utils.executeCmd(cmd, timeout=MAX_WAITTIME, shell=False, output=True) - self.assertTrue(len(ret) and ret[0]) - # wait for server (socket and ready): - self._wait_for_srv(tmp, True, startparams=cmd) - self.assertLogged("Server ready") + # echo from client (inside): + self.assertRaises(ExitException, _exec_client, + (CLIENT,) + startparams + ("echo", "TEST-ECHO",)) + self.assertLogged("TEST-ECHO") + self.assertLogged("Exit with code 0") + self.pruneLog() + # interactive client chat with started server: + INTERACT += [ + "echo INTERACT-ECHO", + "status", + "exit" + ] + self.assertRaises(ExitException, _exec_client, + (CLIENT,) + startparams + ("-i",)) + self.assertLogged("INTERACT-ECHO") + self.assertLogged("Status", "Number of jail:") + self.assertLogged("Exit with code 0") + self.pruneLog() + # test reload and restart over interactive client: + INTERACT += [ + "reload", + "restart", + "exit" + ] + self.assertRaises(ExitException, _exec_client, + (CLIENT,) + startparams + ("-i",)) + self.assertLogged("Reading config files:") + self.assertLogged("Shutdown successful") + self.assertLogged("Server ready") + self.assertLogged("Exit with code 0") + self.pruneLog() + # test reload missing jail (interactive): + INTERACT += [ + "reload ~~unknown~jail~fail~~", + "exit" + ] + self.assertRaises(ExitException, _exec_client, + (CLIENT,) + startparams + ("-i",)) + self.assertLogged("Failed during configuration: No section: '~~unknown~jail~fail~~'") + self.pruneLog() + # test reload missing jail (direct): + self.assertRaises(FailExitException, _exec_client, + (CLIENT,) + startparams + ("reload", "~~unknown~jail~fail~~")) + self.assertLogged("Failed during configuration: No section: '~~unknown~jail~fail~~'") + self.assertLogged("Exit with code -1") self.pruneLog() - try: - # echo from client (inside): - self.assertRaises(ExitException, _exec_client, - (CLIENT,) + startparams + ("echo", "TEST-ECHO",)) - self.assertLogged("TEST-ECHO") - self.assertLogged("Exit with code 0") - self.pruneLog() - # interactive client chat with started server: - INTERACT += [ - "echo INTERACT-ECHO", - "status", - "exit" - ] - self.assertRaises(ExitException, _exec_client, - (CLIENT,) + startparams + ("-i",)) - self.assertLogged("INTERACT-ECHO") - self.assertLogged("Status", "Number of jail:") - self.assertLogged("Exit with code 0") - self.pruneLog() - # test reload and restart over interactive client: - INTERACT += [ - "reload", - "restart", - "exit" - ] - self.assertRaises(ExitException, _exec_client, - (CLIENT,) + startparams + ("-i",)) - self.assertLogged("Reading config files:") - self.assertLogged("Shutdown successful") - self.assertLogged("Server ready") - self.assertLogged("Exit with code 0") - self.pruneLog() - # test reload missing jail (interactive): - INTERACT += [ - "reload ~~unknown~jail~fail~~", - "exit" - ] - self.assertRaises(ExitException, _exec_client, - (CLIENT,) + startparams + ("-i",)) - self.assertLogged("Failed during configuration: No section: '~~unknown~jail~fail~~'") - self.pruneLog() - # test reload missing jail (direct): - self.assertRaises(FailExitException, _exec_client, - (CLIENT,) + startparams + ("reload", "~~unknown~jail~fail~~")) - self.assertLogged("Failed during configuration: No section: '~~unknown~jail~fail~~'") - self.assertLogged("Exit with code -1") - self.pruneLog() - finally: - self.pruneLog() - # stop: - self.assertRaises(ExitException, _exec_client, - (CLIENT,) + startparams + ("stop",)) - self.assertLogged("Shutdown successful") - self.assertLogged("Exit with code 0") finally: - _kill_srv(tmp) + self.pruneLog() + # stop: + self.assertRaises(ExitException, _exec_client, + (CLIENT,) + startparams + ("stop",)) + self.assertLogged("Shutdown successful") + self.assertLogged("Exit with code 0") def _testClientStartForeground(self, tmp, startparams, phase): # start and wait to end (foreground): @@ -424,45 +436,42 @@ class Fail2banClientTest(Fail2banClientServerBase): th.join() @with_tmpdir + @with_kill_srv def testClientFailStart(self, tmp): - try: - # started directly here, so prevent overwrite test cases logger with "INHERITED" - startparams = _start_params(tmp, logtarget="INHERITED") + # started directly here, so prevent overwrite test cases logger with "INHERITED" + startparams = _start_params(tmp, logtarget="INHERITED") - ## wrong config directory - self.assertRaises(FailExitException, _exec_client, - (CLIENT, "--async", "-c", tmp+"/miss", "start",)) - self.assertLogged("Base configuration directory " + tmp+"/miss" + " does not exist") - self.pruneLog() + ## wrong config directory + self.assertRaises(FailExitException, _exec_client, + (CLIENT, "--async", "-c", tmp+"/miss", "start",)) + self.assertLogged("Base configuration directory " + tmp+"/miss" + " does not exist") + self.pruneLog() - ## wrong socket - self.assertRaises(FailExitException, _exec_client, - (CLIENT, "--async", "-c", tmp+"/config", "-s", tmp+"/miss/f2b.sock", "start",)) - self.assertLogged("There is no directory " + tmp+"/miss" + " to contain the socket file") - self.pruneLog() + ## wrong socket + self.assertRaises(FailExitException, _exec_client, + (CLIENT, "--async", "-c", tmp+"/config", "-s", tmp+"/miss/f2b.sock", "start",)) + self.assertLogged("There is no directory " + tmp+"/miss" + " to contain the socket file") + self.pruneLog() - ## not running - self.assertRaises(FailExitException, _exec_client, - (CLIENT, "-c", tmp+"/config", "-s", tmp+"/f2b.sock", "reload",)) - self.assertLogged("Could not find server") - self.pruneLog() + ## not running + self.assertRaises(FailExitException, _exec_client, + (CLIENT, "-c", tmp+"/config", "-s", tmp+"/f2b.sock", "reload",)) + self.assertLogged("Could not find server") + self.pruneLog() - ## already exists: - open(tmp+"/f2b.sock", 'a').close() - self.assertRaises(FailExitException, _exec_client, - (CLIENT, "--async", "-c", tmp+"/config", "-s", tmp+"/f2b.sock", "start",)) - self.assertLogged("Fail2ban seems to be in unexpected state (not running but the socket exists)") - self.pruneLog() - os.remove(tmp+"/f2b.sock") + ## already exists: + open(tmp+"/f2b.sock", 'a').close() + self.assertRaises(FailExitException, _exec_client, + (CLIENT, "--async", "-c", tmp+"/config", "-s", tmp+"/f2b.sock", "start",)) + self.assertLogged("Fail2ban seems to be in unexpected state (not running but the socket exists)") + self.pruneLog() + os.remove(tmp+"/f2b.sock") - ## wrong option: - self.assertRaises(FailExitException, _exec_client, - (CLIENT, "-s",)) - self.assertLogged("Usage: ") - self.pruneLog() - - finally: - _kill_srv(tmp) + ## wrong option: + self.assertRaises(FailExitException, _exec_client, + (CLIENT, "-s",)) + self.assertLogged("Usage: ") + self.pruneLog() def testVisualWait(self): sleeptime = 0.035 @@ -485,34 +494,32 @@ class Fail2banServerTest(Fail2banClientServerBase): self.assertLogged("Report bugs to ") @with_tmpdir + @with_kill_srv def testServerStartBackground(self, tmp): + # to prevent fork of test-cases process, start server in background via command: + startparams = _start_params(tmp, logtarget=tmp+"/f2b.log") + # start (in new process, using the same python version): + cmd = (sys.executable, os.path.join(os.path.join(BIN), SERVER)) + logSys.debug('Start %s ...', cmd) + cmd = cmd + startparams + ("-b",) + ret = Utils.executeCmd(cmd, timeout=MAX_WAITTIME, shell=False, output=True) + self.assertTrue(len(ret) and ret[0]) + # wait for server (socket and ready): + self._wait_for_srv(tmp, True, startparams=cmd) + self.assertLogged("Server ready") + self.pruneLog() try: - # to prevent fork of test-cases process, start server in background via command: - startparams = _start_params(tmp, logtarget=tmp+"/f2b.log") - # start (in new process, using the same python version): - cmd = (sys.executable, os.path.join(os.path.join(BIN), SERVER)) - logSys.debug('Start %s ...', cmd) - cmd = cmd + startparams + ("-b",) - ret = Utils.executeCmd(cmd, timeout=MAX_WAITTIME, shell=False, output=True) - self.assertTrue(len(ret) and ret[0]) - # wait for server (socket and ready): - self._wait_for_srv(tmp, True, startparams=cmd) - self.assertLogged("Server ready") - self.pruneLog() - try: - self.assertRaises(ExitException, _exec_server, - (SERVER,) + startparams + ("echo", "TEST-ECHO",)) - self.assertRaises(FailExitException, _exec_server, - (SERVER,) + startparams + ("~~unknown~cmd~failed~~",)) - finally: - self.pruneLog() - # stop: - self.assertRaises(ExitException, _exec_server, - (SERVER,) + startparams + ("stop",)) - self.assertLogged("Shutdown successful") - self.assertLogged("Exit with code 0") + self.assertRaises(ExitException, _exec_server, + (SERVER,) + startparams + ("echo", "TEST-ECHO",)) + self.assertRaises(FailExitException, _exec_server, + (SERVER,) + startparams + ("~~unknown~cmd~failed~~",)) finally: - _kill_srv(tmp) + self.pruneLog() + # stop: + self.assertRaises(ExitException, _exec_server, + (SERVER,) + startparams + ("stop",)) + self.assertLogged("Shutdown successful") + self.assertLogged("Exit with code 0") def _testServerStartForeground(self, tmp, startparams, phase): # start and wait to end (foreground): @@ -565,30 +572,27 @@ class Fail2banServerTest(Fail2banClientServerBase): th.join() @with_tmpdir + @with_kill_srv def testServerFailStart(self, tmp): - try: - # started directly here, so prevent overwrite test cases logger with "INHERITED" - startparams = _start_params(tmp, logtarget="INHERITED") + # started directly here, so prevent overwrite test cases logger with "INHERITED" + startparams = _start_params(tmp, logtarget="INHERITED") - ## wrong config directory - self.assertRaises(FailExitException, _exec_server, - (SERVER, "-c", tmp+"/miss",)) - self.assertLogged("Base configuration directory " + tmp+"/miss" + " does not exist") - self.pruneLog() + ## wrong config directory + self.assertRaises(FailExitException, _exec_server, + (SERVER, "-c", tmp+"/miss",)) + self.assertLogged("Base configuration directory " + tmp+"/miss" + " does not exist") + self.pruneLog() - ## wrong socket - self.assertRaises(FailExitException, _exec_server, - (SERVER, "-c", tmp+"/config", "-x", "-s", tmp+"/miss/f2b.sock",)) - self.assertLogged("There is no directory " + tmp+"/miss" + " to contain the socket file") - self.pruneLog() + ## wrong socket + self.assertRaises(FailExitException, _exec_server, + (SERVER, "-c", tmp+"/config", "-x", "-s", tmp+"/miss/f2b.sock",)) + self.assertLogged("There is no directory " + tmp+"/miss" + " to contain the socket file") + self.pruneLog() - ## already exists: - open(tmp+"/f2b.sock", 'a').close() - self.assertRaises(FailExitException, _exec_server, - (SERVER, "-c", tmp+"/config", "-s", tmp+"/f2b.sock",)) - self.assertLogged("Fail2ban seems to be in unexpected state (not running but the socket exists)") - self.pruneLog() - os.remove(tmp+"/f2b.sock") - - finally: - _kill_srv(tmp) + ## already exists: + open(tmp+"/f2b.sock", 'a').close() + self.assertRaises(FailExitException, _exec_server, + (SERVER, "-c", tmp+"/config", "-s", tmp+"/f2b.sock",)) + self.assertLogged("Fail2ban seems to be in unexpected state (not running but the socket exists)") + self.pruneLog() + os.remove(tmp+"/f2b.sock") From 06dcad76504f82621a11a5156279d02593538d00 Mon Sep 17 00:00:00 2001 From: sebres Date: Tue, 17 May 2016 12:19:29 +0200 Subject: [PATCH 13/13] fixed mixed indentation (spaces through tabs), duplicate code removed --- bin/fail2ban-testcases | 4 ++-- fail2ban/tests/utils.py | 3 --- 2 files changed, 2 insertions(+), 5 deletions(-) diff --git a/bin/fail2ban-testcases b/bin/fail2ban-testcases index 3b18b7c2..7d122039 100755 --- a/bin/fail2ban-testcases +++ b/bin/fail2ban-testcases @@ -122,7 +122,7 @@ if verbosity > 1: # pragma: no cover if verbosity > 3: fmt = ' | %(module)15.15s-%(levelno)-2d: %(funcName)-20.20s |' + fmt if verbosity > 2: - fmt = ' +%(relativeCreated)5d %(thread)X %(levelname)-5.5s' + fmt + fmt = ' +%(relativeCreated)5d %(thread)X %(levelname)-5.5s' + fmt else: fmt = ' %(asctime)-15s %(thread)X %(levelname)-5.5s' + fmt # @@ -134,7 +134,7 @@ logSys.addHandler(stdout) # if not opts.log_level or opts.log_level != 'critical': # pragma: no cover print("Fail2ban %s test suite. Python %s. Please wait..." \ - % (version, str(sys.version).replace('\n', ''))) + % (version, str(sys.version).replace('\n', ''))) tests = gatherTests(regexps, opts) # diff --git a/fail2ban/tests/utils.py b/fail2ban/tests/utils.py index be8ce015..2d5a38f7 100644 --- a/fail2ban/tests/utils.py +++ b/fail2ban/tests/utils.py @@ -418,9 +418,6 @@ class LogCaptureTestCase(unittest.TestCase): def pruneLog(self): self._log.truncate(0) - def pruneLog(self): - self._log.truncate(0) - def getLog(self): return self._log.getvalue()