diff --git a/.coveragerc b/.coveragerc new file mode 100644 index 00000000..3bffd79a --- /dev/null +++ b/.coveragerc @@ -0,0 +1,4 @@ + +[run] +branch = True +omit = /usr* diff --git a/.gitignore b/.gitignore index 6a0d5e64..c2e979e5 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,8 @@ *~ build +dist *.pyc +htmlcov +.coverage +*.orig +*.rej diff --git a/.travis.yml b/.travis.yml index b68b6c3e..f443ff12 100644 --- a/.travis.yml +++ b/.travis.yml @@ -8,8 +8,11 @@ python: - "3.2" - "3.3" install: - - "pip install pyinotify" + - pip install pyinotify + - if [[ $TRAVIS_PYTHON_VERSION == 2.[6-7] ]] || [[ $TRAVIS_PYTHON_VERSION == 3.3 ]]; then pip install -q coveralls; fi before_script: - if [[ $TRAVIS_PYTHON_VERSION == 3* ]]; then ./fail2ban-2to3; fi script: - - python ./fail2ban-testcases + - if [[ $TRAVIS_PYTHON_VERSION == 2.[6-7] ]] || [[ $TRAVIS_PYTHON_VERSION == 3.3 ]]; then coverage run --rcfile=.travis_coveragerc fail2ban-testcases; else python ./fail2ban-testcases; fi +after_script: + - if [[ $TRAVIS_PYTHON_VERSION == 2.[6-7] ]] || [[ $TRAVIS_PYTHON_VERSION == 3.3 ]]; then coveralls; fi diff --git a/.travis_coveragerc b/.travis_coveragerc new file mode 100644 index 00000000..4d4b7ebd --- /dev/null +++ b/.travis_coveragerc @@ -0,0 +1,7 @@ + +[run] +branch = True +omit = + /usr/* + /home/travis/virtualenv/* + server/filtergamin.py diff --git a/DEVELOP b/DEVELOP index 7a6a7b9b..623aee12 100644 --- a/DEVELOP +++ b/DEVELOP @@ -24,14 +24,100 @@ Request feature. You can find more details on the Fail2Ban wiki Testing ======= -Existing tests can be run by executing `fail2ban-testcases`. +Existing tests can be run by executing `fail2ban-testcases`. This has options +like --log-level that will probably be useful. `fail2ban-testcases --help` for +full options. + +Test cases should cover all usual cases, all exception cases and all inside +/ outside boundary conditions. + +Test cases should cover all branches. The coverage tool will help identify +missing branches. Also see http://nedbatchelder.com/code/coverage/branch.html +for more details. + +Install the package python-coverage to visualise your test coverage. Run the +following (note: on Debian-based systems, the script is called +`python-coverage`): + +coverage run fail2ban-testcases +coverage html + +Then look at htmlcov/index.html and see how much coverage your test cases +exert over the codebase. Full coverage is a good thing however it may not be +complete. Try to ensure tests cover as many independent paths through the +code. + +Manual Execution. To run in a development environment do: + +./fail2ban-client -c config/ -s /tmp/f2b.sock -i start + +some quick commands: + +status +add test pyinotify +status test +set test addaction iptables +set test actionban iptables echo >> /tmp/ban +set test actionunban iptables echo >> /tmp/unban +get test actionban iptables +get test actionunban iptables +set test banip 192.168.2.2 +status test + -Documentation about creating tests (when tests are required and some guidelines -for creating good tests) will be added soon. Coding Standards ================ -Coming Soon. + +Style +----- + +Please use tabs for now. Keep to 80 columns, at least for readable text. + +Tests +----- + +Add tests. They should test all the code you add in a meaning way. + +Coverage +-------- + +Test coverage should always increase as you add code. + +You may use "# pragma: no cover" in the code for branches of code that support +older versions on python. For all other uses of "pragma: no cover" or +"pragma: no branch" document the reason why its not covered. "I haven't written +a test case" isn't a sufficient reason. + +Documentation +------------- + +Ensure this documentation is up to date after changes. Also ensure that the man +pages still are accurate. Ensure that there is sufficient documentation for +your new features to be used. + +Bugs +---- + +Remove them and don't add any more. + +Git +--- + +Use the following tags in your commit messages: + +'BF:' for bug fixes +'DOC:' for documentation fixes +'ENH:' for enhancements +'TST:' for commits concerning tests only (thus not touching the main code-base) + +Multiple tags could be joined with +, e.g. "BF+TST:". + +Adding Actions +-------------- + +If you add an action.d/*.conf file also add a example in config/jail.conf +with enabled=false and maxretry=5 for ssh. Design @@ -127,12 +213,14 @@ FileContainer .__pos Keeps the position pointer + +dnsutils.py +~~~~~~~~~~~ + DNSUtils Utility class for DNS and IP handling - RF-Note: convert to functions within a separate submodule - filter*.py ~~~~~~~~~~ @@ -156,3 +244,35 @@ action.py ~~~~~~~~~ Takes care about executing start/check/ban/unban/stop commands + + +Releasing +========= + +# Ensure the version is correct in ./common/version.py + +# Add/finalize the corresponding entry in the ChangeLog + +# Update man pages + + (cd man ; ./generate-man ) + git commit -m 'update man pages for release' man/* + +# Make sure the tests pass + + ./fail2ban-testcases-all + +# Prepare/upload source and rpm binary distributions + + python setup.py check + python setup.py sdist + python setup.py bdist_rpm + python setup.py upload + +# Run the following and update the wiki with output: + + python -c 'import common.protocol; common.protocol.printWiki()' + +# Email users and development list of release + +TODO notifying distributors etc. diff --git a/MANIFEST b/MANIFEST index 785edd5e..fb6f84e4 100644 --- a/MANIFEST +++ b/MANIFEST @@ -3,6 +3,8 @@ ChangeLog TODO THANKS COPYING +DEVELOP +doc/run-rootless.txt fail2ban-client fail2ban-server fail2ban-testcases @@ -41,6 +43,7 @@ server/banmanager.py server/datetemplate.py server/mytime.py server/failregex.py +testcases/files/testcase-usedns.log testcases/banmanagertestcase.py testcases/failmanagertestcase.py testcases/clientreadertestcase.py @@ -49,6 +52,7 @@ testcases/__init__.py testcases/datedetectortestcase.py testcases/actiontestcase.py testcases/servertestcase.py +testcases/sockettestcase.py testcases/files/testcase01.log testcases/files/testcase02.log testcases/files/testcase03.log @@ -56,6 +60,7 @@ testcases/files/testcase04.log setup.py setup.cfg common/__init__.py +common/exceptions.py common/helpers.py common/version.py common/protocol.py @@ -87,6 +92,17 @@ config/filter.d/vsftpd.conf config/filter.d/webmin-auth.conf config/filter.d/wuftpd.conf config/filter.d/xinetd-fail.conf +config/filter.d/asterisk.conf +config/filter.d/dovecot.conf +config/filter.d/dropbear.conf +config/filter.d/lighttpd-auth.conf +config/filter.d/recidive.conf +config/filter.d/roundcube-auth.conf +config/action.d/dummy.conf +config/action.d/iptables-ipset-proto4.conf +config/action.d/iptables-ipset-proto6.conf +config/action.d/iptables-xt_recent-echo.conf +config/action.d/route.conf config/action.d/complain.conf config/action.d/dshield.conf config/action.d/hostsdeny.conf @@ -109,6 +125,8 @@ config/action.d/sendmail-whois-lines.conf config/action.d/shorewall.conf config/fail2ban.conf man/fail2ban-client.1 +man/fail2ban.1 +man/jail.conf.5 man/fail2ban-client.h2m man/fail2ban-server.1 man/fail2ban-server.h2m diff --git a/README b/README index db97aa8b..cab683f5 100644 --- a/README +++ b/README @@ -51,10 +51,12 @@ call fail2ban-server directly. Configuration: -------------- -You can configure Fail2ban using the files in /etc/fail2ban. It is possible to -configure the server using commands sent to it by fail2ban-client. The available -commands are described in the man page of fail2ban-client. Please refer to it or -to the website: http://www.fail2ban.org +You can configure Fail2Ban using the files in /etc/fail2ban. It is +possible to configure the server using commands sent to it by +fail2ban-client. The available commands are described in the +fail2ban-client(1) manpage. Also see fail2ban(1) manpage for further +references and find even more documentation on the website: +http://www.fail2ban.org Contact: -------- @@ -91,5 +93,5 @@ WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details. You should have received a copy of the GNU General Public License along with -Fail2Ban; if not, write to the Free Software Foundation, Inc., 59 Temple Place, -Suite 330, Boston, MA 02111-1307 USA +Fail2Ban; if not, write to the Free Software Foundation, Inc., 51 Franklin +Street, Fifth Floor, Boston, MA 02110, USA diff --git a/THANKS b/THANKS index ff20d07d..f207d71e 100644 --- a/THANKS +++ b/THANKS @@ -13,6 +13,7 @@ Christian Rauch Christoph Haas Christos Psonis Daniel B. Cid +Daniel Black David Nutter Eric Gerbier Guillaume Delvit @@ -38,6 +39,7 @@ Robert Edeker Russell Odom Sireyessire Stephen Gildea +Steven Hiscocks Tom Pike Tyler Vaclav Misek diff --git a/TODO b/TODO index fc43dc2c..61bdc093 100644 --- a/TODO +++ b/TODO @@ -13,9 +13,12 @@ Legend: # partially done * done -- Removed relative imports +- Run tests though all filters/examples files - (see sshd example file) as unit + test -- Cleanup fail2ban-client and fail2ban-server. Move code to server/ and client/ +* Removed relative imports + +* Cleanup fail2ban-client and fail2ban-server. Move code to server/ and client/ - Add timeout to external commands (signal alarm, watchdog thread, etc) diff --git a/client/actionreader.py b/client/actionreader.py index 581a1b3c..9ad1ef28 100644 --- a/client/actionreader.py +++ b/client/actionreader.py @@ -35,8 +35,8 @@ logSys = logging.getLogger("fail2ban.client.config") class ActionReader(ConfigReader): - def __init__(self, action, name): - ConfigReader.__init__(self) + def __init__(self, action, name, **kwargs): + ConfigReader.__init__(self, **kwargs) self.__file = action[0] self.__cInfo = action[1] self.__name = name diff --git a/client/configreader.py b/client/configreader.py index 063484e8..243c843c 100644 --- a/client/configreader.py +++ b/client/configreader.py @@ -27,7 +27,7 @@ __date__ = "$Date$" __copyright__ = "Copyright (c) 2004 Cyril Jaquier" __license__ = "GPL" -import logging, os +import glob, logging, os from configparserinc import SafeConfigParserWithIncludes from ConfigParser import NoOptionError, NoSectionError @@ -35,36 +35,64 @@ from ConfigParser import NoOptionError, NoSectionError logSys = logging.getLogger("fail2ban.client.config") class ConfigReader(SafeConfigParserWithIncludes): + + DEFAULT_BASEDIR = '/etc/fail2ban' - BASE_DIRECTORY = "/etc/fail2ban/" - - def __init__(self): + def __init__(self, basedir=None): SafeConfigParserWithIncludes.__init__(self) + self.setBaseDir(basedir) self.__opts = None - #@staticmethod - def setBaseDir(folderName): - path = folderName.rstrip('/') - ConfigReader.BASE_DIRECTORY = path + '/' - setBaseDir = staticmethod(setBaseDir) - - #@staticmethod - def getBaseDir(): - return ConfigReader.BASE_DIRECTORY - getBaseDir = staticmethod(getBaseDir) + def setBaseDir(self, basedir): + if basedir is None: + basedir = ConfigReader.DEFAULT_BASEDIR # stock system location + self._basedir = basedir.rstrip('/') + + def getBaseDir(self): + return self._basedir def read(self, filename): - basename = ConfigReader.BASE_DIRECTORY + filename - logSys.debug("Reading " + basename) - bConf = basename + ".conf" - bLocal = basename + ".local" - if os.path.exists(bConf) or os.path.exists(bLocal): - SafeConfigParserWithIncludes.read(self, [bConf, bLocal]) + if not (os.path.exists(self._basedir) and os.access(self._basedir, os.R_OK | os.X_OK)): + raise ValueError("Base configuration directory %s either does not exist " + "or is not accessible" % self._basedir) + basename = os.path.join(self._basedir, filename) + logSys.debug("Reading configs for %s under %s " % (basename, self._basedir)) + config_files = [ basename + ".conf", + basename + ".local" ] + + # choose only existing ones + config_files = filter(os.path.exists, config_files) + + # possible further customizations under a .conf.d directory + config_dir = basename + '.d' + if os.path.exists(config_dir): + if os.path.isdir(config_dir) and os.access(config_dir, os.X_OK | os.R_OK): + # files must carry .conf suffix as well + config_files += sorted(glob.glob('%s/*.conf' % config_dir)) + else: + logSys.warn("%s exists but not a directory or not accessible" + % config_dir) + + # check if files are accessible, warn if any is not accessible + # and remove it from the list + config_files_accessible = [] + for f in config_files: + if os.access(f, os.R_OK): + config_files_accessible.append(f) + else: + logSys.warn("%s exists but not accessible - skipping" % f) + + if len(config_files_accessible): + # at least one config exists and accessible + SafeConfigParserWithIncludes.read(self, config_files_accessible) return True else: - logSys.error(bConf + " and " + bLocal + " do not exist") + logSys.error("Found no accessible config files for %r " % filename + + (["under %s" % self.getBaseDir(), + "among existing ones: " + ', '.join(config_files)][bool(len(config_files))])) + return False - + ## # Read the options. # @@ -94,8 +122,8 @@ class ConfigReader(SafeConfigParserWithIncludes): values[option[1]] = option[2] except NoOptionError: if not option[2] == None: - logSys.warn("'%s' not defined in '%s'. Using default value" - % (option[1], sec)) + logSys.warn("'%s' not defined in '%s'. Using default one: %r" + % (option[1], sec, option[2])) values[option[1]] = option[2] except ValueError: logSys.warn("Wrong value for '" + option[1] + "' in '" + sec + diff --git a/client/configurator.py b/client/configurator.py index 0baff2d8..2097fd54 100644 --- a/client/configurator.py +++ b/client/configurator.py @@ -43,15 +43,19 @@ class Configurator: self.__fail2ban = Fail2banReader() self.__jails = JailsReader() - #@staticmethod - def setBaseDir(folderName): - ConfigReader.setBaseDir(folderName) - setBaseDir = staticmethod(setBaseDir) + def setBaseDir(self, folderName): + self.__fail2ban.setBaseDir(folderName) + self.__jails.setBaseDir(folderName) - #@staticmethod - def getBaseDir(): - return ConfigReader.getBaseDir() - getBaseDir = staticmethod(getBaseDir) + def getBaseDir(self): + fail2ban_basedir = self.__fail2ban.getBaseDir() + jails_basedir = self.__jails.getBaseDir() + if fail2ban_basedir != jails_basedir: + logSys.error("fail2ban.conf and jails.conf readers have differing " + "basedirs: %r and %r. " + "Returning the one for fail2ban.conf" + % (fail2ban_basedir, jails_basedir)) + return fail2ban_basedir def readEarly(self): self.__fail2ban.read() diff --git a/client/fail2banreader.py b/client/fail2banreader.py index 6954b3bf..c8f42976 100644 --- a/client/fail2banreader.py +++ b/client/fail2banreader.py @@ -35,8 +35,8 @@ logSys = logging.getLogger("fail2ban.client.config") class Fail2banReader(ConfigReader): - def __init__(self): - ConfigReader.__init__(self) + def __init__(self, **kwargs): + ConfigReader.__init__(self, **kwargs) def read(self): ConfigReader.read(self, "fail2ban") diff --git a/client/filterreader.py b/client/filterreader.py index b7a72f9c..7dba3579 100644 --- a/client/filterreader.py +++ b/client/filterreader.py @@ -35,8 +35,8 @@ logSys = logging.getLogger("fail2ban.client.config") class FilterReader(ConfigReader): - def __init__(self, fileName, name): - ConfigReader.__init__(self) + def __init__(self, fileName, name, **kwargs): + ConfigReader.__init__(self, **kwargs) self.__file = fileName self.__name = name diff --git a/client/jailreader.py b/client/jailreader.py index be22a78f..3b825186 100644 --- a/client/jailreader.py +++ b/client/jailreader.py @@ -40,10 +40,11 @@ class JailReader(ConfigReader): actionCRE = re.compile("^((?:\w|-|_|\.)+)(?:\[(.*)\])?$") - def __init__(self, name): - ConfigReader.__init__(self) + def __init__(self, name, force_enable=False, **kwargs): + ConfigReader.__init__(self, **kwargs) self.__name = name self.__filter = None + self.__force_enable = force_enable self.__actions = list() def setName(self, value): @@ -53,10 +54,10 @@ class JailReader(ConfigReader): return self.__name def read(self): - ConfigReader.read(self, "jail") + return ConfigReader.read(self, "jail") def isEnabled(self): - return self.__opts["enabled"] + return self.__force_enable or self.__opts["enabled"] def getOptions(self): opts = [["bool", "enabled", "false"], @@ -76,7 +77,8 @@ class JailReader(ConfigReader): if self.isEnabled(): # Read filter - self.__filter = FilterReader(self.__opts["filter"], self.__name) + self.__filter = FilterReader(self.__opts["filter"], self.__name, + basedir=self.getBaseDir()) ret = self.__filter.read() if ret: self.__filter.getOptions(self.__opts) @@ -87,8 +89,10 @@ class JailReader(ConfigReader): # Read action for act in self.__opts["action"].split('\n'): try: + if not act: # skip empty actions + continue splitAct = JailReader.splitAction(act) - action = ActionReader(splitAct, self.__name) + action = ActionReader(splitAct, self.__name, basedir=self.getBaseDir()) ret = action.read() if ret: action.getOptions(self.__opts) @@ -97,8 +101,10 @@ class JailReader(ConfigReader): raise AttributeError("Unable to read action") except Exception, e: logSys.error("Error in action definition " + act) - logSys.debug(e) + logSys.debug("Caught exception: %s" % (e,)) return False + if not len(self.__actions): + logSys.warn("No actions were defined for %s" % self.__name) return True def convert(self): @@ -145,12 +151,20 @@ class JailReader(ConfigReader): def splitAction(action): m = JailReader.actionCRE.match(action) d = dict() - if not m.group(2) == None: + mgroups = m.groups() + if len(mgroups) == 2: + action_name, action_opts = mgroups + elif len(mgroups) == 1: + action_name, action_opts = mgroups[0], None + else: + raise ValueError("While reading action %s we should have got up to " + "2 groups. Got: %r" % (action, mgroups)) + if not action_opts is None: # Huge bad hack :( This method really sucks. TODO Reimplement it. actions = "" escapeChar = None allowComma = False - for c in m.group(2): + for c in action_opts: if c in ('"', "'") and not allowComma: # Start escapeChar = c @@ -175,6 +189,6 @@ class JailReader(ConfigReader): try: d[p[0].strip()] = p[1].strip() except IndexError: - logSys.error("Invalid argument %s in '%s'" % (p, m.group(2))) - return [m.group(1), d] + logSys.error("Invalid argument %s in '%s'" % (p, action_opts)) + return [action_name, d] splitAction = staticmethod(splitAction) diff --git a/client/jailsreader.py b/client/jailsreader.py index bedc5a3c..91e178d6 100644 --- a/client/jailsreader.py +++ b/client/jailsreader.py @@ -36,12 +36,20 @@ logSys = logging.getLogger("fail2ban.client.config") class JailsReader(ConfigReader): - def __init__(self): - ConfigReader.__init__(self) + def __init__(self, force_enable=False, **kwargs): + """ + Parameters + ---------- + force_enable : bool, optional + Passed to JailReader to force enable the jails. + It is for internal use + """ + ConfigReader.__init__(self, **kwargs) self.__jails = list() + self.__force_enable = force_enable def read(self): - ConfigReader.read(self, "jail") + return ConfigReader.read(self, "jail") def getOptions(self, section = None): opts = [] @@ -49,7 +57,7 @@ class JailsReader(ConfigReader): if section: # Get the options of a specific jail. - jail = JailReader(section) + jail = JailReader(section, basedir=self.getBaseDir(), force_enable=self.__force_enable) jail.read() ret = jail.getOptions() if ret: @@ -62,7 +70,7 @@ class JailsReader(ConfigReader): else: # Get the options of all jails. for sec in self.sections(): - jail = JailReader(sec) + jail = JailReader(sec, basedir=self.getBaseDir(), force_enable=self.__force_enable) jail.read() ret = jail.getOptions() if ret: diff --git a/common/protocol.py b/common/protocol.py index 99d608a8..1ac8bef5 100644 --- a/common/protocol.py +++ b/common/protocol.py @@ -40,6 +40,7 @@ protocol = [ ["stop", "stops all jails and terminate the server"], ["status", "gets the current status of the server"], ["ping", "tests if the server is alive"], +["help", "return this output"], ['', "LOGGING", ""], ["set loglevel ", "sets logging level to . 0 is minimal, 4 is debug"], ["get loglevel", "gets the logging level"], diff --git a/config/action.d/complain.conf b/config/action.d/complain.conf index 6677ec49..4c2de92b 100644 --- a/config/action.d/complain.conf +++ b/config/action.d/complain.conf @@ -52,10 +52,7 @@ actioncheck = # Option: actionban # Notes.: command executed when banning an IP. Take care that the # command is executed with Fail2Ban user rights. -# Tags: IP address -# number of failures -# unix timestamp of the last failure -# unix timestamp of the ban time +# Tags: See jail.conf(5) man page # Values: CMD # actionban = ADDRESSES=`whois | perl -e 'while () { next if /^changed|@(ripe|apnic)\.net/io; $m += (/abuse|trouble:|report|spam|security/io?3:0); if (/([a-z0-9_\-\.+]+@[a-z0-9\-]+(\.[[a-z0-9\-]+)+)/io) { while (s/([a-z0-9_\-\.+]+@[a-z0-9\-]+(\.[[a-z0-9\-]+)+)//io) { if ($m) { $a{lc($1)}=$m } else { $b{lc($1)}=$m } } $m=0 } else { $m && --$m } } if (%%a) {print join(",",keys(%%a))} else {print join(",",keys(%%b))}'` @@ -67,9 +64,7 @@ actionban = ADDRESSES=`whois | perl -e 'while () { next if /^changed # Option: actionunban # Notes.: command executed when unbanning an IP. Take care that the # command is executed with Fail2Ban user rights. -# Tags: IP address -# unix timestamp of the ban time -# unix timestamp of the unban time +# Tags: See jail.conf(5) man page # Values: CMD # actionunban = diff --git a/config/action.d/dshield.conf b/config/action.d/dshield.conf index 177329b2..151db28f 100644 --- a/config/action.d/dshield.conf +++ b/config/action.d/dshield.conf @@ -54,9 +54,7 @@ actioncheck = # Option: actionban # Notes.: command executed when banning an IP. Take care that the # command is executed with Fail2Ban user rights. -# Tags: IP address -# number of failures -#