diff --git a/.coveragerc b/.coveragerc index 3bffd79a..19bc3db4 100644 --- a/.coveragerc +++ b/.coveragerc @@ -1,4 +1,11 @@ [run] branch = True -omit = /usr* +source = + config + fail2ban + +[report] +exclude_lines = + pragma: no cover + pragma: systemd no cover diff --git a/.travis.yml b/.travis.yml index bad7e16e..f65e4896 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,29 +1,56 @@ # vim ft=yaml # travis-ci.org definition for Fail2Ban build +# https://travis-ci.org/fail2ban/fail2ban/ language: python python: - - "2.6" - - "2.7" - - "3.2" - - "3.3" - - "3.4" - - "pypy" - - "pypy3" + - 2.6 + - 2.7 + - pypy + - 3.2 + - 3.3 + - 3.4 + - pypy3 before_install: - - if [[ $TRAVIS_PYTHON_VERSION == 2.7 ]]; then travis_retry sudo apt-get update -qq; fi + - if [[ $TRAVIS_PYTHON_VERSION == 2* || $TRAVIS_PYTHON_VERSION == 'pypy' ]]; then export F2B_PY_2=true && echo "Set F2B_PY_2"; fi + - if [[ $TRAVIS_PYTHON_VERSION == 3* || $TRAVIS_PYTHON_VERSION == 'pypy3' ]]; then export F2B_PY_3=true && echo "Set F2B_PY_3"; fi + - travis_retry sudo apt-get update -qq + # Set this so sudo executes the correct python binary + # Anything not using sudo will already have the correct environment + - export VENV_BIN="$VIRTUAL_ENV/bin" && echo "VENV_BIN set to $VENV_BIN" install: + # Install Python packages / dependencies + # coverage + - travis_retry pip install coverage + # coveralls + - travis_retry pip install coveralls + # dnspython or dnspython3 + - if [[ "$F2B_PY_2" ]]; then travis_retry pip install dnspython; fi + - if [[ "$F2B_PY_3" ]]; then travis_retry pip install dnspython3; fi + # gamin - install manually (not in PyPI) - travis-ci system Python is 2.7 + - if [[ $TRAVIS_PYTHON_VERSION == 2.7 ]]; then travis_retry sudo apt-get install -qq python-gamin && cp /usr/share/pyshared/gamin.py /usr/lib/pyshared/python2.7/_gamin.so $VIRTUAL_ENV/lib/python2.7/site-packages/; fi + # pyinotify - travis_retry pip install pyinotify - - if [[ $TRAVIS_PYTHON_VERSION == 2* || $TRAVIS_PYTHON_VERSION == 'pypy' ]]; then travis_retry pip install dnspython; fi - - if [[ $TRAVIS_PYTHON_VERSION == 3* || $TRAVIS_PYTHON_VERSION == 'pypy3' ]]; then travis_retry pip install dnspython3; fi - - if [[ $TRAVIS_PYTHON_VERSION == 2.7 ]]; then travis_retry sudo apt-get install -qq python-gamin; cp /usr/share/pyshared/gamin.py /usr/lib/pyshared/python2.7/_gamin.so $VIRTUAL_ENV/lib/python2.7/site-packages/; fi - - if [[ $TRAVIS_PYTHON_VERSION == 2.7 ]]; then cd ..; travis_retry pip install -q coveralls; cd -; fi - # overcome buggy pypy - - if [[ $TRAVIS_PYTHON_VERSION == pypy ]] ; then dpkg --compare-versions $(pypy --version 2>&1 | awk '/PyPy/{print $2;}') ge 2.5.1 || { d=$PWD; cd /tmp; wget http://buildbot.pypy.org/nightly/trunk/pypy-c-jit-latest-linux64.tar.bz2; tar -xjvf pypy*bz2; cd pypy-*/bin/; export PATH=$PWD:$PATH; cd $d; } ; fi +before_script: + # Manually execute 2to3 for now + - if [[ "$F2B_PY_3" ]]; then ./fail2ban-2to3; fi script: - - if [[ $TRAVIS_PYTHON_VERSION == 2.7 ]]; then coverage run --rcfile=.travis_coveragerc setup.py test; else python setup.py test; fi -# test installation - - sudo python setup.py install + # Keep the legacy setup.py test approach of checking coverage for python2 + - if [[ "$F2B_PY_2" ]]; then coverage run setup.py test; fi + # Coverage doesn't pick up setup.py test with python3, so run it directly + - if [[ "$F2B_PY_3" ]]; then coverage run bin/fail2ban-testcases; fi + # Use $VENV_BIN (not python) or else sudo will always run the system's python (2.7) + - sudo $VENV_BIN/pip install . after_success: -# Coverage config file must be .coveragerc for coveralls - - if [[ $TRAVIS_PYTHON_VERSION == 2.7 ]]; then cp -v .travis_coveragerc .coveragerc; fi - - if [[ $TRAVIS_PYTHON_VERSION == 2.7 ]]; then coveralls; fi + - coveralls +matrix: + fast_finish: true +# Might be worth looking into +#notifications: +# email: true +# irc: +# channels: "irc.freenode.org#fail2ban" +# template: +# - "%{repository}@%{branch}: %{message} (%{build_url})" +# on_success: change +# on_failure: change +# skip_join: true diff --git a/.travis_coveragerc b/.travis_coveragerc deleted file mode 100644 index 49fc3134..00000000 --- a/.travis_coveragerc +++ /dev/null @@ -1,7 +0,0 @@ - -[run] -branch = True -omit = - /usr/* - /home/travis/virtualenv/* - fail2ban/server/filtersystemd.py diff --git a/ChangeLog b/ChangeLog index 8a81399b..3e80e7c3 100644 --- a/ChangeLog +++ b/ChangeLog @@ -6,6 +6,85 @@ Fail2Ban: Changelog =================== +ver. 0.9.3 (2015/08/01) - lets-all-stay-friends +---------- + +- IMPORTANT incompatible changes: + * filter.d/roundcube-auth.conf + - Changed logpath to 'errors' log (was 'userlogins') + * action.d/iptables-common.conf + - All calls to iptables command now use -w switch introduced in + iptables 1.4.20 (some distribution could have patched their + earlier base version as well) to provide this locking mechanism + useful under heavy load to avoid contesting on iptables calls. + If you need to disable, define 'action.d/iptables-common.local' + with empty value for 'lockingopt' in `[Init]` section. + * mail-whois-lines, sendmail-geoip-lines and sendmail-whois-lines + actions now include by default only the first 1000 log lines in + the emails. Adjust to augment the behavior. + +- Fixes: + * reload in interactive mode appends all the jails twice (gh-825) + * reload server/jail failed if database used (but was not changed) and + some jail active (gh-1072) + * filter.d/dovecot.conf - also match unknown user in passwd-file. + Thanks Anton Shestakov + * Fix fail2ban-regex not parsing journalmatch correctly from filter config + * filter.d/asterisk.conf - fix security log support for Asterisk 12+ + * filter.d/roundcube-auth.conf + - Updated regex to work with 'errors' log (1.0.5 and 1.1.1) + - Added regex to work with 'userlogins' log + * action.d/sendmail*.conf - use LC_ALL (superseeding LC_TIME) to override + locale on systems with customized LC_ALL + * performance fix: minimizes connection overhead, close socket only at + communication end (gh-1099) + * unbanip always deletes ip from database (independent of bantime, also if + currently not banned or persistent) + * guarantee order of dbfile to be before dbpurgeage (gh-1048) + * always set 'dbfile' before other database options (gh-1050) + * kill the entire process group of the child process upon timeout (gh-1129). + Otherwise could lead to resource exhaustion due to hanging whois + processes. + * resolve /var/run/fail2ban path in setup.py to help installation + on platforms with /var/run -> /run symlink (gh-1142) + +- New Features: + * RETURN iptables target is now a variable: + * New type of operation: pass2allow, use fail2ban for "knocking", + opening a closed port by swapping blocktype and returntype + * New filters: + - froxlor-auth - Thanks Joern Muehlencord + - apache-pass - filter Apache access log for successful authentication + * New actions: + - shorewall-ipset-proto6 - using proto feature of the Shorewall. Still requires + manual pre-configuration of the shorewall. See the action file for detail. + * New jails: + - pass2allow-ftp - allows FTP traffic after successful HTTP authentication + +- Enhancements: + * action.d/cloudflare.conf - improved documentation on how to allow + multiple CF accounts, and jail.conf got new compound action + definition action_cf_mwl to submit cloudflare report. + * Check access to socket for more detailed logging on error (gh-595) + * fail2ban-testcases man page + * filter.d/apache-badbots.conf, filter.d/nginx-botsearch.conf - add + HEAD method verb + * Revamp of Travis and coverage automated testing + * Added a space between IP address and the following colon + in notification emails for easier text selection + * Character detection heuristics for whois output via optional setting + in mail-whois*.conf. Thanks Thomas Mayer. + Not enabled by default, if _whois_command is set to be + %(_whois_convert_charset)s (e.g. in action.d/mail-whois-common.local), + it + - detects character set of whois output (which is undefined by + RFC 3912) via heuristics of the file command + - converts whois data to UTF-8 character set with iconv + - sends the whois output in UTF-8 character set to mail program + - avoids that heirloom mailx creates binary attachment for input with + unknown character set + + ver. 0.9.2 (2015/04/29) - better-quick-now-than-later ---------- @@ -42,7 +121,7 @@ ver. 0.9.2 (2015/04/29) - better-quick-now-than-later * firewallcmd-* actions: split output into separate lines for grepping (gh-908) * Guard unicode encode/decode issues while storing records in the database. Fixes "binding parameter error (unsupported type)" (gh-973), thanks to kot - for reporting + for reporting * filter.d/sshd added regex for matching openSUSE ssh authentication failure * filter.d/asterisk.conf: - Dropped "Sending fake auth rejection" failregex since it incorrectly diff --git a/DEVELOP b/DEVELOP index 1384a6ac..bb7de5c8 100644 --- a/DEVELOP +++ b/DEVELOP @@ -56,9 +56,12 @@ following (note: on Debian-based systems, the script is called `python-coverage`):: coverage run bin/fail2ban-testcases + coverage report + +Optionally: coverage html -Then look at htmlcov/index.html and see how much coverage your test cases +And then browse htmlcov/index.html and see how much coverage your test cases exert over the code base. Full coverage is a good thing however it may not be complete. Try to ensure tests cover as many independent paths through the code. @@ -88,7 +91,7 @@ Testing can now be done inside a vagrant VM. Vagrantfile provided in source code repository established two VMs: - VM "secure" which can be used for testing fail2ban code. -- VM "attacker" which hcan be used to perform attack against our "secure" VM. +- VM "attacker" which can be used to perform attack against our "secure" VM. Both VMs are sharing the 192.168.200/24 network. If you are using this network take a look into the Vagrantfile and change the IP. diff --git a/README.md b/README.md index 92dedd8c..fe941a63 100644 --- a/README.md +++ b/README.md @@ -2,7 +2,7 @@ / _|__ _(_) |_ ) |__ __ _ _ _ | _/ _` | | |/ /| '_ \/ _` | ' \ |_| \__,_|_|_/___|_.__/\__,_|_||_| - v0.9.2 2015/04/29 + v0.9.3 2015/08/01 ## Fail2Ban: ban hosts that cause multiple authentication errors @@ -37,8 +37,8 @@ Optional: To install, just do: - tar xvfj fail2ban-0.9.2.tar.bz2 - cd fail2ban-0.9.2 + tar xvfj fail2ban-0.9.3.tar.bz2 + cd fail2ban-0.9.3 python setup.py install This will install Fail2Ban into the python library directory. The executable diff --git a/RELEASE b/RELEASE index c008b1c6..d2a0d552 100644 --- a/RELEASE +++ b/RELEASE @@ -61,24 +61,24 @@ Preparation * Which indicates that testcases/files/logs/mysqld.log has been moved or is a directory:: - tar -C /tmp -jxf dist/fail2ban-0.9.2.tar.bz2 + tar -C /tmp -jxf dist/fail2ban-0.9.3.tar.bz2 * clean up current direcory:: - diff -rul --exclude \*.pyc . /tmp/fail2ban-0.9.2/ + diff -rul --exclude \*.pyc . /tmp/fail2ban-0.9.3/ * Only differences should be files that you don't want distributed. * Ensure the tests work from the tarball:: - cd /tmp/fail2ban-0.9.2/ && bin/fail2ban-testcases + cd /tmp/fail2ban-0.9.3/ && bin/fail2ban-testcases * Add/finalize the corresponding entry in the ChangeLog * To generate a list of committers use e.g.:: - git shortlog -sn 0.9.2.. | sed -e 's,^[ 0-9\t]*,,g' | tr '\n' '\|' | sed -e 's:|:, :g' + git shortlog -sn 0.9.3.. | sed -e 's,^[ 0-9\t]*,,g' | tr '\n' '\|' | sed -e 's:|:, :g' * Ensure the top of the ChangeLog has the right version and current date. * Ensure the top entry of the ChangeLog has the right version and current date. @@ -101,7 +101,7 @@ Preparation * Tag the release by using a signed (and annotated) tag. Cut/paste release ChangeLog entry as tag annotation:: - git tag -s 0.9.2 + git tag -s 0.9.3 Pre Release =========== @@ -185,7 +185,7 @@ Post Release Add the following to the top of the ChangeLog:: - ver. 0.9.3 (2014/XX/XXX) - wanna-be-released + ver. 0.9.4 (2014/XX/XXX) - wanna-be-released ----------- - Fixes: @@ -197,5 +197,5 @@ Add the following to the top of the ChangeLog:: Alter the git shortlog command in the previous section to refer to the just released version. -and adjust fail2ban/version.py to carry .dev suffix to signal +and adjust fail2ban/version.py to carry .dev0 suffix to signal a version under development. diff --git a/THANKS b/THANKS index 5ae86a3c..7bf723c5 100644 --- a/THANKS +++ b/THANKS @@ -109,6 +109,7 @@ Stefan Tatschner Stephen Gildea Steven Hiscocks TESTOVIK +Thomas Mayer Tom Pike Tomas Pihl Tony Lawrence diff --git a/bin/fail2ban-client b/bin/fail2ban-client index 866a5287..7f3f5639 100755 --- a/bin/fail2ban-client +++ b/bin/fail2ban-client @@ -22,8 +22,17 @@ __author__ = "Cyril Jaquier" __copyright__ = "Copyright (c) 2004 Cyril Jaquier" __license__ = "GPL" -import sys, string, os, pickle, re, logging, signal -import getopt, time, shlex, socket +import getopt +import logging +import os +import pickle +import re +import shlex +import signal +import socket +import string +import sys +import time from fail2ban.version import version from fail2ban.protocol import printFormatted @@ -144,32 +153,58 @@ class Fail2banClient: return self.__processCmd([["ping"]], False) def __processCmd(self, cmd, showRet = True): - beautifier = Beautifier() - streamRet = True - for c in cmd: - beautifier.setInputCmd(c) - try: - client = CSocket(self.__conf["socket"]) - ret = client.send(c) - if ret[0] == 0: - logSys.debug("OK : " + `ret[1]`) + client = None + try: + beautifier = Beautifier() + streamRet = True + for c in cmd: + beautifier.setInputCmd(c) + try: + if not client: + client = CSocket(self.__conf["socket"]) + ret = client.send(c) + if ret[0] == 0: + logSys.debug("OK : " + `ret[1]`) + if showRet: + print beautifier.beautify(ret[1]) + else: + logSys.error("NOK: " + `ret[1].args`) + if showRet: + print beautifier.beautifyError(ret[1]) + streamRet = False + except socket.error: if showRet: - print beautifier.beautify(ret[1]) - else: - logSys.error("NOK: " + `ret[1].args`) + self.__logSocketError() + return False + except Exception, e: if showRet: - print beautifier.beautifyError(ret[1]) - streamRet = False - except socket.error: - if showRet: - logSys.error("Unable to contact server. Is it running?") - return False - except Exception, e: - if showRet: - logSys.error(e) - return False + logSys.error(e) + return False + finally: + if client: + client.close() return streamRet + def __logSocketError(self): + try: + if os.access(self.__conf["socket"], os.F_OK): + # This doesn't check if path is a socket, + # but socket.error should be raised + if os.access(self.__conf["socket"], os.W_OK): + # Permissions look good, but socket.error was raised + logSys.error("Unable to contact server. Is it running?") + else: + logSys.error("Permission denied to socket: %s," + " (you must be root)", self.__conf["socket"]) + else: + logSys.error("Failed to access socket path: %s." + " Is fail2ban running?", + self.__conf["socket"]) + except Exception as e: + logSys.error("Exception while checking socket access: %s", + self.__conf["socket"]) + logSys.error(e) + ## # Process a command line. # diff --git a/bin/fail2ban-regex b/bin/fail2ban-regex index b337ab5d..b7b0579f 100755 --- a/bin/fail2ban-regex +++ b/bin/fail2ban-regex @@ -29,7 +29,15 @@ __author__ = "Fail2Ban Developers" __copyright__ = "Copyright (c) 2004-2008 Cyril Jaquier, 2012-2014 Yaroslav Halchenko" __license__ = "GPL" -import getopt, sys, time, logging, os, locale, shlex, time, urllib +import getopt +import locale +import logging +import os +import shlex +import sys +import time +import time +import urllib from optparse import OptionParser, Option from ConfigParser import NoOptionError, NoSectionError, MissingSectionHeaderError @@ -297,8 +305,8 @@ class Fail2banRegex(object): "read from %(value)s" % locals() return False elif command[2] == 'addjournalmatch': - journalmatch = command[3] - self.setJournalMatch(shlex.split(journalmatch)) + journalmatch = command[3:] + self.setJournalMatch(journalmatch) elif command[2] == 'datepattern': datepattern = command[3] self.setDatePattern(datepattern) diff --git a/bin/fail2ban-server b/bin/fail2ban-server index ec0c0dbe..f522f418 100755 --- a/bin/fail2ban-server +++ b/bin/fail2ban-server @@ -22,7 +22,9 @@ __author__ = "Cyril Jaquier" __copyright__ = "Copyright (c) 2004 Cyril Jaquier" __license__ = "GPL" -import getopt, sys, os +import getopt +import os +import sys from fail2ban.version import version from fail2ban.server.server import Server diff --git a/bin/fail2ban-testcases b/bin/fail2ban-testcases index 475aa40b..dd6547a5 100755 --- a/bin/fail2ban-testcases +++ b/bin/fail2ban-testcases @@ -25,7 +25,10 @@ __copyright__ = "Copyright (c) 2004 Cyril Jaquier, 2012- Yaroslav Halchenko" __license__ = "GPL" import logging -import unittest, sys, time, os +import os +import sys +import time +import unittest # Check if local fail2ban module exists, and use if it exists by # modifying the path. This is such that tests can be used in dev diff --git a/config/action.d/badips.py b/config/action.d/badips.py index c2a239f5..a1df00a3 100644 --- a/config/action.d/badips.py +++ b/config/action.d/badips.py @@ -35,6 +35,7 @@ else: from fail2ban.server.actions import ActionBase from fail2ban.version import version as f2bVersion + class BadIPsAction(ActionBase): """Fail2Ban action which reports bans to badips.com, and also blacklist bad IPs listed on badips.com by using another action's diff --git a/config/action.d/cloudflare.conf b/config/action.d/cloudflare.conf index 4d5e2dc8..4bc90c97 100644 --- a/config/action.d/cloudflare.conf +++ b/config/action.d/cloudflare.conf @@ -1,10 +1,14 @@ # # Author: Mike Rushton # -# Referenced from from http://www.normyee.net/blog/2012/02/02/adding-cloudflare-support-to-fail2ban by NORM YEE +# IMPORTANT # -# To get your Cloudflare API key: https://www.cloudflare.com/my-account +# Please set jail.local's permission to 640 because it contains your CF API key. # +# This action depends on curl. +# Referenced from http://www.normyee.net/blog/2012/02/02/adding-cloudflare-support-to-fail2ban by NORM YEE +# +# To get your CloudFlare API Key: https://www.cloudflare.com/a/account/my-account [Definition] @@ -34,7 +38,8 @@ actioncheck = #