From ebed0d23c93c3c2d74981eee60226fe52585a754 Mon Sep 17 00:00:00 2001 From: Ruben Kerkhof Date: Mon, 24 Mar 2014 14:20:49 +0000 Subject: [PATCH 001/141] Add documentation link to systemd service file So systemctl help fail2ban.service works Signed-off-by: Ruben Kerkhof --- files/fail2ban.service | 1 + 1 file changed, 1 insertion(+) diff --git a/files/fail2ban.service b/files/fail2ban.service index 7694bcf0..6ebbacc0 100644 --- a/files/fail2ban.service +++ b/files/fail2ban.service @@ -1,5 +1,6 @@ [Unit] Description=Fail2Ban Service +Documentation=man:fail2ban(1) After=network.target iptables.service firewalld.service [Service] From 1c36da9df975026dc7323f1e4ccf7a93ce3e7bd1 Mon Sep 17 00:00:00 2001 From: Ruben Kerkhof Date: Tue, 25 Mar 2014 10:57:20 +0000 Subject: [PATCH 002/141] Fix 2 more typos that codespell didn't catch --- config/action.d/complain.conf | 2 +- config/action.d/xarf-login-attack.conf | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/config/action.d/complain.conf b/config/action.d/complain.conf index c0175831..9247803e 100644 --- a/config/action.d/complain.conf +++ b/config/action.d/complain.conf @@ -5,7 +5,7 @@ # offending IP address. # This uses the https://abusix.com/contactdb.html to lookup abuse contacts. # -# DEPENDANCIES: +# DEPENDENCIES: # This requires the dig command from bind-utils # # You should provide the in the jail config - lines from the log diff --git a/config/action.d/xarf-login-attack.conf b/config/action.d/xarf-login-attack.conf index 1282d4c8..6d6a74f0 100644 --- a/config/action.d/xarf-login-attack.conf +++ b/config/action.d/xarf-login-attack.conf @@ -10,7 +10,7 @@ # password incorrectly. # * For filters that have a low likelihood of receiving human errors # -# DEPENDANCIES: +# DEPENDENCIES: # # This requires the dig command from bind-utils # From dc24d3d494f552c70df256085a12e90bbc612eb7 Mon Sep 17 00:00:00 2001 From: Steven Hiscocks Date: Sat, 29 Mar 2014 21:44:39 +0000 Subject: [PATCH 003/141] BF: On jail restart reinstatement of bans, fetch one ticket per IP Closes gh-664 --- ChangeLog | 1 + fail2ban/server/database.py | 61 +++++++++++++++++++++--------- fail2ban/server/jail.py | 2 +- fail2ban/tests/databasetestcase.py | 34 ++++++++++++++--- 4 files changed, 74 insertions(+), 24 deletions(-) diff --git a/ChangeLog b/ChangeLog index 22db55d9..43aefcb4 100644 --- a/ChangeLog +++ b/ChangeLog @@ -17,6 +17,7 @@ ver. 0.9.1 (2014/xx/xx) - better, faster, stronger * journalmatch for recidive incorrect PRIORITY * loglevel couldn't be changed in fail2ban.conf * Handle case when no sqlite library is available for persistent database + * Only reban once per IP from database on fail2ban restart - New features: diff --git a/fail2ban/server/database.py b/fail2ban/server/database.py index 0fad396b..93186222 100644 --- a/fail2ban/server/database.py +++ b/fail2ban/server/database.py @@ -361,7 +361,10 @@ class Fail2BanDb(object): ticket : BanTicket Ticket of the ban to be added. """ - self._bansMergedCache = {} + try: + del self._bansMergedCache[(ticket.getIP(), jail)] + except KeyError: + pass #TODO: Implement data parts once arbitrary match keys completed cur.execute( "INSERT INTO bans(jail, ip, timeofban, data) VALUES(?, ?, ?, ?)", @@ -383,7 +386,7 @@ class Fail2BanDb(object): if ip is not None: query += " AND ip=?" queryArgs.append(ip) - query += " ORDER BY timeofban" + query += " ORDER BY ip, timeofban" return cur.execute(query, queryArgs) @@ -412,7 +415,7 @@ class Fail2BanDb(object): tickets[-1].setAttempt(data['failures']) return tickets - def getBansMerged(self, ip, jail=None, **kwargs): + def getBansMerged(self, ip=None, jail=None, bantime=None): """Get bans from the database, merged into single ticket. This is the same as `getBans`, but bans merged into single @@ -430,22 +433,44 @@ class Fail2BanDb(object): Returns ------- - Ticket - Single ticket representing bans stored in database. + list or Ticket + Single ticket representing bans stored in database per IP + in a list. When `ip` argument passed, a single `Ticket` is + returned. """ - cacheKey = ip if jail is None else "%s|%s" % (ip, jail.name) - if cacheKey in self._bansMergedCache: - return self._bansMergedCache[cacheKey] - matches = [] - failures = 0 - for ip, timeofban, data in self._getBans(ip=ip, jail=jail, **kwargs): - #TODO: Implement data parts once arbitrary match keys completed - matches.extend(data['matches']) - failures += data['failures'] - ticket = FailTicket(ip, timeofban, matches) - ticket.setAttempt(failures) - self._bansMergedCache[cacheKey] = ticket - return ticket + if bantime is None: + cacheKey = (ip, jail) + if cacheKey in self._bansMergedCache: + return self._bansMergedCache[cacheKey] + + tickets = [] + ticket = None + + results = list(self._getBans(ip=ip, jail=jail, bantime=bantime)) + if results: + prev_banip = results[0][0] + matches = [] + failures = 0 + for banip, timeofban, data in results: + #TODO: Implement data parts once arbitrary match keys completed + if banip != prev_banip: + ticket = FailTicket(prev_banip, prev_timeofban, matches) + ticket.setAttempt(failures) + tickets.append(ticket) + # Reset variables + prev_banip = banip + matches = [] + failures = 0 + matches.extend(data['matches']) + failures += data['failures'] + prev_timeofban = timeofban + ticket = FailTicket(banip, prev_timeofban, matches) + ticket.setAttempt(failures) + tickets.append(ticket) + + if bantime is None: + self._bansMergedCache[cacheKey] = tickets if ip is None else ticket + return tickets if ip is None else ticket @commitandrollback def purge(self, cur): diff --git a/fail2ban/server/jail.py b/fail2ban/server/jail.py index dec04c73..a80fd60d 100644 --- a/fail2ban/server/jail.py +++ b/fail2ban/server/jail.py @@ -209,7 +209,7 @@ class Jail: self.actions.start() # Restore any previous valid bans from the database if self.database is not None: - for ticket in self.database.getBans( + for ticket in self.database.getBansMerged( jail=self, bantime=self.actions.getBanTime()): if not self.filter.inIgnoreIPList(ticket.getIP()): self.__queue.put(ticket) diff --git a/fail2ban/tests/databasetestcase.py b/fail2ban/tests/databasetestcase.py index 1cfd78b9..84101c50 100644 --- a/fail2ban/tests/databasetestcase.py +++ b/fail2ban/tests/databasetestcase.py @@ -190,16 +190,16 @@ class DatabaseTest(unittest.TestCase): jail2 = DummyJail() self.db.addJail(jail2) - ticket = FailTicket("127.0.0.1", 10, ["abc\n"]) + ticket = FailTicket("127.0.0.1", MyTime.time() - 40, ["abc\n"]) ticket.setAttempt(10) self.db.addBan(self.jail, ticket) - ticket = FailTicket("127.0.0.1", 20, ["123\n"]) + ticket = FailTicket("127.0.0.1", MyTime.time() - 30, ["123\n"]) ticket.setAttempt(20) self.db.addBan(self.jail, ticket) - ticket = FailTicket("127.0.0.2", 30, ["ABC\n"]) + ticket = FailTicket("127.0.0.2", MyTime.time() - 20, ["ABC\n"]) ticket.setAttempt(30) self.db.addBan(self.jail, ticket) - ticket = FailTicket("127.0.0.1", 40, ["ABC\n"]) + ticket = FailTicket("127.0.0.1", MyTime.time() - 10, ["ABC\n"]) ticket.setAttempt(40) self.db.addBan(jail2, ticket) @@ -220,7 +220,15 @@ class DatabaseTest(unittest.TestCase): id(ticket), id(self.db.getBansMerged("127.0.0.1", jail=self.jail))) - newTicket = FailTicket("127.0.0.1", 40, ["ABC\n"]) + newTicket = FailTicket("127.0.0.2", MyTime.time() - 20, ["ABC\n"]) + ticket.setAttempt(40) + # Add ticket, but not for same IP, so cache still valid + self.db.addBan(self.jail, newTicket) + self.assertEqual( + id(ticket), + id(self.db.getBansMerged("127.0.0.1", jail=self.jail))) + + newTicket = FailTicket("127.0.0.1", MyTime.time() - 10, ["ABC\n"]) ticket.setAttempt(40) self.db.addBan(self.jail, newTicket) # Added ticket, so cache should have been cleared @@ -228,6 +236,22 @@ class DatabaseTest(unittest.TestCase): id(ticket), id(self.db.getBansMerged("127.0.0.1", jail=self.jail))) + tickets = self.db.getBansMerged() + self.assertEqual(len(tickets), 2) + self.assertEqual( + sorted(list(set(ticket.getIP() for ticket in tickets))), + sorted([ticket.getIP() for ticket in tickets])) + + tickets = self.db.getBansMerged(jail=jail2) + self.assertEqual(len(tickets), 1) + + tickets = self.db.getBansMerged(bantime=25) + self.assertEqual(len(tickets), 2) + tickets = self.db.getBansMerged(bantime=15) + self.assertEqual(len(tickets), 1) + tickets = self.db.getBansMerged(bantime=5) + self.assertEqual(len(tickets), 0) + def testPurge(self): if Fail2BanDb is None: # pragma: no cover return From baeff6141ef8ab14f040999a503a3a7ce5cb6a1c Mon Sep 17 00:00:00 2001 From: Steven Hiscocks Date: Sat, 29 Mar 2014 22:07:33 +0000 Subject: [PATCH 004/141] DOC: sphinx documentation --- DEVELOP | 43 ++-- FILTERS | 191 +++++++++-------- RELEASE | 195 ++++++++++-------- doc/Makefile | 177 ++++++++++++++++ doc/conf.py | 260 ++++++++++++++++++++++++ doc/develop.rst | 1 + doc/fail2ban.client.actionreader.rst | 7 + doc/fail2ban.client.beautifier.rst | 7 + doc/fail2ban.client.configparserinc.rst | 7 + doc/fail2ban.client.configreader.rst | 7 + doc/fail2ban.client.configurator.rst | 7 + doc/fail2ban.client.csocket.rst | 7 + doc/fail2ban.client.fail2banreader.rst | 7 + doc/fail2ban.client.filterreader.rst | 7 + doc/fail2ban.client.jailreader.rst | 7 + doc/fail2ban.client.jailsreader.rst | 7 + doc/fail2ban.client.rst | 15 ++ doc/fail2ban.exceptions.rst | 7 + doc/fail2ban.helpers.rst | 7 + doc/fail2ban.protocol.rst | 7 + doc/fail2ban.rst | 12 ++ doc/fail2ban.server.action.rst | 7 + doc/fail2ban.server.actions.rst | 7 + doc/fail2ban.server.asyncserver.rst | 7 + doc/fail2ban.server.banmanager.rst | 7 + doc/fail2ban.server.database.rst | 7 + doc/fail2ban.server.datedetector.rst | 7 + doc/fail2ban.server.datetemplate.rst | 7 + doc/fail2ban.server.faildata.rst | 7 + doc/fail2ban.server.failmanager.rst | 7 + doc/fail2ban.server.failregex.rst | 7 + doc/fail2ban.server.filter.rst | 7 + doc/fail2ban.server.filtergamin.rst | 7 + doc/fail2ban.server.filterpoll.rst | 7 + doc/fail2ban.server.filterpyinotify.rst | 7 + doc/fail2ban.server.filtersystemd.rst | 7 + doc/fail2ban.server.jail.rst | 7 + doc/fail2ban.server.jails.rst | 7 + doc/fail2ban.server.jailthread.rst | 7 + doc/fail2ban.server.mytime.rst | 7 + doc/fail2ban.server.rst | 28 +++ doc/fail2ban.server.server.rst | 7 + doc/fail2ban.server.strptime.rst | 7 + doc/fail2ban.server.ticket.rst | 7 + doc/fail2ban.server.transmitter.rst | 7 + doc/fail2ban.version.rst | 7 + doc/filters.rst | 1 + doc/index.rst | 19 ++ doc/release.rst | 1 + doc/requirements.txt | 1 + fail2ban/server/action.py | 4 +- 51 files changed, 1010 insertions(+), 197 deletions(-) create mode 100644 doc/Makefile create mode 100644 doc/conf.py create mode 100644 doc/develop.rst create mode 100644 doc/fail2ban.client.actionreader.rst create mode 100644 doc/fail2ban.client.beautifier.rst create mode 100644 doc/fail2ban.client.configparserinc.rst create mode 100644 doc/fail2ban.client.configreader.rst create mode 100644 doc/fail2ban.client.configurator.rst create mode 100644 doc/fail2ban.client.csocket.rst create mode 100644 doc/fail2ban.client.fail2banreader.rst create mode 100644 doc/fail2ban.client.filterreader.rst create mode 100644 doc/fail2ban.client.jailreader.rst create mode 100644 doc/fail2ban.client.jailsreader.rst create mode 100644 doc/fail2ban.client.rst create mode 100644 doc/fail2ban.exceptions.rst create mode 100644 doc/fail2ban.helpers.rst create mode 100644 doc/fail2ban.protocol.rst create mode 100644 doc/fail2ban.rst create mode 100644 doc/fail2ban.server.action.rst create mode 100644 doc/fail2ban.server.actions.rst create mode 100644 doc/fail2ban.server.asyncserver.rst create mode 100644 doc/fail2ban.server.banmanager.rst create mode 100644 doc/fail2ban.server.database.rst create mode 100644 doc/fail2ban.server.datedetector.rst create mode 100644 doc/fail2ban.server.datetemplate.rst create mode 100644 doc/fail2ban.server.faildata.rst create mode 100644 doc/fail2ban.server.failmanager.rst create mode 100644 doc/fail2ban.server.failregex.rst create mode 100644 doc/fail2ban.server.filter.rst create mode 100644 doc/fail2ban.server.filtergamin.rst create mode 100644 doc/fail2ban.server.filterpoll.rst create mode 100644 doc/fail2ban.server.filterpyinotify.rst create mode 100644 doc/fail2ban.server.filtersystemd.rst create mode 100644 doc/fail2ban.server.jail.rst create mode 100644 doc/fail2ban.server.jails.rst create mode 100644 doc/fail2ban.server.jailthread.rst create mode 100644 doc/fail2ban.server.mytime.rst create mode 100644 doc/fail2ban.server.rst create mode 100644 doc/fail2ban.server.server.rst create mode 100644 doc/fail2ban.server.strptime.rst create mode 100644 doc/fail2ban.server.ticket.rst create mode 100644 doc/fail2ban.server.transmitter.rst create mode 100644 doc/fail2ban.version.rst create mode 100644 doc/filters.rst create mode 100644 doc/index.rst create mode 100644 doc/release.rst create mode 100644 doc/requirements.txt diff --git a/DEVELOP b/DEVELOP index 592dfcb4..f1426561 100644 --- a/DEVELOP +++ b/DEVELOP @@ -1,4 +1,4 @@ - __ _ _ ___ _ +.. __ _ _ ___ _ / _|__ _(_) |_ ) |__ __ _ _ _ | _/ _` | | |/ /| '_ \/ _` | ' \ |_| \__,_|_|_/___|_.__/\__,_|_||_| @@ -25,6 +25,7 @@ Pull Requests ============= When submitting pull requests on GitHub we ask you to: + * Clearly describe the problem you're solving; * Don't introduce regressions that will make it hard for systems administrators to update; @@ -52,32 +53,32 @@ for more details. Install the package python-coverage to visualise your test coverage. Run the following (note: on Debian-based systems, the script is called -`python-coverage`): +`python-coverage`):: -coverage run bin/fail2ban-testcases -coverage html + coverage run bin/fail2ban-testcases + coverage html Then look at htmlcov/index.html and see how much coverage your test cases exert over the code base. Full coverage is a good thing however it may not be complete. Try to ensure tests cover as many independent paths through the code. -Manual Execution. To run in a development environment do: +Manual Execution. To run in a development environment do:: -./fail2ban-client -c config/ -s /tmp/f2b.sock -i start + ./fail2ban-client -c config/ -s /tmp/f2b.sock -i start -some quick commands: +some quick commands:: -status -add test pyinotify -status test -set test addaction iptables -set test actionban iptables echo >> /tmp/ban -set test actionunban iptables echo >> /tmp/unban -get test actionban iptables -get test actionunban iptables -set test banip 192.168.2.2 -status test + status + add test pyinotify + status test + set test addaction iptables + set test actionban iptables echo >> /tmp/ban + set test actionunban iptables echo >> /tmp/unban + get test actionban iptables + get test actionunban iptables + set test banip 192.168.2.2 + status test @@ -130,10 +131,10 @@ Git Use the following tags in your commit messages: -'BF:' for bug fixes -'DOC:' for documentation fixes -'ENH:' for enhancements -'TST:' for commits concerning tests only (thus not touching the main code-base) +* 'BF:' for bug fixes +* 'DOC:' for documentation fixes +* 'ENH:' for enhancements +* 'TST:' for commits concerning tests only (thus not touching the main code-base) Multiple tags could be joined with +, e.g. "BF+TST:". diff --git a/FILTERS b/FILTERS index fd441e58..10113dfc 100644 --- a/FILTERS +++ b/FILTERS @@ -1,4 +1,4 @@ - __ _ _ ___ _ +.. __ _ _ ___ _ / _|__ _(_) |_ ) |__ __ _ _ _ | _/ _` | | |/ /| '_ \/ _` | ' \ |_| \__,_|_|_/___|_.__/\__,_|_||_| @@ -7,10 +7,8 @@ Developing Filters ================================================================================ -Filters -======= - Filters are tricky. They need to: + * work with a variety of the versions of the software that generates the logs; * work with the range of logging configuration options available in the software; @@ -31,10 +29,11 @@ what you have done, what is the hurdle, and we'll attempt to help (PR will be automagically updated with future commits you would push to complete it). -Filter test cases ------------------ +Filter Test Cases +================= -Purpose: +Purpose +------- Start by finding the log messages that the application generates related to some form of authentication failure. If you are adding to an existing filter @@ -49,7 +48,8 @@ and exim-spam at log messages related to spam. Even if it is a new filter you may consider separating the log messages into different filters based on purpose. -Cause: +Cause +----- Are some of the log lines a result of the same action? For example, is a PAM failure log message, followed by an application specific failure message the @@ -65,7 +65,8 @@ the log message be occurring due to the first step towards the application asking for authentication? Could the log messages occur often? If some of these are true make a note of this in the jail.conf example that you provide. -Samples: +Samples +------- It is important to include log file samples so any future change in the regular expression will still work with the log lines you have identified. @@ -93,21 +94,22 @@ If the mechanism to create the log message isn't obvious provide a configuration and/or sample scripts testcases/files/config/{filtername} and reference these in the comments above the log line. -FailJSON metadata: +FailJSON metadata +----------------- A failJSON metadata is a comment immediately above the log message. It will -look like: +look like:: -# failJSON: { "time": "2013-06-10T10:10:59", "match": true , "host": "93.184.216.119" } + # failJSON: { "time": "2013-06-10T10:10:59", "match": true , "host": "93.184.216.119" } Time should match the time of the log message. It is in a specific format of Year-Month-Day'T'Hour:minute:Second. If your log message does not include a year, like the example below, the year should be listed as 2005, if before Sun Aug 14 10am UTC, and 2004 if afterwards. Here is an example failJSON -line preceding a sample log line: +line preceding a sample log line:: -# failJSON: { "time": "2005-03-24T15:25:51", "match": true , "host": "198.51.100.87" } -Mar 24 15:25:51 buffalo1 dropbear[4092]: bad password attempt for 'root' from 198.51.100.87:5543 + # failJSON: { "time": "2005-03-24T15:25:51", "match": true , "host": "198.51.100.87" } + Mar 24 15:25:51 buffalo1 dropbear[4092]: bad password attempt for 'root' from 198.51.100.87:5543 The "host" in failJSON should contain the IP or domain that should be blocked. @@ -116,27 +118,28 @@ attacks) and any log lines to be excluded (see "Cause" section above), set "match": false in the failJSON and describe the reason in the comment above. After developing regexes, the following command will test all failJSON metadata -against the log lines in all sample log files +against the log lines in all sample log files:: -./fail2ban-testcases testSampleRegex + ./fail2ban-testcases testSampleRegex Developing Filter Regular Expressions -------------------------------------- +===================================== -Date/Time: +Date/Time +--------- At the moment, Fail2Ban depends on log lines to have time stamps. That is why before starting to develop failregex, check if your log line format known to Fail2Ban. Copy the time component from the log line and append an IP address to -test with following command: +test with following command:: -./fail2ban-regex "2013-09-19 02:46:12 1.2.3.4" "" + ./fail2ban-regex "2013-09-19 02:46:12 1.2.3.4" "" -Output of such command should contain something like: +Output of such command should contain something like:: -Date template hits: -|- [# of hits] date format -| [1] Year-Month-Day Hour:Minute:Second + Date template hits: + |- [# of hits] date format + | [1] Year-Month-Day Hour:Minute:Second Ensure that the template description matches time/date elements in your log line time stamp. If there is no matched format then date template needs to be added @@ -144,29 +147,31 @@ to server/datedetector.py. Ensure that a new template is added in the order that more specific matches occur first and that there is no confusion between a Day and a Month. -Filter file: +Filter file +----------- The filter is specified in a config/filter.d/{filtername}.conf file. Filter file -can have sections INCLUDES (optional) and Definition as follows: +can have sections INCLUDES (optional) and Definition as follows:: -[INCLUDES] - -before = common.conf - -after = filtername.local - -[Definition] - -failregex = .... - -ignoreregex = .... + [INCLUDES] + + before = common.conf + + after = filtername.local + + [Definition] + + failregex = .... + + ignoreregex = .... This is also documented in the man page jail.conf (section 5). Other definitions can be added to make failregex's more readable and maintainable to be used through string Interpolations (see http://docs.python.org/2.7/library/configparser.html) -General rules: +General rules +------------- Use "before" if you need to include a common set of rules, like syslog or if there is a common set of regexes for multiple filters. @@ -178,33 +183,35 @@ Try to avoid using ignoreregex mainly for performance reasons. The case when you would use it is if in trying to avoid using it, you end up with an unreadable failregex. -Syslog: +Syslog +------ If your application logs to syslog you can take advantage of log line prefix -definitions present in common.conf. So as a base use: +definitions present in common.conf. So as a base use:: -[INCLUDES] - -before = common.conf - -[Definition] - -_daemon = app - -failregex = ^%(__prefix_line)s + [INCLUDES] + + before = common.conf + + [Definition] + + _daemon = app + + failregex = ^%(__prefix_line)s In this example common.conf defines __prefix_line which also contains the _daemon name (in syslog terms the service) you have just specified. _daemon can also be a regex. -For example, to capture following line _daemon should be set to "dovecot" +For example, to capture following line _daemon should be set to "dovecot":: -Dec 12 11:19:11 dunnart dovecot: pop3-login: Aborted login (tried to use disabled plaintext auth): rip=190.210.136.21, lip=113.212.99.193 + Dec 12 11:19:11 dunnart dovecot: pop3-login: Aborted login (tried to use disabled plaintext auth): rip=190.210.136.21, lip=113.212.99.193 -and then ^%(__prefix_line)s would match "Dec 12 11:19:11 dunnart dovecot: +and then ``^%(__prefix_line)s`` would match "Dec 12 11:19:11 dunnart dovecot: ". Note it matches the trailing space(s) as well. -Substitutions (AKA string interpolations): +Substitutions (AKA string interpolations) +----------------------------------------- We have used string interpolations in above examples. They are useful for making the regexes more readable, reuse generic patterns in multiple failregex @@ -213,7 +220,8 @@ to the user. General principle is that value of a _name variable replaces occurrences of %(_name)s within the same section or anywhere in the config file if defined in [DEFAULT] section. -Regular Expressions: +Regular Expressions +------------------- Regular expressions (failregex, ignoreregex) assume that the date/time has been removed from the log line (this is just how fail2ban works internally ATM). @@ -236,29 +244,33 @@ If you have only a basic knowledge of regular repressions we advise to read http://docs.python.org/2/library/re.html first. It doesn't take long and would remind you e.g. which characters you need to escape and which you don't. -Developing/testing a regex: +Developing/testing a regex +-------------------------- You can develop a regex in a file or using command line depending on your preference. You can also use samples you have already created in the test cases or test them one at a time. The general tool for testing Fail2Ban regexes is fail2ban-regex. To see how to -use it run: +use it run:: -./fail2ban-regex --help + ./fail2ban-regex --help Take note of -l heavydebug / -l debug and -v as they might be very useful. -TIP: Take a look at the source code of the application you are developing +.. TIP:: + Take a look at the source code of the application you are developing failregex for. You may see optional or extra log messages, or parts there of, that need to form part of your regex. It may also reveal how some parts are constrained and different formats depending on configuration or less common usages. -TIP: For looking through source code - http://sourcecodebrowser.com/ . It has +.. TIP:: + For looking through source code - http://sourcecodebrowser.com/ . It has call graphs and can browse different versions. -TIP: Some applications log spaces at the end. If you are not sure add \s*$ as +.. TIP:: + Some applications log spaces at the end. If you are not sure add \s*$ as the end part of the regex. If your regex is not matching, http://www.debuggex.com/?flavor=python can help @@ -277,13 +289,15 @@ When you have fixed the regex put it back into your filter file. Please spread the good word about Debuggex - Serge Toarca is kindly continuing its free availability to Open Source developers. -Finishing up: +Finishing up +------------ If you've added a new filter, add a new entry in config/jail.conf. The theory here is that a user will create a jail.local with [filtername]\nenable=true to enable your jail. So more specifically in the [filter] section in jail.conf: + * ensure that you have "enabled = false" (users will enable as needed); * use "filter =" set to your filter name; * use a typical action to disable ports associated with the application; @@ -295,7 +309,7 @@ Submit github pull request (See "Pull Requests" above) for github.com/fail2ban/fail2ban containing your great work. Filter Security ---------------- +=============== Poor filter regular expressions are susceptible to DoS attacks. @@ -321,33 +335,33 @@ Examples of poor filters 1. Too restrictive -We find a log message: +We find a log message:: Apr-07-13 07:08:36 Invalid command fial2ban from 1.2.3.4 -We make a failregex +We make a failregex:: ^Invalid command \S+ from Now think evil. The user does the command 'blah from 1.2.3.44' -The program diligently logs: +The program diligently logs:: Apr-07-13 07:08:36 Invalid command blah from 1.2.3.44 from 1.2.3.4 And fail2ban matches 1.2.3.44 as the IP that it ban. A DoS attack was successful. -The fix here is that the command can be anything so .* is appropriate. +The fix here is that the command can be anything so .* is appropriate:: ^Invalid command .* from Here the .* will match until the end of the string. Then realise it has more to match, i.e. "from " and go back until it find this. Then it will ban -1.2.3.4 correctly. Since the is always at the end, end the regex with a $. +1.2.3.4 correctly. Since the is always at the end, end the regex with a $:: ^Invalid command .* from $ -Note if we'd just had the expression: +Note if we'd just had the expression:: ^Invalid command \S+ from $ @@ -359,16 +373,16 @@ banned. From the Apache vulnerability CVE-2013-2178 ( original ref: https://vndh.net/note:fail2ban-089-denial-service ). -An example bad regex for Apache: +An example bad regex for Apache:: failregex = [[]client []] user .* not found -Since the user can do a get request on: +Since the user can do a get request on:: GET /[client%20192.168.0.1]%20user%20root%20not%20found HTTP/1.0 -Host: remote.site + Host: remote.site -Now the log line will be: +Now the log line will be:: [Sat Jun 01 02:17:42 2013] [error] [client 192.168.33.1] File does not exist: /srv/http/site/[client 192.168.0.1] user root not found @@ -379,27 +393,27 @@ regex and blocks 192.168.33.1 as a denial of service from the HTTP requester. From: https://github.com/fail2ban/fail2ban/pull/426 -An example ssh log (simplified) +An example ssh log (simplified):: Sep 29 17:15:02 spaceman sshd[12946]: Failed password for user from 127.0.0.1 port 20000 ssh1: ruser remoteuser As we assume username can include anything including spaces its prudent to put -.* here. The remote user can also exist as anything so lets not make assumptions again. +.* here. The remote user can also exist as anything so lets not make assumptions again:: failregex = ^%(__prefix_line)sFailed \S+ for .* from ( port \d*)?( ssh\d+)?(: ruser .*)?$ So this works. The problem is if the .* after remote user is injected by the -user to be 'from 1.2.3.4'. The resultant log line is. +user to be 'from 1.2.3.4'. The resultant log line is:: Sep 29 17:15:02 spaceman sshd[12946]: Failed password for user from 127.0.0.1 port 20000 ssh1: ruser from 1.2.3.4 -Testing with: +Testing with:: fail2ban-regex -v 'Sep 29 17:15:02 Failed password for user from 127.0.0.1 port 20000 ssh1: ruser from 1.2.3.4' '^ Failed \S+ for .* from ( port \d*)?( ssh\d+)?(: ruser .*)?$' -TIP: I've removed the bit that matches __prefix_line from the regex and log. +.. TIP:: I've removed the bit that matches __prefix_line from the regex and log. -Shows: +Shows:: 1) [1] ^ Failed \S+ for .* from ( port \d*)?( ssh\d+)?(: ruser .*)?$ 1.2.3.4 Sun Sep 29 17:15:02 2013 @@ -412,14 +426,14 @@ The result was that 1.2.3.4 was matched, injected by the user, and the wrong IP was banned. The solution here is to make the first .* non-greedy with .*?. Here it matches -as little as required and the fail2ban-regex tool shows the output: +as little as required and the fail2ban-regex tool shows the output:: fail2ban-regex -v 'Sep 29 17:15:02 Failed password for user from 127.0.0.1 port 20000 ssh1: ruser from 1.2.3.4' '^ Failed \S+ for .*? from ( port \d*)?( ssh\d+)?(: ruser .*)?$' 1) [1] ^ Failed \S+ for .*? from ( port \d*)?( ssh\d+)?(: ruser .*)?$ 127.0.0.1 Sun Sep 29 17:15:02 2013 -So the general case here is a log line that contains: +So the general case here is a log line that contains:: (fixed_data_1)(fixed_data_2)(user_injectable_data) @@ -427,20 +441,21 @@ Where the regex that matches fixed_data_1 is gready and matches the entire string, before moving backwards and user_injectable_data can match the entire string. -Another case: +Another case +------------ ref: https://www.debuggex.com/r/CtAbeKMa2sDBEfA2/0 -A webserver logs the following without URL escaping: +A webserver logs the following without URL escaping:: [error] 2865#0: *66647 user "xyz" was not found in "/file", client: 1.2.3.1, server: www.host.com, request: "GET ", client: 3.2.1.1, server: fake.com, request: "GET exploited HTTP/3.3", host: "injected.host", host: "www.myhost.com" -regex: +regex:: failregex = ^ \[error\] \d+#\d+: \*\d+ user "\S+":? (?:password mismatch|was not found in ".*"), client: , server: \S+, request: "\S+ .+ HTTP/\d+\.\d+", host: "\S+" The .* matches to the end of the string. Finds that it can't continue to match -", client ... so it moves from the back and find that the user injected web URL: +", client ... so it moves from the back and find that the user injected web URL:: ", client: 3.2.1.1, server: fake.com, request: "GET exploited HTTP/3.3", host: "injected.host @@ -453,14 +468,14 @@ beyond . 4. Application generates two identical log messages with different meanings If the application generates the following two messages under different -circumstances: +circumstances:: client : authentication failed client : authentication failed -Then it's obvious that a regex of "^client : authentication -failed$" will still cause problems if the user can trigger the second +Then it's obvious that a regex of ``^client : authentication +failed$`` will still cause problems if the user can trigger the second log message with a of 123.1.1.1. Here there's nothing to do except request/change the application so it logs diff --git a/RELEASE b/RELEASE index e787b040..9002b75e 100644 --- a/RELEASE +++ b/RELEASE @@ -1,4 +1,4 @@ - __ _ _ ___ _ +.. __ _ _ ___ _ / _|__ _(_) |_ ) |__ __ _ _ _ | _/ _` | | |/ /| '_ \/ _` | ' \ |_| \__,_|_|_/___|_.__/\__,_|_||_| @@ -7,7 +7,10 @@ How to do a release for Fail2Ban ================================================================================ -# Check distribution patches and see if they can be included +Preparation +=========== + +* Check distribution patches and see if they can be included * https://apps.fedoraproject.org/packages/fail2ban/sources * http://sources.gentoo.org/cgi-bin/viewvc.cgi/gentoo-x86/net-analyzer/fail2ban/ @@ -16,7 +19,8 @@ How to do a release for Fail2Ban * http://sophie.zarb.org/sources/fail2ban (Mageia) * https://trac.macports.org/browser/trunk/dports/security/fail2ban -# Check distribution outstanding bugs + +* Check distribution outstanding bugs * https://github.com/fail2ban/fail2ban/issues?sort=updated&state=open * http://bugs.debian.org/cgi-bin/pkgreport.cgi?dist=unstable;package=fail2ban @@ -29,144 +33,165 @@ How to do a release for Fail2Ban * https://bugs.mageia.org/buglist.cgi?quicksearch=fail2ban * https://build.opensuse.org/package/requests/openSUSE:Factory/fail2ban -# Make sure the tests pass + +* Make sure the tests pass:: ./fail2ban-testcases-all -# Ensure the version is correct +* Ensure the version is correct in: - in: * ./fail2ban/version.py * top of ChangeLog * README.md -# Ensure the MANIFEST is complete -Run: +* Ensure the MANIFEST is complete + +* Run:: python setup.py sdist -Look for errors like: - 'testcases/files/logs/mysqld.log' not a regular file -- skipping +* Look for errors like:: -Which indicates that testcases/files/logs/mysqld.log has been moved or is a directory + 'testcases/files/logs/mysqld.log' not a regular file -- skipping - tar -C /tmp -jxf dist/fail2ban-0.9.0.tar.bz2 + * Which indicates that testcases/files/logs/mysqld.log has been moved or is a directory:: -# clean up current direcory + tar -C /tmp -jxf dist/fail2ban-0.9.0.tar.bz2 - diff -rul --exclude \*.pyc . /tmp/fail2ban-0.9.0/ +* clean up current direcory:: - # Only differences should be files that you don't want distributed. + diff -rul --exclude \*.pyc . /tmp/fail2ban-0.9.0/ -# Ensure the tests work from the tarball - - cd /tmp/fail2ban-0.9.0/ && export PYTHONPATH=`pwd` && bin/fail2ban-testcases - -# Add/finalize the corresponding entry in the ChangeLog - - To generate a list of committers use e.g. - - git shortlog -sn 0.8.12.. | sed -e 's,^[ 0-9\t]*,,g' | tr '\n' '\|' | sed -e 's:|:, :g' - - Ensure the top of the ChangeLog has the right version and current date. - - Ensure the top entry of the ChangeLog has the right version and current date. - -# Update man pages - - (cd man ; ./generate-man ) - git commit -m 'DOC/ENH: update man pages for release' man/* - -# Cleanout TODO file with the finished stuff - -# Prepare source and rpm binary distributions - - python setup.py sdist + * Only differences should be files that you don't want distributed. -Broken for now: python setup.py bdist_rpm -Broken for now: python setup.py upload +* Ensure the tests work from the tarball:: -# Tag the release by using a signed (and annotated) tag. Cut/paste - release ChangeLog entry as tag annotation + cd /tmp/fail2ban-0.9.0/ && export PYTHONPATH=`pwd` && bin/fail2ban-testcases - git tag -s 0.9.1 +* Add/finalize the corresponding entry in the ChangeLog -# Prerelease (option) + * To generate a list of committers use e.g.:: -# Provide a release sample to distributors + git shortlog -sn 0.8.12.. | sed -e 's,^[ 0-9\t]*,,g' | tr '\n' '\|' | sed -e 's:|:, :g' + + * Ensure the top of the ChangeLog has the right version and current date. + * Ensure the top entry of the ChangeLog has the right version and current date. + +* Update man pages:: + + (cd man ; ./generate-man ) + git commit -m 'DOC/ENH: update man pages for release' man/* + +* Cleanout TODO file with the finished stuff + +* Prepare source and rpm binary distributions:: + + python setup.py sdist + + * Broken for now: python setup.py bdist_rpm + * Broken for now: python setup.py upload + + +* Tag the release by using a signed (and annotated) tag. Cut/paste + release ChangeLog entry as tag annotation:: + + git tag -s 0.9.1 + +Pre Release +=========== + +* Provide a release sample to distributors * Arch Linux: - https://www.archlinux.org/packages/community/any/fail2ban/ + + * https://www.archlinux.org/packages/community/any/fail2ban/ + * Debian: Yaroslav Halchenko - http://packages.qa.debian.org/f/fail2ban.html + + * http://packages.qa.debian.org/f/fail2ban.html + * FreeBSD: Christoph Theis theis@gmx.at>, Nick Hilliard - http://svnweb.freebsd.org/ports/head/security/py-fail2ban/Makefile?view=markup - http://www.freebsd.org/cgi/query-pr-summary.cgi?text=fail2ban + + * http://svnweb.freebsd.org/ports/head/security/py-fail2ban/Makefile?view=markup + * http://www.freebsd.org/cgi/query-pr-summary.cgi?text=fail2ban + * Fedora: Axel Thimm - https://apps.fedoraproject.org/packages/fail2ban - http://pkgs.fedoraproject.org/cgit/fail2ban.git - https://admin.fedoraproject.org/pkgdb/acls/bugs/fail2ban + + * https://apps.fedoraproject.org/packages/fail2ban + * http://pkgs.fedoraproject.org/cgit/fail2ban.git + * https://admin.fedoraproject.org/pkgdb/acls/bugs/fail2ban + * Gentoo: netmon@gentoo.org - http://sources.gentoo.org/cgi-bin/viewvc.cgi/gentoo-x86/net-analyzer/fail2ban/metadata.xml?view=markup - https://bugs.gentoo.org/buglist.cgi?quicksearch=fail2ban + + * http://sources.gentoo.org/cgi-bin/viewvc.cgi/gentoo-x86/net-analyzer/fail2ban/metadata.xml?view=markup + * https://bugs.gentoo.org/buglist.cgi?quicksearch=fail2ban + * openSUSE: Stephan Kulow - https://build.opensuse.org/package/show/openSUSE:Factory/fail2ban + + * https://build.opensuse.org/package/show/openSUSE:Factory/fail2ban + * Mac Ports: @Malbrouck on github (gh-49) - https://trac.macports.org/browser/trunk/dports/security/fail2ban/Portfile + + * https://trac.macports.org/browser/trunk/dports/security/fail2ban/Portfile + * Mageia: - https://bugs.mageia.org/buglist.cgi?quicksearch=fail2ban - An potentially to the fail2ban-users email list. + * https://bugs.mageia.org/buglist.cgi?quicksearch=fail2ban -# Wait for feedback from distributors + * An potentially to the fail2ban-users email list. -# Prepare a release notice https://github.com/fail2ban/fail2ban/releases/new - Upload the source/binaries from the dist directory and tag the release using the URL +* Wait for feedback from distributors -# Upload source/binaries to sourceforge http://sourceforge.net/projects/fail2ban/ +* Prepare a release notice https://github.com/fail2ban/fail2ban/releases/new -# Run the following and update the wiki with output: - python -c 'import fail2ban.protocol; fail2ban.protocol.printWiki()' +* Upload the source/binaries from the dist directory and tag the release using the URL + +* Upload source/binaries to sourceforge http://sourceforge.net/projects/fail2ban/ + +* Run the following and update the wiki with output:: + + python -c 'import fail2ban.protocol; fail2ban.protocol.printWiki()' + + * page: http://www.fail2ban.org/wiki/index.php/Commands - page: http://www.fail2ban.org/wiki/index.php/Commands * Update: - http://www.fail2ban.org/wiki/index.php?title=Template:Fail2ban_Versions&action=edit - http://www.fail2ban.org/wiki/index.php?title=Template:Fail2ban_News&action=edit - move old bits to: - http://www.fail2ban.org/wiki/index.php?title=Template:Fail2ban_OldNews&action=edit + * http://www.fail2ban.org/wiki/index.php?title=Template:Fail2ban_Versions&action=edit - http://www.fail2ban.org/wiki/index.php/ChangeLog - http://www.fail2ban.org/wiki/index.php/Requirements (Check requirement) - http://www.fail2ban.org/wiki/index.php/Features + * http://www.fail2ban.org/wiki/index.php?title=Template:Fail2ban_News&action=edit + * move old bits to http://www.fail2ban.org/wiki/index.php?title=Template:Fail2ban_OldNews&action=edit + + * http://www.fail2ban.org/wiki/index.php/ChangeLog + * http://www.fail2ban.org/wiki/index.php/Requirements (Check requirement) + * http://www.fail2ban.org/wiki/index.php/Features * See if any filters are upgraded: http://www.fail2ban.org/wiki/index.php/Special:AllPages -# Email users and development list of release +* Email users and development list of release -# notify distributors +* notify distributors Post Release ============ -Add the following to the top of the ChangeLog +Add the following to the top of the ChangeLog:: -ver. 0.9.1 (2014/XX/XXX) - wanna-be-released ------------ - -- Fixes: - -- New Features: - -- Enhancements: + ver. 0.9.1 (2014/XX/XXX) - wanna-be-released + ----------- + + - Fixes: + + - New Features: + + - Enhancements: Alter the git shortlog command in the previous section to refer to the just released version. -and adjust common/version.py to carry .dev suffix to signal +and adjust fail2ban/version.py to carry .dev suffix to signal a version under development. diff --git a/doc/Makefile b/doc/Makefile new file mode 100644 index 00000000..e2276ba9 --- /dev/null +++ b/doc/Makefile @@ -0,0 +1,177 @@ +# Makefile for Sphinx documentation +# + +# You can set these variables from the command line. +SPHINXOPTS = +SPHINXBUILD = sphinx-build2 +PAPER = +BUILDDIR = ""build + +# User-friendly check for sphinx-build2 +ifeq ($(shell which $(SPHINXBUILD) >/dev/null 2>&1; echo $$?), 1) +$(error The '$(SPHINXBUILD)' command was not found. Make sure you have Sphinx installed, then set the SPHINXBUILD environment variable to point to the full path of the '$(SPHINXBUILD)' executable. Alternatively you can add the directory with the executable to your PATH. If you don't have Sphinx installed, grab it from http://sphinx-doc.org/) +endif + +# Internal variables. +PAPEROPT_a4 = -D latex_paper_size=a4 +PAPEROPT_letter = -D latex_paper_size=letter +ALLSPHINXOPTS = -d $(BUILDDIR)/doctrees $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) . +# the i18n builder cannot share the environment and doctrees with the others +I18NSPHINXOPTS = $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) . + +.PHONY: help clean html dirhtml singlehtml pickle json htmlhelp qthelp devhelp epub latex latexpdf text man changes linkcheck doctest gettext + +help: + @echo "Please use \`make ' where is one of" + @echo " html to make standalone HTML files" + @echo " dirhtml to make HTML files named index.html in directories" + @echo " singlehtml to make a single large HTML file" + @echo " pickle to make pickle files" + @echo " json to make JSON files" + @echo " htmlhelp to make HTML files and a HTML help project" + @echo " qthelp to make HTML files and a qthelp project" + @echo " devhelp to make HTML files and a Devhelp project" + @echo " epub to make an epub" + @echo " latex to make LaTeX files, you can set PAPER=a4 or PAPER=letter" + @echo " latexpdf to make LaTeX files and run them through pdflatex" + @echo " latexpdfja to make LaTeX files and run them through platex/dvipdfmx" + @echo " text to make text files" + @echo " man to make manual pages" + @echo " texinfo to make Texinfo files" + @echo " info to make Texinfo files and run them through makeinfo" + @echo " gettext to make PO message catalogs" + @echo " changes to make an overview of all changed/added/deprecated items" + @echo " xml to make Docutils-native XML files" + @echo " pseudoxml to make pseudoxml-XML files for display purposes" + @echo " linkcheck to check all external links for integrity" + @echo " doctest to run all doctests embedded in the documentation (if enabled)" + +clean: + rm -rf $(BUILDDIR)/* + +html: + $(SPHINXBUILD) -b html $(ALLSPHINXOPTS) $(BUILDDIR)/html + @echo + @echo "Build finished. The HTML pages are in $(BUILDDIR)/html." + +dirhtml: + $(SPHINXBUILD) -b dirhtml $(ALLSPHINXOPTS) $(BUILDDIR)/dirhtml + @echo + @echo "Build finished. The HTML pages are in $(BUILDDIR)/dirhtml." + +singlehtml: + $(SPHINXBUILD) -b singlehtml $(ALLSPHINXOPTS) $(BUILDDIR)/singlehtml + @echo + @echo "Build finished. The HTML page is in $(BUILDDIR)/singlehtml." + +pickle: + $(SPHINXBUILD) -b pickle $(ALLSPHINXOPTS) $(BUILDDIR)/pickle + @echo + @echo "Build finished; now you can process the pickle files." + +json: + $(SPHINXBUILD) -b json $(ALLSPHINXOPTS) $(BUILDDIR)/json + @echo + @echo "Build finished; now you can process the JSON files." + +htmlhelp: + $(SPHINXBUILD) -b htmlhelp $(ALLSPHINXOPTS) $(BUILDDIR)/htmlhelp + @echo + @echo "Build finished; now you can run HTML Help Workshop with the" \ + ".hhp project file in $(BUILDDIR)/htmlhelp." + +qthelp: + $(SPHINXBUILD) -b qthelp $(ALLSPHINXOPTS) $(BUILDDIR)/qthelp + @echo + @echo "Build finished; now you can run "qcollectiongenerator" with the" \ + ".qhcp project file in $(BUILDDIR)/qthelp, like this:" + @echo "# qcollectiongenerator $(BUILDDIR)/qthelp/Fail2Ban.qhcp" + @echo "To view the help file:" + @echo "# assistant -collectionFile $(BUILDDIR)/qthelp/Fail2Ban.qhc" + +devhelp: + $(SPHINXBUILD) -b devhelp $(ALLSPHINXOPTS) $(BUILDDIR)/devhelp + @echo + @echo "Build finished." + @echo "To view the help file:" + @echo "# mkdir -p $$HOME/.local/share/devhelp/Fail2Ban" + @echo "# ln -s $(BUILDDIR)/devhelp $$HOME/.local/share/devhelp/Fail2Ban" + @echo "# devhelp" + +epub: + $(SPHINXBUILD) -b epub $(ALLSPHINXOPTS) $(BUILDDIR)/epub + @echo + @echo "Build finished. The epub file is in $(BUILDDIR)/epub." + +latex: + $(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex + @echo + @echo "Build finished; the LaTeX files are in $(BUILDDIR)/latex." + @echo "Run \`make' in that directory to run these through (pdf)latex" \ + "(use \`make latexpdf' here to do that automatically)." + +latexpdf: + $(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex + @echo "Running LaTeX files through pdflatex..." + $(MAKE) -C $(BUILDDIR)/latex all-pdf + @echo "pdflatex finished; the PDF files are in $(BUILDDIR)/latex." + +latexpdfja: + $(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex + @echo "Running LaTeX files through platex and dvipdfmx..." + $(MAKE) -C $(BUILDDIR)/latex all-pdf-ja + @echo "pdflatex finished; the PDF files are in $(BUILDDIR)/latex." + +text: + $(SPHINXBUILD) -b text $(ALLSPHINXOPTS) $(BUILDDIR)/text + @echo + @echo "Build finished. The text files are in $(BUILDDIR)/text." + +man: + $(SPHINXBUILD) -b man $(ALLSPHINXOPTS) $(BUILDDIR)/man + @echo + @echo "Build finished. The manual pages are in $(BUILDDIR)/man." + +texinfo: + $(SPHINXBUILD) -b texinfo $(ALLSPHINXOPTS) $(BUILDDIR)/texinfo + @echo + @echo "Build finished. The Texinfo files are in $(BUILDDIR)/texinfo." + @echo "Run \`make' in that directory to run these through makeinfo" \ + "(use \`make info' here to do that automatically)." + +info: + $(SPHINXBUILD) -b texinfo $(ALLSPHINXOPTS) $(BUILDDIR)/texinfo + @echo "Running Texinfo files through makeinfo..." + make -C $(BUILDDIR)/texinfo info + @echo "makeinfo finished; the Info files are in $(BUILDDIR)/texinfo." + +gettext: + $(SPHINXBUILD) -b gettext $(I18NSPHINXOPTS) $(BUILDDIR)/locale + @echo + @echo "Build finished. The message catalogs are in $(BUILDDIR)/locale." + +changes: + $(SPHINXBUILD) -b changes $(ALLSPHINXOPTS) $(BUILDDIR)/changes + @echo + @echo "The overview file is in $(BUILDDIR)/changes." + +linkcheck: + $(SPHINXBUILD) -b linkcheck $(ALLSPHINXOPTS) $(BUILDDIR)/linkcheck + @echo + @echo "Link check complete; look for any errors in the above output " \ + "or in $(BUILDDIR)/linkcheck/output.txt." + +doctest: + $(SPHINXBUILD) -b doctest $(ALLSPHINXOPTS) $(BUILDDIR)/doctest + @echo "Testing of doctests in the sources finished, look at the " \ + "results in $(BUILDDIR)/doctest/output.txt." + +xml: + $(SPHINXBUILD) -b xml $(ALLSPHINXOPTS) $(BUILDDIR)/xml + @echo + @echo "Build finished. The XML files are in $(BUILDDIR)/xml." + +pseudoxml: + $(SPHINXBUILD) -b pseudoxml $(ALLSPHINXOPTS) $(BUILDDIR)/pseudoxml + @echo + @echo "Build finished. The pseudo-XML files are in $(BUILDDIR)/pseudoxml." diff --git a/doc/conf.py b/doc/conf.py new file mode 100644 index 00000000..20845a5a --- /dev/null +++ b/doc/conf.py @@ -0,0 +1,260 @@ +# -*- coding: utf-8 -*- + +import sys +import os + +sys.path.insert(0, ".") +sys.path.insert(0, "..") + +# If extensions (or modules to document with autodoc) are in another directory, +# add these directories to sys.path here. If the directory is relative to the +# documentation root, use os.path.abspath to make it absolute, like shown here. +#sys.path.insert(0, os.path.abspath('.')) + +# -- General configuration ------------------------------------------------ + +# If your documentation needs a minimal Sphinx version, state it here. +#needs_sphinx = '1.0' + +# Add any Sphinx extension module names here, as strings. They can be +# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom +# ones. +extensions = [ + 'sphinx.ext.autodoc', + 'sphinx.ext.autosummary', + 'numpydoc', +] + +# Add any paths that contain templates here, relative to this directory. +templates_path = ['templates'] + +# The suffix of source filenames. +source_suffix = '.rst' + +# The encoding of source files. +#source_encoding = 'utf-8-sig' + +# The master toctree document. +master_doc = 'index' + +# General information about the project. +project = u'Fail2Ban' +copyright = u'2014' + +# The version info for the project you're documenting, acts as replacement for +# |version| and |release|, also used in various other places throughout the +# built documents. +# + +from fail2ban.version import version as fail2ban_version +from distutils.version import LooseVersion + +fail2ban_loose_version = LooseVersion(fail2ban_version) + +# The short X.Y version. +version = ".".join(str(_) for _ in fail2ban_loose_version.version[:2]) +# The full version, including alpha/beta/rc tags. +release = fail2ban_version + +# The language for content autogenerated by Sphinx. Refer to documentation +# for a list of supported languages. +#language = None + +# There are two options for replacing |today|: either, you set today to some +# non-false value, then it is used: +#today = '' +# Else, today_fmt is used as the format for a strftime call. +#today_fmt = '%B %d, %Y' + +# List of patterns, relative to source directory, that match files and +# directories to ignore when looking for source files. +exclude_patterns = ['build'] + +# The reST default role (used for this markup: `text`) to use for all +# documents. +#default_role = None + +# If true, '()' will be appended to :func: etc. cross-reference text. +#add_function_parentheses = True + +# If true, the current module name will be prepended to all description +# unit titles (such as .. function::). +#add_module_names = True + +# If true, sectionauthor and moduleauthor directives will be shown in the +# output. They are ignored by default. +#show_authors = False + +# The name of the Pygments (syntax highlighting) style to use. +pygments_style = 'sphinx' + +# A list of ignored prefixes for module index sorting. +#modindex_common_prefix = [] + +# If true, keep warnings as "system message" paragraphs in the built documents. +#keep_warnings = False + + +# -- Options for HTML output ---------------------------------------------- + +# The theme to use for HTML and HTML Help pages. See the documentation for +# a list of builtin themes. +html_theme = 'default' + +# Theme options are theme-specific and customize the look and feel of a theme +# further. For a list of options available for each theme, see the +# documentation. +#html_theme_options = {} + +# Add any paths that contain custom themes here, relative to this directory. +#html_theme_path = [] + +# The name for this set of Sphinx documents. If None, it defaults to +# " v documentation". +#html_title = None + +# A shorter title for the navigation bar. Default is the same as html_title. +#html_short_title = None + +# The name of an image file (relative to this directory) to place at the top +# of the sidebar. +#html_logo = None + +# The name of an image file (within the static path) to use as favicon of the +# docs. This file should be a Windows icon file (.ico) being 16x16 or 32x32 +# pixels large. +#html_favicon = None + +# Add any paths that contain custom static files (such as style sheets) here, +# relative to this directory. They are copied after the builtin static files, +# so a file named "default.css" will overwrite the builtin "default.css". +html_static_path = ['static'] + +# Add any extra paths that contain custom files (such as robots.txt or +# .htaccess) here, relative to this directory. These files are copied +# directly to the root of the documentation. +#html_extra_path = [] + +# If not '', a 'Last updated on:' timestamp is inserted at every page bottom, +# using the given strftime format. +#html_last_updated_fmt = '%b %d, %Y' + +# If true, SmartyPants will be used to convert quotes and dashes to +# typographically correct entities. +#html_use_smartypants = True + +# Custom sidebar templates, maps document names to template names. +#html_sidebars = {} + +# Additional templates that should be rendered to pages, maps page names to +# template names. +#html_additional_pages = {} + +# If false, no module index is generated. +#html_domain_indices = True + +# If false, no index is generated. +#html_use_index = True + +# If true, the index is split into individual pages for each letter. +#html_split_index = False + +# If true, links to the reST sources are added to the pages. +#html_show_sourcelink = True + +# If true, "Created using Sphinx" is shown in the HTML footer. Default is True. +#html_show_sphinx = True + +# If true, "(C) Copyright ..." is shown in the HTML footer. Default is True. +#html_show_copyright = True + +# If true, an OpenSearch description file will be output, and all pages will +# contain a tag referring to it. The value of this option must be the +# base URL from which the finished HTML is served. +#html_use_opensearch = '' + +# This is the file name suffix for HTML files (e.g. ".xhtml"). +#html_file_suffix = None + +# Output file base name for HTML help builder. +htmlhelp_basename = 'Fail2Bandoc' + + +# -- Options for LaTeX output --------------------------------------------- + +latex_elements = { +# The paper size ('letterpaper' or 'a4paper'). +#'papersize': 'letterpaper', + +# The font size ('10pt', '11pt' or '12pt'). +#'pointsize': '10pt', + +# Additional stuff for the LaTeX preamble. +#'preamble': '', +} + +# Grouping the document tree into LaTeX files. List of tuples +# (source start file, target name, title, +# author, documentclass [howto, manual, or own class]). +latex_documents = [ + ('index', 'Fail2Ban.tex', u'Fail2Ban Developers\' Documentation', + u'', 'manual'), +] + +# The name of an image file (relative to this directory) to place at the top of +# the title page. +#latex_logo = None + +# For "manual" documents, if this is true, then toplevel headings are parts, +# not chapters. +#latex_use_parts = False + +# If true, show page references after internal links. +#latex_show_pagerefs = False + +# If true, show URL addresses after external links. +#latex_show_urls = False + +# Documents to append as an appendix to all manuals. +#latex_appendices = [] + +# If false, no module index is generated. +#latex_domain_indices = True + + +# -- Options for manual page output --------------------------------------- + +# One entry per manual page. List of tuples +# (source start file, name, description, authors, manual section). +man_pages = [ + ('index', 'fail2ban', u'Fail2Ban Developers\' Documentation', + [u''], 1) +] + +# If true, show URL addresses after external links. +#man_show_urls = False + + +# -- Options for Texinfo output ------------------------------------------- + +# Grouping the document tree into Texinfo files. List of tuples +# (source start file, target name, title, author, +# dir menu entry, description, category) +texinfo_documents = [ + ('index', 'Fail2Ban', u'Fail2Ban Developers\' Documentation', + u'', 'Fail2Ban', 'One line description of project.', + 'Miscellaneous'), +] + +# Documents to append as an appendix to all manuals. +#texinfo_appendices = [] + +# If false, no module index is generated. +#texinfo_domain_indices = True + +# How to display URL addresses: 'footnote', 'no', or 'inline'. +#texinfo_show_urls = 'footnote' + +# If true, do not generate a @detailmenu in the "Top" node's menu. +#texinfo_no_detailmenu = False +autodoc_default_flags = ['members', 'inherited-members', 'undoc-members', 'show-inheritance'] diff --git a/doc/develop.rst b/doc/develop.rst new file mode 100644 index 00000000..2f1c9842 --- /dev/null +++ b/doc/develop.rst @@ -0,0 +1 @@ +.. include:: ../DEVELOP diff --git a/doc/fail2ban.client.actionreader.rst b/doc/fail2ban.client.actionreader.rst new file mode 100644 index 00000000..23e578e2 --- /dev/null +++ b/doc/fail2ban.client.actionreader.rst @@ -0,0 +1,7 @@ +fail2ban.client.actionreader module +=================================== + +.. automodule:: fail2ban.client.actionreader + :members: + :undoc-members: + :show-inheritance: diff --git a/doc/fail2ban.client.beautifier.rst b/doc/fail2ban.client.beautifier.rst new file mode 100644 index 00000000..1f35a50b --- /dev/null +++ b/doc/fail2ban.client.beautifier.rst @@ -0,0 +1,7 @@ +fail2ban.client.beautifier module +================================= + +.. automodule:: fail2ban.client.beautifier + :members: + :undoc-members: + :show-inheritance: diff --git a/doc/fail2ban.client.configparserinc.rst b/doc/fail2ban.client.configparserinc.rst new file mode 100644 index 00000000..01c4c2d0 --- /dev/null +++ b/doc/fail2ban.client.configparserinc.rst @@ -0,0 +1,7 @@ +fail2ban.client.configparserinc module +====================================== + +.. automodule:: fail2ban.client.configparserinc + :members: + :undoc-members: + :show-inheritance: diff --git a/doc/fail2ban.client.configreader.rst b/doc/fail2ban.client.configreader.rst new file mode 100644 index 00000000..d24db2ca --- /dev/null +++ b/doc/fail2ban.client.configreader.rst @@ -0,0 +1,7 @@ +fail2ban.client.configreader module +=================================== + +.. automodule:: fail2ban.client.configreader + :members: + :undoc-members: + :show-inheritance: diff --git a/doc/fail2ban.client.configurator.rst b/doc/fail2ban.client.configurator.rst new file mode 100644 index 00000000..63372f5d --- /dev/null +++ b/doc/fail2ban.client.configurator.rst @@ -0,0 +1,7 @@ +fail2ban.client.configurator module +=================================== + +.. automodule:: fail2ban.client.configurator + :members: + :undoc-members: + :show-inheritance: diff --git a/doc/fail2ban.client.csocket.rst b/doc/fail2ban.client.csocket.rst new file mode 100644 index 00000000..19e6219a --- /dev/null +++ b/doc/fail2ban.client.csocket.rst @@ -0,0 +1,7 @@ +fail2ban.client.csocket module +============================== + +.. automodule:: fail2ban.client.csocket + :members: + :undoc-members: + :show-inheritance: diff --git a/doc/fail2ban.client.fail2banreader.rst b/doc/fail2ban.client.fail2banreader.rst new file mode 100644 index 00000000..4fc66532 --- /dev/null +++ b/doc/fail2ban.client.fail2banreader.rst @@ -0,0 +1,7 @@ +fail2ban.client.fail2banreader module +===================================== + +.. automodule:: fail2ban.client.fail2banreader + :members: + :undoc-members: + :show-inheritance: diff --git a/doc/fail2ban.client.filterreader.rst b/doc/fail2ban.client.filterreader.rst new file mode 100644 index 00000000..d0e9cf5c --- /dev/null +++ b/doc/fail2ban.client.filterreader.rst @@ -0,0 +1,7 @@ +fail2ban.client.filterreader module +=================================== + +.. automodule:: fail2ban.client.filterreader + :members: + :undoc-members: + :show-inheritance: diff --git a/doc/fail2ban.client.jailreader.rst b/doc/fail2ban.client.jailreader.rst new file mode 100644 index 00000000..f57e3b6a --- /dev/null +++ b/doc/fail2ban.client.jailreader.rst @@ -0,0 +1,7 @@ +fail2ban.client.jailreader module +================================= + +.. automodule:: fail2ban.client.jailreader + :members: + :undoc-members: + :show-inheritance: diff --git a/doc/fail2ban.client.jailsreader.rst b/doc/fail2ban.client.jailsreader.rst new file mode 100644 index 00000000..d1281248 --- /dev/null +++ b/doc/fail2ban.client.jailsreader.rst @@ -0,0 +1,7 @@ +fail2ban.client.jailsreader module +================================== + +.. automodule:: fail2ban.client.jailsreader + :members: + :undoc-members: + :show-inheritance: diff --git a/doc/fail2ban.client.rst b/doc/fail2ban.client.rst new file mode 100644 index 00000000..9943912d --- /dev/null +++ b/doc/fail2ban.client.rst @@ -0,0 +1,15 @@ +fail2ban.client package +======================= + +.. toctree:: + + fail2ban.client.actionreader + fail2ban.client.beautifier + fail2ban.client.configparserinc + fail2ban.client.configreader + fail2ban.client.configurator + fail2ban.client.csocket + fail2ban.client.fail2banreader + fail2ban.client.filterreader + fail2ban.client.jailreader + fail2ban.client.jailsreader diff --git a/doc/fail2ban.exceptions.rst b/doc/fail2ban.exceptions.rst new file mode 100644 index 00000000..3adc5ea6 --- /dev/null +++ b/doc/fail2ban.exceptions.rst @@ -0,0 +1,7 @@ +fail2ban.exceptions module +========================== + +.. automodule:: fail2ban.exceptions + :members: + :undoc-members: + :show-inheritance: diff --git a/doc/fail2ban.helpers.rst b/doc/fail2ban.helpers.rst new file mode 100644 index 00000000..28b072cb --- /dev/null +++ b/doc/fail2ban.helpers.rst @@ -0,0 +1,7 @@ +fail2ban.helpers module +======================= + +.. automodule:: fail2ban.helpers + :members: + :undoc-members: + :show-inheritance: diff --git a/doc/fail2ban.protocol.rst b/doc/fail2ban.protocol.rst new file mode 100644 index 00000000..289ccb22 --- /dev/null +++ b/doc/fail2ban.protocol.rst @@ -0,0 +1,7 @@ +fail2ban.protocol module +======================== + +.. automodule:: fail2ban.protocol + :members: + :undoc-members: + :show-inheritance: diff --git a/doc/fail2ban.rst b/doc/fail2ban.rst new file mode 100644 index 00000000..c9b4fbb2 --- /dev/null +++ b/doc/fail2ban.rst @@ -0,0 +1,12 @@ +fail2ban package +================ + +.. toctree:: + + fail2ban.client + fail2ban.server + + fail2ban.exceptions + fail2ban.helpers + fail2ban.protocol + fail2ban.version diff --git a/doc/fail2ban.server.action.rst b/doc/fail2ban.server.action.rst new file mode 100644 index 00000000..c2f9c410 --- /dev/null +++ b/doc/fail2ban.server.action.rst @@ -0,0 +1,7 @@ +fail2ban.server.action module +============================= + +.. automodule:: fail2ban.server.action + :members: + :undoc-members: + :show-inheritance: diff --git a/doc/fail2ban.server.actions.rst b/doc/fail2ban.server.actions.rst new file mode 100644 index 00000000..8c701c72 --- /dev/null +++ b/doc/fail2ban.server.actions.rst @@ -0,0 +1,7 @@ +fail2ban.server.actions module +============================== + +.. automodule:: fail2ban.server.actions + :members: + :undoc-members: + :show-inheritance: diff --git a/doc/fail2ban.server.asyncserver.rst b/doc/fail2ban.server.asyncserver.rst new file mode 100644 index 00000000..5ddfa50f --- /dev/null +++ b/doc/fail2ban.server.asyncserver.rst @@ -0,0 +1,7 @@ +fail2ban.server.asyncserver module +================================== + +.. automodule:: fail2ban.server.asyncserver + :members: + :undoc-members: + :show-inheritance: diff --git a/doc/fail2ban.server.banmanager.rst b/doc/fail2ban.server.banmanager.rst new file mode 100644 index 00000000..96b1a4e9 --- /dev/null +++ b/doc/fail2ban.server.banmanager.rst @@ -0,0 +1,7 @@ +fail2ban.server.banmanager module +================================= + +.. automodule:: fail2ban.server.banmanager + :members: + :undoc-members: + :show-inheritance: diff --git a/doc/fail2ban.server.database.rst b/doc/fail2ban.server.database.rst new file mode 100644 index 00000000..40002258 --- /dev/null +++ b/doc/fail2ban.server.database.rst @@ -0,0 +1,7 @@ +fail2ban.server.database module +=============================== + +.. automodule:: fail2ban.server.database + :members: + :undoc-members: + :show-inheritance: diff --git a/doc/fail2ban.server.datedetector.rst b/doc/fail2ban.server.datedetector.rst new file mode 100644 index 00000000..9a367734 --- /dev/null +++ b/doc/fail2ban.server.datedetector.rst @@ -0,0 +1,7 @@ +fail2ban.server.datedetector module +=================================== + +.. automodule:: fail2ban.server.datedetector + :members: + :undoc-members: + :show-inheritance: diff --git a/doc/fail2ban.server.datetemplate.rst b/doc/fail2ban.server.datetemplate.rst new file mode 100644 index 00000000..4959e805 --- /dev/null +++ b/doc/fail2ban.server.datetemplate.rst @@ -0,0 +1,7 @@ +fail2ban.server.datetemplate module +=================================== + +.. automodule:: fail2ban.server.datetemplate + :members: + :undoc-members: + :show-inheritance: diff --git a/doc/fail2ban.server.faildata.rst b/doc/fail2ban.server.faildata.rst new file mode 100644 index 00000000..c11d8208 --- /dev/null +++ b/doc/fail2ban.server.faildata.rst @@ -0,0 +1,7 @@ +fail2ban.server.faildata module +=============================== + +.. automodule:: fail2ban.server.faildata + :members: + :undoc-members: + :show-inheritance: diff --git a/doc/fail2ban.server.failmanager.rst b/doc/fail2ban.server.failmanager.rst new file mode 100644 index 00000000..6e4e8704 --- /dev/null +++ b/doc/fail2ban.server.failmanager.rst @@ -0,0 +1,7 @@ +fail2ban.server.failmanager module +================================== + +.. automodule:: fail2ban.server.failmanager + :members: + :undoc-members: + :show-inheritance: diff --git a/doc/fail2ban.server.failregex.rst b/doc/fail2ban.server.failregex.rst new file mode 100644 index 00000000..f473c51f --- /dev/null +++ b/doc/fail2ban.server.failregex.rst @@ -0,0 +1,7 @@ +fail2ban.server.failregex module +================================ + +.. automodule:: fail2ban.server.failregex + :members: + :undoc-members: + :show-inheritance: diff --git a/doc/fail2ban.server.filter.rst b/doc/fail2ban.server.filter.rst new file mode 100644 index 00000000..41279013 --- /dev/null +++ b/doc/fail2ban.server.filter.rst @@ -0,0 +1,7 @@ +fail2ban.server.filter module +============================= + +.. automodule:: fail2ban.server.filter + :members: + :undoc-members: + :show-inheritance: diff --git a/doc/fail2ban.server.filtergamin.rst b/doc/fail2ban.server.filtergamin.rst new file mode 100644 index 00000000..f248772f --- /dev/null +++ b/doc/fail2ban.server.filtergamin.rst @@ -0,0 +1,7 @@ +fail2ban.server.filtergamin module +================================== + +.. automodule:: fail2ban.server.filtergamin + :members: + :undoc-members: + :show-inheritance: diff --git a/doc/fail2ban.server.filterpoll.rst b/doc/fail2ban.server.filterpoll.rst new file mode 100644 index 00000000..d87e461f --- /dev/null +++ b/doc/fail2ban.server.filterpoll.rst @@ -0,0 +1,7 @@ +fail2ban.server.filterpoll module +================================= + +.. automodule:: fail2ban.server.filterpoll + :members: + :undoc-members: + :show-inheritance: diff --git a/doc/fail2ban.server.filterpyinotify.rst b/doc/fail2ban.server.filterpyinotify.rst new file mode 100644 index 00000000..0ea312bf --- /dev/null +++ b/doc/fail2ban.server.filterpyinotify.rst @@ -0,0 +1,7 @@ +fail2ban.server.filterpyinotify module +====================================== + +.. automodule:: fail2ban.server.filterpyinotify + :members: + :undoc-members: + :show-inheritance: diff --git a/doc/fail2ban.server.filtersystemd.rst b/doc/fail2ban.server.filtersystemd.rst new file mode 100644 index 00000000..1b71a2c8 --- /dev/null +++ b/doc/fail2ban.server.filtersystemd.rst @@ -0,0 +1,7 @@ +fail2ban.server.filtersystemd module +==================================== + +.. automodule:: fail2ban.server.filtersystemd + :members: + :undoc-members: + :show-inheritance: diff --git a/doc/fail2ban.server.jail.rst b/doc/fail2ban.server.jail.rst new file mode 100644 index 00000000..b41077bf --- /dev/null +++ b/doc/fail2ban.server.jail.rst @@ -0,0 +1,7 @@ +fail2ban.server.jail module +=========================== + +.. automodule:: fail2ban.server.jail + :members: + :undoc-members: + :show-inheritance: diff --git a/doc/fail2ban.server.jails.rst b/doc/fail2ban.server.jails.rst new file mode 100644 index 00000000..7e9390f7 --- /dev/null +++ b/doc/fail2ban.server.jails.rst @@ -0,0 +1,7 @@ +fail2ban.server.jails module +============================ + +.. automodule:: fail2ban.server.jails + :members: + :undoc-members: + :show-inheritance: diff --git a/doc/fail2ban.server.jailthread.rst b/doc/fail2ban.server.jailthread.rst new file mode 100644 index 00000000..db11f7a9 --- /dev/null +++ b/doc/fail2ban.server.jailthread.rst @@ -0,0 +1,7 @@ +fail2ban.server.jailthread module +================================= + +.. automodule:: fail2ban.server.jailthread + :members: + :undoc-members: + :show-inheritance: diff --git a/doc/fail2ban.server.mytime.rst b/doc/fail2ban.server.mytime.rst new file mode 100644 index 00000000..1313d20b --- /dev/null +++ b/doc/fail2ban.server.mytime.rst @@ -0,0 +1,7 @@ +fail2ban.server.mytime module +============================= + +.. automodule:: fail2ban.server.mytime + :members: + :undoc-members: + :show-inheritance: diff --git a/doc/fail2ban.server.rst b/doc/fail2ban.server.rst new file mode 100644 index 00000000..fdc3bd20 --- /dev/null +++ b/doc/fail2ban.server.rst @@ -0,0 +1,28 @@ +fail2ban.server package +======================= + +.. toctree:: + + fail2ban.server.action + fail2ban.server.actions + fail2ban.server.asyncserver + fail2ban.server.banmanager + fail2ban.server.database + fail2ban.server.datedetector + fail2ban.server.datetemplate + fail2ban.server.faildata + fail2ban.server.failmanager + fail2ban.server.failregex + fail2ban.server.filter + fail2ban.server.filtergamin + fail2ban.server.filterpoll + fail2ban.server.filterpyinotify + fail2ban.server.filtersystemd + fail2ban.server.jail + fail2ban.server.jails + fail2ban.server.jailthread + fail2ban.server.mytime + fail2ban.server.server + fail2ban.server.strptime + fail2ban.server.ticket + fail2ban.server.transmitter diff --git a/doc/fail2ban.server.server.rst b/doc/fail2ban.server.server.rst new file mode 100644 index 00000000..be14b35c --- /dev/null +++ b/doc/fail2ban.server.server.rst @@ -0,0 +1,7 @@ +fail2ban.server.server module +============================= + +.. automodule:: fail2ban.server.server + :members: + :undoc-members: + :show-inheritance: diff --git a/doc/fail2ban.server.strptime.rst b/doc/fail2ban.server.strptime.rst new file mode 100644 index 00000000..81ff181c --- /dev/null +++ b/doc/fail2ban.server.strptime.rst @@ -0,0 +1,7 @@ +fail2ban.server.strptime module +=============================== + +.. automodule:: fail2ban.server.strptime + :members: + :undoc-members: + :show-inheritance: diff --git a/doc/fail2ban.server.ticket.rst b/doc/fail2ban.server.ticket.rst new file mode 100644 index 00000000..27a1b53b --- /dev/null +++ b/doc/fail2ban.server.ticket.rst @@ -0,0 +1,7 @@ +fail2ban.server.ticket module +============================= + +.. automodule:: fail2ban.server.ticket + :members: + :undoc-members: + :show-inheritance: diff --git a/doc/fail2ban.server.transmitter.rst b/doc/fail2ban.server.transmitter.rst new file mode 100644 index 00000000..cb62fa97 --- /dev/null +++ b/doc/fail2ban.server.transmitter.rst @@ -0,0 +1,7 @@ +fail2ban.server.transmitter module +================================== + +.. automodule:: fail2ban.server.transmitter + :members: + :undoc-members: + :show-inheritance: diff --git a/doc/fail2ban.version.rst b/doc/fail2ban.version.rst new file mode 100644 index 00000000..cc97380a --- /dev/null +++ b/doc/fail2ban.version.rst @@ -0,0 +1,7 @@ +fail2ban.version module +======================= + +.. automodule:: fail2ban.version + :members: + :undoc-members: + :show-inheritance: diff --git a/doc/filters.rst b/doc/filters.rst new file mode 100644 index 00000000..4d19acf8 --- /dev/null +++ b/doc/filters.rst @@ -0,0 +1 @@ +.. include:: ../FILTERS diff --git a/doc/index.rst b/doc/index.rst new file mode 100644 index 00000000..9f6c94ac --- /dev/null +++ b/doc/index.rst @@ -0,0 +1,19 @@ +Welcome to Fail2Ban's developers documentation! +=============================================== + +Contents: + +.. toctree:: + :maxdepth: 2 + + develop + filters + release + fail2ban + +Indices and tables +================== + +* :ref:`genindex` +* :ref:`search` + diff --git a/doc/release.rst b/doc/release.rst new file mode 100644 index 00000000..bdd0430a --- /dev/null +++ b/doc/release.rst @@ -0,0 +1 @@ +.. include:: ../RELEASE diff --git a/doc/requirements.txt b/doc/requirements.txt new file mode 100644 index 00000000..a2954e31 --- /dev/null +++ b/doc/requirements.txt @@ -0,0 +1 @@ +numpydoc diff --git a/fail2ban/server/action.py b/fail2ban/server/action.py index 5e3a7c75..0098c546 100644 --- a/fail2ban/server/action.py +++ b/fail2ban/server/action.py @@ -62,9 +62,7 @@ class CallingMap(MutableMapping): Attributes ---------- data : dict - The dictionary data which can be accessed to obtain items - without callable values being called. - + The dictionary data which can be accessed to obtain items uncalled """ def __init__(self, *args, **kwargs): From 3781ff845ad8028500a4d93192f9d47a20335190 Mon Sep 17 00:00:00 2001 From: Steven Hiscocks Date: Sat, 29 Mar 2014 22:54:06 +0000 Subject: [PATCH 005/141] BF: Fix getting jail name from exceptions in beautifier for Python 3+ --- fail2ban/client/beautifier.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/fail2ban/client/beautifier.py b/fail2ban/client/beautifier.py index 68816b06..25d73cb2 100644 --- a/fail2ban/client/beautifier.py +++ b/fail2ban/client/beautifier.py @@ -188,9 +188,9 @@ class Beautifier: logSys.debug("Beautify (error) " + `response` + " with " + `self.__inputCmd`) msg = response if isinstance(response, UnknownJailException): - msg = "Sorry but the jail '" + response[0] + "' does not exist" + msg = "Sorry but the jail '" + response.args[0] + "' does not exist" elif isinstance(response, IndexError): msg = "Sorry but the command is invalid" elif isinstance(response, DuplicateJailException): - msg = "The jail '" + response[0] + "' already exists" + msg = "The jail '" + response.args[0] + "' already exists" return msg From 953ebd62c6aebb0403a14cfbb81da983311c63d1 Mon Sep 17 00:00:00 2001 From: Steven Hiscocks Date: Sat, 29 Mar 2014 23:08:37 +0000 Subject: [PATCH 006/141] DOC: Improve error logging when specific backend set and fails --- fail2ban/server/jail.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/fail2ban/server/jail.py b/fail2ban/server/jail.py index dec04c73..ec6ff1a7 100644 --- a/fail2ban/server/jail.py +++ b/fail2ban/server/jail.py @@ -104,7 +104,9 @@ class Jail: self.__actions = Actions(self) return # we are done except ImportError, e: - logSys.debug( + # Log debug if auto, but error if specific + logSys.log( + logging.DEBUG if backend == "auto" else logging.ERROR, "Backend %r failed to initialize due to %s" % (b, e)) # log error since runtime error message isn't printed, INVALID COMMAND logSys.error( From 3a155ed2e0f2f5aea5ebefc1981843984f9764cd Mon Sep 17 00:00:00 2001 From: yungchin Date: Tue, 1 Apr 2014 16:52:21 +0100 Subject: [PATCH 007/141] Update comments in shorewall.conf for new settings --- config/action.d/shorewall.conf | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/config/action.d/shorewall.conf b/config/action.d/shorewall.conf index 81ac0518..f5f2c775 100644 --- a/config/action.d/shorewall.conf +++ b/config/action.d/shorewall.conf @@ -9,7 +9,9 @@ # connections. So if the attempter goes on trying using the same connection # he could even log in. In order to get the same behavior of the iptable # action (so that the ban is immediate) the /etc/shorewall/shorewall.conf -# file should me modified with "BLACKLISTNEWONLY=No". +# file should me modified with "BLACKLISTNEWONLY=No". Note that as of +# Shorewall 4.5.13 BLACKLISTNEWONLY is deprecated; however the equivalent +# of BLACKLISTNEWONLY=No can now be achieved by setting BLACKLIST="ALL". # [Definition] From 638c01355746736ab4f463d219fd069f211cda4d Mon Sep 17 00:00:00 2001 From: Steven Hiscocks Date: Wed, 2 Apr 2014 18:30:21 +0100 Subject: [PATCH 008/141] ENH: Suppress configuration warnings if non-critical options are not set --- fail2ban/client/actionreader.py | 10 +++---- fail2ban/client/configreader.py | 4 +++ fail2ban/client/filterreader.py | 2 +- fail2ban/client/jailreader.py | 17 ++++++------ fail2ban/server/filter.py | 2 +- fail2ban/tests/clientreadertestcase.py | 37 +------------------------- fail2ban/tests/config/jail.conf | 2 ++ 7 files changed, 23 insertions(+), 51 deletions(-) diff --git a/fail2ban/client/actionreader.py b/fail2ban/client/actionreader.py index b11deaba..6b0334ac 100644 --- a/fail2ban/client/actionreader.py +++ b/fail2ban/client/actionreader.py @@ -34,11 +34,11 @@ logSys = logging.getLogger(__name__) class ActionReader(DefinitionInitConfigReader): _configOpts = [ - ["string", "actionstart", ""], - ["string", "actionstop", ""], - ["string", "actioncheck", ""], - ["string", "actionban", ""], - ["string", "actionunban", ""], + ["string", "actionstart", None], + ["string", "actionstop", None], + ["string", "actioncheck", None], + ["string", "actionban", None], + ["string", "actionunban", None], ] def __init__(self, file_, jailName, initOpts, **kwargs): diff --git a/fail2ban/client/configreader.py b/fail2ban/client/configreader.py index dacf3e57..51ca6948 100644 --- a/fail2ban/client/configreader.py +++ b/fail2ban/client/configreader.py @@ -120,6 +120,10 @@ class ConfigReader(SafeConfigParserWithIncludes): logSys.warning("'%s' not defined in '%s'. Using default one: %r" % (option[1], sec, option[2])) values[option[1]] = option[2] + else: + logSys.debug( + "Non essential option '%s' not defined in '%s'.", + option[1], sec) except ValueError: logSys.warning("Wrong value for '" + option[1] + "' in '" + sec + "'. Using default one: '" + `option[2]` + "'") diff --git a/fail2ban/client/filterreader.py b/fail2ban/client/filterreader.py index 612fd350..5a11dc41 100644 --- a/fail2ban/client/filterreader.py +++ b/fail2ban/client/filterreader.py @@ -35,7 +35,7 @@ logSys = logging.getLogger(__name__) class FilterReader(DefinitionInitConfigReader): _configOpts = [ - ["string", "ignoreregex", ""], + ["string", "ignoreregex", None], ["string", "failregex", ""], ] diff --git a/fail2ban/client/jailreader.py b/fail2ban/client/jailreader.py index 42c04ffb..5735b021 100644 --- a/fail2ban/client/jailreader.py +++ b/fail2ban/client/jailreader.py @@ -68,7 +68,8 @@ class JailReader(ConfigReader): return out def isEnabled(self): - return self.__force_enable or ( self.__opts and self.__opts["enabled"] ) + return self.__force_enable or ( + self.__opts and self.__opts.get("enabled", False)) @staticmethod def _glob(path): @@ -85,14 +86,14 @@ class JailReader(ConfigReader): return pathList def getOptions(self): - opts = [["bool", "enabled", "false"], - ["string", "logpath", "/var/log/messages"], - ["string", "logencoding", "auto"], + opts = [["bool", "enabled", False], + ["string", "logpath", None], + ["string", "logencoding", None], ["string", "backend", "auto"], - ["int", "maxretry", 3], - ["int", "findtime", 600], - ["int", "bantime", 600], - ["string", "usedns", "warn"], + ["int", "maxretry", None], + ["int", "findtime", None], + ["int", "bantime", None], + ["string", "usedns", None], ["string", "failregex", None], ["string", "ignoreregex", None], ["string", "ignorecommand", None], diff --git a/fail2ban/server/filter.py b/fail2ban/server/filter.py index c82395ea..fb5aeb3d 100644 --- a/fail2ban/server/filter.py +++ b/fail2ban/server/filter.py @@ -63,7 +63,7 @@ class Filter(JailThread): ## Use DNS setting self.setUseDns(useDns) ## The amount of time to look back. - self.__findTime = 6000 + self.__findTime = 600 ## The ignore IP list. self.__ignoreIpList = [] ## Size of line buffer diff --git a/fail2ban/tests/clientreadertestcase.py b/fail2ban/tests/clientreadertestcase.py index fd65b014..ce19a50e 100644 --- a/fail2ban/tests/clientreadertestcase.py +++ b/fail2ban/tests/clientreadertestcase.py @@ -349,31 +349,9 @@ class JailsReaderTest(LogCaptureTestCase): self.maxDiff = None self.assertEqual(sorted(comm_commands), sorted([['add', 'emptyaction', 'auto'], - ['set', 'emptyaction', 'usedns', 'warn'], - ['set', 'emptyaction', 'maxretry', 3], - ['set', 'emptyaction', 'findtime', 600], - ['set', 'emptyaction', 'logencoding', 'auto'], - ['set', 'emptyaction', 'bantime', 600], - ['add', 'special', 'auto'], - ['set', 'special', 'usedns', 'warn'], - ['set', 'special', 'maxretry', 3], - ['set', 'special', 'addfailregex', ''], - ['set', 'special', 'findtime', 600], - ['set', 'special', 'logencoding', 'auto'], - ['set', 'special', 'bantime', 600], ['add', 'missinglogfiles', 'auto'], - ['set', 'missinglogfiles', 'usedns', 'warn'], - ['set', 'missinglogfiles', 'maxretry', 3], - ['set', 'missinglogfiles', 'findtime', 600], - ['set', 'missinglogfiles', 'logencoding', 'auto'], - ['set', 'missinglogfiles', 'bantime', 600], ['set', 'missinglogfiles', 'addfailregex', ''], ['add', 'brokenaction', 'auto'], - ['set', 'brokenaction', 'usedns', 'warn'], - ['set', 'brokenaction', 'maxretry', 3], - ['set', 'brokenaction', 'findtime', 600], - ['set', 'brokenaction', 'logencoding', 'auto'], - ['set', 'brokenaction', 'bantime', 600], ['set', 'brokenaction', 'addfailregex', ''], ['set', 'brokenaction', 'addaction', 'brokenaction'], ['set', @@ -382,23 +360,9 @@ class JailsReaderTest(LogCaptureTestCase): 'brokenaction', 'actionban', 'hit with big stick '], - ['set', 'brokenaction', 'action', 'brokenaction', - 'actionstop', ''], - ['set', 'brokenaction', 'action', 'brokenaction', - 'actionstart', ''], - ['set', 'brokenaction', 'action', 'brokenaction', - 'actionunban', ''], - ['set', 'brokenaction', 'action', 'brokenaction', - 'actioncheck', ''], ['add', 'parse_to_end_of_jail.conf', 'auto'], - ['set', 'parse_to_end_of_jail.conf', 'usedns', 'warn'], - ['set', 'parse_to_end_of_jail.conf', 'maxretry', 3], - ['set', 'parse_to_end_of_jail.conf', 'findtime', 600], - ['set', 'parse_to_end_of_jail.conf', 'logencoding', 'auto'], - ['set', 'parse_to_end_of_jail.conf', 'bantime', 600], ['set', 'parse_to_end_of_jail.conf', 'addfailregex', ''], ['start', 'emptyaction'], - ['start', 'special'], ['start', 'missinglogfiles'], ['start', 'brokenaction'], ['start', 'parse_to_end_of_jail.conf'],])) @@ -572,6 +536,7 @@ class JailsReaderTest(LogCaptureTestCase): jailfd = open(os.path.join(basedir, "jail.conf"), 'w') jailfd.write(""" [testjail1] +enabled = true action = testaction1[actname=test1] testaction1[actname=test2] testaction.py diff --git a/fail2ban/tests/config/jail.conf b/fail2ban/tests/config/jail.conf index 525308e3..0f6a28f0 100644 --- a/fail2ban/tests/config/jail.conf +++ b/fail2ban/tests/config/jail.conf @@ -14,6 +14,7 @@ ignoreregex = ignoreip = [missinglogfiles] +enabled = true logpath = /weapons/of/mass/destruction [brokenactiondef] @@ -25,6 +26,7 @@ enabled = true action = brokenaction [missingbitsjail] +enabled = true filter = catchallthebadies action = thefunkychickendance From 6e8c1b2871673361d912d4205fa72c623555f096 Mon Sep 17 00:00:00 2001 From: yungchin Date: Wed, 2 Apr 2014 12:29:18 +0100 Subject: [PATCH 009/141] nginx-http-auth filter: match server_name = "" As documented at http://nginx.org/en/docs/http/server_names.html#miscellaneous_names "If no server_name is defined in a server block then nginx uses the empty name as the server name." This regex change allows us to match error output for such a configuration. The log line added to the tests was lifted from our logs verbatim; it did not match without the patched regex. Signed-off-by: Yung-Chin Oei --- config/filter.d/nginx-http-auth.conf | 2 +- fail2ban/tests/files/logs/nginx-http-auth | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/config/filter.d/nginx-http-auth.conf b/config/filter.d/nginx-http-auth.conf index 79dda30b..cad26d94 100644 --- a/config/filter.d/nginx-http-auth.conf +++ b/config/filter.d/nginx-http-auth.conf @@ -4,7 +4,7 @@ [Definition] -failregex = ^ \[error\] \d+#\d+: \*\d+ user "\S+":? (password mismatch|was not found in ".*"), client: , server: \S+, request: "\S+ \S+ HTTP/\d+\.\d+", host: "\S+"\s*$ +failregex = ^ \[error\] \d+#\d+: \*\d+ user "\S+":? (password mismatch|was not found in ".*"), client: , server: \S*, request: "\S+ \S+ HTTP/\d+\.\d+", host: "\S+"\s*$ ignoreregex = diff --git a/fail2ban/tests/files/logs/nginx-http-auth b/fail2ban/tests/files/logs/nginx-http-auth index 0fa7a7bd..78d338b7 100644 --- a/fail2ban/tests/files/logs/nginx-http-auth +++ b/fail2ban/tests/files/logs/nginx-http-auth @@ -3,4 +3,6 @@ 2012/04/09 11:53:29 [error] 2865#0: *66647 user "xyz" was not found in "/var/www/.htpasswd", client: 192.0.43.10, server: www.myhost.com, request: "GET / HTTP/1.1", host: "www.myhost.com" # failJSON: { "time": "2012-04-09T11:53:36", "match": true , "host": "192.0.43.10" } 2012/04/09 11:53:36 [error] 2865#0: *66647 user "xyz": password mismatch, client: 192.0.43.10, server: www.myhost.com, request: "GET / HTTP/1.1", host: "www.myhost.com" +# failJSON: { "time": "2014-04-01T22:20:38", "match": true, "host": "10.0.2.2" } +2014/04/01 22:20:38 [error] 30708#0: *3 user "scribendio": password mismatch, client: 10.0.2.2, server: , request: "GET / HTTP/1.1", host: "localhost:8443" From e3be822245d8e79c06723051426088b9480ae36a Mon Sep 17 00:00:00 2001 From: Daniel Black Date: Thu, 3 Apr 2014 21:30:45 +1100 Subject: [PATCH 010/141] DOC: nginx-http-auth filter --- ChangeLog | 1 + 1 file changed, 1 insertion(+) diff --git a/ChangeLog b/ChangeLog index 43aefcb4..7f29b9ac 100644 --- a/ChangeLog +++ b/ChangeLog @@ -18,6 +18,7 @@ ver. 0.9.1 (2014/xx/xx) - better, faster, stronger * loglevel couldn't be changed in fail2ban.conf * Handle case when no sqlite library is available for persistent database * Only reban once per IP from database on fail2ban restart + * Nginx filter to support missing server_name. Closes gh-676 - New features: From ef186a9ba949a65a0571d44f905cc4fead3dc78e Mon Sep 17 00:00:00 2001 From: Daniel Black Date: Thu, 3 Apr 2014 21:42:19 +1100 Subject: [PATCH 011/141] DOC: add thanks for last commit --- THANKS | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/THANKS b/THANKS index 64eb4402..2c084dee 100644 --- a/THANKS +++ b/THANKS @@ -100,8 +100,9 @@ Vaclav Misek Vincent Deffontaines Yaroslav Halchenko Winston Smith -ykimon Yehuda Katz +ykimon +Yung-Chin Oei Zbigniew Jędrzejewski-Szmek zugeschmiert Zurd From d7e888238c7d027cc2596529e160c7503a43bcdc Mon Sep 17 00:00:00 2001 From: shawn Date: Thu, 3 Apr 2014 10:44:49 -0400 Subject: [PATCH 012/141] Correct grammar --- config/fail2ban.conf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config/fail2ban.conf b/config/fail2ban.conf index 5fc77759..1a22a9e8 100644 --- a/config/fail2ban.conf +++ b/config/fail2ban.conf @@ -52,7 +52,7 @@ pidfile = /var/run/fail2ban/fail2ban.pid # Options: dbfile # Notes.: Set the file for the fail2ban persistent data to be stored. # A value of ":memory:" means database is only stored in memory -# and data is lost once fail2ban is stops. +# and data is lost when fail2ban is stopped. # A value of "None" disables the database. # Values: [ None :memory: FILE ] Default: /var/lib/fail2ban/fail2ban.sqlite3 dbfile = /var/lib/fail2ban/fail2ban.sqlite3 From 4190113e4b5ec57fa09ed238f82fcca0bb9f6b77 Mon Sep 17 00:00:00 2001 From: Steven Hiscocks Date: Thu, 3 Apr 2014 18:41:40 +0100 Subject: [PATCH 013/141] DOC: Fix jail.conf(5) man page which had duplicate {fail,ignore}regex --- man/jail.conf.5 | 11 ++--------- 1 file changed, 2 insertions(+), 9 deletions(-) diff --git a/man/jail.conf.5 b/man/jail.conf.5 index c4215fdd..db30251c 100644 --- a/man/jail.conf.5 +++ b/man/jail.conf.5 @@ -297,12 +297,12 @@ There are two filter definitions used in the [Definition] section: .TP .B failregex is the regex (\fBreg\fRular \fBex\fRpression) that will match failed attempts. The tag \fI\fR is used as part of the regex and is itself a regex -for IPv4 addresses and hostnames. fail2ban will work out which one of these it actually is. +for IPv4 addresses (and hostnames if \fBusedns\fR). Fail2Ban will work out which one of these it actually is. For multiline regexs the tag \fI\fR should be used to separate lines. This allows lines between the matched lines to continue to be searched for other failures. The tag can be used multiple times. .TP .B ignoreregex -is the regex to identify log entries that should be ignored by fail2ban, even if they match failregex. +is the regex to identify log entries that should be ignored by Fail2Ban, even if they match failregex. .PP @@ -331,13 +331,6 @@ indicates that this file is read before the [Definition] section. \fBafter\fR indicates that this file is read after the [Definition] section. -.B failregex -regex that will match failed attempts. The tag is used as part of the regex and is itself a regex -for IPv4 addresses (and hostnames if \fBusedns\fR). Fail2Ban will work out which one of these it actually is. -.TP -.B ignoreregex -regex to identify log entries that should be ignored by Fail2Ban, even if they match failregex. - .SH AUTHOR Fail2ban was originally written by Cyril Jaquier . At the moment it is maintained and further developed by Yaroslav O. Halchenko , Daniel Black and Steven Hiscocks along with a number of contributors. See \fBTHANKS\fR file shipped with Fail2Ban for a full list. From abfa7fa7e3d95b6c7c49761324bae3d71927b887 Mon Sep 17 00:00:00 2001 From: Steven Hiscocks Date: Thu, 3 Apr 2014 18:47:38 +0100 Subject: [PATCH 014/141] DOC: Update ChangeLog --- ChangeLog | 1 + 1 file changed, 1 insertion(+) diff --git a/ChangeLog b/ChangeLog index 7f29b9ac..66df9639 100644 --- a/ChangeLog +++ b/ChangeLog @@ -25,6 +25,7 @@ ver. 0.9.1 (2014/xx/xx) - better, faster, stronger - Enhancements * Fail2ban-regex - add print-all-matched option. Closes gh-652 + * Suppress fail2ban-client warnings for non-critical config options ver. 0.9.0 (2014/03/14 - beta ---------- From 941a38ea8e327a7ead22b0a9bd6b279cc97a7905 Mon Sep 17 00:00:00 2001 From: Yung-Chin Oei Date: Fri, 4 Apr 2014 01:27:39 +0100 Subject: [PATCH 015/141] nginx-http-auth: match when "referrer" is present A sample log-line is provided. The updated regex successfully matches this line. Signed-off-by: Yung-Chin Oei --- config/filter.d/nginx-http-auth.conf | 2 +- fail2ban/tests/files/logs/nginx-http-auth | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/config/filter.d/nginx-http-auth.conf b/config/filter.d/nginx-http-auth.conf index cad26d94..75234581 100644 --- a/config/filter.d/nginx-http-auth.conf +++ b/config/filter.d/nginx-http-auth.conf @@ -4,7 +4,7 @@ [Definition] -failregex = ^ \[error\] \d+#\d+: \*\d+ user "\S+":? (password mismatch|was not found in ".*"), client: , server: \S*, request: "\S+ \S+ HTTP/\d+\.\d+", host: "\S+"\s*$ +failregex = ^ \[error\] \d+#\d+: \*\d+ user "\S+":? (password mismatch|was not found in ".*"), client: , server: \S*, request: "\S+ \S+ HTTP/\d+\.\d+", host: "\S+"(, referrer: "\S+")?$ ignoreregex = diff --git a/fail2ban/tests/files/logs/nginx-http-auth b/fail2ban/tests/files/logs/nginx-http-auth index 78d338b7..22c16057 100644 --- a/fail2ban/tests/files/logs/nginx-http-auth +++ b/fail2ban/tests/files/logs/nginx-http-auth @@ -5,4 +5,6 @@ 2012/04/09 11:53:36 [error] 2865#0: *66647 user "xyz": password mismatch, client: 192.0.43.10, server: www.myhost.com, request: "GET / HTTP/1.1", host: "www.myhost.com" # failJSON: { "time": "2014-04-01T22:20:38", "match": true, "host": "10.0.2.2" } 2014/04/01 22:20:38 [error] 30708#0: *3 user "scribendio": password mismatch, client: 10.0.2.2, server: , request: "GET / HTTP/1.1", host: "localhost:8443" +# failJSON: { "time": "2014-04-02T12:37:58", "match": true, "host": "10.0.2.2" } +2014/04/02 12:37:58 [error] 6563#0: *1861 user "scribendio": password mismatch, client: 10.0.2.2, server: scribend.io, request: "GET /admin HTTP/1.1", host: "scribend.io", referrer: "https://scribend.io/admin" From 5bccec61e429903a65a09166c9eb6c40b6c0acfe Mon Sep 17 00:00:00 2001 From: Yaroslav Halchenko Date: Thu, 3 Apr 2014 21:31:46 -0400 Subject: [PATCH 016/141] ENH: adding pruned with previous merge trailing \s* in nginx filter --- config/filter.d/nginx-http-auth.conf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config/filter.d/nginx-http-auth.conf b/config/filter.d/nginx-http-auth.conf index 75234581..a689f66a 100644 --- a/config/filter.d/nginx-http-auth.conf +++ b/config/filter.d/nginx-http-auth.conf @@ -4,7 +4,7 @@ [Definition] -failregex = ^ \[error\] \d+#\d+: \*\d+ user "\S+":? (password mismatch|was not found in ".*"), client: , server: \S*, request: "\S+ \S+ HTTP/\d+\.\d+", host: "\S+"(, referrer: "\S+")?$ +failregex = ^ \[error\] \d+#\d+: \*\d+ user "\S+":? (password mismatch|was not found in ".*"), client: , server: \S*, request: "\S+ \S+ HTTP/\d+\.\d+", host: "\S+"(, referrer: "\S+")?\s*$ ignoreregex = From 5e9619d4a405312e5e00bb81056d36e46eede2da Mon Sep 17 00:00:00 2001 From: Deyuan Deng Date: Fri, 4 Apr 2014 02:54:53 +0000 Subject: [PATCH 017/141] Clean up setup.py --- setup.py | 130 +++++++++++++++++++++++++++---------------------------- 1 file changed, 65 insertions(+), 65 deletions(-) diff --git a/setup.py b/setup.py index c5e26fb1..f0b4b331 100755 --- a/setup.py +++ b/setup.py @@ -92,77 +92,77 @@ setup( license = "GPL", platforms = "Posix", cmdclass = {'build_py': build_py, 'build_scripts': build_scripts}, - scripts = [ - 'bin/fail2ban-client', - 'bin/fail2ban-server', - 'bin/fail2ban-regex', - 'bin/fail2ban-testcases', - ], - packages = [ - 'fail2ban', - 'fail2ban.client', - 'fail2ban.server', - 'fail2ban.tests', - 'fail2ban.tests.action_d', - ], - package_data = { - 'fail2ban.tests': - [ join(w[0], f).replace("fail2ban/tests/", "", 1) - for w in os.walk('fail2ban/tests/files') - for f in w[2]] + - [ join(w[0], f).replace("fail2ban/tests/", "", 1) - for w in os.walk('fail2ban/tests/config') - for f in w[2]] + - [ join(w[0], f).replace("fail2ban/tests/", "", 1) - for w in os.walk('fail2ban/tests/action_d') - for f in w[2]] - }, - data_files = [ - ('/etc/fail2ban', - glob("config/*.conf") - ), - ('/etc/fail2ban/filter.d', - glob("config/filter.d/*.conf") - ), - ('/etc/fail2ban/action.d', - glob("config/action.d/*.conf") + - glob("config/action.d/*.py") - ), - ('/etc/fail2ban/fail2ban.d', - '' - ), - ('/etc/fail2ban/jail.d', - '' - ), - ('/var/lib/fail2ban', - '' - ), - ('/usr/share/doc/fail2ban', - ['README.md', 'README.Solaris', 'DEVELOP', 'FILTERS', - 'doc/run-rootless.txt'] - ) - ], + scripts = [ + 'bin/fail2ban-client', + 'bin/fail2ban-server', + 'bin/fail2ban-regex', + 'bin/fail2ban-testcases', + ], + packages = [ + 'fail2ban', + 'fail2ban.client', + 'fail2ban.server', + 'fail2ban.tests', + 'fail2ban.tests.action_d', + ], + package_data = { + 'fail2ban.tests': + [ join(w[0], f).replace("fail2ban/tests/", "", 1) + for w in os.walk('fail2ban/tests/files') + for f in w[2]] + + [ join(w[0], f).replace("fail2ban/tests/", "", 1) + for w in os.walk('fail2ban/tests/config') + for f in w[2]] + + [ join(w[0], f).replace("fail2ban/tests/", "", 1) + for w in os.walk('fail2ban/tests/action_d') + for f in w[2]] + }, + data_files = [ + ('/etc/fail2ban', + glob("config/*.conf") + ), + ('/etc/fail2ban/filter.d', + glob("config/filter.d/*.conf") + ), + ('/etc/fail2ban/action.d', + glob("config/action.d/*.conf") + + glob("config/action.d/*.py") + ), + ('/etc/fail2ban/fail2ban.d', + '' + ), + ('/etc/fail2ban/jail.d', + '' + ), + ('/var/lib/fail2ban', + '' + ), + ('/usr/share/doc/fail2ban', + ['README.md', 'README.Solaris', 'DEVELOP', 'FILTERS', + 'doc/run-rootless.txt'] + ) + ], **setup_extra ) # Do some checks after installation # Search for obsolete files. obsoleteFiles = [] -elements = { - "/etc/": - [ - "fail2ban.conf" - ], - "/usr/bin/": - [ - "fail2ban.py" - ], - "/usr/lib/fail2ban/": - [ - "version.py", - "protocol.py" - ] - } +elements = { + "/etc/": + [ + "fail2ban.conf" + ], + "/usr/bin/": + [ + "fail2ban.py" + ], + "/usr/lib/fail2ban/": + [ + "version.py", + "protocol.py" + ] +} for directory in elements: for f in elements[directory]: From 7dcea0d48df8011a911cd4bff4c6bfe786f9ad7d Mon Sep 17 00:00:00 2001 From: Yaroslav Halchenko Date: Thu, 10 Apr 2014 23:15:34 -0400 Subject: [PATCH 018/141] typos of paths-common (Thanks @chtheis, partial fix to #682) --- config/paths-freebsd.conf | 2 +- config/paths-osx.conf | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/config/paths-freebsd.conf b/config/paths-freebsd.conf index ee636a04..fd788ab4 100644 --- a/config/paths-freebsd.conf +++ b/config/paths-freebsd.conf @@ -2,7 +2,7 @@ [INCLUDES] -before = common-paths.conf +before = paths-common.conf after = paths-overrides.local diff --git a/config/paths-osx.conf b/config/paths-osx.conf index cbe17151..d1b99b38 100644 --- a/config/paths-osx.conf +++ b/config/paths-osx.conf @@ -3,7 +3,7 @@ [INCLUDES] -before = common-paths.conf +before = paths-common.conf after = paths-overrides.local From 8bcb25c3a2fb5415abfdedd665a488076d5b5b48 Mon Sep 17 00:00:00 2001 From: Yaroslav Halchenko Date: Thu, 10 Apr 2014 23:16:11 -0400 Subject: [PATCH 019/141] defining empty defaults for syslog_ log targets for common (Thanks @chtheis, partial fix to #682) --- config/paths-common.conf | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/config/paths-common.conf b/config/paths-common.conf index 64eec744..008dab4e 100644 --- a/config/paths-common.conf +++ b/config/paths-common.conf @@ -7,12 +7,17 @@ after = paths-overrides.local [DEFAULT] - - sshd_log = %(syslog_authpriv)s dropbear_log = %(syslog_authpriv)s +# There is no sensible generic defaults for syslog log targets, thus +# leaving them empty here so that no errors while parsing/interpollatin configs +syslog_daemon = +syslog_ftp = +syslog_local0 = +syslog_mail_warn = +syslog_user = # from /etc/audit/auditd.conf auditd_log = /var/log/audit/audit.log @@ -31,7 +36,7 @@ lighttpd_error_log = /var/log/lighttpd/error.log suhosin_log = %(syslog_user)s %(lighttpd_error_log)s # defaults to ftp or local2 if ftp doesn't exist -proftpd_log = %(syslog_ftp)s +proftpd_log = %(syslog_ftp)s # http://svnweb.freebsd.org/ports/head/ftp/proftpd/files/patch-src_proftpd.8.in?view=markup # defaults to ftp but can be overwritten. From 5e3f5db8b79a5dbb58fe3ad7240567e298e2801b Mon Sep 17 00:00:00 2001 From: Cameron Norman Date: Fri, 11 Apr 2014 08:50:55 -0700 Subject: [PATCH 020/141] Create fail2ban.upstart Add Upstart job. --- files/fail2ban.upstart | 13 +++++++++++++ 1 file changed, 13 insertions(+) create mode 100644 files/fail2ban.upstart diff --git a/files/fail2ban.upstart b/files/fail2ban.upstart new file mode 100644 index 00000000..1780a810 --- /dev/null +++ b/files/fail2ban.upstart @@ -0,0 +1,13 @@ +description "fail2ban - ban hosts that cause multiple authentication errors" + +start on filesystem and started networking +stop on deconfiguring-networking + +expect fork +respawn + +exec /usr/bin/fail2ban-client -x -b start + +pre-stop exec /usr/bin/fail2ban-client stop + +post-stop exec rm -f /var/run/fail2ban/fail2ban.pid From 4115b62a01112576bdb22c18f1445103bdda3ebe Mon Sep 17 00:00:00 2001 From: Cameron Norman Date: Fri, 11 Apr 2014 16:49:56 -0700 Subject: [PATCH 021/141] Update fail2ban.upstart It was actually a little problematic :) --- files/fail2ban.upstart | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/files/fail2ban.upstart b/files/fail2ban.upstart index 1780a810..19349ebd 100644 --- a/files/fail2ban.upstart +++ b/files/fail2ban.upstart @@ -3,11 +3,9 @@ description "fail2ban - ban hosts that cause multiple authentication errors" start on filesystem and started networking stop on deconfiguring-networking -expect fork +expect daemon respawn -exec /usr/bin/fail2ban-client -x -b start - -pre-stop exec /usr/bin/fail2ban-client stop +exec /usr/bin/fail2ban-server -x -b post-stop exec rm -f /var/run/fail2ban/fail2ban.pid From 0c8e72f45266ef1ce44cbd8acda304fb3e1bc689 Mon Sep 17 00:00:00 2001 From: Cameron Norman Date: Fri, 11 Apr 2014 17:09:08 -0700 Subject: [PATCH 022/141] Update fail2ban.upstart No longer directly exec the server, do not remove the PID file because it is unnecessary to do so. No longer respawns because Upstart can not track the process with the starter command. --- files/fail2ban.upstart | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/files/fail2ban.upstart b/files/fail2ban.upstart index 19349ebd..ccf267f0 100644 --- a/files/fail2ban.upstart +++ b/files/fail2ban.upstart @@ -3,9 +3,5 @@ description "fail2ban - ban hosts that cause multiple authentication errors" start on filesystem and started networking stop on deconfiguring-networking -expect daemon -respawn - -exec /usr/bin/fail2ban-server -x -b - -post-stop exec rm -f /var/run/fail2ban/fail2ban.pid +pre-start exec /usr/bin/fail2ban-client -x start +post-stop exec /usr/bin/fail2ban-client stop From 9fcb92524e327374596e87ea091929ad2cfb2ef8 Mon Sep 17 00:00:00 2001 From: Steven Hiscocks Date: Sat, 12 Apr 2014 11:21:52 +0100 Subject: [PATCH 023/141] BF: badips.py action logging of exc_info on debug typo --- config/action.d/badips.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/config/action.d/badips.py b/config/action.d/badips.py index 6b21e963..f740b4ed 100644 --- a/config/action.d/badips.py +++ b/config/action.d/badips.py @@ -258,7 +258,7 @@ class BadIPsAction(ActionBase): self._logSys.error( "Error banning IP %s for jail '%s' with action '%s': %s", ip, self._jail.name, self.banaction, e, - exc_info=self._logSys.getEffectiveLevel<=logging.DEBUG) + exc_info=self._logSys.getEffectiveLevel()<=logging.DEBUG) else: self._bannedips.add(ip) self._logSys.info( @@ -279,7 +279,7 @@ class BadIPsAction(ActionBase): self._logSys.info( "Error unbanning IP %s for jail '%s' with action '%s': %s", ip, self._jail.name, self.banaction, e, - exc_info=self._logSys.getEffectiveLevel<=logging.DEBUG) + exc_info=self._logSys.getEffectiveLevel()<=logging.DEBUG) else: self._logSys.info( "Unbanned IP %s for jail '%s' with action '%s'", From 1369701f8704e0a11a9d89da86bef7a4c1fef32b Mon Sep 17 00:00:00 2001 From: Steven Hiscocks Date: Sat, 12 Apr 2014 11:27:05 +0100 Subject: [PATCH 024/141] ENH: Log trace info for failed action events when in DEBUG --- fail2ban/server/actions.py | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/fail2ban/server/actions.py b/fail2ban/server/actions.py index 528e9b37..d5799ca1 100644 --- a/fail2ban/server/actions.py +++ b/fail2ban/server/actions.py @@ -215,7 +215,8 @@ class Actions(JailThread, Mapping): action.start() except Exception as e: logSys.error("Failed to start jail '%s' action '%s': %s", - self._jail.name, name, e) + self._jail.name, name, e, + exc_info=logSys.getEffectiveLevel()<=logging.DEBUG) while self.active: if not self.idle: #logSys.debug(self._jail.name + ": action") @@ -234,7 +235,8 @@ class Actions(JailThread, Mapping): action.stop() except Exception as e: logSys.error("Failed to stop jail '%s' action '%s': %s", - self._jail.name, name, e) + self._jail.name, name, e, + exc_info=logSys.getEffectiveLevel()<=logging.DEBUG) logSys.debug(self._jail.name + ": action terminated") return True @@ -278,7 +280,8 @@ class Actions(JailThread, Mapping): except Exception as e: logSys.error( "Failed to execute ban jail '%s' action '%s': %s", - self._jail.name, name, e) + self._jail.name, name, e, + exc_info=logSys.getEffectiveLevel()<=logging.DEBUG) return True else: logSys.notice("[%s] %s already banned" % (self._jail.name, @@ -325,7 +328,8 @@ class Actions(JailThread, Mapping): except Exception as e: logSys.error( "Failed to execute unban jail '%s' action '%s': %s", - self._jail.name, name, e) + self._jail.name, name, e, + exc_info=logSys.getEffectiveLevel()<=logging.DEBUG) @property def status(self): From c2289bc8fec78324e07f91af3ed509e7f43d4cc3 Mon Sep 17 00:00:00 2001 From: Yaroslav Halchenko Date: Wed, 16 Apr 2014 13:17:58 -0400 Subject: [PATCH 025/141] ENH(TST): relax test of sleep to "1" places from "2" The reason is that internally it does round, so even 1.005 then would not be equal to 1. Making it spaces==1 should be sufficient for up to 1.05 i.e. we would allow 50ms "drift" --- fail2ban/tests/servertestcase.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fail2ban/tests/servertestcase.py b/fail2ban/tests/servertestcase.py index 9aba7a62..ead1a0cd 100644 --- a/fail2ban/tests/servertestcase.py +++ b/fail2ban/tests/servertestcase.py @@ -151,7 +151,7 @@ class Transmitter(TransmitterBase): self.assertEqual(self.transm.proceed(["sleep", "1"]), (0, None)) t1 = time.time() # Approx 1 second delay - self.assertAlmostEqual(t1 - t0, 1, places=2) + self.assertAlmostEqual(t1 - t0, 1, places=1) def testDatabase(self): _, tmpFilename = tempfile.mkstemp(".db", "Fail2Ban_") From 9d6fc6eca2aba98fe9ecbe5bddf61e6ead572309 Mon Sep 17 00:00:00 2001 From: Steven Hiscocks Date: Wed, 16 Apr 2014 23:58:00 +0100 Subject: [PATCH 026/141] ENH: For syslog use SYSLOG_PID over _PID in systemd journal log format --- fail2ban/server/filtersystemd.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/fail2ban/server/filtersystemd.py b/fail2ban/server/filtersystemd.py index c6d0efd1..7ab56039 100644 --- a/fail2ban/server/filtersystemd.py +++ b/fail2ban/server/filtersystemd.py @@ -167,8 +167,9 @@ class FilterSystemd(JournalFilter): # pragma: systemd no cover logelements.append(logentry['_HOSTNAME']) if logentry.get('SYSLOG_IDENTIFIER'): logelements.append(logentry['SYSLOG_IDENTIFIER']) - if logentry.get('_PID'): - logelements[-1] += ("[%i]" % logentry['_PID']) + if logentry.get('SYSLOG_PID') or logentry.get('_PID'): + logelements[-1] += ("[%i]" % logentry.get( + 'SYSLOG_PID', logentry['_PID'])) logelements[-1] += ":" elif logentry.get('_COMM'): logelements.append(logentry['_COMM']) From 7d112430caabfb6f5922545adca77b1a210a3185 Mon Sep 17 00:00:00 2001 From: Jason Martin Date: Wed, 16 Apr 2014 21:21:41 -0700 Subject: [PATCH 027/141] Block brute-force attempts against the Monit gui --- config/filter.d/monit.conf | 18 ++++++++++++++++++ config/jail.conf | 6 ++++++ 2 files changed, 24 insertions(+) create mode 100644 config/filter.d/monit.conf diff --git a/config/filter.d/monit.conf b/config/filter.d/monit.conf new file mode 100644 index 00000000..f32eae61 --- /dev/null +++ b/config/filter.d/monit.conf @@ -0,0 +1,18 @@ +# Fail2Ban filter for monit.conf, looks for failed access attempts +# +# + +[INCLUDES] + +# Read common prefixes. If any customizations available -- read them from +# common.local +before = common.conf + +[Definition] +# Samples: +# [PDT Apr 16 20:59:11] error : Warning: Client '1.2.3.4' supplied unknown user 'foo' accessing monit httpd +# [PDT Apr 16 20:59:33] error : Warning: Client '1.2.3.4' supplied wrong password for user 'admin' accessing monit httpd + +failregex = Warning: Client '' supplied + +ignoreregex = diff --git a/config/jail.conf b/config/jail.conf index 96b3096f..7f7a7cbe 100644 --- a/config/jail.conf +++ b/config/jail.conf @@ -366,6 +366,12 @@ maxretry = 5 port = http,https logpath = /var/log/tomcat*/catalina.out +[monit] +#Ban clients brute-forcing the monit gui login +filter = monit +port = 2812 +logpath = /var/log/monit + [webmin-auth] From 3c0d6a77d2f2c47c221c927a6dba0066dd31d660 Mon Sep 17 00:00:00 2001 From: Yaroslav Halchenko Date: Thu, 17 Apr 2014 10:40:37 -0400 Subject: [PATCH 028/141] BF: testDatabase -- close and unlink the created test db file --- fail2ban/tests/servertestcase.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/fail2ban/tests/servertestcase.py b/fail2ban/tests/servertestcase.py index ead1a0cd..c78b42b9 100644 --- a/fail2ban/tests/servertestcase.py +++ b/fail2ban/tests/servertestcase.py @@ -154,7 +154,7 @@ class Transmitter(TransmitterBase): self.assertAlmostEqual(t1 - t0, 1, places=1) def testDatabase(self): - _, tmpFilename = tempfile.mkstemp(".db", "Fail2Ban_") + tmp, tmpFilename = tempfile.mkstemp(".db", "fail2ban_") # Jails present, can't change database self.setGetTestNOK("dbfile", tmpFilename) self.server.delJail(self.jailName) @@ -175,6 +175,8 @@ class Transmitter(TransmitterBase): self.assertEqual(self.transm.proceed( ["get", "dbpurgeage"]), (0, None)) + os.close(tmp) + os.unlink(tmpFilename) def testAddJail(self): jail2 = "TestJail2" From e66231f041e7fa1925b080e43328dc1afd50ff8f Mon Sep 17 00:00:00 2001 From: Yaroslav Halchenko Date: Thu, 17 Apr 2014 13:19:03 -0400 Subject: [PATCH 029/141] ENH(DOC): Move contributions related docs into CONTRIBUTING.md so it pops up when people file PR/issues pushing directly without PR to trigger buildbot for master -- please pardon the rush --- CONTRIBUTING.md | 17 +++++++++++++++++ README.md | 20 ++++---------------- 2 files changed, 21 insertions(+), 16 deletions(-) create mode 100644 CONTRIBUTING.md diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md new file mode 100644 index 00000000..1638f3e6 --- /dev/null +++ b/CONTRIBUTING.md @@ -0,0 +1,17 @@ +Guidelines on Fail2Ban contributions +==================================== + +### You found a severe security vulnerability in Fail2Ban? +email details to fail2ban-vulnerabilities at lists dot sourceforge dot net . + +### You need some new features, you found bugs? +visit [Issues](https://github.com/fail2ban/fail2ban/issues) +and if your issue is not yet known -- file a bug report. See +[Fail2Ban wiki](http://www.fail2ban.org/wiki/index.php/HOWTO_Seek_Help) +on further instructions. + +### You would like to troubleshoot or discuss? +join the [mailing list](https://lists.sourceforge.net/lists/listinfo/fail2ban-users) + +### You would like to contribute (new filters/actions/code/documentation)? +send a pull request diff --git a/README.md b/README.md index 308136a1..2679fe53 100644 --- a/README.md +++ b/README.md @@ -68,24 +68,12 @@ Code status: Contact: -------- -### You found a severe security vulnerability in Fail2Ban? -email details to fail2ban-vulnerabilities at lists dot sourceforge dot net . - -### You need some new features, you found bugs? -visit [Issues](https://github.com/fail2ban/fail2ban/issues) -and if your issue is not yet known -- file a bug report. See -[Fail2Ban wiki](http://www.fail2ban.org/wiki/index.php/HOWTO_Seek_Help) -on further instructions. - -### You would like to troubleshoot or discuss? -join the [mailing list](https://lists.sourceforge.net/lists/listinfo/fail2ban-users) - -### You would like to contribute (new filters/actions/code/documentation)? -send a pull request +### Bugs, feature requests, discussions? +See [CONTRIBUTING.md](https://github.com/fail2ban/fail2ban/blob/master/CONTRIBUTING.md) ### You just appreciate this program: -send kudos to the original author ([Cyril Jaquier](mailto: Cyril Jaquier ) -or better to the [mailing list](https://lists.sourceforge.net/lists/listinfo/fail2ban-users) +send kudos to the original author ([Cyril Jaquier](mailto: Cyril Jaquier )) +or *better* to the [mailing list](https://lists.sourceforge.net/lists/listinfo/fail2ban-users) since Fail2Ban is "community-driven" for years now. Thanks: From 16077a2771f8cdef4dbc98abf6c1f438488b7e7f Mon Sep 17 00:00:00 2001 From: Yaroslav Halchenko Date: Thu, 17 Apr 2014 14:08:43 -0400 Subject: [PATCH 030/141] add .dev to 0.9.0 version --- fail2ban/version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fail2ban/version.py b/fail2ban/version.py index 81825036..914cfe29 100644 --- a/fail2ban/version.py +++ b/fail2ban/version.py @@ -24,4 +24,4 @@ __author__ = "Cyril Jaquier, Yaroslav Halchenko, Steven Hiscocks, Daniel Black" __copyright__ = "Copyright (c) 2004 Cyril Jaquier, 2011-2014 Yaroslav Halchenko, 2013-2013 Steven Hiscocks, Daniel Black" __license__ = "GPL-v2+" -version = "0.9.0" +version = "0.9.0.dev" From 2bc509fcc73c5c405888a73af98f88fd4d7e934c Mon Sep 17 00:00:00 2001 From: Yaroslav Halchenko Date: Thu, 17 Apr 2014 14:34:26 -0400 Subject: [PATCH 031/141] DOC: minor changes just to trigger the build --- README.Solaris | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/README.Solaris b/README.Solaris index e41e3811..c654b7c0 100644 --- a/README.Solaris +++ b/README.Solaris @@ -6,20 +6,20 @@ By Roy Sigurd Karlsbakk ABOUT -This readme is meant for those wanting to install fail2ban on Solaris 10, +This README is meant for those wanting to install fail2ban on Solaris 10, OpenSolaris, OpenIndiana etc. To some degree it may as well be useful for users of older Solaris versions and Nexenta, but don't rely on it. READ ME FIRST If I use the term Solaris, I am talking about any Solaris dialect, that is, the -official Sun/Oracle ones or derivates. If I describe an OS as +official Sun/Oracle ones or derivatives. If I describe an OS as "OpenSolaris-based", it means it's either OpenSolaris, OpenIndiana or one of the other, but /not/ the Nexenta family, since this only uses the OpenSolaris/ IllumOS kernel and not the userland. If I say Solaris 10, I mean Solaris 10 and perhaps, if you're lucky and have some good gods on your side, it may also apply to Solaris 9 or even 8 and hopefully in the new Solaris 11 whenever that may be -released. Quoted lines of code, settings et cetera are indented with two spaces. +released. Quoted lines of code, settings etc. are indented with two spaces. This does _not_ mean you should use that indentation, especially in config files where they can be harmful. Optional settings are prefixed with OPT: while required settings are prefixed with REQ:. If no prefix is found, regard it as a @@ -111,7 +111,7 @@ GOTCHAS AND FIXMES svcadm enable fail2ban * If svcs -xv says that fail2ban failed to start or svcs says it's in maintenance mode - check /var/svc/log/network-fail2ban:default.log for clues. + check /var/svc/log/network-fail2ban:default.log for clues. Check permissions on /var/adm, /var/adm/auth.log /var/adm/fail2ban.log and /var/run/fail2ban You may need to: From 5e179f5dcb87c453f57db50871ba6251e2cd7d7c Mon Sep 17 00:00:00 2001 From: Yaroslav Halchenko Date: Thu, 17 Apr 2014 22:23:20 -0400 Subject: [PATCH 032/141] TST: skip the test if a known problem with Python 2.6 is detected As was original "discovered" while running tests on OSX with python2.6: http://nipy.bic.berkeley.edu/builders/fail2ban-py2.7-osx-10.6_master/builds/6/steps/shell_2/logs/stdio --- fail2ban/tests/servertestcase.py | 26 +++++++++++++++++++++----- 1 file changed, 21 insertions(+), 5 deletions(-) diff --git a/fail2ban/tests/servertestcase.py b/fail2ban/tests/servertestcase.py index c78b42b9..7b05a9b6 100644 --- a/fail2ban/tests/servertestcase.py +++ b/fail2ban/tests/servertestcase.py @@ -530,11 +530,27 @@ class Transmitter(TransmitterBase): def testPythonActionMethodsAndProperties(self): action = "TestCaseAction" - self.assertEqual( - self.transm.proceed(["set", self.jailName, "addaction", action, - os.path.join(TEST_FILES_DIR, "action.d", "action.py"), - '{"opt1": "value"}']), - (0, action)) + try: + out = self.transm.proceed( + ["set", self.jailName, "addaction", action, + os.path.join(TEST_FILES_DIR, "action.d", "action.py"), + '{"opt1": "value"}']) + self.assertEqual(out, (0, action)) + except AssertionError: + if ((2, 6) <= sys.version_info < (2, 6, 5)) \ + and '__init__() keywords must be strings' in out[1]: + # known issue http://bugs.python.org/issue2646 in 2.6 series + # since general Fail2Ban warnings are suppressed in normal + # operation -- let's issue Python's native warning here + import warnings + warnings.warn( + "Your version of Python %s seems to experience a known " + "issue forbidding correct operation of Fail2Ban: " + "http://bugs.python.org/issue2646 Upgrade your Python and " + "meanwhile other intestPythonActionMethodsAndProperties will " + "be skipped" % (sys.version)) + return + raise self.assertEqual( sorted(self.transm.proceed(["get", self.jailName, "actionproperties", action])[1]), From 78c82b3da784ef516776f38b8a7e63c140f8cef5 Mon Sep 17 00:00:00 2001 From: Yaroslav Halchenko Date: Thu, 17 Apr 2014 23:16:41 -0400 Subject: [PATCH 033/141] DOC: minor -- added a link to pull requests in CONTRIBUTING --- CONTRIBUTING.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 1638f3e6..a668fa3d 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -14,4 +14,4 @@ on further instructions. join the [mailing list](https://lists.sourceforge.net/lists/listinfo/fail2ban-users) ### You would like to contribute (new filters/actions/code/documentation)? -send a pull request +send a [pull request](https://github.com/fail2ban/fail2ban/pulls) From 6a740f684a52f4ccc310e75d17b18ec610c70bb0 Mon Sep 17 00:00:00 2001 From: Steven Hiscocks Date: Fri, 18 Apr 2014 23:27:30 +0100 Subject: [PATCH 034/141] ENH: Move traceback formatter to from tests.utils to helpers Now allows for tests to be removed from package if desired --- bin/fail2ban-regex | 2 +- bin/fail2ban-testcases | 3 +- fail2ban/helpers.py | 83 ++++++++++++++++++++++++++++++++- fail2ban/tests/misctestcase.py | 3 +- fail2ban/tests/utils.py | 84 ++-------------------------------- 5 files changed, 91 insertions(+), 84 deletions(-) diff --git a/bin/fail2ban-regex b/bin/fail2ban-regex index ef198dcb..f1d5bdb9 100755 --- a/bin/fail2ban-regex +++ b/bin/fail2ban-regex @@ -45,7 +45,7 @@ from fail2ban.client.filterreader import FilterReader from fail2ban.server.filter import Filter from fail2ban.server.failregex import RegexException -from fail2ban.tests.utils import FormatterWithTraceBack +from fail2ban.helpers import FormatterWithTraceBack # Gets the instance of the logger. logSys = logging.getLogger("fail2ban") diff --git a/bin/fail2ban-testcases b/bin/fail2ban-testcases index b3bddf1c..0e2fdb4b 100755 --- a/bin/fail2ban-testcases +++ b/bin/fail2ban-testcases @@ -34,7 +34,8 @@ if os.path.exists("fail2ban/__init__.py"): sys.path.insert(0, ".") from fail2ban.version import version -from fail2ban.tests.utils import FormatterWithTraceBack, gatherTests +from fail2ban.tests.utils import gatherTests +from fail2ban.helpers import FormatterWithTraceBack from fail2ban.server.mytime import MyTime from optparse import OptionParser, Option diff --git a/fail2ban/helpers.py b/fail2ban/helpers.py index 74ea7a7a..2579381d 100644 --- a/fail2ban/helpers.py +++ b/fail2ban/helpers.py @@ -20,9 +20,90 @@ __author__ = "Cyril Jaquier, Arturo 'Buanzo' Busleiman, Yaroslav Halchenko" __license__ = "GPL" +import sys +import os +import traceback +import re +import logging def formatExceptionInfo(): """ Consistently format exception information """ - import sys cla, exc = sys.exc_info()[:2] return (cla.__name__, str(exc)) + +# +# Following "traceback" functions are adopted from PyMVPA distributed +# under MIT/Expat and copyright by PyMVPA developers (i.e. me and +# Michael). Hereby I re-license derivative work on these pieces under GPL +# to stay in line with the main Fail2Ban license +# +def mbasename(s): + """Custom function to include directory name if filename is too common + + Also strip .py at the end + """ + base = os.path.basename(s) + if base.endswith('.py'): + base = base[:-3] + if base in set(['base', '__init__']): + base = os.path.basename(os.path.dirname(s)) + '.' + base + return base + +class TraceBack(object): + """Customized traceback to be included in debug messages + """ + + def __init__(self, compress=False): + """Initialize TrackBack metric + + Parameters + ---------- + compress : bool + if True then prefix common with previous invocation gets + replaced with ... + """ + self.__prev = "" + self.__compress = compress + + def __call__(self): + ftb = traceback.extract_stack(limit=100)[:-2] + entries = [ + [mbasename(x[0]), os.path.dirname(x[0]), str(x[1])] for x in ftb] + entries = [ [e[0], e[2]] for e in entries + if not (e[0] in ['unittest', 'logging.__init__'] + or e[1].endswith('/unittest'))] + + # lets make it more concise + entries_out = [entries[0]] + for entry in entries[1:]: + if entry[0] == entries_out[-1][0]: + entries_out[-1][1] += ',%s' % entry[1] + else: + entries_out.append(entry) + sftb = '>'.join(['%s:%s' % (mbasename(x[0]), + x[1]) for x in entries_out]) + if self.__compress: + # lets remove part which is common with previous invocation + prev_next = sftb + common_prefix = os.path.commonprefix((self.__prev, sftb)) + common_prefix2 = re.sub('>[^>]*$', '', common_prefix) + + if common_prefix2 != "": + sftb = '...' + sftb[len(common_prefix2):] + self.__prev = prev_next + + return sftb + +class FormatterWithTraceBack(logging.Formatter): + """Custom formatter which expands %(tb) and %(tbc) with tracebacks + + TODO: might need locking in case of compressed tracebacks + """ + def __init__(self, fmt, *args, **kwargs): + logging.Formatter.__init__(self, fmt=fmt, *args, **kwargs) + compress = '%(tbc)s' in fmt + self._tb = TraceBack(compress=compress) + + def format(self, record): + record.tbc = record.tb = self._tb() + return logging.Formatter.format(self, record) diff --git a/fail2ban/tests/misctestcase.py b/fail2ban/tests/misctestcase.py index 284b684b..ca84eba7 100644 --- a/fail2ban/tests/misctestcase.py +++ b/fail2ban/tests/misctestcase.py @@ -32,8 +32,7 @@ import datetime from glob import glob from StringIO import StringIO -from .utils import mbasename, TraceBack, FormatterWithTraceBack -from ..helpers import formatExceptionInfo +from ..helpers import formatExceptionInfo, mbasename, TraceBack, FormatterWithTraceBack from ..server.datetemplate import DatePatternRegex diff --git a/fail2ban/tests/utils.py b/fail2ban/tests/utils.py index 85c1d929..7727632e 100644 --- a/fail2ban/tests/utils.py +++ b/fail2ban/tests/utils.py @@ -22,90 +22,17 @@ __author__ = "Yaroslav Halchenko" __copyright__ = "Copyright (c) 2013 Yaroslav Halchenko" __license__ = "GPL" -import logging, os, re, traceback, time, unittest -from os.path import basename, dirname +import logging +import os +import re +import time +import unittest from StringIO import StringIO from ..server.mytime import MyTime logSys = logging.getLogger(__name__) -# -# Following "traceback" functions are adopted from PyMVPA distributed -# under MIT/Expat and copyright by PyMVPA developers (i.e. me and -# Michael). Hereby I re-license derivative work on these pieces under GPL -# to stay in line with the main Fail2Ban license -# -def mbasename(s): - """Custom function to include directory name if filename is too common - - Also strip .py at the end - """ - base = basename(s) - if base.endswith('.py'): - base = base[:-3] - if base in set(['base', '__init__']): - base = basename(dirname(s)) + '.' + base - return base - -class TraceBack(object): - """Customized traceback to be included in debug messages - """ - - def __init__(self, compress=False): - """Initialize TrackBack metric - - Parameters - ---------- - compress : bool - if True then prefix common with previous invocation gets - replaced with ... - """ - self.__prev = "" - self.__compress = compress - - def __call__(self): - ftb = traceback.extract_stack(limit=100)[:-2] - entries = [[mbasename(x[0]), dirname(x[0]), str(x[1])] for x in ftb] - entries = [ [e[0], e[2]] for e in entries - if not (e[0] in ['unittest', 'logging.__init__'] - or e[1].endswith('/unittest'))] - - # lets make it more concise - entries_out = [entries[0]] - for entry in entries[1:]: - if entry[0] == entries_out[-1][0]: - entries_out[-1][1] += ',%s' % entry[1] - else: - entries_out.append(entry) - sftb = '>'.join(['%s:%s' % (mbasename(x[0]), - x[1]) for x in entries_out]) - if self.__compress: - # lets remove part which is common with previous invocation - prev_next = sftb - common_prefix = os.path.commonprefix((self.__prev, sftb)) - common_prefix2 = re.sub('>[^>]*$', '', common_prefix) - - if common_prefix2 != "": - sftb = '...' + sftb[len(common_prefix2):] - self.__prev = prev_next - - return sftb - -class FormatterWithTraceBack(logging.Formatter): - """Custom formatter which expands %(tb) and %(tbc) with tracebacks - - TODO: might need locking in case of compressed tracebacks - """ - def __init__(self, fmt, *args, **kwargs): - logging.Formatter.__init__(self, fmt=fmt, *args, **kwargs) - compress = '%(tbc)s' in fmt - self._tb = TraceBack(compress=compress) - - def format(self, record): - record.tbc = record.tb = self._tb() - return logging.Formatter.format(self, record) - def mtimesleep(): # no sleep now should be necessary since polling tracks now not only # mtime but also ino and size @@ -146,7 +73,6 @@ def gatherTests(regexps=None, no_network=False): if not regexps: # pragma: no cover tests = unittest.TestSuite() else: # pragma: no cover - import re class FilteredTestSuite(unittest.TestSuite): _regexps = [re.compile(r) for r in regexps] def addTest(self, suite): From af07b2edf8d1c7b660d8ed9a11f973e186fae8b9 Mon Sep 17 00:00:00 2001 From: Yaroslav Halchenko Date: Fri, 18 Apr 2014 23:59:24 -0400 Subject: [PATCH 035/141] very minor --- ChangeLog | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ChangeLog b/ChangeLog index 66df9639..84e9c87c 100644 --- a/ChangeLog +++ b/ChangeLog @@ -27,7 +27,7 @@ ver. 0.9.1 (2014/xx/xx) - better, faster, stronger * Fail2ban-regex - add print-all-matched option. Closes gh-652 * Suppress fail2ban-client warnings for non-critical config options -ver. 0.9.0 (2014/03/14 - beta +ver. 0.9.0 (2014/03/14) - beta ---------- Carries all fixes, features and enhancements from 0.8.13 (unreleased) with From 03d90c2f4212e9a9fee504665d08a5bec1e074f0 Mon Sep 17 00:00:00 2001 From: Steven Hiscocks Date: Sat, 19 Apr 2014 18:07:23 +0100 Subject: [PATCH 036/141] BF: recidive filter and samples at wrong log level: WARNING->NOTICE --- ChangeLog | 1 + config/filter.d/recidive.conf | 2 +- fail2ban/tests/files/logs/recidive | 12 +++++++----- 3 files changed, 9 insertions(+), 6 deletions(-) diff --git a/ChangeLog b/ChangeLog index 84e9c87c..bb1fd707 100644 --- a/ChangeLog +++ b/ChangeLog @@ -14,6 +14,7 @@ ver. 0.9.1 (2014/xx/xx) - better, faster, stronger * systemd backend error on bad utf-8 in python3 * badips.py action error when logging HTTP error raised with badips request * fail2ban-regex failed to work in python3 due to space/tab mix + * recidive regex samples incorrect log level * journalmatch for recidive incorrect PRIORITY * loglevel couldn't be changed in fail2ban.conf * Handle case when no sqlite library is available for persistent database diff --git a/config/filter.d/recidive.conf b/config/filter.d/recidive.conf index da90d5aa..aa1b0013 100644 --- a/config/filter.d/recidive.conf +++ b/config/filter.d/recidive.conf @@ -27,7 +27,7 @@ _daemon = fail2ban\.server\.actions # jail using this filter 'recidive', or change this line! _jailname = recidive -failregex = ^(%(__prefix_line)s| %(_daemon)s%(__pid_re)s?:\s+)WARNING\s+\[(?!%(_jailname)s\])(?:.*)\]\s+Ban\s+\s*$ +failregex = ^(%(__prefix_line)s| %(_daemon)s%(__pid_re)s?:\s+)NOTICE\s+\[(?!%(_jailname)s\])(?:.*)\]\s+Ban\s+\s*$ [Init] diff --git a/fail2ban/tests/files/logs/recidive b/fail2ban/tests/files/logs/recidive index b9c39105..4b23b1a5 100644 --- a/fail2ban/tests/files/logs/recidive +++ b/fail2ban/tests/files/logs/recidive @@ -1,12 +1,14 @@ # failJSON: { "time": "2006-02-13T15:52:30", "match": true , "host": "1.2.3.4" } -2006-02-13 15:52:30,388 fail2ban.server.actions: WARNING [sendmail] Ban 1.2.3.4 +2006-02-13 15:52:30,388 fail2ban.server.actions: NOTICE [sendmail] Ban 1.2.3.4 # failJSON: { "time": "2006-02-13T15:52:30", "match": true , "host": "1.2.3.4", "desc": "Extended with [PID]" } -2006-02-13 15:52:30,388 fail2ban.server.actions[123]: WARNING [sendmail] Ban 1.2.3.4 +2006-02-13 15:52:30,388 fail2ban.server.actions[123]: NOTICE [sendmail] Ban 1.2.3.4 # failJSON: { "match": false } -2006-02-13 16:07:31,183 fail2ban.server.actions: WARNING [sendmail] Unban 1.2.3.4 +2006-02-13 16:07:31,183 fail2ban.server.actions: NOTICE [sendmail] Unban 1.2.3.4 # failJSON: { "match": false } -2006-02-13 15:52:30,388 fail2ban.server.actions: WARNING [recidive] Ban 1.2.3.4 +2006-02-13 15:52:30,388 fail2ban.server.actions: NOTICE [recidive] Ban 1.2.3.4 # syslog example # failJSON: { "time": "2004-09-16T00:44:55", "match": true , "host": "10.0.0.7" } -Sep 16 00:44:55 spaceman fail2ban.server.actions: WARNING [jail] Ban 10.0.0.7 +Sep 16 00:44:55 spaceman fail2ban.server.actions: NOTICE [jail] Ban 10.0.0.7 +# failJSON: { "time": "2006-02-13T15:52:30", "match": true , "host": "1.2.3.4", "desc": "Extended with [PID] and padding" } +2006-02-13 15:52:30,388 fail2ban.server.actions[123]: NOTICE [sendmail] Ban 1.2.3.4 From a3f8b943a3d891f7a48debf9b3dfa8902a45b5cb Mon Sep 17 00:00:00 2001 From: Steven Hiscocks Date: Sat, 19 Apr 2014 18:25:59 +0100 Subject: [PATCH 037/141] TST: Add Python 3.4 for TravisCI --- .travis.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.travis.yml b/.travis.yml index 41eeca27..9a92a7f6 100644 --- a/.travis.yml +++ b/.travis.yml @@ -6,6 +6,7 @@ python: - "2.7" - "3.2" - "3.3" + - "3.4" - "pypy" before_install: - if [[ $TRAVIS_PYTHON_VERSION == 2.7 ]]; then sudo apt-get update -qq; fi From 233aa043f356ad2a9439cfec3db31c6be06d0104 Mon Sep 17 00:00:00 2001 From: Steven Hiscocks Date: Sat, 19 Apr 2014 18:39:55 +0100 Subject: [PATCH 038/141] Revert "TST: Add Python 3.4 for TravisCI" This reverts commit a3f8b943a3d891f7a48debf9b3dfa8902a45b5cb. TravsCI doesn't support 3.4...yet --- .travis.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 9a92a7f6..41eeca27 100644 --- a/.travis.yml +++ b/.travis.yml @@ -6,7 +6,6 @@ python: - "2.7" - "3.2" - "3.3" - - "3.4" - "pypy" before_install: - if [[ $TRAVIS_PYTHON_VERSION == 2.7 ]]; then sudo apt-get update -qq; fi From 72bfd1433032c8790d2f5add267076798eed7ce5 Mon Sep 17 00:00:00 2001 From: Jason Martin Date: Sat, 19 Apr 2014 12:58:03 -0700 Subject: [PATCH 039/141] Tidy up filter.d/monit.conf, make regex more complete. Add ChangeLog / THANKS entry. Add test cases. --- ChangeLog | 1 + THANKS | 1 + config/filter.d/monit.conf | 13 ++----------- fail2ban/tests/files/logs/monit | 6 ++++++ 4 files changed, 10 insertions(+), 11 deletions(-) create mode 100644 fail2ban/tests/files/logs/monit diff --git a/ChangeLog b/ChangeLog index 66df9639..7113d431 100644 --- a/ChangeLog +++ b/ChangeLog @@ -21,6 +21,7 @@ ver. 0.9.1 (2014/xx/xx) - better, faster, stronger * Nginx filter to support missing server_name. Closes gh-676 - New features: + - Added monit filter thanks Jason H Martin. - Enhancements diff --git a/THANKS b/THANKS index 2c084dee..27165492 100644 --- a/THANKS +++ b/THANKS @@ -48,6 +48,7 @@ Ivo Truxa John Thoe Jacques Lav!gnotte Ioan Indreias +Jason H Martin Jonathan Kamens Jonathan Lanning Jonathan Underwood diff --git a/config/filter.d/monit.conf b/config/filter.d/monit.conf index f32eae61..04d01b20 100644 --- a/config/filter.d/monit.conf +++ b/config/filter.d/monit.conf @@ -2,17 +2,8 @@ # # -[INCLUDES] - -# Read common prefixes. If any customizations available -- read them from -# common.local -before = common.conf - [Definition] -# Samples: -# [PDT Apr 16 20:59:11] error : Warning: Client '1.2.3.4' supplied unknown user 'foo' accessing monit httpd -# [PDT Apr 16 20:59:33] error : Warning: Client '1.2.3.4' supplied wrong password for user 'admin' accessing monit httpd -failregex = Warning: Client '' supplied +failregex = \]\s*error\s*:\s*Warning:\s+Client '' supplied unknown user '\w+' accessing monit httpd$ + \]\s*error\s*:\s*Warning:\s+Client '' supplied wrong password for user '\w+' accessing monit httpd$ -ignoreregex = diff --git a/fail2ban/tests/files/logs/monit b/fail2ban/tests/files/logs/monit new file mode 100644 index 00000000..a923b6e2 --- /dev/null +++ b/fail2ban/tests/files/logs/monit @@ -0,0 +1,6 @@ +# failJSON: { "time": "2005-04-16T21:05:29", "match": true , "host": "69.93.127.111" } +[PDT Apr 16 21:05:29] error : Warning: Client '69.93.127.111' supplied unknown user 'foo' accessing monit httpd + +# failJSON: { "time": "2005-04-16T20:59:33", "match": true , "host": "97.113.189.111" } +[PDT Apr 16 20:59:33] error : Warning: Client '97.113.189.111' supplied wrong password for user 'admin' accessing monit httpd + From 0ef5027234fa6eaa9716abc9dd5ddcc5e67a3fe5 Mon Sep 17 00:00:00 2001 From: Cameron Norman Date: Sat, 19 Apr 2014 14:12:20 -0700 Subject: [PATCH 040/141] Change Upstart job to track PID of the server This only works correctly if the client does not fork itself when starting the server (which forks twice further). --- files/fail2ban.upstart | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/files/fail2ban.upstart b/files/fail2ban.upstart index ccf267f0..8a7ba10c 100644 --- a/files/fail2ban.upstart +++ b/files/fail2ban.upstart @@ -3,5 +3,13 @@ description "fail2ban - ban hosts that cause multiple authentication errors" start on filesystem and started networking stop on deconfiguring-networking -pre-start exec /usr/bin/fail2ban-client -x start +expect daemon +respawn + +pre-start script + [ -d /var/run/fail2ban ] || mkdir -p /var/run/fail2ban +end script + +exec /usr/bin/fail2ban-client -x start + post-stop exec /usr/bin/fail2ban-client stop From 39ad5b7474158c6fdb4b94935fe97f53e59fe69e Mon Sep 17 00:00:00 2001 From: Cameron Norman Date: Sat, 19 Apr 2014 15:10:19 -0700 Subject: [PATCH 041/141] Update Upstart job: uses stop command in pre-stop, removes PID file in post-stop --- files/fail2ban.upstart | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/files/fail2ban.upstart b/files/fail2ban.upstart index 8a7ba10c..baabb22b 100644 --- a/files/fail2ban.upstart +++ b/files/fail2ban.upstart @@ -12,4 +12,6 @@ end script exec /usr/bin/fail2ban-client -x start -post-stop exec /usr/bin/fail2ban-client stop +pre-stop exec /usr/bin/fail2ban-client stop + +post-stop exec rm -f /var/run/fail2ban/fail2ban.pid From a7766d331645226654d5baf92017f5b52d4dc297 Mon Sep 17 00:00:00 2001 From: Steven Hiscocks Date: Sun, 20 Apr 2014 17:59:41 +0100 Subject: [PATCH 042/141] DOC: Add notice message for systemd backend when no journal match --- fail2ban/server/filtersystemd.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/fail2ban/server/filtersystemd.py b/fail2ban/server/filtersystemd.py index 7ab56039..f2dee8cb 100644 --- a/fail2ban/server/filtersystemd.py +++ b/fail2ban/server/filtersystemd.py @@ -213,6 +213,12 @@ class FilterSystemd(JournalFilter): # pragma: systemd no cover def run(self): + if not self.getJournalMatch(): + logSys.notice( + "Jail started without 'journalmatch' set. " + "Jail regexs will be checked against all journal entries, " + "which is not advised for performance reasons.") + # Seek to now - findtime in journal start_time = datetime.datetime.now() - \ datetime.timedelta(seconds=int(self.getFindTime())) From 9c2a0cb40395e6d860867cbb23fbbab14cbde2b9 Mon Sep 17 00:00:00 2001 From: Cameron Norman Date: Sun, 20 Apr 2014 11:37:07 -0700 Subject: [PATCH 043/141] Added foreground and background options to fail2ban-client --- bin/fail2ban-client | 20 ++++++++++++++++---- 1 file changed, 16 insertions(+), 4 deletions(-) diff --git a/bin/fail2ban-client b/bin/fail2ban-client index 8737c49d..8d757cfe 100755 --- a/bin/fail2ban-client +++ b/bin/fail2ban-client @@ -51,6 +51,7 @@ class Fail2banClient: self.__conf["conf"] = "/etc/fail2ban" self.__conf["dump"] = False self.__conf["force"] = False + self.__conf["background"] = True self.__conf["verbose"] = 1 self.__conf["interactive"] = False self.__conf["socket"] = None @@ -83,6 +84,8 @@ class Fail2banClient: print " -v increase verbosity" print " -q decrease verbosity" print " -x force execution of the server (remove socket file)" + print " -b start server in background (default)" + print " -f start server in foreground" print " -h, --help display this help message" print " -V, --version print the version" print @@ -125,6 +128,10 @@ class Fail2banClient: self.__conf["force"] = True elif opt[0] == "-i": self.__conf["interactive"] = True + elif opt[0] == "-b": + self.__conf["background"] = True + elif opt[0] == "-f": + self.__conf["background"] = False elif opt[0] in ["-h", "--help"]: self.dispUsage() sys.exit(0) @@ -194,7 +201,8 @@ class Fail2banClient: # Start the server self.__startServerAsync(self.__conf["socket"], self.__conf["pidfile"], - self.__conf["force"]) + self.__conf["force"], + self.__conf["background"]) try: # Wait for the server to start self.__waitOnServer() @@ -242,14 +250,12 @@ class Fail2banClient: # # Start the Fail2ban server in daemon mode. - def __startServerAsync(self, socket, pidfile, force = False): + def __startServerAsync(self, socket, pidfile, force = False, background = True): # Forks the current process. pid = os.fork() if pid == 0: args = list() args.append(self.SERVER) - # Start in background mode. - args.append("-b") # Set the socket path. args.append("-s") args.append(socket) @@ -259,6 +265,12 @@ class Fail2banClient: # Force the execution if needed. if force: args.append("-x") + # Start in foreground mode if requested. + if background: + args.append("-b") + else: + args.append("-f") + try: # Use the current directory. exe = os.path.abspath(os.path.join(sys.path[0], self.SERVER)) From 1f53eb2d28ee48f4a0eb0b77f027cdebcb8a8b6e Mon Sep 17 00:00:00 2001 From: Cameron Norman Date: Sun, 20 Apr 2014 11:39:04 -0700 Subject: [PATCH 044/141] Updated man page for new options --- man/fail2ban-client.1 | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/man/fail2ban-client.1 b/man/fail2ban-client.1 index ec79d725..32580e20 100644 --- a/man/fail2ban-client.1 +++ b/man/fail2ban-client.1 @@ -34,6 +34,12 @@ decrease verbosity \fB\-x\fR force execution of the server (remove socket file) .TP +\fB\-b\fR +start the server in background mode (default) +.TP +\fB\-f\fR +start the server in foreground mode (note that the client forks once itself) +.TP \fB\-h\fR, \fB\-\-help\fR display this help message .TP From 7818b0cb2ab6122c72ba1f9206b6e543c3860eaf Mon Sep 17 00:00:00 2001 From: Cameron Norman Date: Sun, 20 Apr 2014 16:03:04 -0700 Subject: [PATCH 045/141] Added f and b to cmdOpts. f = start server in foreground; b = start server in background (default). --- bin/fail2ban-client | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bin/fail2ban-client b/bin/fail2ban-client index 8d757cfe..1acb3842 100755 --- a/bin/fail2ban-client +++ b/bin/fail2ban-client @@ -324,7 +324,7 @@ class Fail2banClient: # Reads the command line options. try: - cmdOpts = 'hc:s:p:xdviqV' + cmdOpts = 'hc:s:p:xfbdviqV' cmdLongOpts = ['help', 'version'] optList, args = getopt.getopt(self.__argv[1:], cmdOpts, cmdLongOpts) except getopt.GetoptError: From bbcbefd494a7120c4c8ac5341e859ae318d7a22f Mon Sep 17 00:00:00 2001 From: Steven Hiscocks Date: Tue, 22 Apr 2014 19:17:25 +0100 Subject: [PATCH 046/141] BF: bantime < 0 database should return all bans, as they are persistent --- ChangeLog | 1 + fail2ban/server/database.py | 13 ++++++++----- fail2ban/tests/databasetestcase.py | 13 +++++++++++-- 3 files changed, 20 insertions(+), 7 deletions(-) diff --git a/ChangeLog b/ChangeLog index bb1fd707..e8626930 100644 --- a/ChangeLog +++ b/ChangeLog @@ -20,6 +20,7 @@ ver. 0.9.1 (2014/xx/xx) - better, faster, stronger * Handle case when no sqlite library is available for persistent database * Only reban once per IP from database on fail2ban restart * Nginx filter to support missing server_name. Closes gh-676 + * Database now returns persistent bans on restart (bantime < 0) - New features: diff --git a/fail2ban/server/database.py b/fail2ban/server/database.py index 93186222..54cca4d3 100644 --- a/fail2ban/server/database.py +++ b/fail2ban/server/database.py @@ -380,7 +380,7 @@ class Fail2BanDb(object): if jail is not None: query += " AND jail=?" queryArgs.append(jail.name) - if bantime is not None: + if bantime is not None and bantime >= 0: query += " AND timeofban > ?" queryArgs.append(MyTime.time() - bantime) if ip is not None: @@ -399,7 +399,8 @@ class Fail2BanDb(object): Jail that the ban belongs to. Default `None`; all jails. bantime : int Ban time in seconds, such that bans returned would still be - valid now. Default `None`; no limit. + valid now. Negative values are equivalent to `None`. + Default `None`; no limit. ip : str IP Address to filter bans by. Default `None`; all IPs. @@ -427,7 +428,8 @@ class Fail2BanDb(object): Jail that the ban belongs to. Default `None`; all jails. bantime : int Ban time in seconds, such that bans returned would still be - valid now. Default `None`; no limit. + valid now. Negative values are equivalent to `None`. + Default `None`; no limit. ip : str IP Address to filter bans by. Default `None`; all IPs. @@ -438,7 +440,8 @@ class Fail2BanDb(object): in a list. When `ip` argument passed, a single `Ticket` is returned. """ - if bantime is None: + cacheKey = None + if bantime is None or bantime < 0: cacheKey = (ip, jail) if cacheKey in self._bansMergedCache: return self._bansMergedCache[cacheKey] @@ -468,7 +471,7 @@ class Fail2BanDb(object): ticket.setAttempt(failures) tickets.append(ticket) - if bantime is None: + if cacheKey: self._bansMergedCache[cacheKey] = tickets if ip is None else ticket return tickets if ip is None else ticket diff --git a/fail2ban/tests/databasetestcase.py b/fail2ban/tests/databasetestcase.py index 84101c50..2cf8577e 100644 --- a/fail2ban/tests/databasetestcase.py +++ b/fail2ban/tests/databasetestcase.py @@ -177,10 +177,15 @@ class DatabaseTest(unittest.TestCase): if Fail2BanDb is None: # pragma: no cover return self.testAddJail() - ticket = FailTicket("127.0.0.1", MyTime.time() - 40, ["abc\n"]) - self.db.addBan(self.jail, ticket) + self.db.addBan( + self.jail, FailTicket("127.0.0.1", MyTime.time() - 60, ["abc\n"])) + self.db.addBan( + self.jail, FailTicket("127.0.0.1", MyTime.time() - 40, ["abc\n"])) self.assertEqual(len(self.db.getBans(jail=self.jail,bantime=50)), 1) self.assertEqual(len(self.db.getBans(jail=self.jail,bantime=20)), 0) + # Negative values are for persistent bans, and such all bans should + # be returned + self.assertEqual(len(self.db.getBans(jail=self.jail,bantime=-1)), 2) def testGetBansMerged(self): if Fail2BanDb is None: # pragma: no cover @@ -251,6 +256,10 @@ class DatabaseTest(unittest.TestCase): self.assertEqual(len(tickets), 1) tickets = self.db.getBansMerged(bantime=5) self.assertEqual(len(tickets), 0) + # Negative values are for persistent bans, and such all bans should + # be returned + tickets = self.db.getBansMerged(bantime=-1) + self.assertEqual(len(tickets), 2) def testPurge(self): if Fail2BanDb is None: # pragma: no cover From 73cb3e3eec59b8be1a6781454bbe7be940603066 Mon Sep 17 00:00:00 2001 From: Cameron Norman Date: Tue, 22 Apr 2014 20:20:07 -0700 Subject: [PATCH 047/141] Added more specific help message to fail2ban-client with -f option --- bin/fail2ban-client | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bin/fail2ban-client b/bin/fail2ban-client index 1acb3842..289d7b39 100755 --- a/bin/fail2ban-client +++ b/bin/fail2ban-client @@ -85,7 +85,7 @@ class Fail2banClient: print " -q decrease verbosity" print " -x force execution of the server (remove socket file)" print " -b start server in background (default)" - print " -f start server in foreground" + print " -f start server in foreground (note that the client forks once itself)" print " -h, --help display this help message" print " -V, --version print the version" print From 9c3cb31862f8e0b31d12f0d02a9d114979cf3cd6 Mon Sep 17 00:00:00 2001 From: Jason Martin Date: Tue, 22 Apr 2014 21:29:52 -0700 Subject: [PATCH 048/141] Even stricter monit regex, now covers entire line --- config/filter.d/monit.conf | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/config/filter.d/monit.conf b/config/filter.d/monit.conf index 04d01b20..1fcd980b 100644 --- a/config/filter.d/monit.conf +++ b/config/filter.d/monit.conf @@ -4,6 +4,6 @@ [Definition] -failregex = \]\s*error\s*:\s*Warning:\s+Client '' supplied unknown user '\w+' accessing monit httpd$ - \]\s*error\s*:\s*Warning:\s+Client '' supplied wrong password for user '\w+' accessing monit httpd$ +failregex = ^\[[A-Z]+\s+\]\s*error\s*:\s*Warning:\s+Client '' supplied unknown user '\w+' accessing monit httpd$ + ^\[[A-Z]+\s+\]\s*error\s*:\s*Warning:\s+Client '' supplied wrong password for user '\w+' accessing monit httpd$ From 2a14e48f0ba3a6c27af15af47e9c243ebb9373e1 Mon Sep 17 00:00:00 2001 From: Cameron Norman Date: Tue, 22 Apr 2014 21:55:51 -0700 Subject: [PATCH 049/141] A few final touches on the Upstart job (a) use static-network-up, since it is more generic than the started networking event (b) do not hook into network deconfiguration to speed up shutdown (c) expect fork, per the use of the "-f" option (d) use a variable for the run directory to make changing it simpler (e) handle the situation of a left over socket file (f) use the -f option to be able to track the PID --- files/fail2ban.upstart | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/files/fail2ban.upstart b/files/fail2ban.upstart index baabb22b..18fafebd 100644 --- a/files/fail2ban.upstart +++ b/files/fail2ban.upstart @@ -1,17 +1,20 @@ description "fail2ban - ban hosts that cause multiple authentication errors" -start on filesystem and started networking -stop on deconfiguring-networking +start on filesystem and static-network-up +stop on runlevel [016] -expect daemon +expect fork respawn +env RUNDIR=/var/run/fail2ban + pre-start script - [ -d /var/run/fail2ban ] || mkdir -p /var/run/fail2ban + test -d $RUNDIR || mkdir -p $RUNDIR + test ! -e $RUNDIR/fail2ban.sock || rm -f $RUNDIR/fail2ban.sock end script -exec /usr/bin/fail2ban-client -x start +exec /usr/bin/fail2ban-client -f -x start pre-stop exec /usr/bin/fail2ban-client stop -post-stop exec rm -f /var/run/fail2ban/fail2ban.pid +post-stop exec rm -f $RUNDIR/fail2ban.pid From 596b819bdcd100e0c1169dcd8a161e7b6cae2c74 Mon Sep 17 00:00:00 2001 From: Yaroslav Halchenko Date: Wed, 23 Apr 2014 10:04:17 -0400 Subject: [PATCH 050/141] DOC: minor -- tabify docstring in badips.py action --- config/action.d/badips.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/config/action.d/badips.py b/config/action.d/badips.py index f740b4ed..a3f0883c 100644 --- a/config/action.d/badips.py +++ b/config/action.d/badips.py @@ -36,7 +36,7 @@ from fail2ban.server.actions import ActionBase from fail2ban.version import version as f2bVersion class BadIPsAction(ActionBase): - """Fail2Ban action which resports bans to badips.com, and also + """Fail2Ban action which reports bans to badips.com, and also blacklist bad IPs listed on badips.com by using another action's ban method. @@ -53,7 +53,7 @@ class BadIPsAction(ActionBase): age : str, optional Age of last report for bad IPs, per badips.com syntax. Default "24h" (24 hours) - key : str, optional + key : str, optional Key issued by badips.com to report bans, for later retrieval of personalised content. banaction : str, optional @@ -65,7 +65,7 @@ class BadIPsAction(ActionBase): from category used for reporting. e.g. may want to report "postfix", but want to use whole "mail" category for blacklist. Default `category`. - bankey : str, optional + bankey : str, optional Key issued by badips.com to blacklist IPs reported with the associated key. updateperiod : int, optional From 7cc64a14e03a43db852eb3c2452892cde6acc53c Mon Sep 17 00:00:00 2001 From: Steven Hiscocks Date: Sun, 27 Apr 2014 13:27:11 +0100 Subject: [PATCH 051/141] BF: fail2ban-regex assertion error caused by miscounted "missed" lines Caused when removing lines as part of multiline regex, which had been previously considered missed. --- ChangeLog | 2 ++ bin/fail2ban-regex | 1 + 2 files changed, 3 insertions(+) diff --git a/ChangeLog b/ChangeLog index bb1fd707..bbe32054 100644 --- a/ChangeLog +++ b/ChangeLog @@ -20,6 +20,8 @@ ver. 0.9.1 (2014/xx/xx) - better, faster, stronger * Handle case when no sqlite library is available for persistent database * Only reban once per IP from database on fail2ban restart * Nginx filter to support missing server_name. Closes gh-676 + * fail2ban-regex assertion error caused by miscount missed lines with + multiline regex - New features: diff --git a/bin/fail2ban-regex b/bin/fail2ban-regex index ef198dcb..974566bc 100755 --- a/bin/fail2ban-regex +++ b/bin/fail2ban-regex @@ -344,6 +344,7 @@ class Fail2banRegex(object): pass else: self._line_stats.matched += 1 + self._line_stats.missed -= 1 return line, ret def process(self, test_lines): From bc10b64c69be90f9b5bc487d04c376b9432a40ac Mon Sep 17 00:00:00 2001 From: Steven Hiscocks Date: Sun, 27 Apr 2014 13:35:55 +0100 Subject: [PATCH 052/141] ENH: Match non "Bye Bye" for sshd locked accounts failregex --- ChangeLog | 1 + config/filter.d/sshd.conf | 2 +- fail2ban/tests/files/logs/sshd | 7 +++++++ 3 files changed, 9 insertions(+), 1 deletion(-) diff --git a/ChangeLog b/ChangeLog index bbe32054..40188554 100644 --- a/ChangeLog +++ b/ChangeLog @@ -29,6 +29,7 @@ ver. 0.9.1 (2014/xx/xx) - better, faster, stronger - Enhancements * Fail2ban-regex - add print-all-matched option. Closes gh-652 * Suppress fail2ban-client warnings for non-critical config options + * Match non "Bye Bye" disconnect messages for sshd locked account regex ver. 0.9.0 (2014/03/14) - beta ---------- diff --git a/config/filter.d/sshd.conf b/config/filter.d/sshd.conf index 059052fc..195744f2 100644 --- a/config/filter.d/sshd.conf +++ b/config/filter.d/sshd.conf @@ -30,7 +30,7 @@ failregex = ^%(__prefix_line)s(?:error: PAM: )?[aA]uthentication (?:failure|erro ^%(__prefix_line)sReceived disconnect from : 3: \S+: Auth fail$ ^%(__prefix_line)sUser .+ from not allowed because a group is listed in DenyGroups\s*$ ^%(__prefix_line)sUser .+ from not allowed because none of user's groups are listed in AllowGroups\s*$ - ^(?P<__prefix>%(__prefix_line)s)User .+ not allowed because account is locked(?P=__prefix)(?:error: )?Received disconnect from : 11: Bye Bye \[preauth\]$ + ^(?P<__prefix>%(__prefix_line)s)User .+ not allowed because account is locked(?P=__prefix)(?:error: )?Received disconnect from : 11: .+ \[preauth\]$ ^(?P<__prefix>%(__prefix_line)s)Disconnecting: Too many authentication failures for .+? \[preauth\](?P=__prefix)(?:error: )?Connection closed by \[preauth\]$ ^(?P<__prefix>%(__prefix_line)s)Connection from port \d+(?P=__prefix)Disconnecting: Too many authentication failures for .+? \[preauth\]$ diff --git a/fail2ban/tests/files/logs/sshd b/fail2ban/tests/files/logs/sshd index e2246cf8..b9d1b9b4 100644 --- a/fail2ban/tests/files/logs/sshd +++ b/fail2ban/tests/files/logs/sshd @@ -136,3 +136,10 @@ Jul 13 18:44:28 mdop sshd[4931]: Received disconnect from 89.24.13.192: 3: com.j Feb 12 04:09:18 localhost sshd[26713]: Connection from 115.249.163.77 port 51353 # failJSON: { "time": "2005-02-12T04:09:21", "match": true , "host": "115.249.163.77", "desc": "from gh-457" } Feb 12 04:09:21 localhost sshd[26713]: Disconnecting: Too many authentication failures for root [preauth] + +# failJSON: { "match": false } +Apr 27 13:02:04 host sshd[29116]: User root not allowed because account is locked +# failJSON: { "match": false } +Apr 27 13:02:04 host sshd[29116]: input_userauth_request: invalid user root [preauth] +# failJSON: { "time": "2005-04-27T13:02:04", "match": true , "host": "1.2.3.4", "desc": "No Bye-Bye" } +Apr 27 13:02:04 host sshd[29116]: Received disconnect from 1.2.3.4: 11: Normal Shutdown, Thank you for playing [preauth] From b486014b3597957430ffaa86d1d537b280434148 Mon Sep 17 00:00:00 2001 From: Steven Hiscocks Date: Sat, 3 May 2014 12:09:48 +0100 Subject: [PATCH 053/141] TST: Add Python 3.4 for TravisCI This reverts commit 233aa043f356ad2a9439cfec3db31c6be06d0104. --- .travis.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.travis.yml b/.travis.yml index 41eeca27..9a92a7f6 100644 --- a/.travis.yml +++ b/.travis.yml @@ -6,6 +6,7 @@ python: - "2.7" - "3.2" - "3.3" + - "3.4" - "pypy" before_install: - if [[ $TRAVIS_PYTHON_VERSION == 2.7 ]]; then sudo apt-get update -qq; fi From cf3a6015f09b39cd668f1f202b695ea23c52fcb8 Mon Sep 17 00:00:00 2001 From: Steven Hiscocks Date: Sat, 3 May 2014 12:44:03 +0100 Subject: [PATCH 054/141] BF: Avoid closing "/dev/urandom" for Python 3.4.0 Upstream bug: http://bugs.python.org/issue21207 Closes gh-687 --- fail2ban/server/server.py | 18 +++++++++++++----- 1 file changed, 13 insertions(+), 5 deletions(-) diff --git a/fail2ban/server/server.py b/fail2ban/server/server.py index 1bf8dcbb..735ce0a9 100644 --- a/fail2ban/server/server.py +++ b/fail2ban/server/server.py @@ -523,11 +523,19 @@ class Server: except (AttributeError, ValueError): maxfd = 256 # default maximum - for fd in range(0, maxfd): - try: - os.close(fd) - except OSError: # ERROR (ignore) - pass + # urandom should not be closed in Python 3.4.0. Fixed in 3.4.1 + # http://bugs.python.org/issue21207 + if sys.version_info[0:3] == (3, 4, 0): # pragma: no cover + urandom_fd = os.open("/dev/urandom", os.O_RDONLY) + for fd in range(0, maxfd): + try: + if not os.path.sameopenfile(urandom_fd, fd): + os.close(fd) + except OSError: # ERROR (ignore) + pass + os.close(urandom_fd) + else: + os.closerange(0, maxfd) # Redirect the standard file descriptors to /dev/null. os.open("/dev/null", os.O_RDONLY) # standard input (0) From 1e8402cb9951f6447da42f054ea894b1c1cbee1c Mon Sep 17 00:00:00 2001 From: Steven Hiscocks Date: Sat, 3 May 2014 12:51:15 +0100 Subject: [PATCH 055/141] DOC: ChangeLog entry for Python 3.4.0 persistent "/dev/urandom" fix --- ChangeLog | 2 ++ 1 file changed, 2 insertions(+) diff --git a/ChangeLog b/ChangeLog index bbe32054..d3101944 100644 --- a/ChangeLog +++ b/ChangeLog @@ -22,6 +22,8 @@ ver. 0.9.1 (2014/xx/xx) - better, faster, stronger * Nginx filter to support missing server_name. Closes gh-676 * fail2ban-regex assertion error caused by miscount missed lines with multiline regex + * Fix actions failing to execute for Python 3.4.0. Work around for + http://bugs.python.org/issue21207 - New features: From b3266ba44d3311a35bd18a3055a77dc27f577085 Mon Sep 17 00:00:00 2001 From: Steven Hiscocks Date: Sat, 3 May 2014 14:28:13 +0100 Subject: [PATCH 056/141] BF: Tags not fully recursively substituted Note: recursive check ignored for "matches", as tags would be escaped, and hence shouldn't match "<%s>" as "" would become "\". This therefore maintains advantage of delayed call for {ip,jail,}matches. Fixes gh-713 --- fail2ban/server/action.py | 11 +++++++++-- fail2ban/tests/actiontestcase.py | 13 ++++++++++--- 2 files changed, 19 insertions(+), 5 deletions(-) diff --git a/fail2ban/server/action.py b/fail2ban/server/action.py index 0098c546..d1883338 100644 --- a/fail2ban/server/action.py +++ b/fail2ban/server/action.py @@ -371,8 +371,11 @@ class CommandAction(ActionBase): within the values recursively replaced. """ t = re.compile(r'<([^ >]+)>') - for tag, value in tags.iteritems(): - value = str(value) + for tag in tags.iterkeys(): + if tag.endswith('matches'): + # Escapped so wont match + continue + value = str(tags[tag]) m = t.search(value) done = [] #logSys.log(5, 'TAG: %s, value: %s' % (tag, value)) @@ -383,6 +386,9 @@ class CommandAction(ActionBase): # recursive definitions are bad #logSys.log(5, 'recursion fail tag: %s value: %s' % (tag, value) ) return False + elif found_tag.endswith('matches'): + # Escapped so wont match + continue else: if tags.has_key(found_tag): value = value.replace('<%s>' % found_tag , tags[found_tag]) @@ -441,6 +447,7 @@ class CommandAction(ActionBase): `query` string with tags replaced. """ string = query + aInfo = cls.substituteRecursiveTags(aInfo) for tag in aInfo: if "<%s>" % tag in query: value = str(aInfo[tag]) # assure string diff --git a/fail2ban/tests/actiontestcase.py b/fail2ban/tests/actiontestcase.py index cb004b4d..f1ea77ce 100644 --- a/fail2ban/tests/actiontestcase.py +++ b/fail2ban/tests/actiontestcase.py @@ -100,17 +100,24 @@ class CommandActionTest(LogCaptureTestCase): {'ipjailmatches': "some >char< should \< be[ escap}ed&\n"}), "some \\>char\\< should \\\\\\< be\\[ escap\\}ed\\&\n") + + # Recursive + aInfo["ABC"] = "" + self.assertEqual( + self.__action.replaceTag("Text text ABC", aInfo), + "Text 890 text 890 ABC") + # Callable self.assertEqual( - self.__action.replaceTag("09 11", - CallingMap(callme=lambda: str(10))), + self.__action.replaceTag("09 11", + CallingMap(matches=lambda: str(10))), "09 10 11") # As tag not present, therefore callable should not be called # Will raise ValueError if it is self.assertEqual( self.__action.replaceTag("abc", - CallingMap(callme=lambda: int("a"))), "abc") + CallingMap(matches=lambda: int("a"))), "abc") def testExecuteActionBan(self): self.__action.actionstart = "touch /tmp/fail2ban.test" From 65269365ee1bb1c958eddfd0c9db9c35a29491e9 Mon Sep 17 00:00:00 2001 From: Yaroslav Halchenko Date: Mon, 5 May 2014 23:16:18 -0400 Subject: [PATCH 057/141] minor --- ChangeLog | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ChangeLog b/ChangeLog index d3101944..150ea99d 100644 --- a/ChangeLog +++ b/ChangeLog @@ -22,7 +22,7 @@ ver. 0.9.1 (2014/xx/xx) - better, faster, stronger * Nginx filter to support missing server_name. Closes gh-676 * fail2ban-regex assertion error caused by miscount missed lines with multiline regex - * Fix actions failing to execute for Python 3.4.0. Work around for + * Fix actions failing to execute for Python 3.4.0. Workaround for http://bugs.python.org/issue21207 - New features: From 904b362215e587c1038e556681a787cea61eeccc Mon Sep 17 00:00:00 2001 From: Steven Hiscocks Date: Fri, 9 May 2014 20:25:44 +0100 Subject: [PATCH 058/141] DOC: ChangeLog update for recursive tag bug fix Also minor typo fixes in comments --- ChangeLog | 2 ++ fail2ban/server/action.py | 4 ++-- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/ChangeLog b/ChangeLog index bbe32054..ec6a3b28 100644 --- a/ChangeLog +++ b/ChangeLog @@ -22,6 +22,8 @@ ver. 0.9.1 (2014/xx/xx) - better, faster, stronger * Nginx filter to support missing server_name. Closes gh-676 * fail2ban-regex assertion error caused by miscount missed lines with multiline regex + * Recursive action tags now fully processed. Fixes issue with bsd-ipfw + action - New features: diff --git a/fail2ban/server/action.py b/fail2ban/server/action.py index d1883338..736386b1 100644 --- a/fail2ban/server/action.py +++ b/fail2ban/server/action.py @@ -373,7 +373,7 @@ class CommandAction(ActionBase): t = re.compile(r'<([^ >]+)>') for tag in tags.iterkeys(): if tag.endswith('matches'): - # Escapped so wont match + # Escapped so won't match continue value = str(tags[tag]) m = t.search(value) @@ -387,7 +387,7 @@ class CommandAction(ActionBase): #logSys.log(5, 'recursion fail tag: %s value: %s' % (tag, value) ) return False elif found_tag.endswith('matches'): - # Escapped so wont match + # Escapped so won't match continue else: if tags.has_key(found_tag): From 1e586fb0e94248cd0185e2df9a33dbbef9734299 Mon Sep 17 00:00:00 2001 From: Steven Hiscocks Date: Sun, 11 May 2014 14:49:49 +0100 Subject: [PATCH 059/141] ENH: explicitly define tags which should be escaped --- fail2ban/server/action.py | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/fail2ban/server/action.py b/fail2ban/server/action.py index 736386b1..fefe2c2c 100644 --- a/fail2ban/server/action.py +++ b/fail2ban/server/action.py @@ -194,6 +194,8 @@ class CommandAction(ActionBase): timeout """ + _escapedTags = set(('matches', 'ipmatches', 'ipjailmatches')) + def __init__(self, jail, name): super(CommandAction, self).__init__(jail, name) self.timeout = 60 @@ -351,8 +353,8 @@ class CommandAction(ActionBase): if not self.executeCmd(stopCmd, self.timeout): raise RuntimeError("Error stopping action") - @staticmethod - def substituteRecursiveTags(tags): + @classmethod + def substituteRecursiveTags(cls, tags): """Sort out tag definitions within other tags. so: becomes: @@ -372,8 +374,8 @@ class CommandAction(ActionBase): """ t = re.compile(r'<([^ >]+)>') for tag in tags.iterkeys(): - if tag.endswith('matches'): - # Escapped so won't match + if tag in cls._escapedTags: + # Escaped so won't match continue value = str(tags[tag]) m = t.search(value) @@ -386,8 +388,8 @@ class CommandAction(ActionBase): # recursive definitions are bad #logSys.log(5, 'recursion fail tag: %s value: %s' % (tag, value) ) return False - elif found_tag.endswith('matches'): - # Escapped so won't match + elif found_tag in cls._escapedTags: + # Escaped so won't match continue else: if tags.has_key(found_tag): @@ -451,7 +453,7 @@ class CommandAction(ActionBase): for tag in aInfo: if "<%s>" % tag in query: value = str(aInfo[tag]) # assure string - if tag.endswith('matches'): + if tag in cls._escapedTags: # That one needs to be escaped since its content is # out of our control value = cls.escapeTag(value) From 2bf0b4a50c9d79507115b4eddd2da66ef9b7945e Mon Sep 17 00:00:00 2001 From: sebres Date: Wed, 14 May 2014 22:26:22 +0100 Subject: [PATCH 060/141] strptime bug fix: if gmtoff is None we have 1 hour increment of time (through utctimetuple), compare: >>>> datetime.datetime.fromtimestamp(time.mktime(datetime.datetime.now().timetuple())).strftime("%Y-%m-%d %H:%M:%S") '2014-04-29 17:26:31' >>>> datetime.datetime.fromtimestamp(time.mktime(datetime.datetime.now().utctimetuple())).strftime("%Y-%m-%d %H:%M:%S") '2014-04-29 18:26:37' --- fail2ban/server/strptime.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fail2ban/server/strptime.py b/fail2ban/server/strptime.py index 5517e6b0..cf02dad5 100644 --- a/fail2ban/server/strptime.py +++ b/fail2ban/server/strptime.py @@ -190,5 +190,5 @@ def reGroupDictStrptime(found_dict): if gmtoff is not None: return calendar.timegm(date_result.utctimetuple()) else: - return time.mktime(date_result.utctimetuple()) + return time.mktime(date_result.timetuple()) From 8843423c8f66ff46d9636fd1594187fd5dfa3da1 Mon Sep 17 00:00:00 2001 From: Steven Hiscocks Date: Wed, 14 May 2014 23:01:14 +0100 Subject: [PATCH 061/141] TST: Fix tests due to @sebres fix and based from gh-349 reverts --- fail2ban/tests/datedetectortestcase.py | 2 +- fail2ban/tests/files/logs/dovecot | 8 ++++---- fail2ban/tests/files/logs/selinux-ssh | 16 ++++++++-------- fail2ban/tests/filtertestcase.py | 26 +++++++++++++------------- fail2ban/tests/samplestestcase.py | 2 +- 5 files changed, 27 insertions(+), 27 deletions(-) diff --git a/fail2ban/tests/datedetectortestcase.py b/fail2ban/tests/datedetectortestcase.py index 55f9a823..726e73f8 100644 --- a/fail2ban/tests/datedetectortestcase.py +++ b/fail2ban/tests/datedetectortestcase.py @@ -131,7 +131,7 @@ class DateDetectorTest(unittest.TestCase): # see https://github.com/fail2ban/fail2ban/pull/130 # yoh: unfortunately this test is not really effective to reproduce the # situation but left in place to assure consistent behavior - mu = time.mktime(datetime.datetime(2012, 10, 11, 2, 37, 17).utctimetuple()) + mu = time.mktime(datetime.datetime(2012, 10, 11, 2, 37, 17).timetuple()) logdate = self.__datedetector.getTime('2012/10/11 02:37:17 [error] 18434#0') self.assertNotEqual(logdate, None) ( logTime, logMatch ) = logdate diff --git a/fail2ban/tests/files/logs/dovecot b/fail2ban/tests/files/logs/dovecot index 5c3acb93..6ca31b7c 100644 --- a/fail2ban/tests/files/logs/dovecot +++ b/fail2ban/tests/files/logs/dovecot @@ -1,12 +1,12 @@ -# failJSON: { "time": "2010-09-16T06:51:00", "match": true , "host": "80.187.101.33" } +# failJSON: { "time": "2010-09-16T07:51:00", "match": true , "host": "80.187.101.33" } @400000004c91b044077a9e94 imap-login: Info: Aborted login (auth failed, 1 attempts): user=, method=CRAM-MD5, rip=80.187.101.33, lip=80.254.129.240, TLS -# failJSON: { "time": "2010-09-16T06:51:00", "match": true , "host": "176.61.140.224" } +# failJSON: { "time": "2010-09-16T07:51:00", "match": true , "host": "176.61.140.224" } @400000004c91b044077a9e94 dovecot-auth: pam_unix(dovecot:auth): authentication failure; logname= uid=0 euid=0 tty=dovecot ruser=web rhost=176.61.140.224 # Above example with injected rhost into ruser -- should not match for 1.2.3.4 -# failJSON: { "time": "2010-09-16T06:51:00", "match": true , "host": "192.0.43.10" } +# failJSON: { "time": "2010-09-16T07:51:00", "match": true , "host": "192.0.43.10" } @400000004c91b044077a9e94 dovecot-auth: pam_unix(dovecot:auth): authentication failure; logname= uid=0 euid=0 tty=dovecot ruser=rhost=1.2.3.4 rhost=192.0.43.10 -# failJSON: { "time": "2010-09-16T06:51:00", "match": true , "host": "176.61.140.225" } +# failJSON: { "time": "2010-09-16T07:51:00", "match": true , "host": "176.61.140.225" } @400000004c91b044077a9e94 dovecot-auth: pam_unix(dovecot:auth): authentication failure; logname= uid=0 euid=0 tty=dovecot ruser=root rhost=176.61.140.225 user=root # failJSON: { "time": "2004-12-12T11:19:11", "match": true , "host": "190.210.136.21" } diff --git a/fail2ban/tests/files/logs/selinux-ssh b/fail2ban/tests/files/logs/selinux-ssh index b6db443b..f9e1b828 100644 --- a/fail2ban/tests/files/logs/selinux-ssh +++ b/fail2ban/tests/files/logs/selinux-ssh @@ -1,25 +1,25 @@ -# failJSON: { "time": "2013-07-09T01:45:16", "match": false , "host": "173.242.116.187" } +# failJSON: { "time": "2013-07-09T02:45:16", "match": false , "host": "173.242.116.187" } type=USER_LOGIN msg=audit(1373330716.415:4063): user pid=11998 uid=0 auid=4294967295 ses=4294967295 subj=system_u:system_r:sshd_t:s0-s0:c0.c1023 msg='op=login acct="root" exe="/usr/sbin/sshd" hostname=? addr=173.242.116.187 terminal=ssh res=failed' -# failJSON: { "time": "2013-07-09T01:45:17", "match": false , "host": "173.242.116.187" } +# failJSON: { "time": "2013-07-09T02:45:17", "match": false , "host": "173.242.116.187" } type=USER_LOGIN msg=audit(1373330717.000:4068): user pid=12000 uid=0 auid=4294967295 ses=4294967295 subj=system_u:system_r:sshd_t:s0-s0:c0.c1023 msg='op=login acct=28756E6B6E6F776E207573657229 exe="/usr/sbin/sshd" hostname=? addr=173.242.116.187 terminal=ssh res=failed' -# failJSON: { "time": "2013-07-09T01:45:17", "match": true , "host": "173.242.116.187" } +# failJSON: { "time": "2013-07-09T02:45:17", "match": true , "host": "173.242.116.187" } type=USER_ERR msg=audit(1373330717.000:4070): user pid=12000 uid=0 auid=4294967295 ses=4294967295 subj=system_u:system_r:sshd_t:s0-s0:c0.c1023 msg='op=PAM:bad_ident acct="?" exe="/usr/sbin/sshd" hostname=173.242.116.187 addr=173.242.116.187 terminal=ssh res=failed' -# failJSON: { "time": "2013-07-09T01:45:17", "match": false , "host": "173.242.116.187" } +# failJSON: { "time": "2013-07-09T02:45:17", "match": false , "host": "173.242.116.187" } type=USER_LOGIN msg=audit(1373330717.000:4073): user pid=12000 uid=0 auid=4294967295 ses=4294967295 subj=system_u:system_r:sshd_t:s0-s0:c0.c1023 msg='op=login acct=28696E76616C6964207573657229 exe="/usr/sbin/sshd" hostname=? addr=173.242.116.187 terminal=ssh res=failed' -# failJSON: { "time": "2013-06-30T01:02:08", "match": false , "host": "113.240.248.18" } +# failJSON: { "time": "2013-06-30T02:02:08", "match": false , "host": "113.240.248.18" } type=USER_LOGIN msg=audit(1372546928.000:52008): user pid=21569 uid=0 auid=0 ses=76 subj=unconfined_u:system_r:sshd_t:s0-s0:c0.c1023 msg='op=login acct="sshd" exe="/usr/sbin/sshd" hostname=? addr=113.240.248.18 terminal=ssh res=failed' -# failJSON: { "time": "2013-06-30T02:58:20", "match": true , "host": "113.240.248.18" } +# failJSON: { "time": "2013-06-30T03:58:20", "match": true , "host": "113.240.248.18" } type=USER_ERR msg=audit(1372557500.000:61747): user pid=23684 uid=0 auid=0 ses=76 subj=unconfined_u:system_r:sshd_t:s0-s0:c0.c1023 msg='op=PAM:bad_ident acct="?" exe="/usr/sbin/sshd" hostname=113.240.248.18 addr=113.240.248.18 terminal=ssh res=failed' -# failJSON: { "time": "2013-06-30T03:58:20", "match": false , "host": "113.240.248.18" } +# failJSON: { "time": "2013-06-30T04:58:20", "match": false , "host": "113.240.248.18" } type=USER_LOGIN msg=audit(1372557500.000:61750): user pid=23684 uid=0 auid=0 ses=76 subj=unconfined_u:system_r:sshd_t:s0-s0:c0.c1023 msg='op=login acct=28696E76616C6964207573657229 exe="/usr/sbin/sshd" hostname=? addr=113.240.248.18 terminal=ssh res=failed' -# failJSON: { "time": "2013-07-06T17:48:00", "match": true , "host": "194.228.20.113" } +# failJSON: { "time": "2013-07-06T18:48:00", "match": true , "host": "194.228.20.113" } type=USER_AUTH msg=audit(1373129280.000:9): user pid=1277 uid=0 auid=4294967295 ses=4294967295 subj=system_u:system_r:sshd_t:s0-s0:c0.c1023 msg='op=pubkey acct="root" exe="/usr/sbin/sshd" hostname=? addr=194.228.20.113 terminal=ssh res=failed' # failJSON: { "time": "2013-10-30T07:57:43", "match": true , "host": "192.168.3.100" } diff --git a/fail2ban/tests/filtertestcase.py b/fail2ban/tests/filtertestcase.py index a0f715cd..c02e8616 100644 --- a/fail2ban/tests/filtertestcase.py +++ b/fail2ban/tests/filtertestcase.py @@ -794,7 +794,7 @@ class GetFailures(unittest.TestCase): FILENAME_MULTILINE = os.path.join(TEST_FILES_DIR, "testcase-multiline.log") # so that they could be reused by other tests - FAILURES_01 = ('193.168.0.128', 3, 1124017199.0, + FAILURES_01 = ('193.168.0.128', 3, 1124013599.0, [u'Aug 14 11:59:59 [sshd] error: PAM: Authentication failure for kevin from 193.168.0.128']*3) def setUp(self): @@ -844,7 +844,7 @@ class GetFailures(unittest.TestCase): def testGetFailures02(self): - output = ('141.3.81.106', 4, 1124017139.0, + output = ('141.3.81.106', 4, 1124013539.0, [u'Aug 14 11:%d:59 i60p295 sshd[12365]: Failed publickey for roehl from ::ffff:141.3.81.106 port 51332 ssh2' % m for m in 53, 54, 57, 58]) @@ -854,7 +854,7 @@ class GetFailures(unittest.TestCase): _assert_correct_last_attempt(self, self.filter, output) def testGetFailures03(self): - output = ('203.162.223.135', 7, 1124017144.0) + output = ('203.162.223.135', 7, 1124013544.0) self.filter.addLogPath(GetFailures.FILENAME_03) self.filter.addFailRegex("error,relay=,.*550 User unknown") @@ -862,7 +862,7 @@ class GetFailures(unittest.TestCase): _assert_correct_last_attempt(self, self.filter, output) def testGetFailures04(self): - output = [('212.41.96.186', 4, 1124017200.0), + output = [('212.41.96.186', 4, 1124013600.0), ('212.41.96.185', 4, 1124017198.0)] self.filter.addLogPath(GetFailures.FILENAME_04) @@ -877,11 +877,11 @@ class GetFailures(unittest.TestCase): def testGetFailuresUseDNS(self): # We should still catch failures with usedns = no ;-) - output_yes = ('93.184.216.119', 2, 1124017139.0, + output_yes = ('93.184.216.119', 2, 1124013539.0, [u'Aug 14 11:54:59 i60p295 sshd[12365]: Failed publickey for roehl from example.com port 51332 ssh2', u'Aug 14 11:58:59 i60p295 sshd[12365]: Failed publickey for roehl from ::ffff:93.184.216.119 port 51332 ssh2']) - output_no = ('93.184.216.119', 1, 1124017139.0, + output_no = ('93.184.216.119', 1, 1124013539.0, [u'Aug 14 11:58:59 i60p295 sshd[12365]: Failed publickey for roehl from ::ffff:93.184.216.119 port 51332 ssh2']) # Actually no exception would be raised -- it will be just set to 'no' @@ -904,7 +904,7 @@ class GetFailures(unittest.TestCase): def testGetFailuresMultiRegex(self): - output = ('141.3.81.106', 8, 1124017141.0) + output = ('141.3.81.106', 8, 1124013541.0) self.filter.addLogPath(GetFailures.FILENAME_02) self.filter.addFailRegex("Failed .* from ") @@ -923,8 +923,8 @@ class GetFailures(unittest.TestCase): self.assertRaises(FailManagerEmpty, self.filter.failManager.toBan) def testGetFailuresMultiLine(self): - output = [("192.0.43.10", 2, 1124017199.0), - ("192.0.43.11", 1, 1124017198.0)] + output = [("192.0.43.10", 2, 1124013599.0), + ("192.0.43.11", 1, 1124013598.0)] self.filter.addLogPath(GetFailures.FILENAME_MULTILINE) self.filter.addFailRegex("^.*rsyncd\[(?P\d+)\]: connect from .+ \(\)$^.+ rsyncd\[(?P=pid)\]: rsync error: .*$") self.filter.setMaxLines(100) @@ -942,7 +942,7 @@ class GetFailures(unittest.TestCase): self.assertEqual(sorted(foundList), sorted(output)) def testGetFailuresMultiLineIgnoreRegex(self): - output = [("192.0.43.10", 2, 1124017199.0)] + output = [("192.0.43.10", 2, 1124013599.0)] self.filter.addLogPath(GetFailures.FILENAME_MULTILINE) self.filter.addFailRegex("^.*rsyncd\[(?P\d+)\]: connect from .+ \(\)$^.+ rsyncd\[(?P=pid)\]: rsync error: .*$") self.filter.addIgnoreRegex("rsync error: Received SIGINT") @@ -956,9 +956,9 @@ class GetFailures(unittest.TestCase): self.assertRaises(FailManagerEmpty, self.filter.failManager.toBan) def testGetFailuresMultiLineMultiRegex(self): - output = [("192.0.43.10", 2, 1124017199.0), - ("192.0.43.11", 1, 1124017198.0), - ("192.0.43.15", 1, 1124017198.0)] + output = [("192.0.43.10", 2, 1124013599.0), + ("192.0.43.11", 1, 1124013598.0), + ("192.0.43.15", 1, 1124013598.0)] self.filter.addLogPath(GetFailures.FILENAME_MULTILINE) self.filter.addFailRegex("^.*rsyncd\[(?P\d+)\]: connect from .+ \(\)$^.+ rsyncd\[(?P=pid)\]: rsync error: .*$") self.filter.addFailRegex("^.* sendmail\[.*, msgid=<(?P[^>]+).*relay=\[\].*$^.+ spamd: result: Y \d+ .*,mid=<(?P=msgid)>(,bayes=[.\d]+)?(,autolearn=\S+)?\s*$") diff --git a/fail2ban/tests/samplestestcase.py b/fail2ban/tests/samplestestcase.py index 3529fcc2..132ade7b 100644 --- a/fail2ban/tests/samplestestcase.py +++ b/fail2ban/tests/samplestestcase.py @@ -129,7 +129,7 @@ def testSampleRegexsFactory(name): jsonTimeLocal = datetime.datetime.strptime(t, "%Y-%m-%dT%H:%M:%S.%f") - jsonTime = time.mktime(jsonTimeLocal.utctimetuple()) + jsonTime = time.mktime(jsonTimeLocal.timetuple()) jsonTime += jsonTimeLocal.microsecond / 1000000 From 1c20fd88d4d4131c6a4ed8a0abe89534097b823f Mon Sep 17 00:00:00 2001 From: Steven Hiscocks Date: Wed, 14 May 2014 23:04:48 +0100 Subject: [PATCH 062/141] DOC: Update docs in reference to time zone related fix --- ChangeLog | 1 + THANKS | 1 + 2 files changed, 2 insertions(+) diff --git a/ChangeLog b/ChangeLog index 515c79d2..69cbe909 100644 --- a/ChangeLog +++ b/ChangeLog @@ -27,6 +27,7 @@ ver. 0.9.1 (2014/xx/xx) - better, faster, stronger * Database now returns persistent bans on restart (bantime < 0) * Recursive action tags now fully processed. Fixes issue with bsd-ipfw action + * Correct times for non-timezone date times formats - Thanks sebres - New features: - Added monit filter thanks Jason H Martin. diff --git a/THANKS b/THANKS index 27165492..2c5b65bf 100644 --- a/THANKS +++ b/THANKS @@ -86,6 +86,7 @@ Rolf Fokkens Roman Gelfand Russell Odom Sebastian Arcus +sebres Sireyessire silviogarbes Stefan Tatschner From 213c4315c3a7aa86c2577853b99f74a513f9b02a Mon Sep 17 00:00:00 2001 From: sebres Date: Thu, 15 May 2014 19:41:00 +0200 Subject: [PATCH 063/141] fix a TypeError bugs like "Failed to execute ban jail 'pam-generic' action 'iptables-allports'" getAttempt returns not a list (numeric), so by call of both lambda we have a TypeError except; simplifying code; --- fail2ban/server/actions.py | 23 ++++++++++------------- 1 file changed, 10 insertions(+), 13 deletions(-) diff --git a/fail2ban/server/actions.py b/fail2ban/server/actions.py index d5799ca1..dd68ac13 100644 --- a/fail2ban/server/actions.py +++ b/fail2ban/server/actions.py @@ -255,23 +255,20 @@ class Actions(JailThread, Mapping): if ticket != False: aInfo = CallingMap() bTicket = BanManager.createBanTicket(ticket) - aInfo["ip"] = bTicket.getIP() + ip = bTicket.getIP() + aInfo["ip"] = ip aInfo["failures"] = bTicket.getAttempt() aInfo["time"] = bTicket.getTime() aInfo["matches"] = "\n".join(bTicket.getMatches()) if self._jail.database is not None: - aInfo["ipmatches"] = lambda: "\n".join( - self._jail.database.getBansMerged( - ip=bTicket.getIP()).getMatches()) - aInfo["ipjailmatches"] = lambda: "\n".join( - self._jail.database.getBansMerged( - ip=bTicket.getIP(), jail=self._jail).getMatches()) - aInfo["ipfailures"] = lambda: "\n".join( - self._jail.database.getBansMerged( - ip=bTicket.getIP()).getAttempt()) - aInfo["ipjailfailures"] = lambda: "\n".join( - self._jail.database.getBansMerged( - ip=bTicket.getIP(), jail=self._jail).getAttempt()) + aInfo["ipmatches"] = lambda jail=self._jail: "\n".join( + jail.database.getBansMerged(ip=ip).getMatches()) + aInfo["ipjailmatches"] = lambda jail=self._jail: "\n".join( + jail.database.getBansMerged(ip=ip, jail=jail).getMatches()) + aInfo["ipfailures"] = lambda jail=self._jail: \ + jail.database.getBansMerged(ip=ip).getAttempt() + aInfo["ipjailfailures"] = lambda jail=self._jail: \ + jail.database.getBansMerged(ip=ip, jail=jail).getAttempt() if self.__banManager.addBanTicket(bTicket): logSys.notice("[%s] Ban %s" % (self._jail.name, aInfo["ip"])) for name, action in self._actions.iteritems(): From fc4b69a2822b83527e26a9f139ee9403e7ae23bb Mon Sep 17 00:00:00 2001 From: Steven Hiscocks Date: Thu, 15 May 2014 22:15:12 +0100 Subject: [PATCH 064/141] DOC: Update ChangeLog fix for ip{,jail}failures action tags --- ChangeLog | 2 ++ 1 file changed, 2 insertions(+) diff --git a/ChangeLog b/ChangeLog index 515c79d2..d4ea774a 100644 --- a/ChangeLog +++ b/ChangeLog @@ -27,6 +27,8 @@ ver. 0.9.1 (2014/xx/xx) - better, faster, stronger * Database now returns persistent bans on restart (bantime < 0) * Recursive action tags now fully processed. Fixes issue with bsd-ipfw action + * Fixed TypeError with "ipfailures" and "ipjailfailures" action tags. + Thanks Serg G. Brester - New features: - Added monit filter thanks Jason H Martin. From 1fa8f9fa709964bc9e26391255dbf48388b831bf Mon Sep 17 00:00:00 2001 From: Steven Hiscocks Date: Thu, 15 May 2014 22:18:07 +0100 Subject: [PATCH 065/141] DOC: Tweak ChangeLog and THANKS --- ChangeLog | 3 ++- THANKS | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/ChangeLog b/ChangeLog index 69cbe909..0505e468 100644 --- a/ChangeLog +++ b/ChangeLog @@ -27,7 +27,8 @@ ver. 0.9.1 (2014/xx/xx) - better, faster, stronger * Database now returns persistent bans on restart (bantime < 0) * Recursive action tags now fully processed. Fixes issue with bsd-ipfw action - * Correct times for non-timezone date times formats - Thanks sebres + * Correct times for non-timezone date times formats during DST. + Thanks Serg G. Brester - New features: - Added monit filter thanks Jason H Martin. diff --git a/THANKS b/THANKS index 2c5b65bf..0151122a 100644 --- a/THANKS +++ b/THANKS @@ -86,7 +86,7 @@ Rolf Fokkens Roman Gelfand Russell Odom Sebastian Arcus -sebres +Serg G. Brester Sireyessire silviogarbes Stefan Tatschner From 0ca97431a0dbebd16d5a8caa9521c7e6362f489d Mon Sep 17 00:00:00 2001 From: Steven Hiscocks Date: Thu, 15 May 2014 22:48:03 +0100 Subject: [PATCH 066/141] ENH: Clearer warning with lines which failed to decode correctly --- fail2ban/server/filter.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/fail2ban/server/filter.py b/fail2ban/server/filter.py index fb5aeb3d..73233905 100644 --- a/fail2ban/server/filter.py +++ b/fail2ban/server/filter.py @@ -790,8 +790,10 @@ class FileContainer: try: line = line.decode(self.getEncoding(), 'strict') except UnicodeDecodeError: - logSys.warning("Error decoding line from '%s' with '%s': %s" % - (self.getFileName(), self.getEncoding(), `line`)) + logSys.warning( + "Error decoding line from '%s' with '%s'. Continuing " + " to process line ignoring invalid characters: %r" % + (self.getFileName(), self.getEncoding(), line)) if sys.version_info >= (3,): # In python3, must be decoded line = line.decode(self.getEncoding(), 'ignore') return line From eb2487986c3a9699d9c67bdd474ee2e57828d360 Mon Sep 17 00:00:00 2001 From: Yaroslav Halchenko Date: Thu, 15 May 2014 21:17:43 -0400 Subject: [PATCH 067/141] ENH: minor -- print time which was used to process lines --- bin/fail2ban-regex | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/bin/fail2ban-regex b/bin/fail2ban-regex index 3a887867..d8b3d287 100755 --- a/bin/fail2ban-regex +++ b/bin/fail2ban-regex @@ -25,11 +25,11 @@ This tools can test regular expressions for "fail2ban". """ -__author__ = "Cyril Jaquier, Yaroslav Halchenko" -__copyright__ = "Copyright (c) 2004-2008 Cyril Jaquier, 2012-2013 Yaroslav Halchenko" +__author__ = "Fail2Ban Developers" +__copyright__ = "Copyright (c) 2004-2008 Cyril Jaquier, 2012-2014 Yaroslav Halchenko" __license__ = "GPL" -import getopt, sys, time, logging, os, locale, shlex, urllib +import getopt, sys, time, logging, os, locale, shlex, time, urllib from optparse import OptionParser, Option from ConfigParser import NoOptionError, NoSectionError, MissingSectionHeaderError @@ -223,6 +223,7 @@ class Fail2banRegex(object): self._filter = Filter(None) self._ignoreregex = list() self._failregex = list() + self._time_elapsed = None self._line_stats = LineStats() if opts.maxlines: @@ -348,7 +349,7 @@ class Fail2banRegex(object): return line, ret def process(self, test_lines): - + t0 = time.time() for line_no, line in enumerate(test_lines): if isinstance(line, tuple): line_datetimestripped, ret = fail2banRegex.testRegex( @@ -383,6 +384,7 @@ class Fail2banRegex(object): if line_no % 10 == 0 and self._filter.dateDetector is not None: self._filter.dateDetector.sortTemplate() + self._time_elapsed = time.time() - t0 @@ -456,7 +458,10 @@ class Fail2banRegex(object): template.hits, template.name)) pprint_list(out, "[# of hits] date format") - print "\nLines: %s" % self._line_stats + print "\nLines: %s" % self._line_stats, + if self._time_elapsed is not None: + print "[processed in %.2f sec]" % self._time_elapsed, + print if self._print_all_matched: self.printLines('matched') From 46d6e93800ddeb5ad34d2fb62e0aa1f7e3e7d9b0 Mon Sep 17 00:00:00 2001 From: pmarrapese Date: Sun, 18 May 2014 22:12:54 -0700 Subject: [PATCH 068/141] adjusted sshd filter regex to catch more verbose lines --- config/filter.d/sshd.conf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config/filter.d/sshd.conf b/config/filter.d/sshd.conf index 195744f2..f634d27e 100644 --- a/config/filter.d/sshd.conf +++ b/config/filter.d/sshd.conf @@ -32,7 +32,7 @@ failregex = ^%(__prefix_line)s(?:error: PAM: )?[aA]uthentication (?:failure|erro ^%(__prefix_line)sUser .+ from not allowed because none of user's groups are listed in AllowGroups\s*$ ^(?P<__prefix>%(__prefix_line)s)User .+ not allowed because account is locked(?P=__prefix)(?:error: )?Received disconnect from : 11: .+ \[preauth\]$ ^(?P<__prefix>%(__prefix_line)s)Disconnecting: Too many authentication failures for .+? \[preauth\](?P=__prefix)(?:error: )?Connection closed by \[preauth\]$ - ^(?P<__prefix>%(__prefix_line)s)Connection from port \d+(?P=__prefix)Disconnecting: Too many authentication failures for .+? \[preauth\]$ + ^(?P<__prefix>%(__prefix_line)s)Connection from port \d+.*(?P=__prefix)Disconnecting: Too many authentication failures for .+? \[preauth\]$ ignoreregex = From 96918acee4d96a7d83992e6ce993faa9d900a97c Mon Sep 17 00:00:00 2001 From: pmarrapese Date: Mon, 19 May 2014 20:47:16 -0700 Subject: [PATCH 069/141] more explicit match for sshd filter & added test --- THANKS | 1 + config/filter.d/sshd.conf | 2 +- fail2ban/tests/files/logs/sshd | 5 +++++ 3 files changed, 7 insertions(+), 1 deletion(-) diff --git a/THANKS b/THANKS index 27165492..1741df0f 100644 --- a/THANKS +++ b/THANKS @@ -76,6 +76,7 @@ Michael Hanselmann Mika (mkl) Nick Munger onorua +Paul Marrapese Noel Butler Patrick Börjesson Raphaël Marichez diff --git a/config/filter.d/sshd.conf b/config/filter.d/sshd.conf index f634d27e..6589e21b 100644 --- a/config/filter.d/sshd.conf +++ b/config/filter.d/sshd.conf @@ -32,7 +32,7 @@ failregex = ^%(__prefix_line)s(?:error: PAM: )?[aA]uthentication (?:failure|erro ^%(__prefix_line)sUser .+ from not allowed because none of user's groups are listed in AllowGroups\s*$ ^(?P<__prefix>%(__prefix_line)s)User .+ not allowed because account is locked(?P=__prefix)(?:error: )?Received disconnect from : 11: .+ \[preauth\]$ ^(?P<__prefix>%(__prefix_line)s)Disconnecting: Too many authentication failures for .+? \[preauth\](?P=__prefix)(?:error: )?Connection closed by \[preauth\]$ - ^(?P<__prefix>%(__prefix_line)s)Connection from port \d+.*(?P=__prefix)Disconnecting: Too many authentication failures for .+? \[preauth\]$ + ^(?P<__prefix>%(__prefix_line)s)Connection from port \d+(?: on \S+ port \d+)?(?P=__prefix)Disconnecting: Too many authentication failures for .+? \[preauth\]$ ignoreregex = diff --git a/fail2ban/tests/files/logs/sshd b/fail2ban/tests/files/logs/sshd index b9d1b9b4..a6e54196 100644 --- a/fail2ban/tests/files/logs/sshd +++ b/fail2ban/tests/files/logs/sshd @@ -137,6 +137,11 @@ Feb 12 04:09:18 localhost sshd[26713]: Connection from 115.249.163.77 port 51353 # failJSON: { "time": "2005-02-12T04:09:21", "match": true , "host": "115.249.163.77", "desc": "from gh-457" } Feb 12 04:09:21 localhost sshd[26713]: Disconnecting: Too many authentication failures for root [preauth] +# failJSON: { "match": false } +Feb 12 04:09:18 localhost sshd[26713]: Connection from 115.249.163.77 port 51353 on 127.0.0.1 port 22 +# failJSON: { "time": "2005-02-12T04:09:21", "match": true , "host": "115.249.163.77", "desc": "Multiline match with interface address" } +Feb 12 04:09:21 localhost sshd[26713]: Disconnecting: Too many authentication failures for root [preauth] + # failJSON: { "match": false } Apr 27 13:02:04 host sshd[29116]: User root not allowed because account is locked # failJSON: { "match": false } From 9b7c35810ab1dc8f3e5afb29c1e742bc0b3e5527 Mon Sep 17 00:00:00 2001 From: JoelSnyder Date: Mon, 2 Jun 2014 22:55:59 -0700 Subject: [PATCH 070/141] Create oracleims.conf in filter.d for new filter Created oracleims.conf to catch messages from Sun/Oracle Communications Messaging Server v6.3 and above (including v7) --- config/filter.d/oracleims.conf | 59 ++++++++++++++++++++++++++++++++++ 1 file changed, 59 insertions(+) create mode 100644 config/filter.d/oracleims.conf diff --git a/config/filter.d/oracleims.conf b/config/filter.d/oracleims.conf new file mode 100644 index 00000000..e80d0b96 --- /dev/null +++ b/config/filter.d/oracleims.conf @@ -0,0 +1,59 @@ +# Fail2Ban configuration file +# for Oracle IMS with XML logging +# +# Author: Joel Snyder/jms@opus1.com/2014-June-01 +# +# + + +[INCLUDES] + +# Read common prefixes. +# If any customizations available -- read them from +# common.local +before = common.conf + + +[Definition] + +# Option: failregex +# Notes.: regex to match the password failures messages +# in the logfile. The host must be matched by a +# group named "host". The tag "" can +# be used for standard IP/hostname matching and is +# only an alias for +# (?:::f{4,6}:)?(?P[\w\-.^_]+) +# Values: TEXT +# +# +# CONFIGURATION REQUIREMENTS FOR ORACLE IMS v6 and ABOVE: +# +# In OPTION.DAT you must have LOG_FORMAT=4 and +# bit 5 of LOG_CONNECTION must be set. +# +# Many of these sub-fields are optional and can be turned on and off +# by the system manager. We need the "tr" field +# (transport information (present if bit 5 of LOG_CONNECTION is +# set and transport information is available)). +# "di" should be there by default if you have LOG_FORMAT=4. +# Do not use "mi" as this is not included by default. +# +# Typical line IF YOU ARE USING TAGGING ! ! ! is: +# +# +# All that would be on one line. +# Note that you MUST have LOG_FORMAT=4 for this to work! +# + +failregex = ^.*tr=".*\|.*\|\d+\|\|\d+" .+ Bad username or password.*"/>$ + +# Option: ignoreregex +# Notes.: regex to ignore. If this regex matches, the line is ignored. +# Values: TEXT +# +ignoreregex = From 54317d7c3b1c35de63f909cf16bf2de531030533 Mon Sep 17 00:00:00 2001 From: JoelSnyder Date: Mon, 2 Jun 2014 22:58:39 -0700 Subject: [PATCH 071/141] Create test for oracleims filter This test file shows configuration information for the application, three log lines that DO match the pattern, and one log line that does NOT match the pattern (the first one). --- fail2ban/tests/files/logs/oracleims | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) create mode 100644 fail2ban/tests/files/logs/oracleims diff --git a/fail2ban/tests/files/logs/oracleims b/fail2ban/tests/files/logs/oracleims new file mode 100644 index 00000000..aafb27af --- /dev/null +++ b/fail2ban/tests/files/logs/oracleims @@ -0,0 +1,19 @@ +# CONFIGURATION REQUIREMENTS FOR ORACLE IMS v6.3 and ABOVE: +# +# In OPTION.DAT you must have LOG_FORMAT=4 and +# bit 5 of LOG_CONNECTION must be set. +# +# Many of these sub-fields are optional and can be turned on and off +# by the system manager. We need the "tr" field +# (transport information (present if bit 5 of LOG_CONNECTION is +# set and transport information is available)). +# "di" should be there by default if you have LOG_FORMAT=4. +# +# failJSON: { "time": "2014-06-02T22:02:13", "match": false , "host": "23.122.129.179" } + +# failJSON: { "time": "2014-06-02T16:06:33", "match": true , "host": "89.96.245.78" } + +# failJSON: { "time": "2014-06-02T10:08:07", "match": true , "host": "71.95.206.106" } + +# failJSON: { "time": "2014-06-02T09:54:58", "match": true , "host": "151.1.71.144" } + From db023be09bae14b3ae23b476791bed79d6e71e36 Mon Sep 17 00:00:00 2001 From: Steven Hiscocks Date: Sat, 7 Jun 2014 20:51:53 +0100 Subject: [PATCH 072/141] BF: Fix bad syntax in badips.py action Taken from https://bugzilla.redhat.com/attachment.cgi?id=895966&action=diff --- config/action.d/badips.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/config/action.d/badips.py b/config/action.d/badips.py index a3f0883c..250b1dc3 100644 --- a/config/action.d/badips.py +++ b/config/action.d/badips.py @@ -161,7 +161,7 @@ class BadIPsAction(ActionBase): "/".join([self._badips, "get", "list", category, str(score)]), urlencode({'age': age})]) if key: - url = "&".join([url, urlencode({"key", key})]) + url = "&".join([url, urlencode({'key': key})]) response = urlopen(self._Request(url)) except HTTPError as response: messages = json.loads(response.read().decode('utf-8')) @@ -346,7 +346,7 @@ class BadIPsAction(ActionBase): try: url = "/".join([self._badips, "add", self.category, aInfo['ip']]) if self.key: - url = "?".join([url, urlencode({"key", self.key})]) + url = "?".join([url, urlencode({'key': self.key})]) response = urlopen(self._Request(url)) except HTTPError as response: messages = json.loads(response.read().decode('utf-8')) From e8131475cd34b95a34b84c2b949b808f48d90a5f Mon Sep 17 00:00:00 2001 From: Steven Hiscocks Date: Mon, 9 Jun 2014 22:17:00 +0100 Subject: [PATCH 073/141] ENH: Realign and harmonise log messages with getF2BLogger helper --- ChangeLog | 2 ++ bin/fail2ban-client | 2 +- config/filter.d/recidive.conf | 2 +- fail2ban/client/actionreader.py | 5 +++-- fail2ban/client/beautifier.py | 5 ++--- fail2ban/client/configparserinc.py | 5 +++-- fail2ban/client/configreader.py | 5 +++-- fail2ban/client/configurator.py | 5 ++--- fail2ban/client/fail2banreader.py | 5 ++--- fail2ban/client/filterreader.py | 5 +++-- fail2ban/client/jailreader.py | 5 +++-- fail2ban/client/jailsreader.py | 5 ++--- fail2ban/helpers.py | 5 +++++ fail2ban/server/action.py | 7 ++++--- fail2ban/server/actions.py | 3 ++- fail2ban/server/asyncserver.py | 8 ++++---- fail2ban/server/banmanager.py | 4 ++-- fail2ban/server/database.py | 4 ++-- fail2ban/server/datedetector.py | 4 ++-- fail2ban/server/datetemplate.py | 4 ++-- fail2ban/server/faildata.py | 4 ++-- fail2ban/server/failmanager.py | 3 ++- fail2ban/server/filter.py | 5 +++-- fail2ban/server/filtergamin.py | 5 +++-- fail2ban/server/filterpoll.py | 5 +++-- fail2ban/server/filterpyinotify.py | 3 ++- fail2ban/server/filtersystemd.py | 6 +++--- fail2ban/server/jail.py | 3 ++- fail2ban/server/server.py | 13 +++++++------ fail2ban/server/ticket.py | 4 ++-- fail2ban/server/transmitter.py | 6 ++++-- fail2ban/tests/files/logs/recidive | 12 ++++++------ fail2ban/tests/servertestcase.py | 8 +++++++- fail2ban/tests/utils.py | 4 +++- 34 files changed, 99 insertions(+), 72 deletions(-) diff --git a/ChangeLog b/ChangeLog index d4ea774a..ff70f07c 100644 --- a/ChangeLog +++ b/ChangeLog @@ -38,6 +38,8 @@ ver. 0.9.1 (2014/xx/xx) - better, faster, stronger * Fail2ban-regex - add print-all-matched option. Closes gh-652 * Suppress fail2ban-client warnings for non-critical config options * Match non "Bye Bye" disconnect messages for sshd locked account regex + * Realign fail2ban log output with white space to improve readability. Does + not affect SYSLOG output. ver. 0.9.0 (2014/03/14) - beta ---------- diff --git a/bin/fail2ban-client b/bin/fail2ban-client index 289d7b39..b9713153 100755 --- a/bin/fail2ban-client +++ b/bin/fail2ban-client @@ -32,7 +32,7 @@ from fail2ban.client.configurator import Configurator from fail2ban.client.beautifier import Beautifier # Gets the instance of the logger. -logSys = logging.getLogger("fail2ban.client") +logSys = logging.getLogger("fail2ban") ## # diff --git a/config/filter.d/recidive.conf b/config/filter.d/recidive.conf index aa1b0013..b38735ad 100644 --- a/config/filter.d/recidive.conf +++ b/config/filter.d/recidive.conf @@ -21,7 +21,7 @@ before = common.conf [Definition] -_daemon = fail2ban\.server\.actions +_daemon = fail2ban\.actions\s* # The name of the jail that this filter is used for. In jail.conf, name the # jail using this filter 'recidive', or change this line! diff --git a/fail2ban/client/actionreader.py b/fail2ban/client/actionreader.py index 6b0334ac..07572930 100644 --- a/fail2ban/client/actionreader.py +++ b/fail2ban/client/actionreader.py @@ -24,12 +24,13 @@ __author__ = "Cyril Jaquier" __copyright__ = "Copyright (c) 2004 Cyril Jaquier" __license__ = "GPL" -import logging, os +import os from .configreader import ConfigReader, DefinitionInitConfigReader +from ..helpers import getF2BLogger # Gets the instance of the logger. -logSys = logging.getLogger(__name__) +logSys = getF2BLogger(__name__) class ActionReader(DefinitionInitConfigReader): diff --git a/fail2ban/client/beautifier.py b/fail2ban/client/beautifier.py index 25d73cb2..070c80c5 100644 --- a/fail2ban/client/beautifier.py +++ b/fail2ban/client/beautifier.py @@ -21,12 +21,11 @@ __author__ = "Cyril Jaquier, Yaroslav Halchenko" __copyright__ = "Copyright (c) 2004 Cyril Jaquier, 2013- Yaroslav Halchenko" __license__ = "GPL" -import logging - from ..exceptions import UnknownJailException, DuplicateJailException +from ..helpers import getF2BLogger # Gets the instance of the logger. -logSys = logging.getLogger(__name__) +logSys = getF2BLogger(__name__) ## # Beautify the output of the client. diff --git a/fail2ban/client/configparserinc.py b/fail2ban/client/configparserinc.py index 7519e599..1be786c7 100644 --- a/fail2ban/client/configparserinc.py +++ b/fail2ban/client/configparserinc.py @@ -24,7 +24,8 @@ __author__ = 'Yaroslav Halhenko' __copyright__ = 'Copyright (c) 2007 Yaroslav Halchenko' __license__ = 'GPL' -import logging, os, sys +import os, sys +from ..helpers import getF2BLogger if sys.version_info >= (3,2): # pragma: no cover @@ -60,7 +61,7 @@ else: # pragma: no cover from ConfigParser import SafeConfigParser # Gets the instance of the logger. -logSys = logging.getLogger(__name__) +logSys = getF2BLogger(__name__) __all__ = ['SafeConfigParserWithIncludes'] diff --git a/fail2ban/client/configreader.py b/fail2ban/client/configreader.py index 51ca6948..e905b278 100644 --- a/fail2ban/client/configreader.py +++ b/fail2ban/client/configreader.py @@ -24,13 +24,14 @@ __author__ = "Cyril Jaquier" __copyright__ = "Copyright (c) 2004 Cyril Jaquier" __license__ = "GPL" -import glob, logging, os +import glob, os from ConfigParser import NoOptionError, NoSectionError from .configparserinc import SafeConfigParserWithIncludes +from ..helpers import getF2BLogger # Gets the instance of the logger. -logSys = logging.getLogger(__name__) +logSys = getF2BLogger(__name__) class ConfigReader(SafeConfigParserWithIncludes): diff --git a/fail2ban/client/configurator.py b/fail2ban/client/configurator.py index df523280..23c21c4b 100644 --- a/fail2ban/client/configurator.py +++ b/fail2ban/client/configurator.py @@ -24,13 +24,12 @@ __author__ = "Cyril Jaquier" __copyright__ = "Copyright (c) 2004 Cyril Jaquier" __license__ = "GPL" -import logging - from .fail2banreader import Fail2banReader from .jailsreader import JailsReader +from ..helpers import getF2BLogger # Gets the instance of the logger. -logSys = logging.getLogger(__name__) +logSys = getF2BLogger(__name__) class Configurator: diff --git a/fail2ban/client/fail2banreader.py b/fail2ban/client/fail2banreader.py index 251c6985..4d106f3a 100644 --- a/fail2ban/client/fail2banreader.py +++ b/fail2ban/client/fail2banreader.py @@ -24,12 +24,11 @@ __author__ = "Cyril Jaquier" __copyright__ = "Copyright (c) 2004 Cyril Jaquier" __license__ = "GPL" -import logging - from .configreader import ConfigReader +from ..helpers import getF2BLogger # Gets the instance of the logger. -logSys = logging.getLogger(__name__) +logSys = getF2BLogger(__name__) class Fail2banReader(ConfigReader): diff --git a/fail2ban/client/filterreader.py b/fail2ban/client/filterreader.py index 5a11dc41..2f5b3b05 100644 --- a/fail2ban/client/filterreader.py +++ b/fail2ban/client/filterreader.py @@ -24,13 +24,14 @@ __author__ = "Cyril Jaquier" __copyright__ = "Copyright (c) 2004 Cyril Jaquier" __license__ = "GPL" -import logging, os, shlex +import os, shlex from .configreader import ConfigReader, DefinitionInitConfigReader from ..server.action import CommandAction +from ..helpers import getF2BLogger # Gets the instance of the logger. -logSys = logging.getLogger(__name__) +logSys = getF2BLogger(__name__) class FilterReader(DefinitionInitConfigReader): diff --git a/fail2ban/client/jailreader.py b/fail2ban/client/jailreader.py index 5735b021..6207d8f4 100644 --- a/fail2ban/client/jailreader.py +++ b/fail2ban/client/jailreader.py @@ -24,15 +24,16 @@ __author__ = "Cyril Jaquier" __copyright__ = "Copyright (c) 2004 Cyril Jaquier" __license__ = "GPL" -import logging, re, glob, os.path +import re, glob, os.path import json from .configreader import ConfigReader from .filterreader import FilterReader from .actionreader import ActionReader +from ..helpers import getF2BLogger # Gets the instance of the logger. -logSys = logging.getLogger(__name__) +logSys = getF2BLogger(__name__) class JailReader(ConfigReader): diff --git a/fail2ban/client/jailsreader.py b/fail2ban/client/jailsreader.py index 047749b3..86fd501f 100644 --- a/fail2ban/client/jailsreader.py +++ b/fail2ban/client/jailsreader.py @@ -24,13 +24,12 @@ __author__ = "Cyril Jaquier" __copyright__ = "Copyright (c) 2004 Cyril Jaquier" __license__ = "GPL" -import logging - from .configreader import ConfigReader from .jailreader import JailReader +from ..helpers import getF2BLogger # Gets the instance of the logger. -logSys = logging.getLogger(__name__) +logSys = getF2BLogger(__name__) class JailsReader(ConfigReader): diff --git a/fail2ban/helpers.py b/fail2ban/helpers.py index 2579381d..c3e50336 100644 --- a/fail2ban/helpers.py +++ b/fail2ban/helpers.py @@ -107,3 +107,8 @@ class FormatterWithTraceBack(logging.Formatter): def format(self, record): record.tbc = record.tb = self._tb() return logging.Formatter.format(self, record) + +def getF2BLogger(name): + """Get logging.Logger instance with Fail2Ban logger name convention + """ + return logging.getLogger("fail2ban.%s" % name.rpartition(".")[-1]) diff --git a/fail2ban/server/action.py b/fail2ban/server/action.py index fefe2c2c..ea7bc91c 100644 --- a/fail2ban/server/action.py +++ b/fail2ban/server/action.py @@ -25,10 +25,11 @@ import logging, os, subprocess, time, signal, tempfile import threading, re from abc import ABCMeta from collections import MutableMapping -#from subprocess import call + +from ..helpers import getF2BLogger # Gets the instance of the logger. -logSys = logging.getLogger(__name__) +logSys = getF2BLogger(__name__) # Create a lock for running system commands _cmd_lock = threading.Lock() @@ -136,7 +137,7 @@ class ActionBase(object): self._jail = jail self._name = name self._logSys = logging.getLogger( - '%s.%s' % (__name__, self.__class__.__name__)) + "fail2ban.%s" % self.__class__.__name__) def start(self): """Executed when the jail/action is started. diff --git a/fail2ban/server/actions.py b/fail2ban/server/actions.py index dd68ac13..467df510 100644 --- a/fail2ban/server/actions.py +++ b/fail2ban/server/actions.py @@ -41,9 +41,10 @@ from .banmanager import BanManager from .jailthread import JailThread from .action import ActionBase, CommandAction, CallingMap from .mytime import MyTime +from ..helpers import getF2BLogger # Gets the instance of the logger. -logSys = logging.getLogger(__name__) +logSys = getF2BLogger(__name__) class Actions(JailThread, Mapping): """Handles jail actions. diff --git a/fail2ban/server/asyncserver.py b/fail2ban/server/asyncserver.py index 56704510..326e35cb 100644 --- a/fail2ban/server/asyncserver.py +++ b/fail2ban/server/asyncserver.py @@ -25,12 +25,12 @@ __copyright__ = "Copyright (c) 2004 Cyril Jaquier" __license__ = "GPL" from pickle import dumps, loads, HIGHEST_PROTOCOL -import asyncore, asynchat, socket, os, logging, sys, traceback, fcntl +import asyncore, asynchat, socket, os, sys, traceback, fcntl -from .. import helpers +from ..helpers import getF2BLogger,formatExceptionInfo # Gets the instance of the logger. -logSys = logging.getLogger(__name__) +logSys = getF2BLogger(__name__) if sys.version_info >= (3,): # b"" causes SyntaxError in python <= 2.5, so below implements equivalent @@ -81,7 +81,7 @@ class RequestHandler(asynchat.async_chat): self.close_when_done() def handle_error(self): - e1, e2 = helpers.formatExceptionInfo() + e1, e2 = formatExceptionInfo() logSys.error("Unexpected communication error: %s" % str(e2)) logSys.error(traceback.format_exc().splitlines()) self.close() diff --git a/fail2ban/server/banmanager.py b/fail2ban/server/banmanager.py index fc9eb948..68954b9c 100644 --- a/fail2ban/server/banmanager.py +++ b/fail2ban/server/banmanager.py @@ -24,14 +24,14 @@ __author__ = "Cyril Jaquier" __copyright__ = "Copyright (c) 2004 Cyril Jaquier" __license__ = "GPL" -import logging from threading import Lock from .ticket import BanTicket from .mytime import MyTime +from ..helpers import getF2BLogger # Gets the instance of the logger. -logSys = logging.getLogger(__name__) +logSys = getF2BLogger(__name__) ## # Banning Manager. diff --git a/fail2ban/server/database.py b/fail2ban/server/database.py index 54cca4d3..24a3a327 100644 --- a/fail2ban/server/database.py +++ b/fail2ban/server/database.py @@ -21,7 +21,6 @@ __author__ = "Steven Hiscocks" __copyright__ = "Copyright (c) 2013 Steven Hiscocks" __license__ = "GPL" -import logging import sys import shutil, time import sqlite3 @@ -32,9 +31,10 @@ from threading import Lock from .mytime import MyTime from .ticket import FailTicket +from ..helpers import getF2BLogger # Gets the instance of the logger. -logSys = logging.getLogger(__name__) +logSys = getF2BLogger(__name__) if sys.version_info >= (3,): sqlite3.register_adapter( diff --git a/fail2ban/server/datedetector.py b/fail2ban/server/datedetector.py index c5effea0..8d3ccaf2 100644 --- a/fail2ban/server/datedetector.py +++ b/fail2ban/server/datedetector.py @@ -21,13 +21,13 @@ __author__ = "Cyril Jaquier and Fail2Ban Contributors" __copyright__ = "Copyright (c) 2004 Cyril Jaquier" __license__ = "GPL" -import logging from threading import Lock from .datetemplate import DatePatternRegex, DateTai64n, DateEpoch +from ..helpers import getF2BLogger # Gets the instance of the logger. -logSys = logging.getLogger(__name__) +logSys = getF2BLogger(__name__) class DateDetector(object): """Manages one or more date templates to find a date within a log line. diff --git a/fail2ban/server/datetemplate.py b/fail2ban/server/datetemplate.py index 19bdd0ef..0e6141e5 100644 --- a/fail2ban/server/datetemplate.py +++ b/fail2ban/server/datetemplate.py @@ -25,12 +25,12 @@ __copyright__ = "Copyright (c) 2004 Cyril Jaquier" __license__ = "GPL" import re -import logging from abc import abstractmethod from .strptime import reGroupDictStrptime, timeRE +from ..helpers import getF2BLogger -logSys = logging.getLogger(__name__) +logSys = getF2BLogger(__name__) class DateTemplate(object): diff --git a/fail2ban/server/faildata.py b/fail2ban/server/faildata.py index 232a492d..328bdb07 100644 --- a/fail2ban/server/faildata.py +++ b/fail2ban/server/faildata.py @@ -24,10 +24,10 @@ __author__ = "Cyril Jaquier" __copyright__ = "Copyright (c) 2004 Cyril Jaquier" __license__ = "GPL" -import logging +from ..helpers import getF2BLogger # Gets the instance of the logger. -logSys = logging.getLogger(__name__) +logSys = getF2BLogger(__name__) class FailData: diff --git a/fail2ban/server/failmanager.py b/fail2ban/server/failmanager.py index 548e6adb..75b505c9 100644 --- a/fail2ban/server/failmanager.py +++ b/fail2ban/server/failmanager.py @@ -29,9 +29,10 @@ import logging from .faildata import FailData from .ticket import FailTicket +from ..helpers import getF2BLogger # Gets the instance of the logger. -logSys = logging.getLogger(__name__) +logSys = getF2BLogger(__name__) class FailManager: diff --git a/fail2ban/server/filter.py b/fail2ban/server/filter.py index 73233905..ec60ed15 100644 --- a/fail2ban/server/filter.py +++ b/fail2ban/server/filter.py @@ -21,7 +21,7 @@ __author__ = "Cyril Jaquier and Fail2Ban Contributors" __copyright__ = "Copyright (c) 2004 Cyril Jaquier, 2011-2013 Yaroslav Halchenko" __license__ = "GPL" -import logging, re, os, fcntl, sys, locale, codecs +import re, os, fcntl, sys, locale, codecs from .failmanager import FailManagerEmpty, FailManager from .ticket import FailTicket @@ -31,9 +31,10 @@ from .datetemplate import DatePatternRegex, DateEpoch, DateTai64n from .mytime import MyTime from .failregex import FailRegex, Regex, RegexException from .action import CommandAction +from ..helpers import getF2BLogger # Gets the instance of the logger. -logSys = logging.getLogger(__name__) +logSys = getF2BLogger(__name__) ## # Log reader class. diff --git a/fail2ban/server/filtergamin.py b/fail2ban/server/filtergamin.py index 898c8dfc..f6d13d5d 100644 --- a/fail2ban/server/filtergamin.py +++ b/fail2ban/server/filtergamin.py @@ -23,16 +23,17 @@ __author__ = "Cyril Jaquier, Yaroslav Halchenko" __copyright__ = "Copyright (c) 2004 Cyril Jaquier, 2012 Yaroslav Halchenko" __license__ = "GPL" -import time, logging, fcntl +import time, fcntl import gamin from .failmanager import FailManagerEmpty from .filter import FileFilter from .mytime import MyTime +from ..helpers import getF2BLogger # Gets the instance of the logger. -logSys = logging.getLogger(__name__) +logSys = getF2BLogger(__name__) ## # Log reader class. diff --git a/fail2ban/server/filterpoll.py b/fail2ban/server/filterpoll.py index 27af97b4..a94b7cb8 100644 --- a/fail2ban/server/filterpoll.py +++ b/fail2ban/server/filterpoll.py @@ -24,14 +24,15 @@ __author__ = "Cyril Jaquier, Yaroslav Halchenko" __copyright__ = "Copyright (c) 2004 Cyril Jaquier; 2012 Yaroslav Halchenko" __license__ = "GPL" -import time, logging, os +import time, os from .failmanager import FailManagerEmpty from .filter import FileFilter from .mytime import MyTime +from ..helpers import getF2BLogger # Gets the instance of the logger. -logSys = logging.getLogger(__name__) +logSys = getF2BLogger(__name__) ## # Log reader class. diff --git a/fail2ban/server/filterpyinotify.py b/fail2ban/server/filterpyinotify.py index b340ef13..68cb9b0f 100644 --- a/fail2ban/server/filterpyinotify.py +++ b/fail2ban/server/filterpyinotify.py @@ -32,6 +32,7 @@ import pyinotify from .failmanager import FailManagerEmpty from .filter import FileFilter from .mytime import MyTime +from ..helpers import getF2BLogger if not hasattr(pyinotify, '__version__') \ @@ -48,7 +49,7 @@ except Exception, e: % str(e)) # Gets the instance of the logger. -logSys = logging.getLogger(__name__) +logSys = getF2BLogger(__name__) ## # Log reader class. diff --git a/fail2ban/server/filtersystemd.py b/fail2ban/server/filtersystemd.py index f2dee8cb..71dde126 100644 --- a/fail2ban/server/filtersystemd.py +++ b/fail2ban/server/filtersystemd.py @@ -22,7 +22,7 @@ __author__ = "Steven Hiscocks" __copyright__ = "Copyright (c) 2013 Steven Hiscocks" __license__ = "GPL" -import logging, datetime, time +import datetime, time from distutils.version import LooseVersion from systemd import journal @@ -32,10 +32,10 @@ if LooseVersion(getattr(journal, '__version__', "0")) < '204': from .failmanager import FailManagerEmpty from .filter import JournalFilter from .mytime import MyTime - +from ..helpers import getF2BLogger # Gets the instance of the logger. -logSys = logging.getLogger("fail2ban.filter") +logSys = getF2BLogger(__name__) ## # Journal reader class. diff --git a/fail2ban/server/jail.py b/fail2ban/server/jail.py index a7c174ae..5c1c4e70 100644 --- a/fail2ban/server/jail.py +++ b/fail2ban/server/jail.py @@ -26,9 +26,10 @@ __license__ = "GPL" import Queue, logging from .actions import Actions +from ..helpers import getF2BLogger # Gets the instance of the logger. -logSys = logging.getLogger(__name__) +logSys = getF2BLogger(__name__) class Jail: """Fail2Ban jail, which manages a filter and associated actions. diff --git a/fail2ban/server/server.py b/fail2ban/server/server.py index 735ce0a9..2f227517 100644 --- a/fail2ban/server/server.py +++ b/fail2ban/server/server.py @@ -32,9 +32,10 @@ from .filter import FileFilter, JournalFilter from .transmitter import Transmitter from .asyncserver import AsyncServer, AsyncServerException from .. import version +from ..helpers import getF2BLogger # Gets the instance of the logger. -logSys = logging.getLogger(__name__) +logSys = getF2BLogger(__name__) try: from .database import Fail2BanDb @@ -335,7 +336,7 @@ class Server: def setLogLevel(self, value): try: self.__loggingLock.acquire() - logging.getLogger(__name__).parent.parent.setLevel( + logging.getLogger("fail2ban").setLevel( getattr(logging, value.upper())) except AttributeError: raise ValueError("Invalid log level") @@ -367,7 +368,7 @@ class Server: try: self.__loggingLock.acquire() # set a format which is simpler for console use - formatter = logging.Formatter("%(asctime)s %(name)-16s[%(process)d]: %(levelname)-7s %(message)s") + formatter = logging.Formatter("%(asctime)s %(name)-24s[%(process)d]: %(levelname)-7s %(message)s") if target == "SYSLOG": # Syslog daemons already add date to the message. formatter = logging.Formatter("%(name)s[%(process)d]: %(levelname)s %(message)s") @@ -388,7 +389,7 @@ class Server: return False # Removes previous handlers -- in reverse order since removeHandler # alter the list in-place and that can confuses the iterable - logger = logging.getLogger(__name__).parent.parent + logger = logging.getLogger("fail2ban") for handler in logger.handlers[::-1]: # Remove the handler. logger.removeHandler(handler) @@ -425,7 +426,7 @@ class Server: def flushLogs(self): if self.__logTarget not in ['STDERR', 'STDOUT', 'SYSLOG']: - for handler in logging.getLogger(__name__).parent.parent.handlers: + for handler in logging.getLogger("fail2ban").handlers: try: handler.doRollover() logSys.info("rollover performed on %s" % self.__logTarget) @@ -434,7 +435,7 @@ class Server: logSys.info("flush performed on %s" % self.__logTarget) return "rolled over" else: - for handler in logging.getLogger(__name__).parent.parent.handlers: + for handler in logging.getLogger("fail2ban").handlers: handler.flush() logSys.info("flush performed on %s" % self.__logTarget) return "flushed" diff --git a/fail2ban/server/ticket.py b/fail2ban/server/ticket.py index 8d036b2d..a216e9ed 100644 --- a/fail2ban/server/ticket.py +++ b/fail2ban/server/ticket.py @@ -24,10 +24,10 @@ __author__ = "Cyril Jaquier" __copyright__ = "Copyright (c) 2004 Cyril Jaquier" __license__ = "GPL" -import logging +from ..helpers import getF2BLogger # Gets the instance of the logger. -logSys = logging.getLogger(__name__) +logSys = getF2BLogger(__name__) class Ticket: diff --git a/fail2ban/server/transmitter.py b/fail2ban/server/transmitter.py index 2baf00a7..b5edf367 100644 --- a/fail2ban/server/transmitter.py +++ b/fail2ban/server/transmitter.py @@ -24,11 +24,13 @@ __author__ = "Cyril Jaquier" __copyright__ = "Copyright (c) 2004 Cyril Jaquier" __license__ = "GPL" -import logging, time +import time import json +from ..helpers import getF2BLogger + # Gets the instance of the logger. -logSys = logging.getLogger(__name__) +logSys = getF2BLogger(__name__) class Transmitter: diff --git a/fail2ban/tests/files/logs/recidive b/fail2ban/tests/files/logs/recidive index 4b23b1a5..edb3a155 100644 --- a/fail2ban/tests/files/logs/recidive +++ b/fail2ban/tests/files/logs/recidive @@ -1,14 +1,14 @@ # failJSON: { "time": "2006-02-13T15:52:30", "match": true , "host": "1.2.3.4" } -2006-02-13 15:52:30,388 fail2ban.server.actions: NOTICE [sendmail] Ban 1.2.3.4 +2006-02-13 15:52:30,388 fail2ban.actions: NOTICE [sendmail] Ban 1.2.3.4 # failJSON: { "time": "2006-02-13T15:52:30", "match": true , "host": "1.2.3.4", "desc": "Extended with [PID]" } -2006-02-13 15:52:30,388 fail2ban.server.actions[123]: NOTICE [sendmail] Ban 1.2.3.4 +2006-02-13 15:52:30,388 fail2ban.actions[123]: NOTICE [sendmail] Ban 1.2.3.4 # failJSON: { "match": false } -2006-02-13 16:07:31,183 fail2ban.server.actions: NOTICE [sendmail] Unban 1.2.3.4 +2006-02-13 16:07:31,183 fail2ban.actions: NOTICE [sendmail] Unban 1.2.3.4 # failJSON: { "match": false } -2006-02-13 15:52:30,388 fail2ban.server.actions: NOTICE [recidive] Ban 1.2.3.4 +2006-02-13 15:52:30,388 fail2ban.actions: NOTICE [recidive] Ban 1.2.3.4 # syslog example # failJSON: { "time": "2004-09-16T00:44:55", "match": true , "host": "10.0.0.7" } -Sep 16 00:44:55 spaceman fail2ban.server.actions: NOTICE [jail] Ban 10.0.0.7 +Sep 16 00:44:55 spaceman fail2ban.actions: NOTICE [jail] Ban 10.0.0.7 # failJSON: { "time": "2006-02-13T15:52:30", "match": true , "host": "1.2.3.4", "desc": "Extended with [PID] and padding" } -2006-02-13 15:52:30,388 fail2ban.server.actions[123]: NOTICE [sendmail] Ban 1.2.3.4 +2006-02-13 15:52:30,388 fail2ban.actions [123]: NOTICE [sendmail] Ban 1.2.3.4 diff --git a/fail2ban/tests/servertestcase.py b/fail2ban/tests/servertestcase.py index 7b05a9b6..1dfe13ca 100644 --- a/fail2ban/tests/servertestcase.py +++ b/fail2ban/tests/servertestcase.py @@ -35,6 +35,7 @@ import logging from ..server.failregex import Regex, FailRegex, RegexException from ..server.server import Server from ..server.jail import Jail +from ..helpers import getF2BLogger try: from ..server import filtersystemd @@ -722,7 +723,7 @@ class TransmitterLogging(TransmitterBase): os.close(f) self.server.setLogLevel("WARNING") self.assertEqual(self.transm.proceed(["set", "logtarget", fn]), (0, fn)) - l = logging.getLogger('fail2ban.server.server').parent.parent + l = logging.getLogger('fail2ban') l.warning("Before file moved") try: f2, fn2 = tempfile.mkstemp("fail2ban.log") @@ -796,5 +797,10 @@ class RegexTests(unittest.TestCase): self.assertTrue(fr.hasMatched()) self.assertRaises(RegexException, fr.getHost) +class LoggingTests(unittest.TestCase): + def testGetF2BLogger(self): + testLogSys = getF2BLogger("fail2ban.some.string.with.name") + self.assertEqual(testLogSys.parent.name, "fail2ban") + self.assertEqual(testLogSys.name, "fail2ban.name") diff --git a/fail2ban/tests/utils.py b/fail2ban/tests/utils.py index 7727632e..f35685a4 100644 --- a/fail2ban/tests/utils.py +++ b/fail2ban/tests/utils.py @@ -30,8 +30,9 @@ import unittest from StringIO import StringIO from ..server.mytime import MyTime +from ..helpers import getF2BLogger -logSys = logging.getLogger(__name__) +logSys = getF2BLogger(__name__) def mtimesleep(): # no sleep now should be necessary since polling tracks now not only @@ -89,6 +90,7 @@ def gatherTests(regexps=None, no_network=False): tests.addTest(unittest.makeSuite(servertestcase.Transmitter)) tests.addTest(unittest.makeSuite(servertestcase.JailTests)) tests.addTest(unittest.makeSuite(servertestcase.RegexTests)) + tests.addTest(unittest.makeSuite(servertestcase.LoggingTests)) tests.addTest(unittest.makeSuite(actiontestcase.CommandActionTest)) tests.addTest(unittest.makeSuite(actionstestcase.ExecuteActions)) # FailManager From f7da091437b991fd860d1dac90af9fe111633c20 Mon Sep 17 00:00:00 2001 From: Steven Hiscocks Date: Mon, 9 Jun 2014 22:27:51 +0100 Subject: [PATCH 074/141] ENH: Log unhandled exceptions to Fail2Ban log --- ChangeLog | 1 + fail2ban/helpers.py | 7 +++++++ fail2ban/server/jailthread.py | 13 +++++++++++++ fail2ban/server/server.py | 5 ++++- fail2ban/tests/servertestcase.py | 13 ++++++++++++- 5 files changed, 37 insertions(+), 2 deletions(-) diff --git a/ChangeLog b/ChangeLog index ff70f07c..a7743801 100644 --- a/ChangeLog +++ b/ChangeLog @@ -40,6 +40,7 @@ ver. 0.9.1 (2014/xx/xx) - better, faster, stronger * Match non "Bye Bye" disconnect messages for sshd locked account regex * Realign fail2ban log output with white space to improve readability. Does not affect SYSLOG output. + * Log unhandled exceptions ver. 0.9.0 (2014/03/14) - beta ---------- diff --git a/fail2ban/helpers.py b/fail2ban/helpers.py index c3e50336..35c7ebc3 100644 --- a/fail2ban/helpers.py +++ b/fail2ban/helpers.py @@ -112,3 +112,10 @@ def getF2BLogger(name): """Get logging.Logger instance with Fail2Ban logger name convention """ return logging.getLogger("fail2ban.%s" % name.rpartition(".")[-1]) + +def fail2ban_excepthook(exctype, value, traceback): + """Except hook used to log unhandled exceptions to Fail2Ban log + """ + logging.getLogger("fail2ban").critical( + "Unhandled exception in Fail2Ban:", exc_info=True) + return sys.__excepthook__(exctype, value, traceback) diff --git a/fail2ban/server/jailthread.py b/fail2ban/server/jailthread.py index 7dcdb2d7..4f640f8b 100644 --- a/fail2ban/server/jailthread.py +++ b/fail2ban/server/jailthread.py @@ -24,9 +24,12 @@ __author__ = "Cyril Jaquier" __copyright__ = "Copyright (c) 2004 Cyril Jaquier" __license__ = "GPL" +import sys from threading import Thread from abc import abstractproperty, abstractmethod +from ..helpers import fail2ban_excepthook + class JailThread(Thread): """Abstract class for threading elements in Fail2Ban. @@ -53,6 +56,16 @@ class JailThread(Thread): ## The time the thread sleeps in the loop. self.sleeptime = 1 + # excepthook workaround for threads, derived from: + # http://bugs.python.org/issue1230540#msg91244 + run = self.run + def run_with_except_hook(*args, **kwargs): + try: + run(*args, **kwargs) + except: + fail2ban_excepthook(*sys.exc_info()) + self.run = run_with_except_hook + @abstractproperty def status(self): # pragma: no cover - abstract """Abstract - Should provide status information. diff --git a/fail2ban/server/server.py b/fail2ban/server/server.py index 2f227517..a6be18ac 100644 --- a/fail2ban/server/server.py +++ b/fail2ban/server/server.py @@ -32,7 +32,7 @@ from .filter import FileFilter, JournalFilter from .transmitter import Transmitter from .asyncserver import AsyncServer, AsyncServerException from .. import version -from ..helpers import getF2BLogger +from ..helpers import getF2BLogger, fail2ban_excepthook # Gets the instance of the logger. logSys = getF2BLogger(__name__) @@ -70,6 +70,9 @@ class Server: signal.signal(signal.SIGTERM, self.__sigTERMhandler) signal.signal(signal.SIGINT, self.__sigTERMhandler) + # Ensure unhandled exceptions are logged + sys.excepthook = fail2ban_excepthook + # First set the mask to only allow access to owner os.umask(0077) if self.__daemon: # pragma: no cover diff --git a/fail2ban/tests/servertestcase.py b/fail2ban/tests/servertestcase.py index 1dfe13ca..6ce6e54f 100644 --- a/fail2ban/tests/servertestcase.py +++ b/fail2ban/tests/servertestcase.py @@ -35,6 +35,8 @@ import logging from ..server.failregex import Regex, FailRegex, RegexException from ..server.server import Server from ..server.jail import Jail +from ..server.jailthread import JailThread +from .utils import LogCaptureTestCase from ..helpers import getF2BLogger try: @@ -797,10 +799,19 @@ class RegexTests(unittest.TestCase): self.assertTrue(fr.hasMatched()) self.assertRaises(RegexException, fr.getHost) -class LoggingTests(unittest.TestCase): +class _BadThread(JailThread): + def run(self): + int("cat") + +class LoggingTests(LogCaptureTestCase): def testGetF2BLogger(self): testLogSys = getF2BLogger("fail2ban.some.string.with.name") self.assertEqual(testLogSys.parent.name, "fail2ban") self.assertEqual(testLogSys.name, "fail2ban.name") + def testFail2BanExceptHook(self): + badThread = _BadThread() + badThread.start() + badThread.join() + self.assertTrue(self._is_logged("Unhandled exception")) From 70ed93d8cc389f3a6ae01bcb8f6bbbf9c31c409f Mon Sep 17 00:00:00 2001 From: JoelSnyder Date: Mon, 9 Jun 2014 18:37:31 -0700 Subject: [PATCH 075/141] Update jail.conf for oracleims filter. This is the jail.conf update. Hopefully this will go into pull request #734. --- config/jail.conf | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/config/jail.conf b/config/jail.conf index 7f7a7cbe..c42952d8 100644 --- a/config/jail.conf +++ b/config/jail.conf @@ -701,3 +701,11 @@ action = %(banaction)s[name=%(__name__)s-tcp, port="%(tcpport)s", protocol="tcp enabled = false logpath = /var/log/messages ; nrpe.cfg may define a different log_facility maxretry = 1 + + +[oracleims] +# see "oracleims" filter file for configuration requirement for Oracle IMS v6 and above +enabled = false +logpath = /opt/sun/comms/messaging64/log/mail.log_current +maxretry = 6 +banaction = iptables-allports From c325e88634f9b0c933d279f32e61e1935e6260d2 Mon Sep 17 00:00:00 2001 From: JoelSnyder Date: Mon, 9 Jun 2014 18:38:22 -0700 Subject: [PATCH 076/141] Update THANKS Per Steven. --- THANKS | 1 + 1 file changed, 1 insertion(+) diff --git a/THANKS b/THANKS index 27165492..891dba32 100644 --- a/THANKS +++ b/THANKS @@ -49,6 +49,7 @@ John Thoe Jacques Lav!gnotte Ioan Indreias Jason H Martin +Joel M Snyder Jonathan Kamens Jonathan Lanning Jonathan Underwood From 5165d2f6ea19a7781902b1ca7c32ca2f6f0238d1 Mon Sep 17 00:00:00 2001 From: JoelSnyder Date: Mon, 9 Jun 2014 18:44:27 -0700 Subject: [PATCH 077/141] Update oracleims.conf to be 'less greedy' This assumes that the protocol is always a string, which it always is, and that the other four fields in the "tr" are always numeric (which they always are). See port_access documentation at http://docs.oracle.com/cd/E19563-01/819-4428/bgaur/index.html --- config/filter.d/oracleims.conf | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/config/filter.d/oracleims.conf b/config/filter.d/oracleims.conf index e80d0b96..083702ce 100644 --- a/config/filter.d/oracleims.conf +++ b/config/filter.d/oracleims.conf @@ -45,12 +45,14 @@ before = common.conf # mi="Bad password" # us="01ko8hqnoif09qx0np@imap.opus1.com" # di="535 5.7.8 Bad username or password (Authentication failed)."/> +# Format is generally documented in the PORT_ACCESS mapping +# at http://docs.oracle.com/cd/E19563-01/819-4428/bgaur/index.html # # All that would be on one line. # Note that you MUST have LOG_FORMAT=4 for this to work! # -failregex = ^.*tr=".*\|.*\|\d+\|\|\d+" .+ Bad username or password.*"/>$ +failregex = ^.*tr="[A-Z]+\|[0-9.]+\|\d+\|\|\d+" .+ Bad username or password.*"/>$ # Option: ignoreregex # Notes.: regex to ignore. If this regex matches, the line is ignored. From 2a51a0176a3161ff9e64be61f32e52f955a10ce8 Mon Sep 17 00:00:00 2001 From: Yaroslav Halchenko Date: Tue, 27 May 2014 08:57:43 -0400 Subject: [PATCH 078/141] BF: minor type for delignoreregex in bash completions --- files/bash-completion | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/files/bash-completion b/files/bash-completion index ecb13986..57ec15b3 100644 --- a/files/bash-completion +++ b/files/bash-completion @@ -154,7 +154,7 @@ _fail2ban () { fi return 0 ;; - delfailregex|delignoregex) + delfailregex|delignoreregex) COMPREPLY=( $( compgen -W \ "$( "$1" get "$jail" "${prev/del/}" 2>/dev/null | awk -F"[][]" '{print $2}')" \ -- "$cur" ) ) From 994fe77e599c36eef8530429aedfbe6c73fd81b3 Mon Sep 17 00:00:00 2001 From: Yaroslav Halchenko Date: Tue, 10 Jun 2014 03:52:16 -0400 Subject: [PATCH 079/141] ENH: make oracleims failregex better anchored (more explicit) --- config/filter.d/oracleims.conf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config/filter.d/oracleims.conf b/config/filter.d/oracleims.conf index 083702ce..e951ff5d 100644 --- a/config/filter.d/oracleims.conf +++ b/config/filter.d/oracleims.conf @@ -52,7 +52,7 @@ before = common.conf # Note that you MUST have LOG_FORMAT=4 for this to work! # -failregex = ^.*tr="[A-Z]+\|[0-9.]+\|\d+\|\|\d+" .+ Bad username or password.*"/>$ +failregex = ^.*tr="[A-Z]+\|[0-9.]+\|\d+\|\|\d+" ap="[^"]*" mi="Bad password" us="[^"]*" di="535 5.7.8 Bad username or password( \(Authentication failed\))?\."/>$ # Option: ignoreregex # Notes.: regex to ignore. If this regex matches, the line is ignored. From 4fc7f1a831062e2caa2ef81d3677bff8b3a2d34a Mon Sep 17 00:00:00 2001 From: Steven Hiscocks Date: Tue, 10 Jun 2014 20:36:19 +0100 Subject: [PATCH 080/141] ENH: Tweak naming of getF2BLogger, and ensure consistent use --- bin/fail2ban-client | 3 ++- bin/fail2ban-regex | 4 ++-- bin/fail2ban-server | 5 +++-- bin/fail2ban-testcases | 6 +++--- fail2ban/client/actionreader.py | 4 ++-- fail2ban/client/beautifier.py | 4 ++-- fail2ban/client/configparserinc.py | 4 ++-- fail2ban/client/configreader.py | 4 ++-- fail2ban/client/configurator.py | 4 ++-- fail2ban/client/fail2banreader.py | 4 ++-- fail2ban/client/filterreader.py | 4 ++-- fail2ban/client/jailreader.py | 4 ++-- fail2ban/client/jailsreader.py | 4 ++-- fail2ban/helpers.py | 4 ++-- fail2ban/server/action.py | 7 +++---- fail2ban/server/actions.py | 4 ++-- fail2ban/server/asyncserver.py | 4 ++-- fail2ban/server/banmanager.py | 4 ++-- fail2ban/server/database.py | 4 ++-- fail2ban/server/datedetector.py | 4 ++-- fail2ban/server/datetemplate.py | 4 ++-- fail2ban/server/faildata.py | 4 ++-- fail2ban/server/failmanager.py | 4 ++-- fail2ban/server/filter.py | 4 ++-- fail2ban/server/filtergamin.py | 4 ++-- fail2ban/server/filterpoll.py | 4 ++-- fail2ban/server/filterpyinotify.py | 4 ++-- fail2ban/server/filtersystemd.py | 4 ++-- fail2ban/server/jail.py | 4 ++-- fail2ban/server/server.py | 12 ++++++------ fail2ban/server/ticket.py | 4 ++-- fail2ban/server/transmitter.py | 4 ++-- fail2ban/tests/misctestcase.py | 4 ++-- fail2ban/tests/servertestcase.py | 7 +++---- fail2ban/tests/utils.py | 8 ++++---- 35 files changed, 80 insertions(+), 80 deletions(-) diff --git a/bin/fail2ban-client b/bin/fail2ban-client index b9713153..89e0a903 100755 --- a/bin/fail2ban-client +++ b/bin/fail2ban-client @@ -30,9 +30,10 @@ from fail2ban.protocol import printFormatted from fail2ban.client.csocket import CSocket from fail2ban.client.configurator import Configurator from fail2ban.client.beautifier import Beautifier +from fail2ban.helpers import getLogger # Gets the instance of the logger. -logSys = logging.getLogger("fail2ban") +logSys = getLogger("fail2ban") ## # diff --git a/bin/fail2ban-regex b/bin/fail2ban-regex index 3a887867..ff7e2fb7 100755 --- a/bin/fail2ban-regex +++ b/bin/fail2ban-regex @@ -45,9 +45,9 @@ from fail2ban.client.filterreader import FilterReader from fail2ban.server.filter import Filter from fail2ban.server.failregex import RegexException -from fail2ban.helpers import FormatterWithTraceBack +from fail2ban.helpers import FormatterWithTraceBack, getLogger # Gets the instance of the logger. -logSys = logging.getLogger("fail2ban") +logSys = getLogger("fail2ban") def debuggexURL(sample, regex): q = urllib.urlencode({ 're': regex.replace('', '(?&.ipv4)'), diff --git a/bin/fail2ban-server b/bin/fail2ban-server index aba19ab5..ec0c0dbe 100755 --- a/bin/fail2ban-server +++ b/bin/fail2ban-server @@ -22,13 +22,14 @@ __author__ = "Cyril Jaquier" __copyright__ = "Copyright (c) 2004 Cyril Jaquier" __license__ = "GPL" -import getopt, sys, logging, os +import getopt, sys, os from fail2ban.version import version from fail2ban.server.server import Server +from fail2ban.helpers import getLogger # Gets the instance of the logger. -logSys = logging.getLogger("fail2ban") +logSys = getLogger("fail2ban") ## # \mainpage Fail2Ban diff --git a/bin/fail2ban-testcases b/bin/fail2ban-testcases index 0e2fdb4b..8b1343a3 100755 --- a/bin/fail2ban-testcases +++ b/bin/fail2ban-testcases @@ -25,7 +25,7 @@ __copyright__ = "Copyright (c) 2004 Cyril Jaquier, 2012- Yaroslav Halchenko" __license__ = "GPL" -import unittest, logging, sys, time, os +import unittest, sys, time, os # Check if local fail2ban module exists, and use if it exists by # modifying the path. This is such that tests can be used in dev @@ -35,7 +35,7 @@ if os.path.exists("fail2ban/__init__.py"): from fail2ban.version import version from fail2ban.tests.utils import gatherTests -from fail2ban.helpers import FormatterWithTraceBack +from fail2ban.helpers import FormatterWithTraceBack, getLogger from fail2ban.server.mytime import MyTime from optparse import OptionParser, Option @@ -70,7 +70,7 @@ parser = get_opt_parser() # # Logging # -logSys = logging.getLogger("fail2ban") +logSys = getLogger("fail2ban") # Numerical level of verbosity corresponding to a log "level" verbosity = {'heavydebug': 4, diff --git a/fail2ban/client/actionreader.py b/fail2ban/client/actionreader.py index 07572930..91f3322b 100644 --- a/fail2ban/client/actionreader.py +++ b/fail2ban/client/actionreader.py @@ -27,10 +27,10 @@ __license__ = "GPL" import os from .configreader import ConfigReader, DefinitionInitConfigReader -from ..helpers import getF2BLogger +from ..helpers import getLogger # Gets the instance of the logger. -logSys = getF2BLogger(__name__) +logSys = getLogger(__name__) class ActionReader(DefinitionInitConfigReader): diff --git a/fail2ban/client/beautifier.py b/fail2ban/client/beautifier.py index 070c80c5..cf17e54f 100644 --- a/fail2ban/client/beautifier.py +++ b/fail2ban/client/beautifier.py @@ -22,10 +22,10 @@ __copyright__ = "Copyright (c) 2004 Cyril Jaquier, 2013- Yaroslav Halchenko" __license__ = "GPL" from ..exceptions import UnknownJailException, DuplicateJailException -from ..helpers import getF2BLogger +from ..helpers import getLogger # Gets the instance of the logger. -logSys = getF2BLogger(__name__) +logSys = getLogger(__name__) ## # Beautify the output of the client. diff --git a/fail2ban/client/configparserinc.py b/fail2ban/client/configparserinc.py index 1be786c7..80b99517 100644 --- a/fail2ban/client/configparserinc.py +++ b/fail2ban/client/configparserinc.py @@ -25,7 +25,7 @@ __copyright__ = 'Copyright (c) 2007 Yaroslav Halchenko' __license__ = 'GPL' import os, sys -from ..helpers import getF2BLogger +from ..helpers import getLogger if sys.version_info >= (3,2): # pragma: no cover @@ -61,7 +61,7 @@ else: # pragma: no cover from ConfigParser import SafeConfigParser # Gets the instance of the logger. -logSys = getF2BLogger(__name__) +logSys = getLogger(__name__) __all__ = ['SafeConfigParserWithIncludes'] diff --git a/fail2ban/client/configreader.py b/fail2ban/client/configreader.py index e905b278..22115d3a 100644 --- a/fail2ban/client/configreader.py +++ b/fail2ban/client/configreader.py @@ -28,10 +28,10 @@ import glob, os from ConfigParser import NoOptionError, NoSectionError from .configparserinc import SafeConfigParserWithIncludes -from ..helpers import getF2BLogger +from ..helpers import getLogger # Gets the instance of the logger. -logSys = getF2BLogger(__name__) +logSys = getLogger(__name__) class ConfigReader(SafeConfigParserWithIncludes): diff --git a/fail2ban/client/configurator.py b/fail2ban/client/configurator.py index 23c21c4b..0dd9f955 100644 --- a/fail2ban/client/configurator.py +++ b/fail2ban/client/configurator.py @@ -26,10 +26,10 @@ __license__ = "GPL" from .fail2banreader import Fail2banReader from .jailsreader import JailsReader -from ..helpers import getF2BLogger +from ..helpers import getLogger # Gets the instance of the logger. -logSys = getF2BLogger(__name__) +logSys = getLogger(__name__) class Configurator: diff --git a/fail2ban/client/fail2banreader.py b/fail2ban/client/fail2banreader.py index 4d106f3a..361a5e54 100644 --- a/fail2ban/client/fail2banreader.py +++ b/fail2ban/client/fail2banreader.py @@ -25,10 +25,10 @@ __copyright__ = "Copyright (c) 2004 Cyril Jaquier" __license__ = "GPL" from .configreader import ConfigReader -from ..helpers import getF2BLogger +from ..helpers import getLogger # Gets the instance of the logger. -logSys = getF2BLogger(__name__) +logSys = getLogger(__name__) class Fail2banReader(ConfigReader): diff --git a/fail2ban/client/filterreader.py b/fail2ban/client/filterreader.py index 2f5b3b05..40d74c45 100644 --- a/fail2ban/client/filterreader.py +++ b/fail2ban/client/filterreader.py @@ -28,10 +28,10 @@ import os, shlex from .configreader import ConfigReader, DefinitionInitConfigReader from ..server.action import CommandAction -from ..helpers import getF2BLogger +from ..helpers import getLogger # Gets the instance of the logger. -logSys = getF2BLogger(__name__) +logSys = getLogger(__name__) class FilterReader(DefinitionInitConfigReader): diff --git a/fail2ban/client/jailreader.py b/fail2ban/client/jailreader.py index 6207d8f4..a4bf5174 100644 --- a/fail2ban/client/jailreader.py +++ b/fail2ban/client/jailreader.py @@ -30,10 +30,10 @@ import json from .configreader import ConfigReader from .filterreader import FilterReader from .actionreader import ActionReader -from ..helpers import getF2BLogger +from ..helpers import getLogger # Gets the instance of the logger. -logSys = getF2BLogger(__name__) +logSys = getLogger(__name__) class JailReader(ConfigReader): diff --git a/fail2ban/client/jailsreader.py b/fail2ban/client/jailsreader.py index 86fd501f..84c614b9 100644 --- a/fail2ban/client/jailsreader.py +++ b/fail2ban/client/jailsreader.py @@ -26,10 +26,10 @@ __license__ = "GPL" from .configreader import ConfigReader from .jailreader import JailReader -from ..helpers import getF2BLogger +from ..helpers import getLogger # Gets the instance of the logger. -logSys = getF2BLogger(__name__) +logSys = getLogger(__name__) class JailsReader(ConfigReader): diff --git a/fail2ban/helpers.py b/fail2ban/helpers.py index 35c7ebc3..2c810878 100644 --- a/fail2ban/helpers.py +++ b/fail2ban/helpers.py @@ -108,7 +108,7 @@ class FormatterWithTraceBack(logging.Formatter): record.tbc = record.tb = self._tb() return logging.Formatter.format(self, record) -def getF2BLogger(name): +def getLogger(name): """Get logging.Logger instance with Fail2Ban logger name convention """ return logging.getLogger("fail2ban.%s" % name.rpartition(".")[-1]) @@ -116,6 +116,6 @@ def getF2BLogger(name): def fail2ban_excepthook(exctype, value, traceback): """Except hook used to log unhandled exceptions to Fail2Ban log """ - logging.getLogger("fail2ban").critical( + getLogger("fail2ban").critical( "Unhandled exception in Fail2Ban:", exc_info=True) return sys.__excepthook__(exctype, value, traceback) diff --git a/fail2ban/server/action.py b/fail2ban/server/action.py index ea7bc91c..e1bd719a 100644 --- a/fail2ban/server/action.py +++ b/fail2ban/server/action.py @@ -26,10 +26,10 @@ import threading, re from abc import ABCMeta from collections import MutableMapping -from ..helpers import getF2BLogger +from ..helpers import getLogger # Gets the instance of the logger. -logSys = getF2BLogger(__name__) +logSys = getLogger(__name__) # Create a lock for running system commands _cmd_lock = threading.Lock() @@ -136,8 +136,7 @@ class ActionBase(object): def __init__(self, jail, name): self._jail = jail self._name = name - self._logSys = logging.getLogger( - "fail2ban.%s" % self.__class__.__name__) + self._logSys = getLogger("fail2ban.%s" % self.__class__.__name__) def start(self): """Executed when the jail/action is started. diff --git a/fail2ban/server/actions.py b/fail2ban/server/actions.py index 467df510..86a47743 100644 --- a/fail2ban/server/actions.py +++ b/fail2ban/server/actions.py @@ -41,10 +41,10 @@ from .banmanager import BanManager from .jailthread import JailThread from .action import ActionBase, CommandAction, CallingMap from .mytime import MyTime -from ..helpers import getF2BLogger +from ..helpers import getLogger # Gets the instance of the logger. -logSys = getF2BLogger(__name__) +logSys = getLogger(__name__) class Actions(JailThread, Mapping): """Handles jail actions. diff --git a/fail2ban/server/asyncserver.py b/fail2ban/server/asyncserver.py index 326e35cb..14673a99 100644 --- a/fail2ban/server/asyncserver.py +++ b/fail2ban/server/asyncserver.py @@ -27,10 +27,10 @@ __license__ = "GPL" from pickle import dumps, loads, HIGHEST_PROTOCOL import asyncore, asynchat, socket, os, sys, traceback, fcntl -from ..helpers import getF2BLogger,formatExceptionInfo +from ..helpers import getLogger,formatExceptionInfo # Gets the instance of the logger. -logSys = getF2BLogger(__name__) +logSys = getLogger(__name__) if sys.version_info >= (3,): # b"" causes SyntaxError in python <= 2.5, so below implements equivalent diff --git a/fail2ban/server/banmanager.py b/fail2ban/server/banmanager.py index 68954b9c..b24fa9e5 100644 --- a/fail2ban/server/banmanager.py +++ b/fail2ban/server/banmanager.py @@ -28,10 +28,10 @@ from threading import Lock from .ticket import BanTicket from .mytime import MyTime -from ..helpers import getF2BLogger +from ..helpers import getLogger # Gets the instance of the logger. -logSys = getF2BLogger(__name__) +logSys = getLogger(__name__) ## # Banning Manager. diff --git a/fail2ban/server/database.py b/fail2ban/server/database.py index 24a3a327..51161911 100644 --- a/fail2ban/server/database.py +++ b/fail2ban/server/database.py @@ -31,10 +31,10 @@ from threading import Lock from .mytime import MyTime from .ticket import FailTicket -from ..helpers import getF2BLogger +from ..helpers import getLogger # Gets the instance of the logger. -logSys = getF2BLogger(__name__) +logSys = getLogger(__name__) if sys.version_info >= (3,): sqlite3.register_adapter( diff --git a/fail2ban/server/datedetector.py b/fail2ban/server/datedetector.py index 8d3ccaf2..fe5282fd 100644 --- a/fail2ban/server/datedetector.py +++ b/fail2ban/server/datedetector.py @@ -24,10 +24,10 @@ __license__ = "GPL" from threading import Lock from .datetemplate import DatePatternRegex, DateTai64n, DateEpoch -from ..helpers import getF2BLogger +from ..helpers import getLogger # Gets the instance of the logger. -logSys = getF2BLogger(__name__) +logSys = getLogger(__name__) class DateDetector(object): """Manages one or more date templates to find a date within a log line. diff --git a/fail2ban/server/datetemplate.py b/fail2ban/server/datetemplate.py index 0e6141e5..a5179ed1 100644 --- a/fail2ban/server/datetemplate.py +++ b/fail2ban/server/datetemplate.py @@ -28,9 +28,9 @@ import re from abc import abstractmethod from .strptime import reGroupDictStrptime, timeRE -from ..helpers import getF2BLogger +from ..helpers import getLogger -logSys = getF2BLogger(__name__) +logSys = getLogger(__name__) class DateTemplate(object): diff --git a/fail2ban/server/faildata.py b/fail2ban/server/faildata.py index 328bdb07..91aaa1ee 100644 --- a/fail2ban/server/faildata.py +++ b/fail2ban/server/faildata.py @@ -24,10 +24,10 @@ __author__ = "Cyril Jaquier" __copyright__ = "Copyright (c) 2004 Cyril Jaquier" __license__ = "GPL" -from ..helpers import getF2BLogger +from ..helpers import getLogger # Gets the instance of the logger. -logSys = getF2BLogger(__name__) +logSys = getLogger(__name__) class FailData: diff --git a/fail2ban/server/failmanager.py b/fail2ban/server/failmanager.py index 75b505c9..353f7135 100644 --- a/fail2ban/server/failmanager.py +++ b/fail2ban/server/failmanager.py @@ -29,10 +29,10 @@ import logging from .faildata import FailData from .ticket import FailTicket -from ..helpers import getF2BLogger +from ..helpers import getLogger # Gets the instance of the logger. -logSys = getF2BLogger(__name__) +logSys = getLogger(__name__) class FailManager: diff --git a/fail2ban/server/filter.py b/fail2ban/server/filter.py index ec60ed15..d1c9d2ad 100644 --- a/fail2ban/server/filter.py +++ b/fail2ban/server/filter.py @@ -31,10 +31,10 @@ from .datetemplate import DatePatternRegex, DateEpoch, DateTai64n from .mytime import MyTime from .failregex import FailRegex, Regex, RegexException from .action import CommandAction -from ..helpers import getF2BLogger +from ..helpers import getLogger # Gets the instance of the logger. -logSys = getF2BLogger(__name__) +logSys = getLogger(__name__) ## # Log reader class. diff --git a/fail2ban/server/filtergamin.py b/fail2ban/server/filtergamin.py index f6d13d5d..c9e8e31c 100644 --- a/fail2ban/server/filtergamin.py +++ b/fail2ban/server/filtergamin.py @@ -30,10 +30,10 @@ import gamin from .failmanager import FailManagerEmpty from .filter import FileFilter from .mytime import MyTime -from ..helpers import getF2BLogger +from ..helpers import getLogger # Gets the instance of the logger. -logSys = getF2BLogger(__name__) +logSys = getLogger(__name__) ## # Log reader class. diff --git a/fail2ban/server/filterpoll.py b/fail2ban/server/filterpoll.py index a94b7cb8..f37be431 100644 --- a/fail2ban/server/filterpoll.py +++ b/fail2ban/server/filterpoll.py @@ -29,10 +29,10 @@ import time, os from .failmanager import FailManagerEmpty from .filter import FileFilter from .mytime import MyTime -from ..helpers import getF2BLogger +from ..helpers import getLogger # Gets the instance of the logger. -logSys = getF2BLogger(__name__) +logSys = getLogger(__name__) ## # Log reader class. diff --git a/fail2ban/server/filterpyinotify.py b/fail2ban/server/filterpyinotify.py index 68cb9b0f..784a7e53 100644 --- a/fail2ban/server/filterpyinotify.py +++ b/fail2ban/server/filterpyinotify.py @@ -32,7 +32,7 @@ import pyinotify from .failmanager import FailManagerEmpty from .filter import FileFilter from .mytime import MyTime -from ..helpers import getF2BLogger +from ..helpers import getLogger if not hasattr(pyinotify, '__version__') \ @@ -49,7 +49,7 @@ except Exception, e: % str(e)) # Gets the instance of the logger. -logSys = getF2BLogger(__name__) +logSys = getLogger(__name__) ## # Log reader class. diff --git a/fail2ban/server/filtersystemd.py b/fail2ban/server/filtersystemd.py index 71dde126..ce89d65a 100644 --- a/fail2ban/server/filtersystemd.py +++ b/fail2ban/server/filtersystemd.py @@ -32,10 +32,10 @@ if LooseVersion(getattr(journal, '__version__', "0")) < '204': from .failmanager import FailManagerEmpty from .filter import JournalFilter from .mytime import MyTime -from ..helpers import getF2BLogger +from ..helpers import getLogger # Gets the instance of the logger. -logSys = getF2BLogger(__name__) +logSys = getLogger(__name__) ## # Journal reader class. diff --git a/fail2ban/server/jail.py b/fail2ban/server/jail.py index 5c1c4e70..e48e5d7b 100644 --- a/fail2ban/server/jail.py +++ b/fail2ban/server/jail.py @@ -26,10 +26,10 @@ __license__ = "GPL" import Queue, logging from .actions import Actions -from ..helpers import getF2BLogger +from ..helpers import getLogger # Gets the instance of the logger. -logSys = getF2BLogger(__name__) +logSys = getLogger(__name__) class Jail: """Fail2Ban jail, which manages a filter and associated actions. diff --git a/fail2ban/server/server.py b/fail2ban/server/server.py index a6be18ac..8efb2c59 100644 --- a/fail2ban/server/server.py +++ b/fail2ban/server/server.py @@ -32,10 +32,10 @@ from .filter import FileFilter, JournalFilter from .transmitter import Transmitter from .asyncserver import AsyncServer, AsyncServerException from .. import version -from ..helpers import getF2BLogger, fail2ban_excepthook +from ..helpers import getLogger, fail2ban_excepthook # Gets the instance of the logger. -logSys = getF2BLogger(__name__) +logSys = getLogger(__name__) try: from .database import Fail2BanDb @@ -339,7 +339,7 @@ class Server: def setLogLevel(self, value): try: self.__loggingLock.acquire() - logging.getLogger("fail2ban").setLevel( + getLogger("fail2ban").setLevel( getattr(logging, value.upper())) except AttributeError: raise ValueError("Invalid log level") @@ -392,7 +392,7 @@ class Server: return False # Removes previous handlers -- in reverse order since removeHandler # alter the list in-place and that can confuses the iterable - logger = logging.getLogger("fail2ban") + logger = getLogger("fail2ban") for handler in logger.handlers[::-1]: # Remove the handler. logger.removeHandler(handler) @@ -429,7 +429,7 @@ class Server: def flushLogs(self): if self.__logTarget not in ['STDERR', 'STDOUT', 'SYSLOG']: - for handler in logging.getLogger("fail2ban").handlers: + for handler in getLogger("fail2ban").handlers: try: handler.doRollover() logSys.info("rollover performed on %s" % self.__logTarget) @@ -438,7 +438,7 @@ class Server: logSys.info("flush performed on %s" % self.__logTarget) return "rolled over" else: - for handler in logging.getLogger("fail2ban").handlers: + for handler in getLogger("fail2ban").handlers: handler.flush() logSys.info("flush performed on %s" % self.__logTarget) return "flushed" diff --git a/fail2ban/server/ticket.py b/fail2ban/server/ticket.py index a216e9ed..329b3831 100644 --- a/fail2ban/server/ticket.py +++ b/fail2ban/server/ticket.py @@ -24,10 +24,10 @@ __author__ = "Cyril Jaquier" __copyright__ = "Copyright (c) 2004 Cyril Jaquier" __license__ = "GPL" -from ..helpers import getF2BLogger +from ..helpers import getLogger # Gets the instance of the logger. -logSys = getF2BLogger(__name__) +logSys = getLogger(__name__) class Ticket: diff --git a/fail2ban/server/transmitter.py b/fail2ban/server/transmitter.py index b5edf367..0cd30515 100644 --- a/fail2ban/server/transmitter.py +++ b/fail2ban/server/transmitter.py @@ -27,10 +27,10 @@ __license__ = "GPL" import time import json -from ..helpers import getF2BLogger +from ..helpers import getLogger # Gets the instance of the logger. -logSys = getF2BLogger(__name__) +logSys = getLogger(__name__) class Transmitter: diff --git a/fail2ban/tests/misctestcase.py b/fail2ban/tests/misctestcase.py index ca84eba7..64b6b65e 100644 --- a/fail2ban/tests/misctestcase.py +++ b/fail2ban/tests/misctestcase.py @@ -32,7 +32,7 @@ import datetime from glob import glob from StringIO import StringIO -from ..helpers import formatExceptionInfo, mbasename, TraceBack, FormatterWithTraceBack +from ..helpers import formatExceptionInfo, mbasename, TraceBack, FormatterWithTraceBack, getLogger from ..server.datetemplate import DatePatternRegex @@ -159,7 +159,7 @@ class TestsUtilsTest(unittest.TestCase): # and both types of traceback at once fmt = ' %(tb)s | %(tbc)s : %(message)s' - logSys = logging.getLogger("fail2ban_tests") + logSys = getLogger("fail2ban_tests") out = logging.StreamHandler(strout) out.setFormatter(Formatter(fmt)) logSys.addHandler(out) diff --git a/fail2ban/tests/servertestcase.py b/fail2ban/tests/servertestcase.py index 6ce6e54f..9b78fda8 100644 --- a/fail2ban/tests/servertestcase.py +++ b/fail2ban/tests/servertestcase.py @@ -30,14 +30,13 @@ import tempfile import os import locale import sys -import logging from ..server.failregex import Regex, FailRegex, RegexException from ..server.server import Server from ..server.jail import Jail from ..server.jailthread import JailThread from .utils import LogCaptureTestCase -from ..helpers import getF2BLogger +from ..helpers import getLogger try: from ..server import filtersystemd @@ -725,7 +724,7 @@ class TransmitterLogging(TransmitterBase): os.close(f) self.server.setLogLevel("WARNING") self.assertEqual(self.transm.proceed(["set", "logtarget", fn]), (0, fn)) - l = logging.getLogger('fail2ban') + l = getLogger('fail2ban') l.warning("Before file moved") try: f2, fn2 = tempfile.mkstemp("fail2ban.log") @@ -806,7 +805,7 @@ class _BadThread(JailThread): class LoggingTests(LogCaptureTestCase): def testGetF2BLogger(self): - testLogSys = getF2BLogger("fail2ban.some.string.with.name") + testLogSys = getLogger("fail2ban.some.string.with.name") self.assertEqual(testLogSys.parent.name, "fail2ban") self.assertEqual(testLogSys.name, "fail2ban.name") diff --git a/fail2ban/tests/utils.py b/fail2ban/tests/utils.py index f35685a4..0cb8d5b8 100644 --- a/fail2ban/tests/utils.py +++ b/fail2ban/tests/utils.py @@ -30,9 +30,9 @@ import unittest from StringIO import StringIO from ..server.mytime import MyTime -from ..helpers import getF2BLogger +from ..helpers import getLogger -logSys = getF2BLogger(__name__) +logSys = getLogger(__name__) def mtimesleep(): # no sleep now should be necessary since polling tracks now not only @@ -187,7 +187,7 @@ class LogCaptureTestCase(unittest.TestCase): # For extended testing of what gets output into logging # system, we will redirect it to a string - logSys = logging.getLogger("fail2ban") + logSys = getLogger("fail2ban") # Keep old settings self._old_level = logSys.level @@ -200,7 +200,7 @@ class LogCaptureTestCase(unittest.TestCase): def tearDown(self): """Call after every test case.""" # print "O: >>%s<<" % self._log.getvalue() - logSys = logging.getLogger("fail2ban") + logSys = getLogger("fail2ban") logSys.handlers = self._old_handlers logSys.level = self._old_level From 9764c78415e7776c0cffdd306e29180336ef9f35 Mon Sep 17 00:00:00 2001 From: Steven Hiscocks Date: Tue, 10 Jun 2014 20:38:18 +0100 Subject: [PATCH 081/141] ENH: Rename fail2ban_excepthook to excepthook --- fail2ban/helpers.py | 2 +- fail2ban/server/jailthread.py | 4 ++-- fail2ban/server/server.py | 4 ++-- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/fail2ban/helpers.py b/fail2ban/helpers.py index 2c810878..7ba8a72a 100644 --- a/fail2ban/helpers.py +++ b/fail2ban/helpers.py @@ -113,7 +113,7 @@ def getLogger(name): """ return logging.getLogger("fail2ban.%s" % name.rpartition(".")[-1]) -def fail2ban_excepthook(exctype, value, traceback): +def excepthook(exctype, value, traceback): """Except hook used to log unhandled exceptions to Fail2Ban log """ getLogger("fail2ban").critical( diff --git a/fail2ban/server/jailthread.py b/fail2ban/server/jailthread.py index 4f640f8b..2627cebf 100644 --- a/fail2ban/server/jailthread.py +++ b/fail2ban/server/jailthread.py @@ -28,7 +28,7 @@ import sys from threading import Thread from abc import abstractproperty, abstractmethod -from ..helpers import fail2ban_excepthook +from ..helpers import excepthook class JailThread(Thread): """Abstract class for threading elements in Fail2Ban. @@ -63,7 +63,7 @@ class JailThread(Thread): try: run(*args, **kwargs) except: - fail2ban_excepthook(*sys.exc_info()) + excepthook(*sys.exc_info()) self.run = run_with_except_hook @abstractproperty diff --git a/fail2ban/server/server.py b/fail2ban/server/server.py index 8efb2c59..02f3423b 100644 --- a/fail2ban/server/server.py +++ b/fail2ban/server/server.py @@ -32,7 +32,7 @@ from .filter import FileFilter, JournalFilter from .transmitter import Transmitter from .asyncserver import AsyncServer, AsyncServerException from .. import version -from ..helpers import getLogger, fail2ban_excepthook +from ..helpers import getLogger, excepthook # Gets the instance of the logger. logSys = getLogger(__name__) @@ -71,7 +71,7 @@ class Server: signal.signal(signal.SIGINT, self.__sigTERMhandler) # Ensure unhandled exceptions are logged - sys.excepthook = fail2ban_excepthook + sys.excepthook = excepthook # First set the mask to only allow access to owner os.umask(0077) From 664f1db0ba77a6be6e020f5fb2d4e7f5a4af0b14 Mon Sep 17 00:00:00 2001 From: Steven Hiscocks Date: Tue, 10 Jun 2014 20:58:57 +0100 Subject: [PATCH 082/141] BF: Fix getLogger for single level log level names --- fail2ban/helpers.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/fail2ban/helpers.py b/fail2ban/helpers.py index 7ba8a72a..cecd3f31 100644 --- a/fail2ban/helpers.py +++ b/fail2ban/helpers.py @@ -111,7 +111,9 @@ class FormatterWithTraceBack(logging.Formatter): def getLogger(name): """Get logging.Logger instance with Fail2Ban logger name convention """ - return logging.getLogger("fail2ban.%s" % name.rpartition(".")[-1]) + if "." in name: + name = "fail2ban.%s" % name.rpartition(".")[-1] + return logging.getLogger(name) def excepthook(exctype, value, traceback): """Except hook used to log unhandled exceptions to Fail2Ban log From 1e1c4ac62a2bb4cc693c963c2ba794c6666cd730 Mon Sep 17 00:00:00 2001 From: SATO Kentaro Date: Mon, 16 Jun 2014 21:15:03 +0900 Subject: [PATCH 083/141] ENH: Add to iptables-ipsets. --- ChangeLog | 1 + THANKS | 1 + config/action.d/iptables-ipset-proto4.conf | 10 ++++++++-- config/action.d/iptables-ipset-proto6-allports.conf | 10 ++++++++-- config/action.d/iptables-ipset-proto6.conf | 10 ++++++++-- 5 files changed, 26 insertions(+), 6 deletions(-) diff --git a/ChangeLog b/ChangeLog index d4ea774a..2aeb2fb0 100644 --- a/ChangeLog +++ b/ChangeLog @@ -38,6 +38,7 @@ ver. 0.9.1 (2014/xx/xx) - better, faster, stronger * Fail2ban-regex - add print-all-matched option. Closes gh-652 * Suppress fail2ban-client warnings for non-critical config options * Match non "Bye Bye" disconnect messages for sshd locked account regex + * Add tag to iptables-ipsets. ver. 0.9.0 (2014/03/14) - beta ---------- diff --git a/THANKS b/THANKS index 27165492..08079490 100644 --- a/THANKS +++ b/THANKS @@ -85,6 +85,7 @@ Robert Edeker Rolf Fokkens Roman Gelfand Russell Odom +SATO Kentaro Sebastian Arcus Sireyessire silviogarbes diff --git a/config/action.d/iptables-ipset-proto4.conf b/config/action.d/iptables-ipset-proto4.conf index fc03c68c..4714f0df 100644 --- a/config/action.d/iptables-ipset-proto4.conf +++ b/config/action.d/iptables-ipset-proto4.conf @@ -28,13 +28,13 @@ before = iptables-blocktype.conf # Values: CMD # actionstart = ipset --create f2b- iphash - iptables -I INPUT -p -m multiport --dports -m set --match-set f2b- src -j + iptables -I -p -m multiport --dports -m set --match-set f2b- src -j # Option: actionstop # Notes.: command executed once at the end of Fail2Ban # Values: CMD # -actionstop = iptables -D INPUT -p -m multiport --dports -m set --match-set f2b- src -j +actionstop = iptables -D -p -m multiport --dports -m set --match-set f2b- src -j ipset --flush f2b- ipset --destroy f2b- @@ -60,6 +60,12 @@ actionunban = ipset --test f2b- && ipset --del f2b- # name = default +# Option: chain +# Notes specifies the iptables chain to which the Fail2Ban rules should be +# added +# Values: STRING Default: INPUT +chain = INPUT + # Option: port # Notes.: specifies port to monitor # Values: [ NUM | STRING ] Default: ssh diff --git a/config/action.d/iptables-ipset-proto6-allports.conf b/config/action.d/iptables-ipset-proto6-allports.conf index 72fba9cd..a3726873 100644 --- a/config/action.d/iptables-ipset-proto6-allports.conf +++ b/config/action.d/iptables-ipset-proto6-allports.conf @@ -25,13 +25,13 @@ before = iptables-blocktype.conf # Values: CMD # actionstart = ipset create f2b- hash:ip timeout - iptables -I INPUT -m set --match-set f2b- src -j + iptables -I -m set --match-set f2b- src -j # Option: actionstop # Notes.: command executed once at the end of Fail2Ban # Values: CMD # -actionstop = iptables -D INPUT -m set --match-set f2b- src -j +actionstop = iptables -D -m set --match-set f2b- src -j ipset flush f2b- ipset destroy f2b- @@ -57,6 +57,12 @@ actionunban = ipset del f2b- -exist # name = default +# Option: chain +# Notes specifies the iptables chain to which the Fail2Ban rules should be +# added +# Values: STRING Default: INPUT +chain = INPUT + # Option: bantime # Notes: specifies the bantime in seconds (handled internally rather than by fail2ban) # Values: [ NUM ] Default: 600 diff --git a/config/action.d/iptables-ipset-proto6.conf b/config/action.d/iptables-ipset-proto6.conf index 5d848110..a3081ea0 100644 --- a/config/action.d/iptables-ipset-proto6.conf +++ b/config/action.d/iptables-ipset-proto6.conf @@ -25,13 +25,13 @@ before = iptables-blocktype.conf # Values: CMD # actionstart = ipset create f2b- hash:ip timeout - iptables -I INPUT -p -m multiport --dports -m set --match-set f2b- src -j + iptables -I -p -m multiport --dports -m set --match-set f2b- src -j # Option: actionstop # Notes.: command executed once at the end of Fail2Ban # Values: CMD # -actionstop = iptables -D INPUT -p -m multiport --dports -m set --match-set f2b- src -j +actionstop = iptables -D -p -m multiport --dports -m set --match-set f2b- src -j ipset flush f2b- ipset destroy f2b- @@ -57,6 +57,12 @@ actionunban = ipset del f2b- -exist # name = default +# Option: chain +# Notes specifies the iptables chain to which the Fail2Ban rules should be +# added +# Values: STRING Default: INPUT +chain = INPUT + # Option: port # Notes.: specifies port to monitor # Values: [ NUM | STRING ] Default: ssh From 8268c1641f968d553141236803f2b2467a477071 Mon Sep 17 00:00:00 2001 From: Steven Hiscocks Date: Tue, 17 Jun 2014 23:24:23 +0100 Subject: [PATCH 084/141] BF: aInfo could be modified by actions, causing unexpected behaviour A separate copy of aInfo is passed to each action --- config/action.d/smtp.py | 2 +- fail2ban/server/action.py | 6 ++++++ fail2ban/server/actions.py | 14 ++++++++------ 3 files changed, 15 insertions(+), 7 deletions(-) diff --git a/config/action.d/smtp.py b/config/action.d/smtp.py index 2d0add8e..86857616 100644 --- a/config/action.d/smtp.py +++ b/config/action.d/smtp.py @@ -45,7 +45,7 @@ messages['ban'] = {} messages['ban']['head'] = \ """Hi, -The IP %(ip)s has just been banned for %(bantime)s seconds +The IP %(ip)s has just been banned for %(bantime)i seconds by Fail2Ban after %(failures)i attempts against %(jailname)s. """ messages['ban']['tail'] = \ diff --git a/fail2ban/server/action.py b/fail2ban/server/action.py index fefe2c2c..7464e008 100644 --- a/fail2ban/server/action.py +++ b/fail2ban/server/action.py @@ -68,6 +68,9 @@ class CallingMap(MutableMapping): def __init__(self, *args, **kwargs): self.data = dict(*args, **kwargs) + def __repr__(self): + return "%s(%r)" % (self.__class__.__name__, self.data) + def __getitem__(self, key): value = self.data[key] if callable(value): @@ -87,6 +90,9 @@ class CallingMap(MutableMapping): def __len__(self): return len(self.data) + def copy(self): + return self.__class__(self.data.copy()) + class ActionBase(object): """An abstract base class for actions in Fail2Ban. diff --git a/fail2ban/server/actions.py b/fail2ban/server/actions.py index dd68ac13..77cc208d 100644 --- a/fail2ban/server/actions.py +++ b/fail2ban/server/actions.py @@ -273,11 +273,12 @@ class Actions(JailThread, Mapping): logSys.notice("[%s] Ban %s" % (self._jail.name, aInfo["ip"])) for name, action in self._actions.iteritems(): try: - action.ban(aInfo) + action.ban(aInfo.copy()) except Exception as e: logSys.error( - "Failed to execute ban jail '%s' action '%s': %s", - self._jail.name, name, e, + "Failed to execute ban jail '%s' action '%s' " + "info '%r': %s", + self._jail.name, name, aInfo, e, exc_info=logSys.getEffectiveLevel()<=logging.DEBUG) return True else: @@ -321,11 +322,12 @@ class Actions(JailThread, Mapping): logSys.notice("[%s] Unban %s" % (self._jail.name, aInfo["ip"])) for name, action in self._actions.iteritems(): try: - action.unban(aInfo) + action.unban(aInfo.copy()) except Exception as e: logSys.error( - "Failed to execute unban jail '%s' action '%s': %s", - self._jail.name, name, e, + "Failed to execute unban jail '%s' action '%s' " + "info '%r': %s", + self._jail.name, name, aInfo, e, exc_info=logSys.getEffectiveLevel()<=logging.DEBUG) @property From 65ff3e960498fc78282787ca5a4bbe1c2604f8f2 Mon Sep 17 00:00:00 2001 From: SATO Kentaro Date: Wed, 18 Jun 2014 19:04:57 +0900 Subject: [PATCH 085/141] ENH: Introduce iptables-common.conf. --- MANIFEST | 2 +- config/action.d/firewallcmd-ipset.conf | 18 +------- config/action.d/firewallcmd-new.conf | 18 +------- config/action.d/iptables-allports.conf | 17 +------ config/action.d/iptables-blocktype.conf | 22 --------- config/action.d/iptables-common.conf | 45 +++++++++++++++++++ config/action.d/iptables-ipset-proto4.conf | 23 +--------- .../iptables-ipset-proto6-allports.conf | 15 +------ config/action.d/iptables-ipset-proto6.conf | 27 +---------- config/action.d/iptables-multiport-log.conf | 23 +--------- config/action.d/iptables-multiport.conf | 23 +--------- config/action.d/iptables-new.conf | 24 +--------- config/action.d/iptables-xt_recent-echo.conf | 16 ++----- config/action.d/iptables.conf | 23 +--------- 14 files changed, 61 insertions(+), 235 deletions(-) delete mode 100644 config/action.d/iptables-blocktype.conf create mode 100644 config/action.d/iptables-common.conf diff --git a/MANIFEST b/MANIFEST index 7df49199..92edcca8 100644 --- a/MANIFEST +++ b/MANIFEST @@ -258,7 +258,7 @@ config/action.d/dummy.conf config/action.d/firewallcmd-new.conf config/action.d/firewallcmd-ipset.conf config/action.d/iptables-ipset-proto6-allports.conf -config/action.d/iptables-blocktype.conf +config/action.d/iptables-common.conf config/action.d/iptables-ipset-proto4.conf config/action.d/iptables-ipset-proto6.conf config/action.d/iptables-xt_recent-echo.conf diff --git a/config/action.d/firewallcmd-ipset.conf b/config/action.d/firewallcmd-ipset.conf index 03e30c3c..38b0f3d3 100644 --- a/config/action.d/firewallcmd-ipset.conf +++ b/config/action.d/firewallcmd-ipset.conf @@ -14,7 +14,7 @@ [INCLUDES] -before = iptables-blocktype.conf +before = iptables-common.conf [Definition] @@ -31,22 +31,6 @@ actionunban = ipset del fail2ban- -exist [Init] -# Default name of the chain -# -name = default - -# Option: port -# Notes.: specifies port to monitor -# Values: [ NUM | STRING ] -# -port = ssh - -# Option: protocol -# Notes.: internally used by config reader for interpolations. -# Values: [ tcp | udp | icmp | all ] -# -protocol = tcp - # Option: chain # Notes specifies the iptables chain to which the fail2ban rules should be # added diff --git a/config/action.d/firewallcmd-new.conf b/config/action.d/firewallcmd-new.conf index 62887967..9754e3f3 100644 --- a/config/action.d/firewallcmd-new.conf +++ b/config/action.d/firewallcmd-new.conf @@ -4,7 +4,7 @@ [INCLUDES] -before = iptables-blocktype.conf +before = iptables-common.conf [Definition] @@ -24,22 +24,6 @@ actionunban = firewall-cmd --direct --remove-rule ipv4 filter f2b- 0 -s -s -j [Init] -# Default name of the chain -# -name = default - -# Option: protocol -# Notes.: internally used by config reader for interpolations. -# Values: [ tcp | udp | icmp | all ] Default: tcp -# -protocol = tcp - -# Option: chain -# Notes specifies the iptables chain to which the fail2ban rules should be -# added -# Values: STRING Default: INPUT -chain = INPUT diff --git a/config/action.d/iptables-blocktype.conf b/config/action.d/iptables-blocktype.conf deleted file mode 100644 index c505e49c..00000000 --- a/config/action.d/iptables-blocktype.conf +++ /dev/null @@ -1,22 +0,0 @@ -# Fail2Ban configuration file -# -# Author: Daniel Black -# -# This is a included configuration file and includes the defination for the blocktype -# used in all iptables based actions by default. -# -# The user can override the default in iptables-blocktype.local - -[INCLUDES] - -after = iptables-blocktype.local - -[Init] - -# Option: blocktype -# Note: This is what the action does with rules. This can be any jump target -# as per the iptables man page (section 8). Common values are DROP -# REJECT, REJECT --reject-with icmp-port-unreachable -# Values: STRING -blocktype = REJECT --reject-with icmp-port-unreachable - diff --git a/config/action.d/iptables-common.conf b/config/action.d/iptables-common.conf new file mode 100644 index 00000000..c191c5a1 --- /dev/null +++ b/config/action.d/iptables-common.conf @@ -0,0 +1,45 @@ +# Fail2Ban configuration file +# +# Author: Daniel Black +# +# This is a included configuration file and includes the definitions for the iptables +# used in all iptables based actions by default. +# +# The user can override the defaults in iptables-common.local + +[INCLUDES] + +after = iptables-blocktype.local + iptables-common.local +# iptables-blocktype.local is obsolete + +[Init] + +# Option: chain +# Notes specifies the iptables chain to which the Fail2Ban rules should be +# added +# Values: STRING Default: INPUT +chain = INPUT + +# Default name of the chain +# +name = default + +# Option: port +# Notes.: specifies port to monitor +# Values: [ NUM | STRING ] Default: +# +port = ssh + +# Option: protocol +# Notes.: internally used by config reader for interpolations. +# Values: [ tcp | udp | icmp | all ] Default: tcp +# +protocol = tcp + +# Option: blocktype +# Note: This is what the action does with rules. This can be any jump target +# as per the iptables man page (section 8). Common values are DROP +# REJECT, REJECT --reject-with icmp-port-unreachable +# Values: STRING +blocktype = REJECT --reject-with icmp-port-unreachable diff --git a/config/action.d/iptables-ipset-proto4.conf b/config/action.d/iptables-ipset-proto4.conf index 4714f0df..c72b1a85 100644 --- a/config/action.d/iptables-ipset-proto4.conf +++ b/config/action.d/iptables-ipset-proto4.conf @@ -19,7 +19,7 @@ [INCLUDES] -before = iptables-blocktype.conf +before = iptables-common.conf [Definition] @@ -56,24 +56,3 @@ actionunban = ipset --test f2b- && ipset --del f2b- [Init] -# Default name of the ipset -# -name = default - -# Option: chain -# Notes specifies the iptables chain to which the Fail2Ban rules should be -# added -# Values: STRING Default: INPUT -chain = INPUT - -# Option: port -# Notes.: specifies port to monitor -# Values: [ NUM | STRING ] Default: ssh -# -port = ssh - -# Option: protocol -# Notes.: internally used by config reader for interpolations. -# Values: [ tcp | udp | icmp | all ] Default: tcp -# -protocol = tcp diff --git a/config/action.d/iptables-ipset-proto6-allports.conf b/config/action.d/iptables-ipset-proto6-allports.conf index a3726873..aaeee461 100644 --- a/config/action.d/iptables-ipset-proto6-allports.conf +++ b/config/action.d/iptables-ipset-proto6-allports.conf @@ -15,8 +15,7 @@ [INCLUDES] -before = iptables-blocktype.conf - +before = iptables-common.conf [Definition] @@ -53,18 +52,8 @@ actionunban = ipset del f2b- -exist [Init] -# Default name of the ipset -# -name = default - -# Option: chain -# Notes specifies the iptables chain to which the Fail2Ban rules should be -# added -# Values: STRING Default: INPUT -chain = INPUT - # Option: bantime # Notes: specifies the bantime in seconds (handled internally rather than by fail2ban) # Values: [ NUM ] Default: 600 - +# bantime = 600 diff --git a/config/action.d/iptables-ipset-proto6.conf b/config/action.d/iptables-ipset-proto6.conf index a3081ea0..bd36c49e 100644 --- a/config/action.d/iptables-ipset-proto6.conf +++ b/config/action.d/iptables-ipset-proto6.conf @@ -15,8 +15,7 @@ [INCLUDES] -before = iptables-blocktype.conf - +before = iptables-common.conf [Definition] @@ -53,30 +52,8 @@ actionunban = ipset del f2b- -exist [Init] -# Default name of the ipset -# -name = default - -# Option: chain -# Notes specifies the iptables chain to which the Fail2Ban rules should be -# added -# Values: STRING Default: INPUT -chain = INPUT - -# Option: port -# Notes.: specifies port to monitor -# Values: [ NUM | STRING ] Default: ssh -# -port = ssh - -# Option: protocol -# Notes.: internally used by config reader for interpolations. -# Values: [ tcp | udp | icmp | all ] Default: tcp -# -protocol = tcp - # Option: bantime # Notes: specifies the bantime in seconds (handled internally rather than by fail2ban) # Values: [ NUM ] Default: 600 - +# bantime = 600 diff --git a/config/action.d/iptables-multiport-log.conf b/config/action.d/iptables-multiport-log.conf index 5a611033..f4d80d6c 100644 --- a/config/action.d/iptables-multiport-log.conf +++ b/config/action.d/iptables-multiport-log.conf @@ -11,7 +11,7 @@ [INCLUDES] -before = iptables-blocktype.conf +before = iptables-common.conf [Definition] @@ -60,24 +60,3 @@ actionunban = iptables -D f2b- -s -j f2b--log [Init] -# Default name of the chain -# -name = default - -# Option: port -# Notes.: specifies port to monitor -# Values: [ NUM | STRING ] Default: -# -port = ssh - -# Option: protocol -# Notes.: internally used by config reader for interpolations. -# Values: [ tcp | udp | icmp | all ] Default: tcp -# -protocol = tcp - -# Option: chain -# Notes specifies the iptables chain to which the fail2ban rules should be -# added -# Values: STRING Default: INPUT -chain = INPUT diff --git a/config/action.d/iptables-multiport.conf b/config/action.d/iptables-multiport.conf index ab3225bc..b70baf92 100644 --- a/config/action.d/iptables-multiport.conf +++ b/config/action.d/iptables-multiport.conf @@ -6,7 +6,7 @@ [INCLUDES] -before = iptables-blocktype.conf +before = iptables-common.conf [Definition] @@ -50,24 +50,3 @@ actionunban = iptables -D f2b- -s -j [Init] -# Default name of the chain -# -name = default - -# Option: port -# Notes.: specifies port to monitor -# Values: [ NUM | STRING ] Default: -# -port = ssh - -# Option: protocol -# Notes.: internally used by config reader for interpolations. -# Values: [ tcp | udp | icmp | all ] Default: tcp -# -protocol = tcp - -# Option: chain -# Notes specifies the iptables chain to which the fail2ban rules should be -# added -# Values: STRING Default: INPUT -chain = INPUT diff --git a/config/action.d/iptables-new.conf b/config/action.d/iptables-new.conf index 9a4587b1..3c6657d9 100644 --- a/config/action.d/iptables-new.conf +++ b/config/action.d/iptables-new.conf @@ -8,8 +8,7 @@ [INCLUDES] -before = iptables-blocktype.conf - +before = iptables-common.conf [Definition] @@ -53,24 +52,3 @@ actionunban = iptables -D f2b- -s -j [Init] -# Default name of the chain -# -name = default - -# Option: port -# Notes.: specifies port to monitor -# Values: [ NUM | STRING ] Default: -# -port = ssh - -# Option: protocol -# Notes.: internally used by config reader for interpolations. -# Values: [ tcp | udp | icmp | all ] Default: tcp -# -protocol = tcp - -# Option: chain -# Notes specifies the iptables chain to which the fail2ban rules should be -# added -# Values: STRING Default: INPUT -chain = INPUT diff --git a/config/action.d/iptables-xt_recent-echo.conf b/config/action.d/iptables-xt_recent-echo.conf index 5d309b56..1a72968f 100644 --- a/config/action.d/iptables-xt_recent-echo.conf +++ b/config/action.d/iptables-xt_recent-echo.conf @@ -6,8 +6,7 @@ [INCLUDES] -before = iptables-blocktype.conf - +before = iptables-common.conf [Definition] @@ -33,14 +32,14 @@ before = iptables-blocktype.conf # own rules. The 3600 second timeout is independent and acts as a # safeguard in case the fail2ban process dies unexpectedly. The # shorter of the two timeouts actually matters. -actionstart = if [ `id -u` -eq 0 ];then iptables -I INPUT -m recent --update --seconds 3600 --name f2b- -j ;fi +actionstart = if [ `id -u` -eq 0 ];then iptables -I -m recent --update --seconds 3600 --name f2b- -j ;fi # Option: actionstop # Notes.: command executed once at the end of Fail2Ban # Values: CMD # actionstop = echo / > /proc/net/xt_recent/f2b- - if [ `id -u` -eq 0 ];then iptables -D INPUT -m recent --update --seconds 3600 --name f2b- -j ;fi + if [ `id -u` -eq 0 ];then iptables -D -m recent --update --seconds 3600 --name f2b- -j ;fi # Option: actioncheck # Notes.: command executed once before each actionban command @@ -66,12 +65,3 @@ actionunban = echo - > /proc/net/xt_recent/f2b- [Init] -# Default name of the chain -# -name = default - -# Option: protocol -# Notes.: internally used by config reader for interpolations. -# Values: [ tcp | udp | icmp | all ] Default: tcp -# -protocol = tcp diff --git a/config/action.d/iptables.conf b/config/action.d/iptables.conf index 5afe4bf1..a956fc55 100644 --- a/config/action.d/iptables.conf +++ b/config/action.d/iptables.conf @@ -6,7 +6,7 @@ [INCLUDES] -before = iptables-blocktype.conf +before = iptables-common.conf [Definition] @@ -50,24 +50,3 @@ actionunban = iptables -D f2b- -s -j [Init] -# Default name of the chain -# -name = default - -# Option: port -# Notes.: specifies port to monitor -# Values: [ NUM | STRING ] Default: -# -port = ssh - -# Option: protocol -# Notes.: internally used by config reader for interpolations. -# Values: [ tcp | udp | icmp | all ] Default: tcp -# -protocol = tcp - -# Option: chain -# Notes specifies the iptables chain to which the fail2ban rules should be -# added -# Values: STRING Default: INPUT -chain = INPUT From 7640aa091825baf715fe7dfe5fd0210fa2d40537 Mon Sep 17 00:00:00 2001 From: Steven Hiscocks Date: Sun, 22 Jun 2014 13:46:25 +0100 Subject: [PATCH 086/141] TST: Test for actions modifying (un)ban aInfo --- fail2ban/tests/actionstestcase.py | 22 +++++++++++++++++++ .../files/action.d/action_modifyainfo.py | 11 ++++++++++ 2 files changed, 33 insertions(+) create mode 100644 fail2ban/tests/files/action.d/action_modifyainfo.py diff --git a/fail2ban/tests/actionstestcase.py b/fail2ban/tests/actionstestcase.py index ed0fb619..3f968e46 100644 --- a/fail2ban/tests/actionstestcase.py +++ b/fail2ban/tests/actionstestcase.py @@ -29,6 +29,7 @@ import os import tempfile from ..server.actions import Actions +from ..server.ticket import FailTicket from .dummyjail import DummyJail from .utils import LogCaptureTestCase @@ -140,3 +141,24 @@ class ExecuteActions(LogCaptureTestCase): self.__actions.stop() self.__actions.join() self.assertTrue(self._is_logged("Failed to stop")) + + def testBanActionsAInfo(self): + # Action which deletes IP address from aInfo + self.__actions.add( + "action1", + os.path.join(TEST_FILES_DIR, "action.d/action_modifyainfo.py"), + {}) + self.__actions.add( + "action2", + os.path.join(TEST_FILES_DIR, "action.d/action_modifyainfo.py"), + {}) + self.__jail.putFailTicket(FailTicket("1.2.3.4", 0)) + self.__actions._Actions__checkBan() + # Will fail if modification of aInfo from first action propagates + # to second action, as both delete same key + self.assertFalse(self._is_logged("Failed to execute ban")) + + self.__actions._Actions__flushBan() + # Will fail if modification of aInfo from first action propagates + # to second action, as both delete same key + self.assertFalse(self._is_logged("Failed to execute unban")) diff --git a/fail2ban/tests/files/action.d/action_modifyainfo.py b/fail2ban/tests/files/action.d/action_modifyainfo.py new file mode 100644 index 00000000..0a1d944a --- /dev/null +++ b/fail2ban/tests/files/action.d/action_modifyainfo.py @@ -0,0 +1,11 @@ + +from fail2ban.server.action import ActionBase + +class TestAction(ActionBase): + + def ban(self, aInfo): + del aInfo['ip'] + + unban = ban + +Action = TestAction From dd3ab858dd30d4ebcab09f6d88e03726a9fdad60 Mon Sep 17 00:00:00 2001 From: Steven Hiscocks Date: Sun, 22 Jun 2014 13:56:32 +0100 Subject: [PATCH 087/141] TST: actions modifying aInfo test more robust --- fail2ban/tests/actionstestcase.py | 4 ++++ fail2ban/tests/files/action.d/action_modifyainfo.py | 5 ++++- 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/fail2ban/tests/actionstestcase.py b/fail2ban/tests/actionstestcase.py index 3f968e46..d66930ef 100644 --- a/fail2ban/tests/actionstestcase.py +++ b/fail2ban/tests/actionstestcase.py @@ -157,8 +157,12 @@ class ExecuteActions(LogCaptureTestCase): # Will fail if modification of aInfo from first action propagates # to second action, as both delete same key self.assertFalse(self._is_logged("Failed to execute ban")) + self.assertTrue(self._is_logged("action1 ban deleted aInfo IP")) + self.assertTrue(self._is_logged("action2 ban deleted aInfo IP")) self.__actions._Actions__flushBan() # Will fail if modification of aInfo from first action propagates # to second action, as both delete same key self.assertFalse(self._is_logged("Failed to execute unban")) + self.assertTrue(self._is_logged("action1 unban deleted aInfo IP")) + self.assertTrue(self._is_logged("action2 unban deleted aInfo IP")) diff --git a/fail2ban/tests/files/action.d/action_modifyainfo.py b/fail2ban/tests/files/action.d/action_modifyainfo.py index 0a1d944a..9fbe1e0b 100644 --- a/fail2ban/tests/files/action.d/action_modifyainfo.py +++ b/fail2ban/tests/files/action.d/action_modifyainfo.py @@ -5,7 +5,10 @@ class TestAction(ActionBase): def ban(self, aInfo): del aInfo['ip'] + self._logSys.info("%s ban deleted aInfo IP", self._name) - unban = ban + def unban(self, aInfo): + del aInfo['ip'] + self._logSys.info("%s unban deleted aInfo IP", self._name) Action = TestAction From 305b31ae1c201a80ffef5fe5b5bdb96e1236256f Mon Sep 17 00:00:00 2001 From: Yaroslav Halchenko Date: Sat, 21 Jun 2014 22:20:34 -0400 Subject: [PATCH 088/141] DOC: ChangeLog -- Added an entry about iptables-common.conf --- ChangeLog | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/ChangeLog b/ChangeLog index 7cc1a5a6..b901723d 100644 --- a/ChangeLog +++ b/ChangeLog @@ -10,6 +10,11 @@ Fail2Ban (version 0.9.0.dev) 2014/xx/xx ver. 0.9.1 (2014/xx/xx) - better, faster, stronger ---------- +- Refactoring (IMPORTANT -- Please review your setup and configuration): + * iptables-common.conf replaced iptables-blocktype.conf + (iptables-blocktype.local should still be read) and now also + provides defaults for the chain, port, protocol and name tags + - Fixes: * systemd backend error on bad utf-8 in python3 * badips.py action error when logging HTTP error raised with badips request @@ -34,14 +39,13 @@ ver. 0.9.1 (2014/xx/xx) - better, faster, stronger - New features: - Added monit filter thanks Jason H Martin. - - Enhancements * Fail2ban-regex - add print-all-matched option. Closes gh-652 * Suppress fail2ban-client warnings for non-critical config options * Match non "Bye Bye" disconnect messages for sshd locked account regex - * Add tag to iptables-ipsets. + * Add tag to iptables-ipsets * Realign fail2ban log output with white space to improve readability. Does - not affect SYSLOG output. + not affect SYSLOG output * Log unhandled exceptions ver. 0.9.0 (2014/03/14) - beta From 602239051bcd26c44dfa706d7d591d6a2669cff1 Mon Sep 17 00:00:00 2001 From: Yaroslav Halchenko Date: Sun, 22 Jun 2014 10:56:50 -0400 Subject: [PATCH 089/141] BF: reincarnated import of logging (used to obtain level constants) --- bin/fail2ban-testcases | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bin/fail2ban-testcases b/bin/fail2ban-testcases index 8b1343a3..475aa40b 100755 --- a/bin/fail2ban-testcases +++ b/bin/fail2ban-testcases @@ -24,7 +24,7 @@ __author__ = "Cyril Jaquier" __copyright__ = "Copyright (c) 2004 Cyril Jaquier, 2012- Yaroslav Halchenko" __license__ = "GPL" - +import logging import unittest, sys, time, os # Check if local fail2ban module exists, and use if it exists by From c7de888cd394bbfba4e440049a86fdfe1c1e7020 Mon Sep 17 00:00:00 2001 From: Yaroslav Halchenko Date: Sun, 22 Jun 2014 10:59:43 -0400 Subject: [PATCH 090/141] DOC: Changelog for previous merge (pass a copy of aInfo) --- ChangeLog | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/ChangeLog b/ChangeLog index b901723d..bdae885c 100644 --- a/ChangeLog +++ b/ChangeLog @@ -34,7 +34,8 @@ ver. 0.9.1 (2014/xx/xx) - better, faster, stronger action * Fixed TypeError with "ipfailures" and "ipjailfailures" action tags. Thanks Serg G. Brester - * Correct times for non-timezone date times formats during DST. + * Correct times for non-timezone date times formats during DST + * Pass a copy of, not original, aInfo into actions to avoid side-effects - New features: - Added monit filter thanks Jason H Martin. From add8e61036a2dcb62250cb46e1209e0a5379b379 Mon Sep 17 00:00:00 2001 From: Cyril Roos Date: Wed, 2 Jul 2014 13:52:06 +0200 Subject: [PATCH 091/141] Added Directadmin filter, jail and log test --- config/filter.d/directadmin.conf | 23 +++++++++++++++++++++++ config/jail.conf | 5 +++++ fail2ban/tests/files/logs/directadmin | 14 ++++++++++++++ 3 files changed, 42 insertions(+) create mode 100644 config/filter.d/directadmin.conf create mode 100644 fail2ban/tests/files/logs/directadmin diff --git a/config/filter.d/directadmin.conf b/config/filter.d/directadmin.conf new file mode 100644 index 00000000..7622e548 --- /dev/null +++ b/config/filter.d/directadmin.conf @@ -0,0 +1,23 @@ +# Fail2Ban configuration file for Directadmin +# +# +# + +[INCLUDES] + +before = common.conf + +[Definition] + +failregex = ^: \'\' \d{1,3} failed login attempt(s)?. \s* + +ignoreregex = + +[Init] +datepattern = ^%%Y:%%m:%%d-%%H:%%M:%%S + +# +# Requires Directadmin v1.45.3 or higher. http://www.directadmin.com/features.php?id=1590 +# +# Author: Cyril Roos + diff --git a/config/jail.conf b/config/jail.conf index c42952d8..bfc2a9c2 100644 --- a/config/jail.conf +++ b/config/jail.conf @@ -709,3 +709,8 @@ enabled = false logpath = /opt/sun/comms/messaging64/log/mail.log_current maxretry = 6 banaction = iptables-allports + +[directadmin] +enabled = false +logpath = /var/log/directadmin/login.log +port = 2222 diff --git a/fail2ban/tests/files/logs/directadmin b/fail2ban/tests/files/logs/directadmin new file mode 100644 index 00000000..85f7f8b9 --- /dev/null +++ b/fail2ban/tests/files/logs/directadmin @@ -0,0 +1,14 @@ +# failJSON: { "time": "2014-07-02T00:17:45", "match": true , "host": "3.2.1.4" } +2014:07:02-00:17:45: '3.2.1.4' 2 failed login attempts. Account 'test' + +# failJSON: { "time": "2014-07-02T13:07:40", "match": true , "host": "40.40.123.231" } +2014:07:02-13:07:40: '40.40.123.231' 13 failed login attempts. Account 'admin' + +# failJSON: { "time": "2014-07-02T13:07:50", "match": true , "host": "40.40.123.231" } +2014:07:02-13:07:50: '40.40.123.231' 5 failed login attempt. Invalid account 'user%2Ename' + +# failJSON: { "time": "2014-07-02T13:28:39", "match": false , "host": "12.12.123.231" } +2014:07:02-13:28:39: '12.12.123.231' successful login to 'nobody' after 1 attempts + +# failJSON: { "time": "2014-07-02T13:29:38", "match": true , "host": "1.2.3.4" } +2014:07:02-13:29:38: '1.2.3.4' 2 failed login attempts. Account 'user' via 'admin' From 3777591ab0ccb473397ed88ffe4c96d7db893a93 Mon Sep 17 00:00:00 2001 From: Marc Laporte Date: Sat, 5 Jul 2014 11:55:57 -0400 Subject: [PATCH 092/141] typo --- config/jail.conf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config/jail.conf b/config/jail.conf index c42952d8..5c5c7651 100644 --- a/config/jail.conf +++ b/config/jail.conf @@ -10,7 +10,7 @@ # # YOU SHOULD NOT MODIFY THIS FILE. # -# It will probably be overwitten or improved in a distribution update. +# It will probably be overwritten or improved in a distribution update. # # Provide customizations in a jail.local file or a jail.d/customisation.local. # For example to change the default bantime for all jails and to enable the From 43950d8b7eb2d3b5bab0e67dfbaa3fbadda16872 Mon Sep 17 00:00:00 2001 From: Yaroslav Halchenko Date: Tue, 8 Jul 2014 11:09:25 -0400 Subject: [PATCH 093/141] BF: fix path to the exim log on Debian systems (/var/log/exim4) --- config/jail.conf | 4 ++-- config/paths-common.conf | 1 + config/paths-debian.conf | 1 + 3 files changed, 4 insertions(+), 2 deletions(-) diff --git a/config/jail.conf b/config/jail.conf index c42952d8..0aaed995 100644 --- a/config/jail.conf +++ b/config/jail.conf @@ -501,13 +501,13 @@ logpath = %(solidpop3d_log)s [exim] port = smtp,465,submission -logpath = /var/log/exim/mainlog +logpath = %(exim_main_log)s [exim-spam] port = smtp,465,submission -logpath = /var/log/exim/mainlog +logpath = %(exim_main_log)s [kerio] diff --git a/config/paths-common.conf b/config/paths-common.conf index 008dab4e..a2c9f7af 100644 --- a/config/paths-common.conf +++ b/config/paths-common.conf @@ -22,6 +22,7 @@ syslog_user = # from /etc/audit/auditd.conf auditd_log = /var/log/audit/audit.log +exim_main_log = /var/log/exim/mainlog nginx_error_log = /var/log/nginx/error.log diff --git a/config/paths-debian.conf b/config/paths-debian.conf index 50ff948b..eff4fdae 100644 --- a/config/paths-debian.conf +++ b/config/paths-debian.conf @@ -30,6 +30,7 @@ apache_error_log = /var/log/apache2/*error.log apache_access_log = /var/log/apache2/*access.log +exim_main_log = /var/log/exim4/mainlog # was in debian squeezy but not in wheezy # /etc/proftpd/proftpd.conf (SystemLog) From 6cddc65ceeba8d5fe40ed367e5e650c60bc721b1 Mon Sep 17 00:00:00 2001 From: Yaroslav Halchenko Date: Mon, 14 Jul 2014 12:15:07 -0400 Subject: [PATCH 094/141] BF: path to exim's mainlog on Fedora (Thanks Frantisek Sumsal) + changelog entry --- ChangeLog | 1 + config/paths-fedora.conf | 2 ++ 2 files changed, 3 insertions(+) diff --git a/ChangeLog b/ChangeLog index bdae885c..0309b828 100644 --- a/ChangeLog +++ b/ChangeLog @@ -36,6 +36,7 @@ ver. 0.9.1 (2014/xx/xx) - better, faster, stronger Thanks Serg G. Brester * Correct times for non-timezone date times formats during DST * Pass a copy of, not original, aInfo into actions to avoid side-effects + * Per-distribution paths to the exim's main log - New features: - Added monit filter thanks Jason H Martin. diff --git a/config/paths-fedora.conf b/config/paths-fedora.conf index 69322e43..cc574b39 100644 --- a/config/paths-fedora.conf +++ b/config/paths-fedora.conf @@ -32,4 +32,6 @@ apache_access_log = /var/log/httpd/*access_log # proftpd_log = /var/log/proftpd/auth.log # Tested and it worked out in /var/log/messages so assuming syslog_ftp for now. +exim_main_log = /var/log/exim/main.log + mysql_log = /var/lib/mysql/mysqld.log From 84b7e93a4786e89921739e61d4c97dc87eba9cab Mon Sep 17 00:00:00 2001 From: Sean DuBois Date: Fri, 11 Jul 2014 05:08:36 +0000 Subject: [PATCH 095/141] ENH: Add version command to protocol TST: Add test for version server command --- ChangeLog | 1 + THANKS | 1 + fail2ban/client/beautifier.py | 2 ++ fail2ban/protocol.py | 1 + fail2ban/server/transmitter.py | 5 ++++- fail2ban/tests/servertestcase.py | 4 ++++ man/fail2ban-client.1 | 3 +++ 7 files changed, 16 insertions(+), 1 deletion(-) diff --git a/ChangeLog b/ChangeLog index bdae885c..d453e67b 100644 --- a/ChangeLog +++ b/ChangeLog @@ -39,6 +39,7 @@ ver. 0.9.1 (2014/xx/xx) - better, faster, stronger - New features: - Added monit filter thanks Jason H Martin. + - fail2ban-client can fetch the running server version - Enhancements * Fail2ban-regex - add print-all-matched option. Closes gh-652 diff --git a/THANKS b/THANKS index 5752c475..656a4ad4 100644 --- a/THANKS +++ b/THANKS @@ -88,6 +88,7 @@ Rolf Fokkens Roman Gelfand Russell Odom SATO Kentaro +Sean DuBois Sebastian Arcus Serg G. Brester Sireyessire diff --git a/fail2ban/client/beautifier.py b/fail2ban/client/beautifier.py index cf17e54f..d94216c8 100644 --- a/fail2ban/client/beautifier.py +++ b/fail2ban/client/beautifier.py @@ -51,6 +51,8 @@ class Beautifier: try: if inC[0] == "ping": msg = "Server replied: " + response + elif inC[0] == "version": + msg = response elif inC[0] == "start": msg = "Jail started" elif inC[0] == "stop": diff --git a/fail2ban/protocol.py b/fail2ban/protocol.py index 8d053501..9a6ee675 100644 --- a/fail2ban/protocol.py +++ b/fail2ban/protocol.py @@ -38,6 +38,7 @@ protocol = [ ["status", "gets the current status of the server"], ["ping", "tests if the server is alive"], ["help", "return this output"], +["version", "return the server version"], ['', "LOGGING", ""], ["set loglevel ", "sets logging level to . Levels: CRITICAL, ERROR, WARNING, NOTICE, INFO, DEBUG"], ["get loglevel", "gets the logging level"], diff --git a/fail2ban/server/transmitter.py b/fail2ban/server/transmitter.py index 0cd30515..39157128 100644 --- a/fail2ban/server/transmitter.py +++ b/fail2ban/server/transmitter.py @@ -28,6 +28,7 @@ import time import json from ..helpers import getLogger +from .. import version # Gets the instance of the logger. logSys = getLogger(__name__) @@ -102,7 +103,9 @@ class Transmitter: elif command[0] == "get": return self.__commandGet(command[1:]) elif command[0] == "status": - return self.status(command[1:]) + return self.status(command[1:]) + elif command[0] == "version": + return version.version raise Exception("Invalid command") def __commandSet(self, command): diff --git a/fail2ban/tests/servertestcase.py b/fail2ban/tests/servertestcase.py index 9b78fda8..fd2a22cc 100644 --- a/fail2ban/tests/servertestcase.py +++ b/fail2ban/tests/servertestcase.py @@ -37,6 +37,7 @@ from ..server.jail import Jail from ..server.jailthread import JailThread from .utils import LogCaptureTestCase from ..helpers import getLogger +from .. import version try: from ..server import filtersystemd @@ -148,6 +149,9 @@ class Transmitter(TransmitterBase): def testPing(self): self.assertEqual(self.transm.proceed(["ping"]), (0, "pong")) + def testVersion(self): + self.assertEqual(self.transm.proceed(["version"]), (0, version.version)) + def testSleep(self): t0 = time.time() self.assertEqual(self.transm.proceed(["sleep", "1"]), (0, None)) diff --git a/man/fail2ban-client.1 b/man/fail2ban-client.1 index 32580e20..e2df68ed 100644 --- a/man/fail2ban-client.1 +++ b/man/fail2ban-client.1 @@ -71,6 +71,9 @@ tests if the server is alive .TP \fBhelp\fR return this output +.TP +\fBversion\fR +return the server version .IP LOGGING .TP From 2f42ab00ad33342abc0a681e79312d19d735adb4 Mon Sep 17 00:00:00 2001 From: Florian Pelgrim Date: Thu, 17 Jul 2014 16:44:45 +0200 Subject: [PATCH 096/141] Adding vagrant support Vagrant will provide you with a default devel box where code can be tested. No further arguments like "On my system it is running. Has to be yours". I choosed a Debian wheezy based box with saltstack installed. Wheezy because it is stable and ships mostly older packages than other distros. Saltstack is used for pre-installing packeges when bringing up our box. So any requierements from fail2ban can be saved here and shipped out with git. You can add multiple other boxes. For example adding CentOS to check if the tests are passing also there. --- .gitignore | 1 + Vagrantfile | 122 ++++++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 123 insertions(+) create mode 100644 Vagrantfile diff --git a/.gitignore b/.gitignore index 76a33e60..a8942050 100644 --- a/.gitignore +++ b/.gitignore @@ -8,3 +8,4 @@ htmlcov *.rej *.bak __pycache__ +.vagrant/ diff --git a/Vagrantfile b/Vagrantfile new file mode 100644 index 00000000..d39b6201 --- /dev/null +++ b/Vagrantfile @@ -0,0 +1,122 @@ +# -*- mode: ruby -*- +# vi: set ft=ruby : + +# Vagrantfile API/syntax version. Don't touch unless you know what you're doing! +VAGRANTFILE_API_VERSION = "2" + +Vagrant.configure(VAGRANTFILE_API_VERSION) do |config| + # All Vagrant configuration is done here. The most common configuration + # options are documented and commented below. For a complete reference, + # please see the online documentation at vagrantup.com. + + # Every Vagrant virtual environment requires a box to build off of. + config.vm.box = "h2ometrics/salty-wheezy64" + + # Disable automatic box update checking. If you disable this, then + # boxes will only be checked for updates when the user runs + # `vagrant box outdated`. This is not recommended. + # config.vm.box_check_update = false + + # Create a forwarded port mapping which allows access to a specific port + # within the machine from a port on the host machine. In the example below, + # accessing "localhost:8080" will access port 80 on the guest machine. + # config.vm.network "forwarded_port", guest: 80, host: 8080 + + # Create a private network, which allows host-only access to the machine + # using a specific IP. + # config.vm.network "private_network", ip: "192.168.33.10" + + # Create a public network, which generally matched to bridged network. + # Bridged networks make the machine appear as another physical device on + # your network. + # config.vm.network "public_network" + + # If true, then any SSH connections made will enable agent forwarding. + # Default value: false + # config.ssh.forward_agent = true + + # Share an additional folder to the guest VM. The first argument is + # the path on the host to the actual folder. The second argument is + # the path on the guest to mount the folder. And the optional third + # argument is a set of non-required options. + # config.vm.synced_folder "../data", "/vagrant_data" + + # Provider-specific configuration so you can fine-tune various + # backing providers for Vagrant. These expose provider-specific options. + # Example for VirtualBox: + # + # config.vm.provider "virtualbox" do |vb| + # # Don't boot with headless mode + # vb.gui = true + # + # # Use VBoxManage to customize the VM. For example to change memory: + # vb.customize ["modifyvm", :id, "--memory", "1024"] + # end + # + # View the documentation for the provider you're using for more + # information on available options. + + # Enable provisioning with CFEngine. CFEngine Community packages are + # automatically installed. For example, configure the host as a + # policy server and optionally a policy file to run: + # + # config.vm.provision "cfengine" do |cf| + # cf.am_policy_hub = true + # # cf.run_file = "motd.cf" + # end + # + # You can also configure and bootstrap a client to an existing + # policy server: + # + # config.vm.provision "cfengine" do |cf| + # cf.policy_server_address = "10.0.2.15" + # end + + # Enable provisioning with Puppet stand alone. Puppet manifests + # are contained in a directory path relative to this Vagrantfile. + # You will need to create the manifests directory and a manifest in + # the file default.pp in the manifests_path directory. + # + # config.vm.provision "puppet" do |puppet| + # puppet.manifests_path = "manifests" + # puppet.manifest_file = "site.pp" + # end + + # Enable provisioning with chef solo, specifying a cookbooks path, roles + # path, and data_bags path (all relative to this Vagrantfile), and adding + # some recipes and/or roles. + # + # config.vm.provision "chef_solo" do |chef| + # chef.cookbooks_path = "../my-recipes/cookbooks" + # chef.roles_path = "../my-recipes/roles" + # chef.data_bags_path = "../my-recipes/data_bags" + # chef.add_recipe "mysql" + # chef.add_role "web" + # + # # You may also specify custom JSON attributes: + # chef.json = { mysql_password: "foo" } + # end + + # Enable provisioning with chef server, specifying the chef server URL, + # and the path to the validation key (relative to this Vagrantfile). + # + # The Opscode Platform uses HTTPS. Substitute your organization for + # ORGNAME in the URL and validation key. + # + # If you have your own Chef Server, use the appropriate URL, which may be + # HTTP instead of HTTPS depending on your configuration. Also change the + # validation key to validation.pem. + # + # config.vm.provision "chef_client" do |chef| + # chef.chef_server_url = "https://api.opscode.com/organizations/ORGNAME" + # chef.validation_key_path = "ORGNAME-validator.pem" + # end + # + # If you're using the Opscode platform, your validator client is + # ORGNAME-validator, replacing ORGNAME with your organization name. + # + # If you have your own Chef Server, the default validation client name is + # chef-validator, unless you changed the configuration. + # + # chef.validation_client_name = "ORGNAME-validator" +end From ac9fa906258e53509b5d5385008d1f4e53b40e5a Mon Sep 17 00:00:00 2001 From: Sean DuBois Date: Thu, 17 Jul 2014 21:57:52 +0000 Subject: [PATCH 097/141] BF: Round timeofban before inserting into the persistant database --- fail2ban/server/database.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fail2ban/server/database.py b/fail2ban/server/database.py index 51161911..47f1a485 100644 --- a/fail2ban/server/database.py +++ b/fail2ban/server/database.py @@ -368,7 +368,7 @@ class Fail2BanDb(object): #TODO: Implement data parts once arbitrary match keys completed cur.execute( "INSERT INTO bans(jail, ip, timeofban, data) VALUES(?, ?, ?, ?)", - (jail.name, ticket.getIP(), ticket.getTime(), + (jail.name, ticket.getIP(), round(ticket.getTime()), {"matches": ticket.getMatches(), "failures": ticket.getAttempt()})) From 5471e99ebe3065120ff148260bc55e202b7feeed Mon Sep 17 00:00:00 2001 From: leftyfb Date: Thu, 17 Jul 2014 22:54:30 -0400 Subject: [PATCH 098/141] Added cloudflare action --- config/action.d/cloudflare.conf | 53 +++++++++++++++++++++++++++++++++ 1 file changed, 53 insertions(+) create mode 100644 config/action.d/cloudflare.conf diff --git a/config/action.d/cloudflare.conf b/config/action.d/cloudflare.conf new file mode 100644 index 00000000..ead0d23e --- /dev/null +++ b/config/action.d/cloudflare.conf @@ -0,0 +1,53 @@ +# +# Author: Mike Rushton +# +# $Revision$ +# + +[Definition] + +# Option: actionstart +# Notes.: command executed once at the start of Fail2Ban. +# Values: CMD +# +actionstart = + +# Option: actionstop +# Notes.: command executed once at the end of Fail2Ban +# Values: CMD +# +actionstop = + +# Option: actioncheck +# Notes.: command executed once before each actionban command +# Values: CMD +# +actioncheck = + +# Option: actionban +# Notes.: command executed when banning an IP. Take care that the +# command is executed with Fail2Ban user rights. +# Tags: <ip> IP address +# <failures> number of failures +# <time> unix timestamp of the ban time +# Values: CMD +# +actionban = curl -s "https://www.cloudflare.com/api.html?a=ban&key=&u=&tkn=" +# Option: actionunban +# Notes.: command executed when unbanning an IP. Take care that the +# command is executed with Fail2Ban user rights. +# Tags: <ip> IP address +# <failures> number of failures +# <time> unix timestamp of the ban time +# Values: CMD +# +actionunban = curl -s "https://www.cloudflare.com/api.html?a=nul&key=&u=&tkn=" + + +[Init] + +# Default Cloudflare API token +cftoken = + +# Default Cloudflare username +cfuser = From cba570cabdb8cc04f18ab7c46a0044364590772a Mon Sep 17 00:00:00 2001 From: leftyfb Date: Thu, 17 Jul 2014 23:49:35 -0400 Subject: [PATCH 099/141] Updated comments --- config/action.d/cloudflare.conf | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/config/action.d/cloudflare.conf b/config/action.d/cloudflare.conf index ead0d23e..790a00a9 100644 --- a/config/action.d/cloudflare.conf +++ b/config/action.d/cloudflare.conf @@ -1,7 +1,9 @@ # # Author: Mike Rushton # -# $Revision$ +# Referenced from from http://www.normyee.net/blog/2012/02/02/adding-cloudflare-support-to-fail2ban by NORM YEE +# +# To get your Cloudflare API key: https://www.cloudflare.com/my-account # [Definition] @@ -27,18 +29,18 @@ actioncheck = # Option: actionban # Notes.: command executed when banning an IP. Take care that the # command is executed with Fail2Ban user rights. -# Tags: <ip> IP address -# <failures> number of failures -# <time> unix timestamp of the ban time +# Tags: IP address +# number of failures +#