From 68d829c1dde3979073414623c9dd4890dd5ee72c Mon Sep 17 00:00:00 2001
From: Felix Yan <felixonmars@archlinux.org>
Date: Tue, 14 Feb 2017 18:12:06 +0800
Subject: [PATCH 01/18] Add a path configuration for Arch Linux
---
MANIFEST | 1 +
config/paths-arch.conf | 32 ++++++++++++++++++++++++++++++++
2 files changed, 33 insertions(+)
create mode 100644 config/paths-arch.conf
diff --git a/MANIFEST b/MANIFEST
index f03f185b..6832f8cf 100644
--- a/MANIFEST
+++ b/MANIFEST
@@ -148,6 +148,7 @@ config/filter.d/webmin-auth.conf
config/filter.d/wuftpd.conf
config/filter.d/xinetd-fail.conf
config/jail.conf
+config/paths-arch.conf
config/paths-common.conf
config/paths-debian.conf
config/paths-fedora.conf
diff --git a/config/paths-arch.conf b/config/paths-arch.conf
new file mode 100644
index 00000000..270ce65d
--- /dev/null
+++ b/config/paths-arch.conf
@@ -0,0 +1,32 @@
+# Arch
+
+[INCLUDES]
+
+before = paths-common.conf
+
+after = paths-overrides.local
+
+
+[DEFAULT]
+
+apache_error_log = /var/log/httpd/*error_log
+
+apache_access_log = /var/log/httpd/*access_log
+
+exim_main_log = /var/log/exim/main.log
+
+mysql_log = /var/log/mariadb/mariadb.log
+ /var/log/mysqld.log
+
+roundcube_errors_log = /var/log/roundcubemail/errors
+
+# These services will log to the journal via syslog, so use the journal by
+# default.
+syslog_backend = systemd
+sshd_backend = systemd
+dropbear_backend = systemd
+proftpd_backend = systemd
+pureftpd_backend = systemd
+wuftpd_backend = systemd
+postfix_backend = systemd
+dovecot_backend = systemd
From 96c3b06abbb11f7eb8735acd7741d62854e425ba Mon Sep 17 00:00:00 2001
From: sebres <serg.brester@sebres.de>
Date: Fri, 19 May 2017 13:25:49 +0200
Subject: [PATCH 02/18] amend to #1778: repair notifier wait-cycle (too long
timeout in polling, too late check for pending files, too long stop)
---
fail2ban/server/filterpyinotify.py | 32 ++++++++++++++++++++----------
fail2ban/tests/filtertestcase.py | 1 +
2 files changed, 22 insertions(+), 11 deletions(-)
diff --git a/fail2ban/server/filterpyinotify.py b/fail2ban/server/filterpyinotify.py
index 71540f3c..53f2cbda 100644
--- a/fail2ban/server/filterpyinotify.py
+++ b/fail2ban/server/filterpyinotify.py
@@ -82,7 +82,7 @@ class FilterPyinotify(FileFilter):
self.__watchDirs = dict()
self.__pending = dict()
self.__pendingChkTime = 0
- self.__pendingNextTime = 0
+ self.__pendingMinTime = 60
logSys.debug("Created FilterPyinotify")
def callback(self, event, origin=''):
@@ -150,7 +150,7 @@ class FilterPyinotify(FileFilter):
def _addPending(self, path, reason, isDir=False):
if path not in self.__pending:
self.__pending[path] = [Utils.DEFAULT_SLEEP_INTERVAL, isDir];
- self.__pendingNextTime = 0
+ self.__pendingMinTime = 0
if isinstance(reason, pyinotify.Event):
reason = [reason.maskname, reason.pathname]
logSys.log(logging.MSG, "Log absence detected (possibly rotation) for %s, reason: %s of %s",
@@ -165,7 +165,7 @@ class FilterPyinotify(FileFilter):
if not self.__pending:
return
ntm = time.time()
- if ntm < self.__pendingNextTime:
+ if ntm < self.__pendingChkTime + self.__pendingMinTime:
return
found = {}
minTime = 60
@@ -187,7 +187,7 @@ class FilterPyinotify(FileFilter):
del self.__pending[path]
except KeyError: pass
self.__pendingChkTime = time.time()
- self.__pendingNextTime = self.__pendingChkTime + minTime
+ self.__pendingMinTime = minTime
# process now because we've missed it in monitoring:
for path, isDir in found.iteritems():
# refresh monitoring of this:
@@ -295,6 +295,12 @@ class FilterPyinotify(FileFilter):
self.commonError()
self.ticks += 1
+ @property
+ def __notify_maxtout(self):
+ # timeout for pyinotify must be set in milliseconds (fail2ban time values are
+ # floats contain seconds), max 0.5 sec (additionally regards pending check time)
+ return min(self.sleeptime, 0.5, self.__pendingMinTime) * 1000
+
##
# Main loop.
#
@@ -304,9 +310,8 @@ class FilterPyinotify(FileFilter):
def run(self):
prcevent = pyinotify.ProcessEvent()
prcevent.process_default = self.__process_default
- ## timeout for pyinotify must be set in milliseconds (our time values are floats contain seconds)
self.__notifier = pyinotify.Notifier(self.__monitor,
- prcevent, timeout=self.sleeptime * 1000)
+ prcevent, timeout=self.__notify_maxtout)
logSys.debug("[%s] filter started (pyinotifier)", self.jailName)
while self.active:
try:
@@ -314,13 +319,19 @@ class FilterPyinotify(FileFilter):
# slow check events while idle:
if self.idle:
if Utils.wait_for(lambda: not self.active or not self.idle,
- self.sleeptime * 10, self.sleeptime
+ min(self.sleeptime * 10, self.__pendingMinTime),
+ min(self.sleeptime, self.__pendingMinTime)
):
if not self.active: break
# default pyinotify handling using Notifier:
self.__notifier.process_events()
- if Utils.wait_for(lambda: not self.active or self.__notifier.check_events(), self.sleeptime):
+
+ # wait for events / timeout:
+ notify_maxtout = self.__notify_maxtout
+ def __check_events():
+ return not self.active or self.__notifier.check_events(timeout=notify_maxtout)
+ if Utils.wait_for(__check_events, min(self.sleeptime, self.__pendingMinTime)):
if not self.active: break
self.__notifier.read_events()
@@ -347,11 +358,10 @@ class FilterPyinotify(FileFilter):
# Call super.stop() and then stop the 'Notifier'
def stop(self):
- if self.__notifier: # stop the notifier
- self.__notifier.stop()
# stop filter thread:
super(FilterPyinotify, self).stop()
- self.join()
+ if self.__notifier: # stop the notifier
+ self.__notifier.stop()
##
# Wait for exit with cleanup.
diff --git a/fail2ban/tests/filtertestcase.py b/fail2ban/tests/filtertestcase.py
index ec2dea89..20a7ad50 100644
--- a/fail2ban/tests/filtertestcase.py
+++ b/fail2ban/tests/filtertestcase.py
@@ -996,6 +996,7 @@ def get_monitor_failures_testcase(Filter_):
self._wait4failures(2)
# stop before tmpdir deleted (just prevents many monitor events)
self.filter.stop()
+ self.filter.join()
def _test_move_into_file(self, interim_kill=False):
From c7ddf1f940aa30f7d31dc1bf6e4d895713a6e38b Mon Sep 17 00:00:00 2001
From: sebres <serg.brester@sebres.de>
Date: Fri, 19 May 2017 15:36:06 +0200
Subject: [PATCH 03/18] [systemd-backend] implicit closing journal descriptor
by stop filter. Partially cherry-picked from 0.10
(d153555a07ec81dabe4678ea06c8e644afc8ec40)
---
ChangeLog | 2 ++
fail2ban/server/filtersystemd.py | 7 +++++++
2 files changed, 9 insertions(+)
diff --git a/ChangeLog b/ChangeLog
index 8e7ebdb0..c798768d 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -15,6 +15,8 @@ releases.
### Fixes
+* Fix for systemd-backend: fail2ban hits the ulimit (out of file descriptors), see gh-991.
+ Partially back-ported from v.0.10.
### New Features
diff --git a/fail2ban/server/filtersystemd.py b/fail2ban/server/filtersystemd.py
index 908112a7..73db618e 100644
--- a/fail2ban/server/filtersystemd.py
+++ b/fail2ban/server/filtersystemd.py
@@ -291,6 +291,13 @@ class FilterSystemd(JournalFilter): # pragma: systemd no cover
except FailManagerEmpty:
self.failManager.cleanup(MyTime.time())
+ # close journal:
+ try:
+ if self.__journal:
+ self.__journal.close()
+ except Exception as e: # pragma: no cover
+ logSys.error("Close journal failed: %r", e,
+ exc_info=logSys.getEffectiveLevel()<=logging.DEBUG)
logSys.debug((self.jail is not None and self.jail.name
or "jailless") +" filter terminated")
return True
From e8f217390449120a3c3c7fe4ca502b37eb8d2c56 Mon Sep 17 00:00:00 2001
From: Georges Racinet <gracinet@anybox.fr>
Date: Tue, 23 May 2017 17:27:12 +0200
Subject: [PATCH 04/18] New logtimezone jail option
This new option allows to force the time zone on log lines
that don't bear a time zone indication (GitHub issue #1773), so it behaves
actually with respect to log line contents as a default time zone.
For the time being, only fixed offset timezones (UTC or UTC[+-]hhmm) are
supported, but the implementation is designed to later on treat the case
of logical timezones with DST, e.g., Europe/Paris etc.
In particular, the timezone name gets passed all the way to the strptime
module, and the resulting offset is computed for the given log line, even
though for now, it doesn't actually depend on it.
Also, the DateTemplate subclass gets to choose whether to use it or not.
For instance, it doesn't make sense to apply a time zone offset to
Unix timestamps.
The drawback is to introduce an API change for DateTemplate. I hope it's
internal enough for that not being a problem.
---
fail2ban/client/jailreader.py | 1 +
fail2ban/server/datedetector.py | 4 +--
fail2ban/server/datetemplate.py | 16 +++++++----
fail2ban/server/filter.py | 27 ++++++++++++++++--
fail2ban/server/server.py | 6 ++++
fail2ban/server/strptime.py | 39 ++++++++++++++++++++++++--
fail2ban/server/transmitter.py | 6 ++++
fail2ban/tests/clientreadertestcase.py | 12 ++++++++
fail2ban/tests/config/jail.conf | 4 +++
fail2ban/tests/datedetectortestcase.py | 15 ++++++++++
fail2ban/tests/filtertestcase.py | 10 +++++++
fail2ban/tests/servertestcase.py | 4 +++
12 files changed, 133 insertions(+), 11 deletions(-)
diff --git a/fail2ban/client/jailreader.py b/fail2ban/client/jailreader.py
index ca092990..ce0ed3b6 100644
--- a/fail2ban/client/jailreader.py
+++ b/fail2ban/client/jailreader.py
@@ -101,6 +101,7 @@ class JailReader(ConfigReader):
["string", "filter", ""]]
opts = [["bool", "enabled", False],
["string", "logpath", None],
+ ["string", "logtimezone", None],
["string", "logencoding", None],
["string", "backend", "auto"],
["int", "maxretry", None],
diff --git a/fail2ban/server/datedetector.py b/fail2ban/server/datedetector.py
index dd5d198f..39a15828 100644
--- a/fail2ban/server/datedetector.py
+++ b/fail2ban/server/datedetector.py
@@ -423,7 +423,7 @@ class DateDetector(object):
logSys.log(logLevel, " no template.")
return (None, None)
- def getTime(self, line, timeMatch=None):
+ def getTime(self, line, timeMatch=None, default_tz=None):
"""Attempts to return the date on a log line using templates.
This uses the templates' `getDate` method in an attempt to find
@@ -449,7 +449,7 @@ class DateDetector(object):
template = timeMatch[1]
if template is not None:
try:
- date = template.getDate(line, timeMatch[0])
+ date = template.getDate(line, timeMatch[0], default_tz=default_tz)
if date is not None:
if logSys.getEffectiveLevel() <= logLevel: # pragma: no cover - heavy debug
logSys.log(logLevel, " got time %f for %r using template %s",
diff --git a/fail2ban/server/datetemplate.py b/fail2ban/server/datetemplate.py
index 1d0b014b..44ea54a8 100644
--- a/fail2ban/server/datetemplate.py
+++ b/fail2ban/server/datetemplate.py
@@ -158,7 +158,7 @@ class DateTemplate(object):
return dateMatch
@abstractmethod
- def getDate(self, line, dateMatch=None):
+ def getDate(self, line, dateMatch=None, default_tz=None):
"""Abstract method, which should return the date for a log line
This should return the date for a log line, typically taking the
@@ -169,6 +169,8 @@ class DateTemplate(object):
----------
line : str
Log line, of which the date should be extracted from.
+ default_tz: if no explicit time zone is present in the line
+ passing this will interpret it as in that time zone.
Raises
------
@@ -200,13 +202,14 @@ class DateEpoch(DateTemplate):
regex = r"((?P<square>(?<=^\[))?\d{10,11}\b(?:\.\d{3,6})?)(?(square)(?=\]))"
self.setRegex(regex, wordBegin='start', wordEnd=True)
- def getDate(self, line, dateMatch=None):
+ def getDate(self, line, dateMatch=None, default_tz=None):
"""Method to return the date for a log line.
Parameters
----------
line : str
Log line, of which the date should be extracted from.
+ default_tz: ignored, Unix timestamps are time zone independent
Returns
-------
@@ -277,7 +280,7 @@ class DatePatternRegex(DateTemplate):
regex = r'(?iu)' + regex
super(DatePatternRegex, self).setRegex(regex, wordBegin, wordEnd)
- def getDate(self, line, dateMatch=None):
+ def getDate(self, line, dateMatch=None, default_tz=None):
"""Method to return the date for a log line.
This uses a custom version of strptime, using the named groups
@@ -287,6 +290,7 @@ class DatePatternRegex(DateTemplate):
----------
line : str
Log line, of which the date should be extracted from.
+ default_tz: optionally used to correct timezone
Returns
-------
@@ -297,7 +301,8 @@ class DatePatternRegex(DateTemplate):
if not dateMatch:
dateMatch = self.matchDate(line)
if dateMatch:
- return reGroupDictStrptime(dateMatch.groupdict()), dateMatch
+ return (reGroupDictStrptime(dateMatch.groupdict(), default_tz=default_tz),
+ dateMatch)
class DateTai64n(DateTemplate):
@@ -315,13 +320,14 @@ class DateTai64n(DateTemplate):
# We already know the format for TAI64N
self.setRegex("@[0-9a-f]{24}", wordBegin=wordBegin)
- def getDate(self, line, dateMatch=None):
+ def getDate(self, line, dateMatch=None, default_tz=None):
"""Method to return the date for a log line.
Parameters
----------
line : str
Log line, of which the date should be extracted from.
+ default_tz: ignored, since TAI is time zone independent
Returns
-------
diff --git a/fail2ban/server/filter.py b/fail2ban/server/filter.py
index fce02a7a..8d1eb856 100644
--- a/fail2ban/server/filter.py
+++ b/fail2ban/server/filter.py
@@ -40,6 +40,7 @@ from .failregex import FailRegex, Regex, RegexException
from .action import CommandAction
from .utils import Utils
from ..helpers import getLogger, PREFER_ENC
+from .strptime import validateTimeZone
# Gets the instance of the logger.
logSys = getLogger(__name__)
@@ -102,6 +103,8 @@ class Filter(JailThread):
self.checkAllRegex = False
## if true ignores obsolete failures (failure time < now - findTime):
self.checkFindTime = True
+ ## if set, treat log lines without explicit time zone to be in this time zone
+ self.logtimezone = None
## Ticks counter
self.ticks = 0
@@ -307,6 +310,22 @@ class Filter(JailThread):
return pattern, templates[0].name
return None
+ ##
+ # Set the log default time zone
+ #
+ # @param tz the symbolic timezone (for now fixed offset only: UTC[+-]HHMM)
+
+ def setLogTimeZone(self, tz):
+ self.logtimezone = validateTimeZone(tz)
+
+ ##
+ # Get the log default timezone
+ #
+ # @return symbolic timezone (a string)
+
+ def getLogTimeZone(self):
+ return self.logtimezone
+
##
# Set the maximum retry value.
#
@@ -621,7 +640,8 @@ class Filter(JailThread):
self.__lastDate = date
elif timeText:
- dateTimeMatch = self.dateDetector.getTime(timeText, tupleLine[3])
+ dateTimeMatch = self.dateDetector.getTime(timeText, tupleLine[3],
+ default_tz=self.logtimezone)
if dateTimeMatch is None:
logSys.error("findFailure failed to parse timeText: %s", timeText)
@@ -972,7 +992,10 @@ class FileFilter(Filter):
break
(timeMatch, template) = self.dateDetector.matchTime(line)
if timeMatch:
- dateTimeMatch = self.dateDetector.getTime(line[timeMatch.start():timeMatch.end()], (timeMatch, template))
+ dateTimeMatch = self.dateDetector.getTime(
+ line[timeMatch.start():timeMatch.end()],
+ (timeMatch, template),
+ default_tz=self.logtimezone)
else:
nextp = container.tell()
if nextp > maxp:
diff --git a/fail2ban/server/server.py b/fail2ban/server/server.py
index 56367ed7..e3b22c44 100644
--- a/fail2ban/server/server.py
+++ b/fail2ban/server/server.py
@@ -379,6 +379,12 @@ class Server:
def getDatePattern(self, name):
return self.__jails[name].filter.getDatePattern()
+ def setLogTimeZone(self, name, tz):
+ self.__jails[name].filter.setLogTimeZone(tz)
+
+ def getLogTimeZone(self, name):
+ return self.__jails[name].filter.getLogTimeZone()
+
def setIgnoreCommand(self, name, value):
self.__jails[name].filter.setIgnoreCommand(value)
diff --git a/fail2ban/server/strptime.py b/fail2ban/server/strptime.py
index 55bdcc8c..aff9db92 100644
--- a/fail2ban/server/strptime.py
+++ b/fail2ban/server/strptime.py
@@ -17,6 +17,7 @@
# along with Fail2Ban; if not, write to the Free Software
# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
+import re
import time
import calendar
import datetime
@@ -26,6 +27,7 @@ from .mytime import MyTime
locale_time = LocaleTime()
timeRE = TimeRE()
+FIXED_OFFSET_TZ_RE = re.compile(r'UTC(([+-]\d{2})(\d{2}))?$')
def _getYearCentRE(cent=(0,3), distance=3, now=(MyTime.now(), MyTime.alternateNow)):
""" Build century regex for last year and the next years (distance).
@@ -78,7 +80,36 @@ def getTimePatternRE():
names[key] = "%%%s" % key
return (patt, names)
-def reGroupDictStrptime(found_dict, msec=False):
+
+def validateTimeZone(tz):
+ """Validate a timezone.
+
+ For now this accepts only the UTC[+-]hhmm format.
+ In the future, it may be extended for named time zones (such as Europe/Paris)
+ present on the system, if a suitable tz library is present.
+ """
+ m = FIXED_OFFSET_TZ_RE.match(tz)
+ if m is None:
+ raise ValueError("Unknown or unsupported time zone: %r" % tz)
+ return tz
+
+def zone2offset(tz, dt):
+ """Return the proper offset, in minutes according to given timezone at a given time.
+
+ Parameters
+ ----------
+ tz: symbolic timezone (for now only UTC[+-]hhmm is supported, and it's assumed to have
+ been validated already)
+ dt: datetime instance for offset computation
+ """
+ if tz == 'UTC':
+ return 0
+ unsigned = int(tz[4:6])*60 + int(tz[6:])
+ if tz[3] == '-':
+ return -unsigned
+ return unsigned
+
+def reGroupDictStrptime(found_dict, msec=False, default_tz=None):
"""Return time from dictionary of strptime fields
This is tweaked from python built-in _strptime.
@@ -88,7 +119,8 @@ def reGroupDictStrptime(found_dict, msec=False):
found_dict : dict
Dictionary where keys represent the strptime fields, and values the
respective value.
-
+ default_tz : default timezone to apply if nothing relevant is in found_dict
+ (may be a non-fixed one in the future)
Returns
-------
float
@@ -209,6 +241,9 @@ def reGroupDictStrptime(found_dict, msec=False):
# Actully create date
date_result = datetime.datetime(
year, month, day, hour, minute, second, fraction)
+ # Correct timezone if not supplied in the log linge
+ if tzoffset is None and default_tz is not None:
+ tzoffset = zone2offset(default_tz, date_result)
# Add timezone info
if tzoffset is not None:
date_result -= datetime.timedelta(seconds=tzoffset * 60)
diff --git a/fail2ban/server/transmitter.py b/fail2ban/server/transmitter.py
index bc9edd43..ecc2a138 100644
--- a/fail2ban/server/transmitter.py
+++ b/fail2ban/server/transmitter.py
@@ -261,6 +261,10 @@ class Transmitter:
value = command[2]
self.__server.setDatePattern(name, value)
return self.__server.getDatePattern(name)
+ elif command[1] == "logtimezone":
+ value = command[2]
+ self.__server.setLogTimeZone(name, value)
+ return self.__server.getLogTimeZone(name)
elif command[1] == "maxretry":
value = command[2]
self.__server.setMaxRetry(name, int(value))
@@ -363,6 +367,8 @@ class Transmitter:
return self.__server.getFindTime(name)
elif command[1] == "datepattern":
return self.__server.getDatePattern(name)
+ elif command[1] == "logtimezone":
+ return self.__server.getLogTimeZone(name)
elif command[1] == "maxretry":
return self.__server.getMaxRetry(name)
elif command[1] == "maxlines":
diff --git a/fail2ban/tests/clientreadertestcase.py b/fail2ban/tests/clientreadertestcase.py
index bfa68e03..6cc2f659 100644
--- a/fail2ban/tests/clientreadertestcase.py
+++ b/fail2ban/tests/clientreadertestcase.py
@@ -196,6 +196,14 @@ class JailReaderTest(LogCaptureTestCase):
self.assertTrue(jail.isEnabled())
self.assertLogged("Invalid action definition 'joho[foo'")
+ def testJailLogTimeZone(self):
+ jail = JailReader('tz_correct', basedir=IMPERFECT_CONFIG,
+ share_config=IMPERFECT_CONFIG_SHARE_CFG)
+ self.assertTrue(jail.read())
+ self.assertTrue(jail.getOptions())
+ self.assertTrue(jail.isEnabled())
+ self.assertEqual(jail.options['logtimezone'], 'UTC+0200')
+
def testJailFilterBrokenDef(self):
jail = JailReader('brokenfilterdef', basedir=IMPERFECT_CONFIG,
share_config=IMPERFECT_CONFIG_SHARE_CFG)
@@ -533,10 +541,14 @@ class JailsReaderTest(LogCaptureTestCase):
]],
['add', 'parse_to_end_of_jail.conf', 'auto'],
['set', 'parse_to_end_of_jail.conf', 'addfailregex', '<IP>'],
+ ['set', 'tz_correct', 'addfailregex', '<IP>'],
+ ['set', 'tz_correct', 'logtimezone', 'UTC+0200'],
['start', 'emptyaction'],
['start', 'missinglogfiles'],
['start', 'brokenaction'],
['start', 'parse_to_end_of_jail.conf'],
+ ['add', 'tz_correct', 'auto'],
+ ['start', 'tz_correct'],
['config-error',
"Jail 'brokenactiondef' skipped, because of wrong configuration: Invalid action definition 'joho[foo'"],
['config-error',
diff --git a/fail2ban/tests/config/jail.conf b/fail2ban/tests/config/jail.conf
index 64c1b830..3dcbf634 100644
--- a/fail2ban/tests/config/jail.conf
+++ b/fail2ban/tests/config/jail.conf
@@ -47,3 +47,7 @@ action = thefunkychickendance
[parse_to_end_of_jail.conf]
enabled = true
action =
+
+[tz_correct]
+enabled = true
+logtimezone = UTC+0200
diff --git a/fail2ban/tests/datedetectortestcase.py b/fail2ban/tests/datedetectortestcase.py
index 39ab7173..16515f1b 100644
--- a/fail2ban/tests/datedetectortestcase.py
+++ b/fail2ban/tests/datedetectortestcase.py
@@ -89,6 +89,21 @@ class DateDetectorTest(LogCaptureTestCase):
self.assertEqual(datelog, dateUnix)
self.assertEqual(matchlog.group(1), 'Jan 23 21:59:59')
+ def testDefaultTimeZone(self):
+ log = "2017-01-23 15:00:00"
+ datelog, _ = self.datedetector.getTime(log, default_tz='UTC+0300')
+ # so in UTC, it was noon
+ self.assertEqual(datetime.datetime.utcfromtimestamp(datelog),
+ datetime.datetime(2017, 1, 23, 12, 0, 0))
+
+ datelog, _ = self.datedetector.getTime(log, default_tz='UTC')
+ self.assertEqual(datetime.datetime.utcfromtimestamp(datelog),
+ datetime.datetime(2017, 1, 23, 15, 0, 0))
+
+ datelog, _ = self.datedetector.getTime(log, default_tz='UTC-0430')
+ self.assertEqual(datetime.datetime.utcfromtimestamp(datelog),
+ datetime.datetime(2017, 1, 23, 19, 30, 0))
+
def testVariousTimes(self):
"""Test detection of various common date/time formats f2b should understand
"""
diff --git a/fail2ban/tests/filtertestcase.py b/fail2ban/tests/filtertestcase.py
index 745dcca7..cb0edb06 100644
--- a/fail2ban/tests/filtertestcase.py
+++ b/fail2ban/tests/filtertestcase.py
@@ -289,6 +289,16 @@ class BasicFilter(unittest.TestCase):
("^%Y-%m-%d-%H%M%S.%f %z **",
"^Year-Month-Day-24hourMinuteSecond.Microseconds Zone offset **"))
+ def testGetSetLogTimeZone(self):
+ self.assertEqual(self.filter.getLogTimeZone(), None)
+ self.filter.setLogTimeZone('UTC')
+ self.assertEqual(self.filter.getLogTimeZone(), 'UTC')
+ self.filter.setLogTimeZone('UTC-0400')
+ self.assertEqual(self.filter.getLogTimeZone(), 'UTC-0400')
+ self.filter.setLogTimeZone('UTC+0200')
+ self.assertEqual(self.filter.getLogTimeZone(), 'UTC+0200')
+ self.assertRaises(ValueError, self.filter.setLogTimeZone, 'not-a-time-zone')
+
def testAssertWrongTime(self):
self.assertRaises(AssertionError,
lambda: _assert_equal_entries(self,
diff --git a/fail2ban/tests/servertestcase.py b/fail2ban/tests/servertestcase.py
index 32f794a6..68b9951c 100644
--- a/fail2ban/tests/servertestcase.py
+++ b/fail2ban/tests/servertestcase.py
@@ -311,6 +311,10 @@ class Transmitter(TransmitterBase):
"datepattern", "TAI64N", (None, "TAI64N"), jail=self.jailName)
self.setGetTestNOK("datepattern", "%Cat%a%%%g", jail=self.jailName)
+ def testLogTimeZone(self):
+ self.setGetTest("logtimezone", "UTC+0400", "UTC+0400", jail=self.jailName)
+ self.setGetTestNOK("logtimezone", "not-a-time-zone", jail=self.jailName)
+
def testJailUseDNS(self):
self.setGetTest("usedns", "yes", jail=self.jailName)
self.setGetTest("usedns", "warn", jail=self.jailName)
From 5bb6be01636ca515ded2b906bb9c7f72ea85b96f Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Viktor=20Sz=C3=A9pe?= <viktor@szepe.net>
Date: Tue, 30 May 2017 02:05:38 +0200
Subject: [PATCH 05/18] IPv6 address may overlap
---
config/action.d/helpers-common.conf | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
diff --git a/config/action.d/helpers-common.conf b/config/action.d/helpers-common.conf
index b04c7f7a..ccbcb172 100644
--- a/config/action.d/helpers-common.conf
+++ b/config/action.d/helpers-common.conf
@@ -5,7 +5,7 @@
# (printf %%b "Log-excerpt contains 'test':\n"; %(_grep_logs)s; printf %%b "Log-excerpt contains 'test':\n") | mail ...
#
_grep_logs = logpath="<logpath>"; grep <grepopts> -E %(_grep_logs_args)s $logpath | <greplimit>
-_grep_logs_args = '(^|[^0-9])<ip>([^0-9]|$)'
+_grep_logs_args = '(^|[^0-9:])<ip>([^0-9:]|$)'
# Used for actions, that should not by executed if ticket was restored:
_bypass_if_restored = if [ '<restored>' = '1' ]; then exit 0; fi;
@@ -13,4 +13,4 @@ _bypass_if_restored = if [ '<restored>' = '1' ]; then exit 0; fi;
[Init]
greplimit = tail -n <grepmax>
grepmax = 1000
-grepopts = -m <grepmax>
\ No newline at end of file
+grepopts = -m <grepmax>
From 80cc47b75ff14858d9a7bab8410ce982745508da Mon Sep 17 00:00:00 2001
From: "Serg G. Brester" <github@sebres.de>
Date: Tue, 30 May 2017 09:14:43 +0200
Subject: [PATCH 06/18] Update helpers-common.conf
fixed grep pattern: escape dot-char in search-IP and more restrictive boundaries (IPv6-capable)
---
config/action.d/helpers-common.conf | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/config/action.d/helpers-common.conf b/config/action.d/helpers-common.conf
index ccbcb172..5799d9d3 100644
--- a/config/action.d/helpers-common.conf
+++ b/config/action.d/helpers-common.conf
@@ -5,7 +5,7 @@
# (printf %%b "Log-excerpt contains 'test':\n"; %(_grep_logs)s; printf %%b "Log-excerpt contains 'test':\n") | mail ...
#
_grep_logs = logpath="<logpath>"; grep <grepopts> -E %(_grep_logs_args)s $logpath | <greplimit>
-_grep_logs_args = '(^|[^0-9:])<ip>([^0-9:]|$)'
+_grep_logs_args = "(^|[^0-9a-fA-F:])$(echo '<ip>' | sed 's/\./\\./g')([^0-9a-fA-F:]|$)"
# Used for actions, that should not by executed if ticket was restored:
_bypass_if_restored = if [ '<restored>' = '1' ]; then exit 0; fi;
From 228d25c548a3f62d8606054709fcb29ff3367775 Mon Sep 17 00:00:00 2001
From: Marcel Bischoff <marcel@herrbischoff.com>
Date: Tue, 30 May 2017 20:27:44 +0200
Subject: [PATCH 07/18] Update Kerio Connect filter (#1455)
* Update Kerio Connect filter
Fixed regex for some log entries that did not get recognized and some additional error formats are added.
* Add missing colon, GitHub address
* Add filter tests
* Add missing test
---
config/filter.d/kerio.conf | 8 +++++++-
fail2ban/tests/files/logs/kerio | 15 +++++++++++++++
2 files changed, 22 insertions(+), 1 deletion(-)
diff --git a/config/filter.d/kerio.conf b/config/filter.d/kerio.conf
index 313c9b36..ec1c09df 100644
--- a/config/filter.d/kerio.conf
+++ b/config/filter.d/kerio.conf
@@ -3,9 +3,14 @@
[Definition]
failregex = ^ SMTP Spam attack detected from <HOST>,
- ^ IP address <HOST> found in DNS blacklist \S+, mail from \S+ to \S+$
+ ^ IP address <HOST> found in DNS blacklist
^ Relay attempt from IP address <HOST>
^ Attempt to deliver to unknown recipient \S+, from \S+, IP address <HOST>$
+ ^ Failed SMTP login from <HOST>
+ ^ SMTP: User \S+ doesn't exist. Attempt from IP address <HOST>
+ ^ Client with IP address <HOST> has no reverse DNS entry, connection rejected before SMTP greeting$
+ ^ Administration login into Web Administration from <HOST> failed: IP address not allowed$
+ ^ Message from IP address <HOST>, sender \S+ rejected: sender domain does not exist$
ignoreregex =
@@ -16,5 +21,6 @@ datepattern = ^\[%%d/%%b/%%Y %%H:%%M:%%S\]
# DEV NOTES:
#
# Author: A.P. Lawrence
+# Updated by: M. Bischoff <https://github.com/herrbischoff>
#
# Based off: http://aplawrence.com/Kerio/fail2ban.html
diff --git a/fail2ban/tests/files/logs/kerio b/fail2ban/tests/files/logs/kerio
index c9368c22..f7aa0eb7 100644
--- a/fail2ban/tests/files/logs/kerio
+++ b/fail2ban/tests/files/logs/kerio
@@ -25,5 +25,20 @@
# failJSON: { "time": "2013-12-13T01:11:04", "match": true, "host": "218.85.253.185" }
[13/Dec/2013 01:11:04] Attempt to deliver to unknown recipient <marge@aplawrence.com>, from <yu@rrd.com>, IP address 218.85.253.185
+# failJSON: { "time": "2017-05-29T17:29:29", "match": true, "host": "185.140.108.56" }
+[29/May/2017 17:29:29] IP address 185.140.108.56 found in DNS blacklist SpamCop, mail from <noreply-tjgqNffcgPfpbZtpDzasm@oakspaversusa.com> to <info@verinion.com> rejected
+# failJSON: { "time": "2017-05-17T19:43:42", "match": true, "host": "185.140.108.26" }
+[17/May/2017 19:43:42] SMTP: User printer@verinion.com doesn't exist. Attempt from IP address 185.140.108.26.
+# failJSON: { "time": "2017-05-17T19:44:25", "match": true, "host": "184.171.168.211" }
+[17/May/2017 19:44:25] Client with IP address 184.171.168.211 has no reverse DNS entry, connection rejected before SMTP greeting
+
+# failJSON: { "time": "2017-05-17T19:45:27", "match": true, "host": "170.178.167.136" }
+[17/May/2017 19:45:27] Administration login into Web Administration from 170.178.167.136 failed: IP address not allowed
+
+# failJSON: { "time": "2017-05-17T22:14:57", "match": true, "host": "67.211.219.82" }
+[17/May/2017 22:14:57] Message from IP address 67.211.219.82, sender <promo123@goodresponse.site> rejected: sender domain does not exist
+
+# failJSON: { "time": "2017-05-18T07:25:15", "match": true, "host": "212.92.127.112" }
+[18/May/2017 07:25:15] Failed SMTP login from 212.92.127.112 with SASL method CRAM-MD5.
From 5214c1c5d1611e0c4a8f3c960a4b833ee767f7bc Mon Sep 17 00:00:00 2001
From: "Serg G. Brester" <github@sebres.de>
Date: Tue, 30 May 2017 20:31:48 +0200
Subject: [PATCH 08/18] Update changelog (gh-1455)
---
ChangeLog | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/ChangeLog b/ChangeLog
index c798768d..5f472f81 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -21,7 +21,7 @@ releases.
### New Features
### Enhancements
-
+* filter.d/kerio.conf - filter extended with new rules (see gh-1455)
ver. 0.9.7 (2017/05/11) - awaiting-victory
From b93e47b12f69419692fc8275358b498f01b30cbe Mon Sep 17 00:00:00 2001
From: Peter Nowee <peter+github@peternowee.com>
Date: Wed, 31 May 2017 15:24:21 +0200
Subject: [PATCH 09/18] dovecot: Match also when user field is empty
Commit 5678d08 of 2016-11-26 changed:
( user=<\S*>,)?
to:
( user=<[^>]+>,)?
The change from `*` (zero or more times) to `+` (one or more times) may
not have been intended. It will miss lines containing, for example:
Aborted login (tried to use disallowed plaintext auth): user=<>
This commit reverts the `+` back to `*`.
---
config/filter.d/dovecot.conf | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/config/filter.d/dovecot.conf b/config/filter.d/dovecot.conf
index a3dc60bb..68c78c7a 100644
--- a/config/filter.d/dovecot.conf
+++ b/config/filter.d/dovecot.conf
@@ -10,7 +10,7 @@ before = common.conf
_daemon = (auth|dovecot(-auth)?|auth-worker)
failregex = ^%(__prefix_line)s(?:%(__pam_auth)s(?:\(dovecot:auth\))?:)?\s+authentication failure; logname=\S* uid=\S* euid=\S* tty=dovecot ruser=\S* rhost=<HOST>(?:\s+user=\S*)?\s*$
- ^%(__prefix_line)s(?:pop3|imap)-login: (?:Info: )?(?:Aborted login|Disconnected)(?::(?: [^ \(]+)+)? \((?:auth failed, \d+ attempts( in \d+ secs)?|tried to use (disabled|disallowed) \S+ auth)\):( user=<[^>]+>,)?( method=\S+,)? rip=<HOST>(?:, lip=\S+)?(?:, TLS(?: handshaking(?:: SSL_accept\(\) failed: error:[\dA-F]+:SSL routines:[TLS\d]+_GET_CLIENT_HELLO:unknown protocol)?)?(: Disconnected)?)?(, session=<\S+>)?\s*$
+ ^%(__prefix_line)s(?:pop3|imap)-login: (?:Info: )?(?:Aborted login|Disconnected)(?::(?: [^ \(]+)+)? \((?:auth failed, \d+ attempts( in \d+ secs)?|tried to use (disabled|disallowed) \S+ auth)\):( user=<[^>]*>,)?( method=\S+,)? rip=<HOST>(?:, lip=\S+)?(?:, TLS(?: handshaking(?:: SSL_accept\(\) failed: error:[\dA-F]+:SSL routines:[TLS\d]+_GET_CLIENT_HELLO:unknown protocol)?)?(: Disconnected)?)?(, session=<\S+>)?\s*$
^%(__prefix_line)s(?:Info|dovecot: auth\(default\)|auth-worker\(\d+\)): pam\(\S+,<HOST>\): pam_authenticate\(\) failed: (User not known to the underlying authentication module: \d+ Time\(s\)|Authentication failure \(password mismatch\?\))\s*$
^%(__prefix_line)s(?:auth|auth-worker\(\d+\)): (?:pam|passwd-file)\(\S+,<HOST>\): unknown user\s*$
^%(__prefix_line)s(?:auth|auth-worker\(\d+\)): Info: ldap\(\S*,<HOST>,\S*\): invalid credentials\s*$
From 8cb4ae0242986a80e1d3eedd217fafbfeee082b7 Mon Sep 17 00:00:00 2001
From: sebres <serg.brester@sebres.de>
Date: Fri, 9 Jun 2017 13:55:30 +0200
Subject: [PATCH 10/18] Code review and small optimizations, prepared to
provide offset-based time zones for date-detectors (parsing of input-string)
---
fail2ban/server/datedetector.py | 15 ++++++++++++--
fail2ban/server/filter.py | 20 +++++++++----------
fail2ban/server/strptime.py | 27 ++++++++++++++------------
fail2ban/tests/datedetectortestcase.py | 11 ++++++++---
4 files changed, 46 insertions(+), 27 deletions(-)
diff --git a/fail2ban/server/datedetector.py b/fail2ban/server/datedetector.py
index 39a15828..90cfe1fd 100644
--- a/fail2ban/server/datedetector.py
+++ b/fail2ban/server/datedetector.py
@@ -27,6 +27,7 @@ import time
from threading import Lock
from .datetemplate import re, DateTemplate, DatePatternRegex, DateTai64n, DateEpoch
+from .strptime import validateTimeZone
from .utils import Utils
from ..helpers import getLogger
@@ -222,6 +223,8 @@ class DateDetector(object):
self.__firstUnused = 0
# pre-match pattern:
self.__preMatch = None
+ # default TZ (if set, treat log lines without explicit time zone to be in this time zone):
+ self.__default_tz = None
def _appendTemplate(self, template, ignoreDup=False):
name = template.name
@@ -423,7 +426,15 @@ class DateDetector(object):
logSys.log(logLevel, " no template.")
return (None, None)
- def getTime(self, line, timeMatch=None, default_tz=None):
+ @property
+ def default_tz(self):
+ return self.__default_tz
+
+ @default_tz.setter
+ def default_tz(self, value):
+ self.__default_tz = validateTimeZone(value)
+
+ def getTime(self, line, timeMatch=None):
"""Attempts to return the date on a log line using templates.
This uses the templates' `getDate` method in an attempt to find
@@ -449,7 +460,7 @@ class DateDetector(object):
template = timeMatch[1]
if template is not None:
try:
- date = template.getDate(line, timeMatch[0], default_tz=default_tz)
+ date = template.getDate(line, timeMatch[0], default_tz=self.__default_tz)
if date is not None:
if logSys.getEffectiveLevel() <= logLevel: # pragma: no cover - heavy debug
logSys.log(logLevel, " got time %f for %r using template %s",
diff --git a/fail2ban/server/filter.py b/fail2ban/server/filter.py
index 8d1eb856..a8f99998 100644
--- a/fail2ban/server/filter.py
+++ b/fail2ban/server/filter.py
@@ -34,13 +34,12 @@ from .failmanager import FailManagerEmpty, FailManager
from .ipdns import DNSUtils, IPAddr
from .ticket import FailTicket
from .jailthread import JailThread
-from .datedetector import DateDetector
+from .datedetector import DateDetector, validateTimeZone
from .mytime import MyTime
from .failregex import FailRegex, Regex, RegexException
from .action import CommandAction
from .utils import Utils
from ..helpers import getLogger, PREFER_ENC
-from .strptime import validateTimeZone
# Gets the instance of the logger.
logSys = getLogger(__name__)
@@ -88,6 +87,8 @@ class Filter(JailThread):
## Store last time stamp, applicable for multi-line
self.__lastTimeText = ""
self.__lastDate = None
+ ## if set, treat log lines without explicit time zone to be in this time zone
+ self.__logtimezone = None
## External command
self.__ignoreCommand = False
## Default or preferred encoding (to decode bytes from file or journal):
@@ -103,8 +104,6 @@ class Filter(JailThread):
self.checkAllRegex = False
## if true ignores obsolete failures (failure time < now - findTime):
self.checkFindTime = True
- ## if set, treat log lines without explicit time zone to be in this time zone
- self.logtimezone = None
## Ticks counter
self.ticks = 0
@@ -285,6 +284,7 @@ class Filter(JailThread):
return
else:
dd = DateDetector()
+ dd.default_tz = self.__logtimezone
if not isinstance(pattern, (list, tuple)):
pattern = filter(bool, map(str.strip, re.split('\n+', pattern)))
for pattern in pattern:
@@ -316,7 +316,9 @@ class Filter(JailThread):
# @param tz the symbolic timezone (for now fixed offset only: UTC[+-]HHMM)
def setLogTimeZone(self, tz):
- self.logtimezone = validateTimeZone(tz)
+ validateTimeZone(tz); # avoid setting of wrong value, but hold original
+ self.__logtimezone = tz
+ if self.dateDetector: self.dateDetector.default_tz = self.__logtimezone
##
# Get the log default timezone
@@ -324,7 +326,7 @@ class Filter(JailThread):
# @return symbolic timezone (a string)
def getLogTimeZone(self):
- return self.logtimezone
+ return self.__logtimezone
##
# Set the maximum retry value.
@@ -640,8 +642,7 @@ class Filter(JailThread):
self.__lastDate = date
elif timeText:
- dateTimeMatch = self.dateDetector.getTime(timeText, tupleLine[3],
- default_tz=self.logtimezone)
+ dateTimeMatch = self.dateDetector.getTime(timeText, tupleLine[3])
if dateTimeMatch is None:
logSys.error("findFailure failed to parse timeText: %s", timeText)
@@ -994,8 +995,7 @@ class FileFilter(Filter):
if timeMatch:
dateTimeMatch = self.dateDetector.getTime(
line[timeMatch.start():timeMatch.end()],
- (timeMatch, template),
- default_tz=self.logtimezone)
+ (timeMatch, template))
else:
nextp = container.tell()
if nextp > maxp:
diff --git a/fail2ban/server/strptime.py b/fail2ban/server/strptime.py
index aff9db92..68ff2f41 100644
--- a/fail2ban/server/strptime.py
+++ b/fail2ban/server/strptime.py
@@ -27,7 +27,7 @@ from .mytime import MyTime
locale_time = LocaleTime()
timeRE = TimeRE()
-FIXED_OFFSET_TZ_RE = re.compile(r'UTC(([+-]\d{2})(\d{2}))?$')
+FIXED_OFFSET_TZ_RE = re.compile(r'(?:Z|UTC|GMT)?([+-]\d{2}(?:\d{2}))?$')
def _getYearCentRE(cent=(0,3), distance=3, now=(MyTime.now(), MyTime.alternateNow)):
""" Build century regex for last year and the next years (distance).
@@ -84,30 +84,33 @@ def getTimePatternRE():
def validateTimeZone(tz):
"""Validate a timezone.
- For now this accepts only the UTC[+-]hhmm format.
+ For now this accepts only the UTC[+-]hhmm format (UTC has aliases GMT/Z and optional).
In the future, it may be extended for named time zones (such as Europe/Paris)
present on the system, if a suitable tz library is present.
"""
+ if tz is None:
+ return None
m = FIXED_OFFSET_TZ_RE.match(tz)
if m is None:
raise ValueError("Unknown or unsupported time zone: %r" % tz)
- return tz
+ tz = m.group(1)
+ if tz is None or tz == '': # UTC/GMT
+ return 0; # fixed zero offzet
+ return zone2offset(tz, 0)
def zone2offset(tz, dt):
"""Return the proper offset, in minutes according to given timezone at a given time.
Parameters
----------
- tz: symbolic timezone (for now only UTC[+-]hhmm is supported, and it's assumed to have
- been validated already)
- dt: datetime instance for offset computation
+ tz: symbolic timezone or offset (for now only [+-]hhmm is supported, and it's assumed to have
+ been validated already)
+ dt: datetime instance for offset computation
"""
- if tz == 'UTC':
- return 0
- unsigned = int(tz[4:6])*60 + int(tz[6:])
- if tz[3] == '-':
- return -unsigned
- return unsigned
+ if isinstance(tz, basestring):
+ # [+-]1 * (hh*60 + mm)
+ return int(tz[0]+'1') * (int(tz[1:3])*60 + int(tz[3:5]))
+ return tz
def reGroupDictStrptime(found_dict, msec=False, default_tz=None):
"""Return time from dictionary of strptime fields
diff --git a/fail2ban/tests/datedetectortestcase.py b/fail2ban/tests/datedetectortestcase.py
index 16515f1b..6d728b5e 100644
--- a/fail2ban/tests/datedetectortestcase.py
+++ b/fail2ban/tests/datedetectortestcase.py
@@ -91,19 +91,24 @@ class DateDetectorTest(LogCaptureTestCase):
def testDefaultTimeZone(self):
log = "2017-01-23 15:00:00"
- datelog, _ = self.datedetector.getTime(log, default_tz='UTC+0300')
+ dd = self.datedetector
+ dd.default_tz='UTC+0300'; datelog, _ = dd.getTime(log)
# so in UTC, it was noon
self.assertEqual(datetime.datetime.utcfromtimestamp(datelog),
datetime.datetime(2017, 1, 23, 12, 0, 0))
- datelog, _ = self.datedetector.getTime(log, default_tz='UTC')
+ dd.default_tz='UTC'; datelog, _ = dd.getTime(log)
self.assertEqual(datetime.datetime.utcfromtimestamp(datelog),
datetime.datetime(2017, 1, 23, 15, 0, 0))
+ self.assertEqual(dd.default_tz, 0); # utc == 0
- datelog, _ = self.datedetector.getTime(log, default_tz='UTC-0430')
+ dd.default_tz='UTC-0430'; datelog, _ = dd.getTime(log)
self.assertEqual(datetime.datetime.utcfromtimestamp(datelog),
datetime.datetime(2017, 1, 23, 19, 30, 0))
+ self.assertRaises(ValueError, setattr, dd, 'default_tz', 'WRONG-TZ')
+ dd.default_tz = None
+
def testVariousTimes(self):
"""Test detection of various common date/time formats f2b should understand
"""
From 9f41d1e3819de407a5e992bd4699566d60c1e941 Mon Sep 17 00:00:00 2001
From: sebres <serg.brester@sebres.de>
Date: Fri, 9 Jun 2017 14:55:44 +0200
Subject: [PATCH 11/18] Normalize zone2offset (usable within
reGroupDictStrptime), tests simplified and extended with more cases (covers
precedence of input-zone over default, etc.)
---
fail2ban/server/strptime.py | 19 +++++++------
fail2ban/tests/datedetectortestcase.py | 38 ++++++++++++++++----------
2 files changed, 35 insertions(+), 22 deletions(-)
diff --git a/fail2ban/server/strptime.py b/fail2ban/server/strptime.py
index 68ff2f41..56f1c753 100644
--- a/fail2ban/server/strptime.py
+++ b/fail2ban/server/strptime.py
@@ -27,7 +27,7 @@ from .mytime import MyTime
locale_time = LocaleTime()
timeRE = TimeRE()
-FIXED_OFFSET_TZ_RE = re.compile(r'(?:Z|UTC|GMT)?([+-]\d{2}(?:\d{2}))?$')
+FIXED_OFFSET_TZ_RE = re.compile(r'(?:Z|UTC|GMT)?([+-]\d{2}(?::?\d{2})?)?$')
def _getYearCentRE(cent=(0,3), distance=3, now=(MyTime.now(), MyTime.alternateNow)):
""" Build century regex for last year and the next years (distance).
@@ -108,8 +108,15 @@ def zone2offset(tz, dt):
dt: datetime instance for offset computation
"""
if isinstance(tz, basestring):
- # [+-]1 * (hh*60 + mm)
- return int(tz[0]+'1') * (int(tz[1:3])*60 + int(tz[3:5]))
+ if len(tz) <= 3: # short tz (hh only)
+ # [+-]hh --> [+-]hh*60
+ return int(tz)*60
+ if tz[3] != ':':
+ # [+-]hhmm --> [+-]1 * (hh*60 + mm)
+ return int(tz[0]+'1') * (int(tz[1:3])*60 + int(tz[3:5]))
+ else:
+ # [+-]hh:mm --> [+-]1 * (hh*60 + mm)
+ return int(tz[0]+'1') * (int(tz[1:3])*60 + int(tz[4:6]))
return tz
def reGroupDictStrptime(found_dict, msec=False, default_tz=None):
@@ -202,11 +209,7 @@ def reGroupDictStrptime(found_dict, msec=False, default_tz=None):
if z in ("Z", "UTC", "GMT"):
tzoffset = 0
else:
- tzoffset = int(z[1:3]) * 60 # Hours...
- if len(z)>3:
- tzoffset += int(z[-2:]) # ...and minutes
- if z.startswith("-"):
- tzoffset = -tzoffset
+ tzoffset = zone2offset(z, 0); # currently offset-based only
elif key == 'Z':
z = val
if z in ("UTC", "GMT"):
diff --git a/fail2ban/tests/datedetectortestcase.py b/fail2ban/tests/datedetectortestcase.py
index 6d728b5e..cbf1ac21 100644
--- a/fail2ban/tests/datedetectortestcase.py
+++ b/fail2ban/tests/datedetectortestcase.py
@@ -90,21 +90,31 @@ class DateDetectorTest(LogCaptureTestCase):
self.assertEqual(matchlog.group(1), 'Jan 23 21:59:59')
def testDefaultTimeZone(self):
- log = "2017-01-23 15:00:00"
dd = self.datedetector
- dd.default_tz='UTC+0300'; datelog, _ = dd.getTime(log)
- # so in UTC, it was noon
- self.assertEqual(datetime.datetime.utcfromtimestamp(datelog),
- datetime.datetime(2017, 1, 23, 12, 0, 0))
-
- dd.default_tz='UTC'; datelog, _ = dd.getTime(log)
- self.assertEqual(datetime.datetime.utcfromtimestamp(datelog),
- datetime.datetime(2017, 1, 23, 15, 0, 0))
- self.assertEqual(dd.default_tz, 0); # utc == 0
-
- dd.default_tz='UTC-0430'; datelog, _ = dd.getTime(log)
- self.assertEqual(datetime.datetime.utcfromtimestamp(datelog),
- datetime.datetime(2017, 1, 23, 19, 30, 0))
+ dt = datetime.datetime
+ logdt = "2017-01-23 15:00:00"
+ dtUTC = dt(2017, 1, 23, 15, 0)
+ for tz, log, desired in (
+ ('UTC+0300', logdt, dt(2017, 1, 23, 12, 0)), # so in UTC, it was noon
+ ('UTC', logdt, dtUTC), # UTC
+ ('UTC-0430', logdt, dt(2017, 1, 23, 19, 30)),
+ ('GMT+12', logdt, dt(2017, 1, 23, 3, 0)),
+ (None, logdt, dt(2017, 1, 23, 14, 0)), # default CET in our test-framework
+ ('UTC+0300', logdt+' GMT', dtUTC), # GMT wins
+ ('UTC', logdt+' GMT', dtUTC), # GMT wins
+ ('UTC-0430', logdt+' GMT', dtUTC), # GMT wins
+ (None, logdt+' GMT', dtUTC), # GMT wins
+ ('UTC', logdt+' -1045', dt(2017, 1, 24, 1, 45)), # -1045 wins
+ (None, logdt+' -10:45', dt(2017, 1, 24, 1, 45)), # -1045 wins
+ ('UTC', logdt+' +0945', dt(2017, 1, 23, 5, 15)), # +0945 wins
+ (None, logdt+' +09:45', dt(2017, 1, 23, 5, 15)), # +0945 wins
+ (None, logdt+' Z', dtUTC), # Z wins (UTC)
+ ):
+ logSys.debug('== test %r with TZ %r', log, tz)
+ dd.default_tz=tz; datelog, _ = dd.getTime(log)
+ val = dt.utcfromtimestamp(datelog)
+ self.assertEqual(val, desired,
+ "wrong offset %r != %r by %r with TZ %r (%r)" % (val, desired, log, tz, dd.default_tz))
self.assertRaises(ValueError, setattr, dd, 'default_tz', 'WRONG-TZ')
dd.default_tz = None
From 39c4acf6bd0726131bbc3848d514ec66341ce70f Mon Sep 17 00:00:00 2001
From: sebres <serg.brester@sebres.de>
Date: Fri, 9 Jun 2017 15:52:14 +0200
Subject: [PATCH 12/18] small amend white-spaces (no functional changes) + a
bit optimized `zone2offset`
---
fail2ban/server/strptime.py | 22 +++++++++++-----------
fail2ban/tests/datedetectortestcase.py | 10 +++++-----
2 files changed, 16 insertions(+), 16 deletions(-)
diff --git a/fail2ban/server/strptime.py b/fail2ban/server/strptime.py
index 56f1c753..2da27c58 100644
--- a/fail2ban/server/strptime.py
+++ b/fail2ban/server/strptime.py
@@ -107,17 +107,17 @@ def zone2offset(tz, dt):
been validated already)
dt: datetime instance for offset computation
"""
- if isinstance(tz, basestring):
- if len(tz) <= 3: # short tz (hh only)
- # [+-]hh --> [+-]hh*60
- return int(tz)*60
- if tz[3] != ':':
- # [+-]hhmm --> [+-]1 * (hh*60 + mm)
- return int(tz[0]+'1') * (int(tz[1:3])*60 + int(tz[3:5]))
- else:
- # [+-]hh:mm --> [+-]1 * (hh*60 + mm)
- return int(tz[0]+'1') * (int(tz[1:3])*60 + int(tz[4:6]))
- return tz
+ if isinstance(tz, int):
+ return tz
+ if len(tz) <= 3: # short tz (hh only)
+ # [+-]hh --> [+-]hh*60
+ return int(tz)*60
+ if tz[3] != ':':
+ # [+-]hhmm --> [+-]1 * (hh*60 + mm)
+ return (-1 if tz[0] == '-' else 1) * (int(tz[1:3])*60 + int(tz[3:5]))
+ else:
+ # [+-]hh:mm --> [+-]1 * (hh*60 + mm)
+ return (-1 if tz[0] == '-' else 1) * (int(tz[1:3])*60 + int(tz[4:6]))
def reGroupDictStrptime(found_dict, msec=False, default_tz=None):
"""Return time from dictionary of strptime fields
diff --git a/fail2ban/tests/datedetectortestcase.py b/fail2ban/tests/datedetectortestcase.py
index cbf1ac21..56970ac5 100644
--- a/fail2ban/tests/datedetectortestcase.py
+++ b/fail2ban/tests/datedetectortestcase.py
@@ -99,16 +99,16 @@ class DateDetectorTest(LogCaptureTestCase):
('UTC', logdt, dtUTC), # UTC
('UTC-0430', logdt, dt(2017, 1, 23, 19, 30)),
('GMT+12', logdt, dt(2017, 1, 23, 3, 0)),
- (None, logdt, dt(2017, 1, 23, 14, 0)), # default CET in our test-framework
+ (None, logdt, dt(2017, 1, 23, 14, 0)), # default CET in our test-framework
('UTC+0300', logdt+' GMT', dtUTC), # GMT wins
('UTC', logdt+' GMT', dtUTC), # GMT wins
('UTC-0430', logdt+' GMT', dtUTC), # GMT wins
- (None, logdt+' GMT', dtUTC), # GMT wins
+ (None, logdt+' GMT', dtUTC), # GMT wins
('UTC', logdt+' -1045', dt(2017, 1, 24, 1, 45)), # -1045 wins
- (None, logdt+' -10:45', dt(2017, 1, 24, 1, 45)), # -1045 wins
+ (None, logdt+' -10:45', dt(2017, 1, 24, 1, 45)), # -1045 wins
('UTC', logdt+' +0945', dt(2017, 1, 23, 5, 15)), # +0945 wins
- (None, logdt+' +09:45', dt(2017, 1, 23, 5, 15)), # +0945 wins
- (None, logdt+' Z', dtUTC), # Z wins (UTC)
+ (None, logdt+' +09:45', dt(2017, 1, 23, 5, 15)), # +0945 wins
+ (None, logdt+' Z', dtUTC), # Z wins (UTC)
):
logSys.debug('== test %r with TZ %r', log, tz)
dd.default_tz=tz; datelog, _ = dd.getTime(log)
From 030f89bf7a7877224e7095fdf34f898354b25f63 Mon Sep 17 00:00:00 2001
From: sebres <serg.brester@sebres.de>
Date: Fri, 9 Jun 2017 20:29:34 +0200
Subject: [PATCH 13/18] Implemented zone abbreviations (DST, etc.) and
abbr+-offset functionality (accept zones like 'CET+0100'), for the list of
abbreviations see strptime.TZ_STR; Tokens `%z` and `%Z` are more precise now;
Introduced new tokens `%Exz` and `%ExZ` that fully support zone abbreviations
and/or offset-based zones;
# TODO: because python currently does not support mixing of case-sensitive with case-insensitive matching,
# check how TZ (in uppercase) can be combined with %a/%b etc. (that are currently case-insensitive),
# to avoid invalid date-time recognition in strings like '11-Aug-2013 03:36:11.372 error ...'
# with wrong TZ "error", which is at least not backwards compatible.
# Hence %z currently match literal Z|UTC|GMT only (and offset-based), and %Exz - all zone abbreviations.
---
fail2ban/server/strptime.py | 114 ++++++++++++++++++++-----
fail2ban/tests/datedetectortestcase.py | 25 +++++-
2 files changed, 116 insertions(+), 23 deletions(-)
diff --git a/fail2ban/server/strptime.py b/fail2ban/server/strptime.py
index 2da27c58..dbf75d21 100644
--- a/fail2ban/server/strptime.py
+++ b/fail2ban/server/strptime.py
@@ -26,8 +26,9 @@ from _strptime import LocaleTime, TimeRE, _calc_julian_from_U_or_W
from .mytime import MyTime
locale_time = LocaleTime()
-timeRE = TimeRE()
-FIXED_OFFSET_TZ_RE = re.compile(r'(?:Z|UTC|GMT)?([+-]\d{2}(?::?\d{2})?)?$')
+
+TZ_ABBR_RE = r"[A-Z](?:[A-Z]{2,4})?"
+FIXED_OFFSET_TZ_RE = re.compile(r"(%s)?([+-][01]\d(?::?\d{2})?)?$" % (TZ_ABBR_RE,))
def _getYearCentRE(cent=(0,3), distance=3, now=(MyTime.now(), MyTime.alternateNow)):
""" Build century regex for last year and the next years (distance).
@@ -40,10 +41,20 @@ def _getYearCentRE(cent=(0,3), distance=3, now=(MyTime.now(), MyTime.alternateNo
exprset |= set( cent(now[1].year + i) for i in (-1, distance) )
return "(?:%s)" % "|".join(exprset) if len(exprset) > 1 else "".join(exprset)
-#todo: implement literal time zone support like CET, PST, PDT, etc (via pytz):
-#timeRE['z'] = r"%s?(?P<z>Z|[+-]\d{2}(?::?[0-5]\d)?|[A-Z]{3})?" % timeRE['Z']
-timeRE['Z'] = r"(?P<Z>[A-Z]{3,5})"
-timeRE['z'] = r"(?P<z>Z|UTC|GMT|[+-]\d{2}(?::?[0-5]\d)?)"
+timeRE = TimeRE()
+
+# TODO: because python currently does not support mixing of case-sensitive with case-insensitive matching,
+# check how TZ (in uppercase) can be combined with %a/%b etc. (that are currently case-insensitive),
+# to avoid invalid date-time recognition in strings like '11-Aug-2013 03:36:11.372 error ...'
+# with wrong TZ "error", which is at least not backwards compatible.
+# Hence %z currently match literal Z|UTC|GMT only (and offset-based), and %Exz - all zone abbreviations.
+timeRE['Z'] = r"(?P<Z>Z|[A-Z]{3,5})"
+timeRE['z'] = r"(?P<z>Z|UTC|GMT|[+-][01]\d(?::?\d{2})?)"
+
+# Note: this extended tokens supported zone abbreviations, but it can parse 1 or 3-5 char(s) in lowercase,
+# see todo above. Don't use them in default date-patterns (if not anchored, few precise resp. optional).
+timeRE['ExZ'] = r"(?P<Z>%s)" % (TZ_ABBR_RE,)
+timeRE['Exz'] = r"(?P<z>(?:%s)?[+-][01]\d(?::?\d{2})?|%s)" % (TZ_ABBR_RE, TZ_ABBR_RE)
# Extend build-in TimeRE with some exact patterns
# exact two-digit patterns:
@@ -82,20 +93,22 @@ def getTimePatternRE():
def validateTimeZone(tz):
- """Validate a timezone.
+ """Validate a timezone and convert it to offset if it can (offset-based TZ).
- For now this accepts only the UTC[+-]hhmm format (UTC has aliases GMT/Z and optional).
+ For now this accepts the UTC[+-]hhmm format (UTC has aliases GMT/Z and optional).
+ Additionally it accepts all zone abbreviations mentioned below in TZ_STR.
+ Note that currently this zone abbreviations are offset-based and used fixed
+ offset without automatically DST-switch (if CET used then no automatically CEST-switch).
+
In the future, it may be extended for named time zones (such as Europe/Paris)
- present on the system, if a suitable tz library is present.
+ present on the system, if a suitable tz library is present (pytz).
"""
if tz is None:
return None
m = FIXED_OFFSET_TZ_RE.match(tz)
if m is None:
raise ValueError("Unknown or unsupported time zone: %r" % tz)
- tz = m.group(1)
- if tz is None or tz == '': # UTC/GMT
- return 0; # fixed zero offzet
+ tz = m.groups()
return zone2offset(tz, 0)
def zone2offset(tz, dt):
@@ -103,21 +116,29 @@ def zone2offset(tz, dt):
Parameters
----------
- tz: symbolic timezone or offset (for now only [+-]hhmm is supported, and it's assumed to have
- been validated already)
- dt: datetime instance for offset computation
+ tz: symbolic timezone or offset (for now only TZA?([+-]hh:?mm?)? is supported,
+ as value are accepted:
+ int offset;
+ string in form like 'CET+0100' or 'UTC' or '-0400';
+ tuple (or list) in form (zone name, zone offset);
+ dt: datetime instance for offset computation (currently unused)
"""
if isinstance(tz, int):
return tz
- if len(tz) <= 3: # short tz (hh only)
+ if isinstance(tz, basestring):
+ return validateTimeZone(tz)
+ tz, tzo = tz
+ if tzo is None or tzo == '': # without offset
+ return TZ_ABBR_OFFS[tz]
+ if len(tzo) <= 3: # short tzo (hh only)
# [+-]hh --> [+-]hh*60
- return int(tz)*60
- if tz[3] != ':':
+ return TZ_ABBR_OFFS[tz] + int(tzo)*60
+ if tzo[3] != ':':
# [+-]hhmm --> [+-]1 * (hh*60 + mm)
- return (-1 if tz[0] == '-' else 1) * (int(tz[1:3])*60 + int(tz[3:5]))
+ return TZ_ABBR_OFFS[tz] + (-1 if tzo[0] == '-' else 1) * (int(tzo[1:3])*60 + int(tzo[3:5]))
else:
# [+-]hh:mm --> [+-]1 * (hh*60 + mm)
- return (-1 if tz[0] == '-' else 1) * (int(tz[1:3])*60 + int(tz[4:6]))
+ return TZ_ABBR_OFFS[tz] + (-1 if tzo[0] == '-' else 1) * (int(tzo[1:3])*60 + int(tzo[4:6]))
def reGroupDictStrptime(found_dict, msec=False, default_tz=None):
"""Return time from dictionary of strptime fields
@@ -275,3 +296,56 @@ def reGroupDictStrptime(found_dict, msec=False, default_tz=None):
if msec: # pragma: no cover - currently unused
tm += fraction/1000000.0
return tm
+
+
+TZ_ABBR_OFFS = {'':0, None:0}
+TZ_STR = '''
+ -12 Y
+ -11 X NUT SST
+ -10 W CKT HAST HST TAHT TKT
+ -9 V AKST GAMT GIT HADT HNY
+ -8 U AKDT CIST HAY HNP PST PT
+ -7 T HAP HNR MST PDT
+ -6 S CST EAST GALT HAR HNC MDT
+ -5 R CDT COT EASST ECT EST ET HAC HNE PET
+ -4 Q AST BOT CLT COST EDT FKT GYT HAE HNA PYT
+ -3 P ADT ART BRT CLST FKST GFT HAA PMST PYST SRT UYT WGT
+ -2 O BRST FNT PMDT UYST WGST
+ -1 N AZOT CVT EGT
+ 0 Z EGST GMT UTC WET WT
+ 1 A CET DFT WAT WEDT WEST
+ 2 B CAT CEDT CEST EET SAST WAST
+ 3 C EAT EEDT EEST IDT MSK
+ 4 D AMT AZT GET GST KUYT MSD MUT RET SAMT SCT
+ 5 E AMST AQTT AZST HMT MAWT MVT PKT TFT TJT TMT UZT YEKT
+ 6 F ALMT BIOT BTT IOT KGT NOVT OMST YEKST
+ 7 G CXT DAVT HOVT ICT KRAT NOVST OMSST THA WIB
+ 8 H ACT AWST BDT BNT CAST HKT IRKT KRAST MYT PHT SGT ULAT WITA WST
+ 9 I AWDT IRKST JST KST PWT TLT WDT WIT YAKT
+ 10 K AEST ChST PGT VLAT YAKST YAPT
+ 11 L AEDT LHDT MAGT NCT PONT SBT VLAST VUT
+ 12 M ANAST ANAT FJT GILT MAGST MHT NZST PETST PETT TVT WFT
+ 13 FJST NZDT
+ 11.5 NFT
+ 10.5 ACDT LHST
+ 9.5 ACST
+ 6.5 CCT MMT
+ 5.75 NPT
+ 5.5 SLT
+ 4.5 AFT IRDT
+ 3.5 IRST
+ -2.5 HAT NDT
+ -3.5 HNT NST NT
+ -4.5 HLV VET
+ -9.5 MART MIT
+'''
+
+def _init_TZ_ABBR():
+ """Initialized TZ_ABBR_OFFS dictionary (TZ -> offset in minutes)"""
+ for tzline in map(str.split, TZ_STR.split('\n')):
+ if not len(tzline): continue
+ tzoffset = int(float(tzline[0]) * 60)
+ for tz in tzline[1:]:
+ TZ_ABBR_OFFS[tz] = tzoffset
+
+_init_TZ_ABBR()
diff --git a/fail2ban/tests/datedetectortestcase.py b/fail2ban/tests/datedetectortestcase.py
index 56970ac5..02facf30 100644
--- a/fail2ban/tests/datedetectortestcase.py
+++ b/fail2ban/tests/datedetectortestcase.py
@@ -90,16 +90,32 @@ class DateDetectorTest(LogCaptureTestCase):
self.assertEqual(matchlog.group(1), 'Jan 23 21:59:59')
def testDefaultTimeZone(self):
- dd = self.datedetector
+ # use special date-pattern (with %Exz), because %z currently does not supported
+ # zone abbreviations except Z|UTC|GMT.
+ dd = DateDetector()
+ dd.appendTemplate('^%ExY-%Exm-%Exd %H:%M:%S(?: ?%Exz)?')
dt = datetime.datetime
logdt = "2017-01-23 15:00:00"
dtUTC = dt(2017, 1, 23, 15, 0)
for tz, log, desired in (
+ # no TZ in input-string:
('UTC+0300', logdt, dt(2017, 1, 23, 12, 0)), # so in UTC, it was noon
('UTC', logdt, dtUTC), # UTC
('UTC-0430', logdt, dt(2017, 1, 23, 19, 30)),
('GMT+12', logdt, dt(2017, 1, 23, 3, 0)),
(None, logdt, dt(2017, 1, 23, 14, 0)), # default CET in our test-framework
+ # CET:
+ ('CET', logdt, dt(2017, 1, 23, 14, 0)),
+ ('+0100', logdt, dt(2017, 1, 23, 14, 0)),
+ ('CEST-01', logdt, dt(2017, 1, 23, 14, 0)),
+ # CEST:
+ ('CEST', logdt, dt(2017, 1, 23, 13, 0)),
+ ('+0200', logdt, dt(2017, 1, 23, 13, 0)),
+ ('CET+01', logdt, dt(2017, 1, 23, 13, 0)),
+ ('CET+0100', logdt, dt(2017, 1, 23, 13, 0)),
+ # check offset in minutes:
+ ('CET+0130', logdt, dt(2017, 1, 23, 12, 30)),
+ # TZ in input-string have precedence:
('UTC+0300', logdt+' GMT', dtUTC), # GMT wins
('UTC', logdt+' GMT', dtUTC), # GMT wins
('UTC-0430', logdt+' GMT', dtUTC), # GMT wins
@@ -108,13 +124,16 @@ class DateDetectorTest(LogCaptureTestCase):
(None, logdt+' -10:45', dt(2017, 1, 24, 1, 45)), # -1045 wins
('UTC', logdt+' +0945', dt(2017, 1, 23, 5, 15)), # +0945 wins
(None, logdt+' +09:45', dt(2017, 1, 23, 5, 15)), # +0945 wins
- (None, logdt+' Z', dtUTC), # Z wins (UTC)
+ ('UTC+0300', logdt+' Z', dtUTC), # Z wins (UTC)
+ ('GMT+12', logdt+' CET', dt(2017, 1, 23, 14, 0)), # CET wins
+ ('GMT+12', logdt+' CEST', dt(2017, 1, 23, 13, 0)), # CEST wins
+ ('GMT+12', logdt+' CET+0130', dt(2017, 1, 23, 12, 30)), # CET+0130 wins
):
logSys.debug('== test %r with TZ %r', log, tz)
dd.default_tz=tz; datelog, _ = dd.getTime(log)
val = dt.utcfromtimestamp(datelog)
self.assertEqual(val, desired,
- "wrong offset %r != %r by %r with TZ %r (%r)" % (val, desired, log, tz, dd.default_tz))
+ "wrong offset %r != %r by %r with default TZ %r (%r)" % (val, desired, log, tz, dd.default_tz))
self.assertRaises(ValueError, setattr, dd, 'default_tz', 'WRONG-TZ')
dd.default_tz = None
From 12259bb3c7aac2e33b9cc8f601c0e556b5401bd5 Mon Sep 17 00:00:00 2001
From: Georges Racinet <gracinet@anybox.fr>
Date: Fri, 9 Jun 2017 20:39:03 +0200
Subject: [PATCH 14/18] man and ChangeLog for logtimezone
---
ChangeLog | 2 +-
man/jail.conf.5 | 19 +++++++++++++++++++
2 files changed, 20 insertions(+), 1 deletion(-)
diff --git a/ChangeLog b/ChangeLog
index cd4776ee..7c99da91 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -98,7 +98,7 @@ TODO: implementing of options resp. other tasks from PR #1346
e. g. by unban all, reload with removing action, stop, shutdown the system (gh-1743),
the actions having `actionflush` do not execute `actionunban` for each single ticket
* add new command `actionflush` default for several iptables/iptables-ipset actions (and common include);
-
+* add new jail option `logtimezone` to force the timezone on log lines that don't have an explicit one (gh-1773)
ver. 0.10.0-alpha-1 (2016/07/14) - ipv6-support-etc
-----------
diff --git a/man/jail.conf.5 b/man/jail.conf.5
index 5a75369c..a43346e1 100644
--- a/man/jail.conf.5
+++ b/man/jail.conf.5
@@ -177,6 +177,25 @@ Ensure syslog or the program that generates the log file isn't configured to com
.TP
.B logencoding
encoding of log files used for decoding. Default value of "auto" uses current system locale.
+.TP
+.B logtimezone
+Force the time zone for log lines that don't have one.
+
+If this option is not specified, log lines from which no explicit time zone has been found are interpreted by fail2ban in its own system time zone, and that may turn to be inappropriate. While the best practice is to configure the monitored applications to include explicit offsets, this option is meant to handle cases where that is not possible.
+
+The supported time zones in this option are those with fixed offset: Z, UTC[+-]hhmm (you can also use GMT as an alias to UTC).
+
+This option has no effect on log lines on which an explicit time zone has been found.
+Examples:
+
+.RS
+.nf
+ logtimezone = UTC
+ logtimezone = UTC+0200
+ logtimezone = GMT-0100
+.fi
+.RE
+
.TP
.B banaction
banning action (default iptables-multiport) typically specified in the \fI[DEFAULT]\fR section for all jails.
From 23c2d05250db5da1cf111da96115fce0e41c9ea3 Mon Sep 17 00:00:00 2001
From: "Serg G. Brester" <github@sebres.de>
Date: Fri, 9 Jun 2017 20:51:28 +0200
Subject: [PATCH 15/18] Update changelog (new enhancements from gh-1792)
---
ChangeLog | 23 +++++++++++++++++++----
1 file changed, 19 insertions(+), 4 deletions(-)
diff --git a/ChangeLog b/ChangeLog
index 7c99da91..a2eac62b 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -88,17 +88,32 @@ TODO: implementing of options resp. other tasks from PR #1346
the parsing of log-entries contain new-line chars (as single entry);
- if multiline regex however expected (by single-line parsing without buffering) - prefix `(?m)`
could be used in regex to enable it;
-* implemented execution of `actionstart` on demand (conditional), if action depends on `family` (gh-1742):
+* Implemented execution of `actionstart` on demand (conditional), if action depends on `family` (gh-1742):
- new action parameter `actionstart_on_demand` (bool) can be set to prevent/allow starting action
on demand (default retrieved automatically, if some conditional parameter `param?family=...`
presents in action properties), see `action.d/pf.conf` for example;
- additionally `actionstop` will be executed only for families previously executing `actionstart`
(starting on demand only)
-* introduced new command `actionflush`: executed in order to flush all bans at once
+* Introduced new command `actionflush`: executed in order to flush all bans at once
e. g. by unban all, reload with removing action, stop, shutdown the system (gh-1743),
the actions having `actionflush` do not execute `actionunban` for each single ticket
-* add new command `actionflush` default for several iptables/iptables-ipset actions (and common include);
-* add new jail option `logtimezone` to force the timezone on log lines that don't have an explicit one (gh-1773)
+* Add new command `actionflush` default for several iptables/iptables-ipset actions (and common include);
+* Add new jail option `logtimezone` to force the timezone on log lines that don't have an explicit one (gh-1773)
+* Implemented zone abbreviations (like CET, CEST, etc.) and abbr+-offset functionality (accept zones
+ like 'CET+0100'), for the list of abbreviations see strptime.TZ_STR;
+* Tokens `%z` and `%Z` are changed (more precise now);
+* Introduced new tokens `%Exz` and `%ExZ` that fully support zone abbreviations and/or offset-based
+ zones (implemented as enhancement using custom `datepattern`, because may be too dangerous for default
+ patterns and tokens like `%z`);
+ Note: the extended tokens supported zone abbreviations, but it can parse 1 or 3-5 char(s) in lowercase.
+ Don't use them in default date-patterns (if not anchored, few precise resp. optional).
+ Because python currently does not support mixing of case-sensitive with case-insensitive matching,
+ the TZ (in uppercase) cannot be combined with `%a`/`%b` etc (that are currently case-insensitive),
+ to avoid invalid date-time recognition in strings like '11-Aug-2013 03:36:11.372 error ...' with
+ wrong TZ "error".
+ Hence `%z` currently match literal Z|UTC|GMT only (and offset-based), and `%Exz` - all zone
+ abbreviations.
+
ver. 0.10.0-alpha-1 (2016/07/14) - ipv6-support-etc
-----------
From 2b358bc1a4c985c07120d8b4bc7e025339926bdc Mon Sep 17 00:00:00 2001
From: sebres <serg.brester@sebres.de>
Date: Thu, 15 Jun 2017 11:16:19 +0200
Subject: [PATCH 16/18] filter.d/apache-overflows.conf: rewritten without
end-anchor ($), because apache-log could contain very long URLs (and/or
referrer), the parsing of it anchored way may be very vulnerable (at least as
regards the system resources, see gh-1790).
---
config/filter.d/apache-overflows.conf | 6 +++++-
1 file changed, 5 insertions(+), 1 deletion(-)
diff --git a/config/filter.d/apache-overflows.conf b/config/filter.d/apache-overflows.conf
index 74e44b8e..02a2ef20 100644
--- a/config/filter.d/apache-overflows.conf
+++ b/config/filter.d/apache-overflows.conf
@@ -8,11 +8,15 @@ before = apache-common.conf
[Definition]
-failregex = ^%(_apache_error_client)s ((AH0013[456]: )?Invalid (method|URI) in request .*( - possible attempt to establish SSL connection on non-SSL port)?|(AH00565: )?request failed: URI too long \(longer than \d+\)|request failed: erroneous characters after protocol string: .*|AH00566: request failed: invalid characters in URI)(, referer: \S+)?$
+failregex = ^%(_apache_error_client)s (?:(?:AH0013[456]: )?Invalid (method|URI) in request\b|(?:AH00565: )?request failed: URI too long \(longer than \d+\)|request failed: erroneous characters after protocol string:|(?:AH00566: )?request failed: invalid characters in URI\b)
ignoreregex =
# DEV Notes:
+#
+# [sebres] Because this apache-log could contain very long URLs (and/or referrer),
+# the parsing of it anchored way may be very vulnerable (at least as regards
+# the system resources, see gh-1790). Thus rewritten without end-anchor ($).
#
# fgrep -r 'URI too long' httpd-2.*
# httpd-2.2.25/server/protocol.c: "request failed: URI too long (longer than %d)", r->server->limit_req_line);
From e1234a5249d7d9ca604970d80c54353a86b60aba Mon Sep 17 00:00:00 2001
From: sebres <serg.brester@sebres.de>
Date: Thu, 15 Jun 2017 11:40:07 +0200
Subject: [PATCH 17/18] ChangeLog update
---
ChangeLog | 3 +++
1 file changed, 3 insertions(+)
diff --git a/ChangeLog b/ChangeLog
index 5f472f81..e99bab04 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -17,6 +17,9 @@ releases.
### Fixes
* Fix for systemd-backend: fail2ban hits the ulimit (out of file descriptors), see gh-991.
Partially back-ported from v.0.10.
+* filter.d/apache-overflows.conf:
+ - Fixes resources greedy expression (see gh-1790);
+ - Rewritten without end-anchor ($), because of potential vulnerability on very long URLs.
### New Features
From fd32e908e31add6f5eed677ce2c2f3e77e50ee49 Mon Sep 17 00:00:00 2001
From: sebres <serg.brester@sebres.de>
Date: Thu, 15 Jun 2017 18:28:37 +0200
Subject: [PATCH 18/18] fixes restoring of tickets from database for jails with
persistent ban (if `bantime = -1`)
---
fail2ban/server/database.py | 2 +-
fail2ban/tests/databasetestcase.py | 13 +++++++++++++
2 files changed, 14 insertions(+), 1 deletion(-)
diff --git a/fail2ban/server/database.py b/fail2ban/server/database.py
index 77786f57..f4f9b6c2 100644
--- a/fail2ban/server/database.py
+++ b/fail2ban/server/database.py
@@ -593,7 +593,7 @@ class Fail2BanDb(object):
if ip is not None:
query += " AND ip=?"
queryArgs.append(ip)
- if forbantime is not None:
+ if forbantime not in (None, -1): # not specified or persistent (all)
query += " AND timeofban > ?"
queryArgs.append(fromtime - forbantime)
if ip is None:
diff --git a/fail2ban/tests/databasetestcase.py b/fail2ban/tests/databasetestcase.py
index b1c68a8f..cbfc1517 100644
--- a/fail2ban/tests/databasetestcase.py
+++ b/fail2ban/tests/databasetestcase.py
@@ -358,6 +358,19 @@ class DatabaseTest(LogCaptureTestCase):
self.assertEqual(len(tickets), 2)
ticket = self.db.getCurrentBans(jail=None, ip="127.0.0.1");
self.assertEqual(ticket.getIP(), "127.0.0.1")
+
+ # positive case (1 ticket not yet expired):
+ tickets = self.db.getCurrentBans(jail=self.jail, forbantime=15,
+ fromtime=MyTime.time())
+ self.assertEqual(len(tickets), 1)
+ # negative case (all are expired in 1year):
+ tickets = self.db.getCurrentBans(jail=self.jail, forbantime=15,
+ fromtime=MyTime.time() + MyTime.str2seconds("1year"))
+ self.assertEqual(len(tickets), 0)
+ # persistent bantime (-1), so never expired:
+ tickets = self.db.getCurrentBans(jail=self.jail, forbantime=-1,
+ fromtime=MyTime.time() + MyTime.str2seconds("1year"))
+ self.assertEqual(len(tickets), 2)
def testActionWithDB(self):
# test action together with database functionality