amend to #2387:

- common.conf: rewritten using section-based handling round about option logtype;
- option `logtype` extended with `rfc5424` to cover RFC 5424 log-format (see #2309);

config/filter.d/asterisk.conf

@@ -32,10 +32,6 @@ failregex = ^Registration from '[^']*' failed for '<HOST>(:\d+)?' - (?:Wrong pas
 # FreePBX (todo: make optional in v.0.10):
 #            ^(%(__prefix_line)s|\[\]\s*WARNING%(__pid_re)s:?(?:\[C-[\da-f]*\])? )[^:]+: Friendly Scanner from <HOST>$
 
-__extra_timestamp = (?:\[[^\]]+\]\s+)?
-
-__prefix_line_journal = %(known/__prefix_line_journal)s%(__extra_timestamp)s
-
 ignoreregex =
 
 datepattern = {^LN-BEG}
@@ -50,3 +46,10 @@ datepattern = {^LN-BEG}
 # main/logger.c:ast_log_vsyslog - "in {functionname}:" only occurs in syslog
 
 journalmatch = _SYSTEMD_UNIT=asterisk.service
+
+
+[lt_journal]
+
+# asterisk can log timestamp if logs into systemd-journal (optional part matching this timestamp, gh-2383):
+__extra_timestamp = (?:\[[^\]]+\]\s+)?
+__prefix_line = %(known/__prefix_line)s%(__extra_timestamp)s

config/filter.d/common.conf

@@ -10,6 +10,7 @@ after = common.local
 
 [DEFAULT]
 
+# Type of log-file resp. log-format (file, short, journal, rfc542):
 logtype = file
 
 # Daemon definition is to be specialized (if needed) in .conf file
@@ -57,20 +58,32 @@ __date_ambit = (?:\[\])
 #      [bsdverbose]? [hostname] [vserver tag] daemon_id spaces
 #
 # This can be optional (for instance if we match named native log files)
-__prefix_line = <__prefix_line_<logtype>>
-
-# Common line prefixes for logtype "file":
-__prefix_line_file = %(__date_ambit)s?\s*(?:%(__bsd_syslog_verbose)s\s+)?(?:%(__hostname)s\s+)?(?:%(__kernel_prefix)s\s+)?(?:%(__vserver)s\s+)?(?:%(__daemon_combs_re)s\s+)?(?:%(__daemon_extra_re)s\s+)?
-
-# Common (short) line prefix for logtype "journal" (corresponds output of formatJournalEntry):
-__prefix_line_short = \s*(?:%(__hostname)s\s+)?(?:%(_daemon)s%(__pid_re)s?:?\s+)?(?:%(__kernel_prefix)s\s+)?
-__prefix_line_journal = %(__prefix_line_short)s
+__prefix_line = <lt_<logtype>/__prefix_line>
 
 # PAM authentication mechanism check for failures, e.g.: pam_unix, pam_sss,
 # pam_ldap
 __pam_auth = pam_unix
 
 # standardly all formats using prefix have line-begin anchored date:
+datepattern = <lt_<logtype>/datepattern>
+
+[lt_file]
+# Common line prefixes for logtype "file":
+__prefix_line = %(__date_ambit)s?\s*(?:%(__bsd_syslog_verbose)s\s+)?(?:%(__hostname)s\s+)?(?:%(__kernel_prefix)s\s+)?(?:%(__vserver)s\s+)?(?:%(__daemon_combs_re)s\s+)?(?:%(__daemon_extra_re)s\s+)?
 datepattern = {^LN-BEG}
 
-# Author: Yaroslav Halchenko
+[lt_short]
+# Common (short) line prefix for logtype "journal" (corresponds output of formatJournalEntry):
+__prefix_line = \s*(?:%(__hostname)s\s+)?(?:%(_daemon)s%(__pid_re)s?:?\s+)?(?:%(__kernel_prefix)s\s+)?
+datepattern = %(lt_file/datepattern)s
+[lt_journal]
+__prefix_line = %(lt_short/__prefix_line)s
+datepattern = %(lt_short/datepattern)s
+
+[lt_rfc5424]
+# RFC 5424 log-format, see gh-2309:
+#__prefix_line = \s*<__hostname> <__daemon_re> \d+ \S+ \S+\s+
+__prefix_line = \s*<__hostname> <__daemon_re> \d+ \S+ (?:[^\[\]\s]+|(?:\[(?:[^\]"]*|"[^"]*")*\])+)\s+
+datepattern = ^<\d+>\d+\s+{DATE}
+
+# Author: Yaroslav Halchenko, Sergey G. Brester (aka sebres)

config/filter.d/sshd.conf

@@ -106,8 +106,6 @@ maxlines = 1
 
 journalmatch = _SYSTEMD_UNIT=sshd.service + _COMM=sshd
 
-datepattern = {^LN-BEG}
-
 # DEV Notes:
 #
 #   "Failed \S+ for .*? from <HOST>..." failregex uses non-greedy catch-all because

fail2ban/tests/files/logs/sshd

@@ -345,4 +345,12 @@ Nov 26 13:03:39 srv sshd[14738]: fatal: Unable to negotiate with 192.0.2.5 port
 # failJSON: { "time": "2004-11-26T16:47:51", "match": true , "host": "192.0.2.6", "desc": "Disconnected during preauth phase (in extra/aggressive mode)" }
 Nov 26 16:47:51 srv sshd[19320]: Disconnected from authenticating user root 192.0.2.6 port 33553 [preauth]
 
-# addFILE: "sshd-journal"
+# filterOptions: {"test.condition":"name=='sshd'", "logtype": "rfc5424"}
+
+# failJSON: { "time": "2019-07-08T23:40:16", "match": true , "host": "192.0.2.1", "desc": "RFC 5424 format (gh-2309)" }
+<38>1 2019-07-08T17:40:16.954167-04:00 hostname.example.com sshd 57915 - - Failed unknown for invalid user redmond from 192.0.2.1 port 59197 ssh2
+
+# failJSON: { "time": "2019-07-08T23:40:17", "match": true , "host": "192.0.2.2", "desc": "RFC 5424 format with extra data and brackets in value (gh-2309)" }
+<38>1 2019-07-08T17:40:17.954167-04:00 hostname.example.com sshd 55555 - [timeQuality tzKnown="1" isSynced="0"][xxx@123 test="[brackets]"] Failed unknown for invalid user redmond from 192.0.2.2 port 55555 ssh2
+
+# addFILE: "sshd-journal"