diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml new file mode 100644 index 00000000..7a1d31df --- /dev/null +++ b/.github/workflows/main.yml @@ -0,0 +1,66 @@ +name: CI + +# Controls when the action will run. Triggers the workflow on push or pull request +# events but only for the master branch +on: + push: + paths-ignore: + - 'doc/**' + - 'files/**' + - 'man/**' + pull_request: + paths-ignore: + - 'doc/**' + - 'files/**' + - 'man/**' + +# A workflow run is made up of one or more jobs that can run sequentially or in parallel +jobs: + # This workflow contains a single job called "build" + build: + # The type of runner that the job will run on + runs-on: ubuntu-20.04 + strategy: + matrix: + python-version: [2.7, 3.5, 3.6, 3.7, 3.8, 3.9, pypy2, pypy3] + fail-fast: false + # Steps represent a sequence of tasks that will be executed as part of the job + steps: + # Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it + - uses: actions/checkout@v2 + + - name: Set up Python + uses: actions/setup-python@v2 + with: + python-version: ${{ matrix.python-version }} + + - name: Python version + run: | + F2B_PY=$(python -c "import sys; print(sys.version)") + echo "Python: ${{ matrix.python-version }} -- $F2B_PY" + F2B_PY=${F2B_PY:0:1} + echo "Set F2B_PY=$F2B_PY" + echo "F2B_PY=$F2B_PY" >> $GITHUB_ENV + + - name: Install dependencies + run: | + python -m pip install --upgrade pip + if [[ "$F2B_PY" = 3 ]] && ! command -v 2to3x -v 2to3 > /dev/null; then + pip install 2to3 + fi + pip install systemd-python || echo 'systemd not available' + pip install pyinotify || echo 'inotify not available' + + - name: Before scripts + run: | + cd "$GITHUB_WORKSPACE" + # Manually execute 2to3 for now + if [[ "$F2B_PY" = 3 ]]; then echo "2to3 ..." && ./fail2ban-2to3; fi + # (debug) output current preferred encoding: + python -c 'import locale, sys; from fail2ban.helpers import PREFER_ENC; print(PREFER_ENC, locale.getpreferredencoding(), (sys.stdout and sys.stdout.encoding))' + + - name: Test suite + run: if [[ "$F2B_PY" = 2 ]]; then python setup.py test; else python bin/fail2ban-testcases --verbosity=2; fi + + #- name: Test initd scripts + # run: shellcheck -s bash -e SC1090,SC1091 files/debian-initd diff --git a/.travis.yml b/.travis.yml index 158cff99..064b678b 100644 --- a/.travis.yml +++ b/.travis.yml @@ -18,14 +18,14 @@ matrix: - python: 2.7 name: 2.7 (xenial) - python: pypy - dist: trusty - python: 3.3 dist: trusty - python: 3.4 - python: 3.5 - python: 3.6 - python: 3.7 - - python: 3.8-dev + - python: 3.8 + - python: 3.9-dev - python: pypy3.5 before_install: - echo "running under $TRAVIS_PYTHON_VERSION" @@ -69,8 +69,8 @@ script: - if [[ "$F2B_PY" = 3 ]]; then coverage run bin/fail2ban-testcases --verbosity=2; fi # Use $VENV_BIN (not python) or else sudo will always run the system's python (2.7) - sudo $VENV_BIN/pip install . - # Doc files should get installed on Travis under Linux (python >= 3.8 seem to use another path segment) - - if [[ $TRAVIS_PYTHON_VERSION < 3.8 ]]; then test -e /usr/share/doc/fail2ban/FILTERS; fi + # Doc files should get installed on Travis under Linux (some builds/python's seem to use another path segment) + - test -e /usr/share/doc/fail2ban/FILTERS && echo 'found' || echo 'not found' # Test initd script - shellcheck -s bash -e SC1090,SC1091 files/debian-initd after_success: diff --git a/ChangeLog b/ChangeLog index cc0c6608..0a601226 100644 --- a/ChangeLog +++ b/ChangeLog @@ -6,7 +6,7 @@ Fail2Ban: Changelog =================== -ver. 0.11.1 (2020/01/11) - this-is-the-way +ver. 0.11.2-dev (20??/??/??) - development edition ----------- ### Compatibility: @@ -37,6 +37,70 @@ ver. 0.11.1 (2020/01/11) - this-is-the-way - Since v0.10 fail2ban supports the matching of IPv6 addresses, but not all ban actions are IPv6-capable now. +### Fixes +* [stability] prevent race condition - no ban if filter (backend) is continuously busy if + too many messages will be found in log, e. g. initial scan of large log-file or journal (gh-2660) +* pyinotify-backend sporadically avoided initial scanning of log-file by start +* python 3.9 compatibility (and Travis CI support) +* restoring a large number (500+ depending on files ulimit) of current bans when using PyPy fixed +* manual ban is written to database, so can be restored by restart (gh-2647) +* `jail.conf`: don't specify `action` directly in jails (use `action_` or `banaction` instead) +* no mails-action added per default anymore (e. g. to allow that `action = %(action_mw)s` should be specified + per jail or in default section in jail.local), closes gh-2357 +* ensure we've unique action name per jail (also if parameter `actname` is not set but name deviates from standard name, gh-2686) +* don't use `%(banaction)s` interpolation because it can be complex value (containing `[...]` and/or quotes), + so would bother the action interpolation +* fixed type conversion in config readers (take place after all interpolations get ready), that allows to + specify typed parameters variable (as substitutions) as well as to supply it in other sections or as init parameters. +* `action.d/*-ipset*.conf`: several ipset actions fixed (no timeout per default anymore), so no discrepancy + between ipset and fail2ban (removal from ipset will be managed by fail2ban only, gh-2703) +* `action.d/cloudflare.conf`: fixed `actionunban` (considering new-line chars and optionally real json-parsing + with `jq`, gh-2140, gh-2656) +* `action.d/nftables.conf` (type=multiport only): fixed port range selector, replacing `:` with `-` (gh-2763) +* `action.d/firewallcmd-*.conf` (multiport only): fixed port range selector, replacing `:` with `-` (gh-2821) +* `action.d/bsd-ipfw.conf`: fixed selection of rule-no by large list or initial `lowest_rule_num` (gh-2836) +* `filter.d/common.conf`: avoid substitute of default values in related `lt_*` section, `__prefix_line` + should be interpolated in definition section (inside the filter-config, gh-2650) +* `filter.d/courier-smtp.conf`: prefregex extended to consider port in log-message (gh-2697) +* `filter.d/traefik-auth.conf`: filter extended with parameter mode (`normal`, `ddos`, `aggressive`) to handle + the match of username differently (gh-2693): + - `normal`: matches 401 with supplied username only + - `ddos`: matches 401 without supplied username only + - `aggressive`: matches 401 and any variant (with and without username) +* `filter.d/sshd.conf`: normalizing of user pattern in all RE's, allowing empty user (gh-2749) + +### New Features and Enhancements +* fail2ban-regex: + - speedup formatted output (bypass unneeded stats creation) + - extended with prefregex statistic + - more informative output for `datepattern` (e. g. set from filter) - pattern : description +* parsing of action in jail-configs considers space between action-names as separator also + (previously only new-line was allowed), for example `action = a b` would specify 2 actions `a` and `b` +* new filter and jail for GitLab recognizing failed application logins (gh-2689) +* new filter and jail for Grafana recognizing failed application logins (gh-2855) +* new filter and jail for SoftEtherVPN recognizing failed application logins (gh-2723) +* `filter.d/guacamole.conf` extended with `logging` parameter to follow webapp-logging if it's configured (gh-2631) +* `filter.d/bitwarden.conf` enhanced to support syslog (gh-2778) +* introduced new prefix `{UNB}` for `datepattern` to disable word boundaries in regex; +* datetemplate: improved anchor detection for capturing groups `(^...)`; +* datepattern: improved handling with wrong recognized timestamps (timezones, no datepattern, etc) + as well as some warnings signaling user about invalid pattern or zone (gh-2814): + - filter gets mode in-operation, which gets activated if filter starts processing of new messages; + in this mode a timestamp read from log-line that appeared recently (not an old line), deviating too much + from now (up too 24h), will be considered as now (assuming a timezone issue), so could avoid unexpected + bypass of failure (previously exceeding `findtime`); + - better interaction with non-matching optional datepattern or invalid timestamps; + - implements special datepattern `{NONE}` - allow to find failures totally without date-time in log messages, + whereas filter will use now as timestamp (gh-2802) +* performance optimization of `datepattern` (better search algorithm in datedetector, especially for single template); +* fail2ban-client: extended to unban IP range(s) by subnet (CIDR/mask) or hostname (DNS), gh-2791; +* extended capturing of alternate tags in filter, allowing combine of multiple groups to single tuple token with new tag + prefix `` with all value of `` tags (gh-2755) + + +ver. 0.11.1 (2020/01/11) - this-is-the-way +----------- + ### Fixes * purge database will be executed now (within observer). * restoring currently banned ip after service restart fixed diff --git a/MANIFEST b/MANIFEST index 3974184c..630df5ea 100644 --- a/MANIFEST +++ b/MANIFEST @@ -227,6 +227,8 @@ fail2ban/tests/clientreadertestcase.py fail2ban/tests/config/action.d/action.conf fail2ban/tests/config/action.d/brokenaction.conf fail2ban/tests/config/fail2ban.conf +fail2ban/tests/config/filter.d/checklogtype.conf +fail2ban/tests/config/filter.d/checklogtype_test.conf fail2ban/tests/config/filter.d/simple.conf fail2ban/tests/config/filter.d/test.conf fail2ban/tests/config/filter.d/test.local diff --git a/config/action.d/abuseipdb.conf b/config/action.d/abuseipdb.conf index 010af5b5..ed958c86 100644 --- a/config/action.d/abuseipdb.conf +++ b/config/action.d/abuseipdb.conf @@ -21,14 +21,13 @@ # # Example, for ssh bruteforce (in section [sshd] of `jail.local`): # action = %(known/action)s -# %(action_abuseipdb)s[abuseipdb_apikey="my-api-key", abuseipdb_category="18,22"] +# abuseipdb[abuseipdb_apikey="my-api-key", abuseipdb_category="18,22"] # -# See below for catagories. +# See below for categories. # -# Original Ref: https://wiki.shaunc.com/wikka.php?wakka=ReportingToAbuseIPDBWithFail2Ban # Added to fail2ban by Andrew James Collett (ajcollett) -## abuseIPDB Catagories, `the abuseipdb_category` MUST be set in the jail.conf action call. +## abuseIPDB Categories, `the abuseipdb_category` MUST be set in the jail.conf action call. # Example, for ssh bruteforce: action = %(action_abuseipdb)s[abuseipdb_category="18,22"] # ID Title Description # 3 Fraud Orders diff --git a/config/action.d/bsd-ipfw.conf b/config/action.d/bsd-ipfw.conf index 5116b0d8..444192d3 100644 --- a/config/action.d/bsd-ipfw.conf +++ b/config/action.d/bsd-ipfw.conf @@ -14,7 +14,10 @@ # Notes.: command executed on demand at the first ban (or at the start of Fail2Ban if actionstart_on_demand is set to false). # Values: CMD # -actionstart = ipfw show | fgrep -c -m 1 -s 'table()' > /dev/null 2>&1 || ( ipfw show | awk 'BEGIN { b = } { if ($1 < b) {} else if ($1 == b) { b = $1 + 1 } else { e = b } } END { if (e) exit e
else exit b }'; num=$?; ipfw -q add $num from table\(
\) to me ; echo $num > "" ) +actionstart = ipfw show | fgrep -c -m 1 -s 'table(
)' > /dev/null 2>&1 || ( + num=$(ipfw show | awk 'BEGIN { b = } { if ($1 == b) { b = $1 + 1 } } END { print b }'); + ipfw -q add "$num" from table\(
\) to me ; echo "$num" > "" + ) # Option: actionstop diff --git a/config/action.d/cloudflare.conf b/config/action.d/cloudflare.conf index 1c48a37f..361cb177 100644 --- a/config/action.d/cloudflare.conf +++ b/config/action.d/cloudflare.conf @@ -5,7 +5,7 @@ # # Please set jail.local's permission to 640 because it contains your CF API key. # -# This action depends on curl. +# This action depends on curl (and optionally jq). # Referenced from http://www.normyee.net/blog/2012/02/02/adding-cloudflare-support-to-fail2ban by NORM YEE # # To get your CloudFlare API Key: https://www.cloudflare.com/a/account/my-account @@ -43,9 +43,9 @@ actioncheck = # API v1 #actionban = curl -s -o /dev/null https://www.cloudflare.com/api_json.html -d 'a=ban' -d 'tkn=' -d 'email=' -d 'key=' # API v4 -actionban = curl -s -o /dev/null -X POST -H 'X-Auth-Email: ' -H 'X-Auth-Key: ' \ - -H 'Content-Type: application/json' -d '{ "mode": "block", "configuration": { "target": "ip", "value": "" } }' \ - https://api.cloudflare.com/client/v4/user/firewall/access_rules/rules +actionban = curl -s -o /dev/null -X POST <_cf_api_prms> \ + -d '{"mode":"block","configuration":{"target":"ip","value":""},"notes":"Fail2Ban "}' \ + <_cf_api_url> # Option: actionunban # Notes.: command executed when unbanning an IP. Take care that the @@ -58,9 +58,14 @@ actionban = curl -s -o /dev/null -X POST -H 'X-Auth-Email: ' -H 'X-Auth- # API v1 #actionunban = curl -s -o /dev/null https://www.cloudflare.com/api_json.html -d 'a=nul' -d 'tkn=' -d 'email=' -d 'key=' # API v4 -actionunban = curl -s -o /dev/null -X DELETE -H 'X-Auth-Email: ' -H 'X-Auth-Key: ' \ - https://api.cloudflare.com/client/v4/user/firewall/access_rules/rules/$(curl -s -X GET -H 'X-Auth-Email: ' -H 'X-Auth-Key: ' \ - 'https://api.cloudflare.com/client/v4/user/firewall/access_rules/rules?mode=block&configuration_target=ip&configuration_value=&page=1&per_page=1' | cut -d'"' -f6) +actionunban = id=$(curl -s -X GET <_cf_api_prms> \ + "<_cf_api_url>?mode=block&configuration_target=ip&configuration_value=&page=1&per_page=1¬es=Fail2Ban%%20" \ + | { jq -r '.result[0].id' 2>/dev/null || tr -d '\n' | sed -nE 's/^.*"result"\s*:\s*\[\s*\{\s*"id"\s*:\s*"([^"]+)".*$/\1/p'; }) + if [ -z "$id" ]; then echo ": id for cannot be found"; exit 0; fi; + curl -s -o /dev/null -X DELETE <_cf_api_prms> "<_cf_api_url>/$id" + +_cf_api_url = https://api.cloudflare.com/client/v4/user/firewall/access_rules/rules +_cf_api_prms = -H 'X-Auth-Email: ' -H 'X-Auth-Key: ' -H 'Content-Type: application/json' [Init] diff --git a/config/action.d/firewallcmd-ipset.conf b/config/action.d/firewallcmd-ipset.conf index a1065224..c89a0243 100644 --- a/config/action.d/firewallcmd-ipset.conf +++ b/config/action.d/firewallcmd-ipset.conf @@ -18,7 +18,7 @@ before = firewallcmd-common.conf [Definition] -actionstart = ipset create hash:ip timeout +actionstart = ipset create hash:ip timeout firewall-cmd --direct --add-rule filter 0 -m set --match-set src -j actionflush = ipset flush @@ -27,9 +27,9 @@ actionstop = firewall-cmd --direct --remove-rule filter 0 ipset destroy -actionban = ipset add timeout -exist +actionban = ipset add timeout -exist -actionprolong = %(actionban)s +# actionprolong = %(actionban)s actionunban = ipset del -exist @@ -42,11 +42,19 @@ actionunban = ipset del -exist # chain = INPUT_direct -# Option: default-timeout +# Option: default-ipsettime # Notes: specifies default timeout in seconds (handled default ipset timeout only) -# Values: [ NUM ] Default: 600 +# Values: [ NUM ] Default: 0 (no timeout, managed by fail2ban by unban) +default-ipsettime = 0 -default-timeout = 600 +# Option: ipsettime +# Notes: specifies ticket timeout (handled ipset timeout only) +# Values: [ NUM ] Default: 0 (managed by fail2ban by unban) +ipsettime = 0 + +# expresion to caclulate timeout from bantime, example: +# banaction = %(known/banaction)s[ipsettime=''] +timeout-bantime = $([ "" -le 2147483 ] && echo "" || echo 0) # Option: actiontype # Notes.: defines additions to the blocking rule @@ -63,7 +71,7 @@ allports = -p # Option: multiport # Notes.: addition to block access only to specific ports # Usage.: use in jail config: banaction = firewallcmd-ipset[actiontype=] -multiport = -p -m multiport --dports +multiport = -p -m multiport --dports "$(echo '' | sed s/:/-/g)" ipmset = f2b- familyopt = @@ -71,7 +79,7 @@ familyopt = [Init?family=inet6] ipmset = f2b-6 -familyopt = family inet6 +familyopt = family inet6 # DEV NOTES: diff --git a/config/action.d/firewallcmd-multiport.conf b/config/action.d/firewallcmd-multiport.conf index 81540e5b..0c401f1b 100644 --- a/config/action.d/firewallcmd-multiport.conf +++ b/config/action.d/firewallcmd-multiport.conf @@ -11,9 +11,9 @@ before = firewallcmd-common.conf actionstart = firewall-cmd --direct --add-chain filter f2b- firewall-cmd --direct --add-rule filter f2b- 1000 -j RETURN - firewall-cmd --direct --add-rule filter 0 -m conntrack --ctstate NEW -p -m multiport --dports -j f2b- + firewall-cmd --direct --add-rule filter 0 -m conntrack --ctstate NEW -p -m multiport --dports "$(echo '' | sed s/:/-/g)" -j f2b- -actionstop = firewall-cmd --direct --remove-rule filter 0 -m conntrack --ctstate NEW -p -m multiport --dports -j f2b- +actionstop = firewall-cmd --direct --remove-rule filter 0 -m conntrack --ctstate NEW -p -m multiport --dports "$(echo '' | sed s/:/-/g)" -j f2b- firewall-cmd --direct --remove-rules filter f2b- firewall-cmd --direct --remove-chain filter f2b- diff --git a/config/action.d/firewallcmd-new.conf b/config/action.d/firewallcmd-new.conf index b06f5ccd..7b08603c 100644 --- a/config/action.d/firewallcmd-new.conf +++ b/config/action.d/firewallcmd-new.conf @@ -10,9 +10,9 @@ before = firewallcmd-common.conf actionstart = firewall-cmd --direct --add-chain filter f2b- firewall-cmd --direct --add-rule filter f2b- 1000 -j RETURN - firewall-cmd --direct --add-rule filter 0 -m state --state NEW -p -m multiport --dports -j f2b- + firewall-cmd --direct --add-rule filter 0 -m state --state NEW -p -m multiport --dports "$(echo '' | sed s/:/-/g)" -j f2b- -actionstop = firewall-cmd --direct --remove-rule filter 0 -m state --state NEW -p -m multiport --dports -j f2b- +actionstop = firewall-cmd --direct --remove-rule filter 0 -m state --state NEW -p -m multiport --dports "$(echo '' | sed s/:/-/g)" -j f2b- firewall-cmd --direct --remove-rules filter f2b- firewall-cmd --direct --remove-chain filter f2b- diff --git a/config/action.d/firewallcmd-rich-logging.conf b/config/action.d/firewallcmd-rich-logging.conf index badfee83..21e45087 100644 --- a/config/action.d/firewallcmd-rich-logging.conf +++ b/config/action.d/firewallcmd-rich-logging.conf @@ -1,6 +1,6 @@ # Fail2Ban configuration file # -# Author: Donald Yandt +# Authors: Donald Yandt, Sergey G. Brester # # Because of the rich rule commands requires firewalld-0.3.1+ # This action uses firewalld rich-rules which gives you a cleaner iptables since it stores rules according to zones and not @@ -10,36 +10,15 @@ # # If you use the --permanent rule you get a xml file in /etc/firewalld/zones/.xml that can be shared and parsed easliy # -# Example commands to view rules: -# firewall-cmd [--zone=] --list-rich-rules -# firewall-cmd [--zone=] --list-all -# firewall-cmd [--zone=zone] --query-rich-rule='rule' +# This is an derivative of firewallcmd-rich-rules.conf, see there for details and other parameters. [INCLUDES] -before = firewallcmd-common.conf +before = firewallcmd-rich-rules.conf [Definition] -actionstart = - -actionstop = - -actioncheck = - -# you can also use zones and/or service names. -# -# zone example: -# firewall-cmd --zone= --add-rich-rule="rule family='' source address='' port port='' protocol='' log prefix='f2b-' level='' limit value='/m' " -# -# service name example: -# firewall-cmd --zone= --add-rich-rule="rule family='' source address='' service name='' log prefix='f2b-' level='' limit value='/m' " -# -# Because rich rules can only handle single or a range of ports we must split ports and execute the command for each port. Ports can be single and ranges separated by a comma or space for an example: http, https, 22-60, 18 smtp - -actionban = ports=""; for p in $(echo $ports | tr ", " " "); do firewall-cmd --add-rich-rule="rule family='' source address='' port port='$p' protocol='' log prefix='f2b-' level='' limit value='/m' "; done - -actionunban = ports=""; for p in $(echo $ports | tr ", " " "); do firewall-cmd --remove-rich-rule="rule family='' source address='' port port='$p' protocol='' log prefix='f2b-' level='' limit value='/m' "; done +rich-suffix = log prefix='f2b-' level='' limit value='/m' [Init] @@ -48,4 +27,3 @@ level = info # log rate per minute rate = 1 - diff --git a/config/action.d/firewallcmd-rich-rules.conf b/config/action.d/firewallcmd-rich-rules.conf index bed71797..803d7d12 100644 --- a/config/action.d/firewallcmd-rich-rules.conf +++ b/config/action.d/firewallcmd-rich-rules.conf @@ -35,8 +35,10 @@ actioncheck = # # Because rich rules can only handle single or a range of ports we must split ports and execute the command for each port. Ports can be single and ranges separated by a comma or space for an example: http, https, 22-60, 18 smtp -actionban = ports=""; for p in $(echo $ports | tr ", " " "); do firewall-cmd --add-rich-rule="rule family='' source address='' port port='$p' protocol='' "; done +fwcmd_rich_rule = rule family='' source address='' port port='$p' protocol='' %(rich-suffix)s + +actionban = ports="$(echo '' | sed s/:/-/g)"; for p in $(echo $ports | tr ", " " "); do firewall-cmd --add-rich-rule="%(fwcmd_rich_rule)s"; done -actionunban = ports=""; for p in $(echo $ports | tr ", " " "); do firewall-cmd --remove-rich-rule="rule family='' source address='' port port='$p' protocol='' "; done - +actionunban = ports="$(echo '' | sed s/:/-/g)"; for p in $(echo $ports | tr ", " " "); do firewall-cmd --remove-rich-rule="%(fwcmd_rich_rule)s"; done +rich-suffix = \ No newline at end of file diff --git a/config/action.d/iptables-ipset-proto6-allports.conf b/config/action.d/iptables-ipset-proto6-allports.conf index c851233c..67d7947b 100644 --- a/config/action.d/iptables-ipset-proto6-allports.conf +++ b/config/action.d/iptables-ipset-proto6-allports.conf @@ -26,7 +26,7 @@ before = iptables-common.conf # Notes.: command executed on demand at the first ban (or at the start of Fail2Ban if actionstart_on_demand is set to false). # Values: CMD # -actionstart = ipset create hash:ip timeout +actionstart = ipset create hash:ip timeout -I -m set --match-set src -j # Option: actionflush @@ -49,9 +49,9 @@ actionstop = -D -m set --match-set src -j timeout -exist +actionban = ipset add timeout -exist -actionprolong = %(actionban)s +# actionprolong = %(actionban)s # Option: actionunban # Notes.: command executed when unbanning an IP. Take care that the @@ -63,11 +63,19 @@ actionunban = ipset del -exist [Init] -# Option: default-timeout +# Option: default-ipsettime # Notes: specifies default timeout in seconds (handled default ipset timeout only) -# Values: [ NUM ] Default: 600 +# Values: [ NUM ] Default: 0 (no timeout, managed by fail2ban by unban) +default-ipsettime = 0 -default-timeout = 600 +# Option: ipsettime +# Notes: specifies ticket timeout (handled ipset timeout only) +# Values: [ NUM ] Default: 0 (managed by fail2ban by unban) +ipsettime = 0 + +# expresion to caclulate timeout from bantime, example: +# banaction = %(known/banaction)s[ipsettime=''] +timeout-bantime = $([ "" -le 2147483 ] && echo "" || echo 0) ipmset = f2b- familyopt = @@ -76,4 +84,4 @@ familyopt = [Init?family=inet6] ipmset = f2b-6 -familyopt = family inet6 +familyopt = family inet6 diff --git a/config/action.d/iptables-ipset-proto6.conf b/config/action.d/iptables-ipset-proto6.conf index 12c3ddd6..87601027 100644 --- a/config/action.d/iptables-ipset-proto6.conf +++ b/config/action.d/iptables-ipset-proto6.conf @@ -26,7 +26,7 @@ before = iptables-common.conf # Notes.: command executed on demand at the first ban (or at the start of Fail2Ban if actionstart_on_demand is set to false). # Values: CMD # -actionstart = ipset create hash:ip timeout +actionstart = ipset create hash:ip timeout -I -p -m multiport --dports -m set --match-set src -j # Option: actionflush @@ -49,9 +49,9 @@ actionstop = -D -p -m multiport --dports -m # Tags: See jail.conf(5) man page # Values: CMD # -actionban = ipset add timeout -exist +actionban = ipset add timeout -exist -actionprolong = %(actionban)s +# actionprolong = %(actionban)s # Option: actionunban # Notes.: command executed when unbanning an IP. Take care that the @@ -63,11 +63,19 @@ actionunban = ipset del -exist [Init] -# Option: default-timeout +# Option: default-ipsettime # Notes: specifies default timeout in seconds (handled default ipset timeout only) -# Values: [ NUM ] Default: 600 +# Values: [ NUM ] Default: 0 (no timeout, managed by fail2ban by unban) +default-ipsettime = 0 -default-timeout = 600 +# Option: ipsettime +# Notes: specifies ticket timeout (handled ipset timeout only) +# Values: [ NUM ] Default: 0 (managed by fail2ban by unban) +ipsettime = 0 + +# expresion to caclulate timeout from bantime, example: +# banaction = %(known/banaction)s[ipsettime=''] +timeout-bantime = $([ "" -le 2147483 ] && echo "" || echo 0) ipmset = f2b- familyopt = @@ -76,4 +84,4 @@ familyopt = [Init?family=inet6] ipmset = f2b-6 -familyopt = family inet6 +familyopt = family inet6 diff --git a/config/action.d/nftables.conf b/config/action.d/nftables.conf index c1fb8550..77cf3661 100644 --- a/config/action.d/nftables.conf +++ b/config/action.d/nftables.conf @@ -34,7 +34,7 @@ type = multiport rule_match-custom = rule_match-allports = meta l4proto \{ \} -rule_match-multiport = $proto dport \{ \} +rule_match-multiport = $proto dport \{ $(echo '' | sed s/:/-/g) \} match = > # Option: rule_stat diff --git a/config/action.d/nginx-block-map.conf b/config/action.d/nginx-block-map.conf index 0b6aa0ad..ee702907 100644 --- a/config/action.d/nginx-block-map.conf +++ b/config/action.d/nginx-block-map.conf @@ -103,6 +103,8 @@ actionstop = %(actionflush)s actioncheck = -actionban = echo "\\\\ 1;" >> '%(blck_lst_file)s'; %(blck_lst_reload)s +_echo_blck_row = printf '\%%s 1;\n' "" -actionunban = id=$(echo "" | sed -e 's/[]\/$*.^|[]/\\&/g'); sed -i "/^\\\\$id 1;$/d" %(blck_lst_file)s; %(blck_lst_reload)s +actionban = %(_echo_blck_row)s >> '%(blck_lst_file)s'; %(blck_lst_reload)s + +actionunban = id=$(%(_echo_blck_row)s | sed -e 's/[]\/$*.^|[]/\\&/g'); sed -i "/^$id$/d" %(blck_lst_file)s; %(blck_lst_reload)s diff --git a/config/action.d/shorewall-ipset-proto6.conf b/config/action.d/shorewall-ipset-proto6.conf index 45be0c0a..eacb53d9 100644 --- a/config/action.d/shorewall-ipset-proto6.conf +++ b/config/action.d/shorewall-ipset-proto6.conf @@ -51,7 +51,7 @@ # Values: CMD # actionstart = if ! ipset -quiet -name list f2b- >/dev/null; - then ipset -quiet -exist create f2b- hash:ip timeout ; + then ipset -quiet -exist create f2b- hash:ip timeout ; fi # Option: actionstop @@ -66,9 +66,9 @@ actionstop = ipset flush f2b- # Tags: See jail.conf(5) man page # Values: CMD # -actionban = ipset add f2b- timeout -exist +actionban = ipset add f2b- timeout -exist -actionprolong = %(actionban)s +# actionprolong = %(actionban)s # Option: actionunban # Notes.: command executed when unbanning an IP. Take care that the @@ -78,8 +78,16 @@ actionprolong = %(actionban)s # actionunban = ipset del f2b- -exist -# Option: default-timeout +# Option: default-ipsettime # Notes: specifies default timeout in seconds (handled default ipset timeout only) -# Values: [ NUM ] Default: 600 +# Values: [ NUM ] Default: 0 (no timeout, managed by fail2ban by unban) +default-ipsettime = 0 -default-timeout = 600 +# Option: ipsettime +# Notes: specifies ticket timeout (handled ipset timeout only) +# Values: [ NUM ] Default: 0 (managed by fail2ban by unban) +ipsettime = 0 + +# expresion to caclulate timeout from bantime, example: +# banaction = %(known/banaction)s[ipsettime=''] +timeout-bantime = $([ "" -le 2147483 ] && echo "" || echo 0) diff --git a/config/fail2ban.conf b/config/fail2ban.conf index ba0e9204..f3867839 100644 --- a/config/fail2ban.conf +++ b/config/fail2ban.conf @@ -19,7 +19,7 @@ # NOTICE # INFO # DEBUG -# Values: [ LEVEL ] Default: ERROR +# Values: [ LEVEL ] Default: INFO # loglevel = INFO diff --git a/config/filter.d/bitwarden.conf b/config/filter.d/bitwarden.conf index 29bd4be8..b0651c8e 100644 --- a/config/filter.d/bitwarden.conf +++ b/config/filter.d/bitwarden.conf @@ -2,5 +2,12 @@ # Detecting failed login attempts # Logged in bwdata/logs/identity/Identity/log.txt +[INCLUDES] +before = common.conf + [Definition] -failregex = ^\s*\[WRN\]\s+Failed login attempt(?:, 2FA invalid)?\. $ +_daemon = Bitwarden-Identity +failregex = ^%(__prefix_line)s\s*\[(?:W(?:RN|arning)|Bit\.Core\.[^\]]+)\]\s+Failed login attempt(?:, 2FA invalid)?\. $ + +# DEV Notes: +# __prefix_line can result to an empty string, so it can support syslog and non-syslog at once. diff --git a/config/filter.d/common.conf b/config/filter.d/common.conf index 16897c8e..13286038 100644 --- a/config/filter.d/common.conf +++ b/config/filter.d/common.conf @@ -25,7 +25,7 @@ __pid_re = (?:\[\d+\]) # Daemon name (with optional source_file:line or whatever) # EXAMPLES: pam_rhosts_auth, [sshd], pop(pam_unix) -__daemon_re = [\[\(]?%(_daemon)s(?:\(\S+\))?[\]\)]?:? +__daemon_re = [\[\(]?<_daemon>(?:\(\S+\))?[\]\)]?:? # extra daemon info # EXAMPLE: [ID 800047 auth.info] @@ -33,7 +33,7 @@ __daemon_extra_re = \[ID \d+ \S+\] # Combinations of daemon name and PID # EXAMPLES: sshd[31607], pop(pam_unix)[4920] -__daemon_combs_re = (?:%(__pid_re)s?:\s+%(__daemon_re)s|%(__daemon_re)s%(__pid_re)s?:?) +__daemon_combs_re = (?:<__pid_re>?:\s+<__daemon_re>|<__daemon_re><__pid_re>?:?) # Some messages have a kernel prefix with a timestamp # EXAMPLES: kernel: [769570.846956] @@ -69,12 +69,12 @@ datepattern = /datepattern> [lt_file] # Common line prefixes for logtype "file": -__prefix_line = %(__date_ambit)s?\s*(?:%(__bsd_syslog_verbose)s\s+)?(?:%(__hostname)s\s+)?(?:%(__kernel_prefix)s\s+)?(?:%(__vserver)s\s+)?(?:%(__daemon_combs_re)s\s+)?(?:%(__daemon_extra_re)s\s+)? +__prefix_line = <__date_ambit>?\s*(?:<__bsd_syslog_verbose>\s+)?(?:<__hostname>\s+)?(?:<__kernel_prefix>\s+)?(?:<__vserver>\s+)?(?:<__daemon_combs_re>\s+)?(?:<__daemon_extra_re>\s+)? datepattern = {^LN-BEG} [lt_short] # Common (short) line prefix for logtype "journal" (corresponds output of formatJournalEntry): -__prefix_line = \s*(?:%(__hostname)s\s+)?(?:%(_daemon)s%(__pid_re)s?:?\s+)?(?:%(__kernel_prefix)s\s+)? +__prefix_line = \s*(?:<__hostname>\s+)?(?:<_daemon><__pid_re>?:?\s+)?(?:<__kernel_prefix>\s+)? datepattern = %(lt_file/datepattern)s [lt_journal] __prefix_line = %(lt_short/__prefix_line)s diff --git a/config/filter.d/courier-smtp.conf b/config/filter.d/courier-smtp.conf index 888753c4..4b2b8d87 100644 --- a/config/filter.d/courier-smtp.conf +++ b/config/filter.d/courier-smtp.conf @@ -12,7 +12,7 @@ before = common.conf _daemon = courieresmtpd -prefregex = ^%(__prefix_line)serror,relay=,.+$ +prefregex = ^%(__prefix_line)serror,relay=,(?:port=\d+,)?.+$ failregex = ^[^:]*: 550 User (<.*> )?unknown\.?$ ^msg="535 Authentication failed\.",cmd:( AUTH \S+)?( [0-9a-zA-Z\+/=]+)?(?: \S+)$ diff --git a/config/filter.d/gitlab.conf b/config/filter.d/gitlab.conf new file mode 100644 index 00000000..0c614ae5 --- /dev/null +++ b/config/filter.d/gitlab.conf @@ -0,0 +1,6 @@ +# Fail2Ban filter for Gitlab +# Detecting unauthorized access to the Gitlab Web portal +# typically logged in /var/log/gitlab/gitlab-rails/application.log + +[Definition] +failregex = ^: Failed Login: username=.+ ip=$ diff --git a/config/filter.d/grafana.conf b/config/filter.d/grafana.conf new file mode 100644 index 00000000..e7f0f420 --- /dev/null +++ b/config/filter.d/grafana.conf @@ -0,0 +1,9 @@ +# Fail2Ban filter for Grafana +# Detecting unauthorized access +# Typically logged in /var/log/grafana/grafana.log + +[Init] +datepattern = ^t=%%Y-%%m-%%dT%%H:%%M:%%S%%z + +[Definition] +failregex = ^(?: lvl=err?or)? msg="Invalid username or password"(?: uname=(?:"[^"]+"|\S+)| error="[^"]+"| \S+=(?:\S*|"[^"]+"))* remote_addr=$ diff --git a/config/filter.d/guacamole.conf b/config/filter.d/guacamole.conf index 09b4e7b0..bc6dbea9 100644 --- a/config/filter.d/guacamole.conf +++ b/config/filter.d/guacamole.conf @@ -5,21 +5,47 @@ [Definition] -# Option: failregex -# Notes.: regex to match the password failures messages in the logfile. -# Values: TEXT -# +logging = catalina +failregex = /failregex> +maxlines = /maxlines> +datepattern = /datepattern> + +[L_catalina] + failregex = ^.*\nWARNING: Authentication attempt from for user "[^"]*" failed\.$ -# Option: ignoreregex -# Notes.: regex to ignore. If this regex matches, the line is ignored. -# Values: TEXT -# -ignoreregex = - -# "maxlines" is number of log lines to buffer for multi-line regex searches maxlines = 2 datepattern = ^%%b %%d, %%ExY %%I:%%M:%%S %%p ^WARNING:()** - {^LN-BEG} \ No newline at end of file + {^LN-BEG} + +[L_webapp] + +failregex = ^ \[\S+\] WARN \S+ - Authentication attempt from for user "[^"]+" failed. + +maxlines = 1 + +datepattern = ^%%H:%%M:%%S.%%f + +# DEV Notes: +# +# failregex is based on the default pattern given in Guacamole documentation : +# https://guacamole.apache.org/doc/gug/configuring-guacamole.html#webapp-logging +# +# The following logback.xml Guacamole configuration file can then be used accordingly : +# +# +# /var/log/guacamole.log +# +# /var/log/guacamole.%d.log.gz +# 32 +# +# +# %d{HH:mm:ss.SSS} [%thread] %-5level %logger{36} - %msg%n +# +# +# +# +# +# diff --git a/config/filter.d/monit.conf b/config/filter.d/monit.conf index b652a1f4..fdaee9c3 100644 --- a/config/filter.d/monit.conf +++ b/config/filter.d/monit.conf @@ -8,13 +8,17 @@ # common.local before = common.conf +# [DEFAULT] +# logtype = short + [Definition] _daemon = monit +_prefix = Warning|HttpRequest + # Regexp for previous (accessing monit httpd) and new (access denied) versions -failregex = ^\[\s*\]\s*error\s*:\s*Warning:\s+Client '' supplied (?:unknown user '[^']+'|wrong password for user '[^']*') accessing monit httpd$ - ^%(__prefix_line)s\w+: access denied -- client : (?:unknown user '[^']+'|wrong password for user '[^']*'|empty password)$ +failregex = ^%(__prefix_line)s(?:error\s*:\s+)?(?:%(_prefix)s):\s+(?:access denied\s+--\s+)?[Cc]lient '?'?(?:\s+supplied|\s*:)\s+(?:unknown user '[^']+'|wrong password for user '[^']*'|empty password) # Ignore login with empty user (first connect, no user specified) # ignoreregex = %(__prefix_line)s\w+: access denied -- client : (?:unknown user '') diff --git a/config/filter.d/mysqld-auth.conf b/config/filter.d/mysqld-auth.conf index 97b37920..930c9b5a 100644 --- a/config/filter.d/mysqld-auth.conf +++ b/config/filter.d/mysqld-auth.conf @@ -17,7 +17,7 @@ before = common.conf _daemon = mysqld -failregex = ^%(__prefix_line)s(?:(?:\d{6}|\d{4}-\d{2}-\d{2})[ T]\s?\d{1,2}:\d{2}:\d{2} )?(?:\d+ )?\[\w+\] (?:\[[^\]]+\] )*Access denied for user '[^']+'@'' (to database '[^']*'|\(using password: (YES|NO)\))*\s*$ +failregex = ^%(__prefix_line)s(?:(?:\d{6}|\d{4}-\d{2}-\d{2})[ T]\s?\d{1,2}:\d{2}:\d{2} )?(?:\d+ )?\[\w+\] (?:\[[^\]]+\] )*Access denied for user '[^']+'@'' (to database '[^']*'|\(using password: (YES|NO)\))*\s*$ ignoreregex = diff --git a/config/filter.d/postfix.conf b/config/filter.d/postfix.conf index 29866dfa..fb690fb0 100644 --- a/config/filter.d/postfix.conf +++ b/config/filter.d/postfix.conf @@ -37,7 +37,7 @@ mdre-rbl = ^RCPT from [^[]*\[\]%(_port)s: [45]54 [45]\.7\.1 Service unava mdpr-more = %(mdpr-normal)s mdre-more = %(mdre-normal)s -mdpr-ddos = lost connection after(?! DATA) [A-Z]+ +mdpr-ddos = (?:lost connection after(?! DATA) [A-Z]+|disconnect(?= from \S+(?: \S+=\d+)* auth=0/(?:[1-9]|\d\d+))) mdre-ddos = ^from [^[]*\[\]%(_port)s:? mdpr-extra = (?:%(mdpr-auth)s|%(mdpr-normal)s) diff --git a/config/filter.d/proftpd.conf b/config/filter.d/proftpd.conf index a7bd2837..d3f10b18 100644 --- a/config/filter.d/proftpd.conf +++ b/config/filter.d/proftpd.conf @@ -14,16 +14,15 @@ before = common.conf _daemon = proftpd -__suffix_failed_login = (User not authorized for login|No such user found|Incorrect password|Password expired|Account disabled|Invalid shell: '\S+'|User in \S+|Limit (access|configuration) denies login|Not a UserAlias|maximum login length exceeded).? +__suffix_failed_login = ([uU]ser not authorized for login|[nN]o such user found|[iI]ncorrect password|[pP]assword expired|[aA]ccount disabled|[iI]nvalid shell: '\S+'|[uU]ser in \S+|[lL]imit (access|configuration) denies login|[nN]ot a UserAlias|[mM]aximum login length exceeded) -prefregex = ^%(__prefix_line)s%(__hostname)s \(\S+\[\]\)[: -]+ (?:USER|SECURITY|Maximum).+$ +prefregex = ^%(__prefix_line)s%(__hostname)s \(\S+\[\]\)[: -]+ (?:USER|SECURITY|Maximum) .+$ -failregex = ^USER .*: no such user found from \S+ \[\S+\] to \S+:\S+ *$ - ^USER .* \(Login failed\): %(__suffix_failed_login)s\s*$ - ^SECURITY VIOLATION: .* login attempted\. *$ - ^Maximum login attempts \(\d+\) exceeded *$ +failregex = ^USER \S+|.*?(?: \(Login failed\))?: %(__suffix_failed_login)s + ^SECURITY VIOLATION: \S+|.*? login attempted + ^Maximum login attempts \(\d+\) exceeded ignoreregex = diff --git a/config/filter.d/sendmail-auth.conf b/config/filter.d/sendmail-auth.conf index 14995fed..84fcbdda 100644 --- a/config/filter.d/sendmail-auth.conf +++ b/config/filter.d/sendmail-auth.conf @@ -8,11 +8,14 @@ before = common.conf [Definition] _daemon = (?:sendmail|sm-(?:mta|acceptingconnections)) +# "\w{14,20}" will give support for IDs from 14 up to 20 characters long __prefix_line = %(known/__prefix_line)s(?:\w{14,20}: )? +addr = (?:IPv6:|) -# "w{14,20}" will give support for IDs from 14 up to 20 characters long -failregex = ^%(__prefix_line)s(\S+ )?\[(?:IPv6:|)\]( \(may be forged\))?: possible SMTP attack: command=AUTH, count=\d+$ +prefregex = ^%(__prefix_line)s.+$ +failregex = ^(\S+ )?\[%(addr)s\]( \(may be forged\))?: possible SMTP attack: command=AUTH, count=\d+$ + ^AUTH failure \(LOGIN\):(?: [^:]+:)? authentication failure: checkpass failed, user=(?:\S+|.*?), relay=(?:\S+ )?\[%(addr)s\](?: \(may be forged\))?$ ignoreregex = journalmatch = _SYSTEMD_UNIT=sendmail.service diff --git a/config/filter.d/sendmail-reject.conf b/config/filter.d/sendmail-reject.conf index ca171915..e8b766c5 100644 --- a/config/filter.d/sendmail-reject.conf +++ b/config/filter.d/sendmail-reject.conf @@ -21,19 +21,20 @@ before = common.conf _daemon = (?:(sm-(mta|acceptingconnections)|sendmail)) __prefix_line = %(known/__prefix_line)s(?:\w{14,20}: )? +addr = (?:IPv6:|) prefregex = ^%(__prefix_line)s.+$ -cmnfailre = ^ruleset=check_rcpt, arg1=(?P<\S+@\S+>), relay=(\S+ )?\[(?:IPv6:|)\](?: \(may be forged\))?, reject=(550 5\.7\.1 (?P=email)\.\.\. Relaying denied\. (IP name possibly forged \[(\d+\.){3}\d+\]|Proper authentication required\.|IP name lookup failed \[(\d+\.){3}\d+\])|553 5\.1\.8 (?P=email)\.\.\. Domain of sender address \S+ does not exist|550 5\.[71]\.1 (?P=email)\.\.\. (Rejected: .*|User unknown))$ - ^ruleset=check_relay, arg1=(?P\S+), arg2=(?:IPv6:|), relay=((?P=dom) )?\[(\d+\.){3}\d+\](?: \(may be forged\))?, reject=421 4\.3\.2 (Connection rate limit exceeded\.|Too many open connections\.)$ - ^rejecting commands from (\S* )?\[(?:IPv6:|)\] due to pre-greeting traffic after \d+ seconds$ - ^(?:\S+ )?\[(?:IPv6:|)\]: (?:(?i)expn|vrfy) \S+ \[rejected\]$ +cmnfailre = ^ruleset=check_rcpt, arg1=(?P<\S+@\S+>), relay=(\S+ )?\[%(addr)s\](?: \(may be forged\))?, reject=(550 5\.7\.1 (?P=email)\.\.\. Relaying denied\. (IP name possibly forged \[(\d+\.){3}\d+\]|Proper authentication required\.|IP name lookup failed \[(\d+\.){3}\d+\])|553 5\.1\.8 (?P=email)\.\.\. Domain of sender address \S+ does not exist|550 5\.[71]\.1 (?P=email)\.\.\. (Rejected: .*|User unknown))$ + ^ruleset=check_relay, arg1=(?P\S+), arg2=%(addr)s, relay=((?P=dom) )?\[(\d+\.){3}\d+\](?: \(may be forged\))?, reject=421 4\.3\.2 (Connection rate limit exceeded\.|Too many open connections\.)$ + ^rejecting commands from (\S* )?\[%(addr)s\] due to pre-greeting traffic after \d+ seconds$ + ^(?:\S+ )?\[%(addr)s\]: (?:(?i)expn|vrfy) \S+ \[rejected\]$ ^<[^@]+@[^>]+>\.\.\. No such user here$ - ^from=<[^@]+@[^>]+>, size=\d+, class=\d+, nrcpts=\d+, bodytype=\w+, proto=E?SMTP, daemon=MTA, relay=\S+ \[(?:IPv6:|)\]$ + ^from=<[^@]+@[^>]+>, size=\d+, class=\d+, nrcpts=\d+, bodytype=\w+, proto=E?SMTP, daemon=MTA, relay=\S+ \[%(addr)s\]$ mdre-normal = -mdre-extra = ^(?:\S+ )?\[(?:IPv6:|)\](?: \(may be forged\))? did not issue (?:[A-Z]{4}[/ ]?)+during connection to (?:TLS)?M(?:TA|S[PA])(?:-\w+)?$ +mdre-extra = ^(?:\S+ )?\[%(addr)s\](?: \(may be forged\))? did not issue \S+ during connection mdre-aggressive = %(mdre-extra)s diff --git a/config/filter.d/softethervpn.conf b/config/filter.d/softethervpn.conf new file mode 100644 index 00000000..f7e7c0c3 --- /dev/null +++ b/config/filter.d/softethervpn.conf @@ -0,0 +1,9 @@ +# Fail2Ban filter for SoftEtherVPN +# Detecting unauthorized access to SoftEtherVPN +# typically logged in /usr/local/vpnserver/security_log/*/sec.log, or in syslog, depending on configuration + +[INCLUDES] +before = common.conf + +[Definition] +failregex = ^%(__prefix_line)s(?:(?:\([\d\-]+ [\d:.]+\) )?: )?Connection "[^"]+": User authentication failed. The user name that has been provided was "(?:[^"]+|.+)", from \.$ diff --git a/config/filter.d/sshd.conf b/config/filter.d/sshd.conf index d764a076..e7942262 100644 --- a/config/filter.d/sshd.conf +++ b/config/filter.d/sshd.conf @@ -25,7 +25,7 @@ __pref = (?:(?:error|fatal): (?:PAM: )?)? __suff = (?: (?:port \d+|on \S+|\[preauth\])){0,3}\s* __on_port_opt = (?: (?:port \d+|on \S+)){0,2} # close by authenticating user: -__authng_user = (?: (?:invalid|authenticating) user \S+|.+?)? +__authng_user = (?: (?:invalid|authenticating) user \S+|.*?)? # for all possible (also future) forms of "no matching (cipher|mac|MAC|compression method|key exchange method|host key type) found", # see ssherr.c for all possible SSH_ERR_..._ALG_MATCH errors. @@ -40,39 +40,45 @@ prefregex = ^%(__prefix_line)s%(__pref)s.+.* from ( via \S+)?%(__suff)s$ ^User not known to the underlying authentication module for .* from %(__suff)s$ - ^Failed publickey for invalid user (?P\S+)|(?:(?! from ).)*? from %(__on_port_opt)s(?: ssh\d*)?(?(cond_user): |(?:(?:(?! from ).)*)$) - ^Failed \b(?!publickey)\S+ for (?Pinvalid user )?(?P\S+)|(?(cond_inv)(?:(?! from ).)*?|[^:]+) from %(__on_port_opt)s(?: ssh\d*)?(?(cond_user): |(?:(?:(?! from ).)*)$) + > + ^Failed for (?Pinvalid user )?(?P\S+)|(?(cond_inv)(?:(?! from ).)*?|[^:]+) from %(__on_port_opt)s(?: ssh\d*)?(?(cond_user): |(?:(?:(?! from ).)*)$) ^ROOT LOGIN REFUSED FROM ^[iI](?:llegal|nvalid) user .*? from %(__suff)s$ - ^User .+ from not allowed because not listed in AllowUsers%(__suff)s$ - ^User .+ from not allowed because listed in DenyUsers%(__suff)s$ - ^User .+ from not allowed because not in any group%(__suff)s$ + ^User \S+|.*? from not allowed because not listed in AllowUsers%(__suff)s$ + ^User \S+|.*? from not allowed because listed in DenyUsers%(__suff)s$ + ^User \S+|.*? from not allowed because not in any group%(__suff)s$ ^refused connect from \S+ \(\) ^Received disconnect from %(__on_port_opt)s:\s*3: .*: Auth fail%(__suff)s$ - ^User .+ from not allowed because a group is listed in DenyGroups%(__suff)s$ - ^User .+ from not allowed because none of user's groups are listed in AllowGroups%(__suff)s$ + ^User \S+|.*? from not allowed because a group is listed in DenyGroups%(__suff)s$ + ^User \S+|.*? from not allowed because none of user's groups are listed in AllowGroups%(__suff)s$ ^%(__pam_auth)s\(sshd:auth\):\s+authentication failure;(?:\s+(?:(?:logname|e?uid|tty)=\S*)){0,4}\s+ruser=\S*\s+rhost=(?:\s+user=\S*)?%(__suff)s$ - ^(error: )?maximum authentication attempts exceeded for .* from %(__on_port_opt)s(?: ssh\d*)?%(__suff)s$ - ^User .+ not allowed because account is locked%(__suff)s + ^maximum authentication attempts exceeded for .* from %(__on_port_opt)s(?: ssh\d*)?%(__suff)s$ + ^User \S+|.*? not allowed because account is locked%(__suff)s ^Disconnecting(?: from)?(?: (?:invalid|authenticating)) user \S+ %(__on_port_opt)s:\s*Change of username or service not allowed:\s*.*\[preauth\]\s*$ - ^Disconnecting: Too many authentication failures(?: for .+?)?%(__suff)s$ + ^Disconnecting: Too many authentication failures(?: for \S+|.*?)?%(__suff)s$ ^Received disconnect from %(__on_port_opt)s:\s*11: -other> ^Accepted \w+ for \S+ from (?:\s|$) +cmnfailed-any = \S+ +cmnfailed-ignore = \b(?!publickey)\S+ +cmnfailed-invalid = +cmnfailed-nofail = (?:publickey|\S+) +cmnfailed = > + mdre-normal = # used to differentiate "connection closed" with and without `[preauth]` (fail/nofail cases in ddos mode) mdre-normal-other = ^(Connection closed|Disconnected) (?:by|from)%(__authng_user)s (?:%(__suff)s|\s*)$ mdre-ddos = ^Did not receive identification string from + ^kex_exchange_identification: (?:[Cc]lient sent invalid protocol identifier|[Cc]onnection closed by remote host) ^Bad protocol version identification '.*' from - ^Connection reset by ^SSH: Server;Ltype: (?:Authname|Version|Kex);Remote: -\d+;[A-Z]\w+: ^Read from socket failed: Connection reset by peer # same as mdre-normal-other, but as failure (without ) and [preauth] only: -mdre-ddos-other = ^(Connection closed|Disconnected) (?:by|from)%(__authng_user)s %(__on_port_opt)s\s+\[preauth\]\s*$ +mdre-ddos-other = ^(Connection (?:closed|reset)|Disconnected) (?:by|from)%(__authng_user)s %(__on_port_opt)s\s+\[preauth\]\s*$ -mdre-extra = ^Received disconnect from %(__on_port_opt)s:\s*14: No supported authentication methods available +mdre-extra = ^Received disconnect from %(__on_port_opt)s:\s*14: No(?: supported)? authentication methods available ^Unable to negotiate with %(__on_port_opt)s: no matching <__alg_match> found. ^Unable to negotiate a <__alg_match> ^no matching <__alg_match> found: @@ -84,6 +90,17 @@ mdre-aggressive = %(mdre-ddos)s # mdre-extra-other is fully included within mdre-ddos-other: mdre-aggressive-other = %(mdre-ddos-other)s +# Parameter "publickey": nofail (default), invalid, any, ignore +publickey = nofail +# consider failed publickey for invalid users only: +cmnfailre-failed-pub-invalid = ^Failed publickey for invalid user (?P\S+)|(?:(?! from ).)*? from %(__on_port_opt)s(?: ssh\d*)?(?(cond_user): |(?:(?:(?! from ).)*)$) +# consider failed publickey for valid users too (don't need RE, see cmnfailed): +cmnfailre-failed-pub-any = +# same as invalid, but consider failed publickey for valid users too, just as no failure (helper to get IP and user-name only, see cmnfailed): +cmnfailre-failed-pub-nofail = +# don't consider failed publickey as failures (don't need RE, see cmnfailed): +cmnfailre-failed-pub-ignore = + cfooterre = ^Connection from failregex = %(cmnfailre)s diff --git a/config/filter.d/traefik-auth.conf b/config/filter.d/traefik-auth.conf index 8321a138..8022fee1 100644 --- a/config/filter.d/traefik-auth.conf +++ b/config/filter.d/traefik-auth.conf @@ -51,6 +51,26 @@ [Definition] -failregex = ^ \- (?!- )\S+ \[\] \"(GET|POST|HEAD) [^\"]+\" 401\b +# Parameter "method" can be used to specifiy request method +req-method = \S+ +# Usage example (for jail.local): +# filter = traefik-auth[req-method="GET|POST|HEAD"] + +failregex = ^ \- > \[\] \"(?:) [^\"]+\" 401\b ignoreregex = + +# Parameter "mode": normal (default), ddos or aggressive +# Usage example (for jail.local): +# [traefik-auth] +# mode = aggressive +# # or another jail (rewrite filter parameters of jail): +# [traefik-auth-ddos] +# filter = traefik-auth[mode=ddos] +# +mode = normal + +# part of failregex matches user name (must be available in normal mode, must be empty in ddos mode, and both for aggressive mode): +usrre-normal = (?!- )\S+ +usrre-ddos = - +usrre-aggressive = \S+ \ No newline at end of file diff --git a/config/jail.conf b/config/jail.conf index 60131cef..e6961a18 100644 --- a/config/jail.conf +++ b/config/jail.conf @@ -52,7 +52,7 @@ before = paths-debian.conf # to prevent "clever" botnets calculate exact time IP can be unbanned again: #bantime.rndtime = -# "bantime.maxtime" is the max number of seconds using the ban time can reach (don't grows further) +# "bantime.maxtime" is the max number of seconds using the ban time can reach (doesn't grow further) #bantime.maxtime = # "bantime.factor" is a coefficient to calculate exponent growing of the formula or common multiplier, @@ -60,7 +60,7 @@ before = paths-debian.conf # grows by 1, 2, 4, 8, 16 ... #bantime.factor = 1 -# "bantime.formula" used by default to calculate next value of ban time, default value bellow, +# "bantime.formula" used by default to calculate next value of ban time, default value below, # the same ban time growing will be reached by multipliers 1, 2, 4, 8, 16, 32... #bantime.formula = ban.Time * (1<<(ban.Count if ban.Count<20 else 20)) * banFactor # @@ -209,28 +209,28 @@ banaction = iptables-multiport banaction_allports = iptables-allports # The simplest action to take: ban only -action_ = %(banaction)s[name=%(__name__)s, port="%(port)s", protocol="%(protocol)s", chain="%(chain)s"] +action_ = %(banaction)s[port="%(port)s", protocol="%(protocol)s", chain="%(chain)s"] # ban & send an e-mail with whois report to the destemail. -action_mw = %(banaction)s[name=%(__name__)s, port="%(port)s", protocol="%(protocol)s", chain="%(chain)s"] - %(mta)s-whois[name=%(__name__)s, sender="%(sender)s", dest="%(destemail)s", protocol="%(protocol)s", chain="%(chain)s"] +action_mw = %(action_)s + %(mta)s-whois[sender="%(sender)s", dest="%(destemail)s", protocol="%(protocol)s", chain="%(chain)s"] # ban & send an e-mail with whois report and relevant log lines # to the destemail. -action_mwl = %(banaction)s[name=%(__name__)s, port="%(port)s", protocol="%(protocol)s", chain="%(chain)s"] - %(mta)s-whois-lines[name=%(__name__)s, sender="%(sender)s", dest="%(destemail)s", logpath="%(logpath)s", chain="%(chain)s"] +action_mwl = %(action_)s + %(mta)s-whois-lines[sender="%(sender)s", dest="%(destemail)s", logpath="%(logpath)s", chain="%(chain)s"] # See the IMPORTANT note in action.d/xarf-login-attack for when to use this action # # ban & send a xarf e-mail to abuse contact of IP address and include relevant log lines # to the destemail. -action_xarf = %(banaction)s[name=%(__name__)s, port="%(port)s", protocol="%(protocol)s", chain="%(chain)s"] +action_xarf = %(action_)s xarf-login-attack[service=%(__name__)s, sender="%(sender)s", logpath="%(logpath)s", port="%(port)s"] # ban IP on CloudFlare & send an e-mail with whois report and relevant log lines # to the destemail. action_cf_mwl = cloudflare[cfuser="%(cfemail)s", cftoken="%(cfapikey)s"] - %(mta)s-whois-lines[name=%(__name__)s, sender="%(sender)s", dest="%(destemail)s", logpath="%(logpath)s", chain="%(chain)s"] + %(mta)s-whois-lines[sender="%(sender)s", dest="%(destemail)s", logpath="%(logpath)s", chain="%(chain)s"] # Report block via blocklist.de fail2ban reporting service API # @@ -240,7 +240,7 @@ action_cf_mwl = cloudflare[cfuser="%(cfemail)s", cftoken="%(cfapikey)s"] # in your `jail.local` globally (section [DEFAULT]) or per specific jail section (resp. in # corresponding jail.d/my-jail.local file). # -action_blocklist_de = blocklist_de[email="%(sender)s", service=%(filter)s, apikey="%(blocklist_de_apikey)s", agent="%(fail2ban_agent)s"] +action_blocklist_de = blocklist_de[email="%(sender)s", service="%(__name__)s", apikey="%(blocklist_de_apikey)s", agent="%(fail2ban_agent)s"] # Report ban via badips.com, and use as blacklist # @@ -371,7 +371,7 @@ maxretry = 1 [openhab-auth] filter = openhab -action = iptables-allports[name=NoAuthFailures] +banaction = %(banaction_allports)s logpath = /opt/openhab/logs/request.log @@ -478,6 +478,7 @@ backend = %(syslog_backend)s port = http,https logpath = /var/log/tomcat*/catalina.out +#logpath = /var/log/guacamole.log [monit] #Ban clients brute-forcing the monit gui login @@ -744,8 +745,8 @@ logpath = /var/log/named/security.log [nsd] port = 53 -action = %(banaction)s[name=%(__name__)s-tcp, port="%(port)s", protocol="tcp", chain="%(chain)s", actname=%(banaction)s-tcp] - %(banaction)s[name=%(__name__)s-udp, port="%(port)s", protocol="udp", chain="%(chain)s", actname=%(banaction)s-udp] +action_ = %(default/action_)s[name=%(__name__)s-tcp, protocol="tcp"] + %(default/action_)s[name=%(__name__)s-udp, protocol="udp"] logpath = /var/log/nsd.log @@ -756,9 +757,8 @@ logpath = /var/log/nsd.log [asterisk] port = 5060,5061 -action = %(banaction)s[name=%(__name__)s-tcp, port="%(port)s", protocol="tcp", chain="%(chain)s", actname=%(banaction)s-tcp] - %(banaction)s[name=%(__name__)s-udp, port="%(port)s", protocol="udp", chain="%(chain)s", actname=%(banaction)s-udp] - %(mta)s-whois[name=%(__name__)s, dest="%(destemail)s"] +action_ = %(default/action_)s[name=%(__name__)s-tcp, protocol="tcp"] + %(default/action_)s[name=%(__name__)s-udp, protocol="udp"] logpath = /var/log/asterisk/messages maxretry = 10 @@ -766,9 +766,8 @@ maxretry = 10 [freeswitch] port = 5060,5061 -action = %(banaction)s[name=%(__name__)s-tcp, port="%(port)s", protocol="tcp", chain="%(chain)s", actname=%(banaction)s-tcp] - %(banaction)s[name=%(__name__)s-udp, port="%(port)s", protocol="udp", chain="%(chain)s", actname=%(banaction)s-udp] - %(mta)s-whois[name=%(__name__)s, dest="%(destemail)s"] +action_ = %(default/action_)s[name=%(__name__)s-tcp, protocol="tcp"] + %(default/action_)s[name=%(__name__)s-udp, protocol="udp"] logpath = /var/log/freeswitch.log maxretry = 10 @@ -853,11 +852,23 @@ logpath = /var/log/ejabberd/ejabberd.log [counter-strike] logpath = /opt/cstrike/logs/L[0-9]*.log -# Firewall: http://www.cstrike-planet.com/faq/6 tcpport = 27030,27031,27032,27033,27034,27035,27036,27037,27038,27039 udpport = 1200,27000,27001,27002,27003,27004,27005,27006,27007,27008,27009,27010,27011,27012,27013,27014,27015 -action = %(banaction)s[name=%(__name__)s-tcp, port="%(tcpport)s", protocol="tcp", chain="%(chain)s", actname=%(banaction)s-tcp] - %(banaction)s[name=%(__name__)s-udp, port="%(udpport)s", protocol="udp", chain="%(chain)s", actname=%(banaction)s-udp] +action_ = %(default/action_)s[name=%(__name__)s-tcp, port="%(tcpport)s", protocol="tcp"] + %(default/action_)s[name=%(__name__)s-udp, port="%(udpport)s", protocol="udp"] + +[softethervpn] +port = 500,4500 +protocol = udp +logpath = /usr/local/vpnserver/security_log/*/sec.log + +[gitlab] +port = http,https +logpath = /var/log/gitlab/gitlab-rails/application.log + +[grafana] +port = http,https +logpath = /var/log/grafana/grafana.log [bitwarden] port = http,https @@ -909,8 +920,8 @@ findtime = 1 [murmur] # AKA mumble-server port = 64738 -action = %(banaction)s[name=%(__name__)s-tcp, port="%(port)s", protocol=tcp, chain="%(chain)s", actname=%(banaction)s-tcp] - %(banaction)s[name=%(__name__)s-udp, port="%(port)s", protocol=udp, chain="%(chain)s", actname=%(banaction)s-udp] +action_ = %(default/action_)s[name=%(__name__)s-tcp, protocol="tcp"] + %(default/action_)s[name=%(__name__)s-udp, protocol="udp"] logpath = /var/log/mumble-server/mumble-server.log diff --git a/fail2ban/client/actionreader.py b/fail2ban/client/actionreader.py index 80617a50..88b0aca1 100644 --- a/fail2ban/client/actionreader.py +++ b/fail2ban/client/actionreader.py @@ -38,28 +38,32 @@ class ActionReader(DefinitionInitConfigReader): _configOpts = { "actionstart": ["string", None], - "actionstart_on_demand": ["string", None], + "actionstart_on_demand": ["bool", None], "actionstop": ["string", None], "actionflush": ["string", None], "actionreload": ["string", None], "actioncheck": ["string", None], "actionrepair": ["string", None], - "actionrepair_on_unban": ["string", None], + "actionrepair_on_unban": ["bool", None], "actionban": ["string", None], "actionprolong": ["string", None], "actionreban": ["string", None], "actionunban": ["string", None], - "norestored": ["string", None], + "norestored": ["bool", None], } def __init__(self, file_, jailName, initOpts, **kwargs): + # always supply jail name as name parameter if not specified in options: + n = initOpts.get("name") + if n is None: + initOpts["name"] = n = jailName actname = initOpts.get("actname") if actname is None: actname = file_ + # ensure we've unique action name per jail: + if n != jailName: + actname += n[len(jailName):] if n.startswith(jailName) else '-' + n initOpts["actname"] = actname - # always supply jail name as name parameter if not specified in options: - if initOpts.get("name") is None: - initOpts["name"] = jailName self._name = actname DefinitionInitConfigReader.__init__( self, file_, jailName, initOpts, **kwargs) @@ -80,11 +84,6 @@ class ActionReader(DefinitionInitConfigReader): def convert(self): opts = self.getCombined( ignore=CommandAction._escapedTags | set(('timeout', 'bantime'))) - # type-convert only after combined (otherwise boolean converting prevents substitution): - for o in ('norestored', 'actionstart_on_demand', 'actionrepair_on_unban'): - if opts.get(o): - opts[o] = self._convert_to_boolean(opts[o]) - # stream-convert: head = ["set", self._jailName] stream = list() diff --git a/fail2ban/client/configparserinc.py b/fail2ban/client/configparserinc.py index e0f39579..cc4ada0a 100644 --- a/fail2ban/client/configparserinc.py +++ b/fail2ban/client/configparserinc.py @@ -29,7 +29,7 @@ import re import sys from ..helpers import getLogger -if sys.version_info >= (3,2): +if sys.version_info >= (3,): # pragma: 2.x no cover # SafeConfigParser deprecated from Python 3.2 (renamed to ConfigParser) from configparser import ConfigParser as SafeConfigParser, BasicInterpolation, \ @@ -61,7 +61,7 @@ if sys.version_info >= (3,2): return super(BasicInterpolationWithName, self)._interpolate_some( parser, option, accum, rest, section, map, *args, **kwargs) -else: # pragma: no cover +else: # pragma: 3.x no cover from ConfigParser import SafeConfigParser, \ InterpolationMissingOptionError, NoOptionError, NoSectionError @@ -372,7 +372,8 @@ after = 1.conf s2 = alls.get(n) if isinstance(s2, dict): # save previous known values, for possible using in local interpolations later: - self.merge_section('KNOWN/'+n, s2, '') + self.merge_section('KNOWN/'+n, + dict(filter(lambda i: i[0] in s, s2.iteritems())), '') # merge section s2.update(s) else: diff --git a/fail2ban/client/configreader.py b/fail2ban/client/configreader.py index 66b987b2..1b5a56a2 100644 --- a/fail2ban/client/configreader.py +++ b/fail2ban/client/configreader.py @@ -34,6 +34,30 @@ from ..helpers import getLogger, _as_bool, _merge_dicts, substituteRecursiveTags # Gets the instance of the logger. logSys = getLogger(__name__) +CONVERTER = { + "bool": _as_bool, + "int": int, +} +def _OptionsTemplateGen(options): + """Iterator over the options template with default options. + + Each options entry is composed of an array or tuple with: + [[type, name, ?default?], ...] + Or it is a dict: + {name: [type, default], ...} + """ + if isinstance(options, (list,tuple)): + for optname in options: + if len(optname) > 2: + opttype, optname, optvalue = optname + else: + (opttype, optname), optvalue = optname, None + yield opttype, optname, optvalue + else: + for optname in options: + opttype, optvalue = options[optname] + yield opttype, optname, optvalue + class ConfigReader(): """Generic config reader class. @@ -120,6 +144,10 @@ class ConfigReader(): except AttributeError: return False + def has_option(self, sec, opt, withDefault=True): + return self._cfg.has_option(sec, opt) if withDefault \ + else opt in self._cfg._sections.get(sec, {}) + def merge_defaults(self, d): self._cfg.get_defaults().update(d) @@ -224,31 +252,22 @@ class ConfigReaderUnshared(SafeConfigParserWithIncludes): # Or it is a dict: # {name: [type, default], ...} - def getOptions(self, sec, options, pOptions=None, shouldExist=False): + def getOptions(self, sec, options, pOptions=None, shouldExist=False, convert=True): values = dict() if pOptions is None: pOptions = {} # Get only specified options: - for optname in options: - if isinstance(options, (list,tuple)): - if len(optname) > 2: - opttype, optname, optvalue = optname - else: - (opttype, optname), optvalue = optname, None - else: - opttype, optvalue = options[optname] + for opttype, optname, optvalue in _OptionsTemplateGen(options): if optname in pOptions: continue try: - if opttype == "bool": - v = self.getboolean(sec, optname) - if v is None: continue - elif opttype == "int": - v = self.getint(sec, optname) - if v is None: continue - else: - v = self.get(sec, optname, vars=pOptions) + v = self.get(sec, optname, vars=pOptions) values[optname] = v + if convert: + conv = CONVERTER.get(opttype) + if conv: + if v is None: continue + values[optname] = conv(v) except NoSectionError as e: if shouldExist: raise @@ -261,8 +280,8 @@ class ConfigReaderUnshared(SafeConfigParserWithIncludes): logSys.warning("'%s' not defined in '%s'. Using default one: %r" % (optname, sec, optvalue)) values[optname] = optvalue - elif logSys.getEffectiveLevel() <= logLevel: - logSys.log(logLevel, "Non essential option '%s' not defined in '%s'.", optname, sec) + # elif logSys.getEffectiveLevel() <= logLevel: + # logSys.log(logLevel, "Non essential option '%s' not defined in '%s'.", optname, sec) except ValueError: logSys.warning("Wrong value for '" + optname + "' in '" + sec + "'. Using default one: '" + repr(optvalue) + "'") @@ -320,8 +339,9 @@ class DefinitionInitConfigReader(ConfigReader): pOpts = dict() if self._initOpts: pOpts = _merge_dicts(pOpts, self._initOpts) + # type-convert only in combined (otherwise int/bool converting prevents substitution): self._opts = ConfigReader.getOptions( - self, "Definition", self._configOpts, pOpts) + self, "Definition", self._configOpts, pOpts, convert=False) self._pOpts = pOpts if self.has_section("Init"): # get only own options (without options from default): @@ -342,10 +362,21 @@ class DefinitionInitConfigReader(ConfigReader): if opt == '__name__' or opt in self._opts: continue self._opts[opt] = self.get("Definition", opt) + def convertOptions(self, opts, configOpts): + """Convert interpolated combined options to expected type. + """ + for opttype, optname, optvalue in _OptionsTemplateGen(configOpts): + conv = CONVERTER.get(opttype) + if conv: + v = opts.get(optname) + if v is None: continue + try: + opts[optname] = conv(v) + except ValueError: + logSys.warning("Wrong %s value %r for %r. Using default one: %r", + opttype, v, optname, optvalue) + opts[optname] = optvalue - def _convert_to_boolean(self, value): - return _as_bool(value) - def getCombOption(self, optname): """Get combined definition option (as string) using pre-set and init options as preselection (values with higher precedence as specified in section). @@ -380,6 +411,8 @@ class DefinitionInitConfigReader(ConfigReader): ignore=ignore, addrepl=self.getCombOption) if not opts: raise ValueError('recursive tag definitions unable to be resolved') + # convert options after all interpolations: + self.convertOptions(opts, self._configOpts) return opts def convert(self): diff --git a/fail2ban/client/csocket.py b/fail2ban/client/csocket.py index ab3e294b..88795674 100644 --- a/fail2ban/client/csocket.py +++ b/fail2ban/client/csocket.py @@ -48,7 +48,8 @@ class CSocket: def send(self, msg, nonblocking=False, timeout=None): # Convert every list member to string obj = dumps(map(CSocket.convert, msg), HIGHEST_PROTOCOL) - self.__csock.send(obj + CSPROTO.END) + self.__csock.send(obj) + self.__csock.send(CSPROTO.END) return self.receive(self.__csock, nonblocking, timeout) def settimeout(self, timeout): @@ -81,9 +82,12 @@ class CSocket: msg = CSPROTO.EMPTY if nonblocking: sock.setblocking(0) if timeout: sock.settimeout(timeout) - while msg.rfind(CSPROTO.END) == -1: - chunk = sock.recv(512) - if chunk in ('', b''): # python 3.x may return b'' instead of '' - raise RuntimeError("socket connection broken") + bufsize = 1024 + while msg.rfind(CSPROTO.END, -32) == -1: + chunk = sock.recv(bufsize) + if not len(chunk): + raise socket.error(104, 'Connection reset by peer') + if chunk == CSPROTO.END: break msg = msg + chunk + if bufsize < 32768: bufsize <<= 1 return loads(msg) diff --git a/fail2ban/client/fail2banclient.py b/fail2ban/client/fail2banclient.py index 7c90ca40..6ea18fda 100755 --- a/fail2ban/client/fail2banclient.py +++ b/fail2ban/client/fail2banclient.py @@ -168,19 +168,6 @@ class Fail2banClient(Fail2banCmdLine, Thread): if not ret: return None - # verify that directory for the socket file exists - socket_dir = os.path.dirname(self._conf["socket"]) - if not os.path.exists(socket_dir): - logSys.error( - "There is no directory %s to contain the socket file %s." - % (socket_dir, self._conf["socket"])) - return None - if not os.access(socket_dir, os.W_OK | os.X_OK): # pragma: no cover - logSys.error( - "Directory %s exists but not accessible for writing" - % (socket_dir,)) - return None - # Check already running if not self._conf["force"] and os.path.exists(self._conf["socket"]): logSys.error("Fail2ban seems to be in unexpected state (not running but the socket exists)") diff --git a/fail2ban/client/fail2bancmdline.py b/fail2ban/client/fail2bancmdline.py index 1268ee9f..03683cad 100644 --- a/fail2ban/client/fail2bancmdline.py +++ b/fail2ban/client/fail2bancmdline.py @@ -27,15 +27,20 @@ import sys from ..version import version, normVersion from ..protocol import printFormatted -from ..helpers import getLogger, str2LogLevel, getVerbosityFormat +from ..helpers import getLogger, str2LogLevel, getVerbosityFormat, BrokenPipeError # Gets the instance of the logger. logSys = getLogger("fail2ban") def output(s): # pragma: no cover - print(s) + try: + print(s) + except (BrokenPipeError, IOError) as e: # pragma: no cover + if e.errno != 32: # closed / broken pipe + raise -CONFIG_PARAMS = ("socket", "pidfile", "logtarget", "loglevel", "syslogsocket",) +# Config parameters required to start fail2ban which can be also set via command line (overwrite fail2ban.conf), +CONFIG_PARAMS = ("socket", "pidfile", "logtarget", "loglevel", "syslogsocket") # Used to signal - we are in test cases (ex: prevents change logging params, log capturing, etc) PRODUCTION = True @@ -94,9 +99,10 @@ class Fail2banCmdLine(): output("and bans the corresponding IP addresses using firewall rules.") output("") output("Options:") - output(" -c configuration directory") - output(" -s socket path") - output(" -p pidfile path") + output(" -c, --conf configuration directory") + output(" -s, --socket socket path") + output(" -p, --pidfile pidfile path") + output(" --pname name of the process (main thread) to identify instance (default fail2ban-server)") output(" --loglevel logging level") output(" --logtarget logging target, use file-name or stdout, stderr, syslog or sysout.") output(" --syslogsocket auto|") @@ -129,17 +135,15 @@ class Fail2banCmdLine(): """ for opt in optList: o = opt[0] - if o == "-c": + if o in ("-c", "--conf"): self._conf["conf"] = opt[1] - elif o == "-s": + elif o in ("-s", "--socket"): self._conf["socket"] = opt[1] - elif o == "-p": + elif o in ("-p", "--pidfile"): self._conf["pidfile"] = opt[1] - elif o.startswith("--log") or o.startswith("--sys"): - self._conf[ o[2:] ] = opt[1] - elif o in ["-d", "--dp", "--dump-pretty"]: + elif o in ("-d", "--dp", "--dump-pretty"): self._conf["dump"] = True if o == "-d" else 2 - elif o == "-t" or o == "--test": + elif o in ("-t", "--test"): self.cleanConfOnly = True self._conf["test"] = True elif o == "-v": @@ -163,12 +167,14 @@ class Fail2banCmdLine(): from ..server.mytime import MyTime output(MyTime.str2seconds(opt[1])) return True - elif o in ["-h", "--help"]: + elif o in ("-h", "--help"): self.dispUsage() return True - elif o in ["-V", "--version"]: + elif o in ("-V", "--version"): self.dispVersion(o == "-V") return True + elif o.startswith("--"): # other long named params (see also resetConf) + self._conf[ o[2:] ] = opt[1] return None def initCmdLine(self, argv): @@ -185,6 +191,7 @@ class Fail2banCmdLine(): try: cmdOpts = 'hc:s:p:xfbdtviqV' cmdLongOpts = ['loglevel=', 'logtarget=', 'syslogsocket=', 'test', 'async', + 'conf=', 'pidfile=', 'pname=', 'socket=', 'timeout=', 'str2sec=', 'help', 'version', 'dp', '--dump-pretty'] optList, self._args = getopt.getopt(self._argv[1:], cmdOpts, cmdLongOpts) except getopt.GetoptError: @@ -227,7 +234,8 @@ class Fail2banCmdLine(): if not conf: self.configurator.readEarly() conf = self.configurator.getEarlyOptions() - self._conf[o] = conf[o] + if o in conf: + self._conf[o] = conf[o] logSys.info("Using socket file %s", self._conf["socket"]) @@ -304,18 +312,24 @@ class Fail2banCmdLine(): # since method is also exposed in API via globally bound variable @staticmethod def _exit(code=0): - if hasattr(os, '_exit') and os._exit: - os._exit(code) - else: - sys.exit(code) + # implicit flush without to produce broken pipe error (32): + sys.stderr.close() + try: + sys.stdout.flush() + # exit: + if hasattr(sys, 'exit') and sys.exit: + sys.exit(code) + else: + os._exit(code) + except (BrokenPipeError, IOError) as e: # pragma: no cover + if e.errno != 32: # closed / broken pipe + raise @staticmethod def exit(code=0): logSys.debug("Exit with code %s", code) # because of possible buffered output in python, we should flush it before exit: logging.shutdown() - sys.stdout.flush() - sys.stderr.flush() # exit Fail2banCmdLine._exit(code) diff --git a/fail2ban/client/fail2banregex.py b/fail2ban/client/fail2banregex.py index f6a4b141..e7a4e214 100644 --- a/fail2ban/client/fail2banregex.py +++ b/fail2ban/client/fail2banregex.py @@ -21,7 +21,6 @@ Fail2Ban reads log file that contains password failure report and bans the corresponding IP addresses using firewall rules. This tools can test regular expressions for "fail2ban". - """ __author__ = "Fail2Ban Developers" @@ -109,19 +108,22 @@ class _f2bOptParser(OptionParser): def format_help(self, *args, **kwargs): """ Overwritten format helper with full ussage.""" self.usage = '' - return "Usage: " + usage() + __doc__ + """ + return "Usage: " + usage() + "\n" + __doc__ + """ LOG: - string a string representing a log line - filename path to a log file (/var/log/auth.log) - "systemd-journal" search systemd journal (systemd-python required) + string a string representing a log line + filename path to a log file (/var/log/auth.log) + systemd-journal search systemd journal (systemd-python required), + optionally with backend parameters, see `man jail.conf` + for usage and examples (systemd-journal[journalflags=1]). REGEX: - string a string representing a 'failregex' - filename path to a filter file (filter.d/sshd.conf) + string a string representing a 'failregex' + filter name of filter, optionally with options (sshd[mode=aggressive]) + filename path to a filter file (filter.d/sshd.conf) IGNOREREGEX: - string a string representing an 'ignoreregex' - filename path to a filter file (filter.d/sshd.conf) + string a string representing an 'ignoreregex' + filename path to a filter file (filter.d/sshd.conf) \n""" + OptionParser.format_help(self, *args, **kwargs) + """\n Report bugs to https://github.com/fail2ban/fail2ban/issues\n """ + __copyright__ + "\n" @@ -252,6 +254,8 @@ class Fail2banRegex(object): self.share_config=dict() self._filter = Filter(None) + self._prefREMatched = 0 + self._prefREGroups = list() self._ignoreregex = list() self._failregex = list() self._time_elapsed = None @@ -272,6 +276,10 @@ class Fail2banRegex(object): self._filter.returnRawHost = opts.raw self._filter.checkFindTime = False self._filter.checkAllRegex = opts.checkAllRegex and not opts.out + # ignore pending (without ID/IP), added to matches if it hits later (if ID/IP can be retreved) + self._filter.ignorePending = opts.out + # callback to increment ignored RE's by index (during process): + self._filter.onIgnoreRegex = self._onIgnoreRegex self._backend = 'auto' def output(self, line): @@ -288,8 +296,8 @@ class Fail2banRegex(object): self._filter.setDatePattern(pattern) self._datepattern_set = True if pattern is not None: - self.output( "Use datepattern : %s" % ( - self._filter.getDatePattern()[1], ) ) + self.output( "Use datepattern : %s : %s" % ( + pattern, self._filter.getDatePattern()[1], ) ) def setMaxLines(self, v): if not self._maxlines_set: @@ -372,11 +380,8 @@ class Fail2banRegex(object): if not ret: output( "ERROR: failed to load filter %s" % value ) return False - # overwrite default logtype (considering that the filter could specify this too in Definition/Init sections): - if not fltOpt.get('logtype'): - reader.merge_defaults({ - 'logtype': ['file','journal'][int(self._backend.startswith("systemd"))] - }) + # set backend-related options (logtype): + reader.applyAutoOptions(self._backend) # get, interpolate and convert options: reader.getOptions(None) # show real options if expected: @@ -436,71 +441,140 @@ class Fail2banRegex(object): 'add%sRegex' % regextype.title())(regex.getFailRegex()) return True - def testIgnoreRegex(self, line): - found = False - try: - ret = self._filter.ignoreLine([(line, "", "")]) - if ret is not None: - found = True - regex = self._ignoreregex[ret].inc() - except RegexException as e: # pragma: no cover - output( 'ERROR: %s' % e ) - return False - return found + def _onIgnoreRegex(self, idx, ignoreRegex): + self._lineIgnored = True + self._ignoreregex[idx].inc() def testRegex(self, line, date=None): orgLineBuffer = self._filter._Filter__lineBuffer + # duplicate line buffer (list can be changed inplace during processLine): + if self._filter.getMaxLines() > 1: + orgLineBuffer = orgLineBuffer[:] fullBuffer = len(orgLineBuffer) >= self._filter.getMaxLines() - is_ignored = False + is_ignored = self._lineIgnored = False try: found = self._filter.processLine(line, date) lines = [] - line = self._filter.processedLine() ret = [] for match in found: - # Append True/False flag depending if line was matched by - # more than one regex - match.append(len(ret)>1) - regex = self._failregex[match[0]] - regex.inc() - regex.appendIP(match) + if not self._opts.out: + # Append True/False flag depending if line was matched by + # more than one regex + match.append(len(ret)>1) + regex = self._failregex[match[0]] + regex.inc() + regex.appendIP(match) if not match[3].get('nofail'): ret.append(match) else: is_ignored = True + if self._opts.out: # (formated) output - don't need stats: + return None, ret, None + # prefregex stats: + if self._filter.prefRegex: + pre = self._filter.prefRegex + if pre.hasMatched(): + self._prefREMatched += 1 + if self._verbose: + if len(self._prefREGroups) < self._maxlines: + self._prefREGroups.append(pre.getGroups()) + else: + if len(self._prefREGroups) == self._maxlines: + self._prefREGroups.append('...') except RegexException as e: # pragma: no cover output( 'ERROR: %s' % e ) - return False - for bufLine in orgLineBuffer[int(fullBuffer):]: - if bufLine not in self._filter._Filter__lineBuffer: - try: - self._line_stats.missed_lines.pop( - self._line_stats.missed_lines.index("".join(bufLine))) - if self._debuggex: - self._line_stats.missed_lines_timeextracted.pop( - self._line_stats.missed_lines_timeextracted.index( - "".join(bufLine[::2]))) - except ValueError: - pass - # if buffering - add also another lines from match: - if self._print_all_matched: - if not self._debuggex: - self._line_stats.matched_lines.append("".join(bufLine)) - else: - lines.append(bufLine[0] + bufLine[2]) - self._line_stats.matched += 1 - self._line_stats.missed -= 1 + return None, 0, None + if self._filter.getMaxLines() > 1: + for bufLine in orgLineBuffer[int(fullBuffer):]: + if bufLine not in self._filter._Filter__lineBuffer: + try: + self._line_stats.missed_lines.pop( + self._line_stats.missed_lines.index("".join(bufLine))) + if self._debuggex: + self._line_stats.missed_lines_timeextracted.pop( + self._line_stats.missed_lines_timeextracted.index( + "".join(bufLine[::2]))) + except ValueError: + pass + # if buffering - add also another lines from match: + if self._print_all_matched: + if not self._debuggex: + self._line_stats.matched_lines.append("".join(bufLine)) + else: + lines.append(bufLine[0] + bufLine[2]) + self._line_stats.matched += 1 + self._line_stats.missed -= 1 if lines: # pre-lines parsed in multiline mode (buffering) - lines.append(line) + lines.append(self._filter.processedLine()) line = "\n".join(lines) - return line, ret, is_ignored + return line, ret, (is_ignored or self._lineIgnored) + + def _prepaireOutput(self): + """Prepares output- and fetch-function corresponding given '--out' option (format)""" + ofmt = self._opts.out + if ofmt in ('id', 'ip'): + def _out(ret): + for r in ret: + output(r[1]) + elif ofmt == 'msg': + def _out(ret): + for r in ret: + for r in r[3].get('matches'): + if not isinstance(r, basestring): + r = ''.join(r for r in r) + output(r) + elif ofmt == 'row': + def _out(ret): + for r in ret: + output('[%r,\t%r,\t%r],' % (r[1],r[2],dict((k,v) for k, v in r[3].iteritems() if k != 'matches'))) + elif '<' not in ofmt: + def _out(ret): + for r in ret: + output(r[3].get(ofmt)) + else: # extended format with tags substitution: + from ..server.actions import Actions, CommandAction, BanTicket + def _escOut(t, v): + # use safe escape (avoid inject on pseudo tag "\x00msg\x00"): + if t not in ('msg',): + return v.replace('\x00', '\\x00') + return v + def _out(ret): + rows = [] + wrap = {'NL':0} + for r in ret: + ticket = BanTicket(r[1], time=r[2], data=r[3]) + aInfo = Actions.ActionInfo(ticket) + # if msg tag is used - output if single line (otherwise let it as is to wrap multilines later): + def _get_msg(self): + if not wrap['NL'] and len(r[3].get('matches', [])) <= 1: + return self['matches'] + else: # pseudo tag for future replacement: + wrap['NL'] = 1 + return "\x00msg\x00" + aInfo['msg'] = _get_msg + # not recursive interpolation (use safe escape): + v = CommandAction.replaceDynamicTags(ofmt, aInfo, escapeVal=_escOut) + if wrap['NL']: # contains multiline tags (msg): + rows.append((r, v)) + continue + output(v) + # wrap multiline tag (msg) interpolations to single line: + for r, v in rows: + for r in r[3].get('matches'): + if not isinstance(r, basestring): + r = ''.join(r for r in r) + r = v.replace("\x00msg\x00", r) + output(r) + return _out + def process(self, test_lines): t0 = time.time() + if self._opts.out: # get out function + out = self._prepaireOutput() for line in test_lines: if isinstance(line, tuple): - line_datetimestripped, ret, is_ignored = self.testRegex( - line[0], line[1]) + line_datetimestripped, ret, is_ignored = self.testRegex(line[0], line[1]) line = "".join(line[0]) else: line = line.rstrip('\r\n') @@ -508,8 +582,10 @@ class Fail2banRegex(object): # skip comment and empty lines continue line_datetimestripped, ret, is_ignored = self.testRegex(line) - if not is_ignored: - is_ignored = self.testIgnoreRegex(line_datetimestripped) + + if self._opts.out: # (formated) output: + if len(ret) > 0 and not is_ignored: out(ret) + continue if is_ignored: self._line_stats.ignored += 1 @@ -517,42 +593,25 @@ class Fail2banRegex(object): self._line_stats.ignored_lines.append(line) if self._debuggex: self._line_stats.ignored_lines_timeextracted.append(line_datetimestripped) - - if len(ret) > 0: - assert(not is_ignored) - if self._opts.out: - if self._opts.out in ('id', 'ip'): - for ret in ret: - output(ret[1]) - elif self._opts.out == 'msg': - for ret in ret: - output('\n'.join(map(lambda v:''.join(v for v in v), ret[3].get('matches')))) - elif self._opts.out == 'row': - for ret in ret: - output('[%r,\t%r,\t%r],' % (ret[1],ret[2],dict((k,v) for k, v in ret[3].iteritems() if k != 'matches'))) - else: - for ret in ret: - output(ret[3].get(self._opts.out)) - continue + elif len(ret) > 0: self._line_stats.matched += 1 if self._print_all_matched: self._line_stats.matched_lines.append(line) if self._debuggex: self._line_stats.matched_lines_timeextracted.append(line_datetimestripped) else: - if not is_ignored: - self._line_stats.missed += 1 - if not self._print_no_missed and (self._print_all_missed or self._line_stats.missed <= self._maxlines + 1): - self._line_stats.missed_lines.append(line) - if self._debuggex: - self._line_stats.missed_lines_timeextracted.append(line_datetimestripped) + self._line_stats.missed += 1 + if not self._print_no_missed and (self._print_all_missed or self._line_stats.missed <= self._maxlines + 1): + self._line_stats.missed_lines.append(line) + if self._debuggex: + self._line_stats.missed_lines_timeextracted.append(line_datetimestripped) self._line_stats.tested += 1 self._time_elapsed = time.time() - t0 def printLines(self, ltype): lstats = self._line_stats - assert(self._line_stats.missed == lstats.tested - (lstats.matched + lstats.ignored)) + assert(lstats.missed == lstats.tested - (lstats.matched + lstats.ignored)) lines = lstats[ltype] l = lstats[ltype + '_lines'] multiline = self._filter.getMaxLines() > 1 @@ -610,7 +669,18 @@ class Fail2banRegex(object): pprint_list(out, " #) [# of hits] regular expression") return total - # Print title + # Print prefregex: + if self._filter.prefRegex: + #self._filter.prefRegex.hasMatched() + pre = self._filter.prefRegex + out = [pre.getRegex()] + if self._verbose: + for grp in self._prefREGroups: + out.append(" %s" % (grp,)) + output( "\n%s: %d total" % ("Prefregex", self._prefREMatched) ) + pprint_list(out) + + # Print regex's: total = print_failregexes("Failregex", self._failregex) _ = print_failregexes("Ignoreregex", self._ignoreregex) @@ -689,10 +759,10 @@ class Fail2banRegex(object): test_lines = journal_lines_gen(flt, myjournal) else: # if single line parsing (without buffering) - if self._filter.getMaxLines() <= 1: + if self._filter.getMaxLines() <= 1 and '\n' not in cmd_log: self.output( "Use single line : %s" % shortstr(cmd_log.replace("\n", r"\n")) ) test_lines = [ cmd_log ] - else: # multi line parsing (with buffering) + else: # multi line parsing (with and without buffering) test_lines = cmd_log.split("\n") self.output( "Use multi line : %s line(s)" % len(test_lines) ) for i, l in enumerate(test_lines): @@ -712,6 +782,7 @@ class Fail2banRegex(object): def exec_command_line(*args): + logging.exitOnIOError = True parser = get_opt_parser() (opts, args) = parser.parse_args(*args) errors = [] diff --git a/fail2ban/client/filterreader.py b/fail2ban/client/filterreader.py index ede18dca..413f125e 100644 --- a/fail2ban/client/filterreader.py +++ b/fail2ban/client/filterreader.py @@ -53,6 +53,14 @@ class FilterReader(DefinitionInitConfigReader): def getFile(self): return self.__file + def applyAutoOptions(self, backend): + # set init option to backend-related logtype, considering + # that the filter settings may be overwritten in its local: + if (not self._initOpts.get('logtype') and + not self.has_option('Definition', 'logtype', False) + ): + self._initOpts['logtype'] = ['file','journal'][int(backend.startswith("systemd"))] + def convert(self): stream = list() opts = self.getCombined() diff --git a/fail2ban/client/jailreader.py b/fail2ban/client/jailreader.py index aceeeee0..50c1d047 100644 --- a/fail2ban/client/jailreader.py +++ b/fail2ban/client/jailreader.py @@ -149,11 +149,8 @@ class JailReader(ConfigReader): ret = self.__filter.read() if not ret: raise JailDefError("Unable to read the filter %r" % filterName) - if not filterOpt.get('logtype'): - # overwrite default logtype backend-related (considering that the filter settings may be overwritten): - self.__filter.merge_defaults({ - 'logtype': ['file','journal'][int(self.__opts.get('backend', '').startswith("systemd"))] - }) + # set backend-related options (logtype): + self.__filter.applyAutoOptions(self.__opts.get('backend', '')) # merge options from filter as 'known/...' (all options unfiltered): self.__filter.getOptions(self.__opts, all=True) ConfigReader.merge_section(self, self.__name, self.__filter.getCombined(), 'known/') diff --git a/fail2ban/helpers.py b/fail2ban/helpers.py index 213a405f..c45be849 100644 --- a/fail2ban/helpers.py +++ b/fail2ban/helpers.py @@ -208,6 +208,26 @@ class FormatterWithTraceBack(logging.Formatter): return logging.Formatter.format(self, record) +logging.exitOnIOError = False +def __stopOnIOError(logSys=None, logHndlr=None): # pragma: no cover + if logSys and len(logSys.handlers): + logSys.removeHandler(logSys.handlers[0]) + if logHndlr: + logHndlr.close = lambda: None + logging.StreamHandler.flush = lambda self: None + #sys.excepthook = lambda *args: None + if logging.exitOnIOError: + try: + sys.stderr.close() + except: + pass + sys.exit(0) + +try: + BrokenPipeError = BrokenPipeError +except NameError: # pragma: 3.x no cover + BrokenPipeError = IOError + __origLog = logging.Logger._log def __safeLog(self, level, msg, args, **kwargs): """Safe log inject to avoid possible errors by unsafe log-handlers, @@ -223,6 +243,10 @@ def __safeLog(self, level, msg, args, **kwargs): try: # if isEnabledFor(level) already called... __origLog(self, level, msg, args, **kwargs) + except (BrokenPipeError, IOError) as e: # pragma: no cover + if e.errno == 32: # closed / broken pipe + __stopOnIOError(self) + raise except Exception as e: # pragma: no cover - unreachable if log-handler safe in this python-version try: for args in ( @@ -237,6 +261,18 @@ def __safeLog(self, level, msg, args, **kwargs): pass logging.Logger._log = __safeLog +__origLogFlush = logging.StreamHandler.flush +def __safeLogFlush(self): + """Safe flush inject stopping endless logging on closed streams (redirected pipe). + """ + try: + __origLogFlush(self) + except (BrokenPipeError, IOError) as e: # pragma: no cover + if e.errno == 32: # closed / broken pipe + __stopOnIOError(None, self) + raise +logging.StreamHandler.flush = __safeLogFlush + def getLogger(name): """Get logging.Logger instance with Fail2Ban logger name convention """ @@ -267,7 +303,7 @@ def getVerbosityFormat(verbosity, fmt=' %(message)s', addtime=True, padding=True if addtime: fmt = ' %(asctime)-15s' + fmt else: # default (not verbose): - fmt = "%(name)-23.23s [%(process)d]: %(levelname)-7s" + fmt + fmt = "%(name)-24s[%(process)d]: %(levelname)-7s" + fmt if addtime: fmt = "%(asctime)s " + fmt # remove padding if not needed: @@ -291,7 +327,7 @@ def splitwords(s): """ if not s: return [] - return filter(bool, map(str.strip, re.split('[ ,\n]+', s))) + return filter(bool, map(lambda v: v.strip(), re.split('[ ,\n]+', s))) if sys.version_info >= (3,5): eval(compile(r'''if 1: @@ -338,7 +374,7 @@ OPTION_EXTRACT_CRE = re.compile( r'([\w\-_\.]+)=(?:"([^"]*)"|\'([^\']*)\'|([^,\]]*))(?:,|\]\s*\[|$)', re.DOTALL) # split by new-line considering possible new-lines within options [...]: OPTION_SPLIT_CRE = re.compile( - r'(?:[^\[\n]+(?:\s*\[\s*(?:[\w\-_\.]+=(?:"[^"]*"|\'[^\']*\'|[^,\]]*)\s*(?:,|\]\s*\[)?\s*)*\])?\s*|[^\n]+)(?=\n\s*|$)', re.DOTALL) + r'(?:[^\[\s]+(?:\s*\[\s*(?:[\w\-_\.]+=(?:"[^"]*"|\'[^\']*\'|[^,\]]*)\s*(?:,|\]\s*\[)?\s*)*\])?\s*|\S+)(?=\n\s*|\s+|$)', re.DOTALL) def extractOptions(option): match = OPTION_CRE.match(option) @@ -363,8 +399,8 @@ def splitWithOptions(option): # tags () in tagged options. # -# max tag replacement count: -MAX_TAG_REPLACE_COUNT = 10 +# max tag replacement count (considering tag X in tag Y repeat): +MAX_TAG_REPLACE_COUNT = 25 # compiled RE for tag name (replacement name) TAG_CRE = re.compile(r'<([^ <>]+)>') @@ -398,6 +434,7 @@ def substituteRecursiveTags(inptags, conditional='', done = set() noRecRepl = hasattr(tags, "getRawItem") # repeat substitution while embedded-recursive (repFlag is True) + repCounts = {} while True: repFlag = False # substitute each value: @@ -409,7 +446,7 @@ def substituteRecursiveTags(inptags, conditional='', value = orgval = uni_string(tags[tag]) # search and replace all tags within value, that can be interpolated using other tags: m = tre_search(value) - refCounts = {} + rplc = repCounts.get(tag, {}) #logSys.log(5, 'TAG: %s, value: %s' % (tag, value)) while m: # found replacement tag: @@ -419,13 +456,13 @@ def substituteRecursiveTags(inptags, conditional='', m = tre_search(value, m.end()) continue #logSys.log(5, 'found: %s' % rtag) - if rtag == tag or refCounts.get(rtag, 1) > MAX_TAG_REPLACE_COUNT: + if rtag == tag or rplc.get(rtag, 1) > MAX_TAG_REPLACE_COUNT: # recursive definitions are bad #logSys.log(5, 'recursion fail tag: %s value: %s' % (tag, value) ) raise ValueError( "properties contain self referencing definitions " "and cannot be resolved, fail tag: %s, found: %s in %s, value: %s" % - (tag, rtag, refCounts, value)) + (tag, rtag, rplc, value)) repl = None if conditional: repl = tags.get(rtag + '?' + conditional) @@ -445,7 +482,7 @@ def substituteRecursiveTags(inptags, conditional='', value = value.replace('<%s>' % rtag, repl) #logSys.log(5, 'value now: %s' % value) # increment reference count: - refCounts[rtag] = refCounts.get(rtag, 0) + 1 + rplc[rtag] = rplc.get(rtag, 0) + 1 # the next match for replace: m = tre_search(value, m.start()) #logSys.log(5, 'TAG: %s, newvalue: %s' % (tag, value)) @@ -453,6 +490,7 @@ def substituteRecursiveTags(inptags, conditional='', if orgval != value: # check still contains any tag - should be repeated (possible embedded-recursive substitution): if tre_search(value): + repCounts[tag] = rplc repFlag = True # copy return tags dict to prevent modifying of inptags: if id(tags) == id(inptags): diff --git a/fail2ban/protocol.py b/fail2ban/protocol.py index 6b33d30a..0a4c84ed 100644 --- a/fail2ban/protocol.py +++ b/fail2ban/protocol.py @@ -55,6 +55,8 @@ protocol = [ ["stop", "stops all jails and terminate the server"], ["unban --all", "unbans all IP addresses (in all jails and database)"], ["unban ... ", "unbans (in all jails and database)"], +["banned", "return jails with banned IPs as dictionary"], +["banned ... ]", "return list(s) of jails where given IP(s) are banned"], ["status", "gets the current status of the server"], ["ping", "tests if the server is alive"], ["echo", "for internal usage, returns back and outputs a given string"], @@ -120,6 +122,8 @@ protocol = [ ["set action ", "sets the of for the action for "], ["set action [ ]", "calls the with for the action for "], ['', "JAIL INFORMATION", ""], +["get banned", "return banned IPs of "], +["get banned ... ]", "return 1 if IP is banned in otherwise 0, or a list of 1/0 for multiple IPs"], ["get logpath", "gets the list of the monitored files for "], ["get logencoding", "gets the encoding of the log files for "], ["get journalmatch", "gets the journal filter match for "], diff --git a/fail2ban/server/action.py b/fail2ban/server/action.py index d7812b6d..f52e0878 100644 --- a/fail2ban/server/action.py +++ b/fail2ban/server/action.py @@ -404,10 +404,13 @@ class CommandAction(ActionBase): def _getOperation(self, tag, family): # replace operation tag (interpolate all values), be sure family is enclosed as conditional value # (as lambda in addrepl so only if not overwritten in action): - return self.replaceTag(tag, self._properties, + cmd = self.replaceTag(tag, self._properties, conditional=('family='+family if family else ''), - addrepl=(lambda tag:family if tag == 'family' else None), cache=self.__substCache) + if '<' not in cmd or not family: return cmd + # replace family as dynamic tags, important - don't cache, no recursion and auto-escape here: + cmd = self.replaceDynamicTags(cmd, {'family':family}) + return cmd def _operationExecuted(self, tag, family, *args): """ Get, set or delete command of operation considering family. @@ -452,7 +455,18 @@ class CommandAction(ActionBase): ret = True # avoid double execution of same command for both families: if cmd and cmd not in self._operationExecuted(tag, lambda f: f != famoper): - ret = self.executeCmd(cmd, self.timeout) + realCmd = cmd + if self._jail: + # simulate action info with "empty" ticket: + aInfo = getattr(self._jail.actions, 'actionInfo', None) + if not aInfo: + aInfo = self._jail.actions._getActionInfo(None) + setattr(self._jail.actions, 'actionInfo', aInfo) + aInfo['time'] = MyTime.time() + aInfo['family'] = famoper + # replace dynamical tags, important - don't cache, no recursion and auto-escape here + realCmd = self.replaceDynamicTags(cmd, aInfo) + ret = self.executeCmd(realCmd, self.timeout) res &= ret if afterExec: afterExec(famoper, ret) self._operationExecuted(tag, famoper, cmd if ret else None) @@ -806,7 +820,7 @@ class CommandAction(ActionBase): ESCAPE_VN_CRE = re.compile(r"\W") @classmethod - def replaceDynamicTags(cls, realCmd, aInfo): + def replaceDynamicTags(cls, realCmd, aInfo, escapeVal=None): """Replaces dynamical tags in `query` with property values. **Important** @@ -831,16 +845,17 @@ class CommandAction(ActionBase): # array for escaped vars: varsDict = dict() - def escapeVal(tag, value): - # if the value should be escaped: - if cls.ESCAPE_CRE.search(value): - # That one needs to be escaped since its content is - # out of our control - tag = 'f2bV_%s' % cls.ESCAPE_VN_CRE.sub('_', tag) - varsDict[tag] = value # add variable - value = '$'+tag # replacement as variable - # replacement for tag: - return value + if not escapeVal: + def escapeVal(tag, value): + # if the value should be escaped: + if cls.ESCAPE_CRE.search(value): + # That one needs to be escaped since its content is + # out of our control + tag = 'f2bV_%s' % cls.ESCAPE_VN_CRE.sub('_', tag) + varsDict[tag] = value # add variable + value = '$'+tag # replacement as variable + # replacement for tag: + return value # additional replacement as calling map: ADD_REPL_TAGS_CM = CallingMap(ADD_REPL_TAGS) @@ -864,7 +879,7 @@ class CommandAction(ActionBase): tickData = aInfo.get("F-*") if not tickData: tickData = {} def substTag(m): - tag = mapTag2Opt(m.groups()[0]) + tag = mapTag2Opt(m.group(1)) try: value = uni_string(tickData[tag]) except KeyError: diff --git a/fail2ban/server/actions.py b/fail2ban/server/actions.py index 24fea838..967908af 100644 --- a/fail2ban/server/actions.py +++ b/fail2ban/server/actions.py @@ -211,6 +211,14 @@ class Actions(JailThread, Mapping): def getBanTime(self): return self.__banManager.getBanTime() + def getBanned(self, ids): + lst = self.__banManager.getBanList() + if not ids: + return lst + if len(ids) == 1: + return 1 if ids[0] in lst else 0 + return map(lambda ip: 1 if ip in lst else 0, ids) + def getBanList(self, withTime=False): """Returns the list of banned IP addresses. @@ -254,7 +262,7 @@ class Actions(JailThread, Mapping): if ip is None: return self.__flushBan(db) # Multiple IPs: - if isinstance(ip, list): + if isinstance(ip, (list, tuple)): missed = [] cnt = 0 for i in ip: @@ -276,6 +284,14 @@ class Actions(JailThread, Mapping): # Unban the IP. self.__unBan(ticket) else: + # Multiple IPs by subnet or dns: + if not isinstance(ip, IPAddr): + ipa = IPAddr(ip) + if not ipa.isSingle: # subnet (mask/cidr) or raw (may be dns/hostname): + ips = filter(ipa.contains, self.__banManager.getBanList()) + if ips: + return self.removeBannedIP(ips, db, ifexists) + # not found: msg = "%s is not banned" % ip logSys.log(logging.MSG, msg) if ifexists: @@ -322,23 +338,33 @@ class Actions(JailThread, Mapping): self._jail.name, name, e, exc_info=logSys.getEffectiveLevel()<=logging.DEBUG) while self.active: - if self.idle: - logSys.debug("Actions: enter idle mode") - Utils.wait_for(lambda: not self.active or not self.idle, - lambda: False, self.sleeptime) - logSys.debug("Actions: leave idle mode") - continue - # wait for ban (stop if gets inactive): - bancnt = Utils.wait_for(lambda: not self.active or self.__checkBan(), self.sleeptime) - cnt += bancnt - # unban if nothing is banned not later than banned tickets >= banPrecedence - if not bancnt or cnt >= self.banPrecedence: - if self.active: - # let shrink the ban list faster - bancnt *= 2 - self.__checkUnBan(bancnt if bancnt and bancnt < self.unbanMaxCount else self.unbanMaxCount) - cnt = 0 - + try: + if self.idle: + logSys.debug("Actions: enter idle mode") + Utils.wait_for(lambda: not self.active or not self.idle, + lambda: False, self.sleeptime) + logSys.debug("Actions: leave idle mode") + continue + # wait for ban (stop if gets inactive, pending ban or unban): + bancnt = 0 + wt = min(self.sleeptime, self.__banManager._nextUnbanTime - MyTime.time()) + logSys.log(5, "Actions: wait for pending tickets %s (default %s)", wt, self.sleeptime) + if Utils.wait_for(lambda: not self.active or self._jail.hasFailTickets, wt): + bancnt = self.__checkBan() + cnt += bancnt + # unban if nothing is banned not later than banned tickets >= banPrecedence + if not bancnt or cnt >= self.banPrecedence: + if self.active: + # let shrink the ban list faster + bancnt *= 2 + logSys.log(5, "Actions: check-unban %s, bancnt %s, max: %s", bancnt if bancnt and bancnt < self.unbanMaxCount else self.unbanMaxCount, bancnt, self.unbanMaxCount) + self.__checkUnBan(bancnt if bancnt and bancnt < self.unbanMaxCount else self.unbanMaxCount) + cnt = 0 + except Exception as e: # pragma: no cover + logSys.error("[%s] unhandled error in actions thread: %s", + self._jail.name, e, + exc_info=logSys.getEffectiveLevel()<=logging.DEBUG) + self.__flushBan(stop=True) self.stopActions() return True @@ -431,7 +457,9 @@ class Actions(JailThread, Mapping): return mi[idx] if mi[idx] is not None else self.__ticket - def __getActionInfo(self, ticket): + def _getActionInfo(self, ticket): + if not ticket: + ticket = BanTicket("", MyTime.time()) aInfo = Actions.ActionInfo(ticket, self._jail) return aInfo @@ -465,7 +493,7 @@ class Actions(JailThread, Mapping): bTicket = BanTicket.wrap(ticket) btime = ticket.getBanTime(self.__banManager.getBanTime()) ip = bTicket.getIP() - aInfo = self.__getActionInfo(bTicket) + aInfo = self._getActionInfo(bTicket) reason = {} if self.__banManager.addBanTicket(bTicket, reason=reason): cnt += 1 @@ -476,7 +504,7 @@ class Actions(JailThread, Mapping): # do actions : for name, action in self._actions.iteritems(): try: - if ticket.restored and getattr(action, 'norestored', False): + if bTicket.restored and getattr(action, 'norestored', False): continue if not aInfo.immutable: aInfo.reset() action.ban(aInfo) @@ -522,6 +550,8 @@ class Actions(JailThread, Mapping): cnt += self.__reBan(bTicket, actions=rebanacts) else: # pragma: no cover - unexpected: ticket is not banned for some reasons - reban using all actions: cnt += self.__reBan(bTicket) + # add ban to database moved to observer (should previously check not already banned + # and increase ticket time if "bantime.increment" set) if cnt: logSys.debug("Banned %s / %s, %s ticket(s) in %r", cnt, self.__banManager.getBanTotal(), self.__banManager.size(), self._jail.name) @@ -540,7 +570,7 @@ class Actions(JailThread, Mapping): """ actions = actions or self._actions ip = ticket.getIP() - aInfo = self.__getActionInfo(ticket) + aInfo = self._getActionInfo(ticket) if log: logSys.notice("[%s] Reban %s%s", self._jail.name, aInfo["ip"], (', action %r' % actions.keys()[0] if len(actions) == 1 else '')) for name, action in actions.iteritems(): @@ -574,7 +604,7 @@ class Actions(JailThread, Mapping): if not action._prolongable: continue if aInfo is None: - aInfo = self.__getActionInfo(ticket) + aInfo = self._getActionInfo(ticket) if not aInfo.immutable: aInfo.reset() action.prolong(aInfo) except Exception as e: @@ -668,7 +698,7 @@ class Actions(JailThread, Mapping): else: unbactions = actions ip = ticket.getIP() - aInfo = self.__getActionInfo(ticket) + aInfo = self._getActionInfo(ticket) if log: logSys.notice("[%s] Unban %s", self._jail.name, aInfo["ip"]) for name, action in unbactions.iteritems(): @@ -687,13 +717,19 @@ class Actions(JailThread, Mapping): """Status of current and total ban counts and current banned IP list. """ # TODO: Allow this list to be printed as 'status' output - supported_flavors = ["basic", "cymru"] + supported_flavors = ["short", "basic", "cymru"] if flavor is None or flavor not in supported_flavors: logSys.warning("Unsupported extended jail status flavor %r. Supported: %s" % (flavor, supported_flavors)) # Always print this information (basic) - ret = [("Currently banned", self.__banManager.size()), - ("Total banned", self.__banManager.getBanTotal()), - ("Banned IP list", self.__banManager.getBanList())] + if flavor != "short": + banned = self.__banManager.getBanList() + cnt = len(banned) + else: + cnt = self.__banManager.size() + ret = [("Currently banned", cnt), + ("Total banned", self.__banManager.getBanTotal())] + if flavor != "short": + ret += [("Banned IP list", banned)] if flavor == "cymru": cymru_info = self.__banManager.getBanListExtendedCymruInfo() ret += \ diff --git a/fail2ban/server/banmanager.py b/fail2ban/server/banmanager.py index 5770bfd7..9168d5b8 100644 --- a/fail2ban/server/banmanager.py +++ b/fail2ban/server/banmanager.py @@ -57,7 +57,7 @@ class BanManager: ## Total number of banned IP address self.__banTotal = 0 ## The time for next unban process (for performance and load reasons): - self.__nextUnbanTime = BanTicket.MAX_TIME + self._nextUnbanTime = BanTicket.MAX_TIME ## # Set the ban time. @@ -66,7 +66,6 @@ class BanManager: # @param value the time def setBanTime(self, value): - with self.__lock: self.__banTime = int(value) ## @@ -76,7 +75,6 @@ class BanManager: # @return the time def getBanTime(self): - with self.__lock: return self.__banTime ## @@ -85,7 +83,6 @@ class BanManager: # @param value total number def setBanTotal(self, value): - with self.__lock: self.__banTotal = value ## @@ -94,7 +91,6 @@ class BanManager: # @return the total number def getBanTotal(self): - with self.__lock: return self.__banTotal ## @@ -103,21 +99,21 @@ class BanManager: # @return IP list def getBanList(self, ordered=False, withTime=False): + if not ordered: + return list(self.__banList.keys()) with self.__lock: - if not ordered: - return self.__banList.keys() lst = [] for ticket in self.__banList.itervalues(): eob = ticket.getEndOfBanTime(self.__banTime) lst.append((ticket,eob)) - lst.sort(key=lambda t: t[1]) - t2s = MyTime.time2str - if withTime: - return ['%s \t%s + %d = %s' % ( - t[0].getID(), - t2s(t[0].getTime()), t[0].getBanTime(self.__banTime), t2s(t[1]) - ) for t in lst] - return [t[0].getID() for t in lst] + lst.sort(key=lambda t: t[1]) + t2s = MyTime.time2str + if withTime: + return ['%s \t%s + %d = %s' % ( + t[0].getID(), + t2s(t[0].getTime()), t[0].getBanTime(self.__banTime), t2s(t[1]) + ) for t in lst] + return [t[0].getID() for t in lst] ## # Returns a iterator to ban list (used in reload, so idle). @@ -125,8 +121,8 @@ class BanManager: # @return ban list iterator def __iter__(self): - with self.__lock: - return self.__banList.itervalues() + # ensure iterator is safe - traverse over the list in snapshot created within lock (GIL): + return iter(list(self.__banList.values())) ## # Returns normalized value @@ -297,8 +293,8 @@ class BanManager: self.__banTotal += 1 ticket.incrBanCount() # correct next unban time: - if self.__nextUnbanTime > eob: - self.__nextUnbanTime = eob + if self._nextUnbanTime > eob: + self._nextUnbanTime = eob return True ## @@ -329,12 +325,8 @@ class BanManager: def unBanList(self, time, maxCount=0x7fffffff): with self.__lock: - # Permanent banning - if self.__banTime < 0: - return list() - # Check next unban time: - nextUnbanTime = self.__nextUnbanTime + nextUnbanTime = self._nextUnbanTime if nextUnbanTime > time: return list() @@ -347,12 +339,12 @@ class BanManager: if time > eob: unBanList[fid] = ticket if len(unBanList) >= maxCount: # stop search cycle, so reset back the next check time - nextUnbanTime = self.__nextUnbanTime + nextUnbanTime = self._nextUnbanTime break elif nextUnbanTime > eob: nextUnbanTime = eob - self.__nextUnbanTime = nextUnbanTime + self._nextUnbanTime = nextUnbanTime # Removes tickets. if len(unBanList): if len(unBanList) / 2.0 <= len(self.__banList) / 3.0: diff --git a/fail2ban/server/database.py b/fail2ban/server/database.py index 0dd9acb6..ed736a7a 100644 --- a/fail2ban/server/database.py +++ b/fail2ban/server/database.py @@ -489,22 +489,24 @@ class Fail2BanDb(object): If log was already present in database, value of last position in the log file; else `None` """ + return self._addLog(cur, jail, container.getFileName(), container.getPos(), container.getHash()) + + def _addLog(self, cur, jail, name, pos=0, md5=None): lastLinePos = None cur.execute( "SELECT firstlinemd5, lastfilepos FROM logs " "WHERE jail=? AND path=?", - (jail.name, container.getFileName())) + (jail.name, name)) try: firstLineMD5, lastLinePos = cur.fetchone() except TypeError: - firstLineMD5 = False + firstLineMD5 = None - cur.execute( - "INSERT OR REPLACE INTO logs(jail, path, firstlinemd5, lastfilepos) " - "VALUES(?, ?, ?, ?)", - (jail.name, container.getFileName(), - container.getHash(), container.getPos())) - if container.getHash() != firstLineMD5: + if not firstLineMD5 and (pos or md5): + cur.execute( + "INSERT OR REPLACE INTO logs(jail, path, firstlinemd5, lastfilepos) " + "VALUES(?, ?, ?, ?)", (jail.name, name, md5, pos)) + if md5 is not None and md5 != firstLineMD5: lastLinePos = None return lastLinePos @@ -533,7 +535,7 @@ class Fail2BanDb(object): return set(row[0] for row in cur.fetchmany()) @commitandrollback - def updateLog(self, cur, *args, **kwargs): + def updateLog(self, cur, jail, container): """Updates hash and last position in log file. Parameters @@ -543,14 +545,48 @@ class Fail2BanDb(object): container : FileContainer File container of the log file being updated. """ - self._updateLog(cur, *args, **kwargs) + self._updateLog(cur, jail, container.getFileName(), container.getPos(), container.getHash()) - def _updateLog(self, cur, jail, container): + def _updateLog(self, cur, jail, name, pos, md5): cur.execute( "UPDATE logs SET firstlinemd5=?, lastfilepos=? " - "WHERE jail=? AND path=?", - (container.getHash(), container.getPos(), - jail.name, container.getFileName())) + "WHERE jail=? AND path=?", (md5, pos, jail.name, name)) + # be sure it is set (if not available): + if not cur.rowcount: + cur.execute( + "INSERT OR REPLACE INTO logs(jail, path, firstlinemd5, lastfilepos) " + "VALUES(?, ?, ?, ?)", (jail.name, name, md5, pos)) + + @commitandrollback + def getJournalPos(self, cur, jail, name, time=0, iso=None): + """Get journal position from database. + + Parameters + ---------- + jail : Jail + Jail of which the journal belongs to. + name, time, iso : + Journal name (typically systemd-journal) and last known time. + + Returns + ------- + int (or float) + Last position (as time) if it was already present in database; else `None` + """ + return self._addLog(cur, jail, name, time, iso); # no hash, just time as iso + + @commitandrollback + def updateJournal(self, cur, jail, name, time, iso): + """Updates last position (as time) of journal. + + Parameters + ---------- + jail : Jail + Jail of which the journal belongs to. + name, time, iso : + Journal name (typically systemd-journal) and last known time. + """ + self._updateLog(cur, jail, name, time, iso); # no hash, just time as iso @commitandrollback def addBan(self, cur, jail, ticket): @@ -754,7 +790,8 @@ class Fail2BanDb(object): if overalljails or jail is None: query += " GROUP BY ip ORDER BY timeofban DESC LIMIT 1" cur = self._db.cursor() - return cur.execute(query, queryArgs) + # repack iterator as long as in lock: + return list(cur.execute(query, queryArgs)) def _getCurrentBans(self, cur, jail = None, ip = None, forbantime=None, fromtime=None): queryArgs = [] diff --git a/fail2ban/server/datedetector.py b/fail2ban/server/datedetector.py index 5942e3e0..90a70b0d 100644 --- a/fail2ban/server/datedetector.py +++ b/fail2ban/server/datedetector.py @@ -282,6 +282,8 @@ class DateDetector(object): elif "{DATE}" in key: self.addDefaultTemplate(preMatch=pattern, allDefaults=False) return + elif key == "{NONE}": + template = _getPatternTemplate('{UNB}^', key) else: template = _getPatternTemplate(pattern, key) @@ -337,65 +339,76 @@ class DateDetector(object): # if no templates specified - default templates should be used: if not len(self.__templates): self.addDefaultTemplate() - logSys.log(logLevel-1, "try to match time for line: %.120s", line) - match = None + log = logSys.log if logSys.getEffectiveLevel() <= logLevel else lambda *args: None + log(logLevel-1, "try to match time for line: %.120s", line) + # first try to use last template with same start/end position: + match = None + found = None, 0x7fffffff, 0x7fffffff, -1 ignoreBySearch = 0x7fffffff i = self.__lastTemplIdx if i < len(self.__templates): ddtempl = self.__templates[i] template = ddtempl.template if template.flags & (DateTemplate.LINE_BEGIN|DateTemplate.LINE_END): - if logSys.getEffectiveLevel() <= logLevel-1: # pragma: no cover - very-heavy debug - logSys.log(logLevel-1, " try to match last anchored template #%02i ...", i) + log(logLevel-1, " try to match last anchored template #%02i ...", i) match = template.matchDate(line) ignoreBySearch = i else: distance, endpos = self.__lastPos[0], self.__lastEndPos[0] - if logSys.getEffectiveLevel() <= logLevel-1: - logSys.log(logLevel-1, " try to match last template #%02i (from %r to %r): ...%r==%r %s %r==%r...", - i, distance, endpos, - line[distance-1:distance], self.__lastPos[1], - line[distance:endpos], - line[endpos:endpos+1], self.__lastEndPos[1]) - # check same boundaries left/right, otherwise possible collision/pattern switch: - if (line[distance-1:distance] == self.__lastPos[1] and - line[endpos:endpos+1] == self.__lastEndPos[1] - ): + log(logLevel-1, " try to match last template #%02i (from %r to %r): ...%r==%r %s %r==%r...", + i, distance, endpos, + line[distance-1:distance], self.__lastPos[1], + line[distance:endpos], + line[endpos:endpos+1], self.__lastEndPos[2]) + # check same boundaries left/right, outside fully equal, inside only if not alnum (e. g. bound RE + # with space or some special char), otherwise possible collision/pattern switch: + if (( + line[distance-1:distance] == self.__lastPos[1] or + (line[distance] == self.__lastPos[2] and not self.__lastPos[2].isalnum()) + ) and ( + line[endpos:endpos+1] == self.__lastEndPos[2] or + (line[endpos-1] == self.__lastEndPos[1] and not self.__lastEndPos[1].isalnum()) + )): + # search in line part only: + log(logLevel-1, " boundaries are correct, search in part %r", line[distance:endpos]) match = template.matchDate(line, distance, endpos) + else: + log(logLevel-1, " boundaries show conflict, try whole search") + match = template.matchDate(line) + ignoreBySearch = i if match: distance = match.start() endpos = match.end() # if different position, possible collision/pattern switch: if ( + len(self.__templates) == 1 or # single template: template.flags & (DateTemplate.LINE_BEGIN|DateTemplate.LINE_END) or (distance == self.__lastPos[0] and endpos == self.__lastEndPos[0]) ): - logSys.log(logLevel, " matched last time template #%02i", i) + log(logLevel, " matched last time template #%02i", i) else: - logSys.log(logLevel, " ** last pattern collision - pattern change, search ...") + log(logLevel, " ** last pattern collision - pattern change, reserve & search ...") + found = match, distance, endpos, i; # save current best alternative match = None else: - logSys.log(logLevel, " ** last pattern not found - pattern change, search ...") + log(logLevel, " ** last pattern not found - pattern change, search ...") # search template and better match: if not match: - logSys.log(logLevel, " search template (%i) ...", len(self.__templates)) - found = None, 0x7fffffff, 0x7fffffff, -1 + log(logLevel, " search template (%i) ...", len(self.__templates)) i = 0 for ddtempl in self.__templates: - if logSys.getEffectiveLevel() <= logLevel-1: - logSys.log(logLevel-1, " try template #%02i: %s", i, ddtempl.name) if i == ignoreBySearch: i += 1 continue + log(logLevel-1, " try template #%02i: %s", i, ddtempl.name) template = ddtempl.template match = template.matchDate(line) if match: distance = match.start() endpos = match.end() - if logSys.getEffectiveLevel() <= logLevel: - logSys.log(logLevel, " matched time template #%02i (at %r <= %r, %r) %s", - i, distance, ddtempl.distance, self.__lastPos[0], template.name) + log(logLevel, " matched time template #%02i (at %r <= %r, %r) %s", + i, distance, ddtempl.distance, self.__lastPos[0], template.name) ## last (or single) template - fast stop: if i+1 >= len(self.__templates): break @@ -408,7 +421,7 @@ class DateDetector(object): ## [grave] if distance changed, possible date-match was found somewhere ## in body of message, so save this template, and search further: if distance > ddtempl.distance or distance > self.__lastPos[0]: - logSys.log(logLevel, " ** distance collision - pattern change, reserve") + log(logLevel, " ** distance collision - pattern change, reserve") ## shortest of both: if distance < found[1]: found = match, distance, endpos, i @@ -422,7 +435,7 @@ class DateDetector(object): # check other template was found (use this one with shortest distance): if not match and found[0]: match, distance, endpos, i = found - logSys.log(logLevel, " use best time template #%02i", i) + log(logLevel, " use best time template #%02i", i) ddtempl = self.__templates[i] template = ddtempl.template # we've winner, incr hits, set distance, usage, reorder, etc: @@ -432,8 +445,8 @@ class DateDetector(object): ddtempl.distance = distance if self.__firstUnused == i: self.__firstUnused += 1 - self.__lastPos = distance, line[distance-1:distance] - self.__lastEndPos = endpos, line[endpos:endpos+1] + self.__lastPos = distance, line[distance-1:distance], line[distance] + self.__lastEndPos = endpos, line[endpos-1], line[endpos:endpos+1] # if not first - try to reorder current template (bubble up), they will be not sorted anymore: if i and i != self.__lastTemplIdx: i = self._reorderTemplate(i) @@ -442,7 +455,7 @@ class DateDetector(object): return (match, template) # not found: - logSys.log(logLevel, " no template.") + log(logLevel, " no template.") return (None, None) @property diff --git a/fail2ban/server/datetemplate.py b/fail2ban/server/datetemplate.py index 973a8a51..a198e4ed 100644 --- a/fail2ban/server/datetemplate.py +++ b/fail2ban/server/datetemplate.py @@ -36,15 +36,16 @@ logSys = getLogger(__name__) RE_GROUPED = re.compile(r'(? time) + if item.getTime() + self.__maxTime > time) self.__bgSvc.service() def delFailure(self, fid): diff --git a/fail2ban/server/failregex.py b/fail2ban/server/failregex.py index f7dafbef..4032ecdb 100644 --- a/fail2ban/server/failregex.py +++ b/fail2ban/server/failregex.py @@ -87,20 +87,24 @@ RH4TAG = { # default failure groups map for customizable expressions (with different group-id): R_MAP = { - "ID": "fid", - "PORT": "fport", + "id": "fid", + "port": "fport", } def mapTag2Opt(tag): - try: # if should be mapped: - return R_MAP[tag] - except KeyError: - return tag.lower() + tag = tag.lower() + return R_MAP.get(tag, tag) -# alternate names to be merged, e. g. alt_user_1 -> user ... +# complex names: +# ALT_ - alternate names to be merged, e. g. alt_user_1 -> user ... ALTNAME_PRE = 'alt_' -ALTNAME_CRE = re.compile(r'^' + ALTNAME_PRE + r'(.*)(?:_\d+)?$') +# TUPLE_ - names of parts to be combined to single value as tuple +TUPNAME_PRE = 'tuple_' + +COMPLNAME_PRE = (ALTNAME_PRE, TUPNAME_PRE) +COMPLNAME_CRE = re.compile(r'^(' + '|'.join(COMPLNAME_PRE) + r')(.*?)(?:_\d+)?$') + ## # Regular expression class. @@ -127,17 +131,27 @@ class Regex: try: self._regexObj = re.compile(regex, re.MULTILINE if multiline else 0) self._regex = regex - self._altValues = {} + self._altValues = [] + self._tupleValues = [] for k in filter( - lambda k: len(k) > len(ALTNAME_PRE) and k.startswith(ALTNAME_PRE), - self._regexObj.groupindex + lambda k: len(k) > len(COMPLNAME_PRE[0]), self._regexObj.groupindex ): - n = ALTNAME_CRE.match(k).group(1) - self._altValues[k] = n - self._altValues = list(self._altValues.items()) if len(self._altValues) else None + n = COMPLNAME_CRE.match(k) + if n: + g, n = n.group(1), mapTag2Opt(n.group(2)) + if g == ALTNAME_PRE: + self._altValues.append((k,n)) + else: + self._tupleValues.append((k,n)) + self._altValues.sort() + self._tupleValues.sort() + self._altValues = self._altValues if len(self._altValues) else None + self._tupleValues = self._tupleValues if len(self._tupleValues) else None except sre_constants.error: raise RegexException("Unable to compile regular expression '%s'" % regex) + # set fetch handler depending on presence of alternate (or tuple) tags: + self.getGroups = self._getGroupsWithAlt if (self._altValues or self._tupleValues) else self._getGroups def __str__(self): return "%s(%r)" % (self.__class__.__name__, self._regex) @@ -277,18 +291,33 @@ class Regex: # Returns all matched groups. # - def getGroups(self): - if not self._altValues: - return self._matchCache.groupdict() - # merge alternate values (e. g. 'alt_user_1' -> 'user' or 'alt_host' -> 'host'): + def _getGroups(self): + return self._matchCache.groupdict() + + def _getGroupsWithAlt(self): fail = self._matchCache.groupdict() #fail = fail.copy() - for k,n in self._altValues: - v = fail.get(k) - if v and not fail.get(n): - fail[n] = v + # merge alternate values (e. g. 'alt_user_1' -> 'user' or 'alt_host' -> 'host'): + if self._altValues: + for k,n in self._altValues: + v = fail.get(k) + if v and not fail.get(n): + fail[n] = v + # combine tuple values (e. g. 'id', 'tuple_id' ... 'tuple_id_N' -> 'id'): + if self._tupleValues: + for k,n in self._tupleValues: + v = fail.get(k) + t = fail.get(n) + if isinstance(t, tuple): + t += (v,) + else: + t = (t,v,) + fail[n] = t return fail + def getGroups(self): # pragma: no cover - abstract function (replaced in __init__) + pass + ## # Returns skipped lines. # diff --git a/fail2ban/server/filter.py b/fail2ban/server/filter.py index 998fe298..d3261c32 100644 --- a/fail2ban/server/filter.py +++ b/fail2ban/server/filter.py @@ -81,6 +81,7 @@ class Filter(JailThread): ## Ignore own IPs flag: self.__ignoreSelf = True ## The ignore IP list. + self.__ignoreIpSet = set() self.__ignoreIpList = [] ## External command self.__ignoreCommand = False @@ -106,8 +107,16 @@ class Filter(JailThread): self.returnRawHost = False ## check each regex (used for test purposes): self.checkAllRegex = False + ## avoid finding of pending failures (without ID/IP, used in fail2ban-regex): + self.ignorePending = True + ## callback called on ignoreregex match : + self.onIgnoreRegex = None ## if true ignores obsolete failures (failure time < now - findTime): self.checkFindTime = True + ## shows that filter is in operation mode (processing new messages): + self.inOperation = True + ## if true prevents against retarded banning in case of RC by too many failures (disabled only for test purposes): + self.banASAP = True ## Ticks counter self.ticks = 0 ## Thread name: @@ -169,7 +178,7 @@ class Filter(JailThread): # @param value the regular expression def addFailRegex(self, value): - multiLine = self.getMaxLines() > 1 + multiLine = self.__lineBufferSize > 1 try: regex = FailRegex(value, prefRegex=self.__prefRegex, multiline=multiLine, useDns=self.__useDns) @@ -452,10 +461,10 @@ class Filter(JailThread): logSys.info( "[%s] Attempt %s - %s", self.jailName, ip, datetime.datetime.fromtimestamp(unixTime).strftime("%Y-%m-%d %H:%M:%S") ) - self.failManager.addFailure(ticket, len(matches) or 1) - + attempts = self.failManager.addFailure(ticket, len(matches) or 1) # Perform the ban if this attempt is resulted to: - self.performBan(ip) + if attempts >= self.failManager.getMaxRetry(): + self.performBan(ip) return 1 @@ -484,28 +493,36 @@ class Filter(JailThread): # Create IP address object ip = IPAddr(ipstr) # Avoid exact duplicates - if ip in self.__ignoreIpList: - logSys.warn(" Ignore duplicate %r (%r), already in ignore list", ip, ipstr) + if ip in self.__ignoreIpSet or ip in self.__ignoreIpList: + logSys.log(logging.MSG, " Ignore duplicate %r (%r), already in ignore list", ip, ipstr) return # log and append to ignore list logSys.debug(" Add %r to ignore list (%r)", ip, ipstr) - self.__ignoreIpList.append(ip) + # if single IP (not DNS or a subnet) add to set, otherwise to list: + if ip.isSingle: + self.__ignoreIpSet.add(ip) + else: + self.__ignoreIpList.append(ip) def delIgnoreIP(self, ip=None): # clear all: if ip is None: + self.__ignoreIpSet.clear() del self.__ignoreIpList[:] return # delete by ip: logSys.debug(" Remove %r from ignore list", ip) - self.__ignoreIpList.remove(ip) + if ip in self.__ignoreIpSet: + self.__ignoreIpSet.remove(ip) + else: + self.__ignoreIpList.remove(ip) def logIgnoreIp(self, ip, log_ignore, ignore_source="unknown source"): if log_ignore: logSys.info("[%s] Ignore %s by %s", self.jailName, ip, ignore_source) def getIgnoreIP(self): - return self.__ignoreIpList + return self.__ignoreIpList + list(self.__ignoreIpSet) ## # Check if IP address/DNS is in the ignore list. @@ -545,8 +562,11 @@ class Filter(JailThread): if self.__ignoreCache: c.set(key, True) return True + # check if the IP is covered by ignore IP (in set or in subnet/dns): + if ip in self.__ignoreIpSet: + self.logIgnoreIp(ip, log_ignore, ignore_source="ip") + return True for net in self.__ignoreIpList: - # check if the IP is covered by ignore IP if ip.isInNet(net): self.logIgnoreIp(ip, log_ignore, ignore_source=("ip" if net.isValid else "dns")) if self.__ignoreCache: c.set(key, True) @@ -569,29 +589,89 @@ class Filter(JailThread): if self.__ignoreCache: c.set(key, False) return False + def _logWarnOnce(self, nextLTM, *args): + """Log some issue as warning once per day, otherwise level 7""" + if MyTime.time() < getattr(self, nextLTM, 0): + if logSys.getEffectiveLevel() <= 7: logSys.log(7, *(args[0])) + else: + setattr(self, nextLTM, MyTime.time() + 24*60*60) + for args in args: + logSys.warning('[%s] ' + args[0], self.jailName, *args[1:]) + def processLine(self, line, date=None): """Split the time portion from log msg and return findFailures on them """ + logSys.log(7, "Working on line %r", line) + + noDate = False if date: tupleLine = line + self.__lastTimeText = tupleLine[1] + self.__lastDate = date else: - l = line.rstrip('\r\n') - logSys.log(7, "Working on line %r", line) - - (timeMatch, template) = self.dateDetector.matchTime(l) - if timeMatch: - tupleLine = ( - l[:timeMatch.start(1)], - l[timeMatch.start(1):timeMatch.end(1)], - l[timeMatch.end(1):], - (timeMatch, template) - ) + # try to parse date: + timeMatch = self.dateDetector.matchTime(line) + m = timeMatch[0] + if m: + s = m.start(1) + e = m.end(1) + m = line[s:e] + tupleLine = (line[:s], m, line[e:]) + if m: # found and not empty - retrive date: + date = self.dateDetector.getTime(m, timeMatch) + if date is not None: + # Lets get the time part + date = date[0] + self.__lastTimeText = m + self.__lastDate = date + else: + logSys.error("findFailure failed to parse timeText: %s", m) + # matched empty value - date is optional or not available - set it to last known or now: + elif self.__lastDate and self.__lastDate > MyTime.time() - 60: + # set it to last known: + tupleLine = ("", self.__lastTimeText, line) + date = self.__lastDate + else: + # set it to now: + date = MyTime.time() else: - tupleLine = (l, "", "", None) + tupleLine = ("", "", line) + # still no date - try to use last known: + if date is None: + noDate = True + if self.__lastDate and self.__lastDate > MyTime.time() - 60: + tupleLine = ("", self.__lastTimeText, line) + date = self.__lastDate + + if self.checkFindTime: + # if in operation (modifications have been really found): + if self.inOperation: + # if weird date - we'd simulate now for timeing issue (too large deviation from now): + if (date is None or date < MyTime.time() - 60 or date > MyTime.time() + 60): + # log time zone issue as warning once per day: + self._logWarnOnce("_next_simByTimeWarn", + ("Simulate NOW in operation since found time has too large deviation %s ~ %s +/- %s", + date, MyTime.time(), 60), + ("Please check jail has possibly a timezone issue. Line with odd timestamp: %s", + line)) + # simulate now as date: + date = MyTime.time() + self.__lastDate = date + else: + # in initialization (restore) phase, if too old - ignore: + if date is not None and date < MyTime.time() - self.getFindTime(): + # log time zone issue as warning once per day: + self._logWarnOnce("_next_ignByTimeWarn", + ("Ignore line since time %s < %s - %s", + date, MyTime.time(), self.getFindTime()), + ("Please check jail has possibly a timezone issue. Line with odd timestamp: %s", + line)) + # ignore - too old (obsolete) entry: + return [] # save last line (lazy convert of process line tuple to string on demand): self.processedLine = lambda: "".join(tupleLine[::2]) - return self.findFailure(tupleLine, date) + return self.findFailure(tupleLine, date, noDate=noDate) def processLineAndAdd(self, line, date=None): """Processes the line for failures and populates failManager @@ -603,13 +683,20 @@ class Filter(JailThread): fail = element[3] logSys.debug("Processing line with time:%s and ip:%s", unixTime, ip) + # ensure the time is not in the future, e. g. by some estimated (assumed) time: + if self.checkFindTime and unixTime > MyTime.time(): + unixTime = MyTime.time() tick = FailTicket(ip, unixTime, data=fail) if self._inIgnoreIPList(ip, tick): continue logSys.info( "[%s] Found %s - %s", self.jailName, ip, MyTime.time2str(unixTime) ) - self.failManager.addFailure(tick) + attempts = self.failManager.addFailure(tick) + # avoid RC on busy filter (too many failures) - if attempts for IP/ID reached maxretry, + # we can speedup ban, so do it as soon as possible: + if self.banASAP and attempts >= self.failManager.getMaxRetry(): + self.performBan(ip) # report to observer - failure was found, for possibly increasing of it retry counter (asynchronous) if Observers.Main is not None: Observers.Main.add('failureFound', self.failManager, self.jail, tick) @@ -632,20 +719,26 @@ class Filter(JailThread): self._errors //= 2 self.idle = True - ## - # Returns true if the line should be ignored. - # - # Uses ignoreregex. - # @param line: the line - # @return: a boolean - - def ignoreLine(self, tupleLines): - buf = Regex._tupleLinesBuf(tupleLines) + def _ignoreLine(self, buf, orgBuffer, failRegex=None): + # if multi-line buffer - use matched only, otherwise (single line) - original buf: + if failRegex and self.__lineBufferSize > 1: + orgBuffer = failRegex.getMatchedTupleLines() + buf = Regex._tupleLinesBuf(orgBuffer) + # search ignored: + fnd = None for ignoreRegexIndex, ignoreRegex in enumerate(self.__ignoreRegex): - ignoreRegex.search(buf, tupleLines) + ignoreRegex.search(buf, orgBuffer) if ignoreRegex.hasMatched(): - return ignoreRegexIndex - return None + fnd = ignoreRegexIndex + logSys.log(7, " Matched ignoreregex %d and was ignored", fnd) + if self.onIgnoreRegex: self.onIgnoreRegex(fnd, ignoreRegex) + # remove ignored match: + if not self.checkAllRegex or self.__lineBufferSize > 1: + # todo: check ignoreRegex.getUnmatchedTupleLines() would be better (fix testGetFailuresMultiLineIgnoreRegex): + if failRegex: + self.__lineBuffer = failRegex.getUnmatchedTupleLines() + if not self.checkAllRegex: break + return fnd def _updateUsers(self, fail, user=()): users = fail.get('users') @@ -655,54 +748,31 @@ class Filter(JailThread): fail['users'] = users = set() users.add(user) return users - return None - - # # ATM incremental (non-empty only) merge deactivated ... - # @staticmethod - # def _updateFailure(self, mlfidGroups, fail): - # # reset old failure-ids when new types of id available in this failure: - # fids = set() - # for k in ('fid', 'ip4', 'ip6', 'dns'): - # if fail.get(k): - # fids.add(k) - # if fids: - # for k in ('fid', 'ip4', 'ip6', 'dns'): - # if k not in fids: - # try: - # del mlfidGroups[k] - # except: - # pass - # # update not empty values: - # mlfidGroups.update(((k,v) for k,v in fail.iteritems() if v)) + return users def _mergeFailure(self, mlfid, fail, failRegex): mlfidFail = self.mlfidCache.get(mlfid) if self.__mlfidCache else None users = None nfflgs = 0 if fail.get("mlfgained"): - nfflgs |= 9 + nfflgs |= (8|1) if not fail.get('nofail'): fail['nofail'] = fail["mlfgained"] elif fail.get('nofail'): nfflgs |= 1 - if fail.get('mlfforget'): nfflgs |= 2 + if fail.pop('mlfforget', None): nfflgs |= 2 # if multi-line failure id (connection id) known: if mlfidFail: mlfidGroups = mlfidFail[1] # update users set (hold all users of connect): users = self._updateUsers(mlfidGroups, fail.get('user')) # be sure we've correct current state ('nofail' and 'mlfgained' only from last failure) - try: - del mlfidGroups['nofail'] - del mlfidGroups['mlfgained'] - except KeyError: - pass - # # ATM incremental (non-empty only) merge deactivated (for future version only), - # # it can be simulated using alternate value tags, like ..., - # # so previous value 'val' will be overwritten only if 'alt_val' is not empty... - # _updateFailure(mlfidGroups, fail) - # + if mlfidGroups.pop('nofail', None): nfflgs |= 4 + if mlfidGroups.pop('mlfgained', None): nfflgs |= 4 + # if we had no pending failures then clear the matches (they are already provided): + if (nfflgs & 4) == 0 and not mlfidGroups.get('mlfpending', 0): + mlfidGroups.pop("matches", None) # overwrite multi-line failure with all values, available in fail: - mlfidGroups.update(fail) + mlfidGroups.update(((k,v) for k,v in fail.iteritems() if v is not None)) # new merged failure data: fail = mlfidGroups # if forget (disconnect/reset) - remove cached entry: @@ -713,24 +783,19 @@ class Filter(JailThread): mlfidFail = [self.__lastDate, fail] self.mlfidCache.set(mlfid, mlfidFail) # check users in order to avoid reset failure by multiple logon-attempts: - if users and len(users) > 1: - # we've new user, reset 'nofail' because of multiple users attempts: - try: - del fail['nofail'] - nfflgs &= ~1 # reset nofail - except KeyError: - pass + if fail.pop('mlfpending', 0) or users and len(users) > 1: + # we've pending failures or new user, reset 'nofail' because of failures or multiple users attempts: + fail.pop('nofail', None) + fail.pop('mlfgained', None) + nfflgs &= ~(8|1) # reset nofail and gained # merge matches: - if not (nfflgs & 1): # current nofail state (corresponding users) - try: - m = fail.pop("nofail-matches") - m += fail.get("matches", []) - except KeyError: - m = fail.get("matches", []) - if not (nfflgs & 8): # no gain signaled + if (nfflgs & 1) == 0: # current nofail state (corresponding users) + m = fail.pop("nofail-matches", []) + m += fail.get("matches", []) + if (nfflgs & 8) == 0: # no gain signaled m += failRegex.getMatchedTupleLines() fail["matches"] = m - elif not (nfflgs & 2) and (nfflgs & 1): # not mlfforget and nofail: + elif (nfflgs & 3) == 1: # not mlfforget and nofail: fail["nofail-matches"] = fail.get("nofail-matches", []) + failRegex.getMatchedTupleLines() # return merged: return fail @@ -743,7 +808,7 @@ class Filter(JailThread): # to find the logging time. # @return a dict with IP and timestamp. - def findFailure(self, tupleLine, date=None): + def findFailure(self, tupleLine, date, noDate=False): failList = list() ll = logSys.getEffectiveLevel() @@ -753,62 +818,33 @@ class Filter(JailThread): returnRawHost = True cidr = IPAddr.CIDR_RAW - # Checks if we mut ignore this line. - if self.ignoreLine([tupleLine[::2]]) is not None: - # The ignoreregex matched. Return. - if ll <= 7: logSys.log(7, "Matched ignoreregex and was \"%s\" ignored", - "".join(tupleLine[::2])) - return failList - - timeText = tupleLine[1] - if date: - self.__lastTimeText = timeText - self.__lastDate = date - elif timeText: - - dateTimeMatch = self.dateDetector.getTime(timeText, tupleLine[3]) - - if dateTimeMatch is None: - logSys.error("findFailure failed to parse timeText: %s", timeText) - date = self.__lastDate - - else: - # Lets get the time part - date = dateTimeMatch[0] - - self.__lastTimeText = timeText - self.__lastDate = date - else: - timeText = self.__lastTimeText or "".join(tupleLine[::2]) - date = self.__lastDate - - if self.checkFindTime and date is not None and date < MyTime.time() - self.getFindTime(): - if ll <= 5: logSys.log(5, "Ignore line since time %s < %s - %s", - date, MyTime.time(), self.getFindTime()) - return failList - if self.__lineBufferSize > 1: - orgBuffer = self.__lineBuffer = ( - self.__lineBuffer + [tupleLine[:3]])[-self.__lineBufferSize:] + self.__lineBuffer.append(tupleLine) + orgBuffer = self.__lineBuffer = self.__lineBuffer[-self.__lineBufferSize:] else: - orgBuffer = self.__lineBuffer = [tupleLine[:3]] - if ll <= 5: logSys.log(5, "Looking for match of %r", self.__lineBuffer) - buf = Regex._tupleLinesBuf(self.__lineBuffer) + orgBuffer = self.__lineBuffer = [tupleLine] + if ll <= 5: logSys.log(5, "Looking for match of %r", orgBuffer) + buf = Regex._tupleLinesBuf(orgBuffer) + + # Checks if we must ignore this line (only if fewer ignoreregex than failregex). + if self.__ignoreRegex and len(self.__ignoreRegex) < len(self.__failRegex) - 2: + if self._ignoreLine(buf, orgBuffer) is not None: + # The ignoreregex matched. Return. + return failList # Pre-filter fail regex (if available): preGroups = {} if self.__prefRegex: if ll <= 5: logSys.log(5, " Looking for prefregex %r", self.__prefRegex.getRegex()) - self.__prefRegex.search(buf, self.__lineBuffer) + self.__prefRegex.search(buf, orgBuffer) if not self.__prefRegex.hasMatched(): if ll <= 5: logSys.log(5, " Prefregex not matched") return failList preGroups = self.__prefRegex.getGroups() if ll <= 7: logSys.log(7, " Pre-filter matched %s", preGroups) - repl = preGroups.get('content') + repl = preGroups.pop('content', None) # Content replacement: if repl: - del preGroups['content'] self.__lineBuffer, buf = [('', '', repl)], None # Iterates over all the regular expressions. @@ -826,28 +862,21 @@ class Filter(JailThread): # The failregex matched. if ll <= 7: logSys.log(7, " Matched failregex %d: %s", failRegexIndex, fail) # Checks if we must ignore this match. - if self.ignoreLine(failRegex.getMatchedTupleLines()) \ - is not None: + if self.__ignoreRegex and self._ignoreLine(buf, orgBuffer, failRegex) is not None: # The ignoreregex matched. Remove ignored match. - self.__lineBuffer, buf = failRegex.getUnmatchedTupleLines(), None - if ll <= 7: logSys.log(7, " Matched ignoreregex and was ignored") + buf = None if not self.checkAllRegex: break - else: - continue - if date is None: - logSys.warning( - "Found a match for %r but no valid date/time " - "found for %r. Please try setting a custom " - "date pattern (see man page jail.conf(5)). " - "If format is complex, please " - "file a detailed issue on" - " https://github.com/fail2ban/fail2ban/issues " - "in order to get support for this format.", - "\n".join(failRegex.getMatchedLines()), timeText) continue + if noDate: + self._logWarnOnce("_next_noTimeWarn", + ("Found a match but no valid date/time found for %r.", tupleLine[1]), + ("Match without a timestamp: %s", "\n".join(failRegex.getMatchedLines())), + ("Please try setting a custom date pattern (see man page jail.conf(5)).",) + ) + if date is None and self.checkFindTime: continue # we should check all regex (bypass on multi-line, otherwise too complex): - if not self.checkAllRegex or self.getMaxLines() > 1: + if not self.checkAllRegex or self.__lineBufferSize > 1: self.__lineBuffer, buf = failRegex.getUnmatchedTupleLines(), None # merge data if multi-line failure: raw = returnRawHost @@ -892,7 +921,8 @@ class Filter(JailThread): if host is None: if ll <= 7: logSys.log(7, "No failure-id by mlfid %r in regex %s: %s", mlfid, failRegexIndex, fail.get('mlfforget', "waiting for identifier")) - if not self.checkAllRegex: return failList + fail['mlfpending'] = 1; # mark failure is pending + if not self.checkAllRegex and self.ignorePending: return failList ips = [None] # if raw - add single ip or failure-id, # otherwise expand host to multiple ips using dns (or ignore it if not valid): @@ -905,6 +935,9 @@ class Filter(JailThread): # otherwise, try to use dns conversion: else: ips = DNSUtils.textToIp(host, self.__useDns) + # if checkAllRegex we must make a copy (to be sure next RE doesn't change merged/cached failure): + if self.checkAllRegex and mlfid is not None: + fail = fail.copy() # append failure with match to the list: for ip in ips: failList.append([failRegexIndex, ip, date, fail]) @@ -950,7 +983,7 @@ class FileFilter(Filter): log.setPos(lastpos) self.__logs[path] = log logSys.info("Added logfile: %r (pos = %s, hash = %s)" , path, log.getPos(), log.getHash()) - if autoSeek: + if autoSeek and not tail: self.__autoSeek[path] = autoSeek self._addLogPath(path) # backend specific @@ -1034,7 +1067,7 @@ class FileFilter(Filter): # MyTime.time()-self.findTime. When a failure is detected, a FailTicket # is created and is added to the FailManager. - def getFailures(self, filename): + def getFailures(self, filename, inOperation=None): log = self.getLog(filename) if log is None: logSys.error("Unable to get failures in %s", filename) @@ -1079,10 +1112,15 @@ class FileFilter(Filter): if has_content: while not self.idle: line = log.readline() - if not line or not self.active: - # The jail reached the bottom or has been stopped + if not self.active: break; # jail has been stopped + if not line: + # The jail reached the bottom, simply set in operation for this log + # (since we are first time at end of file, growing is only possible after modifications): + log.inOperation = True break - self.processLineAndAdd(line) + # acquire in operation from log and process: + self.inOperation = inOperation if inOperation is not None else log.inOperation + self.processLineAndAdd(line.rstrip('\r\n')) finally: log.close() db = self.jail.database @@ -1220,7 +1258,7 @@ except ImportError: # pragma: no cover class FileContainer: - def __init__(self, filename, encoding, tail = False): + def __init__(self, filename, encoding, tail=False): self.__filename = filename self.setEncoding(encoding) self.__tail = tail @@ -1241,6 +1279,8 @@ class FileContainer: self.__pos = 0 finally: handler.close() + ## shows that log is in operation mode (expecting new messages only from here): + self.inOperation = tail def getFileName(self): return self.__filename @@ -1314,16 +1354,17 @@ class FileContainer: return line.decode(enc, 'strict') except (UnicodeDecodeError, UnicodeEncodeError) as e: global _decode_line_warn - lev = logging.DEBUG - if _decode_line_warn.get(filename, 0) <= MyTime.time(): + lev = 7 + if not _decode_line_warn.get(filename, 0): lev = logging.WARNING - _decode_line_warn[filename] = MyTime.time() + 24*60*60 + _decode_line_warn.set(filename, 1) logSys.log(lev, - "Error decoding line from '%s' with '%s'." - " Consider setting logencoding=utf-8 (or another appropriate" - " encoding) for this jail. Continuing" - " to process line ignoring invalid characters: %r", - filename, enc, line) + "Error decoding line from '%s' with '%s'.", filename, enc) + if logSys.getEffectiveLevel() <= lev: + logSys.log(lev, "Consider setting logencoding=utf-8 (or another appropriate" + " encoding) for this jail. Continuing" + " to process line ignoring invalid characters: %r", + line) # decode with replacing error chars: line = line.decode(enc, 'replace') return line @@ -1344,7 +1385,7 @@ class FileContainer: ## print "D: Closed %s with pos %d" % (handler, self.__pos) ## sys.stdout.flush() -_decode_line_warn = {} +_decode_line_warn = Utils.Cache(maxCount=1000, maxTime=24*60*60); ## diff --git a/fail2ban/server/filtergamin.py b/fail2ban/server/filtergamin.py index 77c81757..078246de 100644 --- a/fail2ban/server/filtergamin.py +++ b/fail2ban/server/filtergamin.py @@ -79,7 +79,8 @@ class FilterGamin(FileFilter): this is a common logic and must be shared/provided by FileFilter """ self.getFailures(path) - self.performBan() + if not self.banASAP: # pragma: no cover + self.performBan() self.__modified = False ## diff --git a/fail2ban/server/filterpoll.py b/fail2ban/server/filterpoll.py index 228a2c8b..7bbdfc5c 100644 --- a/fail2ban/server/filterpoll.py +++ b/fail2ban/server/filterpoll.py @@ -111,13 +111,16 @@ class FilterPoll(FileFilter): modlst = [] Utils.wait_for(lambda: not self.active or self.getModified(modlst), self.sleeptime) + if not self.active: # pragma: no cover - timing + break for filename in modlst: self.getFailures(filename) self.__modified = True self.ticks += 1 if self.__modified: - self.performBan() + if not self.banASAP: # pragma: no cover + self.performBan() self.__modified = False except Exception as e: # pragma: no cover if not self.active: # if not active - error by stop... @@ -139,7 +142,7 @@ class FilterPoll(FileFilter): try: logStats = os.stat(filename) stats = logStats.st_mtime, logStats.st_ino, logStats.st_size - pstats = self.__prevStats.get(filename, (0)) + pstats = self.__prevStats.get(filename, (0,)) if logSys.getEffectiveLevel() <= 4: # we do not want to waste time on strftime etc if not necessary dt = logStats.st_mtime - pstats[0] diff --git a/fail2ban/server/filterpyinotify.py b/fail2ban/server/filterpyinotify.py index ca6b253f..9796e26f 100644 --- a/fail2ban/server/filterpyinotify.py +++ b/fail2ban/server/filterpyinotify.py @@ -140,7 +140,8 @@ class FilterPyinotify(FileFilter): """ if not self.idle: self.getFailures(path) - self.performBan() + if not self.banASAP: # pragma: no cover + self.performBan() self.__modified = False def _addPending(self, path, reason, isDir=False): @@ -187,7 +188,8 @@ class FilterPyinotify(FileFilter): for path, isDir in found.iteritems(): self._delPending(path) # refresh monitoring of this: - self._refreshWatcher(path, isDir=isDir) + if isDir is not None: + self._refreshWatcher(path, isDir=isDir) if isDir: # check all files belong to this dir: for logpath in self.__watchFiles: @@ -270,7 +272,13 @@ class FilterPyinotify(FileFilter): def _addLogPath(self, path): self._addFileWatcher(path) - self._process_file(path) + # initial scan: + if self.active: + # we can execute it right now: + self._process_file(path) + else: + # retard until filter gets started, isDir=None signals special case: process file only (don't need to refresh monitor): + self._addPending(path, ('INITIAL', path), isDir=None) ## # Delete a log path @@ -278,9 +286,9 @@ class FilterPyinotify(FileFilter): # @param path the log file to delete def _delLogPath(self, path): + self._delPending(path) if not self._delFileWatcher(path): # pragma: no cover logSys.error("Failed to remove watch on path: %s", path) - self._delPending(path) path_dir = dirname(path) for k in self.__watchFiles: @@ -290,8 +298,8 @@ class FilterPyinotify(FileFilter): if path_dir: # Remove watches for the directory # since there is no other monitored file under this directory - self._delDirWatcher(path_dir) self._delPending(path_dir) + self._delDirWatcher(path_dir) # pyinotify.ProcessEvent default handler: def __process_default(self, event): diff --git a/fail2ban/server/filtersystemd.py b/fail2ban/server/filtersystemd.py index c2a72598..47fc891e 100644 --- a/fail2ban/server/filtersystemd.py +++ b/fail2ban/server/filtersystemd.py @@ -190,6 +190,13 @@ class FilterSystemd(JournalFilter): # pragma: systemd no cover def getJournalReader(self): return self.__journal + def getJrnEntTime(self, logentry): + """ Returns time of entry as tuple (ISO-str, Posix).""" + date = logentry.get('_SOURCE_REALTIME_TIMESTAMP') + if date is None: + date = logentry.get('__REALTIME_TIMESTAMP') + return (date.isoformat(), time.mktime(date.timetuple()) + date.microsecond/1.0E6) + ## # Format journal log entry into syslog style # @@ -222,9 +229,8 @@ class FilterSystemd(JournalFilter): # pragma: systemd no cover logelements[-1] += v logelements[-1] += ":" if logelements[-1] == "kernel:": - if '_SOURCE_MONOTONIC_TIMESTAMP' in logentry: - monotonic = logentry.get('_SOURCE_MONOTONIC_TIMESTAMP') - else: + monotonic = logentry.get('_SOURCE_MONOTONIC_TIMESTAMP') + if monotonic is None: monotonic = logentry.get('__MONOTONIC_TIMESTAMP')[0] logelements.append("[%12.6f]" % monotonic.total_seconds()) msg = logentry.get('MESSAGE','') @@ -235,13 +241,11 @@ class FilterSystemd(JournalFilter): # pragma: systemd no cover logline = " ".join(logelements) - date = logentry.get('_SOURCE_REALTIME_TIMESTAMP', - logentry.get('__REALTIME_TIMESTAMP')) + date = self.getJrnEntTime(logentry) logSys.log(5, "[%s] Read systemd journal entry: %s %s", self.jailName, - date.isoformat(), logline) + date[0], logline) ## use the same type for 1st argument: - return ((logline[:0], date.isoformat(), logline.replace('\n', '\\n')), - time.mktime(date.timetuple()) + date.microsecond/1.0E6) + return ((logline[:0], date[0], logline.replace('\n', '\\n')), date[1]) def seekToTime(self, date): if not isinstance(date, datetime.datetime): @@ -262,9 +266,12 @@ class FilterSystemd(JournalFilter): # pragma: systemd no cover "Jail regexs will be checked against all journal entries, " "which is not advised for performance reasons.") - # Seek to now - findtime in journal - start_time = datetime.datetime.now() - \ - datetime.timedelta(seconds=int(self.getFindTime())) + # Try to obtain the last known time (position of journal) + start_time = 0 + if self.jail.database is not None: + start_time = self.jail.database.getJournalPos(self.jail, 'systemd-journal') or 0 + # Seek to max(last_known_time, now - findtime) in journal + start_time = max( start_time, MyTime.time() - int(self.getFindTime()) ) self.seekToTime(start_time) # Move back one entry to ensure do not end up in dead space # if start time beyond end of journal @@ -303,16 +310,20 @@ class FilterSystemd(JournalFilter): # pragma: systemd no cover e, exc_info=logSys.getEffectiveLevel() <= logging.DEBUG) self.ticks += 1 if logentry: - self.processLineAndAdd( - *self.formatJournalEntry(logentry)) + line = self.formatJournalEntry(logentry) + self.processLineAndAdd(*line) self.__modified += 1 if self.__modified >= 100: # todo: should be configurable break else: break if self.__modified: - self.performBan() + if not self.banASAP: # pragma: no cover + self.performBan() self.__modified = 0 + # update position in log (time and iso string): + if self.jail.database is not None: + self.jail.database.updateJournal(self.jail, 'systemd-journal', line[1], line[0][1]) except Exception as e: # pragma: no cover if not self.active: # if not active - error by stop... break diff --git a/fail2ban/server/ipdns.py b/fail2ban/server/ipdns.py index 6648dac6..ab3ec2da 100644 --- a/fail2ban/server/ipdns.py +++ b/fail2ban/server/ipdns.py @@ -337,7 +337,7 @@ class IPAddr(object): return repr(self.ntoa) def __str__(self): - return self.ntoa + return self.ntoa if isinstance(self.ntoa, basestring) else str(self.ntoa) def __reduce__(self): """IPAddr pickle-handler, that simply wraps IPAddr to the str @@ -379,6 +379,12 @@ class IPAddr(object): """ return self._family != socket.AF_UNSPEC + @property + def isSingle(self): + """Returns whether the object is a single IP address (not DNS and subnet) + """ + return self._plen == {socket.AF_INET: 32, socket.AF_INET6: 128}.get(self._family, -1000) + def __eq__(self, other): if self._family == IPAddr.CIDR_RAW and not isinstance(other, IPAddr): return self._raw == other @@ -511,6 +517,11 @@ class IPAddr(object): return (self.addr & mask) == net.addr + def contains(self, ip): + """Return whether the object (as network) contains given IP + """ + return isinstance(ip, IPAddr) and (ip == self or ip.isInNet(self)) + # Pre-calculated map: addr to maskplen def __getMaskMap(): m6 = (1 << 128)-1 diff --git a/fail2ban/server/jail.py b/fail2ban/server/jail.py index 9453e205..673b6454 100644 --- a/fail2ban/server/jail.py +++ b/fail2ban/server/jail.py @@ -161,6 +161,10 @@ class Jail(object): """ return self.__db + @database.setter + def database(self, value): + self.__db = value; + @property def filter(self): """The filter which the jail is using to monitor log files. @@ -192,6 +196,12 @@ class Jail(object): ("Actions", self.actions.status(flavor=flavor)), ] + @property + def hasFailTickets(self): + """Retrieve whether queue has tickets to ban. + """ + return not self.__queue.empty() + def putFailTicket(self, ticket): """Add a fail ticket to the jail. diff --git a/fail2ban/server/jailthread.py b/fail2ban/server/jailthread.py index d0430367..94f34542 100644 --- a/fail2ban/server/jailthread.py +++ b/fail2ban/server/jailthread.py @@ -120,3 +120,6 @@ class JailThread(Thread): ## python 2.x replace binding of private __bootstrap method: if sys.version_info < (3,): # pragma: 3.x no cover JailThread._Thread__bootstrap = JailThread._JailThread__bootstrap +## python 3.9, restore isAlive method: +elif not hasattr(JailThread, 'isAlive'): # pragma: 2.x no cover + JailThread.isAlive = JailThread.is_alive diff --git a/fail2ban/server/mytime.py b/fail2ban/server/mytime.py index 98b69bd4..e4b091a7 100644 --- a/fail2ban/server/mytime.py +++ b/fail2ban/server/mytime.py @@ -121,8 +121,11 @@ class MyTime: @return ISO-capable string representation of given unixTime """ - return datetime.datetime.fromtimestamp( - unixTime).replace(microsecond=0).strftime(format) + # consider end of 9999th year (in GMT+23 to avoid year overflow in other TZ) + dt = datetime.datetime.fromtimestamp( + unixTime).replace(microsecond=0 + ) if unixTime < 253402214400 else datetime.datetime(9999, 12, 31, 23, 59, 59) + return dt.strftime(format) ## precreate/precompile primitives used in str2seconds: diff --git a/fail2ban/server/observer.py b/fail2ban/server/observer.py index bd7cbe4a..f5ba20d9 100644 --- a/fail2ban/server/observer.py +++ b/fail2ban/server/observer.py @@ -87,7 +87,7 @@ class ObserverThread(JailThread): except KeyError: raise KeyError("Invalid event index : %s" % i) - def __delitem__(self, name): + def __delitem__(self, i): try: del self._queue[i] except KeyError: @@ -146,9 +146,11 @@ class ObserverThread(JailThread): def pulse_notify(self): """Notify wakeup (sets /and resets/ notify event) """ - if not self._paused and self._notify: - self._notify.set() - #self._notify.clear() + if not self._paused: + n = self._notify + if n: + n.set() + #n.clear() def add(self, *event): """Add a event to queue and notify thread to wake up. @@ -237,6 +239,7 @@ class ObserverThread(JailThread): break ## end of main loop - exit logSys.info("Observer stopped, %s events remaining.", len(self._queue)) + self._notify = None #print("Observer stopped, %s events remaining." % len(self._queue)) except Exception as e: logSys.error('Observer stopped after error: %s', e, exc_info=True) @@ -262,9 +265,8 @@ class ObserverThread(JailThread): if not self.active: super(ObserverThread, self).start() - def stop(self): + def stop(self, wtime=5, forceQuit=True): if self.active and self._notify: - wtime = 5 logSys.info("Observer stop ... try to end queue %s seconds", wtime) #print("Observer stop ....") # just add shutdown job to make possible wait later until full (events remaining) @@ -276,10 +278,15 @@ class ObserverThread(JailThread): #self.pulse_notify() self._notify = None # wait max wtime seconds until full (events remaining) - self.wait_empty(wtime) - n.clear() - self.active = False - self.wait_idle(0.5) + if self.wait_empty(wtime) or forceQuit: + n.clear() + self.active = False; # leave outer (active) loop + self._paused = True; # leave inner (queue) loop + self.__db = None + else: + self._notify = n + return self.wait_idle(min(wtime, 0.5)) and not self.is_full + return True @property def is_full(self): diff --git a/fail2ban/server/server.py b/fail2ban/server/server.py index 15265822..4606d928 100644 --- a/fail2ban/server/server.py +++ b/fail2ban/server/server.py @@ -58,6 +58,23 @@ except ImportError: # pragma: no cover def _thread_name(): return threading.current_thread().__class__.__name__ +try: + FileExistsError +except NameError: # pragma: 3.x no cover + FileExistsError = OSError + +def _make_file_path(name): + """Creates path of file (last level only) on demand""" + name = os.path.dirname(name) + # only if it is absolute (e. g. important for socket, so if unix path): + if os.path.isabs(name): + # be sure path exists (create last level of directory on demand): + try: + os.mkdir(name) + except (OSError, FileExistsError) as e: + if e.errno != 17: # pragma: no cover - not EEXIST is not covered + raise + class Server: @@ -81,8 +98,6 @@ class Server: 'Linux': '/dev/log', } self.__prev_signals = {} - # replace real thread name with short process name (for top/ps/pstree or diagnostic): - prctl_set_th_name('f2b/server') def __sigTERMhandler(self, signum, frame): # pragma: no cover - indirect tested logSys.debug("Caught signal %d. Exiting", signum) @@ -99,7 +114,7 @@ class Server: def start(self, sock, pidfile, force=False, observer=True, conf={}): # First set the mask to only allow access to owner - os.umask(0077) + os.umask(0o077) # Second daemonize before logging etc, because it will close all handles: if self.__daemon: # pragma: no cover logSys.info("Starting in daemon mode") @@ -113,6 +128,9 @@ class Server: logSys.error(err) raise ServerInitializationError(err) # We are daemon. + + # replace main thread (and process) name to identify server (for top/ps/pstree or diagnostic): + prctl_set_th_name(conf.get("pname", "fail2ban-server")) # Set all logging parameters (or use default if not specified): self.__verbose = conf.get("verbose", None) @@ -141,6 +159,7 @@ class Server: # Creates a PID file. try: logSys.debug("Creating PID file %s", pidfile) + _make_file_path(pidfile) pidFile = open(pidfile, 'w') pidFile.write("%s\n" % os.getpid()) pidFile.close() @@ -156,6 +175,7 @@ class Server: # Start the communication logSys.debug("Starting communication") try: + _make_file_path(sock) self.__asyncServer = AsyncServer(self.__transm) self.__asyncServer.onstart = conf.get('onstart') self.__asyncServer.start(sock, force) @@ -193,23 +213,26 @@ class Server: signal.signal(s, sh) # Give observer a small chance to complete its work before exit - if Observers.Main is not None: - Observers.Main.stop() + obsMain = Observers.Main + if obsMain is not None: + if obsMain.stop(forceQuit=False): + obsMain = None + Observers.Main = None # Now stop all the jails self.stopAllJail() + # Stop observer ultimately + if obsMain is not None: + obsMain.stop() + # Explicit close database (server can leave in a thread, # so delayed GC can prevent commiting changes) if self.__db: self.__db.close() self.__db = None - # Stop observer and exit - if Observers.Main is not None: - Observers.Main.stop() - Observers.Main = None - # Stop async + # Stop async and exit if self.__asyncServer is not None: self.__asyncServer.stop() self.__asyncServer = None @@ -517,6 +540,32 @@ class Server: cnt += jail.actions.removeBannedIP(value, ifexists=ifexists) return cnt + def banned(self, name=None, ids=None): + if name is not None: + # single jail: + jails = [self.__jails[name]] + else: + # in all jails: + jails = self.__jails.values() + # check banned ids: + res = [] + if name is None and ids: + for ip in ids: + ret = [] + for jail in jails: + if jail.actions.getBanned([ip]): + ret.append(jail.name) + res.append(ret) + else: + for jail in jails: + ret = jail.actions.getBanned(ids) + if name is not None: + return ret + res.append(ret) + else: + res.append({jail.name: ret}) + return res + def getBanTime(self, name): return self.__jails[name].actions.getBanTime() @@ -777,6 +826,7 @@ class Server: self.__db = None else: if Fail2BanDb is not None: + _make_file_path(filename) self.__db = Fail2BanDb(filename) self.__db.delAllJails() else: # pragma: no cover diff --git a/fail2ban/server/strptime.py b/fail2ban/server/strptime.py index 498d284b..1464a96d 100644 --- a/fail2ban/server/strptime.py +++ b/fail2ban/server/strptime.py @@ -291,9 +291,8 @@ def reGroupDictStrptime(found_dict, msec=False, default_tz=None): date_result -= datetime.timedelta(days=1) if assume_year: if not now: now = MyTime.now() - if date_result > now: - # Could be last year? - # also reset month and day as it's not yesterday... + if date_result > now + datetime.timedelta(days=1): # ignore by timezone issues (+24h) + # assume last year - also reset month and day as it's not yesterday... date_result = date_result.replace( year=year-1, month=month, day=day) diff --git a/fail2ban/server/ticket.py b/fail2ban/server/ticket.py index f67e0d23..f99b6462 100644 --- a/fail2ban/server/ticket.py +++ b/fail2ban/server/ticket.py @@ -227,15 +227,14 @@ class FailTicket(Ticket): def __init__(self, ip=None, time=None, matches=None, data={}, ticket=None): # this class variables: - self._retry = 0 - self._lastReset = None + self._firstTime = None + self._retry = 1 # create/copy using default ticket constructor: Ticket.__init__(self, ip, time, matches, data, ticket) # init: - if ticket is None: - self._lastReset = time if time is not None else self.getTime() - if not self._retry: - self._retry = self._data['failures']; + if not isinstance(ticket, FailTicket): + self._firstTime = time if time is not None else self.getTime() + self._retry = self._data.get('failures', 1) def setRetry(self, value): """ Set artificial retry count, normally equal failures / attempt, @@ -252,7 +251,20 @@ class FailTicket(Ticket): """ Returns failures / attempt count or artificial retry count increased for bad IPs """ - return max(self._retry, self._data['failures']) + return self._retry + + def adjustTime(self, time, maxTime): + """ Adjust time of ticket and current attempts count considering given maxTime + as estimation from rate by previous known interval (if it exceeds the findTime) + """ + if time > self._time: + # expand current interval and attemps count (considering maxTime): + if self._firstTime < time - maxTime: + # adjust retry calculated as estimation from rate by previous known interval: + self._retry = int(round(self._retry / float(time - self._firstTime) * maxTime)) + self._firstTime = time - maxTime + # last time of failure: + self._time = time def inc(self, matches=None, attempt=1, count=1): self._retry += count @@ -264,19 +276,6 @@ class FailTicket(Ticket): else: self._data['matches'] = matches - def setLastTime(self, value): - if value > self._time: - self._time = value - - def getLastTime(self): - return self._time - - def getLastReset(self): - return self._lastReset - - def setLastReset(self, value): - self._lastReset = value - @staticmethod def wrap(o): o.__class__ = FailTicket diff --git a/fail2ban/server/transmitter.py b/fail2ban/server/transmitter.py index f83e9d5f..31b729b0 100644 --- a/fail2ban/server/transmitter.py +++ b/fail2ban/server/transmitter.py @@ -118,6 +118,9 @@ class Transmitter: if len(value) == 1 and value[0] == "--all": return self.__server.setUnbanIP() return self.__server.setUnbanIP(None, value) + elif name == "banned": + # check IP is banned in all jails: + return self.__server.banned(None, command[1:]) elif name == "echo": return command[1:] elif name == "server-status": @@ -274,7 +277,8 @@ class Transmitter: value = command[2] self.__server.setPrefRegex(name, value) if self.__quiet: return - return self.__server.getPrefRegex(name) + v = self.__server.getPrefRegex(name) + return v.getRegex() if v else "" elif command[1] == "addfailregex": value = command[2] self.__server.addFailRegex(name, value, multiple=multiple) @@ -430,7 +434,10 @@ class Transmitter: return None else: return db.purgeage - # Filter + # Jail, Filter + elif command[1] == "banned": + # check IP is banned in all jails: + return self.__server.banned(name, command[2:]) elif command[1] == "logpath": return self.__server.getLogPath(name) elif command[1] == "logencoding": @@ -446,7 +453,8 @@ class Transmitter: elif command[1] == "ignorecache": return self.__server.getIgnoreCache(name) elif command[1] == "prefregex": - return self.__server.getPrefRegex(name) + v = self.__server.getPrefRegex(name) + return v.getRegex() if v else "" elif command[1] == "failregex": return self.__server.getFailRegex(name) elif command[1] == "ignoreregex": diff --git a/fail2ban/server/utils.py b/fail2ban/server/utils.py index d4461a7d..294d147f 100644 --- a/fail2ban/server/utils.py +++ b/fail2ban/server/utils.py @@ -125,6 +125,10 @@ class Utils(): with self.__lock: self._cache.pop(k, None) + def clear(self): + with self.__lock: + self._cache.clear() + @staticmethod def setFBlockMode(fhandle, value): @@ -260,7 +264,6 @@ class Utils(): if stdout is not None and stdout != '' and std_level >= logSys.getEffectiveLevel(): for l in stdout.splitlines(): logSys.log(std_level, "%x -- stdout: %r", realCmdId, uni_decode(l)) - popen.stdout.close() if popen.stderr: try: if retcode is None or retcode < 0: @@ -271,7 +274,9 @@ class Utils(): if stderr is not None and stderr != '' and std_level >= logSys.getEffectiveLevel(): for l in stderr.splitlines(): logSys.log(std_level, "%x -- stderr: %r", realCmdId, uni_decode(l)) - popen.stderr.close() + + if popen.stdout: popen.stdout.close() + if popen.stderr: popen.stderr.close() success = False if retcode in success_codes: diff --git a/fail2ban/tests/actionstestcase.py b/fail2ban/tests/actionstestcase.py index a3c14a4b..7b85ff94 100644 --- a/fail2ban/tests/actionstestcase.py +++ b/fail2ban/tests/actionstestcase.py @@ -96,6 +96,8 @@ class ExecuteActions(LogCaptureTestCase): self.assertLogged("stdout: %r" % 'ip flush', "stdout: %r" % 'ip stop') self.assertEqual(self.__actions.status(),[("Currently banned", 0 ), ("Total banned", 0 ), ("Banned IP list", [] )]) + self.assertEqual(self.__actions.status('short'),[("Currently banned", 0 ), + ("Total banned", 0 )]) def testAddActionPython(self): self.__actions.add( diff --git a/fail2ban/tests/actiontestcase.py b/fail2ban/tests/actiontestcase.py index 1a00c040..d45c3171 100644 --- a/fail2ban/tests/actiontestcase.py +++ b/fail2ban/tests/actiontestcase.py @@ -252,7 +252,7 @@ class CommandActionTest(LogCaptureTestCase): delattr(self.__action, 'ac') # produce self-referencing query except: self.assertRaisesRegexp(ValueError, r"possible self referencing definitions in query", - lambda: self.__action.replaceTag(">>>>>>>>>>>>>>>>>>>>", + lambda: self.__action.replaceTag(""*30, self.__action._properties, conditional="family=inet6") ) diff --git a/fail2ban/tests/banmanagertestcase.py b/fail2ban/tests/banmanagertestcase.py index a5b37ef6..3be31bc5 100644 --- a/fail2ban/tests/banmanagertestcase.py +++ b/fail2ban/tests/banmanagertestcase.py @@ -154,6 +154,21 @@ class AddFailure(unittest.TestCase): finally: self.__banManager.setBanTime(btime) + def testBanList(self): + tickets = [ + BanTicket('192.0.2.1', 1167605999.0), + BanTicket('192.0.2.2', 1167605999.0), + ] + tickets[1].setBanTime(-1) + for t in tickets: + self.__banManager.addBanTicket(t) + self.assertSortedEqual(self.__banManager.getBanList(ordered=True, withTime=True), + [ + '192.0.2.1 \t2006-12-31 23:59:59 + 600 = 2007-01-01 00:09:59', + '192.0.2.2 \t2006-12-31 23:59:59 + -1 = 9999-12-31 23:59:59' + ] + ) + class StatusExtendedCymruInfo(unittest.TestCase): def setUp(self): diff --git a/fail2ban/tests/clientreadertestcase.py b/fail2ban/tests/clientreadertestcase.py index d39860f4..2cfaff77 100644 --- a/fail2ban/tests/clientreadertestcase.py +++ b/fail2ban/tests/clientreadertestcase.py @@ -87,6 +87,21 @@ option = %s self.assertTrue(self.c.read(f)) # we got some now return self.c.getOptions('section', [("int", 'option')])['option'] + def testConvert(self): + self.c.add_section("Definition") + self.c.set("Definition", "a", "1") + self.c.set("Definition", "b", "1") + self.c.set("Definition", "c", "test") + opts = self.c.getOptions("Definition", + (('int', 'a', 0), ('bool', 'b', 0), ('int', 'c', 0))) + self.assertSortedEqual(opts, {'a': 1, 'b': True, 'c': 0}) + opts = self.c.getOptions("Definition", + (('int', 'a'), ('bool', 'b'), ('int', 'c'))) + self.assertSortedEqual(opts, {'a': 1, 'b': True, 'c': None}) + opts = self.c.getOptions("Definition", + {'a': ('int', 0), 'b': ('bool', 0), 'c': ('int', 0)}) + self.assertSortedEqual(opts, {'a': 1, 'b': True, 'c': 0}) + def testInaccessibleFile(self): f = os.path.join(self.d, "d.conf") # inaccessible file self._write('d.conf', 0) @@ -249,6 +264,17 @@ class JailReaderTest(LogCaptureTestCase): def __init__(self, *args, **kwargs): super(JailReaderTest, self).__init__(*args, **kwargs) + def testSplitWithOptions(self): + # covering all separators - new-line and spaces: + for sep in ('\n', '\t', ' '): + self.assertEqual(splitWithOptions('a%sb' % (sep,)), ['a', 'b']) + self.assertEqual(splitWithOptions('a[x=y]%sb' % (sep,)), ['a[x=y]', 'b']) + self.assertEqual(splitWithOptions('a[x=y][z=z]%sb' % (sep,)), ['a[x=y][z=z]', 'b']) + self.assertEqual(splitWithOptions('a[x="y][z"]%sb' % (sep,)), ['a[x="y][z"]', 'b']) + self.assertEqual(splitWithOptions('a[x="y z"]%sb' % (sep,)), ['a[x="y z"]', 'b']) + self.assertEqual(splitWithOptions('a[x="y\tz"]%sb' % (sep,)), ['a[x="y\tz"]', 'b']) + self.assertEqual(splitWithOptions('a[x="y\nz"]%sb' % (sep,)), ['a[x="y\nz"]', 'b']) + def testIncorrectJail(self): jail = JailReader('XXXABSENTXXX', basedir=CONFIG_DIR, share_config=CONFIG_DIR_SHARE_CFG) self.assertRaises(ValueError, jail.read) @@ -328,7 +354,22 @@ class JailReaderTest(LogCaptureTestCase): self.assertFalse(len(o) > 2 and o[2].endswith('regex')) i += 1 if i > usednsidx: break - + + def testLogTypeOfBackendInJail(self): + unittest.F2B.SkipIfCfgMissing(stock=True); # expected include of common.conf + # test twice to check cache works peoperly: + for i in (1, 2): + # backend-related, overwritten in definition, specified in init parameters: + for prefline in ('JRNL', 'FILE', 'TEST', 'INIT'): + jail = JailReader('checklogtype_'+prefline.lower(), basedir=IMPERFECT_CONFIG, + share_config=IMPERFECT_CONFIG_SHARE_CFG, force_enable=True) + self.assertTrue(jail.read()) + self.assertTrue(jail.getOptions()) + stream = jail.convert() + # 'JRNL' for systemd, 'FILE' for file backend, 'TEST' for custom logtype (overwrite it): + self.assertEqual([['set', jail.getName(), 'addfailregex', '^%s failure from $' % prefline]], + [o for o in stream if len(o) > 2 and o[2] == 'addfailregex']) + def testSplitOption(self): # Simple example option = "mail-whois[name=SSH]" @@ -468,14 +509,12 @@ class JailReaderTest(LogCaptureTestCase): self.assertRaises(NoSectionError, c.getOptions, 'test', {}) -class FilterReaderTest(unittest.TestCase): - - def __init__(self, *args, **kwargs): - super(FilterReaderTest, self).__init__(*args, **kwargs) - self.__share_cfg = {} +class FilterReaderTest(LogCaptureTestCase): def testConvert(self): - output = [['multi-set', 'testcase01', 'addfailregex', [ + output = [ + ['set', 'testcase01', 'maxlines', 1], + ['multi-set', 'testcase01', 'addfailregex', [ "^\\s*(?:\\S+ )?(?:kernel: \\[\\d+\\.\\d+\\] )?(?:@vserver_\\S+ )" "?(?:(?:\\[\\d+\\])?:\\s+[\\[\\(]?sshd(?:\\(\\S+\\))?[\\]\\)]?:?|" "[\\[\\(]?sshd(?:\\(\\S+\\))?[\\]\\)]?:?(?:\\[\\d+\\])?:)?\\s*(?:" @@ -497,7 +536,6 @@ class FilterReaderTest(unittest.TestCase): ['set', 'testcase01', 'addjournalmatch', "FIELD= with spaces ", "+", "AFIELD= with + char and spaces"], ['set', 'testcase01', 'datepattern', "%Y %m %d %H:%M:%S"], - ['set', 'testcase01', 'maxlines', 1], # Last for overide test ] filterReader = FilterReader("testcase01", "testcase01", {}) filterReader.setBaseDir(TEST_FILES_DIR) @@ -514,9 +552,18 @@ class FilterReaderTest(unittest.TestCase): filterReader.read() #filterReader.getOptions(["failregex", "ignoreregex"]) filterReader.getOptions(None) - output[-1][-1] = "5" + output[0][-1] = 5; # maxlines = 5 self.assertSortedEqual(filterReader.convert(), output) + def testConvertOptions(self): + filterReader = FilterReader("testcase01", "testcase01", {'maxlines': '', 'test': 'X'}, + share_config=TEST_FILES_DIR_SHARE_CFG, basedir=TEST_FILES_DIR) + filterReader.read() + filterReader.getOptions(None) + opts = filterReader.getCombined(); + self.assertNotEqual(opts['maxlines'], 'X'); # wrong int value 'X' for 'maxlines' + self.assertLogged("Wrong int value 'X' for 'maxlines'. Using default one:") + def testFilterReaderSubstitionDefault(self): output = [['set', 'jailname', 'addfailregex', 'to=sweet@example.com fromip=']] filterReader = FilterReader('substition', "jailname", {}, @@ -526,6 +573,17 @@ class FilterReaderTest(unittest.TestCase): c = filterReader.convert() self.assertSortedEqual(c, output) + def testFilterReaderSubstKnown(self): + # testcase02.conf + testcase02.local, test covering that known/option is not overridden + # with unmodified (not available) value of option from .local config file, so wouldn't + # cause self-recursion if option already has a reference to known/option in .conf file. + filterReader = FilterReader('testcase02', "jailname", {}, + share_config=TEST_FILES_DIR_SHARE_CFG, basedir=TEST_FILES_DIR) + filterReader.read() + filterReader.getOptions(None) + opts = filterReader.getCombined() + self.assertTrue('sshd' in opts['failregex']) + def testFilterReaderSubstitionSet(self): output = [['set', 'jailname', 'addfailregex', 'to=sour@example.com fromip=']] filterReader = FilterReader('substition', "jailname", {'honeypot': 'sour@example.com'}, diff --git a/fail2ban/tests/config/filter.d/checklogtype.conf b/fail2ban/tests/config/filter.d/checklogtype.conf new file mode 100644 index 00000000..4d700fff --- /dev/null +++ b/fail2ban/tests/config/filter.d/checklogtype.conf @@ -0,0 +1,31 @@ +# Fail2Ban configuration file +# + +[INCLUDES] + +# Read common prefixes (logtype is set in default section) +before = ../../../../config/filter.d/common.conf + +[Definition] + +_daemon = test + +failregex = ^/__prefix_line> failure from $ +ignoreregex = + +# following sections define prefix line considering logtype: + +# backend-related (retrieved from backend, overwrite default): +[lt_file] +__prefix_line = FILE + +[lt_journal] +__prefix_line = JRNL + +# specified in definition section of filter (see filter checklogtype_test.conf): +[lt_test] +__prefix_line = TEST + +# specified in init parameter of jail (see ../jail.conf, jail checklogtype_init): +[lt_init] +__prefix_line = INIT diff --git a/fail2ban/tests/config/filter.d/checklogtype_test.conf b/fail2ban/tests/config/filter.d/checklogtype_test.conf new file mode 100644 index 00000000..a76f5fcf --- /dev/null +++ b/fail2ban/tests/config/filter.d/checklogtype_test.conf @@ -0,0 +1,12 @@ +# Fail2Ban configuration file +# + +[INCLUDES] + +# Read common prefixes (logtype is set in default section) +before = checklogtype.conf + +[Definition] + +# overwrite logtype in definition (no backend anymore): +logtype = test \ No newline at end of file diff --git a/fail2ban/tests/config/filter.d/zzz-sshd-obsolete-multiline.conf b/fail2ban/tests/config/filter.d/zzz-sshd-obsolete-multiline.conf index 549797af..ad8adeb6 100644 --- a/fail2ban/tests/config/filter.d/zzz-sshd-obsolete-multiline.conf +++ b/fail2ban/tests/config/filter.d/zzz-sshd-obsolete-multiline.conf @@ -37,7 +37,7 @@ __pam_auth = pam_[a-z]+ cmnfailre = ^%(__prefix_line_sl)s[aA]uthentication (?:failure|error|failed) for .* from ( via \S+)?\s*%(__suff)s$ ^%(__prefix_line_sl)sUser not known to the underlying authentication module for .* from \s*%(__suff)s$ ^%(__prefix_line_sl)sFailed \S+ for invalid user (?P\S+)|(?:(?! from ).)*? from %(__on_port_opt)s(?: ssh\d*)?(?(cond_user): |(?:(?:(?! from ).)*)$) - ^%(__prefix_line_sl)sFailed \b(?!publickey)\S+ for (?Pinvalid user )?(?P\S+)|(?(cond_inv)(?:(?! from ).)*?|[^:]+) from %(__on_port_opt)s(?: ssh\d*)?(?(cond_user): |(?:(?:(?! from ).)*)$) + ^%(__prefix_line_sl)sFailed (?:publickey|\S+) for (?Pinvalid user )?(?P\S+)|(?(cond_inv)(?:(?! from ).)*?|[^:]+) from %(__on_port_opt)s(?: ssh\d*)?(?(cond_user): |(?:(?:(?! from ).)*)$) ^%(__prefix_line_sl)sROOT LOGIN REFUSED FROM ^%(__prefix_line_sl)s[iI](?:llegal|nvalid) user .*? from %(__suff)s$ ^%(__prefix_line_sl)sUser .+ from not allowed because not listed in AllowUsers\s*%(__suff)s$ @@ -57,11 +57,10 @@ mdre-normal = mdre-ddos = ^%(__prefix_line_sl)sDid not receive identification string from ^%(__prefix_line_sl)sBad protocol version identification '.*' from - ^%(__prefix_line_sl)sConnection closed by%(__authng_user)s %(__on_port_opt)s\s+\[preauth\]\s*$ - ^%(__prefix_line_sl)sConnection reset by + ^%(__prefix_line_sl)sConnection (?:closed|reset) by%(__authng_user)s %(__on_port_opt)s\s+\[preauth\]\s*$ ^%(__prefix_line_ml1)sSSH: Server;Ltype: (?:Authname|Version|Kex);Remote: -\d+;[A-Z]\w+:.*%(__prefix_line_ml2)sRead from socket failed: Connection reset by peer%(__suff)s$ -mdre-extra = ^%(__prefix_line_sl)sReceived disconnect from %(__on_port_opt)s:\s*14: No supported authentication methods available +mdre-extra = ^%(__prefix_line_sl)sReceived disconnect from %(__on_port_opt)s:\s*14: No(?: supported)? authentication methods available ^%(__prefix_line_sl)sUnable to negotiate with %(__on_port_opt)s: no matching <__alg_match> found. ^%(__prefix_line_ml1)sConnection from %(__on_port_opt)s%(__prefix_line_ml2)sUnable to negotiate a <__alg_match> ^%(__prefix_line_ml1)sConnection from %(__on_port_opt)s%(__prefix_line_ml2)sno matching <__alg_match> found: diff --git a/fail2ban/tests/config/jail.conf b/fail2ban/tests/config/jail.conf index de5bbbdc..b1a1707b 100644 --- a/fail2ban/tests/config/jail.conf +++ b/fail2ban/tests/config/jail.conf @@ -74,3 +74,28 @@ journalmatch = _COMM=test maxlines = 2 usedns = no enabled = false + +[checklogtype_jrnl] +filter = checklogtype +backend = systemd +action = action +enabled = false + +[checklogtype_file] +filter = checklogtype +backend = polling +logpath = README.md +action = action +enabled = false + +[checklogtype_test] +filter = checklogtype_test +backend = systemd +action = action +enabled = false + +[checklogtype_init] +filter = checklogtype_test[logtype=init] +backend = systemd +action = action +enabled = false diff --git a/fail2ban/tests/databasetestcase.py b/fail2ban/tests/databasetestcase.py index a1df2993..a8e2ceae 100644 --- a/fail2ban/tests/databasetestcase.py +++ b/fail2ban/tests/databasetestcase.py @@ -262,6 +262,15 @@ class DatabaseTest(LogCaptureTestCase): self.db.addLog(self.jail, self.fileContainer), None) os.remove(filename) + def testUpdateJournal(self): + self.testAddJail() # Jail required + # not yet updated: + self.assertEqual(self.db.getJournalPos(self.jail, 'systemd-journal'), None) + # update 3 times (insert and 2 updates) and check it was set (and overwritten): + for t in (1500000000, 1500000001, 1500000002): + self.db.updateJournal(self.jail, 'systemd-journal', t, 'TEST'+str(t)) + self.assertEqual(self.db.getJournalPos(self.jail, 'systemd-journal'), t) + def testAddBan(self): self.testAddJail() ticket = FailTicket("127.0.0.1", 0, ["abc\n"]) @@ -534,6 +543,7 @@ class DatabaseTest(LogCaptureTestCase): # test action together with database functionality self.testAddJail() # Jail required self.jail.database = self.db + self.db.addJail(self.jail) actions = Actions(self.jail) actions.add( "action_checkainfo", diff --git a/fail2ban/tests/datedetectortestcase.py b/fail2ban/tests/datedetectortestcase.py index 458f76ef..d6370fc4 100644 --- a/fail2ban/tests/datedetectortestcase.py +++ b/fail2ban/tests/datedetectortestcase.py @@ -330,6 +330,27 @@ class DateDetectorTest(LogCaptureTestCase): dt = '2005 Jun 03'; self.assertEqual(t.matchDate(dt).group(1), dt) dt = '2005 JUN 03'; self.assertEqual(t.matchDate(dt).group(1), dt) + def testNotAnchoredCollision(self): + # try for patterns with and without word boundaries: + for dp in (r'%H:%M:%S', r'{UNB}%H:%M:%S'): + dd = DateDetector() + dd.appendTemplate(dp) + # boundary of timestamp changes right and left (and time is left and right in line): + for fmt in ('%s test', '%8s test', 'test %s', 'test %8s'): + for dt in ( + '00:01:02', + '00:01:2', + '00:1:2', + '0:1:2', + '00:1:2', + '00:01:2', + '00:01:02', + '0:1:2', + '00:01:02', + ): + t = dd.getTime(fmt % dt) + self.assertEqual((t[0], t[1].group()), (1123970462.0, dt)) + def testAmbiguousInOrderedTemplates(self): dd = self.datedetector for (debit, line, cnt) in ( diff --git a/fail2ban/tests/dummyjail.py b/fail2ban/tests/dummyjail.py index ec960290..fdeced8f 100644 --- a/fail2ban/tests/dummyjail.py +++ b/fail2ban/tests/dummyjail.py @@ -40,7 +40,6 @@ class DummyJail(Jail): self.lock = Lock() self.queue = [] super(DummyJail, self).__init__(name=name, backend=backend) - self.__db = None self.__actions = DummyActions(self) def __len__(self): @@ -55,6 +54,10 @@ class DummyJail(Jail): with self.lock: return bool(self.queue) + @property + def hasFailTickets(self): + return bool(self.queue) + def putFailTicket(self, ticket): with self.lock: self.queue.append(ticket) @@ -74,14 +77,6 @@ class DummyJail(Jail): def idle(self, value): pass - @property - def database(self): - return self.__db; - - @database.setter - def database(self, value): - self.__db = value; - @property def actions(self): return self.__actions; diff --git a/fail2ban/tests/fail2banclienttestcase.py b/fail2ban/tests/fail2banclienttestcase.py index 29adb122..2a38daa6 100644 --- a/fail2ban/tests/fail2banclienttestcase.py +++ b/fail2ban/tests/fail2banclienttestcase.py @@ -37,7 +37,7 @@ from threading import Thread from ..client import fail2banclient, fail2banserver, fail2bancmdline from ..client.fail2bancmdline import Fail2banCmdLine -from ..client.fail2banclient import exec_command_line as _exec_client, VisualWait +from ..client.fail2banclient import exec_command_line as _exec_client, CSocket, VisualWait from ..client.fail2banserver import Fail2banServer, exec_command_line as _exec_server from .. import protocol from ..server import server @@ -343,6 +343,7 @@ def with_foreground_server_thread(startextra={}): # to wait for end of server, default accept any exit code, because multi-threaded, # thus server can exit in-between... def _stopAndWaitForServerEnd(code=(SUCCESS, FAILED)): + tearDownMyTime() # if seems to be down - try to catch end phase (wait a bit for end:True to recognize down state): if not phase.get('end', None) and not os.path.exists(pjoin(tmp, "f2b.pid")): Utils.wait_for(lambda: phase.get('end', None) is not None, MID_WAITTIME) @@ -452,6 +453,14 @@ class Fail2banClientServerBase(LogCaptureTestCase): self.assertRaises(exitType, self.exec_command_line[0], (self.exec_command_line[1:] + startparams + args)) + def execCmdDirect(self, startparams, *args): + sock = startparams[startparams.index('-s')+1] + s = CSocket(sock) + try: + return s.send(args) + finally: + s.close() + # # Common tests # @@ -469,14 +478,14 @@ class Fail2banClientServerBase(LogCaptureTestCase): @with_foreground_server_thread(startextra={'f2b_local':( "[Thread]", - "stacksize = 32" + "stacksize = 128" "", )}) def testStartForeground(self, tmp, startparams): # check thread options were set: self.pruneLog() self.execCmd(SUCCESS, startparams, "get", "thread") - self.assertLogged("{'stacksize': 32}") + self.assertLogged("{'stacksize': 128}") # several commands to server: self.execCmd(SUCCESS, startparams, "ping") self.execCmd(FAILED, startparams, "~~unknown~cmd~failed~~") @@ -646,12 +655,6 @@ class Fail2banClientTest(Fail2banClientServerBase): self.assertLogged("Base configuration directory " + pjoin(tmp, "miss") + " does not exist") self.pruneLog() - ## wrong socket - self.execCmd(FAILED, (), - "--async", "-c", pjoin(tmp, "config"), "-s", pjoin(tmp, "miss/f2b.sock"), "start") - self.assertLogged("There is no directory " + pjoin(tmp, "miss") + " to contain the socket file") - self.pruneLog() - ## not running self.execCmd(FAILED, (), "-c", pjoin(tmp, "config"), "-s", pjoin(tmp, "f2b.sock"), "reload") @@ -747,12 +750,6 @@ class Fail2banServerTest(Fail2banClientServerBase): self.assertLogged("Base configuration directory " + pjoin(tmp, "miss") + " does not exist") self.pruneLog() - ## wrong socket - self.execCmd(FAILED, (), - "-c", pjoin(tmp, "config"), "-x", "-s", pjoin(tmp, "miss/f2b.sock")) - self.assertLogged("There is no directory " + pjoin(tmp, "miss") + " to contain the socket file") - self.pruneLog() - ## already exists: open(pjoin(tmp, "f2b.sock"), 'a').close() self.execCmd(FAILED, (), @@ -891,7 +888,7 @@ class Fail2banServerTest(Fail2banClientServerBase): "action = ", " test-action2[name='%(__name__)s', restore='restored: ', info=', err-code: ']" \ if 2 in actions else "", - " test-action2[name='%(__name__)s', actname=test-action3, _exec_once=1, restore='restored: ']" + " test-action2[name='%(__name__)s', actname=test-action3, _exec_once=1, restore='restored: '," " actionflush=<_use_flush_>]" \ if 3 in actions else "", "logpath = " + test2log, @@ -1004,8 +1001,8 @@ class Fail2banServerTest(Fail2banClientServerBase): # leave action2 just to test restored interpolation: _write_jail_cfg(actions=[2,3]) - # write new failures: self.pruneLog("[test-phase 2b]") + # write new failures: _write_file(test2log, "w+", *( (str(int(MyTime.time())) + " error 403 from 192.0.2.2: test 2",) * 3 + (str(int(MyTime.time())) + " error 403 from 192.0.2.3: test 2",) * 3 + @@ -1018,13 +1015,19 @@ class Fail2banServerTest(Fail2banClientServerBase): self.assertLogged( "2 ticket(s) in 'test-jail2", "5 ticket(s) in 'test-jail1", all=True, wait=MID_WAITTIME) + # ban manually to cover restore in restart (phase 2c): + self.execCmd(SUCCESS, startparams, + "set", "test-jail2", "banip", "192.0.2.9") + self.assertLogged( + "3 ticket(s) in 'test-jail2", wait=MID_WAITTIME) self.assertLogged( "[test-jail1] Ban 192.0.2.2", "[test-jail1] Ban 192.0.2.3", "[test-jail1] Ban 192.0.2.4", "[test-jail1] Ban 192.0.2.8", "[test-jail2] Ban 192.0.2.4", - "[test-jail2] Ban 192.0.2.8", all=True) + "[test-jail2] Ban 192.0.2.8", + "[test-jail2] Ban 192.0.2.9", all=True) # test ips at all not visible for jail2: self.assertNotLogged( "[test-jail2] Found 192.0.2.2", @@ -1034,6 +1037,30 @@ class Fail2banServerTest(Fail2banClientServerBase): all=True) # if observer available wait for it becomes idle (write all tickets to db): _observer_wait_idle() + # test banned command: + self.assertSortedEqual(self.execCmdDirect(startparams, + 'banned'), (0, [ + {'test-jail1': ['192.0.2.4', '192.0.2.1', '192.0.2.8', '192.0.2.3', '192.0.2.2']}, + {'test-jail2': ['192.0.2.4', '192.0.2.9', '192.0.2.8']} + ] + )) + self.assertSortedEqual(self.execCmdDirect(startparams, + 'banned', '192.0.2.1', '192.0.2.4', '192.0.2.222'), (0, [ + ['test-jail1'], ['test-jail1', 'test-jail2'], [] + ] + )) + self.assertSortedEqual(self.execCmdDirect(startparams, + 'get', 'test-jail1', 'banned')[1], [ + '192.0.2.4', '192.0.2.1', '192.0.2.8', '192.0.2.3', '192.0.2.2']) + self.assertSortedEqual(self.execCmdDirect(startparams, + 'get', 'test-jail2', 'banned')[1], [ + '192.0.2.4', '192.0.2.9', '192.0.2.8']) + self.assertEqual(self.execCmdDirect(startparams, + 'get', 'test-jail1', 'banned', '192.0.2.3')[1], 1) + self.assertEqual(self.execCmdDirect(startparams, + 'get', 'test-jail1', 'banned', '192.0.2.9')[1], 0) + self.assertEqual(self.execCmdDirect(startparams, + 'get', 'test-jail1', 'banned', '192.0.2.3', '192.0.2.9')[1], [1, 0]) # rotate logs: _write_file(test1log, "w+") @@ -1046,15 +1073,17 @@ class Fail2banServerTest(Fail2banClientServerBase): self.assertLogged( "Reload finished.", "Restore Ban", - "2 ticket(s) in 'test-jail2", all=True, wait=MID_WAITTIME) + "3 ticket(s) in 'test-jail2", all=True, wait=MID_WAITTIME) # stop/start and unban/restore ban: self.assertLogged( - "Jail 'test-jail2' stopped", - "Jail 'test-jail2' started", "[test-jail2] Unban 192.0.2.4", "[test-jail2] Unban 192.0.2.8", + "[test-jail2] Unban 192.0.2.9", + "Jail 'test-jail2' stopped", + "Jail 'test-jail2' started", "[test-jail2] Restore Ban 192.0.2.4", - "[test-jail2] Restore Ban 192.0.2.8", all=True + "[test-jail2] Restore Ban 192.0.2.8", + "[test-jail2] Restore Ban 192.0.2.9", all=True ) # test restored is 1 (only test-action2): self.assertLogged( @@ -1099,7 +1128,8 @@ class Fail2banServerTest(Fail2banClientServerBase): "Jail 'test-jail2' stopped", "Jail 'test-jail2' started", "[test-jail2] Unban 192.0.2.4", - "[test-jail2] Unban 192.0.2.8", all=True + "[test-jail2] Unban 192.0.2.8", + "[test-jail2] Unban 192.0.2.9", all=True ) # test unban (action2): self.assertLogged( @@ -1173,13 +1203,41 @@ class Fail2banServerTest(Fail2banClientServerBase): self.assertNotLogged("[test-jail1] Found 192.0.2.5") # unban single ips: - self.pruneLog("[test-phase 6]") + self.pruneLog("[test-phase 6a]") self.execCmd(SUCCESS, startparams, "--async", "unban", "192.0.2.5", "192.0.2.6") self.assertLogged( "192.0.2.5 is not banned", "[test-jail1] Unban 192.0.2.6", all=True, wait=MID_WAITTIME ) + # unban ips by subnet (cidr/mask): + self.pruneLog("[test-phase 6b]") + self.execCmd(SUCCESS, startparams, + "--async", "unban", "192.0.2.2/31") + self.assertLogged( + "[test-jail1] Unban 192.0.2.2", + "[test-jail1] Unban 192.0.2.3", all=True, wait=MID_WAITTIME + ) + self.execCmd(SUCCESS, startparams, + "--async", "unban", "192.0.2.8/31", "192.0.2.100/31") + self.assertLogged( + "[test-jail1] Unban 192.0.2.8", + "192.0.2.100/31 is not banned", all=True, wait=MID_WAITTIME) + + # ban/unban subnet(s): + self.pruneLog("[test-phase 6c]") + self.execCmd(SUCCESS, startparams, + "--async", "set", "test-jail1", "banip", "192.0.2.96/28", "192.0.2.112/28") + self.assertLogged( + "[test-jail1] Ban 192.0.2.96/28", + "[test-jail1] Ban 192.0.2.112/28", all=True, wait=MID_WAITTIME + ) + self.execCmd(SUCCESS, startparams, + "--async", "set", "test-jail1", "unbanip", "192.0.2.64/26"); # contains both subnets .96/28 and .112/28 + self.assertLogged( + "[test-jail1] Unban 192.0.2.96/28", + "[test-jail1] Unban 192.0.2.112/28", all=True, wait=MID_WAITTIME + ) # reload all (one jail) with unban all: self.pruneLog("[test-phase 7]") @@ -1190,8 +1248,6 @@ class Fail2banServerTest(Fail2banClientServerBase): self.assertLogged( "Jail 'test-jail1' reloaded", "[test-jail1] Unban 192.0.2.1", - "[test-jail1] Unban 192.0.2.2", - "[test-jail1] Unban 192.0.2.3", "[test-jail1] Unban 192.0.2.4", all=True ) # no restart occurred, no more ban (unbanned all using option "--unban"): @@ -1199,8 +1255,6 @@ class Fail2banServerTest(Fail2banClientServerBase): "Jail 'test-jail1' stopped", "Jail 'test-jail1' started", "[test-jail1] Ban 192.0.2.1", - "[test-jail1] Ban 192.0.2.2", - "[test-jail1] Ban 192.0.2.3", "[test-jail1] Ban 192.0.2.4", all=True ) @@ -1570,6 +1624,37 @@ class Fail2banServerTest(Fail2banClientServerBase): self.assertLogged( "192.0.2.11", "+ 600 =", all=True, wait=MID_WAITTIME) + # test stop with busy observer: + self.pruneLog("[test-phase end) stop on busy observer]") + tearDownMyTime() + a = {'state': 0} + obsMain = Observers.Main + def _long_action(): + logSys.info('++ observer enters busy state ...') + a['state'] = 1 + Utils.wait_for(lambda: a['state'] == 2, MAX_WAITTIME) + obsMain.db_purge(); # does nothing (db is already None) + logSys.info('-- observer leaves busy state.') + obsMain.add('call', _long_action) + obsMain.add('call', lambda: None) + # wait observer enter busy state: + Utils.wait_for(lambda: a['state'] == 1, MAX_WAITTIME) + # overwrite default wait time (normally 5 seconds): + obsMain_stop = obsMain.stop + def _stop(wtime=(0.01 if unittest.F2B.fast else 0.1), forceQuit=True): + return obsMain_stop(wtime, forceQuit) + obsMain.stop = _stop + # stop server and wait for end: + self.stopAndWaitForServerEnd(SUCCESS) + # check observer and db state: + self.assertNotLogged('observer leaves busy state') + self.assertFalse(obsMain.idle) + self.assertEqual(obsMain._ObserverThread__db, None) + # server is exited without wait for observer, stop it now: + a['state'] = 2 + self.assertLogged('observer leaves busy state', wait=True) + obsMain.join() + # test multiple start/stop of the server (threaded in foreground) -- if False: # pragma: no cover @with_foreground_server_thread() diff --git a/fail2ban/tests/fail2banregextestcase.py b/fail2ban/tests/fail2banregextestcase.py index c09c4171..c663c50b 100644 --- a/fail2ban/tests/fail2banregextestcase.py +++ b/fail2ban/tests/fail2banregextestcase.py @@ -81,15 +81,32 @@ def _test_exec_command_line(*args): return _exit_code STR_00 = "Dec 31 11:59:59 [sshd] error: PAM: Authentication failure for kevin from 192.0.2.0" +STR_00_NODT = "[sshd] error: PAM: Authentication failure for kevin from 192.0.2.0" RE_00 = r"(?:(?:Authentication failure|Failed [-/\w+]+) for(?: [iI](?:llegal|nvalid) user)?|[Ii](?:llegal|nvalid) user|ROOT LOGIN REFUSED) .*(?: from|FROM) " -RE_00_ID = r"Authentication failure for .*? from $" -RE_00_USER = r"Authentication failure for .*? from $" +RE_00_ID = r"Authentication failure for .*? from $" +RE_00_USER = r"Authentication failure for .*? from $" FILENAME_01 = os.path.join(TEST_FILES_DIR, "testcase01.log") FILENAME_02 = os.path.join(TEST_FILES_DIR, "testcase02.log") FILENAME_WRONGCHAR = os.path.join(TEST_FILES_DIR, "testcase-wrong-char.log") +# STR_ML_SSHD -- multiline log-excerpt with two sessions: +# 192.0.2.1 (sshd[32307]) makes 2 failed attempts using public keys (without "Disconnecting: Too many authentication"), +# and delayed success on accepted (STR_ML_SSHD_OK) or no success by close on preauth phase (STR_ML_SSHD_FAIL) +# 192.0.2.2 (sshd[32310]) makes 2 failed attempts using public keys (with "Disconnecting: Too many authentication"), +# and closed on preauth phase +STR_ML_SSHD = """Nov 28 09:16:03 srv sshd[32307]: Failed publickey for git from 192.0.2.1 port 57904 ssh2: ECDSA 0e:ff:xx:xx:xx:xx:xx:xx:xx:xx:xx:... +Nov 28 09:16:03 srv sshd[32307]: Failed publickey for git from 192.0.2.1 port 57904 ssh2: RSA 04:bc:xx:xx:xx:xx:xx:xx:xx:xx:xx:... +Nov 28 09:16:03 srv sshd[32307]: Postponed publickey for git from 192.0.2.1 port 57904 ssh2 [preauth] +Nov 28 09:16:05 srv sshd[32310]: Failed publickey for git from 192.0.2.2 port 57910 ssh2: ECDSA 1e:fe:xx:xx:xx:xx:xx:xx:xx:xx:xx:... +Nov 28 09:16:05 srv sshd[32310]: Failed publickey for git from 192.0.2.2 port 57910 ssh2: RSA 14:ba:xx:xx:xx:xx:xx:xx:xx:xx:xx:... +Nov 28 09:16:05 srv sshd[32310]: Disconnecting: Too many authentication failures for git [preauth] +Nov 28 09:16:05 srv sshd[32310]: Connection closed by 192.0.2.2 [preauth]""" +STR_ML_SSHD_OK = "Nov 28 09:16:06 srv sshd[32307]: Accepted publickey for git from 192.0.2.1 port 57904 ssh2: DSA 36:48:xx:xx:xx:xx:xx:xx:xx:xx:xx:..." +STR_ML_SSHD_FAIL = "Nov 28 09:16:06 srv sshd[32307]: Connection closed by 192.0.2.1 [preauth]" + + FILENAME_SSHD = os.path.join(TEST_FILES_DIR, "logs", "sshd") FILTER_SSHD = os.path.join(CONFIG_DIR, 'filter.d', 'sshd.conf') FILENAME_ZZZ_SSHD = os.path.join(TEST_FILES_DIR, 'zzz-sshd-obsolete-multiline.log') @@ -156,7 +173,7 @@ class Fail2banRegexTest(LogCaptureTestCase): "--print-all-matched", FILENAME_01, RE_00 )) - self.assertLogged('Lines: 19 lines, 0 ignored, 13 matched, 6 missed') + self.assertLogged('Lines: 19 lines, 0 ignored, 16 matched, 3 missed') self.assertLogged('Error decoding line'); self.assertLogged('Continuing to process line ignoring invalid characters') @@ -170,7 +187,7 @@ class Fail2banRegexTest(LogCaptureTestCase): "--print-all-matched", "--raw", FILENAME_01, RE_00 )) - self.assertLogged('Lines: 19 lines, 0 ignored, 16 matched, 3 missed') + self.assertLogged('Lines: 19 lines, 0 ignored, 19 matched, 0 missed') def testDirectRE_1raw_noDns(self): self.assertTrue(_test_exec( @@ -178,7 +195,7 @@ class Fail2banRegexTest(LogCaptureTestCase): "--print-all-matched", "--raw", "--usedns=no", FILENAME_01, RE_00 )) - self.assertLogged('Lines: 19 lines, 0 ignored, 13 matched, 6 missed') + self.assertLogged('Lines: 19 lines, 0 ignored, 16 matched, 3 missed') # usage of \S+ causes raw handling automatically: self.pruneLog() self.assertTrue(_test_exec( @@ -291,10 +308,10 @@ class Fail2banRegexTest(LogCaptureTestCase): # self.assertTrue(_test_exec( "--usedns", "no", "-d", "^Epoch", "--print-all-matched", - "1490349000 FAIL: failure\nhost: 192.0.2.35", + "-L", "2", "1490349000 FAIL: failure\nhost: 192.0.2.35", r"^\s*FAIL:\s*.*\nhost:\s+$" )) - self.assertLogged('Lines: 1 lines, 0 ignored, 1 matched, 0 missed') + self.assertLogged('Lines: 2 lines, 0 ignored, 2 matched, 0 missed') def testRegexEpochPatterns(self): self.assertTrue(_test_exec( @@ -324,6 +341,23 @@ class Fail2banRegexTest(LogCaptureTestCase): self.assertTrue(_test_exec('-o', 'id', STR_00, RE_00_ID)) self.assertLogged('kevin') self.pruneLog() + # multiple id combined to a tuple (id, tuple_id): + self.assertTrue(_test_exec('-o', 'id', + '1591983743.667 192.0.2.1 192.0.2.2', + r'^\s* \S+')) + self.assertLogged(str(('192.0.2.1', '192.0.2.2'))) + self.pruneLog() + # multiple id combined to a tuple, id first - (id, tuple_id_1, tuple_id_2): + self.assertTrue(_test_exec('-o', 'id', + '1591983743.667 left 192.0.2.3 right', + r'^\s*\S+ \S+')) + self.pruneLog() + # id had higher precedence as ip-address: + self.assertTrue(_test_exec('-o', 'id', + '1591983743.667 left [192.0.2.4]:12345 right', + r'^\s*\S+ : \S+')) + self.assertLogged(str(('[192.0.2.4]:12345', 'left', 'right'))) + self.pruneLog() # row with id : self.assertTrue(_test_exec('-o', 'row', STR_00, RE_00_ID)) self.assertLogged("['kevin'", "'ip4': '192.0.2.0'", "'fid': 'kevin'", all=True) @@ -340,6 +374,73 @@ class Fail2banRegexTest(LogCaptureTestCase): self.assertTrue(_test_exec('-o', 'user', STR_00, RE_00_USER)) self.assertLogged('kevin') self.pruneLog() + # complex substitution using tags (ip, user, family): + self.assertTrue(_test_exec('-o', ', , ', STR_00, RE_00_USER)) + self.assertLogged('192.0.2.0, kevin, inet4') + self.pruneLog() + + def testNoDateTime(self): + # datepattern doesn't match: + self.assertTrue(_test_exec('-d', '{^LN-BEG}EPOCH', '-o', 'Found-ID:', STR_00_NODT, RE_00_ID)) + self.assertLogged( + "Found a match but no valid date/time found", + "Match without a timestamp:", + "Found-ID:kevin", all=True) + self.pruneLog() + # explicitly no datepattern: + self.assertTrue(_test_exec('-d', '{NONE}', '-o', 'Found-ID:', STR_00_NODT, RE_00_ID)) + self.assertLogged( + "Found-ID:kevin", all=True) + self.assertNotLogged( + "Found a match but no valid date/time found", + "Match without a timestamp:", all=True) + + self.pruneLog() + + def testFrmtOutputWrapML(self): + unittest.F2B.SkipIfCfgMissing(stock=True) + # complex substitution using tags and message (ip, user, msg): + self.assertTrue(_test_exec('-o', ', , ', + '-c', CONFIG_DIR, '--usedns', 'no', + STR_ML_SSHD + "\n" + STR_ML_SSHD_OK, 'sshd[logtype=short, publickey=invalid]')) + # be sure we don't have IP in one line and have it in another: + lines = STR_ML_SSHD.split("\n") + self.assertTrue('192.0.2.2' not in lines[-2] and '192.0.2.2' in lines[-1]) + # but both are in output "merged" with IP and user: + self.assertLogged( + '192.0.2.2, git, '+lines[-2], + '192.0.2.2, git, '+lines[-1], + all=True) + # nothing should be found for 192.0.2.1 (mode is not aggressive): + self.assertNotLogged('192.0.2.1, git, ') + + # test with publickey (nofail) - would not produce output for 192.0.2.1 because accepted: + self.pruneLog("[test-phase 1] mode=aggressive & publickey=nofail + OK (accepted)") + self.assertTrue(_test_exec('-o', ', , ', + '-c', CONFIG_DIR, '--usedns', 'no', + STR_ML_SSHD + "\n" + STR_ML_SSHD_OK, 'sshd[logtype=short, mode=aggressive]')) + self.assertLogged( + '192.0.2.2, git, '+lines[-4], + '192.0.2.2, git, '+lines[-3], + '192.0.2.2, git, '+lines[-2], + '192.0.2.2, git, '+lines[-1], + all=True) + # nothing should be found for 192.0.2.1 (access gained so failures ignored): + self.assertNotLogged('192.0.2.1, git, ') + + # now same test but "accepted" replaced with "closed" on preauth phase: + self.pruneLog("[test-phase 2] mode=aggressive & publickey=nofail + FAIL (closed on preauth)") + self.assertTrue(_test_exec('-o', ', , ', + '-c', CONFIG_DIR, '--usedns', 'no', + STR_ML_SSHD + "\n" + STR_ML_SSHD_FAIL, 'sshd[logtype=short, mode=aggressive]')) + # 192.0.2.1 should be found for every failure (2x failed key + 1x closed): + lines = STR_ML_SSHD.split("\n")[0:2] + STR_ML_SSHD_FAIL.split("\n")[-1:] + self.assertLogged( + '192.0.2.1, git, '+lines[-3], + '192.0.2.1, git, '+lines[-2], + '192.0.2.1, git, '+lines[-1], + all=True) + def testWrongFilterFile(self): # use test log as filter file to cover eror cases... @@ -420,7 +521,7 @@ class Fail2banRegexTest(LogCaptureTestCase): def testLogtypeSystemdJournal(self): # pragma: no cover if not fail2banregex.FilterSystemd: - raise unittest.SkipTest('Skip test because no systemd backand available') + raise unittest.SkipTest('Skip test because no systemd backend available') self.assertTrue(_test_exec( "systemd-journal", FILTER_ZZZ_GEN +'[journalmatch="SYSLOG_IDENTIFIER=\x01\x02dummy\x02\x01",' diff --git a/fail2ban/tests/files/filter.d/testcase02.conf b/fail2ban/tests/files/filter.d/testcase02.conf new file mode 100644 index 00000000..99b3bb45 --- /dev/null +++ b/fail2ban/tests/files/filter.d/testcase02.conf @@ -0,0 +1,12 @@ +[INCLUDES] + +# Read common prefixes. If any customizations available -- read them from +# common.local +before = testcase-common.conf + +[Definition] + +_daemon = sshd +__prefix_line = %(known/__prefix_line)s(?:\w{14,20}: )? + +failregex = %(__prefix_line)s test \ No newline at end of file diff --git a/fail2ban/tests/files/filter.d/testcase02.local b/fail2ban/tests/files/filter.d/testcase02.local new file mode 100644 index 00000000..bfc81d4b --- /dev/null +++ b/fail2ban/tests/files/filter.d/testcase02.local @@ -0,0 +1,4 @@ +[Definition] + +# no options here, coverage for testFilterReaderSubstKnown: +# avoid to overwrite known/option with unmodified (not available) value of option from .local config file \ No newline at end of file diff --git a/fail2ban/tests/files/logs/apache-modsecurity b/fail2ban/tests/files/logs/apache-modsecurity index dbb14863..7e2f8c86 100644 --- a/fail2ban/tests/files/logs/apache-modsecurity +++ b/fail2ban/tests/files/logs/apache-modsecurity @@ -6,3 +6,6 @@ # failJSON: { "time": "2018-09-28T09:18:06", "match": true , "host": "192.0.2.1", "desc": "two client entries in message (gh-2247)" } [Sat Sep 28 09:18:06 2018] [error] [client 192.0.2.1:55555] [client 192.0.2.1] ModSecurity: [file "/etc/httpd/modsecurity.d/10_asl_rules.conf"] [line "635"] [id "340069"] [rev "4"] [msg "Atomicorp.com UNSUPPORTED DELAYED Rules: Web vulnerability scanner"] [severity "CRITICAL"] Access denied with code 403 (phase 2). Pattern match "(?:nessus(?:_is_probing_you_|test)|^/w00tw00t\\\\.at\\\\.)" at REQUEST_URI. [hostname "192.81.249.191"] [uri "/w00tw00t.at.blackhats.romanian.anti-sec:)"] [unique_id "4Q6RdsBR@b4AAA65LRUAAAAA"] + +# failJSON: { "time": "2020-05-09T00:35:52", "match": true , "host": "192.0.2.2", "desc": "new format - apache 2.4 and php-fpm (gh-2717)" } +[Sat May 09 00:35:52.389262 2020] [:error] [pid 22406:tid 139985298601728] [client 192.0.2.2:47762] [client 192.0.2.2] ModSecurity: Access denied with code 401 (phase 2). Operator EQ matched 1 at IP:blocked. [file "/etc/httpd/modsecurity.d/activated_rules/modsecurity_wp_login.conf"] [line "14"] [id "500000"] [msg "Ip address blocked for 15 minutes, more than 5 login attempts in 3 minutes."] [hostname "example.com"] [uri "/wp-login.php"] [unique_id "XrYlGL5IY3I@EoLOgAAAA8"], referer: https://example.com/wp-login.php diff --git a/fail2ban/tests/files/logs/bitwarden b/fail2ban/tests/files/logs/bitwarden index 3642b3bf..0fede6c6 100644 --- a/fail2ban/tests/files/logs/bitwarden +++ b/fail2ban/tests/files/logs/bitwarden @@ -3,3 +3,9 @@ # failJSON: { "time": "2019-11-25T21:39:58", "match": true , "host": "192.168.0.21" } 2019-11-25 21:39:58.464 +01:00 [WRN] Failed login attempt, 2FA invalid. 192.168.0.21 + +# failJSON: { "time": "2019-11-25T21:39:58", "match": true , "host": "192.168.0.21" } +2019-11-25 21:39:58.464 +01:00 [Warning] Failed login attempt, 2FA invalid. 192.168.0.21 + +# failJSON: { "time": "2019-09-24T13:16:50", "match": true , "host": "192.168.0.23" } +2019-09-24T13:16:50 e5a81dbf7fd1 Bitwarden-Identity[1]: [Bit.Core.IdentityServer.ResourceOwnerPasswordValidator] Failed login attempt. 192.168.0.23 diff --git a/fail2ban/tests/files/logs/courier-smtp b/fail2ban/tests/files/logs/courier-smtp index ab99d322..6da0d0a4 100644 --- a/fail2ban/tests/files/logs/courier-smtp +++ b/fail2ban/tests/files/logs/courier-smtp @@ -8,7 +8,9 @@ Jul 4 18:39:39 mail courieresmtpd: error,relay=::ffff:1.2.3.4,from=,to=: 550 User unknown. # failJSON: { "time": "2004-11-21T23:16:17", "match": true , "host": "1.2.3.4" } Nov 21 23:16:17 server courieresmtpd: error,relay=::ffff:1.2.3.4,from=<>,to=<>: 550 User unknown. -# failJSON: { "time": "2004-08-14T12:51:04", "match": true , "host": "1.2.3.4" } +# failJSON: { "time": "2005-08-14T12:51:04", "match": true , "host": "1.2.3.4" } Aug 14 12:51:04 HOSTNAME courieresmtpd: error,relay=::ffff:1.2.3.4,from=,to=: 550 User unknown. -# failJSON: { "time": "2004-08-14T12:51:04", "match": true , "host": "1.2.3.4" } +# failJSON: { "time": "2005-08-14T12:51:04", "match": true , "host": "1.2.3.4" } Aug 14 12:51:04 mail.server courieresmtpd[26762]: error,relay=::ffff:1.2.3.4,msg="535 Authentication failed.",cmd: AUTH PLAIN AAAAABBBBCCCCWxlZA== admin +# failJSON: { "time": "2005-08-14T12:51:05", "match": true , "host": "192.0.2.3" } +Aug 14 12:51:05 mail.server courieresmtpd[425070]: error,relay=::ffff:192.0.2.3,port=43632,msg="535 Authentication failed.",cmd: AUTH LOGIN PlcmSpIp@example.com diff --git a/fail2ban/tests/files/logs/gitlab b/fail2ban/tests/files/logs/gitlab new file mode 100644 index 00000000..70ddc0e8 --- /dev/null +++ b/fail2ban/tests/files/logs/gitlab @@ -0,0 +1,5 @@ +# Access of unauthorized host in /var/log/gitlab/gitlab-rails/application.log +# failJSON: { "time": "2020-04-09T16:04:00", "match": true , "host": "80.10.11.12" } +2020-04-09T14:04:00.667Z: Failed Login: username=admin ip=80.10.11.12 +# failJSON: { "time": "2020-04-09T16:15:09", "match": true , "host": "80.10.11.12" } +2020-04-09T14:15:09.344Z: Failed Login: username=user name ip=80.10.11.12 diff --git a/fail2ban/tests/files/logs/grafana b/fail2ban/tests/files/logs/grafana new file mode 100644 index 00000000..aac86ebc --- /dev/null +++ b/fail2ban/tests/files/logs/grafana @@ -0,0 +1,5 @@ +# Access of unauthorized host in /var/log/grafana/grafana.log +# failJSON: { "time": "2020-10-19T17:44:33", "match": true , "host": "182.56.23.12" } +t=2020-10-19T17:44:33+0200 lvl=eror msg="Invalid username or password" logger=context userId=0 orgId=0 uname= error="Invalid Username or Password" remote_addr=182.56.23.12 +# failJSON: { "time": "2020-10-19T18:44:33", "match": true , "host": "182.56.23.13" } +t=2020-10-19T18:44:33+0200 lvl=eror msg="Invalid username or password" logger=context userId=0 orgId=0 uname= error="User not found" remote_addr=182.56.23.13 diff --git a/fail2ban/tests/files/logs/guacamole b/fail2ban/tests/files/logs/guacamole index 3de67454..ebb7afb0 100644 --- a/fail2ban/tests/files/logs/guacamole +++ b/fail2ban/tests/files/logs/guacamole @@ -10,3 +10,8 @@ WARNING: Authentication attempt from 192.0.2.0 for user "null" failed. apr 16, 2013 8:32:28 AM org.slf4j.impl.JCLLoggerAdapter warn # failJSON: { "time": "2013-04-16T08:32:28", "match": true , "host": "192.0.2.0" } WARNING: Authentication attempt from 192.0.2.0 for user "pippo" failed. + +# filterOptions: {"logging": "webapp"} + +# failJSON: { "time": "2005-08-13T12:57:32", "match": true , "host": "182.23.72.36" } +12:57:32.907 [http-nio-8080-exec-10] WARN o.a.g.r.auth.AuthenticationService - Authentication attempt from 182.23.72.36 for user "guacadmin" failed. diff --git a/fail2ban/tests/files/logs/mysqld-auth b/fail2ban/tests/files/logs/mysqld-auth index 0b0827f9..29faeb71 100644 --- a/fail2ban/tests/files/logs/mysqld-auth +++ b/fail2ban/tests/files/logs/mysqld-auth @@ -33,3 +33,7 @@ Sep 16 21:30:32 catinthehat mysqld: 130916 21:30:32 [Warning] Access denied for 2019-09-06T01:45:18 srv mysqld: 2019-09-06 1:45:18 140581192722176 [Warning] Access denied for user 'global'@'192.0.2.2' (using password: YES) # failJSON: { "time": "2019-09-24T13:16:50", "match": true , "host": "192.0.2.3", "desc": "ISO timestamp within log message" } 2019-09-24T13:16:50 srv mysqld[1234]: 2019-09-24 13:16:50 8756 [Warning] Access denied for user 'root'@'192.0.2.3' (using password: YES) + +# filterOptions: [{"logtype": "file"}, {"logtype": "short"}, {"logtype": "journal"}] +# failJSON: { "match": true , "host": "192.0.2.1", "user":"root", "desc": "mariadb 10.4 log format, gh-2611" } +2020-01-16 21:34:14 4644 [Warning] Access denied for user 'root'@'192.0.2.1' (using password: YES) diff --git a/fail2ban/tests/files/logs/postfix b/fail2ban/tests/files/logs/postfix index d7d37600..6e2dc460 100644 --- a/fail2ban/tests/files/logs/postfix +++ b/fail2ban/tests/files/logs/postfix @@ -137,6 +137,11 @@ Jan 14 16:18:16 xxx postfix/smtpd[14933]: warning: host[192.0.2.5]: SASL CRAM-MD # filterOptions: [{"mode": "ddos"}, {"mode": "aggressive"}] +# failJSON: { "time": "2005-02-10T13:26:34", "match": true , "host": "192.0.2.1" } +Feb 10 13:26:34 srv postfix/smtpd[123]: disconnect from unknown[192.0.2.1] helo=1 auth=0/1 quit=1 commands=2/3 +# failJSON: { "time": "2005-02-10T13:26:34", "match": true , "host": "192.0.2.2" } +Feb 10 13:26:34 srv postfix/smtpd[123]: disconnect from unknown[192.0.2.2] ehlo=1 auth=0/1 rset=1 quit=1 commands=3/4 + # failJSON: { "time": "2005-02-18T09:45:10", "match": true , "host": "192.0.2.10" } Feb 18 09:45:10 xxx postfix/smtpd[42]: lost connection after CONNECT from spammer.example.com[192.0.2.10] # failJSON: { "time": "2005-02-18T09:45:12", "match": true , "host": "192.0.2.42" } diff --git a/fail2ban/tests/files/logs/proftpd b/fail2ban/tests/files/logs/proftpd index b255e91e..8d0d571c 100644 --- a/fail2ban/tests/files/logs/proftpd +++ b/fail2ban/tests/files/logs/proftpd @@ -1,6 +1,6 @@ -# failJSON: { "time": "2005-01-10T00:00:00", "match": true , "host": "123.123.123.123" } +# failJSON: { "time": "2005-01-10T00:00:00", "match": true , "host": "123.123.123.123", "user": "username" } Jan 10 00:00:00 myhost proftpd[12345] myhost.domain.com (123.123.123.123[123.123.123.123]): USER username (Login failed): User in /etc/ftpusers -# failJSON: { "time": "2005-02-01T00:00:00", "match": true , "host": "123.123.123.123" } +# failJSON: { "time": "2005-02-01T00:00:00", "match": true , "host": "123.123.123.123", "user": "username" } Feb 1 00:00:00 myhost proftpd[12345] myhost.domain.com (123.123.123.123[123.123.123.123]): USER username: no such user found from 123.123.123.123 [123.123.123.123] to 234.234.234.234:21 # failJSON: { "time": "2005-06-09T07:30:58", "match": true , "host": "67.227.224.66" } Jun 09 07:30:58 platypus.ace-hosting.com.au proftpd[11864] platypus.ace-hosting.com.au (mail.bloodymonster.net[::ffff:67.227.224.66]): USER username (Login failed): Incorrect password. @@ -12,7 +12,9 @@ Jun 13 22:07:23 platypus.ace-hosting.com.au proftpd[15719] platypus.ace-hosting. Jun 14 00:09:59 platypus.ace-hosting.com.au proftpd[17839] platypus.ace-hosting.com.au (::ffff:59.167.242.100[::ffff:59.167.242.100]): USER platypus.ace-hosting.com.au proftpd[17424] platypus.ace-hosting.com.au (hihoinjection[1.2.3.44]): no such user found from ::ffff:59.167.242.100 [::ffff:59.167.242.100] to ::ffff:113.212.99.194:21 # failJSON: { "time": "2005-05-31T10:53:25", "match": true , "host": "1.2.3.4" } May 31 10:53:25 mail proftpd[15302]: xxxxxxxxxx (::ffff:1.2.3.4[::ffff:1.2.3.4]) - Maximum login attempts (3) exceeded -# failJSON: { "time": "2004-12-05T15:44:32", "match": true , "host": "1.2.3.4" } +# failJSON: { "time": "2004-10-02T15:45:44", "match": true , "host": "192.0.2.13", "user": "Root", "desc": "dot at end is optional (mod_sftp, gh-2246)" } +Oct 2 15:45:44 ftp01 proftpd[5517]: 192.0.2.13 (192.0.2.13[192.0.2.13]) - SECURITY VIOLATION: Root login attempted +# failJSON: { "time": "2004-12-05T15:44:32", "match": true , "host": "1.2.3.4", "user": "jtittle@domain.org" } Dec 5 15:44:32 serv1 proftpd[70944]: serv1.domain.com (example.com[1.2.3.4]) - USER jtittle@domain.org: no such user found from example.com [1.2.3.4] to 1.2.3.4:21 # failJSON: { "time": "2013-11-16T21:59:30", "match": true , "host": "1.2.3.4", "desc": "proftpd-basic 1.3.5~rc3-2.1 on Debian uses date format with milliseconds if logging under /var/log/proftpd/proftpd.log" } -2013-11-16 21:59:30,121 novo proftpd[25891] localhost (andy[1.2.3.4]): USER kjsad: no such user found from andy [1.2.3.5] to ::ffff:192.168.1.14:21 +2013-11-16 21:59:30,121 novo proftpd[25891] localhost (andy[1.2.3.4]): USER kjsad: no such user found from andy [1.2.3.5] to ::ffff:192.168.1.14:21 \ No newline at end of file diff --git a/fail2ban/tests/files/logs/sendmail-auth b/fail2ban/tests/files/logs/sendmail-auth index a7ddd6f8..93bf0b14 100644 --- a/fail2ban/tests/files/logs/sendmail-auth +++ b/fail2ban/tests/files/logs/sendmail-auth @@ -17,3 +17,8 @@ Feb 24 14:00:00 server sendmail[26592]: u0CB32qX026592: [192.0.2.1]: possible SM # failJSON: { "time": "2005-02-24T14:00:01", "match": true , "host": "192.0.2.2", "desc": "long PID, ID longer as 14 chars (gh-2563)" } Feb 24 14:00:01 server sendmail[3529566]: xA32R2PQ3529566: [192.0.2.2]: possible SMTP attack: command=AUTH, count=5 + +# failJSON: { "time": "2005-02-25T04:02:27", "match": true , "host": "192.0.2.3", "desc": "sendmail 8.16.1, AUTH_FAIL_LOG_USER (gh-2757)" } +Feb 25 04:02:27 relay1 sendmail[16664]: 06I02CNi016764: AUTH failure (LOGIN): authentication failure (-13) SASL(-13): authentication failure: checkpass failed, user=user@example.com, relay=example.com [192.0.2.3] (may be forged) +# failJSON: { "time": "2005-02-25T04:02:28", "match": true , "host": "192.0.2.4", "desc": "injection attempt on user name" } +Feb 25 04:02:28 relay1 sendmail[16665]: 06I02CNi016765: AUTH failure (LOGIN): authentication failure (-13) SASL(-13): authentication failure: checkpass failed, user=criminal, relay=[192.0.2.100], relay=[192.0.2.4] (may be forged) diff --git a/fail2ban/tests/files/logs/sendmail-reject b/fail2ban/tests/files/logs/sendmail-reject index f69e4531..99c1877c 100644 --- a/fail2ban/tests/files/logs/sendmail-reject +++ b/fail2ban/tests/files/logs/sendmail-reject @@ -103,3 +103,7 @@ Mar 29 22:51:42 kismet sm-mta[24202]: x2TMpAlI024202: internettl.org [104.152.52 # failJSON: { "time": "2005-03-29T22:51:43", "match": true , "host": "192.0.2.2", "desc": "long PID, ID longer as 14 chars (gh-2563)" } Mar 29 22:51:43 server sendmail[3529565]: xA32R2PQ3529565: [192.0.2.2] did not issue MAIL/EXPN/VRFY/ETRN during connection to MTA +# failJSON: { "time": "2005-03-29T22:51:45", "match": true , "host": "192.0.2.3", "desc": "sendmail 8.15.2 default names IPv4/6 (gh-2787)" } +Mar 29 22:51:45 server sm-mta[50437]: 06QDQnNf050437: example.com [192.0.2.3] did not issue MAIL/EXPN/VRFY/ETRN during connection to IPv4 +# failJSON: { "time": "2005-03-29T22:51:46", "match": true , "host": "2001:DB8::1", "desc": "IPv6" } +Mar 29 22:51:46 server sm-mta[50438]: 06QDQnNf050438: example.com [IPv6:2001:DB8::1] did not issue MAIL/EXPN/VRFY/ETRN during connection to IPv6 \ No newline at end of file diff --git a/fail2ban/tests/files/logs/softethervpn b/fail2ban/tests/files/logs/softethervpn new file mode 100644 index 00000000..dd2a798b --- /dev/null +++ b/fail2ban/tests/files/logs/softethervpn @@ -0,0 +1,7 @@ +# Access of unauthorized host in /usr/local/vpnserver/security_log/*/sec.log +# failJSON: { "time": "2020-05-12T10:53:19", "match": true , "host": "80.10.11.12" } +2020-05-12 10:53:19.781 Connection "CID-72": User authentication failed. The user name that has been provided was "bob", from 80.10.11.12. + +# Access of unauthorized host in syslog +# failJSON: { "time": "2020-05-13T10:53:19", "match": true , "host": "80.10.11.13" } +2020-05-13T10:53:19 localhost [myserver.com/VPN/defaultvpn] (2020-05-13 10:53:19.591) : Connection "CID-594": User authentication failed. The user name that has been provided was "alice", from 80.10.11.13. diff --git a/fail2ban/tests/files/logs/sshd b/fail2ban/tests/files/logs/sshd index a5f64939..5d23f96f 100644 --- a/fail2ban/tests/files/logs/sshd +++ b/fail2ban/tests/files/logs/sshd @@ -134,7 +134,7 @@ Sep 29 17:15:02 spaceman sshd[12946]: Failed password for user from 127.0.0.1 po # failJSON: { "time": "2004-09-29T17:15:02", "match": true , "host": "127.0.0.1", "desc": "Injecting while exhausting initially present {0,100} match length limits set for ruser etc" } Sep 29 17:15:02 spaceman sshd[12946]: Failed password for user from 127.0.0.1 port 20000 ssh1: ruser XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX from 1.2.3.4 # failJSON: { "time": "2004-09-29T17:15:03", "match": true , "host": "aaaa:bbbb:cccc:1234::1:1", "desc": "Injecting while exhausting initially present {0,100} match length limits set for ruser etc" } -Sep 29 17:15:03 spaceman sshd[12946]: Failed password for user from aaaa:bbbb:cccc:1234::1:1 port 20000 ssh1: ruser XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX from 1.2.3.4 +Sep 29 17:15:03 spaceman sshd[12947]: Failed password for user from aaaa:bbbb:cccc:1234::1:1 port 20000 ssh1: ruser XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX from 1.2.3.4 # failJSON: { "time": "2004-11-11T08:04:51", "match": true , "host": "127.0.0.1", "desc": "Injecting on username ssh 'from 10.10.1.1'@localhost" } Nov 11 08:04:51 redbamboo sshd[2737]: Failed password for invalid user from 10.10.1.1 from 127.0.0.1 port 58946 ssh2 @@ -166,9 +166,11 @@ Nov 28 09:16:03 srv sshd[32307]: Connection closed by 192.0.2.1 Nov 28 09:16:05 srv sshd[32310]: Failed publickey for git from 192.0.2.111 port 57910 ssh2: ECDSA 1e:fe:xx:xx:xx:xx:xx:xx:xx:xx:xx:xx:xx:xx:xx:xx # failJSON: { "match": false } Nov 28 09:16:05 srv sshd[32310]: Failed publickey for git from 192.0.2.111 port 57910 ssh2: RSA 14:ba:xx:xx:xx:xx:xx:xx:xx:xx:xx:xx:xx:xx:xx:xx -# failJSON: { "match": false } +# failJSON: { "constraint": "name == 'sshd'", "time": "2004-11-28T09:16:05", "match": true , "attempts": 3, "desc": "Should catch failure - no success/no accepted public key" } Nov 28 09:16:05 srv sshd[32310]: Disconnecting: Too many authentication failures for git [preauth] -# failJSON: { "time": "2004-11-28T09:16:05", "match": true , "host": "192.0.2.111", "desc": "Should catch failure - no success/no accepted public key" } +# failJSON: { "constraint": "opts.get('mode') != 'aggressive'", "match": false, "desc": "Nofail in normal mode, failure already produced above" } +Nov 28 09:16:05 srv sshd[32310]: Connection closed by 192.0.2.111 [preauth] +# failJSON: { "constraint": "opts.get('mode') == 'aggressive'", "time": "2004-11-28T09:16:05", "match": true , "host": "192.0.2.111", "attempts":1, "desc": "Matches in aggressive mode only" } Nov 28 09:16:05 srv sshd[32310]: Connection closed by 192.0.2.111 [preauth] # failJSON: { "match": false } @@ -215,7 +217,7 @@ Apr 27 13:02:04 host sshd[29116]: Received disconnect from 1.2.3.4: 11: Normal S # Match sshd auth errors on OpenSUSE systems (gh-1024) # failJSON: { "match": false, "desc": "No failure until closed or another fail (e. g. F-MLFFORGET by success/accepted password can avoid failure, see gh-2070)" } 2015-04-16T18:02:50.321974+00:00 host sshd[2716]: pam_unix(sshd:auth): authentication failure; logname= uid=0 euid=0 tty=ssh ruser= rhost=192.0.2.112 user=root -# failJSON: { "time": "2015-04-16T20:02:50", "match": true , "host": "192.0.2.112", "desc": "Should catch failure - no success/no accepted password" } +# failJSON: { "constraint": "opts.get('mode') == 'aggressive'", "time": "2015-04-16T20:02:50", "match": true , "host": "192.0.2.112", "desc": "Should catch failure - no success/no accepted password" } 2015-04-16T18:02:50.568798+00:00 host sshd[2716]: Connection closed by 192.0.2.112 [preauth] # disable this test-cases block for obsolete multi-line filter (zzz-sshd-obsolete...): @@ -238,7 +240,7 @@ Mar 7 18:53:20 bar sshd[1556]: Connection closed by 192.0.2.113 Mar 7 18:53:22 bar sshd[1558]: pam_unix(sshd:auth): authentication failure; logname= uid=0 euid=0 tty=ssh ruser=root rhost=192.0.2.114 # failJSON: { "time": "2005-03-07T18:53:23", "match": true , "attempts": 2, "users": ["root", "sudoer"], "host": "192.0.2.114", "desc": "Failure: attempt 2nd user" } Mar 7 18:53:23 bar sshd[1558]: pam_unix(sshd:auth): authentication failure; logname= uid=0 euid=0 tty=ssh ruser=sudoer rhost=192.0.2.114 -# failJSON: { "time": "2005-03-07T18:53:24", "match": true , "attempts": 2, "users": ["root", "sudoer", "known"], "host": "192.0.2.114", "desc": "Failure: attempt 3rd user" } +# failJSON: { "time": "2005-03-07T18:53:24", "match": true , "attempts": 1, "users": ["root", "sudoer", "known"], "host": "192.0.2.114", "desc": "Failure: attempt 3rd user" } Mar 7 18:53:24 bar sshd[1558]: Accepted password for known from 192.0.2.114 port 52100 ssh2 # failJSON: { "match": false , "desc": "No failure" } Mar 7 18:53:24 bar sshd[1558]: pam_unix(sshd:session): session opened for user known by (uid=0) @@ -248,14 +250,14 @@ Mar 7 18:53:24 bar sshd[1558]: pam_unix(sshd:session): session opened for user Mar 7 18:53:32 bar sshd[1559]: pam_unix(sshd:auth): authentication failure; logname= uid=0 euid=0 tty=ssh ruser=root rhost=192.0.2.116 # failJSON: { "match": false , "desc": "Still no failure (second try, same user)" } Mar 7 18:53:32 bar sshd[1559]: pam_unix(sshd:auth): authentication failure; logname= uid=0 euid=0 tty=ssh ruser=root rhost=192.0.2.116 -# failJSON: { "time": "2005-03-07T18:53:34", "match": true , "attempts": 2, "users": ["root", "known"], "host": "192.0.2.116", "desc": "Failure: attempt 2nd user" } +# failJSON: { "time": "2005-03-07T18:53:34", "match": true , "attempts": 3, "users": ["root", "known"], "host": "192.0.2.116", "desc": "Failure: attempt 2nd user" } Mar 7 18:53:34 bar sshd[1559]: Accepted password for known from 192.0.2.116 port 52100 ssh2 # failJSON: { "match": false , "desc": "No failure" } Mar 7 18:53:38 bar sshd[1559]: Connection closed by 192.0.2.116 # failJSON: { "time": "2005-03-19T16:47:48", "match": true , "attempts": 1, "user": "admin", "host": "192.0.2.117", "desc": "Failure: attempt invalid user" } Mar 19 16:47:48 test sshd[5672]: Invalid user admin from 192.0.2.117 port 44004 -# failJSON: { "time": "2005-03-19T16:47:49", "match": true , "attempts": 2, "user": "admin", "host": "192.0.2.117", "desc": "Failure: attempt to change user (disallowed)" } +# failJSON: { "time": "2005-03-19T16:47:49", "match": true , "attempts": 1, "user": "admin", "host": "192.0.2.117", "desc": "Failure: attempt to change user (disallowed)" } Mar 19 16:47:49 test sshd[5672]: Disconnecting invalid user admin 192.0.2.117 port 44004: Change of username or service not allowed: (admin,ssh-connection) -> (user,ssh-connection) [preauth] # failJSON: { "time": "2005-03-19T16:47:50", "match": false, "desc": "Disconnected during preauth phase (no failure in normal mode)" } Mar 19 16:47:50 srv sshd[5672]: Disconnected from authenticating user admin 192.0.2.6 port 33553 [preauth] @@ -294,6 +296,9 @@ Nov 24 23:46:43 host sshd[32686]: fatal: Read from socket failed: Connection res # failJSON: { "time": "2005-03-15T09:20:57", "match": true , "host": "192.0.2.39", "desc": "Singleline for connection reset by" } Mar 15 09:20:57 host sshd[28972]: Connection reset by 192.0.2.39 port 14282 [preauth] +# failJSON: { "time": "2005-03-16T09:29:50", "match": true , "host": "192.0.2.20", "desc": "connection reset by user (gh-2662)" } +Mar 16 09:29:50 host sshd[19131]: Connection reset by authenticating user root 192.0.2.20 port 1558 [preauth] + # failJSON: { "time": "2005-07-17T23:03:05", "match": true , "host": "192.0.2.10", "user": "root", "desc": "user name additionally, gh-2185" } Jul 17 23:03:05 srv sshd[1296]: Connection closed by authenticating user root 192.0.2.10 port 46038 [preauth] # failJSON: { "time": "2005-07-17T23:04:00", "match": true , "host": "192.0.2.11", "user": "test 127.0.0.1", "desc": "check inject on username, gh-2185" } @@ -303,6 +308,13 @@ Jul 17 23:04:01 srv sshd[1300]: Connection closed by authenticating user test 12 # filterOptions: [{"test.condition":"name=='sshd'", "mode": "ddos"}, {"test.condition":"name=='sshd'", "mode": "aggressive"}] +# failJSON: { "match": false } +Feb 17 17:40:17 sshd[19725]: Connection from 192.0.2.10 port 62004 on 192.0.2.10 port 22 +# failJSON: { "time": "2005-02-17T17:40:17", "match": true , "host": "192.0.2.10", "desc": "ddos: port scanner (invalid protocol identifier)" } +Feb 17 17:40:17 sshd[19725]: error: kex_exchange_identification: client sent invalid protocol identifier "" +# failJSON: { "time": "2005-02-17T17:40:18", "match": true , "host": "192.0.2.10", "desc": "ddos: flood attack vector, gh-2850" } +Feb 17 17:40:18 sshd[19725]: error: kex_exchange_identification: Connection closed by remote host + # failJSON: { "time": "2005-03-15T09:21:01", "match": true , "host": "192.0.2.212", "desc": "DDOS mode causes failure on close within preauth stage" } Mar 15 09:21:01 host sshd[2717]: Connection closed by 192.0.2.212 [preauth] # failJSON: { "time": "2005-03-15T09:21:02", "match": true , "host": "192.0.2.212", "desc": "DDOS mode causes failure on close within preauth stage" } @@ -311,6 +323,11 @@ Mar 15 09:21:02 host sshd[2717]: Connection closed by 192.0.2.212 [preauth] # failJSON: { "time": "2005-07-18T17:19:11", "match": true , "host": "192.0.2.4", "desc": "ddos: disconnect on preauth phase, gh-2115" } Jul 18 17:19:11 srv sshd[2101]: Disconnected from 192.0.2.4 port 36985 [preauth] +# failJSON: { "time": "2005-06-06T04:17:04", "match": true , "host": "192.0.2.68", "dns": null, "user": "", "desc": "empty user, gh-2749" } +Jun 6 04:17:04 host sshd[1189074]: Invalid user from 192.0.2.68 port 34916 +# failJSON: { "time": "2005-06-06T04:17:09", "match": true , "host": "192.0.2.68", "dns": null, "user": "", "desc": "empty user, gh-2749" } +Jun 6 04:17:09 host sshd[1189074]: Connection closed by invalid user 192.0.2.68 port 34916 [preauth] + # filterOptions: [{"mode": "extra"}, {"mode": "aggressive"}] # several other cases from gh-864: @@ -320,6 +337,8 @@ Nov 25 01:34:12 srv sshd[123]: Received disconnect from 127.0.0.1: 14: No suppor Nov 25 01:35:13 srv sshd[123]: error: Received disconnect from 127.0.0.1: 14: No supported authentication methods available [preauth] # failJSON: { "time": "2004-11-25T01:35:14", "match": true , "host": "192.168.2.92", "desc": "Optional space after port" } Nov 25 01:35:14 srv sshd[3625]: error: Received disconnect from 192.168.2.92 port 1684:14: No supported authentication methods available [preauth] +# failJSON: { "time": "2004-11-25T01:35:15", "match": true , "host": "192.168.2.93", "desc": "No authentication methods available (supported is optional, gh-2682)" } +Nov 25 01:35:15 srv sshd[3626]: error: Received disconnect from 192.168.2.93 port 1883:14: No authentication methods available [preauth] # gh-1545: # failJSON: { "time": "2004-11-26T13:03:29", "match": true , "host": "192.0.2.1", "desc": "No matching cipher" } @@ -332,7 +351,7 @@ Nov 26 13:03:30 srv sshd[45]: fatal: Unable to negotiate with 192.0.2.2 port 554 Nov 26 15:03:30 host sshd[22440]: Connection from 192.0.2.3 port 39678 on 192.168.1.9 port 22 # failJSON: { "time": "2004-11-26T15:03:31", "match": true , "host": "192.0.2.3", "desc": "Multiline - no matching key exchange method" } Nov 26 15:03:31 host sshd[22440]: fatal: Unable to negotiate a key exchange method [preauth] -# failJSON: { "time": "2004-11-26T15:03:32", "match": true , "host": "192.0.2.3", "filter": "sshd", "desc": "Second attempt within the same connect" } +# failJSON: { "time": "2004-11-26T15:03:32", "match": true , "host": "192.0.2.3", "constraint": "name == 'sshd'", "desc": "Second attempt within the same connect" } Nov 26 15:03:32 host sshd[22440]: fatal: Unable to negotiate a key exchange method [preauth] # gh-1943 (previous OpenSSH log-format) diff --git a/fail2ban/tests/files/logs/sshd-journal b/fail2ban/tests/files/logs/sshd-journal index 07e34efe..d19889d7 100644 --- a/fail2ban/tests/files/logs/sshd-journal +++ b/fail2ban/tests/files/logs/sshd-journal @@ -135,7 +135,7 @@ srv sshd[12946]: Failed password for user from 127.0.0.1 port 20000 ssh1: ruser # failJSON: { "match": true , "host": "127.0.0.1", "desc": "Injecting while exhausting initially present {0,100} match length limits set for ruser etc" } srv sshd[12946]: Failed password for user from 127.0.0.1 port 20000 ssh1: ruser XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX from 1.2.3.4 # failJSON: { "match": true , "host": "aaaa:bbbb:cccc:1234::1:1", "desc": "Injecting while exhausting initially present {0,100} match length limits set for ruser etc" } -srv sshd[12946]: Failed password for user from aaaa:bbbb:cccc:1234::1:1 port 20000 ssh1: ruser XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX from 1.2.3.4 +srv sshd[12947]: Failed password for user from aaaa:bbbb:cccc:1234::1:1 port 20000 ssh1: ruser XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX from 1.2.3.4 # failJSON: { "match": true , "host": "127.0.0.1", "desc": "Injecting on username ssh 'from 10.10.1.1'@localhost" } srv sshd[2737]: Failed password for invalid user from 10.10.1.1 from 127.0.0.1 port 58946 ssh2 @@ -167,9 +167,11 @@ srv sshd[32307]: Connection closed by 192.0.2.1 srv sshd[32310]: Failed publickey for git from 192.0.2.111 port 57910 ssh2: ECDSA 1e:fe:xx:xx:xx:xx:xx:xx:xx:xx:xx:xx:xx:xx:xx:xx # failJSON: { "match": false } srv sshd[32310]: Failed publickey for git from 192.0.2.111 port 57910 ssh2: RSA 14:ba:xx:xx:xx:xx:xx:xx:xx:xx:xx:xx:xx:xx:xx:xx -# failJSON: { "match": false } +# failJSON: { "match": true , "attempts": 3, "desc": "Should catch failure - no success/no accepted public key" } srv sshd[32310]: Disconnecting: Too many authentication failures for git [preauth] -# failJSON: { "match": true , "host": "192.0.2.111", "desc": "Should catch failure - no success/no accepted public key" } +# failJSON: { "constraint": "opts.get('mode') != 'aggressive'", "match": false, "desc": "Nofail in normal mode, failure already produced above" } +srv sshd[32310]: Connection closed by 192.0.2.111 [preauth] +# failJSON: { "constraint": "opts.get('mode') == 'aggressive'", "match": true , "host": "192.0.2.111", "attempts":1, "desc": "Matches in aggressive mode only" } srv sshd[32310]: Connection closed by 192.0.2.111 [preauth] # failJSON: { "match": false } @@ -216,7 +218,7 @@ srv sshd[29116]: Received disconnect from 1.2.3.4: 11: Normal Shutdown, Thank yo # Match sshd auth errors on OpenSUSE systems (gh-1024) # failJSON: { "match": false, "desc": "No failure until closed or another fail (e. g. F-MLFFORGET by success/accepted password can avoid failure, see gh-2070)" } srv sshd[2716]: pam_unix(sshd:auth): authentication failure; logname= uid=0 euid=0 tty=ssh ruser= rhost=192.0.2.112 user=root -# failJSON: { "match": true , "host": "192.0.2.112", "desc": "Should catch failure - no success/no accepted password" } +# failJSON: { "constraint": "opts.get('mode') == 'aggressive'", "match": true , "host": "192.0.2.112", "desc": "Should catch failure - no success/no accepted password" } srv sshd[2716]: Connection closed by 192.0.2.112 [preauth] # filterOptions: [{}] @@ -238,7 +240,7 @@ srv sshd[1556]: Connection closed by 192.0.2.113 srv sshd[1558]: pam_unix(sshd:auth): authentication failure; logname= uid=0 euid=0 tty=ssh ruser=root rhost=192.0.2.114 # failJSON: { "match": true , "attempts": 2, "users": ["root", "sudoer"], "host": "192.0.2.114", "desc": "Failure: attempt 2nd user" } srv sshd[1558]: pam_unix(sshd:auth): authentication failure; logname= uid=0 euid=0 tty=ssh ruser=sudoer rhost=192.0.2.114 -# failJSON: { "match": true , "attempts": 2, "users": ["root", "sudoer", "known"], "host": "192.0.2.114", "desc": "Failure: attempt 3rd user" } +# failJSON: { "match": true , "attempts": 1, "users": ["root", "sudoer", "known"], "host": "192.0.2.114", "desc": "Failure: attempt 3rd user" } srv sshd[1558]: Accepted password for known from 192.0.2.114 port 52100 ssh2 # failJSON: { "match": false , "desc": "No failure" } srv sshd[1558]: pam_unix(sshd:session): session opened for user known by (uid=0) @@ -248,14 +250,14 @@ srv sshd[1558]: pam_unix(sshd:session): session opened for user known by (uid=0) srv sshd[1559]: pam_unix(sshd:auth): authentication failure; logname= uid=0 euid=0 tty=ssh ruser=root rhost=192.0.2.116 # failJSON: { "match": false , "desc": "Still no failure (second try, same user)" } srv sshd[1559]: pam_unix(sshd:auth): authentication failure; logname= uid=0 euid=0 tty=ssh ruser=root rhost=192.0.2.116 -# failJSON: { "match": true , "attempts": 2, "users": ["root", "known"], "host": "192.0.2.116", "desc": "Failure: attempt 2nd user" } +# failJSON: { "match": true , "attempts": 3, "users": ["root", "known"], "host": "192.0.2.116", "desc": "Failure: attempt 2nd user" } srv sshd[1559]: Accepted password for known from 192.0.2.116 port 52100 ssh2 # failJSON: { "match": false , "desc": "No failure" } srv sshd[1559]: Connection closed by 192.0.2.116 # failJSON: { "match": true , "attempts": 1, "user": "admin", "host": "192.0.2.117", "desc": "Failure: attempt invalid user" } srv sshd[5672]: Invalid user admin from 192.0.2.117 port 44004 -# failJSON: { "match": true , "attempts": 2, "user": "admin", "host": "192.0.2.117", "desc": "Failure: attempt to change user (disallowed)" } +# failJSON: { "match": true , "attempts": 1, "user": "admin", "host": "192.0.2.117", "desc": "Failure: attempt to change user (disallowed)" } srv sshd[5672]: Disconnecting invalid user admin 192.0.2.117 port 44004: Change of username or service not allowed: (admin,ssh-connection) -> (user,ssh-connection) [preauth] # failJSON: { "match": false, "desc": "Disconnected during preauth phase (no failure in normal mode)" } srv sshd[5672]: Disconnected from authenticating user admin 192.0.2.6 port 33553 [preauth] @@ -325,7 +327,7 @@ srv sshd[45]: fatal: Unable to negotiate with 192.0.2.2 port 55419: no matching srv sshd[22440]: Connection from 192.0.2.3 port 39678 on 192.168.1.9 port 22 # failJSON: { "match": true , "host": "192.0.2.3", "desc": "Multiline - no matching key exchange method" } srv sshd[22440]: fatal: Unable to negotiate a key exchange method [preauth] -# failJSON: { "match": true , "host": "192.0.2.3", "filter": "sshd", "desc": "Second attempt within the same connect" } +# failJSON: { "match": true , "host": "192.0.2.3", "constraint": "name == 'sshd'", "desc": "Second attempt within the same connect" } srv sshd[22440]: fatal: Unable to negotiate a key exchange method [preauth] # gh-1943 (previous OpenSSH log-format) diff --git a/fail2ban/tests/files/logs/traefik-auth b/fail2ban/tests/files/logs/traefik-auth index 3e7a8987..edfe7306 100644 --- a/fail2ban/tests/files/logs/traefik-auth +++ b/fail2ban/tests/files/logs/traefik-auth @@ -1,6 +1,23 @@ +# filterOptions: [{"mode": "normal"}] + # failJSON: { "match": false } 10.0.0.2 - - [18/Nov/2018:21:34:30 +0000] "GET /dashboard/ HTTP/2.0" 401 17 "-" "Mozilla/5.0 (X11; Linux x86_64; rv:57.0) Gecko/20100101 Firefox/57.0" 72 "Auth for frontend-Host-traefik-0" "/dashboard/" 0ms + +# filterOptions: [{"mode": "ddos"}] + +# failJSON: { "match": false } +10.0.0.2 - username [18/Nov/2018:21:34:30 +0000] "GET /dashboard/ HTTP/2.0" 401 17 "-" "Mozilla/5.0 (X11; Linux x86_64; rv:57.0) Gecko/20100101 Firefox/57.0" 72 "Auth for frontend-Host-traefik-0" "/dashboard/" 0ms + +# filterOptions: [{"mode": "normal"}, {"mode": "aggressive"}] + # failJSON: { "time": "2018-11-18T22:34:34", "match": true , "host": "10.0.0.2" } 10.0.0.2 - username [18/Nov/2018:21:34:34 +0000] "GET /dashboard/ HTTP/2.0" 401 17 "-" "Mozilla/5.0 (X11; Linux x86_64; rv:57.0) Gecko/20100101 Firefox/57.0" 72 "Auth for frontend-Host-traefik-0" "/dashboard/" 0ms +# failJSON: { "time": "2018-11-18T22:34:34", "match": true , "host": "10.0.0.2", "desc": "other request method" } +10.0.0.2 - username [18/Nov/2018:21:34:34 +0000] "TRACE /dashboard/ HTTP/2.0" 401 17 "-" "Mozilla/5.0 (X11; Linux x86_64; rv:57.0) Gecko/20100101 Firefox/57.0" 72 "Auth for frontend-Host-traefik-0" "/dashboard/" 0ms # failJSON: { "match": false } 10.0.0.2 - username [27/Nov/2018:23:33:31 +0000] "GET /dashboard/ HTTP/2.0" 200 716 "-" "Mozilla/5.0 (X11; Linux x86_64; rv:57.0) Gecko/20100101 Firefox/57.0" 118 "Host-traefik-0" "/dashboard/" 4ms + +# filterOptions: [{"mode": "ddos"}, {"mode": "aggressive"}] + +# failJSON: { "time": "2018-11-18T22:34:30", "match": true , "host": "10.0.0.2" } +10.0.0.2 - - [18/Nov/2018:21:34:30 +0000] "GET /dashboard/ HTTP/2.0" 401 17 "-" "Mozilla/5.0 (X11; Linux x86_64; rv:57.0) Gecko/20100101 Firefox/57.0" 72 "Auth for frontend-Host-traefik-0" "/dashboard/" 0ms diff --git a/fail2ban/tests/files/logs/zzz-generic-example b/fail2ban/tests/files/logs/zzz-generic-example index d0c31740..118c7e12 100644 --- a/fail2ban/tests/files/logs/zzz-generic-example +++ b/fail2ban/tests/files/logs/zzz-generic-example @@ -30,8 +30,8 @@ Jun 21 16:55:02 machine kernel: [ 970.699396] @vserver_demo test- # failJSON: { "time": "2005-06-21T16:55:03", "match": true , "host": "192.0.2.3" } [Jun 21 16:55:03] machine kernel: [ 970.699396] @vserver_demo test-demo(pam_unix)[13709] [ID 255 test] F2B: failure from 192.0.2.3 -# -- wrong time direct in journal-line (used last known date): -# failJSON: { "time": "2005-06-21T16:55:03", "match": true , "host": "192.0.2.1" } +# -- wrong time direct in journal-line (used last known date or now, but null because no checkFindTime in samples test factory): +# failJSON: { "time": null, "match": true , "host": "192.0.2.1" } 0000-12-30 00:00:00 server test-demo[47831]: F2B: failure from 192.0.2.1 # -- wrong time after newline in message (plist without escaped newlines): # failJSON: { "match": false } @@ -42,8 +42,8 @@ Jun 22 20:37:04 server test-demo[402]: writeToStorage plist={ applicationDate = "0000-12-30 00:00:00 +0000"; # failJSON: { "match": false } } -# -- wrong time direct in journal-line (used last known date): -# failJSON: { "time": "2005-06-22T20:37:04", "match": true , "host": "192.0.2.2" } +# -- wrong time direct in journal-line (used last known date, but null because no checkFindTime in samples test factory): +# failJSON: { "time": null, "match": true , "host": "192.0.2.2" } 0000-12-30 00:00:00 server test-demo[47831]: F2B: failure from 192.0.2.2 # -- test no zone and UTC/GMT named zone "2005-06-21T14:55:10 UTC" == "2005-06-21T16:55:10 CEST" (diff +2h in CEST): @@ -60,3 +60,6 @@ Jun 22 20:37:04 server test-demo[402]: writeToStorage plist={ [Jun 21 16:56:03] machine test-demo(pam_unix)[13709] F2B: error from 192.0.2.251 # failJSON: { "match": false, "desc": "test 2nd ignoreregex" } [Jun 21 16:56:04] machine test-demo(pam_unix)[13709] F2B: error from 192.0.2.252 + +# failJSON: { "match": false, "desc": "ignore other daemon" } +[Jun 21 16:56:04] machine captain-nemo(pam_unix)[55555] F2B: error from 192.0.2.2 diff --git a/fail2ban/tests/filtertestcase.py b/fail2ban/tests/filtertestcase.py index 790ea417..e8915f7a 100644 --- a/fail2ban/tests/filtertestcase.py +++ b/fail2ban/tests/filtertestcase.py @@ -43,7 +43,8 @@ from ..server.failmanager import FailManagerEmpty from ..server.ipdns import asip, getfqdn, DNSUtils, IPAddr from ..server.mytime import MyTime from ..server.utils import Utils, uni_decode -from .utils import setUpMyTime, tearDownMyTime, mtimesleep, with_tmpdir, LogCaptureTestCase, \ +from .databasetestcase import getFail2BanDb +from .utils import setUpMyTime, tearDownMyTime, mtimesleep, with_alt_time, with_tmpdir, LogCaptureTestCase, \ logSys as DefLogSys, CONFIG_DIR as STOCK_CONF_DIR from .dummyjail import DummyJail @@ -62,10 +63,7 @@ def open(*args): if len(args) == 2: # ~50kB buffer should be sufficient for all tests here. args = args + (50000,) - if sys.version_info >= (3,): - return fopen(*args, **{'encoding': 'utf-8', 'errors': 'ignore'}) - else: - return fopen(*args) + return fopen(*args) def _killfile(f, name): @@ -199,7 +197,7 @@ def _copy_lines_between_files(in_, fout, n=None, skip=0, mode='a', terminal_line # polling filter could detect the change mtimesleep() if isinstance(in_, str): # pragma: no branch - only used with str in test cases - fin = open(in_, 'r') + fin = open(in_, 'rb') else: fin = in_ # Skip @@ -209,7 +207,7 @@ def _copy_lines_between_files(in_, fout, n=None, skip=0, mode='a', terminal_line i = 0 lines = [] while n is None or i < n: - l = fin.readline() + l = FileContainer.decode_line(in_, 'UTF-8', fin.readline()).rstrip('\r\n') if terminal_line is not None and l == terminal_line: break lines.append(l) @@ -217,7 +215,7 @@ def _copy_lines_between_files(in_, fout, n=None, skip=0, mode='a', terminal_line # Write: all at once and flush if isinstance(fout, str): fout = open(fout, mode) - fout.write('\n'.join(lines)) + fout.write('\n'.join(lines)+'\n') fout.flush() if isinstance(in_, str): # pragma: no branch - only used with str in test cases # Opened earlier, therefore must close it @@ -237,7 +235,7 @@ def _copy_lines_to_journal(in_, fields={},n=None, skip=0, terminal_line=""): # p Returns None """ if isinstance(in_, str): # pragma: no branch - only used with str in test cases - fin = open(in_, 'r') + fin = open(in_, 'rb') else: fin = in_ # Required for filtering @@ -248,7 +246,7 @@ def _copy_lines_to_journal(in_, fields={},n=None, skip=0, terminal_line=""): # p # Read/Write i = 0 while n is None or i < n: - l = fin.readline() + l = FileContainer.decode_line(in_, 'UTF-8', fin.readline()).rstrip('\r\n') if terminal_line is not None and l == terminal_line: break journal.send(MESSAGE=l.strip(), **fields) @@ -396,11 +394,13 @@ class IgnoreIP(LogCaptureTestCase): finally: tearDownMyTime() - def testTimeJump(self): + def _testTimeJump(self, inOperation=False): try: self.filter.addFailRegex('^') self.filter.setDatePattern(r'{^LN-BEG}%Y-%m-%d %H:%M:%S(?:\s*%Z)?\s') self.filter.setFindTime(10); # max 10 seconds back + self.filter.setMaxRetry(5); # don't ban here + self.filter.inOperation = inOperation # self.pruneLog('[phase 1] DST time jump') # check local time jump (DST hole): @@ -431,6 +431,47 @@ class IgnoreIP(LogCaptureTestCase): self.assertNotLogged('Ignore line') finally: tearDownMyTime() + def testTimeJump(self): + self._testTimeJump(inOperation=False) + def testTimeJump_InOperation(self): + self._testTimeJump(inOperation=True) + + def testWrongTimeZone(self): + try: + self.filter.addFailRegex('fail from $') + self.filter.setDatePattern(r'{^LN-BEG}%Y-%m-%d %H:%M:%S(?:\s*%Z)?\s') + self.filter.setMaxRetry(5); # don't ban here + self.filter.inOperation = True; # real processing (all messages are new) + # current time is 1h later than log-entries: + MyTime.setTime(1572138000+3600) + # + self.pruneLog("[phase 1] simulate wrong TZ") + for i in (1,2,3): + self.filter.processLineAndAdd('2019-10-27 02:00:00 fail from 192.0.2.15'); # +3 = 3 + self.assertLogged( + "Simulate NOW in operation since found time has too large deviation", + "Please check jail has possibly a timezone issue.", + "192.0.2.15:1", "192.0.2.15:2", "192.0.2.15:3", + "Total # of detected failures: 3.", wait=True) + # + self.pruneLog("[phase 2] wrong TZ given in log") + for i in (1,2,3): + self.filter.processLineAndAdd('2019-10-27 04:00:00 GMT fail from 192.0.2.16'); # +3 = 6 + self.assertLogged( + "192.0.2.16:1", "192.0.2.16:2", "192.0.2.16:3", + "Total # of detected failures: 6.", all=True, wait=True) + self.assertNotLogged("Found a match but no valid date/time found") + # + self.pruneLog("[phase 3] other timestamp (don't match datepattern), regex matches") + for i in range(3): + self.filter.processLineAndAdd('27.10.2019 04:00:00 fail from 192.0.2.17'); # +3 = 9 + self.assertLogged( + "Found a match but no valid date/time found", + "Match without a timestamp:", + "192.0.2.17:1", "192.0.2.17:2", "192.0.2.17:3", + "Total # of detected failures: 9.", all=True, wait=True) + finally: + tearDownMyTime() def testAddAttempt(self): self.filter.setMaxRetry(3) @@ -759,6 +800,7 @@ class LogFileMonitor(LogCaptureTestCase): _, self.name = tempfile.mkstemp('fail2ban', 'monitorfailures') self.file = open(self.name, 'a') self.filter = FilterPoll(DummyJail()) + self.filter.banASAP = False # avoid immediate ban in this tests self.filter.addLogPath(self.name, autoSeek=False) self.filter.active = True self.filter.addFailRegex(r"(?:(?:Authentication failure|Failed [-/\w+]+) for(?: [iI](?:llegal|nvalid) user)?|[Ii](?:llegal|nvalid) user|ROOT LOGIN REFUSED) .*(?: from|FROM) ") @@ -878,7 +920,7 @@ class LogFileMonitor(LogCaptureTestCase): self.assertRaises(FailManagerEmpty, self.filter.failManager.toBan) # and it should have not been enough - _copy_lines_between_files(GetFailures.FILENAME_01, self.file, skip=5) + _copy_lines_between_files(GetFailures.FILENAME_01, self.file, skip=12, n=3) self.filter.getFailures(self.name) _assert_correct_last_attempt(self, self.filter, GetFailures.FAILURES_01) @@ -897,7 +939,7 @@ class LogFileMonitor(LogCaptureTestCase): # filter "marked" as the known beginning, otherwise it # would not detect "rotation" self.file = _copy_lines_between_files(GetFailures.FILENAME_01, self.name, - skip=3, mode='w') + skip=12, n=3, mode='w') self.filter.getFailures(self.name) #self.assertRaises(FailManagerEmpty, self.filter.failManager.toBan) _assert_correct_last_attempt(self, self.filter, GetFailures.FAILURES_01) @@ -916,7 +958,7 @@ class LogFileMonitor(LogCaptureTestCase): # move aside, but leaving the handle still open... os.rename(self.name, self.name + '.bak') - _copy_lines_between_files(GetFailures.FILENAME_01, self.name, skip=14).close() + _copy_lines_between_files(GetFailures.FILENAME_01, self.name, skip=14, n=1).close() self.filter.getFailures(self.name) _assert_correct_last_attempt(self, self.filter, GetFailures.FAILURES_01) self.assertEqual(self.filter.failManager.getFailTotal(), 3) @@ -976,6 +1018,7 @@ def get_monitor_failures_testcase(Filter_): self.file = open(self.name, 'a') self.jail = DummyJail() self.filter = Filter_(self.jail) + self.filter.banASAP = False # avoid immediate ban in this tests self.filter.addLogPath(self.name, autoSeek=False) # speedup search using exact date pattern: self.filter.setDatePattern(r'^(?:%a )?%b %d %H:%M:%S(?:\.%f)?(?: %ExY)?') @@ -1026,13 +1069,13 @@ def get_monitor_failures_testcase(Filter_): self.assertRaises(FailManagerEmpty, self.filter.failManager.toBan) # Now let's feed it with entries from the file - _copy_lines_between_files(GetFailures.FILENAME_01, self.file, n=5) + _copy_lines_between_files(GetFailures.FILENAME_01, self.file, n=12) self.assertRaises(FailManagerEmpty, self.filter.failManager.toBan) # and our dummy jail is empty as well self.assertFalse(len(self.jail)) # since it should have not been enough - _copy_lines_between_files(GetFailures.FILENAME_01, self.file, skip=5) + _copy_lines_between_files(GetFailures.FILENAME_01, self.file, skip=12, n=3) if idle: self.waitForTicks(1) self.assertTrue(self.isEmpty(1)) @@ -1051,7 +1094,7 @@ def get_monitor_failures_testcase(Filter_): #return # just for fun let's copy all of them again and see if that results # in a new ban - _copy_lines_between_files(GetFailures.FILENAME_01, self.file, n=100) + _copy_lines_between_files(GetFailures.FILENAME_01, self.file, skip=12, n=3) self.assert_correct_last_attempt(GetFailures.FAILURES_01) def test_rewrite_file(self): @@ -1065,7 +1108,7 @@ def get_monitor_failures_testcase(Filter_): # filter "marked" as the known beginning, otherwise it # would not detect "rotation" self.file = _copy_lines_between_files(GetFailures.FILENAME_01, self.name, - skip=3, mode='w') + skip=12, n=3, mode='w') self.assert_correct_last_attempt(GetFailures.FAILURES_01) def _wait4failures(self, count=2): @@ -1086,13 +1129,13 @@ def get_monitor_failures_testcase(Filter_): # move aside, but leaving the handle still open... os.rename(self.name, self.name + '.bak') - _copy_lines_between_files(GetFailures.FILENAME_01, self.name, skip=14).close() + _copy_lines_between_files(GetFailures.FILENAME_01, self.name, skip=14, n=1).close() self.assert_correct_last_attempt(GetFailures.FAILURES_01) self.assertEqual(self.filter.failManager.getFailTotal(), 3) # now remove the moved file _killfile(None, self.name + '.bak') - _copy_lines_between_files(GetFailures.FILENAME_01, self.name, n=100).close() + _copy_lines_between_files(GetFailures.FILENAME_01, self.name, skip=12, n=3).close() self.assert_correct_last_attempt(GetFailures.FAILURES_01) self.assertEqual(self.filter.failManager.getFailTotal(), 6) @@ -1168,8 +1211,7 @@ def get_monitor_failures_testcase(Filter_): def _test_move_into_file(self, interim_kill=False): # if we move a new file into the location of an old (monitored) file - _copy_lines_between_files(GetFailures.FILENAME_01, self.name, - n=100).close() + _copy_lines_between_files(GetFailures.FILENAME_01, self.name).close() # make sure that it is monitored first self.assert_correct_last_attempt(GetFailures.FAILURES_01) self.assertEqual(self.filter.failManager.getFailTotal(), 3) @@ -1180,14 +1222,14 @@ def get_monitor_failures_testcase(Filter_): # now create a new one to override old one _copy_lines_between_files(GetFailures.FILENAME_01, self.name + '.new', - n=100).close() + skip=12, n=3).close() os.rename(self.name + '.new', self.name) self.assert_correct_last_attempt(GetFailures.FAILURES_01) self.assertEqual(self.filter.failManager.getFailTotal(), 6) # and to make sure that it now monitored for changes _copy_lines_between_files(GetFailures.FILENAME_01, self.name, - n=100).close() + skip=12, n=3).close() self.assert_correct_last_attempt(GetFailures.FAILURES_01) self.assertEqual(self.filter.failManager.getFailTotal(), 9) @@ -1206,7 +1248,7 @@ def get_monitor_failures_testcase(Filter_): # create a bogus file in the same directory and see if that doesn't affect open(self.name + '.bak2', 'w').close() - _copy_lines_between_files(GetFailures.FILENAME_01, self.name, n=100).close() + _copy_lines_between_files(GetFailures.FILENAME_01, self.name, skip=12, n=3).close() self.assert_correct_last_attempt(GetFailures.FAILURES_01) self.assertEqual(self.filter.failManager.getFailTotal(), 6) _killfile(None, self.name + '.bak2') @@ -1238,8 +1280,8 @@ def get_monitor_failures_testcase(Filter_): self.assert_correct_last_attempt(GetFailures.FAILURES_01, count=6) # was needed if we write twice above # now copy and get even more - _copy_lines_between_files(GetFailures.FILENAME_01, self.file, n=100) - # check for 3 failures (not 9), because 6 already get above... + _copy_lines_between_files(GetFailures.FILENAME_01, self.file, skip=12, n=3) + # check for 3 failures (not 9), because 6 already get above... self.assert_correct_last_attempt(GetFailures.FAILURES_01) # total count in this test: self.assertEqual(self.filter.failManager.getFailTotal(), 12) @@ -1274,6 +1316,7 @@ def get_monitor_failures_journal_testcase(Filter_): # pragma: systemd no cover def _initFilter(self, **kwargs): self._getRuntimeJournal() # check journal available self.filter = Filter_(self.jail, **kwargs) + self.filter.banASAP = False # avoid immediate ban in this tests self.filter.addJournalMatch([ "SYSLOG_IDENTIFIER=fail2ban-testcases", "TEST_FIELD=1", @@ -1397,6 +1440,52 @@ def get_monitor_failures_journal_testcase(Filter_): # pragma: systemd no cover self.test_file, self.journal_fields, skip=5, n=4) self.assert_correct_ban("193.168.0.128", 3) + @with_alt_time + def test_grow_file_with_db(self): + + def _gen_falure(ip): + # insert new failures ans check it is monitored: + fields = self.journal_fields + fields.update(TEST_JOURNAL_FIELDS) + journal.send(MESSAGE="error: PAM: Authentication failure for test from "+ip, **fields) + self.waitForTicks(1) + self.assert_correct_ban(ip, 1) + + # coverage for update log: + self.jail.database = getFail2BanDb(':memory:') + self.jail.database.addJail(self.jail) + MyTime.setTime(time.time()) + self._test_grow_file() + # stop: + self.filter.stop() + self.filter.join() + MyTime.setTime(time.time() + 2) + # update log manually (should cause a seek to end of log without wait for next second): + self.jail.database.updateJournal(self.jail, 'systemd-journal', MyTime.time(), 'TEST') + # check seek to last (simulated) position succeeds (without bans of previous copied tickets): + self._failTotal = 0 + self._initFilter() + self.filter.setMaxRetry(1) + self.filter.start() + self.waitForTicks(1) + # check new IP but no old IPs found: + _gen_falure("192.0.2.5") + self.assertFalse(self.jail.getFailTicket()) + + # now the same with increased time (check now - findtime case): + self.filter.stop() + self.filter.join() + MyTime.setTime(time.time() + 10000) + self._failTotal = 0 + self._initFilter() + self.filter.setMaxRetry(1) + self.filter.start() + self.waitForTicks(1) + MyTime.setTime(time.time() + 3) + # check new IP but no old IPs found: + _gen_falure("192.0.2.6") + self.assertFalse(self.jail.getFailTicket()) + def test_delJournalMatch(self): self._initFilter() self.filter.start() @@ -1481,6 +1570,7 @@ class GetFailures(LogCaptureTestCase): setUpMyTime() self.jail = DummyJail() self.filter = FileFilter(self.jail) + self.filter.banASAP = False # avoid immediate ban in this tests self.filter.active = True # speedup search using exact date pattern: self.filter.setDatePattern(r'^(?:%a )?%b %d %H:%M:%S(?:\.%f)?(?: %ExY)?') @@ -1536,9 +1626,9 @@ class GetFailures(LogCaptureTestCase): # We first adjust logfile/failures to end with CR+LF fname = tempfile.mktemp(prefix='tmp_fail2ban', suffix='crlf') # poor man unix2dos: - fin, fout = open(GetFailures.FILENAME_01), open(fname, 'w') - for l in fin.readlines(): - fout.write('%s\r\n' % l.rstrip('\n')) + fin, fout = open(GetFailures.FILENAME_01, 'rb'), open(fname, 'wb') + for l in fin.read().splitlines(): + fout.write(l + b'\r\n') fin.close() fout.close() @@ -1557,16 +1647,24 @@ class GetFailures(LogCaptureTestCase): _assert_correct_last_attempt(self, self.filter, output) def testGetFailures03(self): - output = ('203.162.223.135', 7, 1124013544.0) + output = ('203.162.223.135', 6, 1124013600.0) self.filter.addLogPath(GetFailures.FILENAME_03, autoSeek=0) self.filter.addFailRegex(r"error,relay=,.*550 User unknown") self.filter.getFailures(GetFailures.FILENAME_03) _assert_correct_last_attempt(self, self.filter, output) + def testGetFailures03_InOperation(self): + output = ('203.162.223.135', 9, 1124013600.0) + + self.filter.addLogPath(GetFailures.FILENAME_03, autoSeek=0) + self.filter.addFailRegex(r"error,relay=,.*550 User unknown") + self.filter.getFailures(GetFailures.FILENAME_03, inOperation=True) + _assert_correct_last_attempt(self, self.filter, output) + def testGetFailures03_Seek1(self): # same test as above but with seek to 'Aug 14 11:55:04' - so other output ... - output = ('203.162.223.135', 5, 1124013544.0) + output = ('203.162.223.135', 3, 1124013600.0) self.filter.addLogPath(GetFailures.FILENAME_03, autoSeek=output[2] - 4*60) self.filter.addFailRegex(r"error,relay=,.*550 User unknown") @@ -1575,7 +1673,7 @@ class GetFailures(LogCaptureTestCase): def testGetFailures03_Seek2(self): # same test as above but with seek to 'Aug 14 11:59:04' - so other output ... - output = ('203.162.223.135', 1, 1124013544.0) + output = ('203.162.223.135', 2, 1124013600.0) self.filter.setMaxRetry(1) self.filter.addLogPath(GetFailures.FILENAME_03, autoSeek=output[2]) @@ -1603,6 +1701,7 @@ class GetFailures(LogCaptureTestCase): _assert_correct_last_attempt(self, self.filter, output) def testGetFailuresWrongChar(self): + self.filter.checkFindTime = False # write wrong utf-8 char: fname = tempfile.mktemp(prefix='tmp_fail2ban', suffix='crlf') fout = fopen(fname, 'wb') @@ -1623,6 +1722,8 @@ class GetFailures(LogCaptureTestCase): for enc in (None, 'utf-8', 'ascii'): if enc is not None: self.tearDown();self.setUp(); + if DefLogSys.getEffectiveLevel() > 7: DefLogSys.setLevel(7); # ensure decode_line logs always + self.filter.checkFindTime = False; self.filter.setLogEncoding(enc); # speedup search using exact date pattern: self.filter.setDatePattern(r'^%ExY-%Exm-%Exd %ExH:%ExM:%ExS') @@ -1670,6 +1771,7 @@ class GetFailures(LogCaptureTestCase): self.pruneLog("[test-phase useDns=%s]" % useDns) jail = DummyJail() filter_ = FileFilter(jail, useDns=useDns) + filter_.banASAP = False # avoid immediate ban in this tests filter_.active = True filter_.failManager.setMaxRetry(1) # we might have just few failures @@ -1849,7 +1951,9 @@ class DNSUtilsNetworkTests(unittest.TestCase): ip4 = IPAddr('192.0.2.1') ip6 = IPAddr('2001:DB8::') self.assertTrue(ip4.isIPv4) + self.assertTrue(ip4.isSingle) self.assertTrue(ip6.isIPv6) + self.assertTrue(ip6.isSingle) self.assertTrue(asip('192.0.2.1').isIPv4) self.assertTrue(id(asip(ip4)) == id(ip4)) @@ -1858,6 +1962,7 @@ class DNSUtilsNetworkTests(unittest.TestCase): r = IPAddr('xxx', IPAddr.CIDR_RAW) self.assertFalse(r.isIPv4) self.assertFalse(r.isIPv6) + self.assertFalse(r.isSingle) self.assertTrue(r.isValid) self.assertEqual(r, 'xxx') self.assertEqual('xxx', str(r)) @@ -1866,6 +1971,7 @@ class DNSUtilsNetworkTests(unittest.TestCase): r = IPAddr('1:2', IPAddr.CIDR_RAW) self.assertFalse(r.isIPv4) self.assertFalse(r.isIPv6) + self.assertFalse(r.isSingle) self.assertTrue(r.isValid) self.assertEqual(r, '1:2') self.assertEqual('1:2', str(r)) @@ -1888,7 +1994,7 @@ class DNSUtilsNetworkTests(unittest.TestCase): def testUseDns(self): res = DNSUtils.textToIp('www.example.com', 'no') self.assertSortedEqual(res, []) - unittest.F2B.SkipIfNoNetwork() + #unittest.F2B.SkipIfNoNetwork() res = DNSUtils.textToIp('www.example.com', 'warn') # sort ipaddr, IPv4 is always smaller as IPv6 self.assertSortedEqual(res, ['93.184.216.34', '2606:2800:220:1:248:1893:25c8:1946']) @@ -1897,7 +2003,7 @@ class DNSUtilsNetworkTests(unittest.TestCase): self.assertSortedEqual(res, ['93.184.216.34', '2606:2800:220:1:248:1893:25c8:1946']) def testTextToIp(self): - unittest.F2B.SkipIfNoNetwork() + #unittest.F2B.SkipIfNoNetwork() # Test hostnames hostnames = [ 'www.example.com', @@ -1921,7 +2027,7 @@ class DNSUtilsNetworkTests(unittest.TestCase): self.assertTrue(isinstance(ip, IPAddr)) def testIpToName(self): - unittest.F2B.SkipIfNoNetwork() + #unittest.F2B.SkipIfNoNetwork() res = DNSUtils.ipToName('8.8.4.4') self.assertTrue(res.endswith(('.google', '.google.com'))) # same as above, but with IPAddr: @@ -1943,8 +2049,10 @@ class DNSUtilsNetworkTests(unittest.TestCase): self.assertEqual(res.addr, 167772160L) res = IPAddr('10.0.0.1', cidr=32L) self.assertEqual(res.addr, 167772161L) + self.assertTrue(res.isSingle) res = IPAddr('10.0.0.1', cidr=31L) self.assertEqual(res.addr, 167772160L) + self.assertFalse(res.isSingle) self.assertEqual(IPAddr('10.0.0.0').hexdump, '0a000000') self.assertEqual(IPAddr('1::2').hexdump, '00010000000000000000000000000002') @@ -1969,6 +2077,8 @@ class DNSUtilsNetworkTests(unittest.TestCase): def testIPAddr_InInet(self): ip4net = IPAddr('93.184.0.1/24') ip6net = IPAddr('2606:2800:220:1:248:1893:25c8:0/120') + self.assertFalse(ip4net.isSingle) + self.assertFalse(ip6net.isSingle) # ip4: self.assertTrue(IPAddr('93.184.0.1').isInNet(ip4net)) self.assertTrue(IPAddr('93.184.0.255').isInNet(ip4net)) @@ -2064,6 +2174,7 @@ class DNSUtilsNetworkTests(unittest.TestCase): ) def testIPAddr_CompareDNS(self): + #unittest.F2B.SkipIfNoNetwork() ips = IPAddr('example.com') self.assertTrue(IPAddr("93.184.216.34").isInNet(ips)) self.assertTrue(IPAddr("2606:2800:220:1:248:1893:25c8:1946").isInNet(ips)) diff --git a/fail2ban/tests/misctestcase.py b/fail2ban/tests/misctestcase.py index cd27ad92..458e9a23 100644 --- a/fail2ban/tests/misctestcase.py +++ b/fail2ban/tests/misctestcase.py @@ -34,7 +34,7 @@ from StringIO import StringIO from utils import LogCaptureTestCase, logSys as DefLogSys from ..helpers import formatExceptionInfo, mbasename, TraceBack, FormatterWithTraceBack, getLogger, \ - splitwords, uni_decode, uni_string + getVerbosityFormat, splitwords, uni_decode, uni_string from ..server.mytime import MyTime @@ -66,6 +66,8 @@ class HelpersTest(unittest.TestCase): self.assertEqual(splitwords(' 1, 2 , '), ['1', '2']) self.assertEqual(splitwords(' 1\n 2'), ['1', '2']) self.assertEqual(splitwords(' 1\n 2, 3'), ['1', '2', '3']) + # string as unicode: + self.assertEqual(splitwords(u' 1\n 2, 3'), ['1', '2', '3']) if sys.version_info >= (2,7): @@ -199,7 +201,8 @@ class TestsUtilsTest(LogCaptureTestCase): uni_decode((b'test\xcf' if sys.version_info >= (3,) else u'test\xcf')) uni_string(b'test\xcf') uni_string('test\xcf') - uni_string(u'test\xcf') + if sys.version_info < (3,) and 'PyPy' not in sys.version: + uni_string(u'test\xcf') def testSafeLogging(self): # logging should be exception-safe, to avoid possible errors (concat, str. conversion, representation failures, etc) @@ -388,12 +391,28 @@ class TestsUtilsTest(LogCaptureTestCase): self.assertSortedEqual(['Z', {'A': ['B', 'C'], 'B': ['E', 'F']}], [{'B': ['F', 'E'], 'A': ['C', 'B']}, 'Z'], level=-1) self.assertRaises(AssertionError, lambda: self.assertSortedEqual( - ['Z', {'A': ['B', 'C'], 'B': ['E', 'F']}], [{'B': ['F', 'E'], 'A': ['C', 'B']}, 'Z'])) + ['Z', {'A': ['B', 'C'], 'B': ['E', 'F']}], [{'B': ['F', 'E'], 'A': ['C', 'B']}, 'Z'], + nestedOnly=True)) + self.assertSortedEqual( + (0, [['A1'], ['A2', 'A1'], []]), + (0, [['A1'], ['A1', 'A2'], []]), + ) + self.assertSortedEqual(list('ABC'), list('CBA')) + self.assertRaises(AssertionError, self.assertSortedEqual, ['ABC'], ['CBA']) + self.assertRaises(AssertionError, self.assertSortedEqual, [['ABC']], [['CBA']]) self._testAssertionErrorRE(r"\['A'\] != \['C', 'B'\]", self.assertSortedEqual, ['A'], ['C', 'B']) self._testAssertionErrorRE(r"\['A', 'B'\] != \['B', 'C'\]", self.assertSortedEqual, ['A', 'B'], ['C', 'B']) + def testVerbosityFormat(self): + self.assertEqual(getVerbosityFormat(1), + '%(asctime)s %(name)-24s[%(process)d]: %(levelname)-7s %(message)s') + self.assertEqual(getVerbosityFormat(1, padding=False), + '%(asctime)s %(name)s[%(process)d]: %(levelname)s %(message)s') + self.assertEqual(getVerbosityFormat(1, addtime=False, padding=False), + '%(name)s[%(process)d]: %(levelname)s %(message)s') + def testFormatterWithTraceBack(self): strout = StringIO() Formatter = FormatterWithTraceBack diff --git a/fail2ban/tests/observertestcase.py b/fail2ban/tests/observertestcase.py index 8e944454..e379ccd1 100644 --- a/fail2ban/tests/observertestcase.py +++ b/fail2ban/tests/observertestcase.py @@ -36,7 +36,6 @@ from ..server.failmanager import FailManager from ..server.observer import Observers, ObserverThread from ..server.utils import Utils from .utils import LogCaptureTestCase -from ..server.filter import Filter from .dummyjail import DummyJail from .databasetestcase import getFail2BanDb, Fail2BanDb @@ -224,7 +223,7 @@ class BanTimeIncrDB(LogCaptureTestCase): jail.actions.setBanTime(10) jail.setBanTimeExtra('increment', 'true') jail.setBanTimeExtra('multipliers', '1 2 4 8 16 32 64 128 256 512 1024 2048') - ip = "127.0.0.2" + ip = "192.0.2.1" # used as start and fromtime (like now but time independence, cause test case can run slow): stime = int(MyTime.time()) ticket = FailTicket(ip, stime, []) @@ -385,10 +384,12 @@ class BanTimeIncrDB(LogCaptureTestCase): # two separate jails : jail1 = DummyJail(backend='polling') + jail1.filter.ignoreSelf = False jail1.setBanTimeExtra('increment', 'true') jail1.database = self.db self.db.addJail(jail1) jail2 = DummyJail(name='DummyJail-2', backend='polling') + jail2.filter.ignoreSelf = False jail2.database = self.db self.db.addJail(jail2) ticket1 = FailTicket(ip, stime, []) @@ -477,7 +478,7 @@ class BanTimeIncrDB(LogCaptureTestCase): self.assertEqual(tickets, []) # add failure: - ip = "127.0.0.2" + ip = "192.0.2.1" ticket = FailTicket(ip, stime-120, []) failManager = FailManager() failManager.setMaxRetry(3) diff --git a/fail2ban/tests/samplestestcase.py b/fail2ban/tests/samplestestcase.py index 1039b65e..b99dd06c 100644 --- a/fail2ban/tests/samplestestcase.py +++ b/fail2ban/tests/samplestestcase.py @@ -32,7 +32,7 @@ import sys import time import unittest from ..server.failregex import Regex -from ..server.filter import Filter +from ..server.filter import Filter, FileContainer from ..client.filterreader import FilterReader from .utils import setUpMyTime, tearDownMyTime, TEST_NOW, CONFIG_DIR @@ -157,10 +157,11 @@ def testSampleRegexsFactory(name, basedir): while i < len(filenames): filename = filenames[i]; i += 1; logFile = fileinput.FileInput(os.path.join(TEST_FILES_DIR, "logs", - filename)) + filename), mode='rb') ignoreBlock = False for line in logFile: + line = FileContainer.decode_line(logFile.filename(), 'UTF-8', line) jsonREMatch = re.match("^#+ ?(failJSON|(?:file|filter)Options|addFILE):(.+)$", line) if jsonREMatch: try: @@ -202,6 +203,7 @@ def testSampleRegexsFactory(name, basedir): raise ValueError("%s: %s:%i" % (e, logFile.filename(), logFile.filelineno())) line = next(logFile) + line = FileContainer.decode_line(logFile.filename(), 'UTF-8', line) elif ignoreBlock or line.startswith("#") or not line.strip(): continue else: # pragma: no cover - normally unreachable @@ -214,37 +216,41 @@ def testSampleRegexsFactory(name, basedir): flt = self._readFilter(fltName, name, basedir, opts=None) self._filterTests = [(fltName, flt, {})] + line = line.rstrip('\r\n') # process line using several filter options (if specified in the test-file): for fltName, flt, opts in self._filterTests: + # Bypass if constraint (as expression) is not valid: + if faildata.get('constraint') and not eval(faildata['constraint']): + continue flt, regexsUsedIdx = flt regexList = flt.getFailRegex() - failregex = -1 try: fail = {} # for logtype "journal" we don't need parse timestamp (simulate real systemd-backend handling): - checktime = True if opts.get('logtype') != 'journal': ret = flt.processLine(line) else: # simulate journal processing, time is known from journal (formatJournalEntry): - checktime = False if opts.get('test.prefix-line'): # journal backends creates common prefix-line: line = opts.get('test.prefix-line') + line - ret = flt.processLine(('', TEST_NOW_STR, line.rstrip('\r\n')), TEST_NOW) - if not ret: - # Bypass if filter constraint specified: - if faildata.get('filter') and name != faildata.get('filter'): - continue - # Check line is flagged as none match - self.assertFalse(faildata.get('match', True), - "Line not matched when should have") - continue + ret = flt.processLine(('', TEST_NOW_STR, line), TEST_NOW) + if ret: + # filter matched only (in checkAllRegex mode it could return 'nofail' too): + found = [] + for ret in ret: + failregex, fid, fail2banTime, fail = ret + # bypass pending and nofail: + if fid is None or fail.get('nofail'): + regexsUsedIdx.add(failregex) + regexsUsedRe.add(regexList[failregex]) + continue + found.append(ret) + ret = found - failregex, fid, fail2banTime, fail = ret[0] - # Bypass no failure helpers-regexp: - if not faildata.get('match', False) and (fid is None or fail.get('nofail')): - regexsUsedIdx.add(failregex) - regexsUsedRe.add(regexList[failregex]) + if not ret: + # Check line is flagged as none match + self.assertFalse(faildata.get('match', False), + "Line not matched when should have") continue # Check line is flagged to match @@ -253,39 +259,41 @@ def testSampleRegexsFactory(name, basedir): self.assertEqual(len(ret), 1, "Multiple regexs matched %r" % (map(lambda x: x[0], ret))) - # Verify match captures (at least fid/host) and timestamp as expected - for k, v in faildata.iteritems(): - if k not in ("time", "match", "desc", "filter"): - fv = fail.get(k, None) - if fv is None: - # Fallback for backwards compatibility (previously no fid, was host only): - if k == "host": - fv = fid - # special case for attempts counter: - if k == "attempts": - fv = len(fail.get('matches', {})) - # compare sorted (if set) - if isinstance(fv, (set, list, dict)): - self.assertSortedEqual(fv, v) - continue - self.assertEqual(fv, v) + for ret in ret: + failregex, fid, fail2banTime, fail = ret + # Verify match captures (at least fid/host) and timestamp as expected + for k, v in faildata.iteritems(): + if k not in ("time", "match", "desc", "constraint"): + fv = fail.get(k, None) + if fv is None: + # Fallback for backwards compatibility (previously no fid, was host only): + if k == "host": + fv = fid + # special case for attempts counter: + if k == "attempts": + fv = len(fail.get('matches', {})) + # compare sorted (if set) + if isinstance(fv, (set, list, dict)): + self.assertSortedEqual(fv, v) + continue + self.assertEqual(fv, v) - t = faildata.get("time", None) - if checktime or t is not None: - try: - jsonTimeLocal = datetime.datetime.strptime(t, "%Y-%m-%dT%H:%M:%S") - except ValueError: - jsonTimeLocal = datetime.datetime.strptime(t, "%Y-%m-%dT%H:%M:%S.%f") - jsonTime = time.mktime(jsonTimeLocal.timetuple()) - jsonTime += jsonTimeLocal.microsecond / 1000000.0 - self.assertEqual(fail2banTime, jsonTime, - "UTC Time mismatch %s (%s) != %s (%s) (diff %.3f seconds)" % - (fail2banTime, time.strftime("%Y-%m-%dT%H:%M:%S", time.gmtime(fail2banTime)), - jsonTime, time.strftime("%Y-%m-%dT%H:%M:%S", time.gmtime(jsonTime)), - fail2banTime - jsonTime) ) + t = faildata.get("time", None) + if t is not None: + try: + jsonTimeLocal = datetime.datetime.strptime(t, "%Y-%m-%dT%H:%M:%S") + except ValueError: + jsonTimeLocal = datetime.datetime.strptime(t, "%Y-%m-%dT%H:%M:%S.%f") + jsonTime = time.mktime(jsonTimeLocal.timetuple()) + jsonTime += jsonTimeLocal.microsecond / 1000000.0 + self.assertEqual(fail2banTime, jsonTime, + "UTC Time mismatch %s (%s) != %s (%s) (diff %.3f seconds)" % + (fail2banTime, time.strftime("%Y-%m-%dT%H:%M:%S", time.gmtime(fail2banTime)), + jsonTime, time.strftime("%Y-%m-%dT%H:%M:%S", time.gmtime(jsonTime)), + fail2banTime - jsonTime) ) - regexsUsedIdx.add(failregex) - regexsUsedRe.add(regexList[failregex]) + regexsUsedIdx.add(failregex) + regexsUsedRe.add(regexList[failregex]) except AssertionError as e: # pragma: no cover import pprint raise AssertionError("%s: %s on: %s:%i, line:\n %sregex (%s):\n %s\n" diff --git a/fail2ban/tests/servertestcase.py b/fail2ban/tests/servertestcase.py index 55e72455..d2bf8bdc 100644 --- a/fail2ban/tests/servertestcase.py +++ b/fail2ban/tests/servertestcase.py @@ -557,6 +557,9 @@ class Transmitter(TransmitterBase): jail=self.jailName) self.setGetTest("ignorecache", '', None, jail=self.jailName) + def testJailPrefRegex(self): + self.setGetTest("prefregex", "^Test", jail=self.jailName) + def testJailRegex(self): self.jailAddDelRegexTest("failregex", [ @@ -1361,11 +1364,11 @@ class ServerConfigReaderTests(LogCaptureTestCase): ), 'ip4-start': ( r"`nft add set inet f2b-table addr-set-j-w-nft-mp \{ type ipv4_addr\; \}`", - r"`nft add rule inet f2b-table f2b-chain $proto dport \{ http,https \} ip saddr @addr-set-j-w-nft-mp reject`", + r"`nft add rule inet f2b-table f2b-chain $proto dport \{ $(echo 'http,https' | sed s/:/-/g) \} ip saddr @addr-set-j-w-nft-mp reject`", ), 'ip6-start': ( r"`nft add set inet f2b-table addr6-set-j-w-nft-mp \{ type ipv6_addr\; \}`", - r"`nft add rule inet f2b-table f2b-chain $proto dport \{ http,https \} ip6 saddr @addr6-set-j-w-nft-mp reject`", + r"`nft add rule inet f2b-table f2b-chain $proto dport \{ $(echo 'http,https' | sed s/:/-/g) \} ip6 saddr @addr6-set-j-w-nft-mp reject`", ), 'flush': ( "`{ nft flush set inet f2b-table addr-set-j-w-nft-mp 2> /dev/null; } || ", @@ -1445,9 +1448,10 @@ class ServerConfigReaderTests(LogCaptureTestCase): ), }), # dummy -- - ('j-dummy', 'dummy[name=%(__name__)s, init="==", target="/tmp/fail2ban.dummy"]', { + ('j-dummy', '''dummy[name=%(__name__)s, init="=='/'==bt:==bc:==", target="/tmp/fail2ban.dummy"]''', { 'ip4': ('family: inet4',), 'ip6': ('family: inet6',), 'start': ( + '''`printf %b "=='/'==bt:600==bc:0==\\n"''', ## empty family (independent in this action, same for both), no ip on start, initial bantime and bancount '`echo "[j-dummy] dummy /tmp/fail2ban.dummy -- started"`', ), 'flush': ( @@ -1574,10 +1578,10 @@ class ServerConfigReaderTests(LogCaptureTestCase): ), }), # iptables-ipset-proto6 -- - ('j-w-iptables-ipset', 'iptables-ipset-proto6[name=%(__name__)s, bantime="10m", default-timeout=0, port="http", protocol="tcp", chain=""]', { + ('j-w-iptables-ipset', 'iptables-ipset-proto6[name=%(__name__)s, port="http", protocol="tcp", chain=""]', { 'ip4': (' f2b-j-w-iptables-ipset ',), 'ip6': (' f2b-j-w-iptables-ipset6 ',), 'ip4-start': ( - "`ipset create f2b-j-w-iptables-ipset hash:ip timeout 0`", + "`ipset create f2b-j-w-iptables-ipset hash:ip timeout 0 `", "`iptables -w -I INPUT -p tcp -m multiport --dports http -m set --match-set f2b-j-w-iptables-ipset src -j REJECT --reject-with icmp-port-unreachable`", ), 'ip6-start': ( @@ -1597,23 +1601,23 @@ class ServerConfigReaderTests(LogCaptureTestCase): "`ipset destroy f2b-j-w-iptables-ipset6`", ), 'ip4-ban': ( - r"`ipset add f2b-j-w-iptables-ipset 192.0.2.1 timeout 600 -exist`", + r"`ipset add f2b-j-w-iptables-ipset 192.0.2.1 timeout 0 -exist`", ), 'ip4-unban': ( r"`ipset del f2b-j-w-iptables-ipset 192.0.2.1 -exist`", ), 'ip6-ban': ( - r"`ipset add f2b-j-w-iptables-ipset6 2001:db8:: timeout 600 -exist`", + r"`ipset add f2b-j-w-iptables-ipset6 2001:db8:: timeout 0 -exist`", ), 'ip6-unban': ( r"`ipset del f2b-j-w-iptables-ipset6 2001:db8:: -exist`", ), }), # iptables-ipset-proto6-allports -- - ('j-w-iptables-ipset-ap', 'iptables-ipset-proto6-allports[name=%(__name__)s, bantime="10m", default-timeout=0, chain=""]', { + ('j-w-iptables-ipset-ap', 'iptables-ipset-proto6-allports[name=%(__name__)s, chain=""]', { 'ip4': (' f2b-j-w-iptables-ipset-ap ',), 'ip6': (' f2b-j-w-iptables-ipset-ap6 ',), 'ip4-start': ( - "`ipset create f2b-j-w-iptables-ipset-ap hash:ip timeout 0`", + "`ipset create f2b-j-w-iptables-ipset-ap hash:ip timeout 0 `", "`iptables -w -I INPUT -m set --match-set f2b-j-w-iptables-ipset-ap src -j REJECT --reject-with icmp-port-unreachable`", ), 'ip6-start': ( @@ -1633,13 +1637,13 @@ class ServerConfigReaderTests(LogCaptureTestCase): "`ipset destroy f2b-j-w-iptables-ipset-ap6`", ), 'ip4-ban': ( - r"`ipset add f2b-j-w-iptables-ipset-ap 192.0.2.1 timeout 600 -exist`", + r"`ipset add f2b-j-w-iptables-ipset-ap 192.0.2.1 timeout 0 -exist`", ), 'ip4-unban': ( r"`ipset del f2b-j-w-iptables-ipset-ap 192.0.2.1 -exist`", ), 'ip6-ban': ( - r"`ipset add f2b-j-w-iptables-ipset-ap6 2001:db8:: timeout 600 -exist`", + r"`ipset add f2b-j-w-iptables-ipset-ap6 2001:db8:: timeout 0 -exist`", ), 'ip6-unban': ( r"`ipset del f2b-j-w-iptables-ipset-ap6 2001:db8:: -exist`", @@ -1842,18 +1846,18 @@ class ServerConfigReaderTests(LogCaptureTestCase): 'ip4-start': ( "`firewall-cmd --direct --add-chain ipv4 filter f2b-j-w-fwcmd-mp`", "`firewall-cmd --direct --add-rule ipv4 filter f2b-j-w-fwcmd-mp 1000 -j RETURN`", - "`firewall-cmd --direct --add-rule ipv4 filter INPUT_direct 0 -m conntrack --ctstate NEW -p tcp -m multiport --dports http,https -j f2b-j-w-fwcmd-mp`", + """`firewall-cmd --direct --add-rule ipv4 filter INPUT_direct 0 -m conntrack --ctstate NEW -p tcp -m multiport --dports "$(echo 'http,https' | sed s/:/-/g)" -j f2b-j-w-fwcmd-mp`""", ), 'ip6-start': ( "`firewall-cmd --direct --add-chain ipv6 filter f2b-j-w-fwcmd-mp`", "`firewall-cmd --direct --add-rule ipv6 filter f2b-j-w-fwcmd-mp 1000 -j RETURN`", - "`firewall-cmd --direct --add-rule ipv6 filter INPUT_direct 0 -m conntrack --ctstate NEW -p tcp -m multiport --dports http,https -j f2b-j-w-fwcmd-mp`", + """`firewall-cmd --direct --add-rule ipv6 filter INPUT_direct 0 -m conntrack --ctstate NEW -p tcp -m multiport --dports "$(echo 'http,https' | sed s/:/-/g)" -j f2b-j-w-fwcmd-mp`""", ), 'stop': ( - "`firewall-cmd --direct --remove-rule ipv4 filter INPUT_direct 0 -m conntrack --ctstate NEW -p tcp -m multiport --dports http,https -j f2b-j-w-fwcmd-mp`", + """`firewall-cmd --direct --remove-rule ipv4 filter INPUT_direct 0 -m conntrack --ctstate NEW -p tcp -m multiport --dports "$(echo 'http,https' | sed s/:/-/g)" -j f2b-j-w-fwcmd-mp`""", "`firewall-cmd --direct --remove-rules ipv4 filter f2b-j-w-fwcmd-mp`", "`firewall-cmd --direct --remove-chain ipv4 filter f2b-j-w-fwcmd-mp`", - "`firewall-cmd --direct --remove-rule ipv6 filter INPUT_direct 0 -m conntrack --ctstate NEW -p tcp -m multiport --dports http,https -j f2b-j-w-fwcmd-mp`", + """`firewall-cmd --direct --remove-rule ipv6 filter INPUT_direct 0 -m conntrack --ctstate NEW -p tcp -m multiport --dports "$(echo 'http,https' | sed s/:/-/g)" -j f2b-j-w-fwcmd-mp`""", "`firewall-cmd --direct --remove-rules ipv6 filter f2b-j-w-fwcmd-mp`", "`firewall-cmd --direct --remove-chain ipv6 filter f2b-j-w-fwcmd-mp`", ), @@ -1917,50 +1921,50 @@ class ServerConfigReaderTests(LogCaptureTestCase): ), }), # firewallcmd-ipset (multiport) -- - ('j-w-fwcmd-ipset', 'firewallcmd-ipset[name=%(__name__)s, bantime="10m", default-timeout=0, port="http", protocol="tcp", chain=""]', { + ('j-w-fwcmd-ipset', 'firewallcmd-ipset[name=%(__name__)s, port="http", protocol="tcp", chain=""]', { 'ip4': (' f2b-j-w-fwcmd-ipset ',), 'ip6': (' f2b-j-w-fwcmd-ipset6 ',), 'ip4-start': ( - "`ipset create f2b-j-w-fwcmd-ipset hash:ip timeout 0`", - "`firewall-cmd --direct --add-rule ipv4 filter INPUT_direct 0 -p tcp -m multiport --dports http -m set --match-set f2b-j-w-fwcmd-ipset src -j REJECT --reject-with icmp-port-unreachable`", + "`ipset create f2b-j-w-fwcmd-ipset hash:ip timeout 0 `", + """`firewall-cmd --direct --add-rule ipv4 filter INPUT_direct 0 -p tcp -m multiport --dports "$(echo 'http' | sed s/:/-/g)" -m set --match-set f2b-j-w-fwcmd-ipset src -j REJECT --reject-with icmp-port-unreachable`""", ), 'ip6-start': ( "`ipset create f2b-j-w-fwcmd-ipset6 hash:ip timeout 0 family inet6`", - "`firewall-cmd --direct --add-rule ipv6 filter INPUT_direct 0 -p tcp -m multiport --dports http -m set --match-set f2b-j-w-fwcmd-ipset6 src -j REJECT --reject-with icmp6-port-unreachable`", + """`firewall-cmd --direct --add-rule ipv6 filter INPUT_direct 0 -p tcp -m multiport --dports "$(echo 'http' | sed s/:/-/g)" -m set --match-set f2b-j-w-fwcmd-ipset6 src -j REJECT --reject-with icmp6-port-unreachable`""", ), 'flush': ( "`ipset flush f2b-j-w-fwcmd-ipset`", "`ipset flush f2b-j-w-fwcmd-ipset6`", ), 'stop': ( - "`firewall-cmd --direct --remove-rule ipv4 filter INPUT_direct 0 -p tcp -m multiport --dports http -m set --match-set f2b-j-w-fwcmd-ipset src -j REJECT --reject-with icmp-port-unreachable`", + """`firewall-cmd --direct --remove-rule ipv4 filter INPUT_direct 0 -p tcp -m multiport --dports "$(echo 'http' | sed s/:/-/g)" -m set --match-set f2b-j-w-fwcmd-ipset src -j REJECT --reject-with icmp-port-unreachable`""", "`ipset flush f2b-j-w-fwcmd-ipset`", "`ipset destroy f2b-j-w-fwcmd-ipset`", - "`firewall-cmd --direct --remove-rule ipv6 filter INPUT_direct 0 -p tcp -m multiport --dports http -m set --match-set f2b-j-w-fwcmd-ipset6 src -j REJECT --reject-with icmp6-port-unreachable`", + """`firewall-cmd --direct --remove-rule ipv6 filter INPUT_direct 0 -p tcp -m multiport --dports "$(echo 'http' | sed s/:/-/g)" -m set --match-set f2b-j-w-fwcmd-ipset6 src -j REJECT --reject-with icmp6-port-unreachable`""", "`ipset flush f2b-j-w-fwcmd-ipset6`", "`ipset destroy f2b-j-w-fwcmd-ipset6`", ), 'ip4-ban': ( - r"`ipset add f2b-j-w-fwcmd-ipset 192.0.2.1 timeout 600 -exist`", + r"`ipset add f2b-j-w-fwcmd-ipset 192.0.2.1 timeout 0 -exist`", ), 'ip4-unban': ( r"`ipset del f2b-j-w-fwcmd-ipset 192.0.2.1 -exist`", ), 'ip6-ban': ( - r"`ipset add f2b-j-w-fwcmd-ipset6 2001:db8:: timeout 600 -exist`", + r"`ipset add f2b-j-w-fwcmd-ipset6 2001:db8:: timeout 0 -exist`", ), 'ip6-unban': ( r"`ipset del f2b-j-w-fwcmd-ipset6 2001:db8:: -exist`", ), }), # firewallcmd-ipset (allports) -- - ('j-w-fwcmd-ipset-ap', 'firewallcmd-ipset[name=%(__name__)s, bantime="10m", actiontype=, protocol="tcp", chain=""]', { + ('j-w-fwcmd-ipset-ap', 'firewallcmd-ipset[name=%(__name__)s, actiontype=, protocol="tcp", chain=""]', { 'ip4': (' f2b-j-w-fwcmd-ipset-ap ',), 'ip6': (' f2b-j-w-fwcmd-ipset-ap6 ',), 'ip4-start': ( - "`ipset create f2b-j-w-fwcmd-ipset-ap hash:ip timeout 600`", + "`ipset create f2b-j-w-fwcmd-ipset-ap hash:ip timeout 0 `", "`firewall-cmd --direct --add-rule ipv4 filter INPUT_direct 0 -p tcp -m set --match-set f2b-j-w-fwcmd-ipset-ap src -j REJECT --reject-with icmp-port-unreachable`", ), 'ip6-start': ( - "`ipset create f2b-j-w-fwcmd-ipset-ap6 hash:ip timeout 600 family inet6`", + "`ipset create f2b-j-w-fwcmd-ipset-ap6 hash:ip timeout 0 family inet6`", "`firewall-cmd --direct --add-rule ipv6 filter INPUT_direct 0 -p tcp -m set --match-set f2b-j-w-fwcmd-ipset-ap6 src -j REJECT --reject-with icmp6-port-unreachable`", ), 'flush': ( @@ -1976,18 +1980,50 @@ class ServerConfigReaderTests(LogCaptureTestCase): "`ipset destroy f2b-j-w-fwcmd-ipset-ap6`", ), 'ip4-ban': ( - r"`ipset add f2b-j-w-fwcmd-ipset-ap 192.0.2.1 timeout 600 -exist`", + r"`ipset add f2b-j-w-fwcmd-ipset-ap 192.0.2.1 timeout 0 -exist`", ), 'ip4-unban': ( r"`ipset del f2b-j-w-fwcmd-ipset-ap 192.0.2.1 -exist`", ), 'ip6-ban': ( - r"`ipset add f2b-j-w-fwcmd-ipset-ap6 2001:db8:: timeout 600 -exist`", + r"`ipset add f2b-j-w-fwcmd-ipset-ap6 2001:db8:: timeout 0 -exist`", ), 'ip6-unban': ( r"`ipset del f2b-j-w-fwcmd-ipset-ap6 2001:db8:: -exist`", ), }), + # firewallcmd-rich-rules -- + ('j-fwcmd-rr', 'firewallcmd-rich-rules[port="22:24", protocol="tcp"]', { + 'ip4': ("family='ipv4'", "icmp-port-unreachable",), 'ip6': ("family='ipv6'", 'icmp6-port-unreachable',), + 'ip4-ban': ( + """`ports="$(echo '22:24' | sed s/:/-/g)"; for p in $(echo $ports | tr ", " " "); do firewall-cmd --add-rich-rule="rule family='ipv4' source address='192.0.2.1' port port='$p' protocol='tcp' reject type='icmp-port-unreachable'"; done`""", + ), + 'ip4-unban': ( + """`ports="$(echo '22:24' | sed s/:/-/g)"; for p in $(echo $ports | tr ", " " "); do firewall-cmd --remove-rich-rule="rule family='ipv4' source address='192.0.2.1' port port='$p' protocol='tcp' reject type='icmp-port-unreachable'"; done`""", + ), + 'ip6-ban': ( + """ `ports="$(echo '22:24' | sed s/:/-/g)"; for p in $(echo $ports | tr ", " " "); do firewall-cmd --add-rich-rule="rule family='ipv6' source address='2001:db8::' port port='$p' protocol='tcp' reject type='icmp6-port-unreachable'"; done`""", + ), + 'ip6-unban': ( + """`ports="$(echo '22:24' | sed s/:/-/g)"; for p in $(echo $ports | tr ", " " "); do firewall-cmd --remove-rich-rule="rule family='ipv6' source address='2001:db8::' port port='$p' protocol='tcp' reject type='icmp6-port-unreachable'"; done`""", + ), + }), + # firewallcmd-rich-logging -- + ('j-fwcmd-rl', 'firewallcmd-rich-logging[port="22:24", protocol="tcp"]', { + 'ip4': ("family='ipv4'", "icmp-port-unreachable",), 'ip6': ("family='ipv6'", 'icmp6-port-unreachable',), + 'ip4-ban': ( + """`ports="$(echo '22:24' | sed s/:/-/g)"; for p in $(echo $ports | tr ", " " "); do firewall-cmd --add-rich-rule="rule family='ipv4' source address='192.0.2.1' port port='$p' protocol='tcp' log prefix='f2b-j-fwcmd-rl' level='info' limit value='1/m' reject type='icmp-port-unreachable'"; done`""", + ), + 'ip4-unban': ( + """`ports="$(echo '22:24' | sed s/:/-/g)"; for p in $(echo $ports | tr ", " " "); do firewall-cmd --remove-rich-rule="rule family='ipv4' source address='192.0.2.1' port port='$p' protocol='tcp' log prefix='f2b-j-fwcmd-rl' level='info' limit value='1/m' reject type='icmp-port-unreachable'"; done`""", + ), + 'ip6-ban': ( + """ `ports="$(echo '22:24' | sed s/:/-/g)"; for p in $(echo $ports | tr ", " " "); do firewall-cmd --add-rich-rule="rule family='ipv6' source address='2001:db8::' port port='$p' protocol='tcp' log prefix='f2b-j-fwcmd-rl' level='info' limit value='1/m' reject type='icmp6-port-unreachable'"; done`""", + ), + 'ip6-unban': ( + """`ports="$(echo '22:24' | sed s/:/-/g)"; for p in $(echo $ports | tr ", " " "); do firewall-cmd --remove-rich-rule="rule family='ipv6' source address='2001:db8::' port port='$p' protocol='tcp' log prefix='f2b-j-fwcmd-rl' level='info' limit value='1/m' reject type='icmp6-port-unreachable'"; done`""", + ), + }), ) server = TestServer() transm = server._Server__transm diff --git a/fail2ban/tests/sockettestcase.py b/fail2ban/tests/sockettestcase.py index 69bf8d8b..e3a07998 100644 --- a/fail2ban/tests/sockettestcase.py +++ b/fail2ban/tests/sockettestcase.py @@ -87,7 +87,7 @@ class Socket(LogCaptureTestCase): def _stopServerThread(self): serverThread = self.serverThread # wait for end of thread : - Utils.wait_for(lambda: not serverThread.isAlive() + Utils.wait_for(lambda: not serverThread.is_alive() or serverThread.join(Utils.DEFAULT_SLEEP_TIME), unittest.F2B.maxWaitTime(10)) self.serverThread = None @@ -98,7 +98,7 @@ class Socket(LogCaptureTestCase): self.server.close() # wait for end of thread : self._stopServerThread() - self.assertFalse(serverThread.isAlive()) + self.assertFalse(serverThread.is_alive()) # clean : self.server.stop() self.assertFalse(self.server.isActive()) @@ -139,7 +139,7 @@ class Socket(LogCaptureTestCase): self.server.stop() # wait for end of thread : self._stopServerThread() - self.assertFalse(serverThread.isAlive()) + self.assertFalse(serverThread.is_alive()) self.assertFalse(self.server.isActive()) self.assertFalse(os.path.exists(self.sock_name)) @@ -153,7 +153,7 @@ class Socket(LogCaptureTestCase): org_handler = RequestHandler.found_terminator try: RequestHandler.found_terminator = lambda self: self.close() - self.assertRaisesRegexp(RuntimeError, r"socket connection broken", + self.assertRaisesRegexp(Exception, r"reset by peer|Broken pipe", lambda: client.send(testMessage, timeout=unittest.F2B.maxWaitTime(10))) finally: RequestHandler.found_terminator = org_handler @@ -169,7 +169,7 @@ class Socket(LogCaptureTestCase): org_handler = RequestHandler.found_terminator try: RequestHandler.found_terminator = lambda self: TestMsgError() - #self.assertRaisesRegexp(RuntimeError, r"socket connection broken", client.send, testMessage) + #self.assertRaisesRegexp(Exception, r"reset by peer|Broken pipe", client.send, testMessage) self.assertEqual(client.send(testMessage), 'ERROR: test unpickle error') finally: RequestHandler.found_terminator = org_handler @@ -180,7 +180,7 @@ class Socket(LogCaptureTestCase): self.server.stop() # wait for end of thread : self._stopServerThread() - self.assertFalse(serverThread.isAlive()) + self.assertFalse(serverThread.is_alive()) def testLoopErrors(self): # replace poll handler to produce error in loop-cycle: @@ -216,7 +216,7 @@ class Socket(LogCaptureTestCase): self.server.stop() # wait for end of thread : self._stopServerThread() - self.assertFalse(serverThread.isAlive()) + self.assertFalse(serverThread.is_alive()) self.assertFalse(self.server.isActive()) self.assertFalse(os.path.exists(self.sock_name)) diff --git a/fail2ban/tests/tickettestcase.py b/fail2ban/tests/tickettestcase.py index 277c2f28..d7d5f19a 100644 --- a/fail2ban/tests/tickettestcase.py +++ b/fail2ban/tests/tickettestcase.py @@ -69,10 +69,10 @@ class TicketTests(unittest.TestCase): self.assertEqual(ft.getTime(), tm) self.assertEqual(ft.getMatches(), matches2) ft.setAttempt(2) - self.assertEqual(ft.getAttempt(), 2) - # retry is max of set retry and failures: - self.assertEqual(ft.getRetry(), 2) ft.setRetry(1) + self.assertEqual(ft.getAttempt(), 2) + self.assertEqual(ft.getRetry(), 1) + ft.setRetry(2) self.assertEqual(ft.getRetry(), 2) ft.setRetry(3) self.assertEqual(ft.getRetry(), 3) @@ -86,13 +86,21 @@ class TicketTests(unittest.TestCase): self.assertEqual(ft.getRetry(), 14) self.assertEqual(ft.getMatches(), matches3) # last time (ignore if smaller as time): - self.assertEqual(ft.getLastTime(), tm) - ft.setLastTime(tm-60) self.assertEqual(ft.getTime(), tm) - self.assertEqual(ft.getLastTime(), tm) - ft.setLastTime(tm+60) + ft.adjustTime(tm-60, 3600) + self.assertEqual(ft.getTime(), tm) + self.assertEqual(ft.getRetry(), 14) + ft.adjustTime(tm+60, 3600) self.assertEqual(ft.getTime(), tm+60) - self.assertEqual(ft.getLastTime(), tm+60) + self.assertEqual(ft.getRetry(), 14) + ft.adjustTime(tm+3600, 3600) + self.assertEqual(ft.getTime(), tm+3600) + self.assertEqual(ft.getRetry(), 14) + # adjust time so interval is larger than find time (3600), so reset retry count: + ft.adjustTime(tm+7200, 3600) + self.assertEqual(ft.getTime(), tm+7200) + self.assertEqual(ft.getRetry(), 7); # estimated attempts count + self.assertEqual(ft.getAttempt(), 4); # real known failure count ft.setData('country', 'DE') self.assertEqual(ft.getData(), {'matches': ['first', 'second', 'third'], 'failures': 4, 'country': 'DE'}) @@ -102,10 +110,10 @@ class TicketTests(unittest.TestCase): self.assertEqual(ft, ft2) self.assertEqual(ft.getData(), ft2.getData()) self.assertEqual(ft2.getAttempt(), 4) - self.assertEqual(ft2.getRetry(), 14) + self.assertEqual(ft2.getRetry(), 7) self.assertEqual(ft2.getMatches(), matches3) self.assertEqual(ft2.getTime(), ft.getTime()) - self.assertEqual(ft2.getLastTime(), ft.getLastTime()) + self.assertEqual(ft2.getTime(), ft.getTime()) self.assertEqual(ft2.getBanTime(), ft.getBanTime()) def testTicketFlags(self): diff --git a/fail2ban/tests/utils.py b/fail2ban/tests/utils.py index fcfddba7..0c5ed139 100644 --- a/fail2ban/tests/utils.py +++ b/fail2ban/tests/utils.py @@ -39,7 +39,7 @@ from cStringIO import StringIO from functools import wraps from ..helpers import getLogger, str2LogLevel, getVerbosityFormat, uni_decode -from ..server.ipdns import DNSUtils +from ..server.ipdns import IPAddr, DNSUtils from ..server.mytime import MyTime from ..server.utils import Utils # for action_d.test_smtp : @@ -292,15 +292,15 @@ def initTests(opts): unittest.F2B.SkipIfFast = F2B_SkipIfFast else: # smaller inertance inside test-cases (litle speedup): - Utils.DEFAULT_SLEEP_TIME = 0.25 - Utils.DEFAULT_SLEEP_INTERVAL = 0.025 + Utils.DEFAULT_SLEEP_TIME = 0.025 + Utils.DEFAULT_SLEEP_INTERVAL = 0.005 Utils.DEFAULT_SHORT_INTERVAL = 0.0005 # sleep intervals are large - use replacement for sleep to check time to sleep: _org_sleep = time.sleep def _new_sleep(v): - if v > max(1, Utils.DEFAULT_SLEEP_TIME): # pragma: no cover + if v > 0.25: # pragma: no cover raise ValueError('[BAD-CODE] To long sleep interval: %s, try to use conditional Utils.wait_for instead' % v) - _org_sleep(min(v, Utils.DEFAULT_SLEEP_TIME)) + _org_sleep(v) time.sleep = _new_sleep # --no-network : if unittest.F2B.no_network: # pragma: no cover @@ -331,13 +331,21 @@ def initTests(opts): c.set('2001:db8::ffff', 'test-other') c.set('87.142.124.10', 'test-host') if unittest.F2B.no_network: # pragma: no cover - # precache all wrong dns to ip's used in test cases: + # precache all ip to dns used in test cases: + c.set('192.0.2.888', None) + c.set('8.8.4.4', 'dns.google') + c.set('8.8.4.4', 'dns.google') + # precache all dns to ip's used in test cases: c = DNSUtils.CACHE_nameToIp for i in ( ('999.999.999.999', set()), ('abcdef.abcdef', set()), ('192.168.0.', set()), ('failed.dns.ch', set()), + ('doh1.2.3.4.buga.xxxxx.yyy.invalid', set()), + ('1.2.3.4.buga.xxxxx.yyy.invalid', set()), + ('example.com', set([IPAddr('2606:2800:220:1:248:1893:25c8:1946'), IPAddr('93.184.216.34')])), + ('www.example.com', set([IPAddr('2606:2800:220:1:248:1893:25c8:1946'), IPAddr('93.184.216.34')])), ): c.set(*i) # if fast - precache all host names as localhost addresses (speed-up getSelfIPs/ignoreself): @@ -552,7 +560,7 @@ if not hasattr(unittest.TestCase, 'assertDictEqual'): self.fail(msg) unittest.TestCase.assertDictEqual = assertDictEqual -def assertSortedEqual(self, a, b, level=1, nestedOnly=True, key=repr, msg=None): +def assertSortedEqual(self, a, b, level=1, nestedOnly=False, key=repr, msg=None): """Compare complex elements (like dict, list or tuple) in sorted order until level 0 not reached (initial level = -1 meant all levels), or if nestedOnly set to True and some of the objects still contains nested lists or dicts. @@ -562,6 +570,13 @@ def assertSortedEqual(self, a, b, level=1, nestedOnly=True, key=repr, msg=None): if isinstance(v, dict): return any(isinstance(v, (dict, list, tuple)) for v in v.itervalues()) return any(isinstance(v, (dict, list, tuple)) for v in v) + if nestedOnly: + _nest_sorted = sorted + else: + def _nest_sorted(v, key=key): + if isinstance(v, (set, list, tuple)): + return sorted(list(_nest_sorted(v, key) for v in v), key=key) + return v # level comparison routine: def _assertSortedEqual(a, b, level, nestedOnly, key): # first the lengths: @@ -580,8 +595,8 @@ def assertSortedEqual(self, a, b, level=1, nestedOnly=True, key=repr, msg=None): elif v1 != v2: raise ValueError('%r != %r' % (a, b)) else: # list, tuple, something iterable: - a = sorted(a, key=key) - b = sorted(b, key=key) + a = _nest_sorted(a, key=key) + b = _nest_sorted(b, key=key) for v1, v2 in zip(a, b): if isinstance(v1, (dict, list, tuple)) and isinstance(v2, (dict, list, tuple)): _assertSortedEqual(v1, v2, level-1 if level != 0 else 0, nestedOnly, key) diff --git a/fail2ban/version.py b/fail2ban/version.py index e5efbe77..77b81097 100644 --- a/fail2ban/version.py +++ b/fail2ban/version.py @@ -24,7 +24,7 @@ __author__ = "Cyril Jaquier, Yaroslav Halchenko, Steven Hiscocks, Daniel Black" __copyright__ = "Copyright (c) 2004 Cyril Jaquier, 2005-2016 Yaroslav Halchenko, 2013-2014 Steven Hiscocks, Daniel Black" __license__ = "GPL-v2+" -version = "0.11.1" +version = "0.11.2-dev" def normVersion(): """ Returns fail2ban version in normalized machine-readable format""" diff --git a/files/fail2ban.service.in b/files/fail2ban.service.in index 5e540545..9a245c61 100644 --- a/files/fail2ban.service.in +++ b/files/fail2ban.service.in @@ -6,6 +6,7 @@ PartOf=iptables.service firewalld.service ip6tables.service ipset.service nftabl [Service] Type=simple +Environment="PYTHONNOUSERSITE=1" ExecStartPre=/bin/mkdir -p /run/fail2ban ExecStart=@BINDIR@/fail2ban-server -xf start # if should be logged in systemd journal, use following line or set logtarget to sysout in fail2ban.local diff --git a/man/fail2ban-client.1 b/man/fail2ban-client.1 index 745c080a..372c2b7a 100644 --- a/man/fail2ban-client.1 +++ b/man/fail2ban-client.1 @@ -1,24 +1,27 @@ .\" DO NOT MODIFY THIS FILE! It was generated by help2man 1.47.4. -.TH FAIL2BAN-CLIENT "1" "January 2020" "fail2ban-client v0.11.1" "User Commands" +.TH FAIL2BAN-CLIENT "1" "February 2020" "fail2ban-client v0.11.2-dev" "User Commands" .SH NAME fail2ban-client \- configure and control the server .SH SYNOPSIS .B fail2ban-client [\fI\,OPTIONS\/\fR] \fI\,\/\fR .SH DESCRIPTION -Fail2Ban v0.11.1 reads log file that contains password failure report +Fail2Ban v0.11.2\-dev reads log file that contains password failure report and bans the corresponding IP addresses using firewall rules. .SH OPTIONS .TP -\fB\-c\fR +\fB\-c\fR, \fB\-\-conf\fR configuration directory .TP -\fB\-s\fR +\fB\-s\fR, \fB\-\-socket\fR socket path .TP -\fB\-p\fR +\fB\-p\fR, \fB\-\-pidfile\fR pidfile path .TP +\fB\-\-pname\fR +name of the process (main thread) to identify instance (default fail2ban\-server) +.TP \fB\-\-loglevel\fR logging level .TP @@ -108,6 +111,14 @@ jails and database) unbans (in all jails and database) .TP +\fBbanned\fR +return jails with banned IPs as +dictionary +.TP +\fBbanned ... ]\fR +return list(s) of jails where +given IP(s) are banned +.TP \fBstatus\fR gets the current status of the server @@ -353,6 +364,14 @@ for .IP JAIL INFORMATION .TP +\fBget banned\fR +return banned IPs of +.TP +\fBget banned ... ]\fR +return 1 if IP is banned in +otherwise 0, or a list of 1/0 for +multiple IPs +.TP \fBget logpath\fR gets the list of the monitored files for diff --git a/man/fail2ban-python.1 b/man/fail2ban-python.1 index 2c9746ee..5a9c96e7 100644 --- a/man/fail2ban-python.1 +++ b/man/fail2ban-python.1 @@ -1,5 +1,5 @@ .\" DO NOT MODIFY THIS FILE! It was generated by help2man 1.47.4. -.TH FAIL2BAN-PYTHON "1" "January 2020" "fail2ban-python 0.11.1" "User Commands" +.TH FAIL2BAN-PYTHON "1" "February 2020" "fail2ban-python 0.11.2-dev" "User Commands" .SH NAME fail2ban-python \- a helper for Fail2Ban to assure that the same Python is used .SH DESCRIPTION diff --git a/man/fail2ban-regex.1 b/man/fail2ban-regex.1 index a703130c..d122aebc 100644 --- a/man/fail2ban-regex.1 +++ b/man/fail2ban-regex.1 @@ -1,5 +1,5 @@ .\" DO NOT MODIFY THIS FILE! It was generated by help2man 1.47.4. -.TH FAIL2BAN-REGEX "1" "January 2020" "fail2ban-regex 0.11.1" "User Commands" +.TH FAIL2BAN-REGEX "1" "February 2020" "fail2ban-regex 0.11.2-dev" "User Commands" .SH NAME fail2ban-regex \- test Fail2ban "failregex" option .SH SYNOPSIS @@ -18,13 +18,18 @@ a string representing a log line filename path to a log file (\fI\,/var/log/auth.log\/\fP) .TP -"systemd\-journal" -search systemd journal (systemd\-python required) +systemd\-journal +search systemd journal (systemd\-python required), +optionally with backend parameters, see `man jail.conf` +for usage and examples (systemd\-journal[journalflags=1]). .SS "REGEX:" .TP string a string representing a 'failregex' .TP +filter +name of filter, optionally with options (sshd[mode=aggressive]) +.TP filename path to a filter file (filter.d/sshd.conf) .SS "IGNOREREGEX:" diff --git a/man/fail2ban-server.1 b/man/fail2ban-server.1 index 418b46dd..8099ffe9 100644 --- a/man/fail2ban-server.1 +++ b/man/fail2ban-server.1 @@ -1,24 +1,27 @@ .\" DO NOT MODIFY THIS FILE! It was generated by help2man 1.47.4. -.TH FAIL2BAN-SERVER "1" "January 2020" "fail2ban-server v0.11.1" "User Commands" +.TH FAIL2BAN-SERVER "1" "February 2020" "fail2ban-server v0.11.2-dev" "User Commands" .SH NAME fail2ban-server \- start the server .SH SYNOPSIS .B fail2ban-server [\fI\,OPTIONS\/\fR] .SH DESCRIPTION -Fail2Ban v0.11.1 reads log file that contains password failure report +Fail2Ban v0.11.2\-dev reads log file that contains password failure report and bans the corresponding IP addresses using firewall rules. .SH OPTIONS .TP -\fB\-c\fR +\fB\-c\fR, \fB\-\-conf\fR configuration directory .TP -\fB\-s\fR +\fB\-s\fR, \fB\-\-socket\fR socket path .TP -\fB\-p\fR +\fB\-p\fR, \fB\-\-pidfile\fR pidfile path .TP +\fB\-\-pname\fR +name of the process (main thread) to identify instance (default fail2ban\-server) +.TP \fB\-\-loglevel\fR logging level .TP diff --git a/man/fail2ban-testcases.1 b/man/fail2ban-testcases.1 index 6c32cb9e..643c8c25 100644 --- a/man/fail2ban-testcases.1 +++ b/man/fail2ban-testcases.1 @@ -1,5 +1,5 @@ .\" DO NOT MODIFY THIS FILE! It was generated by help2man 1.47.4. -.TH FAIL2BAN-TESTCASES "1" "January 2020" "fail2ban-testcases 0.11.1" "User Commands" +.TH FAIL2BAN-TESTCASES "1" "February 2020" "fail2ban-testcases 0.11.2-dev" "User Commands" .SH NAME fail2ban-testcases \- run Fail2Ban unit-tests .SH SYNOPSIS diff --git a/man/jail.conf.5 b/man/jail.conf.5 index 662b3f48..d7722124 100644 --- a/man/jail.conf.5 +++ b/man/jail.conf.5 @@ -130,7 +130,7 @@ Comments: use '#' for comment lines and '; ' (space is important) for inline com The items that can be set in section [Definition] are: .TP .B loglevel -verbosity level of log output: CRITICAL, ERROR, WARNING, NOTICE, INFO, DEBUG, TRACEDEBUG, HEAVYDEBUG or corresponding numeric value (50-5). Default: ERROR (equal 40) +verbosity level of log output: CRITICAL, ERROR, WARNING, NOTICE, INFO, DEBUG, TRACEDEBUG, HEAVYDEBUG or corresponding numeric value (50-5). Default: INFO (equal 20) .TP .B logtarget log target: filename, SYSLOG, STDERR or STDOUT. Default: STDOUT if not set in fail2ban.conf/fail2ban.local @@ -298,7 +298,14 @@ requires Gamin (a file alteration monitor) to be installed. If Gamin is not inst uses a polling algorithm which does not require external libraries. .TP .B systemd -uses systemd python library to access the systemd journal. Specifying \fBlogpath\fR is not valid for this backend and instead utilises \fBjournalmatch\fR from the jails associated filter config. +uses systemd python library to access the systemd journal. Specifying \fBlogpath\fR is not valid for this backend and instead utilises \fBjournalmatch\fR from the jails associated filter config. Multiple systemd-specific flags can be passed to the backend, including \fBjournalpath\fR and \fBjournalfiles\fR, to explicitly set the path to a directory or set of files. \fBjournalflags\fR, which by default is 4 and excludes user session files, can be set to include them with \fBjournalflags=1\fR, see the python-systemd documentation for other settings and further details. Examples: +.PP +.RS +.nf +backend = systemd[journalpath=/run/log/journal/machine-1] +backend = systemd[journalfiles="/path/to/system.journal, /path/to/user.journal"] +backend = systemd[journalflags=1] +.fi .SS Actions Each jail can be configured with only a single filter, but may have multiple actions. By default, the name of a action is the action filename, and in the case of Python actions, the ".py" file extension is stripped. Where multiple of the same action are to be used, the \fBactname\fR option can be assigned to the action to avoid duplication e.g.: @@ -460,11 +467,27 @@ Similar to actions, filters have an [Init] section which can be overridden in \f specifies the maximum number of lines to buffer to match multi-line regexs. For some log formats this will not required to be changed. Other logs may require to increase this value if a particular log file is frequently written to. .TP \fBdatepattern\fR -specifies a custom date pattern/regex as an alternative to the default date detectors e.g. %Y-%m-%d %H:%M(?::%S)?. For a list of valid format directives, see Python library documentation for strptime behaviour. -.br -Also, special values of \fIEpoch\fR (UNIX Timestamp), \fITAI64N\fR and \fIISO8601\fR can be used. +specifies a custom date pattern/regex as an alternative to the default date detectors e.g. %%Y-%%m-%%d %%H:%%M(?::%%S)?. +For a list of valid format directives, see Python library documentation for strptime behaviour. .br \fBNOTE:\fR due to config file string substitution, that %'s must be escaped by an % in config files. +.br +Also, special values of \fIEpoch\fR (UNIX Timestamp), \fITAI64N\fR and \fIISO8601\fR can be used as datepattern. +.br +Normally the regexp generated for datepattern additionally gets word-start and word-end boundaries to avoid accidental match inside of some word in a message. +There are several prefixes and words with special meaning that could be specified with custom datepattern to control resulting regex: +.RS +.IP +\fI{DEFAULT}\fR - can be used to add default date patterns of fail2ban. +.IP +\fI{DATE}\fR - can be used as part of regex that will be replaced with default date patterns. +.IP +\fI{^LN-BEG}\fR - prefix (similar to \fI^\fR) changing word-start boundary to line-start boundary (ignoring up to 2 characters). If used as value (not as a prefix), it will also set all default date patterns (similar to \fI{DEFAULT}\fR), but anchored at begin of message line. +.IP +\fI{UNB}\fR - prefix to disable automatic word boundaries in regex. +.IP +\fI{NONE}\fR - value would allow to find failures totally without date-time in log message. Filter will use now as a timestamp (or last known timestamp from previous line with timestamp). +.RE .TP \fBjournalmatch\fR specifies the systemd journal match used to filter the journal entries. See \fBjournalctl(1)\fR and \fBsystemd.journal-fields(7)\fR for matches syntax and more details on special journal fields. This option is only valid for the \fIsystemd\fR backend. diff --git a/setup.py b/setup.py index e476c5dd..ce1eedf6 100755 --- a/setup.py +++ b/setup.py @@ -134,28 +134,13 @@ class install_command_f2b(install): cmdclass = self.distribution.cmdclass cmdclass['build_py'] = build_py_2to3 cmdclass['build_scripts'] = build_scripts_2to3 - if not self.without_tests: - self.distribution.scripts += [ - 'bin/fail2ban-testcases', - ] + if self.without_tests: + self.distribution.scripts.remove('bin/fail2ban-testcases') - self.distribution.packages += [ - 'fail2ban.tests', - 'fail2ban.tests.action_d', - ] + self.distribution.packages.remove('fail2ban.tests') + self.distribution.packages.remove('fail2ban.tests.action_d') - self.distribution.package_data = { - 'fail2ban.tests': - [ join(w[0], f).replace("fail2ban/tests/", "", 1) - for w in os.walk('fail2ban/tests/files') - for f in w[2]] + - [ join(w[0], f).replace("fail2ban/tests/", "", 1) - for w in os.walk('fail2ban/tests/config') - for f in w[2]] + - [ join(w[0], f).replace("fail2ban/tests/", "", 1) - for w in os.walk('fail2ban/tests/action_d') - for f in w[2]] - } + del self.distribution.package_data['fail2ban.tests'] install.finalize_options(self) def run(self): install.run(self) @@ -239,13 +224,28 @@ setup( 'bin/fail2ban-client', 'bin/fail2ban-server', 'bin/fail2ban-regex', + 'bin/fail2ban-testcases', # 'bin/fail2ban-python', -- link (binary), will be installed via install_scripts_f2b wrapper ], packages = [ 'fail2ban', 'fail2ban.client', 'fail2ban.server', + 'fail2ban.tests', + 'fail2ban.tests.action_d', ], + package_data = { + 'fail2ban.tests': + [ join(w[0], f).replace("fail2ban/tests/", "", 1) + for w in os.walk('fail2ban/tests/files') + for f in w[2]] + + [ join(w[0], f).replace("fail2ban/tests/", "", 1) + for w in os.walk('fail2ban/tests/config') + for f in w[2]] + + [ join(w[0], f).replace("fail2ban/tests/", "", 1) + for w in os.walk('fail2ban/tests/action_d') + for f in w[2]] + }, data_files = [ ('/etc/fail2ban', glob("config/*.conf")