From 1418bcdf5bb01dd210f4d37b5c2de8ccd0c658e4 Mon Sep 17 00:00:00 2001 From: sebres Date: Tue, 29 Sep 2020 12:35:49 +0200 Subject: [PATCH 1/8] `action.d/bsd-ipfw.conf`: fixed selection of rule-no by large list or initial `lowest_rule_num`, exit code can't be larger than 255 (gh-2836) --- ChangeLog | 1 + config/action.d/bsd-ipfw.conf | 5 ++++- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/ChangeLog b/ChangeLog index 361b81d5..c3e2c6d4 100644 --- a/ChangeLog +++ b/ChangeLog @@ -52,6 +52,7 @@ ver. 0.10.6-dev (20??/??/??) - development edition with `jq`, gh-2140, gh-2656) * `action.d/nftables.conf` (type=multiport only): fixed port range selector, replacing `:` with `-` (gh-2763) * `action.d/firewallcmd-*.conf` (multiport only): fixed port range selector, replacing `:` with `-` (gh-2821) +* `action.d/bsd-ipfw.conf`: fixed selection of rule-no by large list or initial `lowest_rule_num` (gh-2836) * `filter.d/common.conf`: avoid substitute of default values in related `lt_*` section, `__prefix_line` should be interpolated in definition section (inside the filter-config, gh-2650) * `filter.d/courier-smtp.conf`: prefregex extended to consider port in log-message (gh-2697) diff --git a/config/action.d/bsd-ipfw.conf b/config/action.d/bsd-ipfw.conf index 5116b0d8..7f04fe7c 100644 --- a/config/action.d/bsd-ipfw.conf +++ b/config/action.d/bsd-ipfw.conf @@ -14,7 +14,10 @@ # Notes.: command executed on demand at the first ban (or at the start of Fail2Ban if actionstart_on_demand is set to false). # Values: CMD # -actionstart = ipfw show | fgrep -c -m 1 -s 'table()' > /dev/null 2>&1 || ( ipfw show | awk 'BEGIN { b = } { if ($1 < b) {} else if ($1 == b) { b = $1 + 1 } else { e = b } } END { if (e) exit e
else exit b }'; num=$?; ipfw -q add $num from table\(
\) to me ; echo $num > "" ) +actionstart = ipfw show | fgrep -c -m 1 -s 'table(
)' > /dev/null 2>&1 || ( + num=$(ipfw show | awk 'BEGIN { b = } { if ($1 < b) {} else if ($1 == b) { b = $1 + 1 } else { e = b } } END { if (e) print e
else print b }'); + ipfw -q add "$num" from table\(
\) to me ; echo "$num" > "" + ) # Option: actionstop From 2817a8144c685d3ca18b7ca27064179ac318924e Mon Sep 17 00:00:00 2001 From: sebres Date: Tue, 29 Sep 2020 13:33:40 +0200 Subject: [PATCH 2/8] `action.d/bsd-ipfw.conf`: small amend (gh-2836) simplifying awk condition/code (position starts from `` and increases whilst used) --- config/action.d/bsd-ipfw.conf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config/action.d/bsd-ipfw.conf b/config/action.d/bsd-ipfw.conf index 7f04fe7c..444192d3 100644 --- a/config/action.d/bsd-ipfw.conf +++ b/config/action.d/bsd-ipfw.conf @@ -15,7 +15,7 @@ # Values: CMD # actionstart = ipfw show | fgrep -c -m 1 -s 'table(
)' > /dev/null 2>&1 || ( - num=$(ipfw show | awk 'BEGIN { b = } { if ($1 < b) {} else if ($1 == b) { b = $1 + 1 } else { e = b } } END { if (e) print e
else print b }'); + num=$(ipfw show | awk 'BEGIN { b = } { if ($1 == b) { b = $1 + 1 } } END { print b }'); ipfw -q add "$num" from table\(
\) to me ; echo "$num" > "" ) From c8059bf9b35a167ad7bd4579f83495f9dda50c79 Mon Sep 17 00:00:00 2001 From: sebres Date: Tue, 29 Sep 2020 16:27:17 +0200 Subject: [PATCH 3/8] ban/unban: increase responsiveness of actions thread by (un)banning process, better waiting timeout considering pending tickets for unban (_nextUnbanTime) --- fail2ban/server/actions.py | 46 ++++++++++++++++++++--------------- fail2ban/server/banmanager.py | 16 +++++------- 2 files changed, 33 insertions(+), 29 deletions(-) diff --git a/fail2ban/server/actions.py b/fail2ban/server/actions.py index f14d8d7b..b7b95b44 100644 --- a/fail2ban/server/actions.py +++ b/fail2ban/server/actions.py @@ -327,25 +327,33 @@ class Actions(JailThread, Mapping): self._jail.name, name, e, exc_info=logSys.getEffectiveLevel()<=logging.DEBUG) while self.active: - if self.idle: - logSys.debug("Actions: enter idle mode") - Utils.wait_for(lambda: not self.active or not self.idle, - lambda: False, self.sleeptime) - logSys.debug("Actions: leave idle mode") - continue - # wait for ban (stop if gets inactive): - bancnt = 0 - if Utils.wait_for(lambda: not self.active or self._jail.hasFailTickets, self.sleeptime): - bancnt = self.__checkBan() - cnt += bancnt - # unban if nothing is banned not later than banned tickets >= banPrecedence - if not bancnt or cnt >= self.banPrecedence: - if self.active: - # let shrink the ban list faster - bancnt *= 2 - self.__checkUnBan(bancnt if bancnt and bancnt < self.unbanMaxCount else self.unbanMaxCount) - cnt = 0 - + try: + if self.idle: + logSys.debug("Actions: enter idle mode") + Utils.wait_for(lambda: not self.active or not self.idle, + lambda: False, self.sleeptime) + logSys.debug("Actions: leave idle mode") + continue + # wait for ban (stop if gets inactive, pending ban or unban): + bancnt = 0 + wt = min(self.sleeptime, self.__banManager._nextUnbanTime - MyTime.time()) + logSys.log(5, "Actions: wait for pending tickets %s (default %s)", wt, self.sleeptime) + if Utils.wait_for(lambda: not self.active or self._jail.hasFailTickets, wt): + bancnt = self.__checkBan() + cnt += bancnt + # unban if nothing is banned not later than banned tickets >= banPrecedence + if not bancnt or cnt >= self.banPrecedence: + if self.active: + # let shrink the ban list faster + bancnt *= 2 + logSys.log(5, "Actions: check-unban %s, bancnt %s, max: %s", bancnt if bancnt and bancnt < self.unbanMaxCount else self.unbanMaxCount, bancnt, self.unbanMaxCount) + self.__checkUnBan(bancnt if bancnt and bancnt < self.unbanMaxCount else self.unbanMaxCount) + cnt = 0 + except Exception as e: # pragma: no cover + logSys.error("[%s] unhandled error in actions thread: %s", + self._jail.name, e, + exc_info=logSys.getEffectiveLevel()<=logging.DEBUG) + self.__flushBan(stop=True) self.stopActions() return True diff --git a/fail2ban/server/banmanager.py b/fail2ban/server/banmanager.py index fe38dec6..575d648b 100644 --- a/fail2ban/server/banmanager.py +++ b/fail2ban/server/banmanager.py @@ -57,7 +57,7 @@ class BanManager: ## Total number of banned IP address self.__banTotal = 0 ## The time for next unban process (for performance and load reasons): - self.__nextUnbanTime = BanTicket.MAX_TIME + self._nextUnbanTime = BanTicket.MAX_TIME ## # Set the ban time. @@ -290,8 +290,8 @@ class BanManager: self.__banList[fid] = ticket self.__banTotal += 1 # correct next unban time: - if self.__nextUnbanTime > eob: - self.__nextUnbanTime = eob + if self._nextUnbanTime > eob: + self._nextUnbanTime = eob return True ## @@ -322,12 +322,8 @@ class BanManager: def unBanList(self, time, maxCount=0x7fffffff): with self.__lock: - # Permanent banning - if self.__banTime < 0: - return list() - # Check next unban time: - nextUnbanTime = self.__nextUnbanTime + nextUnbanTime = self._nextUnbanTime if nextUnbanTime > time: return list() @@ -340,12 +336,12 @@ class BanManager: if time > eob: unBanList[fid] = ticket if len(unBanList) >= maxCount: # stop search cycle, so reset back the next check time - nextUnbanTime = self.__nextUnbanTime + nextUnbanTime = self._nextUnbanTime break elif nextUnbanTime > eob: nextUnbanTime = eob - self.__nextUnbanTime = nextUnbanTime + self._nextUnbanTime = nextUnbanTime # Removes tickets. if len(unBanList): if len(unBanList) / 2.0 <= len(self.__banList) / 3.0: From 02525d7b6f96c2841c24c4dbfcfe14e4a3d1ef63 Mon Sep 17 00:00:00 2001 From: sebres Date: Thu, 8 Oct 2020 21:07:51 +0200 Subject: [PATCH 4/8] filter.d/sshd.conf: mode `ddos` (and `aggressive`) extended with new rule closing flood attack vector, matching: error: kex_exchange_identification: Connection closed by remote host (gh-2850) --- config/filter.d/sshd.conf | 2 +- fail2ban/tests/files/logs/sshd | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/config/filter.d/sshd.conf b/config/filter.d/sshd.conf index 4c86dca0..e7942262 100644 --- a/config/filter.d/sshd.conf +++ b/config/filter.d/sshd.conf @@ -71,7 +71,7 @@ mdre-normal = mdre-normal-other = ^(Connection closed|Disconnected) (?:by|from)%(__authng_user)s (?:%(__suff)s|\s*)$ mdre-ddos = ^Did not receive identification string from - ^kex_exchange_identification: client sent invalid protocol identifier + ^kex_exchange_identification: (?:[Cc]lient sent invalid protocol identifier|[Cc]onnection closed by remote host) ^Bad protocol version identification '.*' from ^SSH: Server;Ltype: (?:Authname|Version|Kex);Remote: -\d+;[A-Z]\w+: ^Read from socket failed: Connection reset by peer diff --git a/fail2ban/tests/files/logs/sshd b/fail2ban/tests/files/logs/sshd index 9fff416a..5d23f96f 100644 --- a/fail2ban/tests/files/logs/sshd +++ b/fail2ban/tests/files/logs/sshd @@ -312,6 +312,8 @@ Jul 17 23:04:01 srv sshd[1300]: Connection closed by authenticating user test 12 Feb 17 17:40:17 sshd[19725]: Connection from 192.0.2.10 port 62004 on 192.0.2.10 port 22 # failJSON: { "time": "2005-02-17T17:40:17", "match": true , "host": "192.0.2.10", "desc": "ddos: port scanner (invalid protocol identifier)" } Feb 17 17:40:17 sshd[19725]: error: kex_exchange_identification: client sent invalid protocol identifier "" +# failJSON: { "time": "2005-02-17T17:40:18", "match": true , "host": "192.0.2.10", "desc": "ddos: flood attack vector, gh-2850" } +Feb 17 17:40:18 sshd[19725]: error: kex_exchange_identification: Connection closed by remote host # failJSON: { "time": "2005-03-15T09:21:01", "match": true , "host": "192.0.2.212", "desc": "DDOS mode causes failure on close within preauth stage" } Mar 15 09:21:01 host sshd[2717]: Connection closed by 192.0.2.212 [preauth] From a07e6fe1a2d1b8945d8a84abf3022061c9cc4528 Mon Sep 17 00:00:00 2001 From: "Sergey G. Brester" Date: Fri, 30 Oct 2020 13:19:56 +0100 Subject: [PATCH 5/8] reduce default `maxmatches` from 50 to 5: avoid too large memory consumption if `maxretry` is large and many failures don't cause ban (but accumulated in fail-manager with all the matched lines); closes gh-2843 --- fail2ban/server/failmanager.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fail2ban/server/failmanager.py b/fail2ban/server/failmanager.py index 3e81e8b5..4173a233 100644 --- a/fail2ban/server/failmanager.py +++ b/fail2ban/server/failmanager.py @@ -43,7 +43,7 @@ class FailManager: self.__maxRetry = 3 self.__maxTime = 600 self.__failTotal = 0 - self.maxMatches = 50 + self.maxMatches = 5 self.__bgSvc = BgService() def setFailTotal(self, value): From 7cb6412f68afd2590bb86d3d9939e0d21ad00367 Mon Sep 17 00:00:00 2001 From: "Sergey G. Brester" Date: Tue, 3 Nov 2020 13:15:42 +0100 Subject: [PATCH 6/8] 1st try of GH actions flow (CI only, no coverage atm) --- .github/workflows/main.yml | 64 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 64 insertions(+) create mode 100644 .github/workflows/main.yml diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml new file mode 100644 index 00000000..154a75ae --- /dev/null +++ b/.github/workflows/main.yml @@ -0,0 +1,64 @@ +name: CI + +# Controls when the action will run. Triggers the workflow on push or pull request +# events but only for the master branch +on: + push: + paths-ignore: + - 'doc/**' + - 'files/**' + - 'man/**' + pull_request: + paths-ignore: + - 'doc/**' + - 'files/**' + - 'man/**' + +# A workflow run is made up of one or more jobs that can run sequentially or in parallel +jobs: + # This workflow contains a single job called "build" + build: + # The type of runner that the job will run on + runs-on: ubuntu-20.04 + strategy: + matrix: + python-version: [2.7, 3.5, 3.6, 3.7, 3.8, 3.9, pypy2, pypy3] + fail-fast: false + # Steps represent a sequence of tasks that will be executed as part of the job + steps: + # Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it + - uses: actions/checkout@v2 + + - name: Set up Python + uses: actions/setup-python@v2 + with: + python-version: ${{ matrix.python-version }} + + - name: Python version + run: | + F2B_PY=$(python -c "import sys; print(sys.version)") + echo "Python: ${{ matrix.python-version }} -- $F2B_PY" + F2B_PY=${F2B_PY:0:1} + echo "Set F2B_PY=$F2B_PY" + echo "F2B_PY=$F2B_PY" >> $GITHUB_ENV + + - name: Install dependencies + run: | + if [[ "$F2B_PY" = 3 ]] && ! command -v 2to3x -v 2to3 > /dev/null; then + python -m pip install --upgrade pip + pip install 2to3 + fi + + - name: Before scripts + run: | + cd "$GITHUB_WORKSPACE" + # Manually execute 2to3 for now + if [[ "$F2B_PY" = 3 ]]; then echo "2to3 ..." && ./fail2ban-2to3; fi + # (debug) output current preferred encoding: + python -c 'import locale, sys; from fail2ban.helpers import PREFER_ENC; print(PREFER_ENC, locale.getpreferredencoding(), (sys.stdout and sys.stdout.encoding))' + + - name: Test suite + run: if [[ "$F2B_PY" = 2 ]]; then python setup.py test; else python bin/fail2ban-testcases --verbosity=2; fi + + #- name: Test initd scripts + # run: shellcheck -s bash -e SC1090,SC1091 files/debian-initd From 7f0010be68d0ca778ae97698bdd893c859b282eb Mon Sep 17 00:00:00 2001 From: sebres Date: Tue, 3 Nov 2020 15:51:49 +0100 Subject: [PATCH 7/8] attempt to install systemd-python module --- .github/workflows/main.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 154a75ae..7a1d31df 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -44,10 +44,12 @@ jobs: - name: Install dependencies run: | + python -m pip install --upgrade pip if [[ "$F2B_PY" = 3 ]] && ! command -v 2to3x -v 2to3 > /dev/null; then - python -m pip install --upgrade pip pip install 2to3 fi + pip install systemd-python || echo 'systemd not available' + pip install pyinotify || echo 'inotify not available' - name: Before scripts run: | From 55d6408b13c75100134f123cc283a801a384fd6c Mon Sep 17 00:00:00 2001 From: sebres Date: Thu, 5 Nov 2020 15:31:11 +0100 Subject: [PATCH 8/8] tweaks to speedup test-cases (test-suite seems to be time stable now, so we could shorten sleeping intervals) --- fail2ban/tests/utils.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/fail2ban/tests/utils.py b/fail2ban/tests/utils.py index 47e5b909..b54581f5 100644 --- a/fail2ban/tests/utils.py +++ b/fail2ban/tests/utils.py @@ -292,15 +292,15 @@ def initTests(opts): unittest.F2B.SkipIfFast = F2B_SkipIfFast else: # smaller inertance inside test-cases (litle speedup): - Utils.DEFAULT_SLEEP_TIME = 0.25 - Utils.DEFAULT_SLEEP_INTERVAL = 0.025 + Utils.DEFAULT_SLEEP_TIME = 0.025 + Utils.DEFAULT_SLEEP_INTERVAL = 0.005 Utils.DEFAULT_SHORT_INTERVAL = 0.0005 # sleep intervals are large - use replacement for sleep to check time to sleep: _org_sleep = time.sleep def _new_sleep(v): - if v > max(1, Utils.DEFAULT_SLEEP_TIME): # pragma: no cover + if v > 0.25: # pragma: no cover raise ValueError('[BAD-CODE] To long sleep interval: %s, try to use conditional Utils.wait_for instead' % v) - _org_sleep(min(v, Utils.DEFAULT_SLEEP_TIME)) + _org_sleep(v) time.sleep = _new_sleep # --no-network : if unittest.F2B.no_network: # pragma: no cover