Merge branch '0.11' into master

.github/workflows/main.yml

@@ -44,10 +44,12 @@ jobs:
 
       - name: Install dependencies
         run: |
-          if [[ "$F2B_PY" = 3 ]] && ! command -v 2to3x -v 2to3 > /dev/null; then
           python -m pip install --upgrade pip
+          if [[ "$F2B_PY" = 3 ]] && ! command -v 2to3x -v 2to3 > /dev/null; then
             pip install 2to3
           fi
+          pip install systemd-python || echo 'systemd not available'
+          pip install pyinotify || echo 'inotify not available'
 
       - name: Before scripts
         run: |

ChangeLog

@@ -36,6 +36,7 @@ ver. 1.0.1-dev-1 (20??/??/??) - development nightly edition
    with `jq`, gh-2140, gh-2656)
 * `action.d/nftables.conf` (type=multiport only): fixed port range selector, replacing `:` with `-` (gh-2763)
 * `action.d/firewallcmd-*.conf` (multiport only): fixed port range selector, replacing `:` with `-` (gh-2821)
+* `action.d/bsd-ipfw.conf`: fixed selection of rule-no by large list or initial `lowest_rule_num` (gh-2836)
 * `filter.d/common.conf`: avoid substitute of default values in related `lt_*` section, `__prefix_line`
   should be interpolated in definition section (inside the filter-config, gh-2650)
 * `filter.d/courier-smtp.conf`: prefregex extended to consider port in log-message (gh-2697)

config/action.d/bsd-ipfw.conf

@@ -14,7 +14,10 @@
 # Notes.:  command executed on demand at the first ban (or at the start of Fail2Ban if actionstart_on_demand is set to false).
 # Values:  CMD
 #
-actionstart = ipfw show | fgrep -c -m 1 -s 'table(<table>)' > /dev/null 2>&1 || ( ipfw show | awk 'BEGIN { b = <lowest_rule_num> } { if ($1 < b) {} else if ($1 == b) { b = $1 + 1 } else { e = b } } END { if (e) exit e <br> else exit b }'; num=$?; ipfw -q add $num <blocktype> <block> from table\(<table>\) to me <port>; echo $num > "<startstatefile>" )
+actionstart = ipfw show | fgrep -c -m 1 -s 'table(<table>)' > /dev/null 2>&1 || (
+    num=$(ipfw show | awk 'BEGIN { b = <lowest_rule_num> } { if ($1 == b) { b = $1 + 1 } } END { print b }');
+    ipfw -q add "$num" <blocktype> <block> from table\(<table>\) to me <port>; echo "$num" > "<startstatefile>"
+  )
 
 
 # Option:  actionstop

config/filter.d/sshd.conf

@@ -71,7 +71,7 @@ mdre-normal =
 mdre-normal-other = ^<F-NOFAIL><F-MLFFORGET>(Connection closed|Disconnected)</F-MLFFORGET></F-NOFAIL> (?:by|from)%(__authng_user)s <HOST>(?:%(__suff)s|\s*)$
 
 mdre-ddos = ^Did not receive identification string from <HOST>
-            ^kex_exchange_identification: client sent invalid protocol identifier
+            ^kex_exchange_identification: (?:[Cc]lient sent invalid protocol identifier|[Cc]onnection closed by remote host)
             ^Bad protocol version identification '.*' from <HOST>
             ^<F-NOFAIL>SSH: Server;Ltype:</F-NOFAIL> (?:Authname|Version|Kex);Remote: <HOST>-\d+;[A-Z]\w+:
             ^Read from socket failed: Connection <F-MLFFORGET>reset</F-MLFFORGET> by peer

fail2ban/server/actions.py

@@ -338,15 +338,18 @@ class Actions(JailThread, Mapping):
 					self._jail.name, name, e,
 					exc_info=logSys.getEffectiveLevel()<=logging.DEBUG)
 		while self.active:
+			try:
 				if self.idle:
 					logSys.debug("Actions: enter idle mode")
 					Utils.wait_for(lambda: not self.active or not self.idle,
 						lambda: False, self.sleeptime)
 					logSys.debug("Actions: leave idle mode")
 					continue
-			# wait for ban (stop if gets inactive):
+				# wait for ban (stop if gets inactive, pending ban or unban):
 				bancnt = 0
-			if Utils.wait_for(lambda: not self.active or self._jail.hasFailTickets, self.sleeptime):
+				wt = min(self.sleeptime, self.__banManager._nextUnbanTime - MyTime.time())
+				logSys.log(5, "Actions: wait for pending tickets %s (default %s)", wt, self.sleeptime)
+				if Utils.wait_for(lambda: not self.active or self._jail.hasFailTickets, wt):
 					bancnt = self.__checkBan()
 					cnt += bancnt
 				# unban if nothing is banned not later than banned tickets >= banPrecedence
@@ -354,8 +357,13 @@ class Actions(JailThread, Mapping):
 					if self.active:
 						# let shrink the ban list faster
 						bancnt *= 2
+						logSys.log(5, "Actions: check-unban %s, bancnt %s, max: %s", bancnt if bancnt and bancnt < self.unbanMaxCount else self.unbanMaxCount, bancnt, self.unbanMaxCount)
 						self.__checkUnBan(bancnt if bancnt and bancnt < self.unbanMaxCount else self.unbanMaxCount)
 					cnt = 0
+			except Exception as e: # pragma: no cover
+				logSys.error("[%s] unhandled error in actions thread: %s",
+					self._jail.name, e,
+					exc_info=logSys.getEffectiveLevel()<=logging.DEBUG)
 
 		self.__flushBan(stop=True)
 		self.stopActions()

fail2ban/server/banmanager.py

@@ -57,7 +57,7 @@ class BanManager:
 		## Total number of banned IP address
 		self.__banTotal = 0
 		## The time for next unban process (for performance and load reasons):
-		self.__nextUnbanTime = BanTicket.MAX_TIME
+		self._nextUnbanTime = BanTicket.MAX_TIME
 	
 	##
 	# Set the ban time.
@@ -293,8 +293,8 @@ class BanManager:
 			self.__banTotal += 1
 			ticket.incrBanCount()
 			# correct next unban time:
-			if self.__nextUnbanTime > eob:
+			if self._nextUnbanTime > eob:
-				self.__nextUnbanTime = eob
+				self._nextUnbanTime = eob
 			return True
 
 	##
@@ -325,12 +325,8 @@ class BanManager:
 	
 	def unBanList(self, time, maxCount=0x7fffffff):
 		with self.__lock:
-			# Permanent banning
-			if self.__banTime < 0:
-				return list()
-
 			# Check next unban time:
-			nextUnbanTime = self.__nextUnbanTime
+			nextUnbanTime = self._nextUnbanTime
 			if nextUnbanTime > time:
 				return list()
 
@@ -343,12 +339,12 @@ class BanManager:
 				if time > eob:
 					unBanList[fid] = ticket
 					if len(unBanList) >= maxCount: # stop search cycle, so reset back the next check time
-						nextUnbanTime = self.__nextUnbanTime
+						nextUnbanTime = self._nextUnbanTime
 						break
 				elif nextUnbanTime > eob:
 					nextUnbanTime = eob
 
-			self.__nextUnbanTime = nextUnbanTime
+			self._nextUnbanTime = nextUnbanTime
 			# Removes tickets.
 			if len(unBanList):
 				if len(unBanList) / 2.0 <= len(self.__banList) / 3.0:

fail2ban/server/failmanager.py

@@ -43,7 +43,7 @@ class FailManager:
 		self.__maxRetry = 3
 		self.__maxTime = 600
 		self.__failTotal = 0
-		self.maxMatches = 50
+		self.maxMatches = 5
 		self.__bgSvc = BgService()
 	
 	def setFailTotal(self, value):

fail2ban/tests/files/logs/sshd

@@ -312,6 +312,8 @@ Jul 17 23:04:01 srv sshd[1300]: Connection closed by authenticating user test 12
 Feb 17 17:40:17 sshd[19725]: Connection from 192.0.2.10 port 62004 on 192.0.2.10 port 22
 # failJSON: { "time": "2005-02-17T17:40:17", "match": true , "host": "192.0.2.10", "desc": "ddos: port scanner (invalid protocol identifier)" }
 Feb 17 17:40:17 sshd[19725]: error: kex_exchange_identification: client sent invalid protocol identifier ""
+# failJSON: { "time": "2005-02-17T17:40:18", "match": true , "host": "192.0.2.10", "desc": "ddos: flood attack vector, gh-2850" }
+Feb 17 17:40:18 sshd[19725]: error: kex_exchange_identification: Connection closed by remote host
 
 # failJSON: { "time": "2005-03-15T09:21:01", "match": true , "host": "192.0.2.212", "desc": "DDOS mode causes failure on close within preauth stage" }
 Mar 15 09:21:01 host sshd[2717]: Connection closed by 192.0.2.212 [preauth]

fail2ban/tests/utils.py

@@ -292,15 +292,15 @@ def initTests(opts):
 		unittest.F2B.SkipIfFast = F2B_SkipIfFast
 	else:
 		# smaller inertance inside test-cases (litle speedup):
-		Utils.DEFAULT_SLEEP_TIME = 0.25
+		Utils.DEFAULT_SLEEP_TIME = 0.025
-		Utils.DEFAULT_SLEEP_INTERVAL = 0.025
+		Utils.DEFAULT_SLEEP_INTERVAL = 0.005
 		Utils.DEFAULT_SHORT_INTERVAL = 0.0005
 		# sleep intervals are large - use replacement for sleep to check time to sleep:
 		_org_sleep = time.sleep
 		def _new_sleep(v):
-			if v > max(1, Utils.DEFAULT_SLEEP_TIME): # pragma: no cover
+			if v > 0.25: # pragma: no cover
 				raise ValueError('[BAD-CODE] To long sleep interval: %s, try to use conditional Utils.wait_for instead' % v)
-			_org_sleep(min(v, Utils.DEFAULT_SLEEP_TIME))
+			_org_sleep(v)
 		time.sleep = _new_sleep
 	# --no-network :
 	if unittest.F2B.no_network: # pragma: no cover