Merge remote-tracking branch 'remotes/gh-upstream/0.10' into 0.11

.github/workflows/main.yml

@@ -22,7 +22,7 @@ jobs:
     runs-on: ubuntu-20.04
     strategy:
       matrix:
-        python-version: [2.7, 3.5, 3.6, 3.7, 3.8, 3.9, '3.10', '3.11.0-alpha.1', pypy2, pypy3]
+        python-version: [2.7, 3.5, 3.6, 3.7, 3.8, 3.9, '3.10', '3.11.0-beta.3', pypy2, pypy3]
       fail-fast: false
     # Steps represent a sequence of tasks that will be executed as part of the job
     steps:
@@ -61,6 +61,8 @@ jobs:
           #sudo apt-get -y install python${F2B_PY/2/}-systemd || echo 'systemd not available'
           sudo apt-get -y install libsystemd-dev || echo 'systemd dependencies seems to be unavailable'
           python -m pip install systemd-python || echo 'systemd not available'
+          #readline if available as module:
+          python -c 'import readline' 2> /dev/null || python -m pip install readline || echo 'readline not available'
 
       - name: Before scripts
         run: |

fail2ban/client/fail2banregex.py

@@ -334,7 +334,7 @@ class Fail2banRegex(object):
 		fltFile = None
 		fltOpt = {}
 		if regextype == 'fail':
-			if re.search(r'^(?ms)/{0,3}[\w/_\-.]+(?:\[.*\])?$', value):
+			if re.search(r'(?ms)^/{0,3}[\w/_\-.]+(?:\[.*\])?$', value):
 				try:
 					fltName, fltOpt = extractOptions(value)
 					if "." in fltName[~5:]:

fail2ban/server/actions.py

@@ -664,7 +664,7 @@ class Actions(JailThread, Mapping):
 				if hasattr(action, 'consistencyCheck'):
 					def _beforeRepair():
 						if stop and not getattr(action, 'actionrepair_on_unban', None): # don't need repair on stop
-							self._logSys.error("Invariant check failed. Flush is impossible.")
+							logSys.error("Invariant check failed. Flush is impossible.")
 							return False
 						return True
 					action.consistencyCheck(_beforeRepair)

fail2ban/server/datetemplate.py

@@ -35,6 +35,7 @@ logSys = getLogger(__name__)
 # check already grouped contains "(", but ignores char "\(" and conditional "(?(id)...)":
 RE_GROUPED = re.compile(r'(?<!(?:\(\?))(?<!\\)\((?!\?)')
 RE_GROUP = ( re.compile(r'^((?:\(\?\w+\))?\^?(?:\(\?\w+\))?)(.*?)(\$?)$'), r"\1(\2)\3" )
+RE_GLOBALFLAGS = re.compile(r'((?:^|(?!<\\))\(\?[a-z]+\))')
 
 RE_EXLINE_NO_BOUNDS = re.compile(r'^\{UNB\}')
 RE_EXLINE_BOUND_BEG = re.compile(r'^\{\^LN-BEG\}')
@@ -110,6 +111,11 @@ class DateTemplate(object):
 		# because it may be very slow in negative case (by long log-lines not matching pattern)
 
 		regex = regex.strip()
+		# cut global flags like (?iu) from RE in order to pre-set it after processing:
+		gf = RE_GLOBALFLAGS.search(regex)
+		if gf:
+			regex = RE_GLOBALFLAGS.sub('', regex, count=1)
+		# check word boundaries needed:
 		boundBegin = wordBegin and not RE_NO_WRD_BOUND_BEG.search(regex)
 		boundEnd = wordEnd and not RE_NO_WRD_BOUND_END.search(regex)
 		# if no group add it now, should always have a group(1):
@@ -135,6 +141,8 @@ class DateTemplate(object):
 			self.flags |= DateTemplate.LINE_END
 		# remove possible special pattern "**" in front and end of regex:
 		regex = RE_DEL_WRD_BOUNDS[0].sub(RE_DEL_WRD_BOUNDS[1], regex)
+		if gf: # restore global flags:
+			regex = gf.group(1) + regex
 		self._regex = regex
 		logSys.log(4, '  constructed regex %s', regex)
 		self._cRegex = None

fail2ban/server/failregex.py

@@ -91,6 +91,13 @@ R_MAP = {
 	"port": "fport",
 }
 
+# map global flags like ((?i)xxx) or (?:(?i)xxx) to local flags (?i:xxx) if supported by RE-engine in this python version:
+try:
+	re.search("^re(?i:val)$", "reVAL")
+	R_GLOB2LOCFLAGS = ( re.compile(r"(?<!\\)\((?:\?:)?(\(\?[a-z]+)\)"), r"\1:" )
+except:
+	R_GLOB2LOCFLAGS = ()
+
 def mapTag2Opt(tag):
 	tag = tag.lower()
 	return R_MAP.get(tag, tag)
@@ -128,6 +135,9 @@ class Regex:
 		#
 		if regex.lstrip() == '':
 			raise RegexException("Cannot add empty regex")
+		# special handling wrapping global flags to local flags:
+		if R_GLOB2LOCFLAGS:
+			regex = R_GLOB2LOCFLAGS[0].sub(R_GLOB2LOCFLAGS[1], regex)
 		try:
 			self._regexObj = re.compile(regex, re.MULTILINE if multiline else 0)
 			self._regex = regex

fail2ban/tests/fail2banclienttestcase.py

@@ -600,6 +600,11 @@ class Fail2banClientTest(Fail2banClientServerBase):
 				os.kill(pid, signal.SIGCONT)
 			self.assertLogged("timed out")
 			self.pruneLog()
+			# check readline module available (expected by interactive client)
+			try:
+				import readline
+			except ImportError as e:
+				raise unittest.SkipTest('Skip test because of import error: %s' % e)
 			# interactive client chat with started server:
 			INTERACT += [
 				"echo INTERACT-ECHO",