From ee3c787cc68f6d043c8702c8f53a566277e2ee02 Mon Sep 17 00:00:00 2001
From: sebres <serg.brester@sebres.de>
Date: Fri, 13 Jan 2017 19:06:17 +0100
Subject: [PATCH 1/7] Recognize restored (from database) tickets after restart
 (tell action restored state of the ticket); Prevent executing of several
 actions (e.g. mail, send-mail etc) on restart (bans were already notified).
 Test cases extended (smtp and by restart in ServerReloadTest). Closes gh-1141
 Closes gh-921

---
 config/action.d/complain.conf                 |  3 +-
 config/action.d/dshield.conf                  | 10 ++++-
 config/action.d/helpers-common.conf           |  3 ++
 config/action.d/mail-buffered.conf            |  8 +++-
 config/action.d/mail-whois-lines.conf         |  4 +-
 config/action.d/mail-whois.conf               |  4 +-
 config/action.d/mail.conf                     |  7 +++-
 config/action.d/sendmail-buffered.conf        |  4 +-
 config/action.d/sendmail-geoip-lines.conf     |  3 +-
 .../sendmail-whois-ipjailmatches.conf         |  4 +-
 config/action.d/sendmail-whois-ipmatches.conf |  4 +-
 config/action.d/sendmail-whois-lines.conf     |  3 +-
 config/action.d/sendmail-whois-matches.conf   |  4 +-
 config/action.d/sendmail-whois.conf           |  4 +-
 config/action.d/sendmail.conf                 |  4 +-
 config/action.d/smtp.py                       |  2 +
 config/action.d/xarf-login-attack.conf        |  7 +++-
 fail2ban/server/action.py                     |  4 +-
 fail2ban/server/actions.py                    |  4 ++
 fail2ban/tests/action_d/test_smtp.py          | 15 ++++++-
 fail2ban/tests/fail2banclienttestcase.py      | 42 +++++++++++++++----
 21 files changed, 114 insertions(+), 29 deletions(-)

diff --git a/config/action.d/complain.conf b/config/action.d/complain.conf
index e4ceb35f..d0156a44 100644
--- a/config/action.d/complain.conf
+++ b/config/action.d/complain.conf
@@ -58,7 +58,8 @@ actioncheck =
 # Tags:    See jail.conf(5) man page
 # Values:  CMD
 #
-actionban = oifs=${IFS}; 
+actionban = %(_bypass_if_restored)s
+            oifs=${IFS}; 
               IFS=.; SEP_IP=( <ip> ); set -- ${SEP_IP}; ADDRESSES=$(dig +short -t txt -q $4.$3.$2.$1.abuse-contacts.abusix.org); 
               IFS=,; ADDRESSES=$(echo $ADDRESSES)
             IFS=${oifs}
diff --git a/config/action.d/dshield.conf b/config/action.d/dshield.conf
index a0041986..35eaa3be 100644
--- a/config/action.d/dshield.conf
+++ b/config/action.d/dshield.conf
@@ -26,6 +26,10 @@
 # configure how often the buffer is flushed).
 #
 
+[INCLUDES]
+
+before = helpers-common.conf
+
 [Definition]
 
 # Option:  actionstart
@@ -64,7 +68,8 @@ actioncheck =
 # few seconds out, are incorrect. See
 # http://sourceforge.net/tracker/index.php?func=detail&aid=2017795&group_id=121032&atid=689047
 #
-actionban = TZONE=`date +%%z | sed 's/\([+-]..\)\(..\)/\1:\2/'`
+actionban = %(_bypass_if_restored)s
+            TZONE=`date +%%z | sed 's/\([+-]..\)\(..\)/\1:\2/'`
             DATETIME="`perl -e '@t=localtime(<time>);printf "%%4d-%%02d-%%02d %%02d:%%02d:%%02d",1900+$t[5],$t[4]+1,$t[3],$t[2],$t[1],$t[0]'` $TZONE"
 	    PROTOCOL=`awk '{IGNORECASE=1;if($1=="<protocol>"){print $2;exit}}' /etc/protocols`
 	    if [ -z "$PROTOCOL" ]; then PROTOCOL=<protocol>; fi
@@ -91,7 +96,8 @@ actionban = TZONE=`date +%%z | sed 's/\([+-]..\)\(..\)/\1:\2/'`
 # Tags:    See jail.conf(5) man page
 # Values:  CMD
 #
-actionunban = if [ -f <tmpfile>.first ]; then
+actionunban = %(_bypass_if_restored)s
+              if [ -f <tmpfile>.first ]; then
                   NOW=`date +%%s`
                   LOGAGE=$(($NOW - `cat <tmpfile>.first`))
                   if [ $LOGAGE -gt <maxbufferage> ]; then
diff --git a/config/action.d/helpers-common.conf b/config/action.d/helpers-common.conf
index 7fa8e9e4..b04c7f7a 100644
--- a/config/action.d/helpers-common.conf
+++ b/config/action.d/helpers-common.conf
@@ -7,6 +7,9 @@
 _grep_logs = logpath="<logpath>"; grep <grepopts> -E %(_grep_logs_args)s $logpath | <greplimit>
 _grep_logs_args = '(^|[^0-9])<ip>([^0-9]|$)'
 
+# Used for actions, that should not by executed if ticket was restored:
+_bypass_if_restored = if [ '<restored>' = '1' ]; then exit 0; fi;
+
 [Init]
 greplimit = tail -n <grepmax>
 grepmax = 1000
diff --git a/config/action.d/mail-buffered.conf b/config/action.d/mail-buffered.conf
index 914d4a5a..c1b3f565 100644
--- a/config/action.d/mail-buffered.conf
+++ b/config/action.d/mail-buffered.conf
@@ -4,6 +4,11 @@
 #
 #
 
+[INCLUDES]
+
+before = helpers-common.conf
+
+
 [Definition]
 
 # Option:  actionstart
@@ -45,7 +50,8 @@ actioncheck =
 # Tags:    See jail.conf(5) man page
 # Values:  CMD
 #
-actionban = printf %%b "`date`: <ip> (<failures> failures)\n" >> <tmpfile>
+actionban = %(_bypass_if_restored)s
+            printf %%b "`date`: <ip> (<failures> failures)\n" >> <tmpfile>
             LINE=$( wc -l <tmpfile> | awk '{ print $1 }' )
             if [ $LINE -ge <lines> ]; then
                 printf %%b "Hi,\n
diff --git a/config/action.d/mail-whois-lines.conf b/config/action.d/mail-whois-lines.conf
index cbd970c9..c471a44a 100644
--- a/config/action.d/mail-whois-lines.conf
+++ b/config/action.d/mail-whois-lines.conf
@@ -52,7 +52,9 @@ _ban_mail_content = ( printf %%b "Hi,\n
             printf %%b "\n
             Regards,\n
             Fail2Ban" )
-actionban = %(_ban_mail_content)s | <mailcmd> "[Fail2Ban] <name>: banned <ip> from  `uname -n`" <dest>
+
+actionban = %(_bypass_if_restored)s
+            %(_ban_mail_content)s | <mailcmd> "[Fail2Ban] <name>: banned <ip> from  `uname -n`" <dest>
 
 # Option:  actionunban
 # Notes.:  command executed when unbanning an IP. Take care that the
diff --git a/config/action.d/mail-whois.conf b/config/action.d/mail-whois.conf
index 018c327d..045da5ec 100644
--- a/config/action.d/mail-whois.conf
+++ b/config/action.d/mail-whois.conf
@@ -7,6 +7,7 @@
 [INCLUDES]
 
 before = mail-whois-common.conf
+         helpers-common.conf
 
 [Definition]
 
@@ -40,7 +41,8 @@ actioncheck =
 # Tags:    See jail.conf(5) man page
 # Values:  CMD
 #
-actionban = printf %%b "Hi,\n
+actionban = %(_bypass_if_restored)s
+            printf %%b "Hi,\n
             The IP <ip> has just been banned by Fail2Ban after
             <failures> attempts against <name>.\n\n
             Here is more information about <ip> :\n
diff --git a/config/action.d/mail.conf b/config/action.d/mail.conf
index 7bf51a1d..b76c7fd3 100644
--- a/config/action.d/mail.conf
+++ b/config/action.d/mail.conf
@@ -4,6 +4,10 @@
 #
 #
 
+[INCLUDES]
+
+before = helpers-common.conf
+
 [Definition]
 
 # Option:  actionstart
@@ -36,7 +40,8 @@ actioncheck =
 # Tags:    See jail.conf(5) man page
 # Values:  CMD
 #
-actionban = printf %%b "Hi,\n
+actionban = %(_bypass_if_restored)s
+            printf %%b "Hi,\n
             The IP <ip> has just been banned by Fail2Ban after
             <failures> attempts against <name>.\n
             Regards,\n
diff --git a/config/action.d/sendmail-buffered.conf b/config/action.d/sendmail-buffered.conf
index 80eb20a3..67b84251 100644
--- a/config/action.d/sendmail-buffered.conf
+++ b/config/action.d/sendmail-buffered.conf
@@ -7,6 +7,7 @@
 [INCLUDES]
 
 before = sendmail-common.conf
+         helpers-common.conf
 
 [Definition]
 
@@ -58,7 +59,8 @@ actioncheck =
 # Tags:    See jail.conf(5) man page
 # Values:  CMD
 #
-actionban = printf %%b "`date`: <ip> (<failures> failures)\n" >> <tmpfile>
+actionban = %(_bypass_if_restored)s
+            printf %%b "`date`: <ip> (<failures> failures)\n" >> <tmpfile>
             LINE=$( wc -l <tmpfile> | awk '{ print $1 }' )
             if [ $LINE -ge <lines> ]; then
                 printf %%b "Subject: [Fail2Ban] <name>: summary from `uname -n`
diff --git a/config/action.d/sendmail-geoip-lines.conf b/config/action.d/sendmail-geoip-lines.conf
index a5616e9f..a0de8f17 100644
--- a/config/action.d/sendmail-geoip-lines.conf
+++ b/config/action.d/sendmail-geoip-lines.conf
@@ -20,7 +20,8 @@ before = sendmail-common.conf
 # Tags:    See jail.conf(5) man page
 # Values:  CMD
 #
-actionban = ( printf %%b "Subject: [Fail2Ban] <name>: banned <ip> from `uname -n`
+actionban = %(_bypass_if_restored)s
+            ( printf %%b "Subject: [Fail2Ban] <name>: banned <ip> from `uname -n`
             Date: `LC_ALL=C date +"%%a, %%d %%h %%Y %%T %%z"`
             From: <sendername> <<sender>>
             To: <dest>\n
diff --git a/config/action.d/sendmail-whois-ipjailmatches.conf b/config/action.d/sendmail-whois-ipjailmatches.conf
index 689ffe45..2a8fcbcd 100644
--- a/config/action.d/sendmail-whois-ipjailmatches.conf
+++ b/config/action.d/sendmail-whois-ipjailmatches.conf
@@ -7,6 +7,7 @@
 [INCLUDES]
 
 before = sendmail-common.conf
+         helpers-common.conf
 
 [Definition]
 
@@ -16,7 +17,8 @@ before = sendmail-common.conf
 # Tags:    See jail.conf(5) man page
 # Values:  CMD
 #
-actionban = printf %%b "Subject: [Fail2Ban] <name>: banned <ip> from `uname -n`
+actionban = %(_bypass_if_restored)s
+            printf %%b "Subject: [Fail2Ban] <name>: banned <ip> from `uname -n`
             Date: `LC_ALL=C date +"%%a, %%d %%h %%Y %%T %%z"`
             From: <sendername> <<sender>>
             To: <dest>\n
diff --git a/config/action.d/sendmail-whois-ipmatches.conf b/config/action.d/sendmail-whois-ipmatches.conf
index b06e6db6..b6de1a8d 100644
--- a/config/action.d/sendmail-whois-ipmatches.conf
+++ b/config/action.d/sendmail-whois-ipmatches.conf
@@ -7,6 +7,7 @@
 [INCLUDES]
 
 before = sendmail-common.conf
+         helpers-common.conf
 
 [Definition]
 
@@ -16,7 +17,8 @@ before = sendmail-common.conf
 # Tags:    See jail.conf(5) man page
 # Values:  CMD
 #
-actionban = printf %%b "Subject: [Fail2Ban] <name>: banned <ip> from `uname -n`
+actionban = %(_bypass_if_restored)s
+            printf %%b "Subject: [Fail2Ban] <name>: banned <ip> from `uname -n`
             Date: `LC_ALL=C date +"%%a, %%d %%h %%Y %%T %%z"`
             From: <sendername> <<sender>>
             To: <dest>\n
diff --git a/config/action.d/sendmail-whois-lines.conf b/config/action.d/sendmail-whois-lines.conf
index e1c85928..77972acc 100644
--- a/config/action.d/sendmail-whois-lines.conf
+++ b/config/action.d/sendmail-whois-lines.conf
@@ -17,7 +17,8 @@ before = sendmail-common.conf
 # Tags:    See jail.conf(5) man page
 # Values:  CMD
 #
-actionban = ( printf %%b "Subject: [Fail2Ban] <name>: banned <ip> from `uname -n`
+actionban = %(_bypass_if_restored)s
+            ( printf %%b "Subject: [Fail2Ban] <name>: banned <ip> from `uname -n`
             Date: `LC_ALL=C date +"%%a, %%d %%h %%Y %%T %%z"`
             From: <sendername> <<sender>>
             To: <dest>\n
diff --git a/config/action.d/sendmail-whois-matches.conf b/config/action.d/sendmail-whois-matches.conf
index 8bca5937..7122d923 100644
--- a/config/action.d/sendmail-whois-matches.conf
+++ b/config/action.d/sendmail-whois-matches.conf
@@ -7,6 +7,7 @@
 [INCLUDES]
 
 before = sendmail-common.conf
+         helpers-common.conf
 
 [Definition]
 
@@ -16,7 +17,8 @@ before = sendmail-common.conf
 # Tags:    See jail.conf(5) man page
 # Values:  CMD
 #
-actionban = printf %%b "Subject: [Fail2Ban] <name>: banned <ip> from `uname -n`
+actionban = %(_bypass_if_restored)s
+            printf %%b "Subject: [Fail2Ban] <name>: banned <ip> from `uname -n`
             Date: `LC_ALL=C date +"%%a, %%d %%h %%Y %%T %%z"`
             From: <sendername> <<sender>>
             To: <dest>\n
diff --git a/config/action.d/sendmail-whois.conf b/config/action.d/sendmail-whois.conf
index 55b80bc5..82bc202e 100644
--- a/config/action.d/sendmail-whois.conf
+++ b/config/action.d/sendmail-whois.conf
@@ -7,6 +7,7 @@
 [INCLUDES]
 
 before = sendmail-common.conf
+         helpers-common.conf
 
 [Definition]
 
@@ -16,7 +17,8 @@ before = sendmail-common.conf
 # Tags:    See jail.conf(5) man page
 # Values:  CMD
 #
-actionban = printf %%b "Subject: [Fail2Ban] <name>: banned <ip> from `uname -n`
+actionban = %(_bypass_if_restored)s
+            printf %%b "Subject: [Fail2Ban] <name>: banned <ip> from `uname -n`
             Date: `LC_ALL=C date +"%%a, %%d %%h %%Y %%T %%z"`
             From: <sendername> <<sender>>
             To: <dest>\n
diff --git a/config/action.d/sendmail.conf b/config/action.d/sendmail.conf
index 5f5670c3..15f49766 100644
--- a/config/action.d/sendmail.conf
+++ b/config/action.d/sendmail.conf
@@ -7,6 +7,7 @@
 [INCLUDES]
 
 before = sendmail-common.conf
+         helpers-common.conf
 
 [Definition]
 
@@ -16,7 +17,8 @@ before = sendmail-common.conf
 # Tags:    See jail.conf(5) man page
 # Values:  CMD
 #
-actionban = printf %%b "Subject: [Fail2Ban] <name>: banned <ip> from `uname -n`
+actionban = %(_bypass_if_restored)s
+            printf %%b "Subject: [Fail2Ban] <name>: banned <ip> from `uname -n`
             Date: `LC_ALL=C date +"%%a, %%d %%h %%Y %%T %%z"`
             From: <sendername> <<sender>>
             To: <dest>\n
diff --git a/config/action.d/smtp.py b/config/action.d/smtp.py
index 2429cf48..65d50011 100644
--- a/config/action.d/smtp.py
+++ b/config/action.d/smtp.py
@@ -211,6 +211,8 @@ class SMTPAction(ActionBase):
 			Dictionary which includes information in relation to
 			the ban.
 		"""
+		if aInfo.get('restored'):
+			return
 		aInfo.update(self.message_values)
 		message = "".join([
 			messages['ban']['head'],
diff --git a/config/action.d/xarf-login-attack.conf b/config/action.d/xarf-login-attack.conf
index 3ab73817..13a5b70c 100644
--- a/config/action.d/xarf-login-attack.conf
+++ b/config/action.d/xarf-login-attack.conf
@@ -30,6 +30,10 @@
 #
 #
 
+[INCLUDES]
+
+before = helpers-common.conf
+
 [Definition]
 
 actionstart =
@@ -38,7 +42,8 @@ actionstop =
 
 actioncheck =
 
-actionban = oifs=${IFS}; IFS=.;SEP_IP=( <ip> ); set -- ${SEP_IP}; ADDRESSES=$(dig +short -t txt -q $4.$3.$2.$1.abuse-contacts.abusix.org); IFS=${oifs}
+actionban = %(_bypass_if_restored)s
+            oifs=${IFS}; IFS=.;SEP_IP=( <ip> ); set -- ${SEP_IP}; ADDRESSES=$(dig +short -t txt -q $4.$3.$2.$1.abuse-contacts.abusix.org); IFS=${oifs}
             IP=<ip>
             FROM=<sender>
             SERVICE=<service>
diff --git a/fail2ban/server/action.py b/fail2ban/server/action.py
index 62aa51dc..41c58068 100644
--- a/fail2ban/server/action.py
+++ b/fail2ban/server/action.py
@@ -219,9 +219,9 @@ class CommandAction(ActionBase):
 			self.timeout = 60
 			## Command executed in order to initialize the system.
 			self.actionstart = ''
-			## Command executed when an IP address gets banned.
+			## Command executed when ticket gets banned.
 			self.actionban = ''
-			## Command executed when an IP address gets removed.
+			## Command executed when ticket gets removed.
 			self.actionunban = ''
 			## Command executed in order to check requirements.
 			self.actioncheck = ''
diff --git a/fail2ban/server/actions.py b/fail2ban/server/actions.py
index dd4c97c2..cc832491 100644
--- a/fail2ban/server/actions.py
+++ b/fail2ban/server/actions.py
@@ -348,6 +348,8 @@ class Actions(JailThread, Mapping):
 			aInfo["failures"] = bTicket.getAttempt()
 			aInfo["time"] = bTicket.getTime()
 			aInfo["matches"] = "\n".join(bTicket.getMatches())
+			# to bypass actions, that should not be executed for restored tickets
+			aInfo["restored"] = 1 if ticket.restored else 0
 			if self._jail.database is not None:
 				mi4ip = lambda overalljails=False, self=self, \
 					mi={'ip':ip, 'ticket':bTicket}: self.__getBansMerged(mi, overalljails)
@@ -449,6 +451,8 @@ class Actions(JailThread, Mapping):
 		aInfo["failures"] = ticket.getAttempt()
 		aInfo["time"] = ticket.getTime()
 		aInfo["matches"] = "".join(ticket.getMatches())
+		# to bypass actions, that should not be executed for restored tickets
+		aInfo["restored"] = 1 if ticket.restored else 0
 		if actions is None:
 			logSys.notice("[%s] Unban %s", self._jail.name, aInfo["ip"])
 		for name, action in unbactions.iteritems():
diff --git a/fail2ban/tests/action_d/test_smtp.py b/fail2ban/tests/action_d/test_smtp.py
index d9ad0f3a..1dba454c 100644
--- a/fail2ban/tests/action_d/test_smtp.py
+++ b/fail2ban/tests/action_d/test_smtp.py
@@ -94,16 +94,21 @@ class SMTPActionTest(unittest.TestCase):
 			"Subject: [Fail2Ban] %s: stopped" %
 				self.jail.name in self.smtpd.data)
 
-	def testBan(self):
+	def _testBan(self, restored=False):
 		aInfo = {
 			'ip': "127.0.0.2",
 			'failures': 3,
 			'matches': "Test fail 1\n",
 			'ipjailmatches': "Test fail 1\nTest Fail2\n",
 			'ipmatches': "Test fail 1\nTest Fail2\nTest Fail3\n",
-			}
+		}
+		if restored:
+			aInfo['restored'] = 1
 
 		self.action.ban(aInfo)
+		if restored: # no mail, should raises attribute error:
+			self.assertRaises(AttributeError, lambda: self.smtpd.mailfrom)
+			return
 		self.assertEqual(self.smtpd.mailfrom, "fail2ban")
 		self.assertEqual(self.smtpd.rcpttos, ["root"])
 		subject = "Subject: [Fail2Ban] %s: banned %s" % (
@@ -123,6 +128,12 @@ class SMTPActionTest(unittest.TestCase):
 		self.action.matches = "ipmatches"
 		self.action.ban(aInfo)
 		self.assertIn(aInfo['ipmatches'], self.smtpd.data)
+	
+	def testBan(self):
+		self._testBan()
+
+	def testNOPByRestored(self):
+		self._testBan(restored=True)
 
 	def testOptions(self):
 		self.action.start()
diff --git a/fail2ban/tests/fail2banclienttestcase.py b/fail2ban/tests/fail2banclienttestcase.py
index e6cc46cb..5d285b76 100644
--- a/fail2ban/tests/fail2banclienttestcase.py
+++ b/fail2ban/tests/fail2banclienttestcase.py
@@ -756,9 +756,10 @@ class Fail2banServerTest(Fail2banClientServerBase):
 				return
 			_write_file(fn, "w",
 				"[Definition]",
+				"restore = ",
 				"actionstart =  echo '[<name>] %s: ** start'" % actname, start,
 				"actionreload = echo '[<name>] %s: .. reload'" % actname, reload,
-				"actionban =    echo '[<name>] %s: ++ ban <ip>'" % actname, ban,
+				"actionban =    echo '[<name>] %s: ++ ban <ip> %%(restore)s'" % actname, ban,
 				"actionunban =  echo '[<name>] %s: -- unban <ip>'" % actname, unban,
 				"actionstop =   echo '[<name>] %s: __ stop'" % actname, stop,
 			)
@@ -777,17 +778,22 @@ class Fail2banServerTest(Fail2banClientServerBase):
 				"",
 				"[test-jail1]", "backend = " + backend, "filter =", 
 				"action = ",
-				"         test-action1[name='%(__name__)s']" if 1 in actions else "",
-				"         test-action2[name='%(__name__)s']" if 2 in actions else "",
+				"         test-action1[name='%(__name__)s']" \
+					if 1 in actions else "",
+				"         test-action2[name='%(__name__)s', restore='restored: <restored>']" \
+					if 2 in actions else "",
 				"logpath = " + test1log,
 				"          " + test2log if 2 in enabled else "",
 				"          " + test3log if 2 in enabled else "",
 				"failregex = ^\s*failure (401|403) from <HOST>",
-				"            ^\s*error (401|403) from <HOST>" if 2 in enabled else "",
+				"            ^\s*error (401|403) from <HOST>" \
+					if 2 in enabled else "",
 				"enabled = true" if 1 in enabled else "",
 				"",
 				"[test-jail2]", "backend = " + backend, "filter =", 
-				"action =",
+				"action = ",
+				"         test-action2[name='%(__name__)s', restore='restored: <restored>']" \
+					if 2 in actions else "",
 				"logpath = " + test2log,
 				"enabled = true" if 2 in enabled else "",
 			)
@@ -821,6 +827,10 @@ class Fail2banServerTest(Fail2banClientServerBase):
 		self.assertLogged(
 			"stdout: '[test-jail1] test-action1: ** start'", 
 			"stdout: '[test-jail1] test-action2: ** start'", all=True)
+		# test restored is 0:
+		self.assertLogged(
+			"stdout: '[test-jail1] test-action2: ++ ban 192.0.2.1 restored: 0'",
+			all=True, wait=MID_WAITTIME)
 
 		# broken jail was logged (in client and server log):
 		self.assertLogged(
@@ -882,10 +892,10 @@ class Fail2banServerTest(Fail2banClientServerBase):
 		self.assertNotLogged(
 			"stdout: '[test-jail1] test-action1: -- unban 192.0.2.1'")
 		
-		# don't need both actions anymore:
+		# don't need action1 anymore:
 		_write_action_cfg(actname="test-action1", allow=False)
-		_write_action_cfg(actname="test-action2", allow=False)
-		_write_jail_cfg(actions=[])
+		# leave action2 just to test restored interpolation:
+		_write_jail_cfg(actions=[2])
 		
 		# write new failures:
 		self.pruneLog("[test-phase 2b]")
@@ -913,7 +923,8 @@ class Fail2banServerTest(Fail2banClientServerBase):
 			"[test-jail2] Found 192.0.2.2", 
 			"[test-jail2] Ban 192.0.2.2",
 			"[test-jail2] Found 192.0.2.3", 
-			"[test-jail2] Ban 192.0.2.3", all=True)
+			"[test-jail2] Ban 192.0.2.3", 
+			all=True)
 
 		# rotate logs:
 		_write_file(test1log, "w+")
@@ -936,6 +947,19 @@ class Fail2banServerTest(Fail2banClientServerBase):
 			"[test-jail2] Restore Ban 192.0.2.4",
 			"[test-jail2] Restore Ban 192.0.2.8", all=True
 		)
+		# test restored is 1:
+		self.assertLogged(
+			"stdout: '[test-jail2] test-action2: ++ ban 192.0.2.4 restored: 1'",
+			"stdout: '[test-jail2] test-action2: ++ ban 192.0.2.8 restored: 1'",
+			all=True, wait=MID_WAITTIME)
+		self.assertNotLogged(
+			"stdout: '[test-jail2] test-action2: ++ ban 192.0.2.4 restored: 0'",
+			"stdout: '[test-jail2] test-action2: ++ ban 192.0.2.8 restored: 0'",
+			all=True)
+
+		# don't need actions anymore:
+		_write_action_cfg(actname="test-action2", allow=False)
+		_write_jail_cfg(actions=[])
 
 		# restart jail with unban all:
 		self.pruneLog("[test-phase 2d]")

From de49f0c27f3391cbb5e5eade6324365af2f7bfa5 Mon Sep 17 00:00:00 2001
From: sebres <serg.brester@sebres.de>
Date: Fri, 13 Jan 2017 19:33:16 +0100
Subject: [PATCH 2/7] ChangeLog entry

---
 ChangeLog | 22 ++++++++++++++++++++++
 1 file changed, 22 insertions(+)

diff --git a/ChangeLog b/ChangeLog
index 2c202c07..8e7b5522 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -165,6 +165,28 @@ fail2ban-client set loglevel INFO
   - faster match and fewer searching of appropriate templates
     (DateDetector.matchTime calls rarer DateTemplate.matchDate now);
   - several standard filters extended with exact prefixed or anchored date templates;
+* Added possibility to recognize restored state of the tickets (see gh-1669).
+  For the shell-based actions it is enough to add following script-piece at begin 
+  of `actionban` (or `actionunban`), to prevent execution for the restored 
+  (after restart) tickets.
+  `if [ '<restored>' = '1' ]; then exit 0; fi;`
+  Several actions extended now:
+  - complain.conf
+  - dshield.conf
+  - mail-buffered.conf
+  - mail-whois-lines.conf
+  - mail-whois.conf
+  - mail.conf
+  - sendmail-buffered.conf
+  - sendmail-geoip-lines.conf
+  - sendmail-whois-ipjailmatches.conf
+  - sendmail-whois-ipmatches.conf
+  - sendmail-whois-lines.conf
+  - sendmail-whois-matches.conf
+  - sendmail-whois.conf
+  - sendmail.conf
+  - smtp.py
+  - xarf-login-attack.conf
 * fail2ban-testcases:
   - `assertLogged` extended with parameter wait (to wait up to specified timeout,
     before we throw assert exception) + test cases rewritten using that

From 2ed2e7810d1f6c594b2d596857c80660dec8b584 Mon Sep 17 00:00:00 2001
From: sebres <serg.brester@sebres.de>
Date: Sat, 14 Jan 2017 00:17:23 +0100
Subject: [PATCH 3/7] normalization of DefinitionInitConfigReader (action /
 filter): client-side interpolation, etc.

---
 fail2ban/client/actionreader.py        | 17 ++++++++++++++---
 fail2ban/client/configreader.py        | 23 +++++++++++++++++++++++
 fail2ban/client/filterreader.py        | 16 +++-------------
 fail2ban/client/jailreader.py          |  3 ++-
 fail2ban/server/action.py              | 15 ++++++++++-----
 fail2ban/tests/clientreadertestcase.py | 12 +++++-------
 6 files changed, 57 insertions(+), 29 deletions(-)

diff --git a/fail2ban/client/actionreader.py b/fail2ban/client/actionreader.py
index e5025fa3..55ceda93 100644
--- a/fail2ban/client/actionreader.py
+++ b/fail2ban/client/actionreader.py
@@ -43,10 +43,15 @@ class ActionReader(DefinitionInitConfigReader):
 		"actionrepair": ["string", None],
 		"actionban": ["string", None],
 		"actionunban": ["string", None],
+		"norestored": ["string", None],
 	}
 
 	def __init__(self, file_, jailName, initOpts, **kwargs):
-		self._name = initOpts.get("actname", file_)
+		actname = initOpts.get("actname")
+		if actname is None:
+			actname = file_
+			initOpts["actname"] = actname
+		self._name = actname
 		DefinitionInitConfigReader.__init__(
 			self, file_, jailName, initOpts, **kwargs)
 
@@ -64,16 +69,22 @@ class ActionReader(DefinitionInitConfigReader):
 		return self._name
 
 	def convert(self):
+		opts = self.getCombined(ignore=('timeout', 'bantime'))
+		# type-convert only after combined (otherwise boolean converting prevents substitution):
+		if opts.get('norestored'):
+			opts['norestored'] = self._convert_to_boolean(opts['norestored'])
+		# stream-convert:
 		head = ["set", self._jailName]
 		stream = list()
 		stream.append(head + ["addaction", self._name])
 		multi = []
-		for opt, optval in self._opts.iteritems():
+		for opt, optval in opts.iteritems():
 			if opt in self._configOpts:
 				multi.append([opt, optval])
 		if self._initOpts:
 			for opt, optval in self._initOpts.iteritems():
-				multi.append([opt, optval])
+				if opt not in self._configOpts:
+					multi.append([opt, optval])
 		if len(multi) > 1:
 			stream.append(["multi-set", self._jailName, "action", self._name, multi])
 		elif len(multi):
diff --git a/fail2ban/client/configreader.py b/fail2ban/client/configreader.py
index a72ca1e9..7840cd12 100644
--- a/fail2ban/client/configreader.py
+++ b/fail2ban/client/configreader.py
@@ -30,6 +30,7 @@ from ConfigParser import NoOptionError, NoSectionError
 
 from .configparserinc import sys, SafeConfigParserWithIncludes, logLevel
 from ..helpers import getLogger
+from ..server.action import CommandAction
 
 # Gets the instance of the logger.
 logSys = getLogger(__name__)
@@ -319,6 +320,28 @@ class DefinitionInitConfigReader(ConfigReader):
 					self._initOpts['known/'+opt] = v
 				if not opt in self._initOpts:
 					self._initOpts[opt] = v
+
+	def _convert_to_boolean(self, value):
+		return value.lower() in ("1", "yes", "true", "on")
+	
+	def getCombined(self, ignore=()):
+		combinedopts = self._opts
+		ignore = set(ignore).copy()
+		if self._initOpts:
+			combinedopts = _merge_dicts(self._opts, self._initOpts)
+		if not len(combinedopts):
+			return {}
+		# ignore conditional options:
+		for n in combinedopts:
+			cond = SafeConfigParserWithIncludes.CONDITIONAL_RE.match(n)
+			if cond:
+				n, cond = cond.groups()
+				ignore.add(n)
+		# substiture options already specified direct:
+		opts = CommandAction.substituteRecursiveTags(combinedopts, ignore=ignore)
+		if not opts:
+			raise ValueError('recursive tag definitions unable to be resolved')
+		return opts
 	
 	def convert(self):
 		raise NotImplementedError
diff --git a/fail2ban/client/filterreader.py b/fail2ban/client/filterreader.py
index cdf0d8af..d89ef5ad 100644
--- a/fail2ban/client/filterreader.py
+++ b/fail2ban/client/filterreader.py
@@ -27,8 +27,7 @@ __license__ = "GPL"
 import os
 import shlex
 
-from .configreader import DefinitionInitConfigReader, _merge_dicts
-from ..server.action import CommandAction
+from .configreader import DefinitionInitConfigReader
 from ..helpers import getLogger
 
 # Gets the instance of the logger.
@@ -52,17 +51,6 @@ class FilterReader(DefinitionInitConfigReader):
 	def getFile(self):
 		return self.__file
 
-	def getCombined(self):
-		combinedopts = self._opts
-		if self._initOpts:
-			combinedopts = _merge_dicts(self._opts, self._initOpts)
-		if not len(combinedopts):
-			return {}
-		opts = CommandAction.substituteRecursiveTags(combinedopts)
-		if not opts:
-			raise ValueError('recursive tag definitions unable to be resolved')
-		return opts
-	
 	def convert(self):
 		stream = list()
 		opts = self.getCombined()
@@ -70,6 +58,7 @@ class FilterReader(DefinitionInitConfigReader):
 			return stream
 		for opt, value in opts.iteritems():
 			if opt in ("failregex", "ignoreregex"):
+				if value is None: continue
 				multi = []
 				for regex in value.split('\n'):
 					# Do not send a command if the rule is empty.
@@ -87,6 +76,7 @@ class FilterReader(DefinitionInitConfigReader):
 				stream.append(["set", self._jailName, "datepattern", value])
 			# Do not send a command if the match is empty.
 			elif opt == 'journalmatch':
+				if value is None: continue
 				for match in value.split("\n"):
 					if match == '': continue
 					stream.append(
diff --git a/fail2ban/client/jailreader.py b/fail2ban/client/jailreader.py
index b223532e..f4adadbf 100644
--- a/fail2ban/client/jailreader.py
+++ b/fail2ban/client/jailreader.py
@@ -136,7 +136,8 @@ class JailReader(ConfigReader):
 				if not filterName:
 					raise JailDefError("Invalid filter definition %r" % flt)
 				self.__filter = FilterReader(
-					filterName, self.__name, filterOpt, share_config=self.share_config, basedir=self.getBaseDir())
+					filterName, self.__name, filterOpt, 
+					share_config=self.share_config, basedir=self.getBaseDir())
 				ret = self.__filter.read()
 				# merge options from filter as 'known/...':
 				self.__filter.getOptions(self.__opts)
diff --git a/fail2ban/server/action.py b/fail2ban/server/action.py
index 41c58068..46a19cd1 100644
--- a/fail2ban/server/action.py
+++ b/fail2ban/server/action.py
@@ -365,7 +365,7 @@ class CommandAction(ActionBase):
 		return self._executeOperation('<actionreload>', 'reloading')
 
 	@classmethod
-	def substituteRecursiveTags(cls, inptags, conditional=''):
+	def substituteRecursiveTags(cls, inptags, conditional='', ignore=()):
 		"""Sort out tag definitions within other tags.
 		Since v.0.9.2 supports embedded interpolation (see test cases for examples).
 
@@ -387,21 +387,26 @@ class CommandAction(ActionBase):
 		# copy return tags dict to prevent modifying of inptags:
 		tags = inptags.copy()
 		t = TAG_CRE
+		ignore = set(ignore)
+		done = cls._escapedTags.copy() | ignore
 		# repeat substitution while embedded-recursive (repFlag is True)
-		done = cls._escapedTags.copy()
 		while True:
 			repFlag = False
 			# substitute each value:
 			for tag in tags.iterkeys():
-				# ignore escaped or already done:
+				# ignore escaped or already done (or in ignore list):
 				if tag in done: continue
-				value = str(tags[tag])
+				value = orgval = str(tags[tag])
 				# search and replace all tags within value, that can be interpolated using other tags:
 				m = t.search(value)
 				refCounts = {}
 				#logSys.log(5, 'TAG: %s, value: %s' % (tag, value))
 				while m:
 					found_tag = m.group(1)
+					# don't replace tags that should be currently ignored (pre-replacement):
+					if found_tag in ignore: 
+						m = t.search(value, m.end())
+						continue
 					#logSys.log(5, 'found: %s' % found_tag)
 					if found_tag == tag or refCounts.get(found_tag, 1) > MAX_TAG_REPLACE_COUNT:
 						# recursive definitions are bad
@@ -429,7 +434,7 @@ class CommandAction(ActionBase):
 					m = t.search(value, m.start())
 				#logSys.log(5, 'TAG: %s, newvalue: %s' % (tag, value))
 				# was substituted?
-				if tags[tag] != value:
+				if orgval != value:
 					# check still contains any tag - should be repeated (possible embedded-recursive substitution):
 					if t.search(value):
 						repFlag = True
diff --git a/fail2ban/tests/clientreadertestcase.py b/fail2ban/tests/clientreadertestcase.py
index 39622cd0..f8bd3975 100644
--- a/fail2ban/tests/clientreadertestcase.py
+++ b/fail2ban/tests/clientreadertestcase.py
@@ -347,7 +347,7 @@ class FilterReaderTest(unittest.TestCase):
 			['set', 'testcase01', 'addjournalmatch',
 				"FIELD= with spaces ", "+", "AFIELD= with + char and spaces"],
 			['set', 'testcase01', 'datepattern', "%Y %m %d %H:%M:%S"],
-			['set', 'testcase01', 'maxlines', "1"], # Last for overide test
+			['set', 'testcase01', 'maxlines', 1], # Last for overide test
 		]
 		filterReader = FilterReader("testcase01", "testcase01", {})
 		filterReader.setBaseDir(TEST_FILES_DIR)
@@ -517,12 +517,10 @@ class JailsReaderTest(LogCaptureTestCase):
 			 ['add', 'brokenaction', 'auto'],
 			 ['set', 'brokenaction', 'addfailregex', '<IP>'],
 			 ['set', 'brokenaction', 'addaction', 'brokenaction'],
-			 ['set',
-			  'brokenaction',
-			  'action',
-			  'brokenaction',
-			  'actionban',
-			  'hit with big stick <ip>'],
+			 ['multi-set', 'brokenaction', 'action', 'brokenaction', [
+				 ['actionban', 'hit with big stick <ip>'],
+				 ['actname', 'brokenaction']
+			 ]],
 			 ['add', 'parse_to_end_of_jail.conf', 'auto'],
 			 ['set', 'parse_to_end_of_jail.conf', 'addfailregex', '<IP>'],
 			 ['start', 'emptyaction'],

From 0aa241d303fa64510252bf92988f6615baa9f8ab Mon Sep 17 00:00:00 2001
From: sebres <serg.brester@sebres.de>
Date: Mon, 16 Jan 2017 09:05:45 +0100
Subject: [PATCH 4/7] Another way to recognize restored tickets - new option
 `norestored` of action introduced; Complete prevents executing of ban/unban
 operations for actions where norestored = true.

---
 fail2ban/server/actions.py               |  4 +++
 fail2ban/tests/fail2banclienttestcase.py | 32 ++++++++++++++++--------
 2 files changed, 25 insertions(+), 11 deletions(-)

diff --git a/fail2ban/server/actions.py b/fail2ban/server/actions.py
index cc832491..5bd00c85 100644
--- a/fail2ban/server/actions.py
+++ b/fail2ban/server/actions.py
@@ -363,6 +363,8 @@ class Actions(JailThread, Mapping):
 				logSys.notice("[%s] %sBan %s", self._jail.name, ('' if not bTicket.restored else 'Restore '), ip)
 				for name, action in self._actions.iteritems():
 					try:
+						if ticket.restored and getattr(action, 'norestored', False):
+							continue
 						action.ban(aInfo.copy())
 					except Exception as e:
 						logSys.error(
@@ -457,6 +459,8 @@ class Actions(JailThread, Mapping):
 			logSys.notice("[%s] Unban %s", self._jail.name, aInfo["ip"])
 		for name, action in unbactions.iteritems():
 			try:
+				if ticket.restored and getattr(action, 'norestored', False):
+					continue
 				logSys.debug("[%s] action %r: unban %s", self._jail.name, name, aInfo["ip"])
 				action.unban(aInfo.copy())
 			except Exception as e:
diff --git a/fail2ban/tests/fail2banclienttestcase.py b/fail2ban/tests/fail2banclienttestcase.py
index 5d285b76..b8417be5 100644
--- a/fail2ban/tests/fail2banclienttestcase.py
+++ b/fail2ban/tests/fail2banclienttestcase.py
@@ -755,13 +755,17 @@ class Fail2banServerTest(Fail2banClientServerBase):
 				os.remove(fn)
 				return
 			_write_file(fn, "w",
+				"[DEFAULT]",
+				"_exec_once = 0",
+				"",
 				"[Definition]",
+				"norestored = %(_exec_once)s",
 				"restore = ",
-				"actionstart =  echo '[<name>] %s: ** start'" % actname, start,
-				"actionreload = echo '[<name>] %s: .. reload'" % actname, reload,
-				"actionban =    echo '[<name>] %s: ++ ban <ip> %%(restore)s'" % actname, ban,
-				"actionunban =  echo '[<name>] %s: -- unban <ip>'" % actname, unban,
-				"actionstop =   echo '[<name>] %s: __ stop'" % actname, stop,
+				"actionstart =  echo '[%(name)s] %(actname)s: ** start'", start,
+				"actionreload = echo '[%(name)s] %(actname)s: .. reload'", reload,
+				"actionban =    echo '[%(name)s] %(actname)s: ++ ban <ip> %(restore)s'", ban,
+				"actionunban =  echo '[%(name)s] %(actname)s: -- unban <ip>'", unban,
+				"actionstop =   echo '[%(name)s] %(actname)s: __ stop'", stop,
 			)
 			if unittest.F2B.log_level <= logging.DEBUG: # pragma: no cover
 				_out_file(fn)
@@ -782,6 +786,8 @@ class Fail2banServerTest(Fail2banClientServerBase):
 					if 1 in actions else "",
 				"         test-action2[name='%(__name__)s', restore='restored: <restored>']" \
 					if 2 in actions else "",
+				"         test-action2[name='%(__name__)s', actname=test-action3, _exec_once=1, restore='restored: <restored>']" \
+					if 3 in actions else "",
 				"logpath = " + test1log,
 				"          " + test2log if 2 in enabled else "",
 				"          " + test3log if 2 in enabled else "",
@@ -794,6 +800,8 @@ class Fail2banServerTest(Fail2banClientServerBase):
 				"action = ",
 				"         test-action2[name='%(__name__)s', restore='restored: <restored>']" \
 					if 2 in actions else "",
+				"         test-action2[name='%(__name__)s', actname=test-action3, _exec_once=1, restore='restored: <restored>']" \
+					if 3 in actions else "",
 				"logpath = " + test2log,
 				"enabled = true" if 2 in enabled else "",
 			)
@@ -804,7 +812,7 @@ class Fail2banServerTest(Fail2banClientServerBase):
 		_write_action_cfg(actname="test-action1")
 		_write_action_cfg(actname="test-action2")
 
-		_write_jail_cfg(enabled=[1], actions=[1,2])
+		_write_jail_cfg(enabled=[1], actions=[1,2,3])
 		# append one wrong configured jail:
 		_write_file(pjoin(cfg, "jail.conf"), "a", "", "[broken-jail]", 
 			"", "filter = broken-jail-filter", "enabled = true")
@@ -827,9 +835,10 @@ class Fail2banServerTest(Fail2banClientServerBase):
 		self.assertLogged(
 			"stdout: '[test-jail1] test-action1: ** start'", 
 			"stdout: '[test-jail1] test-action2: ** start'", all=True)
-		# test restored is 0:
+		# test restored is 0 (both actions available):
 		self.assertLogged(
 			"stdout: '[test-jail1] test-action2: ++ ban 192.0.2.1 restored: 0'",
+			"stdout: '[test-jail1] test-action3: ++ ban 192.0.2.1 restored: 0'",
 			all=True, wait=MID_WAITTIME)
 
 		# broken jail was logged (in client and server log):
@@ -895,7 +904,7 @@ class Fail2banServerTest(Fail2banClientServerBase):
 		# don't need action1 anymore:
 		_write_action_cfg(actname="test-action1", allow=False)
 		# leave action2 just to test restored interpolation:
-		_write_jail_cfg(actions=[2])
+		_write_jail_cfg(actions=[2,3])
 		
 		# write new failures:
 		self.pruneLog("[test-phase 2b]")
@@ -947,14 +956,15 @@ class Fail2banServerTest(Fail2banClientServerBase):
 			"[test-jail2] Restore Ban 192.0.2.4",
 			"[test-jail2] Restore Ban 192.0.2.8", all=True
 		)
-		# test restored is 1:
+		# test restored is 1 (only test-action2):
 		self.assertLogged(
 			"stdout: '[test-jail2] test-action2: ++ ban 192.0.2.4 restored: 1'",
 			"stdout: '[test-jail2] test-action2: ++ ban 192.0.2.8 restored: 1'",
 			all=True, wait=MID_WAITTIME)
+		# test test-action3 not executed at all (norestored check):
 		self.assertNotLogged(
-			"stdout: '[test-jail2] test-action2: ++ ban 192.0.2.4 restored: 0'",
-			"stdout: '[test-jail2] test-action2: ++ ban 192.0.2.8 restored: 0'",
+			"stdout: '[test-jail2] test-action3: ++ ban 192.0.2.4 restored: 1'",
+			"stdout: '[test-jail2] test-action3: ++ ban 192.0.2.8 restored: 1'",
 			all=True)
 
 		# don't need actions anymore:

From 8b82c6669e5d33fc41c72e59e7352f199e74c8bf Mon Sep 17 00:00:00 2001
From: sebres <serg.brester@sebres.de>
Date: Mon, 16 Jan 2017 09:34:10 +0100
Subject: [PATCH 5/7] provide name of action to fail-message (e. g. if
 interpolation fails)

---
 fail2ban/tests/clientreadertestcase.py | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/fail2ban/tests/clientreadertestcase.py b/fail2ban/tests/clientreadertestcase.py
index f8bd3975..4a692598 100644
--- a/fail2ban/tests/clientreadertestcase.py
+++ b/fail2ban/tests/clientreadertestcase.py
@@ -546,7 +546,10 @@ class JailsReaderTest(LogCaptureTestCase):
 				actionName = os.path.basename(actionConfig).replace('.conf', '')
 				actionReader = ActionReader(actionName, "TEST", {}, basedir=CONFIG_DIR)
 				self.assertTrue(actionReader.read())
-				actionReader.getOptions({})	  # populate _opts
+				try:
+					actionReader.getOptions({})	  # populate _opts
+				except Exception as e: # pragma: no cover
+					self.fail("action %r\n%s: %s" % (actionName, type(e).__name__, e))
 				if not actionName.endswith('-common'):
 					self.assertIn('Definition', actionReader.sections(),
 						msg="Action file %r is lacking [Definition] section" % actionConfig)

From 74a6afadd565b679f73b88a6ebf81a358c0ee155 Mon Sep 17 00:00:00 2001
From: sebres <serg.brester@sebres.de>
Date: Mon, 16 Jan 2017 09:40:48 +0100
Subject: [PATCH 6/7] Mail-actions switched to use new option "norestored"
 instead of checking of variable `restored` during shell execution (prevents
 executing of such actions at all).

---
 config/action.d/complain.conf                     |  6 ++++--
 config/action.d/dshield.conf                      | 13 +++++--------
 config/action.d/mail-buffered.conf                | 11 ++++-------
 config/action.d/mail-whois-lines.conf             |  6 ++++--
 config/action.d/mail-whois.conf                   |  7 ++++---
 config/action.d/mail.conf                         | 10 ++++------
 config/action.d/sendmail-buffered.conf            |  7 ++++---
 config/action.d/sendmail-geoip-lines.conf         |  6 ++++--
 config/action.d/sendmail-whois-ipjailmatches.conf |  7 ++++---
 config/action.d/sendmail-whois-ipmatches.conf     |  7 ++++---
 config/action.d/sendmail-whois-lines.conf         |  6 ++++--
 config/action.d/sendmail-whois-matches.conf       |  7 ++++---
 config/action.d/sendmail-whois.conf               |  7 ++++---
 config/action.d/sendmail.conf                     |  7 ++++---
 config/action.d/smtp.py                           |  3 +++
 config/action.d/xarf-login-attack.conf            | 10 ++++------
 16 files changed, 64 insertions(+), 56 deletions(-)

diff --git a/config/action.d/complain.conf b/config/action.d/complain.conf
index d0156a44..84dbaf39 100644
--- a/config/action.d/complain.conf
+++ b/config/action.d/complain.conf
@@ -34,6 +34,9 @@ before = helpers-common.conf
 
 [Definition]
 
+# bypass ban/unban for restored tickets
+norestored = 1
+
 # Option:  actionstart
 # Notes.:  command executed once at the start of Fail2Ban.
 # Values:  CMD
@@ -58,8 +61,7 @@ actioncheck =
 # Tags:    See jail.conf(5) man page
 # Values:  CMD
 #
-actionban = %(_bypass_if_restored)s
-            oifs=${IFS}; 
+actionban = oifs=${IFS}; 
               IFS=.; SEP_IP=( <ip> ); set -- ${SEP_IP}; ADDRESSES=$(dig +short -t txt -q $4.$3.$2.$1.abuse-contacts.abusix.org); 
               IFS=,; ADDRESSES=$(echo $ADDRESSES)
             IFS=${oifs}
diff --git a/config/action.d/dshield.conf b/config/action.d/dshield.conf
index 35eaa3be..4f2e09ca 100644
--- a/config/action.d/dshield.conf
+++ b/config/action.d/dshield.conf
@@ -26,12 +26,11 @@
 # configure how often the buffer is flushed).
 #
 
-[INCLUDES]
-
-before = helpers-common.conf
-
 [Definition]
 
+# bypass ban/unban for restored tickets
+norestored = 1
+
 # Option:  actionstart
 # Notes.:  command executed once at the start of Fail2Ban.
 # Values:  CMD
@@ -68,8 +67,7 @@ actioncheck =
 # few seconds out, are incorrect. See
 # http://sourceforge.net/tracker/index.php?func=detail&aid=2017795&group_id=121032&atid=689047
 #
-actionban = %(_bypass_if_restored)s
-            TZONE=`date +%%z | sed 's/\([+-]..\)\(..\)/\1:\2/'`
+actionban = TZONE=`date +%%z | sed 's/\([+-]..\)\(..\)/\1:\2/'`
             DATETIME="`perl -e '@t=localtime(<time>);printf "%%4d-%%02d-%%02d %%02d:%%02d:%%02d",1900+$t[5],$t[4]+1,$t[3],$t[2],$t[1],$t[0]'` $TZONE"
 	    PROTOCOL=`awk '{IGNORECASE=1;if($1=="<protocol>"){print $2;exit}}' /etc/protocols`
 	    if [ -z "$PROTOCOL" ]; then PROTOCOL=<protocol>; fi
@@ -96,8 +94,7 @@ actionban = %(_bypass_if_restored)s
 # Tags:    See jail.conf(5) man page
 # Values:  CMD
 #
-actionunban = %(_bypass_if_restored)s
-              if [ -f <tmpfile>.first ]; then
+actionunban = if [ -f <tmpfile>.first ]; then
                   NOW=`date +%%s`
                   LOGAGE=$(($NOW - `cat <tmpfile>.first`))
                   if [ $LOGAGE -gt <maxbufferage> ]; then
diff --git a/config/action.d/mail-buffered.conf b/config/action.d/mail-buffered.conf
index c1b3f565..e74db9cc 100644
--- a/config/action.d/mail-buffered.conf
+++ b/config/action.d/mail-buffered.conf
@@ -4,13 +4,11 @@
 #
 #
 
-[INCLUDES]
-
-before = helpers-common.conf
-
-
 [Definition]
 
+# bypass ban/unban for restored tickets
+norestored = 1
+
 # Option:  actionstart
 # Notes.:  command executed once at the start of Fail2Ban.
 # Values:  CMD
@@ -50,8 +48,7 @@ actioncheck =
 # Tags:    See jail.conf(5) man page
 # Values:  CMD
 #
-actionban = %(_bypass_if_restored)s
-            printf %%b "`date`: <ip> (<failures> failures)\n" >> <tmpfile>
+actionban = printf %%b "`date`: <ip> (<failures> failures)\n" >> <tmpfile>
             LINE=$( wc -l <tmpfile> | awk '{ print $1 }' )
             if [ $LINE -ge <lines> ]; then
                 printf %%b "Hi,\n
diff --git a/config/action.d/mail-whois-lines.conf b/config/action.d/mail-whois-lines.conf
index c471a44a..0852ba8f 100644
--- a/config/action.d/mail-whois-lines.conf
+++ b/config/action.d/mail-whois-lines.conf
@@ -11,6 +11,9 @@ before = mail-whois-common.conf
 
 [Definition]
 
+# bypass ban/unban for restored tickets
+norestored = 1
+
 # Option:  actionstart
 # Notes.:  command executed once at the start of Fail2Ban.
 # Values:  CMD
@@ -53,8 +56,7 @@ _ban_mail_content = ( printf %%b "Hi,\n
             Regards,\n
             Fail2Ban" )
 
-actionban = %(_bypass_if_restored)s
-            %(_ban_mail_content)s | <mailcmd> "[Fail2Ban] <name>: banned <ip> from  `uname -n`" <dest>
+actionban = %(_ban_mail_content)s | <mailcmd> "[Fail2Ban] <name>: banned <ip> from  `uname -n`" <dest>
 
 # Option:  actionunban
 # Notes.:  command executed when unbanning an IP. Take care that the
diff --git a/config/action.d/mail-whois.conf b/config/action.d/mail-whois.conf
index 045da5ec..553bfb69 100644
--- a/config/action.d/mail-whois.conf
+++ b/config/action.d/mail-whois.conf
@@ -7,10 +7,12 @@
 [INCLUDES]
 
 before = mail-whois-common.conf
-         helpers-common.conf
 
 [Definition]
 
+# bypass ban/unban for restored tickets
+norestored = 1
+
 # Option:  actionstart
 # Notes.:  command executed once at the start of Fail2Ban.
 # Values:  CMD
@@ -41,8 +43,7 @@ actioncheck =
 # Tags:    See jail.conf(5) man page
 # Values:  CMD
 #
-actionban = %(_bypass_if_restored)s
-            printf %%b "Hi,\n
+actionban = printf %%b "Hi,\n
             The IP <ip> has just been banned by Fail2Ban after
             <failures> attempts against <name>.\n\n
             Here is more information about <ip> :\n
diff --git a/config/action.d/mail.conf b/config/action.d/mail.conf
index b76c7fd3..4715ecc5 100644
--- a/config/action.d/mail.conf
+++ b/config/action.d/mail.conf
@@ -4,12 +4,11 @@
 #
 #
 
-[INCLUDES]
-
-before = helpers-common.conf
-
 [Definition]
 
+# bypass ban/unban for restored tickets
+norestored = 1
+
 # Option:  actionstart
 # Notes.:  command executed once at the start of Fail2Ban.
 # Values:  CMD
@@ -40,8 +39,7 @@ actioncheck =
 # Tags:    See jail.conf(5) man page
 # Values:  CMD
 #
-actionban = %(_bypass_if_restored)s
-            printf %%b "Hi,\n
+actionban = printf %%b "Hi,\n
             The IP <ip> has just been banned by Fail2Ban after
             <failures> attempts against <name>.\n
             Regards,\n
diff --git a/config/action.d/sendmail-buffered.conf b/config/action.d/sendmail-buffered.conf
index 67b84251..a91a6957 100644
--- a/config/action.d/sendmail-buffered.conf
+++ b/config/action.d/sendmail-buffered.conf
@@ -7,10 +7,12 @@
 [INCLUDES]
 
 before = sendmail-common.conf
-         helpers-common.conf
 
 [Definition]
 
+# bypass ban/unban for restored tickets
+norestored = 1
+
 # Option:  actionstart
 # Notes.:  command executed once at the start of Fail2Ban.
 # Values:  CMD
@@ -59,8 +61,7 @@ actioncheck =
 # Tags:    See jail.conf(5) man page
 # Values:  CMD
 #
-actionban = %(_bypass_if_restored)s
-            printf %%b "`date`: <ip> (<failures> failures)\n" >> <tmpfile>
+actionban = printf %%b "`date`: <ip> (<failures> failures)\n" >> <tmpfile>
             LINE=$( wc -l <tmpfile> | awk '{ print $1 }' )
             if [ $LINE -ge <lines> ]; then
                 printf %%b "Subject: [Fail2Ban] <name>: summary from `uname -n`
diff --git a/config/action.d/sendmail-geoip-lines.conf b/config/action.d/sendmail-geoip-lines.conf
index a0de8f17..34c3aedd 100644
--- a/config/action.d/sendmail-geoip-lines.conf
+++ b/config/action.d/sendmail-geoip-lines.conf
@@ -11,6 +11,9 @@ before = sendmail-common.conf
 
 [Definition]
 
+# bypass ban/unban for restored tickets
+norestored = 1
+
 # Option:  actionban
 # Notes.:  Command executed when banning an IP. Take care that the
 #          command is executed with Fail2Ban user rights.
@@ -20,8 +23,7 @@ before = sendmail-common.conf
 # Tags:    See jail.conf(5) man page
 # Values:  CMD
 #
-actionban = %(_bypass_if_restored)s
-            ( printf %%b "Subject: [Fail2Ban] <name>: banned <ip> from `uname -n`
+actionban = ( printf %%b "Subject: [Fail2Ban] <name>: banned <ip> from `uname -n`
             Date: `LC_ALL=C date +"%%a, %%d %%h %%Y %%T %%z"`
             From: <sendername> <<sender>>
             To: <dest>\n
diff --git a/config/action.d/sendmail-whois-ipjailmatches.conf b/config/action.d/sendmail-whois-ipjailmatches.conf
index 2a8fcbcd..5bcefe89 100644
--- a/config/action.d/sendmail-whois-ipjailmatches.conf
+++ b/config/action.d/sendmail-whois-ipjailmatches.conf
@@ -7,18 +7,19 @@
 [INCLUDES]
 
 before = sendmail-common.conf
-         helpers-common.conf
 
 [Definition]
 
+# bypass ban/unban for restored tickets
+norestored = 1
+
 # Option:  actionban
 # Notes.:  command executed when banning an IP. Take care that the
 #          command is executed with Fail2Ban user rights.
 # Tags:    See jail.conf(5) man page
 # Values:  CMD
 #
-actionban = %(_bypass_if_restored)s
-            printf %%b "Subject: [Fail2Ban] <name>: banned <ip> from `uname -n`
+actionban = printf %%b "Subject: [Fail2Ban] <name>: banned <ip> from `uname -n`
             Date: `LC_ALL=C date +"%%a, %%d %%h %%Y %%T %%z"`
             From: <sendername> <<sender>>
             To: <dest>\n
diff --git a/config/action.d/sendmail-whois-ipmatches.conf b/config/action.d/sendmail-whois-ipmatches.conf
index b6de1a8d..4a8edcb7 100644
--- a/config/action.d/sendmail-whois-ipmatches.conf
+++ b/config/action.d/sendmail-whois-ipmatches.conf
@@ -7,18 +7,19 @@
 [INCLUDES]
 
 before = sendmail-common.conf
-         helpers-common.conf
 
 [Definition]
 
+# bypass ban/unban for restored tickets
+norestored = 1
+
 # Option:  actionban
 # Notes.:  command executed when banning an IP. Take care that the
 #          command is executed with Fail2Ban user rights.
 # Tags:    See jail.conf(5) man page
 # Values:  CMD
 #
-actionban = %(_bypass_if_restored)s
-            printf %%b "Subject: [Fail2Ban] <name>: banned <ip> from `uname -n`
+actionban = printf %%b "Subject: [Fail2Ban] <name>: banned <ip> from `uname -n`
             Date: `LC_ALL=C date +"%%a, %%d %%h %%Y %%T %%z"`
             From: <sendername> <<sender>>
             To: <dest>\n
diff --git a/config/action.d/sendmail-whois-lines.conf b/config/action.d/sendmail-whois-lines.conf
index 77972acc..e3a1c974 100644
--- a/config/action.d/sendmail-whois-lines.conf
+++ b/config/action.d/sendmail-whois-lines.conf
@@ -11,14 +11,16 @@ before = sendmail-common.conf
 
 [Definition]
 
+# bypass ban/unban for restored tickets
+norestored = 1
+
 # Option:  actionban
 # Notes.:  command executed when banning an IP. Take care that the
 #          command is executed with Fail2Ban user rights.
 # Tags:    See jail.conf(5) man page
 # Values:  CMD
 #
-actionban = %(_bypass_if_restored)s
-            ( printf %%b "Subject: [Fail2Ban] <name>: banned <ip> from `uname -n`
+actionban = ( printf %%b "Subject: [Fail2Ban] <name>: banned <ip> from `uname -n`
             Date: `LC_ALL=C date +"%%a, %%d %%h %%Y %%T %%z"`
             From: <sendername> <<sender>>
             To: <dest>\n
diff --git a/config/action.d/sendmail-whois-matches.conf b/config/action.d/sendmail-whois-matches.conf
index 7122d923..fc4ba061 100644
--- a/config/action.d/sendmail-whois-matches.conf
+++ b/config/action.d/sendmail-whois-matches.conf
@@ -7,18 +7,19 @@
 [INCLUDES]
 
 before = sendmail-common.conf
-         helpers-common.conf
 
 [Definition]
 
+# bypass ban/unban for restored tickets
+norestored = 1
+
 # Option:  actionban
 # Notes.:  command executed when banning an IP. Take care that the
 #          command is executed with Fail2Ban user rights.
 # Tags:    See jail.conf(5) man page
 # Values:  CMD
 #
-actionban = %(_bypass_if_restored)s
-            printf %%b "Subject: [Fail2Ban] <name>: banned <ip> from `uname -n`
+actionban = printf %%b "Subject: [Fail2Ban] <name>: banned <ip> from `uname -n`
             Date: `LC_ALL=C date +"%%a, %%d %%h %%Y %%T %%z"`
             From: <sendername> <<sender>>
             To: <dest>\n
diff --git a/config/action.d/sendmail-whois.conf b/config/action.d/sendmail-whois.conf
index 82bc202e..b8d99423 100644
--- a/config/action.d/sendmail-whois.conf
+++ b/config/action.d/sendmail-whois.conf
@@ -7,18 +7,19 @@
 [INCLUDES]
 
 before = sendmail-common.conf
-         helpers-common.conf
 
 [Definition]
 
+# bypass ban/unban for restored tickets
+norestored = 1
+
 # Option:  actionban
 # Notes.:  command executed when banning an IP. Take care that the
 #          command is executed with Fail2Ban user rights.
 # Tags:    See jail.conf(5) man page
 # Values:  CMD
 #
-actionban = %(_bypass_if_restored)s
-            printf %%b "Subject: [Fail2Ban] <name>: banned <ip> from `uname -n`
+actionban = printf %%b "Subject: [Fail2Ban] <name>: banned <ip> from `uname -n`
             Date: `LC_ALL=C date +"%%a, %%d %%h %%Y %%T %%z"`
             From: <sendername> <<sender>>
             To: <dest>\n
diff --git a/config/action.d/sendmail.conf b/config/action.d/sendmail.conf
index 15f49766..62c94439 100644
--- a/config/action.d/sendmail.conf
+++ b/config/action.d/sendmail.conf
@@ -7,18 +7,19 @@
 [INCLUDES]
 
 before = sendmail-common.conf
-         helpers-common.conf
 
 [Definition]
 
+# bypass ban/unban for restored tickets
+norestored = 1
+
 # Option:  actionban
 # Notes.:  command executed when banning an IP. Take care that the
 #          command is executed with Fail2Ban user rights.
 # Tags:    See jail.conf(5) man page
 # Values:  CMD
 #
-actionban = %(_bypass_if_restored)s
-            printf %%b "Subject: [Fail2Ban] <name>: banned <ip> from `uname -n`
+actionban = printf %%b "Subject: [Fail2Ban] <name>: banned <ip> from `uname -n`
             Date: `LC_ALL=C date +"%%a, %%d %%h %%Y %%T %%z"`
             From: <sendername> <<sender>>
             To: <dest>\n
diff --git a/config/action.d/smtp.py b/config/action.d/smtp.py
index 65d50011..aa85b259 100644
--- a/config/action.d/smtp.py
+++ b/config/action.d/smtp.py
@@ -126,6 +126,9 @@ class SMTPAction(ActionBase):
 			bantime = self._jail.actions.getBanTime,
 			)
 
+		# bypass ban/unban for restored tickets
+		self.norestored = 1
+
 	def _sendMessage(self, subject, text):
 		"""Sends message based on arguments and instance's properties.
 
diff --git a/config/action.d/xarf-login-attack.conf b/config/action.d/xarf-login-attack.conf
index 13a5b70c..5274cdaf 100644
--- a/config/action.d/xarf-login-attack.conf
+++ b/config/action.d/xarf-login-attack.conf
@@ -30,20 +30,18 @@
 #
 #
 
-[INCLUDES]
-
-before = helpers-common.conf
-
 [Definition]
 
+# bypass ban/unban for restored tickets
+norestored = 1
+
 actionstart =
 
 actionstop =
 
 actioncheck =
 
-actionban = %(_bypass_if_restored)s
-            oifs=${IFS}; IFS=.;SEP_IP=( <ip> ); set -- ${SEP_IP}; ADDRESSES=$(dig +short -t txt -q $4.$3.$2.$1.abuse-contacts.abusix.org); IFS=${oifs}
+actionban = oifs=${IFS}; IFS=.;SEP_IP=( <ip> ); set -- ${SEP_IP}; ADDRESSES=$(dig +short -t txt -q $4.$3.$2.$1.abuse-contacts.abusix.org); IFS=${oifs}
             IP=<ip>
             FROM=<sender>
             SERVICE=<service>

From f35da076df5d0c02947f5e8cf5f6092925f6e55f Mon Sep 17 00:00:00 2001
From: sebres <serg.brester@sebres.de>
Date: Mon, 16 Jan 2017 09:55:01 +0100
Subject: [PATCH 7/7] ChangeLog entry

---
 ChangeLog | 11 +++++++----
 1 file changed, 7 insertions(+), 4 deletions(-)

diff --git a/ChangeLog b/ChangeLog
index 8e7b5522..37d74955 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -166,11 +166,14 @@ fail2ban-client set loglevel INFO
     (DateDetector.matchTime calls rarer DateTemplate.matchDate now);
   - several standard filters extended with exact prefixed or anchored date templates;
 * Added possibility to recognize restored state of the tickets (see gh-1669).
-  For the shell-based actions it is enough to add following script-piece at begin 
-  of `actionban` (or `actionunban`), to prevent execution for the restored 
-  (after restart) tickets.
+  New option `norestored` introduced, to ignore restored tickets (after restart).
+  To avoid execution of ban/unban for the restored tickets, `norestored = true`
+  could be added in definition section of action.
+  For conditional usage in the shell-based actions an interpolation `<restored>` 
+  could be used also. E. g. it is enough to add following script-piece at begin
+  of `actionban` (or `actionunban`) to prevent execution:
   `if [ '<restored>' = '1' ]; then exit 0; fi;`
-  Several actions extended now:
+  Several actions extended now using `norestored` option:
   - complain.conf
   - dshield.conf
   - mail-buffered.conf