Merge branch 'badips-blacklist' into 0.9

Conflicts:
	ChangeLog
        - entires added in both branches.

Change:
        config/action.d/badips.py
        - jail.getName() changed to jail.name

ChangeLog

@@ -72,6 +72,8 @@ configuration before relying on it.
    * Custom date formats (strptime) can now be set in filters and jail.conf
    * Python based actions can now be created.
      - SMTP action for sending emails on jail start, stop and ban.
+   * Added action to use badips.com reporting and blacklist
+     - Requires Python 2.7+
 
 - Enhancements
    * Jail names increased to 26 characters and iptables prefix reduced

config/action.d/badips.py

@@ -0,0 +1,348 @@
+# emacs: -*- mode: python; py-indent-offset: 4; indent-tabs-mode: t -*-
+# vi: set ft=python sts=4 ts=4 sw=4 noet :
+
+# This file is part of Fail2Ban.
+#
+# Fail2Ban is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# Fail2Ban is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with Fail2Ban; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
+
+import sys
+if sys.version_info < (2, 7):
+	raise ImportError("badips.py action requires Python >= 2.7")
+import json
+from functools import partial
+import threading
+import logging
+if sys.version_info >= (3, ):
+	from urllib.request import Request, urlopen
+	from urllib.parse import urlencode
+	from urllib.error import HTTPError
+else:
+	from urllib2 import Request, urlopen, HTTPError
+	from urllib import urlencode
+
+from fail2ban.server.actions import ActionBase
+from fail2ban.version import version as f2bVersion
+
+class BadIPsAction(ActionBase):
+	"""Fail2Ban action which resports bans to badips.com, and also
+	blacklist bad IPs listed on badips.com by using another action's
+	ban method.
+
+	Parameters
+	----------
+	jail : Jail
+		The jail which the action belongs to.
+	name : str
+		Name assigned to the action.
+	category : str
+		Valid badips.com category for reporting failures.
+	score : int, optional
+		Minimum score for bad IPs. Default 5.
+	age : str, optional
+		Age of last report for bad IPs, per badips.com syntax.
+		Default "24h" (24 hours)
+	banaction : str, optional
+		Name of banaction to use for blacklisting bad IPs. If `None`,
+		no blacklist of IPs will take place.
+		Default `None`.
+	bancategory : str, optional
+		Name of category to use for blacklisting, which can differ
+		from category used for reporting. e.g. may want to report
+		"postfix", but want to use whole "mail" category for blacklist.
+		Default `category`.
+	updateperiod : int, optional
+		Time in seconds between updating bad IPs blacklist.
+		Default 900 (15 minutes)
+
+	Raises
+	------
+	ValueError
+		If invalid `category`, `score`, `banaction` or `updateperiod`.
+	"""
+
+	_badips = "http://www.badips.com"
+	_Request = partial(
+		Request, headers={'User-Agent': "Fail2Ban %s" % f2bVersion})
+
+	def __init__(self, jail, name, category, score=5, age="24h",
+		banaction=None, bancategory=None, updateperiod=900):
+		super(BadIPsAction, self).__init__(jail, name)
+
+		self.category = category
+		self.score = score
+		self.age = age
+		self.banaction = banaction
+		self.bancategory = bancategory or category
+		self.updateperiod = updateperiod
+
+		self._bannedips = set()
+		# Used later for threading.Timer for updating badips
+		self._timer = None
+
+	@classmethod
+	def getCategories(cls, incParents=False):
+		"""Get badips.com categories.
+
+		Returns
+		-------
+		set
+			Set of categories.
+
+		Raises
+		------
+		HTTPError
+			Any issues with badips.com request.
+		"""
+		try:
+			response = urlopen(
+				cls._Request("/".join([cls._badips, "get", "categories"])))
+		except HTTPError as response:
+			messages = json.loads(response.read().decode('utf-8'))
+			self._logSys.error(
+				"Failed to fetch categories. badips.com response: '%s'",
+				messages['err'])
+			raise
+		else:
+			categories = json.loads(response.read().decode('utf-8'))['categories']
+			categories_names = set(
+				value['Name'] for value in categories)
+			if incParents:
+				categories_names.update(set(
+					value['Parent'] for value in categories
+					if "Parent" in value))
+			return categories_names
+
+	@classmethod
+	def getList(cls, category, score, age):
+		"""Get badips.com list of bad IPs.
+
+		Parameters
+		----------
+		category : str
+			Valid badips.com category.
+		score : int
+			Minimum score for bad IPs.
+		age : str
+			Age of last report for bad IPs, per badips.com syntax.
+
+		Returns
+		-------
+		set
+			Set of bad IPs.
+
+		Raises
+		------
+		HTTPError
+			Any issues with badips.com request.
+		"""
+		try:
+			response = urlopen(cls._Request("?".join([
+				"/".join([cls._badips, "get", "list", category, str(score)]),
+				urlencode({'age': age})])))
+		except HTTPError as response:
+			messages = json.loads(response.read().decode('utf-8'))
+			self._logSys.error(
+				"Failed to fetch bad IP list. badips.com response: '%s'",
+				messages['err'])
+			raise
+		else:
+			return set(response.read().decode('utf-8').split())
+
+	@property
+	def category(self):
+		"""badips.com category for reporting IPs.
+		"""
+		return self._category
+
+	@category.setter
+	def category(self, category):
+		if category not in self.getCategories():
+			self._logSys.error("Category name '%s' not valid. "
+				"see badips.com for list of valid categories",
+				category)
+			raise ValueError("Invalid category: %s" % category)
+		self._category = category
+
+	@property
+	def bancategory(self):
+		"""badips.com bancategory for fetching IPs.
+		"""
+		return self._bancategory
+
+	@bancategory.setter
+	def bancategory(self, bancategory):
+		if bancategory not in self.getCategories(incParents=True):
+			self._logSys.error("Category name '%s' not valid. "
+				"see badips.com for list of valid categories",
+				bancategory)
+			raise ValueError("Invalid bancategory: %s" % bancategory)
+		self._bancategory = bancategory
+
+	@property
+	def score(self):
+		"""badips.com minimum score for fetching IPs.
+		"""
+		return self._score
+
+	@score.setter
+	def score(self, score):
+		score = int(score)
+		if 0 <= score <= 5:
+			self._score = score
+		else:
+			raise ValueError("Score must be 0-5")
+
+	@property
+	def banaction(self):
+		"""Jail action to use for banning/unbanning.
+		"""
+		return self._banaction
+
+	@banaction.setter
+	def banaction(self, banaction):
+		if banaction is not None and banaction not in self._jail.actions:
+			self._logSys.error("Action name '%s' not in jail '%s'",
+				banaction, self._jail.name)
+			raise ValueError("Invalid banaction")
+		self._banaction = banaction
+
+	@property
+	def updateperiod(self):
+		"""Period in seconds between banned bad IPs will be updated.
+		"""
+		return self._updateperiod
+
+	@updateperiod.setter
+	def updateperiod(self, updateperiod):
+		updateperiod = int(updateperiod)
+		if updateperiod > 0:
+			self._updateperiod = updateperiod
+		else:
+			raise ValueError("Update period must be integer greater than 0")
+
+	def _banIPs(self, ips):
+		for ip in ips:
+			try:
+				self._jail.actions[self.banaction].ban({
+					'ip': ip,
+					'failures': 0,
+					'matches': "",
+					'ipmatches': "",
+					'ipjailmatches': "",
+				})
+			except Exception as e:
+				self._logSys.error(
+					"Error banning IP %s for jail '%s' with action '%s': %s",
+					ip, self._jail.name, self.banaction, e,
+					exc_info=self._logSys.getEffectiveLevel<=logging.DEBUG)
+			else:
+				self._bannedips.add(ip)
+				self._logSys.info(
+					"Banned IP %s for jail '%s' with action '%s'",
+					ip, self._jail.name, self.banaction)
+
+	def _unbanIPs(self, ips):
+		for ip in ips:
+			try:
+				self._jail.actions[self.banaction].unban({
+					'ip': ip,
+					'failures': 0,
+					'matches': "",
+					'ipmatches': "",
+					'ipjailmatches': "",
+				})
+			except Exception as e:
+				self._logSys.info(
+					"Error unbanning IP %s for jail '%s' with action '%s': %s",
+					ip, self._jail.name, self.banaction, e,
+					exc_info=self._logSys.getEffectiveLevel<=logging.DEBUG)
+			else:
+				self._logSys.info(
+					"Unbanned IP %s for jail '%s' with action '%s'",
+					ip, self._jail.name, self.banaction)
+			finally:
+				self._bannedips.remove(ip)
+
+	def start(self):
+		"""If `banaction` set, blacklists bad IPs.
+		"""
+		if self.banaction is not None:
+			self.update()
+
+	def update(self):
+		"""If `banaction` set, updates blacklisted IPs.
+
+		Queries badips.com for list of bad IPs, removing IPs from the
+		blacklist if no longer present, and adds new bad IPs to the
+		blacklist.
+		"""
+		if self.banaction is not None:
+			if self._timer:
+				self._timer.cancel()
+				self._timer = None
+
+			try:
+				ips = self.getList(self.bancategory, self.score, self.age)
+				# Remove old IPs no longer listed
+				self._unbanIPs(self._bannedips - ips)
+				# Add new IPs which are now listed
+				self._banIPs(ips - self._bannedips)
+
+				self._logSys.info(
+					"Updated IPs for jail '%s'. Update again in %i seconds",
+					self._jail.name, self.updateperiod)
+			finally:
+				self._timer = threading.Timer(self.updateperiod, self.update)
+				self._timer.start()
+
+	def stop(self):
+		"""If `banaction` set, clears blacklisted IPs.
+		"""
+		if self.banaction is not None:
+			if self._timer:
+				self._timer.cancel()
+				self._timer = None
+			self._unbanIPs(self._bannedips.copy())
+
+	def ban(self, aInfo):
+		"""Reports banned IP to badips.com.
+
+		Parameters
+		----------
+		aInfo : dict
+			Dictionary which includes information in relation to
+			the ban.
+
+		Raises
+		------
+		HTTPError
+			Any issues with badips.com request.
+		"""
+		try:
+			response = urlopen(self._Request(
+			"/".join([self._badips, "add", self.category, aInfo['ip']])))
+		except HTTPError as response:
+			messages = json.loads(response.read().decode('utf-8'))
+			self._logSys.error(
+				"Response from badips.com report: '%s'",
+				messages['err'])
+			raise
+		else:
+			messages = json.loads(response.read().decode('utf-8'))
+			self._logSys.info(
+				"Response from badips.com report: '%s'",
+				messages['suc'])
+
+Action = BadIPsAction

config/jail.conf

@@ -180,6 +180,12 @@ action_xarf = %(banaction)s[name=%(__name__)s, port="%(port)s", protocol="%(prot
 #
 action_blocklist_de  = blocklist_de[email="%(sender)s", service=%(filter)s, apikey="%(blocklist_de_apikey)s"]
 
+# Report ban via badips.com, and use as blacklist
+#
+# See BadIPsAction docstring in config/action.d/badips.py for
+# documentation for this action.
+#
+action_badips = badips.py[category="%(name)s", banaction="%(banaction)s"]
 
 # Choose default action.  To change, just override value of 'action' with the
 # interpolation to the chosen action shortcut (e.g.  action_mw, action_mwl, etc) in jail.local

fail2ban/server/actions.py

@@ -32,6 +32,10 @@ if sys.version_info >= (3, 3):
 else:
 	import imp
 from collections import Mapping
+try:
+	from collections import OrderedDict
+except ImportError:
+	OrderedDict = None
 
 from .banmanager import BanManager
 from .jailthread import JailThread
@@ -73,7 +77,10 @@ class Actions(JailThread, Mapping):
 		JailThread.__init__(self)
 		## The jail which contains this action.
 		self._jail = jail
-		self._actions = dict()
+		if OrderedDict is not None:
+			self._actions = OrderedDict()
+		else:
+			self._actions = dict()
 		## The ban manager.
 		self.__banManager = BanManager()
 
@@ -219,7 +226,10 @@ class Actions(JailThread, Mapping):
 			else:
 				time.sleep(self.sleeptime)
 		self.__flushBan()
-		for name, action in self._actions.iteritems():
+
+		actions = self._actions.items()
+		actions.reverse()
+		for name, action in actions:
 			try:
 				action.stop()
 			except Exception as e:

fail2ban/tests/action_d/test_badips.py

@@ -0,0 +1,98 @@
+# emacs: -*- mode: python; py-indent-offset: 4; indent-tabs-mode: t -*-
+# vi: set ft=python sts=4 ts=4 sw=4 noet :
+
+# This file is part of Fail2Ban.
+#
+# Fail2Ban is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# Fail2Ban is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with Fail2Ban; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
+
+import os
+import unittest
+import sys
+if sys.version_info >= (3, 3):
+	import importlib
+else:
+	import imp
+
+from ..dummyjail import DummyJail
+
+if os.path.exists('config/fail2ban.conf'):
+	CONFIG_DIR = "config"
+else:
+	CONFIG_DIR='/etc/fail2ban'
+
+if sys.version_info >= (2,7):
+	class BadIPsActionTest(unittest.TestCase):
+
+		def setUp(self):
+			"""Call before every test case."""
+			self.jail = DummyJail()
+
+			self.jail.actions.add("test")
+
+			pythonModule = os.path.join(CONFIG_DIR, "action.d", "badips.py")
+			self.jail.actions.add("badips", pythonModule, initOpts={
+				'category': "ssh",
+				'banaction': "test",
+				})
+			self.action = self.jail.actions["badips"]
+
+		def tearDown(self):
+			"""Call after every test case."""
+			# Must cancel timer!
+			if self.action._timer:
+				self.action._timer.cancel()
+
+		def testCategory(self):
+			categories = self.action.getCategories()
+			self.assertTrue("ssh" in categories)
+			self.assertTrue(len(categories) >= 10)
+
+			self.assertRaises(
+				ValueError, setattr, self.action, "category",
+				"invalid-category")
+
+			# Not valid for reporting category...
+			self.assertRaises(
+				ValueError, setattr, self.action, "category", "mail")
+			# but valid for blacklisting.
+			self.action.bancategory = "mail"
+
+		def testScore(self):
+			self.assertRaises(ValueError, setattr, self.action, "score", -5)
+			self.action.score = 5
+			self.action.score = "5"
+
+		def testBanaction(self):
+			self.assertRaises(
+				ValueError, setattr, self.action, "banaction",
+				"invalid-action")
+			self.action.banaction = "test"
+
+		def testUpdateperiod(self):
+			self.assertRaises(
+				ValueError, setattr, self.action, "updateperiod", -50)
+			self.assertRaises(
+				ValueError, setattr, self.action, "updateperiod", 0)
+			self.action.updateperiod = 900
+			self.action.updateperiod = "900"
+
+		def testStart(self):
+			self.action.start()
+			self.assertTrue(len(self.action._bannedips) > 10)
+
+		def testStop(self):
+			self.testStart()
+			self.action.stop()
+			self.assertTrue(len(self.action._bannedips) == 0)