Merge pull request #924 from leeclemens/ENH/StatusExtendedInfo

Add extended info to status output using Cymru

.travis.yml

@@ -13,6 +13,8 @@ before_install:
   - if [[ $TRAVIS_PYTHON_VERSION == 2.7 ]]; then travis_retry sudo apt-get update -qq; fi
 install:
   - travis_retry pip install pyinotify
+  - if [[ $TRAVIS_PYTHON_VERSION == 2* || $TRAVIS_PYTHON_VERSION == 'pypy' ]]; then travis_retry pip install dnspython; fi
+  - if [[ $TRAVIS_PYTHON_VERSION == 3* || $TRAVIS_PYTHON_VERSION == 'pypy3' ]]; then travis_retry pip install dnspython3; fi
   - if [[ $TRAVIS_PYTHON_VERSION == 2.7 ]]; then travis_retry sudo apt-get install -qq python-gamin; cp /usr/share/pyshared/gamin.py /usr/lib/pyshared/python2.7/_gamin.so $VIRTUAL_ENV/lib/python2.7/site-packages/; fi
   - if [[ $TRAVIS_PYTHON_VERSION == 2.7 ]]; then cd ..; travis_retry pip install -q coveralls; cd -; fi
 script:

ChangeLog

@@ -44,7 +44,12 @@ ver. 0.9.2 (2014/XX/XXX) - wanna-be-released
    - Monit config for fail2ban in /files/monit
    - New actions:
      - action.d/firewallcmd-multiport and action.d/firewallcmd-allports Thanks Donald Yandt
-     
+   - New status argument, flavor:
+     - fail2ban-client status <jail> [flavor]
+       - empty or "basic" works as-is
+       - "cymru" additionally prints (ASN, Country RIR) per banned IP
+       - Requires dnspython or dnspython3
+
 - Enhancements:
    * Enable multiport for firewallcmd-new action.  Closes gh-834
    * files/debian-initd migrated from the debian branch and should be

README.md

@@ -33,6 +33,7 @@ Optional:
   - Linux >= 2.6.13
 - [gamin >= 0.0.21](http://www.gnome.org/~veillard/gamin)
 - [systemd >= 204](http://www.freedesktop.org/wiki/Software/systemd)
+- [dnspython](http://www.dnspython.org/)
 
 To install, just do:
 

fail2ban/protocol.py

@@ -54,7 +54,7 @@ protocol = [
 ["add <JAIL> <BACKEND>", "creates <JAIL> using <BACKEND>"], 
 ["start <JAIL>", "starts the jail <JAIL>"], 
 ["stop <JAIL>", "stops the jail <JAIL>. The jail is removed"], 
-["status <JAIL>", "gets the current status of <JAIL>"],
+["status <JAIL> [FLAVOR]", "gets the current status of <JAIL>, with optional flavor or extended info"],
 ['', "JAIL CONFIGURATION", ""],
 ["set <JAIL> idle on|off", "sets the idle state of <JAIL>"], 
 ["set <JAIL> addignoreip <IP>", "adds <IP> to the ignore list of <JAIL>"], 

fail2ban/server/actions.py

@@ -370,11 +370,21 @@ class Actions(JailThread, Mapping):
 					self._jail.name, name, aInfo, e,
 					exc_info=logSys.getEffectiveLevel()<=logging.DEBUG)
 
-	@property
-	def status(self):
-		"""Status of active bans, and total ban counts.
+	def status(self, flavor="basic"):
+		"""Status of current and total ban counts and current banned IP list.
 		"""
-		ret = [("Currently banned", self.__banManager.size()), 
+		# TODO: Allow this list to be printed as 'status' output
+		supported_flavors = ["basic", "cymru"]
+		if flavor is None or flavor not in supported_flavors:
+			logSys.warning("Unsupported extended jail status flavor %r. Supported: %s" % (flavor, supported_flavors))
+		# Always print this information (basic)
+		ret = [("Currently banned", self.__banManager.size()),
 			   ("Total banned", self.__banManager.getBanTotal()),
 			   ("Banned IP list", self.__banManager.getBanList())]
+		if flavor == "cymru":
+			cymru_info = self.__banManager.getBanListExtendedCymruInfo()
+			ret += \
+				[("Banned ASN list", self.__banManager.geBanListExtendedASN(cymru_info)),
+				 ("Banned Country list", self.__banManager.geBanListExtendedCountry(cymru_info)),
+				 ("Banned RIR list", self.__banManager.geBanListExtendedRIR(cymru_info))]
 		return ret

fail2ban/server/banmanager.py

@@ -118,6 +118,124 @@ class BanManager:
 		finally:
 			self.__lock.release()
 
+	##
+	# Returns normalized value
+	#
+	# @return value or "unknown" if value is None or empty string
+
+	@staticmethod
+	def handleBlankResult(value):
+		if value is None or len(value) == 0:
+			return "unknown"
+		else:
+			return value
+
+	##
+	# Returns Cymru DNS query information
+	#
+	# @return {"asn": [], "country": [], "rir": []} dict for self.__banList IPs
+
+	def getBanListExtendedCymruInfo(self):
+		return_dict = {"asn": [], "country": [], "rir": []}
+		try:
+			import dns.exception
+			import dns.resolver
+		except ImportError:
+			logSys.error("dnspython package is required but could not be imported")
+			return_dict["asn"].append("error")
+			return_dict["country"].append("error")
+			return_dict["rir"].append("error")
+			return return_dict
+		self.__lock.acquire()
+		try:
+			for banData in self.__banList:
+				ip = banData.getIP()
+				# Reference: http://www.team-cymru.org/Services/ip-to-asn.html#dns
+				# TODO: IPv6 compatibility
+				reversed_ip = ".".join(reversed(ip.split(".")))
+				question = "%s.origin.asn.cymru.com" % reversed_ip
+				try:
+					answers = dns.resolver.query(question, "TXT")
+					for rdata in answers:
+						asn, net, country, rir, changed =\
+							[answer.strip("'\" ") for answer in rdata.to_text().split("|")]
+						asn = self.handleBlankResult(asn)
+						country = self.handleBlankResult(country)
+						rir = self.handleBlankResult(rir)
+						return_dict["asn"].append(self.handleBlankResult(asn))
+						return_dict["country"].append(self.handleBlankResult(country))
+						return_dict["rir"].append(self.handleBlankResult(rir))
+				except dns.resolver.NXDOMAIN:
+					return_dict["asn"].append("nxdomain")
+					return_dict["country"].append("nxdomain")
+					return_dict["rir"].append("nxdomain")
+				except dns.exception.DNSException as dnse:
+					logSys.error("Unhandled DNSException querying Cymru for %s TXT" % question)
+					logSys.exception(dnse)
+				except Exception as e:
+					logSys.error("Unhandled Exception querying Cymru for %s TXT" % question)
+					logSys.exception(e)
+		except Exception as e:
+			logSys.error("Failure looking up extended Cymru info")
+			logSys.exception(e)
+		finally:
+			self.__lock.release()
+		return return_dict
+
+	##
+	# Returns list of Banned ASNs from Cymru info
+	#
+	# Use getBanListExtendedCymruInfo() to provide cymru_info
+	#
+	# @return list of Banned ASNs
+
+	def geBanListExtendedASN(self, cymru_info):
+		self.__lock.acquire()
+		try:
+			return [asn for asn in cymru_info["asn"]]
+		except Exception as e:
+			logSys.error("Failed to lookup ASN")
+			logSys.exception(e)
+			return []
+		finally:
+			self.__lock.release()
+
+	##
+	# Returns list of Banned Countries from Cymru info
+	#
+	# Use getBanListExtendedCymruInfo() to provide cymru_info
+	#
+	# @return list of Banned Countries
+
+	def geBanListExtendedCountry(self, cymru_info):
+		self.__lock.acquire()
+		try:
+			return [country for country in cymru_info["country"]]
+		except Exception as e:
+			logSys.error("Failed to lookup Country")
+			logSys.exception(e)
+			return []
+		finally:
+			self.__lock.release()
+
+	##
+	# Returns list of Banned RIRs from Cymru info
+	#
+	# Use getBanListExtendedCymruInfo() to provide cymru_info
+	#
+	# @return list of Banned RIRs
+
+	def geBanListExtendedRIR(self, cymru_info):
+		self.__lock.acquire()
+		try:
+			return [rir for rir in cymru_info["rir"]]
+		except Exception as e:
+			logSys.error("Failed to lookup RIR")
+			logSys.exception(e)
+			return []
+		finally:
+			self.__lock.release()
+
 	##
 	# Create a ban ticket.
 	#

fail2ban/server/filter.py

@@ -529,8 +529,7 @@ class Filter(JailThread):
 						logSys.error(e)
 		return failList
 
-	@property
-	def status(self):
+	def status(self, flavor="basic"):
 		"""Status of failures detected by filter.
 		"""
 		ret = [("Currently failed", self.failManager.size()),
@@ -686,11 +685,10 @@ class FileFilter(Filter):
 			db.updateLog(self.jail, container)
 		return True
 
-	@property
-	def status(self):
+	def status(self, flavor="basic"):
 		"""Status of Filter plus files being monitored.
 		"""
-		ret = super(FileFilter, self).status
+		ret = super(FileFilter, self).status(flavor=flavor)
 		path = [m.getFileName() for m in self.getLogPath()]
 		ret.append(("File list", path))
 		return ret

fail2ban/server/filtersystemd.py

@@ -259,9 +259,8 @@ class FilterSystemd(JournalFilter): # pragma: systemd no cover
                       or "jailless") +" filter terminated")
 		return True
 
-	@property
-	def status(self):
-		ret = super(FilterSystemd, self).status
+	def status(self, flavor="basic"):
+		ret = super(FilterSystemd, self).status(flavor=flavor)
 		ret.append(("Journal matches",
 			[" + ".join(" ".join(match) for match in self.__matches)]))
 		return ret

fail2ban/server/jail.py

@@ -174,13 +174,12 @@ class Jail:
 		self.filter.idle = value
 		self.actions.idle = value
 
-	@property
-	def status(self):
+	def status(self, flavor="basic"):
 		"""The status of the jail.
 		"""
 		return [
-			("Filter", self.filter.status),
-			("Actions", self.actions.status),
+			("Filter", self.filter.status(flavor=flavor)),
+			("Actions", self.actions.status(flavor=flavor)),
 			]
 
 	def putFailTicket(self, ticket):

fail2ban/server/jailthread.py

@@ -26,7 +26,7 @@ __license__ = "GPL"
 
 import sys
 from threading import Thread
-from abc import abstractproperty, abstractmethod
+from abc import abstractmethod
 
 from ..helpers import excepthook
 
@@ -66,8 +66,8 @@ class JailThread(Thread):
 				excepthook(*sys.exc_info())
 		self.run = run_with_except_hook
 
-	@abstractproperty
-	def status(self): # pragma: no cover - abstract
+	@abstractmethod
+	def status(self, flavor="basic"): # pragma: no cover - abstract
 		"""Abstract - Should provide status information.
 		"""
 		pass

fail2ban/server/server.py

@@ -320,9 +320,9 @@ class Server:
 		finally:
 			self.__lock.release()
 	
-	def statusJail(self, name):
-		return self.__jails[name].status
-	
+	def statusJail(self, name, flavor="basic"):
+		return self.__jails[name].status(flavor=flavor)
+
 	# Logging
 	
 	##

fail2ban/server/transmitter.py

@@ -333,5 +333,8 @@ class Transmitter:
 		elif len(command) == 1:
 			name = command[0]
 			return self.__server.statusJail(name)
+		elif len(command) == 2:
+			name = command[0]
+			flavor = command[1]
+			return self.__server.statusJail(name, flavor=flavor)
 		raise Exception("Invalid command (no status)")
-	

fail2ban/tests/actionstestcase.py

@@ -86,7 +86,7 @@ class ExecuteActions(LogCaptureTestCase):
 
 		self.__actions.stop()
 		self.__actions.join()
-		self.assertEqual(self.__actions.status,[("Currently banned", 0 ),
+		self.assertEqual(self.__actions.status(),[("Currently banned", 0 ),
                ("Total banned", 0 ), ("Banned IP list", [] )])
 
 

fail2ban/tests/banmanagertestcase.py

@@ -30,7 +30,6 @@ from ..server.banmanager import BanManager
 from ..server.ticket import BanTicket
 
 class AddFailure(unittest.TestCase):
-
 	def setUp(self):
 		"""Call before every test case."""
 		self.__ticket = BanTicket('193.168.0.128', 1167605999.0)
@@ -39,19 +38,77 @@ class AddFailure(unittest.TestCase):
 
 	def tearDown(self):
 		"""Call after every test case."""
-	
+		pass
+
 	def testAdd(self):
 		self.assertEqual(self.__banManager.size(), 1)
-	
+
 	def testAddDuplicate(self):
 		self.assertFalse(self.__banManager.addBanTicket(self.__ticket))
 		self.assertEqual(self.__banManager.size(), 1)
-		
+
 	def testInListOK(self):
 		ticket = BanTicket('193.168.0.128', 1167605999.0)
 		self.assertTrue(self.__banManager._inBanList(ticket))
-	
+
 	def testInListNOK(self):
 		ticket = BanTicket('111.111.1.111', 1167605999.0)
 		self.assertFalse(self.__banManager._inBanList(ticket))
-		
+
+
+class StatusExtendedCymruInfo(unittest.TestCase):
+	def setUp(self):
+		"""Call before every test case."""
+		self.__ban_ip = "93.184.216.34"
+		self.__asn = "15133"
+		self.__country = "EU"
+		self.__rir = "ripencc"
+		ticket = BanTicket(self.__ban_ip, 1167605999.0)
+		self.__banManager = BanManager()
+		self.assertTrue(self.__banManager.addBanTicket(ticket))
+
+	def tearDown(self):
+		"""Call after every test case."""
+		pass
+
+	def testCymruInfo(self):
+		cymru_info = self.__banManager.getBanListExtendedCymruInfo()
+		if "assertDictEqual" in dir(self):
+			self.assertDictEqual(cymru_info, {"asn": [self.__asn],
+											  "country": [self.__country],
+											  "rir": [self.__rir]})
+		else:
+			# Python 2.6 does not support assertDictEqual()
+			self.assertEqual(cymru_info["asn"], [self.__asn])
+			self.assertEqual(cymru_info["country"], [self.__country])
+			self.assertEqual(cymru_info["rir"], [self.__rir])
+
+	def testCymruInfoASN(self):
+		self.assertEqual(
+			self.__banManager.geBanListExtendedASN(self.__banManager.getBanListExtendedCymruInfo()),
+			[self.__asn])
+
+	def testCymruInfoCountry(self):
+		self.assertEqual(
+			self.__banManager.geBanListExtendedCountry(self.__banManager.getBanListExtendedCymruInfo()),
+			[self.__country])
+
+	def testCymruInfoRIR(self):
+		self.assertEqual(
+			self.__banManager.geBanListExtendedRIR(self.__banManager.getBanListExtendedCymruInfo()),
+			[self.__rir])
+
+	def testCymruInfoNxdomain(self):
+		ticket = BanTicket("10.0.0.0", 1167605999.0)
+		self.__banManager = BanManager()
+		self.assertTrue(self.__banManager.addBanTicket(ticket))
+		cymru_info = self.__banManager.getBanListExtendedCymruInfo()
+		if "assertDictEqual" in dir(self):
+			self.assertDictEqual(cymru_info, {"asn": ["nxdomain"],
+											  "country": ["nxdomain"],
+											  "rir": ["nxdomain"]})
+		else:
+			# Python 2.6 does not support assertDictEqual()
+			self.assertEqual(cymru_info["asn"], ["nxdomain"])
+			self.assertEqual(cymru_info["country"], ["nxdomain"])
+			self.assertEqual(cymru_info["rir"], ["nxdomain"])

fail2ban/tests/servertestcase.py

@@ -474,6 +474,64 @@ class Transmitter(TransmitterBase):
 			)
 		)
 
+	def testJailStatusBasic(self):
+		self.assertEqual(self.transm.proceed(["status", self.jailName, "basic"]),
+			(0,
+				[
+					('Filter', [
+						('Currently failed', 0),
+						('Total failed', 0),
+						('File list', [])]
+					),
+					('Actions', [
+						('Currently banned', 0),
+						('Total banned', 0),
+						('Banned IP list', [])]
+					)
+				]
+			)
+		)
+
+	def testJailStatusBasicKwarg(self):
+		self.assertEqual(self.transm.proceed(["status", self.jailName, "INVALID"]),
+			(0,
+				[
+					('Filter', [
+						('Currently failed', 0),
+						('Total failed', 0),
+						('File list', [])]
+					),
+					('Actions', [
+						('Currently banned', 0),
+						('Total banned', 0),
+						('Banned IP list', [])]
+					)
+				]
+			)
+		)
+
+	def testJailStatusCymru(self):
+		self.assertEqual(self.transm.proceed(["status", self.jailName, "cymru"]),
+			(0,
+				[
+					('Filter', [
+						('Currently failed', 0),
+						('Total failed', 0),
+						('File list', [])]
+					),
+					('Actions', [
+						('Currently banned', 0),
+						('Total banned', 0),
+						('Banned IP list', []),
+						('Banned ASN list', []),
+						('Banned Country list', []),
+						('Banned RIR list', [])]
+					)
+				]
+			)
+		)
+
+
 	def testAction(self):
 		action = "TestCaseAction"
 		cmdList = [

fail2ban/tests/utils.py

@@ -107,6 +107,11 @@ def gatherTests(regexps=None, no_network=False):
 	tests.addTest(unittest.makeSuite(failmanagertestcase.AddFailure))
 	# BanManager
 	tests.addTest(unittest.makeSuite(banmanagertestcase.AddFailure))
+	try:
+		import dns
+		tests.addTest(unittest.makeSuite(banmanagertestcase.StatusExtendedCymruInfo))
+	except ImportError:
+		pass
 	# ClientReaders
 	tests.addTest(unittest.makeSuite(clientreadertestcase.ConfigReaderTest))
 	tests.addTest(unittest.makeSuite(clientreadertestcase.JailReaderTest))