From 625bcef8a1be6bbcdd4d4522aafe46d917d8e472 Mon Sep 17 00:00:00 2001 From: Lee Clemens Date: Fri, 23 Jan 2015 15:12:04 -0500 Subject: [PATCH 1/8] Add extended info to status output using Cyrmu --- .travis.yml | 2 + ChangeLog | 6 +- fail2ban/protocol.py | 1 + fail2ban/server/actions.py | 13 +++- fail2ban/server/banmanager.py | 112 +++++++++++++++++++++++++++ fail2ban/server/jail.py | 9 +++ fail2ban/server/jailthread.py | 6 ++ fail2ban/server/server.py | 5 +- fail2ban/server/transmitter.py | 7 +- fail2ban/tests/banmanagertestcase.py | 50 ++++++++++-- fail2ban/tests/servertestcase.py | 25 ++++++ fail2ban/tests/utils.py | 1 + 12 files changed, 227 insertions(+), 10 deletions(-) diff --git a/.travis.yml b/.travis.yml index bd2d294c..ce14e0ee 100644 --- a/.travis.yml +++ b/.travis.yml @@ -12,6 +12,8 @@ before_install: - if [[ $TRAVIS_PYTHON_VERSION == 2.7 ]]; then sudo apt-get update -qq; fi install: - pip install pyinotify + - if [[ $TRAVIS_PYTHON_VERSION == 2* || $TRAVIS_PYTHON_VERSION == 'pypy' ]]; then travis_retry pip install dnspython; fi + - if [[ $TRAVIS_PYTHON_VERSION == 3* || $TRAVIS_PYTHON_VERSION == 'pypy3' ]]; then travis_retry pip install dnspython3; fi - if [[ $TRAVIS_PYTHON_VERSION == 2.7 ]]; then sudo apt-get install -qq python-gamin; cp /usr/share/pyshared/gamin.py /usr/lib/pyshared/python2.7/_gamin.so $VIRTUAL_ENV/lib/python2.7/site-packages/; fi - if [[ $TRAVIS_PYTHON_VERSION == 2.7 ]]; then cd ..; pip install -q coveralls; cd -; fi script: diff --git a/ChangeLog b/ChangeLog index f01ed930..3013dac8 100644 --- a/ChangeLog +++ b/ChangeLog @@ -41,7 +41,11 @@ ver. 0.9.2 (2014/XX/XXX) - wanna-be-released - Monit config for fail2ban in /files/monit - New actions: - action.d/firewallcmd-multiport and action.d/firewallcmd-allports Thanks Donald Yandt - + - New status commands: + - fail2ban-client status extended + - prints Cymru data (ASN, Country RIR) per banned IP + - Requires dnspython or dnspython3 + - Enhancements: * Enable multiport for firewallcmd-new action. Closes gh-834 * files/debian-initd migrated from the debian branch and should be diff --git a/fail2ban/protocol.py b/fail2ban/protocol.py index 9a6ee675..dc7a7c6b 100644 --- a/fail2ban/protocol.py +++ b/fail2ban/protocol.py @@ -55,6 +55,7 @@ protocol = [ ["start ", "starts the jail "], ["stop ", "stops the jail . The jail is removed"], ["status ", "gets the current status of "], +["status extended", "gets the current status of with extended info"], ['', "JAIL CONFIGURATION", ""], ["set idle on|off", "sets the idle state of "], ["set addignoreip ", "adds to the ignore list of "], diff --git a/fail2ban/server/actions.py b/fail2ban/server/actions.py index a212aaf1..f4d5c18b 100644 --- a/fail2ban/server/actions.py +++ b/fail2ban/server/actions.py @@ -372,9 +372,20 @@ class Actions(JailThread, Mapping): @property def status(self): - """Status of active bans, and total ban counts. + """Status of current and total ban counts and current banned IP list. """ ret = [("Currently banned", self.__banManager.size()), ("Total banned", self.__banManager.getBanTotal()), ("Banned IP list", self.__banManager.getBanList())] return ret + + @property + def statusExtended(self): + """Jail status plus banned IPs' ASN, Country and RIR + """ + cymru_info = self.__banManager.getBanListExtendedCymruInfo() + ret = self.status +\ + [("Banned ASN list", self.__banManager.geBanListExtendedASN(cymru_info)), + ("Banned Country list", self.__banManager.geBanListExtendedCountry(cymru_info)), + ("Banned RIR list", self.__banManager.geBanListExtendedRIR(cymru_info))] + return ret diff --git a/fail2ban/server/banmanager.py b/fail2ban/server/banmanager.py index c21cad45..81ac7c4f 100644 --- a/fail2ban/server/banmanager.py +++ b/fail2ban/server/banmanager.py @@ -26,6 +26,9 @@ __license__ = "GPL" from threading import Lock +import dns.exception +import dns.resolver + from .ticket import BanTicket from .mytime import MyTime from ..helpers import getLogger @@ -118,6 +121,115 @@ class BanManager: finally: self.__lock.release() + ## + # Returns normalized value + # + # @return value or "unknown" if value is None or empty string + + @staticmethod + def handleBlankResult(value): + if value is None or len(value) == 0: + return "unknown" + else: + return value + + ## + # Returns Cymru DNS query information + # + # @return {"asn": [], "country": [], "rir": []} dict for self.__banList IPs + + def getBanListExtendedCymruInfo(self): + self.__lock.acquire() + return_dict = {"asn": [], "country": [], "rir": []} + try: + for banData in self.__banList: + ip = banData.getIP() + # Reference: http://www.team-cymru.org/Services/ip-to-asn.html#dns + # TODO: IPv6 compatibility + reversed_ip = ".".join(reversed(ip.split("."))) + question = "%s.origin.asn.cymru.com" % reversed_ip + try: + answers = dns.resolver.query(question, "TXT") + for rdata in answers: + asn, net, country, rir, changed =\ + [answer.strip("'\" ") for answer in rdata.to_text().split("|")] + asn = self.handleBlankResult(asn) + country = self.handleBlankResult(country) + rir = self.handleBlankResult(rir) + return_dict["asn"].append(self.handleBlankResult(asn)) + return_dict["country"].append(self.handleBlankResult(country)) + return_dict["rir"].append(self.handleBlankResult(rir)) + except dns.resolver.NXDOMAIN: + return_dict["asn"].append("nxdomain") + return_dict["country"].append("nxdomain") + return_dict["rir"].append("nxdomain") + except dns.exception.DNSException as dnse: + logSys.error("Unhandled DNSException querying Cymru for %s TXT" % question) + logSys.exception(dnse) + except Exception as e: + logSys.error("Unhandled Exception querying Cymru for %s TXT" % question) + logSys.exception(e) + except Exception as e: + logSys.error("Failure looking up extended Cymru info") + logSys.exception(e) + finally: + self.__lock.release() + return return_dict + + ## + # Returns list of Banned ASNs from Cymru info + # + # Use getBanListExtendedCymruInfo() to provide cymru_info + # + # @return list of Banned ASNs + + def geBanListExtendedASN(self, cymru_info): + self.__lock.acquire() + try: + return [asn for asn in cymru_info["asn"]] + except Exception as e: + logSys.error("Failed to lookup ASN") + logSys.exception(e) + return [] + finally: + self.__lock.release() + + ## + # Returns list of Banned Countries from Cymru info + # + # Use getBanListExtendedCymruInfo() to provide cymru_info + # + # @return list of Banned Countries + + def geBanListExtendedCountry(self, cymru_info): + self.__lock.acquire() + try: + return [country for country in cymru_info["country"]] + except Exception as e: + logSys.error("Failed to lookup Country") + logSys.exception(e) + return [] + finally: + self.__lock.release() + + ## + # Returns list of Banned RIRs from Cymru info + # + # Use getBanListExtendedCymruInfo() to provide cymru_info + # + # @return list of Banned RIRs + + def geBanListExtendedRIR(self, cymru_info): + self.__lock.acquire() + try: + return [rir for rir in cymru_info["rir"]] + except Exception as e: + logSys.error("Failed to lookup RIR") + logSys.exception(e) + return [] + finally: + self.__lock.release() + ## # Create a ban ticket. # diff --git a/fail2ban/server/jail.py b/fail2ban/server/jail.py index e48e5d7b..df28945e 100644 --- a/fail2ban/server/jail.py +++ b/fail2ban/server/jail.py @@ -183,6 +183,15 @@ class Jail: ("Actions", self.actions.status), ] + @property + def statusExtended(self): + """The extended status of the jail. + """ + return [ + ("Filter", self.filter.status), + ("Actions", self.actions.statusExtended), + ] + def putFailTicket(self, ticket): """Add a fail ticket to the jail. diff --git a/fail2ban/server/jailthread.py b/fail2ban/server/jailthread.py index 2627cebf..123a2891 100644 --- a/fail2ban/server/jailthread.py +++ b/fail2ban/server/jailthread.py @@ -72,6 +72,12 @@ class JailThread(Thread): """ pass + @abstractproperty + def statusExtended(self): # pragma: no cover - abstract + """Abstract - Should provide extended status information. + """ + pass + def start(self): """Sets active flag and starts thread. """ diff --git a/fail2ban/server/server.py b/fail2ban/server/server.py index 02f3423b..c8840608 100644 --- a/fail2ban/server/server.py +++ b/fail2ban/server/server.py @@ -322,7 +322,10 @@ class Server: def statusJail(self, name): return self.__jails[name].status - + + def statusJailExtended(self, name): + return self.__jails[name].statusExtended + # Logging ## diff --git a/fail2ban/server/transmitter.py b/fail2ban/server/transmitter.py index 39157128..05498646 100644 --- a/fail2ban/server/transmitter.py +++ b/fail2ban/server/transmitter.py @@ -333,5 +333,10 @@ class Transmitter: elif len(command) == 1: name = command[0] return self.__server.statusJail(name) + elif len(command) == 2: + name = command[0] + if command[1] == "extended": + return self.__server.statusJailExtended(name) + else: + raise Exception("Invalid command (invalid status extension)") raise Exception("Invalid command (no status)") - diff --git a/fail2ban/tests/banmanagertestcase.py b/fail2ban/tests/banmanagertestcase.py index 7dcb73a7..d4d106a9 100644 --- a/fail2ban/tests/banmanagertestcase.py +++ b/fail2ban/tests/banmanagertestcase.py @@ -30,7 +30,6 @@ from ..server.banmanager import BanManager from ..server.ticket import BanTicket class AddFailure(unittest.TestCase): - def setUp(self): """Call before every test case.""" self.__ticket = BanTicket('193.168.0.128', 1167605999.0) @@ -39,19 +38,58 @@ class AddFailure(unittest.TestCase): def tearDown(self): """Call after every test case.""" - + def testAdd(self): self.assertEqual(self.__banManager.size(), 1) - + def testAddDuplicate(self): self.assertFalse(self.__banManager.addBanTicket(self.__ticket)) self.assertEqual(self.__banManager.size(), 1) - + def testInListOK(self): ticket = BanTicket('193.168.0.128', 1167605999.0) self.assertTrue(self.__banManager._inBanList(ticket)) - + def testInListNOK(self): ticket = BanTicket('111.111.1.111', 1167605999.0) self.assertFalse(self.__banManager._inBanList(ticket)) - + + +class StatusExtendedCymruInfo(unittest.TestCase): + def setUp(self): + """Call before every test case.""" + self.__ban_ip = "93.184.216.34" + self.__asn = "15133" + self.__country = "EU" + self.__rir = "ripencc" + self.__ticket = BanTicket(self.__ban_ip, 1167605999.0) + self.__banManager = BanManager() + self.assertTrue(self.__banManager.addBanTicket(self.__ticket)) + + def tearDown(self): + """Call after every test case.""" + + def testCymruInfo(self): + cymru_info = self.__banManager.getBanListExtendedCymruInfo() + if "assertDictEqual" in dir(self): + self.assertDictEqual(cymru_info, {"asn": [self.__asn], "country": [self.__country], "rir": [self.__rir]}) + else: + # Python 2.6 does not support assertDictEqual() + self.assertEqual(cymru_info["asn"], [self.__asn]) + self.assertEqual(cymru_info["country"], [self.__country]) + self.assertEqual(cymru_info["rir"], [self.__rir]) + + def testCymruInfoASN(self): + self.assertEqual( + self.__banManager.geBanListExtendedASN(self.__banManager.getBanListExtendedCymruInfo()), + [self.__asn]) + + def testCymruInfoCountry(self): + self.assertEqual( + self.__banManager.geBanListExtendedCountry(self.__banManager.getBanListExtendedCymruInfo()), + [self.__country]) + + def testCymruInfoRIR(self): + self.assertEqual( + self.__banManager.geBanListExtendedRIR(self.__banManager.getBanListExtendedCymruInfo()), + [self.__rir]) diff --git a/fail2ban/tests/servertestcase.py b/fail2ban/tests/servertestcase.py index e018ed2f..3bfcc303 100644 --- a/fail2ban/tests/servertestcase.py +++ b/fail2ban/tests/servertestcase.py @@ -474,6 +474,27 @@ class Transmitter(TransmitterBase): ) ) + def testJailStatusExtended(self): + self.assertEqual(self.transm.proceed(["status", self.jailName, "extended"]), + (0, + [ + ('Filter', [ + ('Currently failed', 0), + ('Total failed', 0), + ('File list', [])] + ), + ('Actions', [ + ('Currently banned', 0), + ('Total banned', 0), + ('Banned IP list', []), + ('Banned ASN list', []), + ('Banned Country list', []), + ('Banned RIR list', [])] + ) + ] + ) + ) + def testAction(self): action = "TestCaseAction" cmdList = [ @@ -601,6 +622,10 @@ class Transmitter(TransmitterBase): self.assertEqual( self.transm.proceed(["status", "INVALID", "COMMAND"])[0],1) + def testStatusJailExtendedNOK(self): + self.assertEqual( + self.transm.proceed(["status", self.jailName, "INVALID_COMMAND"])[0],1) + def testJournalMatch(self): if not filtersystemd: # pragma: no cover if sys.version_info >= (2, 7): diff --git a/fail2ban/tests/utils.py b/fail2ban/tests/utils.py index b56c0988..42edf369 100644 --- a/fail2ban/tests/utils.py +++ b/fail2ban/tests/utils.py @@ -107,6 +107,7 @@ def gatherTests(regexps=None, no_network=False): tests.addTest(unittest.makeSuite(failmanagertestcase.AddFailure)) # BanManager tests.addTest(unittest.makeSuite(banmanagertestcase.AddFailure)) + tests.addTest(unittest.makeSuite(banmanagertestcase.StatusExtendedCymruInfo)) # ClientReaders tests.addTest(unittest.makeSuite(clientreadertestcase.ConfigReaderTest)) tests.addTest(unittest.makeSuite(clientreadertestcase.JailReaderTest)) From a8f057eab749d3fbf3974edf73ff9b4b78a32a05 Mon Sep 17 00:00:00 2001 From: Lee Clemens Date: Fri, 23 Jan 2015 15:27:33 -0500 Subject: [PATCH 2/8] Add `pass` to empty methods --- fail2ban/tests/banmanagertestcase.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/fail2ban/tests/banmanagertestcase.py b/fail2ban/tests/banmanagertestcase.py index d4d106a9..9cf820a3 100644 --- a/fail2ban/tests/banmanagertestcase.py +++ b/fail2ban/tests/banmanagertestcase.py @@ -38,6 +38,7 @@ class AddFailure(unittest.TestCase): def tearDown(self): """Call after every test case.""" + pass def testAdd(self): self.assertEqual(self.__banManager.size(), 1) @@ -68,6 +69,7 @@ class StatusExtendedCymruInfo(unittest.TestCase): def tearDown(self): """Call after every test case.""" + pass def testCymruInfo(self): cymru_info = self.__banManager.getBanListExtendedCymruInfo() From a80ff26e151c295bb5cc1661fa5402c0115a8e97 Mon Sep 17 00:00:00 2001 From: Lee Clemens Date: Fri, 23 Jan 2015 15:40:00 -0500 Subject: [PATCH 3/8] Conditionally test fail2ban-client status extended when dnspython is installed --- fail2ban/tests/utils.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/fail2ban/tests/utils.py b/fail2ban/tests/utils.py index 42edf369..dbe5cee3 100644 --- a/fail2ban/tests/utils.py +++ b/fail2ban/tests/utils.py @@ -107,7 +107,11 @@ def gatherTests(regexps=None, no_network=False): tests.addTest(unittest.makeSuite(failmanagertestcase.AddFailure)) # BanManager tests.addTest(unittest.makeSuite(banmanagertestcase.AddFailure)) - tests.addTest(unittest.makeSuite(banmanagertestcase.StatusExtendedCymruInfo)) + try: + import dns + tests.addTest(unittest.makeSuite(banmanagertestcase.StatusExtendedCymruInfo)) + except ImportError: + pass # ClientReaders tests.addTest(unittest.makeSuite(clientreadertestcase.ConfigReaderTest)) tests.addTest(unittest.makeSuite(clientreadertestcase.JailReaderTest)) From 8fd9d6a770bdfc90d59da6c1493d82e0b1a2752e Mon Sep 17 00:00:00 2001 From: Lee Clemens Date: Fri, 23 Jan 2015 15:43:58 -0500 Subject: [PATCH 4/8] add dnspython[3] optional dependencies --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index 8210837e..4915c95b 100644 --- a/README.md +++ b/README.md @@ -33,6 +33,7 @@ Optional: - Linux >= 2.6.13 - [gamin >= 0.0.21](http://www.gnome.org/~veillard/gamin) - [systemd >= 204](http://www.freedesktop.org/wiki/Software/systemd) +- [dnspython or dnspython3](http://www.dnspython.org/) To install, just do: From 256c8711210b17a63d225610afc03a313cef311d Mon Sep 17 00:00:00 2001 From: Lee Clemens Date: Fri, 23 Jan 2015 16:00:33 -0500 Subject: [PATCH 5/8] conditionally import dnspython --- fail2ban/server/banmanager.py | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/fail2ban/server/banmanager.py b/fail2ban/server/banmanager.py index 81ac7c4f..2b2fd39a 100644 --- a/fail2ban/server/banmanager.py +++ b/fail2ban/server/banmanager.py @@ -26,9 +26,6 @@ __license__ = "GPL" from threading import Lock -import dns.exception -import dns.resolver - from .ticket import BanTicket from .mytime import MyTime from ..helpers import getLogger @@ -139,8 +136,17 @@ class BanManager: # @return {"asn": [], "country": [], "rir": []} dict for self.__banList IPs def getBanListExtendedCymruInfo(self): - self.__lock.acquire() return_dict = {"asn": [], "country": [], "rir": []} + try: + import dns.exception + import dns.resolver + except ImportError: + logSys.error("dnspython package is required but could not be imported") + return_dict["asn"].append("error") + return_dict["country"].append("error") + return_dict["rir"].append("error") + return return_dict + self.__lock.acquire() try: for banData in self.__banList: ip = banData.getIP() From ce0fe8b907000a02eca1b8edddd897325580a1f6 Mon Sep 17 00:00:00 2001 From: Lee Clemens Date: Fri, 23 Jan 2015 16:00:49 -0500 Subject: [PATCH 6/8] test dnspython nxdomain returned --- fail2ban/tests/banmanagertestcase.py | 23 ++++++++++++++++++++--- 1 file changed, 20 insertions(+), 3 deletions(-) diff --git a/fail2ban/tests/banmanagertestcase.py b/fail2ban/tests/banmanagertestcase.py index 9cf820a3..3f7fe97e 100644 --- a/fail2ban/tests/banmanagertestcase.py +++ b/fail2ban/tests/banmanagertestcase.py @@ -63,9 +63,9 @@ class StatusExtendedCymruInfo(unittest.TestCase): self.__asn = "15133" self.__country = "EU" self.__rir = "ripencc" - self.__ticket = BanTicket(self.__ban_ip, 1167605999.0) + ticket = BanTicket(self.__ban_ip, 1167605999.0) self.__banManager = BanManager() - self.assertTrue(self.__banManager.addBanTicket(self.__ticket)) + self.assertTrue(self.__banManager.addBanTicket(ticket)) def tearDown(self): """Call after every test case.""" @@ -74,7 +74,9 @@ class StatusExtendedCymruInfo(unittest.TestCase): def testCymruInfo(self): cymru_info = self.__banManager.getBanListExtendedCymruInfo() if "assertDictEqual" in dir(self): - self.assertDictEqual(cymru_info, {"asn": [self.__asn], "country": [self.__country], "rir": [self.__rir]}) + self.assertDictEqual(cymru_info, {"asn": [self.__asn], + "country": [self.__country], + "rir": [self.__rir]}) else: # Python 2.6 does not support assertDictEqual() self.assertEqual(cymru_info["asn"], [self.__asn]) @@ -95,3 +97,18 @@ class StatusExtendedCymruInfo(unittest.TestCase): self.assertEqual( self.__banManager.geBanListExtendedRIR(self.__banManager.getBanListExtendedCymruInfo()), [self.__rir]) + + def testCymruInfoNxdomain(self): + ticket = BanTicket("10.0.0.0", 1167605999.0) + self.__banManager = BanManager() + self.assertTrue(self.__banManager.addBanTicket(ticket)) + cymru_info = self.__banManager.getBanListExtendedCymruInfo() + if "assertDictEqual" in dir(self): + self.assertDictEqual(cymru_info, {"asn": [self.__asn, "nxdomain"], + "country": [self.__country, "nxdomain"], + "rir": [self.__rir, "nxdomain"]}) + else: + # Python 2.6 does not support assertDictEqual() + self.assertEqual(cymru_info["asn"], [self.__asn, "nxdomain"]) + self.assertEqual(cymru_info["country"], [self.__country, "nxdomain"]) + self.assertEqual(cymru_info["rir"], [self.__rir, "nxdomain"]) From 03db257c434e93b80742d33ee68773471d70c617 Mon Sep 17 00:00:00 2001 From: Lee Clemens Date: Fri, 23 Jan 2015 16:18:09 -0500 Subject: [PATCH 7/8] fix test of new banManager's instance --- fail2ban/tests/banmanagertestcase.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/fail2ban/tests/banmanagertestcase.py b/fail2ban/tests/banmanagertestcase.py index 3f7fe97e..c4dd42cf 100644 --- a/fail2ban/tests/banmanagertestcase.py +++ b/fail2ban/tests/banmanagertestcase.py @@ -104,11 +104,11 @@ class StatusExtendedCymruInfo(unittest.TestCase): self.assertTrue(self.__banManager.addBanTicket(ticket)) cymru_info = self.__banManager.getBanListExtendedCymruInfo() if "assertDictEqual" in dir(self): - self.assertDictEqual(cymru_info, {"asn": [self.__asn, "nxdomain"], - "country": [self.__country, "nxdomain"], - "rir": [self.__rir, "nxdomain"]}) + self.assertDictEqual(cymru_info, {"asn": ["nxdomain"], + "country": ["nxdomain"], + "rir": ["nxdomain"]}) else: # Python 2.6 does not support assertDictEqual() - self.assertEqual(cymru_info["asn"], [self.__asn, "nxdomain"]) - self.assertEqual(cymru_info["country"], [self.__country, "nxdomain"]) - self.assertEqual(cymru_info["rir"], [self.__rir, "nxdomain"]) + self.assertEqual(cymru_info["asn"], ["nxdomain"]) + self.assertEqual(cymru_info["country"], ["nxdomain"]) + self.assertEqual(cymru_info["rir"], ["nxdomain"]) From 13b2aea48a370b84075ca4ce80ea0f7ce9d69491 Mon Sep 17 00:00:00 2001 From: Lee Clemens Date: Sat, 24 Jan 2015 12:22:34 -0500 Subject: [PATCH 8/8] Remove version specific package --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 4915c95b..17a78ee5 100644 --- a/README.md +++ b/README.md @@ -33,7 +33,7 @@ Optional: - Linux >= 2.6.13 - [gamin >= 0.0.21](http://www.gnome.org/~veillard/gamin) - [systemd >= 204](http://www.freedesktop.org/wiki/Software/systemd) -- [dnspython or dnspython3](http://www.dnspython.org/) +- [dnspython](http://www.dnspython.org/) To install, just do: