Make fail2ban a full blown module. Close gh-26

AKA: Merge remote-tracking branch 'github_kwirk_fail2ban/module' into 0.9

* github_kwirk_fail2ban/module:
  BF: Added test filter.d files to setup.py package data
  TST: Fix up tests from multiline elements broken in previous merge
  TST: clientreader test now use /etc/fail2ban/ if no local config/
  ENH+TST: Move fail2ban-* scripts to bin/
  TST+ENH: Move testcases to part of fail2ban module
  TST: Update Travis CI coverage config for python module structure
  ENH+BF+TST+DOC: Make fail2ban a python module

Conflicts:
	.travis.yml -- the line for PYTHONPATH
	.travis_coveragerc -- now we do cover gamin tests
pull/170/merge
Yaroslav Halchenko 2013-04-09 19:39:58 -04:00
commit e630b77fca
84 changed files with 174 additions and 168 deletions

View File

@ -13,6 +13,6 @@ install:
- if [[ $TRAVIS_PYTHON_VERSION == 2.7 ]]; then pip install -q coveralls; fi
script:
- if [[ $TRAVIS_PYTHON_VERSION == 2.7 ]]; then export PYTHONPATH="$PYTHONPATH:/usr/share/pyshared:/usr/lib/pyshared/python2.7"; fi
- if [[ $TRAVIS_PYTHON_VERSION == 2.7 ]]; then coverage run --rcfile=.travis_coveragerc fail2ban-testcases; else python ./fail2ban-testcases; fi
- if [[ $TRAVIS_PYTHON_VERSION == 2.[6-7] ]]; then coverage run --rcfile=.travis_coveragerc bin/fail2ban-testcases; else python bin/fail2ban-testcases; fi
after_script:
- if [[ $TRAVIS_PYTHON_VERSION == 2.7 ]]; then coveralls; fi

12
DEVELOP
View File

@ -24,9 +24,9 @@ Request feature. You can find more details on the Fail2Ban wiki
Testing
=======
Existing tests can be run by executing `fail2ban-testcases`. This has options
like --log-level that will probably be useful. `fail2ban-testcases --help` for
full options.
Existing tests can be run by executing `bin/fail2ban-testcases`. This has
options like --log-level that will probably be useful.
`bin/fail2ban-testcases --help` forfull options.
Test cases should cover all usual cases, all exception cases and all inside
/ outside boundary conditions.
@ -39,7 +39,7 @@ Install the package python-coverage to visualise your test coverage. Run the
following (note: on Debian-based systems, the script is called
`python-coverage`):
coverage run fail2ban-testcases
coverage run bin/fail2ban-testcases
coverage html
Then look at htmlcov/index.html and see how much coverage your test cases
@ -249,7 +249,7 @@ Takes care about executing start/check/ban/unban/stop commands
Releasing
=========
# Ensure the version is correct in ./common/version.py
# Ensure the version is correct in ./fail2ban/version.py
# Add/finalize the corresponding entry in the ChangeLog
@ -271,7 +271,7 @@ Releasing
# Run the following and update the wiki with output:
python -c 'import common.protocol; common.protocol.printWiki()'
python -c 'import fail2ban.protocol; fail2ban.protocol.printWiki()'
# Email users and development list of release

113
MANIFEST
View File

@ -5,64 +5,65 @@ THANKS
COPYING
DEVELOP
doc/run-rootless.txt
fail2ban-client
fail2ban-server
fail2ban-testcases
fail2ban-regex
client/configreader.py
client/configparserinc.py
client/jailreader.py
client/fail2banreader.py
client/jailsreader.py
client/beautifier.py
client/filterreader.py
client/actionreader.py
client/__init__.py
client/configurator.py
client/csocket.py
server/asyncserver.py
server/filter.py
server/filterpyinotify.py
server/filtergamin.py
server/filterpoll.py
server/iso8601.py
server/server.py
server/actions.py
server/faildata.py
server/failmanager.py
server/datedetector.py
server/jailthread.py
server/transmitter.py
server/action.py
server/ticket.py
server/jail.py
server/jails.py
server/__init__.py
server/banmanager.py
server/datetemplate.py
server/mytime.py
server/failregex.py
testcases/files/testcase-usedns.log
testcases/banmanagertestcase.py
testcases/failmanagertestcase.py
testcases/clientreadertestcase.py
testcases/filtertestcase.py
testcases/__init__.py
testcases/datedetectortestcase.py
testcases/actiontestcase.py
testcases/servertestcase.py
testcases/sockettestcase.py
testcases/files/testcase01.log
testcases/files/testcase02.log
testcases/files/testcase03.log
testcases/files/testcase04.log
bin/fail2ban-client
bin/fail2ban-server
bin/fail2ban-testcases
bin/fail2ban-regex
fail2ban/client/configreader.py
fail2ban/client/configparserinc.py
fail2ban/client/jailreader.py
fail2ban/client/fail2banreader.py
fail2ban/client/jailsreader.py
fail2ban/client/beautifier.py
fail2ban/client/filterreader.py
fail2ban/client/actionreader.py
fail2ban/client/__init__.py
fail2ban/client/configurator.py
fail2ban/client/csocket.py
fail2ban/server/asyncserver.py
fail2ban/server/filter.py
fail2ban/server/filterpyinotify.py
fail2ban/server/filtergamin.py
fail2ban/server/filterpoll.py
fail2ban/server/iso8601.py
fail2ban/server/server.py
fail2ban/server/actions.py
fail2ban/server/faildata.py
fail2ban/server/failmanager.py
fail2ban/server/datedetector.py
fail2ban/server/jailthread.py
fail2ban/server/transmitter.py
fail2ban/server/action.py
fail2ban/server/ticket.py
fail2ban/server/jail.py
fail2ban/server/jails.py
fail2ban/server/__init__.py
fail2ban/server/banmanager.py
fail2ban/server/datetemplate.py
fail2ban/server/mytime.py
fail2ban/server/failregex.py
fail2ban/tests/banmanagertestcase.py
fail2ban/tests/failmanagertestcase.py
fail2ban/tests/clientreadertestcase.py
fail2ban/tests/filtertestcase.py
fail2ban/tests/__init__.py
fail2ban/tests/datedetectortestcase.py
fail2ban/tests/actiontestcase.py
fail2ban/tests/servertestcase.py
fail2ban/tests/sockettestcase.py
fail2ban/tests/utils.py
fail2ban/tests/files/testcase01.log
fail2ban/tests/files/testcase02.log
fail2ban/tests/files/testcase03.log
fail2ban/tests/files/testcase04.log
fail2ban/tests/files/testcase-usedns.log
setup.py
setup.cfg
common/__init__.py
common/exceptions.py
common/helpers.py
common/version.py
common/protocol.py
fail2ban/__init__.py
fail2ban/exceptions.py
fail2ban/helpers.py
fail2ban/version.py
fail2ban/protocol.py
config/jail.conf
config/filter.d/common.conf
config/filter.d/apache-auth.conf

View File

@ -25,19 +25,11 @@ __license__ = "GPL"
import sys, string, os, pickle, re, logging, signal
import getopt, time, shlex, socket
# Inserts our own modules path first in the list
# fix for bug #343821
try:
from common.version import version
except ImportError, e:
sys.path.insert(1, "/usr/share/fail2ban")
from common.version import version
# Now we can import the rest of modules
from common.protocol import printFormatted
from client.csocket import CSocket
from client.configurator import Configurator
from client.beautifier import Beautifier
from fail2ban.version import version
from fail2ban.protocol import printFormatted
from fail2ban.client.csocket import CSocket
from fail2ban.client.configurator import Configurator
from fail2ban.client.beautifier import Beautifier
# Gets the instance of the logger.
logSys = logging.getLogger("fail2ban.client")

View File

@ -23,19 +23,12 @@ __copyright__ = "Copyright (c) 2004 Cyril Jaquier, 2012 Yaroslav Halchenko"
__license__ = "GPL"
import getopt, sys, time, logging, os
# Inserts our own modules path first in the list
# fix for bug #343821
try:
from common.version import version
except ImportError, e:
sys.path.insert(1, "/usr/share/fail2ban")
from common.version import version
from client.configparserinc import SafeConfigParserWithIncludes
from ConfigParser import NoOptionError, NoSectionError, MissingSectionHeaderError
from server.filter import Filter
from server.failregex import RegexException
from fail2ban.version import version
from fail2ban.client.configparserinc import SafeConfigParserWithIncludes
from fail2ban.server.filter import Filter
from fail2ban.server.failregex import RegexException
# Gets the instance of the logger.
logSys = logging.getLogger("fail2ban.regex")

View File

@ -24,15 +24,8 @@ __license__ = "GPL"
import getopt, sys, logging, os
# Inserts our own modules path first in the list
# fix for bug #343821
try:
from common.version import version
except ImportError, e:
sys.path.insert(1, "/usr/share/fail2ban")
from common.version import version
from server.server import Server
from fail2ban.version import version
from fail2ban.server.server import Server
# Gets the instance of the logger.
logSys = logging.getLogger("fail2ban")

View File

@ -27,18 +27,23 @@ __license__ = "GPL"
import unittest, logging, sys, time, os
from common.version import version
from testcases import banmanagertestcase
from testcases import clientreadertestcase
from testcases import failmanagertestcase
from testcases import filtertestcase
from testcases import servertestcase
from testcases import datedetectortestcase
from testcases import actiontestcase
from testcases import sockettestcase
# Check if local fail2ban module exists, and use if it exists by
# modifying the path. This is such that tests can be used in dev
# environment.
if os.path.exists("fail2ban/__init__.py"):
sys.path.insert(0, ".")
from fail2ban.version import version
from fail2ban.tests import banmanagertestcase
from fail2ban.tests import clientreadertestcase
from fail2ban.tests import failmanagertestcase
from fail2ban.tests import filtertestcase
from fail2ban.tests import servertestcase
from fail2ban.tests import datedetectortestcase
from fail2ban.tests import actiontestcase
from fail2ban.tests import sockettestcase
from testcases.utils import FormatterWithTraceBack
from server.mytime import MyTime
from fail2ban.tests.utils import FormatterWithTraceBack
from fail2ban.server.mytime import MyTime
from optparse import OptionParser, Option
@ -169,20 +174,20 @@ tests.addTest(unittest.makeSuite(datedetectortestcase.DateDetectorTest))
# Extensive use-tests of different available filters backends
#
from server.filterpoll import FilterPoll
from fail2ban.server.filterpoll import FilterPoll
filters = [FilterPoll] # always available
# Additional filters available only if external modules are available
# yoh: Since I do not know better way for parametric tests
# with good old unittest
try:
from server.filtergamin import FilterGamin
from fail2ban.server.filtergamin import FilterGamin
filters.append(FilterGamin)
except Exception, e: # pragma: no cover
print "I: Skipping gamin backend testing. Got exception '%s'" % e
try:
from server.filterpyinotify import FilterPyinotify
from fail2ban.server.filterpyinotify import FilterPyinotify
filters.append(FilterPyinotify)
except Exception, e: # pragma: no cover
print "I: Skipping pyinotify backend testing. Got exception '%s'" % e

View File

@ -9,7 +9,7 @@ for python in /usr/{,local/}bin/python2.[0-9]{,.*}{,-dbg}
do
[ -e "$python" ] || continue
echo "Testing using $python"
$python ./fail2ban-testcases "$@" || failed+=" $python"
$python bin/fail2ban-testcases "$@" || failed+=" $python"
done
if [ ! -z "$failed" ]; then

View File

@ -23,7 +23,7 @@ __license__ = "GPL"
import logging
from common.exceptions import UnknownJailException, DuplicateJailException
from fail2ban.exceptions import UnknownJailException, DuplicateJailException
# Gets the instance of the logger.
logSys = logging.getLogger("fail2ban.client.config")

View File

@ -28,9 +28,10 @@ __copyright__ = "Copyright (c) 2004 Cyril Jaquier"
__license__ = "GPL"
from pickle import dumps, loads, HIGHEST_PROTOCOL
from common import helpers
import asyncore, asynchat, socket, os, logging, sys, traceback
from fail2ban import helpers
# Gets the instance of the logger.
logSys = logging.getLogger("fail2ban.server")

View File

@ -21,7 +21,7 @@ __author__ = "Cyril Jaquier, Yaroslav Halchenko"
__copyright__ = "Copyright (c) 2004 Cyril Jaquier, 2013- Yaroslav Halchenko"
__license__ = "GPL"
from common.exceptions import DuplicateJailException, UnknownJailException
from fail2ban.exceptions import DuplicateJailException, UnknownJailException
from jail import Jail
from threading import Lock

View File

@ -32,7 +32,7 @@ from jails import Jails
from transmitter import Transmitter
from asyncserver import AsyncServer
from asyncserver import AsyncServerException
from common import version
from fail2ban import version
import logging, logging.handlers, sys, os, signal
# Gets the instance of the logger.

View File

@ -29,9 +29,10 @@ __license__ = "GPL"
import unittest, time
import logging, sys
from server.action import Action
from StringIO import StringIO
from fail2ban.server.action import Action
class ExecuteAction(unittest.TestCase):
def setUp(self):

View File

@ -28,8 +28,9 @@ __copyright__ = "Copyright (c) 2004 Cyril Jaquier"
__license__ = "GPL"
import unittest
from server.banmanager import BanManager
from server.ticket import BanTicket
from fail2ban.server.banmanager import BanManager
from fail2ban.server.ticket import BanTicket
class AddFailure(unittest.TestCase):

View File

@ -22,11 +22,18 @@ __copyright__ = "Copyright (c) 2004 Cyril Jaquier, 2011-2013 Yaroslav Halchenko"
__license__ = "GPL"
import os, shutil, tempfile, unittest
from client.configreader import ConfigReader
from client.jailreader import JailReader
from client.filterreader import FilterReader
from client.jailsreader import JailsReader
from client.configurator import Configurator
from fail2ban.client.configreader import ConfigReader
from fail2ban.client.jailreader import JailReader
from fail2ban.client.filterreader import FilterReader
from fail2ban.client.jailsreader import JailsReader
from fail2ban.client.configurator import Configurator
TEST_FILES_DIR = os.path.join(os.path.dirname(__file__), "files")
if os.path.exists('config/fail2ban.conf'):
CONFIG_DIR='config'
else:
CONFIG_DIR='/etc/fail2ban'
class ConfigReaderTest(unittest.TestCase):
@ -99,7 +106,7 @@ option = %s
class JailReaderTest(unittest.TestCase):
def testStockSSHJail(self):
jail = JailReader('ssh-iptables', basedir='config') # we are running tests from root project dir atm
jail = JailReader('ssh-iptables', basedir=CONFIG_DIR) # we are running tests from root project dir atm
self.assertTrue(jail.read())
self.assertTrue(jail.getOptions())
self.assertFalse(jail.isEnabled())
@ -134,7 +141,7 @@ class FilterReaderTest(unittest.TestCase):
['set', 'testcase01', 'addignoreregex',
"^.+ john from host 192.168.1.1\\s*$"]]
filterReader = FilterReader("testcase01", "testcase01")
filterReader.setBaseDir("testcases/files/")
filterReader.setBaseDir(TEST_FILES_DIR)
filterReader.read()
#filterReader.getOptions(["failregex", "ignoreregex"])
filterReader.getOptions(None)
@ -149,7 +156,7 @@ class JailsReaderTest(unittest.TestCase):
self.assertRaises(ValueError, reader.read)
def testReadStockJailConf(self):
jails = JailsReader(basedir='config') # we are running tests from root project dir atm
jails = JailsReader(basedir=CONFIG_DIR) # we are running tests from root project dir atm
self.assertTrue(jails.read()) # opens fine
self.assertTrue(jails.getOptions()) # reads fine
comm_commands = jails.convert()
@ -160,7 +167,7 @@ class JailsReaderTest(unittest.TestCase):
def testReadStockJailConfForceEnabled(self):
# more of a smoke test to make sure that no obvious surprises
# on users' systems when enabling shipped jails
jails = JailsReader(basedir='config', force_enable=True) # we are running tests from root project dir atm
jails = JailsReader(basedir=CONFIG_DIR, force_enable=True) # we are running tests from root project dir atm
self.assertTrue(jails.read()) # opens fine
self.assertTrue(jails.getOptions()) # reads fine
comm_commands = jails.convert()
@ -182,8 +189,8 @@ class JailsReaderTest(unittest.TestCase):
def testConfigurator(self):
configurator = Configurator()
configurator.setBaseDir('config')
self.assertEqual(configurator.getBaseDir(), 'config')
configurator.setBaseDir(CONFIG_DIR)
self.assertEqual(configurator.getBaseDir(), CONFIG_DIR)
configurator.readEarly()
opts = configurator.getEarlyOptions()
@ -196,4 +203,4 @@ class JailsReaderTest(unittest.TestCase):
# otherwise just a code smoke test)
configurator._Configurator__jails.setBaseDir('/tmp')
self.assertEqual(configurator._Configurator__jails.getBaseDir(), '/tmp')
self.assertEqual(configurator.getBaseDir(), 'config')
self.assertEqual(configurator.getBaseDir(), CONFIG_DIR)

View File

@ -28,8 +28,9 @@ __copyright__ = "Copyright (c) 2004 Cyril Jaquier"
__license__ = "GPL"
import unittest
from server.datedetector import DateDetector
from server.datetemplate import DateTemplate
from fail2ban.server.datedetector import DateDetector
from fail2ban.server.datetemplate import DateTemplate
class DateDetectorTest(unittest.TestCase):

View File

@ -28,8 +28,9 @@ __copyright__ = "Copyright (c) 2004 Cyril Jaquier"
__license__ = "GPL"
import unittest, socket, time, pickle
from server.failmanager import FailManager, FailManagerEmpty
from server.ticket import FailTicket
from fail2ban.server.failmanager import FailManager, FailManagerEmpty
from fail2ban.server.ticket import FailTicket
class AddFailure(unittest.TestCase):

View File

@ -29,11 +29,13 @@ import sys
import time
import tempfile
from server.jail import Jail
from server.filterpoll import FilterPoll
from server.filter import FileFilter, DNSUtils
from server.failmanager import FailManager
from server.failmanager import FailManagerEmpty
from fail2ban.server.jail import Jail
from fail2ban.server.filterpoll import FilterPoll
from fail2ban.server.filter import FileFilter, DNSUtils
from fail2ban.server.failmanager import FailManager
from fail2ban.server.failmanager import FailManagerEmpty
TEST_FILES_DIR = os.path.join(os.path.dirname(__file__), "files")
#
# Useful helpers
@ -182,7 +184,7 @@ class IgnoreIP(unittest.TestCase):
class LogFile(unittest.TestCase):
FILENAME = "testcases/files/testcase01.log"
FILENAME = os.path.join(TEST_FILES_DIR, "testcase01.log")
def setUp(self):
"""Call before every test case."""
@ -522,12 +524,12 @@ def get_monitor_failures_testcase(Filter_):
class GetFailures(unittest.TestCase):
FILENAME_01 = "testcases/files/testcase01.log"
FILENAME_02 = "testcases/files/testcase02.log"
FILENAME_03 = "testcases/files/testcase03.log"
FILENAME_04 = "testcases/files/testcase04.log"
FILENAME_USEDNS = "testcases/files/testcase-usedns.log"
FILENAME_MULTILINE = "testcases/files/testcase-multiline.log"
FILENAME_01 = os.path.join(TEST_FILES_DIR, "testcase01.log")
FILENAME_02 = os.path.join(TEST_FILES_DIR, "testcase02.log")
FILENAME_03 = os.path.join(TEST_FILES_DIR, "testcase03.log")
FILENAME_04 = os.path.join(TEST_FILES_DIR, "testcase04.log")
FILENAME_USEDNS = os.path.join(TEST_FILES_DIR, "testcase-usedns.log")
FILENAME_MULTILINE = os.path.join(TEST_FILES_DIR, "testcase-multiline.log")
# so that they could be reused by other tests
FAILURES_01 = ('193.168.0.128', 3, 1124013599.0,

View File

@ -28,8 +28,11 @@ __copyright__ = "Copyright (c) 2004 Cyril Jaquier"
__license__ = "GPL"
import unittest, socket, time, tempfile, os
from server.server import Server
from common.exceptions import UnknownJailException
from fail2ban.server.server import Server
from fail2ban.exceptions import UnknownJailException
TEST_FILES_DIR = os.path.join(os.path.dirname(__file__), "files")
class StartStop(unittest.TestCase):
@ -272,14 +275,14 @@ class Transmitter(TransmitterBase):
self.jailAddDelTest(
"logpath",
[
"testcases/files/testcase01.log",
"testcases/files/testcase02.log",
"testcases/files/testcase03.log",
os.path.join(TEST_FILES_DIR, "testcase01.log"),
os.path.join(TEST_FILES_DIR, "testcase02.log"),
os.path.join(TEST_FILES_DIR, "testcase03.log"),
],
self.jailName
)
# Try duplicates
value = "testcases/files/testcase04.log"
value = os.path.join(TEST_FILES_DIR, "testcase04.log")
self.assertEqual(
self.transm.proceed(["set", self.jailName, "addlogpath", value]),
(0, [value]))

View File

@ -28,8 +28,9 @@ __copyright__ = "Copyright (c) 2013 Steven Hiscocks"
__license__ = "GPL"
import unittest, time, tempfile, os, threading
from server.asyncserver import AsyncServer, AsyncServerException
from client.csocket import CSocket
from fail2ban.server.asyncserver import AsyncServer, AsyncServerException
from fail2ban.client.csocket import CSocket
class Socket(unittest.TestCase):

View File

@ -1,6 +1,3 @@
[install]
install-purelib=/usr/share/fail2ban
[sdist]
formats=bztar

View File

@ -23,11 +23,12 @@ __copyright__ = "Copyright (c) 2004 Cyril Jaquier"
__license__ = "GPL"
from distutils.core import setup
from common.version import version
from os.path import isfile, join, isdir
from sys import argv
import sys
from glob import glob
from fail2ban.version import version
longdesc = '''
Fail2Ban scans log files like /var/log/pwdfail or
/var/log/apache/error_log and bans IP that makes
@ -38,7 +39,7 @@ commands.'''
setup(
name = "fail2ban",
version = version,
description = "Ban IPs that make too many password failure",
description = "Ban IPs that make too many password failures",
long_description = longdesc,
author = "Cyril Jaquier",
author_email = "cyril.jaquier@fail2ban.org",
@ -46,15 +47,21 @@ setup(
license = "GPL",
platforms = "Posix",
scripts = [
'fail2ban-client',
'fail2ban-server',
'fail2ban-regex'
'bin/fail2ban-client',
'bin/fail2ban-server',
'bin/fail2ban-regex',
'bin/fail2ban-testcases',
],
packages = [
'common',
'client',
'server'
'fail2ban',
'fail2ban.client',
'fail2ban.server',
'fail2ban.tests',
],
package_data = {
'fail2ban.tests':
['files/*.log', 'files/filter.d/*.conf'],
},
data_files = [
('/etc/fail2ban',
glob("config/*.conf")
@ -117,7 +124,7 @@ if isdir("/usr/lib/fail2ban"):
print
# Update config file
if argv[1] == "install":
if sys.argv[1] == "install":
print
print "Please do not forget to update your configuration files."
print "They are in /etc/fail2ban/."