Merge pull request #1104 from leeclemens/travis-rework

Improve Travis setup and coverage reporting (all versions report)

.coveragerc

@@ -1,4 +1,11 @@
 
 [run]
 branch = True
-omit = /usr*
+source =
+    config
+    fail2ban
+
+[report]
+exclude_lines =
+    pragma: no cover
+    pragma: systemd no cover

.travis.yml

@@ -1,29 +1,56 @@
 # vim ft=yaml
 # travis-ci.org definition for Fail2Ban build
+# https://travis-ci.org/fail2ban/fail2ban/
 language: python
 python:
-  - "2.6"
-  - "2.7"
-  - "3.2"
-  - "3.3"
-  - "3.4"
-#  - "pypy" # disabled for now due to frequent unreproducable failures # gh-1089
-  - "pypy3"
+  - 2.6
+  - 2.7
+  - pypy
+  - 3.2
+  - 3.3
+  - 3.4
+  - pypy3
 before_install:
-  - if [[ $TRAVIS_PYTHON_VERSION == 2.7 ]]; then travis_retry sudo apt-get update -qq; fi
+  - if [[ $TRAVIS_PYTHON_VERSION == 2* || $TRAVIS_PYTHON_VERSION == 'pypy' ]]; then export F2B_PY_2=true && echo "Set F2B_PY_2"; fi
+  - if [[ $TRAVIS_PYTHON_VERSION == 3* || $TRAVIS_PYTHON_VERSION == 'pypy3' ]]; then export F2B_PY_3=true && echo "Set F2B_PY_3"; fi
+  - travis_retry sudo apt-get update -qq
+  # Set this so sudo executes the correct python binary
+  #   Anything not using sudo will already have the correct environment
+  - export VENV_BIN="$VIRTUAL_ENV/bin" && echo "VENV_BIN set to $VENV_BIN"
 install:
+  # Install Python packages / dependencies
+  #   coverage
+  - travis_retry pip install coverage
+  #   coveralls
+  - travis_retry pip install coveralls
+  #   dnspython or dnspython3
+  - if [[ "$F2B_PY_2" ]]; then travis_retry pip install dnspython; fi
+  - if [[ "$F2B_PY_3" ]]; then travis_retry pip install dnspython3; fi
+  #   gamin - install manually (not in PyPI) - travis-ci system Python is 2.7
+  - if [[ $TRAVIS_PYTHON_VERSION == 2.7 ]]; then travis_retry sudo apt-get install -qq python-gamin && cp /usr/share/pyshared/gamin.py /usr/lib/pyshared/python2.7/_gamin.so $VIRTUAL_ENV/lib/python2.7/site-packages/; fi
+  #   pyinotify
   - travis_retry pip install pyinotify
-  - if [[ $TRAVIS_PYTHON_VERSION == 2* || $TRAVIS_PYTHON_VERSION == 'pypy' ]]; then travis_retry pip install dnspython; fi
-  - if [[ $TRAVIS_PYTHON_VERSION == 3* || $TRAVIS_PYTHON_VERSION == 'pypy3' ]]; then travis_retry pip install dnspython3; fi
-  - if [[ $TRAVIS_PYTHON_VERSION == 2.7 ]]; then travis_retry sudo apt-get install -qq python-gamin; cp /usr/share/pyshared/gamin.py /usr/lib/pyshared/python2.7/_gamin.so $VIRTUAL_ENV/lib/python2.7/site-packages/; fi
-  - if [[ $TRAVIS_PYTHON_VERSION == 2.7 ]]; then cd ..; travis_retry pip install -q coveralls; cd -; fi
-  # overcome buggy pypy
-  - if [[ $TRAVIS_PYTHON_VERSION == pypy ]] ; then dpkg --compare-versions $(pypy --version 2>&1 | awk '/PyPy/{print $2;}') ge 2.5.1 || { d=$PWD; cd /tmp; wget --no-check-certificate https://bitbucket.org/pypy/pypy/downloads/pypy-2.5.1-linux64.tar.bz2; tar -xjvf pypy*bz2; cd pypy-*/bin/; export PATH=$PWD:$PATH; cd $d; } ; fi
+before_script:
+  # Manually execute 2to3 for now
+  - if [[ "$F2B_PY_3" ]]; then ./fail2ban-2to3; fi
 script:
-  - if [[ $TRAVIS_PYTHON_VERSION == 2.7 ]]; then coverage run --rcfile=.travis_coveragerc setup.py test; else python setup.py test; fi
-# test installation
-  - sudo python setup.py install
+  # Keep the legacy setup.py test approach of checking coverage for python2
+  - if [[ "$F2B_PY_2" ]]; then coverage run setup.py test; fi
+  # Coverage doesn't pick up setup.py test with python3, so run it directly
+  - if [[ "$F2B_PY_3" ]]; then coverage run bin/fail2ban-testcases; fi
+  # Use $VENV_BIN (not python) or else sudo will always run the system's python (2.7)
+  - sudo $VENV_BIN/pip install .
 after_success:
-# Coverage config file must be .coveragerc for coveralls
-  - if [[ $TRAVIS_PYTHON_VERSION == 2.7 ]]; then cp -v .travis_coveragerc .coveragerc; fi
-  - if [[ $TRAVIS_PYTHON_VERSION == 2.7 ]]; then coveralls; fi
+  - coveralls
+matrix:
+  fast_finish: true
+# Might be worth looking into
+#notifications:
+#  email: true
+#  irc:
+#    channels: "irc.freenode.org#fail2ban"
+#    template:
+#      - "%{repository}@%{branch}: %{message} (%{build_url})"
+#    on_success: change
+#    on_failure: change
+#    skip_join: true

.travis_coveragerc

@@ -1,7 +0,0 @@
-
-[run]
-branch = True
-omit =
-    /usr/*
-    /home/travis/virtualenv/*
-    fail2ban/server/filtersystemd.py

ChangeLog

@@ -41,6 +41,7 @@ ver. 0.9.3 (2015/XX/XXX) - wanna-be-released
    * fail2ban-testcases man page
    * filter.d/apache-badbots.conf, filter.d/nginx-botsearch.conf - add
      HEAD method verb
+   * Revamp of Travis and coverage automated testing
 
 
 ver. 0.9.2 (2015/04/29) - better-quick-now-than-later

DEVELOP

@@ -56,9 +56,12 @@ following (note: on Debian-based systems, the script is called
 `python-coverage`)::
 
   coverage run bin/fail2ban-testcases
+  coverage report
+
+Optionally:
   coverage html
 
-Then look at htmlcov/index.html and see how much coverage your test cases
+And then browse htmlcov/index.html and see how much coverage your test cases
 exert over the code base. Full coverage is a good thing however it may not be
 complete. Try to ensure tests cover as many independent paths through the
 code.

fail2ban/tests/misctestcase.py

@@ -155,8 +155,10 @@ class TestsUtilsTest(unittest.TestCase):
 				# we must be calling it from setup or nosetests but using at least
 				# nose's core etc
 				self.assertTrue('>' in s, msg="no '>' in %r" % s)
-			else:
-				self.assertFalse('>' in s, msg="'>' present in %r" % s)  # There is only "fail2ban-testcases" in this case, no true traceback
+			elif not ('coverage' in s):
+				# There is only "fail2ban-testcases" in this case, no true traceback
+				self.assertFalse('>' in s, msg="'>' present in %r" % s)
+
 			self.assertTrue(':' in s, msg="no ':' in %r" % s)
 
 	def testFormatterWithTraceBack(self):