From 3ec725a2ba50784ee3f17aa16e0e4eff63a89715 Mon Sep 17 00:00:00 2001 From: Simon Brown Date: Mon, 26 Oct 2015 17:35:38 -0700 Subject: [PATCH 01/52] Created file From https://github.com/beezwax/filemaker-fail2ban/blob/master/fail2ban/filter.d/screensharingd.conf --- config/filter.d/screensharingd.conf | 33 +++++++++++++++++++++++++++++ 1 file changed, 33 insertions(+) create mode 100644 config/filter.d/screensharingd.conf diff --git a/config/filter.d/screensharingd.conf b/config/filter.d/screensharingd.conf new file mode 100644 index 00000000..c0b5d32c --- /dev/null +++ b/config/filter.d/screensharingd.conf @@ -0,0 +1,33 @@ +# Fail2Ban configuration file +# +# Author: Simon Brown +# +# $Revision: 1 $ +# +# Filter for Mac OS X Screen Sharing service + +[INCLUDES] + +# Read common prefixes. If any customizations available -- read them from +# common.local +before = common.conf + + +[Definition] + +_daemon = screensharingd + +# Option: failregex +# Notes.: regex to match the password failures messages in the logfile. The +# host must be matched by a group named "host". The tag "" can +# be used for standard IP/hostname matching and is only an alias for +# (?:::f{4,6}:)?(?P[\w\-.^_]+) +# Values: TEXT +# +failregex = ^.+ screensharingd.+: Authentication: FAILED :: User Name: .+ :: Viewer Address: :: .*$ + +# Option: ignoreregex +# Notes.: regex to ignore. If this regex matches, the line is ignored. +# Values: TEXT +# +ignoreregex = From 80546c61642248445b5140bf5f156dca058cd858 Mon Sep 17 00:00:00 2001 From: Simon Brown Date: Mon, 26 Oct 2015 17:50:49 -0700 Subject: [PATCH 02/52] Added in settings for screensharingd filter --- config/jail.conf | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/config/jail.conf b/config/jail.conf index fd7f376e..e056ea3c 100644 --- a/config/jail.conf +++ b/config/jail.conf @@ -786,3 +786,10 @@ returntype = DROP bantime = 3600 maxretry = 1 findtime = 1 + +[screensharing] +# For Mac OS Screen Sharing Service +enabled = true +filter = screensharingd +logpath = %(system_log)s +maxretry = 4 From de14946542f0cbc8eece09329ca03418086c36da Mon Sep 17 00:00:00 2001 From: Simon Brown Date: Mon, 26 Oct 2015 18:02:07 -0700 Subject: [PATCH 03/52] Added new path variable for system.log Logging location for the majority of Mac OS daemons. --- config/paths-osx.conf | 2 ++ 1 file changed, 2 insertions(+) diff --git a/config/paths-osx.conf b/config/paths-osx.conf index d1b99b38..4f4df632 100644 --- a/config/paths-osx.conf +++ b/config/paths-osx.conf @@ -25,3 +25,5 @@ syslog_authpriv = /var/log/secure.log #syslog_local0 = +# Default Mac OS log location for syslog output. +system_log = /var/log/system.log From d17d837b8c855091f7cca3231c8ac05c4963553b Mon Sep 17 00:00:00 2001 From: Simon Brown Date: Tue, 27 Oct 2015 10:28:07 -0700 Subject: [PATCH 04/52] Update jail.conf Added logencoding to screensharing jail to avoid encoding error messages in fail2ban log --- config/jail.conf | 1 + 1 file changed, 1 insertion(+) diff --git a/config/jail.conf b/config/jail.conf index e056ea3c..2b9e9eaf 100644 --- a/config/jail.conf +++ b/config/jail.conf @@ -792,4 +792,5 @@ findtime = 1 enabled = true filter = screensharingd logpath = %(system_log)s +logencoding=utf-8 maxretry = 4 From 4c3f778b826248809273d1287271e599a1f0f64e Mon Sep 17 00:00:00 2001 From: Simon Brown Date: Tue, 27 Oct 2015 10:33:30 -0700 Subject: [PATCH 05/52] Replaced .* with literal Per Serg's suggestions. Possible I'm missing some auth attempt types, but I couldn't find anything where literal wasn't sufficient. --- config/filter.d/screensharingd.conf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config/filter.d/screensharingd.conf b/config/filter.d/screensharingd.conf index c0b5d32c..fc9f6aed 100644 --- a/config/filter.d/screensharingd.conf +++ b/config/filter.d/screensharingd.conf @@ -24,7 +24,7 @@ _daemon = screensharingd # (?:::f{4,6}:)?(?P[\w\-.^_]+) # Values: TEXT # -failregex = ^.+ screensharingd.+: Authentication: FAILED :: User Name: .+ :: Viewer Address: :: .*$ +failregex = ^.+ screensharingd.+: Authentication: FAILED :: User Name: .+ :: Viewer Address: :: Type: DH$ # Option: ignoreregex # Notes.: regex to ignore. If this regex matches, the line is ignored. From b3a18631e214ad970e87d2704e3d8027bb76359d Mon Sep 17 00:00:00 2001 From: Simon Brown Date: Tue, 27 Oct 2015 10:43:43 -0700 Subject: [PATCH 06/52] Sample log for test case --- fail2ban/tests/files/logs/screenshare | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 fail2ban/tests/files/logs/screenshare diff --git a/fail2ban/tests/files/logs/screenshare b/fail2ban/tests/files/logs/screenshare new file mode 100644 index 00000000..d3736f82 --- /dev/null +++ b/fail2ban/tests/files/logs/screenshare @@ -0,0 +1,5 @@ +Oct 27 09:24:46 test1.beezwax.net screensharingd[1170]: Authentication: SUCCEEDED :: User Name: simon :: Viewer Address: 192.168.5.247 :: Type: DH +Oct 27 09:25:41 test1.beezwax.net screensharingd[1170]: Authentication: FAILED :: User Name: sdfsdfs () mro :: Viewer Address: 192.168.5.247 :: Type: DH +Oct 27 09:25:49 test1.beezwax.net screensharingd[1170]: Authentication: FAILED :: User Name: sdfsdfs :: :: Viewer Address: 192.168.5.247 :: Type: DH +Oct 27 09:25:58 test1.beezwax.net screensharingd[1170]: Authentication: FAILED :: User Name: sdfsdfs _+ :: Viewer Address: 192.168.5.247 :: Type: DH +Oct 27 09:26:09 test1.beezwax.net screensharingd[1170]: Authentication: FAILED :: User Name: sdfsdfs @! s:: :: Viewer Address: 192.168.5.247 :: Type: DH From 3e4a77a5687550a47888720856d0b652cf6745d9 Mon Sep 17 00:00:00 2001 From: Simon Brown Date: Tue, 27 Oct 2015 12:31:51 -0700 Subject: [PATCH 07/52] Added json metadata --- fail2ban/tests/files/logs/screenshare | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/fail2ban/tests/files/logs/screenshare b/fail2ban/tests/files/logs/screenshare index d3736f82..73425b43 100644 --- a/fail2ban/tests/files/logs/screenshare +++ b/fail2ban/tests/files/logs/screenshare @@ -1,5 +1,14 @@ +# failJSON: { "time": "Oct 27 09:24:46", "match": false , "host": "192.168.5.247" } Oct 27 09:24:46 test1.beezwax.net screensharingd[1170]: Authentication: SUCCEEDED :: User Name: simon :: Viewer Address: 192.168.5.247 :: Type: DH +# failJSON: { "time": "Oct 27 09:25:41", "match": true , "host": "192.168.5.247" } Oct 27 09:25:41 test1.beezwax.net screensharingd[1170]: Authentication: FAILED :: User Name: sdfsdfs () mro :: Viewer Address: 192.168.5.247 :: Type: DH +# failJSON: { "time": "Oct 27 09:25:49", "match": true , "host": "192.168.5.247" } Oct 27 09:25:49 test1.beezwax.net screensharingd[1170]: Authentication: FAILED :: User Name: sdfsdfs :: :: Viewer Address: 192.168.5.247 :: Type: DH -Oct 27 09:25:58 test1.beezwax.net screensharingd[1170]: Authentication: FAILED :: User Name: sdfsdfs _+ :: Viewer Address: 192.168.5.247 :: Type: DH +# failJSON: { "time": "Oct 27 09:26:09", "match": true , "host": "192.168.5.247" } Oct 27 09:26:09 test1.beezwax.net screensharingd[1170]: Authentication: FAILED :: User Name: sdfsdfs @! s:: :: Viewer Address: 192.168.5.247 :: Type: DH +# failJSON: { "time": "Oct 27 09:23:20", "match": false , "host": "192.168.5.247" } +Oct 27 09:23:20 fm100 com.apple.xpc.launchd[1] (com.apple.xpc.launchd.domain.pid.SecurityAgent.1156): Path not allowed in target domain: type = uid, path = /System/Library/Frameworks/AppKit.framework/Versions/C/XPCServices/SandboxedServiceRunner.xpc/Contents/MacOS/SandboxedServiceRunner error = 1: Operation not permitted, origin = /System/Library/Frameworks/Security.framework/Versions/A/XPCServices/SecurityAgent.xpc +# failJSON: { "time": "Oct 27 09:23:20", "match": false , "host": "192.168.5.247" } +Oct 27 09:23:20 fm100.beezwax.net ManagedClient[1155]: MCXCCacheMCXRecordAndGraph(): vproc_swap_integer(NULL, VPROC_GSK_PERUSER_SUSPEND, &(uid=1027), NULL) failed +# failJSON: { "time": "Oct 27 12:26:44", "match": false , "host": "192.168.5.247" } +Oct 27 12:26:44 fm100.beezwax.net digest-service[3828]: digest-request: kdc failed with 36150275 proto=unknown From 6a5f10ee72eed02fb8aa66af6beb4c7e57fc6c97 Mon Sep 17 00:00:00 2001 From: Simon Brown Date: Tue, 27 Oct 2015 16:27:14 -0700 Subject: [PATCH 08/52] name change & new sample data changed name to match daemon, log samples with year --- fail2ban/tests/files/logs/screenshare | 14 -------------- fail2ban/tests/files/logs/screenshared | 18 ++++++++++++++++++ 2 files changed, 18 insertions(+), 14 deletions(-) delete mode 100644 fail2ban/tests/files/logs/screenshare create mode 100644 fail2ban/tests/files/logs/screenshared diff --git a/fail2ban/tests/files/logs/screenshare b/fail2ban/tests/files/logs/screenshare deleted file mode 100644 index 73425b43..00000000 --- a/fail2ban/tests/files/logs/screenshare +++ /dev/null @@ -1,14 +0,0 @@ -# failJSON: { "time": "Oct 27 09:24:46", "match": false , "host": "192.168.5.247" } -Oct 27 09:24:46 test1.beezwax.net screensharingd[1170]: Authentication: SUCCEEDED :: User Name: simon :: Viewer Address: 192.168.5.247 :: Type: DH -# failJSON: { "time": "Oct 27 09:25:41", "match": true , "host": "192.168.5.247" } -Oct 27 09:25:41 test1.beezwax.net screensharingd[1170]: Authentication: FAILED :: User Name: sdfsdfs () mro :: Viewer Address: 192.168.5.247 :: Type: DH -# failJSON: { "time": "Oct 27 09:25:49", "match": true , "host": "192.168.5.247" } -Oct 27 09:25:49 test1.beezwax.net screensharingd[1170]: Authentication: FAILED :: User Name: sdfsdfs :: :: Viewer Address: 192.168.5.247 :: Type: DH -# failJSON: { "time": "Oct 27 09:26:09", "match": true , "host": "192.168.5.247" } -Oct 27 09:26:09 test1.beezwax.net screensharingd[1170]: Authentication: FAILED :: User Name: sdfsdfs @! s:: :: Viewer Address: 192.168.5.247 :: Type: DH -# failJSON: { "time": "Oct 27 09:23:20", "match": false , "host": "192.168.5.247" } -Oct 27 09:23:20 fm100 com.apple.xpc.launchd[1] (com.apple.xpc.launchd.domain.pid.SecurityAgent.1156): Path not allowed in target domain: type = uid, path = /System/Library/Frameworks/AppKit.framework/Versions/C/XPCServices/SandboxedServiceRunner.xpc/Contents/MacOS/SandboxedServiceRunner error = 1: Operation not permitted, origin = /System/Library/Frameworks/Security.framework/Versions/A/XPCServices/SecurityAgent.xpc -# failJSON: { "time": "Oct 27 09:23:20", "match": false , "host": "192.168.5.247" } -Oct 27 09:23:20 fm100.beezwax.net ManagedClient[1155]: MCXCCacheMCXRecordAndGraph(): vproc_swap_integer(NULL, VPROC_GSK_PERUSER_SUSPEND, &(uid=1027), NULL) failed -# failJSON: { "time": "Oct 27 12:26:44", "match": false , "host": "192.168.5.247" } -Oct 27 12:26:44 fm100.beezwax.net digest-service[3828]: digest-request: kdc failed with 36150275 proto=unknown diff --git a/fail2ban/tests/files/logs/screenshared b/fail2ban/tests/files/logs/screenshared new file mode 100644 index 00000000..63e1dfb4 --- /dev/null +++ b/fail2ban/tests/files/logs/screenshared @@ -0,0 +1,18 @@ +# NOTE: dates here include years -- this is NOT the typical configuration for the system.log +# file on Mac OS. However, matches will not pass unless year is included. +# +# failJSON: { "match": false } +Oct 27 2015 09:24:46 test1.beezwax.net screensharingd[1170]: Authentication: SUCCEEDED :: User Name: simon :: Viewer Address: 192.168.5.247 :: Type: DH +# failJSON: { "match": false } +Oct 27 2015 09:23:20 test1 com.apple.xpc.launchd[1] (com.apple.xpc.launchd.domain.pid.SecurityAgent.1156): Path not allowed in target domain: type = uid, path = /System/Library/Frameworks/AppKit.framework/Versions/C/XPCServices/SandboxedServiceRunner.xpc/Contents/MacOS/SandboxedServiceRunner error = 1: Operation not permitted, origin = /System/Library/Frameworks/Security.framework/Versions/A/XPCServices/SecurityAgent.xpc +# failJSON: { "match": false } +Oct 27 2015 09:23:20 test1.beezwax.net ManagedClient[1155]: MCXCCacheMCXRecordAndGraph(): vproc_swap_integer(NULL, VPROC_GSK_PERUSER_SUSPEND, &(uid=1027), NULL) failed +# failJSON: { "match": false } +Oct 27 2015 12:26:44 test1.beezwax.net digest-service[3828]: digest-request: kdc failed with 36150275 proto=unknown +# +# failJSON: { "time": "2015-10-27T12:35:40", "match": true , "host": "192.168.5.247" } +Oct 27 2015 12:35:40 test1.beezwax.net screensharingd[1170]: Authentication: FAILED :: User Name: sdfsdfs () mro :: Viewer Address: 192.168.5.247 :: Type: DH +# failJSON: { "time": "2015-10-27T12:35:50", "match": true , "host": "192.168.5.247" } +Oct 27 2015 12:35:50 test1.beezwax.net screensharingd[1170]: Authentication: FAILED :: User Name: brown_s :: :: Viewer Address: 192.168.5.247 :: Type: DH +# failJSON: { "time": "2015-10-27T12:26:01", "match": true , "host": "192.168.5.247" } +Oct 27 2015 12:26:01 test1.beezwax.net screensharingd[1170]: Authentication: FAILED :: User Name: brown @! s:: :: Viewer Address: 192.168.5.247 :: Type: DH From 3dd1c305ce536d42fba397ac516abc3dd43d5f35 Mon Sep 17 00:00:00 2001 From: Simon Brown Date: Tue, 27 Oct 2015 21:20:12 -0700 Subject: [PATCH 09/52] added entry for new screensharingd filter --- ChangeLog | 2 ++ 1 file changed, 2 insertions(+) diff --git a/ChangeLog b/ChangeLog index 391eabc6..a6e53a02 100644 --- a/ChangeLog +++ b/ChangeLog @@ -39,6 +39,8 @@ ver. 0.9.4 (2015/XX/XXX) - wanna-be-released (gh-1226) * Added filter for openhab domotic software authentication failure with the rest api and web interface (gh-1223) + * Added filter for Mac OS screen sharing (VNC) daemon + ver. 0.9.3 (2015/08/01) - lets-all-stay-friends ---------- From 4b4d5a95b7af7aaca0c8bc9858bd78c6d3b75320 Mon Sep 17 00:00:00 2001 From: Simon Brown Date: Tue, 27 Oct 2015 21:30:20 -0700 Subject: [PATCH 10/52] Changed regex prequel Use standard prefix macro instead of literal daemon name. --- config/filter.d/screensharingd.conf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config/filter.d/screensharingd.conf b/config/filter.d/screensharingd.conf index fc9f6aed..fbaff542 100644 --- a/config/filter.d/screensharingd.conf +++ b/config/filter.d/screensharingd.conf @@ -24,7 +24,7 @@ _daemon = screensharingd # (?:::f{4,6}:)?(?P[\w\-.^_]+) # Values: TEXT # -failregex = ^.+ screensharingd.+: Authentication: FAILED :: User Name: .+ :: Viewer Address: :: Type: DH$ +failregex = ^%(__prefix_line)sAuthentication: Authentication: FAILED :: User Name: .+ :: Viewer Address: :: Type: DH$ # Option: ignoreregex # Notes.: regex to ignore. If this regex matches, the line is ignored. From acee68a9ee94a084576e91f74f226efb41b231d7 Mon Sep 17 00:00:00 2001 From: Simon Brown Date: Wed, 28 Oct 2015 15:11:11 -0700 Subject: [PATCH 11/52] Made screensharing jail off by default Also added note about requiring paths-osx.conf. --- config/jail.conf | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/config/jail.conf b/config/jail.conf index 2b9e9eaf..65ffdef1 100644 --- a/config/jail.conf +++ b/config/jail.conf @@ -788,8 +788,9 @@ maxretry = 1 findtime = 1 [screensharing] -# For Mac OS Screen Sharing Service -enabled = true +# For Mac OS Screen Sharing Service (VNC) +# Requires the 'before' statement in the [INCLUDE] section to include paths-osx.conf +enabled = false filter = screensharingd logpath = %(system_log)s logencoding=utf-8 From c936d19805aae62fa71fb8087313b889713901be Mon Sep 17 00:00:00 2001 From: Simon Brown Date: Wed, 28 Oct 2015 15:30:31 -0700 Subject: [PATCH 12/52] Fixed name (again?) --- fail2ban/tests/files/logs/{screenshared => screensharingd} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename fail2ban/tests/files/logs/{screenshared => screensharingd} (100%) diff --git a/fail2ban/tests/files/logs/screenshared b/fail2ban/tests/files/logs/screensharingd similarity index 100% rename from fail2ban/tests/files/logs/screenshared rename to fail2ban/tests/files/logs/screensharingd From bed28eaa62b6c1737c2030ce2ced56dc5ef20316 Mon Sep 17 00:00:00 2001 From: Simon Brown Date: Wed, 28 Oct 2015 15:32:58 -0700 Subject: [PATCH 13/52] clarified comments on sample log format --- fail2ban/tests/files/logs/screensharingd | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/fail2ban/tests/files/logs/screensharingd b/fail2ban/tests/files/logs/screensharingd index 63e1dfb4..c43a5c41 100644 --- a/fail2ban/tests/files/logs/screensharingd +++ b/fail2ban/tests/files/logs/screensharingd @@ -1,5 +1,5 @@ -# NOTE: dates here include years -- this is NOT the typical configuration for the system.log -# file on Mac OS. However, matches will not pass unless year is included. +# NOTE: dates here include years -- this is not the typical configuration for the system.log +# file on Mac OS. However, without it the test routines will use 2004 as the year and matches will not pass. # # failJSON: { "match": false } Oct 27 2015 09:24:46 test1.beezwax.net screensharingd[1170]: Authentication: SUCCEEDED :: User Name: simon :: Viewer Address: 192.168.5.247 :: Type: DH From cabd46f069d7fc585b02c0ac88e7069af202e419 Mon Sep 17 00:00:00 2001 From: Simon Brown Date: Wed, 28 Oct 2015 20:58:25 -0700 Subject: [PATCH 14/52] Fixed blatant typo in regex However, still failing test, even though ```PYTHONPATH=. fail2ban-regex -v fail2ban/tests/files/logs/screensharingd /etc/fail2ban/filter.d/screensharingd.conf``` gives desired result --- config/filter.d/screensharingd.conf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config/filter.d/screensharingd.conf b/config/filter.d/screensharingd.conf index fbaff542..c5e3e858 100644 --- a/config/filter.d/screensharingd.conf +++ b/config/filter.d/screensharingd.conf @@ -24,7 +24,7 @@ _daemon = screensharingd # (?:::f{4,6}:)?(?P[\w\-.^_]+) # Values: TEXT # -failregex = ^%(__prefix_line)sAuthentication: Authentication: FAILED :: User Name: .+ :: Viewer Address: :: Type: DH$ +failregex = ^%(__prefix_line)sAuthentication: FAILED :: User Name: .+ :: Viewer Address: :: Type: DH$ # Option: ignoreregex # Notes.: regex to ignore. If this regex matches, the line is ignored. From 65bc5cf6ba08f64fc13c03f5c44d7911a7f0179f Mon Sep 17 00:00:00 2001 From: Simon Brown Date: Thu, 29 Oct 2015 09:03:01 -0700 Subject: [PATCH 15/52] Now using a literal logpath for screensharing jail --- config/jail.conf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config/jail.conf b/config/jail.conf index 65ffdef1..7470039c 100644 --- a/config/jail.conf +++ b/config/jail.conf @@ -792,6 +792,6 @@ findtime = 1 # Requires the 'before' statement in the [INCLUDE] section to include paths-osx.conf enabled = false filter = screensharingd -logpath = %(system_log)s +logpath = /var/log/system.log logencoding=utf-8 maxretry = 4 From 5839a3bd80c73a5ff6a0f725d4453963505eb68a Mon Sep 17 00:00:00 2001 From: Simon Brown Date: Thu, 29 Oct 2015 16:07:54 -0700 Subject: [PATCH 16/52] Removed includes comment for screensharing jail --- config/jail.conf | 1 - 1 file changed, 1 deletion(-) diff --git a/config/jail.conf b/config/jail.conf index 7470039c..2aaea7ea 100644 --- a/config/jail.conf +++ b/config/jail.conf @@ -789,7 +789,6 @@ findtime = 1 [screensharing] # For Mac OS Screen Sharing Service (VNC) -# Requires the 'before' statement in the [INCLUDE] section to include paths-osx.conf enabled = false filter = screensharingd logpath = /var/log/system.log From d16ad805975afdade337dc052f88a198866efe3e Mon Sep 17 00:00:00 2001 From: Simon Brown Date: Mon, 2 Nov 2015 09:06:32 -0800 Subject: [PATCH 17/52] removed false matches For non-screensharingd related messages --- fail2ban/tests/files/logs/screensharingd | 6 ------ 1 file changed, 6 deletions(-) diff --git a/fail2ban/tests/files/logs/screensharingd b/fail2ban/tests/files/logs/screensharingd index c43a5c41..0ec0ebd6 100644 --- a/fail2ban/tests/files/logs/screensharingd +++ b/fail2ban/tests/files/logs/screensharingd @@ -3,12 +3,6 @@ # # failJSON: { "match": false } Oct 27 2015 09:24:46 test1.beezwax.net screensharingd[1170]: Authentication: SUCCEEDED :: User Name: simon :: Viewer Address: 192.168.5.247 :: Type: DH -# failJSON: { "match": false } -Oct 27 2015 09:23:20 test1 com.apple.xpc.launchd[1] (com.apple.xpc.launchd.domain.pid.SecurityAgent.1156): Path not allowed in target domain: type = uid, path = /System/Library/Frameworks/AppKit.framework/Versions/C/XPCServices/SandboxedServiceRunner.xpc/Contents/MacOS/SandboxedServiceRunner error = 1: Operation not permitted, origin = /System/Library/Frameworks/Security.framework/Versions/A/XPCServices/SecurityAgent.xpc -# failJSON: { "match": false } -Oct 27 2015 09:23:20 test1.beezwax.net ManagedClient[1155]: MCXCCacheMCXRecordAndGraph(): vproc_swap_integer(NULL, VPROC_GSK_PERUSER_SUSPEND, &(uid=1027), NULL) failed -# failJSON: { "match": false } -Oct 27 2015 12:26:44 test1.beezwax.net digest-service[3828]: digest-request: kdc failed with 36150275 proto=unknown # # failJSON: { "time": "2015-10-27T12:35:40", "match": true , "host": "192.168.5.247" } Oct 27 2015 12:35:40 test1.beezwax.net screensharingd[1170]: Authentication: FAILED :: User Name: sdfsdfs () mro :: Viewer Address: 192.168.5.247 :: Type: DH From 3e16f33dbe840d507b687517699072a5e100101c Mon Sep 17 00:00:00 2001 From: Simon Brown Date: Mon, 2 Nov 2015 09:08:47 -0800 Subject: [PATCH 18/52] Removed old svn revision comment --- config/filter.d/screensharingd.conf | 2 -- 1 file changed, 2 deletions(-) diff --git a/config/filter.d/screensharingd.conf b/config/filter.d/screensharingd.conf index c5e3e858..4cd76465 100644 --- a/config/filter.d/screensharingd.conf +++ b/config/filter.d/screensharingd.conf @@ -2,8 +2,6 @@ # # Author: Simon Brown # -# $Revision: 1 $ -# # Filter for Mac OS X Screen Sharing service [INCLUDES] From 69bb532db0c9c34bf7c68ba0ed7adb769206f58d Mon Sep 17 00:00:00 2001 From: Simon Brown Date: Mon, 2 Nov 2015 09:26:45 -0800 Subject: [PATCH 19/52] removed system.log --- config/paths-osx.conf | 3 --- 1 file changed, 3 deletions(-) diff --git a/config/paths-osx.conf b/config/paths-osx.conf index 4f4df632..180923c0 100644 --- a/config/paths-osx.conf +++ b/config/paths-osx.conf @@ -24,6 +24,3 @@ syslog_authpriv = /var/log/secure.log #syslog_daemon = #syslog_local0 = - -# Default Mac OS log location for syslog output. -system_log = /var/log/system.log From dfaf82d68a39662e475b3a0403a64de04fda7e76 Mon Sep 17 00:00:00 2001 From: Yaroslav Halchenko Date: Fri, 18 Dec 2015 09:23:12 -0500 Subject: [PATCH 20/52] Changelog entry for PartOf in .service fix --- ChangeLog | 3 +++ 1 file changed, 3 insertions(+) diff --git a/ChangeLog b/ChangeLog index 90bcfcb3..74a88115 100644 --- a/ChangeLog +++ b/ChangeLog @@ -68,6 +68,9 @@ ver. 0.9.4 (2015/XX/XXX) - wanna-be-released * Performance improvements while monitoring large number of files (gh-1265). Use associative array (dict) for monitored log files to speed up lookup operations. Thanks @kshetragia + * Specified that fail2ban is PartOf iptables.service firewalld.service in + .service file -- would reload fail2ban if those services are restarted + ver. 0.9.3 (2015/08/01) - lets-all-stay-friends ---------- From 28c98322933f36d5ba4ea12ea638fb573b53dbc6 Mon Sep 17 00:00:00 2001 From: Yaroslav Halchenko Date: Tue, 29 Dec 2015 19:43:52 -0500 Subject: [PATCH 21/52] RF: harmonize jail.conf (no explicit enabled=false in jails, match filter name for screesharingd, etc) --- config/jail.conf | 26 ++++++-------------------- 1 file changed, 6 insertions(+), 20 deletions(-) diff --git a/config/jail.conf b/config/jail.conf index 02543705..41a43e56 100644 --- a/config/jail.conf +++ b/config/jail.conf @@ -240,7 +240,6 @@ backend = %(dropbear_backend)s port = ssh logpath = %(auditd_log)s -maxretry = 5 # @@ -266,7 +265,6 @@ maxretry = 1 port = http,https logpath = %(apache_error_log)s -maxretry = 6 [apache-overflows] @@ -304,18 +302,21 @@ port = http,https logpath = %(apache_error_log)s maxretry = 2 + [apache-shellshock] port = http,https logpath = %(apache_error_log)s maxretry = 1 + [openhab-auth] filter = openhab action = iptables-allports[name=NoAuthFailures] logpath = /opt/openhab/logs/request.log + [nginx-http-auth] port = http,https @@ -335,6 +336,7 @@ port = http,https logpath = %(nginx_error_log)s maxretry = 2 + # Ban attackers that try to use PHP's URL-fopen() functionality # through GET/POST variables. - Experimental, with more than a year # of usage in production environments. @@ -399,7 +401,6 @@ logpath = /var/log/sogo/sogo.log logpath = /var/log/tine20/tine20.log port = http,https -maxretry = 5 # @@ -420,7 +421,6 @@ logpath = /var/log/tomcat*/catalina.out [monit] #Ban clients brute-forcing the monit gui login -filter = monit port = 2812 logpath = /var/log/monit @@ -473,7 +473,6 @@ backend = %(proftpd_backend)s port = ftp,ftp-data,ftps,ftps-data logpath = %(pureftpd_log)s backend = %(pureftpd_backend)s -maxretry = 6 [gssftpd] @@ -481,7 +480,6 @@ maxretry = 6 port = ftp,ftp-data,ftps,ftps-data logpath = %(syslog_daemon)s backend = %(syslog_backend)s -maxretry = 6 [wuftpd] @@ -489,7 +487,6 @@ maxretry = 6 port = ftp,ftp-data,ftps,ftps-data logpath = %(wuftpd_log)s backend = %(wuftpd_backend)s -maxretry = 6 [vsftpd] @@ -724,7 +721,6 @@ maxretry = 10 port = 3306 logpath = %(mysql_log)s backend = %(mysql_backend)s -maxretry = 5 # Jail for more extended banning of persistent abusers @@ -740,7 +736,6 @@ logpath = /var/log/fail2ban.log banaction = %(banaction_allports)s bantime = 604800 ; 1 week findtime = 86400 ; 1 day -maxretry = 5 # Generic filter for PAM. Has to be used with action which bans all @@ -786,7 +781,6 @@ action = %(banaction)s[name=%(__name__)s-tcp, port="%(tcpport)s", protocol="tcp # nobody except your own Nagios server should ever probe nrpe [nagios] -enabled = false logpath = %(syslog_daemon)s ; nrpe.cfg may define a different log_facility backend = %(syslog_backend)s maxretry = 1 @@ -794,18 +788,14 @@ maxretry = 1 [oracleims] # see "oracleims" filter file for configuration requirement for Oracle IMS v6 and above -enabled = false logpath = /opt/sun/comms/messaging64/log/mail.log_current -maxretry = 6 banaction = %(banaction_allports)s [directadmin] -enabled = false logpath = /var/log/directadmin/login.log port = 2222 [portsentry] -enabled = false logpath = /var/lib/portsentry/portsentry.history maxretry = 1 @@ -826,16 +816,12 @@ findtime = 1 [murmur] # AKA mumble-server port = 64738 -filter = murmur action = %(banaction)s[name=%(__name__)s-tcp, port="%(port)s", protocol=tcp, chain="%(chain)s", actname=%(banaction)s-tcp] %(banaction)s[name=%(__name__)s-udp, port="%(port)s", protocol=udp, chain="%(chain)s", actname=%(banaction)s-udp] logpath = /var/log/mumble-server/mumble-server.log -[screensharing] +[screensharingd] # For Mac OS Screen Sharing Service (VNC) -enabled = false -filter = screensharingd logpath = /var/log/system.log -logencoding=utf-8 -maxretry = 4 +logencoding = utf-8 From b76aede40d4202bdf5b5c8908162429b30370e33 Mon Sep 17 00:00:00 2001 From: Yaroslav Halchenko Date: Tue, 29 Dec 2015 19:56:45 -0500 Subject: [PATCH 22/52] ENH(TST): verify that passed bantime is non-0 and int --- fail2ban/tests/clientreadertestcase.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/fail2ban/tests/clientreadertestcase.py b/fail2ban/tests/clientreadertestcase.py index d19090be..e6860a47 100644 --- a/fail2ban/tests/clientreadertestcase.py +++ b/fail2ban/tests/clientreadertestcase.py @@ -594,6 +594,12 @@ class JailsReaderTest(LogCaptureTestCase): # by default we have lots of jails ;) self.assertTrue(len(comm_commands)) + # some common sanity checks for commands + for command in comm_commands: + if len(command) >= 3 and [command[0], command[2]] == ['set', 'bantime']: + self.assertTrue(isinstance(command[3], int)) + self.assertTrue(command[3] > 0) + # and we know even some of them by heart for j in ['sshd', 'recidive']: # by default we have 'auto' backend ATM From cf334421bdd287e4b5fe81344d50386f1f2d8f4a Mon Sep 17 00:00:00 2001 From: sebres Date: Wed, 30 Dec 2015 20:17:12 +0100 Subject: [PATCH 23/52] Provides fail2ban version to jail (as interpolation variable during parse of jail.conf); BF: use `fail2ban_agent` as user-agent in actions badips, blocklist_de, etc. (closes #1271, closes #1272) --- config/action.d/badips.conf | 2 +- config/action.d/blocklist_de.conf | 2 +- config/action.d/mynetwatchman.conf | 8 +++++-- config/jail.conf | 11 +++++++-- fail2ban/client/jailreader.py | 5 ++++ fail2ban/tests/clientreadertestcase.py | 32 +++++++++++++++++++++++++- 6 files changed, 53 insertions(+), 7 deletions(-) diff --git a/config/action.d/badips.conf b/config/action.d/badips.conf index 4a5c0f97..70b46546 100644 --- a/config/action.d/badips.conf +++ b/config/action.d/badips.conf @@ -10,7 +10,7 @@ [Definition] -actionban = curl --fail --user-agent "fail2ban v0.8.12" http://www.badips.com/add// +actionban = curl --fail --user-agent "" http://www.badips.com/add// [Init] diff --git a/config/action.d/blocklist_de.conf b/config/action.d/blocklist_de.conf index 6d520694..2f31d8b9 100644 --- a/config/action.d/blocklist_de.conf +++ b/config/action.d/blocklist_de.conf @@ -54,7 +54,7 @@ actioncheck = # Tags: See jail.conf(5) man page # Values: CMD # -actionban = curl --fail --data-urlencode 'server=' --data 'apikey=' --data 'service=' --data 'ip=' --data-urlencode 'logs=' --data 'format=text' --user-agent "fail2ban v0.8.12" "https://www.blocklist.de/en/httpreports.html" +actionban = curl --fail --data-urlencode 'server=' --data 'apikey=' --data 'service=' --data 'ip=' --data-urlencode 'logs=' --data 'format=text' --user-agent "" "https://www.blocklist.de/en/httpreports.html" # Option: actionunban # Notes.: command executed when unbanning an IP. Take care that the diff --git a/config/action.d/mynetwatchman.conf b/config/action.d/mynetwatchman.conf index 5245a4e3..8f3edf9e 100644 --- a/config/action.d/mynetwatchman.conf +++ b/config/action.d/mynetwatchman.conf @@ -111,13 +111,17 @@ myip = `ip -4 addr show dev eth0 | grep inet | head -n 1 | sed -r 's/.*inet ([0- # protocol = tcp +# Option: agent +# Default: Fail2ban +agent = Fail2ban + # Option: getcmd # Notes.: A command to fetch a URL. Should output page to STDOUT # Values: CMD Default: wget # -getcmd = wget --no-verbose --tries=3 --waitretry=10 --connect-timeout=10 --read-timeout=60 --retry-connrefused --output-document=- --user-agent=Fail2Ban +getcmd = wget --no-verbose --tries=3 --waitretry=10 --connect-timeout=10 --read-timeout=60 --retry-connrefused --output-document=- --user-agent= # Alternative value: -# getcmd = curl --silent --show-error --retry 3 --connect-timeout 10 --max-time 60 --user-agent Fail2Ban +# getcmd = curl --silent --show-error --retry 3 --connect-timeout 10 --max-time 60 --user-agent # Option: srcport # Notes.: The source port of the attack. You're unlikely to have this info, so diff --git a/config/jail.conf b/config/jail.conf index 02543705..1d560288 100644 --- a/config/jail.conf +++ b/config/jail.conf @@ -146,6 +146,9 @@ chain = INPUT # Usually should be overridden in a particular jail port = 0:65535 +# Format of user-agent https://tools.ietf.org/html/rfc7231#section-5.5.3 +fail2ban_agent = Fail2Ban/%(fail2ban_version)s + # # Action shortcuts. To be used to define action parameter @@ -187,7 +190,7 @@ action_cf_mwl = cloudflare[cfuser="%(cfemail)s", cftoken="%(cfapikey)s"] # [Init] # blocklist_de_apikey = {api key from registration] # -action_blocklist_de = blocklist_de[email="%(sender)s", service=%(filter)s, apikey="%(blocklist_de_apikey)s"] +action_blocklist_de = blocklist_de[email="%(sender)s", service=%(filter)s, apikey="%(blocklist_de_apikey)s", agent="%(fail2ban_agent)s"] # Report ban via badips.com, and use as blacklist # @@ -197,7 +200,11 @@ action_blocklist_de = blocklist_de[email="%(sender)s", service=%(filter)s, apik # NOTE: This action relies on banaction being present on start and therefore # should be last action defined for a jail. # -action_badips = badips.py[category="%(name)s", banaction="%(banaction)s"] +action_badips = badips.py[category="%(__name__)s", banaction="%(banaction)s", agent="%(fail2ban_agent)s"] +# +# Report ban via badips.com (uses action.d/badips.conf for reporting only) +# +action_badips_report = badips[category="%(__name__)s", agent="%(fail2ban_agent)s"] # Choose default action. To change, just override value of 'action' with the # interpolation to the chosen action shortcut (e.g. action_mw, action_mwl, etc) in jail.local diff --git a/fail2ban/client/jailreader.py b/fail2ban/client/jailreader.py index 54ac59fa..56b8889c 100644 --- a/fail2ban/client/jailreader.py +++ b/fail2ban/client/jailreader.py @@ -32,6 +32,7 @@ import re from .configreader import ConfigReaderUnshared, ConfigReader from .filterreader import FilterReader from .actionreader import ActionReader +from ..version import version from ..helpers import getLogger from ..helpers import splitcommaspace @@ -108,6 +109,10 @@ class JailReader(ConfigReader): ["string", "filter", ""], ["string", "action", ""]] + # Before interpolation (substitution) add static options always available as default: + defsec = self._cfg.get_defaults() + defsec["fail2ban_version"] = version + # Read first options only needed for merge defaults ('known/...' from filter): self.__opts = ConfigReader.getOptions(self, self.__name, opts1st) if not self.__opts: diff --git a/fail2ban/tests/clientreadertestcase.py b/fail2ban/tests/clientreadertestcase.py index d19090be..fcabc727 100644 --- a/fail2ban/tests/clientreadertestcase.py +++ b/fail2ban/tests/clientreadertestcase.py @@ -28,18 +28,20 @@ import re import shutil import tempfile import unittest -from ..client.configreader import ConfigReaderUnshared +from ..client.configreader import ConfigReader, ConfigReaderUnshared from ..client import configparserinc from ..client.jailreader import JailReader from ..client.filterreader import FilterReader from ..client.jailsreader import JailsReader from ..client.actionreader import ActionReader from ..client.configurator import Configurator +from ..version import version from .utils import LogCaptureTestCase TEST_FILES_DIR = os.path.join(os.path.dirname(__file__), "files") from .utils import CONFIG_DIR +CONFIG_DIR_TESTSHARE_CFG = {} STOCK = os.path.exists(os.path.join('config','fail2ban.conf')) @@ -251,6 +253,34 @@ class JailReaderTest(LogCaptureTestCase): result = JailReader.extractOptions(option) self.assertEqual(expected, result) + def testVersionAgent(self): + jail = JailReader('blocklisttest', force_enable=True, basedir=CONFIG_DIR) + # emulate jail.read(), because such jail not exists: + ConfigReader.read(jail, "jail"); + sections = jail._cfg.get_sections() + sections['blocklisttest'] = dict((('__name__', 'blocklisttest'), + ('filter', ''), ('failregex', '^test $'), + ('sender', 'f2b-test@example.com'), ('blocklist_de_apikey', 'test-key'), + ('action', + '%(action_blocklist_de)s\n' + '%(action_badips_report)s\n' + '%(action_badips)s\n' + 'mynetwatchman[port=1234,protocol=udp,agent="%(fail2ban_agent)s"]' + ), + )) + # get options: + self.assertTrue(jail.getOptions()) + # convert and get stream + stream = jail.convert() + # get action and retrieve agent from it, compare with agent saved in version: + act = [o for o in stream if len(o) > 4 and (o[4] == 'agent' or o[4].endswith('badips.py'))] + useragent = 'Fail2Ban/%s' % version + self.assertEqual(len(act), 4) + self.assertEqual(act[0], ['set', 'blocklisttest', 'action', 'blocklist_de', 'agent', useragent]) + self.assertEqual(act[1], ['set', 'blocklisttest', 'action', 'badips', 'agent', useragent]) + self.assertEqual(eval(act[2][5]).get('agent', ''), useragent) + self.assertEqual(act[3], ['set', 'blocklisttest', 'action', 'mynetwatchman', 'agent', useragent]) + def testGlob(self): d = tempfile.mkdtemp(prefix="f2b-temp") # Generate few files From e133762a282e8059ad89b5130a3123ea6e0f526f Mon Sep 17 00:00:00 2001 From: Jordan Moeser Date: Thu, 31 Dec 2015 11:16:23 +1000 Subject: [PATCH 24/52] Added HAProxy HTTP Auth filter --- ChangeLog | 2 ++ config/filter.d/haproxy-http-auth.conf | 37 +++++++++++++++++++++ config/jail.conf | 9 +++++ fail2ban/tests/files/logs/haproxy-http-auth | 4 +++ 4 files changed, 52 insertions(+) create mode 100644 config/filter.d/haproxy-http-auth.conf create mode 100644 fail2ban/tests/files/logs/haproxy-http-auth diff --git a/ChangeLog b/ChangeLog index 8b5344be..81575f62 100644 --- a/ChangeLog +++ b/ChangeLog @@ -44,6 +44,8 @@ ver. 0.9.4 (2015/XX/XXX) - wanna-be-released request processing rate (ngx_http_limit_req_module) - murmur - ban hosts that repeatedly attempt to connect to murmur/mumble-server with an invalid server password or certificate. + - haproxy-http-auth - filter to match failed HTTP Authentications against a + HAProxy server * New jails: - murmur - bans TCP and UDP from the bad host on the default murmur port. * sshd filter got new failregex to match "maximum authentication diff --git a/config/filter.d/haproxy-http-auth.conf b/config/filter.d/haproxy-http-auth.conf new file mode 100644 index 00000000..7c371dde --- /dev/null +++ b/config/filter.d/haproxy-http-auth.conf @@ -0,0 +1,37 @@ +# Fail2Ban filter configuration file to match failed login attempts to +# HAProxy HTTP Authentication protected servers. +# +# PLEASE NOTE - When a user first hits the HTTP Auth a 401 is returned by the server +# which prompts their browser to ask for login details. +# This initial 401 is logged by HAProxy. +# In other words, even successful logins will have at least 1 fail regex match. +# Please keep this in mind when setting findtime and maxretry for jails. +# +# Author: Jordan Moeser +# + +[INCLUDES] + +# Read common prefixes. If any customizations available -- read them from +# common.local +before = common.conf + + +[Definition] + +_daemon = haproxy + +# Option: failregex +# Notes.: regex to match the password failures messages in the logfile. The +# host must be matched by a group named "host". The tag "" can +# be used for standard IP/hostname matching and is only an alias for +# (?:::f{4,6}:)?(?P[\w\-.^_]+) +# Values: TEXT +# +failregex = ^%(__prefix_line)s.*NOSRV.*401 + +# Option: ignoreregex +# Notes.: regex to ignore. If this regex matches, the line is ignored. +# Values: TEXT +# +ignoreregex = diff --git a/config/jail.conf b/config/jail.conf index 02543705..9788bd25 100644 --- a/config/jail.conf +++ b/config/jail.conf @@ -839,3 +839,12 @@ filter = screensharingd logpath = /var/log/system.log logencoding=utf-8 maxretry = 4 + +[haproxy-http-auth] +# HAProxy by default doesn't log to file you'll need to set it up to forward +# logs to a syslog server which would then write them to disk. +# See "haproxy-http-auth" filter for a brief cautionary note when setting +# maxretry and findtime. +enabled = false +filter = haproxy-http-auth +logpath = /var/log/haproxy.log diff --git a/fail2ban/tests/files/logs/haproxy-http-auth b/fail2ban/tests/files/logs/haproxy-http-auth new file mode 100644 index 00000000..298f1972 --- /dev/null +++ b/fail2ban/tests/files/logs/haproxy-http-auth @@ -0,0 +1,4 @@ +# failJSON: { "match": false } +Nov 14 22:45:27 test haproxy[760]: 192.168.33.1:58444 [14/Nov/2015:22:45:25.439] main app/app1 1939/0/1/0/1940 403 5168 - - ---- 3/3/0/0/0 0/0 "GET / HTTP/1.1" +# failJSON: { "time": "2004-11-14T22:45:11", "match": true , "host": "192.168.33.1" } +Nov 14 22:45:11 test haproxy[760]: 192.168.33.1:58430 [14/Nov/2015:22:45:11.608] main main/ -1/-1/-1/-1/0 401 248 - - PR-- 0/0/0/0/0 0/0 "GET / HTTP/1.1" From ac311214329c3e8100ba9b37091de01475dd522f Mon Sep 17 00:00:00 2001 From: sebres Date: Thu, 31 Dec 2015 02:30:25 +0100 Subject: [PATCH 25/52] amend to fix fail2ban-version: correct user-agent for badips.py "Fail2Ban/ver", changeable within jail/config now; --- config/action.d/badips.py | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/config/action.d/badips.py b/config/action.d/badips.py index 99e1866a..5248e994 100644 --- a/config/action.d/badips.py +++ b/config/action.d/badips.py @@ -21,7 +21,6 @@ import sys if sys.version_info < (2, 7): raise ImportError("badips.py action requires Python >= 2.7") import json -from functools import partial import threading import logging if sys.version_info >= (3, ): @@ -72,6 +71,9 @@ class BadIPsAction(ActionBase): updateperiod : int, optional Time in seconds between updating bad IPs blacklist. Default 900 (15 minutes) + agent : str, optional + User agent transmitted to server. + Default `Fail2Ban/ver.` Raises ------ @@ -80,13 +82,14 @@ class BadIPsAction(ActionBase): """ _badips = "http://www.badips.com" - _Request = partial( - Request, headers={'User-Agent': "Fail2Ban %s" % f2bVersion}) + def _Request(self, url, **argv): + return Request(url, headers={'User-Agent': self.agent}, **argv) def __init__(self, jail, name, category, score=3, age="24h", key=None, - banaction=None, bancategory=None, bankey=None, updateperiod=900): + banaction=None, bancategory=None, bankey=None, updateperiod=900, agent=None): super(BadIPsAction, self).__init__(jail, name) + self.agent = agent if agent is not None else ("Fail2Ban/%s" % f2bVersion) self.category = category self.score = score self.age = age From 618e97bce8a1ad359e191d84835f3400152acbed Mon Sep 17 00:00:00 2001 From: Alexander Belykh Date: Mon, 4 Jan 2016 01:36:28 +0600 Subject: [PATCH 26/52] Add nftables actions --- config/action.d/nftables-allports.conf | 52 +++++++++++++++++++++++++ config/action.d/nftables-common.conf | 50 ++++++++++++++++++++++++ config/action.d/nftables-multiport.conf | 51 ++++++++++++++++++++++++ 3 files changed, 153 insertions(+) create mode 100644 config/action.d/nftables-allports.conf create mode 100644 config/action.d/nftables-common.conf create mode 100644 config/action.d/nftables-multiport.conf diff --git a/config/action.d/nftables-allports.conf b/config/action.d/nftables-allports.conf new file mode 100644 index 00000000..34622f65 --- /dev/null +++ b/config/action.d/nftables-allports.conf @@ -0,0 +1,52 @@ +# Fail2Ban configuration file +# +# Author: Cyril Jaquier +# Modified: Yaroslav O. Halchenko +# made active on all ports from original iptables.conf +# Modified: Alexander Belykh +# adapted for nftables +# + +[INCLUDES] + +before = nftables-common.conf + +[Definition] + +# Option: actionstart +# Notes.: command executed once at the start of Fail2Ban. +# Values: CMD +# +actionstart = add set filter f2b- { type ipv4_addr\; } + insert rule filter ip protocol ip saddr @f2b- + +# Option: actionstop +# Notes.: command executed once at the end of Fail2Ban +# Values: CMD +# +actionstop = HANDLE_ID=$( --handle --numeric list chain filter | grep -m1 'ip saddr @f2b- # handle' | grep -oe ' handle [0-9]*'); delete rule filter $HANDLE_ID + delete set filter f2b- +# Option: actioncheck +# Notes.: command executed once before each actionban command +# Values: CMD +# +actioncheck = list chain filter | grep -q '@f2b-[ \t]' + +# Option: actionban +# Notes.: command executed when banning an IP. Take care that the +# command is executed with Fail2Ban user rights. +# Tags: See jail.conf(5) man page +# Values: CMD +# +actionban = add element filter f2b- { } + +# Option: actionunban +# Notes.: command executed when unbanning an IP. Take care that the +# command is executed with Fail2Ban user rights. +# Tags: See jail.conf(5) man page +# Values: CMD +# +actionunban = delete element filter f2b- { } + +[Init] + diff --git a/config/action.d/nftables-common.conf b/config/action.d/nftables-common.conf new file mode 100644 index 00000000..a0534d35 --- /dev/null +++ b/config/action.d/nftables-common.conf @@ -0,0 +1,50 @@ +# Fail2Ban configuration file +# +# Author: Daniel Black +# Modified: Alexander Belykh +# adapted for nftables +# +# This is a included configuration file and includes the definitions for the nftables +# used in all nftables based actions by default. +# +# The user can override the defaults in nftables-common.local + +[INCLUDES] + +after = nftables-common.local + +[Init] + +# Option: chain +# Notes specifies the nftables chain to which the Fail2Ban rules should be +# added +# Values: STRING Default: input +chain = input + +# Default name of the filtering set +# +name = default + +# Option: port +# Notes.: specifies port to monitor +# Values: [ NUM | STRING ] Default: +# +port = ssh + +# Option: protocol +# Notes.: internally used by config reader for interpolations. +# Values: [ tcp | udp ] Default: tcp +# +protocol = tcp + +# Option: blocktype +# Note: This is what the action does with rules. This can be any jump target +# as per the nftables man page (section 8). Common values are drop +# reject, reject with icmp type host-unreachable +# Values: STRING +blocktype = reject + +# Option: nftables +# Notes.: Actual command to be executed, including common to all calls options +# Values: STRING +nftables = nft diff --git a/config/action.d/nftables-multiport.conf b/config/action.d/nftables-multiport.conf new file mode 100644 index 00000000..ad61bf63 --- /dev/null +++ b/config/action.d/nftables-multiport.conf @@ -0,0 +1,51 @@ +# Fail2Ban configuration file +# +# Author: Cyril Jaquier +# Modified by Yaroslav Halchenko for multiport banning +# Modified: Alexander Belykh +# adapted for nftables +# + +[INCLUDES] + +before = nftables-common.conf + +[Definition] + +# Option: actionstart +# Notes.: command executed once at the start of Fail2Ban. +# Values: CMD +# +actionstart = add set filter f2b- { type ipv4_addr\; } + insert rule filter dport { } ip saddr @f2b- + +# Option: actionstop +# Notes.: command executed once at the end of Fail2Ban +# Values: CMD +# +actionstop = HANDLE_ID=$( --handle --numeric list chain filter | grep -m1 'ip saddr @f2b- # handle' | grep -oe ' handle [0-9]*'); delete rule filter $HANDLE_ID + delete set filter f2b- +# Option: actioncheck +# Notes.: command executed once before each actionban command +# Values: CMD +# +actioncheck = list chain filter | grep -q '@f2b-[ \t]' + +# Option: actionban +# Notes.: command executed when banning an IP. Take care that the +# command is executed with Fail2Ban user rights. +# Tags: See jail.conf(5) man page +# Values: CMD +# +actionban = add element filter f2b- { } + +# Option: actionunban +# Notes.: command executed when unbanning an IP. Take care that the +# command is executed with Fail2Ban user rights. +# Tags: See jail.conf(5) man page +# Values: CMD +# +actionunban = delete element filter f2b- { } + +[Init] + From 69f5623f83180dda4e9461f553fd676f029b3ba9 Mon Sep 17 00:00:00 2001 From: sebres Date: Mon, 4 Jan 2016 09:30:32 +0100 Subject: [PATCH 27/52] code simplifying (remove duplication): agent will be always supplied as parameter from jail.conf --- config/action.d/badips.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/config/action.d/badips.py b/config/action.d/badips.py index 5248e994..025289ca 100644 --- a/config/action.d/badips.py +++ b/config/action.d/badips.py @@ -32,7 +32,6 @@ else: from urllib import urlencode from fail2ban.server.actions import ActionBase -from fail2ban.version import version as f2bVersion class BadIPsAction(ActionBase): @@ -86,10 +85,10 @@ class BadIPsAction(ActionBase): return Request(url, headers={'User-Agent': self.agent}, **argv) def __init__(self, jail, name, category, score=3, age="24h", key=None, - banaction=None, bancategory=None, bankey=None, updateperiod=900, agent=None): + banaction=None, bancategory=None, bankey=None, updateperiod=900, agent="Fail2Ban"): super(BadIPsAction, self).__init__(jail, name) - self.agent = agent if agent is not None else ("Fail2Ban/%s" % f2bVersion) + self.agent = agent self.category = category self.score = score self.age = age From 25a09352e48a303d66d1e207e2e3b51b3d2e9793 Mon Sep 17 00:00:00 2001 From: sebres Date: Mon, 4 Jan 2016 14:46:43 +0100 Subject: [PATCH 28/52] + ChangeLog entry --- ChangeLog | 3 +++ 1 file changed, 3 insertions(+) diff --git a/ChangeLog b/ChangeLog index 8b5344be..36c727fe 100644 --- a/ChangeLog +++ b/ChangeLog @@ -28,6 +28,7 @@ ver. 0.9.4 (2015/XX/XXX) - wanna-be-released for python version < 3.x (gh-1248) * Use postfix_log logpath for postfix-rbl jail * filters.d/postfix.conf - add 'Sender address rejected: Domain not found' failregex + * use `fail2ban_agent` as user-agent in actions badips, blocklist_de, etc (gh-1271) - New Features: * New interpolation feature for definition config readers - `` @@ -72,6 +73,8 @@ ver. 0.9.4 (2015/XX/XXX) - wanna-be-released operations. Thanks @kshetragia * Specified that fail2ban is PartOf iptables.service firewalld.service in .service file -- would reload fail2ban if those services are restarted + * Provides new default `fail2ban_version` and interpolation variable + `fail2ban_agent` in jail.conf ver. 0.9.3 (2015/08/01) - lets-all-stay-friends From f7f91a8bd45d83fa106c2c827e91d53d1e9b75dc Mon Sep 17 00:00:00 2001 From: Alexander Belykh Date: Tue, 5 Jan 2016 19:03:47 +0600 Subject: [PATCH 29/52] Refactor common code out of nftables-multiport/allports.conf --- config/action.d/nftables-allports.conf | 38 +++----------------- config/action.d/nftables-common.conf | 46 +++++++++++++++++++++++++ config/action.d/nftables-multiport.conf | 41 ++++------------------ 3 files changed, 56 insertions(+), 69 deletions(-) diff --git a/config/action.d/nftables-allports.conf b/config/action.d/nftables-allports.conf index 34622f65..afd0ca84 100644 --- a/config/action.d/nftables-allports.conf +++ b/config/action.d/nftables-allports.conf @@ -13,40 +13,10 @@ before = nftables-common.conf [Definition] -# Option: actionstart -# Notes.: command executed once at the start of Fail2Ban. -# Values: CMD +# Option: nftables_mode +# Notes.: additional expressions for nftables filter rule +# Values: nftables expressions # -actionstart = add set filter f2b- { type ipv4_addr\; } - insert rule filter ip protocol ip saddr @f2b- - -# Option: actionstop -# Notes.: command executed once at the end of Fail2Ban -# Values: CMD -# -actionstop = HANDLE_ID=$( --handle --numeric list chain filter | grep -m1 'ip saddr @f2b- # handle' | grep -oe ' handle [0-9]*'); delete rule filter $HANDLE_ID - delete set filter f2b- -# Option: actioncheck -# Notes.: command executed once before each actionban command -# Values: CMD -# -actioncheck = list chain filter | grep -q '@f2b-[ \t]' - -# Option: actionban -# Notes.: command executed when banning an IP. Take care that the -# command is executed with Fail2Ban user rights. -# Tags: See jail.conf(5) man page -# Values: CMD -# -actionban = add element filter f2b- { } - -# Option: actionunban -# Notes.: command executed when unbanning an IP. Take care that the -# command is executed with Fail2Ban user rights. -# Tags: See jail.conf(5) man page -# Values: CMD -# -actionunban = delete element filter f2b- { } +nftables_mode = ip protocol [Init] - diff --git a/config/action.d/nftables-common.conf b/config/action.d/nftables-common.conf index a0534d35..e65618ef 100644 --- a/config/action.d/nftables-common.conf +++ b/config/action.d/nftables-common.conf @@ -1,6 +1,9 @@ # Fail2Ban configuration file # # Author: Daniel Black +# Author: Cyril Jaquier +# Modified: Yaroslav O. Halchenko +# made active on all ports from original iptables.conf # Modified: Alexander Belykh # adapted for nftables # @@ -13,6 +16,49 @@ after = nftables-common.local +[Definition] + +# Option: nftables_mode +# Notes.: additional expressions for nftables filter rule +# Values: nftables expressions +# +nftables_mode = dport { } + +# Option: actionstart +# Notes.: command executed once at the start of Fail2Ban. +# Values: CMD +# +actionstart = add set filter f2b- { type ipv4_addr\; } + insert rule filter %(nftables_mode)s ip saddr @f2b- + +# Option: actionstop +# Notes.: command executed once at the end of Fail2Ban +# Values: CMD +# +actionstop = HANDLE_ID=$( --handle --numeric list chain filter | grep -m1 'ip saddr @f2b- # handle' | grep -oe ' handle [0-9]*'); delete rule filter $HANDLE_ID + delete set filter f2b- +# Option: actioncheck +# Notes.: command executed once before each actionban command +# Values: CMD +# +actioncheck = list chain filter | grep -q '@f2b-[ \t]' + +# Option: actionban +# Notes.: command executed when banning an IP. Take care that the +# command is executed with Fail2Ban user rights. +# Tags: See jail.conf(5) man page +# Values: CMD +# +actionban = add element filter f2b- { } + +# Option: actionunban +# Notes.: command executed when unbanning an IP. Take care that the +# command is executed with Fail2Ban user rights. +# Tags: See jail.conf(5) man page +# Values: CMD +# +actionunban = delete element filter f2b- { } + [Init] # Option: chain diff --git a/config/action.d/nftables-multiport.conf b/config/action.d/nftables-multiport.conf index ad61bf63..3c6455e2 100644 --- a/config/action.d/nftables-multiport.conf +++ b/config/action.d/nftables-multiport.conf @@ -1,7 +1,8 @@ # Fail2Ban configuration file # # Author: Cyril Jaquier -# Modified by Yaroslav Halchenko for multiport banning +# Modified: Yaroslav O. Halchenko +# made active on all ports from original iptables.conf # Modified: Alexander Belykh # adapted for nftables # @@ -12,40 +13,10 @@ before = nftables-common.conf [Definition] -# Option: actionstart -# Notes.: command executed once at the start of Fail2Ban. -# Values: CMD +# Option: nftables_mode +# Notes.: additional expressions for nftables filter rule +# Values: nftables expressions # -actionstart = add set filter f2b- { type ipv4_addr\; } - insert rule filter dport { } ip saddr @f2b- - -# Option: actionstop -# Notes.: command executed once at the end of Fail2Ban -# Values: CMD -# -actionstop = HANDLE_ID=$( --handle --numeric list chain filter | grep -m1 'ip saddr @f2b- # handle' | grep -oe ' handle [0-9]*'); delete rule filter $HANDLE_ID - delete set filter f2b- -# Option: actioncheck -# Notes.: command executed once before each actionban command -# Values: CMD -# -actioncheck = list chain filter | grep -q '@f2b-[ \t]' - -# Option: actionban -# Notes.: command executed when banning an IP. Take care that the -# command is executed with Fail2Ban user rights. -# Tags: See jail.conf(5) man page -# Values: CMD -# -actionban = add element filter f2b- { } - -# Option: actionunban -# Notes.: command executed when unbanning an IP. Take care that the -# command is executed with Fail2Ban user rights. -# Tags: See jail.conf(5) man page -# Values: CMD -# -actionunban = delete element filter f2b- { } +nftables_mode = dport { } [Init] - From cb2d70d7a8a47a8f42e6c2a443b27cc150d669ba Mon Sep 17 00:00:00 2001 From: Alexander Belykh Date: Tue, 5 Jan 2016 19:04:44 +0600 Subject: [PATCH 30/52] Add ChangeLog entry for new nftables actions --- ChangeLog | 3 +++ 1 file changed, 3 insertions(+) diff --git a/ChangeLog b/ChangeLog index 8b5344be..6d42a837 100644 --- a/ChangeLog +++ b/ChangeLog @@ -37,6 +37,9 @@ ver. 0.9.4 (2015/XX/XXX) - wanna-be-released filter.d/*.local file. As extension to interpolation `%(known/parameter)s`, that does not works for filter and action init parameters + * New actions: + - nftables-multiport and nftables-allports - filtering using nftables + framework. Note: it requires a pre-existing chain for the filtering rule. * New filters: - openhab - domotic software authentication failure with the rest api and web interface (gh-1223) From 1983e155808f30eb0eb775061252e3dad1bd2b19 Mon Sep 17 00:00:00 2001 From: Alexander Belykh Date: Wed, 6 Jan 2016 16:55:29 +0600 Subject: [PATCH 31/52] Add empty line between parameters in nftables-common.conf --- config/action.d/nftables-common.conf | 1 + 1 file changed, 1 insertion(+) diff --git a/config/action.d/nftables-common.conf b/config/action.d/nftables-common.conf index e65618ef..534eec2f 100644 --- a/config/action.d/nftables-common.conf +++ b/config/action.d/nftables-common.conf @@ -37,6 +37,7 @@ actionstart = add set filter f2b- { type ipv4_addr\; } # actionstop = HANDLE_ID=$( --handle --numeric list chain filter | grep -m1 'ip saddr @f2b- # handle' | grep -oe ' handle [0-9]*'); delete rule filter $HANDLE_ID delete set filter f2b- + # Option: actioncheck # Notes.: command executed once before each actionban command # Values: CMD From 260c30535d71c5ee1d07f843536bd4e9f6163358 Mon Sep 17 00:00:00 2001 From: Alexander Belykh Date: Wed, 6 Jan 2016 17:13:30 +0600 Subject: [PATCH 32/52] Escape curly braces in nftables actions --- config/action.d/nftables-common.conf | 8 ++++---- config/action.d/nftables-multiport.conf | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/config/action.d/nftables-common.conf b/config/action.d/nftables-common.conf index 534eec2f..157f3c77 100644 --- a/config/action.d/nftables-common.conf +++ b/config/action.d/nftables-common.conf @@ -22,13 +22,13 @@ after = nftables-common.local # Notes.: additional expressions for nftables filter rule # Values: nftables expressions # -nftables_mode = dport { } +nftables_mode = dport \{ \} # Option: actionstart # Notes.: command executed once at the start of Fail2Ban. # Values: CMD # -actionstart = add set filter f2b- { type ipv4_addr\; } +actionstart = add set filter f2b- \{ type ipv4_addr\; \} insert rule filter %(nftables_mode)s ip saddr @f2b- # Option: actionstop @@ -50,7 +50,7 @@ actioncheck = list chain filter | grep -q '@f2b-[ \t]' # Tags: See jail.conf(5) man page # Values: CMD # -actionban = add element filter f2b- { } +actionban = add element filter f2b- \{ \} # Option: actionunban # Notes.: command executed when unbanning an IP. Take care that the @@ -58,7 +58,7 @@ actionban = add element filter f2b- { } # Tags: See jail.conf(5) man page # Values: CMD # -actionunban = delete element filter f2b- { } +actionunban = delete element filter f2b- \{ \} [Init] diff --git a/config/action.d/nftables-multiport.conf b/config/action.d/nftables-multiport.conf index 3c6455e2..d1afafb3 100644 --- a/config/action.d/nftables-multiport.conf +++ b/config/action.d/nftables-multiport.conf @@ -17,6 +17,6 @@ before = nftables-common.conf # Notes.: additional expressions for nftables filter rule # Values: nftables expressions # -nftables_mode = dport { } +nftables_mode = dport \{ \} [Init] From 9779eeb986e323608207f9790556d7b9ce6816f1 Mon Sep 17 00:00:00 2001 From: Alexander Belykh Date: Wed, 6 Jan 2016 17:33:14 +0600 Subject: [PATCH 33/52] Add nftables_type/family/table parameters --- config/action.d/nftables-common.conf | 32 ++++++++++++++++++++++------ 1 file changed, 25 insertions(+), 7 deletions(-) diff --git a/config/action.d/nftables-common.conf b/config/action.d/nftables-common.conf index 157f3c77..26e35892 100644 --- a/config/action.d/nftables-common.conf +++ b/config/action.d/nftables-common.conf @@ -28,21 +28,21 @@ nftables_mode = dport \{ \} # Notes.: command executed once at the start of Fail2Ban. # Values: CMD # -actionstart = add set filter f2b- \{ type ipv4_addr\; \} - insert rule filter %(nftables_mode)s ip saddr @f2b- +actionstart = add set f2b- \{ type \; \} + insert rule %(nftables_mode)s ip saddr @f2b- # Option: actionstop # Notes.: command executed once at the end of Fail2Ban # Values: CMD # -actionstop = HANDLE_ID=$( --handle --numeric list chain filter | grep -m1 'ip saddr @f2b- # handle' | grep -oe ' handle [0-9]*'); delete rule filter $HANDLE_ID - delete set filter f2b- +actionstop = HANDLE_ID=$( --handle --numeric list chain | grep -m1 'ip saddr @f2b- # handle' | grep -oe ' handle [0-9]*'); delete rule $HANDLE_ID + delete set f2b- # Option: actioncheck # Notes.: command executed once before each actionban command # Values: CMD # -actioncheck = list chain filter | grep -q '@f2b-[ \t]' +actioncheck = list chain | grep -q '@f2b-[ \t]' # Option: actionban # Notes.: command executed when banning an IP. Take care that the @@ -50,7 +50,7 @@ actioncheck = list chain filter | grep -q '@f2b-[ \t]' # Tags: See jail.conf(5) man page # Values: CMD # -actionban = add element filter f2b- \{ \} +actionban = add element f2b- \{ \} # Option: actionunban # Notes.: command executed when unbanning an IP. Take care that the @@ -58,10 +58,28 @@ actionban = add element filter f2b- \{ \} # Tags: See jail.conf(5) man page # Values: CMD # -actionunban = delete element filter f2b- \{ \} +actionunban = delete element f2b- \{ \} [Init] +# Option: nftables_type +# Notes.: address type to work with +# Values: [ipv4_addr | ipv6_addr] Default: ipv4_addr +# +nftables_type = ipv4_addr + +# Option: nftables_family +# Notes.: address family to work in +# Values: [ip | ip6 | inet] Default: inet +# +nftables_family = inet + +# Option: nftables_table +# Notes.: table in the address family to work in +# Values: STRING Default: filter +# +nftables_table = filter + # Option: chain # Notes specifies the nftables chain to which the Fail2Ban rules should be # added From 985e8938a4a7aa4181851ad962cfe0c6a3a8fba3 Mon Sep 17 00:00:00 2001 From: Alexander Belykh Date: Wed, 6 Jan 2016 17:39:54 +0600 Subject: [PATCH 34/52] Refactor nftables actionstop into smaller parts --- config/action.d/nftables-common.conf | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/config/action.d/nftables-common.conf b/config/action.d/nftables-common.conf index 26e35892..80657c5c 100644 --- a/config/action.d/nftables-common.conf +++ b/config/action.d/nftables-common.conf @@ -31,11 +31,15 @@ nftables_mode = dport \{ \} actionstart = add set f2b- \{ type \; \} insert rule %(nftables_mode)s ip saddr @f2b- +_nft_list = --handle --numeric list chain +_nft_get_handle_id = grep -m1 'ip saddr @f2b- # handle' | grep -oe ' handle [0-9]*' + # Option: actionstop # Notes.: command executed once at the end of Fail2Ban # Values: CMD # -actionstop = HANDLE_ID=$( --handle --numeric list chain | grep -m1 'ip saddr @f2b- # handle' | grep -oe ' handle [0-9]*'); delete rule $HANDLE_ID +actionstop = HANDLE_ID=$(%(_nft_list)s | %(_nft_get_handle_id)s) + delete rule $HANDLE_ID delete set f2b- # Option: actioncheck From 40c0bed82c649ad7731bbe33ba773a25e55a6163 Mon Sep 17 00:00:00 2001 From: local Date: Sun, 10 Jan 2016 00:05:03 +0100 Subject: [PATCH 35/52] action_mw, action_mwl, action_cf_mwl ignore the "sender" option when sending a notification email. This commit adds "sender="%(sender)s"" to the three actions to correct this issue. --- config/jail.conf | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/config/jail.conf b/config/jail.conf index 7560f582..2bf28f6d 100644 --- a/config/jail.conf +++ b/config/jail.conf @@ -164,12 +164,12 @@ action_ = %(banaction)s[name=%(__name__)s, bantime="%(bantime)s", port="%(port)s # ban & send an e-mail with whois report to the destemail. action_mw = %(banaction)s[name=%(__name__)s, bantime="%(bantime)s", port="%(port)s", protocol="%(protocol)s", chain="%(chain)s"] - %(mta)s-whois[name=%(__name__)s, dest="%(destemail)s", protocol="%(protocol)s", chain="%(chain)s"] + %(mta)s-whois[name=%(__name__)s, sender="%(sender)s", dest="%(destemail)s", protocol="%(protocol)s", chain="%(chain)s"] # ban & send an e-mail with whois report and relevant log lines # to the destemail. action_mwl = %(banaction)s[name=%(__name__)s, bantime="%(bantime)s", port="%(port)s", protocol="%(protocol)s", chain="%(chain)s"] - %(mta)s-whois-lines[name=%(__name__)s, dest="%(destemail)s", logpath=%(logpath)s, chain="%(chain)s"] + %(mta)s-whois-lines[name=%(__name__)s, sender="%(sender)s", dest="%(destemail)s", logpath=%(logpath)s, chain="%(chain)s"] # See the IMPORTANT note in action.d/xarf-login-attack for when to use this action # @@ -181,7 +181,7 @@ action_xarf = %(banaction)s[name=%(__name__)s, bantime="%(bantime)s", port="%(po # ban IP on CloudFlare & send an e-mail with whois report and relevant log lines # to the destemail. action_cf_mwl = cloudflare[cfuser="%(cfemail)s", cftoken="%(cfapikey)s"] - %(mta)s-whois-lines[name=%(__name__)s, dest="%(destemail)s", logpath=%(logpath)s, chain="%(chain)s"] + %(mta)s-whois-lines[name=%(__name__)s, sender="%(sender)s", dest="%(destemail)s", logpath=%(logpath)s, chain="%(chain)s"] # Report block via blocklist.de fail2ban reporting service API # From 58a8736e0f2e35eba89b3b236fbfafae84194821 Mon Sep 17 00:00:00 2001 From: local Date: Sun, 10 Jan 2016 00:10:05 +0100 Subject: [PATCH 36/52] Updating changelog. --- ChangeLog | 1 + 1 file changed, 1 insertion(+) diff --git a/ChangeLog b/ChangeLog index 36c727fe..2220097e 100644 --- a/ChangeLog +++ b/ChangeLog @@ -29,6 +29,7 @@ ver. 0.9.4 (2015/XX/XXX) - wanna-be-released * Use postfix_log logpath for postfix-rbl jail * filters.d/postfix.conf - add 'Sender address rejected: Domain not found' failregex * use `fail2ban_agent` as user-agent in actions badips, blocklist_de, etc (gh-1271) + * Fix ignoring the sender option by action_mw, action_mwl and action_c_mwl - New Features: * New interpolation feature for definition config readers - `` From d7b46509d86cbf7fce447a7439bcfcfc91801cc8 Mon Sep 17 00:00:00 2001 From: Jordan Moeser Date: Tue, 12 Jan 2016 08:37:33 +1000 Subject: [PATCH 37/52] Update haproxy-http-auth.conf Updated failregex to be more strict --- config/filter.d/haproxy-http-auth.conf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config/filter.d/haproxy-http-auth.conf b/config/filter.d/haproxy-http-auth.conf index 7c371dde..298ca292 100644 --- a/config/filter.d/haproxy-http-auth.conf +++ b/config/filter.d/haproxy-http-auth.conf @@ -28,7 +28,7 @@ _daemon = haproxy # (?:::f{4,6}:)?(?P[\w\-.^_]+) # Values: TEXT # -failregex = ^%(__prefix_line)s.*NOSRV.*401 +failregex = ^%(__prefix_line)s.* -1/-1/-1/-1/\+*\d* 401 # Option: ignoreregex # Notes.: regex to ignore. If this regex matches, the line is ignored. From 294a7790a9a32f98448e1e2a2d7a5cfdf741c35f Mon Sep 17 00:00:00 2001 From: Louis Sautier Date: Thu, 28 Jan 2016 23:40:34 +0100 Subject: [PATCH 38/52] gentoo-initd: do not hide useful output Gentoo applies a patch for this: https://bugs.gentoo.org/show_bug.cgi?id=536320 --- files/gentoo-initd | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/files/gentoo-initd b/files/gentoo-initd index 98c5edf9..e939b987 100755 --- a/files/gentoo-initd +++ b/files/gentoo-initd @@ -34,19 +34,19 @@ start() { # remove stalled sock file after system crash # bug 347477 rm -f /var/run/fail2ban/fail2ban.sock || return 1 - ${FAIL2BAN} start &> /dev/null + ${FAIL2BAN} start eend $? "Failed to start fail2ban" } stop() { ebegin "Stopping fail2ban" - ${FAIL2BAN} stop &> /dev/null + ${FAIL2BAN} stop eend $? "Failed to stop fail2ban" } reload() { ebegin "Reloading fail2ban" - ${FAIL2BAN} reload > /dev/null + ${FAIL2BAN} reload eend $? "Failed to reload fail2ban" } From 869d99dd377ff45efa5796bb3be2500e41f32dc3 Mon Sep 17 00:00:00 2001 From: Louis Sautier Date: Thu, 28 Jan 2016 23:52:02 +0100 Subject: [PATCH 39/52] Remove compression and count from logrotate Initially reported at https://bugs.gentoo.org/show_bug.cgi?id=549856 --- ChangeLog | 2 ++ files/fail2ban-logrotate | 4 +--- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/ChangeLog b/ChangeLog index 4460f94c..69245dc9 100644 --- a/ChangeLog +++ b/ChangeLog @@ -30,6 +30,8 @@ ver. 0.9.4 (2015/XX/XXX) - wanna-be-released * filters.d/postfix.conf - add 'Sender address rejected: Domain not found' failregex * use `fail2ban_agent` as user-agent in actions badips, blocklist_de, etc (gh-1271) * Fix ignoring the sender option by action_mw, action_mwl and action_c_mwl + * Remove compression and rotation count from logrotate (inherit them from + the global logrotate config) - New Features: * New interpolation feature for definition config readers - `` diff --git a/files/fail2ban-logrotate b/files/fail2ban-logrotate index 8d94a8b3..13a94537 100644 --- a/files/fail2ban-logrotate +++ b/files/fail2ban-logrotate @@ -6,11 +6,9 @@ # https://github.com/fail2ban/fail2ban/blob/debian/debian/fail2ban.logrotate /var/log/fail2ban.log { - rotate 7 missingok notifempty - compress postrotate - /usr/bin/fail2ban-client flushlogs 1>/dev/null || true + /usr/bin/fail2ban-client flushlogs >/dev/null || true endscript } From b5a07741c82acdb9448584a0d39d3df09d28e162 Mon Sep 17 00:00:00 2001 From: Pierre GINDRAUD Date: Mon, 8 Feb 2016 11:08:10 +0100 Subject: [PATCH 40/52] Add new regex into postfix filter. The new regexp is able to detect bad formatted SMTP EHLO command --- ChangeLog | 1 + config/filter.d/postfix.conf | 1 + fail2ban/tests/files/logs/postfix | 3 +++ 3 files changed, 5 insertions(+) diff --git a/ChangeLog b/ChangeLog index 4460f94c..644a6d16 100644 --- a/ChangeLog +++ b/ChangeLog @@ -79,6 +79,7 @@ ver. 0.9.4 (2015/XX/XXX) - wanna-be-released .service file -- would reload fail2ban if those services are restarted * Provides new default `fail2ban_version` and interpolation variable `fail2ban_agent` in jail.conf + * Enhance filter 'postfix' to ban incoming SMTP client with no fqdn hostname ver. 0.9.3 (2015/08/01) - lets-all-stay-friends diff --git a/config/filter.d/postfix.conf b/config/filter.d/postfix.conf index f6a8578b..25141863 100644 --- a/config/filter.d/postfix.conf +++ b/config/filter.d/postfix.conf @@ -15,6 +15,7 @@ _daemon = postfix/(submission/)?smtp(d|s) failregex = ^%(__prefix_line)sNOQUEUE: reject: RCPT from \S+\[\]: 554 5\.7\.1 .*$ ^%(__prefix_line)sNOQUEUE: reject: RCPT from \S+\[\]: 450 4\.7\.1 Client host rejected: cannot find your hostname, (\[\S*\]); from=<\S*> to=<\S+> proto=ESMTP helo=<\S*>$ ^%(__prefix_line)sNOQUEUE: reject: RCPT from \S+\[\]: 450 4\.7\.1 : Helo command rejected: Host not found; from=<> to=<> proto=ESMTP helo= *$ + ^%(__prefix_line)sNOQUEUE: reject: EHLO from \S+\[\]: 504 5\.5\.2 <\S+>: Helo command rejected: need fully-qualified hostname; ^%(__prefix_line)sNOQUEUE: reject: VRFY from \S+\[\]: 550 5\.1\.1 .*$ ^%(__prefix_line)sNOQUEUE: reject: RCPT from \S+\[\]: 450 4\.1\.8 <\S*>: Sender address rejected: Domain not found; from=<\S*> to=<\S+> proto=ESMTP helo=<\S*>$ ^%(__prefix_line)simproper command pipelining after \S+ from [^[]*\[\]:?$ diff --git a/fail2ban/tests/files/logs/postfix b/fail2ban/tests/files/logs/postfix index 4934a29e..800c7f0c 100644 --- a/fail2ban/tests/files/logs/postfix +++ b/fail2ban/tests/files/logs/postfix @@ -26,3 +26,6 @@ Dec 21 21:17:29 xxx postfix/smtpd[7150]: NOQUEUE: reject: RCPT from badserver.ex # failJSON: { "time": "2004-11-22T22:33:44", "match": true , "host": "1.2.3.4" } Nov 22 22:33:44 xxx postfix/smtpd[11111]: NOQUEUE: reject: RCPT from 1-2-3-4.example.com[1.2.3.4]: 450 4.1.8 : Sender address rejected: Domain not found; from= to= proto=ESMTP helo=<1-2-3-4.example.com> + +# failJSON: { "time": "2005-01-31T13:55:24", "match": true , "host": "78.107.251.238" } +Jan 31 13:55:24 xxx postfix/smtpd[3462]: NOQUEUE: reject: EHLO from s271272.static.corbina.ru[78.107.251.238]: 504 5.5.2 : Helo command rejected: need fully-qualified hostname; proto=SMTP helo= From 257b7049d859c45f514fd1d9c83eed85cdc48384 Mon Sep 17 00:00:00 2001 From: 3eBoP Date: Thu, 28 Jan 2016 14:41:10 +0100 Subject: [PATCH 41/52] Update asterisk filter: changed regex for "Call from ...". Sometimes extension can have a plus symbol (+) because they can be phone number. Closes #1309 --- ChangeLog | 1 + config/filter.d/asterisk.conf | 2 +- fail2ban/tests/files/logs/asterisk | 8 ++++++++ 3 files changed, 10 insertions(+), 1 deletion(-) diff --git a/ChangeLog b/ChangeLog index 644a6d16..e128a91a 100644 --- a/ChangeLog +++ b/ChangeLog @@ -30,6 +30,7 @@ ver. 0.9.4 (2015/XX/XXX) - wanna-be-released * filters.d/postfix.conf - add 'Sender address rejected: Domain not found' failregex * use `fail2ban_agent` as user-agent in actions badips, blocklist_de, etc (gh-1271) * Fix ignoring the sender option by action_mw, action_mwl and action_c_mwl + * Changed filter.d/asterisk regex for "Call from ..." (few vulnerable now) - New Features: * New interpolation feature for definition config readers - `` diff --git a/config/filter.d/asterisk.conf b/config/filter.d/asterisk.conf index b446c44e..6ce65c4f 100644 --- a/config/filter.d/asterisk.conf +++ b/config/filter.d/asterisk.conf @@ -19,7 +19,7 @@ iso8601 = \d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\.\d+[+-]\d{4} log_prefix= (?:NOTICE|SECURITY)%(__pid_re)s:?(?:\[C-[\da-f]*\])? \S+:\d*( in \w+:)? failregex = ^(%(__prefix_line)s|\[\]\s*)%(log_prefix)s Registration from '[^']*' failed for '(:\d+)?' - (Wrong password|Username/auth name mismatch|No matching peer found|Not a local domain|Device does not match ACL|Peer is not supposed to register|ACL error \(permit/deny\)|Not a local domain)$ - ^(%(__prefix_line)s|\[\]\s*)%(log_prefix)s Call from '[^']*' \(:\d+\) to extension '\d+' rejected because extension not found in context 'default'\.$ + ^(%(__prefix_line)s|\[\]\s*)%(log_prefix)s Call from '[^']*' \(:\d+\) to extension '[\d+]+' rejected because extension not found in context 'default'\.$ ^(%(__prefix_line)s|\[\]\s*)%(log_prefix)s Host failed to authenticate as '[^']*'$ ^(%(__prefix_line)s|\[\]\s*)%(log_prefix)s No registration for peer '[^']*' \(from \)$ ^(%(__prefix_line)s|\[\]\s*)%(log_prefix)s Host failed MD5 authentication for '[^']*' \([^)]+\)$ diff --git a/fail2ban/tests/files/logs/asterisk b/fail2ban/tests/files/logs/asterisk index ab018ba9..57b53a02 100644 --- a/fail2ban/tests/files/logs/asterisk +++ b/fail2ban/tests/files/logs/asterisk @@ -59,3 +59,11 @@ Nov 4 18:30:40 localhost asterisk[32229]: NOTICE[32257]: chan_sip.c:23417 in han # match UTF-8 in SessionID # failJSON: { "time": "2015-05-25T07:52:36", "match": true, "host": "10.250.251.252" } [2015-05-25 07:52:36] SECURITY[6988] res_security_log.c: SecurityEvent="InvalidAccountID",EventTV="2015-05-25T07:52:36.888+0300",Severity="Error",Service="PJSIP",EventVersion="1",AccountID="70000180",SessionID="Негодяй",LocalAddress="IPV4/UDP/1.2.3.4/5060",RemoteAddress="IPV4/UDP/10.250.251.252/5061" + +# match phone numbers with + symbol +# failJSON: { "time": "2016-01-28T10:22:27", "match": true , "host": "1.2.3.4" } +[2016-01-28 10:22:27] NOTICE[3477][C-000003bb] chan_sip.c: Call from '' (1.2.3.4:10836) to extension '++441772285411' rejected because extension not found in context 'default'. +# failJSON: { "time": "2016-01-28T10:23:57", "match": true , "host": "1.2.3.4" } +[2016-01-28 10:23:57] NOTICE[3477][C-000003bc] chan_sip.c: Call from '' (1.2.3.4:10836) to extension '+441772285403' rejected because extension not found in context 'default'. +# failJSON: { "time": "2016-01-28T10:34:31", "match": true , "host": "1.2.3.4" } +[2016-01-28 10:34:31] NOTICE[3477][C-000003c3] chan_sip.c: Call from '' (1.2.3.4:10836) to extension '0+441772285407' rejected because extension not found in context 'default'. From d8e81eb417ae0e91b077c5c8b2fad26a9ff6de87 Mon Sep 17 00:00:00 2001 From: sebres Date: Mon, 8 Feb 2016 11:47:21 +0100 Subject: [PATCH 42/52] regexp rewritten (few vulnerable as previous) + test case added --- config/filter.d/asterisk.conf | 2 +- fail2ban/tests/files/logs/asterisk | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/config/filter.d/asterisk.conf b/config/filter.d/asterisk.conf index 6ce65c4f..3975fb29 100644 --- a/config/filter.d/asterisk.conf +++ b/config/filter.d/asterisk.conf @@ -19,7 +19,7 @@ iso8601 = \d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\.\d+[+-]\d{4} log_prefix= (?:NOTICE|SECURITY)%(__pid_re)s:?(?:\[C-[\da-f]*\])? \S+:\d*( in \w+:)? failregex = ^(%(__prefix_line)s|\[\]\s*)%(log_prefix)s Registration from '[^']*' failed for '(:\d+)?' - (Wrong password|Username/auth name mismatch|No matching peer found|Not a local domain|Device does not match ACL|Peer is not supposed to register|ACL error \(permit/deny\)|Not a local domain)$ - ^(%(__prefix_line)s|\[\]\s*)%(log_prefix)s Call from '[^']*' \(:\d+\) to extension '[\d+]+' rejected because extension not found in context 'default'\.$ + ^(%(__prefix_line)s|\[\]\s*)%(log_prefix)s Call from '[^']*' \(:\d+\) to extension '[^']*' rejected because extension not found in context ^(%(__prefix_line)s|\[\]\s*)%(log_prefix)s Host failed to authenticate as '[^']*'$ ^(%(__prefix_line)s|\[\]\s*)%(log_prefix)s No registration for peer '[^']*' \(from \)$ ^(%(__prefix_line)s|\[\]\s*)%(log_prefix)s Host failed MD5 authentication for '[^']*' \([^)]+\)$ diff --git a/fail2ban/tests/files/logs/asterisk b/fail2ban/tests/files/logs/asterisk index 57b53a02..aa32a290 100644 --- a/fail2ban/tests/files/logs/asterisk +++ b/fail2ban/tests/files/logs/asterisk @@ -60,10 +60,10 @@ Nov 4 18:30:40 localhost asterisk[32229]: NOTICE[32257]: chan_sip.c:23417 in han # failJSON: { "time": "2015-05-25T07:52:36", "match": true, "host": "10.250.251.252" } [2015-05-25 07:52:36] SECURITY[6988] res_security_log.c: SecurityEvent="InvalidAccountID",EventTV="2015-05-25T07:52:36.888+0300",Severity="Error",Service="PJSIP",EventVersion="1",AccountID="70000180",SessionID="Негодяй",LocalAddress="IPV4/UDP/1.2.3.4/5060",RemoteAddress="IPV4/UDP/10.250.251.252/5061" -# match phone numbers with + symbol +# match phone numbers with + symbol (and without number, or other context) # failJSON: { "time": "2016-01-28T10:22:27", "match": true , "host": "1.2.3.4" } [2016-01-28 10:22:27] NOTICE[3477][C-000003bb] chan_sip.c: Call from '' (1.2.3.4:10836) to extension '++441772285411' rejected because extension not found in context 'default'. -# failJSON: { "time": "2016-01-28T10:23:57", "match": true , "host": "1.2.3.4" } -[2016-01-28 10:23:57] NOTICE[3477][C-000003bc] chan_sip.c: Call from '' (1.2.3.4:10836) to extension '+441772285403' rejected because extension not found in context 'default'. # failJSON: { "time": "2016-01-28T10:34:31", "match": true , "host": "1.2.3.4" } [2016-01-28 10:34:31] NOTICE[3477][C-000003c3] chan_sip.c: Call from '' (1.2.3.4:10836) to extension '0+441772285407' rejected because extension not found in context 'default'. +# failJSON: { "time": "2016-01-28T10:34:33", "match": true , "host": "1.2.3.4" } +[2016-01-28 10:34:33] NOTICE[3477][C-000003c3] chan_sip.c: Call from '' (1.2.3.4:10836) to extension '' rejected because extension not found in context 'my-context'. From 44490664f5616046eca504aab3f4b78f3fdadf23 Mon Sep 17 00:00:00 2001 From: sebres Date: Tue, 9 Feb 2016 14:23:40 +0100 Subject: [PATCH 43/52] try to start server in foreground # Conflicts: # fail2ban/server/server.py --- bin/fail2ban-client | 117 ++++++++++++++++++++++-------- bin/fail2ban-server | 3 +- fail2ban/client/fail2banreader.py | 9 ++- fail2ban/server/server.py | 18 ++--- 4 files changed, 106 insertions(+), 41 deletions(-) diff --git a/bin/fail2ban-client b/bin/fail2ban-client index 7f3f5639..4a1bab39 100755 --- a/bin/fail2ban-client +++ b/bin/fail2ban-client @@ -54,6 +54,7 @@ class Fail2banClient: PROMPT = "fail2ban> " def __init__(self): + self.__server = None self.__argv = None self.__stream = None self.__configurator = Configurator() @@ -89,13 +90,16 @@ class Fail2banClient: print " -c configuration directory" print " -s socket path" print " -p pidfile path" + print " --loglevel logging level" + print " --logtarget |STDOUT|STDERR|SYSLOG" + print " --syslogsocket auto|file" print " -d dump configuration. For debugging" print " -i interactive mode" print " -v increase verbosity" print " -q decrease verbosity" print " -x force execution of the server (remove socket file)" print " -b start server in background (default)" - print " -f start server in foreground (note that the client forks once itself)" + print " -f start server in foreground" print " -h, --help display this help message" print " -V, --version print the version" print @@ -128,6 +132,8 @@ class Fail2banClient: self.__conf["socket"] = opt[1] elif opt[0] == "-p": self.__conf["pidfile"] = opt[1] + elif opt[0].startswith("--log") or opt[0].startswith("--sys"): + self.__conf[ opt[0][2:] ] = opt[1] elif opt[0] == "-d": self.__conf["dump"] = True elif opt[0] == "-v": @@ -234,24 +240,32 @@ class Fail2banClient: "Directory %s exists but not accessible for writing" % (socket_dir,)) return False - # Start the server - self.__startServerAsync(self.__conf["socket"], - self.__conf["pidfile"], - self.__conf["force"], - self.__conf["background"]) - try: - # Wait for the server to start - self.__waitOnServer() - # Configure the server - self.__processCmd(self.__stream, False) - return True - except ServerExecutionException: - logSys.error("Could not start server. Maybe an old " - "socket file is still present. Try to " - "remove " + self.__conf["socket"] + ". If " - "you used fail2ban-client to start the " - "server, adding the -x option will do it") + + # Check already running + if not self.__conf["force"] and os.path.exists(self.__conf["socket"]): + logSys.error("Fail2ban seems to be in unexpected state (not running but socket exists)") return False + + # Start the server + t = None + if self.__conf["background"]: + # Start server daemon as fork of client process: + self.__startServerAsync() + # Send config stream to server: + return self.__processStartStreamAfterWait() + else: + # In foreground mode we should start server/client communication in other thread: + from threading import Thread + t = Thread(target=Fail2banClient.__processStartStreamAfterWait, args=(self,)) + t.start() + # Start server direct here in main thread: + try: + self.__startServerDirect() + except KeyboardInterrupt: + None + + return True + elif len(cmd) == 1 and cmd[0] == "reload": if self.__ping(): ret = self.__readConfig() @@ -281,12 +295,50 @@ class Fail2banClient: return self.__processCmd([cmd]) + def __processStartStreamAfterWait(self): + try: + # Wait for the server to start + self.__waitOnServer() + # Configure the server + self.__processCmd(self.__stream, False) + except ServerExecutionException: + logSys.error("Could not start server. Maybe an old " + "socket file is still present. Try to " + "remove " + self.__conf["socket"] + ". If " + "you used fail2ban-client to start the " + "server, adding the -x option will do it") + if not self.__conf["background"]: + self.__server.quit() + sys.exit(-1) + return False + return True + + + ## + # Start Fail2Ban server in main thread without fork (foreground). + # + # Start the Fail2ban server in foreground (daemon mode or not). + + def __startServerDirect(self): + from fail2ban.server.server import Server + try: + self.__server = Server(False) + self.__server.start(self.__conf["socket"], + self.__conf["pidfile"], self.__conf["force"], + conf=self.__conf) + except Exception, e: + logSys.exception(e) + if self.__server: + self.__server.quit() + sys.exit(-1) + + ## # Start Fail2Ban server. # # Start the Fail2ban server in daemon mode. - def __startServerAsync(self, socket, pidfile, force = False, background = True): + def __startServerAsync(self): # Forks the current process. pid = os.fork() if pid == 0: @@ -294,18 +346,15 @@ class Fail2banClient: args.append(self.SERVER) # Set the socket path. args.append("-s") - args.append(socket) + args.append(self.__conf["socket"]) # Set the pidfile args.append("-p") - args.append(pidfile) + args.append(self.__conf["pidfile"]) # Force the execution if needed. - if force: + if self.__conf["force"]: args.append("-x") - # Start in foreground mode if requested. - if background: - args.append("-b") - else: - args.append("-f") + # Start in background as requested. + args.append("-b") try: # Use the current directory. @@ -361,7 +410,7 @@ class Fail2banClient: # Reads the command line options. try: cmdOpts = 'hc:s:p:xfbdviqV' - cmdLongOpts = ['help', 'version'] + cmdLongOpts = ['loglevel', 'logtarget', 'syslogsocket', 'help', 'version'] optList, args = getopt.getopt(self.__argv[1:], cmdOpts, cmdLongOpts) except getopt.GetoptError: self.dispUsage() @@ -396,7 +445,17 @@ class Fail2banClient: self.__conf["socket"] = conf["socket"] if self.__conf["pidfile"] is None: self.__conf["pidfile"] = conf["pidfile"] - logSys.info("Using socket file " + self.__conf["socket"]) + if self.__conf.get("logtarget", None) is None: + self.__conf["logtarget"] = conf["logtarget"] + if self.__conf.get("loglevel", None) is None: + self.__conf["loglevel"] = conf["loglevel"] + if self.__conf.get("syslogsocket", None) is None: + self.__conf["syslogsocket"] = conf["syslogsocket"] + + logSys.info("Using socket file %s", self.__conf["socket"]) + + logSys.info("Using pid file %s, [%s] logging to %s", + self.__conf["pidfile"], self.__conf["loglevel"], self.__conf["logtarget"]) if self.__conf["dump"]: ret = self.__readConfig() diff --git a/bin/fail2ban-server b/bin/fail2ban-server index f522f418..0b8b6418 100755 --- a/bin/fail2ban-server +++ b/bin/fail2ban-server @@ -129,7 +129,8 @@ class Fail2banServer: return True except Exception, e: logSys.exception(e) - self.__server.quit() + if self.__server: + self.__server.quit() return False if __name__ == "__main__": diff --git a/fail2ban/client/fail2banreader.py b/fail2ban/client/fail2banreader.py index c55f65ea..b3012c9b 100644 --- a/fail2ban/client/fail2banreader.py +++ b/fail2ban/client/fail2banreader.py @@ -40,8 +40,13 @@ class Fail2banReader(ConfigReader): ConfigReader.read(self, "fail2ban") def getEarlyOptions(self): - opts = [["string", "socket", "/var/run/fail2ban/fail2ban.sock"], - ["string", "pidfile", "/var/run/fail2ban/fail2ban.pid"]] + opts = [ + ["string", "socket", "/var/run/fail2ban/fail2ban.sock"], + ["string", "pidfile", "/var/run/fail2ban/fail2ban.pid"], + ["string", "loglevel", "INFO"], + ["string", "logtarget", "/var/log/fail2ban.log"], + ["string", "syslogsocket", "auto"] + ] return ConfigReader.getOptions(self, "Definition", opts) def getOptions(self): diff --git a/fail2ban/server/server.py b/fail2ban/server/server.py index 3bdfd71b..923b6ba3 100644 --- a/fail2ban/server/server.py +++ b/fail2ban/server/server.py @@ -67,10 +67,6 @@ class Server: 'FreeBSD': '/var/run/log', 'Linux': '/dev/log', } - self.setSyslogSocket("auto") - # Set logging level - self.setLogLevel("INFO") - self.setLogTarget("STDOUT") def __sigTERMhandler(self, signum, frame): logSys.debug("Caught signal %d. Exiting" % signum) @@ -80,7 +76,12 @@ class Server: logSys.debug("Caught signal %d. Flushing logs" % signum) self.flushLogs() - def start(self, sock, pidfile, force = False): + def start(self, sock, pidfile, force=False, conf={}): + # First set all logging parameters: + self.setSyslogSocket(conf.get("syslogsocket", "auto")) + self.setLogLevel(conf.get("loglevel", "INFO")) + self.setLogTarget(conf.get("logtarget", "STDOUT")) + logSys.info("Starting Fail2ban v%s", version.version) # Install signal handlers @@ -392,8 +393,9 @@ class Server: # @param target the logging target def setLogTarget(self, target): - try: - self.__loggingLock.acquire() + with self.__loggingLock: + if self.__logTarget == target: + return True # set a format which is simpler for console use formatter = logging.Formatter("%(asctime)s %(name)-24s[%(process)d]: %(levelname)-7s %(message)s") if target == "SYSLOG": @@ -461,8 +463,6 @@ class Server: # Sets the logging target. self.__logTarget = target return True - finally: - self.__loggingLock.release() ## # Sets the syslog socket. From 3fda77227efcd8b65b06e2e96b9eada558bfc17a Mon Sep 17 00:00:00 2001 From: sebres Date: Wed, 10 Feb 2016 21:00:00 +0100 Subject: [PATCH 44/52] temporary commit (move client/server from bin) --- bin/fail2ban-client => fail2ban/client/fail2banclient.py | 0 bin/fail2ban-server => fail2ban/client/fail2banserver.py | 0 2 files changed, 0 insertions(+), 0 deletions(-) rename bin/fail2ban-client => fail2ban/client/fail2banclient.py (100%) rename bin/fail2ban-server => fail2ban/client/fail2banserver.py (100%) diff --git a/bin/fail2ban-client b/fail2ban/client/fail2banclient.py similarity index 100% rename from bin/fail2ban-client rename to fail2ban/client/fail2banclient.py diff --git a/bin/fail2ban-server b/fail2ban/client/fail2banserver.py similarity index 100% rename from bin/fail2ban-server rename to fail2ban/client/fail2banserver.py From 4d696d69a035902eadaa2e8449d8ec08cccb359d Mon Sep 17 00:00:00 2001 From: sebres Date: Thu, 11 Feb 2016 08:56:12 +0100 Subject: [PATCH 45/52] starting of the server (and client/server communication behavior during start and daemonize) completely rewritten: - client/server functionality moved away from bin and using now the common interface (introduced in fail2bancmdline); - start in foreground fixed; - server can act as client corresponding command line; - command "restart" added: in opposite to "reload" in reality restarts the server (new process); - several client/server bugs during starting process fixed. --- MANIFEST | 4 + bin/fail2ban-client | 37 ++ bin/fail2ban-server | 37 ++ fail2ban/client/beautifier.py | 2 + fail2ban/client/fail2banclient.py | 578 +++++++++++------------------ fail2ban/client/fail2bancmdline.py | 245 ++++++++++++ fail2ban/client/fail2banserver.py | 234 +++++++----- fail2ban/protocol.py | 6 +- fail2ban/server/asyncserver.py | 39 +- fail2ban/server/server.py | 95 ++--- fail2ban/server/transmitter.py | 2 + 11 files changed, 775 insertions(+), 504 deletions(-) create mode 100755 bin/fail2ban-client create mode 100755 bin/fail2ban-server mode change 100755 => 100644 fail2ban/client/fail2banclient.py create mode 100644 fail2ban/client/fail2bancmdline.py mode change 100755 => 100644 fail2ban/client/fail2banserver.py diff --git a/MANIFEST b/MANIFEST index fb70bb4b..f77caad6 100644 --- a/MANIFEST +++ b/MANIFEST @@ -165,7 +165,11 @@ fail2ban/client/configparserinc.py fail2ban/client/configreader.py fail2ban/client/configurator.py fail2ban/client/csocket.py +fail2ban/client/fail2banclient.py +fail2ban/client/fail2bancmdline.py fail2ban/client/fail2banreader.py +fail2ban/client/fail2banregex.py +fail2ban/client/fail2banserver.py fail2ban/client/filterreader.py fail2ban/client/jailreader.py fail2ban/client/jailsreader.py diff --git a/bin/fail2ban-client b/bin/fail2ban-client new file mode 100755 index 00000000..19e76a98 --- /dev/null +++ b/bin/fail2ban-client @@ -0,0 +1,37 @@ +#!/usr/bin/python +# emacs: -*- mode: python; py-indent-offset: 4; indent-tabs-mode: t -*- +# vi: set ft=python sts=4 ts=4 sw=4 noet : + +# This file is part of Fail2Ban. +# +# Fail2Ban is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. +# +# Fail2Ban is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with Fail2Ban; if not, write to the Free Software +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + +""" +Fail2Ban reads log file that contains password failure report +and bans the corresponding IP addresses using firewall rules. + +This tools starts/stops fail2ban server or does client/server communication, +to change/read parameters of the server or jails. + +""" + +__author__ = "Fail2Ban Developers" +__copyright__ = "Copyright (c) 2004-2008 Cyril Jaquier, 2012-2014 Yaroslav Halchenko, 2014-2016 Serg G. Brester" +__license__ = "GPL" + +from fail2ban.client.fail2banclient import exec_command_line + +if __name__ == "__main__": + exec_command_line() diff --git a/bin/fail2ban-server b/bin/fail2ban-server new file mode 100755 index 00000000..8e64d865 --- /dev/null +++ b/bin/fail2ban-server @@ -0,0 +1,37 @@ +#!/usr/bin/python +# emacs: -*- mode: python; py-indent-offset: 4; indent-tabs-mode: t -*- +# vi: set ft=python sts=4 ts=4 sw=4 noet : + +# This file is part of Fail2Ban. +# +# Fail2Ban is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. +# +# Fail2Ban is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with Fail2Ban; if not, write to the Free Software +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + +""" +Fail2Ban reads log file that contains password failure report +and bans the corresponding IP addresses using firewall rules. + +This tools starts/stops fail2ban server or does client/server communication, +to change/read parameters of the server or jails. + +""" + +__author__ = "Fail2Ban Developers" +__copyright__ = "Copyright (c) 2004-2008 Cyril Jaquier, 2012-2014 Yaroslav Halchenko, 2014-2016 Serg G. Brester" +__license__ = "GPL" + +from fail2ban.client.fail2banserver import exec_command_line + +if __name__ == "__main__": + exec_command_line() diff --git a/fail2ban/client/beautifier.py b/fail2ban/client/beautifier.py index 812fbe65..08ff484d 100644 --- a/fail2ban/client/beautifier.py +++ b/fail2ban/client/beautifier.py @@ -68,6 +68,8 @@ class Beautifier: msg = "Added jail " + response elif inC[0] == "flushlogs": msg = "logs: " + response + elif inC[0] == "echo": + msg = ' '.join(msg) elif inC[0:1] == ['status']: if len(inC) > 1: # Display information diff --git a/fail2ban/client/fail2banclient.py b/fail2ban/client/fail2banclient.py old mode 100755 new mode 100644 index 4a1bab39..7adbab95 --- a/fail2ban/client/fail2banclient.py +++ b/fail2ban/client/fail2banclient.py @@ -1,7 +1,7 @@ #!/usr/bin/python # emacs: -*- mode: python; py-indent-offset: 4; indent-tabs-mode: t -*- # vi: set ft=python sts=4 ts=4 sw=4 noet : - +# # This file is part of Fail2Ban. # # Fail2Ban is free software; you can redistribute it and/or modify @@ -17,99 +17,38 @@ # You should have received a copy of the GNU General Public License # along with Fail2Ban; if not, write to the Free Software # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. - -__author__ = "Cyril Jaquier" -__copyright__ = "Copyright (c) 2004 Cyril Jaquier" +__author__ = "Fail2Ban Developers" +__copyright__ = "Copyright (c) 2004-2008 Cyril Jaquier, 2012-2014 Yaroslav Halchenko, 2014-2016 Serg G. Brester" __license__ = "GPL" -import getopt -import logging import os -import pickle -import re import shlex import signal import socket -import string import sys import time -from fail2ban.version import version -from fail2ban.protocol import printFormatted -from fail2ban.client.csocket import CSocket -from fail2ban.client.configurator import Configurator -from fail2ban.client.beautifier import Beautifier -from fail2ban.helpers import getLogger +from threading import Thread -# Gets the instance of the logger. -logSys = getLogger("fail2ban") +from ..version import version +from .csocket import CSocket +from .beautifier import Beautifier +from .fail2bancmdline import Fail2banCmdLine, logSys, exit ## # # @todo This class needs cleanup. -class Fail2banClient: +class Fail2banClient(Fail2banCmdLine, Thread): - SERVER = "fail2ban-server" PROMPT = "fail2ban> " def __init__(self): - self.__server = None - self.__argv = None - self.__stream = None - self.__configurator = Configurator() - self.__conf = dict() - self.__conf["conf"] = "/etc/fail2ban" - self.__conf["dump"] = False - self.__conf["force"] = False - self.__conf["background"] = True - self.__conf["verbose"] = 1 - self.__conf["interactive"] = False - self.__conf["socket"] = None - self.__conf["pidfile"] = None - - def dispVersion(self): - print "Fail2Ban v" + version - print - print "Copyright (c) 2004-2008 Cyril Jaquier, 2008- Fail2Ban Contributors" - print "Copyright of modifications held by their respective authors." - print "Licensed under the GNU General Public License v2 (GPL)." - print - print "Written by Cyril Jaquier ." - print "Many contributions by Yaroslav O. Halchenko ." - - def dispUsage(self): - """ Prints Fail2Ban command line options and exits - """ - print "Usage: "+self.__argv[0]+" [OPTIONS] " - print - print "Fail2Ban v" + version + " reads log file that contains password failure report" - print "and bans the corresponding IP addresses using firewall rules." - print - print "Options:" - print " -c configuration directory" - print " -s socket path" - print " -p pidfile path" - print " --loglevel logging level" - print " --logtarget |STDOUT|STDERR|SYSLOG" - print " --syslogsocket auto|file" - print " -d dump configuration. For debugging" - print " -i interactive mode" - print " -v increase verbosity" - print " -q decrease verbosity" - print " -x force execution of the server (remove socket file)" - print " -b start server in background (default)" - print " -f start server in foreground" - print " -h, --help display this help message" - print " -V, --version print the version" - print - print "Command:" - - # Prints the protocol - printFormatted() - - print - print "Report bugs to https://github.com/fail2ban/fail2ban/issues" + Fail2banCmdLine.__init__(self) + Thread.__init__(self) + self._alive = True + self._server = None + self._beautifier = None def dispInteractive(self): print "Fail2Ban v" + version + " reads log file that contains password failure report" @@ -120,58 +59,32 @@ class Fail2banClient: # Print a new line because we probably come from wait print logSys.warning("Caught signal %d. Exiting" % signum) - sys.exit(-1) - - def __getCmdLineOptions(self, optList): - """ Gets the command line options - """ - for opt in optList: - if opt[0] == "-c": - self.__conf["conf"] = opt[1] - elif opt[0] == "-s": - self.__conf["socket"] = opt[1] - elif opt[0] == "-p": - self.__conf["pidfile"] = opt[1] - elif opt[0].startswith("--log") or opt[0].startswith("--sys"): - self.__conf[ opt[0][2:] ] = opt[1] - elif opt[0] == "-d": - self.__conf["dump"] = True - elif opt[0] == "-v": - self.__conf["verbose"] = self.__conf["verbose"] + 1 - elif opt[0] == "-q": - self.__conf["verbose"] = self.__conf["verbose"] - 1 - elif opt[0] == "-x": - self.__conf["force"] = True - elif opt[0] == "-i": - self.__conf["interactive"] = True - elif opt[0] == "-b": - self.__conf["background"] = True - elif opt[0] == "-f": - self.__conf["background"] = False - elif opt[0] in ["-h", "--help"]: - self.dispUsage() - sys.exit(0) - elif opt[0] in ["-V", "--version"]: - self.dispVersion() - sys.exit(0) + exit(-1) def __ping(self): return self.__processCmd([["ping"]], False) - def __processCmd(self, cmd, showRet = True): + @property + def beautifier(self): + if self._beautifier: + return self._beautifier + self._beautifier = Beautifier() + return self._beautifier + + def __processCmd(self, cmd, showRet=True): client = None try: - beautifier = Beautifier() + beautifier = self.beautifier streamRet = True for c in cmd: beautifier.setInputCmd(c) try: if not client: - client = CSocket(self.__conf["socket"]) + client = CSocket(self._conf["socket"]) ret = client.send(c) if ret[0] == 0: logSys.debug("OK : " + `ret[1]`) - if showRet: + if showRet or c[0] == 'echo': print beautifier.beautify(ret[1]) else: logSys.error("NOK: " + `ret[1].args`) @@ -179,38 +92,126 @@ class Fail2banClient: print beautifier.beautifyError(ret[1]) streamRet = False except socket.error: - if showRet: + if showRet or self._conf["verbose"] > 1: self.__logSocketError() return False except Exception, e: - if showRet: + if showRet or self._conf["verbose"] > 1: logSys.error(e) return False finally: if client: client.close() + if showRet or c[0] == 'echo': + sys.stdout.flush() return streamRet def __logSocketError(self): try: - if os.access(self.__conf["socket"], os.F_OK): + if os.access(self._conf["socket"], os.F_OK): # This doesn't check if path is a socket, # but socket.error should be raised - if os.access(self.__conf["socket"], os.W_OK): + if os.access(self._conf["socket"], os.W_OK): # Permissions look good, but socket.error was raised logSys.error("Unable to contact server. Is it running?") else: logSys.error("Permission denied to socket: %s," - " (you must be root)", self.__conf["socket"]) + " (you must be root)", self._conf["socket"]) else: logSys.error("Failed to access socket path: %s." " Is fail2ban running?", - self.__conf["socket"]) + self._conf["socket"]) except Exception as e: logSys.error("Exception while checking socket access: %s", - self.__conf["socket"]) + self._conf["socket"]) logSys.error(e) + ## + def __prepareStartServer(self): + if self.__ping(): + logSys.error("Server already running") + return None + + # Read the config + ret, stream = self.readConfig() + # Do not continue if configuration is not 100% valid + if not ret: + return None + + # verify that directory for the socket file exists + socket_dir = os.path.dirname(self._conf["socket"]) + if not os.path.exists(socket_dir): + logSys.error( + "There is no directory %s to contain the socket file %s." + % (socket_dir, self._conf["socket"])) + return None + if not os.access(socket_dir, os.W_OK | os.X_OK): + logSys.error( + "Directory %s exists but not accessible for writing" + % (socket_dir,)) + return None + + # Check already running + if not self._conf["force"] and os.path.exists(self._conf["socket"]): + logSys.error("Fail2ban seems to be in unexpected state (not running but the socket exists)") + return None + + stream.append(['echo', 'Server ready']) + return stream + + ## + def __startServer(self, background=True): + from .fail2banserver import Fail2banServer + stream = self.__prepareStartServer() + self._alive = True + if not stream: + return False + # Start the server or just initialize started one: + try: + if background: + # Start server daemon as fork of client process: + Fail2banServer.startServerAsync(self._conf) + # Send config stream to server: + if not self.__processStartStreamAfterWait(stream, False): + return False + else: + # In foreground mode we should make server/client communication in different threads: + Thread(target=Fail2banClient.__processStartStreamAfterWait, args=(self, stream, False)).start() + # Mark current (main) thread as daemon: + self.setDaemon(True) + # Start server direct here in main thread (not fork): + self._server = Fail2banServer.startServerDirect(self._conf, False) + + except Exception as e: + print + logSys.error("Exception while starting server foreground") + logSys.error(e) + finally: + self._alive = False + + return True + + ## + def configureServer(self, async=True, phase=None): + # if asynchron start this operation in the new thread: + if async: + return Thread(target=Fail2banClient.configureServer, args=(self, False, phase)).start() + # prepare: read config, check configuration is valid, etc.: + if phase is not None: + phase['start'] = True + logSys.debug('-- client phase %s', phase) + stream = self.__prepareStartServer() + if phase is not None: + phase['ready'] = phase['start'] = (True if stream else False) + logSys.debug('-- client phase %s', phase) + if not stream: + return False + # configure server with config stream: + ret = self.__processStartStreamAfterWait(stream, False) + if phase is not None: + phase['done'] = ret + return ret + ## # Process a command line. # @@ -219,251 +220,101 @@ class Fail2banClient: def __processCommand(self, cmd): if len(cmd) == 1 and cmd[0] == "start": - if self.__ping(): - logSys.error("Server already running") + + ret = self.__startServer(self._conf["background"]) + if not ret: return False - else: - # Read the config - ret = self.__readConfig() - # Do not continue if configuration is not 100% valid - if not ret: - return False - # verify that directory for the socket file exists - socket_dir = os.path.dirname(self.__conf["socket"]) - if not os.path.exists(socket_dir): - logSys.error( - "There is no directory %s to contain the socket file %s." - % (socket_dir, self.__conf["socket"])) - return False - if not os.access(socket_dir, os.W_OK | os.X_OK): - logSys.error( - "Directory %s exists but not accessible for writing" - % (socket_dir,)) - return False + return ret - # Check already running - if not self.__conf["force"] and os.path.exists(self.__conf["socket"]): - logSys.error("Fail2ban seems to be in unexpected state (not running but socket exists)") - return False + elif len(cmd) == 1 and cmd[0] == "restart": - # Start the server - t = None - if self.__conf["background"]: - # Start server daemon as fork of client process: - self.__startServerAsync() - # Send config stream to server: - return self.__processStartStreamAfterWait() + if self._conf.get("interactive", False): + print(' ## stop ... ') + self.__processCommand(['stop']) + self.__waitOnServer(False) + # in interactive mode reset config, to make full-reload if there something changed: + if self._conf.get("interactive", False): + print(' ## load configuration ... ') + self.resetConf() + ret = self.initCmdLine(self._argv) + if ret is not None: + return ret + if self._conf.get("interactive", False): + print(' ## start ... ') + return self.__processCommand(['start']) + + elif len(cmd) >= 1 and cmd[0] == "reload": + if self.__ping(): + if len(cmd) == 1: + jail = 'all' + ret, stream = self.readConfig() else: - # In foreground mode we should start server/client communication in other thread: - from threading import Thread - t = Thread(target=Fail2banClient.__processStartStreamAfterWait, args=(self,)) - t.start() - # Start server direct here in main thread: - try: - self.__startServerDirect() - except KeyboardInterrupt: - None - - return True - - elif len(cmd) == 1 and cmd[0] == "reload": - if self.__ping(): - ret = self.__readConfig() - # Do not continue if configuration is not 100% valid - if not ret: - return False - self.__processCmd([['stop', 'all']], False) - # Configure the server - return self.__processCmd(self.__stream, False) - else: - logSys.error("Could not find server") - return False - elif len(cmd) == 2 and cmd[0] == "reload": - if self.__ping(): - jail = cmd[1] - ret = self.__readConfig(jail) + jail = cmd[1] + ret, stream = self.readConfig(jail) # Do not continue if configuration is not 100% valid if not ret: return False self.__processCmd([['stop', jail]], False) # Configure the server - return self.__processCmd(self.__stream, False) + return self.__processCmd(stream, True) else: logSys.error("Could not find server") return False + else: return self.__processCmd([cmd]) - def __processStartStreamAfterWait(self): + def __processStartStreamAfterWait(self, *args): try: # Wait for the server to start self.__waitOnServer() - # Configure the server - self.__processCmd(self.__stream, False) + # Configure the server + self.__processCmd(*args) except ServerExecutionException: logSys.error("Could not start server. Maybe an old " "socket file is still present. Try to " - "remove " + self.__conf["socket"] + ". If " + "remove " + self._conf["socket"] + ". If " "you used fail2ban-client to start the " "server, adding the -x option will do it") - if not self.__conf["background"]: - self.__server.quit() - sys.exit(-1) + if self._server: + self._server.quit() + exit(-1) return False return True - - ## - # Start Fail2Ban server in main thread without fork (foreground). - # - # Start the Fail2ban server in foreground (daemon mode or not). - - def __startServerDirect(self): - from fail2ban.server.server import Server - try: - self.__server = Server(False) - self.__server.start(self.__conf["socket"], - self.__conf["pidfile"], self.__conf["force"], - conf=self.__conf) - except Exception, e: - logSys.exception(e) - if self.__server: - self.__server.quit() - sys.exit(-1) - - - ## - # Start Fail2Ban server. - # - # Start the Fail2ban server in daemon mode. - - def __startServerAsync(self): - # Forks the current process. - pid = os.fork() - if pid == 0: - args = list() - args.append(self.SERVER) - # Set the socket path. - args.append("-s") - args.append(self.__conf["socket"]) - # Set the pidfile - args.append("-p") - args.append(self.__conf["pidfile"]) - # Force the execution if needed. - if self.__conf["force"]: - args.append("-x") - # Start in background as requested. - args.append("-b") - - try: - # Use the current directory. - exe = os.path.abspath(os.path.join(sys.path[0], self.SERVER)) - logSys.debug("Starting %r with args %r" % (exe, args)) - os.execv(exe, args) - except OSError: - try: - # Use the PATH env. - logSys.warning("Initial start attempt failed. Starting %r with the same args" % (self.SERVER,)) - os.execvp(self.SERVER, args) - except OSError: - logSys.error("Could not start %s" % self.SERVER) - os.exit(-1) - - def __waitOnServer(self): - # Wait for the server to start - cnt = 0 - if self.__conf["verbose"] > 1: - pos = 0 - delta = 1 - mask = "[ ]" - while not self.__ping(): - # Wonderful visual :) - if self.__conf["verbose"] > 1: - pos += delta - sys.stdout.write("\rINFO " + mask[:pos] + '#' + mask[pos+1:] + - " Waiting on the server...") - sys.stdout.flush() - if pos > len(mask)-3: - delta = -1 - elif pos < 2: - delta = 1 - # The server has 30 seconds to start. - if cnt >= 300: - if self.__conf["verbose"] > 1: - sys.stdout.write('\n') - raise ServerExecutionException("Failed to start server") - time.sleep(0.1) - cnt += 1 - if self.__conf["verbose"] > 1: - sys.stdout.write('\n') - + def __waitOnServer(self, alive=True, maxtime=30): + # Wait for the server to start (the server has 30 seconds to answer ping) + starttime = time.time() + with VisualWait(self._conf["verbose"]) as vis: + while self._alive and not self.__ping() == alive or ( + not alive and os.path.exists(self._conf["socket"]) + ): + now = time.time() + # Wonderful visual :) + if now > starttime + 1: + vis.heartbeat() + # f end time reached: + if now - starttime >= maxtime: + raise ServerExecutionException("Failed to start server") + time.sleep(0.1) def start(self, argv): - # Command line options - self.__argv = argv - # Install signal handlers signal.signal(signal.SIGTERM, self.__sigTERMhandler) signal.signal(signal.SIGINT, self.__sigTERMhandler) - # Reads the command line options. - try: - cmdOpts = 'hc:s:p:xfbdviqV' - cmdLongOpts = ['loglevel', 'logtarget', 'syslogsocket', 'help', 'version'] - optList, args = getopt.getopt(self.__argv[1:], cmdOpts, cmdLongOpts) - except getopt.GetoptError: - self.dispUsage() - return False + # Command line options + if self._argv is None: + ret = self.initCmdLine(argv) + if ret is not None: + return ret - self.__getCmdLineOptions(optList) - - verbose = self.__conf["verbose"] - if verbose <= 0: - logSys.setLevel(logging.ERROR) - elif verbose == 1: - logSys.setLevel(logging.WARNING) - elif verbose == 2: - logSys.setLevel(logging.INFO) - else: - logSys.setLevel(logging.DEBUG) - # Add the default logging handler to dump to stderr - logout = logging.StreamHandler(sys.stderr) - # set a format which is simpler for console use - formatter = logging.Formatter('%(levelname)-6s %(message)s') - # tell the handler to use this format - logout.setFormatter(formatter) - logSys.addHandler(logout) - - # Set the configuration path - self.__configurator.setBaseDir(self.__conf["conf"]) - - # Set socket path - self.__configurator.readEarly() - conf = self.__configurator.getEarlyOptions() - if self.__conf["socket"] is None: - self.__conf["socket"] = conf["socket"] - if self.__conf["pidfile"] is None: - self.__conf["pidfile"] = conf["pidfile"] - if self.__conf.get("logtarget", None) is None: - self.__conf["logtarget"] = conf["logtarget"] - if self.__conf.get("loglevel", None) is None: - self.__conf["loglevel"] = conf["loglevel"] - if self.__conf.get("syslogsocket", None) is None: - self.__conf["syslogsocket"] = conf["syslogsocket"] - - logSys.info("Using socket file %s", self.__conf["socket"]) - - logSys.info("Using pid file %s, [%s] logging to %s", - self.__conf["pidfile"], self.__conf["loglevel"], self.__conf["logtarget"]) - - if self.__conf["dump"]: - ret = self.__readConfig() - self.dumpConfig(self.__stream) - return ret + # Commands + args = self._args # Interactive mode - if self.__conf["interactive"]: + if self._conf.get("interactive", False): try: import readline except ImportError: @@ -498,35 +349,56 @@ class Fail2banClient: return False return self.__processCommand(args) - def __readConfig(self, jail=None): - # Read the configuration - # TODO: get away from stew of return codes and exception - # handling -- handle via exceptions - try: - self.__configurator.Reload() - self.__configurator.readAll() - ret = self.__configurator.getOptions(jail) - self.__configurator.convertToProtocol() - self.__stream = self.__configurator.getConfigStream() - except Exception, e: - logSys.error("Failed during configuration: %s" % e) - ret = False - return ret - - @staticmethod - def dumpConfig(cmd): - for c in cmd: - print c - return True - class ServerExecutionException(Exception): pass -if __name__ == "__main__": # pragma: no cover - can't test main + +## +# Wonderful visual :) +# + +class _VisualWait: + pos = 0 + delta = 1 + maxpos = 10 + def __enter__(self): + return self + def __exit__(self, *args): + if self.pos: + sys.stdout.write('\r'+(' '*(35+self.maxpos))+'\r') + sys.stdout.flush() + def heartbeat(self): + if not self.pos: + sys.stdout.write("\nINFO [#" + (' '*self.maxpos) + "] Waiting on the server...\r\x1b[8C") + self.pos += self.delta + if self.delta > 0: + s = " #\x1b[1D" if self.pos > 1 else "# \x1b[2D" + else: + s = "\x1b[1D# \x1b[2D" + sys.stdout.write(s) + sys.stdout.flush() + if self.pos > self.maxpos: + self.delta = -1 + elif self.pos < 2: + self.delta = 1 +class _NotVisualWait: + def __enter__(self): + return self + def __exit__(self, *args): + pass + def heartbeat(self): + pass + +def VisualWait(verbose): + return _VisualWait() if verbose > 1 else _NotVisualWait() + + +def exec_command_line(): # pragma: no cover - can't test main client = Fail2banClient() # Exit with correct return value if client.start(sys.argv): - sys.exit(0) + exit(0) else: - sys.exit(-1) + exit(-1) + diff --git a/fail2ban/client/fail2bancmdline.py b/fail2ban/client/fail2bancmdline.py new file mode 100644 index 00000000..3628f695 --- /dev/null +++ b/fail2ban/client/fail2bancmdline.py @@ -0,0 +1,245 @@ +#!/usr/bin/python +# emacs: -*- mode: python; py-indent-offset: 4; indent-tabs-mode: t -*- +# vi: set ft=python sts=4 ts=4 sw=4 noet : +# +# This file is part of Fail2Ban. +# +# Fail2Ban is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. +# +# Fail2Ban is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with Fail2Ban; if not, write to the Free Software +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +__author__ = "Fail2Ban Developers" +__copyright__ = "Copyright (c) 2004-2008 Cyril Jaquier, 2012-2014 Yaroslav Halchenko, 2014-2016 Serg G. Brester" +__license__ = "GPL" + +import getopt +import logging +import os +import sys + +from ..version import version +from ..protocol import printFormatted +from ..helpers import getLogger + +# Gets the instance of the logger. +logSys = getLogger("fail2ban") + +CONFIG_PARAMS = ("socket", "pidfile", "logtarget", "loglevel", "syslogsocket",) + + +class Fail2banCmdLine(): + + def __init__(self): + self._argv = self._args = None + self._configurator = None + self.resetConf() + + def resetConf(self): + self._conf = { + "async": False, + "conf": "/etc/fail2ban", + "force": False, + "background": True, + "verbose": 1, + "socket": None, + "pidfile": None + } + + @property + def configurator(self): + if self._configurator: + return self._configurator + # New configurator + from .configurator import Configurator + self._configurator = Configurator() + # Set the configuration path + self._configurator.setBaseDir(self._conf["conf"]) + return self._configurator + + + def applyMembers(self, obj): + for o in obj.__dict__: + self.__dict__[o] = obj.__dict__[o] + + def dispVersion(self): + print "Fail2Ban v" + version + print + print "Copyright (c) 2004-2008 Cyril Jaquier, 2008- Fail2Ban Contributors" + print "Copyright of modifications held by their respective authors." + print "Licensed under the GNU General Public License v2 (GPL)." + print + print "Written by Cyril Jaquier ." + print "Many contributions by Yaroslav O. Halchenko ." + + def dispUsage(self): + """ Prints Fail2Ban command line options and exits + """ + caller = os.path.basename(self._argv[0]) + print "Usage: "+caller+" [OPTIONS]" + (" " if not caller.endswith('server') else "") + print + print "Fail2Ban v" + version + " reads log file that contains password failure report" + print "and bans the corresponding IP addresses using firewall rules." + print + print "Options:" + print " -c configuration directory" + print " -s socket path" + print " -p pidfile path" + print " --loglevel logging level" + print " --logtarget |STDOUT|STDERR|SYSLOG" + print " --syslogsocket auto|" + print " -d dump configuration. For debugging" + print " -i interactive mode" + print " -v increase verbosity" + print " -q decrease verbosity" + print " -x force execution of the server (remove socket file)" + print " -b start server in background (default)" + print " -f start server in foreground" + print " --async start server in async mode (for internal usage only, don't read configuration)" + print " -h, --help display this help message" + print " -V, --version print the version" + + if not caller.endswith('server'): + print + print "Command:" + # Prints the protocol + printFormatted() + + print + print "Report bugs to https://github.com/fail2ban/fail2ban/issues" + + def __getCmdLineOptions(self, optList): + """ Gets the command line options + """ + for opt in optList: + o = opt[0] + if o == "--async": + self._conf["async"] = True + if o == "-c": + self._conf["conf"] = opt[1] + elif o == "-s": + self._conf["socket"] = opt[1] + elif o == "-p": + self._conf["pidfile"] = opt[1] + elif o.startswith("--log") or o.startswith("--sys"): + self._conf[ o[2:] ] = opt[1] + elif o == "-d": + self._conf["dump"] = True + elif o == "-v": + self._conf["verbose"] += 1 + elif o == "-q": + self._conf["verbose"] -= 1 + elif o == "-x": + self._conf["force"] = True + elif o == "-i": + self._conf["interactive"] = True + elif o == "-b": + self._conf["background"] = True + elif o == "-f": + self._conf["background"] = False + elif o in ["-h", "--help"]: + self.dispUsage() + exit(0) + elif o in ["-V", "--version"]: + self.dispVersion() + exit(0) + + def initCmdLine(self, argv): + # First time? + initial = (self._argv is None) + + # Command line options + self._argv = argv + + # Reads the command line options. + try: + cmdOpts = 'hc:s:p:xfbdviqV' + cmdLongOpts = ['loglevel=', 'logtarget=', 'syslogsocket=', 'async', 'help', 'version'] + optList, self._args = getopt.getopt(self._argv[1:], cmdOpts, cmdLongOpts) + except getopt.GetoptError: + self.dispUsage() + exit(-1) + + self.__getCmdLineOptions(optList) + + if initial: + verbose = self._conf["verbose"] + if verbose <= 0: + logSys.setLevel(logging.ERROR) + elif verbose == 1: + logSys.setLevel(logging.WARNING) + elif verbose == 2: + logSys.setLevel(logging.INFO) + else: + logSys.setLevel(logging.DEBUG) + # Add the default logging handler to dump to stderr + logout = logging.StreamHandler(sys.stderr) + # set a format which is simpler for console use + formatter = logging.Formatter('%(levelname)-6s %(message)s') + # tell the handler to use this format + logout.setFormatter(formatter) + logSys.addHandler(logout) + + # Set expected parameters (like socket, pidfile, etc) from configuration, + # if those not yet specified, in which read configuration only if needed here: + conf = None + for o in CONFIG_PARAMS: + if self._conf.get(o, None) is None: + if not conf: + self.configurator.readEarly() + conf = self.configurator.getEarlyOptions() + self._conf[o] = conf[o] + + logSys.info("Using socket file %s", self._conf["socket"]) + + logSys.info("Using pid file %s, [%s] logging to %s", + self._conf["pidfile"], self._conf["loglevel"], self._conf["logtarget"]) + + if self._conf.get("dump", False): + ret, stream = self.readConfig() + self.dumpConfig(stream) + return ret + + # Nothing to do here, process in client/server + return None + + def readConfig(self, jail=None): + # Read the configuration + # TODO: get away from stew of return codes and exception + # handling -- handle via exceptions + stream = None + try: + self.configurator.Reload() + self.configurator.readAll() + ret = self.configurator.getOptions(jail) + self.configurator.convertToProtocol() + stream = self.configurator.getConfigStream() + except Exception, e: + logSys.error("Failed during configuration: %s" % e) + ret = False + return ret, stream + + @staticmethod + def dumpConfig(cmd): + for c in cmd: + print c + return True + + @staticmethod + def exit(code=0): + logSys.debug("Exit with code %s", code) + if os._exit: + os._exit(code) + else: + sys.exit(code) + +# global exit handler: +exit = Fail2banCmdLine.exit \ No newline at end of file diff --git a/fail2ban/client/fail2banserver.py b/fail2ban/client/fail2banserver.py old mode 100755 new mode 100644 index 0b8b6418..6c1dd694 --- a/fail2ban/client/fail2banserver.py +++ b/fail2ban/client/fail2banserver.py @@ -1,7 +1,7 @@ #!/usr/bin/python # emacs: -*- mode: python; py-indent-offset: 4; indent-tabs-mode: t -*- # vi: set ft=python sts=4 ts=4 sw=4 noet : - +# # This file is part of Fail2Ban. # # Fail2Ban is free software; you can redistribute it and/or modify @@ -17,125 +17,171 @@ # You should have received a copy of the GNU General Public License # along with Fail2Ban; if not, write to the Free Software # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. - -__author__ = "Cyril Jaquier" -__copyright__ = "Copyright (c) 2004 Cyril Jaquier" +__author__ = "Fail2Ban Developers" +__copyright__ = "Copyright (c) 2004-2008 Cyril Jaquier, 2012-2014 Yaroslav Halchenko, 2014-2016 Serg G. Brester" __license__ = "GPL" -import getopt import os import sys -from fail2ban.version import version -from fail2ban.server.server import Server -from fail2ban.helpers import getLogger - -# Gets the instance of the logger. -logSys = getLogger("fail2ban") +from ..version import version +from ..server.server import Server, ServerDaemonize +from ..server.utils import Utils +from .fail2bancmdline import Fail2banCmdLine, logSys, exit +SERVER = "fail2ban-server" ## # \mainpage Fail2Ban # # \section Introduction # -# Fail2ban is designed to protect your server against brute force attacks. -# Its first goal was to protect a SSH server. +class Fail2banServer(Fail2banCmdLine): -class Fail2banServer: + # def __init__(self): + # Fail2banCmdLine.__init__(self) - def __init__(self): - self.__server = None - self.__argv = None - self.__conf = dict() - self.__conf["background"] = True - self.__conf["force"] = False - self.__conf["socket"] = "/var/run/fail2ban/fail2ban.sock" - self.__conf["pidfile"] = "/var/run/fail2ban/fail2ban.pid" + ## + # Start Fail2Ban server in main thread without fork (foreground). + # + # Start the Fail2ban server in foreground (daemon mode or not). - def dispVersion(self): - print "Fail2Ban v" + version - print - print "Copyright (c) 2004-2008 Cyril Jaquier, 2008- Fail2Ban Contributors" - print "Copyright of modifications held by their respective authors." - print "Licensed under the GNU General Public License v2 (GPL)." - print - print "Written by Cyril Jaquier ." - print "Many contributions by Yaroslav O. Halchenko ." + @staticmethod + def startServerDirect(conf, daemon=True): + server = None + try: + # Start it in foreground (current thread, not new process), + # server object will internally fork self if daemon is True + server = Server(daemon) + server.start(conf["socket"], + conf["pidfile"], conf["force"], + conf=conf) + except ServerDaemonize: + pass + except Exception, e: + logSys.exception(e) + if server: + server.quit() + exit(-1) - def dispUsage(self): - """ Prints Fail2Ban command line options and exits - """ - print "Usage: "+self.__argv[0]+" [OPTIONS]" - print - print "Fail2Ban v" + version + " reads log file that contains password failure report" - print "and bans the corresponding IP addresses using firewall rules." - print - print "Only use this command for debugging purpose. Start the server with" - print "fail2ban-client instead. The default behaviour is to start the server" - print "in background." - print - print "Options:" - print " -b start in background" - print " -f start in foreground" - print " -s socket path" - print " -p pidfile path" - print " -x force execution of the server (remove socket file)" - print " -h, --help display this help message" - print " -V, --version print the version" - print - print "Report bugs to https://github.com/fail2ban/fail2ban/issues" + return server - def __getCmdLineOptions(self, optList): - """ Gets the command line options - """ - for opt in optList: - if opt[0] == "-b": - self.__conf["background"] = True - if opt[0] == "-f": - self.__conf["background"] = False - if opt[0] == "-s": - self.__conf["socket"] = opt[1] - if opt[0] == "-p": - self.__conf["pidfile"] = opt[1] - if opt[0] == "-x": - self.__conf["force"] = True - if opt[0] in ["-h", "--help"]: - self.dispUsage() - sys.exit(0) - if opt[0] in ["-V", "--version"]: - self.dispVersion() - sys.exit(0) + ## + # Start Fail2Ban server. + # + # Start the Fail2ban server in daemon mode (background, start from client). + + @staticmethod + def startServerAsync(conf): + # Forks the current process. + pid = os.fork() + if pid == 0: + args = list() + args.append(SERVER) + # Start async (don't read config) and in background as requested. + args.append("--async") + args.append("-b") + # Set the socket path. + args.append("-s") + args.append(conf["socket"]) + # Set the pidfile + args.append("-p") + args.append(conf["pidfile"]) + # Force the execution if needed. + if conf["force"]: + args.append("-x") + # Logging parameters: + for o in ('loglevel', 'logtarget', 'syslogsocket'): + args.append("--"+o) + args.append(conf[o]) + + try: + # Use the current directory. + exe = os.path.abspath(os.path.join(sys.path[0], SERVER)) + logSys.debug("Starting %r with args %r", exe, args) + os.execv(exe, args) + except OSError: + try: + # Use the PATH env. + logSys.warning("Initial start attempt failed. Starting %r with the same args", SERVER) + os.execvp(SERVER, args) + except OSError: + exit(-1) + + def _Fail2banClient(self): + from .fail2banclient import Fail2banClient + cli = Fail2banClient() + cli.applyMembers(self) + return cli def start(self, argv): # Command line options - self.__argv = argv + ret = self.initCmdLine(argv) + if ret is not None: + return ret - # Reads the command line options. + # Commands + args = self._args + + cli = None + # If client mode - whole processing over client: + if len(args) or self._conf.get("interactive", False): + cli = self._Fail2banClient() + return cli.start(argv) + + # Start the server: + server = None try: - cmdOpts = 'bfs:p:xhV' - cmdLongOpts = ['help', 'version'] - optList, args = getopt.getopt(self.__argv[1:], cmdOpts, cmdLongOpts) - except getopt.GetoptError: - self.dispUsage() - sys.exit(-1) + # async = True, if started from client, should fork, daemonize, etc... + # background = True, if should start in new process, otherwise start in foreground + async = self._conf.get("async", False) + background = self._conf["background"] + # If was started not from the client: + if not async: + # Start new thread with client to read configuration and + # transfer it to the server: + cli = self._Fail2banClient() + phase = dict() + logSys.debug('Configure via async client thread') + cli.configureServer(async=True, phase=phase) + # wait up to 30 seconds, do not continue if configuration is not 100% valid: + Utils.wait_for(lambda: phase.get('ready', None) is not None, 30) + if not phase.get('start', False): + return False - self.__getCmdLineOptions(optList) + # Start server, daemonize it, etc. + if async or not background: + server = Fail2banServer.startServerDirect(self._conf, background) + else: + Fail2banServer.startServerAsync(self._conf) + if cli: + cli._server = server + + # wait for client answer "done": + if not async and cli: + Utils.wait_for(lambda: phase.get('done', None) is not None, 30) + if not phase.get('done', False): + if server: + server.quit() + exit(-1) + logSys.debug('Starting server done') - try: - self.__server = Server(self.__conf["background"]) - self.__server.start(self.__conf["socket"], - self.__conf["pidfile"], - self.__conf["force"]) - return True except Exception, e: logSys.exception(e) - if self.__server: - self.__server.quit() - return False + if server: + server.quit() + exit(-1) -if __name__ == "__main__": + return True + + @staticmethod + def exit(code=0): # pragma: no cover + if code != 0: + logSys.error("Could not start %s", SERVER) + exit(code) + +def exec_command_line(): # pragma: no cover - can't test main server = Fail2banServer() if server.start(sys.argv): - sys.exit(0) + exit(0) else: - sys.exit(-1) + exit(-1) diff --git a/fail2ban/protocol.py b/fail2ban/protocol.py index 5d9fdd65..857d5fa6 100644 --- a/fail2ban/protocol.py +++ b/fail2ban/protocol.py @@ -42,11 +42,13 @@ CSPROTO = dotdict({ protocol = [ ['', "BASIC", ""], ["start", "starts the server and the jails"], -["reload", "reloads the configuration"], +["restart", "restarts the server"], +["reload", "reloads the configuration without restart"], ["reload ", "reloads the jail "], ["stop", "stops all jails and terminate the server"], ["status", "gets the current status of the server"], -["ping", "tests if the server is alive"], +["ping", "tests if the server is alive"], +["echo", "for internal usage, returns back and outputs a given string"], ["help", "return this output"], ["version", "return the server version"], ['', "LOGGING", ""], diff --git a/fail2ban/server/asyncserver.py b/fail2ban/server/asyncserver.py index ad37544a..6454ef1c 100644 --- a/fail2ban/server/asyncserver.py +++ b/fail2ban/server/asyncserver.py @@ -95,6 +95,7 @@ def loop(active, timeout=None, use_poll=False): # Use poll instead of loop, because of recognition of active flag, # because of loop timeout mistake: different in poll and poll2 (sec vs ms), # and to prevent sporadical errors like EBADF 'Bad file descriptor' etc. (see gh-161) + errCount = 0 if timeout is None: timeout = Utils.DEFAULT_SLEEP_TIME poll = asyncore.poll @@ -107,11 +108,20 @@ def loop(active, timeout=None, use_poll=False): while active(): try: poll(timeout) + if errCount: + errCount -= 1 except Exception as e: # pragma: no cover - if e.args[0] in (errno.ENOTCONN, errno.EBADF): # (errno.EBADF, 'Bad file descriptor') - logSys.info('Server connection was closed: %s', str(e)) - else: - logSys.error('Server connection was closed: %s', str(e)) + if not active(): + break + errCount += 1 + if errCount < 20: + if e.args[0] in (errno.ENOTCONN, errno.EBADF): # (errno.EBADF, 'Bad file descriptor') + logSys.info('Server connection was closed: %s', str(e)) + else: + logSys.error('Server connection was closed: %s', str(e)) + elif errCount == 20: + logSys.info('Too many errors - stop logging connection errors') + logSys.exception(e) ## @@ -162,7 +172,7 @@ class AsyncServer(asyncore.dispatcher): logSys.error("Fail2ban seems to be already running") if force: logSys.warning("Forcing execution of the server") - os.remove(sock) + self._remove_sock() else: raise AsyncServerException("Server already running") # Creates the socket. @@ -175,20 +185,22 @@ class AsyncServer(asyncore.dispatcher): AsyncServer.__markCloseOnExec(self.socket) self.listen(1) # Sets the init flag. - self.__init = self.__active = True + self.__init = self.__loop = self.__active = True # Event loop as long as active: - loop(lambda: self.__active) + loop(lambda: self.__loop) + self.__active = False # Cleanup all self.stop() def close(self): if self.__active: + self.__loop = False asyncore.dispatcher.close(self) # Remove socket (file) only if it was created: if self.__init and os.path.exists(self.__sock): logSys.debug("Removed socket file " + self.__sock) - os.remove(self.__sock) + self._remove_sock() logSys.debug("Socket shutdown") self.__active = False @@ -201,6 +213,17 @@ class AsyncServer(asyncore.dispatcher): def isActive(self): return self.__active + + ## + # Safe remove (in multithreaded mode): + + def _remove_sock(self): + try: + os.remove(self.__sock) + except OSError as e: + if e.errno != errno.ENOENT: + raise + ## # Marks socket as close-on-exec to avoid leaking file descriptors when # running actions involving command execution. diff --git a/fail2ban/server/server.py b/fail2ban/server/server.py index 923b6ba3..72279b2d 100644 --- a/fail2ban/server/server.py +++ b/fail2ban/server/server.py @@ -67,6 +67,11 @@ class Server: 'FreeBSD': '/var/run/log', 'Linux': '/dev/log', } + # todo: remove that, if test cases are fixed + self.setSyslogSocket("auto") + # Set logging level + self.setLogLevel("INFO") + self.setLogTarget("STDOUT") def __sigTERMhandler(self, signum, frame): logSys.debug("Caught signal %d. Exiting" % signum) @@ -77,13 +82,27 @@ class Server: self.flushLogs() def start(self, sock, pidfile, force=False, conf={}): - # First set all logging parameters: - self.setSyslogSocket(conf.get("syslogsocket", "auto")) - self.setLogLevel(conf.get("loglevel", "INFO")) - self.setLogTarget(conf.get("logtarget", "STDOUT")) + # First set the mask to only allow access to owner + os.umask(0077) + # Second daemonize before logging etc, because it will close all handles: + if self.__daemon: # pragma: no cover + logSys.info("Starting in daemon mode") + ret = self.__createDaemon() + if not ret: + logSys.error("Could not create daemon") + raise ServerInitializationError("Could not create daemon") + + # Set all logging parameters (or use default if not specified): + self.setSyslogSocket(conf.get("syslogsocket", self.__syslogSocket)) + self.setLogLevel(conf.get("loglevel", self.__logLevel)) + self.setLogTarget(conf.get("logtarget", self.__logTarget)) + logSys.info("-"*50) logSys.info("Starting Fail2ban v%s", version.version) + if self.__daemon: # pragma: no cover + logSys.info("Daemon started") + # Install signal handlers signal.signal(signal.SIGTERM, self.__sigTERMhandler) signal.signal(signal.SIGINT, self.__sigTERMhandler) @@ -92,17 +111,6 @@ class Server: # Ensure unhandled exceptions are logged sys.excepthook = excepthook - # First set the mask to only allow access to owner - os.umask(0077) - if self.__daemon: # pragma: no cover - logSys.info("Starting in daemon mode") - ret = self.__createDaemon() - if ret: - logSys.info("Daemon started") - else: - logSys.error("Could not create daemon") - raise ServerInitializationError("Could not create daemon") - # Creates a PID file. try: logSys.debug("Creating PID file %s" % pidfile) @@ -139,11 +147,8 @@ class Server: self.stopAllJail() # Only now shutdown the logging. - try: - self.__loggingLock.acquire() + with self.__loggingLock: logging.shutdown() - finally: - self.__loggingLock.release() def addJail(self, name, backend): self.__jails.add(name, backend, self.__db) @@ -362,16 +367,15 @@ class Server: # @param value the level def setLogLevel(self, value): - try: - self.__loggingLock.acquire() - getLogger("fail2ban").setLevel( - getattr(logging, value.upper())) - except AttributeError: - raise ValueError("Invalid log level") - else: - self.__logLevel = value.upper() - finally: - self.__loggingLock.release() + value = value.upper() + with self.__loggingLock: + if self.__logLevel == value: + return + try: + getLogger("fail2ban").setLevel(getattr(logging, value)) + self.__logLevel = value + except AttributeError: + raise ValueError("Invalid log level") ## # Get the logging level. @@ -380,11 +384,8 @@ class Server: # @return the log level def getLogLevel(self): - try: - self.__loggingLock.acquire() + with self.__loggingLock: return self.__logLevel - finally: - self.__loggingLock.release() ## # Sets the logging target. @@ -470,24 +471,21 @@ class Server: # syslogsocket is the full path to the syslog socket # @param syslogsocket the syslog socket path def setSyslogSocket(self, syslogsocket): - self.__syslogSocket = syslogsocket - # Conditionally reload, logtarget depends on socket path when SYSLOG - return self.__logTarget != "SYSLOG"\ - or self.setLogTarget(self.__logTarget) + with self.__loggingLock: + if self.__syslogSocket == syslogsocket: + return True + self.__syslogSocket = syslogsocket + # Conditionally reload, logtarget depends on socket path when SYSLOG + return self.__logTarget != "SYSLOG"\ + or self.setLogTarget(self.__logTarget) def getLogTarget(self): - try: - self.__loggingLock.acquire() + with self.__loggingLock: return self.__logTarget - finally: - self.__loggingLock.release() def getSyslogSocket(self): - try: - self.__loggingLock.acquire() + with self.__loggingLock: return self.__syslogSocket - finally: - self.__loggingLock.release() def flushLogs(self): if self.__logTarget not in ['STDERR', 'STDOUT', 'SYSLOG']: @@ -542,7 +540,7 @@ class Server: # child process, and this makes sure that it is effect even if the parent # terminates quickly. signal.signal(signal.SIGHUP, signal.SIG_IGN) - + try: # Fork a child process so the parent can exit. This will return control # to the command line or shell. This is required so that the new process @@ -583,7 +581,7 @@ class Server: os._exit(0) # Exit parent (the first child) of the second child. else: os._exit(0) # Exit parent of the first child. - + # Close all open files. Try the system configuration variable, SC_OPEN_MAX, # for the maximum number of open files to close. If it doesn't exist, use # the default value (configurable). @@ -615,3 +613,6 @@ class Server: class ServerInitializationError(Exception): pass + +class ServerDaemonize(Exception): + pass \ No newline at end of file diff --git a/fail2ban/server/transmitter.py b/fail2ban/server/transmitter.py index 4c4c32f7..2194f591 100644 --- a/fail2ban/server/transmitter.py +++ b/fail2ban/server/transmitter.py @@ -93,6 +93,8 @@ class Transmitter: name = command[1] self.__server.stopJail(name) return None + elif command[0] == "echo": + return command[1:] elif command[0] == "sleep": value = command[1] time.sleep(float(value)) From f1208777561cfc7cbb3be948fe28d649ad90d433 Mon Sep 17 00:00:00 2001 From: sebres Date: Thu, 11 Feb 2016 17:57:23 +0100 Subject: [PATCH 46/52] client/server (bin) test cases introduced, ultimate closes #1121, closes #1139 small code review and fixing of some bugs during client-server communication process (in the test cases); --- bin/fail2ban-client | 4 +- bin/fail2ban-server | 4 +- fail2ban/client/fail2banclient.py | 146 ++++---- fail2ban/client/fail2bancmdline.py | 190 ++++++----- fail2ban/client/fail2banserver.py | 41 ++- fail2ban/protocol.py | 29 +- fail2ban/server/server.py | 46 ++- fail2ban/server/utils.py | 6 +- fail2ban/tests/fail2banclienttestcase.py | 411 +++++++++++++++++++++++ fail2ban/tests/fail2banregextestcase.py | 12 - fail2ban/tests/utils.py | 25 ++ 11 files changed, 710 insertions(+), 204 deletions(-) create mode 100644 fail2ban/tests/fail2banclienttestcase.py diff --git a/bin/fail2ban-client b/bin/fail2ban-client index 19e76a98..f5ae7946 100755 --- a/bin/fail2ban-client +++ b/bin/fail2ban-client @@ -31,7 +31,7 @@ __author__ = "Fail2Ban Developers" __copyright__ = "Copyright (c) 2004-2008 Cyril Jaquier, 2012-2014 Yaroslav Halchenko, 2014-2016 Serg G. Brester" __license__ = "GPL" -from fail2ban.client.fail2banclient import exec_command_line +from fail2ban.client.fail2banclient import exec_command_line, sys if __name__ == "__main__": - exec_command_line() + exec_command_line(sys.argv) diff --git a/bin/fail2ban-server b/bin/fail2ban-server index 8e64d865..ffafabe2 100755 --- a/bin/fail2ban-server +++ b/bin/fail2ban-server @@ -31,7 +31,7 @@ __author__ = "Fail2Ban Developers" __copyright__ = "Copyright (c) 2004-2008 Cyril Jaquier, 2012-2014 Yaroslav Halchenko, 2014-2016 Serg G. Brester" __license__ = "GPL" -from fail2ban.client.fail2banserver import exec_command_line +from fail2ban.client.fail2banserver import exec_command_line, sys if __name__ == "__main__": - exec_command_line() + exec_command_line(sys.argv) diff --git a/fail2ban/client/fail2banclient.py b/fail2ban/client/fail2banclient.py index 7adbab95..736f8fd2 100644 --- a/fail2ban/client/fail2banclient.py +++ b/fail2ban/client/fail2banclient.py @@ -28,12 +28,20 @@ import socket import sys import time +import threading from threading import Thread from ..version import version from .csocket import CSocket from .beautifier import Beautifier -from .fail2bancmdline import Fail2banCmdLine, logSys, exit +from .fail2bancmdline import Fail2banCmdLine, ExitException, logSys, exit, output + +MAX_WAITTIME = 30 + + +def _thread_name(): + return threading.current_thread().__class__.__name__ + ## # @@ -51,13 +59,13 @@ class Fail2banClient(Fail2banCmdLine, Thread): self._beautifier = None def dispInteractive(self): - print "Fail2Ban v" + version + " reads log file that contains password failure report" - print "and bans the corresponding IP addresses using firewall rules." - print + output("Fail2Ban v" + version + " reads log file that contains password failure report") + output("and bans the corresponding IP addresses using firewall rules.") + output("") def __sigTERMhandler(self, signum, frame): # Print a new line because we probably come from wait - print + output("") logSys.warning("Caught signal %d. Exiting" % signum) exit(-1) @@ -85,11 +93,11 @@ class Fail2banClient(Fail2banCmdLine, Thread): if ret[0] == 0: logSys.debug("OK : " + `ret[1]`) if showRet or c[0] == 'echo': - print beautifier.beautify(ret[1]) + output(beautifier.beautify(ret[1])) else: logSys.error("NOK: " + `ret[1].args`) if showRet: - print beautifier.beautifyError(ret[1]) + output(beautifier.beautifyError(ret[1])) streamRet = False except socket.error: if showRet or self._conf["verbose"] > 1: @@ -182,10 +190,13 @@ class Fail2banClient(Fail2banCmdLine, Thread): # Start server direct here in main thread (not fork): self._server = Fail2banServer.startServerDirect(self._conf, False) + except ExitException: + pass except Exception as e: - print - logSys.error("Exception while starting server foreground") + output("") + logSys.error("Exception while starting server " + ("background" if background else "foreground")) logSys.error(e) + return False finally: self._alive = False @@ -229,18 +240,18 @@ class Fail2banClient(Fail2banCmdLine, Thread): elif len(cmd) == 1 and cmd[0] == "restart": if self._conf.get("interactive", False): - print(' ## stop ... ') + output(' ## stop ... ') self.__processCommand(['stop']) self.__waitOnServer(False) # in interactive mode reset config, to make full-reload if there something changed: if self._conf.get("interactive", False): - print(' ## load configuration ... ') + output(' ## load configuration ... ') self.resetConf() ret = self.initCmdLine(self._argv) if ret is not None: return ret if self._conf.get("interactive", False): - print(' ## start ... ') + output(' ## start ... ') return self.__processCommand(['start']) elif len(cmd) >= 1 and cmd[0] == "reload": @@ -283,7 +294,9 @@ class Fail2banClient(Fail2banCmdLine, Thread): return False return True - def __waitOnServer(self, alive=True, maxtime=30): + def __waitOnServer(self, alive=True, maxtime=None): + if maxtime is None: + maxtime = MAX_WAITTIME # Wait for the server to start (the server has 30 seconds to answer ping) starttime = time.time() with VisualWait(self._conf["verbose"]) as vis: @@ -301,53 +314,59 @@ class Fail2banClient(Fail2banCmdLine, Thread): def start(self, argv): # Install signal handlers - signal.signal(signal.SIGTERM, self.__sigTERMhandler) - signal.signal(signal.SIGINT, self.__sigTERMhandler) + _prev_signals = {} + if _thread_name() == '_MainThread': + for s in (signal.SIGTERM, signal.SIGINT): + _prev_signals[s] = signal.getsignal(s) + signal.signal(s, self.__sigTERMhandler) + try: + # Command line options + if self._argv is None: + ret = self.initCmdLine(argv) + if ret is not None: + return ret - # Command line options - if self._argv is None: - ret = self.initCmdLine(argv) - if ret is not None: - return ret + # Commands + args = self._args - # Commands - args = self._args - - # Interactive mode - if self._conf.get("interactive", False): - try: - import readline - except ImportError: - logSys.error("Readline not available") - return False - try: - ret = True - if len(args) > 0: - ret = self.__processCommand(args) - if ret: - readline.parse_and_bind("tab: complete") - self.dispInteractive() - while True: - cmd = raw_input(self.PROMPT) - if cmd == "exit" or cmd == "quit": - # Exit - return True - if cmd == "help": - self.dispUsage() - elif not cmd == "": - try: - self.__processCommand(shlex.split(cmd)) - except Exception, e: - logSys.error(e) - except (EOFError, KeyboardInterrupt): - print - return True - # Single command mode - else: - if len(args) < 1: - self.dispUsage() - return False - return self.__processCommand(args) + # Interactive mode + if self._conf.get("interactive", False): + try: + import readline + except ImportError: + logSys.error("Readline not available") + return False + try: + ret = True + if len(args) > 0: + ret = self.__processCommand(args) + if ret: + readline.parse_and_bind("tab: complete") + self.dispInteractive() + while True: + cmd = raw_input(self.PROMPT) + if cmd == "exit" or cmd == "quit": + # Exit + return True + if cmd == "help": + self.dispUsage() + elif not cmd == "": + try: + self.__processCommand(shlex.split(cmd)) + except Exception, e: + logSys.error(e) + except (EOFError, KeyboardInterrupt): + output("") + return True + # Single command mode + else: + if len(args) < 1: + self.dispUsage() + return False + return self.__processCommand(args) + finally: + for s, sh in _prev_signals.iteritems(): + signal.signal(s, sh) class ServerExecutionException(Exception): @@ -361,7 +380,8 @@ class ServerExecutionException(Exception): class _VisualWait: pos = 0 delta = 1 - maxpos = 10 + def __init__(self, maxpos=10): + self.maxpos = maxpos def __enter__(self): return self def __exit__(self, *args): @@ -390,14 +410,14 @@ class _NotVisualWait: def heartbeat(self): pass -def VisualWait(verbose): - return _VisualWait() if verbose > 1 else _NotVisualWait() +def VisualWait(verbose, *args, **kwargs): + return _VisualWait(*args, **kwargs) if verbose > 1 else _NotVisualWait() -def exec_command_line(): # pragma: no cover - can't test main +def exec_command_line(argv): client = Fail2banClient() # Exit with correct return value - if client.start(sys.argv): + if client.start(argv): exit(0) else: exit(-1) diff --git a/fail2ban/client/fail2bancmdline.py b/fail2ban/client/fail2bancmdline.py index 3628f695..abbf8363 100644 --- a/fail2ban/client/fail2bancmdline.py +++ b/fail2ban/client/fail2bancmdline.py @@ -33,7 +33,12 @@ from ..helpers import getLogger # Gets the instance of the logger. logSys = getLogger("fail2ban") +def output(s): + print(s) + CONFIG_PARAMS = ("socket", "pidfile", "logtarget", "loglevel", "syslogsocket",) +# Used to signal - we are in test cases (ex: prevents change logging params, log capturing, etc) +PRODUCTION = True class Fail2banCmdLine(): @@ -71,50 +76,50 @@ class Fail2banCmdLine(): self.__dict__[o] = obj.__dict__[o] def dispVersion(self): - print "Fail2Ban v" + version - print - print "Copyright (c) 2004-2008 Cyril Jaquier, 2008- Fail2Ban Contributors" - print "Copyright of modifications held by their respective authors." - print "Licensed under the GNU General Public License v2 (GPL)." - print - print "Written by Cyril Jaquier ." - print "Many contributions by Yaroslav O. Halchenko ." + output("Fail2Ban v" + version) + output("") + output("Copyright (c) 2004-2008 Cyril Jaquier, 2008- Fail2Ban Contributors") + output("Copyright of modifications held by their respective authors.") + output("Licensed under the GNU General Public License v2 (GPL).") + output("") + output("Written by Cyril Jaquier .") + output("Many contributions by Yaroslav O. Halchenko .") def dispUsage(self): """ Prints Fail2Ban command line options and exits """ caller = os.path.basename(self._argv[0]) - print "Usage: "+caller+" [OPTIONS]" + (" " if not caller.endswith('server') else "") - print - print "Fail2Ban v" + version + " reads log file that contains password failure report" - print "and bans the corresponding IP addresses using firewall rules." - print - print "Options:" - print " -c configuration directory" - print " -s socket path" - print " -p pidfile path" - print " --loglevel logging level" - print " --logtarget |STDOUT|STDERR|SYSLOG" - print " --syslogsocket auto|" - print " -d dump configuration. For debugging" - print " -i interactive mode" - print " -v increase verbosity" - print " -q decrease verbosity" - print " -x force execution of the server (remove socket file)" - print " -b start server in background (default)" - print " -f start server in foreground" - print " --async start server in async mode (for internal usage only, don't read configuration)" - print " -h, --help display this help message" - print " -V, --version print the version" + output("Usage: "+caller+" [OPTIONS]" + (" " if not caller.endswith('server') else "")) + output("") + output("Fail2Ban v" + version + " reads log file that contains password failure report") + output("and bans the corresponding IP addresses using firewall rules.") + output("") + output("Options:") + output(" -c configuration directory") + output(" -s socket path") + output(" -p pidfile path") + output(" --loglevel logging level") + output(" --logtarget |STDOUT|STDERR|SYSLOG") + output(" --syslogsocket auto|") + output(" -d dump configuration. For debugging") + output(" -i interactive mode") + output(" -v increase verbosity") + output(" -q decrease verbosity") + output(" -x force execution of the server (remove socket file)") + output(" -b start server in background (default)") + output(" -f start server in foreground") + output(" --async start server in async mode (for internal usage only, don't read configuration)") + output(" -h, --help display this help message") + output(" -V, --version print the version") if not caller.endswith('server'): - print - print "Command:" + output("") + output("Command:") # Prints the protocol printFormatted() - print - print "Report bugs to https://github.com/fail2ban/fail2ban/issues" + output("") + output("Report bugs to https://github.com/fail2ban/fail2ban/issues") def __getCmdLineOptions(self, optList): """ Gets the command line options @@ -147,69 +152,78 @@ class Fail2banCmdLine(): self._conf["background"] = False elif o in ["-h", "--help"]: self.dispUsage() - exit(0) + return True elif o in ["-V", "--version"]: self.dispVersion() - exit(0) + return True + return None def initCmdLine(self, argv): - # First time? - initial = (self._argv is None) - - # Command line options - self._argv = argv - - # Reads the command line options. try: - cmdOpts = 'hc:s:p:xfbdviqV' - cmdLongOpts = ['loglevel=', 'logtarget=', 'syslogsocket=', 'async', 'help', 'version'] - optList, self._args = getopt.getopt(self._argv[1:], cmdOpts, cmdLongOpts) - except getopt.GetoptError: - self.dispUsage() - exit(-1) + # First time? + initial = (self._argv is None) - self.__getCmdLineOptions(optList) + # Command line options + self._argv = argv + logSys.info("Using start params %s", argv[1:]) - if initial: - verbose = self._conf["verbose"] - if verbose <= 0: - logSys.setLevel(logging.ERROR) - elif verbose == 1: - logSys.setLevel(logging.WARNING) - elif verbose == 2: - logSys.setLevel(logging.INFO) - else: - logSys.setLevel(logging.DEBUG) - # Add the default logging handler to dump to stderr - logout = logging.StreamHandler(sys.stderr) - # set a format which is simpler for console use - formatter = logging.Formatter('%(levelname)-6s %(message)s') - # tell the handler to use this format - logout.setFormatter(formatter) - logSys.addHandler(logout) + # Reads the command line options. + try: + cmdOpts = 'hc:s:p:xfbdviqV' + cmdLongOpts = ['loglevel=', 'logtarget=', 'syslogsocket=', 'async', 'help', 'version'] + optList, self._args = getopt.getopt(self._argv[1:], cmdOpts, cmdLongOpts) + except getopt.GetoptError: + self.dispUsage() + return False - # Set expected parameters (like socket, pidfile, etc) from configuration, - # if those not yet specified, in which read configuration only if needed here: - conf = None - for o in CONFIG_PARAMS: - if self._conf.get(o, None) is None: - if not conf: - self.configurator.readEarly() - conf = self.configurator.getEarlyOptions() - self._conf[o] = conf[o] + ret = self.__getCmdLineOptions(optList) + if ret is not None: + return ret - logSys.info("Using socket file %s", self._conf["socket"]) + if initial and PRODUCTION: # pragma: no cover - can't test + verbose = self._conf["verbose"] + if verbose <= 0: + logSys.setLevel(logging.ERROR) + elif verbose == 1: + logSys.setLevel(logging.WARNING) + elif verbose == 2: + logSys.setLevel(logging.INFO) + else: + logSys.setLevel(logging.DEBUG) + # Add the default logging handler to dump to stderr + logout = logging.StreamHandler(sys.stderr) + # set a format which is simpler for console use + formatter = logging.Formatter('%(levelname)-6s %(message)s') + # tell the handler to use this format + logout.setFormatter(formatter) + logSys.addHandler(logout) - logSys.info("Using pid file %s, [%s] logging to %s", - self._conf["pidfile"], self._conf["loglevel"], self._conf["logtarget"]) + # Set expected parameters (like socket, pidfile, etc) from configuration, + # if those not yet specified, in which read configuration only if needed here: + conf = None + for o in CONFIG_PARAMS: + if self._conf.get(o, None) is None: + if not conf: + self.configurator.readEarly() + conf = self.configurator.getEarlyOptions() + self._conf[o] = conf[o] - if self._conf.get("dump", False): - ret, stream = self.readConfig() - self.dumpConfig(stream) - return ret + logSys.info("Using socket file %s", self._conf["socket"]) - # Nothing to do here, process in client/server - return None + logSys.info("Using pid file %s, [%s] logging to %s", + self._conf["pidfile"], self._conf["loglevel"], self._conf["logtarget"]) + + if self._conf.get("dump", False): + ret, stream = self.readConfig() + self.dumpConfig(stream) + return ret + + # Nothing to do here, process in client/server + return None + except Exception as e: + output("ERROR: %s" % (e,)) + #logSys.exception(e) + return False def readConfig(self, jail=None): # Read the configuration @@ -242,4 +256,8 @@ class Fail2banCmdLine(): sys.exit(code) # global exit handler: -exit = Fail2banCmdLine.exit \ No newline at end of file +exit = Fail2banCmdLine.exit + + +class ExitException: + pass \ No newline at end of file diff --git a/fail2ban/client/fail2banserver.py b/fail2ban/client/fail2banserver.py index 6c1dd694..da8e57b8 100644 --- a/fail2ban/client/fail2banserver.py +++ b/fail2ban/client/fail2banserver.py @@ -29,7 +29,11 @@ from ..server.server import Server, ServerDaemonize from ..server.utils import Utils from .fail2bancmdline import Fail2banCmdLine, logSys, exit +MAX_WAITTIME = 30 + SERVER = "fail2ban-server" + + ## # \mainpage Fail2Ban # @@ -72,8 +76,15 @@ class Fail2banServer(Fail2banCmdLine): @staticmethod def startServerAsync(conf): - # Forks the current process. - pid = os.fork() + # Directory of client (to try the first start from the same directory as client): + startdir = sys.path[0] + if startdir in ("", "."): # may be uresolved in test-cases, so get bin-directory: + startdir = os.path.dirname(sys.argv[0]) + # Forks the current process, don't fork if async specified (ex: test cases) + pid = 0 + frk = not conf["async"] + if frk: + pid = os.fork() if pid == 0: args = list() args.append(SERVER) @@ -96,14 +107,20 @@ class Fail2banServer(Fail2banCmdLine): try: # Use the current directory. - exe = os.path.abspath(os.path.join(sys.path[0], SERVER)) + exe = os.path.abspath(os.path.join(startdir, SERVER)) logSys.debug("Starting %r with args %r", exe, args) - os.execv(exe, args) - except OSError: + if frk: + os.execv(exe, args) + else: + os.spawnv(os.P_NOWAITO, exe, args) + except OSError as e: try: # Use the PATH env. - logSys.warning("Initial start attempt failed. Starting %r with the same args", SERVER) - os.execvp(SERVER, args) + logSys.warning("Initial start attempt failed (%s). Starting %r with the same args", e, SERVER) + if frk: + os.execvp(SERVER, args) + else: + os.spawnvp(os.P_NOWAITO, SERVER, args) except OSError: exit(-1) @@ -143,8 +160,8 @@ class Fail2banServer(Fail2banCmdLine): phase = dict() logSys.debug('Configure via async client thread') cli.configureServer(async=True, phase=phase) - # wait up to 30 seconds, do not continue if configuration is not 100% valid: - Utils.wait_for(lambda: phase.get('ready', None) is not None, 30) + # wait up to MAX_WAITTIME, do not continue if configuration is not 100% valid: + Utils.wait_for(lambda: phase.get('ready', None) is not None, MAX_WAITTIME) if not phase.get('start', False): return False @@ -158,7 +175,7 @@ class Fail2banServer(Fail2banCmdLine): # wait for client answer "done": if not async and cli: - Utils.wait_for(lambda: phase.get('done', None) is not None, 30) + Utils.wait_for(lambda: phase.get('done', None) is not None, MAX_WAITTIME) if not phase.get('done', False): if server: server.quit() @@ -179,9 +196,9 @@ class Fail2banServer(Fail2banCmdLine): logSys.error("Could not start %s", SERVER) exit(code) -def exec_command_line(): # pragma: no cover - can't test main +def exec_command_line(argv): server = Fail2banServer() - if server.start(sys.argv): + if server.start(argv): exit(0) else: exit(-1) diff --git a/fail2ban/protocol.py b/fail2ban/protocol.py index 857d5fa6..648666a1 100644 --- a/fail2ban/protocol.py +++ b/fail2ban/protocol.py @@ -26,6 +26,9 @@ __license__ = "GPL" import textwrap +def output(s): + print(s) + ## # Describes the protocol used to communicate with the server. @@ -143,7 +146,7 @@ def printFormatted(): firstHeading = False for m in protocol: if m[0] == '' and firstHeading: - print + output("") firstHeading = True first = True if len(m[0]) >= MARGIN: @@ -154,7 +157,7 @@ def printFormatted(): first = False else: line = ' ' * (INDENT + MARGIN) + n.strip() - print line + output(line) ## @@ -165,20 +168,20 @@ def printWiki(): for m in protocol: if m[0] == '': if firstHeading: - print "|}" + output("|}") __printWikiHeader(m[1], m[2]) firstHeading = True else: - print "|-" - print "| " + m[0] + " || || " + m[1] - print "|}" + output("|-") + output("| " + m[0] + " || || " + m[1]) + output("|}") def __printWikiHeader(section, desc): - print - print "=== " + section + " ===" - print - print desc - print - print "{|" - print "| '''Command''' || || '''Description'''" + output("") + output("=== " + section + " ===") + output("") + output(desc) + output("") + output("{|") + output("| '''Command''' || || '''Description'''") diff --git a/fail2ban/server/server.py b/fail2ban/server/server.py index 72279b2d..a542659b 100644 --- a/fail2ban/server/server.py +++ b/fail2ban/server/server.py @@ -24,6 +24,7 @@ __author__ = "Cyril Jaquier" __copyright__ = "Copyright (c) 2004 Cyril Jaquier" __license__ = "GPL" +import threading from threading import Lock, RLock import logging import logging.handlers @@ -42,6 +43,10 @@ from ..helpers import getLogger, excepthook # Gets the instance of the logger. logSys = getLogger(__name__) +DEF_SYSLOGSOCKET = "auto" +DEF_LOGLEVEL = "INFO" +DEF_LOGTARGET = "STDOUT" + try: from .database import Fail2BanDb except ImportError: # pragma: no cover @@ -49,6 +54,10 @@ except ImportError: # pragma: no cover Fail2BanDb = None +def _thread_name(): + return threading.current_thread().__class__.__name__ + + class Server: def __init__(self, daemon = False): @@ -67,11 +76,7 @@ class Server: 'FreeBSD': '/var/run/log', 'Linux': '/dev/log', } - # todo: remove that, if test cases are fixed - self.setSyslogSocket("auto") - # Set logging level - self.setLogLevel("INFO") - self.setLogTarget("STDOUT") + self.__prev_signals = {} def __sigTERMhandler(self, signum, frame): logSys.debug("Caught signal %d. Exiting" % signum) @@ -93,9 +98,12 @@ class Server: raise ServerInitializationError("Could not create daemon") # Set all logging parameters (or use default if not specified): - self.setSyslogSocket(conf.get("syslogsocket", self.__syslogSocket)) - self.setLogLevel(conf.get("loglevel", self.__logLevel)) - self.setLogTarget(conf.get("logtarget", self.__logTarget)) + self.setSyslogSocket(conf.get("syslogsocket", + self.__syslogSocket if self.__syslogSocket is not None else DEF_SYSLOGSOCKET)) + self.setLogLevel(conf.get("loglevel", + self.__logLevel if self.__logLevel is not None else DEF_LOGLEVEL)) + self.setLogTarget(conf.get("logtarget", + self.__logTarget if self.__logTarget is not None else DEF_LOGTARGET)) logSys.info("-"*50) logSys.info("Starting Fail2ban v%s", version.version) @@ -104,10 +112,10 @@ class Server: logSys.info("Daemon started") # Install signal handlers - signal.signal(signal.SIGTERM, self.__sigTERMhandler) - signal.signal(signal.SIGINT, self.__sigTERMhandler) - signal.signal(signal.SIGUSR1, self.__sigUSR1handler) - + if _thread_name() == '_MainThread': + for s in (signal.SIGTERM, signal.SIGINT, signal.SIGUSR1): + self.__prev_signals[s] = signal.getsignal(s) + signal.signal(s, self.__sigTERMhandler if s != signal.SIGUSR1 else self.__sigUSR1handler) # Ensure unhandled exceptions are logged sys.excepthook = excepthook @@ -150,6 +158,10 @@ class Server: with self.__loggingLock: logging.shutdown() + # Restore default signal handlers: + for s, sh in self.__prev_signals.iteritems(): + signal.signal(s, sh) + def addJail(self, name, backend): self.__jails.add(name, backend, self.__db) if self.__db is not None: @@ -395,8 +407,13 @@ class Server: def setLogTarget(self, target): with self.__loggingLock: + # don't set new handlers if already the same + # or if "INHERITED" (foreground worker of the test cases, to prevent stop logging): if self.__logTarget == target: return True + if target == "INHERITED": + self.__logTarget = target + return True # set a format which is simpler for console use formatter = logging.Formatter("%(asctime)s %(name)-24s[%(process)d]: %(levelname)-7s %(message)s") if target == "SYSLOG": @@ -539,7 +556,10 @@ class Server: # We need to set this in the parent process, so it gets inherited by the # child process, and this makes sure that it is effect even if the parent # terminates quickly. - signal.signal(signal.SIGHUP, signal.SIG_IGN) + if _thread_name() == '_MainThread': + for s in (signal.SIGHUP,): + self.__prev_signals[s] = signal.getsignal(s) + signal.signal(s, signal.SIG_IGN) try: # Fork a child process so the parent can exit. This will return control diff --git a/fail2ban/server/utils.py b/fail2ban/server/utils.py index 45d1c09d..a8496f8e 100644 --- a/fail2ban/server/utils.py +++ b/fail2ban/server/utils.py @@ -125,6 +125,7 @@ class Utils(): timeout_expr = lambda: time.time() - stime <= timeout else: timeout_expr = timeout + popen = None try: popen = subprocess.Popen( realCmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=shell, @@ -151,7 +152,10 @@ class Utils(): if retcode is None and not Utils.pid_exists(pgid): retcode = signal.SIGKILL except OSError as e: - logSys.error("%s -- failed with %s" % (realCmd, e)) + stderr = "%s -- failed with %s" % (realCmd, e) + logSys.error(stderr) + if not popen: + return False if not output else (False, stdout, stderr, retcode) std_level = retcode == 0 and logging.DEBUG or logging.ERROR # if we need output (to return or to log it): diff --git a/fail2ban/tests/fail2banclienttestcase.py b/fail2ban/tests/fail2banclienttestcase.py new file mode 100644 index 00000000..7db48fe8 --- /dev/null +++ b/fail2ban/tests/fail2banclienttestcase.py @@ -0,0 +1,411 @@ +# emacs: -*- mode: python; py-indent-offset: 4; indent-tabs-mode: t -*- +# vi: set ft=python sts=4 ts=4 sw=4 noet : + +# This file is part of Fail2Ban. +# +# Fail2Ban is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. +# +# Fail2Ban is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with Fail2Ban; if not, write to the Free Software +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + +# Fail2Ban developers + +__author__ = "Serg Brester" +__copyright__ = "Copyright (c) 2014- Serg G. Brester (sebres), 2008- Fail2Ban Contributors" +__license__ = "GPL" + +import fileinput +import os +import re +import time +import unittest + +from threading import Thread + +from ..client import fail2banclient, fail2banserver, fail2bancmdline +from ..client.fail2banclient import Fail2banClient, exec_command_line as _exec_client, VisualWait +from ..client.fail2banserver import Fail2banServer, exec_command_line as _exec_server +from .. import protocol +from ..server import server +from ..server.utils import Utils +from .utils import LogCaptureTestCase, logSys, withtmpdir, shutil, logging + + +STOCK_CONF_DIR = "config" +STOCK = os.path.exists(os.path.join(STOCK_CONF_DIR,'fail2ban.conf')) +TEST_CONF_DIR = os.path.join(os.path.dirname(__file__), "config") +if STOCK: + CONF_DIR = STOCK_CONF_DIR +else: + CONF_DIR = TEST_CONF_DIR + +CLIENT = "fail2ban-client" +SERVER = "fail2ban-server" +BIN = os.path.join(os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))), "bin") + +MAX_WAITTIME = 10 +MAX_WAITTIME = unittest.F2B.maxWaitTime(MAX_WAITTIME) + +## +# Several wrappers and settings for proper testing: +# + +fail2banclient.MAX_WAITTIME = \ +fail2banserver.MAX_WAITTIME = MAX_WAITTIME + + +fail2bancmdline.logSys = \ +fail2banclient.logSys = \ +fail2banserver.logSys = logSys + +LOG_LEVEL = logSys.level + +server.DEF_LOGTARGET = "/dev/null" + +def _test_output(*args): + logSys.info(args[0]) +fail2bancmdline.output = \ +fail2banclient.output = \ +fail2banserver.output = \ +protocol.output = _test_output + +def _test_exit(code=0): + logSys.debug("Exit with code %s", code) + if code == 0: + raise ExitException() + else: + raise FailExitException() +fail2bancmdline.exit = \ +fail2banclient.exit = \ +fail2banserver.exit = _test_exit + +INTERACT = [] +def _test_raw_input(*args): + if len(INTERACT): + #print('--- interact command: ', INTERACT[0]) + return INTERACT.pop(0) + else: + return "exit" +fail2banclient.raw_input = _test_raw_input + +# prevents change logging params, log capturing, etc: +fail2bancmdline.PRODUCTION = False + + +class ExitException(fail2bancmdline.ExitException): + pass +class FailExitException(fail2bancmdline.ExitException): + pass + + +def _out_file(fn): # pragma: no cover + logSys.debug('---- ' + fn + ' ----') + for line in fileinput.input(fn): + line = line.rstrip('\n') + logSys.debug(line) + logSys.debug('-'*30) + +def _start_params(tmp, use_stock=False, logtarget="/dev/null"): + cfg = tmp+"/config" + if use_stock and STOCK: + # copy config: + def ig_dirs(dir, files): + return [f for f in files if not os.path.isfile(os.path.join(dir, f))] + shutil.copytree(STOCK_CONF_DIR, cfg, ignore=ig_dirs) + os.symlink(STOCK_CONF_DIR+"/action.d", cfg+"/action.d") + os.symlink(STOCK_CONF_DIR+"/filter.d", cfg+"/filter.d") + # replace fail2ban params (database with memory): + r = re.compile(r'^dbfile\s*=') + for line in fileinput.input(cfg+"/fail2ban.conf", inplace=True): + line = line.rstrip('\n') + if r.match(line): + line = "dbfile = :memory:" + print(line) + # replace jail params (polling as backend to be fast in initialize): + r = re.compile(r'^backend\s*=') + for line in fileinput.input(cfg+"/jail.conf", inplace=True): + line = line.rstrip('\n') + if r.match(line): + line = "backend = polling" + print(line) + else: + # just empty config directory without anything (only fail2ban.conf/jail.conf): + os.mkdir(cfg) + f = open(cfg+"/fail2ban.conf", "wb") + f.write('\n'.join(( + "[Definition]", + "loglevel = INFO", + "logtarget = " + logtarget, + "syslogsocket = auto", + "socket = "+tmp+"/f2b.sock", + "pidfile = "+tmp+"/f2b.pid", + "backend = polling", + "dbfile = :memory:", + "dbpurgeage = 1d", + "", + ))) + f.close() + f = open(cfg+"/jail.conf", "wb") + f.write('\n'.join(( + "[INCLUDES]", "", + "[DEFAULT]", "", + "", + ))) + f.close() + if LOG_LEVEL < logging.DEBUG: # if HEAVYDEBUG + _out_file(cfg+"/fail2ban.conf") + _out_file(cfg+"/jail.conf") + # parameters: + return ("-c", cfg, + "--logtarget", logtarget, "--loglevel", "DEBUG", "--syslogsocket", "auto", + "-s", tmp+"/f2b.sock", "-p", tmp+"/f2b.pid") + + +class Fail2banClientTest(LogCaptureTestCase): + + def setUp(self): + """Call before every test case.""" + LogCaptureTestCase.setUp(self) + + def tearDown(self): + """Call after every test case.""" + LogCaptureTestCase.tearDown(self) + + def testClientUsage(self): + self.assertRaises(ExitException, _exec_client, + (CLIENT, "-h",)) + self.assertLogged("Usage: " + CLIENT) + self.assertLogged("Report bugs to ") + + @withtmpdir + def testClientStartBackgroundInside(self, tmp): + # always add "--async" by start inside, should don't fork by async (not replace client with server, just start in new process) + # (we can't fork the test cases process): + startparams = _start_params(tmp, True) + # start: + self.assertRaises(ExitException, _exec_client, + (CLIENT, "--async", "-b") + startparams + ("start",)) + self.assertLogged("Server ready") + self.assertLogged("Exit with code 0") + try: + self.assertRaises(ExitException, _exec_client, + (CLIENT,) + startparams + ("echo", "TEST-ECHO",)) + self.assertRaises(FailExitException, _exec_client, + (CLIENT,) + startparams + ("~~unknown~cmd~failed~~",)) + finally: + self.pruneLog() + # stop: + self.assertRaises(ExitException, _exec_client, + (CLIENT,) + startparams + ("stop",)) + self.assertLogged("Shutdown successful") + self.assertLogged("Exit with code 0") + + @withtmpdir + def testClientStartBackgroundCall(self, tmp): + global INTERACT + startparams = _start_params(tmp) + # start (without async in new process): + cmd = os.path.join(os.path.join(BIN), CLIENT) + logSys.debug('Start %s ...', cmd) + Utils.executeCmd((cmd,) + startparams + ("start",), + timeout=MAX_WAITTIME, shell=False, output=False) + self.pruneLog() + try: + # echo from client (inside): + self.assertRaises(ExitException, _exec_client, + (CLIENT,) + startparams + ("echo", "TEST-ECHO",)) + self.assertLogged("TEST-ECHO") + self.assertLogged("Exit with code 0") + self.pruneLog() + # interactive client chat with started server: + INTERACT += [ + "echo INTERACT-ECHO", + "status", + "exit" + ] + self.assertRaises(ExitException, _exec_client, + (CLIENT,) + startparams + ("-i",)) + self.assertLogged("INTERACT-ECHO") + self.assertLogged("Status", "Number of jail:") + self.assertLogged("Exit with code 0") + self.pruneLog() + # test reload and restart over interactive client: + INTERACT += [ + "reload", + "restart", + "exit" + ] + self.assertRaises(ExitException, _exec_client, + (CLIENT,) + startparams + ("-i",)) + self.assertLogged("Reading config files:") + self.assertLogged("Shutdown successful") + self.assertLogged("Server ready") + self.assertLogged("Exit with code 0") + self.pruneLog() + finally: + self.pruneLog() + # stop: + self.assertRaises(ExitException, _exec_client, + (CLIENT,) + startparams + ("stop",)) + self.assertLogged("Shutdown successful") + self.assertLogged("Exit with code 0") + + def _testClientStartForeground(self, tmp, startparams, phase): + # start and wait to end (foreground): + phase['start'] = True + self.assertRaises(ExitException, _exec_client, + (CLIENT, "-f") + startparams + ("start",)) + # end : + phase['end'] = True + + @withtmpdir + def testClientStartForeground(self, tmp): + # started directly here, so prevent overwrite test cases logger with "INHERITED" + startparams = _start_params(tmp, logtarget="INHERITED") + # because foreground block execution - start it in thread: + phase = dict() + Thread(name="_TestCaseWorker", + target=Fail2banClientTest._testClientStartForeground, args=(self, tmp, startparams, phase)).start() + try: + # wait for start thread: + Utils.wait_for(lambda: phase.get('start', None) is not None, MAX_WAITTIME) + self.assertTrue(phase.get('start', None)) + # wait for server (socket): + Utils.wait_for(lambda: os.path.exists(tmp+"/f2b.sock"), MAX_WAITTIME) + self.assertLogged("Starting communication") + self.assertRaises(ExitException, _exec_client, + (CLIENT,) + startparams + ("ping",)) + self.assertRaises(FailExitException, _exec_client, + (CLIENT,) + startparams + ("~~unknown~cmd~failed~~",)) + self.assertRaises(ExitException, _exec_client, + (CLIENT,) + startparams + ("echo", "TEST-ECHO",)) + finally: + self.pruneLog() + # stop: + self.assertRaises(ExitException, _exec_client, + (CLIENT,) + startparams + ("stop",)) + # wait for end: + Utils.wait_for(lambda: phase.get('end', None) is not None, MAX_WAITTIME) + self.assertTrue(phase.get('end', None)) + self.assertLogged("Shutdown successful", "Exiting Fail2ban") + self.assertLogged("Exit with code 0") + + @withtmpdir + def testClientFailStart(self, tmp): + self.assertRaises(FailExitException, _exec_client, + (CLIENT, "--async", "-c", tmp+"/miss", "start",)) + self.assertLogged("Base configuration directory " + tmp+"/miss" + " does not exist") + + self.assertRaises(FailExitException, _exec_client, + (CLIENT, "--async", "-c", CONF_DIR, "-s", tmp+"/miss/f2b.sock", "start",)) + self.assertLogged("There is no directory " + tmp+"/miss" + " to contain the socket file") + + def testVisualWait(self): + sleeptime = 0.035 + for verbose in (2, 0): + cntr = 15 + with VisualWait(verbose, 5) as vis: + while cntr: + vis.heartbeat() + if verbose and not unittest.F2B.fast: + time.sleep(sleeptime) + cntr -= 1 + + +class Fail2banServerTest(LogCaptureTestCase): + + def setUp(self): + """Call before every test case.""" + LogCaptureTestCase.setUp(self) + + def tearDown(self): + """Call after every test case.""" + LogCaptureTestCase.tearDown(self) + + def testServerUsage(self): + self.assertRaises(ExitException, _exec_server, + (SERVER, "-h",)) + self.assertLogged("Usage: " + SERVER) + self.assertLogged("Report bugs to ") + + @withtmpdir + def testServerStartBackground(self, tmp): + # don't add "--async" by start, because if will fork current process by daemonize + # (we can't fork the test cases process), + # because server started internal communication in new thread use INHERITED as logtarget here: + startparams = _start_params(tmp, logtarget="INHERITED") + # start: + self.assertRaises(ExitException, _exec_server, + (SERVER, "-b") + startparams) + self.assertLogged("Server ready") + self.assertLogged("Exit with code 0") + try: + self.assertRaises(ExitException, _exec_server, + (SERVER,) + startparams + ("echo", "TEST-ECHO",)) + self.assertRaises(FailExitException, _exec_server, + (SERVER,) + startparams + ("~~unknown~cmd~failed~~",)) + finally: + self.pruneLog() + # stop: + self.assertRaises(ExitException, _exec_server, + (SERVER,) + startparams + ("stop",)) + self.assertLogged("Shutdown successful") + self.assertLogged("Exit with code 0") + + def _testServerStartForeground(self, tmp, startparams, phase): + # start and wait to end (foreground): + phase['start'] = True + self.assertRaises(ExitException, _exec_server, + (SERVER, "-f") + startparams + ("start",)) + # end : + phase['end'] = True + @withtmpdir + def testServerStartForeground(self, tmp): + # started directly here, so prevent overwrite test cases logger with "INHERITED" + startparams = _start_params(tmp, logtarget="INHERITED") + # because foreground block execution - start it in thread: + phase = dict() + Thread(name="_TestCaseWorker", + target=Fail2banServerTest._testServerStartForeground, args=(self, tmp, startparams, phase)).start() + try: + # wait for start thread: + Utils.wait_for(lambda: phase.get('start', None) is not None, MAX_WAITTIME) + self.assertTrue(phase.get('start', None)) + # wait for server (socket): + Utils.wait_for(lambda: os.path.exists(tmp+"/f2b.sock"), MAX_WAITTIME) + self.assertLogged("Starting communication") + self.assertRaises(ExitException, _exec_server, + (SERVER,) + startparams + ("ping",)) + self.assertRaises(FailExitException, _exec_server, + (SERVER,) + startparams + ("~~unknown~cmd~failed~~",)) + self.assertRaises(ExitException, _exec_server, + (SERVER,) + startparams + ("echo", "TEST-ECHO",)) + finally: + self.pruneLog() + # stop: + self.assertRaises(ExitException, _exec_server, + (SERVER,) + startparams + ("stop",)) + # wait for end: + Utils.wait_for(lambda: phase.get('end', None) is not None, MAX_WAITTIME) + self.assertTrue(phase.get('end', None)) + self.assertLogged("Shutdown successful", "Exiting Fail2ban") + self.assertLogged("Exit with code 0") + + @withtmpdir + def testServerFailStart(self, tmp): + self.assertRaises(FailExitException, _exec_server, + (SERVER, "-c", tmp+"/miss",)) + self.assertLogged("Base configuration directory " + tmp+"/miss" + " does not exist") + + self.assertRaises(FailExitException, _exec_server, + (SERVER, "-c", CONF_DIR, "-s", tmp+"/miss/f2b.sock",)) + self.assertLogged("There is no directory " + tmp+"/miss" + " to contain the socket file") diff --git a/fail2ban/tests/fail2banregextestcase.py b/fail2ban/tests/fail2banregextestcase.py index 2fd362c7..a1dcb4da 100644 --- a/fail2ban/tests/fail2banregextestcase.py +++ b/fail2ban/tests/fail2banregextestcase.py @@ -23,19 +23,7 @@ __author__ = "Serg Brester" __copyright__ = "Copyright (c) 2015 Serg G. Brester (sebres), 2008- Fail2Ban Contributors" __license__ = "GPL" -from __builtin__ import open as fopen -import unittest -import getpass import os -import sys -import time -import tempfile -import uuid - -try: - from systemd import journal -except ImportError: - journal = None from ..client import fail2banregex from ..client.fail2banregex import Fail2banRegex, get_opt_parser, output diff --git a/fail2ban/tests/utils.py b/fail2ban/tests/utils.py index ce6d638a..9cdace42 100644 --- a/fail2ban/tests/utils.py +++ b/fail2ban/tests/utils.py @@ -26,10 +26,14 @@ import logging import optparse import os import re +import tempfile +import shutil import sys import time import unittest + from StringIO import StringIO +from functools import wraps from ..helpers import getLogger from ..server.filter import DNSUtils @@ -71,6 +75,17 @@ class F2B(optparse.Values): return wtime +def withtmpdir(f): + @wraps(f) + def wrapper(self, *args, **kwargs): + tmp = tempfile.mkdtemp(prefix="f2b-temp") + try: + return f(self, tmp, *args, **kwargs) + finally: + # clean up + shutil.rmtree(tmp) + return wrapper + def initTests(opts): unittest.F2B = F2B(opts) # --fast : @@ -145,6 +160,7 @@ def gatherTests(regexps=None, opts=None): from . import misctestcase from . import databasetestcase from . import samplestestcase + from . import fail2banclienttestcase from . import fail2banregextestcase if not regexps: # pragma: no cover @@ -223,6 +239,9 @@ def gatherTests(regexps=None, opts=None): # Filter Regex tests with sample logs tests.addTest(unittest.makeSuite(samplestestcase.FilterSamplesRegex)) + # bin/fail2ban-client, bin/fail2ban-server + tests.addTest(unittest.makeSuite(fail2banclienttestcase.Fail2banClientTest)) + tests.addTest(unittest.makeSuite(fail2banclienttestcase.Fail2banServerTest)) # bin/fail2ban-regex tests.addTest(unittest.makeSuite(fail2banregextestcase.Fail2banRegexTest)) @@ -293,8 +312,11 @@ class LogCaptureTestCase(unittest.TestCase): # Let's log everything into a string self._log = StringIO() logSys.handlers = [logging.StreamHandler(self._log)] + if self._old_level <= logging.DEBUG: + print("") if self._old_level < logging.DEBUG: # so if HEAVYDEBUG etc -- show them! logSys.handlers += self._old_handlers + logSys.debug('--'*40) logSys.setLevel(getattr(logging, 'DEBUG')) def tearDown(self): @@ -340,6 +362,9 @@ class LogCaptureTestCase(unittest.TestCase): raise AssertionError("All of the %r were found present in the log: %r" % (s, logged)) + def pruneLog(self): + self._log.truncate(0) + def getLog(self): return self._log.getvalue() From 0e11d81adb6796efc5d70d29f5a70920dbdd0a03 Mon Sep 17 00:00:00 2001 From: sebres Date: Thu, 11 Feb 2016 21:15:03 +0100 Subject: [PATCH 47/52] several bug fixed: fork in client-server test cases prohibited, all worker threads daemonized (to prevent hanging on exit). --- fail2ban/client/fail2banclient.py | 27 +- fail2ban/client/fail2banserver.py | 11 +- fail2ban/server/jailthread.py | 2 + fail2ban/server/server.py | 6 +- fail2ban/tests/action_d/test_smtp.py | 1 + fail2ban/tests/fail2banclienttestcase.py | 373 ++++++++++++++--------- fail2ban/tests/utils.py | 3 +- 7 files changed, 255 insertions(+), 168 deletions(-) diff --git a/fail2ban/client/fail2banclient.py b/fail2ban/client/fail2banclient.py index 736f8fd2..4f4caf79 100644 --- a/fail2ban/client/fail2banclient.py +++ b/fail2ban/client/fail2banclient.py @@ -34,7 +34,7 @@ from threading import Thread from ..version import version from .csocket import CSocket from .beautifier import Beautifier -from .fail2bancmdline import Fail2banCmdLine, ExitException, logSys, exit, output +from .fail2bancmdline import Fail2banCmdLine, ExitException, PRODUCTION, logSys, exit, output MAX_WAITTIME = 30 @@ -108,8 +108,13 @@ class Fail2banClient(Fail2banCmdLine, Thread): logSys.error(e) return False finally: + # prevent errors by close during shutdown (on exit command): if client: - client.close() + try : + client.close() + except Exception as e: + if showRet or self._conf["verbose"] > 1: + logSys.debug(e) if showRet or c[0] == 'echo': sys.stdout.flush() return streamRet @@ -184,7 +189,9 @@ class Fail2banClient(Fail2banCmdLine, Thread): return False else: # In foreground mode we should make server/client communication in different threads: - Thread(target=Fail2banClient.__processStartStreamAfterWait, args=(self, stream, False)).start() + th = Thread(target=Fail2banClient.__processStartStreamAfterWait, args=(self, stream, False)) + th.daemon = True + th.start() # Mark current (main) thread as daemon: self.setDaemon(True) # Start server direct here in main thread (not fork): @@ -197,8 +204,6 @@ class Fail2banClient(Fail2banCmdLine, Thread): logSys.error("Exception while starting server " + ("background" if background else "foreground")) logSys.error(e) return False - finally: - self._alive = False return True @@ -206,7 +211,9 @@ class Fail2banClient(Fail2banCmdLine, Thread): def configureServer(self, async=True, phase=None): # if asynchron start this operation in the new thread: if async: - return Thread(target=Fail2banClient.configureServer, args=(self, False, phase)).start() + th = Thread(target=Fail2banClient.configureServer, args=(self, False, phase)) + th.daemon = True + return th.start() # prepare: read config, check configuration is valid, etc.: if phase is not None: phase['start'] = True @@ -290,7 +297,6 @@ class Fail2banClient(Fail2banCmdLine, Thread): "server, adding the -x option will do it") if self._server: self._server.quit() - exit(-1) return False return True @@ -299,10 +305,12 @@ class Fail2banClient(Fail2banCmdLine, Thread): maxtime = MAX_WAITTIME # Wait for the server to start (the server has 30 seconds to answer ping) starttime = time.time() + logSys.debug("__waitOnServer: %r", (alive, maxtime)) with VisualWait(self._conf["verbose"]) as vis: - while self._alive and not self.__ping() == alive or ( + while self._alive and ( + not self.__ping() == alive or ( not alive and os.path.exists(self._conf["socket"]) - ): + )): now = time.time() # Wonderful visual :) if now > starttime + 1: @@ -365,6 +373,7 @@ class Fail2banClient(Fail2banCmdLine, Thread): return False return self.__processCommand(args) finally: + self._alive = False for s, sh in _prev_signals.iteritems(): signal.signal(s, sh) diff --git a/fail2ban/client/fail2banserver.py b/fail2ban/client/fail2banserver.py index da8e57b8..ac927251 100644 --- a/fail2ban/client/fail2banserver.py +++ b/fail2ban/client/fail2banserver.py @@ -25,15 +25,14 @@ import os import sys from ..version import version -from ..server.server import Server, ServerDaemonize +from ..server.server import Server from ..server.utils import Utils -from .fail2bancmdline import Fail2banCmdLine, logSys, exit +from .fail2bancmdline import Fail2banCmdLine, logSys, PRODUCTION, exit MAX_WAITTIME = 30 SERVER = "fail2ban-server" - ## # \mainpage Fail2Ban # @@ -51,6 +50,7 @@ class Fail2banServer(Fail2banCmdLine): @staticmethod def startServerDirect(conf, daemon=True): + logSys.debug("-- direct starting of server in %s, deamon: %s", os.getpid(), daemon) server = None try: # Start it in foreground (current thread, not new process), @@ -59,8 +59,6 @@ class Fail2banServer(Fail2banCmdLine): server.start(conf["socket"], conf["pidfile"], conf["force"], conf=conf) - except ServerDaemonize: - pass except Exception, e: logSys.exception(e) if server: @@ -82,9 +80,10 @@ class Fail2banServer(Fail2banCmdLine): startdir = os.path.dirname(sys.argv[0]) # Forks the current process, don't fork if async specified (ex: test cases) pid = 0 - frk = not conf["async"] + frk = not conf["async"] and PRODUCTION if frk: pid = os.fork() + logSys.debug("-- async starting of server in %s, fork: %s - %s", os.getpid(), frk, pid) if pid == 0: args = list() args.append(SERVER) diff --git a/fail2ban/server/jailthread.py b/fail2ban/server/jailthread.py index eb43e453..39a86c2b 100644 --- a/fail2ban/server/jailthread.py +++ b/fail2ban/server/jailthread.py @@ -51,6 +51,8 @@ class JailThread(Thread): def __init__(self, name=None): super(JailThread, self).__init__(name=name) + ## Should going with main thread also: + self.daemon = True ## Control the state of the thread. self.active = False ## Control the idle state of the thread. diff --git a/fail2ban/server/server.py b/fail2ban/server/server.py index a542659b..9f56f8a8 100644 --- a/fail2ban/server/server.py +++ b/fail2ban/server/server.py @@ -341,6 +341,9 @@ class Server: def getBanTime(self, name): return self.__jails[name].actions.getBanTime() + def isStarted(self): + self.__asyncServer.isActive() + def isAlive(self, jailnum=None): if jailnum is not None and len(self.__jails) != jailnum: return 0 @@ -633,6 +636,3 @@ class Server: class ServerInitializationError(Exception): pass - -class ServerDaemonize(Exception): - pass \ No newline at end of file diff --git a/fail2ban/tests/action_d/test_smtp.py b/fail2ban/tests/action_d/test_smtp.py index 37fe0138..1385fe82 100644 --- a/fail2ban/tests/action_d/test_smtp.py +++ b/fail2ban/tests/action_d/test_smtp.py @@ -65,6 +65,7 @@ class SMTPActionTest(unittest.TestCase): self._active = True self._loop_thread = threading.Thread( target=asyncserver.loop, kwargs={'active': lambda: self._active}) + self._loop_thread.daemon = True self._loop_thread.start() def tearDown(self): diff --git a/fail2ban/tests/fail2banclienttestcase.py b/fail2ban/tests/fail2banclienttestcase.py index 7db48fe8..fd8a074b 100644 --- a/fail2ban/tests/fail2banclienttestcase.py +++ b/fail2ban/tests/fail2banclienttestcase.py @@ -98,7 +98,9 @@ def _test_raw_input(*args): fail2banclient.raw_input = _test_raw_input # prevents change logging params, log capturing, etc: -fail2bancmdline.PRODUCTION = False +fail2bancmdline.PRODUCTION = \ +fail2banclient.PRODUCTION = \ +fail2banserver.PRODUCTION = False class ExitException(fail2bancmdline.ExitException): @@ -117,9 +119,9 @@ def _out_file(fn): # pragma: no cover def _start_params(tmp, use_stock=False, logtarget="/dev/null"): cfg = tmp+"/config" if use_stock and STOCK: - # copy config: + # copy config (sub-directories as alias): def ig_dirs(dir, files): - return [f for f in files if not os.path.isfile(os.path.join(dir, f))] + return [f for f in files if os.path.isdir(os.path.join(dir, f))] shutil.copytree(STOCK_CONF_DIR, cfg, ignore=ig_dirs) os.symlink(STOCK_CONF_DIR+"/action.d", cfg+"/action.d") os.symlink(STOCK_CONF_DIR+"/filter.d", cfg+"/filter.d") @@ -169,6 +171,47 @@ def _start_params(tmp, use_stock=False, logtarget="/dev/null"): "--logtarget", logtarget, "--loglevel", "DEBUG", "--syslogsocket", "auto", "-s", tmp+"/f2b.sock", "-p", tmp+"/f2b.pid") +def _kill_srv(pidfile): # pragma: no cover + def _pid_exists(pid): + try: + os.kill(pid, 0) + return True + except OSError: + return False + logSys.debug("-- cleanup: %r", (pidfile, os.path.isdir(pidfile))) + if os.path.isdir(pidfile): + piddir = pidfile + pidfile = piddir + "/f2b.pid" + if not os.path.isfile(pidfile): + pidfile = piddir + "/fail2ban.pid" + if not os.path.isfile(pidfile): + logSys.debug("--- cleanup: no pidfile for %r", piddir) + return True + f = pid = None + try: + logSys.debug("--- cleanup pidfile: %r", pidfile) + f = open(pidfile) + pid = f.read().split()[1] + pid = int(pid) + logSys.debug("--- cleanup pid: %r", pid) + if pid <= 0: + raise ValueError('pid %s of %s is invalid' % (pid, pidfile)) + if not _pid_exists(pid): + return True + ## try to preper stop (have signal handler): + os.kill(pid, signal.SIGTERM) + ## check still exists after small timeout: + if not Utils.wait_for(lambda: not _pid_exists(pid), MAX_WAITTIME / 3): + ## try to kill hereafter: + os.kill(pid, signal.SIGKILL) + return not _pid_exists(pid) + except Exception as e: + sysLog.debug(e) + finally: + if f is not None: + f.close() + return True + class Fail2banClientTest(LogCaptureTestCase): @@ -188,126 +231,144 @@ class Fail2banClientTest(LogCaptureTestCase): @withtmpdir def testClientStartBackgroundInside(self, tmp): - # always add "--async" by start inside, should don't fork by async (not replace client with server, just start in new process) - # (we can't fork the test cases process): - startparams = _start_params(tmp, True) - # start: - self.assertRaises(ExitException, _exec_client, - (CLIENT, "--async", "-b") + startparams + ("start",)) - self.assertLogged("Server ready") - self.assertLogged("Exit with code 0") try: + # always add "--async" by start inside, should don't fork by async (not replace client with server, just start in new process) + # (we can't fork the test cases process): + startparams = _start_params(tmp, True) + # start: self.assertRaises(ExitException, _exec_client, - (CLIENT,) + startparams + ("echo", "TEST-ECHO",)) - self.assertRaises(FailExitException, _exec_client, - (CLIENT,) + startparams + ("~~unknown~cmd~failed~~",)) - finally: - self.pruneLog() - # stop: - self.assertRaises(ExitException, _exec_client, - (CLIENT,) + startparams + ("stop",)) - self.assertLogged("Shutdown successful") + (CLIENT, "--async", "-b") + startparams + ("start",)) + self.assertLogged("Server ready") self.assertLogged("Exit with code 0") + try: + self.assertRaises(ExitException, _exec_client, + (CLIENT,) + startparams + ("echo", "TEST-ECHO",)) + self.assertRaises(FailExitException, _exec_client, + (CLIENT,) + startparams + ("~~unknown~cmd~failed~~",)) + finally: + self.pruneLog() + # stop: + self.assertRaises(ExitException, _exec_client, + (CLIENT,) + startparams + ("stop",)) + self.assertLogged("Shutdown successful") + self.assertLogged("Exit with code 0") + finally: + _kill_srv(tmp) @withtmpdir def testClientStartBackgroundCall(self, tmp): - global INTERACT - startparams = _start_params(tmp) - # start (without async in new process): - cmd = os.path.join(os.path.join(BIN), CLIENT) - logSys.debug('Start %s ...', cmd) - Utils.executeCmd((cmd,) + startparams + ("start",), - timeout=MAX_WAITTIME, shell=False, output=False) - self.pruneLog() try: - # echo from client (inside): - self.assertRaises(ExitException, _exec_client, - (CLIENT,) + startparams + ("echo", "TEST-ECHO",)) - self.assertLogged("TEST-ECHO") - self.assertLogged("Exit with code 0") - self.pruneLog() - # interactive client chat with started server: - INTERACT += [ - "echo INTERACT-ECHO", - "status", - "exit" - ] - self.assertRaises(ExitException, _exec_client, - (CLIENT,) + startparams + ("-i",)) - self.assertLogged("INTERACT-ECHO") - self.assertLogged("Status", "Number of jail:") - self.assertLogged("Exit with code 0") - self.pruneLog() - # test reload and restart over interactive client: - INTERACT += [ - "reload", - "restart", - "exit" - ] - self.assertRaises(ExitException, _exec_client, - (CLIENT,) + startparams + ("-i",)) - self.assertLogged("Reading config files:") - self.assertLogged("Shutdown successful") - self.assertLogged("Server ready") - self.assertLogged("Exit with code 0") + global INTERACT + startparams = _start_params(tmp) + # start (without async in new process): + cmd = os.path.join(os.path.join(BIN), CLIENT) + logSys.debug('Start %s ...', cmd) + Utils.executeCmd((cmd,) + startparams + ("start",), + timeout=MAX_WAITTIME, shell=False, output=False) self.pruneLog() + try: + # echo from client (inside): + self.assertRaises(ExitException, _exec_client, + (CLIENT,) + startparams + ("echo", "TEST-ECHO",)) + self.assertLogged("TEST-ECHO") + self.assertLogged("Exit with code 0") + self.pruneLog() + # interactive client chat with started server: + INTERACT += [ + "echo INTERACT-ECHO", + "status", + "exit" + ] + self.assertRaises(ExitException, _exec_client, + (CLIENT,) + startparams + ("-i",)) + self.assertLogged("INTERACT-ECHO") + self.assertLogged("Status", "Number of jail:") + self.assertLogged("Exit with code 0") + self.pruneLog() + # test reload and restart over interactive client: + INTERACT += [ + "reload", + "restart", + "exit" + ] + self.assertRaises(ExitException, _exec_client, + (CLIENT,) + startparams + ("-i",)) + self.assertLogged("Reading config files:") + self.assertLogged("Shutdown successful") + self.assertLogged("Server ready") + self.assertLogged("Exit with code 0") + self.pruneLog() + finally: + self.pruneLog() + # stop: + self.assertRaises(ExitException, _exec_client, + (CLIENT,) + startparams + ("stop",)) + self.assertLogged("Shutdown successful") + self.assertLogged("Exit with code 0") finally: - self.pruneLog() - # stop: - self.assertRaises(ExitException, _exec_client, - (CLIENT,) + startparams + ("stop",)) - self.assertLogged("Shutdown successful") - self.assertLogged("Exit with code 0") + _kill_srv(tmp) def _testClientStartForeground(self, tmp, startparams, phase): # start and wait to end (foreground): + logSys.debug("-- start of test worker") phase['start'] = True self.assertRaises(ExitException, _exec_client, (CLIENT, "-f") + startparams + ("start",)) # end : phase['end'] = True + logSys.debug("-- end of test worker") @withtmpdir def testClientStartForeground(self, tmp): - # started directly here, so prevent overwrite test cases logger with "INHERITED" - startparams = _start_params(tmp, logtarget="INHERITED") - # because foreground block execution - start it in thread: - phase = dict() - Thread(name="_TestCaseWorker", - target=Fail2banClientTest._testClientStartForeground, args=(self, tmp, startparams, phase)).start() + th = None try: - # wait for start thread: - Utils.wait_for(lambda: phase.get('start', None) is not None, MAX_WAITTIME) - self.assertTrue(phase.get('start', None)) - # wait for server (socket): - Utils.wait_for(lambda: os.path.exists(tmp+"/f2b.sock"), MAX_WAITTIME) - self.assertLogged("Starting communication") - self.assertRaises(ExitException, _exec_client, - (CLIENT,) + startparams + ("ping",)) - self.assertRaises(FailExitException, _exec_client, - (CLIENT,) + startparams + ("~~unknown~cmd~failed~~",)) - self.assertRaises(ExitException, _exec_client, - (CLIENT,) + startparams + ("echo", "TEST-ECHO",)) + # started directly here, so prevent overwrite test cases logger with "INHERITED" + startparams = _start_params(tmp, logtarget="INHERITED") + # because foreground block execution - start it in thread: + phase = dict() + th = Thread(name="_TestCaseWorker", + target=Fail2banClientTest._testClientStartForeground, args=(self, tmp, startparams, phase)) + th.daemon = True + th.start() + try: + # wait for start thread: + Utils.wait_for(lambda: phase.get('start', None) is not None, MAX_WAITTIME) + self.assertTrue(phase.get('start', None)) + # wait for server (socket): + Utils.wait_for(lambda: os.path.exists(tmp+"/f2b.sock"), MAX_WAITTIME) + self.assertLogged("Starting communication") + self.assertRaises(ExitException, _exec_client, + (CLIENT,) + startparams + ("ping",)) + self.assertRaises(FailExitException, _exec_client, + (CLIENT,) + startparams + ("~~unknown~cmd~failed~~",)) + self.assertRaises(ExitException, _exec_client, + (CLIENT,) + startparams + ("echo", "TEST-ECHO",)) + finally: + self.pruneLog() + # stop: + self.assertRaises(ExitException, _exec_client, + (CLIENT,) + startparams + ("stop",)) + # wait for end: + Utils.wait_for(lambda: phase.get('end', None) is not None, MAX_WAITTIME) + self.assertTrue(phase.get('end', None)) + self.assertLogged("Shutdown successful", "Exiting Fail2ban") finally: - self.pruneLog() - # stop: - self.assertRaises(ExitException, _exec_client, - (CLIENT,) + startparams + ("stop",)) - # wait for end: - Utils.wait_for(lambda: phase.get('end', None) is not None, MAX_WAITTIME) - self.assertTrue(phase.get('end', None)) - self.assertLogged("Shutdown successful", "Exiting Fail2ban") - self.assertLogged("Exit with code 0") + _kill_srv(tmp) + if th: + th.join() @withtmpdir def testClientFailStart(self, tmp): - self.assertRaises(FailExitException, _exec_client, - (CLIENT, "--async", "-c", tmp+"/miss", "start",)) - self.assertLogged("Base configuration directory " + tmp+"/miss" + " does not exist") + try: + self.assertRaises(FailExitException, _exec_client, + (CLIENT, "--async", "-c", tmp+"/miss", "start",)) + self.assertLogged("Base configuration directory " + tmp+"/miss" + " does not exist") - self.assertRaises(FailExitException, _exec_client, - (CLIENT, "--async", "-c", CONF_DIR, "-s", tmp+"/miss/f2b.sock", "start",)) - self.assertLogged("There is no directory " + tmp+"/miss" + " to contain the socket file") + self.assertRaises(FailExitException, _exec_client, + (CLIENT, "--async", "-c", CONF_DIR, "-s", tmp+"/miss/f2b.sock", "start",)) + self.assertLogged("There is no directory " + tmp+"/miss" + " to contain the socket file") + finally: + _kill_srv(tmp) def testVisualWait(self): sleeptime = 0.035 @@ -339,73 +400,89 @@ class Fail2banServerTest(LogCaptureTestCase): @withtmpdir def testServerStartBackground(self, tmp): - # don't add "--async" by start, because if will fork current process by daemonize - # (we can't fork the test cases process), - # because server started internal communication in new thread use INHERITED as logtarget here: - startparams = _start_params(tmp, logtarget="INHERITED") - # start: - self.assertRaises(ExitException, _exec_server, - (SERVER, "-b") + startparams) - self.assertLogged("Server ready") - self.assertLogged("Exit with code 0") try: + # don't add "--async" by start, because if will fork current process by daemonize + # (we can't fork the test cases process), + # because server started internal communication in new thread use INHERITED as logtarget here: + startparams = _start_params(tmp, logtarget="INHERITED") + # start: self.assertRaises(ExitException, _exec_server, - (SERVER,) + startparams + ("echo", "TEST-ECHO",)) - self.assertRaises(FailExitException, _exec_server, - (SERVER,) + startparams + ("~~unknown~cmd~failed~~",)) - finally: - self.pruneLog() - # stop: - self.assertRaises(ExitException, _exec_server, - (SERVER,) + startparams + ("stop",)) - self.assertLogged("Shutdown successful") + (SERVER, "-b") + startparams) + self.assertLogged("Server ready") self.assertLogged("Exit with code 0") + try: + self.assertRaises(ExitException, _exec_server, + (SERVER,) + startparams + ("echo", "TEST-ECHO",)) + self.assertRaises(FailExitException, _exec_server, + (SERVER,) + startparams + ("~~unknown~cmd~failed~~",)) + finally: + self.pruneLog() + # stop: + self.assertRaises(ExitException, _exec_server, + (SERVER,) + startparams + ("stop",)) + self.assertLogged("Shutdown successful") + self.assertLogged("Exit with code 0") + finally: + _kill_srv(tmp) def _testServerStartForeground(self, tmp, startparams, phase): # start and wait to end (foreground): + logSys.debug("-- start of test worker") phase['start'] = True self.assertRaises(ExitException, _exec_server, (SERVER, "-f") + startparams + ("start",)) # end : phase['end'] = True + logSys.debug("-- end of test worker") + @withtmpdir def testServerStartForeground(self, tmp): - # started directly here, so prevent overwrite test cases logger with "INHERITED" - startparams = _start_params(tmp, logtarget="INHERITED") - # because foreground block execution - start it in thread: - phase = dict() - Thread(name="_TestCaseWorker", - target=Fail2banServerTest._testServerStartForeground, args=(self, tmp, startparams, phase)).start() + th = None try: - # wait for start thread: - Utils.wait_for(lambda: phase.get('start', None) is not None, MAX_WAITTIME) - self.assertTrue(phase.get('start', None)) - # wait for server (socket): - Utils.wait_for(lambda: os.path.exists(tmp+"/f2b.sock"), MAX_WAITTIME) - self.assertLogged("Starting communication") - self.assertRaises(ExitException, _exec_server, - (SERVER,) + startparams + ("ping",)) - self.assertRaises(FailExitException, _exec_server, - (SERVER,) + startparams + ("~~unknown~cmd~failed~~",)) - self.assertRaises(ExitException, _exec_server, - (SERVER,) + startparams + ("echo", "TEST-ECHO",)) + # started directly here, so prevent overwrite test cases logger with "INHERITED" + startparams = _start_params(tmp, logtarget="INHERITED") + # because foreground block execution - start it in thread: + phase = dict() + th = Thread(name="_TestCaseWorker", + target=Fail2banServerTest._testServerStartForeground, args=(self, tmp, startparams, phase)) + th.daemon = True + th.start() + try: + # wait for start thread: + Utils.wait_for(lambda: phase.get('start', None) is not None, MAX_WAITTIME) + self.assertTrue(phase.get('start', None)) + # wait for server (socket): + Utils.wait_for(lambda: os.path.exists(tmp+"/f2b.sock"), MAX_WAITTIME) + self.assertLogged("Starting communication") + self.assertRaises(ExitException, _exec_server, + (SERVER,) + startparams + ("ping",)) + self.assertRaises(FailExitException, _exec_server, + (SERVER,) + startparams + ("~~unknown~cmd~failed~~",)) + self.assertRaises(ExitException, _exec_server, + (SERVER,) + startparams + ("echo", "TEST-ECHO",)) + finally: + self.pruneLog() + # stop: + self.assertRaises(ExitException, _exec_server, + (SERVER,) + startparams + ("stop",)) + # wait for end: + Utils.wait_for(lambda: phase.get('end', None) is not None, MAX_WAITTIME) + self.assertTrue(phase.get('end', None)) + self.assertLogged("Shutdown successful", "Exiting Fail2ban") finally: - self.pruneLog() - # stop: - self.assertRaises(ExitException, _exec_server, - (SERVER,) + startparams + ("stop",)) - # wait for end: - Utils.wait_for(lambda: phase.get('end', None) is not None, MAX_WAITTIME) - self.assertTrue(phase.get('end', None)) - self.assertLogged("Shutdown successful", "Exiting Fail2ban") - self.assertLogged("Exit with code 0") + _kill_srv(tmp) + if th: + th.join() @withtmpdir def testServerFailStart(self, tmp): - self.assertRaises(FailExitException, _exec_server, - (SERVER, "-c", tmp+"/miss",)) - self.assertLogged("Base configuration directory " + tmp+"/miss" + " does not exist") + try: + self.assertRaises(FailExitException, _exec_server, + (SERVER, "-c", tmp+"/miss",)) + self.assertLogged("Base configuration directory " + tmp+"/miss" + " does not exist") - self.assertRaises(FailExitException, _exec_server, - (SERVER, "-c", CONF_DIR, "-s", tmp+"/miss/f2b.sock",)) - self.assertLogged("There is no directory " + tmp+"/miss" + " to contain the socket file") + self.assertRaises(FailExitException, _exec_server, + (SERVER, "-c", CONF_DIR, "-s", tmp+"/miss/f2b.sock",)) + self.assertLogged("There is no directory " + tmp+"/miss" + " to contain the socket file") + finally: + _kill_srv(tmp) diff --git a/fail2ban/tests/utils.py b/fail2ban/tests/utils.py index 9cdace42..1a54d37f 100644 --- a/fail2ban/tests/utils.py +++ b/fail2ban/tests/utils.py @@ -312,9 +312,8 @@ class LogCaptureTestCase(unittest.TestCase): # Let's log everything into a string self._log = StringIO() logSys.handlers = [logging.StreamHandler(self._log)] - if self._old_level <= logging.DEBUG: + if self._old_level <= logging.DEBUG: # so if DEBUG etc -- show them (and log it in travis)! print("") - if self._old_level < logging.DEBUG: # so if HEAVYDEBUG etc -- show them! logSys.handlers += self._old_handlers logSys.debug('--'*40) logSys.setLevel(getattr(logging, 'DEBUG')) From 6cd19894e9b2630be3fdc2a60ff4acb39fe55c6d Mon Sep 17 00:00:00 2001 From: sebres Date: Fri, 12 Feb 2016 21:51:32 +0100 Subject: [PATCH 48/52] some compatibility fixes (prevent forking of testcase-process, code review), wait 4 server ready, test cases fixed (py2/py3) --- fail2ban/client/fail2banclient.py | 80 ++++++++++------ fail2ban/client/fail2bancmdline.py | 8 +- fail2ban/client/fail2banserver.py | 108 +++++++++++++-------- fail2ban/server/server.py | 36 ++++--- fail2ban/tests/fail2banclienttestcase.py | 116 +++++++++++++---------- fail2ban/tests/utils.py | 6 +- 6 files changed, 219 insertions(+), 135 deletions(-) diff --git a/fail2ban/client/fail2banclient.py b/fail2ban/client/fail2banclient.py index 4f4caf79..ad5cc57e 100644 --- a/fail2ban/client/fail2banclient.py +++ b/fail2ban/client/fail2banclient.py @@ -34,14 +34,18 @@ from threading import Thread from ..version import version from .csocket import CSocket from .beautifier import Beautifier -from .fail2bancmdline import Fail2banCmdLine, ExitException, PRODUCTION, logSys, exit, output +from .fail2bancmdline import Fail2banCmdLine, ServerExecutionException, ExitException, \ + logSys, PRODUCTION, exit, output MAX_WAITTIME = 30 +PROMPT = "fail2ban> " def _thread_name(): return threading.current_thread().__class__.__name__ +def input_command(): + return raw_input(PROMPT) ## # @@ -49,8 +53,6 @@ def _thread_name(): class Fail2banClient(Fail2banCmdLine, Thread): - PROMPT = "fail2ban> " - def __init__(self): Fail2banCmdLine.__init__(self) Thread.__init__(self) @@ -91,11 +93,11 @@ class Fail2banClient(Fail2banCmdLine, Thread): client = CSocket(self._conf["socket"]) ret = client.send(c) if ret[0] == 0: - logSys.debug("OK : " + `ret[1]`) + logSys.debug("OK : %r", ret[1]) if showRet or c[0] == 'echo': output(beautifier.beautify(ret[1])) else: - logSys.error("NOK: " + `ret[1].args`) + logSys.error("NOK: %r", ret[1].args) if showRet: output(beautifier.beautifyError(ret[1])) streamRet = False @@ -202,7 +204,10 @@ class Fail2banClient(Fail2banCmdLine, Thread): except Exception as e: output("") logSys.error("Exception while starting server " + ("background" if background else "foreground")) - logSys.error(e) + if self._conf["verbose"] > 1: + logSys.exception(e) + else: + logSys.error(e) return False return True @@ -249,7 +254,9 @@ class Fail2banClient(Fail2banCmdLine, Thread): if self._conf.get("interactive", False): output(' ## stop ... ') self.__processCommand(['stop']) - self.__waitOnServer(False) + if not self.__waitOnServer(False): + logSys.error("Could not stop server") + return False # in interactive mode reset config, to make full-reload if there something changed: if self._conf.get("interactive", False): output(' ## load configuration ... ') @@ -286,10 +293,14 @@ class Fail2banClient(Fail2banCmdLine, Thread): def __processStartStreamAfterWait(self, *args): try: # Wait for the server to start - self.__waitOnServer() + if not self.__waitOnServer(): + logSys.error("Could not find server, waiting failed") + return False # Configure the server self.__processCmd(*args) - except ServerExecutionException: + except ServerExecutionException as e: + if self._conf["verbose"] > 1: + logSys.exception(e) logSys.error("Could not start server. Maybe an old " "socket file is still present. Try to " "remove " + self._conf["socket"] + ". If " @@ -306,11 +317,13 @@ class Fail2banClient(Fail2banCmdLine, Thread): # Wait for the server to start (the server has 30 seconds to answer ping) starttime = time.time() logSys.debug("__waitOnServer: %r", (alive, maxtime)) + test = lambda: os.path.exists(self._conf["socket"]) and self.__ping() with VisualWait(self._conf["verbose"]) as vis: - while self._alive and ( - not self.__ping() == alive or ( - not alive and os.path.exists(self._conf["socket"]) - )): + sltime = 0.0125 / 2 + while self._alive: + runf = test() + if runf == alive: + return True now = time.time() # Wonderful visual :) if now > starttime + 1: @@ -318,7 +331,9 @@ class Fail2banClient(Fail2banCmdLine, Thread): # f end time reached: if now - starttime >= maxtime: raise ServerExecutionException("Failed to start server") - time.sleep(0.1) + sltime = min(sltime * 2, 0.5) + time.sleep(sltime) + return False def start(self, argv): # Install signal handlers @@ -332,27 +347,31 @@ class Fail2banClient(Fail2banCmdLine, Thread): if self._argv is None: ret = self.initCmdLine(argv) if ret is not None: - return ret + if ret: + return True + raise ServerExecutionException("Init of command line failed") # Commands args = self._args # Interactive mode if self._conf.get("interactive", False): - try: - import readline - except ImportError: - logSys.error("Readline not available") - return False + # no readline in test: + if PRODUCTION: # pragma: no cover + try: + import readline + except ImportError: + raise ServerExecutionException("Readline not available") try: ret = True if len(args) > 0: ret = self.__processCommand(args) if ret: - readline.parse_and_bind("tab: complete") + if PRODUCTION: # pragma: no cover + readline.parse_and_bind("tab: complete") self.dispInteractive() while True: - cmd = raw_input(self.PROMPT) + cmd = input_command() if cmd == "exit" or cmd == "quit": # Exit return True @@ -362,26 +381,31 @@ class Fail2banClient(Fail2banCmdLine, Thread): try: self.__processCommand(shlex.split(cmd)) except Exception, e: - logSys.error(e) + if self._conf["verbose"] > 1: + logSys.exception(e) + else: + logSys.error(e) except (EOFError, KeyboardInterrupt): output("") - return True + raise # Single command mode else: if len(args) < 1: self.dispUsage() return False return self.__processCommand(args) + except Exception as e: + if self._conf["verbose"] > 1: + logSys.exception(e) + else: + logSys.error(e) + return False finally: self._alive = False for s, sh in _prev_signals.iteritems(): signal.signal(s, sh) -class ServerExecutionException(Exception): - pass - - ## # Wonderful visual :) # diff --git a/fail2ban/client/fail2bancmdline.py b/fail2ban/client/fail2bancmdline.py index abbf8363..58fd47c2 100644 --- a/fail2ban/client/fail2bancmdline.py +++ b/fail2ban/client/fail2bancmdline.py @@ -259,5 +259,9 @@ class Fail2banCmdLine(): exit = Fail2banCmdLine.exit -class ExitException: - pass \ No newline at end of file +class ExitException(Exception): + pass + + +class ServerExecutionException(Exception): + pass diff --git a/fail2ban/client/fail2banserver.py b/fail2ban/client/fail2banserver.py index ac927251..73e528ca 100644 --- a/fail2ban/client/fail2banserver.py +++ b/fail2ban/client/fail2banserver.py @@ -24,10 +24,8 @@ __license__ = "GPL" import os import sys -from ..version import version -from ..server.server import Server -from ..server.utils import Utils -from .fail2bancmdline import Fail2banCmdLine, logSys, PRODUCTION, exit +from .fail2bancmdline import Fail2banCmdLine, ServerExecutionException, \ + logSys, PRODUCTION, exit MAX_WAITTIME = 30 @@ -44,13 +42,14 @@ class Fail2banServer(Fail2banCmdLine): # Fail2banCmdLine.__init__(self) ## - # Start Fail2Ban server in main thread without fork (foreground). + # Start Fail2Ban server in main thread without fork (direct, it can fork itself in Server if daemon=True). # - # Start the Fail2ban server in foreground (daemon mode or not). + # Start the Fail2ban server in background/foreground (daemon mode or not). @staticmethod def startServerDirect(conf, daemon=True): logSys.debug("-- direct starting of server in %s, deamon: %s", os.getpid(), daemon) + from ..server.server import Server server = None try: # Start it in foreground (current thread, not new process), @@ -59,11 +58,14 @@ class Fail2banServer(Fail2banCmdLine): server.start(conf["socket"], conf["pidfile"], conf["force"], conf=conf) - except Exception, e: - logSys.exception(e) - if server: - server.quit() - exit(-1) + except Exception as e: + try: + if server: + server.quit() + except Exception as e2: + if conf["verbose"] > 1: + logSys.exception(e2) + raise return server @@ -74,10 +76,6 @@ class Fail2banServer(Fail2banCmdLine): @staticmethod def startServerAsync(conf): - # Directory of client (to try the first start from the same directory as client): - startdir = sys.path[0] - if startdir in ("", "."): # may be uresolved in test-cases, so get bin-directory: - startdir = os.path.dirname(sys.argv[0]) # Forks the current process, don't fork if async specified (ex: test cases) pid = 0 frk = not conf["async"] and PRODUCTION @@ -103,25 +101,43 @@ class Fail2banServer(Fail2banCmdLine): for o in ('loglevel', 'logtarget', 'syslogsocket'): args.append("--"+o) args.append(conf[o]) - try: - # Use the current directory. - exe = os.path.abspath(os.path.join(startdir, SERVER)) + # Directory of client (to try the first start from current or the same directory as client, and from relative bin): + exe = Fail2banServer.getServerPath() + if not frk: + # Wrapr args to use the same python version in client/server (important for multi-python systems): + args[0] = exe + exe = sys.executable + args[0:0] = [exe] logSys.debug("Starting %r with args %r", exe, args) if frk: - os.execv(exe, args) + return os.execv(exe, args) else: - os.spawnv(os.P_NOWAITO, exe, args) + # use P_WAIT instead of P_NOWAIT (to prevent defunct-zomby process), it startet as daemon, so parent exit fast after fork): + return os.spawnv(os.P_WAIT, exe, args) except OSError as e: - try: - # Use the PATH env. - logSys.warning("Initial start attempt failed (%s). Starting %r with the same args", e, SERVER) - if frk: - os.execvp(SERVER, args) - else: - os.spawnvp(os.P_NOWAITO, SERVER, args) - except OSError: - exit(-1) + # Use the PATH env. + logSys.warning("Initial start attempt failed (%s). Starting %r with the same args", e, SERVER) + if frk: + return os.execvp(SERVER, args) + else: + del args[0] + args[0] = SERVER + return os.spawnvp(os.P_WAIT, SERVER, args) + return pid + + @staticmethod + def getServerPath(): + startdir = sys.path[0] + exe = os.path.abspath(os.path.join(startdir, SERVER)) + if not os.path.isfile(exe): # may be uresolved in test-cases, so get relative starter (client): + startdir = os.path.dirname(sys.argv[0]) + exe = os.path.abspath(os.path.join(startdir, SERVER)) + if not os.path.isfile(exe): # may be uresolved in test-cases, so try to get relative bin-directory: + startdir = os.path.dirname(os.path.abspath(__file__)) + startdir = os.path.join(os.path.dirname(os.path.dirname(startdir)), "bin") + exe = os.path.abspath(os.path.join(startdir, SERVER)) + return exe def _Fail2banClient(self): from .fail2banclient import Fail2banClient @@ -139,18 +155,24 @@ class Fail2banServer(Fail2banCmdLine): args = self._args cli = None - # If client mode - whole processing over client: - if len(args) or self._conf.get("interactive", False): - cli = self._Fail2banClient() - return cli.start(argv) + # Just start: + if len(args) == 1 and args[0] == 'start' and not self._conf.get("interactive", False): + pass + else: + # If client mode - whole processing over client: + if len(args) or self._conf.get("interactive", False): + cli = self._Fail2banClient() + return cli.start(argv) # Start the server: server = None try: - # async = True, if started from client, should fork, daemonize, etc... - # background = True, if should start in new process, otherwise start in foreground - async = self._conf.get("async", False) + from ..server.utils import Utils + # background = True, if should be new process running in background, otherwise start in foreground + # process will be forked in daemonize, inside of Server module. + # async = True, if started from client, should... background = self._conf["background"] + async = self._conf.get("async", False) # If was started not from the client: if not async: # Start new thread with client to read configuration and @@ -162,13 +184,14 @@ class Fail2banServer(Fail2banCmdLine): # wait up to MAX_WAITTIME, do not continue if configuration is not 100% valid: Utils.wait_for(lambda: phase.get('ready', None) is not None, MAX_WAITTIME) if not phase.get('start', False): - return False + raise ServerExecutionException('Async configuration of server failed') # Start server, daemonize it, etc. - if async or not background: - server = Fail2banServer.startServerDirect(self._conf, background) - else: - Fail2banServer.startServerAsync(self._conf) + pid = os.getpid() + server = Fail2banServer.startServerDirect(self._conf, background) + # If forked - just exit other processes + if pid != os.getpid(): + os._exit(0) if cli: cli._server = server @@ -182,7 +205,8 @@ class Fail2banServer(Fail2banCmdLine): logSys.debug('Starting server done') except Exception, e: - logSys.exception(e) + if self._conf["verbose"] > 1: + logSys.exception(e) if server: server.quit() exit(-1) diff --git a/fail2ban/server/server.py b/fail2ban/server/server.py index 9f56f8a8..0ddaea35 100644 --- a/fail2ban/server/server.py +++ b/fail2ban/server/server.py @@ -93,9 +93,15 @@ class Server: if self.__daemon: # pragma: no cover logSys.info("Starting in daemon mode") ret = self.__createDaemon() - if not ret: - logSys.error("Could not create daemon") - raise ServerInitializationError("Could not create daemon") + # If forked parent - return here (parent process will configure server later): + if ret is None: + return False + # If error: + if not ret[0]: + err = "Could not create daemon %s", ret[1:] + logSys.error(err) + raise ServerInitializationError(err) + # We are daemon. # Set all logging parameters (or use default if not specified): self.setSyslogSocket(conf.get("syslogsocket", @@ -159,8 +165,12 @@ class Server: logging.shutdown() # Restore default signal handlers: - for s, sh in self.__prev_signals.iteritems(): - signal.signal(s, sh) + if _thread_name() == '_MainThread': + for s, sh in self.__prev_signals.iteritems(): + signal.signal(s, sh) + + # Prevent to call quit twice: + self.quit = lambda: False def addJail(self, name, backend): self.__jails.add(name, backend, self.__db) @@ -559,10 +569,9 @@ class Server: # We need to set this in the parent process, so it gets inherited by the # child process, and this makes sure that it is effect even if the parent # terminates quickly. - if _thread_name() == '_MainThread': - for s in (signal.SIGHUP,): - self.__prev_signals[s] = signal.getsignal(s) - signal.signal(s, signal.SIG_IGN) + for s in (signal.SIGHUP,): + self.__prev_signals[s] = signal.getsignal(s) + signal.signal(s, signal.SIG_IGN) try: # Fork a child process so the parent can exit. This will return control @@ -573,7 +582,7 @@ class Server: # PGID. pid = os.fork() except OSError, e: - return((e.errno, e.strerror)) # ERROR (return a tuple) + return (False, (e.errno, e.strerror)) # ERROR (return a tuple) if pid == 0: # The first child. @@ -594,7 +603,7 @@ class Server: # preventing the daemon from ever acquiring a controlling terminal. pid = os.fork() # Fork a second child. except OSError, e: - return((e.errno, e.strerror)) # ERROR (return a tuple) + return (False, (e.errno, e.strerror)) # ERROR (return a tuple) if (pid == 0): # The second child. # Ensure that the daemon doesn't keep any directory in use. Failure @@ -603,7 +612,8 @@ class Server: else: os._exit(0) # Exit parent (the first child) of the second child. else: - os._exit(0) # Exit parent of the first child. + # Signal to exit, parent of the first child. + return None # Close all open files. Try the system configuration variable, SC_OPEN_MAX, # for the maximum number of open files to close. If it doesn't exist, use @@ -631,7 +641,7 @@ class Server: os.open("/dev/null", os.O_RDONLY) # standard input (0) os.open("/dev/null", os.O_RDWR) # standard output (1) os.open("/dev/null", os.O_RDWR) # standard error (2) - return True + return (True,) class ServerInitializationError(Exception): diff --git a/fail2ban/tests/fail2banclienttestcase.py b/fail2ban/tests/fail2banclienttestcase.py index fd8a074b..6fee732b 100644 --- a/fail2ban/tests/fail2banclienttestcase.py +++ b/fail2ban/tests/fail2banclienttestcase.py @@ -26,6 +26,7 @@ __license__ = "GPL" import fileinput import os import re +import sys import time import unittest @@ -50,10 +51,9 @@ else: CLIENT = "fail2ban-client" SERVER = "fail2ban-server" -BIN = os.path.join(os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))), "bin") +BIN = os.path.dirname(Fail2banServer.getServerPath()) -MAX_WAITTIME = 10 -MAX_WAITTIME = unittest.F2B.maxWaitTime(MAX_WAITTIME) +MAX_WAITTIME = 30 if not unittest.F2B.fast else 5 ## # Several wrappers and settings for proper testing: @@ -67,8 +67,6 @@ fail2bancmdline.logSys = \ fail2banclient.logSys = \ fail2banserver.logSys = logSys -LOG_LEVEL = logSys.level - server.DEF_LOGTARGET = "/dev/null" def _test_output(*args): @@ -89,13 +87,13 @@ fail2banclient.exit = \ fail2banserver.exit = _test_exit INTERACT = [] -def _test_raw_input(*args): +def _test_input_command(*args): if len(INTERACT): - #print('--- interact command: ', INTERACT[0]) + #logSys.debug('--- interact command: %r', INTERACT[0]) return INTERACT.pop(0) else: return "exit" -fail2banclient.raw_input = _test_raw_input +fail2banclient.input_command = _test_input_command # prevents change logging params, log capturing, etc: fail2bancmdline.PRODUCTION = \ @@ -142,7 +140,7 @@ def _start_params(tmp, use_stock=False, logtarget="/dev/null"): else: # just empty config directory without anything (only fail2ban.conf/jail.conf): os.mkdir(cfg) - f = open(cfg+"/fail2ban.conf", "wb") + f = open(cfg+"/fail2ban.conf", "w") f.write('\n'.join(( "[Definition]", "loglevel = INFO", @@ -156,20 +154,20 @@ def _start_params(tmp, use_stock=False, logtarget="/dev/null"): "", ))) f.close() - f = open(cfg+"/jail.conf", "wb") + f = open(cfg+"/jail.conf", "w") f.write('\n'.join(( "[INCLUDES]", "", "[DEFAULT]", "", "", ))) f.close() - if LOG_LEVEL < logging.DEBUG: # if HEAVYDEBUG + if logSys.level < logging.DEBUG: # if HEAVYDEBUG _out_file(cfg+"/fail2ban.conf") _out_file(cfg+"/jail.conf") - # parameters: - return ("-c", cfg, - "--logtarget", logtarget, "--loglevel", "DEBUG", "--syslogsocket", "auto", - "-s", tmp+"/f2b.sock", "-p", tmp+"/f2b.pid") + # parameters (sock/pid and config, increase verbosity, set log, etc.): + return ("-c", cfg, "-s", tmp+"/f2b.sock", "-p", tmp+"/f2b.pid", + "-vv", "--logtarget", logtarget, "--loglevel", "DEBUG", "--syslogsocket", "auto", + ) def _kill_srv(pidfile): # pragma: no cover def _pid_exists(pid): @@ -206,14 +204,14 @@ def _kill_srv(pidfile): # pragma: no cover os.kill(pid, signal.SIGKILL) return not _pid_exists(pid) except Exception as e: - sysLog.debug(e) + logSys.debug(e) finally: if f is not None: f.close() return True -class Fail2banClientTest(LogCaptureTestCase): +class Fail2banClientServerBase(LogCaptureTestCase): def setUp(self): """Call before every test case.""" @@ -223,6 +221,21 @@ class Fail2banClientTest(LogCaptureTestCase): """Call after every test case.""" LogCaptureTestCase.tearDown(self) + def _wait_for_srv(self, tmp, ready=True, startparams=None): + sock = tmp+"/f2b.sock" + # wait for server (socket): + ret = Utils.wait_for(lambda: os.path.exists(sock), MAX_WAITTIME) + if not ret: + raise Exception('Unexpected: Socket file does not exists.\nStart failed: %r' % (startparams,)) + if ready: + # wait for communication with worker ready: + ret = Utils.wait_for(lambda: "Server ready" in self.getLog(), MAX_WAITTIME) + if not ret: + raise Exception('Unexpected: Server ready was not found.\nStart failed: %r' % (startparams,)) + + +class Fail2banClientTest(Fail2banClientServerBase): + def testClientUsage(self): self.assertRaises(ExitException, _exec_client, (CLIENT, "-h",)) @@ -232,12 +245,13 @@ class Fail2banClientTest(LogCaptureTestCase): @withtmpdir def testClientStartBackgroundInside(self, tmp): try: - # always add "--async" by start inside, should don't fork by async (not replace client with server, just start in new process) - # (we can't fork the test cases process): + # use once the stock configuration (to test starting also) startparams = _start_params(tmp, True) # start: self.assertRaises(ExitException, _exec_client, - (CLIENT, "--async", "-b") + startparams + ("start",)) + (CLIENT, "-b") + startparams + ("start",)) + # wait for server (socket and ready): + self._wait_for_srv(tmp, True, startparams=startparams) self.assertLogged("Server ready") self.assertLogged("Exit with code 0") try: @@ -245,6 +259,11 @@ class Fail2banClientTest(LogCaptureTestCase): (CLIENT,) + startparams + ("echo", "TEST-ECHO",)) self.assertRaises(FailExitException, _exec_client, (CLIENT,) + startparams + ("~~unknown~cmd~failed~~",)) + self.pruneLog() + # start again (should fail): + self.assertRaises(FailExitException, _exec_client, + (CLIENT, "-b") + startparams + ("start",)) + self.assertLogged("Server already running") finally: self.pruneLog() # stop: @@ -260,11 +279,14 @@ class Fail2banClientTest(LogCaptureTestCase): try: global INTERACT startparams = _start_params(tmp) - # start (without async in new process): - cmd = os.path.join(os.path.join(BIN), CLIENT) + # start (in new process, using the same python version): + cmd = (sys.executable, os.path.join(os.path.join(BIN), CLIENT)) logSys.debug('Start %s ...', cmd) - Utils.executeCmd((cmd,) + startparams + ("start",), - timeout=MAX_WAITTIME, shell=False, output=False) + cmd = cmd + startparams + ("start",) + Utils.executeCmd(cmd, timeout=MAX_WAITTIME, shell=False, output=True) + # wait for server (socket and ready): + self._wait_for_srv(tmp, True, startparams=cmd) + self.assertLogged("Server ready") self.pruneLog() try: # echo from client (inside): @@ -312,7 +334,7 @@ class Fail2banClientTest(LogCaptureTestCase): # start and wait to end (foreground): logSys.debug("-- start of test worker") phase['start'] = True - self.assertRaises(ExitException, _exec_client, + self.assertRaises(fail2bancmdline.ExitException, _exec_client, (CLIENT, "-f") + startparams + ("start",)) # end : phase['end'] = True @@ -334,9 +356,10 @@ class Fail2banClientTest(LogCaptureTestCase): # wait for start thread: Utils.wait_for(lambda: phase.get('start', None) is not None, MAX_WAITTIME) self.assertTrue(phase.get('start', None)) - # wait for server (socket): - Utils.wait_for(lambda: os.path.exists(tmp+"/f2b.sock"), MAX_WAITTIME) - self.assertLogged("Starting communication") + # wait for server (socket and ready): + self._wait_for_srv(tmp, True, startparams=startparams) + self.pruneLog() + # several commands to server: self.assertRaises(ExitException, _exec_client, (CLIENT,) + startparams + ("ping",)) self.assertRaises(FailExitException, _exec_client, @@ -382,15 +405,7 @@ class Fail2banClientTest(LogCaptureTestCase): cntr -= 1 -class Fail2banServerTest(LogCaptureTestCase): - - def setUp(self): - """Call before every test case.""" - LogCaptureTestCase.setUp(self) - - def tearDown(self): - """Call after every test case.""" - LogCaptureTestCase.tearDown(self) +class Fail2banServerTest(Fail2banClientServerBase): def testServerUsage(self): self.assertRaises(ExitException, _exec_server, @@ -401,15 +416,17 @@ class Fail2banServerTest(LogCaptureTestCase): @withtmpdir def testServerStartBackground(self, tmp): try: - # don't add "--async" by start, because if will fork current process by daemonize - # (we can't fork the test cases process), - # because server started internal communication in new thread use INHERITED as logtarget here: - startparams = _start_params(tmp, logtarget="INHERITED") - # start: - self.assertRaises(ExitException, _exec_server, - (SERVER, "-b") + startparams) + # to prevent fork of test-cases process, start server in background via command: + startparams = _start_params(tmp) + # start (in new process, using the same python version): + cmd = (sys.executable, os.path.join(os.path.join(BIN), SERVER)) + logSys.debug('Start %s ...', cmd) + cmd = cmd + startparams + ("-b",) + Utils.executeCmd(cmd, timeout=MAX_WAITTIME, shell=False, output=True) + # wait for server (socket and ready): + self._wait_for_srv(tmp, True, startparams=cmd) self.assertLogged("Server ready") - self.assertLogged("Exit with code 0") + self.pruneLog() try: self.assertRaises(ExitException, _exec_server, (SERVER,) + startparams + ("echo", "TEST-ECHO",)) @@ -429,7 +446,7 @@ class Fail2banServerTest(LogCaptureTestCase): # start and wait to end (foreground): logSys.debug("-- start of test worker") phase['start'] = True - self.assertRaises(ExitException, _exec_server, + self.assertRaises(fail2bancmdline.ExitException, _exec_server, (SERVER, "-f") + startparams + ("start",)) # end : phase['end'] = True @@ -451,9 +468,10 @@ class Fail2banServerTest(LogCaptureTestCase): # wait for start thread: Utils.wait_for(lambda: phase.get('start', None) is not None, MAX_WAITTIME) self.assertTrue(phase.get('start', None)) - # wait for server (socket): - Utils.wait_for(lambda: os.path.exists(tmp+"/f2b.sock"), MAX_WAITTIME) - self.assertLogged("Starting communication") + # wait for server (socket and ready): + self._wait_for_srv(tmp, True, startparams=startparams) + self.pruneLog() + # several commands to server: self.assertRaises(ExitException, _exec_server, (SERVER,) + startparams + ("ping",)) self.assertRaises(FailExitException, _exec_server, diff --git a/fail2ban/tests/utils.py b/fail2ban/tests/utils.py index 1a54d37f..b171511d 100644 --- a/fail2ban/tests/utils.py +++ b/fail2ban/tests/utils.py @@ -54,6 +54,10 @@ if not CONFIG_DIR: else: CONFIG_DIR = '/etc/fail2ban' +# In not installed env (setup, test-cases) use fail2ban modules from main directory: +if 1 or os.environ.get('PYTHONPATH', None) is None: + os.putenv('PYTHONPATH', os.path.dirname(os.path.dirname(os.path.dirname( + os.path.abspath(__file__))))) class F2B(optparse.Values): def __init__(self, opts={}): @@ -315,7 +319,7 @@ class LogCaptureTestCase(unittest.TestCase): if self._old_level <= logging.DEBUG: # so if DEBUG etc -- show them (and log it in travis)! print("") logSys.handlers += self._old_handlers - logSys.debug('--'*40) + logSys.debug('='*10 + ' %s ' + '='*20, self.id()) logSys.setLevel(getattr(logging, 'DEBUG')) def tearDown(self): From 4ec70d7851bb4ff8a103ac335fc446592401f1f8 Mon Sep 17 00:00:00 2001 From: sebres Date: Mon, 15 Feb 2016 19:19:31 +0100 Subject: [PATCH 49/52] code review, timeout fix, better tracing (and test coverage) by start of server/client (with or without fork) --- fail2ban/client/fail2banclient.py | 21 ++-- fail2ban/client/fail2bancmdline.py | 22 +++-- fail2ban/client/fail2banserver.py | 21 ++-- fail2ban/server/utils.py | 12 ++- fail2ban/tests/actiontestcase.py | 4 +- fail2ban/tests/fail2banclienttestcase.py | 116 ++++++++++++++++++----- 6 files changed, 139 insertions(+), 57 deletions(-) diff --git a/fail2ban/client/fail2banclient.py b/fail2ban/client/fail2banclient.py index ad5cc57e..23d31dc5 100644 --- a/fail2ban/client/fail2banclient.py +++ b/fail2ban/client/fail2banclient.py @@ -37,14 +37,13 @@ from .beautifier import Beautifier from .fail2bancmdline import Fail2banCmdLine, ServerExecutionException, ExitException, \ logSys, PRODUCTION, exit, output -MAX_WAITTIME = 30 PROMPT = "fail2ban> " def _thread_name(): return threading.current_thread().__class__.__name__ -def input_command(): +def input_command(): # pragma: no cover return raw_input(PROMPT) ## @@ -101,13 +100,19 @@ class Fail2banClient(Fail2banCmdLine, Thread): if showRet: output(beautifier.beautifyError(ret[1])) streamRet = False - except socket.error: + except socket.error as e: if showRet or self._conf["verbose"] > 1: - self.__logSocketError() + if showRet or c != ["ping"]: + self.__logSocketError() + else: + logSys.debug(" -- ping failed -- %r", e) return False - except Exception, e: + except Exception as e: # pragma: no cover if showRet or self._conf["verbose"] > 1: - logSys.error(e) + if self._conf["verbose"] > 1: + logSys.exception(e) + else: + logSys.error(e) return False finally: # prevent errors by close during shutdown (on exit command): @@ -123,7 +128,7 @@ class Fail2banClient(Fail2banCmdLine, Thread): def __logSocketError(self): try: - if os.access(self._conf["socket"], os.F_OK): + if os.access(self._conf["socket"], os.F_OK): # pragma: no cover # This doesn't check if path is a socket, # but socket.error should be raised if os.access(self._conf["socket"], os.W_OK): @@ -313,7 +318,7 @@ class Fail2banClient(Fail2banCmdLine, Thread): def __waitOnServer(self, alive=True, maxtime=None): if maxtime is None: - maxtime = MAX_WAITTIME + maxtime = self._conf["timeout"] # Wait for the server to start (the server has 30 seconds to answer ping) starttime = time.time() logSys.debug("__waitOnServer: %r", (alive, maxtime)) diff --git a/fail2ban/client/fail2bancmdline.py b/fail2ban/client/fail2bancmdline.py index 58fd47c2..a7754bc3 100644 --- a/fail2ban/client/fail2bancmdline.py +++ b/fail2ban/client/fail2bancmdline.py @@ -33,13 +33,14 @@ from ..helpers import getLogger # Gets the instance of the logger. logSys = getLogger("fail2ban") -def output(s): +def output(s): # pragma: no cover print(s) CONFIG_PARAMS = ("socket", "pidfile", "logtarget", "loglevel", "syslogsocket",) # Used to signal - we are in test cases (ex: prevents change logging params, log capturing, etc) PRODUCTION = True +MAX_WAITTIME = 30 class Fail2banCmdLine(): @@ -56,7 +57,8 @@ class Fail2banCmdLine(): "background": True, "verbose": 1, "socket": None, - "pidfile": None + "pidfile": None, + "timeout": MAX_WAITTIME } @property @@ -109,6 +111,7 @@ class Fail2banCmdLine(): output(" -b start server in background (default)") output(" -f start server in foreground") output(" --async start server in async mode (for internal usage only, don't read configuration)") + output(" --timeout timeout to wait for the server (for internal usage only, don't read configuration)") output(" -h, --help display this help message") output(" -V, --version print the version") @@ -126,8 +129,6 @@ class Fail2banCmdLine(): """ for opt in optList: o = opt[0] - if o == "--async": - self._conf["async"] = True if o == "-c": self._conf["conf"] = opt[1] elif o == "-s": @@ -150,6 +151,11 @@ class Fail2banCmdLine(): self._conf["background"] = True elif o == "-f": self._conf["background"] = False + elif o == "--async": + self._conf["async"] = True + elif o == "-timeout": + from ..mytime import MyTime + self._conf["timeout"] = MyTime.str2seconds(opt[1]) elif o in ["-h", "--help"]: self.dispUsage() return True @@ -170,7 +176,7 @@ class Fail2banCmdLine(): # Reads the command line options. try: cmdOpts = 'hc:s:p:xfbdviqV' - cmdLongOpts = ['loglevel=', 'logtarget=', 'syslogsocket=', 'async', 'help', 'version'] + cmdLongOpts = ['loglevel=', 'logtarget=', 'syslogsocket=', 'async', 'timeout=', 'help', 'version'] optList, self._args = getopt.getopt(self._argv[1:], cmdOpts, cmdLongOpts) except getopt.GetoptError: self.dispUsage() @@ -180,6 +186,8 @@ class Fail2banCmdLine(): if ret is not None: return ret + logSys.debug("-- conf: %r, args: %r", self._conf, self._args) + if initial and PRODUCTION: # pragma: no cover - can't test verbose = self._conf["verbose"] if verbose <= 0: @@ -244,11 +252,11 @@ class Fail2banCmdLine(): @staticmethod def dumpConfig(cmd): for c in cmd: - print c + output(c) return True @staticmethod - def exit(code=0): + def exit(code=0): # pragma: no cover - can't test logSys.debug("Exit with code %s", code) if os._exit: os._exit(code) diff --git a/fail2ban/client/fail2banserver.py b/fail2ban/client/fail2banserver.py index 73e528ca..a6c8a7c5 100644 --- a/fail2ban/client/fail2banserver.py +++ b/fail2ban/client/fail2banserver.py @@ -27,8 +27,6 @@ import sys from .fail2bancmdline import Fail2banCmdLine, ServerExecutionException, \ logSys, PRODUCTION, exit -MAX_WAITTIME = 30 - SERVER = "fail2ban-server" ## @@ -114,16 +112,17 @@ class Fail2banServer(Fail2banCmdLine): return os.execv(exe, args) else: # use P_WAIT instead of P_NOWAIT (to prevent defunct-zomby process), it startet as daemon, so parent exit fast after fork): - return os.spawnv(os.P_WAIT, exe, args) - except OSError as e: + ret = os.spawnv(os.P_WAIT, exe, args) + if ret != 0: + raise OSError(ret, "Unknown error by executing server %r with %r" % (args[1], exe)) + return 0 + except OSError as e: # pragma: no cover + if not frk: #not PRODUCTION: + raise # Use the PATH env. logSys.warning("Initial start attempt failed (%s). Starting %r with the same args", e, SERVER) if frk: return os.execvp(SERVER, args) - else: - del args[0] - args[0] = SERVER - return os.spawnvp(os.P_WAIT, SERVER, args) return pid @staticmethod @@ -181,8 +180,8 @@ class Fail2banServer(Fail2banCmdLine): phase = dict() logSys.debug('Configure via async client thread') cli.configureServer(async=True, phase=phase) - # wait up to MAX_WAITTIME, do not continue if configuration is not 100% valid: - Utils.wait_for(lambda: phase.get('ready', None) is not None, MAX_WAITTIME) + # wait, do not continue if configuration is not 100% valid: + Utils.wait_for(lambda: phase.get('ready', None) is not None, self._conf["timeout"]) if not phase.get('start', False): raise ServerExecutionException('Async configuration of server failed') @@ -197,7 +196,7 @@ class Fail2banServer(Fail2banCmdLine): # wait for client answer "done": if not async and cli: - Utils.wait_for(lambda: phase.get('done', None) is not None, MAX_WAITTIME) + Utils.wait_for(lambda: phase.get('done', None) is not None, self._conf["timeout"]) if not phase.get('done', False): if server: server.quit() diff --git a/fail2ban/server/utils.py b/fail2ban/server/utils.py index a8496f8e..fcf54d5f 100644 --- a/fail2ban/server/utils.py +++ b/fail2ban/server/utils.py @@ -168,8 +168,10 @@ class Utils(): stdout = popen.stdout.read() except IOError as e: logSys.error(" ... -- failed to read stdout %s", e) - if stdout is not None and stdout != '': - logSys.log(std_level, "%s -- stdout: %r", realCmd, stdout) + if stdout is not None and stdout != '' and std_level >= logSys.getEffectiveLevel(): + logSys.log(std_level, "%s -- stdout:", realCmd) + for l in stdout.splitlines(): + logSys.log(std_level, " -- stdout: %r", l) popen.stdout.close() if popen.stderr: try: @@ -178,8 +180,10 @@ class Utils(): stderr = popen.stderr.read() except IOError as e: logSys.error(" ... -- failed to read stderr %s", e) - if stderr is not None and stderr != '': - logSys.log(std_level, "%s -- stderr: %r", realCmd, stderr) + if stderr is not None and stderr != '' and std_level >= logSys.getEffectiveLevel(): + logSys.log(std_level, "%s -- stderr:", realCmd) + for l in stderr.splitlines(): + logSys.log(std_level, " -- stderr: %r", l) popen.stderr.close() if retcode == 0: diff --git a/fail2ban/tests/actiontestcase.py b/fail2ban/tests/actiontestcase.py index 6d8fcc82..1872eb1f 100644 --- a/fail2ban/tests/actiontestcase.py +++ b/fail2ban/tests/actiontestcase.py @@ -271,11 +271,11 @@ class CommandActionTest(LogCaptureTestCase): def testCaptureStdOutErr(self): CommandAction.executeCmd('echo "How now brown cow"') - self.assertLogged("'How now brown cow\\n'") + self.assertLogged("stdout: 'How now brown cow'\n", "stdout: b'How now brown cow'\n") CommandAction.executeCmd( 'echo "The rain in Spain stays mainly in the plain" 1>&2') self.assertLogged( - "'The rain in Spain stays mainly in the plain\\n'") + "stderr: 'The rain in Spain stays mainly in the plain'\n", "stderr: b'The rain in Spain stays mainly in the plain'\n") def testCallingMap(self): mymap = CallingMap(callme=lambda: str(10), error=lambda: int('a'), diff --git a/fail2ban/tests/fail2banclienttestcase.py b/fail2ban/tests/fail2banclienttestcase.py index 6fee732b..aa507fe6 100644 --- a/fail2ban/tests/fail2banclienttestcase.py +++ b/fail2ban/tests/fail2banclienttestcase.py @@ -43,11 +43,6 @@ from .utils import LogCaptureTestCase, logSys, withtmpdir, shutil, logging STOCK_CONF_DIR = "config" STOCK = os.path.exists(os.path.join(STOCK_CONF_DIR,'fail2ban.conf')) -TEST_CONF_DIR = os.path.join(os.path.dirname(__file__), "config") -if STOCK: - CONF_DIR = STOCK_CONF_DIR -else: - CONF_DIR = TEST_CONF_DIR CLIENT = "fail2ban-client" SERVER = "fail2ban-server" @@ -59,9 +54,7 @@ MAX_WAITTIME = 30 if not unittest.F2B.fast else 5 # Several wrappers and settings for proper testing: # -fail2banclient.MAX_WAITTIME = \ -fail2banserver.MAX_WAITTIME = MAX_WAITTIME - +fail2bancmdline.MAX_WAITTIME = MAX_WAITTIME-1 fail2bancmdline.logSys = \ fail2banclient.logSys = \ @@ -167,6 +160,7 @@ def _start_params(tmp, use_stock=False, logtarget="/dev/null"): # parameters (sock/pid and config, increase verbosity, set log, etc.): return ("-c", cfg, "-s", tmp+"/f2b.sock", "-p", tmp+"/f2b.pid", "-vv", "--logtarget", logtarget, "--loglevel", "DEBUG", "--syslogsocket", "auto", + "--timeout", str(fail2bancmdline.MAX_WAITTIME), ) def _kill_srv(pidfile): # pragma: no cover @@ -199,7 +193,7 @@ def _kill_srv(pidfile): # pragma: no cover ## try to preper stop (have signal handler): os.kill(pid, signal.SIGTERM) ## check still exists after small timeout: - if not Utils.wait_for(lambda: not _pid_exists(pid), MAX_WAITTIME / 3): + if not Utils.wait_for(lambda: not _pid_exists(pid), 1): ## try to kill hereafter: os.kill(pid, signal.SIGKILL) return not _pid_exists(pid) @@ -222,25 +216,50 @@ class Fail2banClientServerBase(LogCaptureTestCase): LogCaptureTestCase.tearDown(self) def _wait_for_srv(self, tmp, ready=True, startparams=None): - sock = tmp+"/f2b.sock" - # wait for server (socket): - ret = Utils.wait_for(lambda: os.path.exists(sock), MAX_WAITTIME) - if not ret: - raise Exception('Unexpected: Socket file does not exists.\nStart failed: %r' % (startparams,)) - if ready: - # wait for communication with worker ready: - ret = Utils.wait_for(lambda: "Server ready" in self.getLog(), MAX_WAITTIME) + try: + sock = tmp+"/f2b.sock" + # wait for server (socket): + ret = Utils.wait_for(lambda: os.path.exists(sock), MAX_WAITTIME) if not ret: - raise Exception('Unexpected: Server ready was not found.\nStart failed: %r' % (startparams,)) + raise Exception('Unexpected: Socket file does not exists.\nStart failed: %r' % (startparams,)) + if ready: + # wait for communication with worker ready: + ret = Utils.wait_for(lambda: "Server ready" in self.getLog(), MAX_WAITTIME) + if not ret: + raise Exception('Unexpected: Server ready was not found.\nStart failed: %r' % (startparams,)) + except: # pragma: no cover + log = tmp+"/f2b.log" + if os.path.isfile(log): + _out_file(log) + else: + logSys.debug("No log file %s to examine details of error", log) + raise class Fail2banClientTest(Fail2banClientServerBase): + def testConsistency(self): + self.assertTrue(os.path.isfile(os.path.join(os.path.join(BIN), CLIENT))) + self.assertTrue(os.path.isfile(os.path.join(os.path.join(BIN), SERVER))) + def testClientUsage(self): self.assertRaises(ExitException, _exec_client, (CLIENT, "-h",)) self.assertLogged("Usage: " + CLIENT) self.assertLogged("Report bugs to ") + self.pruneLog() + self.assertRaises(ExitException, _exec_client, + (CLIENT, "-vq", "-V",)) + self.assertLogged("Fail2Ban v" + fail2bancmdline.version) + + @withtmpdir + def testClientDump(self, tmp): + # use here the stock configuration (if possible) + startparams = _start_params(tmp, True) + self.assertRaises(ExitException, _exec_client, + ((CLIENT,) + startparams + ("-vvd",))) + self.assertLogged("Loading files") + self.assertLogged("logtarget") @withtmpdir def testClientStartBackgroundInside(self, tmp): @@ -271,6 +290,13 @@ class Fail2banClientTest(Fail2banClientServerBase): (CLIENT,) + startparams + ("stop",)) self.assertLogged("Shutdown successful") self.assertLogged("Exit with code 0") + + self.pruneLog() + # stop again (should fail): + self.assertRaises(FailExitException, _exec_client, + (CLIENT,) + startparams + ("stop",)) + self.assertLogged("Failed to access socket path") + self.assertLogged("Is fail2ban running?") finally: _kill_srv(tmp) @@ -278,12 +304,13 @@ class Fail2banClientTest(Fail2banClientServerBase): def testClientStartBackgroundCall(self, tmp): try: global INTERACT - startparams = _start_params(tmp) + startparams = _start_params(tmp, logtarget=tmp+"/f2b.log") # start (in new process, using the same python version): cmd = (sys.executable, os.path.join(os.path.join(BIN), CLIENT)) logSys.debug('Start %s ...', cmd) - cmd = cmd + startparams + ("start",) - Utils.executeCmd(cmd, timeout=MAX_WAITTIME, shell=False, output=True) + cmd = cmd + startparams + ("--async", "start",) + ret = Utils.executeCmd(cmd, timeout=MAX_WAITTIME, shell=False, output=True) + self.assertTrue(len(ret) and ret[0]) # wait for server (socket and ready): self._wait_for_srv(tmp, True, startparams=cmd) self.assertLogged("Server ready") @@ -383,13 +410,35 @@ class Fail2banClientTest(Fail2banClientServerBase): @withtmpdir def testClientFailStart(self, tmp): try: + # started directly here, so prevent overwrite test cases logger with "INHERITED" + startparams = _start_params(tmp, logtarget="INHERITED") + + ## wrong config directory self.assertRaises(FailExitException, _exec_client, (CLIENT, "--async", "-c", tmp+"/miss", "start",)) self.assertLogged("Base configuration directory " + tmp+"/miss" + " does not exist") + self.pruneLog() + ## wrong socket self.assertRaises(FailExitException, _exec_client, - (CLIENT, "--async", "-c", CONF_DIR, "-s", tmp+"/miss/f2b.sock", "start",)) + (CLIENT, "--async", "-c", tmp+"/config", "-s", tmp+"/miss/f2b.sock", "start",)) self.assertLogged("There is no directory " + tmp+"/miss" + " to contain the socket file") + self.pruneLog() + + ## already exists: + open(tmp+"/f2b.sock", 'a').close() + self.assertRaises(FailExitException, _exec_client, + (CLIENT, "--async", "-c", tmp+"/config", "-s", tmp+"/f2b.sock", "start",)) + self.assertLogged("Fail2ban seems to be in unexpected state (not running but the socket exists)") + self.pruneLog() + os.remove(tmp+"/f2b.sock") + + ## wrong option: + self.assertRaises(FailExitException, _exec_client, + (CLIENT, "-s",)) + self.assertLogged("Usage: ") + self.pruneLog() + finally: _kill_srv(tmp) @@ -417,12 +466,13 @@ class Fail2banServerTest(Fail2banClientServerBase): def testServerStartBackground(self, tmp): try: # to prevent fork of test-cases process, start server in background via command: - startparams = _start_params(tmp) + startparams = _start_params(tmp, logtarget=tmp+"/f2b.log") # start (in new process, using the same python version): cmd = (sys.executable, os.path.join(os.path.join(BIN), SERVER)) logSys.debug('Start %s ...', cmd) cmd = cmd + startparams + ("-b",) - Utils.executeCmd(cmd, timeout=MAX_WAITTIME, shell=False, output=True) + ret = Utils.executeCmd(cmd, timeout=MAX_WAITTIME, shell=False, output=True) + self.assertTrue(len(ret) and ret[0]) # wait for server (socket and ready): self._wait_for_srv(tmp, True, startparams=cmd) self.assertLogged("Server ready") @@ -495,12 +545,28 @@ class Fail2banServerTest(Fail2banClientServerBase): @withtmpdir def testServerFailStart(self, tmp): try: + # started directly here, so prevent overwrite test cases logger with "INHERITED" + startparams = _start_params(tmp, logtarget="INHERITED") + + ## wrong config directory self.assertRaises(FailExitException, _exec_server, (SERVER, "-c", tmp+"/miss",)) self.assertLogged("Base configuration directory " + tmp+"/miss" + " does not exist") + self.pruneLog() + ## wrong socket self.assertRaises(FailExitException, _exec_server, - (SERVER, "-c", CONF_DIR, "-s", tmp+"/miss/f2b.sock",)) + (SERVER, "-c", tmp+"/config", "-x", "-s", tmp+"/miss/f2b.sock",)) self.assertLogged("There is no directory " + tmp+"/miss" + " to contain the socket file") + self.pruneLog() + + ## already exists: + open(tmp+"/f2b.sock", 'a').close() + self.assertRaises(FailExitException, _exec_server, + (SERVER, "-c", tmp+"/config", "-s", tmp+"/f2b.sock",)) + self.assertLogged("Fail2ban seems to be in unexpected state (not running but the socket exists)") + self.pruneLog() + os.remove(tmp+"/f2b.sock") + finally: _kill_srv(tmp) From 0fef5022f098b220a815387dfca63f8084538ccd Mon Sep 17 00:00:00 2001 From: sebres Date: Mon, 15 Feb 2016 20:41:20 +0100 Subject: [PATCH 50/52] reader bug fix: prevent to silent "load" of not existing jail; coverage of test cases increased; --- fail2ban/client/configreader.py | 4 +++- fail2ban/client/fail2banclient.py | 20 ++++++++++---------- fail2ban/client/fail2banserver.py | 24 ++++++++++++------------ fail2ban/client/jailreader.py | 2 +- fail2ban/tests/fail2banclienttestcase.py | 21 +++++++++++++++++++++ 5 files changed, 47 insertions(+), 24 deletions(-) diff --git a/fail2ban/client/configreader.py b/fail2ban/client/configreader.py index c6dd1b60..f333cdc1 100644 --- a/fail2ban/client/configreader.py +++ b/fail2ban/client/configreader.py @@ -208,7 +208,7 @@ class ConfigReaderUnshared(SafeConfigParserWithIncludes): # 1 -> the name of the option # 2 -> the default value for the option - def getOptions(self, sec, options, pOptions=None): + def getOptions(self, sec, options, pOptions=None, shouldExist=False): values = dict() for option in options: try: @@ -222,6 +222,8 @@ class ConfigReaderUnshared(SafeConfigParserWithIncludes): continue values[option[1]] = v except NoSectionError, e: + if shouldExist: + raise # No "Definition" section or wrong basedir logSys.error(e) values[option[1]] = option[2] diff --git a/fail2ban/client/fail2banclient.py b/fail2ban/client/fail2banclient.py index 23d31dc5..a8e0a331 100644 --- a/fail2ban/client/fail2banclient.py +++ b/fail2ban/client/fail2banclient.py @@ -64,7 +64,7 @@ class Fail2banClient(Fail2banCmdLine, Thread): output("and bans the corresponding IP addresses using firewall rules.") output("") - def __sigTERMhandler(self, signum, frame): + def __sigTERMhandler(self, signum, frame): # pragma: no cover # Print a new line because we probably come from wait output("") logSys.warning("Caught signal %d. Exiting" % signum) @@ -141,7 +141,7 @@ class Fail2banClient(Fail2banCmdLine, Thread): logSys.error("Failed to access socket path: %s." " Is fail2ban running?", self._conf["socket"]) - except Exception as e: + except Exception as e: # pragma: no cover logSys.error("Exception while checking socket access: %s", self._conf["socket"]) logSys.error(e) @@ -165,7 +165,7 @@ class Fail2banClient(Fail2banCmdLine, Thread): "There is no directory %s to contain the socket file %s." % (socket_dir, self._conf["socket"])) return None - if not os.access(socket_dir, os.W_OK | os.X_OK): + if not os.access(socket_dir, os.W_OK | os.X_OK): # pragma: no cover logSys.error( "Directory %s exists but not accessible for writing" % (socket_dir,)) @@ -204,9 +204,9 @@ class Fail2banClient(Fail2banCmdLine, Thread): # Start server direct here in main thread (not fork): self._server = Fail2banServer.startServerDirect(self._conf, False) - except ExitException: + except ExitException: # pragma: no cover pass - except Exception as e: + except Exception as e: # pragma: no cover output("") logSys.error("Exception while starting server " + ("background" if background else "foreground")) if self._conf["verbose"] > 1: @@ -259,7 +259,7 @@ class Fail2banClient(Fail2banCmdLine, Thread): if self._conf.get("interactive", False): output(' ## stop ... ') self.__processCommand(['stop']) - if not self.__waitOnServer(False): + if not self.__waitOnServer(False): # pragma: no cover logSys.error("Could not stop server") return False # in interactive mode reset config, to make full-reload if there something changed: @@ -298,12 +298,12 @@ class Fail2banClient(Fail2banCmdLine, Thread): def __processStartStreamAfterWait(self, *args): try: # Wait for the server to start - if not self.__waitOnServer(): + if not self.__waitOnServer(): # pragma: no cover logSys.error("Could not find server, waiting failed") return False # Configure the server self.__processCmd(*args) - except ServerExecutionException as e: + except ServerExecutionException as e: # pragma: no cover if self._conf["verbose"] > 1: logSys.exception(e) logSys.error("Could not start server. Maybe an old " @@ -385,12 +385,12 @@ class Fail2banClient(Fail2banCmdLine, Thread): elif not cmd == "": try: self.__processCommand(shlex.split(cmd)) - except Exception, e: + except Exception, e: # pragma: no cover if self._conf["verbose"] > 1: logSys.exception(e) else: logSys.error(e) - except (EOFError, KeyboardInterrupt): + except (EOFError, KeyboardInterrupt): # pragma: no cover output("") raise # Single command mode diff --git a/fail2ban/client/fail2banserver.py b/fail2ban/client/fail2banserver.py index a6c8a7c5..a511e017 100644 --- a/fail2ban/client/fail2banserver.py +++ b/fail2ban/client/fail2banserver.py @@ -56,7 +56,7 @@ class Fail2banServer(Fail2banCmdLine): server.start(conf["socket"], conf["pidfile"], conf["force"], conf=conf) - except Exception as e: + except Exception as e: # pragma: no cover try: if server: server.quit() @@ -77,7 +77,7 @@ class Fail2banServer(Fail2banCmdLine): # Forks the current process, don't fork if async specified (ex: test cases) pid = 0 frk = not conf["async"] and PRODUCTION - if frk: + if frk: # pragma: no cover pid = os.fork() logSys.debug("-- async starting of server in %s, fork: %s - %s", os.getpid(), frk, pid) if pid == 0: @@ -108,22 +108,20 @@ class Fail2banServer(Fail2banCmdLine): exe = sys.executable args[0:0] = [exe] logSys.debug("Starting %r with args %r", exe, args) - if frk: - return os.execv(exe, args) + if frk: # pragma: no cover + os.execv(exe, args) else: # use P_WAIT instead of P_NOWAIT (to prevent defunct-zomby process), it startet as daemon, so parent exit fast after fork): ret = os.spawnv(os.P_WAIT, exe, args) - if ret != 0: + if ret != 0: # pragma: no cover raise OSError(ret, "Unknown error by executing server %r with %r" % (args[1], exe)) - return 0 except OSError as e: # pragma: no cover if not frk: #not PRODUCTION: raise # Use the PATH env. logSys.warning("Initial start attempt failed (%s). Starting %r with the same args", e, SERVER) - if frk: - return os.execvp(SERVER, args) - return pid + if frk: # pragma: no cover + os.execvp(SERVER, args) @staticmethod def getServerPath(): @@ -189,7 +187,7 @@ class Fail2banServer(Fail2banCmdLine): pid = os.getpid() server = Fail2banServer.startServerDirect(self._conf, background) # If forked - just exit other processes - if pid != os.getpid(): + if pid != os.getpid(): # pragma: no cover os._exit(0) if cli: cli._server = server @@ -198,7 +196,7 @@ class Fail2banServer(Fail2banCmdLine): if not async and cli: Utils.wait_for(lambda: phase.get('done', None) is not None, self._conf["timeout"]) if not phase.get('done', False): - if server: + if server: # pragma: no cover server.quit() exit(-1) logSys.debug('Starting server done') @@ -206,7 +204,9 @@ class Fail2banServer(Fail2banCmdLine): except Exception, e: if self._conf["verbose"] > 1: logSys.exception(e) - if server: + else: + logSys.error(e) + if server: # pragma: no cover server.quit() exit(-1) diff --git a/fail2ban/client/jailreader.py b/fail2ban/client/jailreader.py index 46f910e3..be53b3f3 100644 --- a/fail2ban/client/jailreader.py +++ b/fail2ban/client/jailreader.py @@ -109,7 +109,7 @@ class JailReader(ConfigReader): ["string", "action", ""]] # Read first options only needed for merge defaults ('known/...' from filter): - self.__opts = ConfigReader.getOptions(self, self.__name, opts1st) + self.__opts = ConfigReader.getOptions(self, self.__name, opts1st, shouldExist=True) if not self.__opts: return False diff --git a/fail2ban/tests/fail2banclienttestcase.py b/fail2ban/tests/fail2banclienttestcase.py index aa507fe6..82ffe9b3 100644 --- a/fail2ban/tests/fail2banclienttestcase.py +++ b/fail2ban/tests/fail2banclienttestcase.py @@ -347,6 +347,21 @@ class Fail2banClientTest(Fail2banClientServerBase): self.assertLogged("Server ready") self.assertLogged("Exit with code 0") self.pruneLog() + # test reload missing jail (interactive): + INTERACT += [ + "reload ~~unknown~jail~fail~~", + "exit" + ] + self.assertRaises(ExitException, _exec_client, + (CLIENT,) + startparams + ("-i",)) + self.assertLogged("Failed during configuration: No section: '~~unknown~jail~fail~~'") + self.pruneLog() + # test reload missing jail (direct): + self.assertRaises(FailExitException, _exec_client, + (CLIENT,) + startparams + ("reload", "~~unknown~jail~fail~~")) + self.assertLogged("Failed during configuration: No section: '~~unknown~jail~fail~~'") + self.assertLogged("Exit with code -1") + self.pruneLog() finally: self.pruneLog() # stop: @@ -425,6 +440,12 @@ class Fail2banClientTest(Fail2banClientServerBase): self.assertLogged("There is no directory " + tmp+"/miss" + " to contain the socket file") self.pruneLog() + ## not running + self.assertRaises(FailExitException, _exec_client, + (CLIENT, "-c", tmp+"/config", "-s", tmp+"/f2b.sock", "reload",)) + self.assertLogged("Could not find server") + self.pruneLog() + ## already exists: open(tmp+"/f2b.sock", 'a').close() self.assertRaises(FailExitException, _exec_client, From 95af3c63ac0d0f29ff48bac9be98918907d33cd2 Mon Sep 17 00:00:00 2001 From: sebres Date: Fri, 12 Feb 2016 21:57:12 +0100 Subject: [PATCH 51/52] increase readability and details level by increased verbosity --- bin/fail2ban-testcases | 12 +++++++---- fail2ban/server/server.py | 13 +++++++---- fail2ban/tests/actiontestcase.py | 2 +- fail2ban/tests/servertestcase.py | 37 +++++++++++++++++++++----------- fail2ban/tests/utils.py | 1 + 5 files changed, 43 insertions(+), 22 deletions(-) diff --git a/bin/fail2ban-testcases b/bin/fail2ban-testcases index 606b0b06..3b18b7c2 100755 --- a/bin/fail2ban-testcases +++ b/bin/fail2ban-testcases @@ -119,10 +119,14 @@ else: # Custom log format for the verbose tests runs if verbosity > 1: # pragma: no cover - stdout.setFormatter(Formatter(' %(asctime)-15s %(thread)s' + fmt)) -else: # pragma: no cover - # just prefix with the space - stdout.setFormatter(Formatter(fmt)) + if verbosity > 3: + fmt = ' | %(module)15.15s-%(levelno)-2d: %(funcName)-20.20s |' + fmt + if verbosity > 2: + fmt = ' +%(relativeCreated)5d %(thread)X %(levelname)-5.5s' + fmt + else: + fmt = ' %(asctime)-15s %(thread)X %(levelname)-5.5s' + fmt +# +stdout.setFormatter(Formatter(fmt)) logSys.addHandler(stdout) # diff --git a/fail2ban/server/server.py b/fail2ban/server/server.py index 0ddaea35..933b3e72 100644 --- a/fail2ban/server/server.py +++ b/fail2ban/server/server.py @@ -67,7 +67,8 @@ class Server: self.__db = None self.__daemon = daemon self.__transm = Transmitter(self) - self.__asyncServer = AsyncServer(self.__transm) + #self.__asyncServer = AsyncServer(self.__transm) + self.__asyncServer = None self.__logLevel = None self.__logTarget = None self.__syslogSocket = None @@ -137,6 +138,7 @@ class Server: # Start the communication logSys.debug("Starting communication") try: + self.__asyncServer = AsyncServer(self.__transm) self.__asyncServer.start(sock, force) except AsyncServerException, e: logSys.error("Could not start server: %s", e) @@ -155,14 +157,17 @@ class Server: # communications first (which should be ok anyways since we # are exiting) # See https://github.com/fail2ban/fail2ban/issues/7 - self.__asyncServer.stop() + if self.__asyncServer is not None: + self.__asyncServer.stop() + self.__asyncServer = None # Now stop all the jails self.stopAllJail() # Only now shutdown the logging. - with self.__loggingLock: - logging.shutdown() + if self.__logTarget is not None: + with self.__loggingLock: + logging.shutdown() # Restore default signal handlers: if _thread_name() == '_MainThread': diff --git a/fail2ban/tests/actiontestcase.py b/fail2ban/tests/actiontestcase.py index 1872eb1f..39984169 100644 --- a/fail2ban/tests/actiontestcase.py +++ b/fail2ban/tests/actiontestcase.py @@ -44,8 +44,8 @@ class CommandActionTest(LogCaptureTestCase): def tearDown(self): """Call after every test case.""" - LogCaptureTestCase.tearDown(self) self.__action.stop() + LogCaptureTestCase.tearDown(self) def testSubstituteRecursiveTags(self): aInfo = { diff --git a/fail2ban/tests/servertestcase.py b/fail2ban/tests/servertestcase.py index 96734262..4b9542c1 100644 --- a/fail2ban/tests/servertestcase.py +++ b/fail2ban/tests/servertestcase.py @@ -62,25 +62,18 @@ class TransmitterBase(unittest.TestCase): def setUp(self): """Call before every test case.""" + #super(TransmitterBase, self).setUp() self.transm = self.server._Server__transm - self.tmp_files = [] - sock_fd, sock_name = tempfile.mkstemp('fail2ban.sock', 'transmitter') - os.close(sock_fd) - self.tmp_files.append(sock_name) - pidfile_fd, pidfile_name = tempfile.mkstemp( - 'fail2ban.pid', 'transmitter') - os.close(pidfile_fd) - self.tmp_files.append(pidfile_name) - self.server.start(sock_name, pidfile_name, force=False) + # To test thransmitter we don't need to start server... + #self.server.start('/dev/null', '/dev/null', force=False) self.jailName = "TestJail1" self.server.addJail(self.jailName, FAST_BACKEND) def tearDown(self): """Call after every test case.""" + # stop jails, etc. self.server.quit() - for f in self.tmp_files: - if os.path.exists(f): - os.remove(f) + #super(TransmitterBase, self).tearDown() def setGetTest(self, cmd, inValue, outValue=(None,), outCode=0, jail=None, repr_=False): """Process set/get commands and compare both return values @@ -792,10 +785,10 @@ class TransmitterLogging(TransmitterBase): def setUp(self): self.server = Server() + super(TransmitterLogging, self).setUp() self.server.setLogTarget("/dev/null") self.server.setLogLevel("CRITICAL") self.server.setSyslogSocket("auto") - super(TransmitterLogging, self).setUp() def testLogTarget(self): logTargets = [] @@ -963,3 +956,21 @@ class LoggingTests(LogCaptureTestCase): sys.__excepthook__ = prev_exchook self.assertEqual(len(x), 1) self.assertEqual(x[0][0], RuntimeError) + + def testStartFailedSockExists(self): + tmp_files = [] + sock_fd, sock_name = tempfile.mkstemp('fail2ban.sock', 'f2b-test') + os.close(sock_fd) + tmp_files.append(sock_name) + pidfile_fd, pidfile_name = tempfile.mkstemp('fail2ban.pid', 'f2b-test') + os.close(pidfile_fd) + tmp_files.append(pidfile_name) + server = TestServer() + try: + server.start(sock_name, pidfile_name, force=False) + self.assertLogged("Server already running") + finally: + server.quit() + for f in tmp_files: + if os.path.exists(f): + os.remove(f) diff --git a/fail2ban/tests/utils.py b/fail2ban/tests/utils.py index b171511d..218cb160 100644 --- a/fail2ban/tests/utils.py +++ b/fail2ban/tests/utils.py @@ -325,6 +325,7 @@ class LogCaptureTestCase(unittest.TestCase): def tearDown(self): """Call after every test case.""" # print "O: >>%s<<" % self._log.getvalue() + self.pruneLog() logSys = getLogger("fail2ban") logSys.handlers = self._old_handlers logSys.level = self._old_level From 70c329e235a1dc8562a8eee485a097bf35155875 Mon Sep 17 00:00:00 2001 From: sebres Date: Mon, 15 Feb 2016 19:22:18 +0100 Subject: [PATCH 52/52] increase verbosity for travis/py3 (currently "debug", use "heavydebug" for more details if needed) --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index adb41e7d..71a0a05f 100644 --- a/.travis.yml +++ b/.travis.yml @@ -38,7 +38,7 @@ script: # Keep the legacy setup.py test approach of checking coverage for python2 - if [[ "$F2B_PY_2" ]]; then coverage run setup.py test; fi # Coverage doesn't pick up setup.py test with python3, so run it directly - - if [[ "$F2B_PY_3" ]]; then coverage run bin/fail2ban-testcases; fi + - if [[ "$F2B_PY_3" ]]; then coverage run bin/fail2ban-testcases -l debug; fi # Use $VENV_BIN (not python) or else sudo will always run the system's python (2.7) - sudo $VENV_BIN/pip install . after_success: