Add HEAD method verb to apache-badbots, nginx-badbots

pull/1101/head
Viktor Szépe 2015-07-07 17:45:40 +02:00
parent 052418a110
commit a3b8257b73
5 changed files with 23 additions and 3 deletions

View File

@ -12,6 +12,8 @@ ver. 0.9.3 (2015/XX/XXX) - wanna-be-released
- IMPORTANT incompatible changes:
* filter.d/roundcube-auth.conf
- Changed logpath to 'errors' log (was 'userlogins')
* filter.d/apache-badbots.conf, filter.d/nginx-botsearch.conf - add
HEAD method verb
- Fixes:
* reload in interactive mode appends all the jails twice (gh-825)

View File

@ -10,7 +10,7 @@
badbotscustom = EmailCollector|WebEMailExtrac|TrackBack/1\.02|sogou music spider
badbots = Atomic_Email_Hunter/4\.0|atSpider/1\.0|autoemailspider|bwh3_user_agent|China Local Browse 2\.6|ContactBot/0\.2|ContentSmartz|DataCha0s/2\.0|DBrowse 1\.4b|DBrowse 1\.4d|Demo Bot DOT 16b|Demo Bot Z 16b|DSurf15a 01|DSurf15a 71|DSurf15a 81|DSurf15a VA|EBrowse 1\.4b|Educate Search VxB|EmailSiphon|EmailSpider|EmailWolf 1\.00|ESurf15a 15|ExtractorPro|Franklin Locator 1\.8|FSurf15a 01|Full Web Bot 0416B|Full Web Bot 0516B|Full Web Bot 2816B|Guestbook Auto Submitter|Industry Program 1\.0\.x|ISC Systems iRc Search 2\.1|IUPUI Research Bot v 1\.9a|LARBIN-EXPERIMENTAL \(efp@gmx\.net\)|LetsCrawl\.com/1\.0 +http\://letscrawl\.com/|Lincoln State Web Browser|LMQueueBot/0\.2|LWP\:\:Simple/5\.803|Mac Finder 1\.0\.xx|MFC Foundation Class Library 4\.0|Microsoft URL Control - 6\.00\.8xxx|Missauga Locate 1\.0\.0|Missigua Locator 1\.9|Missouri College Browse|Mizzu Labs 2\.2|Mo College 1\.9|MVAClient|Mozilla/2\.0 \(compatible; NEWT ActiveX; Win32\)|Mozilla/3\.0 \(compatible; Indy Library\)|Mozilla/3\.0 \(compatible; scan4mail \(advanced version\) http\://www\.peterspages\.net/?scan4mail\)|Mozilla/4\.0 \(compatible; Advanced Email Extractor v2\.xx\)|Mozilla/4\.0 \(compatible; Iplexx Spider/1\.0 http\://www\.iplexx\.at\)|Mozilla/4\.0 \(compatible; MSIE 5\.0; Windows NT; DigExt; DTS Agent|Mozilla/4\.0 efp@gmx\.net|Mozilla/5\.0 \(Version\: xxxx Type\:xx\)|NameOfAgent \(CMS Spider\)|NASA Search 1\.0|Nsauditor/1\.x|PBrowse 1\.4b|PEval 1\.4b|Poirot|Port Huron Labs|Production Bot 0116B|Production Bot 2016B|Production Bot DOT 3016B|Program Shareware 1\.0\.2|PSurf15a 11|PSurf15a 51|PSurf15a VA|psycheclone|RSurf15a 41|RSurf15a 51|RSurf15a 81|searchbot admin@google\.com|ShablastBot 1\.0|snap\.com beta crawler v0|Snapbot/1\.0|Snapbot/1\.0 \(Snap Shots, +http\://www\.snap\.com\)|sogou develop spider|Sogou Orion spider/3\.0\(+http\://www\.sogou\.com/docs/help/webmasters\.htm#07\)|sogou spider|Sogou web spider/3\.0\(+http\://www\.sogou\.com/docs/help/webmasters\.htm#07\)|sohu agent|SSurf15a 11 |TSurf15a 11|Under the Rainbow 2\.2|User-Agent\: Mozilla/4\.0 \(compatible; MSIE 6\.0; Windows NT 5\.1\)|VadixBot|WebVulnCrawl\.unknown/1\.0 libwww-perl/5\.803|Wells Search II|WEP Search 00
failregex = ^<HOST> -.*"(GET|POST).*HTTP.*"(?:%(badbots)s|%(badbotscustom)s)"$
failregex = ^<HOST> -.*"(GET|POST|HEAD).*HTTP.*"(?:%(badbots)s|%(badbotscustom)s)"$
ignoreregex =

View File

@ -8,8 +8,8 @@ before = botsearch-common.conf
[Definition]
failregex = ^<HOST> \- \S+ \[\] \"(GET|POST) \/<block> \S+\" 404 .+$
^ \[error\] \d+#\d+: \*\d+ (\S+ )?\"\S+\" (failed|is not found) \(2\: No such file or directory\), client\: <HOST>\, server\: \S*\, request: \"(GET|POST) \/<block> \S+\"\, .*?$
failregex = ^<HOST> \- \S+ \[\] \"(GET|POST|HEAD) \/<block> \S+\" 404 .+$
^ \[error\] \d+#\d+: \*\d+ (\S+ )?\"\S+\" (failed|is not found) \(2\: No such file or directory\), client\: <HOST>\, server\: \S*\, request: \"(GET|POST|HEAD) \/<block> \S+\"\, .*?$
ignoreregex =

View File

@ -1,2 +1,8 @@
# failJSON: { "time": "2007-03-05T14:39:21", "match": true , "host": "1.2.3.4" }
1.2.3.4 - - [05/Mar/2007:14:39:21 +0100] "POST /123.html/trackback/ HTTP/1.0" 301 459 "http://www.mydomain.tld/123.html/trackback" "TrackBack/1.02"
# failJSON: { "time": "2007-03-05T14:40:21", "match": true , "host": "1.2.3.4" }
1.2.3.4 - - [05/Mar/2007:14:40:21 +0100] "GET /123.html/trackback/ HTTP/1.0" 301 459 "http://www.mydomain.tld/123.html/trackback" "TrackBack/1.02"
# failJSON: { "time": "2007-03-05T14:41:21", "match": true , "host": "1.2.3.4" }
1.2.3.4 - - [05/Mar/2007:14:41:21 +0100] "HEAD /123.html/trackback/ HTTP/1.0" 301 459 "http://www.mydomain.tld/123.html/trackback" "TrackBack/1.02"

View File

@ -10,6 +10,12 @@
# failJSON: { "time": "2015-01-20T19:53:28", "match": true , "host": "12.34.56.78" }
12.34.56.78 - - [20/Jan/2015:19:53:28 +0100] "GET //admin/pma/scripts/setup.php HTTP/1.1" 404 47 "-" "-" "-"
# failJSON: { "time": "2015-01-20T19:54:28", "match": true , "host": "12.34.56.78" }
12.34.56.78 - - [20/Jan/2015:19:54:28 +0100] "POST //admin/pma/scripts/setup.php HTTP/1.1" 404 47 "-" "-" "-"
# failJSON: { "time": "2015-01-20T19:55:28", "match": true , "host": "12.34.56.78" }
12.34.56.78 - - [20/Jan/2015:19:55:28 +0100] "HEAD //admin/pma/scripts/setup.php HTTP/1.1" 404 47 "-" "-" "-"
# failJSON: { "time": "2015-01-20T01:17:07", "match": true , "host": "7.8.9.10" }
7.8.9.10 - root [20/Jan/2015:01:17:07 +0100] "GET /cgi-bin/recent.cgi HTTP/1.1" 404 162 "-" "-" "-"
@ -19,5 +25,11 @@
# failJSON: { "time": "2015-01-21T10:56:10", "match": true , "host": "5.7.9.2" }
2015/01/21 10:56:10 [error] 2833#0: *16336 open() "/var/www/site/cgi-bin/php4" failed (2: No such file or directory), client: 5.7.9.2, server: localhost, request: "GET /cgi-bin/php4 HTTP/1.1", host: "1.2.3.4"
# failJSON: { "time": "2015-01-21T10:57:10", "match": true , "host": "5.7.9.2" }
2015/01/21 10:57:10 [error] 2833#0: *16336 open() "/var/www/site/cgi-bin/php4" failed (2: No such file or directory), client: 5.7.9.2, server: localhost, request: "POST /cgi-bin/php4 HTTP/1.1", host: "1.2.3.4"
# failJSON: { "time": "2015-01-21T10:58:10", "match": true , "host": "5.7.9.2" }
2015/01/21 10:58:10 [error] 2833#0: *16336 open() "/var/www/site/cgi-bin/php4" failed (2: No such file or directory), client: 5.7.9.2, server: localhost, request: "HEAD /cgi-bin/php4 HTTP/1.1", host: "1.2.3.4"
# failJSON: { "time": "2015-01-21T15:02:27", "match": true , "host": "5.7.9.2" }
2015/01/21 15:02:27 [error] 2833#0: *16813 "/var/www/site/roundcube/" is not found (2: No such file or directory), client: 5.7.9.2, server: localhost, request: "GET /roundcube/ HTTP/1.1", host: "1.2.3.4"