From b468593e93d5d2f2ffbdf739527650cc75b1b23b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Hasan=20=C3=87ALI=C5=9EIR?= Date: Sat, 22 Feb 2025 21:34:17 +0300 Subject: [PATCH 1/9] Create fail2ban-abuseipdb.sh --- files/fail2ban-abuseipdb.sh | 255 ++++++++++++++++++++++++++++++++++++ 1 file changed, 255 insertions(+) create mode 100644 files/fail2ban-abuseipdb.sh diff --git a/files/fail2ban-abuseipdb.sh b/files/fail2ban-abuseipdb.sh new file mode 100644 index 00000000..6dfc865d --- /dev/null +++ b/files/fail2ban-abuseipdb.sh @@ -0,0 +1,255 @@ +#!/usr/bin/env bash +# +# Description: +# This script serves as a Fail2Ban `actionban` to report offending IP addresses to AbuseIPDB with custom COMMENTS. +# Mainly whenever Fail2Ban restarts, it calls the actionban function for each IP stored in the database file. +# If you restart your server often this script prevents duplicate reporting to AbuseIPDB by maintaining a local list of already reported IPs. +# Before reporting, it checks both the local list and AbuseIPDB to ensure the IP hasn't +# been reported previously, thereby avoiding redundant entries and potential API rate limiting. +# +# Integration with Fail2Ban: +# - Place this script in the Fail2Ban `action.d` directory named 'abuseipdb_fail2ban_actionban.sh' +# - Configure Fail2Ban to use this script as the `actionban` for relevant jails. +# - First edit 'abuseipdb.conf' in '/etc/fail2ban/action.d/abuseipdb.conf' and add following rule, +# - actionban = /etc/fail2ban/action.d/abuseipdb_fail2ban_actionban.sh \ +# "" "" "" "" "" +# - Also make sure you set your 'abuseipdb_apikey' in the configuration file. +# - Final step, adjust your jails accordingly. Check the below jail example to also reporting with custom comment via 'tp_comment' +# +# Example jail in 'jail.local': +# [nginx-botsearch] +# enabled = true +# logpath = /var/log/nginx/*.log +# port = http,https +# backend = polling +# tp_comment = Fail2Ban - NGINX bad requests 400-401-403-404-444, high level vulnerability scanning, commonly xmlrpc_attack, wp-login brute force, excessive crawling/scraping +# maxretry = 3 +# findtime = 1d +# bantime = 7200 +# action = %(action_mwl)s +# %(action_abuseipdb)s[matches="%(tp_comment)s", abuseipdb_apikey="YOUR_API_KEY", abuseipdb_category="21,15", bantime="%(bantime)s"] +# +# Start from scratch practise: +# - Stop Fail2ban +# - Clear all existing firewall reject rules from iptables +# - Delete Fail2ban SQLite database and truncate fail2ban.log +# - Force logrotate +# - Follow above implementation steps and adjust all your jails accordingly +# - Start Fail2ban +# +# Usage: +# This script is intended to be called by Fail2Ban automatically (actionban) and accepts the following arguments: +# 1. - Your AbuseIPDB API key. +# 2. - A comment describing the reason for reporting the IP. +# 3. - The IP address to report. +# 4. - Comma-separated list of abuse categories as per AbuseIPDB's API. +# 5. - Duration for which the IP should be banned (e.g., 600 for 10 minutes). +# +# Manual Usage: +# For testing purpose before production; +# /etc/fail2ban/action.d/abuseipdb_fail2ban_actionban.sh "your_api_key" "Failed SSH login attempts" "192.0.2.1" "18" "600" +# +# Arguments: +# APIKEY - Required. +# COMMENT - Required. +# IP - Required. +# CATEGORIES - Required. +# BANTIME - Required. +# +# Configurations (Need to set): +# - REPORTED_IP_LIST_FILE: Path to the local file storing reported IPs and their ban times. +# - LOG_FILE: Path to the log file where actions and events are recorded. +# +# Dependencies: +# - curl: For making API requests to AbuseIPDB. +# - jq: For parsing JSON responses. +# - flock: Prevent data corruption. +# +# General Considerations: +# - The script does not interact with or rely on Fail2Ban’s SQLite database, as the Fail2ban database setup can vary across different environments. +# - The script performs two API calls to AbuseIPDB for each ban action in order to determine whether the IP should be reported. +# - First Call: /v2/check → Checks if the IP is already reported. +# - Second Call: /v2/report → Reports the IP if necessary. +# - These two endpoints have separate daily limits, so does not affect your reporting quota. +# - When multiple instances of the script try to write to the REPORTED_IP_LIST_FILE concurrently we need to prevent data corruption. 'flock' used for that reason. +# - Never delete, truncate or try to sync REPORTED_IP_LIST_FILE with Fail2ban SQLite. +# - Even you reset fail2ban SQLite, continue to keep REPORTED_IP_LIST_FILE same, It serves as the script’s local tracking system. +# - When you go production keep watching '/var/log/abuseipdb-report.log' for any abnormal fails. +# +# Return Codes: +# 0 - IP is reported +# 1 - IP is not reported +# +# Exit Codes: +# 1 - API-related failure +# +# Author: +# Hasan ÇALIŞIR - PSAUXIT +# hasan.calisir@psauxit.com +# https://www.psauxit.com/ + +# Arguments +APIKEY="$1" +COMMENT="$2" +IP="$3" +CATEGORIES="$4" +BANTIME="$5" + +# Configuration +REPORTED_IP_LIST_FILE="/etc/fail2ban/action.d/abuseipdb-reported-ip-list" +LOG_FILE="/var/log/abuseipdb-report.log" + +# Set defaults +is_found_local=0 +shouldBanIP=1 + +# Log messages +log_message() { + local message="$1" + echo "$(date +"%Y-%m-%d %H:%M:%S") - ${message}" >> "${LOG_FILE}" +} + +# Arguments validation +if [[ -z "$1" || -z "$2" || -z "$3" || -z "$4" || -z "$5" ]]; then + log_message "FATAL: Usage: $0 " + exit 1 +fi + +# Ensure the directory for the reported IP list exists +REPORTED_IP_DIR=$(dirname "${REPORTED_IP_LIST_FILE}") +if [[ ! -d "${REPORTED_IP_DIR}" ]]; then + mkdir -p "${REPORTED_IP_DIR}" || { log_message "FATAL: Could not create directory ${REPORTED_IP_DIR}"; exit 1; } +fi + +# Ensure the reported IP list file exists and is writable +if [[ ! -f "${REPORTED_IP_LIST_FILE}" ]]; then + touch "${REPORTED_IP_LIST_FILE}" || { log_message "FATAL: Could not create file ${REPORTED_IP_LIST_FILE}"; exit 1; } +fi + +if [[ ! -w "${REPORTED_IP_LIST_FILE}" ]]; then + log_message "FATAL: File ${REPORTED_IP_LIST_FILE} is not writable." + exit 1 +fi + +# Check runtime dependencies +dependencies=("curl" "jq" "flock") +for dep in "${dependencies[@]}"; do + if ! command -v "${dep}" &>/dev/null; then + log_message "FATAL: -${dep} is not installed. Please install -${dep} to proceed." + exit 1 + fi +done + +# Check if the IP is listed on AbuseIPDB +check_ip_in_abuseipdb() { + local response http_status body total_reports error_detail + local delimiter="HTTP_STATUS:" + + # Perform the API call and capture both response and HTTP status + response=$(curl -s -w "${delimiter}%{http_code}" -G "https://api.abuseipdb.com/api/v2/check" \ + --data-urlencode "ipAddress=${IP}" \ + -H "Key: ${APIKEY}" \ + -H "Accept: application/json" 2>&1) + + if [[ $? -ne 0 ]]; then + log_message "ERROR CHECK: API failure. Response: ${response}" + exit 1 + fi + + # Separate the HTTP status code from the response body + http_status=$(echo "${response}" | tr -d '\n' | sed -e "s/.*${delimiter}//") + body=$(echo "${response}" | sed -e "s/${delimiter}[0-9]*//") + + # Check if the response body is empty + if [[ -z "${body}" ]]; then + log_message "ERROR CHECK: API response empty." + exit 1 + fi + + # Validate that the response is valid JSON + if ! echo "${body}" | jq . >/dev/null 2>&1; then + log_message "ERROR CHECK: API response malformed. Response: ${body}" + exit 1 + fi + + # Handle different HTTP status codes + if [[ "${http_status}" =~ ^[0-9]+$ ]]; then + if [[ "${http_status}" -ge 200 && "${http_status}" -lt 300 ]]; then + # Successful HTTP response; check for API-level errors + if echo "${body}" | jq -e '.errors | length > 0' >/dev/null 2>&1; then + error_detail=$(echo "${body}" | jq -r '.errors[].detail') + log_message "ERROR CHECK: API returned errors. Detail: ${error_detail}" + exit 1 + fi + else + # Non-successful HTTP status + log_message "ERROR CHECK: API returned ${http_status}. Response: ${body}" + exit 1 + fi + else + log_message "ERROR CHECK: HTTP status '${http_status}' is not numeric." + exit 1 + fi + + # Extract totalReports + total_reports=$(echo "${body}" | jq '.data.totalReports') + + # Finally, check the IP listed on AbuseIPDB + if [[ "${total_reports}" -gt 0 ]]; then + return 0 # IP is reported + else + return 1 # IP is not reported + fi +} + +# Report to AbuseIpDB +report_ip_to_abuseipdb() { + local response + response=$(curl --fail -s 'https://api.abuseipdb.com/api/v2/report' \ + -H 'Accept: application/json' \ + -H "Key: ${APIKEY}" \ + --data-urlencode "comment=${COMMENT}" \ + --data-urlencode "ip=${IP}" \ + --data "categories=${CATEGORIES}" 2>&1) + + # API call fail + if [[ $? -ne 0 ]]; then + log_message "ERROR REPORT: API failure. Response: ${response}" + exit 1 + else + log_message "SUCCESS REPORT: Reported IP ${IP} to AbuseIPDB. Local list updated." + fi +} + +# Should Ban IP +if grep -m 1 -q -E "^IP=${IP}[[:space:]]+L=[0-9\-]+" "${REPORTED_IP_LIST_FILE}"; then + # IP found locally, check if it's still listed on AbuseIPDB + if check_ip_in_abuseipdb; then + # IP is still listed on AbuseIPDB, no need to report again + log_message "INFO: IP ${IP} has already been reported and remains on AbuseIPDB. No duplicate report made." + shouldBanIP=0 + else + # IP is reported before but not listed on AbuseIPDB, report it again + log_message "INFO: IP ${IP} has already been reported but is no longer listed on AbuseIPDB. Reporting it again." + shouldBanIP=1 + is_found_local=1 + fi +else + # New comer, welcome to hell + shouldBanIP=1 +fi + +# Report to AbuseIpdb +if [[ "${shouldBanIP}" -eq 1 ]]; then + # Add the new ban entry to local list kindly + if [[ "${is_found_local}" -eq 0 ]]; then + exec 200<> "${REPORTED_IP_LIST_FILE}" # Open with read/write access + flock -x 200 # Lock + echo "IP=${IP} L=${BANTIME}" >> "${REPORTED_IP_LIST_FILE}" # Write + flock -u 200 # Release the lock + exec 200>&- # Close the file descriptor + fi + + # Report IP + report_ip_to_abuseipdb +fi From b423631825a2eb36ff8f10dcc4d2b0f806cfd5f1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Hasan=20=C3=87ALI=C5=9EIR?= Date: Sat, 22 Feb 2025 22:27:40 +0300 Subject: [PATCH 2/9] fail2ban: Update AbuseIPDB actionban script instructions - Clarified integration steps for placing the script in 'action.d' - Updated example configuration with correct script name 'fail2ban-abuseipdb.sh' - Emphasized the need to set 'abuseipdb_apikey' in the config file - Improved formatting and readability for better user understanding No functional changes, just documentation updates. --- files/fail2ban-abuseipdb.sh | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/files/fail2ban-abuseipdb.sh b/files/fail2ban-abuseipdb.sh index 6dfc865d..991505b1 100644 --- a/files/fail2ban-abuseipdb.sh +++ b/files/fail2ban-abuseipdb.sh @@ -8,13 +8,13 @@ # been reported previously, thereby avoiding redundant entries and potential API rate limiting. # # Integration with Fail2Ban: -# - Place this script in the Fail2Ban `action.d` directory named 'abuseipdb_fail2ban_actionban.sh' -# - Configure Fail2Ban to use this script as the `actionban` for relevant jails. +# - Place this script in the 'action.d' directory of Fail2Ban. +# - Configure Fail2Ban to use this script as the `actionban`. # - First edit 'abuseipdb.conf' in '/etc/fail2ban/action.d/abuseipdb.conf' and add following rule, -# - actionban = /etc/fail2ban/action.d/abuseipdb_fail2ban_actionban.sh \ +# - actionban = /etc/fail2ban/action.d/fail2ban-abuseipdb.sh \ # "" "" "" "" "" # - Also make sure you set your 'abuseipdb_apikey' in the configuration file. -# - Final step, adjust your jails accordingly. Check the below jail example to also reporting with custom comment via 'tp_comment' +# - Adjust your jails accordingly. Check the below jail example to also reporting with custom comment via 'tp_comment' # # Example jail in 'jail.local': # [nginx-botsearch] From 2962bb0e56253ca7097b0b9ef7545b5b6ba5fa6b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Hasan=20=C3=87ALI=C5=9EIR?= Date: Wed, 26 Feb 2025 12:47:31 +0300 Subject: [PATCH 3/9] Fail2Ban AbuseIPDB: Override main config with enhancements MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Added an override configuration to enhance Fail2Ban’s AbuseIPDB integration. - Introduced a local banned IP list for better isolation from Fail2Ban. - Optimized API calls (`/v2/check` → `/v2/report`) to reduce redundant reports. - Ensured `norestored=1` handling to prevent re-reporting after restarts. - Improved logging and added custom comments to avoid sensitive data exposure. This override provides more control, efficiency, and security while maintaining compatibility with the main configuration. --- files/abuseipdb.local | 63 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 63 insertions(+) create mode 100644 files/abuseipdb.local diff --git a/files/abuseipdb.local b/files/abuseipdb.local new file mode 100644 index 00000000..b698afdc --- /dev/null +++ b/files/abuseipdb.local @@ -0,0 +1,63 @@ +# Fail2Ban AbuseIPDB Integration (Enhanced) +# +# Author: Hasan CALISIR +# GitHub: https://github.com/hsntgm +# +# Description: +# This configuration enhances Fail2Ban's integration with AbuseIPDB, +# providing users with improved control, flexibility, and security when reporting abusive IPs. +# +# Key Enhancements: +# - Implements a **local banned IP list** to ensure **complete isolation** from Fail2Ban, +# enabling the script to manage and track IP bans without relying solely on Fail2Ban's internal ban management. +# - Performs **two API calls**: +# 1. `/v2/check`: Verifies if the IP is already reported to AbuseIPDB. +# 2. `/v2/report`: Reports the IP to AbuseIPDB if necessary, ensuring efficient use of API calls. +# - Supports **Fail2Ban's `norestored=1` feature** to prevent redundant reports on Fail2Ban restart. +# This feature ensures that once an IP is reported, it is not reported again upon Fail2Ban restart. +# - **Prevents redundant reporting** by checking the local list before making a report to AbuseIPDB. +# - Provides **custom comments** for IP reports, helping to avoid the leakage of sensitive information. +# + +[Definition] +# Option: norestored +###################### +# Notes.: Ensure norestored is set to 0 +# We control this at the script level to provide users with more control over how restored tickets are handled. +# We will also be able to log all triggered events by norestored +# Do not modify this value directly. Instead, adjust 'BYPASS_FAIL2BAN' below as needed. +# norestored = 0 + + +# Option: User defined settings +###################### +# Notes.: * Path to the main local banned IP list used by the action script. Not logrotate your main IP list log. +# * Path to the log file where actions and events are recorded by the action script +# * Rely on Fail2Ban for restarts (0) or completely isolate it by bypassing Fail2Ban (1) +# ! Bypassing Fail2Ban on restarts (BYPASS_FAIL2BAN = 1) can overhelm your server and AbuseIPDB API on restarts +# ! Use this option if you want to completely isolate from Fail2Ban and rely solely on the local banned IP list for reporting. + +# BANNED_IP_LIST = "/var/log/abuseipdb/abuseipdb-banned.log" +# LOG_FILE = "/var/log/abuseipdb/abuseipdb.log" +# BYPASS_FAIL2BAN = 0 + + +# Option: actionstart +###################### +# Notes.: DO NOT MODIFY, JUST UNCOMMENT +# actionstart = nohup /etc/fail2ban/action.d/fail2ban_abuseipdb.sh \ +# "--actionstart" "" "" & + + +# Option: actionban +###################### +# Notes.: DO NOT MODIFY, JUST UNCOMMENT +# actionban = /etc/fail2ban/action.d/fail2ban_abuseipdb.sh \ +# "" "" "" "" "" "" "" "" "" + + +[Init] +# Option: abuseipdb_apikey +###################### +# Notes Set your API key and COMMENT OUT +# abuseipdb_apikey = From 63788d02d648a42a7afbfa7317e54b907d3e3435 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Hasan=20=C3=87ALI=C5=9EIR?= Date: Wed, 26 Feb 2025 13:01:31 +0300 Subject: [PATCH 4/9] re-organized script for both actionstart & actionban - Reorganized script to be used by both 'actionstart' and 'actionban' in 'abuseipdb.local' - Isolated heavy 'actionstart' tasks using nohup to prevent latency - Removed redundant API checks to improve performance and reduce overhead - Implemented a lock mechanism to prevent 'actionban' execution if 'actionstart' fails - Ensured 'actionban' does not run at runtime due to missing dependencies or permission issues --- files/fail2ban-abuseipdb.sh | 294 ++++++++++++++++++++---------------- 1 file changed, 160 insertions(+), 134 deletions(-) diff --git a/files/fail2ban-abuseipdb.sh b/files/fail2ban-abuseipdb.sh index 991505b1..ba4caffb 100644 --- a/files/fail2ban-abuseipdb.sh +++ b/files/fail2ban-abuseipdb.sh @@ -1,107 +1,92 @@ #!/usr/bin/env bash # # Description: -# This script serves as a Fail2Ban `actionban` to report offending IP addresses to AbuseIPDB with custom COMMENTS. -# Mainly whenever Fail2Ban restarts, it calls the actionban function for each IP stored in the database file. -# If you restart your server often this script prevents duplicate reporting to AbuseIPDB by maintaining a local list of already reported IPs. -# Before reporting, it checks both the local list and AbuseIPDB to ensure the IP hasn't -# been reported previously, thereby avoiding redundant entries and potential API rate limiting. +# This script acts as a Fail2Ban `actionstart|actionban` to report offending IPs to AbuseIPDB. +# It allows for 'custom comments' to prevent leaking sensitive information. The main goal is to +# avoid relying on Fail2Ban and instead use a local banned IP list for complete isolation. +# It can also be used with Fail2Ban's `norestored=1` feature to rely on Fail2Ban for preventing +# redundant report actions on restarts. Users can toggle this behavior as needed. +# +# The script performs two API calls for each ban action: +# 1. **/v2/check** - Checks if the IP has already been reported. +# 2. **/v2/report** - Reports the IP if necessary and updates the local banned IP list. +# These two endpoints have separate daily limits, so they do not impact your reporting quota. +# +# To view any failures, check `/var/log/abuseipdb/abuseipdb.log`. # # Integration with Fail2Ban: -# - Place this script in the 'action.d' directory of Fail2Ban. -# - Configure Fail2Ban to use this script as the `actionban`. -# - First edit 'abuseipdb.conf' in '/etc/fail2ban/action.d/abuseipdb.conf' and add following rule, -# - actionban = /etc/fail2ban/action.d/fail2ban-abuseipdb.sh \ -# "" "" "" "" "" -# - Also make sure you set your 'abuseipdb_apikey' in the configuration file. -# - Adjust your jails accordingly. Check the below jail example to also reporting with custom comment via 'tp_comment' +# 1. Edit only 'abuseipdb.local' in 'action.d/abuseipdb.local' and uncomment pre-configured settings. +# 2. Adjust your jails to prevent leaking sensitive information in custom comments via 'tp_comment'. # -# Example jail in 'jail.local': -# [nginx-botsearch] -# enabled = true -# logpath = /var/log/nginx/*.log -# port = http,https -# backend = polling -# tp_comment = Fail2Ban - NGINX bad requests 400-401-403-404-444, high level vulnerability scanning, commonly xmlrpc_attack, wp-login brute force, excessive crawling/scraping -# maxretry = 3 -# findtime = 1d -# bantime = 7200 -# action = %(action_mwl)s -# %(action_abuseipdb)s[matches="%(tp_comment)s", abuseipdb_apikey="YOUR_API_KEY", abuseipdb_category="21,15", bantime="%(bantime)s"] -# -# Start from scratch practise: -# - Stop Fail2ban -# - Clear all existing firewall reject rules from iptables -# - Delete Fail2ban SQLite database and truncate fail2ban.log -# - Force logrotate -# - Follow above implementation steps and adjust all your jails accordingly -# - Start Fail2ban +# Example 'jail' configuration in 'jail.local' to prevent leaking sensitive information in AbuseIPDB reports: +# [nginx-botsearch] +# enabled = true +# logpath = /var/log/nginx/*.log +# port = http,https +# backend = polling +# tp_comment = Fail2Ban - NGINX bad requests 400-401-403-404-444, high level vulnerability scanning +# maxretry = 3 +# findtime = 1d +# bantime = 7200 +# action = %(action_mwl)s +# %(action_abuseipdb)s[matches="%(tp_comment)s", abuseipdb_apikey="YOUR_API_KEY", abuseipdb_category="21,15", bantime="%(bantime)s"] # # Usage: -# This script is intended to be called by Fail2Ban automatically (actionban) and accepts the following arguments: -# 1. - Your AbuseIPDB API key. -# 2. - A comment describing the reason for reporting the IP. -# 3. - The IP address to report. -# 4. - Comma-separated list of abuse categories as per AbuseIPDB's API. -# 5. - Duration for which the IP should be banned (e.g., 600 for 10 minutes). -# +# This script is designed to be triggered automatically by Fail2Ban (`actionstart|actionban`). # Manual Usage: -# For testing purpose before production; -# /etc/fail2ban/action.d/abuseipdb_fail2ban_actionban.sh "your_api_key" "Failed SSH login attempts" "192.0.2.1" "18" "600" +# - For testing purpose before production; +# /etc/fail2ban/action.d/fail2ban_abuseipdb.sh "your_api_key" "Failed SSH login attempts" "192.0.2.1" "18" "600" # # Arguments: -# APIKEY - Required. -# COMMENT - Required. -# IP - Required. -# CATEGORIES - Required. -# BANTIME - Required. -# -# Configurations (Need to set): -# - REPORTED_IP_LIST_FILE: Path to the local file storing reported IPs and their ban times. -# - LOG_FILE: Path to the log file where actions and events are recorded. +# $1 APIKEY - Required (Core). Retrieved automatically from the Fail2Ban 'jail'. | Your AbuseIPDB API key. +# $2 COMMENT - Required (Core). Retrieved automatically from the Fail2Ban 'jail'. | A custom comment to prevent the leakage of sensitive data when reporting +# $3 IP - Required (Core). Retrieved automatically from the Fail2Ban 'jail'. | The IP address to report. +# $4 CATEGORIES - Required (Core). Retrieved automatically from the Fail2Ban 'jail'. | Abuse categories as per AbuseIPDB's API +# $5 BANTIME - Required (Core). Retrieved automatically from the Fail2Ban 'jail'. | Ban duration +# $6 RESTORED - Required (Core). Retrieved automatically from the Fail2Ban '' | Status of restored tickets +# $7 BYPASS_FAIL2BAN - Required (User defined). Must be defined in 'action.d/abuseipdb.local'. | Bypassing Fail2Ban on restarts +# $8 LOCAL_LIST - Required (User defined). Must be defined in 'action.d/abuseipdb.local'. | Path to the main banned IP list used by the script +# $9 LOG_FILE - Required (User defined). Must be defined in 'action.d/abuseipdb.local'. | Path to the log file where actions and events are recorded by the script # # Dependencies: -# - curl: For making API requests to AbuseIPDB. -# - jq: For parsing JSON responses. -# - flock: Prevent data corruption. -# -# General Considerations: -# - The script does not interact with or rely on Fail2Ban’s SQLite database, as the Fail2ban database setup can vary across different environments. -# - The script performs two API calls to AbuseIPDB for each ban action in order to determine whether the IP should be reported. -# - First Call: /v2/check → Checks if the IP is already reported. -# - Second Call: /v2/report → Reports the IP if necessary. -# - These two endpoints have separate daily limits, so does not affect your reporting quota. -# - When multiple instances of the script try to write to the REPORTED_IP_LIST_FILE concurrently we need to prevent data corruption. 'flock' used for that reason. -# - Never delete, truncate or try to sync REPORTED_IP_LIST_FILE with Fail2ban SQLite. -# - Even you reset fail2ban SQLite, continue to keep REPORTED_IP_LIST_FILE same, It serves as the script’s local tracking system. -# - When you go production keep watching '/var/log/abuseipdb-report.log' for any abnormal fails. +# curl: For making API requests to AbuseIPDB. +# jq: For parsing JSON responses. +# flock: Prevent data corruption. # # Return Codes: -# 0 - IP is reported -# 1 - IP is not reported +# 0 - 'AbuseIPDB' IP is reported. +# 1 - 'AbuseIPDB' IP is not reported. # # Exit Codes: -# 1 - API-related failure +# 0 - 'norestored' restored tickets enabled. +# 0 - 'actionstart' tasks completed. +# 1 - 'actionstart' tasks cannot completed and lock file created. +# 1 - 'AbuseIPDB' API-related failure. # # Author: -# Hasan ÇALIŞIR - PSAUXIT +# Hasan ÇALIŞIR # hasan.calisir@psauxit.com -# https://www.psauxit.com/ +# https://github.com/hsntgm -# Arguments +# This script is used for both: 'actionstart' and 'actionban' in 'action.d/abuseipdb.local' +# It dynamically assigns arguments based on the action type +# and provides default values for missing user settings to prevent failures. APIKEY="$1" COMMENT="$2" IP="$3" CATEGORIES="$4" BANTIME="$5" - -# Configuration -REPORTED_IP_LIST_FILE="/etc/fail2ban/action.d/abuseipdb-reported-ip-list" -LOG_FILE="/var/log/abuseipdb-report.log" - -# Set defaults -is_found_local=0 -shouldBanIP=1 +RESTORED="${6}" +BYPASS_FAIL2BAN="${7:-0}" +if [[ "$1" == "--actionstart" ]]; then + # When triggered by 'actionstart' + REPORTED_IP_LIST_FILE="${2:-/var/log/abuseipdb/abuseipdb-banned.log}" + LOG_FILE="${3:-/var/log/abuseipdb/abuseipdb.log}" +else + # When triggered by 'actionban' + REPORTED_IP_LIST_FILE="${8:-/var/log/abuseipdb/abuseipdb-banned.log}" + LOG_FILE="${9:-/var/log/abuseipdb/abuseipdb.log}" +fi # Log messages log_message() { @@ -109,40 +94,93 @@ log_message() { echo "$(date +"%Y-%m-%d %H:%M:%S") - ${message}" >> "${LOG_FILE}" } -# Arguments validation -if [[ -z "$1" || -z "$2" || -z "$3" || -z "$4" || -z "$5" ]]; then - log_message "FATAL: Usage: $0 " - exit 1 +# Define lock file +LOCK_FILE="/tmp/abuseipdb_actionstart.lock" + +# Function to remove lock file if it exists +remove_lock() { + if [[ -f "${LOCK_FILE}" ]]; then + rm -f "${LOCK_FILE:?}" + fi +} + +# Function to create lock file +create_lock() { + if [[ ! -f "${LOCK_FILE}" ]]; then + touch "${LOCK_FILE}" + fi +} + +# Check if the script was triggered by 'actionstart' early in execution. +# This ensures necessary checks is performed before proceeding only once. +# This check runs on background always with 'nohup' to prevent latency. +# We listen exit codes carefully to allow or not further runtime 'actionban' events +if [[ "$1" == "--actionstart" ]]; then + # Trap exit signal to create/remove lock file based on exit status + trap 'if [[ $? -ne 0 ]]; then create_lock; else remove_lock; fi' EXIT + + # Ensure the directory for the reported IP list exists + LOG_DIR=$(dirname "${REPORTED_IP_LIST_FILE}") + if [[ ! -d "${LOG_DIR}" ]]; then + mkdir -p "${LOG_DIR}" || exit 1 + fi + + # Ensure the reported IP list and log file exist + for file in "${REPORTED_IP_LIST_FILE}" "${LOG_FILE}"; do + if [[ ! -f "${file}" ]]; then + touch "${file}" || exit 1 + fi + done + + # Check runtime dependencies + dependencies=("curl" "jq" "flock") + for dep in "${dependencies[@]}"; do + if ! command -v "${dep}" &>/dev/null; then + log_message "FATAL: -${dep} is not installed. Please install -${dep} to proceed." + exit 1 + fi + done + + # Tasks completed, quit nicely + exit 0 fi -# Ensure the directory for the reported IP list exists -REPORTED_IP_DIR=$(dirname "${REPORTED_IP_LIST_FILE}") -if [[ ! -d "${REPORTED_IP_DIR}" ]]; then - mkdir -p "${REPORTED_IP_DIR}" || { log_message "FATAL: Could not create directory ${REPORTED_IP_DIR}"; exit 1; } -fi - -# Ensure the reported IP list file exists and is writable -if [[ ! -f "${REPORTED_IP_LIST_FILE}" ]]; then - touch "${REPORTED_IP_LIST_FILE}" || { log_message "FATAL: Could not create file ${REPORTED_IP_LIST_FILE}"; exit 1; } -fi - -if [[ ! -w "${REPORTED_IP_LIST_FILE}" ]]; then - log_message "FATAL: File ${REPORTED_IP_LIST_FILE} is not writable." - exit 1 -fi - -# Check runtime dependencies -dependencies=("curl" "jq" "flock") -for dep in "${dependencies[@]}"; do - if ! command -v "${dep}" &>/dev/null; then - log_message "FATAL: -${dep} is not installed. Please install -${dep} to proceed." +# If the 'actionstart' failed, prevent 'actionban'. +# This stops 'actionban' from being triggered during runtime due to missing dependencies or permission issues. +# A failed initial 'actionstart' check indicates a failure to report to AbuseIPDB. +if [[ -f "${LOCK_FILE}" ]]; then + if [[ -f "${LOG_FILE}" ]]; then + log_message "FATAL: Failed due to a permission issue or missing dependency. Reporting to AbuseIPDB failed." + exit 1 + else exit 1 fi -done +fi -# Check if the IP is listed on AbuseIPDB +# If 'BYPASS_FAIL2BAN' is disabled, Fail2Ban will be relied upon during restarts. +# This prevents duplicate reports when Fail2Ban is restarted. +# This setting is 'OPTIONAL' and can be overridden in 'action.d/abuseipdb.local'. +# If enabled, Fail2Ban is bypassed completely, +# and script takes full control to determine which IP to report based on +# the local banned IP list even on Fail2Ban restarts. +if [[ "${BYPASS_FAIL2BAN}" -eq 0 ]]; then + if [[ "${RESTORED}" -eq 1 ]]; then + log_message "INFO NORESTORED: IP ${IP} has already been reported. No duplicate report made after restart." + exit 0 + fi +fi + +# Validate core arguments: Ensure all required core args are provided. +# These values are expected to be passed by Fail2Ban 'jail' during execution. +# Also for manual testing purpose before production. +if [[ -z "$1" || -z "$2" || -z "$3" || -z "$4" || -z "$5" ]]; then + log_message "FATAL: Missing core argument" + exit 1 +fi + +# Function to check if the IP is listed on AbuseIPDB check_ip_in_abuseipdb() { - local response http_status body total_reports error_detail + local response http_status body total_reports local delimiter="HTTP_STATUS:" # Perform the API call and capture both response and HTTP status @@ -160,35 +198,19 @@ check_ip_in_abuseipdb() { http_status=$(echo "${response}" | tr -d '\n' | sed -e "s/.*${delimiter}//") body=$(echo "${response}" | sed -e "s/${delimiter}[0-9]*//") - # Check if the response body is empty - if [[ -z "${body}" ]]; then - log_message "ERROR CHECK: API response empty." - exit 1 - fi - - # Validate that the response is valid JSON - if ! echo "${body}" | jq . >/dev/null 2>&1; then - log_message "ERROR CHECK: API response malformed. Response: ${body}" - exit 1 - fi - # Handle different HTTP status codes if [[ "${http_status}" =~ ^[0-9]+$ ]]; then - if [[ "${http_status}" -ge 200 && "${http_status}" -lt 300 ]]; then - # Successful HTTP response; check for API-level errors - if echo "${body}" | jq -e '.errors | length > 0' >/dev/null 2>&1; then - error_detail=$(echo "${body}" | jq -r '.errors[].detail') - log_message "ERROR CHECK: API returned errors. Detail: ${error_detail}" - exit 1 - fi - else - # Non-successful HTTP status - log_message "ERROR CHECK: API returned ${http_status}. Response: ${body}" + # Handle rate-limiting (HTTP 429) + if [[ "${http_status}" -eq 429 ]]; then + log_message "ERROR CHECK: API returned HTTP 429 (Too Many Requests). Response: ${body}" + exit 1 + fi + + # Handle other non-200 responses + if [[ "${http_status}" -ne 200 ]]; then + log_message "ERROR CHECK: API returned HTTP status ${http_status}. Response: ${body}" exit 1 fi - else - log_message "ERROR CHECK: HTTP status '${http_status}' is not numeric." - exit 1 fi # Extract totalReports @@ -202,7 +224,7 @@ check_ip_in_abuseipdb() { fi } -# Report to AbuseIpDB +# Function to report AbuseIpDB report_ip_to_abuseipdb() { local response response=$(curl --fail -s 'https://api.abuseipdb.com/api/v2/report' \ @@ -221,6 +243,11 @@ report_ip_to_abuseipdb() { fi } + +# Set defaults +is_found_local=0 +shouldBanIP=1 + # Should Ban IP if grep -m 1 -q -E "^IP=${IP}[[:space:]]+L=[0-9\-]+" "${REPORTED_IP_LIST_FILE}"; then # IP found locally, check if it's still listed on AbuseIPDB @@ -235,11 +262,10 @@ if grep -m 1 -q -E "^IP=${IP}[[:space:]]+L=[0-9\-]+" "${REPORTED_IP_LIST_FILE}"; is_found_local=1 fi else - # New comer, welcome to hell shouldBanIP=1 fi -# Report to AbuseIpdb +# Let's report to AbuseIpdb if [[ "${shouldBanIP}" -eq 1 ]]; then # Add the new ban entry to local list kindly if [[ "${is_found_local}" -eq 0 ]]; then From d13660c588f032a0401b10ed914e17b5459b3dd3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Hasan=20=C3=87ALI=C5=9EIR?= Date: Mon, 3 Mar 2025 18:13:45 +0300 Subject: [PATCH 5/9] migrate banned IPs to SQLite DB and prevent actionban latency - Replace local file storage with AbuseIPDB SQLite database. - Offload heavy tasks to background to avoid latency during concurrent actionban calls. - Add global lock to ensure actionstart runs only once across all jails. --- files/fail2ban-abuseipdb.sh | 377 +++++++++++++++++++++++------------- 1 file changed, 237 insertions(+), 140 deletions(-) diff --git a/files/fail2ban-abuseipdb.sh b/files/fail2ban-abuseipdb.sh index ba4caffb..ca822bd6 100644 --- a/files/fail2ban-abuseipdb.sh +++ b/files/fail2ban-abuseipdb.sh @@ -3,9 +3,9 @@ # Description: # This script acts as a Fail2Ban `actionstart|actionban` to report offending IPs to AbuseIPDB. # It allows for 'custom comments' to prevent leaking sensitive information. The main goal is to -# avoid relying on Fail2Ban and instead use a local banned IP list for complete isolation. +# avoid relying on Fail2Ban and instead use a separate AbuseIPDB SQLite database for complete isolation. # It can also be used with Fail2Ban's `norestored=1` feature to rely on Fail2Ban for preventing -# redundant report actions on restarts. Users can toggle this behavior as needed. +# redundant reporting on restarts. Users can toggle this behavior as needed. # # The script performs two API calls for each ban action: # 1. **/v2/check** - Checks if the IP has already been reported. @@ -33,7 +33,7 @@ # # Usage: # This script is designed to be triggered automatically by Fail2Ban (`actionstart|actionban`). -# Manual Usage: +# For testing (manual execution): # - For testing purpose before production; # /etc/fail2ban/action.d/fail2ban_abuseipdb.sh "your_api_key" "Failed SSH login attempts" "192.0.2.1" "18" "600" # @@ -45,46 +45,34 @@ # $5 BANTIME - Required (Core). Retrieved automatically from the Fail2Ban 'jail'. | Ban duration # $6 RESTORED - Required (Core). Retrieved automatically from the Fail2Ban '' | Status of restored tickets # $7 BYPASS_FAIL2BAN - Required (User defined). Must be defined in 'action.d/abuseipdb.local'. | Bypassing Fail2Ban on restarts -# $8 LOCAL_LIST - Required (User defined). Must be defined in 'action.d/abuseipdb.local'. | Path to the main banned IP list used by the script -# $9 LOG_FILE - Required (User defined). Must be defined in 'action.d/abuseipdb.local'. | Path to the log file where actions and events are recorded by the script +# $2|$8 SQLITE_DB - Required (User defined). Must be defined in 'action.d/abuseipdb.local'. | Path to the main AbuseIPDB SQLite database +# $3|$9 LOG_FILE - Required (User defined). Must be defined in 'action.d/abuseipdb.local'. | Path to the log file where actions and events are recorded by the script # # Dependencies: # curl: For making API requests to AbuseIPDB. # jq: For parsing JSON responses. -# flock: Prevent data corruption. -# -# Return Codes: -# 0 - 'AbuseIPDB' IP is reported. -# 1 - 'AbuseIPDB' IP is not reported. -# -# Exit Codes: -# 0 - 'norestored' restored tickets enabled. -# 0 - 'actionstart' tasks completed. -# 1 - 'actionstart' tasks cannot completed and lock file created. -# 1 - 'AbuseIPDB' API-related failure. +# sqlite3: Local AbuseIPDB db. # # Author: # Hasan ÇALIŞIR -# hasan.calisir@psauxit.com # https://github.com/hsntgm -# This script is used for both: 'actionstart' and 'actionban' in 'action.d/abuseipdb.local' -# It dynamically assigns arguments based on the action type -# and provides default values for missing user settings to prevent failures. +####################################### +# HELPERS: (START) +####################################### + APIKEY="$1" COMMENT="$2" IP="$3" CATEGORIES="$4" BANTIME="$5" -RESTORED="${6}" +RESTORED="$6" BYPASS_FAIL2BAN="${7:-0}" if [[ "$1" == "--actionstart" ]]; then - # When triggered by 'actionstart' - REPORTED_IP_LIST_FILE="${2:-/var/log/abuseipdb/abuseipdb-banned.log}" + SQLITE_DB="${2:-/var/lib/fail2ban/abuseipdb/fail2ban_abuseipdb}" LOG_FILE="${3:-/var/log/abuseipdb/abuseipdb.log}" else - # When triggered by 'actionban' - REPORTED_IP_LIST_FILE="${8:-/var/log/abuseipdb/abuseipdb-banned.log}" + SQLITE_DB="${8:-/var/lib/fail2ban/abuseipdb/fail2ban_abuseipdb}" LOG_FILE="${9:-/var/log/abuseipdb/abuseipdb.log}" fi @@ -94,137 +82,206 @@ log_message() { echo "$(date +"%Y-%m-%d %H:%M:%S") - ${message}" >> "${LOG_FILE}" } -# Define lock file -LOCK_FILE="/tmp/abuseipdb_actionstart.lock" +# Lock files for 'actionstart' status +LOCK_BAN="/tmp/abuseipdb_actionstart.lock" +LOCK_DONE="/tmp/abuseipdb_actionstart.done" -# Function to remove lock file if it exists +# Remove lock file remove_lock() { - if [[ -f "${LOCK_FILE}" ]]; then - rm -f "${LOCK_FILE:?}" - fi + [[ -f "${LOCK_BAN}" ]] && rm -f "${LOCK_BAN}" } -# Function to create lock file +# Create lock file create_lock() { - if [[ ! -f "${LOCK_FILE}" ]]; then - touch "${LOCK_FILE}" - fi + [[ ! -f "${LOCK_BAN}" ]] && touch "${LOCK_BAN}" } -# Check if the script was triggered by 'actionstart' early in execution. -# This ensures necessary checks is performed before proceeding only once. -# This check runs on background always with 'nohup' to prevent latency. -# We listen exit codes carefully to allow or not further runtime 'actionban' events +# Pre-defined SQLite PRAGMAS +SQLITE_PRAGMAS=" + PRAGMA journal_mode=WAL; + PRAGMA synchronous=NORMAL; + PRAGMA temp_store=MEMORY; + PRAGMA locking_mode=NORMAL; + PRAGMA cache_size=-256000; + PRAGMA busy_timeout=10000; +" + +####################################### +# HELPERS: (END) +####################################### + +####################################### +# ACTIONSTART: (START) +####################################### + +######################################## +# Triggered by 'actionstart' +# to perform necessary checks +# and AbuseIPDB SQLite initialization. +# +# - Ensures required checks are done. +# - Runs in the background with 'nohup' +# on initial start to prevent latency. +# - Listens for exit codes to control +# further 'actionban' events via the +# 'lock' mechanism. +# - Check 'abuseipdb.local' for +# integration details. +######################################## + if [[ "$1" == "--actionstart" ]]; then - # Trap exit signal to create/remove lock file based on exit status - trap 'if [[ $? -ne 0 ]]; then create_lock; else remove_lock; fi' EXIT + if [[ ! -f "${LOCK_DONE}" ]]; then + trap 'if [[ $? -ne 0 ]]; then create_lock; else remove_lock; fi' EXIT - # Ensure the directory for the reported IP list exists - LOG_DIR=$(dirname "${REPORTED_IP_LIST_FILE}") - if [[ ! -d "${LOG_DIR}" ]]; then - mkdir -p "${LOG_DIR}" || exit 1 - fi - - # Ensure the reported IP list and log file exist - for file in "${REPORTED_IP_LIST_FILE}" "${LOG_FILE}"; do - if [[ ! -f "${file}" ]]; then - touch "${file}" || exit 1 + SQLITE_DIR=$(dirname "${SQLITE_DB}") + if [[ ! -d "${SQLITE_DIR}" ]]; then + mkdir -p "${SQLITE_DIR}" || exit 1 fi - done - # Check runtime dependencies - dependencies=("curl" "jq" "flock") - for dep in "${dependencies[@]}"; do - if ! command -v "${dep}" &>/dev/null; then - log_message "FATAL: -${dep} is not installed. Please install -${dep} to proceed." + if [[ ! -f "${LOG_FILE}" ]]; then + touch "${LOG_FILE}" || exit 1 + fi + + for dep in curl jq sqlite3; do + if ! command -v "${dep}" &>/dev/null; then + log_message "ERROR: ${dep} is not installed. Please install ${dep}" + exit 1 + fi + done + + if [[ ! -f "${SQLITE_DB}" ]]; then + sqlite3 "${SQLITE_DB}" " + ${SQLITE_PRAGMAS} + CREATE TABLE IF NOT EXISTS banned_ips (ip TEXT PRIMARY KEY, bantime INTEGER); + CREATE INDEX IF NOT EXISTS idx_ip ON banned_ips(ip); + " + fi + + result=$(sqlite3 "file:${SQLITE_DB}?mode=ro" "SELECT name FROM sqlite_master WHERE type='table' AND name='banned_ips';") + if ! [[ -n "${result}" ]]; then + log_message "ERROR: AbuseIPDB database initialization failed." + rm -f "${SQLITE_DB:?}" exit 1 + else + log_message "SUCCESS: The AbuseIPDB database is now ready for connection." fi - done - # Tasks completed, quit nicely - exit 0 -fi - -# If the 'actionstart' failed, prevent 'actionban'. -# This stops 'actionban' from being triggered during runtime due to missing dependencies or permission issues. -# A failed initial 'actionstart' check indicates a failure to report to AbuseIPDB. -if [[ -f "${LOCK_FILE}" ]]; then - if [[ -f "${LOG_FILE}" ]]; then - log_message "FATAL: Failed due to a permission issue or missing dependency. Reporting to AbuseIPDB failed." - exit 1 + touch "${LOCK_DONE}" || exit 1 + exit 0 else - exit 1 - fi -fi - -# If 'BYPASS_FAIL2BAN' is disabled, Fail2Ban will be relied upon during restarts. -# This prevents duplicate reports when Fail2Ban is restarted. -# This setting is 'OPTIONAL' and can be overridden in 'action.d/abuseipdb.local'. -# If enabled, Fail2Ban is bypassed completely, -# and script takes full control to determine which IP to report based on -# the local banned IP list even on Fail2Ban restarts. -if [[ "${BYPASS_FAIL2BAN}" -eq 0 ]]; then - if [[ "${RESTORED}" -eq 1 ]]; then - log_message "INFO NORESTORED: IP ${IP} has already been reported. No duplicate report made after restart." exit 0 fi fi -# Validate core arguments: Ensure all required core args are provided. -# These values are expected to be passed by Fail2Ban 'jail' during execution. -# Also for manual testing purpose before production. -if [[ -z "$1" || -z "$2" || -z "$3" || -z "$4" || -z "$5" ]]; then - log_message "FATAL: Missing core argument" +####################################### +# ACTIONSTART: (END) +####################################### + +####################################### +# ACTIONBAN: (START) +####################################### + +####################################### +# 1) Prevent 'actionban' if +# 'actionstart' fails. +# +# If 'actionstart' fails, block +# 'actionban' to prevent issues from +# missing dependencies or permission +# errors. +####################################### + +####################################### +# 2) Fail2Ban restart handling & +# duplicate report prevention. +# +# - If 'BYPASS_FAIL2BAN' is disabled, +# Fail2Ban manages reports on restart +# and prevents duplicate submissions. +# - This setting can be overridden in +# 'action.d/abuseipdb.local'. +# - If enabled, Fail2Ban is bypassed, +# and the script independently +# decides which IPs to report based +# on the AbuseIPDB db, even after +# restarts. +####################################### + +####################################### +# 3) Core argument validation +# +# - Ensures all required arguments +# are provided. +# - Expected from Fail2Ban 'jail' or +# for manual testing before +# production deployment. +####################################### + +####################################### +# EARLY CHECKS: (START) +####################################### + +if [[ -f "${LOCK_BAN}" ]]; then + [[ -f "${LOG_FILE}" ]] && log_message "ERROR: Initialization failed! (actionstart). Reporting for IP ${IP} is blocked." exit 1 fi -# Function to check if the IP is listed on AbuseIPDB -check_ip_in_abuseipdb() { - local response http_status body total_reports - local delimiter="HTTP_STATUS:" +if [[ "${BYPASS_FAIL2BAN}" -eq 0 && "${RESTORED}" -eq 1 ]]; then + log_message "INFO: IP ${IP} already reported." + exit 0 +fi - # Perform the API call and capture both response and HTTP status +if [[ -z "${APIKEY}" || -z "${COMMENT}" || -z "${IP}" || -z "${CATEGORIES}" || -z "${BANTIME}" ]]; then + log_message "ERROR: Missing core argument(s)." + exit 1 +fi + +####################################### +# EARLY CHECKS: (END) +####################################### + +####################################### +# FUNCTIONS: (START) +####################################### + +check_ip_in_abuseipdb() { + local response http_status body total_reports delimiter="HTTP_STATUS:" response=$(curl -s -w "${delimiter}%{http_code}" -G "https://api.abuseipdb.com/api/v2/check" \ --data-urlencode "ipAddress=${IP}" \ -H "Key: ${APIKEY}" \ -H "Accept: application/json" 2>&1) if [[ $? -ne 0 ]]; then - log_message "ERROR CHECK: API failure. Response: ${response}" - exit 1 + log_message "ERROR: API failure. Response: ${response}" + return 1 fi - # Separate the HTTP status code from the response body http_status=$(echo "${response}" | tr -d '\n' | sed -e "s/.*${delimiter}//") body=$(echo "${response}" | sed -e "s/${delimiter}[0-9]*//") - # Handle different HTTP status codes if [[ "${http_status}" =~ ^[0-9]+$ ]]; then - # Handle rate-limiting (HTTP 429) if [[ "${http_status}" -eq 429 ]]; then - log_message "ERROR CHECK: API returned HTTP 429 (Too Many Requests). Response: ${body}" - exit 1 + log_message "ERROR: Rate limited (HTTP 429). Response: ${body}" + return 1 fi - # Handle other non-200 responses if [[ "${http_status}" -ne 200 ]]; then - log_message "ERROR CHECK: API returned HTTP status ${http_status}. Response: ${body}" - exit 1 + log_message "ERROR: HTTP ${http_status}. Response: ${body}" + return 1 fi + else + log_message "ERROR: API failure. Response: ${response}" + return 1 fi - # Extract totalReports total_reports=$(echo "${body}" | jq '.data.totalReports') - - # Finally, check the IP listed on AbuseIPDB if [[ "${total_reports}" -gt 0 ]]; then - return 0 # IP is reported + return 0 else - return 1 # IP is not reported + return 1 fi } -# Function to report AbuseIpDB report_ip_to_abuseipdb() { local response response=$(curl --fail -s 'https://api.abuseipdb.com/api/v2/report' \ @@ -234,48 +291,88 @@ report_ip_to_abuseipdb() { --data-urlencode "ip=${IP}" \ --data "categories=${CATEGORIES}" 2>&1) - # API call fail if [[ $? -ne 0 ]]; then - log_message "ERROR REPORT: API failure. Response: ${response}" - exit 1 + log_message "ERROR: API failure. Response: ${response} for IP: ${IP}" else - log_message "SUCCESS REPORT: Reported IP ${IP} to AbuseIPDB. Local list updated." + log_message "SUCCESS: Reported IP ${IP} to AbuseIPDB." fi } +check_ip_in_db() { + local ip=$1 result + result=$(sqlite3 "file:${SQLITE_DB}?mode=ro" " + ${SQLITE_PRAGMAS} + SELECT 1 FROM banned_ips WHERE ip = '${ip}' LIMIT 1;" + ) -# Set defaults -is_found_local=0 -shouldBanIP=1 + if [[ $? -ne 0 ]]; then + log_message "ERROR: AbuseIPDB database query failed while checking IP ${ip}. Response: ${result}" + return 1 + fi -# Should Ban IP -if grep -m 1 -q -E "^IP=${IP}[[:space:]]+L=[0-9\-]+" "${REPORTED_IP_LIST_FILE}"; then - # IP found locally, check if it's still listed on AbuseIPDB - if check_ip_in_abuseipdb; then - # IP is still listed on AbuseIPDB, no need to report again - log_message "INFO: IP ${IP} has already been reported and remains on AbuseIPDB. No duplicate report made." - shouldBanIP=0 + if [[ -n "${result}" ]]; then + return 0 else - # IP is reported before but not listed on AbuseIPDB, report it again - log_message "INFO: IP ${IP} has already been reported but is no longer listed on AbuseIPDB. Reporting it again." - shouldBanIP=1 - is_found_local=1 + return 1 fi -else +} + +insert_ip_to_db() { + local ip=$1 + local bantime=$2 + sqlite3 "${SQLITE_DB}" " + ${SQLITE_PRAGMAS} + BEGIN IMMEDIATE; + INSERT INTO banned_ips (ip, bantime) + VALUES ('${ip}', ${bantime}) + ON CONFLICT(ip) DO UPDATE SET bantime=${bantime}; + COMMIT; + " + + if [[ $? -ne 0 ]]; then + log_message "ERROR: Failed to insert or update IP ${ip} in the AbuseIPDB database." + fi +} + +####################################### +# FUNCTIONS: (END) +####################################### + +####################################### +# MAIN (START) +####################################### + +( + is_found_local=0 shouldBanIP=1 -fi -# Let's report to AbuseIpdb -if [[ "${shouldBanIP}" -eq 1 ]]; then - # Add the new ban entry to local list kindly - if [[ "${is_found_local}" -eq 0 ]]; then - exec 200<> "${REPORTED_IP_LIST_FILE}" # Open with read/write access - flock -x 200 # Lock - echo "IP=${IP} L=${BANTIME}" >> "${REPORTED_IP_LIST_FILE}" # Write - flock -u 200 # Release the lock - exec 200>&- # Close the file descriptor + if check_ip_in_db $IP; then + is_found_local=1 + if check_ip_in_abuseipdb; then + log_message "INFO: IP ${IP} has already been reported and remains on AbuseIPDB." + shouldBanIP=0 + else + log_message "INFO: IP ${IP} has already been reported but is no longer listed on AbuseIPDB." + shouldBanIP=1 + fi + else + shouldBanIP=1 fi - # Report IP - report_ip_to_abuseipdb -fi + if [[ "${shouldBanIP}" -eq 1 ]]; then + if [[ "${is_found_local}" -eq 0 ]]; then + insert_ip_to_db $IP $BANTIME + fi + report_ip_to_abuseipdb + fi +) >> "${LOG_FILE}" 2>&1 & + +####################################### +# MAIN (END) +####################################### + +####################################### +# ACTIONBAN: (END) +####################################### + +exit 0 From fa4ce4acbaf609f51d4cb90086d9d4a59fa8e63b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Hasan=20=C3=87ALI=C5=9EIR?= Date: Mon, 3 Mar 2025 18:28:46 +0300 Subject: [PATCH 6/9] update abuseipdb.local accordingly - Replace local file storage with AbuseIPDB SQLite database. - Add info about preventing leaking sensitive information on reports --- files/abuseipdb.local | 24 ++++++++++++++++++------ 1 file changed, 18 insertions(+), 6 deletions(-) diff --git a/files/abuseipdb.local b/files/abuseipdb.local index b698afdc..0c1b80c3 100644 --- a/files/abuseipdb.local +++ b/files/abuseipdb.local @@ -8,8 +8,8 @@ # providing users with improved control, flexibility, and security when reporting abusive IPs. # # Key Enhancements: -# - Implements a **local banned IP list** to ensure **complete isolation** from Fail2Ban, -# enabling the script to manage and track IP bans without relying solely on Fail2Ban's internal ban management. +# - Implements a **AbuseIPDB SQLite DB** to ensure **complete isolation** from Fail2Ban, +# enabling the script to manage and track IP bans without relying solely on Fail2Ban's internal DB and ban management. # - Performs **two API calls**: # 1. `/v2/check`: Verifies if the IP is already reported to AbuseIPDB. # 2. `/v2/report`: Reports the IP to AbuseIPDB if necessary, ensuring efficient use of API calls. @@ -18,6 +18,18 @@ # - **Prevents redundant reporting** by checking the local list before making a report to AbuseIPDB. # - Provides **custom comments** for IP reports, helping to avoid the leakage of sensitive information. # +# Example 'jail' configuration in 'jail.local' to prevent leaking sensitive information in AbuseIPDB reports: +# [nginx-botsearch] +# enabled = true +# logpath = /var/log/nginx/*.log +# port = http,https +# backend = polling +# tp_comment = Fail2Ban - NGINX bad requests 400-401-403-404-444, high level vulnerability scanning +# maxretry = 3 +# findtime = 1d +# bantime = 7200 +# action = %(action_mwl)s +# %(action_abuseipdb)s[matches="%(tp_comment)s", abuseipdb_apikey="YOUR_API_KEY", abuseipdb_category="21,15", bantime="%(bantime)s"] [Definition] # Option: norestored @@ -37,7 +49,7 @@ # ! Bypassing Fail2Ban on restarts (BYPASS_FAIL2BAN = 1) can overhelm your server and AbuseIPDB API on restarts # ! Use this option if you want to completely isolate from Fail2Ban and rely solely on the local banned IP list for reporting. -# BANNED_IP_LIST = "/var/log/abuseipdb/abuseipdb-banned.log" +# SQLITE_DB="/var/lib/fail2ban/abuseipdb/fail2ban_abuseipdb" # LOG_FILE = "/var/log/abuseipdb/abuseipdb.log" # BYPASS_FAIL2BAN = 0 @@ -46,18 +58,18 @@ ###################### # Notes.: DO NOT MODIFY, JUST UNCOMMENT # actionstart = nohup /etc/fail2ban/action.d/fail2ban_abuseipdb.sh \ -# "--actionstart" "" "" & +# "--actionstart" "" "" & # Option: actionban ###################### # Notes.: DO NOT MODIFY, JUST UNCOMMENT # actionban = /etc/fail2ban/action.d/fail2ban_abuseipdb.sh \ -# "" "" "" "" "" "" "" "" "" +# "" "" "" "" "" "" "" "" "" [Init] # Option: abuseipdb_apikey ###################### -# Notes Set your API key and COMMENT OUT +# Notes Set your API key and UNCOMMENT # abuseipdb_apikey = From 28c2d6685d2acd4cae4154d0e89856d70a32c527 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Hasan=20=C3=87ALI=C5=9EIR?= Date: Mon, 3 Mar 2025 21:21:44 +0300 Subject: [PATCH 7/9] fix script naming - rename fail2ban_abuseipdb.sh --> fail2ban-abuseipdb.sh - update descriptions --- files/abuseipdb.local | 45 +++++++++++++++++++++---------------------- 1 file changed, 22 insertions(+), 23 deletions(-) diff --git a/files/abuseipdb.local b/files/abuseipdb.local index 0c1b80c3..a5d9b653 100644 --- a/files/abuseipdb.local +++ b/files/abuseipdb.local @@ -4,19 +4,18 @@ # GitHub: https://github.com/hsntgm # # Description: -# This configuration enhances Fail2Ban's integration with AbuseIPDB, -# providing users with improved control, flexibility, and security when reporting abusive IPs. +# Enhanced AbuseIPDB integration for Fail2Ban with improved control, +# tracking, and isolation for IP abuse reporting. # -# Key Enhancements: -# - Implements a **AbuseIPDB SQLite DB** to ensure **complete isolation** from Fail2Ban, -# enabling the script to manage and track IP bans without relying solely on Fail2Ban's internal DB and ban management. -# - Performs **two API calls**: -# 1. `/v2/check`: Verifies if the IP is already reported to AbuseIPDB. -# 2. `/v2/report`: Reports the IP to AbuseIPDB if necessary, ensuring efficient use of API calls. -# - Supports **Fail2Ban's `norestored=1` feature** to prevent redundant reports on Fail2Ban restart. -# This feature ensures that once an IP is reported, it is not reported again upon Fail2Ban restart. -# - **Prevents redundant reporting** by checking the local list before making a report to AbuseIPDB. -# - Provides **custom comments** for IP reports, helping to avoid the leakage of sensitive information. +# Key Features: +# - Isolated AbuseIPDB SQLite database to track reports independently. +# - Dual API calls: /v2/check and /v2/report for efficient reporting. +# - Optimizes AbuseIPDB daily API usage by reducing unnecessary report calls +# - Supports norestored=1 to avoid duplicate reports on restart. +# - Maintains a separate, long-term persistent database of banned IPs to ensure accurate tracking +# and avoid reliance on Fail2Ban’s bantimes or ban state, even if Fail2Ban’s +# internal ban management becomes inconsistent over time. +# - Customizable report comments to avoid leaking sensitive info. # # Example 'jail' configuration in 'jail.local' to prevent leaking sensitive information in AbuseIPDB reports: # [nginx-botsearch] @@ -31,6 +30,7 @@ # action = %(action_mwl)s # %(action_abuseipdb)s[matches="%(tp_comment)s", abuseipdb_apikey="YOUR_API_KEY", abuseipdb_category="21,15", bantime="%(bantime)s"] + [Definition] # Option: norestored ###################### @@ -43,33 +43,32 @@ # Option: User defined settings ###################### -# Notes.: * Path to the main local banned IP list used by the action script. Not logrotate your main IP list log. -# * Path to the log file where actions and events are recorded by the action script +# Notes.: * Path to AbuseIPDB SQLite database used by the action script. +# * Path to the log file where actions and events are recorded by the action script. # * Rely on Fail2Ban for restarts (0) or completely isolate it by bypassing Fail2Ban (1) -# ! Bypassing Fail2Ban on restarts (BYPASS_FAIL2BAN = 1) can overhelm your server and AbuseIPDB API on restarts -# ! Use this option if you want to completely isolate from Fail2Ban and rely solely on the local banned IP list for reporting. - -# SQLITE_DB="/var/lib/fail2ban/abuseipdb/fail2ban_abuseipdb" +# ! Bypassing Fail2Ban on restarts (BYPASS_FAIL2BAN = 1) can overhelm your server and AbuseIPDB API on restarts. +# ! SET 1 if you want to completely isolate from Fail2Ban and rely solely on the AbuseIPDB SQLite database for reporting on restart. +# SQLITE_DB = "/var/lib/fail2ban/abuseipdb/fail2ban_abuseipdb" # LOG_FILE = "/var/log/abuseipdb/abuseipdb.log" # BYPASS_FAIL2BAN = 0 # Option: actionstart ###################### -# Notes.: DO NOT MODIFY, JUST UNCOMMENT -# actionstart = nohup /etc/fail2ban/action.d/fail2ban_abuseipdb.sh \ +# Notes.: Uncomment and leave as-is +# actionstart = nohup /etc/fail2ban/action.d/fail2ban-abuseipdb.sh \ # "--actionstart" "" "" & # Option: actionban ###################### -# Notes.: DO NOT MODIFY, JUST UNCOMMENT -# actionban = /etc/fail2ban/action.d/fail2ban_abuseipdb.sh \ +# Notes.: Uncomment and leave as-is +# actionban = /etc/fail2ban/action.d/fail2ban-abuseipdb.sh \ # "" "" "" "" "" "" "" "" "" [Init] # Option: abuseipdb_apikey ###################### -# Notes Set your API key and UNCOMMENT +# Notes Set your API key and uncomment # abuseipdb_apikey = From 00bb41b864709ae34e3511a5e6d1b6acb6a15e88 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Hasan=20=C3=87ALI=C5=9EIR?= Date: Wed, 5 Mar 2025 20:04:46 +0300 Subject: [PATCH 8/9] refactor AbuseIPDB integration with improved concurrency and error handling MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This refactors the AbuseIPDB integration for Fail2Ban with major improvements: - Introduced separate lock files (LOCK_INIT, LOCK_BAN, LOCK_DONE) to better handle concurrent initialization and prevent race conditions during restarts. - LOCK_BAN → serializes ban reports to the API (during actionban). - LOCK_DONE → can signal completion or be used for future synchronization (like restart-safe exits). - LOCK_INIT with flock in actionstart to prevent concurrent initialization, ensuring SQLite and log file integrity during parallel Fail2Ban restarts or multiple jail startups. - Enhanced argument validation for both actionstart and actionban to prevent silent failures. - Improved database initialization checks, ensuring proper creation of directories and log files. - Added persistent SQLite pragmas for performance optimization under concurrent access. - Refined error handling and logging for API interactions, including better detection of rate-limiting (HTTP 429) and invalid responses. - Implemented consistent whitespace trimming and sanitization on IP addresses and bantime inputs. - Improved modularity with dedicated helper functions, reducing code duplication and improving maintainability. - Ensured background execution with better log redirection and failure tracking. - Verify local DB insertions, aborting the process on failure to prevent incomplete or invalid state. - Roll back local DB entries if AbuseIPDB reporting fails, ensuring no orphaned records remain. - Replace basic info logs with clear status and error messages to improve traceability and debugging. - Maintain high integrity between the local database and AbuseIPDB by only proceeding when all previous steps succeed. - Shift from a "continue regardless" flow to a controlled stop on any critical error, ensuring system reliability. Previously, the script assumed success of key steps, risking stale database entries, silent API call failures, and duplicate reports after Fail2Ban restarts. These changes improve reliability, prevent data corruption under high concurrency, and ensure accurate synchronization between local db and AbuseIPDB API. --- files/fail2ban-abuseipdb.sh | 343 +++++++++++++++++++++++------------- 1 file changed, 221 insertions(+), 122 deletions(-) diff --git a/files/fail2ban-abuseipdb.sh b/files/fail2ban-abuseipdb.sh index ca822bd6..86bbb3c5 100644 --- a/files/fail2ban-abuseipdb.sh +++ b/files/fail2ban-abuseipdb.sh @@ -68,6 +68,7 @@ CATEGORIES="$4" BANTIME="$5" RESTORED="$6" BYPASS_FAIL2BAN="${7:-0}" + if [[ "$1" == "--actionstart" ]]; then SQLITE_DB="${2:-/var/lib/fail2ban/abuseipdb/fail2ban_abuseipdb}" LOG_FILE="${3:-/var/log/abuseipdb/abuseipdb.log}" @@ -76,35 +77,26 @@ else LOG_FILE="${9:-/var/log/abuseipdb/abuseipdb.log}" fi -# Log messages log_message() { local message="$1" echo "$(date +"%Y-%m-%d %H:%M:%S") - ${message}" >> "${LOG_FILE}" } -# Lock files for 'actionstart' status -LOCK_BAN="/tmp/abuseipdb_actionstart.lock" -LOCK_DONE="/tmp/abuseipdb_actionstart.done" +LOCK_INIT="/tmp/abuseipdb_actionstart_init.lock" +LOCK_BAN="/tmp/abuseipdb_actionstart_ban.lock" +LOCK_DONE="/tmp/abuseipdb_actionstart_done.lock" -# Remove lock file remove_lock() { [[ -f "${LOCK_BAN}" ]] && rm -f "${LOCK_BAN}" } -# Create lock file create_lock() { [[ ! -f "${LOCK_BAN}" ]] && touch "${LOCK_BAN}" } -# Pre-defined SQLite PRAGMAS -SQLITE_PRAGMAS=" - PRAGMA journal_mode=WAL; - PRAGMA synchronous=NORMAL; - PRAGMA temp_store=MEMORY; - PRAGMA locking_mode=NORMAL; - PRAGMA cache_size=-256000; - PRAGMA busy_timeout=10000; -" +SQLITE_NON_PERSISTENT_PRAGMAS="PRAGMA synchronous=NORMAL; \ +PRAGMA locking_mode=NORMAL; \ +PRAGMA busy_timeout=10000;" ####################################### # HELPERS: (END) @@ -124,53 +116,72 @@ SQLITE_PRAGMAS=" # on initial start to prevent latency. # - Listens for exit codes to control # further 'actionban' events via the -# 'lock' mechanism. -# - Check 'abuseipdb.local' for -# integration details. +# 'LOCK_BAN' mechanism. +# - Use 'LOCK_INIT' and 'LOCK_DONE' to +# manage concurrent calls on restarts. ######################################## if [[ "$1" == "--actionstart" ]]; then - if [[ ! -f "${LOCK_DONE}" ]]; then - trap 'if [[ $? -ne 0 ]]; then create_lock; else remove_lock; fi' EXIT - - SQLITE_DIR=$(dirname "${SQLITE_DB}") - if [[ ! -d "${SQLITE_DIR}" ]]; then - mkdir -p "${SQLITE_DIR}" || exit 1 - fi - - if [[ ! -f "${LOG_FILE}" ]]; then - touch "${LOG_FILE}" || exit 1 - fi - - for dep in curl jq sqlite3; do - if ! command -v "${dep}" &>/dev/null; then - log_message "ERROR: ${dep} is not installed. Please install ${dep}" - exit 1 - fi - done - - if [[ ! -f "${SQLITE_DB}" ]]; then - sqlite3 "${SQLITE_DB}" " - ${SQLITE_PRAGMAS} - CREATE TABLE IF NOT EXISTS banned_ips (ip TEXT PRIMARY KEY, bantime INTEGER); - CREATE INDEX IF NOT EXISTS idx_ip ON banned_ips(ip); - " - fi - - result=$(sqlite3 "file:${SQLITE_DB}?mode=ro" "SELECT name FROM sqlite_master WHERE type='table' AND name='banned_ips';") - if ! [[ -n "${result}" ]]; then - log_message "ERROR: AbuseIPDB database initialization failed." - rm -f "${SQLITE_DB:?}" - exit 1 - else - log_message "SUCCESS: The AbuseIPDB database is now ready for connection." - fi - - touch "${LOCK_DONE}" || exit 1 +( + flock -n 200 || { + [[ -f "${LOG_FILE}" ]] && log_message "WARNING: Another initialization is already running. Exiting." exit 0 - else + } + + if [[ -f "${LOCK_DONE}" ]]; then + log_message "INFO: Initialization already completed. Skipping further checks." exit 0 fi + + trap 'if [[ $? -ne 0 ]]; then create_lock; else remove_lock; fi' EXIT + + SQLITE_DIR=$(dirname "${SQLITE_DB}") + if [[ ! -d "${SQLITE_DIR}" ]]; then + mkdir -p "${SQLITE_DIR}" || exit 1 + fi + + LOG_DIR=$(dirname "${LOG_FILE}") + if [[ ! -d "${LOG_DIR}" ]]; then + mkdir -p "${LOG_DIR}" || exit 1 + fi + + + if [[ ! -f "${LOG_FILE}" ]]; then + touch "${LOG_FILE}" || exit 1 + fi + + for dep in curl jq sqlite3; do + if ! command -v "${dep}" &>/dev/null; then + log_message "ERROR: ${dep} is not installed. Please install ${dep}" + exit 1 + fi + done + + if [[ ! -f "${SQLITE_DB}" ]]; then + log_message "INFO: AbuseIPDB database not found. Initializing..." + sqlite3 "${SQLITE_DB}" " + PRAGMA journal_mode=WAL; + CREATE TABLE IF NOT EXISTS banned_ips ( + ip TEXT PRIMARY KEY, + bantime INTEGER + ); + CREATE INDEX IF NOT EXISTS idx_ip ON banned_ips(ip); + " &>/dev/null + log_message "INFO: AbuseIPDB database is initialized!" + fi + + table=$(sqlite3 "${SQLITE_DB}" "SELECT name FROM sqlite_master WHERE type='table' AND name='banned_ips';") + if ! [[ -n "${table}" ]]; then + log_message "ERROR: AbuseIPDB database initialization failed." + exit 1 + fi + + touch "${LOCK_DONE}" || exit 1 + log_message "SUCCESS: All (actionstart) checks completed!" + exit 0 + +) 200>"${LOCK_INIT}" + exit 0 fi ####################################### @@ -182,17 +193,7 @@ fi ####################################### ####################################### -# 1) Prevent 'actionban' if -# 'actionstart' fails. -# -# If 'actionstart' fails, block -# 'actionban' to prevent issues from -# missing dependencies or permission -# errors. -####################################### - -####################################### -# 2) Fail2Ban restart handling & +# 1) Fail2Ban restart handling & # duplicate report prevention. # # - If 'BYPASS_FAIL2BAN' is disabled, @@ -203,8 +204,18 @@ fi # - If enabled, Fail2Ban is bypassed, # and the script independently # decides which IPs to report based -# on the AbuseIPDB db, even after -# restarts. +# on the local AbuseIPDB SQLite db, +# even after restarts. +####################################### + +####################################### +# 2) Prevent 'actionban' if +# 'actionstart' fails. +# +# - If 'actionstart' fails, block +# 'actionban' to prevent issues from +# missing dependencies or permission +# errors. ####################################### ####################################### @@ -221,16 +232,16 @@ fi # EARLY CHECKS: (START) ####################################### +if [[ "${BYPASS_FAIL2BAN}" -eq 0 && "${RESTORED}" -eq 1 ]]; then + log_message "INFO: (RESTART) IP ${IP} was already reported in the previous Fail2Ban session." + exit 0 +fi + if [[ -f "${LOCK_BAN}" ]]; then [[ -f "${LOG_FILE}" ]] && log_message "ERROR: Initialization failed! (actionstart). Reporting for IP ${IP} is blocked." exit 1 fi -if [[ "${BYPASS_FAIL2BAN}" -eq 0 && "${RESTORED}" -eq 1 ]]; then - log_message "INFO: IP ${IP} already reported." - exit 0 -fi - if [[ -z "${APIKEY}" || -z "${COMMENT}" || -z "${IP}" || -z "${CATEGORIES}" || -z "${BANTIME}" ]]; then log_message "ERROR: Missing core argument(s)." exit 1 @@ -246,82 +257,133 @@ fi check_ip_in_abuseipdb() { local response http_status body total_reports delimiter="HTTP_STATUS:" - response=$(curl -s -w "${delimiter}%{http_code}" -G "https://api.abuseipdb.com/api/v2/check" \ + if ! response=$(curl -sS -w "${delimiter}%{http_code}" -G "https://api.abuseipdb.com/api/v2/check" \ --data-urlencode "ipAddress=${IP}" \ -H "Key: ${APIKEY}" \ - -H "Accept: application/json" 2>&1) - - if [[ $? -ne 0 ]]; then - log_message "ERROR: API failure. Response: ${response}" - return 1 + -H "Accept: application/json" 2>&1); then + log_message "ERROR: curl failed. Response: ${response}" + return 2 fi - http_status=$(echo "${response}" | tr -d '\n' | sed -e "s/.*${delimiter}//") - body=$(echo "${response}" | sed -e "s/${delimiter}[0-9]*//") + http_status="${response##*${delimiter}}" + body="${response%"${delimiter}${http_status}"}" - if [[ "${http_status}" =~ ^[0-9]+$ ]]; then + + if [[ ! "${http_status}" =~ ^[0-9]+$ ]]; then + log_message "ERROR: Invalid HTTP status in Response: ${response}" + return 2 + fi + + if [[ "${http_status}" -ne 200 ]]; then if [[ "${http_status}" -eq 429 ]]; then log_message "ERROR: Rate limited (HTTP 429). Response: ${body}" - return 1 - fi - - if [[ "${http_status}" -ne 200 ]]; then + else log_message "ERROR: HTTP ${http_status}. Response: ${body}" - return 1 fi - else - log_message "ERROR: API failure. Response: ${response}" - return 1 + return 2 fi - total_reports=$(echo "${body}" | jq '.data.totalReports') - if [[ "${total_reports}" -gt 0 ]]; then + total_reports=$(jq -r '.data.totalReports // 0' <<< "${body}") + if (( total_reports > 0 )); then return 0 - else - return 1 fi + return 1 +} + +convert_bantime() { + local bantime=$1 + local time_value + local time_unit + + if [[ "${bantime}" =~ ^[0-9]+$ ]]; then + echo "${bantime}" + return 0 + fi + + time_value="${bantime%"${bantime##*[0-9]}"}" + time_unit="${bantime#${time_value}}" + + [[ -z "$time_unit" ]] && time_unit="s" + + case "$time_unit" in + s) return $time_value ;; + m) return $((time_value * 60)) ;; + h) return $((time_value * 3600)) ;; + d) return $((time_value * 86400)) ;; + w) return $((time_value * 604800)) ;; + y) return $((time_value * 31536000)) ;; + *) return $time_value ;; + esac } report_ip_to_abuseipdb() { - local response - response=$(curl --fail -s 'https://api.abuseipdb.com/api/v2/report' \ + local response http_status body delimiter="HTTP_STATUS:" + if ! response=$(curl -sS -w "${delimiter}%{http_code}" "https://api.abuseipdb.com/api/v2/report" \ -H 'Accept: application/json' \ -H "Key: ${APIKEY}" \ --data-urlencode "comment=${COMMENT}" \ --data-urlencode "ip=${IP}" \ - --data "categories=${CATEGORIES}" 2>&1) - - if [[ $? -ne 0 ]]; then - log_message "ERROR: API failure. Response: ${response} for IP: ${IP}" - else - log_message "SUCCESS: Reported IP ${IP} to AbuseIPDB." + --data "categories=${CATEGORIES}" 2>&1); then + log_message "ERROR: curl failed. Response: ${response}" + return 1 fi + + http_status="${response##*${delimiter}}" + body="${response%"${delimiter}${http_status}"}" + + if [[ ! "${http_status}" =~ ^[0-9]+$ ]]; then + log_message "ERROR: Invalid HTTP status in response: ${response}" + return 1 + fi + + if [[ "${http_status}" -ne 200 ]]; then + if [[ "${http_status}" -eq 429 ]]; then + log_message "ERROR: Rate limited (HTTP 429). Response: ${body}" + else + log_message "ERROR: HTTP ${http_status}. Response: ${body}" + fi + return 1 + fi + + log_message "SUCCESS: Reported IP ${IP} to AbuseIPDB." + return 0 } check_ip_in_db() { local ip=$1 result - result=$(sqlite3 "file:${SQLITE_DB}?mode=ro" " - ${SQLITE_PRAGMAS} - SELECT 1 FROM banned_ips WHERE ip = '${ip}' LIMIT 1;" - ) + ip="${ip%"${ip##*[![:space:]]}"}" + ip="${ip#"${ip%%[^[:space:]]*}"}" + ip="${ip//\'/}" + ip="${ip//\"/}" - if [[ $? -ne 0 ]]; then - log_message "ERROR: AbuseIPDB database query failed while checking IP ${ip}. Response: ${result}" - return 1 - fi + sqlite3 "${SQLITE_DB}" "${SQLITE_NON_PERSISTENT_PRAGMAS}" &>/dev/null + result=$(sqlite3 "${SQLITE_DB}" "SELECT EXISTS(SELECT 1 FROM banned_ips WHERE ip = '${ip}');") - if [[ -n "${result}" ]]; then + if [[ "${result}" -eq 1 ]]; then return 0 - else + elif [[ "${result}" -eq 0 ]]; then return 1 + else + return 2 fi } insert_ip_to_db() { - local ip=$1 - local bantime=$2 + local ip=$1 bantime=$2 + bantime=$(convert_bantime "${bantime}") + + bantime="${bantime%"${bantime##*[![:space:]]}"}" + bantime="${bantime#"${bantime%%[^[:space:]]*}"}" + bantime="${bantime//\'/}" + bantime="${bantime//\"/}" + + ip="${ip%"${ip##*[![:space:]]}"}" + ip="${ip#"${ip%%[^[:space:]]*}"}" + ip="${ip//\'/}" + ip="${ip//\"/}" + + sqlite3 "${SQLITE_DB}" "${SQLITE_NON_PERSISTENT_PRAGMAS}" &>/dev/null sqlite3 "${SQLITE_DB}" " - ${SQLITE_PRAGMAS} BEGIN IMMEDIATE; INSERT INTO banned_ips (ip, bantime) VALUES ('${ip}', ${bantime}) @@ -329,9 +391,31 @@ insert_ip_to_db() { COMMIT; " + # TO-DO: Better handle SQLite INSERT ops. exit statuses + # $? -ne 0 | I think not the best approach here. if [[ $? -ne 0 ]]; then - log_message "ERROR: Failed to insert or update IP ${ip} in the AbuseIPDB database." + return 1 fi + return 0 +} + +delete_ip_from_db() { + local ip=$1 + ip="${ip%"${ip##*[![:space:]]}"}" + ip="${ip#"${ip%%[^[:space:]]*}"}" + ip="${ip//\'/}" + ip="${ip//\"/}" + + sqlite3 "${SQLITE_DB}" "${SQLITE_NON_PERSISTENT_PRAGMAS}" &>/dev/null + sqlite3 "${SQLITE_DB}" " + BEGIN IMMEDIATE; + DELETE FROM banned_ips WHERE ip='${ip}'; + COMMIT; + " + + # TO-DO: Do we need to listen exit status DELETE + # I don't think so for now. + log_message "INFO: IP ${ip} deleted from the AbuseIPDB SQLite database." } ####################################### @@ -352,18 +436,33 @@ insert_ip_to_db() { log_message "INFO: IP ${IP} has already been reported and remains on AbuseIPDB." shouldBanIP=0 else - log_message "INFO: IP ${IP} has already been reported but is no longer listed on AbuseIPDB." - shouldBanIP=1 + status=$? + if [[ "${status}" -eq 1 ]]; then + log_message "INFO: IP ${IP} has already been reported but is no longer listed on AbuseIPDB. Resubmitting..." + else + log_message "ERROR: Failed to check IP ${IP} in the AbuseIPDB API. Skipping report." + exit 1 + fi fi else - shouldBanIP=1 + status=$? + if [[ "${status}" -eq 2 ]]; then + log_message "ERROR: Failed to check IP ${IP} in the local database. Skipping report." + exit 1 + fi fi if [[ "${shouldBanIP}" -eq 1 ]]; then if [[ "${is_found_local}" -eq 0 ]]; then - insert_ip_to_db $IP $BANTIME + if ! insert_ip_to_db $IP $BANTIME; then + log_message "ERROR: Failed to insert IP ${IP} into the local database. Skipping report." + exit 1 + fi + fi + + if ! report_ip_to_abuseipdb; then + delete_ip_from_db $IP fi - report_ip_to_abuseipdb fi ) >> "${LOG_FILE}" 2>&1 & From f8a269eaadf3040ee026bb1f68a614eaf3afe781 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Hasan=20=C3=87ALI=C5=9EIR?= Date: Wed, 5 Mar 2025 21:15:10 +0300 Subject: [PATCH 9/9] fix convert_bantime MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Swapped returns for echoes — Bash, not PHP --- files/fail2ban-abuseipdb.sh | 21 ++++++++------------- 1 file changed, 8 insertions(+), 13 deletions(-) diff --git a/files/fail2ban-abuseipdb.sh b/files/fail2ban-abuseipdb.sh index 86bbb3c5..a21a3425 100644 --- a/files/fail2ban-abuseipdb.sh +++ b/files/fail2ban-abuseipdb.sh @@ -268,7 +268,6 @@ check_ip_in_abuseipdb() { http_status="${response##*${delimiter}}" body="${response%"${delimiter}${http_status}"}" - if [[ ! "${http_status}" =~ ^[0-9]+$ ]]; then log_message "ERROR: Invalid HTTP status in Response: ${response}" return 2 @@ -291,10 +290,7 @@ check_ip_in_abuseipdb() { } convert_bantime() { - local bantime=$1 - local time_value - local time_unit - + local bantime=$1 time_value time_unit if [[ "${bantime}" =~ ^[0-9]+$ ]]; then echo "${bantime}" return 0 @@ -304,15 +300,14 @@ convert_bantime() { time_unit="${bantime#${time_value}}" [[ -z "$time_unit" ]] && time_unit="s" - case "$time_unit" in - s) return $time_value ;; - m) return $((time_value * 60)) ;; - h) return $((time_value * 3600)) ;; - d) return $((time_value * 86400)) ;; - w) return $((time_value * 604800)) ;; - y) return $((time_value * 31536000)) ;; - *) return $time_value ;; + s) echo "$time_value" ;; + m) echo "$((time_value * 60))" ;; + h) echo "$((time_value * 3600))" ;; + d) echo "$((time_value * 86400))" ;; + w) echo "$((time_value * 604800))" ;; + y) echo "$((time_value * 31536000))" ;; + *) echo "${time_value}" ;; esac }