migrate banned IPs to SQLite DB and prevent actionban latency

- Replace local file storage with AbuseIPDB SQLite database.
- Offload heavy tasks to background to avoid latency during concurrent actionban calls.
- Add global lock to ensure actionstart runs only once across all jails.

files/fail2ban-abuseipdb.sh

@@ -3,9 +3,9 @@
 # Description:
 #   This script acts as a Fail2Ban `actionstart|actionban` to report offending IPs to AbuseIPDB.
 #   It allows for 'custom comments' to prevent leaking sensitive information. The main goal is to
-#   avoid relying on Fail2Ban and instead use a local banned IP list for complete isolation.
+#   avoid relying on Fail2Ban and instead use a separate AbuseIPDB SQLite database for complete isolation.
 #   It can also be used with Fail2Ban's `norestored=1` feature to rely on Fail2Ban for preventing
-#   redundant report actions on restarts. Users can toggle this behavior as needed.
+#   redundant reporting on restarts. Users can toggle this behavior as needed.
 #
 #   The script performs two API calls for each ban action:
 #     1. **/v2/check**  - Checks if the IP has already been reported.
@@ -33,7 +33,7 @@
 #
 # Usage:
 #   This script is designed to be triggered automatically by Fail2Ban (`actionstart|actionban`).
-#   Manual Usage:
+#   For testing (manual execution):
 #    - For testing purpose before production;
 #       /etc/fail2ban/action.d/fail2ban_abuseipdb.sh "your_api_key" "Failed SSH login attempts" "192.0.2.1" "18" "600"
 #
@@ -45,46 +45,34 @@
 #   $5 BANTIME         - Required (Core). Retrieved automatically from the Fail2Ban 'jail'.        | Ban duration
 #   $6 RESTORED        - Required (Core). Retrieved automatically from the Fail2Ban '<restored>'   | Status of restored tickets
 #   $7 BYPASS_FAIL2BAN - Required (User defined). Must be defined in 'action.d/abuseipdb.local'.   | Bypassing Fail2Ban on restarts
-#   $8 LOCAL_LIST      - Required (User defined). Must be defined in 'action.d/abuseipdb.local'.   | Path to the main banned IP list used by the script
+#   $2|$8 SQLITE_DB    - Required (User defined). Must be defined in 'action.d/abuseipdb.local'.   | Path to the main AbuseIPDB SQLite database
-#   $9 LOG_FILE        - Required (User defined). Must be defined in 'action.d/abuseipdb.local'.   | Path to the log file where actions and events are recorded by the script
+#   $3|$9 LOG_FILE     - Required (User defined). Must be defined in 'action.d/abuseipdb.local'.   | Path to the log file where actions and events are recorded by the script
 #
 # Dependencies:
 #   curl: For making API requests to AbuseIPDB.
 #   jq: For parsing JSON responses.
-#   flock: Prevent data corruption.
+#   sqlite3: Local AbuseIPDB db.
-#
-# Return Codes:
-#   0 - 'AbuseIPDB' IP is reported.
-#   1 - 'AbuseIPDB' IP is not reported.
-#
-# Exit Codes:
-#   0 - 'norestored'  restored tickets enabled.
-#   0 - 'actionstart' tasks completed.
-#   1 - 'actionstart' tasks cannot completed and lock file created.
-#   1 - 'AbuseIPDB'   API-related failure.
 #
 # Author:
 #   Hasan ÇALIŞIR
-#   hasan.calisir@psauxit.com
 #   https://github.com/hsntgm
 
-# This script is used for both: 'actionstart' and 'actionban' in 'action.d/abuseipdb.local'
+#######################################
-# It dynamically assigns arguments based on the action type
+# HELPERS: (START)
-# and provides default values for missing user settings to prevent failures.
+#######################################
+
 APIKEY="$1"
 COMMENT="$2"
 IP="$3"
 CATEGORIES="$4"
 BANTIME="$5"
-RESTORED="${6}"
+RESTORED="$6"
 BYPASS_FAIL2BAN="${7:-0}"
 if [[ "$1" == "--actionstart" ]]; then
-    # When triggered by 'actionstart'
+    SQLITE_DB="${2:-/var/lib/fail2ban/abuseipdb/fail2ban_abuseipdb}"
-    REPORTED_IP_LIST_FILE="${2:-/var/log/abuseipdb/abuseipdb-banned.log}"
     LOG_FILE="${3:-/var/log/abuseipdb/abuseipdb.log}"
 else
-    # When triggered by 'actionban'
+    SQLITE_DB="${8:-/var/lib/fail2ban/abuseipdb/fail2ban_abuseipdb}"
-    REPORTED_IP_LIST_FILE="${8:-/var/log/abuseipdb/abuseipdb-banned.log}"
     LOG_FILE="${9:-/var/log/abuseipdb/abuseipdb.log}"
 fi
 
@@ -94,137 +82,206 @@ log_message() {
     echo "$(date +"%Y-%m-%d %H:%M:%S") - ${message}" >> "${LOG_FILE}"
 }
 
-# Define lock file
+# Lock files for 'actionstart' status
-LOCK_FILE="/tmp/abuseipdb_actionstart.lock"
+LOCK_BAN="/tmp/abuseipdb_actionstart.lock"
+LOCK_DONE="/tmp/abuseipdb_actionstart.done"
 
-# Function to remove lock file if it exists
+# Remove lock file
 remove_lock() {
-    if [[ -f "${LOCK_FILE}" ]]; then
+    [[ -f "${LOCK_BAN}" ]] && rm -f "${LOCK_BAN}"
-        rm -f "${LOCK_FILE:?}"
-    fi
 }
 
-# Function to create lock file
+# Create lock file
 create_lock() {
-    if [[ ! -f "${LOCK_FILE}" ]]; then
+    [[ ! -f "${LOCK_BAN}" ]] && touch "${LOCK_BAN}"
-        touch "${LOCK_FILE}"
-    fi
 }
 
-# Check if the script was triggered by 'actionstart' early in execution.
+# Pre-defined SQLite PRAGMAS
-# This ensures necessary checks is performed before proceeding only once.
+SQLITE_PRAGMAS="
-# This check runs on background always with 'nohup' to prevent latency.
+    PRAGMA journal_mode=WAL;
-# We listen exit codes carefully to allow or not further runtime 'actionban' events
+    PRAGMA synchronous=NORMAL;
+    PRAGMA temp_store=MEMORY;
+    PRAGMA locking_mode=NORMAL;
+    PRAGMA cache_size=-256000;
+    PRAGMA busy_timeout=10000;
+"
+
+#######################################
+# HELPERS: (END)
+#######################################
+
+#######################################
+# ACTIONSTART: (START)
+#######################################
+
+########################################
+#  Triggered by 'actionstart'
+#  to perform necessary checks
+#  and AbuseIPDB SQLite initialization.
+#
+# - Ensures required checks are done.
+# - Runs in the background with 'nohup'
+#   on initial start to prevent latency.
+# - Listens for exit codes to control
+#   further 'actionban' events via the
+#   'lock' mechanism.
+# - Check 'abuseipdb.local' for
+#   integration details.
+########################################
+
 if [[ "$1" == "--actionstart" ]]; then
-    # Trap exit signal to create/remove lock file based on exit status
+    if [[ ! -f "${LOCK_DONE}" ]]; then
-    trap 'if [[ $? -ne 0 ]]; then create_lock; else remove_lock; fi' EXIT
+        trap 'if [[ $? -ne 0 ]]; then create_lock; else remove_lock; fi' EXIT
 
-    # Ensure the directory for the reported IP list exists
+        SQLITE_DIR=$(dirname "${SQLITE_DB}")
-    LOG_DIR=$(dirname "${REPORTED_IP_LIST_FILE}")
+        if [[ ! -d "${SQLITE_DIR}" ]]; then
-    if [[ ! -d "${LOG_DIR}" ]]; then
+            mkdir -p "${SQLITE_DIR}" || exit 1
-        mkdir -p "${LOG_DIR}" || exit 1
-    fi
-
-    # Ensure the reported IP list and log file exist
-    for file in "${REPORTED_IP_LIST_FILE}" "${LOG_FILE}"; do
-        if [[ ! -f "${file}" ]]; then
-            touch "${file}" || exit 1
         fi
-    done
 
-    # Check runtime dependencies
+        if [[ ! -f "${LOG_FILE}" ]]; then
-    dependencies=("curl" "jq" "flock")
+            touch "${LOG_FILE}" || exit 1
-    for dep in "${dependencies[@]}"; do
+        fi
-        if ! command -v "${dep}" &>/dev/null; then
+
-            log_message "FATAL: -${dep} is not installed. Please install -${dep} to proceed."
+        for dep in curl jq sqlite3; do
+            if ! command -v "${dep}" &>/dev/null; then
+                log_message "ERROR: ${dep} is not installed. Please install ${dep}"
+                exit 1
+            fi
+        done
+
+        if [[ ! -f "${SQLITE_DB}" ]]; then
+            sqlite3 "${SQLITE_DB}" "
+                ${SQLITE_PRAGMAS}
+                CREATE TABLE IF NOT EXISTS banned_ips (ip TEXT PRIMARY KEY, bantime INTEGER);
+                CREATE INDEX IF NOT EXISTS idx_ip ON banned_ips(ip);
+            "
+        fi
+
+        result=$(sqlite3 "file:${SQLITE_DB}?mode=ro" "SELECT name FROM sqlite_master WHERE type='table' AND name='banned_ips';")
+        if ! [[ -n "${result}" ]]; then
+            log_message "ERROR: AbuseIPDB database initialization failed."
+            rm -f "${SQLITE_DB:?}"
             exit 1
+        else
+            log_message "SUCCESS: The AbuseIPDB database is now ready for connection."
         fi
-    done
 
-    # Tasks completed, quit nicely
+        touch "${LOCK_DONE}" || exit 1
-    exit 0
+        exit 0
-fi
-
-# If the 'actionstart' failed, prevent 'actionban'.
-# This stops 'actionban' from being triggered during runtime due to missing dependencies or permission issues.
-# A failed initial 'actionstart' check indicates a failure to report to AbuseIPDB.
-if [[ -f "${LOCK_FILE}" ]]; then
-    if [[ -f "${LOG_FILE}" ]]; then
-        log_message "FATAL: Failed due to a permission issue or missing dependency. Reporting to AbuseIPDB failed."
-        exit 1
     else
-        exit 1
-    fi
-fi
-
-# If 'BYPASS_FAIL2BAN' is disabled, Fail2Ban will be relied upon during restarts.
-# This prevents duplicate reports when Fail2Ban is restarted.
-# This setting is 'OPTIONAL' and can be overridden in 'action.d/abuseipdb.local'.
-# If enabled, Fail2Ban is bypassed completely,
-# and script takes full control to determine which IP to report based on
-# the local banned IP list even on Fail2Ban restarts.
-if [[ "${BYPASS_FAIL2BAN}" -eq 0 ]]; then
-    if [[ "${RESTORED}" -eq 1 ]]; then
-        log_message "INFO NORESTORED: IP ${IP} has already been reported. No duplicate report made after restart."
         exit 0
     fi
 fi
 
-# Validate core arguments: Ensure all required core args are provided.
+#######################################
-# These values are expected to be passed by Fail2Ban 'jail' during execution.
+# ACTIONSTART: (END)
-# Also for manual testing purpose before production.
+#######################################
-if [[ -z "$1" || -z "$2" || -z "$3" || -z "$4" || -z "$5" ]]; then
+
-    log_message "FATAL: Missing core argument"
+#######################################
+# ACTIONBAN: (START)
+#######################################
+
+#######################################
+# 1) Prevent 'actionban' if
+# 'actionstart' fails.
+#
+# If 'actionstart' fails, block
+# 'actionban' to prevent issues from
+# missing dependencies or permission
+# errors.
+#######################################
+
+#######################################
+# 2) Fail2Ban restart handling &
+# duplicate report prevention.
+#
+# - If 'BYPASS_FAIL2BAN' is disabled,
+#   Fail2Ban manages reports on restart
+#   and prevents duplicate submissions.
+# - This setting can be overridden in
+#   'action.d/abuseipdb.local'.
+# - If enabled, Fail2Ban is bypassed,
+#   and the script independently
+#   decides which IPs to report based
+#   on the AbuseIPDB db, even after
+#   restarts.
+#######################################
+
+#######################################
+# 3) Core argument validation
+#
+# - Ensures all required arguments
+#   are provided.
+# - Expected from Fail2Ban 'jail' or
+#   for manual testing before
+#   production deployment.
+#######################################
+
+#######################################
+# EARLY CHECKS: (START)
+#######################################
+
+if [[ -f "${LOCK_BAN}" ]]; then
+    [[ -f "${LOG_FILE}" ]] && log_message "ERROR: Initialization failed! (actionstart). Reporting for IP ${IP} is blocked."
     exit 1
 fi
 
-# Function to check if the IP is listed on AbuseIPDB
+if [[ "${BYPASS_FAIL2BAN}" -eq 0 && "${RESTORED}" -eq 1 ]]; then
-check_ip_in_abuseipdb() {
+    log_message "INFO: IP ${IP} already reported."
-    local response http_status body total_reports
+    exit 0
-    local delimiter="HTTP_STATUS:"
+fi
 
-    # Perform the API call and capture both response and HTTP status
+if [[ -z "${APIKEY}" || -z "${COMMENT}" || -z "${IP}" || -z "${CATEGORIES}" || -z "${BANTIME}" ]]; then
+    log_message "ERROR: Missing core argument(s)."
+    exit 1
+fi
+
+#######################################
+# EARLY CHECKS: (END)
+#######################################
+
+#######################################
+# FUNCTIONS: (START)
+#######################################
+
+check_ip_in_abuseipdb() {
+    local response http_status body total_reports delimiter="HTTP_STATUS:"
     response=$(curl -s -w "${delimiter}%{http_code}" -G "https://api.abuseipdb.com/api/v2/check" \
         --data-urlencode "ipAddress=${IP}" \
         -H "Key: ${APIKEY}" \
         -H "Accept: application/json" 2>&1)
 
     if [[ $? -ne 0 ]]; then
-        log_message "ERROR CHECK: API failure. Response: ${response}"
+        log_message "ERROR: API failure. Response: ${response}"
-        exit 1
+        return 1
     fi
 
-    # Separate the HTTP status code from the response body
     http_status=$(echo "${response}" | tr -d '\n' | sed -e "s/.*${delimiter}//")
     body=$(echo "${response}" | sed -e "s/${delimiter}[0-9]*//")
 
-    # Handle different HTTP status codes
     if [[ "${http_status}" =~ ^[0-9]+$ ]]; then
-        # Handle rate-limiting (HTTP 429)
         if [[ "${http_status}" -eq 429 ]]; then
-            log_message "ERROR CHECK: API returned HTTP 429 (Too Many Requests). Response: ${body}"
+            log_message "ERROR: Rate limited (HTTP 429). Response: ${body}"
-            exit 1
+            return 1
         fi
 
-        # Handle other non-200 responses
         if [[ "${http_status}" -ne 200 ]]; then
-            log_message "ERROR CHECK: API returned HTTP status ${http_status}. Response: ${body}"
+            log_message "ERROR: HTTP ${http_status}. Response: ${body}"
-            exit 1
+            return 1
         fi
+    else
+        log_message "ERROR: API failure. Response: ${response}"
+        return 1
     fi
 
-    # Extract totalReports
     total_reports=$(echo "${body}" | jq '.data.totalReports')
-
-    # Finally, check the IP listed on AbuseIPDB
     if [[ "${total_reports}" -gt 0 ]]; then
-        return 0 # IP is reported
+        return 0
     else
-        return 1 # IP is not reported
+        return 1
     fi
 }
 
-# Function to report AbuseIpDB
 report_ip_to_abuseipdb() {
     local response
     response=$(curl --fail -s 'https://api.abuseipdb.com/api/v2/report' \
@@ -234,48 +291,88 @@ report_ip_to_abuseipdb() {
         --data-urlencode "ip=${IP}" \
         --data "categories=${CATEGORIES}" 2>&1)
 
-    # API call fail
     if [[ $? -ne 0 ]]; then
-        log_message "ERROR REPORT: API failure. Response: ${response}"
+        log_message "ERROR: API failure. Response: ${response} for IP: ${IP}"
-        exit 1
     else
-        log_message "SUCCESS REPORT: Reported IP ${IP} to AbuseIPDB. Local list updated."
+        log_message "SUCCESS: Reported IP ${IP} to AbuseIPDB."
     fi
 }
 
+check_ip_in_db() {
+    local ip=$1 result
+    result=$(sqlite3 "file:${SQLITE_DB}?mode=ro" "
+        ${SQLITE_PRAGMAS}
+        SELECT 1 FROM banned_ips WHERE ip = '${ip}' LIMIT 1;"
+    )
 
-# Set defaults
+    if [[ $? -ne 0 ]]; then
-is_found_local=0
+        log_message "ERROR: AbuseIPDB database query failed while checking IP ${ip}. Response: ${result}"
-shouldBanIP=1
+        return 1
+    fi
 
-# Should Ban IP
+    if [[ -n "${result}" ]]; then
-if grep -m 1 -q -E "^IP=${IP}[[:space:]]+L=[0-9\-]+" "${REPORTED_IP_LIST_FILE}"; then
+        return 0
-    # IP found locally, check if it's still listed on AbuseIPDB
-    if check_ip_in_abuseipdb; then
-        # IP is still listed on AbuseIPDB, no need to report again
-        log_message "INFO: IP ${IP} has already been reported and remains on AbuseIPDB. No duplicate report made."
-        shouldBanIP=0
     else
-        # IP is reported before but not listed on AbuseIPDB, report it again
+        return 1
-        log_message "INFO: IP ${IP} has already been reported but is no longer listed on AbuseIPDB. Reporting it again."
-        shouldBanIP=1
-        is_found_local=1
     fi
-else
+}
+
+insert_ip_to_db() {
+    local ip=$1
+    local bantime=$2
+    sqlite3 "${SQLITE_DB}" "
+        ${SQLITE_PRAGMAS}
+        BEGIN IMMEDIATE;
+        INSERT INTO banned_ips (ip, bantime)
+        VALUES ('${ip}', ${bantime})
+        ON CONFLICT(ip) DO UPDATE SET bantime=${bantime};
+        COMMIT;
+    "
+
+    if [[ $? -ne 0 ]]; then
+        log_message "ERROR: Failed to insert or update IP ${ip} in the AbuseIPDB database."
+    fi
+}
+
+#######################################
+# FUNCTIONS: (END)
+#######################################
+
+#######################################
+# MAIN (START)
+#######################################
+
+(
+    is_found_local=0
     shouldBanIP=1
-fi
 
-# Let's report to AbuseIpdb
+    if check_ip_in_db $IP; then
-if [[ "${shouldBanIP}" -eq 1 ]]; then
+        is_found_local=1
-    # Add the new ban entry to local list kindly
+        if check_ip_in_abuseipdb; then
-    if [[ "${is_found_local}" -eq 0 ]]; then
+            log_message "INFO: IP ${IP} has already been reported and remains on AbuseIPDB."
-        exec 200<> "${REPORTED_IP_LIST_FILE}"                      # Open with read/write access
+            shouldBanIP=0
-        flock -x 200                                               # Lock
+        else
-        echo "IP=${IP} L=${BANTIME}" >> "${REPORTED_IP_LIST_FILE}" # Write
+            log_message "INFO: IP ${IP} has already been reported but is no longer listed on AbuseIPDB."
-        flock -u 200                                               # Release the lock
+            shouldBanIP=1
-        exec 200>&-                                                # Close the file descriptor
+        fi
+    else
+        shouldBanIP=1
     fi
 
-    # Report IP
+    if [[ "${shouldBanIP}" -eq 1 ]]; then
-    report_ip_to_abuseipdb
+        if [[ "${is_found_local}" -eq 0 ]]; then
-fi
+            insert_ip_to_db $IP $BANTIME
+        fi
+        report_ip_to_abuseipdb
+    fi
+) >> "${LOG_FILE}" 2>&1 &
+
+#######################################
+# MAIN (END)
+#######################################
+
+#######################################
+# ACTIONBAN: (END)
+#######################################
+
+exit 0