From d13660c588f032a0401b10ed914e17b5459b3dd3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Hasan=20=C3=87ALI=C5=9EIR?= Date: Mon, 3 Mar 2025 18:13:45 +0300 Subject: [PATCH] migrate banned IPs to SQLite DB and prevent actionban latency - Replace local file storage with AbuseIPDB SQLite database. - Offload heavy tasks to background to avoid latency during concurrent actionban calls. - Add global lock to ensure actionstart runs only once across all jails. --- files/fail2ban-abuseipdb.sh | 377 +++++++++++++++++++++++------------- 1 file changed, 237 insertions(+), 140 deletions(-) diff --git a/files/fail2ban-abuseipdb.sh b/files/fail2ban-abuseipdb.sh index ba4caffb..ca822bd6 100644 --- a/files/fail2ban-abuseipdb.sh +++ b/files/fail2ban-abuseipdb.sh @@ -3,9 +3,9 @@ # Description: # This script acts as a Fail2Ban `actionstart|actionban` to report offending IPs to AbuseIPDB. # It allows for 'custom comments' to prevent leaking sensitive information. The main goal is to -# avoid relying on Fail2Ban and instead use a local banned IP list for complete isolation. +# avoid relying on Fail2Ban and instead use a separate AbuseIPDB SQLite database for complete isolation. # It can also be used with Fail2Ban's `norestored=1` feature to rely on Fail2Ban for preventing -# redundant report actions on restarts. Users can toggle this behavior as needed. +# redundant reporting on restarts. Users can toggle this behavior as needed. # # The script performs two API calls for each ban action: # 1. **/v2/check** - Checks if the IP has already been reported. @@ -33,7 +33,7 @@ # # Usage: # This script is designed to be triggered automatically by Fail2Ban (`actionstart|actionban`). -# Manual Usage: +# For testing (manual execution): # - For testing purpose before production; # /etc/fail2ban/action.d/fail2ban_abuseipdb.sh "your_api_key" "Failed SSH login attempts" "192.0.2.1" "18" "600" # @@ -45,46 +45,34 @@ # $5 BANTIME - Required (Core). Retrieved automatically from the Fail2Ban 'jail'. | Ban duration # $6 RESTORED - Required (Core). Retrieved automatically from the Fail2Ban '' | Status of restored tickets # $7 BYPASS_FAIL2BAN - Required (User defined). Must be defined in 'action.d/abuseipdb.local'. | Bypassing Fail2Ban on restarts -# $8 LOCAL_LIST - Required (User defined). Must be defined in 'action.d/abuseipdb.local'. | Path to the main banned IP list used by the script -# $9 LOG_FILE - Required (User defined). Must be defined in 'action.d/abuseipdb.local'. | Path to the log file where actions and events are recorded by the script +# $2|$8 SQLITE_DB - Required (User defined). Must be defined in 'action.d/abuseipdb.local'. | Path to the main AbuseIPDB SQLite database +# $3|$9 LOG_FILE - Required (User defined). Must be defined in 'action.d/abuseipdb.local'. | Path to the log file where actions and events are recorded by the script # # Dependencies: # curl: For making API requests to AbuseIPDB. # jq: For parsing JSON responses. -# flock: Prevent data corruption. -# -# Return Codes: -# 0 - 'AbuseIPDB' IP is reported. -# 1 - 'AbuseIPDB' IP is not reported. -# -# Exit Codes: -# 0 - 'norestored' restored tickets enabled. -# 0 - 'actionstart' tasks completed. -# 1 - 'actionstart' tasks cannot completed and lock file created. -# 1 - 'AbuseIPDB' API-related failure. +# sqlite3: Local AbuseIPDB db. # # Author: # Hasan ÇALIŞIR -# hasan.calisir@psauxit.com # https://github.com/hsntgm -# This script is used for both: 'actionstart' and 'actionban' in 'action.d/abuseipdb.local' -# It dynamically assigns arguments based on the action type -# and provides default values for missing user settings to prevent failures. +####################################### +# HELPERS: (START) +####################################### + APIKEY="$1" COMMENT="$2" IP="$3" CATEGORIES="$4" BANTIME="$5" -RESTORED="${6}" +RESTORED="$6" BYPASS_FAIL2BAN="${7:-0}" if [[ "$1" == "--actionstart" ]]; then - # When triggered by 'actionstart' - REPORTED_IP_LIST_FILE="${2:-/var/log/abuseipdb/abuseipdb-banned.log}" + SQLITE_DB="${2:-/var/lib/fail2ban/abuseipdb/fail2ban_abuseipdb}" LOG_FILE="${3:-/var/log/abuseipdb/abuseipdb.log}" else - # When triggered by 'actionban' - REPORTED_IP_LIST_FILE="${8:-/var/log/abuseipdb/abuseipdb-banned.log}" + SQLITE_DB="${8:-/var/lib/fail2ban/abuseipdb/fail2ban_abuseipdb}" LOG_FILE="${9:-/var/log/abuseipdb/abuseipdb.log}" fi @@ -94,137 +82,206 @@ log_message() { echo "$(date +"%Y-%m-%d %H:%M:%S") - ${message}" >> "${LOG_FILE}" } -# Define lock file -LOCK_FILE="/tmp/abuseipdb_actionstart.lock" +# Lock files for 'actionstart' status +LOCK_BAN="/tmp/abuseipdb_actionstart.lock" +LOCK_DONE="/tmp/abuseipdb_actionstart.done" -# Function to remove lock file if it exists +# Remove lock file remove_lock() { - if [[ -f "${LOCK_FILE}" ]]; then - rm -f "${LOCK_FILE:?}" - fi + [[ -f "${LOCK_BAN}" ]] && rm -f "${LOCK_BAN}" } -# Function to create lock file +# Create lock file create_lock() { - if [[ ! -f "${LOCK_FILE}" ]]; then - touch "${LOCK_FILE}" - fi + [[ ! -f "${LOCK_BAN}" ]] && touch "${LOCK_BAN}" } -# Check if the script was triggered by 'actionstart' early in execution. -# This ensures necessary checks is performed before proceeding only once. -# This check runs on background always with 'nohup' to prevent latency. -# We listen exit codes carefully to allow or not further runtime 'actionban' events +# Pre-defined SQLite PRAGMAS +SQLITE_PRAGMAS=" + PRAGMA journal_mode=WAL; + PRAGMA synchronous=NORMAL; + PRAGMA temp_store=MEMORY; + PRAGMA locking_mode=NORMAL; + PRAGMA cache_size=-256000; + PRAGMA busy_timeout=10000; +" + +####################################### +# HELPERS: (END) +####################################### + +####################################### +# ACTIONSTART: (START) +####################################### + +######################################## +# Triggered by 'actionstart' +# to perform necessary checks +# and AbuseIPDB SQLite initialization. +# +# - Ensures required checks are done. +# - Runs in the background with 'nohup' +# on initial start to prevent latency. +# - Listens for exit codes to control +# further 'actionban' events via the +# 'lock' mechanism. +# - Check 'abuseipdb.local' for +# integration details. +######################################## + if [[ "$1" == "--actionstart" ]]; then - # Trap exit signal to create/remove lock file based on exit status - trap 'if [[ $? -ne 0 ]]; then create_lock; else remove_lock; fi' EXIT + if [[ ! -f "${LOCK_DONE}" ]]; then + trap 'if [[ $? -ne 0 ]]; then create_lock; else remove_lock; fi' EXIT - # Ensure the directory for the reported IP list exists - LOG_DIR=$(dirname "${REPORTED_IP_LIST_FILE}") - if [[ ! -d "${LOG_DIR}" ]]; then - mkdir -p "${LOG_DIR}" || exit 1 - fi - - # Ensure the reported IP list and log file exist - for file in "${REPORTED_IP_LIST_FILE}" "${LOG_FILE}"; do - if [[ ! -f "${file}" ]]; then - touch "${file}" || exit 1 + SQLITE_DIR=$(dirname "${SQLITE_DB}") + if [[ ! -d "${SQLITE_DIR}" ]]; then + mkdir -p "${SQLITE_DIR}" || exit 1 fi - done - # Check runtime dependencies - dependencies=("curl" "jq" "flock") - for dep in "${dependencies[@]}"; do - if ! command -v "${dep}" &>/dev/null; then - log_message "FATAL: -${dep} is not installed. Please install -${dep} to proceed." + if [[ ! -f "${LOG_FILE}" ]]; then + touch "${LOG_FILE}" || exit 1 + fi + + for dep in curl jq sqlite3; do + if ! command -v "${dep}" &>/dev/null; then + log_message "ERROR: ${dep} is not installed. Please install ${dep}" + exit 1 + fi + done + + if [[ ! -f "${SQLITE_DB}" ]]; then + sqlite3 "${SQLITE_DB}" " + ${SQLITE_PRAGMAS} + CREATE TABLE IF NOT EXISTS banned_ips (ip TEXT PRIMARY KEY, bantime INTEGER); + CREATE INDEX IF NOT EXISTS idx_ip ON banned_ips(ip); + " + fi + + result=$(sqlite3 "file:${SQLITE_DB}?mode=ro" "SELECT name FROM sqlite_master WHERE type='table' AND name='banned_ips';") + if ! [[ -n "${result}" ]]; then + log_message "ERROR: AbuseIPDB database initialization failed." + rm -f "${SQLITE_DB:?}" exit 1 + else + log_message "SUCCESS: The AbuseIPDB database is now ready for connection." fi - done - # Tasks completed, quit nicely - exit 0 -fi - -# If the 'actionstart' failed, prevent 'actionban'. -# This stops 'actionban' from being triggered during runtime due to missing dependencies or permission issues. -# A failed initial 'actionstart' check indicates a failure to report to AbuseIPDB. -if [[ -f "${LOCK_FILE}" ]]; then - if [[ -f "${LOG_FILE}" ]]; then - log_message "FATAL: Failed due to a permission issue or missing dependency. Reporting to AbuseIPDB failed." - exit 1 + touch "${LOCK_DONE}" || exit 1 + exit 0 else - exit 1 - fi -fi - -# If 'BYPASS_FAIL2BAN' is disabled, Fail2Ban will be relied upon during restarts. -# This prevents duplicate reports when Fail2Ban is restarted. -# This setting is 'OPTIONAL' and can be overridden in 'action.d/abuseipdb.local'. -# If enabled, Fail2Ban is bypassed completely, -# and script takes full control to determine which IP to report based on -# the local banned IP list even on Fail2Ban restarts. -if [[ "${BYPASS_FAIL2BAN}" -eq 0 ]]; then - if [[ "${RESTORED}" -eq 1 ]]; then - log_message "INFO NORESTORED: IP ${IP} has already been reported. No duplicate report made after restart." exit 0 fi fi -# Validate core arguments: Ensure all required core args are provided. -# These values are expected to be passed by Fail2Ban 'jail' during execution. -# Also for manual testing purpose before production. -if [[ -z "$1" || -z "$2" || -z "$3" || -z "$4" || -z "$5" ]]; then - log_message "FATAL: Missing core argument" +####################################### +# ACTIONSTART: (END) +####################################### + +####################################### +# ACTIONBAN: (START) +####################################### + +####################################### +# 1) Prevent 'actionban' if +# 'actionstart' fails. +# +# If 'actionstart' fails, block +# 'actionban' to prevent issues from +# missing dependencies or permission +# errors. +####################################### + +####################################### +# 2) Fail2Ban restart handling & +# duplicate report prevention. +# +# - If 'BYPASS_FAIL2BAN' is disabled, +# Fail2Ban manages reports on restart +# and prevents duplicate submissions. +# - This setting can be overridden in +# 'action.d/abuseipdb.local'. +# - If enabled, Fail2Ban is bypassed, +# and the script independently +# decides which IPs to report based +# on the AbuseIPDB db, even after +# restarts. +####################################### + +####################################### +# 3) Core argument validation +# +# - Ensures all required arguments +# are provided. +# - Expected from Fail2Ban 'jail' or +# for manual testing before +# production deployment. +####################################### + +####################################### +# EARLY CHECKS: (START) +####################################### + +if [[ -f "${LOCK_BAN}" ]]; then + [[ -f "${LOG_FILE}" ]] && log_message "ERROR: Initialization failed! (actionstart). Reporting for IP ${IP} is blocked." exit 1 fi -# Function to check if the IP is listed on AbuseIPDB -check_ip_in_abuseipdb() { - local response http_status body total_reports - local delimiter="HTTP_STATUS:" +if [[ "${BYPASS_FAIL2BAN}" -eq 0 && "${RESTORED}" -eq 1 ]]; then + log_message "INFO: IP ${IP} already reported." + exit 0 +fi - # Perform the API call and capture both response and HTTP status +if [[ -z "${APIKEY}" || -z "${COMMENT}" || -z "${IP}" || -z "${CATEGORIES}" || -z "${BANTIME}" ]]; then + log_message "ERROR: Missing core argument(s)." + exit 1 +fi + +####################################### +# EARLY CHECKS: (END) +####################################### + +####################################### +# FUNCTIONS: (START) +####################################### + +check_ip_in_abuseipdb() { + local response http_status body total_reports delimiter="HTTP_STATUS:" response=$(curl -s -w "${delimiter}%{http_code}" -G "https://api.abuseipdb.com/api/v2/check" \ --data-urlencode "ipAddress=${IP}" \ -H "Key: ${APIKEY}" \ -H "Accept: application/json" 2>&1) if [[ $? -ne 0 ]]; then - log_message "ERROR CHECK: API failure. Response: ${response}" - exit 1 + log_message "ERROR: API failure. Response: ${response}" + return 1 fi - # Separate the HTTP status code from the response body http_status=$(echo "${response}" | tr -d '\n' | sed -e "s/.*${delimiter}//") body=$(echo "${response}" | sed -e "s/${delimiter}[0-9]*//") - # Handle different HTTP status codes if [[ "${http_status}" =~ ^[0-9]+$ ]]; then - # Handle rate-limiting (HTTP 429) if [[ "${http_status}" -eq 429 ]]; then - log_message "ERROR CHECK: API returned HTTP 429 (Too Many Requests). Response: ${body}" - exit 1 + log_message "ERROR: Rate limited (HTTP 429). Response: ${body}" + return 1 fi - # Handle other non-200 responses if [[ "${http_status}" -ne 200 ]]; then - log_message "ERROR CHECK: API returned HTTP status ${http_status}. Response: ${body}" - exit 1 + log_message "ERROR: HTTP ${http_status}. Response: ${body}" + return 1 fi + else + log_message "ERROR: API failure. Response: ${response}" + return 1 fi - # Extract totalReports total_reports=$(echo "${body}" | jq '.data.totalReports') - - # Finally, check the IP listed on AbuseIPDB if [[ "${total_reports}" -gt 0 ]]; then - return 0 # IP is reported + return 0 else - return 1 # IP is not reported + return 1 fi } -# Function to report AbuseIpDB report_ip_to_abuseipdb() { local response response=$(curl --fail -s 'https://api.abuseipdb.com/api/v2/report' \ @@ -234,48 +291,88 @@ report_ip_to_abuseipdb() { --data-urlencode "ip=${IP}" \ --data "categories=${CATEGORIES}" 2>&1) - # API call fail if [[ $? -ne 0 ]]; then - log_message "ERROR REPORT: API failure. Response: ${response}" - exit 1 + log_message "ERROR: API failure. Response: ${response} for IP: ${IP}" else - log_message "SUCCESS REPORT: Reported IP ${IP} to AbuseIPDB. Local list updated." + log_message "SUCCESS: Reported IP ${IP} to AbuseIPDB." fi } +check_ip_in_db() { + local ip=$1 result + result=$(sqlite3 "file:${SQLITE_DB}?mode=ro" " + ${SQLITE_PRAGMAS} + SELECT 1 FROM banned_ips WHERE ip = '${ip}' LIMIT 1;" + ) -# Set defaults -is_found_local=0 -shouldBanIP=1 + if [[ $? -ne 0 ]]; then + log_message "ERROR: AbuseIPDB database query failed while checking IP ${ip}. Response: ${result}" + return 1 + fi -# Should Ban IP -if grep -m 1 -q -E "^IP=${IP}[[:space:]]+L=[0-9\-]+" "${REPORTED_IP_LIST_FILE}"; then - # IP found locally, check if it's still listed on AbuseIPDB - if check_ip_in_abuseipdb; then - # IP is still listed on AbuseIPDB, no need to report again - log_message "INFO: IP ${IP} has already been reported and remains on AbuseIPDB. No duplicate report made." - shouldBanIP=0 + if [[ -n "${result}" ]]; then + return 0 else - # IP is reported before but not listed on AbuseIPDB, report it again - log_message "INFO: IP ${IP} has already been reported but is no longer listed on AbuseIPDB. Reporting it again." - shouldBanIP=1 - is_found_local=1 + return 1 fi -else +} + +insert_ip_to_db() { + local ip=$1 + local bantime=$2 + sqlite3 "${SQLITE_DB}" " + ${SQLITE_PRAGMAS} + BEGIN IMMEDIATE; + INSERT INTO banned_ips (ip, bantime) + VALUES ('${ip}', ${bantime}) + ON CONFLICT(ip) DO UPDATE SET bantime=${bantime}; + COMMIT; + " + + if [[ $? -ne 0 ]]; then + log_message "ERROR: Failed to insert or update IP ${ip} in the AbuseIPDB database." + fi +} + +####################################### +# FUNCTIONS: (END) +####################################### + +####################################### +# MAIN (START) +####################################### + +( + is_found_local=0 shouldBanIP=1 -fi -# Let's report to AbuseIpdb -if [[ "${shouldBanIP}" -eq 1 ]]; then - # Add the new ban entry to local list kindly - if [[ "${is_found_local}" -eq 0 ]]; then - exec 200<> "${REPORTED_IP_LIST_FILE}" # Open with read/write access - flock -x 200 # Lock - echo "IP=${IP} L=${BANTIME}" >> "${REPORTED_IP_LIST_FILE}" # Write - flock -u 200 # Release the lock - exec 200>&- # Close the file descriptor + if check_ip_in_db $IP; then + is_found_local=1 + if check_ip_in_abuseipdb; then + log_message "INFO: IP ${IP} has already been reported and remains on AbuseIPDB." + shouldBanIP=0 + else + log_message "INFO: IP ${IP} has already been reported but is no longer listed on AbuseIPDB." + shouldBanIP=1 + fi + else + shouldBanIP=1 fi - # Report IP - report_ip_to_abuseipdb -fi + if [[ "${shouldBanIP}" -eq 1 ]]; then + if [[ "${is_found_local}" -eq 0 ]]; then + insert_ip_to_db $IP $BANTIME + fi + report_ip_to_abuseipdb + fi +) >> "${LOG_FILE}" 2>&1 & + +####################################### +# MAIN (END) +####################################### + +####################################### +# ACTIONBAN: (END) +####################################### + +exit 0