mirror of https://github.com/fail2ban/fail2ban
migrate banned IPs to SQLite DB and prevent actionban latency
- Replace local file storage with AbuseIPDB SQLite database. - Offload heavy tasks to background to avoid latency during concurrent actionban calls. - Add global lock to ensure actionstart runs only once across all jails.pull/3948/head
Hasan ÇALIŞIR
GitHub
parent
b5314961e8
commit
d13660c588
|
|
@ -3,9 +3,9 @@
|
|||
# Description:
|
||||
# This script acts as a Fail2Ban `actionstart|actionban` to report offending IPs to AbuseIPDB.
|
||||
# It allows for 'custom comments' to prevent leaking sensitive information. The main goal is to
|
||||
# avoid relying on Fail2Ban and instead use a local banned IP list for complete isolation.
|
||||
# avoid relying on Fail2Ban and instead use a separate AbuseIPDB SQLite database for complete isolation.
|
||||
# It can also be used with Fail2Ban's `norestored=1` feature to rely on Fail2Ban for preventing
|
||||
# redundant report actions on restarts. Users can toggle this behavior as needed.
|
||||
# redundant reporting on restarts. Users can toggle this behavior as needed.
|
||||
#
|
||||
# The script performs two API calls for each ban action:
|
||||
# 1. **/v2/check** - Checks if the IP has already been reported.
|
||||
|
|
@ -33,7 +33,7 @@
|
|||
#
|
||||
# Usage:
|
||||
# This script is designed to be triggered automatically by Fail2Ban (`actionstart|actionban`).
|
||||
# Manual Usage:
|
||||
# For testing (manual execution):
|
||||
# - For testing purpose before production;
|
||||
# /etc/fail2ban/action.d/fail2ban_abuseipdb.sh "your_api_key" "Failed SSH login attempts" "192.0.2.1" "18" "600"
|
||||
#
|
||||
|
|
@ -45,46 +45,34 @@
|
|||
# $5 BANTIME - Required (Core). Retrieved automatically from the Fail2Ban 'jail'. | Ban duration
|
||||
# $6 RESTORED - Required (Core). Retrieved automatically from the Fail2Ban '<restored>' | Status of restored tickets
|
||||
# $7 BYPASS_FAIL2BAN - Required (User defined). Must be defined in 'action.d/abuseipdb.local'. | Bypassing Fail2Ban on restarts
|
||||
# $8 LOCAL_LIST - Required (User defined). Must be defined in 'action.d/abuseipdb.local'. | Path to the main banned IP list used by the script
|
||||
# $9 LOG_FILE - Required (User defined). Must be defined in 'action.d/abuseipdb.local'. | Path to the log file where actions and events are recorded by the script
|
||||
# $2|$8 SQLITE_DB - Required (User defined). Must be defined in 'action.d/abuseipdb.local'. | Path to the main AbuseIPDB SQLite database
|
||||
# $3|$9 LOG_FILE - Required (User defined). Must be defined in 'action.d/abuseipdb.local'. | Path to the log file where actions and events are recorded by the script
|
||||
#
|
||||
# Dependencies:
|
||||
# curl: For making API requests to AbuseIPDB.
|
||||
# jq: For parsing JSON responses.
|
||||
# flock: Prevent data corruption.
|
||||
#
|
||||
# Return Codes:
|
||||
# 0 - 'AbuseIPDB' IP is reported.
|
||||
# 1 - 'AbuseIPDB' IP is not reported.
|
||||
#
|
||||
# Exit Codes:
|
||||
# 0 - 'norestored' restored tickets enabled.
|
||||
# 0 - 'actionstart' tasks completed.
|
||||
# 1 - 'actionstart' tasks cannot completed and lock file created.
|
||||
# 1 - 'AbuseIPDB' API-related failure.
|
||||
# sqlite3: Local AbuseIPDB db.
|
||||
#
|
||||
# Author:
|
||||
# Hasan ÇALIŞIR
|
||||
# hasan.calisir@psauxit.com
|
||||
# https://github.com/hsntgm
|
||||
|
||||
# This script is used for both: 'actionstart' and 'actionban' in 'action.d/abuseipdb.local'
|
||||
# It dynamically assigns arguments based on the action type
|
||||
# and provides default values for missing user settings to prevent failures.
|
||||
#######################################
|
||||
# HELPERS: (START)
|
||||
#######################################
|
||||
|
||||
APIKEY="$1"
|
||||
COMMENT="$2"
|
||||
IP="$3"
|
||||
CATEGORIES="$4"
|
||||
BANTIME="$5"
|
||||
RESTORED="${6}"
|
||||
RESTORED="$6"
|
||||
BYPASS_FAIL2BAN="${7:-0}"
|
||||
if [[ "$1" == "--actionstart" ]]; then
|
||||
# When triggered by 'actionstart'
|
||||
REPORTED_IP_LIST_FILE="${2:-/var/log/abuseipdb/abuseipdb-banned.log}"
|
||||
SQLITE_DB="${2:-/var/lib/fail2ban/abuseipdb/fail2ban_abuseipdb}"
|
||||
LOG_FILE="${3:-/var/log/abuseipdb/abuseipdb.log}"
|
||||
else
|
||||
# When triggered by 'actionban'
|
||||
REPORTED_IP_LIST_FILE="${8:-/var/log/abuseipdb/abuseipdb-banned.log}"
|
||||
SQLITE_DB="${8:-/var/lib/fail2ban/abuseipdb/fail2ban_abuseipdb}"
|
||||
LOG_FILE="${9:-/var/log/abuseipdb/abuseipdb.log}"
|
||||
fi
|
||||
|
||||
|
|
@ -94,137 +82,206 @@ log_message() {
|
|||
echo "$(date +"%Y-%m-%d %H:%M:%S") - ${message}" >> "${LOG_FILE}"
|
||||
}
|
||||
|
||||
# Define lock file
|
||||
LOCK_FILE="/tmp/abuseipdb_actionstart.lock"
|
||||
# Lock files for 'actionstart' status
|
||||
LOCK_BAN="/tmp/abuseipdb_actionstart.lock"
|
||||
LOCK_DONE="/tmp/abuseipdb_actionstart.done"
|
||||
|
||||
# Function to remove lock file if it exists
|
||||
# Remove lock file
|
||||
remove_lock() {
|
||||
if [[ -f "${LOCK_FILE}" ]]; then
|
||||
rm -f "${LOCK_FILE:?}"
|
||||
fi
|
||||
[[ -f "${LOCK_BAN}" ]] && rm -f "${LOCK_BAN}"
|
||||
}
|
||||
|
||||
# Function to create lock file
|
||||
# Create lock file
|
||||
create_lock() {
|
||||
if [[ ! -f "${LOCK_FILE}" ]]; then
|
||||
touch "${LOCK_FILE}"
|
||||
fi
|
||||
[[ ! -f "${LOCK_BAN}" ]] && touch "${LOCK_BAN}"
|
||||
}
|
||||
|
||||
# Check if the script was triggered by 'actionstart' early in execution.
|
||||
# This ensures necessary checks is performed before proceeding only once.
|
||||
# This check runs on background always with 'nohup' to prevent latency.
|
||||
# We listen exit codes carefully to allow or not further runtime 'actionban' events
|
||||
# Pre-defined SQLite PRAGMAS
|
||||
SQLITE_PRAGMAS="
|
||||
PRAGMA journal_mode=WAL;
|
||||
PRAGMA synchronous=NORMAL;
|
||||
PRAGMA temp_store=MEMORY;
|
||||
PRAGMA locking_mode=NORMAL;
|
||||
PRAGMA cache_size=-256000;
|
||||
PRAGMA busy_timeout=10000;
|
||||
"
|
||||
|
||||
#######################################
|
||||
# HELPERS: (END)
|
||||
#######################################
|
||||
|
||||
#######################################
|
||||
# ACTIONSTART: (START)
|
||||
#######################################
|
||||
|
||||
########################################
|
||||
# Triggered by 'actionstart'
|
||||
# to perform necessary checks
|
||||
# and AbuseIPDB SQLite initialization.
|
||||
#
|
||||
# - Ensures required checks are done.
|
||||
# - Runs in the background with 'nohup'
|
||||
# on initial start to prevent latency.
|
||||
# - Listens for exit codes to control
|
||||
# further 'actionban' events via the
|
||||
# 'lock' mechanism.
|
||||
# - Check 'abuseipdb.local' for
|
||||
# integration details.
|
||||
########################################
|
||||
|
||||
if [[ "$1" == "--actionstart" ]]; then
|
||||
# Trap exit signal to create/remove lock file based on exit status
|
||||
trap 'if [[ $? -ne 0 ]]; then create_lock; else remove_lock; fi' EXIT
|
||||
if [[ ! -f "${LOCK_DONE}" ]]; then
|
||||
trap 'if [[ $? -ne 0 ]]; then create_lock; else remove_lock; fi' EXIT
|
||||
|
||||
# Ensure the directory for the reported IP list exists
|
||||
LOG_DIR=$(dirname "${REPORTED_IP_LIST_FILE}")
|
||||
if [[ ! -d "${LOG_DIR}" ]]; then
|
||||
mkdir -p "${LOG_DIR}" || exit 1
|
||||
fi
|
||||
|
||||
# Ensure the reported IP list and log file exist
|
||||
for file in "${REPORTED_IP_LIST_FILE}" "${LOG_FILE}"; do
|
||||
if [[ ! -f "${file}" ]]; then
|
||||
touch "${file}" || exit 1
|
||||
SQLITE_DIR=$(dirname "${SQLITE_DB}")
|
||||
if [[ ! -d "${SQLITE_DIR}" ]]; then
|
||||
mkdir -p "${SQLITE_DIR}" || exit 1
|
||||
fi
|
||||
done
|
||||
|
||||
# Check runtime dependencies
|
||||
dependencies=("curl" "jq" "flock")
|
||||
for dep in "${dependencies[@]}"; do
|
||||
if ! command -v "${dep}" &>/dev/null; then
|
||||
log_message "FATAL: -${dep} is not installed. Please install -${dep} to proceed."
|
||||
if [[ ! -f "${LOG_FILE}" ]]; then
|
||||
touch "${LOG_FILE}" || exit 1
|
||||
fi
|
||||
|
||||
for dep in curl jq sqlite3; do
|
||||
if ! command -v "${dep}" &>/dev/null; then
|
||||
log_message "ERROR: ${dep} is not installed. Please install ${dep}"
|
||||
exit 1
|
||||
fi
|
||||
done
|
||||
|
||||
if [[ ! -f "${SQLITE_DB}" ]]; then
|
||||
sqlite3 "${SQLITE_DB}" "
|
||||
${SQLITE_PRAGMAS}
|
||||
CREATE TABLE IF NOT EXISTS banned_ips (ip TEXT PRIMARY KEY, bantime INTEGER);
|
||||
CREATE INDEX IF NOT EXISTS idx_ip ON banned_ips(ip);
|
||||
"
|
||||
fi
|
||||
|
||||
result=$(sqlite3 "file:${SQLITE_DB}?mode=ro" "SELECT name FROM sqlite_master WHERE type='table' AND name='banned_ips';")
|
||||
if ! [[ -n "${result}" ]]; then
|
||||
log_message "ERROR: AbuseIPDB database initialization failed."
|
||||
rm -f "${SQLITE_DB:?}"
|
||||
exit 1
|
||||
else
|
||||
log_message "SUCCESS: The AbuseIPDB database is now ready for connection."
|
||||
fi
|
||||
done
|
||||
|
||||
# Tasks completed, quit nicely
|
||||
exit 0
|
||||
fi
|
||||
|
||||
# If the 'actionstart' failed, prevent 'actionban'.
|
||||
# This stops 'actionban' from being triggered during runtime due to missing dependencies or permission issues.
|
||||
# A failed initial 'actionstart' check indicates a failure to report to AbuseIPDB.
|
||||
if [[ -f "${LOCK_FILE}" ]]; then
|
||||
if [[ -f "${LOG_FILE}" ]]; then
|
||||
log_message "FATAL: Failed due to a permission issue or missing dependency. Reporting to AbuseIPDB failed."
|
||||
exit 1
|
||||
touch "${LOCK_DONE}" || exit 1
|
||||
exit 0
|
||||
else
|
||||
exit 1
|
||||
fi
|
||||
fi
|
||||
|
||||
# If 'BYPASS_FAIL2BAN' is disabled, Fail2Ban will be relied upon during restarts.
|
||||
# This prevents duplicate reports when Fail2Ban is restarted.
|
||||
# This setting is 'OPTIONAL' and can be overridden in 'action.d/abuseipdb.local'.
|
||||
# If enabled, Fail2Ban is bypassed completely,
|
||||
# and script takes full control to determine which IP to report based on
|
||||
# the local banned IP list even on Fail2Ban restarts.
|
||||
if [[ "${BYPASS_FAIL2BAN}" -eq 0 ]]; then
|
||||
if [[ "${RESTORED}" -eq 1 ]]; then
|
||||
log_message "INFO NORESTORED: IP ${IP} has already been reported. No duplicate report made after restart."
|
||||
exit 0
|
||||
fi
|
||||
fi
|
||||
|
||||
# Validate core arguments: Ensure all required core args are provided.
|
||||
# These values are expected to be passed by Fail2Ban 'jail' during execution.
|
||||
# Also for manual testing purpose before production.
|
||||
if [[ -z "$1" || -z "$2" || -z "$3" || -z "$4" || -z "$5" ]]; then
|
||||
log_message "FATAL: Missing core argument"
|
||||
#######################################
|
||||
# ACTIONSTART: (END)
|
||||
#######################################
|
||||
|
||||
#######################################
|
||||
# ACTIONBAN: (START)
|
||||
#######################################
|
||||
|
||||
#######################################
|
||||
# 1) Prevent 'actionban' if
|
||||
# 'actionstart' fails.
|
||||
#
|
||||
# If 'actionstart' fails, block
|
||||
# 'actionban' to prevent issues from
|
||||
# missing dependencies or permission
|
||||
# errors.
|
||||
#######################################
|
||||
|
||||
#######################################
|
||||
# 2) Fail2Ban restart handling &
|
||||
# duplicate report prevention.
|
||||
#
|
||||
# - If 'BYPASS_FAIL2BAN' is disabled,
|
||||
# Fail2Ban manages reports on restart
|
||||
# and prevents duplicate submissions.
|
||||
# - This setting can be overridden in
|
||||
# 'action.d/abuseipdb.local'.
|
||||
# - If enabled, Fail2Ban is bypassed,
|
||||
# and the script independently
|
||||
# decides which IPs to report based
|
||||
# on the AbuseIPDB db, even after
|
||||
# restarts.
|
||||
#######################################
|
||||
|
||||
#######################################
|
||||
# 3) Core argument validation
|
||||
#
|
||||
# - Ensures all required arguments
|
||||
# are provided.
|
||||
# - Expected from Fail2Ban 'jail' or
|
||||
# for manual testing before
|
||||
# production deployment.
|
||||
#######################################
|
||||
|
||||
#######################################
|
||||
# EARLY CHECKS: (START)
|
||||
#######################################
|
||||
|
||||
if [[ -f "${LOCK_BAN}" ]]; then
|
||||
[[ -f "${LOG_FILE}" ]] && log_message "ERROR: Initialization failed! (actionstart). Reporting for IP ${IP} is blocked."
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# Function to check if the IP is listed on AbuseIPDB
|
||||
check_ip_in_abuseipdb() {
|
||||
local response http_status body total_reports
|
||||
local delimiter="HTTP_STATUS:"
|
||||
if [[ "${BYPASS_FAIL2BAN}" -eq 0 && "${RESTORED}" -eq 1 ]]; then
|
||||
log_message "INFO: IP ${IP} already reported."
|
||||
exit 0
|
||||
fi
|
||||
|
||||
# Perform the API call and capture both response and HTTP status
|
||||
if [[ -z "${APIKEY}" || -z "${COMMENT}" || -z "${IP}" || -z "${CATEGORIES}" || -z "${BANTIME}" ]]; then
|
||||
log_message "ERROR: Missing core argument(s)."
|
||||
exit 1
|
||||
fi
|
||||
|
||||
#######################################
|
||||
# EARLY CHECKS: (END)
|
||||
#######################################
|
||||
|
||||
#######################################
|
||||
# FUNCTIONS: (START)
|
||||
#######################################
|
||||
|
||||
check_ip_in_abuseipdb() {
|
||||
local response http_status body total_reports delimiter="HTTP_STATUS:"
|
||||
response=$(curl -s -w "${delimiter}%{http_code}" -G "https://api.abuseipdb.com/api/v2/check" \
|
||||
--data-urlencode "ipAddress=${IP}" \
|
||||
-H "Key: ${APIKEY}" \
|
||||
-H "Accept: application/json" 2>&1)
|
||||
|
||||
if [[ $? -ne 0 ]]; then
|
||||
log_message "ERROR CHECK: API failure. Response: ${response}"
|
||||
exit 1
|
||||
log_message "ERROR: API failure. Response: ${response}"
|
||||
return 1
|
||||
fi
|
||||
|
||||
# Separate the HTTP status code from the response body
|
||||
http_status=$(echo "${response}" | tr -d '\n' | sed -e "s/.*${delimiter}//")
|
||||
body=$(echo "${response}" | sed -e "s/${delimiter}[0-9]*//")
|
||||
|
||||
# Handle different HTTP status codes
|
||||
if [[ "${http_status}" =~ ^[0-9]+$ ]]; then
|
||||
# Handle rate-limiting (HTTP 429)
|
||||
if [[ "${http_status}" -eq 429 ]]; then
|
||||
log_message "ERROR CHECK: API returned HTTP 429 (Too Many Requests). Response: ${body}"
|
||||
exit 1
|
||||
log_message "ERROR: Rate limited (HTTP 429). Response: ${body}"
|
||||
return 1
|
||||
fi
|
||||
|
||||
# Handle other non-200 responses
|
||||
if [[ "${http_status}" -ne 200 ]]; then
|
||||
log_message "ERROR CHECK: API returned HTTP status ${http_status}. Response: ${body}"
|
||||
exit 1
|
||||
log_message "ERROR: HTTP ${http_status}. Response: ${body}"
|
||||
return 1
|
||||
fi
|
||||
else
|
||||
log_message "ERROR: API failure. Response: ${response}"
|
||||
return 1
|
||||
fi
|
||||
|
||||
# Extract totalReports
|
||||
total_reports=$(echo "${body}" | jq '.data.totalReports')
|
||||
|
||||
# Finally, check the IP listed on AbuseIPDB
|
||||
if [[ "${total_reports}" -gt 0 ]]; then
|
||||
return 0 # IP is reported
|
||||
return 0
|
||||
else
|
||||
return 1 # IP is not reported
|
||||
return 1
|
||||
fi
|
||||
}
|
||||
|
||||
# Function to report AbuseIpDB
|
||||
report_ip_to_abuseipdb() {
|
||||
local response
|
||||
response=$(curl --fail -s 'https://api.abuseipdb.com/api/v2/report' \
|
||||
|
|
@ -234,48 +291,88 @@ report_ip_to_abuseipdb() {
|
|||
--data-urlencode "ip=${IP}" \
|
||||
--data "categories=${CATEGORIES}" 2>&1)
|
||||
|
||||
# API call fail
|
||||
if [[ $? -ne 0 ]]; then
|
||||
log_message "ERROR REPORT: API failure. Response: ${response}"
|
||||
exit 1
|
||||
log_message "ERROR: API failure. Response: ${response} for IP: ${IP}"
|
||||
else
|
||||
log_message "SUCCESS REPORT: Reported IP ${IP} to AbuseIPDB. Local list updated."
|
||||
log_message "SUCCESS: Reported IP ${IP} to AbuseIPDB."
|
||||
fi
|
||||
}
|
||||
|
||||
check_ip_in_db() {
|
||||
local ip=$1 result
|
||||
result=$(sqlite3 "file:${SQLITE_DB}?mode=ro" "
|
||||
${SQLITE_PRAGMAS}
|
||||
SELECT 1 FROM banned_ips WHERE ip = '${ip}' LIMIT 1;"
|
||||
)
|
||||
|
||||
# Set defaults
|
||||
is_found_local=0
|
||||
shouldBanIP=1
|
||||
if [[ $? -ne 0 ]]; then
|
||||
log_message "ERROR: AbuseIPDB database query failed while checking IP ${ip}. Response: ${result}"
|
||||
return 1
|
||||
fi
|
||||
|
||||
# Should Ban IP
|
||||
if grep -m 1 -q -E "^IP=${IP}[[:space:]]+L=[0-9\-]+" "${REPORTED_IP_LIST_FILE}"; then
|
||||
# IP found locally, check if it's still listed on AbuseIPDB
|
||||
if check_ip_in_abuseipdb; then
|
||||
# IP is still listed on AbuseIPDB, no need to report again
|
||||
log_message "INFO: IP ${IP} has already been reported and remains on AbuseIPDB. No duplicate report made."
|
||||
shouldBanIP=0
|
||||
if [[ -n "${result}" ]]; then
|
||||
return 0
|
||||
else
|
||||
# IP is reported before but not listed on AbuseIPDB, report it again
|
||||
log_message "INFO: IP ${IP} has already been reported but is no longer listed on AbuseIPDB. Reporting it again."
|
||||
shouldBanIP=1
|
||||
is_found_local=1
|
||||
return 1
|
||||
fi
|
||||
else
|
||||
}
|
||||
|
||||
insert_ip_to_db() {
|
||||
local ip=$1
|
||||
local bantime=$2
|
||||
sqlite3 "${SQLITE_DB}" "
|
||||
${SQLITE_PRAGMAS}
|
||||
BEGIN IMMEDIATE;
|
||||
INSERT INTO banned_ips (ip, bantime)
|
||||
VALUES ('${ip}', ${bantime})
|
||||
ON CONFLICT(ip) DO UPDATE SET bantime=${bantime};
|
||||
COMMIT;
|
||||
"
|
||||
|
||||
if [[ $? -ne 0 ]]; then
|
||||
log_message "ERROR: Failed to insert or update IP ${ip} in the AbuseIPDB database."
|
||||
fi
|
||||
}
|
||||
|
||||
#######################################
|
||||
# FUNCTIONS: (END)
|
||||
#######################################
|
||||
|
||||
#######################################
|
||||
# MAIN (START)
|
||||
#######################################
|
||||
|
||||
(
|
||||
is_found_local=0
|
||||
shouldBanIP=1
|
||||
fi
|
||||
|
||||
# Let's report to AbuseIpdb
|
||||
if [[ "${shouldBanIP}" -eq 1 ]]; then
|
||||
# Add the new ban entry to local list kindly
|
||||
if [[ "${is_found_local}" -eq 0 ]]; then
|
||||
exec 200<> "${REPORTED_IP_LIST_FILE}" # Open with read/write access
|
||||
flock -x 200 # Lock
|
||||
echo "IP=${IP} L=${BANTIME}" >> "${REPORTED_IP_LIST_FILE}" # Write
|
||||
flock -u 200 # Release the lock
|
||||
exec 200>&- # Close the file descriptor
|
||||
if check_ip_in_db $IP; then
|
||||
is_found_local=1
|
||||
if check_ip_in_abuseipdb; then
|
||||
log_message "INFO: IP ${IP} has already been reported and remains on AbuseIPDB."
|
||||
shouldBanIP=0
|
||||
else
|
||||
log_message "INFO: IP ${IP} has already been reported but is no longer listed on AbuseIPDB."
|
||||
shouldBanIP=1
|
||||
fi
|
||||
else
|
||||
shouldBanIP=1
|
||||
fi
|
||||
|
||||
# Report IP
|
||||
report_ip_to_abuseipdb
|
||||
fi
|
||||
if [[ "${shouldBanIP}" -eq 1 ]]; then
|
||||
if [[ "${is_found_local}" -eq 0 ]]; then
|
||||
insert_ip_to_db $IP $BANTIME
|
||||
fi
|
||||
report_ip_to_abuseipdb
|
||||
fi
|
||||
) >> "${LOG_FILE}" 2>&1 &
|
||||
|
||||
#######################################
|
||||
# MAIN (END)
|
||||
#######################################
|
||||
|
||||
#######################################
|
||||
# ACTIONBAN: (END)
|
||||
#######################################
|
||||
|
||||
exit 0
|
||||
|
|
|
|||
Loading…
Reference in New Issue