From 63788d02d648a42a7afbfa7317e54b907d3e3435 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Hasan=20=C3=87ALI=C5=9EIR?= Date: Wed, 26 Feb 2025 13:01:31 +0300 Subject: [PATCH] re-organized script for both actionstart & actionban - Reorganized script to be used by both 'actionstart' and 'actionban' in 'abuseipdb.local' - Isolated heavy 'actionstart' tasks using nohup to prevent latency - Removed redundant API checks to improve performance and reduce overhead - Implemented a lock mechanism to prevent 'actionban' execution if 'actionstart' fails - Ensured 'actionban' does not run at runtime due to missing dependencies or permission issues --- files/fail2ban-abuseipdb.sh | 294 ++++++++++++++++++++---------------- 1 file changed, 160 insertions(+), 134 deletions(-) diff --git a/files/fail2ban-abuseipdb.sh b/files/fail2ban-abuseipdb.sh index 991505b1..ba4caffb 100644 --- a/files/fail2ban-abuseipdb.sh +++ b/files/fail2ban-abuseipdb.sh @@ -1,107 +1,92 @@ #!/usr/bin/env bash # # Description: -# This script serves as a Fail2Ban `actionban` to report offending IP addresses to AbuseIPDB with custom COMMENTS. -# Mainly whenever Fail2Ban restarts, it calls the actionban function for each IP stored in the database file. -# If you restart your server often this script prevents duplicate reporting to AbuseIPDB by maintaining a local list of already reported IPs. -# Before reporting, it checks both the local list and AbuseIPDB to ensure the IP hasn't -# been reported previously, thereby avoiding redundant entries and potential API rate limiting. +# This script acts as a Fail2Ban `actionstart|actionban` to report offending IPs to AbuseIPDB. +# It allows for 'custom comments' to prevent leaking sensitive information. The main goal is to +# avoid relying on Fail2Ban and instead use a local banned IP list for complete isolation. +# It can also be used with Fail2Ban's `norestored=1` feature to rely on Fail2Ban for preventing +# redundant report actions on restarts. Users can toggle this behavior as needed. +# +# The script performs two API calls for each ban action: +# 1. **/v2/check** - Checks if the IP has already been reported. +# 2. **/v2/report** - Reports the IP if necessary and updates the local banned IP list. +# These two endpoints have separate daily limits, so they do not impact your reporting quota. +# +# To view any failures, check `/var/log/abuseipdb/abuseipdb.log`. # # Integration with Fail2Ban: -# - Place this script in the 'action.d' directory of Fail2Ban. -# - Configure Fail2Ban to use this script as the `actionban`. -# - First edit 'abuseipdb.conf' in '/etc/fail2ban/action.d/abuseipdb.conf' and add following rule, -# - actionban = /etc/fail2ban/action.d/fail2ban-abuseipdb.sh \ -# "" "" "" "" "" -# - Also make sure you set your 'abuseipdb_apikey' in the configuration file. -# - Adjust your jails accordingly. Check the below jail example to also reporting with custom comment via 'tp_comment' +# 1. Edit only 'abuseipdb.local' in 'action.d/abuseipdb.local' and uncomment pre-configured settings. +# 2. Adjust your jails to prevent leaking sensitive information in custom comments via 'tp_comment'. # -# Example jail in 'jail.local': -# [nginx-botsearch] -# enabled = true -# logpath = /var/log/nginx/*.log -# port = http,https -# backend = polling -# tp_comment = Fail2Ban - NGINX bad requests 400-401-403-404-444, high level vulnerability scanning, commonly xmlrpc_attack, wp-login brute force, excessive crawling/scraping -# maxretry = 3 -# findtime = 1d -# bantime = 7200 -# action = %(action_mwl)s -# %(action_abuseipdb)s[matches="%(tp_comment)s", abuseipdb_apikey="YOUR_API_KEY", abuseipdb_category="21,15", bantime="%(bantime)s"] -# -# Start from scratch practise: -# - Stop Fail2ban -# - Clear all existing firewall reject rules from iptables -# - Delete Fail2ban SQLite database and truncate fail2ban.log -# - Force logrotate -# - Follow above implementation steps and adjust all your jails accordingly -# - Start Fail2ban +# Example 'jail' configuration in 'jail.local' to prevent leaking sensitive information in AbuseIPDB reports: +# [nginx-botsearch] +# enabled = true +# logpath = /var/log/nginx/*.log +# port = http,https +# backend = polling +# tp_comment = Fail2Ban - NGINX bad requests 400-401-403-404-444, high level vulnerability scanning +# maxretry = 3 +# findtime = 1d +# bantime = 7200 +# action = %(action_mwl)s +# %(action_abuseipdb)s[matches="%(tp_comment)s", abuseipdb_apikey="YOUR_API_KEY", abuseipdb_category="21,15", bantime="%(bantime)s"] # # Usage: -# This script is intended to be called by Fail2Ban automatically (actionban) and accepts the following arguments: -# 1. - Your AbuseIPDB API key. -# 2. - A comment describing the reason for reporting the IP. -# 3. - The IP address to report. -# 4. - Comma-separated list of abuse categories as per AbuseIPDB's API. -# 5. - Duration for which the IP should be banned (e.g., 600 for 10 minutes). -# +# This script is designed to be triggered automatically by Fail2Ban (`actionstart|actionban`). # Manual Usage: -# For testing purpose before production; -# /etc/fail2ban/action.d/abuseipdb_fail2ban_actionban.sh "your_api_key" "Failed SSH login attempts" "192.0.2.1" "18" "600" +# - For testing purpose before production; +# /etc/fail2ban/action.d/fail2ban_abuseipdb.sh "your_api_key" "Failed SSH login attempts" "192.0.2.1" "18" "600" # # Arguments: -# APIKEY - Required. -# COMMENT - Required. -# IP - Required. -# CATEGORIES - Required. -# BANTIME - Required. -# -# Configurations (Need to set): -# - REPORTED_IP_LIST_FILE: Path to the local file storing reported IPs and their ban times. -# - LOG_FILE: Path to the log file where actions and events are recorded. +# $1 APIKEY - Required (Core). Retrieved automatically from the Fail2Ban 'jail'. | Your AbuseIPDB API key. +# $2 COMMENT - Required (Core). Retrieved automatically from the Fail2Ban 'jail'. | A custom comment to prevent the leakage of sensitive data when reporting +# $3 IP - Required (Core). Retrieved automatically from the Fail2Ban 'jail'. | The IP address to report. +# $4 CATEGORIES - Required (Core). Retrieved automatically from the Fail2Ban 'jail'. | Abuse categories as per AbuseIPDB's API +# $5 BANTIME - Required (Core). Retrieved automatically from the Fail2Ban 'jail'. | Ban duration +# $6 RESTORED - Required (Core). Retrieved automatically from the Fail2Ban '' | Status of restored tickets +# $7 BYPASS_FAIL2BAN - Required (User defined). Must be defined in 'action.d/abuseipdb.local'. | Bypassing Fail2Ban on restarts +# $8 LOCAL_LIST - Required (User defined). Must be defined in 'action.d/abuseipdb.local'. | Path to the main banned IP list used by the script +# $9 LOG_FILE - Required (User defined). Must be defined in 'action.d/abuseipdb.local'. | Path to the log file where actions and events are recorded by the script # # Dependencies: -# - curl: For making API requests to AbuseIPDB. -# - jq: For parsing JSON responses. -# - flock: Prevent data corruption. -# -# General Considerations: -# - The script does not interact with or rely on Fail2Ban’s SQLite database, as the Fail2ban database setup can vary across different environments. -# - The script performs two API calls to AbuseIPDB for each ban action in order to determine whether the IP should be reported. -# - First Call: /v2/check → Checks if the IP is already reported. -# - Second Call: /v2/report → Reports the IP if necessary. -# - These two endpoints have separate daily limits, so does not affect your reporting quota. -# - When multiple instances of the script try to write to the REPORTED_IP_LIST_FILE concurrently we need to prevent data corruption. 'flock' used for that reason. -# - Never delete, truncate or try to sync REPORTED_IP_LIST_FILE with Fail2ban SQLite. -# - Even you reset fail2ban SQLite, continue to keep REPORTED_IP_LIST_FILE same, It serves as the script’s local tracking system. -# - When you go production keep watching '/var/log/abuseipdb-report.log' for any abnormal fails. +# curl: For making API requests to AbuseIPDB. +# jq: For parsing JSON responses. +# flock: Prevent data corruption. # # Return Codes: -# 0 - IP is reported -# 1 - IP is not reported +# 0 - 'AbuseIPDB' IP is reported. +# 1 - 'AbuseIPDB' IP is not reported. # # Exit Codes: -# 1 - API-related failure +# 0 - 'norestored' restored tickets enabled. +# 0 - 'actionstart' tasks completed. +# 1 - 'actionstart' tasks cannot completed and lock file created. +# 1 - 'AbuseIPDB' API-related failure. # # Author: -# Hasan ÇALIŞIR - PSAUXIT +# Hasan ÇALIŞIR # hasan.calisir@psauxit.com -# https://www.psauxit.com/ +# https://github.com/hsntgm -# Arguments +# This script is used for both: 'actionstart' and 'actionban' in 'action.d/abuseipdb.local' +# It dynamically assigns arguments based on the action type +# and provides default values for missing user settings to prevent failures. APIKEY="$1" COMMENT="$2" IP="$3" CATEGORIES="$4" BANTIME="$5" - -# Configuration -REPORTED_IP_LIST_FILE="/etc/fail2ban/action.d/abuseipdb-reported-ip-list" -LOG_FILE="/var/log/abuseipdb-report.log" - -# Set defaults -is_found_local=0 -shouldBanIP=1 +RESTORED="${6}" +BYPASS_FAIL2BAN="${7:-0}" +if [[ "$1" == "--actionstart" ]]; then + # When triggered by 'actionstart' + REPORTED_IP_LIST_FILE="${2:-/var/log/abuseipdb/abuseipdb-banned.log}" + LOG_FILE="${3:-/var/log/abuseipdb/abuseipdb.log}" +else + # When triggered by 'actionban' + REPORTED_IP_LIST_FILE="${8:-/var/log/abuseipdb/abuseipdb-banned.log}" + LOG_FILE="${9:-/var/log/abuseipdb/abuseipdb.log}" +fi # Log messages log_message() { @@ -109,40 +94,93 @@ log_message() { echo "$(date +"%Y-%m-%d %H:%M:%S") - ${message}" >> "${LOG_FILE}" } -# Arguments validation -if [[ -z "$1" || -z "$2" || -z "$3" || -z "$4" || -z "$5" ]]; then - log_message "FATAL: Usage: $0 " - exit 1 +# Define lock file +LOCK_FILE="/tmp/abuseipdb_actionstart.lock" + +# Function to remove lock file if it exists +remove_lock() { + if [[ -f "${LOCK_FILE}" ]]; then + rm -f "${LOCK_FILE:?}" + fi +} + +# Function to create lock file +create_lock() { + if [[ ! -f "${LOCK_FILE}" ]]; then + touch "${LOCK_FILE}" + fi +} + +# Check if the script was triggered by 'actionstart' early in execution. +# This ensures necessary checks is performed before proceeding only once. +# This check runs on background always with 'nohup' to prevent latency. +# We listen exit codes carefully to allow or not further runtime 'actionban' events +if [[ "$1" == "--actionstart" ]]; then + # Trap exit signal to create/remove lock file based on exit status + trap 'if [[ $? -ne 0 ]]; then create_lock; else remove_lock; fi' EXIT + + # Ensure the directory for the reported IP list exists + LOG_DIR=$(dirname "${REPORTED_IP_LIST_FILE}") + if [[ ! -d "${LOG_DIR}" ]]; then + mkdir -p "${LOG_DIR}" || exit 1 + fi + + # Ensure the reported IP list and log file exist + for file in "${REPORTED_IP_LIST_FILE}" "${LOG_FILE}"; do + if [[ ! -f "${file}" ]]; then + touch "${file}" || exit 1 + fi + done + + # Check runtime dependencies + dependencies=("curl" "jq" "flock") + for dep in "${dependencies[@]}"; do + if ! command -v "${dep}" &>/dev/null; then + log_message "FATAL: -${dep} is not installed. Please install -${dep} to proceed." + exit 1 + fi + done + + # Tasks completed, quit nicely + exit 0 fi -# Ensure the directory for the reported IP list exists -REPORTED_IP_DIR=$(dirname "${REPORTED_IP_LIST_FILE}") -if [[ ! -d "${REPORTED_IP_DIR}" ]]; then - mkdir -p "${REPORTED_IP_DIR}" || { log_message "FATAL: Could not create directory ${REPORTED_IP_DIR}"; exit 1; } -fi - -# Ensure the reported IP list file exists and is writable -if [[ ! -f "${REPORTED_IP_LIST_FILE}" ]]; then - touch "${REPORTED_IP_LIST_FILE}" || { log_message "FATAL: Could not create file ${REPORTED_IP_LIST_FILE}"; exit 1; } -fi - -if [[ ! -w "${REPORTED_IP_LIST_FILE}" ]]; then - log_message "FATAL: File ${REPORTED_IP_LIST_FILE} is not writable." - exit 1 -fi - -# Check runtime dependencies -dependencies=("curl" "jq" "flock") -for dep in "${dependencies[@]}"; do - if ! command -v "${dep}" &>/dev/null; then - log_message "FATAL: -${dep} is not installed. Please install -${dep} to proceed." +# If the 'actionstart' failed, prevent 'actionban'. +# This stops 'actionban' from being triggered during runtime due to missing dependencies or permission issues. +# A failed initial 'actionstart' check indicates a failure to report to AbuseIPDB. +if [[ -f "${LOCK_FILE}" ]]; then + if [[ -f "${LOG_FILE}" ]]; then + log_message "FATAL: Failed due to a permission issue or missing dependency. Reporting to AbuseIPDB failed." + exit 1 + else exit 1 fi -done +fi -# Check if the IP is listed on AbuseIPDB +# If 'BYPASS_FAIL2BAN' is disabled, Fail2Ban will be relied upon during restarts. +# This prevents duplicate reports when Fail2Ban is restarted. +# This setting is 'OPTIONAL' and can be overridden in 'action.d/abuseipdb.local'. +# If enabled, Fail2Ban is bypassed completely, +# and script takes full control to determine which IP to report based on +# the local banned IP list even on Fail2Ban restarts. +if [[ "${BYPASS_FAIL2BAN}" -eq 0 ]]; then + if [[ "${RESTORED}" -eq 1 ]]; then + log_message "INFO NORESTORED: IP ${IP} has already been reported. No duplicate report made after restart." + exit 0 + fi +fi + +# Validate core arguments: Ensure all required core args are provided. +# These values are expected to be passed by Fail2Ban 'jail' during execution. +# Also for manual testing purpose before production. +if [[ -z "$1" || -z "$2" || -z "$3" || -z "$4" || -z "$5" ]]; then + log_message "FATAL: Missing core argument" + exit 1 +fi + +# Function to check if the IP is listed on AbuseIPDB check_ip_in_abuseipdb() { - local response http_status body total_reports error_detail + local response http_status body total_reports local delimiter="HTTP_STATUS:" # Perform the API call and capture both response and HTTP status @@ -160,35 +198,19 @@ check_ip_in_abuseipdb() { http_status=$(echo "${response}" | tr -d '\n' | sed -e "s/.*${delimiter}//") body=$(echo "${response}" | sed -e "s/${delimiter}[0-9]*//") - # Check if the response body is empty - if [[ -z "${body}" ]]; then - log_message "ERROR CHECK: API response empty." - exit 1 - fi - - # Validate that the response is valid JSON - if ! echo "${body}" | jq . >/dev/null 2>&1; then - log_message "ERROR CHECK: API response malformed. Response: ${body}" - exit 1 - fi - # Handle different HTTP status codes if [[ "${http_status}" =~ ^[0-9]+$ ]]; then - if [[ "${http_status}" -ge 200 && "${http_status}" -lt 300 ]]; then - # Successful HTTP response; check for API-level errors - if echo "${body}" | jq -e '.errors | length > 0' >/dev/null 2>&1; then - error_detail=$(echo "${body}" | jq -r '.errors[].detail') - log_message "ERROR CHECK: API returned errors. Detail: ${error_detail}" - exit 1 - fi - else - # Non-successful HTTP status - log_message "ERROR CHECK: API returned ${http_status}. Response: ${body}" + # Handle rate-limiting (HTTP 429) + if [[ "${http_status}" -eq 429 ]]; then + log_message "ERROR CHECK: API returned HTTP 429 (Too Many Requests). Response: ${body}" + exit 1 + fi + + # Handle other non-200 responses + if [[ "${http_status}" -ne 200 ]]; then + log_message "ERROR CHECK: API returned HTTP status ${http_status}. Response: ${body}" exit 1 fi - else - log_message "ERROR CHECK: HTTP status '${http_status}' is not numeric." - exit 1 fi # Extract totalReports @@ -202,7 +224,7 @@ check_ip_in_abuseipdb() { fi } -# Report to AbuseIpDB +# Function to report AbuseIpDB report_ip_to_abuseipdb() { local response response=$(curl --fail -s 'https://api.abuseipdb.com/api/v2/report' \ @@ -221,6 +243,11 @@ report_ip_to_abuseipdb() { fi } + +# Set defaults +is_found_local=0 +shouldBanIP=1 + # Should Ban IP if grep -m 1 -q -E "^IP=${IP}[[:space:]]+L=[0-9\-]+" "${REPORTED_IP_LIST_FILE}"; then # IP found locally, check if it's still listed on AbuseIPDB @@ -235,11 +262,10 @@ if grep -m 1 -q -E "^IP=${IP}[[:space:]]+L=[0-9\-]+" "${REPORTED_IP_LIST_FILE}"; is_found_local=1 fi else - # New comer, welcome to hell shouldBanIP=1 fi -# Report to AbuseIpdb +# Let's report to AbuseIpdb if [[ "${shouldBanIP}" -eq 1 ]]; then # Add the new ban entry to local list kindly if [[ "${is_found_local}" -eq 0 ]]; then