shell/backup-media.sh

#!/bin/bash

set -e

# Load the unified backup metrics library
SCRIPT_DIR="$(dirname "$(readlink -f "$0")")"
LIB_DIR="$SCRIPT_DIR/lib"
if [[ -f "$LIB_DIR/unified-backup-metrics.sh" ]]; then
    # shellcheck source=lib/unified-backup-metrics.sh
    source "$LIB_DIR/unified-backup-metrics.sh"
    METRICS_ENABLED=true
else
    echo "Warning: Unified backup metrics library not found at $LIB_DIR/unified-backup-metrics.sh"
    METRICS_ENABLED=false
fi

# Color codes for output
RED='\033[0;31m'
GREEN='\033[0;32m'
YELLOW='\033[1;33m'
BLUE='\033[0;34m'
CYAN='\033[0;36m'
NC='\033[0m' # No Color

# Performance tracking variables
# Configuration
MAX_BACKUP_AGE_DAYS=30
MAX_BACKUPS_TO_KEEP=10
BACKUP_ROOT="/mnt/share/media/backups"
LOG_ROOT="/mnt/share/media/backups/logs"
SCRIPT_DIR="$(dirname "$(readlink -f "$0")")"
JSON_LOG_FILE="${SCRIPT_DIR}/logs/media-backup.json"
PERFORMANCE_LOG_FILE="${SCRIPT_DIR}/logs/media-backup-performance.json"

# Script options
PARALLEL_BACKUPS=true
VERIFY_BACKUPS=true
PERFORMANCE_MONITORING=true
WEBHOOK_URL="https://notify.peterwood.rocks/lab"
INTERACTIVE_MODE=false
DRY_RUN=false

# Show help function
show_help() {
    cat << EOF
Media Services Backup Script

Usage: $0 [OPTIONS]

OPTIONS:
    --dry-run           Show what would be backed up without actually doing it
    --no-verify         Skip backup verification
    --sequential        Run backups sequentially instead of in parallel
    --interactive       Ask for confirmation before each backup
    --webhook URL       Custom webhook URL for notifications
    -h, --help          Show this help message

EXAMPLES:
    $0                  # Run full backup with default settings
    $0 --dry-run        # Preview what would be backed up
    $0 --sequential     # Run backups one at a time
    $0 --no-verify      # Skip verification for faster backup

SERVICES BACKED UP:
    - Sonarr (TV Shows)
    - Radarr (Movies)
    - Prowlarr (Indexers)
    - Audiobookshelf (Audiobooks)
    - Tautulli (Plex Statistics)
    - SABnzbd (Downloads)
    - Jellyseerr (Requests)

EOF
}

# Parse command line arguments
while [[ $# -gt 0 ]]; do
    case $1 in
        --dry-run)
            DRY_RUN=true
            shift
            ;;
        --no-verify)
            VERIFY_BACKUPS=false
            shift
            ;;
        --sequential)
            PARALLEL_BACKUPS=false
            shift
            ;;
        --interactive)
            INTERACTIVE_MODE=true
            shift
            ;;
        --webhook)
            WEBHOOK_URL="$2"
            shift 2
            ;;
        -h|--help)
            show_help
            exit 0
            ;;
        *)
            echo "Unknown option: $1"
            show_help
            exit 1
            ;;
    esac
done

# Create necessary directories
mkdir -p "${SCRIPT_DIR}/logs"
mkdir -p "${BACKUP_ROOT}"/{sonarr,radarr,prowlarr,audiobookshelf,tautulli,sabnzbd,jellyseerr}

# Log files
LOG_FILE="${LOG_ROOT}/media-backup-$(date +%Y%m%d_%H%M%S).log"
MARKDOWN_LOG="${LOG_ROOT}/media-backup-$(date +%Y%m%d_%H%M%S).md"

# Define media services and their backup configurations
declare -A MEDIA_SERVICES=(
    ["sonarr"]="/config/Backups/scheduled"
    ["radarr"]="/config/Backups/scheduled"
    ["prowlarr"]="/config/Backups/scheduled"
    ["audiobookshelf"]="/metadata/backups"
    ["tautulli"]="/config/backups"
    ["sabnzbd"]="/config/sabnzbd.ini"
    ["jellyseerr_db"]="/config/db/db.sqlite3"
    ["jellyseerr_settings"]="/config/settings.json"
)

# Service-specific backup destinations
declare -A BACKUP_DESTINATIONS=(
    ["sonarr"]="${BACKUP_ROOT}/sonarr/"
    ["radarr"]="${BACKUP_ROOT}/radarr/"
    ["prowlarr"]="${BACKUP_ROOT}/prowlarr/"
    ["audiobookshelf"]="${BACKUP_ROOT}/audiobookshelf/"
    ["tautulli"]="${BACKUP_ROOT}/tautulli/"
    ["sabnzbd"]="${BACKUP_ROOT}/sabnzbd/sabnzbd_$(date +%Y%m%d).ini"
    ["jellyseerr_db"]="${BACKUP_ROOT}/jellyseerr/jellyseerr_db_$(date +%Y%m%d_%H%M%S).sqlite3"
    ["jellyseerr_settings"]="${BACKUP_ROOT}/jellyseerr/settings_$(date +%Y%m%d_%H%M%S).json"
)

# Logging functions
log_message() {
    local message="$1"
    local timestamp
    timestamp=$(date '+%Y-%m-%d %H:%M:%S')
    echo -e "${CYAN}[${timestamp}]${NC} ${message}"
    echo "[${timestamp}] $message" >> "${LOG_FILE}" 2>/dev/null || true
}

log_error() {
    local message="$1"
    local timestamp
    timestamp=$(date '+%Y-%m-%d %H:%M:%S')
    echo -e "${RED}[${timestamp}] ERROR:${NC} ${message}" >&2
    echo "[${timestamp}] ERROR: $message" >> "${LOG_FILE}" 2>/dev/null || true
}

log_success() {
    local message="$1"
    local timestamp
    timestamp=$(date '+%Y-%m-%d %H:%M:%S')
    echo -e "${GREEN}[${timestamp}] SUCCESS:${NC} ${message}"
    echo "[${timestamp}] SUCCESS: $message" >> "${LOG_FILE}" 2>/dev/null || true
}

log_warning() {
    local message="$1"
    local timestamp
    timestamp=$(date '+%Y-%m-%d %H:%M:%S')
    echo -e "${YELLOW}[${timestamp}] WARNING:${NC} ${message}"
    echo "[${timestamp}] WARNING: $message" >> "${LOG_FILE}" 2>/dev/null || true
}

log_info() {
    local message="$1"
    local timestamp
    timestamp=$(date '+%Y-%m-%d %H:%M:%S')
    echo -e "${BLUE}[${timestamp}] INFO:${NC} ${message}"
    echo "[${timestamp}] INFO: $message" >> "${LOG_FILE}" 2>/dev/null || true
}

# Check if NAS mount is accessible
check_nas_mount() {
    local mount_point="/mnt/share/media"

    if ! mountpoint -q "$mount_point"; then
        log_error "NAS not mounted at $mount_point"
        log_info "Please mount the NAS first with: sudo mount $mount_point"
        log_info "Or check the mounting setup guide in docs/nas-mount-setup-guide.md"
        return 1
    fi

    # Test write access to backup directory
    if [ ! -w "$BACKUP_ROOT" ]; then
        log_error "No write access to backup directory: $BACKUP_ROOT"
        log_info "Check NAS mount permissions and credentials"
        return 1
    fi

    log_success "NAS mount check passed: $mount_point is accessible"
    return 0
}

# Performance tracking functions
track_performance() {
    if [ "$PERFORMANCE_MONITORING" != true ]; then
        return 0
    fi

    local operation="$1"
    local start_time="$2"
    local end_time="${3:-$(date +%s)}"
    local duration=$((end_time - start_time))

    # Initialize performance log if it doesn't exist
    if [ ! -f "$PERFORMANCE_LOG_FILE" ]; then
        echo "[]" > "$PERFORMANCE_LOG_FILE"
    fi

    # Add performance entry with lock protection
    local entry
    entry=$(jq -n \
        --arg timestamp "$(date -Iseconds)" \
        --arg operation "$operation" \
        --arg duration "$duration" \
        --arg hostname "$(hostname)" \
        '{
            timestamp: $timestamp,
            operation: $operation,
            duration: ($duration | tonumber),
            hostname: $hostname
        }')

    if command -v jq > /dev/null 2>&1; then
        local lock_file="${PERFORMANCE_LOG_FILE}.lock"
        local max_wait=10
        local wait_count=0

        while [ $wait_count -lt $max_wait ]; do
            if (set -C; echo $$ > "$lock_file") 2>/dev/null; then
                break
            fi
            sleep 0.1
            ((wait_count++))
        done

        if [ $wait_count -lt $max_wait ]; then
            if jq --argjson entry "$entry" '. += [$entry]' "$PERFORMANCE_LOG_FILE" > "${PERFORMANCE_LOG_FILE}.tmp" 2>/dev/null; then
                mv "${PERFORMANCE_LOG_FILE}.tmp" "$PERFORMANCE_LOG_FILE"
            else
                rm -f "${PERFORMANCE_LOG_FILE}.tmp"
            fi
            rm -f "$lock_file"
        fi
    fi

    log_info "Performance: $operation completed in ${duration}s"
}

# Initialize JSON log file
initialize_json_log() {
    if [ ! -f "${JSON_LOG_FILE}" ] || ! jq empty "${JSON_LOG_FILE}" 2>/dev/null; then
        echo "{}" > "${JSON_LOG_FILE}"
        log_message "Initialized JSON log file"
    fi
}

# Enhanced function to log file details with markdown formatting
log_file_details() {
    local service="$1"
    local src="$2"
    local dest="$3"
    local status="$4"
    local size=""
    local checksum=""

    # Calculate size if backup was successful
    if [ "$status" == "SUCCESS" ] && [ -e "$dest" ]; then
        size=$(du -sh "$dest" 2>/dev/null | cut -f1 || echo "Unknown")
        if [ "$VERIFY_BACKUPS" == true ]; then
            checksum=$(find "$dest" -type f -exec md5sum {} \; 2>/dev/null | md5sum | cut -d' ' -f1 || echo "N/A")
        fi
    else
        size="N/A"
        checksum="N/A"
    fi

    # Use a lock file for markdown log to prevent race conditions
    local markdown_lock="${MARKDOWN_LOG}.lock"
    local max_wait=30
    local wait_count=0

    while [ $wait_count -lt $max_wait ]; do
        if (set -C; echo $$ > "$markdown_lock") 2>/dev/null; then
            break
        fi
        sleep 0.1
        ((wait_count++))
    done

    if [ $wait_count -lt $max_wait ]; then
        # Log to markdown file safely
        {
            echo "## $service Backup"
            echo "- **Status**: $status"
            echo "- **Source**: \$($src\)"
            echo "- **Destination**: \$($dest\)"
            echo "- **Size**: $size"
            echo "- **Checksum**: $checksum"
            echo "- **Timestamp**: $(date '+%Y-%m-%d %H:%M:%S')"
            echo ""
        } >> "$MARKDOWN_LOG"

        rm -f "$markdown_lock"
    else
        log_warning "Could not acquire markdown log lock for $service"
    fi

    # Log to JSON
    if command -v jq > /dev/null 2>&1; then
        update_backup_log "$service" "$src" "$dest" "$status" "$size" "$checksum"
    fi
}

# Update backup log in JSON format
update_backup_log() {
    local service="$1"
    local src="$2"
    local dest="$3"
    local status="$4"
    local size="$5"
    local checksum="$6"
    local timestamp
    timestamp=$(date -Iseconds)

    if ! command -v jq > /dev/null 2>&1; then
        return 0
    fi

    # Use a lock file for parallel safety
    local lock_file="${JSON_LOG_FILE}.lock"
    local max_wait=30
    local wait_count=0

    while [ $wait_count -lt $max_wait ]; do
        if (set -C; echo $$ > "$lock_file") 2>/dev/null; then
            break
        fi
        sleep 0.1
        ((wait_count++))
    done

    if [ $wait_count -ge $max_wait ]; then
        log_warning "Could not acquire lock for JSON log update"
        return 1
    fi

    # Create entry for this backup
    local entry
    entry=$(jq -n \
        --arg service "$service" \
        --arg src "$src" \
        --arg dest "$dest" \
        --arg status "$status" \
        --arg size "$size" \
        --arg checksum "$checksum" \
        --arg timestamp "$timestamp" \
        '{
            service: $service,
            source: $src,
            destination: $dest,
            status: $status,
            size: $size,
            checksum: $checksum,
            timestamp: $timestamp
        }')

    # Update JSON log safely
    if jq --argjson entry "$entry" --arg service "$service" \
        '.[$service] = $entry' "$JSON_LOG_FILE" > "${JSON_LOG_FILE}.tmp" 2>/dev/null; then
        mv "${JSON_LOG_FILE}.tmp" "$JSON_LOG_FILE"
    else
        rm -f "${JSON_LOG_FILE}.tmp"
    fi

    # Remove lock file
    rm -f "$lock_file"
}

# Check if Docker container is running
check_container_running() {
    local container="$1"

    if ! docker ps --format "table {{.Names}}" | grep -q "^${container}$"; then
        log_warning "Container '$container' is not running"
        return 1
    fi

    return 0
}

# Verify backup integrity
verify_backup() {
    local src_container="$1"
    local src_path="$2"
    local dest_path="$3"

    if [ "$VERIFY_BACKUPS" != true ]; then
        return 0
    fi

    log_info "Verifying backup integrity for $src_container:$src_path"

    # For files, compare checksums
    if [[ "$src_path" == *.ini ]] || [[ "$src_path" == *.json ]]; then
        local src_checksum
        local dest_checksum
        src_checksum=$(docker exec "$src_container" md5sum "$src_path" 2>/dev/null | cut -d' ' -f1 || echo "")
        dest_checksum=$(md5sum "$dest_path" 2>/dev/null | cut -d' ' -f1 || echo "")

        if [ -n "$src_checksum" ] && [ -n "$dest_checksum" ] && [ "$src_checksum" == "$dest_checksum" ]; then
            log_success "Backup verification passed for $src_container:$src_path"
            return 0
        else
            log_error "Backup verification failed for $src_container:$src_path"
            return 1
        fi
    fi

    # For directories, check if they exist and have content
    if [ -d "$dest_path" ]; then
        local file_count
        file_count=$(find "$dest_path" -type f 2>/dev/null | wc -l)
        if [ "$file_count" -gt 0 ]; then
            log_success "Backup verification passed for $src_container:$src_path ($file_count files)"
            return 0
        else
            log_error "Backup verification failed: no files found in $dest_path"
            return 1
        fi
    fi

    log_warning "Unable to verify backup for $src_container:$src_path"
    return 0
}

# Backup a single service
backup_service() {
    local service="$1"
    local container="$1"
    local backup_start_time
    backup_start_time=$(date +%s)

    log_message "Starting backup for service: $service"

    # Handle Jellyseerr services with specialized backup function
    if [[ "$service" == jellyseerr_* ]]; then
        return $(backup_jellyseerr_service "$service")
    fi

    # Handle special cases for container names
    case "$service" in
        jellyseerr_db|jellyseerr_settings)
            container="jellyseerr"
            ;;
    esac

    # Check if container is running
    if ! check_container_running "$container"; then
        log_file_details "$service" "${container}:${MEDIA_SERVICES[$service]}" "${BACKUP_DESTINATIONS[$service]}" "FAILED - Container not running"
        return 1
    fi

    local src_path="${MEDIA_SERVICES[$service]}"
    local dest_path="${BACKUP_DESTINATIONS[$service]}"

    # Perform the backup
    if [ "$DRY_RUN" == true ]; then
        log_info "DRY RUN: Would backup $container:$src_path to $dest_path"
        log_file_details "$service" "$container:$src_path" "$dest_path" "DRY RUN"
        return 0
    fi

    if [ "$INTERACTIVE_MODE" == true ]; then
        echo -n "Backup $service? (y/N): "
        read -r response
        if [[ ! "$response" =~ ^[Yy]$ ]]; then
            log_info "Skipping $service backup (user choice)"
            return 0
        fi
    fi

    # Execute docker cp command
    local docker_cmd="docker cp $container:$src_path $dest_path"
    log_info "Executing: $docker_cmd"

    if $docker_cmd 2>&1 | tee -a "$LOG_FILE"; then
        log_success "Backup completed for $service"

        # File-level metrics tracking (success)
        if [[ "$METRICS_ENABLED" == "true" ]]; then
            local file_size checksum
            if [ -f "$dest_path" ]; then
                file_size=$(stat -c%s "$dest_path" 2>/dev/null || echo "0")
                checksum=$(md5sum "$dest_path" 2>/dev/null | cut -d' ' -f1 || echo "")
                metrics_add_file "$dest_path" "success" "$file_size" "$checksum"
            elif [ -d "$dest_path" ]; then
                # For directories, sum file sizes and add one entry for the directory
                file_size=$(find "$dest_path" -type f -exec stat -c%s {} + 2>/dev/null | awk '{s+=$1} END {print s}' || echo "0")
                metrics_add_file "$dest_path" "success" "$file_size"
            fi
        fi

        # Verify the backup
        if verify_backup "$container" "$src_path" "$dest_path"; then
            log_file_details "$service" "$container:$src_path" "$dest_path" "SUCCESS"
            track_performance "backup_${service}" "$backup_start_time"
            return 0
        else
            log_file_details "$service" "$container:$src_path" "$dest_path" "VERIFICATION_FAILED"
            # File-level metrics tracking (verification failed)
            if [[ "$METRICS_ENABLED" == "true" ]]; then
                local file_size
                if [ -f "$dest_path" ]; then
                    file_size=$(stat -c%s "$dest_path" 2>/dev/null || echo "0")
                    metrics_add_file "$dest_path" "failed" "$file_size" "" "Verification failed"
                elif [ -d "$dest_path" ]; then
                    file_size=$(find "$dest_path" -type f -exec stat -c%s {} + 2>/dev/null | awk '{s+=$1} END {print s}' || echo "0")
                    metrics_add_file "$dest_path" "failed" "$file_size" "" "Verification failed"
                fi
            fi
            return 1
        fi
    else
        log_error "Backup failed for $service"
        log_file_details "$service" "$container:$src_path" "$dest_path" "FAILED"
        # File-level metrics tracking (backup failed)
        if [[ "$METRICS_ENABLED" == "true" ]]; then
            local file_size
            if [ -f "$dest_path" ]; then
                file_size=$(stat -c%s "$dest_path" 2>/dev/null || echo "0")
                metrics_add_file "$dest_path" "failed" "$file_size" "" "Backup failed"
            elif [ -d "$dest_path" ]; then
                file_size=$(find "$dest_path" -type f -exec stat -c%s {} + 2>/dev/null | awk '{s+=$1} END {print s}' || echo "0")
                metrics_add_file "$dest_path" "failed" "$file_size" "" "Backup failed"
            fi
        fi
        return 1
    fi
}

# Specialized Jellyseerr backup function using SQLite CLI method
backup_jellyseerr_service() {
    local service="$1"
    local container="jellyseerr"
    local backup_start_time
    backup_start_time=$(date +%s)

    log_message "Starting specialized Jellyseerr backup for: $service"

    # Check if container is running
    if ! check_container_running "$container"; then
        log_file_details "$service" "${container}:${MEDIA_SERVICES[$service]}" "${BACKUP_DESTINATIONS[$service]}" "FAILED - Container not running"
        return 1
    fi

    local src_path="${MEDIA_SERVICES[$service]}"
    local dest_path="${BACKUP_DESTINATIONS[$service]}"

    # Create destination directory
    mkdir -p "$(dirname "$dest_path")"

    # Perform the backup
    if [ "$DRY_RUN" == true ]; then
        log_info "DRY RUN: Would backup $container:$src_path to $dest_path"
        log_file_details "$service" "$container:$src_path" "$dest_path" "DRY RUN"
        return 0
    fi

    if [ "$INTERACTIVE_MODE" == true ]; then
        echo -n "Backup $service? (y/N): "
        read -r response
        if [[ ! "$response" =~ ^[Yy]$ ]]; then
            log_info "Skipping $service backup (user choice)"
            return 0
        fi
    fi

    # Handle different backup methods based on service type
    local backup_success=false

    if [[ "$service" == "jellyseerr_db" ]]; then
        # Use SQLite CLI method for database backup (recommended by Jellyseerr docs)
        log_info "Using SQLite CLI method for database backup"

        # Create a temporary backup file path inside the container
        local temp_backup="/tmp/jellyseerr_db_$(date +%Y%m%d_%H%M%S).sqlite3"

        # Execute SQLite backup command inside the container
        local sqlite_cmd="docker exec $container sqlite3 $src_path \".backup '$temp_backup'\""
        log_info "Executing: $sqlite_cmd"

        if eval "$sqlite_cmd" 2>&1 | tee -a "$LOG_FILE"; then
            # Copy the backup file from container to host
            local copy_cmd="docker cp $container:$temp_backup $dest_path"
            log_info "Executing: $copy_cmd"

            if $copy_cmd 2>&1 | tee -a "$LOG_FILE"; then
                # Clean up temporary file in container
                docker exec "$container" rm -f "$temp_backup" 2>/dev/null || true
                backup_success=true
            else
                log_error "Failed to copy database backup from container"
                # Clean up temporary file in container
                docker exec "$container" rm -f "$temp_backup" 2>/dev/null || true
            fi
        else
            log_error "SQLite backup command failed"
        fi

    elif [[ "$service" == "jellyseerr_settings" ]]; then
        # Standard file copy for settings
        local docker_cmd="docker cp $container:$src_path $dest_path"
        log_info "Executing: $docker_cmd"

        if $docker_cmd 2>&1 | tee -a "$LOG_FILE"; then
            backup_success=true
        else
            log_error "Settings backup failed"
        fi
    else
        log_error "Unknown Jellyseerr service type: $service"
        return 1
    fi

    if [ "$backup_success" == true ]; then
        log_success "Backup completed for $service"

        # File-level metrics tracking (success)
        if [[ "$METRICS_ENABLED" == "true" ]]; then
            local file_size checksum
            if [ -f "$dest_path" ]; then
                file_size=$(stat -c%s "$dest_path" 2>/dev/null || echo "0")
                checksum=$(md5sum "$dest_path" 2>/dev/null | cut -d' ' -f1 || echo "")
                metrics_add_file "$dest_path" "success" "$file_size" "$checksum"
            fi
        fi

        # Verify the backup
        if verify_jellyseerr_backup "$container" "$src_path" "$dest_path" "$service"; then
            log_file_details "$service" "$container:$src_path" "$dest_path" "SUCCESS"
            track_performance "backup_${service}" "$backup_start_time"
            return 0
        else
            log_file_details "$service" "$container:$src_path" "$dest_path" "VERIFICATION_FAILED"
            # File-level metrics tracking (verification failed)
            if [[ "$METRICS_ENABLED" == "true" ]]; then
                local file_size
                if [ -f "$dest_path" ]; then
                    file_size=$(stat -c%s "$dest_path" 2>/dev/null || echo "0")
                    metrics_add_file "$dest_path" "failed" "$file_size" "" "Verification failed"
                fi
            fi
            return 1
        fi
    else
        log_error "Backup failed for $service"
        log_file_details "$service" "$container:$src_path" "$dest_path" "FAILED"
        # File-level metrics tracking (backup failed)
        if [[ "$METRICS_ENABLED" == "true" ]]; then
            local file_size
            if [ -f "$dest_path" ]; then
                file_size=$(stat -c%s "$dest_path" 2>/dev/null || echo "0")
                metrics_add_file "$dest_path" "failed" "$file_size" "" "Backup failed"
            fi
        fi
        return 1
    fi
}

# Specialized verification for Jellyseerr backups
verify_jellyseerr_backup() {
    local src_container="$1"
    local src_path="$2"
    local dest_path="$3"
    local service="$4"

    if [ "$VERIFY_BACKUPS" != true ]; then
        return 0
    fi

    log_info "Verifying Jellyseerr backup for $service"

    # Check if backup file exists and has reasonable size
    if [ ! -f "$dest_path" ]; then
        log_error "Backup file not found: $dest_path"
        return 1
    fi

    local file_size
    file_size=$(stat -c%s "$dest_path" 2>/dev/null || echo "0")

    if [ "$file_size" -eq 0 ]; then
        log_error "Backup file is empty: $dest_path"
        return 1
    fi

    # Service-specific verification
    if [[ "$service" == "jellyseerr_db" ]]; then
        # For SQLite database, verify it's a valid SQLite file
        if command -v file >/dev/null 2>&1; then
            if file "$dest_path" | grep -q "SQLite"; then
                log_success "Database backup verification passed: Valid SQLite file ($file_size bytes)"
                return 0
            else
                log_error "Database backup verification failed: Not a valid SQLite file"
                return 1
            fi
        else
            # Fallback: just check file size is reasonable (>1KB for a database)
            if [ "$file_size" -gt 1024 ]; then
                log_success "Database backup verification passed: File size check ($file_size bytes)"
                return 0
            else
                log_error "Database backup verification failed: File too small ($file_size bytes)"
                return 1
            fi
        fi
    elif [[ "$service" == "jellyseerr_settings" ]]; then
        # For settings JSON, verify it's valid JSON
        if command -v jq >/dev/null 2>&1; then
            if jq empty "$dest_path" 2>/dev/null; then
                log_success "Settings backup verification passed: Valid JSON file ($file_size bytes)"
                return 0
            else
                log_error "Settings backup verification failed: Invalid JSON file"
                return 1
            fi
        else
            # Fallback: check if it looks like JSON (starts with { or [)
            if head -c 1 "$dest_path" | grep -q "[{\[]"; then
                log_success "Settings backup verification passed: JSON format check ($file_size bytes)"
                return 0
            else
                log_error "Settings backup verification failed: Does not appear to be JSON"
                return 1
            fi
        fi
    fi

    log_warning "Unable to verify backup for $service"
    return 0
}

# Clean up old backups based on age and count
cleanup_old_backups() {
    log_message "Cleaning up old backups..."

    for service_dir in "${BACKUP_ROOT}"/*; do
        if [ ! -d "$service_dir" ]; then
            continue
        fi

        local service
        service=$(basename "$service_dir")
        log_info "Cleaning up old backups for $service"

        # Remove backups older than MAX_BACKUP_AGE_DAYS
        find "$service_dir" -type f -mtime +${MAX_BACKUP_AGE_DAYS} -delete 2>/dev/null || true
        find "$service_dir" -type d -empty -mtime +${MAX_BACKUP_AGE_DAYS} -delete 2>/dev/null || true

        # Keep only the most recent MAX_BACKUPS_TO_KEEP backups
        find "$service_dir" -type f -name "*.ini" -o -name "*.json" | sort -r | tail -n +$((MAX_BACKUPS_TO_KEEP + 1)) | xargs rm -f 2>/dev/null || true

        # Clean up old dated directories (for jellyseerr)
        find "$service_dir" -type d -name "backup_*" | sort -r | tail -n +$((MAX_BACKUPS_TO_KEEP + 1)) | xargs rm -rf 2>/dev/null || true
    done

    # Clean up old log files
    find "$LOG_ROOT" -name "media-backup-*.log" -mtime +${MAX_BACKUP_AGE_DAYS} -delete 2>/dev/null || true
    find "$LOG_ROOT" -name "media-backup-*.md" -mtime +${MAX_BACKUP_AGE_DAYS} -delete 2>/dev/null || true

    log_success "Cleanup completed"
}

# Check disk space
check_disk_space() {
    local required_space_mb=1000  # Minimum 1GB free space

    local available_space_kb
    available_space_kb=$(df "$BACKUP_ROOT" | awk 'NR==2 {print $4}')
    local available_space_mb=$((available_space_kb / 1024))

    if [ "$available_space_mb" -lt "$required_space_mb" ]; then
        log_error "Insufficient disk space. Available: ${available_space_mb}MB, Required: ${required_space_mb}MB"
        return 1
    fi

    log_info "Disk space check passed. Available: ${available_space_mb}MB"
    return 0
}

# Send enhanced notification
send_notification() {
    local title="$1"
    local message="$2"
    local status="${3:-info}"
    local hostname
    hostname=$(hostname)
    local total_services=${#MEDIA_SERVICES[@]}
    local success_count="$4"
    local failed_count="$5"

    # Enhanced message with statistics
    local enhanced_message="$message\n\nServices: $total_services\nSuccessful: $success_count\nFailed: $failed_count\nHost: $hostname"

    # Console notification
    case "$status" in
        "success") log_success "$title: $enhanced_message" ;;
        "error") log_error "$title: $enhanced_message" ;;
        "warning") log_warning "$title: $enhanced_message" ;;
        *) log_info "$title: $enhanced_message" ;;
    esac

    # Webhook notification
    if [ -n "$WEBHOOK_URL" ] && [ "$DRY_RUN" != true ]; then
        local tags="backup,media,${hostname}"
        [ "$failed_count" -gt 0 ] && tags="${tags},errors"

        curl -s \
            -H "tags:${tags}" \
            -d "$enhanced_message" \
            "$WEBHOOK_URL" 2>/dev/null || log_warning "Failed to send webhook notification"
    fi
}

# Generate backup summary report
generate_summary_report() {
    local success_count="$1"
    local failed_count="$2"
    local total_time="$3"

    log_message "=== BACKUP SUMMARY REPORT ==="
    log_message "Total Services: ${#MEDIA_SERVICES[@]}"
    log_message "Successful Backups: $success_count"
    log_message "Failed Backups: $failed_count"
    log_message "Total Time: ${total_time}s"
    log_message "Log File: $LOG_FILE"
    log_message "Markdown Report: $MARKDOWN_LOG"

    if [ "$PERFORMANCE_MONITORING" == true ]; then
        log_message "Performance Log: $PERFORMANCE_LOG_FILE"
    fi

    # Add summary to markdown log
    {
        echo "# Media Backup Summary Report"
        echo "**Date**: $(date '+%Y-%m-%d %H:%M:%S')"
        echo "**Host**: $(hostname)"
        echo "**Total Services**: ${#MEDIA_SERVICES[@]}"
        echo "**Successful**: $success_count"
        echo "**Failed**: $failed_count"
        echo "**Duration**: ${total_time}s"
        echo ""
    } >> "$MARKDOWN_LOG"
}

# Main backup execution function
main() {
    local script_start_time
    script_start_time=$(date +%s)

    log_message "=== MEDIA SERVICES BACKUP STARTED ==="
    log_message "Host: $(hostname)"
    log_message "Timestamp: $(date '+%Y-%m-%d %H:%M:%S')"
    log_message "Dry Run: $DRY_RUN"
    log_message "Parallel Mode: $PARALLEL_BACKUPS"
    log_message "Verify Backups: $VERIFY_BACKUPS"

    # Initialize metrics if enabled
    if [[ "$METRICS_ENABLED" == "true" ]]; then
        metrics_backup_start "media-services" "Media services backup (Sonarr, Radarr, etc.)" "$BACKUP_ROOT"
        metrics_status_update "initializing" "Preparing media services backup"
    fi

    # Initialize logging
    initialize_json_log

    # Initialize markdown log
    {
        echo "# Media Services Backup Report"
        echo "**Started**: $(date '+%Y-%m-%d %H:%M:%S')"
        echo "**Host**: $(hostname)"
        echo ""
    } > "$MARKDOWN_LOG"

    # Update metrics for pre-flight checks
    if [[ "$METRICS_ENABLED" == "true" ]]; then
        metrics_status_update "checking" "Running pre-flight checks"
    fi

    # Pre-flight checks
    if ! check_nas_mount; then
        if [[ "$METRICS_ENABLED" == "true" ]]; then
            metrics_backup_complete "failed" "NAS mount not accessible"
        fi
        send_notification "Media Backup Failed" "NAS mount not accessible at $MOUNT_POINT" "error" 0 1
        exit 1
    fi

    if ! check_disk_space; then
        if [[ "$METRICS_ENABLED" == "true" ]]; then
            metrics_backup_complete "failed" "Insufficient disk space"
        fi
        send_notification "Media Backup Failed" "Insufficient disk space" "error" 0 1
        exit 1
    fi

    # Check if Docker is running
    if ! docker info >/dev/null 2>&1; then
        log_error "Docker is not running or accessible"
        if [[ "$METRICS_ENABLED" == "true" ]]; then
            metrics_backup_complete "failed" "Docker is not accessible"
        fi
        send_notification "Media Backup Failed" "Docker is not accessible" "error" 0 1
        exit 1
    fi

    # Check if NAS mount is accessible
    if ! check_nas_mount; then
        if [[ "$METRICS_ENABLED" == "true" ]]; then
            metrics_backup_complete "failed" "NAS mount not accessible"
        fi
        send_notification "Media Backup Failed" "NAS mount not accessible" "error" 0 1
        exit 1
    fi

    local success_count=0
    local failed_count=0
    local backup_results=()

    if [ "$PARALLEL_BACKUPS" == true ]; then
        log_message "Running backups in parallel mode"

        # Update metrics for parallel backup phase
        if [[ "$METRICS_ENABLED" == "true" ]]; then
            metrics_status_update "backing_up" "Running media service backups in parallel"
        fi

        # Create temporary file for collecting results
        local temp_results
        temp_results=$(mktemp)
        local pids=()

        # Start backup jobs in parallel
        for service in "${!MEDIA_SERVICES[@]}"; do
            backup_service_wrapper "$service" "$temp_results" &
            pids+=($!)
            log_info "Started backup job for $service (PID: $!)"
        done

        # Wait for all jobs to complete
        for pid in "${pids[@]}"; do
            wait "$pid"
            log_info "Backup job completed (PID: $pid)"
        done

        # Collect results
        while IFS= read -r result; do
            if [[ "$result" == SUCCESS:* ]]; then
                ((success_count++))
                backup_results+=("✓ ${result#SUCCESS:}")
            elif [[ "$result" == FAILED:* ]]; then
                ((failed_count++))
                backup_results+=("✗ ${result#FAILED:}")
            fi
        done < "$temp_results"

        rm -f "$temp_results"

    else
        log_message "Running backups in sequential mode"

        # Update metrics for sequential backup phase
        if [[ "$METRICS_ENABLED" == "true" ]]; then
            metrics_status_update "backing_up" "Running media service backups sequentially"
        fi

        # Run backups sequentially
        for service in "${!MEDIA_SERVICES[@]}"; do
            if backup_service "$service"; then
                ((success_count++))
                backup_results+=("✓ $service")
            else
                ((failed_count++))
                backup_results+=("✗ $service")
            fi
        done
    fi

    # Calculate total time
    local script_end_time
    script_end_time=$(date +%s)
    local total_time=$((script_end_time - script_start_time))

    # Track overall performance
    track_performance "full_media_backup" "$script_start_time" "$script_end_time"

    # Update metrics for cleanup phase
    if [[ "$METRICS_ENABLED" == "true" ]]; then
        if [ "$DRY_RUN" != true ]; then
            metrics_status_update "cleaning_up" "Cleaning up old backup files"
        else
            metrics_status_update "completed" "Dry run completed successfully"
        fi
    fi

    # Clean up old backups (only if not dry run)
    if [ "$DRY_RUN" != true ]; then
        cleanup_old_backups
    fi

    # Generate summary report
    generate_summary_report "$success_count" "$failed_count" "$total_time"

    # Add results to markdown log
    {
        echo "## Backup Results"
        for result in "${backup_results[@]}"; do
            echo "- $result"
        done
        echo ""
        echo "**Completed**: $(date '+%Y-%m-%d %H:%M:%S')"
        echo "**Duration**: ${total_time}s"
    } >> "$MARKDOWN_LOG"

    # Send notification
    local status="success"
    local message="Media backup completed"

    if [ "$failed_count" -gt 0 ]; then
        status="warning"
        message="Media backup completed with $failed_count failures"
    fi

    if [ "$DRY_RUN" == true ]; then
        message="Media backup dry run completed"
        status="info"
    fi

    send_notification "Media Backup Complete" "$message" "$status" "$success_count" "$failed_count"

    # Finalize metrics
    if [[ "$METRICS_ENABLED" == "true" ]]; then
        if [ "$failed_count" -gt 0 ]; then
            metrics_backup_complete "completed_with_errors" "Media backup completed with $failed_count failures"
        elif [ "$DRY_RUN" == true ]; then
            metrics_backup_complete "success" "Media backup dry run completed successfully"
        else
            metrics_backup_complete "success" "Media backup completed successfully"
        fi
    fi

    # Exit with error code if any backups failed
    if [ "$failed_count" -gt 0 ]; then
        exit 1
    fi

    log_success "All media backups completed successfully!"
    exit 0
}

# Trap to handle script interruption
trap 'log_error "Script interrupted"; exit 130' INT TERM

# Run main function
main "$@"