#!/bin/bash ################################################################################ # Plex Media Server Enhanced Backup Script ################################################################################ # # Author: Peter Wood # Description: Comprehensive backup solution for Plex Media Server with advanced # database integrity checking, automated repair capabilities, # performance monitoring, and multi-channel notifications. # # Features: # - Database integrity verification with automatic repair # - WAL (Write-Ahead Logging) file handling # - Performance monitoring with JSON logging # - Parallel verification for improved speed # - Multi-channel notifications (webhook, email, console) # - Comprehensive error handling and recovery # - Automated cleanup of old backups # # Related Scripts: # - restore-plex.sh: Restore from backups created by this script # - validate-plex-backups.sh: Validate backup integrity and health # - monitor-plex-backup.sh: Real-time monitoring dashboard # - test-plex-backup.sh: Comprehensive testing suite # - plex.sh: General Plex service management # # Usage: # ./backup-plex.sh # Standard backup with auto-repair # ./backup-plex.sh --disable-auto-repair # Backup without auto-repair # ./backup-plex.sh --check-integrity # Integrity check only # ./backup-plex.sh --non-interactive # Automated mode for cron jobs # # Dependencies: # - Plex Media Server # - sqlite3 or Plex SQLite binary # - curl (for webhook notifications) # - jq (for JSON processing) # - sendmail (optional, for email notifications) # # Exit Codes: # 0 - Success # 1 - General error # 2 - Database integrity issues # 3 - Service management failure # 4 - Backup creation failure # ################################################################################ # NOTE: Removed 'set -e' to allow graceful error handling in repair operations # Critical operations use explicit error checking instead of automatic exit # Color codes for output RED='\033[0;31m' GREEN='\033[0;32m' YELLOW='\033[1;33m' BLUE='\033[0;34m' CYAN='\033[0;36m' NC='\033[0m' # No Color # Performance tracking variables (removed unused variables) # Configuration MAX_BACKUP_AGE_DAYS=30 MAX_BACKUPS_TO_KEEP=10 BACKUP_ROOT="/mnt/share/media/backups/plex" SHARED_LOG_ROOT="/mnt/share/media/backups/logs" # Get script directory with proper error handling if ! SCRIPT_PATH="$(readlink -f "$0")"; then echo "Error: Failed to resolve script path" >&2 exit 1 fi SCRIPT_DIR="$(dirname "$SCRIPT_PATH")" LOCAL_LOG_ROOT="${SCRIPT_DIR}/logs" PERFORMANCE_LOG_FILE="${LOCAL_LOG_ROOT}/plex-backup-performance.json" # Backup strategy configuration - Always perform full backups # Plex SQLite path (custom Plex SQLite binary) PLEX_SQLITE="/usr/lib/plexmediaserver/Plex SQLite" # Script options AUTO_REPAIR=false # CHANGED: Default to disabled to prevent corruption loops INTEGRITY_CHECK_ONLY=false INTERACTIVE_MODE=false PARALLEL_VERIFICATION=true PERFORMANCE_MONITORING=true WEBHOOK_URL="https://notify.peterwood.rocks/lab" EMAIL_RECIPIENT="" # Parse command line arguments while [[ $# -gt 0 ]]; do case $1 in --auto-repair) AUTO_REPAIR=true INTERACTIVE_MODE=false shift ;; --disable-auto-repair) AUTO_REPAIR=false shift ;; --non-interactive) INTERACTIVE_MODE=false shift ;; --interactive) INTERACTIVE_MODE=true shift ;; --no-parallel) PARALLEL_VERIFICATION=false shift ;; --no-performance) PERFORMANCE_MONITORING=false shift ;; --webhook=*) WEBHOOK_URL="${1#*=}" shift ;; --email=*) EMAIL_RECIPIENT="${1#*=}" shift ;; -h|--help) echo "Usage: $0 [OPTIONS]" echo "Options:" echo " --auto-repair Force enable automatic database repair (default: enabled)" echo " --disable-auto-repair Disable automatic database repair" echo " --check-integrity Only check database integrity, don't backup" echo " --non-interactive Run in non-interactive mode (for automation)" echo " --interactive Run in interactive mode (prompts for repair decisions)" echo " --no-parallel Disable parallel verification (slower but safer)" echo " --no-performance Disable performance monitoring" echo " --webhook=URL Send notifications to webhook URL" echo " --email=ADDRESS Send notifications to email address" echo " -h, --help Show this help message" echo "" echo "Database Integrity & Repair:" echo " By default, the script automatically detects and attempts to repair" echo " corrupted databases before backup. Use --disable-auto-repair to" echo " skip repair and backup corrupted databases as-is." echo "" exit 0 ;; *) echo "Unknown option: $1" exit 1 ;; esac done # Create logs directory mkdir -p "${SCRIPT_DIR}/logs" # Define Plex files and their nicknames declare -A PLEX_FILES=( ["database"]="/var/lib/plexmediaserver/Library/Application Support/Plex Media Server/Plug-in Support/Databases/com.plexapp.plugins.library.db" ["blobs"]="/var/lib/plexmediaserver/Library/Application Support/Plex Media Server/Plug-in Support/Databases/com.plexapp.plugins.library.blobs.db" ["preferences"]="/var/lib/plexmediaserver/Library/Application Support/Plex Media Server/Preferences.xml" ) # Logging functions log_message() { local message="$1" local timestamp timestamp=$(date '+%Y-%m-%d %H:%M:%S') echo -e "${CYAN}[${timestamp}]${NC} ${message}" mkdir -p "${LOCAL_LOG_ROOT}" # Ensure acedanger owns the log directory sudo chown acedanger:acedanger "${LOCAL_LOG_ROOT}" 2>/dev/null || true local log_file="${LOCAL_LOG_ROOT}/plex-backup-$(date '+%Y-%m-%d').log" echo "[${timestamp}] ${message}" >> "$log_file" 2>/dev/null || true # Ensure acedanger owns the log file sudo chown acedanger:acedanger "$log_file" 2>/dev/null || true } log_error() { local message="$1" local timestamp timestamp=$(date '+%Y-%m-%d %H:%M:%S') echo -e "${RED}[${timestamp}] ERROR:${NC} ${message}" mkdir -p "${LOCAL_LOG_ROOT}" local log_file="${LOCAL_LOG_ROOT}/plex-backup-$(date '+%Y-%m-%d').log" echo "[${timestamp}] ERROR: ${message}" >> "$log_file" 2>/dev/null || true # Ensure acedanger owns the log file sudo chown acedanger:acedanger "$log_file" 2>/dev/null || true } log_success() { local message="$1" local timestamp timestamp=$(date '+%Y-%m-%d %H:%M:%S') echo -e "${GREEN}[${timestamp}] SUCCESS:${NC} ${message}" mkdir -p "$LOCAL_LOG_ROOT" local log_file="${LOCAL_LOG_ROOT}/plex-backup-$(date '+%Y-%m-%d').log" echo "[${timestamp}] SUCCESS: $message" >> "$log_file" 2>/dev/null || true # Ensure acedanger owns the log file sudo chown acedanger:acedanger "$log_file" 2>/dev/null || true } log_warning() { local message="$1" local timestamp timestamp=$(date '+%Y-%m-%d %H:%M:%S') echo -e "${YELLOW}[${timestamp}] WARNING:${NC} ${message}" mkdir -p "$LOCAL_LOG_ROOT" local log_file="${LOCAL_LOG_ROOT}/plex-backup-$(date '+%Y-%m-%d').log" echo "[${timestamp}] WARNING: $message" >> "$log_file" 2>/dev/null || true # Ensure acedanger owns the log file sudo chown acedanger:acedanger "$log_file" 2>/dev/null || true } log_info() { local message="$1" local timestamp timestamp=$(date '+%Y-%m-%d %H:%M:%S') echo -e "${BLUE}[${timestamp}] INFO:${NC} ${message}" mkdir -p "$LOCAL_LOG_ROOT" local log_file="${LOCAL_LOG_ROOT}/plex-backup-$(date '+%Y-%m-%d').log" echo "[${timestamp}] INFO: $message" >> "$log_file" 2>/dev/null || true # Ensure acedanger owns the log file sudo chown acedanger:acedanger "$log_file" 2>/dev/null || true } # Performance tracking functions track_performance() { if [[ "$PERFORMANCE_MONITORING" != true ]]; then return 0 fi local operation="$1" local start_time="$2" local end_time="${3:-$(date +%s)}" local duration=$((end_time - start_time)) # Initialize performance log if it doesn't exist if [ ! -f "$PERFORMANCE_LOG_FILE" ]; then mkdir -p "$(dirname "$PERFORMANCE_LOG_FILE")" # Ensure acedanger owns the log directory sudo chown -R acedanger:acedanger "$(dirname "$PERFORMANCE_LOG_FILE")" 2>/dev/null || true echo "[]" > "$PERFORMANCE_LOG_FILE" # Ensure acedanger owns the performance log file sudo chown acedanger:acedanger "$PERFORMANCE_LOG_FILE" 2>/dev/null || true fi # Add performance entry local entry local timestamp if ! timestamp="$(date -Iseconds)"; then timestamp="$(date)" # Fallback to basic date format fi entry=$(jq -n \ --arg operation "$operation" \ --arg duration "$duration" \ --arg timestamp "$timestamp" \ '{ operation: $operation, duration_seconds: ($duration | tonumber), timestamp: $timestamp }') jq --argjson entry "$entry" '. += [$entry]' "$PERFORMANCE_LOG_FILE" > "${PERFORMANCE_LOG_FILE}.tmp" && \ mv "${PERFORMANCE_LOG_FILE}.tmp" "$PERFORMANCE_LOG_FILE" # Ensure acedanger owns the performance log file sudo chown acedanger:acedanger "$PERFORMANCE_LOG_FILE" 2>/dev/null || true log_info "Performance: $operation completed in ${duration}s" } # Initialize log directory initialize_logs() { mkdir -p "$(dirname "$PERFORMANCE_LOG_FILE")" # Ensure acedanger owns the log directory sudo chown -R acedanger:acedanger "${LOCAL_LOG_ROOT}" 2>/dev/null || true if [ ! -f "$PERFORMANCE_LOG_FILE" ]; then echo "[]" > "$PERFORMANCE_LOG_FILE" # Ensure acedanger owns the performance log file sudo chown acedanger:acedanger "$PERFORMANCE_LOG_FILE" 2>/dev/null || true log_message "Initialized performance log file" fi } # Log synchronization functions sync_logs_to_shared() { local sync_start_time sync_start_time=$(date +%s) log_info "Starting log synchronization to shared location" # Ensure shared log directory exists if ! mkdir -p "$SHARED_LOG_ROOT" 2>/dev/null; then log_warning "Could not create shared log directory: $SHARED_LOG_ROOT" return 1 fi # Check if shared location is accessible if [ ! -w "$SHARED_LOG_ROOT" ]; then log_warning "Shared log directory is not writable: $SHARED_LOG_ROOT" return 1 fi # Sync log files (one-way: local -> shared) local sync_count=0 local error_count=0 for log_file in "$LOCAL_LOG_ROOT"/*.log "$LOCAL_LOG_ROOT"/*.json; do if [ -f "$log_file" ]; then local filename filename=$(basename "$log_file") local shared_file="$SHARED_LOG_ROOT/$filename" # Only copy if file doesn't exist in shared location or local is newer if [ ! -f "$shared_file" ] || [ "$log_file" -nt "$shared_file" ]; then if cp "$log_file" "$shared_file" 2>/dev/null; then ((sync_count++)) log_info "Synced: $filename" else ((error_count++)) log_warning "Failed to sync: $filename" fi fi fi done local sync_end_time sync_end_time=$(date +%s) local sync_duration=$((sync_end_time - sync_start_time)) if [ $error_count -eq 0 ]; then log_success "Log sync completed: $sync_count files synced in ${sync_duration}s" else log_warning "Log sync completed with errors: $sync_count synced, $error_count failed in ${sync_duration}s" fi return $error_count } # Cleanup old local logs (30 day retention) cleanup_old_local_logs() { local cleanup_start_time cleanup_start_time=$(date +%s) log_info "Starting cleanup of old local logs (30+ days)" if [ ! -d "$LOCAL_LOG_ROOT" ]; then log_info "Local log directory does not exist, nothing to clean up" return 0 fi local cleanup_count=0 local error_count=0 # Find and remove log files older than 30 days while IFS= read -r -d '' old_file; do local filename filename=$(basename "$old_file") if rm "$old_file" 2>/dev/null; then ((cleanup_count++)) log_info "Removed old log: $filename" else ((error_count++)) log_warning "Failed to remove old log: $filename" fi done < <(find "$LOCAL_LOG_ROOT" -name "*.log" -mtime +30 -print0 2>/dev/null) # Also clean up old performance log entries (keep structure, remove old entries) if [ -f "$PERFORMANCE_LOG_FILE" ]; then local thirty_days_ago thirty_days_ago=$(date -d '30 days ago' -Iseconds) local temp_perf_file="${PERFORMANCE_LOG_FILE}.cleanup.tmp" if jq --arg cutoff "$thirty_days_ago" '[.[] | select(.timestamp >= $cutoff)]' "$PERFORMANCE_LOG_FILE" > "$temp_perf_file" 2>/dev/null; then local old_count old_count=$(jq length "$PERFORMANCE_LOG_FILE" 2>/dev/null || echo "0") local new_count new_count=$(jq length "$temp_perf_file" 2>/dev/null || echo "0") local removed_count=$((old_count - new_count)) if [ "$removed_count" -gt 0 ]; then mv "$temp_perf_file" "$PERFORMANCE_LOG_FILE" log_info "Cleaned up $removed_count old performance entries" ((cleanup_count += removed_count)) else rm -f "$temp_perf_file" fi else rm -f "$temp_perf_file" log_warning "Failed to clean up old performance log entries" ((error_count++)) fi fi local cleanup_end_time cleanup_end_time=$(date +%s) local cleanup_duration=$((cleanup_end_time - cleanup_start_time)) if [ $cleanup_count -gt 0 ]; then log_success "Cleanup completed: $cleanup_count items removed in ${cleanup_duration}s" else log_info "Cleanup completed: no old items found to remove in ${cleanup_duration}s" fi return $error_count } # Enhanced notification system send_notification() { local title="$1" local message="$2" local status="${3:-info}" # success, error, warning, info local hostname hostname=$(hostname) # Console notification case "$status" in success) log_success "$title: $message" ;; error) log_error "$title: $message" ;; warning) log_warning "$title: $message" ;; *) log_info "$title: $message" ;; esac # Webhook notification if [ -n "$WEBHOOK_URL" ]; then local tags="backup,plex,${hostname}" [ "$status" == "error" ] && tags="${tags},errors" [ "$status" == "warning" ] && tags="${tags},warnings" # Clean message without newlines or timestamps for webhook local webhook_message="$message" curl -s \ -H "tags:${tags}" \ -d "$webhook_message" \ "$WEBHOOK_URL" 2>/dev/null || log_warning "Failed to send webhook notification" fi # Email notification (if sendmail is available) if [ -n "$EMAIL_RECIPIENT" ] && command -v sendmail > /dev/null 2>&1; then { echo "To: $EMAIL_RECIPIENT" echo "Subject: Plex Backup - $title" echo "Content-Type: text/plain" echo "" echo "Host: $hostname" echo "Time: $(date)" echo "Status: $status" echo "" echo "$message" } | sendmail "$EMAIL_RECIPIENT" 2>/dev/null || true fi } # Format backed up files list for notifications format_backed_up_files() { local files=("$@") local count=${#files[@]} if [ "$count" -eq 0 ]; then echo "no files" elif [ "$count" -eq 1 ]; then echo "${files[0]}" elif [ "$count" -eq 2 ]; then echo "${files[0]} and ${files[1]}" else local last_file="${files[-1]}" local other_files=("${files[@]:0:$((count-1))}") local other_files_str other_files_str=$(IFS=', '; echo "${other_files[*]}") echo "${other_files_str}, and ${last_file}" fi } # Enhanced checksum calculation with caching calculate_checksum() { local file="$1" # Use /tmp for cache files to avoid permission issues local cache_dir="/tmp/plex-backup-cache" local cache_file="$cache_dir/${file//\//_}.md5" local file_mtime file_mtime=$(stat -c %Y "$file" 2>/dev/null || echo "0") # Create cache directory if it doesn't exist mkdir -p "$cache_dir" 2>/dev/null || true # Check if cached checksum exists and is newer than file if [ -f "$cache_file" ]; then local cache_mtime cache_mtime=$(stat -c %Y "$cache_file" 2>/dev/null || echo "0") if [ "$cache_mtime" -gt "$file_mtime" ]; then local cached_checksum cached_checksum=$(cat "$cache_file" 2>/dev/null) if [[ -n "$cached_checksum" && "$cached_checksum" =~ ^[a-f0-9]{32}$ ]]; then echo "$cached_checksum" return 0 fi fi fi # Calculate new checksum local checksum if ! checksum=$(md5sum "$file" 2>/dev/null | cut -d' ' -f1); then checksum="" fi # Check if we got a valid checksum (not empty and looks like md5) if [[ -n "$checksum" && "$checksum" =~ ^[a-f0-9]{32}$ ]]; then # Cache the checksum echo "$checksum" > "$cache_file" 2>/dev/null || true echo "$checksum" return 0 fi # If normal access failed or returned empty, try with sudo if ! checksum=$(sudo md5sum "$file" 2>/dev/null | cut -d' ' -f1); then checksum="" fi # Check if sudo checksum is valid if [[ -n "$checksum" && "$checksum" =~ ^[a-f0-9]{32}$ ]]; then # Cache the checksum with appropriate permissions echo "$checksum" | sudo tee "$cache_file" >/dev/null 2>&1 || true echo "$checksum" return 0 fi # If both fail, return error indicator echo "PERMISSION_DENIED" return 1 } # WAL file handling for backup operations (different from repair-specific function) handle_wal_files() { local operation="$1" local backup_path="$2" case "$operation" in "checkpoint") log_message "Performing WAL checkpoint..." local checkpoint_errors=0 for nickname in "${!PLEX_FILES[@]}"; do local file="${PLEX_FILES[$nickname]}" # Only checkpoint database files if [[ "$file" == *".db" ]] && [ -f "$file" ]; then local db_name db_name=$(basename "$file") log_info "Checkpointing WAL for $db_name..." # Perform WAL checkpoint with TRUNCATE to ensure all data is moved to main DB if sudo "$PLEX_SQLITE" "$file" "PRAGMA wal_checkpoint(TRUNCATE);" >/dev/null 2>&1; then log_success "WAL checkpoint completed for $db_name" else log_warning "WAL checkpoint failed for $db_name" ((checkpoint_errors++)) fi fi done if [ "$checkpoint_errors" -gt 0 ]; then log_warning "WAL checkpoint completed with $checkpoint_errors errors" return 1 else log_success "All WAL checkpoints completed successfully" return 0 fi ;; "backup") if [ -z "$backup_path" ]; then log_error "Backup path required for WAL file backup" return 1 fi log_message "Backing up WAL and SHM files..." local wal_files_backed_up=0 local wal_backup_errors=0 for nickname in "${!PLEX_FILES[@]}"; do local file="${PLEX_FILES[$nickname]}" # Only process database files if [[ "$file" == *".db" ]] && [ -f "$file" ]; then local wal_file="${file}-wal" local shm_file="${file}-shm" # Backup WAL file if it exists if [ -f "$wal_file" ]; then local wal_basename wal_basename=$(basename "$wal_file") local backup_file="$backup_path/$wal_basename" if sudo cp "$wal_file" "$backup_file"; then # Force filesystem sync to prevent corruption sync sudo chown plex:plex "$backup_file" log_success "Backed up WAL file: $wal_basename" ((wal_files_backed_up++)) else log_error "Failed to backup WAL file: $wal_basename" ((wal_backup_errors++)) fi fi # Backup SHM file if it exists if [ -f "$shm_file" ]; then local shm_basename shm_basename=$(basename "$shm_file") local backup_file="$backup_path/$shm_basename" if sudo cp "$shm_file" "$backup_file"; then # Force filesystem sync to prevent corruption sync sudo chown plex:plex "$backup_file" log_success "Backed up SHM file: $shm_basename" ((wal_files_backed_up++)) else log_error "Failed to backup SHM file: $shm_basename" ((wal_backup_errors++)) fi fi fi done if [ "$wal_files_backed_up" -gt 0 ]; then log_success "Backed up $wal_files_backed_up WAL/SHM files" else log_info "No WAL/SHM files found to backup" fi if [ "$wal_backup_errors" -gt 0 ]; then log_error "WAL file backup completed with $wal_backup_errors errors" return 1 else return 0 fi ;; *) log_error "Unknown WAL operation: $operation" return 1 ;; esac } # Check database integrity using shared repair script check_database_integrity() { local db_file="$1" local db_name db_name=$(basename "$db_file") local repair_script="${SCRIPT_DIR}/plex-database-repair.sh" log_message "Checking database integrity: $db_name" # Check if shared repair script exists if [[ ! -f "$repair_script" ]]; then log_error "Database repair script not found at: $repair_script" return 2 fi # Use shared repair script for integrity checking if "$repair_script" check "$db_file" >/dev/null 2>&1; then log_success "Database integrity check passed: $db_name" return 0 else local exit_code=$? if [[ $exit_code -eq 2 ]]; then log_error "Critical error during integrity check for $db_name" return 2 else log_warning "Database integrity issues detected in $db_name" return 1 fi fi } # Preventive corruption detection before severe corruption occurs detect_early_corruption() { local db_file="$1" local db_name db_name=$(basename "$db_file") log_message "Performing early corruption detection for: $db_name" # Check for early warning signs of corruption local warning_count=0 # 1. Check for WAL file size anomalies local wal_file="${db_file}-wal" if [ -f "$wal_file" ]; then local wal_size wal_size=$(stat -f%z "$wal_file" 2>/dev/null || stat -c%s "$wal_file" 2>/dev/null || echo "0") local db_size db_size=$(stat -f%z "$db_file" 2>/dev/null || stat -c%s "$db_file" 2>/dev/null || echo "0") # If WAL file is more than 10% of database size, it might indicate issues if [ "$wal_size" -gt 0 ] && [ "$db_size" -gt 0 ]; then local wal_ratio=$((wal_size * 100 / db_size)) if [ "$wal_ratio" -gt 10 ]; then log_warning "WAL file unusually large: ${wal_ratio}% of database size" ((warning_count++)) fi else log_info "Unable to determine file sizes for WAL analysis" fi fi # 2. Quick integrity check focused on critical issues local quick_check if ! quick_check=$(sudo "$PLEX_SQLITE" "$db_file" "PRAGMA quick_check(5);" 2>&1); then log_warning "Failed to execute quick integrity check for $db_name" ((warning_count++)) elif ! echo "$quick_check" | grep -q "^ok$"; then log_warning "Quick integrity check failed for $db_name" log_warning "Issues found: $quick_check" ((warning_count++)) fi # 3. Check for foreign key violations (common early corruption sign) local fk_check if fk_check=$(sudo "$PLEX_SQLITE" "$db_file" "PRAGMA foreign_key_check;" 2>/dev/null); then if [ -n "$fk_check" ]; then log_warning "Foreign key violations detected in $db_name" ((warning_count++)) fi else log_info "Foreign key check unavailable for $db_name" fi # 4. Check database statistics for anomalies if ! sudo "$PLEX_SQLITE" "$db_file" "PRAGMA compile_options;" >/dev/null 2>&1; then log_warning "Database statistics check failed for $db_name" ((warning_count++)) fi if [ "$warning_count" -gt 0 ]; then log_warning "Early corruption indicators detected ($warning_count warnings) in $db_name" log_warning "Consider performing preventive maintenance or monitoring more closely" return 1 else log_success "Early corruption detection passed for $db_name" return 0 fi } # Enhanced database repair with multiple recovery strategies # Database repair using shared repair script repair_database() { local db_file="$1" local force_repair="${2:-false}" local db_name db_name=$(basename "$db_file") local repair_script="${SCRIPT_DIR}/plex-database-repair.sh" log_message "Attempting to repair corrupted database: $db_name" # Check if shared repair script exists if [[ ! -f "$repair_script" ]]; then log_error "Database repair script not found at: $repair_script" return 2 fi # Use the shared repair script local repair_command="repair" if [[ "$force_repair" == "true" ]]; then repair_command="force-repair" fi log_message "Using shared repair script for database repair..." if "$repair_script" "$repair_command" "$db_file"; then log_success "Database repaired successfully using shared repair script" return 0 else local exit_code=$? if [[ $exit_code -eq 2 ]]; then log_error "Critical error during database repair" return 2 else log_error "Database repair failed" log_warning "Will backup corrupted database - manual intervention may be needed" return 1 fi fi } # Parallel verification function verify_files_parallel() { local backup_dir="$1" local -a pids=() local temp_dir temp_dir=$(mktemp -d) local verification_errors=0 local max_jobs=4 # Limit concurrent jobs to prevent system overload local job_count=0 if [[ "$PARALLEL_VERIFICATION" != true ]]; then # Fall back to sequential verification for nickname in "${!PLEX_FILES[@]}"; do local src_file="${PLEX_FILES[$nickname]}" local dest_file dest_file="$backup_dir/$(basename "$src_file")" if [ -f "$dest_file" ]; then if ! verify_backup "$src_file" "$dest_file"; then verification_errors=$((verification_errors + 1)) fi fi done rm -rf "$temp_dir" 2>/dev/null || true return $verification_errors fi log_info "Starting parallel verification in $backup_dir (max $max_jobs concurrent jobs)" # Start verification jobs in parallel with job control for nickname in "${!PLEX_FILES[@]}"; do local src_file="${PLEX_FILES[$nickname]}" local dest_file dest_file="$backup_dir/$(basename "$src_file")" if [ -f "$dest_file" ]; then # Wait if we've reached the job limit if [ $job_count -ge $max_jobs ]; then wait "${pids[0]}" 2>/dev/null || true pids=("${pids[@]:1}") # Remove first element job_count=$((job_count - 1)) fi ( local result_file="$temp_dir/$nickname.result" if verify_backup "$src_file" "$dest_file"; then echo "0" > "$result_file" else echo "1" > "$result_file" fi ) & pids+=($!) job_count=$((job_count + 1)) fi done # Wait for all remaining verification jobs to complete for pid in "${pids[@]}"; do wait "$pid" 2>/dev/null || true done # Collect results for nickname in "${!PLEX_FILES[@]}"; do local result_file="$temp_dir/$nickname.result" if [ -f "$result_file" ]; then local result result=$(cat "$result_file" 2>/dev/null || echo "1") if [ "$result" != "0" ]; then verification_errors=$((verification_errors + 1)) fi fi done # Cleanup rm -rf "$temp_dir" 2>/dev/null || true return $verification_errors } # Enhanced backup verification with multiple retry strategies and corruption detection verify_backup() { local src="$1" local dest="$2" local max_retries=3 local retry_count=0 log_message "Verifying backup integrity: $(basename "$src")" # Calculate destination checksum first (this doesn't change) local dest_checksum local dest_result=0 if ! dest_checksum=$(sudo md5sum "$dest" 2>/dev/null | cut -d' ' -f1); then dest_result=1 dest_checksum="" fi if [[ $dest_result -ne 0 ]] || [[ ! "$dest_checksum" =~ ^[a-f0-9]{32}$ ]]; then log_error "Failed to calculate destination checksum for $(basename "$dest")" return 1 fi # Retry loop for source checksum calculation while [ $retry_count -lt $max_retries ]; do # Calculate source checksum (without caching to get current state) local src_checksum local src_result=0 if ! src_checksum=$(sudo md5sum "$src" 2>/dev/null | cut -d' ' -f1); then src_result=1 src_checksum="" fi if [[ $src_result -ne 0 ]] || [[ ! "$src_checksum" =~ ^[a-f0-9]{32}$ ]]; then log_error "Failed to calculate source checksum for $(basename "$src") (attempt $((retry_count + 1)))" ((retry_count++)) if [[ $retry_count -lt $max_retries ]]; then log_warning "Retrying checksum calculation in 2 seconds..." sleep 2 continue else return 1 fi fi if [ "$src_checksum" == "$dest_checksum" ]; then log_success "Backup verification passed: $(basename "$src")" log_info "Source checksum: $src_checksum" log_info "Backup checksum: $dest_checksum" return 0 else # If checksums don't match, wait and try again ((retry_count++)) if [ $retry_count -lt $max_retries ]; then log_warning "Checksum mismatch for $(basename "$src") (attempt $retry_count/$max_retries), retrying in 3 seconds..." sleep 3 else log_error "Backup verification failed after $max_retries attempts: $(basename "$src")" log_error "Source checksum: $src_checksum" log_error "Backup checksum: $dest_checksum" # For database files, perform additional integrity check on backup if [[ "$dest" == *.db ]]; then log_warning "Database file checksum mismatch - checking backup integrity..." if sudo "$PLEX_SQLITE" "$dest" "PRAGMA integrity_check;" 2>/dev/null | grep -q "ok"; then log_warning "Backup database integrity is valid despite checksum mismatch" log_warning "Accepting backup (source file may have been modified after copy)" return 0 else log_error "Backup database is also corrupted - backup failed" return 1 fi fi return 1 fi fi done return 1 } # Enhanced service management with SAFE shutdown procedures and extended timeouts # CRITICAL SAFETY NOTE: This function was modified to remove dangerous force-kill operations # that were causing database corruption. Now uses only graceful shutdown methods. manage_plex_service() { local action="$1" local force_stop="${2:-false}" local operation_start operation_start=$(date +%s) log_message "Managing Plex service: $action" case "$action" in stop) # Check if already stopped if ! sudo systemctl is-active --quiet plexmediaserver.service; then log_info "Plex service is already stopped" track_performance "service_stop" "$operation_start" return 0 fi # First try normal stop with extended timeout if sudo systemctl stop plexmediaserver.service; then log_success "Plex service stop command issued" # Wait for clean shutdown with progress indicator (extended timeout) local wait_time=0 local max_wait=30 # Increased from 15 to 30 seconds while [ $wait_time -lt $max_wait ]; do if ! sudo systemctl is-active --quiet plexmediaserver.service; then log_success "Plex service confirmed stopped (${wait_time}s)" track_performance "service_stop" "$operation_start" return 0 fi sleep 1 wait_time=$((wait_time + 1)) echo -n "." done echo # If normal stop failed and force_stop is enabled, try extended graceful shutdown if [ "$force_stop" = "true" ]; then log_warning "Normal stop failed, attempting extended graceful shutdown..." local plex_pids plex_pids=$(pgrep -f "Plex Media Server" 2>/dev/null || true) if [ -n "$plex_pids" ]; then log_message "Found Plex processes: $plex_pids" log_message "Sending graceful termination signal and waiting longer..." # Send TERM signal for graceful shutdown if sudo pkill -TERM -f "Plex Media Server" 2>/dev/null || true; then # Extended wait for graceful shutdown (up to 60 seconds) local extended_wait=0 local max_extended_wait=60 while [ $extended_wait -lt $max_extended_wait ]; do plex_pids=$(pgrep -f "Plex Media Server" 2>/dev/null || true) if [ -z "$plex_pids" ]; then log_success "Plex service gracefully stopped after extended wait (${extended_wait}s)" track_performance "service_extended_stop" "$operation_start" return 0 fi sleep 2 extended_wait=$((extended_wait + 2)) echo -n "." done echo # If still running after extended wait, log error but don't force kill plex_pids=$(pgrep -f "Plex Media Server" 2>/dev/null || true) if [ -n "$plex_pids" ]; then log_error "Plex processes still running after ${max_extended_wait}s graceful shutdown attempt" log_error "Refusing to force-kill processes to prevent database corruption" log_error "Manual intervention may be required: PIDs $plex_pids" return 1 fi else log_error "Failed to send TERM signal to Plex processes" return 1 fi else log_success "No Plex processes found running" track_performance "service_stop" "$operation_start" return 0 fi else log_warning "Plex service may not have stopped cleanly after ${max_wait}s" # Check one more time if service actually stopped with extended timeout sleep 2 if ! sudo systemctl is-active --quiet plexmediaserver.service; then log_success "Plex service stopped (delayed confirmation)" track_performance "service_stop" "$operation_start" return 0 else log_warning "Plex service still appears to be running after ${max_wait}s" return 1 fi fi else log_error "Failed to issue stop command for Plex service" return 1 fi ;; start) # Check if service is already running if sudo systemctl is-active --quiet plexmediaserver.service; then log_info "Plex service is already running" track_performance "service_start" "$operation_start" return 0 fi if sudo systemctl start plexmediaserver.service; then log_success "Plex service start command issued" # Wait for service to be fully running with progress indicator (extended timeout) local wait_time=0 local max_wait=45 # Increased from 30 to 45 seconds for database initialization while [ $wait_time -lt $max_wait ]; do if sudo systemctl is-active --quiet plexmediaserver.service; then # Additional verification: wait for full service readiness sleep 3 if sudo systemctl is-active --quiet plexmediaserver.service; then # Final check: ensure service is stable and not in restart loop sleep 2 if sudo systemctl is-active --quiet plexmediaserver.service; then log_success "Plex service confirmed running and stable (${wait_time}s)" track_performance "service_start" "$operation_start" return 0 fi fi fi sleep 1 wait_time=$((wait_time + 1)) echo -n "." done echo log_error "Plex service failed to start within ${max_wait}s" # Get service status for debugging local service_status service_status=$(sudo systemctl status plexmediaserver.service --no-pager -l 2>&1 | head -10 || echo "Failed to get status") log_error "Service status: $service_status" return 1 else log_error "Failed to start Plex service" return 1 fi ;; *) log_error "Invalid service action: $action" return 1 ;; esac } # Check available disk space check_disk_space() { local backup_dir="$1" local required_space_mb="$2" local available_space_kb available_space_kb=$(df "$backup_dir" | awk 'NR==2 {print $4}') local available_space_mb=$((available_space_kb / 1024)) if [ "$available_space_mb" -lt "$required_space_mb" ]; then log_error "Insufficient disk space. Required: ${required_space_mb}MB, Available: ${available_space_mb}MB" return 1 fi log_message "Disk space check passed. Available: ${available_space_mb}MB" return 0 } # Estimate backup size estimate_backup_size() { local total_size=0 for nickname in "${!PLEX_FILES[@]}"; do local file="${PLEX_FILES[$nickname]}" if [ -f "$file" ]; then local size_kb size_kb=$(du -k "$file" 2>/dev/null | cut -f1) total_size=$((total_size + size_kb)) fi done echo $((total_size / 1024)) # Return size in MB } # Generate performance report generate_performance_report() { if [ "$PERFORMANCE_MONITORING" != true ] || [ ! -f "$PERFORMANCE_LOG_FILE" ]; then return 0 fi log_info "Performance Summary:" # Recent performance data (last 10 entries) jq -r '.[-10:] | .[] | " \(.operation): \(.duration_seconds)s (\(.timestamp))"' "$PERFORMANCE_LOG_FILE" 2>/dev/null || true # Calculate averages for common operations local avg_backup avg_backup=$(jq '[.[] | select(.operation == "backup") | .duration_seconds] | if length > 0 then add/length else 0 end' "$PERFORMANCE_LOG_FILE" 2>/dev/null || echo "0") local avg_verification avg_verification=$(jq '[.[] | select(.operation == "verification") | .duration_seconds] | if length > 0 then add/length else 0 end' "$PERFORMANCE_LOG_FILE" 2>/dev/null || echo "0") local avg_service_stop avg_service_stop=$(jq '[.[] | select(.operation == "service_stop") | .duration_seconds] | if length > 0 then add/length else 0 end' "$PERFORMANCE_LOG_FILE" 2>/dev/null || echo "0") local avg_service_start avg_service_start=$(jq '[.[] | select(.operation == "service_start") | .duration_seconds] | if length > 0 then add/length else 0 end' "$PERFORMANCE_LOG_FILE" 2>/dev/null || echo "0") if [ "$avg_backup" != "0" ]; then log_info "Average backup time: ${avg_backup}s" fi if [ "$avg_verification" != "0" ]; then log_info "Average verification time: ${avg_verification}s" fi if [ "$avg_service_stop" != "0" ]; then log_info "Average service stop time: ${avg_service_stop}s" fi if [ "$avg_service_start" != "0" ]; then log_info "Average service start time: ${avg_service_start}s" fi } # Clean old backups cleanup_old_backups() { log_message "Cleaning up old backups..." # Remove backups older than MAX_BACKUP_AGE_DAYS find "${BACKUP_ROOT}" -maxdepth 1 -type f -name "plex-backup-*.tar.gz" -mtime +${MAX_BACKUP_AGE_DAYS} -delete 2>/dev/null || true # Keep only MAX_BACKUPS_TO_KEEP most recent backups local backup_count backup_count=$(find "${BACKUP_ROOT}" -maxdepth 1 -type f -name "plex-backup-*.tar.gz" | wc -l) if [ "$backup_count" -gt "$MAX_BACKUPS_TO_KEEP" ]; then local excess_count=$((backup_count - MAX_BACKUPS_TO_KEEP)) log_message "Removing $excess_count old backup(s)..." find "${BACKUP_ROOT}" -maxdepth 1 -type f -name "plex-backup-*.tar.gz" -printf '%T@ %p\n' | \ sort -n | head -n "$excess_count" | cut -d' ' -f2- | \ xargs -r rm -f fi # Clean up any remaining dated directories from old backup structure find "${BACKUP_ROOT}" -maxdepth 1 -type d -name "????????" -exec rm -rf {} \; 2>/dev/null || true log_message "Backup cleanup completed" } # Database integrity check only check_integrity_only() { log_message "Starting database integrity check at $(date)" # Stop Plex service - NEVER use force stop for integrity checks to prevent corruption if ! manage_plex_service stop; then log_error "Failed to stop Plex service gracefully" log_error "Cannot perform integrity check while service may be running" log_error "Manual intervention required - please stop Plex service manually" return 1 fi # Handle WAL files first handle_wal_files "checkpoint" local db_integrity_issues=0 local databases_checked=0 for nickname in "${!PLEX_FILES[@]}"; do local file="${PLEX_FILES[$nickname]}" # Only check database files if [[ "$file" == *".db" ]] && [ -f "$file" ]; then databases_checked=$((databases_checked + 1)) log_message "Checking integrity of $(basename "$file")..." if ! check_database_integrity "$file"; then db_integrity_issues=$((db_integrity_issues + 1)) log_warning "Database integrity issues found in $(basename "$file")" # Determine if we should attempt repair local should_repair=false if [ "$AUTO_REPAIR" = true ]; then should_repair=true log_message "Auto-repair enabled, attempting repair..." elif [ "$INTERACTIVE_MODE" = true ]; then read -p "Attempt to repair $(basename "$file")? [y/N]: " -n 1 -r -t 30 local read_result=$? echo if [ $read_result -eq 0 ] && [[ $REPLY =~ ^[Yy]$ ]]; then should_repair=true elif [ $read_result -ne 0 ]; then log_warning "Read timeout or error, defaulting to no repair" fi else log_warning "Non-interactive mode: skipping repair for $(basename "$file")" fi if [ "$should_repair" = true ]; then if repair_database "$file"; then log_success "Database repair successful for $(basename "$file")" # Re-check integrity after repair if check_database_integrity "$file"; then log_success "Post-repair integrity check passed for $(basename "$file")" else log_warning "Post-repair integrity check still shows issues for $(basename "$file")" fi else log_error "Database repair failed for $(basename "$file")" fi fi else log_success "Database integrity check passed for $(basename "$file")" fi fi done # Start Plex service manage_plex_service start # Summary log_message "Integrity check completed at $(date)" log_message "Databases checked: $databases_checked" log_message "Databases with issues: $db_integrity_issues" if [ "$db_integrity_issues" -gt 0 ]; then log_warning "Integrity check completed with issues found" exit 1 else log_success "All database integrity checks passed" exit 0 fi } # Main backup function main() { local overall_start overall_start=$(date +%s) log_message "Starting enhanced Plex backup process at $(date)" send_notification "Backup Started" "Plex backup process initiated" "info" # Create necessary directories mkdir -p "${BACKUP_ROOT}" mkdir -p "${LOCAL_LOG_ROOT}" # Ensure acedanger owns the log directories sudo chown -R acedanger:acedanger "${LOCAL_LOG_ROOT}" 2>/dev/null || true # Initialize logs initialize_logs # Check if only doing integrity check if [ "$INTEGRITY_CHECK_ONLY" = true ]; then check_integrity_only # shellcheck disable=SC2317 return $? fi # Estimate backup size local estimated_size_mb estimated_size_mb=$(estimate_backup_size) log_message "Estimated backup size: ${estimated_size_mb}MB" # Check disk space (require 2x estimated size for safety) local required_space_mb=$((estimated_size_mb * 2)) if ! check_disk_space "${BACKUP_ROOT}" "$required_space_mb"; then log_error "Aborting backup due to insufficient disk space" exit 1 fi # Stop Plex service manage_plex_service stop local backup_errors=0 local files_backed_up=0 local backed_up_files=() # Array to track successfully backed up files local BACKUP_PATH="${BACKUP_ROOT}" # Ensure backup root directory exists mkdir -p "$BACKUP_PATH" # Handle WAL files and check database integrity before backup log_message "Performing WAL checkpoint and checking database integrity before backup..." handle_wal_files "checkpoint" local db_integrity_issues=0 for nickname in "${!PLEX_FILES[@]}"; do local file="${PLEX_FILES[$nickname]}" # Only check database files if [[ "$file" == *".db" ]] && [ -f "$file" ]; then if ! check_database_integrity "$file"; then db_integrity_issues=$((db_integrity_issues + 1)) log_warning "Database integrity issues found in $(basename "$file")" # Always attempt repair when corruption is detected (default behavior) local should_repair=true local repair_attempted=false # Override repair behavior only if explicitly disabled if [ "$AUTO_REPAIR" = false ]; then should_repair=false log_warning "Auto-repair explicitly disabled, skipping repair" elif [ "$INTERACTIVE_MODE" = true ]; then read -p "Database $(basename "$file") has integrity issues. Attempt repair before backup? [Y/n]: " -n 1 -r -t 30 local read_result=$? echo if [ $read_result -eq 0 ] && [[ $REPLY =~ ^[Nn]$ ]]; then should_repair=false log_message "User declined repair for $(basename "$file")" elif [ $read_result -ne 0 ]; then log_message "Read timeout, proceeding with default repair" fi else log_message "Auto-repair enabled by default, attempting repair..." fi if [ "$should_repair" = true ]; then repair_attempted=true log_message "Attempting to repair corrupted database: $(basename "$file")" if repair_database "$file"; then log_success "Database repair successful for $(basename "$file")" # Re-verify integrity after repair if check_database_integrity "$file"; then log_success "Post-repair integrity verification passed for $(basename "$file")" # Decrement issue count since repair was successful db_integrity_issues=$((db_integrity_issues - 1)) else log_warning "Post-repair integrity check still shows issues for $(basename "$file")" log_warning "Will backup corrupted database - manual intervention may be needed" fi else log_error "Database repair failed for $(basename "$file")" log_warning "Will backup corrupted database - manual intervention may be needed" backup_errors=$((backup_errors + 1)) fi else log_warning "Skipping repair - will backup database with known integrity issues" fi # Log repair attempt for monitoring purposes if [ "$repair_attempted" = true ]; then send_notification "Database Repair" "Attempted repair of $(basename "$file")" "warning" fi fi fi done # Handle WAL files backup handle_wal_files "backup" "$BACKUP_PATH" # Backup files - always perform full backup local backup_start backup_start=$(date +%s) for nickname in "${!PLEX_FILES[@]}"; do local file="${PLEX_FILES[$nickname]}" if [ -f "$file" ]; then log_message "Backing up: $(basename "$file")" # Create backup filename without timestamp (use original filename) local backup_file backup_file="${BACKUP_PATH}/$(basename "$file")" # Copy file if sudo cp "$file" "$backup_file"; then # Force filesystem sync to prevent corruption sync # Ensure proper ownership of backup file sudo chown plex:plex "$backup_file" log_success "Copied: $(basename "$file")" # Verify backup if verify_backup "$file" "$backup_file"; then log_success "Verified: $(basename "$file")" files_backed_up=$((files_backed_up + 1)) # Add friendly filename to backed up files list case "$(basename "$file")" in "com.plexapp.plugins.library.db") backed_up_files+=("library.db") ;; "com.plexapp.plugins.library.blobs.db") backed_up_files+=("blobs.db") ;; "Preferences.xml") backed_up_files+=("Preferences.xml") ;; *) backed_up_files+=("$(basename "$file")") ;; esac else log_error "Verification failed: $(basename "$file")" backup_errors=$((backup_errors + 1)) # Remove failed backup rm -f "$backup_file" fi else log_error "Failed to copy: $(basename "$file")" backup_errors=$((backup_errors + 1)) fi else log_warning "File not found: $file" fi done # Start Plex service manage_plex_service start # Create archive if files were backed up if [ "$files_backed_up" -gt 0 ]; then log_message "Creating compressed archive..." # Check backup root directory is writable if [ ! -w "$BACKUP_ROOT" ]; then log_error "Backup root directory is not writable: $BACKUP_ROOT" backup_errors=$((backup_errors + 1)) else local temp_archive temp_archive="/tmp/plex-backup-$(date '+%Y%m%d_%H%M%S').tar.gz" local final_archive final_archive="${BACKUP_ROOT}/plex-backup-$(date '+%Y%m%d_%H%M%S').tar.gz" log_info "Temporary archive: $temp_archive" log_info "Final archive: $final_archive" # Create archive in /tmp first, containing only the backed up files local temp_dir temp_dir="/tmp/plex-backup-staging-$(date '+%Y%m%d_%H%M%S')" if ! mkdir -p "$temp_dir"; then log_error "Failed to create staging directory: $temp_dir" backup_errors=$((backup_errors + 1)) else log_info "Created staging directory: $temp_dir" # Copy backed up files to staging directory local files_staged=0 for nickname in "${!PLEX_FILES[@]}"; do local file="${PLEX_FILES[$nickname]}" local backup_file backup_file="${BACKUP_PATH}/$(basename "$file")" if [ -f "$backup_file" ]; then if cp "$backup_file" "$temp_dir/"; then files_staged=$((files_staged + 1)) log_info "Staged for archive: $(basename "$backup_file")" else log_warning "Failed to stage file: $(basename "$backup_file")" fi else log_warning "Backup file not found for staging: $(basename "$backup_file")" fi done # Check if any files were staged if [ "$files_staged" -eq 0 ]; then log_error "No files were staged for archive creation" rm -rf "$temp_dir" backup_errors=$((backup_errors + 1)) else log_info "Staged $files_staged files for archive creation" # Check disk space in /tmp local temp_available_kb temp_available_kb=$(df /tmp | awk 'NR==2 {print $4}') local temp_available_mb=$((temp_available_kb / 1024)) local staging_size_mb staging_size_mb=$(du -sm "$temp_dir" | cut -f1) log_info "/tmp available space: ${temp_available_mb}MB, staging directory size: ${staging_size_mb}MB" # Check if we have enough space (require 3x staging size for compression) local required_space_mb=$((staging_size_mb * 3)) if [ "$temp_available_mb" -lt "$required_space_mb" ]; then log_error "Insufficient space in /tmp for archive creation. Required: ${required_space_mb}MB, Available: ${temp_available_mb}MB" rm -rf "$temp_dir" backup_errors=$((backup_errors + 1)) else # Create archive with detailed error logging log_info "Creating archive: $(basename "$temp_archive")" local tar_output tar_output=$(tar -czf "$temp_archive" -C "$temp_dir" . 2>&1) local tar_exit_code=$? # Force filesystem sync after archive creation sync if [ $tar_exit_code -eq 0 ]; then # Verify archive was actually created and has reasonable size if [ -f "$temp_archive" ]; then local archive_size_mb archive_size_mb=$(du -sm "$temp_archive" | cut -f1) log_success "Archive created successfully: $(basename "$temp_archive") (${archive_size_mb}MB)" # Test archive integrity before moving if tar -tzf "$temp_archive" >/dev/null 2>&1; then log_success "Archive integrity verified" # Move the completed archive to the backup root if mv "$temp_archive" "$final_archive"; then # Force filesystem sync after final move sync log_success "Archive moved to final location: $(basename "$final_archive")" # Remove individual backup files and staging directory rm -rf "$temp_dir" for nickname in "${!PLEX_FILES[@]}"; do local file="${PLEX_FILES[$nickname]}" local backup_file backup_file="${BACKUP_PATH}/$(basename "$file")" rm -f "$backup_file" "$backup_file.md5" done else log_error "Failed to move archive to final location: $final_archive" log_error "Temporary archive remains at: $temp_archive" rm -rf "$temp_dir" backup_errors=$((backup_errors + 1)) fi else log_error "Archive integrity check failed - archive may be corrupted" log_error "Archive size: ${archive_size_mb}MB" rm -f "$temp_archive" rm -rf "$temp_dir" backup_errors=$((backup_errors + 1)) fi else log_error "Archive file was not created despite tar success" rm -rf "$temp_dir" backup_errors=$((backup_errors + 1)) fi else log_error "Failed to create archive (tar exit code: $tar_exit_code)" if [ -n "$tar_output" ]; then log_error "Tar command output: $tar_output" fi # Additional diagnostic information log_error "Staging directory contents:" find "$temp_dir" -ls 2>&1 | while IFS= read -r line; do log_error " $line" done local temp_usage temp_usage=$(df -h /tmp | awk 'NR==2 {print "Used: " $3 "/" $2 " (" $5 ")"}') log_error "Temp filesystem status: $temp_usage" rm -rf "$temp_dir" backup_errors=$((backup_errors + 1)) fi fi fi fi fi # Send notification local files_list files_list=$(format_backed_up_files "${backed_up_files[@]}") send_notification "Backup Completed" "Successfully backed up $files_list" "success" else log_message "No files needed backup" fi # Cleanup old backups cleanup_old_backups # Track overall backup performance if [ "$files_backed_up" -gt 0 ]; then track_performance "full_backup" "$backup_start" fi track_performance "total_script" "$overall_start" # Generate performance report generate_performance_report # Final summary local total_time=$(($(date +%s) - overall_start)) log_message "Backup process completed at $(date)" log_message "Total execution time: ${total_time}s" log_message "Files backed up: $files_backed_up" log_message "Errors encountered: $backup_errors" # Sync logs to shared location and cleanup old local logs log_info "Post-backup: synchronizing logs and cleaning up old files" sync_logs_to_shared cleanup_old_local_logs if [ "$backup_errors" -gt 0 ]; then log_error "Backup completed with errors" send_notification "Backup Error" "Backup completed with $backup_errors errors" "error" exit 1 else log_success "Enhanced backup completed successfully" local files_list files_list=$(format_backed_up_files "${backed_up_files[@]}") send_notification "Backup Success" "$files_list backed up successfully in ${total_time}s" "success" fi } # Trap to ensure Plex is restarted on script exit trap 'manage_plex_service start' EXIT # Run main function main "$@"