feat: Add comprehensive Plex recovery validation script

- Introduced `validate-plex-recovery.sh` for validating Plex database recovery.
- Implemented checks for service status, database integrity, web interface accessibility, API functionality, and recent logs.
- Added detailed recovery summary and next steps for users.

fix: Improve Debian patching script for compatibility

- Enhanced `debian-patches.sh` to securely download and execute bootstrap scripts.
- Updated package mapping logic and ensured proper permissions for patched files.

fix: Update Docker test scripts for better permission handling

- Modified `run-docker-tests.sh` to set appropriate permissions on logs directory.
- Ensured log files have correct permissions after test runs.

fix: Enhance setup scripts for secure installations

- Updated `setup.sh` to securely download and execute installation scripts for zoxide and nvm.
- Improved error handling for failed downloads.

fix: Refine startup script for log directory permissions

- Adjusted `startup.sh` to set proper permissions for log directories and files.

chore: Revamp update-containers.sh for better error handling and logging

- Rewrote `update-containers.sh` to include detailed logging and error handling.
- Added validation for Docker image names and improved overall script robustness.

plex/backup-plex.sh

@@ -1,5 +1,52 @@
 #!/bin/bash
 
+################################################################################
+# Plex Media Server Enhanced Backup Script
+################################################################################
+#
+# Author: Peter Wood <peter@peterwood.dev>
+# Description: Comprehensive backup solution for Plex Media Server with advanced
+#              database integrity checking, automated repair capabilities,
+#              performance monitoring, and multi-channel notifications.
+#
+# Features:
+#   - Database integrity verification with automatic repair
+#   - WAL (Write-Ahead Logging) file handling
+#   - Performance monitoring with JSON logging
+#   - Parallel verification for improved speed
+#   - Multi-channel notifications (webhook, email, console)
+#   - Comprehensive error handling and recovery
+#   - Automated cleanup of old backups
+#
+# Related Scripts:
+#   - restore-plex.sh: Restore from backups created by this script
+#   - validate-plex-backups.sh: Validate backup integrity and health
+#   - monitor-plex-backup.sh: Real-time monitoring dashboard
+#   - test-plex-backup.sh: Comprehensive testing suite
+#   - plex.sh: General Plex service management
+#
+# Usage:
+#   ./backup-plex.sh                    # Standard backup with auto-repair
+#   ./backup-plex.sh --disable-auto-repair  # Backup without auto-repair
+#   ./backup-plex.sh --check-integrity  # Integrity check only
+#   ./backup-plex.sh --non-interactive  # Automated mode for cron jobs
+#
+# Dependencies:
+#   - Plex Media Server
+#   - sqlite3 or Plex SQLite binary
+#   - curl (for webhook notifications)
+#   - jq (for JSON processing)
+#   - sendmail (optional, for email notifications)
+#
+# Exit Codes:
+#   0 - Success
+#   1 - General error
+#   2 - Database integrity issues
+#   3 - Service management failure
+#   4 - Backup creation failure
+#
+################################################################################
+
 set -e
 
 # Color codes for output
@@ -32,7 +79,7 @@ PERFORMANCE_LOG_FILE="${LOCAL_LOG_ROOT}/plex-backup-performance.json"
 PLEX_SQLITE="/usr/lib/plexmediaserver/Plex SQLite"
 
 # Script options
-AUTO_REPAIR=false
+AUTO_REPAIR=true  # Default to enabled for automatic corruption detection and repair
 INTEGRITY_CHECK_ONLY=false
 INTERACTIVE_MODE=false
 PARALLEL_VERIFICATION=true
@@ -48,6 +95,10 @@ while [[ $# -gt 0 ]]; do
             INTERACTIVE_MODE=false
             shift
             ;;
+        --disable-auto-repair)
+            AUTO_REPAIR=false
+            shift
+            ;;
         --check-integrity)
             INTEGRITY_CHECK_ONLY=true
             shift
@@ -79,15 +130,22 @@ while [[ $# -gt 0 ]]; do
         -h|--help)
             echo "Usage: $0 [OPTIONS]"
             echo "Options:"
-            echo "  --auto-repair        Automatically attempt to repair corrupted databases"
+            echo "  --auto-repair        Force enable automatic database repair (default: enabled)"
+            echo "  --disable-auto-repair Disable automatic database repair"
             echo "  --check-integrity    Only check database integrity, don't backup"
             echo "  --non-interactive    Run in non-interactive mode (for automation)"
+            echo "  --interactive        Run in interactive mode (prompts for repair decisions)"
             echo "  --no-parallel        Disable parallel verification (slower but safer)"
             echo "  --no-performance     Disable performance monitoring"
             echo "  --webhook=URL        Send notifications to webhook URL"
             echo "  --email=ADDRESS      Send notifications to email address"
             echo "  -h, --help          Show this help message"
             echo ""
+            echo "Database Integrity & Repair:"
+            echo "  By default, the script automatically detects and attempts to repair"
+            echo "  corrupted databases before backup. Use --disable-auto-repair to"
+            echo "  skip repair and backup corrupted databases as-is."
+            echo ""
             exit 0
             ;;
         *)
@@ -1100,32 +1158,56 @@ main() {
                 db_integrity_issues=$((db_integrity_issues + 1))
                 log_warning "Database integrity issues found in $(basename "$file")"
 
-                # Determine if we should attempt repair
-                local should_repair=false
+                # Always attempt repair when corruption is detected (default behavior)
+                local should_repair=true
+                local repair_attempted=false
 
-                if [ "$AUTO_REPAIR" = true ]; then
-                    should_repair=true
-                    log_message "Auto-repair enabled, attempting repair..."
+                # Override repair behavior only if explicitly disabled
+                if [ "$AUTO_REPAIR" = false ]; then
+                    should_repair=false
+                    log_warning "Auto-repair explicitly disabled, skipping repair"
                 elif [ "$INTERACTIVE_MODE" = true ]; then
-                    read -p "Database $(basename "$file") has integrity issues. Attempt repair before backup? [y/N]: " -n 1 -r -t 30
+                    read -p "Database $(basename "$file") has integrity issues. Attempt repair before backup? [Y/n]: " -n 1 -r -t 30
                     local read_result=$?
                     echo
-                    if [ $read_result -eq 0 ] && [[ $REPLY =~ ^[Yy]$ ]]; then
-                        should_repair=true
+                    if [ $read_result -eq 0 ] && [[ $REPLY =~ ^[Nn]$ ]]; then
+                        should_repair=false
+                        log_message "User declined repair for $(basename "$file")"
                     elif [ $read_result -ne 0 ]; then
-                        log_warning "Read timeout or error, defaulting to no repair"
+                        log_message "Read timeout, proceeding with default repair"
                     fi
                 else
-                    log_warning "Non-interactive mode: backing up database with integrity issues"
+                    log_message "Auto-repair enabled by default, attempting repair..."
                 fi
 
                 if [ "$should_repair" = true ]; then
+                    repair_attempted=true
+                    log_message "Attempting to repair corrupted database: $(basename "$file")"
+
                     if repair_database "$file"; then
                         log_success "Database repair successful for $(basename "$file")"
+
+                        # Re-verify integrity after repair
+                        if check_database_integrity_with_wal "$file"; then
+                            log_success "Post-repair integrity verification passed for $(basename "$file")"
+                            # Decrement issue count since repair was successful
+                            db_integrity_issues=$((db_integrity_issues - 1))
+                        else
+                            log_warning "Post-repair integrity check still shows issues for $(basename "$file")"
+                            log_warning "Will backup with known integrity issues"
+                        fi
                     else
                         log_error "Database repair failed for $(basename "$file")"
+                        log_warning "Will backup corrupted database - manual intervention may be needed"
                         backup_errors=$((backup_errors + 1))
                     fi
+                else
+                    log_warning "Skipping repair - will backup database with known integrity issues"
+                fi
+
+                # Log repair attempt for monitoring purposes
+                if [ "$repair_attempted" = true ]; then
+                    send_notification "Database Repair" "Attempted repair of $(basename "$file")" "warning"
                 fi
             fi
         fi