feat: Enhance database repair functionality with recovery options and nuclear recovery script

plex/plex.sh

@@ -117,13 +117,14 @@ show_loading() {
     printf "\r%s%s %s %s%s\n" "${CYAN}" "${HOURGLASS}" "${message}" "${CHECKMARK}" "${RESET}"
 }
 
-# 🔧 Function to repair database issues
+# 🔧 Enhanced function to repair database issues
 repair_database() {
     print_status "${INFO}" "Attempting to repair Plex database..." "${BLUE}"
     
     local db_dir="/var/lib/plexmediaserver/Library/Application Support/Plex Media Server/Plug-in Support/Databases"
     local main_db="$db_dir/com.plexapp.plugins.library.db"
     local backup_db="$db_dir/com.plexapp.plugins.library.db.backup.$(date +%Y%m%d_%H%M%S)"
+    local corrupted_dir="$db_dir/corrupted-$(date +%Y%m%d_%H%M%S)"
     
     if [[ ! -f "$main_db" ]]; then
         print_status "${CROSS}" "Main database not found at: $main_db" "${RED}"
@@ -133,7 +134,119 @@ repair_database() {
     # Stop Plex service first
     print_status "${INFO}" "Stopping Plex service..." "${BLUE}"
     sudo systemctl stop "$PLEX_SERVICE" 2>/dev/null || true
-    sleep 2
+    sleep 3
+    
+    # Check if critical tables exist
+    print_status "${INFO}" "Checking database structure..." "${BLUE}"
+    local has_metadata_table=false
+    if sudo -u plex sqlite3 "$main_db" "SELECT name FROM sqlite_master WHERE type='table' AND name='metadata_items';" 2>/dev/null | grep -q metadata_items; then
+        has_metadata_table=true
+    fi
+    
+    if [[ "$has_metadata_table" == "false" ]]; then
+        print_status "${CROSS}" "Critical table 'metadata_items' is missing! Database is severely corrupted." "${RED}"
+        print_status "${INFO}" "Attempting recovery from available backups..." "${YELLOW}"
+        
+        # Find the best recovery candidate
+        local recovery_db=""
+        local recovery_candidates=(
+            "$db_dir/com.plexapp.plugins.library.db.recovery-"*
+            "$db_dir/com.plexapp.plugins.library.db.20"*
+        )
+        
+        for candidate in "${recovery_candidates[@]}"; do
+            if [[ -f "$candidate" && "$candidate" != *"tmp"* && "$candidate" != *"empty"* ]]; then
+                # Test if this candidate has the metadata_items table
+                if sudo -u plex sqlite3 "$candidate" "SELECT name FROM sqlite_master WHERE type='table' AND name='metadata_items';" 2>/dev/null | grep -q metadata_items; then
+                    recovery_db="$candidate"
+                    break
+                fi
+            fi
+        done
+        
+        if [[ -n "$recovery_db" ]]; then
+            print_status "${CHECKMARK}" "Found recovery database: $(basename "$recovery_db")" "${GREEN}"
+            
+            # Move corrupted database to backup location
+            print_status "${INFO}" "Moving corrupted database to backup location..." "${BLUE}"
+            sudo mkdir -p "$corrupted_dir"
+            sudo mv "$main_db" "$corrupted_dir/"
+            sudo mv "$main_db-shm" "$corrupted_dir/" 2>/dev/null || true
+            sudo mv "$main_db-wal" "$corrupted_dir/" 2>/dev/null || true
+            
+            # Copy recovery database as new main database
+            print_status "${INFO}" "Restoring database from recovery file..." "${BLUE}"
+            if sudo cp "$recovery_db" "$main_db"; then
+                sudo chown plex:plex "$main_db"
+                sudo chmod 644 "$main_db"
+                print_status "${CHECKMARK}" "Database restored successfully!" "${GREEN}"
+                
+                # Verify the restored database
+                print_status "${INFO}" "Verifying restored database..." "${BLUE}"
+                local integrity_result
+                integrity_result=$(sudo -u plex sqlite3 "$main_db" "PRAGMA integrity_check;" 2>&1)
+                
+                if echo "$integrity_result" | grep -q "ok"; then
+                    print_status "${CHECKMARK}" "Restored database integrity verified!" "${GREEN}"
+                    return 0
+                elif echo "$integrity_result" | grep -q "no such collation sequence: icu"; then
+                    print_status "${CROSS}" "ICU collation sequence issue detected!" "${YELLOW}"
+                    print_status "${INFO}" "Attempting ICU-aware recovery..." "${BLUE}"
+                    
+                    # Try ICU-aware recovery script
+                    local icu_script="${SCRIPT_DIR}/icu-aware-recovery.sh"
+                    if [[ -f "$icu_script" ]]; then
+                        if "$icu_script" --auto; then
+                            print_status "${CHECKMARK}" "ICU-aware recovery completed!" "${GREEN}"
+                            return 0
+                        else
+                            print_status "${CROSS}" "ICU-aware recovery failed!" "${RED}"
+                        fi
+                    else
+                        print_status "${INFO}" "ICU recovery script not found, trying manual fix..." "${YELLOW}"
+                        
+                        # Try to recreate database without ICU dependencies
+                        local temp_db="/tmp/plex_temp_$(date +%Y%m%d_%H%M%S).db"
+                        print_status "${INFO}" "Attempting to dump and recreate database..." "${BLUE}"
+                        
+                        if sudo -u plex sqlite3 "$recovery_db" ".dump" | grep -v "COLLATE icu_" | sudo -u plex sqlite3 "$temp_db"; then
+                            print_status "${INFO}" "Database dump successful, replacing main database..." "${BLUE}"
+                            sudo mv "$temp_db" "$main_db"
+                            sudo chown plex:plex "$main_db"
+                            sudo chmod 644 "$main_db"
+                            
+                            # Verify the recreated database
+                            if sudo -u plex sqlite3 "$main_db" "PRAGMA integrity_check;" 2>/dev/null | grep -q "ok"; then
+                                print_status "${CHECKMARK}" "Database recreated successfully without ICU!" "${GREEN}"
+                                return 0
+                            fi
+                        fi
+                        
+                        # Clean up temp file if it exists
+                        sudo rm -f "$temp_db" 2>/dev/null || true
+                    fi
+                    
+                    print_status "${CROSS}" "Failed to resolve ICU collation issues!" "${RED}"
+                    return 1
+                else
+                    print_status "${CROSS}" "Restored database failed integrity check!" "${RED}"
+                    print_status "${INFO}" "Integrity check result:" "${YELLOW}"
+                    echo -e "${DIM}${YELLOW}   $integrity_result${RESET}"
+                    return 1
+                fi
+            else
+                print_status "${CROSS}" "Failed to restore database!" "${RED}"
+                return 1
+            fi
+        else
+            print_status "${CROSS}" "No valid recovery databases found!" "${RED}"
+            print_status "${INFO}" "Available options:" "${YELLOW}"
+            echo -e "${DIM}${YELLOW}   1. Check manual backups in /mnt/share/media/backups/plex/${RESET}"
+            echo -e "${DIM}${YELLOW}   2. Let Plex rebuild database (will lose all metadata)${RESET}"
+            echo -e "${DIM}${YELLOW}   3. Run: sudo rm '$main_db' && sudo systemctl start plexmediaserver${RESET}"
+            return 1
+        fi
+    fi
     
     # Create backup of current database
     print_status "${INFO}" "Creating backup of current database..." "${BLUE}"
@@ -146,11 +259,11 @@ repair_database() {
     
     # Try to vacuum the database
     print_status "${INFO}" "Running VACUUM on database..." "${BLUE}"
-    if sudo -u plex sqlite3 "$main_db" "VACUUM;"; then
+    if sudo -u plex sqlite3 "$main_db" "VACUUM;" 2>/dev/null; then
         print_status "${CHECKMARK}" "Database VACUUM completed successfully" "${GREEN}"
         
         # Test integrity again
-        if sudo -u plex sqlite3 "$main_db" "PRAGMA integrity_check;" | grep -q "ok"; then
+        if sudo -u plex sqlite3 "$main_db" "PRAGMA integrity_check;" 2>/dev/null | grep -q "ok"; then
             print_status "${CHECKMARK}" "Database integrity restored!" "${GREEN}"
             print_status "${INFO}" "You can now try starting Plex again" "${BLUE}"
             return 0
@@ -161,11 +274,23 @@ repair_database() {
         print_status "${CROSS}" "VACUUM operation failed" "${RED}"
     fi
     
-    # If VACUUM failed, suggest restore options
+    # Try reindex as last resort
-    print_status "${INFO}" "VACUUM repair failed. Consider these options:" "${YELLOW}"
+    print_status "${INFO}" "Attempting REINDEX operation..." "${BLUE}"
-    echo -e "${DIM}${YELLOW}   1. Restore from a backup using restore-plex.sh${RESET}"
+    if sudo -u plex sqlite3 "$main_db" "REINDEX;" 2>/dev/null; then
-    echo -e "${DIM}${YELLOW}   2. Delete corrupted database (Plex will rebuild, but you'll lose metadata)${RESET}"
+        print_status "${CHECKMARK}" "Database REINDEX completed" "${GREEN}"
-    echo -e "${DIM}${YELLOW}   3. Check for corrupted database backups in: $db_dir/corrupted-*/${RESET}"
+        
+        # Test integrity one more time
+        if sudo -u plex sqlite3 "$main_db" "PRAGMA integrity_check;" 2>/dev/null | grep -q "ok"; then
+            print_status "${CHECKMARK}" "Database integrity restored after REINDEX!" "${GREEN}"
+            return 0
+        fi
+    fi
+    
+    print_status "${CROSS}" "All repair attempts failed" "${RED}"
+    print_status "${INFO}" "Manual intervention required. Consider:" "${YELLOW}"
+    echo -e "${DIM}${YELLOW}   1. Restore from external backup using restore-plex.sh${RESET}"
+    echo -e "${DIM}${YELLOW}   2. Use nuclear recovery: ./nuclear-plex-recovery.sh${RESET}"
+    echo -e "${DIM}${YELLOW}   3. Check corrupted database moved to: $corrupted_dir${RESET}"
     
     return 1
 }
@@ -385,53 +510,94 @@ show_help() {
     echo -e "  ${BLUE}${BOLD}restart${RESET}   ${RECYCLE} Restart Plex Media Server"
     echo -e "  ${CYAN}${BOLD}status${RESET}    ${INFO} Show detailed service status"
     echo -e "  ${RED}${BOLD}repair${RESET}    [!] Repair database corruption issues"
+    echo -e "  ${RED}${BOLD}nuclear${RESET}   [!!] Nuclear database recovery (last resort)"
     echo -e "  ${PURPLE}${BOLD}help${RESET}      [*] Show this help message"
     echo ""
     echo -e "${DIM}${WHITE}Examples:${RESET}"
     echo -e "  ${DIM}${SCRIPT_NAME} start     # Start the Plex service${RESET}"
     echo -e "  ${DIM}${SCRIPT_NAME} status    # Show current status${RESET}"
     echo -e "  ${DIM}${SCRIPT_NAME} repair    # Fix database issues${RESET}"
+    echo -e "  ${DIM}${SCRIPT_NAME} nuclear   # Complete database replacement${RESET}"
     echo ""
 }
 
-# Database repair function using shared script
+# Nuclear database recovery function
-repair_plex() {
+nuclear_recovery() {
-    print_header "DATABASE REPAIR"
+    print_header
-    print_status "${INFO}" "Attempting to repair Plex database..." "${YELLOW}"
+    print_status "${INFO}" "Starting nuclear database recovery..." "${RED}"
     
-    local db_dir="/var/lib/plexmediaserver/Library/Application Support/Plex Media Server/Plug-in Support/Databases"
+    local nuclear_script="${SCRIPT_DIR}/nuclear-plex-recovery.sh"
-    local main_db="$db_dir/com.plexapp.plugins.library.db"
-    local repair_script="${SCRIPT_DIR}/plex-database-repair.sh"
     
-    if [[ ! -f "$repair_script" ]]; then
+    if [[ ! -f "$nuclear_script" ]]; then
-        print_status "${CROSS}" "Database repair script not found: $repair_script" "${RED}"
+        print_status "${CROSS}" "Nuclear recovery script not found: $nuclear_script" "${RED}"
+        print_status "${INFO}" "This script should be in the same directory as plex.sh" "${YELLOW}"
         return 2
     fi
     
-    if [[ ! -f "$main_db" ]]; then
+    # Warning message
-        print_status "${CROSS}" "Main database not found at: $main_db" "${RED}"
+    echo -e "\n${RED}${BOLD}⚠️  WARNING: NUCLEAR RECOVERY ⚠️${RESET}"
-        return 1
+    echo -e "${RED}This will completely replace your Plex database with a backup!${RESET}"
+    echo -e "${RED}All changes since the backup was created will be lost!${RESET}"
+    echo -e "${YELLOW}This should only be used when standard repair methods have failed.${RESET}\n"
+    
+    # Get user confirmation
+    echo -e "${CYAN}Do you want to proceed with nuclear recovery? ${RESET}"
+    echo -e "${DIM}Type 'YES' (uppercase) to confirm: ${RESET}"
+    read -r confirmation
+    
+    if [[ "$confirmation" != "YES" ]]; then
+        print_status "${INFO}" "Nuclear recovery cancelled by user" "${YELLOW}"
+        return 0
     fi
     
-    # Stop Plex service first
+    print_status "${INFO}" "Executing nuclear recovery script..." "${BLUE}"
-    print_status "${INFO}" "Stopping Plex service for repair..." "${BLUE}"
+    
-    if ! sudo systemctl stop "$PLEX_SERVICE"; then
+    # Execute the nuclear recovery script
-        print_status "${CROSS}" "Failed to stop Plex service!" "${RED}"
+    if sudo "$nuclear_script" --auto; then
-        return 1
+        print_status "${CHECKMARK}" "Nuclear recovery completed successfully!" "${GREEN}"
+        print_status "${INFO}" "Your Plex server should now be operational" "${GREEN}"
+        print_footer
+        return 0
+    else
+        local exit_code=$?
+        print_status "${CROSS}" "Nuclear recovery failed!" "${RED}"
+        
+        case $exit_code in
+            2) print_status "${INFO}" "Backup file issues - check backup integrity" "${YELLOW}" ;;
+            3) print_status "${INFO}" "Database replacement failure - check permissions" "${YELLOW}" ;;
+            4) print_status "${INFO}" "Service management failure - check systemctl" "${YELLOW}" ;;
+            5) print_status "${INFO}" "Rollback performed due to failure" "${YELLOW}" ;;
+            *) print_status "${INFO}" "Unknown error occurred during recovery" "${YELLOW}" ;;
+        esac
+        
+        print_footer
+        return $exit_code
     fi
+}
 
-    # Run the repair
+# Database repair function
-    print_status "${INFO}" "Running database repair (this may take a while)..." "${BLUE}"
+repair_plex() {
+    print_header
+    print_status "${INFO}" "Starting Plex database repair..." "${YELLOW}"
     
-    if "$repair_script" repair "$main_db"; then
+    # Run the enhanced repair function
+    if repair_database; then
         print_status "${CHECKMARK}" "Database repair completed successfully!" "${GREEN}"
         
         # Try to start the service
         print_status "${INFO}" "Starting Plex service..." "${BLUE}"
         if sudo systemctl start "$PLEX_SERVICE"; then
-            print_status "${CHECKMARK}" "Plex service started successfully!" "${GREEN}"
+            # Wait for service to fully start
-            print_footer
+            sleep 5
-            return 0
+            if systemctl is-active --quiet "$PLEX_SERVICE"; then
+                print_status "${CHECKMARK}" "Plex service started successfully!" "${GREEN}"
+                print_footer
+                return 0
+            else
+                print_status "${CROSS}" "Service started but may not be fully operational!" "${YELLOW}"
+                show_detailed_status
+                return 1
+            fi
         else
             print_status "${CROSS}" "Failed to start Plex service after repair!" "${RED}"
             return 1
@@ -440,7 +606,7 @@ repair_plex() {
         local repair_exit_code=$?
         print_status "${CROSS}" "Database repair failed!" "${RED}"
         
-        # Try to start the service anyway
+        # Try to start the service anyway in case partial repair helped
         print_status "${INFO}" "Attempting to start Plex service anyway..." "${BLUE}"
         sudo systemctl start "$PLEX_SERVICE" 2>/dev/null || true
         
@@ -492,6 +658,9 @@ main() {
         "repair"|"fix")
             repair_plex
             ;;
+        "nuclear"|"nuke")
+            nuclear_recovery
+            ;;
         "help"|"--help"|"-h")
             print_header
             show_help