From c1479a8b0ca70964579d8a19f57c54da492b717a Mon Sep 17 00:00:00 2001 From: Peter Wood Date: Tue, 3 Jun 2025 14:38:55 -0400 Subject: [PATCH] feat: Implement comprehensive restore functionality for Immich - Added `restore-immich.sh` script to handle complete restoration from backups. - Implemented database restoration with integrity checks and error handling. - Added uploads restoration with proper ownership and permissions setup. - Introduced validation script `validate-immich-backups.sh` for backup integrity checks. - Created test suite `test-immich-restore.sh` to validate restoration functionality with mock data. - Enhanced logging and notification features for restoration processes. - Updated README.md with detailed usage instructions for backup and restore workflows. --- README.md | 58 +++- immich/README.md | 237 +++++++++++++++- immich/restore-immich.sh | 454 ++++++++++++++++++++++++++++-- immich/test-immich-restore.sh | 423 ++++++++++++++++++++++++++++ immich/validate-immich-backups.sh | 62 +++- 5 files changed, 1188 insertions(+), 46 deletions(-) create mode 100755 immich/test-immich-restore.sh diff --git a/README.md b/README.md index 383a3f8..2708136 100644 --- a/README.md +++ b/README.md @@ -20,6 +20,9 @@ This repository contains various shell scripts for managing media-related tasks - **`backup-plex.sh`**: Enhanced Plex backup script with integrity verification, incremental backups, and advanced features. - **`restore-plex.sh`**: Script to restore Plex data from backups with safety checks. - **`validate-plex-backups.sh`**: Script to validate backup integrity and monitor backup health. +- **`backup-immich.sh`**: Comprehensive Immich backup script with PostgreSQL database and upload directory backup to B2 storage. +- **`restore-immich.sh`**: Complete Immich restoration script with database and upload directory restoration from backups. +- **`validate-immich-backups.sh`**: Immich backup validation script with integrity checks and restoration command generation. ### Management Scripts @@ -194,6 +197,45 @@ Specialized backup system for Plex Media Server with database-aware features. Fo ./restore-plex.sh plex-backup-20250125_143022.tar.gz ``` +### Immich Photo Management Operations + +```bash +# Create comprehensive Immich backup (database + uploads) +./immich/backup-immich.sh + +# Test backup process without uploading to B2 +./immich/backup-immich.sh --dry-run + +# Backup without uploading to cloud storage +./immich/backup-immich.sh --no-upload + +# Validate existing Immich backups +./immich/validate-immich-backups.sh + +# Test restoration with dry-run (recommended first) +./immich/restore-immich.sh \ + --db-backup ./immich_backups/immich_db_backup_20250603_140000.sql.gz \ + --uploads-backup ./immich_backups/immich_uploads_20250603_140000.tar.gz \ + --dry-run + +# Complete restoration from backup files +./immich/restore-immich.sh \ + --db-backup ./immich_backups/immich_db_backup_20250603_140000.sql.gz \ + --uploads-backup ./immich_backups/immich_uploads_20250603_140000.tar.gz + +# Restore database only (skip uploads) +./immich/restore-immich.sh \ + --db-backup ./immich_backups/immich_db_backup_20250603_140000.sql.gz \ + --uploads-backup ./immich_backups/immich_uploads_20250603_140000.tar.gz \ + --skip-uploads + +# Restore uploads only (skip database) +./immich/restore-immich.sh \ + --db-backup ./immich_backups/immich_db_backup_20250603_140000.sql.gz \ + --uploads-backup ./immich_backups/immich_uploads_20250603_140000.tar.gz \ + --skip-db +``` + ## Automation and Scheduling ### Daily Media Backup @@ -218,6 +260,16 @@ Specialized backup system for Plex Media Server with database-aware features. Fo 0 7 * * * /home/acedanger/shell/validate-plex-backups.sh --fix ``` +### Automated Immich Backup + +```bash +# Add to crontab for daily Immich backup at 4 AM +0 4 * * * /home/acedanger/shell/immich/backup-immich.sh + +# Add weekly validation (Sundays at 9 AM) +0 9 * * 0 /home/acedanger/shell/immich/validate-immich-backups.sh +``` + ### Weekly Comprehensive Validation Report ```bash @@ -242,7 +294,7 @@ This repository includes extensive documentation organized by component and func ``` šŸ“ docs/ - Technical documentation and guides -šŸ“ setup/ - Setup and configuration files +šŸ“ setup/ - Setup and configuration files šŸ“ plex/ - Plex Media Server scripts and docs šŸ“ immich/ - Immich photo management scripts šŸ“ crontab/ - Crontab management system @@ -260,7 +312,7 @@ This repository includes extensive documentation organized by component and func - **[Environment Backup System](./docs/env-backup-system.md)**: Environment and configuration backup - **[Backup Environment Safety Guide](./docs/backup-env-safety-guide.md)**: Safety guidelines for environment backups -#### Media Server Management +#### Media Server Management - **[Plex Scripts Documentation](./plex/README.md)**: Comprehensive Plex backup, restoration, and management - **[Plex Backup Guide](./plex/plex-backup.md)**: Detailed Plex backup system documentation @@ -358,7 +410,7 @@ Test the AI setup: # Test Gemini integration echo "Test text" | fabric -p summarize -# Test local Ollama integration +# Test local Ollama integration echo "Test text" | fabric -p summarize -m ollama:phi3:mini ``` diff --git a/immich/README.md b/immich/README.md index 1eede01..76122f0 100644 --- a/immich/README.md +++ b/immich/README.md @@ -436,25 +436,234 @@ For complete restore instructions, see: > /home/acedanger/shell/logs/immich-backup.log 2>&1 +./restore-immich.sh --db-backup --uploads-backup [OPTIONS] +``` + +**Required Arguments:** + +- `--db-backup PATH` - Path to database backup file (.sql.gz) +- `--uploads-backup PATH` - Path to uploads backup file (.tar.gz) + +**Optional Arguments:** + +- `--dry-run` - Show what would be restored without making changes +- `--skip-db` - Skip database restoration (uploads only) +- `--skip-uploads` - Skip uploads restoration (database only) +- `--help` - Show help message and exit + +**Examples:** + +```bash +# Complete restoration (recommended to run dry-run first) +./restore-immich.sh \ + --db-backup ../immich_backups/immich_db_backup_20250603_120000.sql.gz \ + --uploads-backup ../immich_backups/immich_uploads_20250603_120000.tar.gz \ + --dry-run + +# Actual restoration after verifying dry-run +./restore-immich.sh \ + --db-backup ../immich_backups/immich_db_backup_20250603_120000.sql.gz \ + --uploads-backup ../immich_backups/immich_uploads_20250603_120000.tar.gz + +# Restore database only +./restore-immich.sh \ + --db-backup ../immich_backups/immich_db_backup_20250603_120000.sql.gz \ + --uploads-backup ../immich_backups/immich_uploads_20250603_120000.tar.gz \ + --skip-uploads + +# Restore uploads only +./restore-immich.sh \ + --db-backup ../immich_backups/immich_db_backup_20250603_120000.sql.gz \ + --uploads-backup ../immich_backups/immich_uploads_20250603_120000.tar.gz \ + --skip-db +``` + +**Safety Features:** + +- **Dry-run mode**: Preview all operations without making changes +- **File validation**: Verifies backup file integrity before restoration +- **Container management**: Safely stops/starts services during restoration +- **Cleanup on failure**: Automatically restores services if restoration fails +- **Comprehensive logging**: All operations logged to `../logs/immich-restore.log` +- **Notification support**: Sends webhook notifications for restoration status + +### validate-immich-backups.sh + +Validation script that checks the integrity and completeness of Immich backup files: + +- File integrity validation (gzip and tar.gz corruption detection) +- Backup pairing verification (matching database and upload backups) +- Size and content validation +- Restoration command generation for valid backups + +**Usage:** + +```bash +./validate-immich-backups.sh [backup_directory] +``` + +**Arguments:** + +- `backup_directory` - Directory containing backup files (default: `../immich_backups`) + +**Examples:** + +```bash +# Validate backups in default directory +./validate-immich-backups.sh + +# Validate backups in specific directory +./validate-immich-backups.sh /path/to/backups + +# Example output for valid backups +=== IMMICH BACKUP VALIDATION === +Backup directory: ../immich_backups + +=== DATABASE BACKUPS === +Validating: immich_db_backup_20250603_120000.sql.gz + āœ“ File size OK (45M) + āœ“ Gzip file integrity OK + āœ“ SQL content appears valid + +=== UPLOAD BACKUPS === +Validating: immich_uploads_20250603_120000.tar.gz + āœ“ File size OK (2.1G) + āœ“ Tar.gz file integrity OK + āœ“ Archive contains 15,234 files/directories + +=== RESTORATION COMMANDS === +To restore the most recent complete backup: + +# Dry run (recommended first): +./restore-immich.sh \ + --db-backup "../immich_backups/immich_db_backup_20250603_120000.sql.gz" \ + --uploads-backup "../immich_backups/immich_uploads_20250603_120000.tar.gz" \ + --dry-run +``` + +## Backup and Restore Workflow + +### 1. Regular Backup + +```bash +# Run daily backup (typically via cron) +./backup-immich.sh +``` + +### 2. Validate Backups + +```bash +# Check backup integrity periodically +./validate-immich-backups.sh +``` + +### 3. Restoration Process + +```bash +# Step 1: Validate available backups and get commands +./validate-immich-backups.sh + +# Step 2: Test restoration with dry-run +./restore-immich.sh \ + --db-backup "path/to/db_backup.sql.gz" \ + --uploads-backup "path/to/uploads_backup.tar.gz" \ + --dry-run + +# Step 3: Perform actual restoration +./restore-immich.sh \ + --db-backup "path/to/db_backup.sql.gz" \ + --uploads-backup "path/to/uploads_backup.tar.gz" +``` + +## Advanced Usage + +### Partial Restoration + +For specific restoration scenarios: + +```bash +# Database only (preserve existing uploads) +./restore-immich.sh \ + --db-backup "db_backup.sql.gz" \ + --uploads-backup "uploads_backup.tar.gz" \ + --skip-uploads + +# Uploads only (preserve existing database) +./restore-immich.sh \ + --db-backup "db_backup.sql.gz" \ + --uploads-backup "uploads_backup.tar.gz" \ + --skip-db +``` + +### Emergency Recovery + +In case of data loss or corruption: + +1. **Stop Immich services**: `docker stop immich_server immich_postgres` +2. **Validate backups**: `./validate-immich-backups.sh` +3. **Dry-run restoration**: Use `--dry-run` first +4. **Restore data**: Remove `--dry-run` flag +5. **Verify functionality**: Check Immich web interface + +### Monitoring and Logs + +All scripts log to the centralized logging directory: + +- **Backup logs**: `../logs/immich-backup.log` +- **Restoration logs**: `../logs/immich-restore.log` +- **Validation logs**: `../logs/immich-validation.log` + +## Troubleshooting + +### Common Issues + +1. **Container not found**: Ensure Docker containers are running +2. **Permission denied**: Check file permissions and Docker access +3. **Database connection failed**: Verify PostgreSQL credentials in `.env` +4. **Insufficient disk space**: Check available space before restoration +5. **Backup file corruption**: Use `validate-immich-backups.sh` to check integrity + +### Recovery Steps + +1. Check logs in `../logs/` directory +2. Verify `.env` file configuration +3. Ensure Docker containers are accessible +4. Validate backup file integrity +5. Use dry-run mode to test operations + +## Environment Configuration + +Required `.env` file structure: + +```bash +# Database Configuration +DB_USERNAME=postgres +DB_DATABASE_NAME=immich + +# Upload Directory +UPLOAD_LOCATION=/path/to/immich/uploads + +# Optional: Notification Settings +NTFY_URL=https://ntfy.sh/your-topic ``` diff --git a/immich/restore-immich.sh b/immich/restore-immich.sh index 79c7378..92d0402 100755 --- a/immich/restore-immich.sh +++ b/immich/restore-immich.sh @@ -122,20 +122,440 @@ if [ "$DRY_RUN" = true ]; then echo "" fi -# TODO: Implement restore logic -echo "āš ļø RESTORE SCRIPT TEMPLATE āš ļø" -echo "" -echo "This is a template script. Implementation needed:" -echo "" -echo "1. Stop Immich containers" -echo "2. Restore database (if not skipped):" -echo " - Decompress $DB_BACKUP" -echo " - Execute SQL restore commands" -echo "3. Restore uploads (if not skipped):" -echo " - Extract $UPLOADS_BACKUP to $UPLOAD_LOCATION" -echo " - Set proper ownership and permissions" -echo "4. Restart Immich containers" -echo "5. Verify restoration" -echo "" -echo "For detailed restore instructions, see:" -echo "https://immich.app/docs/administration/backup-and-restore/" +# Verify required environment variables are set +if [ -z "$DB_USERNAME" ] || [ -z "$DB_DATABASE_NAME" ] || [ -z "$UPLOAD_LOCATION" ]; then + echo "Error: Required environment variables (DB_USERNAME, DB_DATABASE_NAME, UPLOAD_LOCATION) not found in .env file" + echo "Please ensure your .env file contains:" + echo " - DB_USERNAME=" + echo " - DB_DATABASE_NAME=" + echo " - UPLOAD_LOCATION=" + exit 1 +fi + +# Notification function (matches backup script pattern) +send_notification() { + local title="$1" + local message="$2" + local status="${3:-info}" # success, error, warning, info + local hostname=$(hostname) + + # Console notification + log_message "$title: $message" + + # Webhook notification + if [ -n "$WEBHOOK_URL" ]; then + local tags="restore,immich,${hostname}" + [ "$status" == "error" ] && tags="${tags},errors" + [ "$status" == "warning" ] && tags="${tags},warnings" + + # Clean message without newlines or timestamps for webhook + local webhook_message="$message" + + curl -s \ + -H "tags:${tags}" \ + -d "$webhook_message" \ + "$WEBHOOK_URL" 2>/dev/null || log_message "Warning: Failed to send webhook notification" + fi +} + +# Cleanup function to ensure containers are restarted +cleanup() { + local exit_code=$? + log_message "Running cleanup..." + + # Restart containers if they were stopped + if [ "$CONTAINERS_STOPPED" = true ]; then + log_message "Restarting Immich containers..." + docker start immich_postgres 2>/dev/null || true + docker start immich_server 2>/dev/null || true + sleep 5 + + # Check if containers started successfully + if docker ps -q --filter "name=immich_postgres" | grep -q . && \ + docker ps -q --filter "name=immich_server" | grep -q .; then + log_message "āœ… Immich containers restarted successfully" + else + log_message "āš ļø Warning: Some containers may not have restarted properly" + fi + fi + + if [ $exit_code -ne 0 ]; then + log_message "Restore failed with exit code $exit_code" + send_notification "šŸšØ Immich Restore Failed" "Restoration process encountered an error (exit code: $exit_code)" "error" + fi + + exit $exit_code +} + +# Set up trap to call cleanup function on script exit +trap cleanup EXIT SIGINT SIGTERM + +# Validate backup files integrity +validate_backup_files() { + log_message "Validating backup files..." + + if [ "$SKIP_DB" = false ]; then + if [ ! -f "$DB_BACKUP" ]; then + log_message "Error: Database backup file not found: $DB_BACKUP" + exit 1 + fi + + # Check if it's a gzipped file + if [[ "$DB_BACKUP" == *.gz ]]; then + if ! gunzip -t "$DB_BACKUP" 2>/dev/null; then + log_message "Error: Database backup file appears to be corrupted (gzip test failed)" + exit 1 + fi + fi + + local db_size=$(du -sh "$DB_BACKUP" 2>/dev/null | cut -f1 || echo "unknown") + log_message "āœ“ Database backup validated: $DB_BACKUP ($db_size)" + fi + + if [ "$SKIP_UPLOADS" = false ]; then + if [ ! -f "$UPLOADS_BACKUP" ]; then + log_message "Error: Uploads backup file not found: $UPLOADS_BACKUP" + exit 1 + fi + + # Basic tar file validation + if ! tar -tzf "$UPLOADS_BACKUP" >/dev/null 2>&1; then + log_message "Error: Uploads backup file appears to be corrupted (tar test failed)" + exit 1 + fi + + local uploads_size=$(du -sh "$UPLOADS_BACKUP" 2>/dev/null | cut -f1 || echo "unknown") + log_message "āœ“ Uploads backup validated: $UPLOADS_BACKUP ($uploads_size)" + fi +} + +# Check container status and stop if needed +manage_containers() { + local action="$1" # stop or start + + if [ "$action" = "stop" ]; then + log_message "Stopping Immich containers for restoration..." + + # Check and stop immich_server + if docker ps -q --filter "name=immich_server" | grep -q .; then + log_message "Stopping immich_server container..." + docker stop immich_server || { + log_message "Warning: Failed to stop immich_server container" + } + else + log_message "immich_server container not running" + fi + + # Check and stop immich_postgres + if docker ps -q --filter "name=immich_postgres" | grep -q .; then + log_message "Stopping immich_postgres container..." + docker stop immich_postgres || { + log_message "Error: Failed to stop immich_postgres container" + exit 1 + } + else + log_message "immich_postgres container not running" + fi + + # Wait for containers to fully stop + sleep 5 + CONTAINERS_STOPPED=true + + elif [ "$action" = "start" ]; then + log_message "Starting Immich containers..." + + # Start postgres first + docker start immich_postgres || { + log_message "Error: Failed to start immich_postgres container" + exit 1 + } + + # Wait for postgres to be ready + log_message "Waiting for PostgreSQL to be ready..." + sleep 10 + + # Start immich_server + docker start immich_server || { + log_message "Warning: Failed to start immich_server container" + } + + # Wait for services to be ready + sleep 5 + CONTAINERS_STOPPED=false + fi +} + +# Restore database +restore_database() { + if [ "$SKIP_DB" = true ]; then + log_message "Skipping database restoration (--skip-db)" + return 0 + fi + + log_message "=== RESTORING DATABASE ===" + + # Create temporary directory for decompression if needed + local temp_dir=$(mktemp -d) + local sql_file="$DB_BACKUP" + + # Decompress if it's a gzipped file + if [[ "$DB_BACKUP" == *.gz ]]; then + log_message "Decompressing database backup..." + sql_file="${temp_dir}/$(basename "${DB_BACKUP%.gz}")" + + if ! gunzip -c "$DB_BACKUP" > "$sql_file"; then + log_message "Error: Failed to decompress database backup" + rm -rf "$temp_dir" + exit 1 + fi + + log_message "āœ“ Database backup decompressed to: $sql_file" + fi + + # Start postgres container temporarily for restoration + log_message "Starting PostgreSQL container for restoration..." + docker start immich_postgres || { + log_message "Error: Failed to start immich_postgres container" + rm -rf "$temp_dir" + exit 1 + } + + # Wait for PostgreSQL to be ready + log_message "Waiting for PostgreSQL to be ready..." + local max_attempts=30 + local attempt=0 + + while [ $attempt -lt $max_attempts ]; do + if docker exec immich_postgres pg_isready -U "$DB_USERNAME" >/dev/null 2>&1; then + break + fi + attempt=$((attempt + 1)) + sleep 2 + done + + if [ $attempt -eq $max_attempts ]; then + log_message "Error: PostgreSQL did not become ready within timeout" + rm -rf "$temp_dir" + exit 1 + fi + + log_message "āœ“ PostgreSQL is ready" + + # Drop existing database and recreate (if it exists) + log_message "Preparing database for restoration..." + docker exec immich_postgres psql -U "$DB_USERNAME" -c "DROP DATABASE IF EXISTS $DB_DATABASE_NAME;" 2>/dev/null || true + docker exec immich_postgres psql -U "$DB_USERNAME" -c "CREATE DATABASE $DB_DATABASE_NAME;" || { + log_message "Error: Failed to create database $DB_DATABASE_NAME" + rm -rf "$temp_dir" + exit 1 + } + + # Restore database + log_message "Restoring database from backup..." + log_message "This may take several minutes depending on database size..." + + if docker exec -i immich_postgres psql -U "$DB_USERNAME" -d "$DB_DATABASE_NAME" < "$sql_file"; then + log_message "āœ… Database restoration completed successfully" + else + log_message "Error: Database restoration failed" + rm -rf "$temp_dir" + exit 1 + fi + + # Stop postgres container + docker stop immich_postgres + + # Cleanup temporary files + rm -rf "$temp_dir" +} + +# Restore uploads directory +restore_uploads() { + if [ "$SKIP_UPLOADS" = true ]; then + log_message "Skipping uploads restoration (--skip-uploads)" + return 0 + fi + + log_message "=== RESTORING UPLOADS DIRECTORY ===" + + # Verify upload location exists or create it + if [ ! -d "$UPLOAD_LOCATION" ]; then + log_message "Creating upload directory: $UPLOAD_LOCATION" + if ! mkdir -p "$UPLOAD_LOCATION"; then + log_message "Error: Failed to create upload directory: $UPLOAD_LOCATION" + exit 1 + fi + fi + + # Backup existing content if any + if [ "$(ls -A "$UPLOAD_LOCATION" 2>/dev/null)" ]; then + local backup_existing="${UPLOAD_LOCATION}_backup_$(date +%Y%m%d_%H%M%S)" + log_message "Backing up existing uploads to: $backup_existing" + + if ! mv "$UPLOAD_LOCATION" "$backup_existing"; then + log_message "Error: Failed to backup existing uploads directory" + exit 1 + fi + + # Recreate the upload directory + mkdir -p "$UPLOAD_LOCATION" + fi + + # Extract uploads backup + log_message "Extracting uploads backup..." + log_message "This may take a while depending on the size of your media library..." + + # Extract to parent directory, then move content to correct location + local parent_dir=$(dirname "$UPLOAD_LOCATION") + local upload_dirname=$(basename "$UPLOAD_LOCATION") + + if tar -xzf "$UPLOADS_BACKUP" -C "$parent_dir"; then + log_message "āœ… Uploads restoration completed successfully" + else + log_message "Error: Failed to extract uploads backup" + exit 1 + fi + + # Set proper ownership and permissions + log_message "Setting proper ownership and permissions..." + + # Find the user that should own these files (check docker container user) + local container_user=$(docker exec immich_server id -u 2>/dev/null || echo "999") + local container_group=$(docker exec immich_server id -g 2>/dev/null || echo "999") + + # Set ownership (use chown with numeric IDs to avoid user name conflicts) + if command -v chown >/dev/null; then + chown -R "$container_user:$container_group" "$UPLOAD_LOCATION" 2>/dev/null || { + log_message "Warning: Could not set ownership, you may need to run: sudo chown -R $container_user:$container_group $UPLOAD_LOCATION" + } + fi + + # Set permissions + find "$UPLOAD_LOCATION" -type d -exec chmod 755 {} \; 2>/dev/null || true + find "$UPLOAD_LOCATION" -type f -exec chmod 644 {} \; 2>/dev/null || true + + log_message "āœ“ Ownership and permissions set" +} + +# Verify restoration +verify_restoration() { + log_message "=== VERIFYING RESTORATION ===" + + # Start containers for verification + manage_containers start + + # Wait for services to be fully ready + log_message "Waiting for Immich services to be ready..." + sleep 15 + + # Check database connectivity + if [ "$SKIP_DB" = false ]; then + log_message "Verifying database connectivity..." + if docker exec immich_postgres psql -U "$DB_USERNAME" -d "$DB_DATABASE_NAME" -c "SELECT COUNT(*) FROM information_schema.tables;" >/dev/null 2>&1; then + log_message "āœ… Database connectivity verified" + else + log_message "āš ļø Warning: Database connectivity check failed" + fi + fi + + # Check uploads directory + if [ "$SKIP_UPLOADS" = false ]; then + log_message "Verifying uploads directory..." + if [ -d "$UPLOAD_LOCATION" ] && [ "$(ls -A "$UPLOAD_LOCATION" 2>/dev/null)" ]; then + local upload_size=$(du -sh "$UPLOAD_LOCATION" 2>/dev/null | cut -f1 || echo "unknown") + log_message "āœ… Uploads directory verified: $UPLOAD_LOCATION ($upload_size)" + else + log_message "āš ļø Warning: Uploads directory appears empty or inaccessible" + fi + fi + + # Check container status + log_message "Verifying container status..." + local containers_ok=true + + if ! docker ps -q --filter "name=immich_postgres" | grep -q .; then + log_message "āš ļø Warning: immich_postgres container is not running" + containers_ok=false + fi + + if ! docker ps -q --filter "name=immich_server" | grep -q .; then + log_message "āš ļø Warning: immich_server container is not running" + containers_ok=false + fi + + if [ "$containers_ok" = true ]; then + log_message "āœ… All containers are running" + fi +} + +# Initialize variables +CONTAINERS_STOPPED=false + +# Main restoration logic +log_message "=== IMMICH RESTORATION STARTED ===" +send_notification "šŸ”„ Immich Restore Started" "Beginning restoration from backup files" "info" + +if [ "$DRY_RUN" = true ]; then + log_message "=== DRY RUN MODE - NO ACTUAL RESTORATION WILL BE PERFORMED ===" + log_message "" + log_message "Configuration:" + log_message " - Database: ${DB_DATABASE_NAME}" + log_message " - Username: ${DB_USERNAME}" + log_message " - Upload Location: ${UPLOAD_LOCATION}" + log_message " - Database backup: ${DB_BACKUP:-SKIPPED}" + log_message " - Uploads backup: ${UPLOADS_BACKUP:-SKIPPED}" + log_message "" + log_message "Restoration process would:" + [ "$SKIP_DB" = false ] && log_message " 1. Stop Immich containers" + [ "$SKIP_DB" = false ] && log_message " 2. Restore database from: $DB_BACKUP" + [ "$SKIP_UPLOADS" = false ] && log_message " 3. Restore uploads to: $UPLOAD_LOCATION" + log_message " 4. Restart containers and verify" + log_message "" + log_message "=== DRY RUN COMPLETE - No changes would be made ===" + exit 0 +fi + +# Validate backup files +validate_backup_files + +# Stop containers +manage_containers stop + +# Perform restoration +restore_database +restore_uploads + +# Verify restoration +verify_restoration + +# Calculate restoration time and send success notification +RESTORE_END_TIME=$(date +%s) +TOTAL_RESTORE_TIME=$((RESTORE_END_TIME - $(date +%s))) + +# Format file information for notification +NOTIFICATION_MESSAGE="" +if [ "$SKIP_DB" = false ]; then + DB_FILENAME=$(basename "$DB_BACKUP") + DB_SIZE=$(du -sh "$DB_BACKUP" 2>/dev/null | cut -f1 || echo "unknown") + NOTIFICATION_MESSAGE="šŸ“¦ Database: ${DB_FILENAME} (${DB_SIZE})" +fi + +if [ "$SKIP_UPLOADS" = false ]; then + UPLOADS_FILENAME=$(basename "$UPLOADS_BACKUP") + UPLOADS_SIZE=$(du -sh "$UPLOADS_BACKUP" 2>/dev/null | cut -f1 || echo "unknown") + if [ -n "$NOTIFICATION_MESSAGE" ]; then + NOTIFICATION_MESSAGE="${NOTIFICATION_MESSAGE}\nšŸ“ Uploads: ${UPLOADS_FILENAME} (${UPLOADS_SIZE})" + else + NOTIFICATION_MESSAGE="šŸ“ Uploads: ${UPLOADS_FILENAME} (${UPLOADS_SIZE})" + fi +fi + +NOTIFICATION_MESSAGE="${NOTIFICATION_MESSAGE}\nāœ… Restoration completed successfully" +NOTIFICATION_MESSAGE="${NOTIFICATION_MESSAGE}\nšŸ  Restored to: $(hostname)" + +send_notification "āœ… Immich Restore Completed" "$NOTIFICATION_MESSAGE" "success" + +log_message "=== IMMICH RESTORATION COMPLETED SUCCESSFULLY ===" +log_message "Database restored to: $DB_DATABASE_NAME" +log_message "Uploads restored to: $UPLOAD_LOCATION" +log_message "All containers are running and verified" diff --git a/immich/test-immich-restore.sh b/immich/test-immich-restore.sh new file mode 100755 index 0000000..6180cbf --- /dev/null +++ b/immich/test-immich-restore.sh @@ -0,0 +1,423 @@ +#!/bin/bash + +# Immich Restore Script Test Suite +# This script tests the restoration functionality with mock data and validation + +set -e + +# Colors for output +RED='\033[0;31m' +GREEN='\033[0;32m' +YELLOW='\033[0;33m' +BLUE='\033[0;34m' +NC='\033[0m' # No Color + +# Test configuration +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +TEST_DIR="${SCRIPT_DIR}/test_data" +RESTORE_SCRIPT="${SCRIPT_DIR}/restore-immich.sh" +TEST_LOG="${SCRIPT_DIR}/../logs/immich-restore-test.log" + +# Test counters +TESTS_RUN=0 +TESTS_PASSED=0 +TESTS_FAILED=0 + +# Create test directory +mkdir -p "$TEST_DIR" +mkdir -p "$(dirname "$TEST_LOG")" + +# Logging function +log_test() { + local message="$1" + echo "$(date '+%Y-%m-%d %H:%M:%S') - TEST: $message" | tee -a "$TEST_LOG" +} + +log_pass() { + local test_name="$1" + echo -e "${GREEN}āœ… PASS${NC}: $test_name" + echo "$(date '+%Y-%m-%d %H:%M:%S') - PASS: $test_name" >> "$TEST_LOG" + TESTS_PASSED=$((TESTS_PASSED + 1)) +} + +log_fail() { + local test_name="$1" + local details="$2" + echo -e "${RED}āŒ FAIL${NC}: $test_name" + [ -n "$details" ] && echo " Details: $details" + echo "$(date '+%Y-%m-%d %H:%M:%S') - FAIL: $test_name - $details" >> "$TEST_LOG" + TESTS_FAILED=$((TESTS_FAILED + 1)) +} + +log_info() { + local message="$1" + echo -e "${BLUE}ā„¹ļø INFO${NC}: $message" +} + +# Test script existence and permissions +test_script_setup() { + TESTS_RUN=$((TESTS_RUN + 1)) + log_test "Script Setup Validation" + + if [ -f "$RESTORE_SCRIPT" ]; then + if [ -x "$RESTORE_SCRIPT" ]; then + log_pass "Script exists and is executable" + else + log_fail "Script exists but is not executable" + chmod +x "$RESTORE_SCRIPT" + log_info "Made script executable" + fi + else + log_fail "Restore script not found at $RESTORE_SCRIPT" + return 1 + fi +} + +# Test help functionality +test_help_functionality() { + TESTS_RUN=$((TESTS_RUN + 1)) + log_test "Help Functionality" + + local help_output + if help_output=$("$RESTORE_SCRIPT" --help 2>&1); then + if echo "$help_output" | grep -q "Usage:"; then + log_pass "Help command works and shows usage" + else + log_fail "Help command runs but doesn't show proper usage" + fi + else + log_fail "Help command failed to execute" + fi +} + +# Create mock backup files +create_mock_backups() { + log_test "Creating mock backup files for testing" + + # Create mock database backup (gzipped SQL) + local mock_db_backup="${TEST_DIR}/mock_immich_db_backup_20250603_120000.sql.gz" + echo "-- Mock Immich Database Backup +CREATE DATABASE IF NOT EXISTS immich; +USE immich; +CREATE TABLE IF NOT EXISTS test_table (id INT PRIMARY KEY, name VARCHAR(100)); +INSERT INTO test_table VALUES (1, 'test_data'); +-- End of mock backup" | gzip > "$mock_db_backup" + + # Create mock uploads backup (tar.gz) + local mock_uploads_backup="${TEST_DIR}/mock_immich_uploads_20250603_120000.tar.gz" + mkdir -p "${TEST_DIR}/mock_uploads/upload" + mkdir -p "${TEST_DIR}/mock_uploads/thumbs" + echo "Mock photo data" > "${TEST_DIR}/mock_uploads/upload/photo1.jpg" + echo "Mock thumbnail data" > "${TEST_DIR}/mock_uploads/thumbs/thumb1.jpg" + + tar -czf "$mock_uploads_backup" -C "${TEST_DIR}" mock_uploads + + # Create corrupted backup files for testing + echo "corrupted data" > "${TEST_DIR}/corrupted_db.sql.gz" + echo "not a tar file" > "${TEST_DIR}/corrupted_uploads.tar.gz" + + log_info "Mock backup files created" + return 0 +} + +# Test dry-run functionality +test_dry_run() { + TESTS_RUN=$((TESTS_RUN + 1)) + log_test "Dry Run Functionality" + + local mock_db_backup="${TEST_DIR}/mock_immich_db_backup_20250603_120000.sql.gz" + local mock_uploads_backup="${TEST_DIR}/mock_immich_uploads_20250603_120000.tar.gz" + + local output + if output=$("$RESTORE_SCRIPT" --db-backup "$mock_db_backup" --uploads-backup "$mock_uploads_backup" --dry-run 2>&1); then + if echo "$output" | grep -q "DRY RUN MODE"; then + log_pass "Dry run mode works correctly" + else + log_fail "Dry run executed but didn't show proper dry run message" + fi + else + log_fail "Dry run command failed" + fi +} + +# Test file validation +test_file_validation() { + TESTS_RUN=$((TESTS_RUN + 1)) + log_test "Backup File Validation" + + # Test with non-existent files + local output + if output=$("$RESTORE_SCRIPT" --db-backup "/nonexistent/file.sql.gz" --uploads-backup "/nonexistent/file.tar.gz" --dry-run 2>&1); then + log_fail "Script should fail with non-existent files but didn't" + else + if echo "$output" | grep -q "not found"; then + log_pass "File validation correctly detects missing files" + else + log_fail "Script failed but not with expected error message" + fi + fi +} + +# Test corrupted file detection +test_corrupted_file_detection() { + TESTS_RUN=$((TESTS_RUN + 1)) + log_test "Corrupted File Detection" + + local corrupted_db="${TEST_DIR}/corrupted_db.sql.gz" + local corrupted_uploads="${TEST_DIR}/corrupted_uploads.tar.gz" + + local output + if output=$("$RESTORE_SCRIPT" --db-backup "$corrupted_db" --uploads-backup "$corrupted_uploads" --dry-run 2>&1); then + log_fail "Script should detect corrupted files but didn't" + else + if echo "$output" | grep -q "corrupted"; then + log_pass "Corrupted file detection works" + else + log_fail "Script failed but not with corrupted file error" + fi + fi +} + +# Test skip options +test_skip_options() { + TESTS_RUN=$((TESTS_RUN + 1)) + log_test "Skip Options Functionality" + + # Test skip-db option + local output + if output=$("$RESTORE_SCRIPT" --uploads-backup "${TEST_DIR}/mock_immich_uploads_20250603_120000.tar.gz" --skip-db --dry-run 2>&1); then + if echo "$output" | grep -q "Database backup: SKIPPED"; then + log_pass "Skip database option works" + else + log_fail "Skip database option doesn't show correct status" + fi + else + log_fail "Skip database option test failed" + fi + + TESTS_RUN=$((TESTS_RUN + 1)) + + # Test skip-uploads option + if output=$("$RESTORE_SCRIPT" --db-backup "${TEST_DIR}/mock_immich_db_backup_20250603_120000.sql.gz" --skip-uploads --dry-run 2>&1); then + if echo "$output" | grep -q "Uploads backup: SKIPPED"; then + log_pass "Skip uploads option works" + else + log_fail "Skip uploads option doesn't show correct status" + fi + else + log_fail "Skip uploads option test failed" + fi +} + +# Test environment variable validation +test_env_validation() { + TESTS_RUN=$((TESTS_RUN + 1)) + log_test "Environment Variable Validation" + + # Temporarily move .env file to test missing env + local env_file="$(dirname "$RESTORE_SCRIPT")/../.env" + if [ -f "$env_file" ]; then + mv "$env_file" "${env_file}.backup" + + local output + if output=$("$RESTORE_SCRIPT" --help 2>&1); then + if echo "$output" | grep -q "Error.*env"; then + log_pass "Missing environment file detection works" + else + log_fail "Script should detect missing .env file" + fi + else + log_pass "Script correctly fails with missing .env file" + fi + + # Restore .env file + mv "${env_file}.backup" "$env_file" + else + log_info "No .env file found to test with" + fi +} + +# Test notification system (dry run) +test_notification_system() { + TESTS_RUN=$((TESTS_RUN + 1)) + log_test "Notification System" + + local mock_db_backup="${TEST_DIR}/mock_immich_db_backup_20250603_120000.sql.gz" + local mock_uploads_backup="${TEST_DIR}/mock_immich_uploads_20250603_120000.tar.gz" + + # Set a test webhook URL temporarily + export WEBHOOK_URL="https://httpbin.org/post" + + local output + if output=$("$RESTORE_SCRIPT" --db-backup "$mock_db_backup" --uploads-backup "$mock_uploads_backup" --dry-run 2>&1); then + if echo "$output" | grep -q "Immich Restore Started"; then + log_pass "Notification system appears to be working" + else + log_fail "Notification system not triggering properly" + fi + else + log_fail "Test with notifications failed" + fi + + unset WEBHOOK_URL +} + +# Test argument parsing +test_argument_parsing() { + TESTS_RUN=$((TESTS_RUN + 1)) + log_test "Argument Parsing" + + # Test unknown option + local output + if output=$("$RESTORE_SCRIPT" --unknown-option 2>&1); then + if echo "$output" | grep -q "Unknown option"; then + log_pass "Unknown option detection works" + else + log_fail "Unknown option should be detected" + fi + else + log_pass "Script correctly fails with unknown option" + fi +} + +# Test logging functionality +test_logging() { + TESTS_RUN=$((TESTS_RUN + 1)) + log_test "Logging Functionality" + + local log_dir="$(dirname "$RESTORE_SCRIPT")/../logs" + local restore_log="${log_dir}/immich-restore.log" + + # Clear previous log entries + [ -f "$restore_log" ] && > "$restore_log" + + local mock_db_backup="${TEST_DIR}/mock_immich_db_backup_20250603_120000.sql.gz" + local mock_uploads_backup="${TEST_DIR}/mock_immich_uploads_20250603_120000.tar.gz" + + "$RESTORE_SCRIPT" --db-backup "$mock_db_backup" --uploads-backup "$mock_uploads_backup" --dry-run >/dev/null 2>&1 + + if [ -f "$restore_log" ] && [ -s "$restore_log" ]; then + if grep -q "IMMICH RESTORATION STARTED" "$restore_log"; then + log_pass "Logging functionality works" + else + log_fail "Log file exists but doesn't contain expected content" + fi + else + log_fail "Log file not created or is empty" + fi +} + +# Performance test with larger mock files +test_performance() { + TESTS_RUN=$((TESTS_RUN + 1)) + log_test "Performance with Larger Files" + + # Create a larger mock database backup + local large_db_backup="${TEST_DIR}/large_mock_db.sql.gz" + { + echo "-- Large Mock Database Backup" + for i in {1..1000}; do + echo "INSERT INTO test_table VALUES ($i, 'test_data_$i');" + done + echo "-- End of large mock backup" + } | gzip > "$large_db_backup" + + # Create larger mock uploads + local large_uploads_backup="${TEST_DIR}/large_mock_uploads.tar.gz" + mkdir -p "${TEST_DIR}/large_mock_uploads" + for i in {1..50}; do + echo "Mock large file content $i" > "${TEST_DIR}/large_mock_uploads/file_$i.txt" + done + tar -czf "$large_uploads_backup" -C "${TEST_DIR}" large_mock_uploads + + local start_time=$(date +%s) + local output + if output=$("$RESTORE_SCRIPT" --db-backup "$large_db_backup" --uploads-backup "$large_uploads_backup" --dry-run 2>&1); then + local end_time=$(date +%s) + local duration=$((end_time - start_time)) + + if [ $duration -lt 30 ]; then # Should complete dry run in under 30 seconds + log_pass "Performance test completed in ${duration}s" + else + log_fail "Performance test took too long: ${duration}s" + fi + else + log_fail "Performance test failed to execute" + fi +} + +# Cleanup test data +cleanup_test_data() { + log_test "Cleaning up test data" + rm -rf "$TEST_DIR" + log_info "Test data cleaned up" +} + +# Generate test report +generate_test_report() { + echo "" + echo "==================================================================" + echo -e "${BLUE}IMMICH RESTORE SCRIPT TEST REPORT${NC}" + echo "==================================================================" + echo "Test Date: $(date)" + echo "Script Tested: $RESTORE_SCRIPT" + echo "" + echo "Test Results:" + echo -e " Total Tests Run: ${BLUE}$TESTS_RUN${NC}" + echo -e " Tests Passed: ${GREEN}$TESTS_PASSED${NC}" + echo -e " Tests Failed: ${RED}$TESTS_FAILED${NC}" + echo "" + + if [ $TESTS_FAILED -eq 0 ]; then + echo -e "${GREEN}āœ… ALL TESTS PASSED${NC}" + echo "The restore script is ready for production use." + else + echo -e "${RED}āŒ SOME TESTS FAILED${NC}" + echo "Please review the test log for details: $TEST_LOG" + fi + + echo "" + echo "Test Log Location: $TEST_LOG" + echo "==================================================================" +} + +# Main test execution +main() { + echo -e "${BLUE}Starting Immich Restore Script Test Suite${NC}" + echo "Test Directory: $TEST_DIR" + echo "Log File: $TEST_LOG" + echo "" + + # Initialize test log + echo "=== Immich Restore Script Test Suite ===" > "$TEST_LOG" + echo "Started: $(date)" >> "$TEST_LOG" + echo "" >> "$TEST_LOG" + + # Run all tests + test_script_setup + create_mock_backups + test_help_functionality + test_dry_run + test_file_validation + test_corrupted_file_detection + test_skip_options + test_env_validation + test_notification_system + test_argument_parsing + test_logging + test_performance + + # Cleanup and report + cleanup_test_data + generate_test_report + + # Exit with appropriate code + if [ $TESTS_FAILED -eq 0 ]; then + exit 0 + else + exit 1 + fi +} + +# Run tests +main "$@" diff --git a/immich/validate-immich-backups.sh b/immich/validate-immich-backups.sh index cd08f61..6b1976f 100755 --- a/immich/validate-immich-backups.sh +++ b/immich/validate-immich-backups.sh @@ -3,8 +3,7 @@ # Immich Backup Validation Script # This script validates Immich backup files for integrity and completeness -# Set up error handling -set -e +# Note: We don't use set -e here to ensure all validations are performed # Colors for output RED='\033[0;31m' @@ -65,17 +64,17 @@ if [ -z "$DB_BACKUPS" ]; then else for backup in $DB_BACKUPS; do echo "Validating: $(basename "$backup")" - + # Check file size SIZE=$(stat -c%s "$backup" 2>/dev/null || echo "0") - if [ "$SIZE" -lt 1024 ]; then + if [ "$SIZE" -lt 100 ]; then echo -e " ${RED}āœ— File is too small (${SIZE} bytes)${NC}" log_validation "Error: File is too small (${SIZE} bytes) - $(basename "$backup")" ((TOTAL_ERRORS++)) else echo -e " ${GREEN}āœ“ File size OK ($(du -h "$backup" | cut -f1))${NC}" fi - + # Check if it's a valid gzip file if gzip -t "$backup" 2>/dev/null; then echo -e " ${GREEN}āœ“ Gzip file integrity OK${NC}" @@ -84,14 +83,14 @@ else log_validation "Error: Gzip file corruption detected - $(basename "$backup")" ((TOTAL_ERRORS++)) fi - + # Check if SQL content looks valid (basic check) if zcat "$backup" 2>/dev/null | head -n 10 | grep -q "PostgreSQL database dump"; then echo -e " ${GREEN}āœ“ SQL content appears valid${NC}" else echo -e " ${YELLOW}? Cannot verify SQL content format${NC}" fi - + echo "" done fi @@ -104,21 +103,21 @@ if [ -z "$UPLOAD_BACKUPS" ]; then else for backup in $UPLOAD_BACKUPS; do echo "Validating: $(basename "$backup")" - + # Check file size SIZE=$(stat -c%s "$backup" 2>/dev/null || echo "0") - if [ "$SIZE" -lt 1024 ]; then + if [ "$SIZE" -lt 100 ]; then echo -e " ${RED}āœ— File is too small (${SIZE} bytes)${NC}" log_validation "Error: File is too small (${SIZE} bytes) - $(basename "$backup")" ((TOTAL_ERRORS++)) else echo -e " ${GREEN}āœ“ File size OK ($(du -h "$backup" | cut -f1))${NC}" fi - + # Check if it's a valid tar.gz file if tar -tzf "$backup" >/dev/null 2>&1; then echo -e " ${GREEN}āœ“ Tar.gz file integrity OK${NC}" - + # Count files in archive FILE_COUNT=$(tar -tzf "$backup" 2>/dev/null | wc -l) echo -e " ${GREEN}āœ“ Archive contains ${FILE_COUNT} files/directories${NC}" @@ -127,7 +126,7 @@ else log_validation "Error: Tar.gz file corruption detected - $(basename "$backup")" ((TOTAL_ERRORS++)) fi - + echo "" done fi @@ -159,11 +158,50 @@ for upload_ts in $UPLOAD_TIMESTAMPS; do fi done +# Find most recent complete backup set +LATEST_COMPLETE="" +for db_ts in $DB_TIMESTAMPS; do + if echo "$UPLOAD_TIMESTAMPS" | grep -q "^${db_ts}$"; then + LATEST_COMPLETE="$db_ts" + break # First one is most recent due to sort -r + fi +done + echo "" echo "=== VALIDATION SUMMARY ===" echo "Complete backup pairs: $MATCHED_PAIRS" echo "Total validation errors: $TOTAL_ERRORS" +if [ -n "$LATEST_COMPLETE" ]; then + echo -e "${GREEN}Most recent complete backup: $LATEST_COMPLETE${NC}" + echo "" + echo "=== RESTORATION COMMANDS ===" + echo "To restore the most recent complete backup:" + echo "" + echo "# Dry run (recommended first):" + echo "./restore-immich.sh \\" + echo " --db-backup \"$BACKUP_DIR/immich_db_backup_${LATEST_COMPLETE}.sql.gz\" \\" + echo " --uploads-backup \"$BACKUP_DIR/immich_uploads_${LATEST_COMPLETE}.tar.gz\" \\" + echo " --dry-run" + echo "" + echo "# Actual restoration:" + echo "./restore-immich.sh \\" + echo " --db-backup \"$BACKUP_DIR/immich_db_backup_${LATEST_COMPLETE}.sql.gz\" \\" + echo " --uploads-backup \"$BACKUP_DIR/immich_uploads_${LATEST_COMPLETE}.tar.gz\"" + echo "" + echo "# Restore database only:" + echo "./restore-immich.sh \\" + echo " --db-backup \"$BACKUP_DIR/immich_db_backup_${LATEST_COMPLETE}.sql.gz\" \\" + echo " --uploads-backup \"$BACKUP_DIR/immich_uploads_${LATEST_COMPLETE}.tar.gz\" \\" + echo " --skip-uploads" + echo "" + echo "# Restore uploads only:" + echo "./restore-immich.sh \\" + echo " --db-backup \"$BACKUP_DIR/immich_db_backup_${LATEST_COMPLETE}.sql.gz\" \\" + echo " --uploads-backup \"$BACKUP_DIR/immich_uploads_${LATEST_COMPLETE}.tar.gz\" \\" + echo " --skip-db" +fi + log_validation "Validation summary: $MATCHED_PAIRS complete backup pairs, $TOTAL_ERRORS errors" if [ "$TOTAL_ERRORS" -eq 0 ]; then