shell/immich/backup-immich.sh

#!/bin/bash

# Immich Complete Backup Script
# This script creates a complete backup of the Immich installation including:
# 1. Postgres database (using pg_dumpall as recommended by Immich)
# 2. User upload directories (photos, videos, and metadata)
# Based on recommendations from https://immich.app/docs/administration/backup-and-restore/

# Set up error handling
set -e

# Function to ensure server is unpaused even if script fails
cleanup() {
    local exit_code=$?
    echo "Running cleanup..."

    # Check if immich_server is paused and unpause it if needed
    if [ "${IMMICH_SERVER_RUNNING:-true}" = true ] && docker inspect --format='{{.State.Status}}' immich_server 2>/dev/null | grep -q "paused"; then
        echo "Unpausing immich_server container during cleanup..."
        docker unpause immich_server 2>/dev/null || true
    fi

    if [ $exit_code -ne 0 ]; then
        echo "Script failed with exit code $exit_code"
        send_notification "🚨 Immich Backup Failed" "Backup process encountered an error (exit code: $exit_code)" "error"
    fi

    exit $exit_code
}

# Set up trap to call cleanup function on script exit (normal or error)
trap cleanup EXIT SIGINT SIGTERM

# Load environment variables from the .env file
ENV_FILE="$(dirname "$0")/../.env"
if [ -f "$ENV_FILE" ]; then
    echo "Loading environment variables from $ENV_FILE"
    source "$ENV_FILE"
else
    echo "Error: .env file not found in $(dirname "$0")/.."
    exit 1
fi

# Verify required environment variables are set
if [ -z "$DB_USERNAME" ] || [ -z "$DB_DATABASE_NAME" ] || [ -z "$UPLOAD_LOCATION" ]; then
    echo "Error: Required environment variables (DB_USERNAME, DB_DATABASE_NAME, UPLOAD_LOCATION) not found in .env file"
    echo "Please ensure your .env file contains:"
    echo "  - DB_USERNAME=<database_username>"
    echo "  - DB_DATABASE_NAME=<database_name>"
    echo "  - UPLOAD_LOCATION=<path_to_upload_directory>"
    exit 1
fi

# Help function
show_help() {
    cat << EOF
Immich Complete Backup Script
This script creates a complete backup of the Immich installation including:
1. Postgres database (using pg_dumpall as recommended by Immich)
2. User upload directories (photos, videos, and metadata)

USAGE:
    $(basename "$0") [OPTIONS]

OPTIONS:
    --help, -h          Show this help message and exit
    --dry-run          Show what would be backed up without performing actual backup
    --no-upload        Skip B2 upload (local backup only)
    --verbose          Enable verbose logging

CONFIGURATION:
    This script requires a .env file in the parent directory with:
    - DB_USERNAME=<database_username>
    - DB_DATABASE_NAME=<database_name>  
    - UPLOAD_LOCATION=<path_to_upload_directory>

OPTIONAL B2 CONFIGURATION:
    - B2_APPLICATION_KEY_ID=<your_b2_app_key_id>
    - B2_APPLICATION_KEY=<your_b2_app_key>
    - B2_BUCKET_NAME=<your_b2_bucket_name>

OPTIONAL WEBHOOK CONFIGURATION:
    - WEBHOOK_URL=<your_notification_webhook_url>

EXAMPLES:
    $(basename "$0")              # Run full backup
    $(basename "$0") --help       # Show this help
    $(basename "$0") --dry-run    # Preview backup without executing
    $(basename "$0") --no-upload  # Backup locally only (skip B2)

RESTORE INSTRUCTIONS:
    https://immich.app/docs/administration/backup-and-restore/

EOF
}

# Parse command line arguments
DRY_RUN=false
NO_UPLOAD=false
VERBOSE=false

while [[ $# -gt 0 ]]; do
    case $1 in
        --help|-h)
            show_help
            exit 0
            ;;
        --dry-run)
            DRY_RUN=true
            shift
            ;;
        --no-upload)
            NO_UPLOAD=true
            shift
            ;;
        --verbose)
            VERBOSE=true
            shift
            ;;
        *)
            echo "Error: Unknown option $1"
            echo "Use --help for usage information"
            exit 1
            ;;
    esac
done

# B2 CLI tool path
B2_CLI="$(dirname "$0")/b2-linux"

# Notification function
send_notification() {
    local title="$1"
    local message="$2"
    local status="${3:-info}"  # success, error, warning, info
    local hostname=$(hostname)
    
    # Console notification
    log_message "$title: $message"
    
    # Webhook notification
    if [ -n "$WEBHOOK_URL" ]; then
        local tags="backup,immich,${hostname}"
        [ "$status" == "error" ] && tags="${tags},errors"
        [ "$status" == "warning" ] && tags="${tags},warnings"
        
        # Clean message without newlines or timestamps for webhook
        local webhook_message="$message"
        
        curl -s \
            -H "tags:${tags}" \
            -d "$webhook_message" \
            "$WEBHOOK_URL" 2>/dev/null || log_message "Warning: Failed to send webhook notification"
    fi
}

# Function to upload to Backblaze B2
upload_to_b2() {
    local file_path="$1"
    local filename=$(basename "$file_path")
    
    # Check if B2 is configured
    if [ -z "$B2_APPLICATION_KEY_ID" ] || [ -z "$B2_APPLICATION_KEY" ] || [ -z "$B2_BUCKET_NAME" ]; then
        log_message "B2 upload skipped: B2 credentials not configured in .env file"
        return 0
    fi
    
    # Check if B2 CLI exists
    if [ ! -f "$B2_CLI" ]; then
        log_message "Error: B2 CLI not found at $B2_CLI"
        return 1
    fi
    
    log_message "Uploading $filename to B2 bucket: $B2_BUCKET_NAME"
    
    # Authorize B2 account
    if ! "$B2_CLI" authorize-account "$B2_APPLICATION_KEY_ID" "$B2_APPLICATION_KEY" 2>/dev/null; then
        log_message "Error: Failed to authorize B2 account"
        return 1
    fi
    
    # Upload file to B2
    if "$B2_CLI" upload-file "$B2_BUCKET_NAME" "$file_path" "immich-backups/$filename" 2>/dev/null; then
        log_message "✅ Successfully uploaded $filename to B2"
        return 0
    else
        log_message "❌ Failed to upload $filename to B2"
        return 1
    fi
}

# Initialize container status variables
IMMICH_SERVER_RUNNING=true

# Set up logging to central logs directory
LOG_DIR="$(dirname "$0")/../logs"
mkdir -p "$LOG_DIR"
LOG_FILE="${LOG_DIR}/immich-backup.log"

# Function to log with timestamp
log_message() {
    echo "$(date '+%Y-%m-%d %H:%M:%S') - $1" | tee -a "$LOG_FILE"
}

# Function to log without timestamp (for progress/status)
log_status() {
    echo "$1" | tee -a "$LOG_FILE"
}

# Create backup directory if it doesn't exist
BACKUP_DIR="$(dirname "$0")/../immich_backups"
mkdir -p "$BACKUP_DIR"

# Generate timestamp for the backup filename
TIMESTAMP=$(date +"%Y%m%d_%H%M%S")
DB_BACKUP_FILENAME="immich_db_backup_${TIMESTAMP}.sql"
DB_BACKUP_PATH="${BACKUP_DIR}/${DB_BACKUP_FILENAME}"
UPLOAD_BACKUP_PATH="${BACKUP_DIR}/immich_uploads_${TIMESTAMP}.tar.gz"

# Handle dry-run mode
if [ "$DRY_RUN" = true ]; then
    echo ""
    echo "=== DRY RUN MODE - NO ACTUAL BACKUP WILL BE PERFORMED ==="
    echo ""
    echo "Configuration:"
    echo "  - Database: ${DB_DATABASE_NAME}"
    echo "  - Username: ${DB_USERNAME}"
    echo "  - Upload Location: ${UPLOAD_LOCATION}"
    echo "  - Container: immich_postgres"
    echo "  - Backup Directory: ${BACKUP_DIR}"
    echo ""
    echo "Would create:"
    echo "  - Database backup: ${DB_BACKUP_PATH}.gz"
    echo "  - Upload backup: ${UPLOAD_BACKUP_PATH}"
    echo ""
    
    # Check container status in dry-run
    echo "Container Status Check:"
    if docker ps -q --filter "name=immich_server" | grep -q .; then
        echo "  ✓ immich_server: Running (would pause during backup)"
    else
        echo "  ! immich_server: Not running or not found"
    fi
    
    if docker ps -q --filter "name=immich_postgres" | grep -q .; then
        echo "  ✓ immich_postgres: Running"
    else
        echo "  ✗ immich_postgres: Not running - backup would fail!"
        exit 1
    fi
    
    # Check upload directory
    if [ -d "${UPLOAD_LOCATION}" ]; then
        UPLOAD_SIZE=$(du -sh "${UPLOAD_LOCATION}" 2>/dev/null | cut -f1 || echo "unknown")
        echo "  ✓ Upload directory: ${UPLOAD_LOCATION} (${UPLOAD_SIZE})"
    else
        echo "  ✗ Upload directory: ${UPLOAD_LOCATION} does not exist - backup would fail!"
        exit 1
    fi
    
    # Check B2 configuration
    echo ""
    echo "B2 Upload Configuration:"
    if [ "$NO_UPLOAD" = true ]; then
        echo "  ! B2 upload disabled by --no-upload flag"
    elif [ -n "$B2_APPLICATION_KEY_ID" ] && [ -n "$B2_APPLICATION_KEY" ] && [ -n "$B2_BUCKET_NAME" ]; then
        echo "  ✓ B2 configured - would upload to bucket: ${B2_BUCKET_NAME}"
        if [ -f "$B2_CLI" ]; then
            echo "  ✓ B2 CLI found at: ${B2_CLI}"
        else
            echo "  ✗ B2 CLI not found at: ${B2_CLI} - upload would fail!"
        fi
    else
        echo "  ! B2 not configured - would skip upload"
    fi
    
    echo ""
    echo "=== DRY RUN COMPLETE - No files were created or modified ==="
    exit 0
fi

log_message "Starting complete backup of Immich installation..."
log_message "Using settings from .env file:"
log_message "  - Database: ${DB_DATABASE_NAME}"
log_message "  - Username: ${DB_USERNAME}"
log_message "  - Upload Location: ${UPLOAD_LOCATION}"
log_message "  - Container: immich_postgres"
log_message "  - Backup Directory: ${BACKUP_DIR}"

if [ "$NO_UPLOAD" = true ]; then
    log_message "  - B2 Upload: DISABLED (--no-upload flag)"
fi

if [ "$VERBOSE" = true ]; then
    log_message "  - Verbose logging: ENABLED"
fi

# Send start notification
send_notification "🚀 Immich Backup Started" "Starting complete backup of Immich database and uploads directory" "info"

# Check if the Immich server container exists and is running
log_status "Checking immich_server container status..."
if docker ps -q --filter "name=immich_server" | grep -q .; then
    log_message "Pausing immich_server container to minimize database writes..."
    if ! docker pause immich_server; then
        log_message "Failed to pause immich_server container."
        # Continue with backup instead of exiting
    fi
else
    log_message "Note: immich_server container not found or not running. Continuing with backup anyway."
    # Set a flag so we don't try to unpause it later
    IMMICH_SERVER_RUNNING=false
fi

# Check if the Postgres container exists and is running
log_status "Checking postgres container status..."
if ! docker ps -q --filter "name=immich_postgres" | grep -q .; then
    log_message "Error: immich_postgres container is not running. Cannot proceed with backup."
    exit 1
fi

# Check if the Immich server container exists and is running
log_status "Checking immich_server container status..."
if docker ps -q --filter "name=immich_server" | grep -q .; then
    log_message "Pausing immich_server container to minimize database writes during backup..."
    if ! docker pause immich_server; then
        log_message "Failed to pause immich_server container."
        # Continue with backup instead of exiting
    fi
else
    log_message "Note: immich_server container not found or not running. Continuing with backup anyway."
    # Set a flag so we don't try to unpause it later
    IMMICH_SERVER_RUNNING=false
fi

echo ""
echo "=== PHASE 1: DATABASE BACKUP ==="
log_message "Taking database backup using pg_dumpall as recommended by Immich documentation..."
# Use pg_dumpall with recommended flags: --clean and --if-exists
docker exec -t immich_postgres pg_dumpall \
    --clean \
    --if-exists \
    --username="${DB_USERNAME}" \
    > "${DB_BACKUP_PATH}"

# Check if the dump was successful
if [ $? -ne 0 ] || [ ! -s "${DB_BACKUP_PATH}" ]; then
    log_message "Error: Database backup failed or created an empty file."
    exit 1
fi

log_message "Database backup completed successfully!"

# Compress the database backup file
log_message "Compressing database backup file..."
if ! gzip -f "${DB_BACKUP_PATH}"; then
    log_message "Warning: Failed to compress database backup file."
fi

echo ""
echo "=== PHASE 2: UPLOAD DIRECTORY BACKUP ==="
log_message "Backing up user upload directory: ${UPLOAD_LOCATION}"

# Verify the upload location exists
if [ ! -d "${UPLOAD_LOCATION}" ]; then
    log_message "Error: Upload location ${UPLOAD_LOCATION} does not exist!"
    exit 1
fi

# Create compressed archive of the upload directory
# According to Immich docs, we need to backup the entire UPLOAD_LOCATION
# which includes: upload/, profile/, thumbs/, encoded-video/, library/, backups/
log_message "Creating compressed archive of upload directory..."
log_message "This may take a while depending on the size of your media library..."

# Use tar with progress indication and exclude any existing backup files in the upload location
if ! tar --exclude="${UPLOAD_LOCATION}/backups/*.tar.gz" \
     --exclude="${UPLOAD_LOCATION}/backups/*.sql.gz" \
     -czf "${UPLOAD_BACKUP_PATH}" \
     -C "$(dirname "${UPLOAD_LOCATION}")" \
     "$(basename "${UPLOAD_LOCATION}")"; then
    log_message "Error: Failed to create upload directory backup."
    exit 1
fi

log_message "Upload directory backup completed successfully!"

# Resume the Immich server only if it was running and we paused it
if [ "${IMMICH_SERVER_RUNNING:-true}" = true ]; then
    log_status "Resuming immich_server container..."
    if ! docker unpause immich_server 2>/dev/null; then
        log_message "Note: No need to unpause immich_server container."
    fi
fi

# Resume the Immich server only if it was running and we paused it
if [ "${IMMICH_SERVER_RUNNING:-true}" = true ]; then
    log_status "Resuming immich_server container..."
    if ! docker unpause immich_server 2>/dev/null; then
        log_message "Note: No need to unpause immich_server container."
    fi
fi

echo ""
echo "=== BACKUP COMPLETED SUCCESSFULLY! ==="
echo "Database backup saved to: ${DB_BACKUP_PATH}.gz"
echo "Upload directory backup saved to: ${UPLOAD_BACKUP_PATH}"

# Calculate backup sizes
DB_BACKUP_SIZE=$(du -h "${DB_BACKUP_PATH}.gz" 2>/dev/null | cut -f1 || echo "Unknown")
UPLOAD_BACKUP_SIZE=$(du -h "${UPLOAD_BACKUP_PATH}" 2>/dev/null | cut -f1 || echo "Unknown")

echo ""
echo "=== BACKUP SUMMARY ==="
echo "Database backup size: ${DB_BACKUP_SIZE}"
echo "Upload directory backup size: ${UPLOAD_BACKUP_SIZE}"

# Upload to B2 (if configured and not disabled)
echo ""
if [ "$NO_UPLOAD" = true ]; then
    echo "=== SKIPPING B2 UPLOAD (--no-upload flag) ==="
    log_message "B2 upload skipped due to --no-upload flag"
    B2_UPLOAD_SUCCESS="skipped"
else
    echo "=== UPLOADING TO BACKBLAZE B2 ==="
    B2_UPLOAD_SUCCESS=true

    # Upload database backup
    if ! upload_to_b2 "${DB_BACKUP_PATH}.gz"; then
        B2_UPLOAD_SUCCESS=false
    fi

    # Upload uploads backup
    if ! upload_to_b2 "${UPLOAD_BACKUP_PATH}"; then
        B2_UPLOAD_SUCCESS=false
    fi
fi

# Prepare notification message
DB_FILENAME=$(basename "${DB_BACKUP_PATH}.gz")
UPLOAD_FILENAME=$(basename "${UPLOAD_BACKUP_PATH}")
NOTIFICATION_MESSAGE="📦 Database: ${DB_FILENAME} (${DB_BACKUP_SIZE})
📁 Uploads: ${UPLOAD_FILENAME} (${UPLOAD_BACKUP_SIZE})"

if [ "$B2_UPLOAD_SUCCESS" = "skipped" ]; then
    NOTIFICATION_MESSAGE="${NOTIFICATION_MESSAGE}
💾 Local backup only (B2 upload skipped)"
    send_notification "✅ Immich Backup Completed (Local Only)" "$NOTIFICATION_MESSAGE" "success"
elif [ "$B2_UPLOAD_SUCCESS" = true ] && [ -n "$B2_BUCKET_NAME" ]; then
    NOTIFICATION_MESSAGE="${NOTIFICATION_MESSAGE}
☁️ Successfully uploaded to B2 bucket: ${B2_BUCKET_NAME}"
    send_notification "✅ Immich Backup Completed" "$NOTIFICATION_MESSAGE" "success"
elif [ -n "$B2_BUCKET_NAME" ]; then
    NOTIFICATION_MESSAGE="${NOTIFICATION_MESSAGE}
⚠️ B2 upload failed - files saved locally only"
    send_notification "⚠️ Immich Backup Completed (B2 Upload Failed)" "$NOTIFICATION_MESSAGE" "warning"
else
    send_notification "✅ Immich Backup Completed" "$NOTIFICATION_MESSAGE" "success"
fi

# Show backup information
echo ""
echo "=== BACKUP INVENTORY ==="
find "${BACKUP_DIR}" -name "*.gz" | wc -l | xargs echo "Total number of backup files:"
du -sh "${BACKUP_DIR}" | cut -f1 | xargs echo "Total backup directory size:"

# List recent backups
echo ""
echo "Recent backups:"
find "${BACKUP_DIR}" -name "*.gz" -mtime -7 | sort

# Health check: Verify backup file sizes
DB_BACKUP_SIZE_KB=$(du -k "${DB_BACKUP_PATH}.gz" 2>/dev/null | cut -f1 || echo "0")
UPLOAD_BACKUP_SIZE_KB=$(du -k "${UPLOAD_BACKUP_PATH}" 2>/dev/null | cut -f1 || echo "0")

echo ""
echo "=== BACKUP VALIDATION ==="
if [ "${DB_BACKUP_SIZE_KB}" -lt 100 ]; then
    echo "WARNING: Database backup file is smaller than expected (${DB_BACKUP_SIZE_KB}KB). Please verify its integrity."
else
    echo "✓ Database backup size appears normal (${DB_BACKUP_SIZE_KB}KB)"
fi

if [ "${UPLOAD_BACKUP_SIZE_KB}" -lt 1024 ]; then
    echo "WARNING: Upload directory backup file is smaller than expected (${UPLOAD_BACKUP_SIZE_KB}KB). Please verify its integrity."
else
    echo "✓ Upload directory backup size appears normal (${UPLOAD_BACKUP_SIZE_KB}KB)"
fi

# Optional: Remove old backups (older than 30 days)
echo ""
echo "=== CLEANUP ==="
OLD_BACKUPS=$(find "${BACKUP_DIR}" -name "*.gz" -mtime +30 | wc -l)
if [ "${OLD_BACKUPS}" -gt 0 ]; then
    echo "Found ${OLD_BACKUPS} backup files older than 30 days."
    echo "To remove them automatically, uncomment the cleanup line in this script."
    # Uncomment the next line to automatically remove old backups
    # find "${BACKUP_DIR}" -name "*.gz" -mtime +30 -delete
else
    echo "No old backup files found (older than 30 days)."
fi

echo ""
echo "=== RESTORE INSTRUCTIONS ==="
echo "To restore from this backup:"
echo "1. Database restore instructions: https://immich.app/docs/administration/backup-and-restore/#database"
echo "2. Upload directory: Extract ${UPLOAD_BACKUP_PATH} to your UPLOAD_LOCATION"
echo ""
echo "IMPORTANT: For a complete restore, you need BOTH the database backup AND the upload directory backup."
echo "The database contains metadata, while the upload directory contains your actual photos and videos."