mirror of
https://github.com/acedanger/shell.git
synced 2025-12-05 22:50:18 -08:00
6d726cb015
- Created a base HTML template for consistent layout across pages. - Developed a dashboard page to display backup service metrics and statuses. - Implemented a log viewer for detailed log file inspection. - Added error handling page for better user experience during failures. - Introduced service detail page to show specific service metrics and actions. - Enhanced log filtering and viewing capabilities. - Integrated auto-refresh functionality for real-time updates on metrics. - Created integration and unit test scripts for backup metrics functionality.
695 lines
24 KiB
Bash
Executable File
695 lines
24 KiB
Bash
Executable File
#!/bin/bash
|
|
|
|
# Immich Complete Backup Script
|
|
# This script creates a complete backup of the Immich installation including:
|
|
# 1. Postgres database (using pg_dumpall as recommended by Immich)
|
|
# 2. User upload directories (photos, videos, and metadata)
|
|
# Based on recommendations from https://immich.app/docs/administration/backup-and-restore/
|
|
|
|
# Set up error handling
|
|
set -e
|
|
|
|
# Load the unified backup metrics library
|
|
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
|
LIB_DIR="$(dirname "$SCRIPT_DIR")/lib"
|
|
if [[ -f "$LIB_DIR/unified-backup-metrics.sh" ]]; then
|
|
# shellcheck source=../lib/unified-backup-metrics.sh
|
|
source "$LIB_DIR/unified-backup-metrics.sh"
|
|
METRICS_ENABLED=true
|
|
else
|
|
echo "Warning: Unified backup metrics library not found at $LIB_DIR/unified-backup-metrics.sh"
|
|
METRICS_ENABLED=false
|
|
fi
|
|
|
|
# Function to ensure server is unpaused even if script fails
|
|
cleanup() {
|
|
local exit_code=$?
|
|
echo "Running cleanup..."
|
|
|
|
# Finalize metrics if enabled
|
|
if [[ "$METRICS_ENABLED" == "true" ]]; then
|
|
if [[ $exit_code -eq 0 ]]; then
|
|
metrics_backup_complete "success" "Immich backup completed successfully"
|
|
else
|
|
metrics_backup_complete "failed" "Immich backup failed during execution"
|
|
fi
|
|
fi
|
|
|
|
# Check if immich_server is paused and unpause it if needed
|
|
if [ "${IMMICH_SERVER_RUNNING:-true}" = true ] && docker inspect --format='{{.State.Status}}' immich_server 2>/dev/null | grep -q "paused"; then
|
|
echo "Unpausing immich_server container during cleanup..."
|
|
docker unpause immich_server 2>/dev/null || true
|
|
fi
|
|
|
|
if [ $exit_code -ne 0 ]; then
|
|
echo "Script failed with exit code $exit_code"
|
|
send_notification "🚨 Immich Backup Failed" "Backup process encountered an error (exit code: $exit_code)" "error"
|
|
fi
|
|
|
|
exit $exit_code
|
|
}
|
|
|
|
# Set up trap to call cleanup function on script exit (normal or error)
|
|
trap cleanup EXIT SIGINT SIGTERM
|
|
|
|
# Load environment variables from the .env file
|
|
ENV_FILE="$(dirname "$0")/../.env"
|
|
if [ -f "$ENV_FILE" ]; then
|
|
echo "Loading environment variables from $ENV_FILE"
|
|
# shellcheck source=/dev/null
|
|
source "$ENV_FILE"
|
|
else
|
|
echo "Error: .env file not found in $(dirname "$0")/.."
|
|
exit 1
|
|
fi
|
|
|
|
# Verify required environment variables are set
|
|
if [ -z "$DB_USERNAME" ] || [ -z "$DB_DATABASE_NAME" ] || [ -z "$UPLOAD_LOCATION" ]; then
|
|
echo "Error: Required environment variables (DB_USERNAME, DB_DATABASE_NAME, UPLOAD_LOCATION) not found in .env file"
|
|
echo "Please ensure your .env file contains:"
|
|
echo " - DB_USERNAME=<database_username>"
|
|
echo " - DB_DATABASE_NAME=<database_name>"
|
|
echo " - UPLOAD_LOCATION=<path_to_upload_directory>"
|
|
exit 1
|
|
fi
|
|
|
|
# Help function
|
|
show_help() {
|
|
cat << EOF
|
|
Immich Complete Backup Script
|
|
This script creates a complete backup of the Immich installation including:
|
|
1. Postgres database (using pg_dumpall as recommended by Immich)
|
|
2. User upload directories (photos, videos, and metadata)
|
|
|
|
USAGE:
|
|
$(basename "$0") [OPTIONS]
|
|
|
|
OPTIONS:
|
|
--help, -h Show this help message and exit
|
|
--dry-run Show what would be backed up without performing actual backup
|
|
--no-upload Skip B2 upload (local backup only)
|
|
--verbose Enable verbose logging
|
|
|
|
CONFIGURATION:
|
|
This script requires a .env file in the parent directory with:
|
|
- DB_USERNAME=<database_username>
|
|
- DB_DATABASE_NAME=<database_name>
|
|
- UPLOAD_LOCATION=<path_to_upload_directory>
|
|
|
|
OPTIONAL B2 CONFIGURATION:
|
|
- B2_APPLICATION_KEY_ID=<your_b2_app_key_id>
|
|
- B2_APPLICATION_KEY=<your_b2_app_key>
|
|
- B2_BUCKET_NAME=<your_b2_bucket_name>
|
|
|
|
OPTIONAL WEBHOOK CONFIGURATION:
|
|
- WEBHOOK_URL=<your_notification_webhook_url>
|
|
|
|
EXAMPLES:
|
|
$(basename "$0") # Run full backup
|
|
$(basename "$0") --help # Show this help
|
|
$(basename "$0") --dry-run # Preview backup without executing
|
|
$(basename "$0") --no-upload # Backup locally only (skip B2)
|
|
|
|
RESTORE INSTRUCTIONS:
|
|
https://immich.app/docs/administration/backup-and-restore/
|
|
|
|
EOF
|
|
}
|
|
|
|
# Parse command line arguments
|
|
DRY_RUN=false
|
|
NO_UPLOAD=false
|
|
VERBOSE=false
|
|
|
|
while [[ $# -gt 0 ]]; do
|
|
case $1 in
|
|
--help|-h)
|
|
show_help
|
|
exit 0
|
|
;;
|
|
--dry-run)
|
|
DRY_RUN=true
|
|
shift
|
|
;;
|
|
--no-upload)
|
|
NO_UPLOAD=true
|
|
shift
|
|
;;
|
|
--verbose)
|
|
VERBOSE=true
|
|
shift
|
|
;;
|
|
*)
|
|
echo "Error: Unknown option $1"
|
|
echo "Use --help for usage information"
|
|
exit 1
|
|
;;
|
|
esac
|
|
done
|
|
|
|
# B2 CLI tool path
|
|
B2_CLI="$(dirname "$0")/b2-linux"
|
|
|
|
# Notification function
|
|
send_notification() {
|
|
local title="$1"
|
|
local message="$2"
|
|
local status="${3:-info}" # success, error, warning, info
|
|
local hostname
|
|
hostname=$(hostname)
|
|
|
|
# Console notification
|
|
log_message "$title: $message"
|
|
|
|
# Webhook notification
|
|
if [ -n "$WEBHOOK_URL" ]; then
|
|
local tags="backup,immich,${hostname}"
|
|
[ "$status" == "error" ] && tags="${tags},errors"
|
|
[ "$status" == "warning" ] && tags="${tags},warnings"
|
|
|
|
# Clean message without newlines or timestamps for webhook
|
|
local webhook_message="$message"
|
|
|
|
curl -s \
|
|
-H "tags:${tags}" \
|
|
-d "$webhook_message" \
|
|
"$WEBHOOK_URL" 2>/dev/null || log_message "Warning: Failed to send webhook notification"
|
|
fi
|
|
}
|
|
|
|
# Function to upload to Backblaze B2
|
|
upload_to_b2() {
|
|
local file_path="$1"
|
|
local filename
|
|
filename=$(basename "$file_path")
|
|
|
|
# Check if B2 is configured
|
|
if [ -z "$B2_APPLICATION_KEY_ID" ] || [ -z "$B2_APPLICATION_KEY" ] || [ -z "$B2_BUCKET_NAME" ]; then
|
|
log_message "B2 upload skipped: B2 credentials not configured in .env file"
|
|
return 0
|
|
fi
|
|
|
|
# Check if B2 CLI exists
|
|
if [ ! -f "$B2_CLI" ]; then
|
|
log_message "Error: B2 CLI not found at $B2_CLI"
|
|
return 1
|
|
fi
|
|
|
|
log_message "Uploading $filename to B2 bucket: $B2_BUCKET_NAME"
|
|
|
|
# Authorize B2 account
|
|
if ! "$B2_CLI" authorize-account "$B2_APPLICATION_KEY_ID" "$B2_APPLICATION_KEY" 2>/dev/null; then
|
|
log_message "Error: Failed to authorize B2 account"
|
|
return 1
|
|
fi
|
|
|
|
# Upload file to B2
|
|
if "$B2_CLI" upload-file "$B2_BUCKET_NAME" "$file_path" "immich-backups/$filename" 2>/dev/null; then
|
|
log_message "✅ Successfully uploaded $filename to B2"
|
|
return 0
|
|
else
|
|
log_message "❌ Failed to upload $filename to B2"
|
|
return 1
|
|
fi
|
|
}
|
|
|
|
# Initialize container status variables
|
|
IMMICH_SERVER_RUNNING=true
|
|
|
|
# Set up logging to central logs directory
|
|
LOG_DIR="$(dirname "$0")/../logs"
|
|
mkdir -p "$LOG_DIR"
|
|
LOG_FILE="${LOG_DIR}/immich-backup.log"
|
|
|
|
# Function to log with timestamp
|
|
log_message() {
|
|
echo "$(date '+%Y-%m-%d %H:%M:%S') - $1" | tee -a "$LOG_FILE"
|
|
}
|
|
|
|
# Function to log without timestamp (for progress/status)
|
|
log_status() {
|
|
echo "$1" | tee -a "$LOG_FILE"
|
|
}
|
|
|
|
# Create backup directory if it doesn't exist
|
|
BACKUP_DIR="$(dirname "$0")/../immich_backups"
|
|
mkdir -p "$BACKUP_DIR"
|
|
|
|
# Generate timestamp for the backup filename
|
|
TIMESTAMP=$(date +"%Y%m%d_%H%M%S")
|
|
DB_BACKUP_FILENAME="immich_db_backup_${TIMESTAMP}.sql"
|
|
DB_BACKUP_PATH="${BACKUP_DIR}/${DB_BACKUP_FILENAME}"
|
|
UPLOAD_BACKUP_PATH="${BACKUP_DIR}/immich_uploads_${TIMESTAMP}.tar.gz"
|
|
|
|
# Handle dry-run mode
|
|
if [ "$DRY_RUN" = true ]; then
|
|
echo ""
|
|
echo "=== DRY RUN MODE - NO ACTUAL BACKUP WILL BE PERFORMED ==="
|
|
echo ""
|
|
echo "Configuration:"
|
|
echo " - Database: ${DB_DATABASE_NAME}"
|
|
echo " - Username: ${DB_USERNAME}"
|
|
echo " - Upload Location: ${UPLOAD_LOCATION}"
|
|
echo " - Container: immich_postgres"
|
|
echo " - Backup Directory: ${BACKUP_DIR}"
|
|
echo ""
|
|
echo "Would create:"
|
|
echo " - Database backup: ${DB_BACKUP_PATH}.gz"
|
|
echo " - Upload backup: ${UPLOAD_BACKUP_PATH}"
|
|
echo ""
|
|
echo "Workflow:"
|
|
echo " 1. Create local backups"
|
|
echo " 2. Copy to shared storage: /mnt/share/media/backups/immich/"
|
|
echo " 3. Upload to B2 (if configured)"
|
|
echo " 4. Delete local backups (keep shared copies)"
|
|
echo ""
|
|
|
|
# Check container status in dry-run
|
|
echo "Container Status Check:"
|
|
if docker ps -q --filter "name=immich_server" | grep -q .; then
|
|
echo " ✓ immich_server: Running (would pause during backup)"
|
|
else
|
|
echo " ! immich_server: Not running or not found"
|
|
fi
|
|
|
|
if docker ps -q --filter "name=immich_postgres" | grep -q .; then
|
|
echo " ✓ immich_postgres: Running"
|
|
else
|
|
echo " ✗ immich_postgres: Not running - backup would fail!"
|
|
exit 1
|
|
fi
|
|
|
|
# Check upload directory
|
|
if [ -d "${UPLOAD_LOCATION}" ]; then
|
|
UPLOAD_SIZE=$(du -sh "${UPLOAD_LOCATION}" 2>/dev/null | cut -f1 || echo "unknown")
|
|
echo " ✓ Upload directory: ${UPLOAD_LOCATION} (${UPLOAD_SIZE})"
|
|
else
|
|
echo " ✗ Upload directory: ${UPLOAD_LOCATION} does not exist - backup would fail!"
|
|
exit 1
|
|
fi
|
|
|
|
# Check B2 configuration
|
|
echo ""
|
|
echo "B2 Upload Configuration:"
|
|
if [ "$NO_UPLOAD" = true ]; then
|
|
echo " ! B2 upload disabled by --no-upload flag"
|
|
elif [ -n "$B2_APPLICATION_KEY_ID" ] && [ -n "$B2_APPLICATION_KEY" ] && [ -n "$B2_BUCKET_NAME" ]; then
|
|
echo " ✓ B2 configured - would upload to bucket: ${B2_BUCKET_NAME}"
|
|
if [ -f "$B2_CLI" ]; then
|
|
echo " ✓ B2 CLI found at: ${B2_CLI}"
|
|
else
|
|
echo " ✗ B2 CLI not found at: ${B2_CLI} - upload would fail!"
|
|
fi
|
|
else
|
|
echo " ! B2 not configured - would skip upload"
|
|
fi
|
|
|
|
# Check shared storage directory
|
|
echo ""
|
|
echo "Shared Storage Check:"
|
|
if [ -d "/mnt/share/media/backups" ]; then
|
|
echo " ✓ Shared storage accessible: /mnt/share/media/backups"
|
|
if [ -w "/mnt/share/media/backups" ]; then
|
|
echo " ✓ Shared storage writable - would copy backups before B2 upload"
|
|
else
|
|
echo " ⚠ Shared storage not writable - backups would remain in ${BACKUP_DIR}"
|
|
fi
|
|
else
|
|
echo " ⚠ Shared storage not accessible: /mnt/share/media/backups"
|
|
echo " Backups would remain in ${BACKUP_DIR}"
|
|
fi
|
|
|
|
echo ""
|
|
echo "=== DRY RUN COMPLETE - No files were created or modified ==="
|
|
exit 0
|
|
fi
|
|
|
|
log_message "Starting complete backup of Immich installation..."
|
|
log_message "Using settings from .env file:"
|
|
log_message " - Database: ${DB_DATABASE_NAME}"
|
|
log_message " - Username: ${DB_USERNAME}"
|
|
log_message " - Upload Location: ${UPLOAD_LOCATION}"
|
|
log_message " - Container: immich_postgres"
|
|
log_message " - Backup Directory: ${BACKUP_DIR}"
|
|
|
|
if [ "$NO_UPLOAD" = true ]; then
|
|
log_message " - B2 Upload: DISABLED (--no-upload flag)"
|
|
fi
|
|
|
|
if [ "$VERBOSE" = true ]; then
|
|
log_message " - Verbose logging: ENABLED"
|
|
fi
|
|
|
|
# Send start notification
|
|
send_notification "🚀 Immich Backup Started" "Starting complete backup of Immich database and uploads directory" "info"
|
|
|
|
# Initialize backup metrics if enabled
|
|
if [[ "$METRICS_ENABLED" == "true" ]]; then
|
|
metrics_backup_start "immich" "Immich photo management system backup"
|
|
metrics_update_status "running" "Preparing backup environment"
|
|
fi
|
|
|
|
# Check if the Immich server container exists and is running
|
|
log_status "Checking immich_server container status..."
|
|
if docker ps -q --filter "name=immich_server" | grep -q .; then
|
|
log_message "Pausing immich_server container to minimize database writes..."
|
|
if ! docker pause immich_server; then
|
|
log_message "Failed to pause immich_server container."
|
|
# Continue with backup instead of exiting
|
|
fi
|
|
else
|
|
log_message "Note: immich_server container not found or not running. Continuing with backup anyway."
|
|
# Set a flag so we don't try to unpause it later
|
|
IMMICH_SERVER_RUNNING=false
|
|
fi
|
|
|
|
# Check if the Postgres container exists and is running
|
|
log_status "Checking postgres container status..."
|
|
if ! docker ps -q --filter "name=immich_postgres" | grep -q .; then
|
|
log_message "Error: immich_postgres container is not running. Cannot proceed with backup."
|
|
exit 1
|
|
fi
|
|
|
|
echo ""
|
|
echo "=== PHASE 1: DATABASE BACKUP ==="
|
|
|
|
# Update metrics for database backup phase
|
|
if [[ "$METRICS_ENABLED" == "true" ]]; then
|
|
metrics_update_status "running" "Starting database backup"
|
|
fi
|
|
|
|
log_message "Taking database backup using pg_dumpall as recommended by Immich documentation..."
|
|
# Use pg_dumpall with recommended flags: --clean and --if-exists
|
|
if ! docker exec -t immich_postgres pg_dumpall \
|
|
--clean \
|
|
--if-exists \
|
|
--username="${DB_USERNAME}" \
|
|
> "${DB_BACKUP_PATH}" || [ ! -s "${DB_BACKUP_PATH}" ]; then
|
|
log_message "Error: Database backup failed or created an empty file."
|
|
exit 1
|
|
fi
|
|
|
|
log_message "Database backup completed successfully!"
|
|
|
|
# Update metrics for database backup completion
|
|
if [[ "$METRICS_ENABLED" == "true" ]]; then
|
|
metrics_file_backup_complete "${DB_BACKUP_PATH}" "database" "success"
|
|
fi
|
|
|
|
# Compress the database backup file
|
|
log_message "Compressing database backup file..."
|
|
if ! gzip -f "${DB_BACKUP_PATH}"; then
|
|
log_message "Warning: Failed to compress database backup file."
|
|
fi
|
|
|
|
echo ""
|
|
echo "=== PHASE 2: UPLOAD DIRECTORY BACKUP ==="
|
|
|
|
# Update metrics for uploads backup phase
|
|
if [[ "$METRICS_ENABLED" == "true" ]]; then
|
|
metrics_update_status "running" "Starting upload directory backup"
|
|
fi
|
|
|
|
log_message "Backing up user upload directory: ${UPLOAD_LOCATION}"
|
|
|
|
# Verify the upload location exists
|
|
if [ ! -d "${UPLOAD_LOCATION}" ]; then
|
|
log_message "Error: Upload location ${UPLOAD_LOCATION} does not exist!"
|
|
exit 1
|
|
fi
|
|
|
|
# Create compressed archive of the upload directory
|
|
# According to Immich docs, we need to backup the entire UPLOAD_LOCATION
|
|
# which includes: upload/, profile/, thumbs/, encoded-video/, library/, backups/
|
|
|
|
# Update metrics for upload backup phase
|
|
if [[ "$METRICS_ENABLED" == "true" ]]; then
|
|
metrics_update_status "running" "Starting upload directory backup"
|
|
fi
|
|
|
|
log_message "Creating compressed archive of upload directory..."
|
|
log_message "This may take a while depending on the size of your media library..."
|
|
|
|
# Use tar with progress indication and exclude any existing backup files in the upload location
|
|
if ! tar --exclude="${UPLOAD_LOCATION}/backups/*.tar.gz" \
|
|
--exclude="${UPLOAD_LOCATION}/backups/*.sql.gz" \
|
|
-czf "${UPLOAD_BACKUP_PATH}" \
|
|
-C "$(dirname "${UPLOAD_LOCATION}")" \
|
|
"$(basename "${UPLOAD_LOCATION}")"; then
|
|
log_message "Error: Failed to create upload directory backup."
|
|
exit 1
|
|
fi
|
|
|
|
log_message "Upload directory backup completed successfully!"
|
|
|
|
# Update metrics for uploads backup completion
|
|
if [[ "$METRICS_ENABLED" == "true" ]]; then
|
|
metrics_file_backup_complete "${UPLOAD_BACKUP_PATH}" "uploads" "success"
|
|
fi
|
|
|
|
# Resume the Immich server only if it was running and we paused it
|
|
if [ "${IMMICH_SERVER_RUNNING:-true}" = true ]; then
|
|
log_status "Resuming immich_server container..."
|
|
if ! docker unpause immich_server 2>/dev/null; then
|
|
log_message "Note: No need to unpause immich_server container."
|
|
fi
|
|
fi
|
|
|
|
echo ""
|
|
echo "=== COPYING BACKUPS TO SHARED STORAGE ==="
|
|
|
|
# Update metrics for shared storage phase
|
|
if [[ "$METRICS_ENABLED" == "true" ]]; then
|
|
metrics_update_status "running" "Copying backups to shared storage"
|
|
fi
|
|
|
|
SHARED_BACKUP_DIR="/mnt/share/media/backups/immich"
|
|
|
|
# Initialize COPY_SUCCESS before use
|
|
COPY_SUCCESS=false
|
|
|
|
# Create shared backup directory if it doesn't exist
|
|
if ! mkdir -p "$SHARED_BACKUP_DIR"; then
|
|
log_message "Warning: Failed to create shared backup directory: $SHARED_BACKUP_DIR"
|
|
log_message "Backup files remain only in: $BACKUP_DIR"
|
|
COPY_SUCCESS=false
|
|
else
|
|
log_message "Copying backup files to shared storage: $SHARED_BACKUP_DIR"
|
|
COPY_SUCCESS=true
|
|
|
|
# Copy database backup
|
|
if [ -f "${DB_BACKUP_PATH}.gz" ]; then
|
|
if cp "${DB_BACKUP_PATH}.gz" "$SHARED_BACKUP_DIR/"; then
|
|
log_message "✅ Copied database backup to shared storage"
|
|
else
|
|
log_message "❌ Failed to copy database backup to shared storage"
|
|
COPY_SUCCESS=false
|
|
fi
|
|
fi
|
|
|
|
# Copy uploads backup
|
|
if [ -f "${UPLOAD_BACKUP_PATH}" ]; then
|
|
if cp "${UPLOAD_BACKUP_PATH}" "$SHARED_BACKUP_DIR/"; then
|
|
log_message "✅ Copied uploads backup to shared storage"
|
|
else
|
|
log_message "❌ Failed to copy uploads backup to shared storage"
|
|
COPY_SUCCESS=false
|
|
fi
|
|
fi
|
|
|
|
if [ "$COPY_SUCCESS" = true ]; then
|
|
log_message "All backup files successfully copied to shared storage"
|
|
else
|
|
log_message "Some backup files failed to copy to shared storage"
|
|
fi
|
|
fi
|
|
|
|
# Calculate backup sizes
|
|
DB_BACKUP_SIZE=$(du -h "${DB_BACKUP_PATH}.gz" 2>/dev/null | cut -f1 || echo "Unknown")
|
|
UPLOAD_BACKUP_SIZE=$(du -h "${UPLOAD_BACKUP_PATH}" 2>/dev/null | cut -f1 || echo "Unknown")
|
|
|
|
echo ""
|
|
echo "=== BACKUP COMPLETED SUCCESSFULLY! ==="
|
|
if [ "$COPY_SUCCESS" = true ]; then
|
|
echo "Database backup saved to: ${SHARED_BACKUP_DIR}/$(basename "${DB_BACKUP_PATH}.gz")"
|
|
echo "Upload directory backup saved to: ${SHARED_BACKUP_DIR}/$(basename "${UPLOAD_BACKUP_PATH}")"
|
|
echo "(Local backup files have been cleaned up)"
|
|
else
|
|
echo "Database backup saved to: ${DB_BACKUP_PATH}.gz"
|
|
echo "Upload directory backup saved to: ${UPLOAD_BACKUP_PATH}"
|
|
echo "(Local backup files retained due to copy failure)"
|
|
fi
|
|
|
|
echo ""
|
|
echo "=== BACKUP SUMMARY ==="
|
|
echo "Database backup size: ${DB_BACKUP_SIZE}"
|
|
echo "Upload directory backup size: ${UPLOAD_BACKUP_SIZE}"
|
|
|
|
# Upload to B2 (if configured and not disabled)
|
|
echo ""
|
|
if [ "$NO_UPLOAD" = true ]; then
|
|
echo "=== SKIPPING B2 UPLOAD (--no-upload flag) ==="
|
|
log_message "B2 upload skipped due to --no-upload flag"
|
|
B2_UPLOAD_SUCCESS="skipped"
|
|
else
|
|
echo "=== UPLOADING TO BACKBLAZE B2 ==="
|
|
|
|
# Update metrics for B2 upload phase
|
|
if [[ "$METRICS_ENABLED" == "true" ]]; then
|
|
metrics_update_status "running" "Uploading backups to Backblaze B2"
|
|
fi
|
|
|
|
B2_UPLOAD_SUCCESS=true
|
|
|
|
# Upload database backup from local location
|
|
if ! upload_to_b2 "${DB_BACKUP_PATH}.gz"; then
|
|
B2_UPLOAD_SUCCESS=false
|
|
fi
|
|
|
|
# Upload uploads backup from local location
|
|
if ! upload_to_b2 "${UPLOAD_BACKUP_PATH}"; then
|
|
B2_UPLOAD_SUCCESS=false
|
|
fi
|
|
fi
|
|
|
|
# Clean up local backup files after successful copy and B2 upload
|
|
echo ""
|
|
echo "=== CLEANING UP LOCAL BACKUPS ==="
|
|
if [ "$COPY_SUCCESS" = true ]; then
|
|
log_message "Removing local backup files (copies exist in shared storage)..."
|
|
|
|
# Remove local database backup
|
|
if [ -f "${DB_BACKUP_PATH}.gz" ]; then
|
|
if rm "${DB_BACKUP_PATH}.gz"; then
|
|
log_message "✅ Removed local database backup"
|
|
else
|
|
log_message "⚠ Failed to remove local database backup: ${DB_BACKUP_PATH}.gz"
|
|
fi
|
|
fi
|
|
|
|
# Remove local uploads backup
|
|
if [ -f "${UPLOAD_BACKUP_PATH}" ]; then
|
|
if rm "${UPLOAD_BACKUP_PATH}"; then
|
|
log_message "✅ Removed local uploads backup"
|
|
else
|
|
log_message "⚠ Failed to remove local uploads backup: ${UPLOAD_BACKUP_PATH}"
|
|
fi
|
|
fi
|
|
|
|
log_message "Local cleanup completed - backups are now in shared storage"
|
|
|
|
# Store original paths for health checks before updating
|
|
ORIGINAL_DB_PATH="${DB_BACKUP_PATH}.gz"
|
|
ORIGINAL_UPLOAD_PATH="${UPLOAD_BACKUP_PATH}"
|
|
else
|
|
log_message "Skipping local cleanup - copy to shared storage failed"
|
|
log_message "Local backup files retained in: $BACKUP_DIR"
|
|
|
|
# Keep original paths for health checks
|
|
ORIGINAL_DB_PATH="${DB_BACKUP_PATH}.gz"
|
|
ORIGINAL_UPLOAD_PATH="${UPLOAD_BACKUP_PATH}"
|
|
fi
|
|
|
|
# Prepare notification message
|
|
DB_FILENAME=$(basename "${DB_BACKUP_PATH}" .gz)
|
|
UPLOAD_FILENAME=$(basename "${UPLOAD_BACKUP_PATH}")
|
|
NOTIFICATION_MESSAGE="📦 Database: ${DB_FILENAME} (${DB_BACKUP_SIZE})
|
|
📁 Uploads: ${UPLOAD_FILENAME} (${UPLOAD_BACKUP_SIZE})"
|
|
|
|
# Add storage location info to notification
|
|
if [ "$COPY_SUCCESS" = true ]; then
|
|
NOTIFICATION_MESSAGE="${NOTIFICATION_MESSAGE}
|
|
💾 Stored in: ${SHARED_BACKUP_DIR} (local files cleaned up)"
|
|
else
|
|
NOTIFICATION_MESSAGE="${NOTIFICATION_MESSAGE}
|
|
💾 Located in: ${BACKUP_DIR} (copy to shared storage failed)"
|
|
fi
|
|
|
|
if [ "$B2_UPLOAD_SUCCESS" = "skipped" ]; then
|
|
NOTIFICATION_MESSAGE="${NOTIFICATION_MESSAGE}
|
|
💾 Local backup only (B2 upload skipped)"
|
|
send_notification "✅ Immich Backup Completed (Local Only)" "$NOTIFICATION_MESSAGE" "success"
|
|
elif [ "$B2_UPLOAD_SUCCESS" = true ] && [ -n "$B2_BUCKET_NAME" ]; then
|
|
NOTIFICATION_MESSAGE="${NOTIFICATION_MESSAGE}
|
|
☁️ Successfully uploaded to B2 bucket: ${B2_BUCKET_NAME}"
|
|
send_notification "✅ Immich Backup Completed" "$NOTIFICATION_MESSAGE" "success"
|
|
elif [ -n "$B2_BUCKET_NAME" ]; then
|
|
NOTIFICATION_MESSAGE="${NOTIFICATION_MESSAGE}
|
|
⚠️ B2 upload failed - files saved locally only"
|
|
send_notification "⚠️ Immich Backup Completed (B2 Upload Failed)" "$NOTIFICATION_MESSAGE" "warning"
|
|
else
|
|
send_notification "✅ Immich Backup Completed" "$NOTIFICATION_MESSAGE" "success"
|
|
fi
|
|
|
|
# Show backup information
|
|
echo ""
|
|
echo "=== BACKUP INVENTORY ==="
|
|
if [ "$COPY_SUCCESS" = true ]; then
|
|
INVENTORY_DIR="$SHARED_BACKUP_DIR"
|
|
echo "Backup location: $SHARED_BACKUP_DIR"
|
|
else
|
|
INVENTORY_DIR="$BACKUP_DIR"
|
|
echo "Backup location: $BACKUP_DIR"
|
|
fi
|
|
|
|
find "${INVENTORY_DIR}" -name "*.gz" -o -name "*.tar.gz" | wc -l | xargs echo "Total number of backup files:"
|
|
du -sh "${INVENTORY_DIR}" | cut -f1 | xargs echo "Total backup directory size:"
|
|
|
|
# List recent backups
|
|
echo ""
|
|
echo "Recent backups:"
|
|
find "${INVENTORY_DIR}" -name "*.gz" -o -name "*.tar.gz" -mtime -7 | sort
|
|
|
|
# Health check: Verify backup file sizes
|
|
if [ "$COPY_SUCCESS" = true ]; then
|
|
DB_BACKUP_SIZE_KB=$(du -k "${SHARED_BACKUP_DIR}/$(basename "${ORIGINAL_DB_PATH}")" 2>/dev/null | cut -f1 || echo "0")
|
|
UPLOAD_BACKUP_SIZE_KB=$(du -k "${SHARED_BACKUP_DIR}/$(basename "${ORIGINAL_UPLOAD_PATH}")" 2>/dev/null | cut -f1 || echo "0")
|
|
else
|
|
DB_BACKUP_SIZE_KB=$(du -k "${ORIGINAL_DB_PATH}" 2>/dev/null | cut -f1 || echo "0")
|
|
UPLOAD_BACKUP_SIZE_KB=$(du -k "${ORIGINAL_UPLOAD_PATH}" 2>/dev/null | cut -f1 || echo "0")
|
|
fi
|
|
|
|
echo ""
|
|
echo "=== BACKUP VALIDATION ==="
|
|
if [ "${DB_BACKUP_SIZE_KB}" -lt 100 ]; then
|
|
echo "WARNING: Database backup file is smaller than expected (${DB_BACKUP_SIZE_KB}KB). Please verify its integrity."
|
|
else
|
|
echo "✓ Database backup size appears normal (${DB_BACKUP_SIZE_KB}KB)"
|
|
fi
|
|
|
|
if [ "${UPLOAD_BACKUP_SIZE_KB}" -lt 1024 ]; then
|
|
echo "WARNING: Upload directory backup file is smaller than expected (${UPLOAD_BACKUP_SIZE_KB}KB). Please verify its integrity."
|
|
else
|
|
echo "✓ Upload directory backup size appears normal (${UPLOAD_BACKUP_SIZE_KB}KB)"
|
|
fi
|
|
|
|
# Optional: Remove old backups (older than 30 days)
|
|
echo ""
|
|
echo "=== CLEANUP ==="
|
|
OLD_BACKUPS=$(find "${INVENTORY_DIR}" -name "*.gz" -o -name "*.tar.gz" -mtime +30 | wc -l)
|
|
if [ "${OLD_BACKUPS}" -gt 0 ]; then
|
|
echo "Found ${OLD_BACKUPS} backup files older than 30 days in ${INVENTORY_DIR}."
|
|
echo "To remove them automatically, uncomment the cleanup line in this script."
|
|
# Uncomment the next line to automatically remove old backups
|
|
# find "${INVENTORY_DIR}" -name "*.gz" -o -name "*.tar.gz" -mtime +30 -delete
|
|
else
|
|
echo "No old backup files found (older than 30 days) in ${INVENTORY_DIR}."
|
|
fi
|
|
|
|
echo ""
|
|
echo "=== RESTORE INSTRUCTIONS ==="
|
|
echo "To restore from this backup:"
|
|
echo "1. Database restore instructions: https://immich.app/docs/administration/backup-and-restore/#database"
|
|
if [ "$COPY_SUCCESS" = true ]; then
|
|
echo "2. Upload directory: Extract ${SHARED_BACKUP_DIR}/$(basename "${UPLOAD_BACKUP_PATH}") to your UPLOAD_LOCATION"
|
|
echo ""
|
|
echo "Backup files are located in: ${SHARED_BACKUP_DIR}"
|
|
else
|
|
echo "2. Upload directory: Extract ${UPLOAD_BACKUP_PATH} to your UPLOAD_LOCATION"
|
|
echo ""
|
|
echo "Backup files are located in: ${BACKUP_DIR}"
|
|
fi
|
|
echo ""
|
|
echo "IMPORTANT: For a complete restore, you need BOTH the database backup AND the upload directory backup."
|
|
echo "The database contains metadata, while the upload directory contains your actual photos and videos."
|