diff --git a/backup-karakeep.sh b/backup-karakeep.sh new file mode 100755 index 0000000..b0d7256 --- /dev/null +++ b/backup-karakeep.sh @@ -0,0 +1,868 @@ +#!/bin/bash + +set -e + +# Load the unified backup metrics library +SCRIPT_DIR="$(dirname "$(readlink -f "$0")")" +LIB_DIR="$SCRIPT_DIR/lib" +if [[ -f "$LIB_DIR/unified-backup-metrics.sh" ]]; then + # shellcheck source=lib/unified-backup-metrics.sh + source "$LIB_DIR/unified-backup-metrics.sh" + METRICS_ENABLED=true +else + echo "Warning: Unified backup metrics library not found at $LIB_DIR/unified-backup-metrics.sh" + METRICS_ENABLED=false +fi + +# Color codes for output +RED='\033[0;31m' +GREEN='\033[0;32m' +YELLOW='\033[1;33m' +BLUE='\033[0;34m' +CYAN='\033[0;36m' +NC='\033[0m' # No Color + +# Configuration +MAX_BACKUP_AGE_DAYS=30 +MAX_BACKUPS_TO_KEEP=10 +COMPOSE_DIR="/home/acedanger/docker/karakeep" +BACKUP_ROOT="/mnt/share/media/backups/karakeep" +LOCAL_BACKUP_DIR="/home/acedanger/backups/karakeep" +LOG_ROOT="${SCRIPT_DIR}/logs" +JSON_LOG_FILE="${SCRIPT_DIR}/logs/karakeep-backup.json" +PERFORMANCE_LOG_FILE="${SCRIPT_DIR}/logs/karakeep-backup-performance.json" + +NAS_LOG_DIR="/mnt/share/media/backups/logs" + +# Volume configuration: volume_name -> mount_path inside container +declare -A KARAKEEP_VOLUMES=( + ["hoarder_data"]="/data" + ["hoarder_meilisearch"]="/meili_data" +) + +# Script options +VERIFY_BACKUPS=true +PERFORMANCE_MONITORING=true +WEBHOOK_URL="https://notify.peterwood.rocks/lab" +INTERACTIVE_MODE=false +DRY_RUN=false +STOP_CONTAINERS=true # Stop containers before backup for consistency +SKIP_NAS=false + +# show help function +show_help() { + cat << EOF +Karakeep Services Backup Script + +Usage: $0 [OPTIONS] + +OPTIONS: + --dry-run Show what would be backed up without actually doing it + --no-verify Skip backup verification + --no-stop Do hot backup without stopping containers (less safe) + --no-nas Skip copying to NAS, keep local backups only + --interactive Ask for confirmation before each backup + --webhook URL Custom webhook URL for notifications + -h, --help Show this help message + +EXAMPLES: + $0 # Run full backup with container stop/start + $0 --dry-run # Preview what would be backed up + $0 --no-stop # Hot backup without stopping containers + $0 --no-nas # Local backup only (skip NAS copy) + $0 --no-verify # Skip verification for faster backup + +VOLUMES BACKED UP: + - hoarder_data (Karakeep app data: bookmarks, assets, database) + - hoarder_meilisearch (Meilisearch search index) + +COMPOSE DIRECTORY: + $COMPOSE_DIR + +EOF +} + +# Parse command line arguments +while [[ $# -gt 0 ]]; do + case $1 in + --dry-run) + DRY_RUN=true + shift + ;; + --no-verify) + VERIFY_BACKUPS=false + shift + ;; + --no-stop) + STOP_CONTAINERS=false + shift + ;; + --no-nas) + SKIP_NAS=true + shift + ;; + --interactive) + INTERACTIVE_MODE=true + shift + ;; + --webhook) + WEBHOOK_URL="$2" + shift 2 + ;; + -h|--help) + show_help + exit 0 + ;; + *) + echo "Unknown option: $1" + show_help + exit 1 + ;; + esac +done + +# Timestamp for this backup run +BACKUP_TIMESTAMP=$(date +%Y%m%d_%H%M%S) +BACKUP_DEST="${LOCAL_BACKUP_DIR}/${BACKUP_TIMESTAMP}" + +# Create necessary directories +mkdir -p "${LOG_ROOT}" +mkdir -p "${LOCAL_BACKUP_DIR}" + +# Log files +LOG_FILE="${LOG_ROOT}/karakeep-backup-${BACKUP_TIMESTAMP}.log" +MARKDOWN_LOG="${LOG_ROOT}/karakeep-backup-${BACKUP_TIMESTAMP}.md" + +# Logging functions +log_message() { + local message="$1" + local timestamp + timestamp=$(date '+%Y-%m-%d %H:%M:%S') + echo -e "${CYAN}[${timestamp}]${NC} ${message}" + echo "[${timestamp}] $message" >> "${LOG_FILE}" 2>/dev/null || true +} + +log_error() { + local message="$1" + local timestamp + timestamp=$(date '+%Y-%m-%d %H:%M:%S') + echo -e "${RED}[${timestamp}] ERROR:${NC} ${message}" >&2 + echo "[${timestamp}] ERROR: $message" >> "${LOG_FILE}" 2>/dev/null || true +} + +log_success() { + local message="$1" + local timestamp + timestamp=$(date '+%Y-%m-%d %H:%M:%S') + echo -e "${GREEN}[${timestamp}] SUCCESS:${NC} ${message}" + echo "[${timestamp}] SUCCESS: $message" >> "${LOG_FILE}" 2>/dev/null || true +} + +log_warning() { + local message="$1" + local timestamp + timestamp=$(date '+%Y-%m-%d %H:%M:%S') + echo -e "${YELLOW}[${timestamp}] WARNING:${NC} ${message}" + echo "[${timestamp}] WARNING: $message" >> "${LOG_FILE}" 2>/dev/null || true +} + +log_info() { + local message="$1" + local timestamp + timestamp=$(date '+%Y-%m-%d %H:%M:%S') + echo -e "${BLUE}[${timestamp}] INFO:${NC} ${message}" + echo "[${timestamp}] INFO: $message" >> "${LOG_FILE}" 2>/dev/null || true +} + +# Performance tracking +track_performance() { + if [ "$PERFORMANCE_MONITORING" != true ]; then + return 0 + fi + + local operation="$1" + local start_time="$2" + local end_time="${3:-$(date +%s)}" + local duration=$((end_time - start_time)) + + if [ ! -f "$PERFORMANCE_LOG_FILE" ]; then + echo "[]" > "$PERFORMANCE_LOG_FILE" + fi + + if command -v jq > /dev/null 2>&1; then + local entry + entry=$(jq -n \ + --arg timestamp "$(date -Iseconds)" \ + --arg operation "$operation" \ + --arg duration "$duration" \ + --arg hostname "$(hostname)" \ + '{ + timestamp: $timestamp, + operation: $operation, + duration: ($duration | tonumber), + hostname: $hostname + }') + + local lock_file="${PERFORMANCE_LOG_FILE}.lock" + local max_wait=10 + local wait_count=0 + + while [ $wait_count -lt $max_wait ]; do + if (set -C; echo $$ > "$lock_file") 2>/dev/null; then + break + fi + sleep 0.1 + ((wait_count++)) + done + + if [ $wait_count -lt $max_wait ]; then + if jq --argjson entry "$entry" '. += [$entry]' "$PERFORMANCE_LOG_FILE" > "${PERFORMANCE_LOG_FILE}.tmp" 2>/dev/null; then + mv "${PERFORMANCE_LOG_FILE}.tmp" "$PERFORMANCE_LOG_FILE" + else + rm -f "${PERFORMANCE_LOG_FILE}.tmp" + fi + rm -f "$lock_file" + fi + fi + + log_info "Performance: $operation completed in ${duration}s" +} + +# Initialize JSON log file +initialize_json_log() { + if [ ! -f "${JSON_LOG_FILE}" ] || ! jq empty "${JSON_LOG_FILE}" 2>/dev/null; then + echo "{}" > "${JSON_LOG_FILE}" + log_message "Initialized JSON log file" + fi +} + +# Log backup details with markdown formatting +log_file_details() { + local volume="$1" + local dest="$2" + local status="$3" + local size="" + local checksum="" + + if [ "$status" == "SUCCESS" ] && [ -e "$dest" ]; then + size=$(du -sh "$dest" 2>/dev/null | cut -f1 || echo "Unknown") + if [ "$VERIFY_BACKUPS" == true ]; then + checksum=$(md5sum "$dest" 2>/dev/null | cut -d' ' -f1 || echo "N/A") + fi + else + size="N/A" + checksum="N/A" + fi + + local markdown_lock="${MARKDOWN_LOG}.lock" + local max_wait=30 + local wait_count=0 + + while [ $wait_count -lt $max_wait ]; do + if (set -C; echo $$ > "$markdown_lock") 2>/dev/null; then + break + fi + sleep 0.1 + ((wait_count++)) + done + + if [ $wait_count -lt $max_wait ]; then + { + echo "## Volume: $volume" + echo "- **Status**: $status" + echo "- **Destination**: $dest" + echo "- **Size**: $size" + echo "- **Checksum**: $checksum" + echo "- **Timestamp**: $(date '+%Y-%m-%d %H:%M:%S')" + echo "" + } >> "$MARKDOWN_LOG" + rm -f "$markdown_lock" + else + log_warning "Could not acquire markdown log lock for $volume" + fi + + if command -v jq > /dev/null 2>&1; then + update_backup_log "$volume" "$dest" "$status" "$size" "$checksum" + fi +} + +# Update backup log in JSON format +update_backup_log() { + local volume="$1" + local dest="$2" + local status="$3" + local size="$4" + local checksum="$5" + local timestamp + timestamp=$(date -Iseconds) + + if ! command -v jq > /dev/null 2>&1; then + return 0 + fi + + local lock_file="${JSON_LOG_FILE}.lock" + local max_wait=30 + local wait_count=0 + + while [ $wait_count -lt $max_wait ]; do + if (set -C; echo $$ > "$lock_file") 2>/dev/null; then + break + fi + sleep 0.1 + ((wait_count++)) + done + + if [ $wait_count -ge $max_wait ]; then + log_warning "Could not acquire lock for JSON log update" + return 1 + fi + + local entry + entry=$(jq -n \ + --arg volume "$volume" \ + --arg dest "$dest" \ + --arg status "$status" \ + --arg size "$size" \ + --arg checksum "$checksum" \ + --arg timestamp "$timestamp" \ + '{ + volume: $volume, + destination: $dest, + status: $status, + size: $size, + checksum: $checksum, + timestamp: $timestamp + }') + + if jq --argjson entry "$entry" --arg volume "$volume" \ + '.[$volume] = $entry' "$JSON_LOG_FILE" > "${JSON_LOG_FILE}.tmp" 2>/dev/null; then + mv "${JSON_LOG_FILE}.tmp" "$JSON_LOG_FILE" + else + rm -f "${JSON_LOG_FILE}.tmp" + fi + + rm -f "$lock_file" +} + +# Check if NAS mount is accessible +check_nas_mount() { + local mount_point="/mnt/share/media" + + if ! mountpoint -q "$mount_point"; then + log_warning "NAS not mounted at $mount_point - backups will be local only" + return 1 + fi + + if [ ! -w "$(dirname "$BACKUP_ROOT")" ]; then + log_warning "No write access to NAS backup path: $BACKUP_ROOT" + return 1 + fi + + log_success "NAS mount check passed: $mount_point is accessible" + return 0 +} + +# Verify backup archive integrity +verify_backup() { + local volume="$1" + local archive="$2" + + if [ "$VERIFY_BACKUPS" != true ]; then + return 0 + fi + + log_info "Verifying backup archive: $archive" + + if [ ! -f "$archive" ]; then + log_error "Backup archive not found: $archive" + return 1 + fi + + local file_size + file_size=$(stat -c%s "$archive" 2>/dev/null || echo "0") + + if [ "$file_size" -eq 0 ]; then + log_error "Backup archive is empty: $archive" + return 1 + fi + + # Verify gzip container integrity (reads entire file, checks CRC). + # gzip -t is more reliable than tar -tzf, which can exit non-zero on + # tar warnings (e.g. special files) even when the archive is valid. + if ! gzip -t "$archive" 2>/dev/null; then + log_error "Backup archive failed integrity check: $archive" + return 1 + fi + + log_success "Backup verification passed for $volume (${file_size} bytes, gzip integrity OK)" + return 0 +} + +# Check disk space at backup destination +check_disk_space() { + local destination="$1" + local required_space_mb="${2:-500}" + + local available_space_kb + available_space_kb=$(df "$(dirname "$destination")" 2>/dev/null | awk 'NR==2 {print $4}' || echo "0") + local available_space_mb=$((available_space_kb / 1024)) + + if [ "$available_space_mb" -lt "$required_space_mb" ]; then + log_error "Insufficient disk space at $(dirname "$destination"). Available: ${available_space_mb}MB, Required: ${required_space_mb}MB" + return 1 + fi + + log_info "Disk space check passed at $(dirname "$destination"). Available: ${available_space_mb}MB" + return 0 +} + +# Check if Docker volume exists +check_volume_exists() { + local volume_name="$1" + + if ! docker volume inspect "$volume_name" > /dev/null 2>&1; then + log_error "Docker volume '$volume_name' not found" + return 1 + fi + + return 0 +} + +# Backup a single named Docker volume to a tar.gz archive +backup_volume() { + local volume_name="$1" + local mount_path="${KARAKEEP_VOLUMES[$volume_name]}" + local archive="${BACKUP_DEST}/${volume_name}.tar.gz" + local backup_start_time + backup_start_time=$(date +%s) + + log_message "Starting backup for volume: $volume_name (${mount_path})" + + if [ "$DRY_RUN" == true ]; then + log_info "DRY RUN: Would backup volume $volume_name -> $archive" + log_file_details "$volume_name" "$archive" "DRY RUN" + return 0 + fi + + if [ "$INTERACTIVE_MODE" == true ]; then + echo -n "Backup volume $volume_name? (y/N): " + read -r response + if [[ ! "$response" =~ ^[Yy]$ ]]; then + log_info "Skipping $volume_name backup (user choice)" + return 0 + fi + fi + + # Confirm volume exists + if ! check_volume_exists "$volume_name"; then + log_file_details "$volume_name" "$archive" "FAILED - Volume not found" + return 1 + fi + + # Create destination directory + mkdir -p "$BACKUP_DEST" + + log_info "Archiving volume $volume_name to $archive" + + # Use a minimal Alpine container to tar up the volume contents + if docker run --rm \ + --volume "${volume_name}:${mount_path}:ro" \ + alpine \ + tar czf - -C "$(dirname "$mount_path")" "$(basename "$mount_path")" \ + > "$archive" 2>>"$LOG_FILE"; then + + log_success "Volume archive created: $archive" + + # File-level metrics tracking + if [[ "$METRICS_ENABLED" == "true" ]]; then + local file_size + file_size=$(stat -c%s "$archive" 2>/dev/null || echo "0") + local checksum + checksum=$(md5sum "$archive" 2>/dev/null | cut -d' ' -f1 || echo "") + metrics_add_file "$archive" "success" "$file_size" "$checksum" + fi + + if verify_backup "$volume_name" "$archive"; then + log_file_details "$volume_name" "$archive" "SUCCESS" + track_performance "backup_${volume_name}" "$backup_start_time" + return 0 + else + log_file_details "$volume_name" "$archive" "VERIFICATION_FAILED" + if [[ "$METRICS_ENABLED" == "true" ]]; then + local file_size + file_size=$(stat -c%s "$archive" 2>/dev/null || echo "0") + metrics_add_file "$archive" "failed" "$file_size" "" "Verification failed" + fi + return 1 + fi + else + log_error "Failed to archive volume: $volume_name" + rm -f "$archive" + log_file_details "$volume_name" "$archive" "FAILED" + if [[ "$METRICS_ENABLED" == "true" ]]; then + metrics_add_file "$archive" "failed" "0" "" "Archive creation failed" + fi + return 1 + fi +} + +# Stop Karakeep containers before backup +stop_containers() { + log_message "Stopping Karakeep containers for consistent backup..." + + if [ ! -f "$COMPOSE_DIR/docker-compose.yml" ]; then + log_error "docker-compose.yml not found at $COMPOSE_DIR" + return 1 + fi + + local compose_output + if ! compose_output=$(docker compose -f "$COMPOSE_DIR/docker-compose.yml" --progress plain stop 2>&1); then + echo "$compose_output" | tee -a "$LOG_FILE" > /dev/null + log_error "Failed to stop Karakeep containers" + return 1 + fi + echo "$compose_output" | tee -a "$LOG_FILE" > /dev/null + + log_success "Karakeep containers stopped" + return 0 +} + +# Start Karakeep containers after backup +start_containers() { + log_message "Starting Karakeep containers..." + + local compose_output + if ! compose_output=$(docker compose -f "$COMPOSE_DIR/docker-compose.yml" --progress plain start 2>&1); then + echo "$compose_output" | tee -a "$LOG_FILE" > /dev/null + log_error "Failed to start Karakeep containers - manual intervention required" + return 1 + fi + echo "$compose_output" | tee -a "$LOG_FILE" > /dev/null + + log_success "Karakeep containers started" + return 0 +} + +# Copy backup to NAS +copy_to_nas() { + local src="$1" + local nas_dest="${BACKUP_ROOT}/$(basename "$src")" + + log_info "Copying backup to NAS: $nas_dest" + mkdir -p "$BACKUP_ROOT" + + if cp -r "$src" "$nas_dest" 2>>"$LOG_FILE"; then + log_success "Backup copied to NAS: $nas_dest" + return 0 + else + log_error "Failed to copy backup to NAS" + return 1 + fi +} + +# Copy log files for this run to the NAS logs directory +copy_logs_to_nas() { + if [ "$SKIP_NAS" == true ]; then + return 0 + fi + + if ! mountpoint -q "/mnt/share/media" 2>/dev/null; then + log_warning "NAS not mounted - skipping log copy to NAS" + return 1 + fi + + if [ ! -d "$NAS_LOG_DIR" ]; then + if ! mkdir -p "$NAS_LOG_DIR" 2>/dev/null; then + log_warning "Could not create NAS log directory: $NAS_LOG_DIR" + return 1 + fi + fi + + local copied=0 + for log_file in "${LOG_ROOT}/karakeep-backup-${BACKUP_TIMESTAMP}.log" \ + "${LOG_ROOT}/karakeep-backup-${BACKUP_TIMESTAMP}.md"; do + if [ -f "$log_file" ]; then + if cp "$log_file" "$NAS_LOG_DIR/" 2>/dev/null; then + log_info "Copied log to NAS: $NAS_LOG_DIR/$(basename "$log_file")" + copied=$((copied + 1)) + else + log_warning "Failed to copy log to NAS: $log_file" + fi + fi + done + + [ "$copied" -gt 0 ] && log_success "Copied $copied log file(s) to NAS: $NAS_LOG_DIR" + return 0 +} + +# Clean up old backups +cleanup_old_backups() { + log_message "Cleaning up old backups..." + + # Clean local backups: keep only MAX_BACKUPS_TO_KEEP most recent timestamped dirs + find "$LOCAL_BACKUP_DIR" -maxdepth 1 -mindepth 1 -type d | sort -r | \ + tail -n +$((MAX_BACKUPS_TO_KEEP + 1)) | xargs rm -rf 2>/dev/null || true + + # Clean local backups older than MAX_BACKUP_AGE_DAYS + find "$LOCAL_BACKUP_DIR" -maxdepth 1 -mindepth 1 -type d -mtime +${MAX_BACKUP_AGE_DAYS} | \ + xargs rm -rf 2>/dev/null || true + + # Clean NAS backups if accessible + if check_nas_mount && [ "$SKIP_NAS" != true ]; then + find "$BACKUP_ROOT" -maxdepth 1 -mindepth 1 -type d | sort -r | \ + tail -n +$((MAX_BACKUPS_TO_KEEP + 1)) | xargs rm -rf 2>/dev/null || true + + find "$BACKUP_ROOT" -maxdepth 1 -mindepth 1 -type d -mtime +${MAX_BACKUP_AGE_DAYS} | \ + xargs rm -rf 2>/dev/null || true + + # Clean old NAS karakeep logs + find "$NAS_LOG_DIR" -maxdepth 1 -name "karakeep-backup-*.log" -mtime +${MAX_BACKUP_AGE_DAYS} -delete 2>/dev/null || true + find "$NAS_LOG_DIR" -maxdepth 1 -name "karakeep-backup-*.md" -mtime +${MAX_BACKUP_AGE_DAYS} -delete 2>/dev/null || true + fi + + # Clean up old local log files + find "$LOG_ROOT" -name "karakeep-backup-*.log" -mtime +${MAX_BACKUP_AGE_DAYS} -delete 2>/dev/null || true + find "$LOG_ROOT" -name "karakeep-backup-*.md" -mtime +${MAX_BACKUP_AGE_DAYS} -delete 2>/dev/null || true + + log_success "Cleanup completed" +} + +# Send notification +send_notification() { + local title="$1" + local message="$2" + local status="${3:-info}" + local success_count="${4:-0}" + local failed_count="${5:-0}" + local hostname + hostname=$(hostname) + + local enhanced_message + printf -v enhanced_message "%s\n\nVolumes: %d\nSuccessful: %d\nFailed: %d\nHost: %s\nBackup: %s" \ + "$message" "${#KARAKEEP_VOLUMES[@]}" "$success_count" "$failed_count" "$hostname" "$BACKUP_DEST" + + case "$status" in + "success") log_success "$title: $message" ;; + "error") log_error "$title: $message" ;; + "warning") log_warning "$title: $message" ;; + *) log_info "$title: $message" ;; + esac + + if [ -n "$WEBHOOK_URL" ] && [ "$DRY_RUN" != true ]; then + local tags="backup,karakeep,${hostname}" + [ "$failed_count" -gt 0 ] && tags="${tags},errors" + + curl -s \ + -H "tags:${tags}" \ + -d "$enhanced_message" \ + "$WEBHOOK_URL" 2>/dev/null || log_warning "Failed to send webhook notification" + fi +} + +# Generate backup summary report +generate_summary_report() { + local success_count="$1" + local failed_count="$2" + local total_time="$3" + + log_message "=== BACKUP SUMMARY REPORT ===" + log_message "Total Volumes: ${#KARAKEEP_VOLUMES[@]}" + log_message "Successful Backups: $success_count" + log_message "Failed Backups: $failed_count" + log_message "Total Time: ${total_time}s" + log_message "Backup Directory: $BACKUP_DEST" + log_message "Log File: $LOG_FILE" + log_message "Markdown Report: $MARKDOWN_LOG" + + { + echo "# Karakeep Backup Summary Report" + echo "**Date**: $(date '+%Y-%m-%d %H:%M:%S')" + echo "**Host**: $(hostname)" + echo "**Total Volumes**: ${#KARAKEEP_VOLUMES[@]}" + echo "**Successful**: $success_count" + echo "**Failed**: $failed_count" + echo "**Duration**: ${total_time}s" + echo "**Backup Directory**: $BACKUP_DEST" + echo "" + } >> "$MARKDOWN_LOG" +} + +# Main backup execution +main() { + local script_start_time + script_start_time=$(date +%s) + local containers_stopped=false + + log_message "=== KARAKEEP BACKUP STARTED ===" + log_message "Host: $(hostname)" + log_message "Timestamp: $BACKUP_TIMESTAMP" + log_message "Dry Run: $DRY_RUN" + log_message "Stop Containers: $STOP_CONTAINERS" + log_message "Verify Backups: $VERIFY_BACKUPS" + log_message "Backup Destination: $BACKUP_DEST" + + # Initialize metrics if enabled + if [[ "$METRICS_ENABLED" == "true" ]]; then + metrics_backup_start "karakeep" "Karakeep volume backup (hoarder_data, hoarder_meilisearch)" "$LOCAL_BACKUP_DIR" + metrics_status_update "initializing" "Preparing Karakeep backup" + fi + + # Initialize logging + initialize_json_log + + { + echo "# Karakeep Backup Report" + echo "**Started**: $(date '+%Y-%m-%d %H:%M:%S')" + echo "**Host**: $(hostname)" + echo "**Backup Timestamp**: $BACKUP_TIMESTAMP" + echo "" + } > "$MARKDOWN_LOG" + + # Pre-flight: Docker available? + if ! docker info > /dev/null 2>&1; then + log_error "Docker is not running or not accessible" + if [[ "$METRICS_ENABLED" == "true" ]]; then + metrics_backup_complete "failed" "Docker is not accessible" + fi + send_notification "Karakeep Backup Failed" "Docker is not accessible" "error" 0 "${#KARAKEEP_VOLUMES[@]}" + exit 1 + fi + + # Pre-flight: disk space check on local backup dir + if [[ "$METRICS_ENABLED" == "true" ]]; then + metrics_status_update "checking" "Running pre-flight checks" + fi + + mkdir -p "$LOCAL_BACKUP_DIR" + if ! check_disk_space "$LOCAL_BACKUP_DIR" 500; then + if [[ "$METRICS_ENABLED" == "true" ]]; then + metrics_backup_complete "failed" "Insufficient local disk space" + fi + send_notification "Karakeep Backup Failed" "Insufficient local disk space" "error" 0 "${#KARAKEEP_VOLUMES[@]}" + exit 1 + fi + + # Ensure containers are restarted on unexpected exit + trap 'if [[ "$containers_stopped" == "true" ]]; then log_warning "Restarting containers after unexpected exit..."; start_containers || true; fi' EXIT INT TERM + + # Stop containers for consistent snapshot + if [ "$STOP_CONTAINERS" == true ] && [ "$DRY_RUN" != true ]; then + if [[ "$METRICS_ENABLED" == "true" ]]; then + metrics_status_update "backing_up" "Stopping containers for consistent backup" + fi + if ! stop_containers; then + log_warning "Could not stop containers - proceeding with hot backup" + else + containers_stopped=true + fi + fi + + # Back up each volume + if [[ "$METRICS_ENABLED" == "true" ]]; then + metrics_status_update "backing_up" "Archiving Karakeep volumes" + fi + + local success_count=0 + local failed_count=0 + local backup_results=() + + for volume_name in "${!KARAKEEP_VOLUMES[@]}"; do + if backup_volume "$volume_name"; then + success_count=$((success_count + 1)) + backup_results+=("✓ $volume_name") + else + failed_count=$((failed_count + 1)) + backup_results+=("✗ $volume_name") + fi + done + + # Restart containers as soon as volumes are archived + if [ "$containers_stopped" == true ]; then + if ! start_containers; then + log_error "CRITICAL: Failed to restart Karakeep containers after backup" + send_notification "Karakeep Backup WARNING" "Containers failed to restart after backup - manual intervention required" "error" "$success_count" "$failed_count" + fi + containers_stopped=false + fi + + # Copy to NAS if available and not skipped + if [ "$SKIP_NAS" != true ] && [ "$DRY_RUN" != true ]; then + if [[ "$METRICS_ENABLED" == "true" ]]; then + metrics_status_update "backing_up" "Copying backup to NAS" + fi + if check_nas_mount; then + if ! copy_to_nas "$BACKUP_DEST"; then + log_warning "NAS copy failed - local backup is still available at $BACKUP_DEST" + fi + else + log_warning "NAS not available - backup retained locally at $BACKUP_DEST" + fi + fi + + # Calculate elapsed time + local script_end_time + script_end_time=$(date +%s) + local total_time=$((script_end_time - script_start_time)) + + track_performance "full_karakeep_backup" "$script_start_time" "$script_end_time" + + # Clean up old backups + if [ "$DRY_RUN" != true ]; then + if [[ "$METRICS_ENABLED" == "true" ]]; then + metrics_status_update "cleaning_up" "Removing old backup archives" + fi + cleanup_old_backups + fi + + # Generate summary + generate_summary_report "$success_count" "$failed_count" "$total_time" + + { + echo "## Backup Results" + for result in "${backup_results[@]}"; do + echo "- $result" + done + echo "" + echo "**Completed**: $(date '+%Y-%m-%d %H:%M:%S')" + echo "**Duration**: ${total_time}s" + } >> "$MARKDOWN_LOG" + + # Copy logs to NAS + if [ "$DRY_RUN" != true ]; then + copy_logs_to_nas + fi + + # Send notification + local status="success" + local message="Karakeep backup completed successfully (${success_count}/${#KARAKEEP_VOLUMES[@]} volumes)" + + if [ "$DRY_RUN" == true ]; then + message="Karakeep backup dry run completed" + status="info" + elif [ "$failed_count" -gt 0 ]; then + status="warning" + message="Karakeep backup completed with $failed_count failure(s)" + fi + + send_notification "Karakeep Backup Complete" "$message" "$status" "$success_count" "$failed_count" + + # Finalize metrics + if [[ "$METRICS_ENABLED" == "true" ]]; then + if [ "$failed_count" -gt 0 ]; then + metrics_backup_complete "completed_with_errors" "Karakeep backup completed with $failed_count failure(s)" + elif [ "$DRY_RUN" == true ]; then + metrics_backup_complete "success" "Karakeep backup dry run completed" + else + metrics_backup_complete "success" "Karakeep backup completed successfully" + fi + fi + + if [ "$failed_count" -gt 0 ]; then + exit 1 + fi + + log_success "All Karakeep volume backups completed successfully!" + exit 0 +} + +main "$@"