mirror of
https://github.com/acedanger/shell.git
synced 2025-12-06 03:20:12 -08:00
6d726cb015
- Created a base HTML template for consistent layout across pages. - Developed a dashboard page to display backup service metrics and statuses. - Implemented a log viewer for detailed log file inspection. - Added error handling page for better user experience during failures. - Introduced service detail page to show specific service metrics and actions. - Enhanced log filtering and viewing capabilities. - Integrated auto-refresh functionality for real-time updates on metrics. - Created integration and unit test scripts for backup metrics functionality.
850 lines
26 KiB
Bash
Executable File
850 lines
26 KiB
Bash
Executable File
#!/bin/bash
|
|
|
|
set -e
|
|
|
|
# Load the unified backup metrics library
|
|
SCRIPT_DIR="$(dirname "$(readlink -f "$0")")"
|
|
LIB_DIR="$SCRIPT_DIR/lib"
|
|
if [[ -f "$LIB_DIR/unified-backup-metrics.sh" ]]; then
|
|
# shellcheck source=lib/unified-backup-metrics.sh
|
|
source "$LIB_DIR/unified-backup-metrics.sh"
|
|
METRICS_ENABLED=true
|
|
else
|
|
echo "Warning: Unified backup metrics library not found at $LIB_DIR/unified-backup-metrics.sh"
|
|
METRICS_ENABLED=false
|
|
fi
|
|
|
|
# Color codes for output
|
|
RED='\033[0;31m'
|
|
GREEN='\033[0;32m'
|
|
YELLOW='\033[1;33m'
|
|
BLUE='\033[0;34m'
|
|
CYAN='\033[0;36m'
|
|
NC='\033[0m' # No Color
|
|
|
|
# Performance tracking variables
|
|
# Configuration
|
|
MAX_BACKUP_AGE_DAYS=30
|
|
MAX_BACKUPS_TO_KEEP=10
|
|
BACKUP_ROOT="/mnt/share/media/backups"
|
|
LOG_ROOT="/mnt/share/media/backups/logs"
|
|
SCRIPT_DIR="$(dirname "$(readlink -f "$0")")"
|
|
JSON_LOG_FILE="${SCRIPT_DIR}/logs/media-backup.json"
|
|
PERFORMANCE_LOG_FILE="${SCRIPT_DIR}/logs/media-backup-performance.json"
|
|
|
|
# Script options
|
|
PARALLEL_BACKUPS=true
|
|
VERIFY_BACKUPS=true
|
|
PERFORMANCE_MONITORING=true
|
|
WEBHOOK_URL="https://notify.peterwood.rocks/lab"
|
|
INTERACTIVE_MODE=false
|
|
DRY_RUN=false
|
|
|
|
# Show help function
|
|
show_help() {
|
|
cat << EOF
|
|
Media Services Backup Script
|
|
|
|
Usage: $0 [OPTIONS]
|
|
|
|
OPTIONS:
|
|
--dry-run Show what would be backed up without actually doing it
|
|
--no-verify Skip backup verification
|
|
--sequential Run backups sequentially instead of in parallel
|
|
--interactive Ask for confirmation before each backup
|
|
--webhook URL Custom webhook URL for notifications
|
|
-h, --help Show this help message
|
|
|
|
EXAMPLES:
|
|
$0 # Run full backup with default settings
|
|
$0 --dry-run # Preview what would be backed up
|
|
$0 --sequential # Run backups one at a time
|
|
$0 --no-verify # Skip verification for faster backup
|
|
|
|
SERVICES BACKED UP:
|
|
- Sonarr (TV Shows)
|
|
- Radarr (Movies)
|
|
- Prowlarr (Indexers)
|
|
- Audiobookshelf (Audiobooks)
|
|
- Tautulli (Plex Statistics)
|
|
- SABnzbd (Downloads)
|
|
- Jellyseerr (Requests)
|
|
|
|
EOF
|
|
}
|
|
|
|
# Parse command line arguments
|
|
while [[ $# -gt 0 ]]; do
|
|
case $1 in
|
|
--dry-run)
|
|
DRY_RUN=true
|
|
shift
|
|
;;
|
|
--no-verify)
|
|
VERIFY_BACKUPS=false
|
|
shift
|
|
;;
|
|
--sequential)
|
|
PARALLEL_BACKUPS=false
|
|
shift
|
|
;;
|
|
--interactive)
|
|
INTERACTIVE_MODE=true
|
|
shift
|
|
;;
|
|
--webhook)
|
|
WEBHOOK_URL="$2"
|
|
shift 2
|
|
;;
|
|
-h|--help)
|
|
show_help
|
|
exit 0
|
|
;;
|
|
*)
|
|
echo "Unknown option: $1"
|
|
show_help
|
|
exit 1
|
|
;;
|
|
esac
|
|
done
|
|
|
|
# Create necessary directories
|
|
mkdir -p "${SCRIPT_DIR}/logs"
|
|
mkdir -p "${BACKUP_ROOT}"/{sonarr,radarr,prowlarr,audiobookshelf,tautulli,sabnzbd,jellyseerr}
|
|
|
|
# Log files
|
|
LOG_FILE="${LOG_ROOT}/media-backup-$(date +%Y%m%d_%H%M%S).log"
|
|
MARKDOWN_LOG="${LOG_ROOT}/media-backup-$(date +%Y%m%d_%H%M%S).md"
|
|
|
|
# Define media services and their backup configurations
|
|
declare -A MEDIA_SERVICES=(
|
|
["sonarr"]="/config/Backups/scheduled"
|
|
["radarr"]="/config/Backups/scheduled"
|
|
["prowlarr"]="/config/Backups/scheduled"
|
|
["audiobookshelf"]="/metadata/backups"
|
|
["tautulli"]="/config/backups"
|
|
["sabnzbd"]="/config/sabnzbd.ini"
|
|
["jellyseerr_db"]="/config/db/"
|
|
["jellyseerr_settings"]="/config/settings.json"
|
|
)
|
|
|
|
# Service-specific backup destinations
|
|
declare -A BACKUP_DESTINATIONS=(
|
|
["sonarr"]="${BACKUP_ROOT}/sonarr/"
|
|
["radarr"]="${BACKUP_ROOT}/radarr/"
|
|
["prowlarr"]="${BACKUP_ROOT}/prowlarr/"
|
|
["audiobookshelf"]="${BACKUP_ROOT}/audiobookshelf/"
|
|
["tautulli"]="${BACKUP_ROOT}/tautulli/"
|
|
["sabnzbd"]="${BACKUP_ROOT}/sabnzbd/sabnzbd_$(date +%Y%m%d).ini"
|
|
["jellyseerr_db"]="${BACKUP_ROOT}/jellyseerr/backup_$(date +%Y%m%d)/"
|
|
["jellyseerr_settings"]="${BACKUP_ROOT}/jellyseerr/backup_$(date +%Y%m%d)/"
|
|
)
|
|
|
|
# Logging functions
|
|
log_message() {
|
|
local message="$1"
|
|
local timestamp
|
|
timestamp=$(date '+%Y-%m-%d %H:%M:%S')
|
|
echo -e "${CYAN}[${timestamp}]${NC} ${message}"
|
|
echo "[${timestamp}] $message" >> "${LOG_FILE}" 2>/dev/null || true
|
|
}
|
|
|
|
log_error() {
|
|
local message="$1"
|
|
local timestamp
|
|
timestamp=$(date '+%Y-%m-%d %H:%M:%S')
|
|
echo -e "${RED}[${timestamp}] ERROR:${NC} ${message}" >&2
|
|
echo "[${timestamp}] ERROR: $message" >> "${LOG_FILE}" 2>/dev/null || true
|
|
}
|
|
|
|
log_success() {
|
|
local message="$1"
|
|
local timestamp
|
|
timestamp=$(date '+%Y-%m-%d %H:%M:%S')
|
|
echo -e "${GREEN}[${timestamp}] SUCCESS:${NC} ${message}"
|
|
echo "[${timestamp}] SUCCESS: $message" >> "${LOG_FILE}" 2>/dev/null || true
|
|
}
|
|
|
|
log_warning() {
|
|
local message="$1"
|
|
local timestamp
|
|
timestamp=$(date '+%Y-%m-%d %H:%M:%S')
|
|
echo -e "${YELLOW}[${timestamp}] WARNING:${NC} ${message}"
|
|
echo "[${timestamp}] WARNING: $message" >> "${LOG_FILE}" 2>/dev/null || true
|
|
}
|
|
|
|
log_info() {
|
|
local message="$1"
|
|
local timestamp
|
|
timestamp=$(date '+%Y-%m-%d %H:%M:%S')
|
|
echo -e "${BLUE}[${timestamp}] INFO:${NC} ${message}"
|
|
echo "[${timestamp}] INFO: $message" >> "${LOG_FILE}" 2>/dev/null || true
|
|
}
|
|
|
|
# Performance tracking functions
|
|
track_performance() {
|
|
if [ "$PERFORMANCE_MONITORING" != true ]; then
|
|
return 0
|
|
fi
|
|
|
|
local operation="$1"
|
|
local start_time="$2"
|
|
local end_time="${3:-$(date +%s)}"
|
|
local duration=$((end_time - start_time))
|
|
|
|
# Initialize performance log if it doesn't exist
|
|
if [ ! -f "$PERFORMANCE_LOG_FILE" ]; then
|
|
echo "[]" > "$PERFORMANCE_LOG_FILE"
|
|
fi
|
|
|
|
# Add performance entry with lock protection
|
|
local entry
|
|
entry=$(jq -n \
|
|
--arg timestamp "$(date -Iseconds)" \
|
|
--arg operation "$operation" \
|
|
--arg duration "$duration" \
|
|
--arg hostname "$(hostname)" \
|
|
'{
|
|
timestamp: $timestamp,
|
|
operation: $operation,
|
|
duration: ($duration | tonumber),
|
|
hostname: $hostname
|
|
}')
|
|
|
|
if command -v jq > /dev/null 2>&1; then
|
|
local lock_file="${PERFORMANCE_LOG_FILE}.lock"
|
|
local max_wait=10
|
|
local wait_count=0
|
|
|
|
while [ $wait_count -lt $max_wait ]; do
|
|
if (set -C; echo $$ > "$lock_file") 2>/dev/null; then
|
|
break
|
|
fi
|
|
sleep 0.1
|
|
((wait_count++))
|
|
done
|
|
|
|
if [ $wait_count -lt $max_wait ]; then
|
|
if jq --argjson entry "$entry" '. += [$entry]' "$PERFORMANCE_LOG_FILE" > "${PERFORMANCE_LOG_FILE}.tmp" 2>/dev/null; then
|
|
mv "${PERFORMANCE_LOG_FILE}.tmp" "$PERFORMANCE_LOG_FILE"
|
|
else
|
|
rm -f "${PERFORMANCE_LOG_FILE}.tmp"
|
|
fi
|
|
rm -f "$lock_file"
|
|
fi
|
|
fi
|
|
|
|
log_info "Performance: $operation completed in ${duration}s"
|
|
}
|
|
|
|
# Initialize JSON log file
|
|
initialize_json_log() {
|
|
if [ ! -f "${JSON_LOG_FILE}" ] || ! jq empty "${JSON_LOG_FILE}" 2>/dev/null; then
|
|
echo "{}" > "${JSON_LOG_FILE}"
|
|
log_message "Initialized JSON log file"
|
|
fi
|
|
}
|
|
|
|
# Enhanced function to log file details with markdown formatting
|
|
log_file_details() {
|
|
local service="$1"
|
|
local src="$2"
|
|
local dest="$3"
|
|
local status="$4"
|
|
local size=""
|
|
local checksum=""
|
|
|
|
# Calculate size if backup was successful
|
|
if [ "$status" == "SUCCESS" ] && [ -e "$dest" ]; then
|
|
size=$(du -sh "$dest" 2>/dev/null | cut -f1 || echo "Unknown")
|
|
if [ "$VERIFY_BACKUPS" == true ]; then
|
|
checksum=$(find "$dest" -type f -exec md5sum {} \; 2>/dev/null | md5sum | cut -d' ' -f1 || echo "N/A")
|
|
fi
|
|
else
|
|
size="N/A"
|
|
checksum="N/A"
|
|
fi
|
|
|
|
# Use a lock file for markdown log to prevent race conditions
|
|
local markdown_lock="${MARKDOWN_LOG}.lock"
|
|
local max_wait=30
|
|
local wait_count=0
|
|
|
|
while [ $wait_count -lt $max_wait ]; do
|
|
if (set -C; echo $$ > "$markdown_lock") 2>/dev/null; then
|
|
break
|
|
fi
|
|
sleep 0.1
|
|
((wait_count++))
|
|
done
|
|
|
|
if [ $wait_count -lt $max_wait ]; then
|
|
# Log to markdown file safely
|
|
{
|
|
echo "## $service Backup"
|
|
echo "- **Status**: $status"
|
|
echo "- **Source**: \$($src\)"
|
|
echo "- **Destination**: \$($dest\)"
|
|
echo "- **Size**: $size"
|
|
echo "- **Checksum**: $checksum"
|
|
echo "- **Timestamp**: $(date '+%Y-%m-%d %H:%M:%S')"
|
|
echo ""
|
|
} >> "$MARKDOWN_LOG"
|
|
|
|
rm -f "$markdown_lock"
|
|
else
|
|
log_warning "Could not acquire markdown log lock for $service"
|
|
fi
|
|
|
|
# Log to JSON
|
|
if command -v jq > /dev/null 2>&1; then
|
|
update_backup_log "$service" "$src" "$dest" "$status" "$size" "$checksum"
|
|
fi
|
|
}
|
|
|
|
# Update backup log in JSON format
|
|
update_backup_log() {
|
|
local service="$1"
|
|
local src="$2"
|
|
local dest="$3"
|
|
local status="$4"
|
|
local size="$5"
|
|
local checksum="$6"
|
|
local timestamp
|
|
timestamp=$(date -Iseconds)
|
|
|
|
if ! command -v jq > /dev/null 2>&1; then
|
|
return 0
|
|
fi
|
|
|
|
# Use a lock file for parallel safety
|
|
local lock_file="${JSON_LOG_FILE}.lock"
|
|
local max_wait=30
|
|
local wait_count=0
|
|
|
|
while [ $wait_count -lt $max_wait ]; do
|
|
if (set -C; echo $$ > "$lock_file") 2>/dev/null; then
|
|
break
|
|
fi
|
|
sleep 0.1
|
|
((wait_count++))
|
|
done
|
|
|
|
if [ $wait_count -ge $max_wait ]; then
|
|
log_warning "Could not acquire lock for JSON log update"
|
|
return 1
|
|
fi
|
|
|
|
# Create entry for this backup
|
|
local entry
|
|
entry=$(jq -n \
|
|
--arg service "$service" \
|
|
--arg src "$src" \
|
|
--arg dest "$dest" \
|
|
--arg status "$status" \
|
|
--arg size "$size" \
|
|
--arg checksum "$checksum" \
|
|
--arg timestamp "$timestamp" \
|
|
'{
|
|
service: $service,
|
|
source: $src,
|
|
destination: $dest,
|
|
status: $status,
|
|
size: $size,
|
|
checksum: $checksum,
|
|
timestamp: $timestamp
|
|
}')
|
|
|
|
# Update JSON log safely
|
|
if jq --argjson entry "$entry" --arg service "$service" \
|
|
'.[$service] = $entry' "$JSON_LOG_FILE" > "${JSON_LOG_FILE}.tmp" 2>/dev/null; then
|
|
mv "${JSON_LOG_FILE}.tmp" "$JSON_LOG_FILE"
|
|
else
|
|
rm -f "${JSON_LOG_FILE}.tmp"
|
|
fi
|
|
|
|
# Remove lock file
|
|
rm -f "$lock_file"
|
|
}
|
|
|
|
# Check if Docker container is running
|
|
check_container_running() {
|
|
local container="$1"
|
|
|
|
if ! docker ps --format "table {{.Names}}" | grep -q "^${container}$"; then
|
|
log_warning "Container '$container' is not running"
|
|
return 1
|
|
fi
|
|
|
|
return 0
|
|
}
|
|
|
|
# Verify backup integrity
|
|
verify_backup() {
|
|
local src_container="$1"
|
|
local src_path="$2"
|
|
local dest_path="$3"
|
|
|
|
if [ "$VERIFY_BACKUPS" != true ]; then
|
|
return 0
|
|
fi
|
|
|
|
log_info "Verifying backup integrity for $src_container:$src_path"
|
|
|
|
# For files, compare checksums
|
|
if [[ "$src_path" == *.ini ]] || [[ "$src_path" == *.json ]]; then
|
|
local src_checksum
|
|
local dest_checksum
|
|
src_checksum=$(docker exec "$src_container" md5sum "$src_path" 2>/dev/null | cut -d' ' -f1 || echo "")
|
|
dest_checksum=$(md5sum "$dest_path" 2>/dev/null | cut -d' ' -f1 || echo "")
|
|
|
|
if [ -n "$src_checksum" ] && [ -n "$dest_checksum" ] && [ "$src_checksum" == "$dest_checksum" ]; then
|
|
log_success "Backup verification passed for $src_container:$src_path"
|
|
return 0
|
|
else
|
|
log_error "Backup verification failed for $src_container:$src_path"
|
|
return 1
|
|
fi
|
|
fi
|
|
|
|
# For directories, check if they exist and have content
|
|
if [ -d "$dest_path" ]; then
|
|
local file_count
|
|
file_count=$(find "$dest_path" -type f 2>/dev/null | wc -l)
|
|
if [ "$file_count" -gt 0 ]; then
|
|
log_success "Backup verification passed for $src_container:$src_path ($file_count files)"
|
|
return 0
|
|
else
|
|
log_error "Backup verification failed: no files found in $dest_path"
|
|
return 1
|
|
fi
|
|
fi
|
|
|
|
log_warning "Unable to verify backup for $src_container:$src_path"
|
|
return 0
|
|
}
|
|
|
|
# Backup a single service
|
|
backup_service() {
|
|
local service="$1"
|
|
local container="$1"
|
|
local backup_start_time
|
|
backup_start_time=$(date +%s)
|
|
|
|
log_message "Starting backup for service: $service"
|
|
|
|
# Handle special cases for container names
|
|
case "$service" in
|
|
jellyseerr_db|jellyseerr_settings)
|
|
container="jellyseerr"
|
|
;;
|
|
esac
|
|
|
|
# Check if container is running
|
|
if ! check_container_running "$container"; then
|
|
log_file_details "$service" "${container}:${MEDIA_SERVICES[$service]}" "${BACKUP_DESTINATIONS[$service]}" "FAILED - Container not running"
|
|
return 1
|
|
fi
|
|
|
|
local src_path="${MEDIA_SERVICES[$service]}"
|
|
local dest_path="${BACKUP_DESTINATIONS[$service]}"
|
|
|
|
# Create destination directory for jellyseerr
|
|
if [[ "$service" == jellyseerr_* ]]; then
|
|
mkdir -p "$(dirname "$dest_path")"
|
|
fi
|
|
|
|
# Perform the backup
|
|
if [ "$DRY_RUN" == true ]; then
|
|
log_info "DRY RUN: Would backup $container:$src_path to $dest_path"
|
|
log_file_details "$service" "$container:$src_path" "$dest_path" "DRY RUN"
|
|
return 0
|
|
fi
|
|
|
|
if [ "$INTERACTIVE_MODE" == true ]; then
|
|
echo -n "Backup $service? (y/N): "
|
|
read -r response
|
|
if [[ ! "$response" =~ ^[Yy]$ ]]; then
|
|
log_info "Skipping $service backup (user choice)"
|
|
return 0
|
|
fi
|
|
fi
|
|
|
|
# Execute docker cp command
|
|
local docker_cmd="docker cp $container:$src_path $dest_path"
|
|
log_info "Executing: $docker_cmd"
|
|
|
|
if $docker_cmd 2>&1 | tee -a "$LOG_FILE"; then
|
|
log_success "Backup completed for $service"
|
|
|
|
# File-level metrics tracking (success)
|
|
if [[ "$METRICS_ENABLED" == "true" ]]; then
|
|
local file_size checksum
|
|
if [ -f "$dest_path" ]; then
|
|
file_size=$(stat -c%s "$dest_path" 2>/dev/null || echo "0")
|
|
checksum=$(md5sum "$dest_path" 2>/dev/null | cut -d' ' -f1 || echo "")
|
|
metrics_add_file "$dest_path" "success" "$file_size" "$checksum"
|
|
elif [ -d "$dest_path" ]; then
|
|
# For directories, sum file sizes and add one entry for the directory
|
|
file_size=$(find "$dest_path" -type f -exec stat -c%s {} + 2>/dev/null | awk '{s+=$1} END {print s}' || echo "0")
|
|
metrics_add_file "$dest_path" "success" "$file_size"
|
|
fi
|
|
fi
|
|
|
|
# Verify the backup
|
|
if verify_backup "$container" "$src_path" "$dest_path"; then
|
|
log_file_details "$service" "$container:$src_path" "$dest_path" "SUCCESS"
|
|
track_performance "backup_${service}" "$backup_start_time"
|
|
return 0
|
|
else
|
|
log_file_details "$service" "$container:$src_path" "$dest_path" "VERIFICATION_FAILED"
|
|
# File-level metrics tracking (verification failed)
|
|
if [[ "$METRICS_ENABLED" == "true" ]]; then
|
|
local file_size
|
|
if [ -f "$dest_path" ]; then
|
|
file_size=$(stat -c%s "$dest_path" 2>/dev/null || echo "0")
|
|
metrics_add_file "$dest_path" "failed" "$file_size" "" "Verification failed"
|
|
elif [ -d "$dest_path" ]; then
|
|
file_size=$(find "$dest_path" -type f -exec stat -c%s {} + 2>/dev/null | awk '{s+=$1} END {print s}' || echo "0")
|
|
metrics_add_file "$dest_path" "failed" "$file_size" "" "Verification failed"
|
|
fi
|
|
fi
|
|
return 1
|
|
fi
|
|
else
|
|
log_error "Backup failed for $service"
|
|
log_file_details "$service" "$container:$src_path" "$dest_path" "FAILED"
|
|
# File-level metrics tracking (backup failed)
|
|
if [[ "$METRICS_ENABLED" == "true" ]]; then
|
|
local file_size
|
|
if [ -f "$dest_path" ]; then
|
|
file_size=$(stat -c%s "$dest_path" 2>/dev/null || echo "0")
|
|
metrics_add_file "$dest_path" "failed" "$file_size" "" "Backup failed"
|
|
elif [ -d "$dest_path" ]; then
|
|
file_size=$(find "$dest_path" -type f -exec stat -c%s {} + 2>/dev/null | awk '{s+=$1} END {print s}' || echo "0")
|
|
metrics_add_file "$dest_path" "failed" "$file_size" "" "Backup failed"
|
|
fi
|
|
fi
|
|
return 1
|
|
fi
|
|
}
|
|
|
|
# Backup service wrapper for parallel execution
|
|
backup_service_wrapper() {
|
|
local service="$1"
|
|
local temp_file="$2"
|
|
|
|
if backup_service "$service"; then
|
|
echo "SUCCESS:$service" >> "$temp_file"
|
|
else
|
|
echo "FAILED:$service" >> "$temp_file"
|
|
fi
|
|
}
|
|
|
|
# Clean old backups based on age and count
|
|
cleanup_old_backups() {
|
|
log_message "Cleaning up old backups..."
|
|
|
|
for service_dir in "${BACKUP_ROOT}"/*; do
|
|
if [ ! -d "$service_dir" ]; then
|
|
continue
|
|
fi
|
|
|
|
local service
|
|
service=$(basename "$service_dir")
|
|
log_info "Cleaning up old backups for $service"
|
|
|
|
# Remove backups older than MAX_BACKUP_AGE_DAYS
|
|
find "$service_dir" -type f -mtime +${MAX_BACKUP_AGE_DAYS} -delete 2>/dev/null || true
|
|
find "$service_dir" -type d -empty -mtime +${MAX_BACKUP_AGE_DAYS} -delete 2>/dev/null || true
|
|
|
|
# Keep only the most recent MAX_BACKUPS_TO_KEEP backups
|
|
find "$service_dir" -type f -name "*.ini" -o -name "*.json" | sort -r | tail -n +$((MAX_BACKUPS_TO_KEEP + 1)) | xargs rm -f 2>/dev/null || true
|
|
|
|
# Clean up old dated directories (for jellyseerr)
|
|
find "$service_dir" -type d -name "backup_*" | sort -r | tail -n +$((MAX_BACKUPS_TO_KEEP + 1)) | xargs rm -rf 2>/dev/null || true
|
|
done
|
|
|
|
# Clean up old log files
|
|
find "$LOG_ROOT" -name "media-backup-*.log" -mtime +${MAX_BACKUP_AGE_DAYS} -delete 2>/dev/null || true
|
|
find "$LOG_ROOT" -name "media-backup-*.md" -mtime +${MAX_BACKUP_AGE_DAYS} -delete 2>/dev/null || true
|
|
|
|
log_success "Cleanup completed"
|
|
}
|
|
|
|
# Check disk space
|
|
check_disk_space() {
|
|
local required_space_mb=1000 # Minimum 1GB free space
|
|
|
|
local available_space_kb
|
|
available_space_kb=$(df "$BACKUP_ROOT" | awk 'NR==2 {print $4}')
|
|
local available_space_mb=$((available_space_kb / 1024))
|
|
|
|
if [ "$available_space_mb" -lt "$required_space_mb" ]; then
|
|
log_error "Insufficient disk space. Available: ${available_space_mb}MB, Required: ${required_space_mb}MB"
|
|
return 1
|
|
fi
|
|
|
|
log_info "Disk space check passed. Available: ${available_space_mb}MB"
|
|
return 0
|
|
}
|
|
|
|
# Send enhanced notification
|
|
send_notification() {
|
|
local title="$1"
|
|
local message="$2"
|
|
local status="${3:-info}"
|
|
local hostname
|
|
hostname=$(hostname)
|
|
local total_services=${#MEDIA_SERVICES[@]}
|
|
local success_count="$4"
|
|
local failed_count="$5"
|
|
|
|
# Enhanced message with statistics
|
|
local enhanced_message="$message\n\nServices: $total_services\nSuccessful: $success_count\nFailed: $failed_count\nHost: $hostname"
|
|
|
|
# Console notification
|
|
case "$status" in
|
|
"success") log_success "$title: $enhanced_message" ;;
|
|
"error") log_error "$title: $enhanced_message" ;;
|
|
"warning") log_warning "$title: $enhanced_message" ;;
|
|
*) log_info "$title: $enhanced_message" ;;
|
|
esac
|
|
|
|
# Webhook notification
|
|
if [ -n "$WEBHOOK_URL" ] && [ "$DRY_RUN" != true ]; then
|
|
local tags="backup,media,${hostname}"
|
|
[ "$failed_count" -gt 0 ] && tags="${tags},errors"
|
|
|
|
curl -s \
|
|
-H "tags:${tags}" \
|
|
-d "$enhanced_message" \
|
|
"$WEBHOOK_URL" 2>/dev/null || log_warning "Failed to send webhook notification"
|
|
fi
|
|
}
|
|
|
|
# Generate backup summary report
|
|
generate_summary_report() {
|
|
local success_count="$1"
|
|
local failed_count="$2"
|
|
local total_time="$3"
|
|
|
|
log_message "=== BACKUP SUMMARY REPORT ==="
|
|
log_message "Total Services: ${#MEDIA_SERVICES[@]}"
|
|
log_message "Successful Backups: $success_count"
|
|
log_message "Failed Backups: $failed_count"
|
|
log_message "Total Time: ${total_time}s"
|
|
log_message "Log File: $LOG_FILE"
|
|
log_message "Markdown Report: $MARKDOWN_LOG"
|
|
|
|
if [ "$PERFORMANCE_MONITORING" == true ]; then
|
|
log_message "Performance Log: $PERFORMANCE_LOG_FILE"
|
|
fi
|
|
|
|
# Add summary to markdown log
|
|
{
|
|
echo "# Media Backup Summary Report"
|
|
echo "**Date**: $(date '+%Y-%m-%d %H:%M:%S')"
|
|
echo "**Host**: $(hostname)"
|
|
echo "**Total Services**: ${#MEDIA_SERVICES[@]}"
|
|
echo "**Successful**: $success_count"
|
|
echo "**Failed**: $failed_count"
|
|
echo "**Duration**: ${total_time}s"
|
|
echo ""
|
|
} >> "$MARKDOWN_LOG"
|
|
}
|
|
|
|
# Main backup execution function
|
|
main() {
|
|
local script_start_time
|
|
script_start_time=$(date +%s)
|
|
|
|
log_message "=== MEDIA SERVICES BACKUP STARTED ==="
|
|
log_message "Host: $(hostname)"
|
|
log_message "Timestamp: $(date '+%Y-%m-%d %H:%M:%S')"
|
|
log_message "Dry Run: $DRY_RUN"
|
|
log_message "Parallel Mode: $PARALLEL_BACKUPS"
|
|
log_message "Verify Backups: $VERIFY_BACKUPS"
|
|
|
|
# Initialize metrics if enabled
|
|
if [[ "$METRICS_ENABLED" == "true" ]]; then
|
|
metrics_backup_start "media-services" "Media services backup (Sonarr, Radarr, etc.)" "$BACKUP_ROOT"
|
|
metrics_status_update "initializing" "Preparing media services backup"
|
|
fi
|
|
|
|
# Initialize logging
|
|
initialize_json_log
|
|
|
|
# Initialize markdown log
|
|
{
|
|
echo "# Media Services Backup Report"
|
|
echo "**Started**: $(date '+%Y-%m-%d %H:%M:%S')"
|
|
echo "**Host**: $(hostname)"
|
|
echo ""
|
|
} > "$MARKDOWN_LOG"
|
|
|
|
# Update metrics for pre-flight checks
|
|
if [[ "$METRICS_ENABLED" == "true" ]]; then
|
|
metrics_status_update "checking" "Running pre-flight checks"
|
|
fi
|
|
|
|
# Pre-flight checks
|
|
if ! check_disk_space; then
|
|
if [[ "$METRICS_ENABLED" == "true" ]]; then
|
|
metrics_backup_complete "failed" "Insufficient disk space"
|
|
fi
|
|
send_notification "Media Backup Failed" "Insufficient disk space" "error" 0 1
|
|
exit 1
|
|
fi
|
|
|
|
# Check if Docker is running
|
|
if ! docker info >/dev/null 2>&1; then
|
|
log_error "Docker is not running or accessible"
|
|
if [[ "$METRICS_ENABLED" == "true" ]]; then
|
|
metrics_backup_complete "failed" "Docker is not accessible"
|
|
fi
|
|
send_notification "Media Backup Failed" "Docker is not accessible" "error" 0 1
|
|
exit 1
|
|
fi
|
|
|
|
local success_count=0
|
|
local failed_count=0
|
|
local backup_results=()
|
|
|
|
if [ "$PARALLEL_BACKUPS" == true ]; then
|
|
log_message "Running backups in parallel mode"
|
|
|
|
# Update metrics for parallel backup phase
|
|
if [[ "$METRICS_ENABLED" == "true" ]]; then
|
|
metrics_status_update "backing_up" "Running media service backups in parallel"
|
|
fi
|
|
|
|
# Create temporary file for collecting results
|
|
local temp_results
|
|
temp_results=$(mktemp)
|
|
local pids=()
|
|
|
|
# Start backup jobs in parallel
|
|
for service in "${!MEDIA_SERVICES[@]}"; do
|
|
backup_service_wrapper "$service" "$temp_results" &
|
|
pids+=($!)
|
|
log_info "Started backup job for $service (PID: $!)"
|
|
done
|
|
|
|
# Wait for all jobs to complete
|
|
for pid in "${pids[@]}"; do
|
|
wait "$pid"
|
|
log_info "Backup job completed (PID: $pid)"
|
|
done
|
|
|
|
# Collect results
|
|
while IFS= read -r result; do
|
|
if [[ "$result" == SUCCESS:* ]]; then
|
|
((success_count++))
|
|
backup_results+=("✓ ${result#SUCCESS:}")
|
|
elif [[ "$result" == FAILED:* ]]; then
|
|
((failed_count++))
|
|
backup_results+=("✗ ${result#FAILED:}")
|
|
fi
|
|
done < "$temp_results"
|
|
|
|
rm -f "$temp_results"
|
|
|
|
else
|
|
log_message "Running backups in sequential mode"
|
|
|
|
# Update metrics for sequential backup phase
|
|
if [[ "$METRICS_ENABLED" == "true" ]]; then
|
|
metrics_status_update "backing_up" "Running media service backups sequentially"
|
|
fi
|
|
|
|
# Run backups sequentially
|
|
for service in "${!MEDIA_SERVICES[@]}"; do
|
|
if backup_service "$service"; then
|
|
((success_count++))
|
|
backup_results+=("✓ $service")
|
|
else
|
|
((failed_count++))
|
|
backup_results+=("✗ $service")
|
|
fi
|
|
done
|
|
fi
|
|
|
|
# Calculate total time
|
|
local script_end_time
|
|
script_end_time=$(date +%s)
|
|
local total_time=$((script_end_time - script_start_time))
|
|
|
|
# Track overall performance
|
|
track_performance "full_media_backup" "$script_start_time" "$script_end_time"
|
|
|
|
# Update metrics for cleanup phase
|
|
if [[ "$METRICS_ENABLED" == "true" ]]; then
|
|
if [ "$DRY_RUN" != true ]; then
|
|
metrics_status_update "cleaning_up" "Cleaning up old backup files"
|
|
else
|
|
metrics_status_update "completed" "Dry run completed successfully"
|
|
fi
|
|
fi
|
|
|
|
# Clean up old backups (only if not dry run)
|
|
if [ "$DRY_RUN" != true ]; then
|
|
cleanup_old_backups
|
|
fi
|
|
|
|
# Generate summary report
|
|
generate_summary_report "$success_count" "$failed_count" "$total_time"
|
|
|
|
# Add results to markdown log
|
|
{
|
|
echo "## Backup Results"
|
|
for result in "${backup_results[@]}"; do
|
|
echo "- $result"
|
|
done
|
|
echo ""
|
|
echo "**Completed**: $(date '+%Y-%m-%d %H:%M:%S')"
|
|
echo "**Duration**: ${total_time}s"
|
|
} >> "$MARKDOWN_LOG"
|
|
|
|
# Send notification
|
|
local status="success"
|
|
local message="Media backup completed"
|
|
|
|
if [ "$failed_count" -gt 0 ]; then
|
|
status="warning"
|
|
message="Media backup completed with $failed_count failures"
|
|
fi
|
|
|
|
if [ "$DRY_RUN" == true ]; then
|
|
message="Media backup dry run completed"
|
|
status="info"
|
|
fi
|
|
|
|
send_notification "Media Backup Complete" "$message" "$status" "$success_count" "$failed_count"
|
|
|
|
# Finalize metrics
|
|
if [[ "$METRICS_ENABLED" == "true" ]]; then
|
|
if [ "$failed_count" -gt 0 ]; then
|
|
metrics_backup_complete "completed_with_errors" "Media backup completed with $failed_count failures"
|
|
elif [ "$DRY_RUN" == true ]; then
|
|
metrics_backup_complete "success" "Media backup dry run completed successfully"
|
|
else
|
|
metrics_backup_complete "success" "Media backup completed successfully"
|
|
fi
|
|
fi
|
|
|
|
# Exit with error code if any backups failed
|
|
if [ "$failed_count" -gt 0 ]; then
|
|
exit 1
|
|
fi
|
|
|
|
log_success "All media backups completed successfully!"
|
|
exit 0
|
|
}
|
|
|
|
# Trap to handle script interruption
|
|
trap 'log_error "Script interrupted"; exit 130' INT TERM
|
|
|
|
# Run main function
|
|
main "$@"
|