mirror of
https://github.com/acedanger/shell.git
synced 2025-12-06 04:30:13 -08:00
6d726cb015
- Created a base HTML template for consistent layout across pages. - Developed a dashboard page to display backup service metrics and statuses. - Implemented a log viewer for detailed log file inspection. - Added error handling page for better user experience during failures. - Introduced service detail page to show specific service metrics and actions. - Enhanced log filtering and viewing capabilities. - Integrated auto-refresh functionality for real-time updates on metrics. - Created integration and unit test scripts for backup metrics functionality.
338 lines
10 KiB
Bash
Executable File
338 lines
10 KiB
Bash
Executable File
#!/bin/bash
|
|
|
|
# backup-docker.sh - Comprehensive Docker volumes backup script
|
|
# Author: Shell Repository
|
|
# Description: Backup Docker container volumes with proper error handling, logging, and metrics
|
|
|
|
set -e
|
|
|
|
# Load the unified backup metrics library
|
|
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
|
LIB_DIR="$SCRIPT_DIR/lib"
|
|
if [[ -f "$LIB_DIR/unified-backup-metrics.sh" ]]; then
|
|
# shellcheck source=lib/unified-backup-metrics.sh
|
|
source "$LIB_DIR/unified-backup-metrics.sh"
|
|
METRICS_ENABLED=true
|
|
else
|
|
echo "Warning: Unified backup metrics library not found at $LIB_DIR/unified-backup-metrics.sh"
|
|
METRICS_ENABLED=false
|
|
fi
|
|
|
|
# Colors for output
|
|
GREEN='\033[0;32m'
|
|
YELLOW='\033[0;33m'
|
|
RED='\033[0;31m'
|
|
BLUE='\033[0;34m'
|
|
NC='\033[0m' # No Color
|
|
|
|
# Configuration
|
|
BACKUP_ROOT="/home/acedanger/backup/docker-data"
|
|
LOG_FILE="$SCRIPT_DIR/logs/docker-backup.log"
|
|
NOTIFICATION_URL="https://notify.peterwood.rocks/lab"
|
|
|
|
# Container definitions: container_name:volume_path:description
|
|
declare -A CONTAINERS=(
|
|
["vaultwarden"]="/var/lib/docker/volumes/vaultwarden_data/_data:Password manager data"
|
|
["uptime-kuma"]="/var/lib/docker/volumes/uptime-kuma/_data:Uptime monitoring data"
|
|
# ["paperless-ng"]="/var/lib/docker/volumes/paperless-ng_data/_data:Document management data"
|
|
# ["paperless-media"]="/var/lib/docker/volumes/paperless-ng_media/_data:Document media files"
|
|
# ["paperless-pgdata"]="/var/lib/docker/volumes/paperless-ng_pgdata/_data:PostgreSQL database"
|
|
)
|
|
|
|
# Ensure directories exist
|
|
mkdir -p "$(dirname "$LOG_FILE")"
|
|
mkdir -p "$BACKUP_ROOT"
|
|
|
|
# Logging function
|
|
log() {
|
|
echo "$(date '+%Y-%m-%d %H:%M:%S') - $1" | tee -a "$LOG_FILE"
|
|
}
|
|
|
|
# Cleanup function for metrics finalization
|
|
cleanup() {
|
|
if [[ "$METRICS_ENABLED" == "true" ]]; then
|
|
if [[ -n "$1" && "$1" == "error" ]]; then
|
|
metrics_backup_complete "failed" "Docker backup failed during execution"
|
|
else
|
|
metrics_backup_complete "success" "Docker volumes backup completed successfully"
|
|
fi
|
|
fi
|
|
}
|
|
|
|
# Set up cleanup trap
|
|
trap 'cleanup error' ERR
|
|
|
|
# Check if container is running
|
|
check_container_running() {
|
|
local container="$1"
|
|
if docker ps --format "table {{.Names}}" | grep -q "^${container}$"; then
|
|
return 0
|
|
else
|
|
return 1
|
|
fi
|
|
}
|
|
|
|
# Stop container safely
|
|
stop_container() {
|
|
local container="$1"
|
|
|
|
log "Stopping container: $container"
|
|
if [[ "$METRICS_ENABLED" == "true" ]]; then
|
|
metrics_status_update "stopping_service" "Stopping container: $container"
|
|
fi
|
|
|
|
if ! docker stop "$container" >/dev/null 2>&1; then
|
|
log "Warning: Failed to stop container $container or container not running"
|
|
return 1
|
|
fi
|
|
|
|
# Wait for container to fully stop
|
|
local max_wait=30
|
|
local wait_count=0
|
|
while [ $wait_count -lt $max_wait ]; do
|
|
if ! docker ps -q --filter "name=$container" | grep -q .; then
|
|
log "Container $container stopped successfully"
|
|
return 0
|
|
fi
|
|
wait_count=$((wait_count + 1))
|
|
sleep 1
|
|
done
|
|
|
|
log "Warning: Container $container may not have stopped completely"
|
|
return 1
|
|
}
|
|
|
|
# Start container safely
|
|
start_container() {
|
|
local container="$1"
|
|
|
|
log "Starting container: $container"
|
|
if [[ "$METRICS_ENABLED" == "true" ]]; then
|
|
metrics_status_update "starting_service" "Starting container: $container"
|
|
fi
|
|
|
|
if ! docker start "$container" >/dev/null 2>&1; then
|
|
log "Error: Failed to start container $container"
|
|
return 1
|
|
fi
|
|
|
|
# Wait for container to be running
|
|
local max_wait=30
|
|
local wait_count=0
|
|
while [ $wait_count -lt $max_wait ]; do
|
|
if docker ps -q --filter "name=$container" | grep -q .; then
|
|
log "Container $container started successfully"
|
|
return 0
|
|
fi
|
|
wait_count=$((wait_count + 1))
|
|
sleep 1
|
|
done
|
|
|
|
log "Warning: Container $container may not have started properly"
|
|
return 1
|
|
}
|
|
|
|
# Backup container volume
|
|
backup_container_volume() {
|
|
local container="$1"
|
|
local volume_path="$2"
|
|
local description="$3"
|
|
local backup_file="$BACKUP_ROOT/${container}-data-bk-$(date +%Y%m%d).tar.gz"
|
|
|
|
log "Starting backup for $container ($description)"
|
|
|
|
# Check if volume path exists
|
|
if [ ! -d "$volume_path" ]; then
|
|
log "Error: Volume path does not exist: $volume_path"
|
|
return 1
|
|
fi
|
|
|
|
# Check if container was running
|
|
local was_running=false
|
|
if check_container_running "$container"; then
|
|
was_running=true
|
|
if ! stop_container "$container"; then
|
|
log "Error: Failed to stop container $container"
|
|
return 1
|
|
fi
|
|
else
|
|
log "Container $container is not running, proceeding with backup"
|
|
fi
|
|
|
|
# Create backup
|
|
log "Creating backup archive: $(basename "$backup_file")"
|
|
if [[ "$METRICS_ENABLED" == "true" ]]; then
|
|
metrics_status_update "backing_up" "Creating archive for $container"
|
|
fi
|
|
|
|
if tar -czf "$backup_file" -C "$(dirname "$volume_path")" "$(basename "$volume_path")" 2>/dev/null; then
|
|
local backup_size
|
|
backup_size=$(du -h "$backup_file" | cut -f1)
|
|
log "Backup completed successfully: $(basename "$backup_file") ($backup_size)"
|
|
|
|
# Track file completion in metrics
|
|
if [[ "$METRICS_ENABLED" == "true" ]]; then
|
|
local file_size_bytes
|
|
file_size_bytes=$(stat -c%s "$backup_file" 2>/dev/null || echo "0")
|
|
metrics_file_backup_complete "$(basename "$backup_file")" "$file_size_bytes" "created"
|
|
fi
|
|
else
|
|
log "Error: Failed to create backup for $container"
|
|
# Try to restart container even if backup failed
|
|
if [ "$was_running" = true ]; then
|
|
start_container "$container" || true
|
|
fi
|
|
return 1
|
|
fi
|
|
|
|
# Restart container if it was running
|
|
if [ "$was_running" = true ]; then
|
|
if ! start_container "$container"; then
|
|
log "Error: Failed to restart container $container after backup"
|
|
return 1
|
|
fi
|
|
fi
|
|
|
|
return 0
|
|
}
|
|
|
|
# Send notification
|
|
send_notification() {
|
|
local status="$1"
|
|
local message="$2"
|
|
local failed_containers="$3"
|
|
|
|
local tags="backup,docker,${HOSTNAME}"
|
|
local priority="default"
|
|
|
|
if [ "$status" = "failed" ]; then
|
|
priority="high"
|
|
tags="${tags},error"
|
|
fi
|
|
|
|
# Add successful container names to tags
|
|
for container in "${!CONTAINERS[@]}"; do
|
|
if [[ ! " $failed_containers " =~ " $container " ]]; then
|
|
tags="${tags},$container"
|
|
fi
|
|
done
|
|
|
|
curl -s \
|
|
-H "priority:$priority" \
|
|
-H "tags:$tags" \
|
|
-d "$message" \
|
|
"$NOTIFICATION_URL" || log "Warning: Failed to send notification"
|
|
}
|
|
|
|
# Check dependencies
|
|
check_dependencies() {
|
|
local missing_deps=()
|
|
|
|
if ! command -v docker >/dev/null 2>&1; then
|
|
missing_deps+=("docker")
|
|
fi
|
|
|
|
if ! command -v tar >/dev/null 2>&1; then
|
|
missing_deps+=("tar")
|
|
fi
|
|
|
|
if ! command -v curl >/dev/null 2>&1; then
|
|
missing_deps+=("curl")
|
|
fi
|
|
|
|
if [ ${#missing_deps[@]} -ne 0 ]; then
|
|
log "Error: Missing required dependencies: ${missing_deps[*]}"
|
|
exit 1
|
|
fi
|
|
|
|
# Check if Docker daemon is running
|
|
if ! docker info >/dev/null 2>&1; then
|
|
log "Error: Docker daemon is not running or not accessible"
|
|
exit 1
|
|
fi
|
|
}
|
|
|
|
# Main backup function
|
|
main() {
|
|
log "=== Docker Volumes Backup Started ==="
|
|
|
|
# Initialize metrics if enabled
|
|
if [[ "$METRICS_ENABLED" == "true" ]]; then
|
|
metrics_backup_start "docker-volumes" "Docker container volumes backup" "$BACKUP_ROOT"
|
|
metrics_status_update "initializing" "Preparing Docker volumes backup"
|
|
fi
|
|
|
|
# Check dependencies
|
|
check_dependencies
|
|
|
|
# Check backup directory space
|
|
local available_space_gb
|
|
available_space_gb=$(df -BG "$BACKUP_ROOT" | awk 'NR==2 {print $4}' | sed 's/G//')
|
|
if [ "$available_space_gb" -lt 5 ]; then
|
|
log "Warning: Low disk space in backup directory: ${available_space_gb}GB available"
|
|
fi
|
|
|
|
local successful_backups=0
|
|
local failed_backups=0
|
|
local failed_containers=()
|
|
|
|
# Update metrics for backup phase
|
|
if [[ "$METRICS_ENABLED" == "true" ]]; then
|
|
metrics_status_update "backing_up" "Backing up Docker container volumes"
|
|
fi
|
|
|
|
# Backup each container
|
|
for container in "${!CONTAINERS[@]}"; do
|
|
local volume_info="${CONTAINERS[$container]}"
|
|
local volume_path="${volume_info%%:*}"
|
|
local description="${volume_info##*:}"
|
|
|
|
if backup_container_volume "$container" "$volume_path" "$description"; then
|
|
((successful_backups++))
|
|
else
|
|
((failed_backups++))
|
|
failed_containers+=("$container")
|
|
fi
|
|
done
|
|
|
|
# Update metrics for completion
|
|
if [[ "$METRICS_ENABLED" == "true" ]]; then
|
|
if [ $failed_backups -eq 0 ]; then
|
|
metrics_status_update "completed" "All Docker backups completed successfully"
|
|
else
|
|
metrics_status_update "completed_with_errors" "Docker backup completed with $failed_backups failures"
|
|
fi
|
|
fi
|
|
|
|
# Summary
|
|
log "=== Docker Volumes Backup Summary ==="
|
|
log "Successful backups: $successful_backups"
|
|
log "Failed backups: $failed_backups"
|
|
|
|
if [ ${#failed_containers[@]} -gt 0 ]; then
|
|
log "Failed containers: ${failed_containers[*]}"
|
|
fi
|
|
|
|
# Send notification
|
|
if [ $failed_backups -eq 0 ]; then
|
|
log "All backups completed successfully!"
|
|
send_notification "success" "Completed backup of all Docker containers ($successful_backups services)" ""
|
|
else
|
|
log "Some backups failed!"
|
|
send_notification "failed" "Docker backup completed with errors: $failed_backups failed, $successful_backups succeeded" "${failed_containers[*]}"
|
|
fi
|
|
|
|
# Finalize metrics
|
|
if [[ "$METRICS_ENABLED" == "true" ]]; then
|
|
cleanup
|
|
fi
|
|
|
|
log "=== Docker Volumes Backup Finished ==="
|
|
|
|
# Exit with error code if any backups failed
|
|
exit $failed_backups
|
|
}
|
|
|
|
# Run main function
|
|
main "$@"
|