mirror of
https://github.com/acedanger/shell.git
synced 2025-12-05 22:50:18 -08:00
feat: Add base HTML template and implement dashboard, logs, and service views
- Created a base HTML template for consistent layout across pages. - Developed a dashboard page to display backup service metrics and statuses. - Implemented a log viewer for detailed log file inspection. - Added error handling page for better user experience during failures. - Introduced service detail page to show specific service metrics and actions. - Enhanced log filtering and viewing capabilities. - Integrated auto-refresh functionality for real-time updates on metrics. - Created integration and unit test scripts for backup metrics functionality.
This commit is contained in:
Peter Wood
354
backup-docker.sh
354
backup-docker.sh
@@ -1,25 +1,337 @@
|
||||
#! /bin/bash
|
||||
#!/bin/bash
|
||||
|
||||
# vaultwarden
|
||||
docker stop vaultwarden
|
||||
tar zcf "/home/acedanger/backup/docker-data/vaultwarden-data-bk-$(date +%Y%m%d).tar.gz" /var/lib/docker/volumes/vaultwarden_data/_data
|
||||
docker start vaultwarden
|
||||
# backup-docker.sh - Comprehensive Docker volumes backup script
|
||||
# Author: Shell Repository
|
||||
# Description: Backup Docker container volumes with proper error handling, logging, and metrics
|
||||
|
||||
# paperless
|
||||
#docker stop paperless-ng_broker_1 paperless-ng_db_1 paperless-ng_webserver_1
|
||||
#tar zcf /home/acedanger/backup/docker-data/paperless-data-bk-`date +%Y%m%d`.tar.gz /var/lib/docker/volumes/paperless-ng_data/_data
|
||||
#tar zcf /home/acedanger/backup/docker-data/paperless-media-bk-`date +%Y%m%d`.tar.gz /var/lib/docker/volumes/paperless-ng_media/_data
|
||||
#tar zcf /home/acedanger/backup/docker-data/paperless-pgdata-bk-`date +%Y%m%d`.tar.gz /var/lib/docker/volumes/paperless-ng_pgdata/_data
|
||||
#docker start paperless-ng_broker_1 paperless-ng_db_1 paperless-ng_webserver_1
|
||||
set -e
|
||||
|
||||
# uptime-kuma
|
||||
docker stop uptime-kuma
|
||||
tar zcf "/home/acedanger/backup/docker-data/uptime-kuma-data-bk-$(date +%Y%m%d).tar.gz" /var/lib/docker/volumes/uptime-kuma/_data
|
||||
docker start uptime-kuma
|
||||
# Load the unified backup metrics library
|
||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
LIB_DIR="$SCRIPT_DIR/lib"
|
||||
if [[ -f "$LIB_DIR/unified-backup-metrics.sh" ]]; then
|
||||
# shellcheck source=lib/unified-backup-metrics.sh
|
||||
source "$LIB_DIR/unified-backup-metrics.sh"
|
||||
METRICS_ENABLED=true
|
||||
else
|
||||
echo "Warning: Unified backup metrics library not found at $LIB_DIR/unified-backup-metrics.sh"
|
||||
METRICS_ENABLED=false
|
||||
fi
|
||||
|
||||
# send a notification to https://notify.peterwood.rocks\lab
|
||||
curl \
|
||||
-H priority:default \
|
||||
-H tags:backup,docker,vaultwarden,uptime-kuma,"${HOSTNAME}" \
|
||||
-d "Completed backup of vaultwarden, uptime-kuma" \
|
||||
https://notify.peterwood.rocks/lab
|
||||
# Colors for output
|
||||
GREEN='\033[0;32m'
|
||||
YELLOW='\033[0;33m'
|
||||
RED='\033[0;31m'
|
||||
BLUE='\033[0;34m'
|
||||
NC='\033[0m' # No Color
|
||||
|
||||
# Configuration
|
||||
BACKUP_ROOT="/home/acedanger/backup/docker-data"
|
||||
LOG_FILE="$SCRIPT_DIR/logs/docker-backup.log"
|
||||
NOTIFICATION_URL="https://notify.peterwood.rocks/lab"
|
||||
|
||||
# Container definitions: container_name:volume_path:description
|
||||
declare -A CONTAINERS=(
|
||||
["vaultwarden"]="/var/lib/docker/volumes/vaultwarden_data/_data:Password manager data"
|
||||
["uptime-kuma"]="/var/lib/docker/volumes/uptime-kuma/_data:Uptime monitoring data"
|
||||
# ["paperless-ng"]="/var/lib/docker/volumes/paperless-ng_data/_data:Document management data"
|
||||
# ["paperless-media"]="/var/lib/docker/volumes/paperless-ng_media/_data:Document media files"
|
||||
# ["paperless-pgdata"]="/var/lib/docker/volumes/paperless-ng_pgdata/_data:PostgreSQL database"
|
||||
)
|
||||
|
||||
# Ensure directories exist
|
||||
mkdir -p "$(dirname "$LOG_FILE")"
|
||||
mkdir -p "$BACKUP_ROOT"
|
||||
|
||||
# Logging function
|
||||
log() {
|
||||
echo "$(date '+%Y-%m-%d %H:%M:%S') - $1" | tee -a "$LOG_FILE"
|
||||
}
|
||||
|
||||
# Cleanup function for metrics finalization
|
||||
cleanup() {
|
||||
if [[ "$METRICS_ENABLED" == "true" ]]; then
|
||||
if [[ -n "$1" && "$1" == "error" ]]; then
|
||||
metrics_backup_complete "failed" "Docker backup failed during execution"
|
||||
else
|
||||
metrics_backup_complete "success" "Docker volumes backup completed successfully"
|
||||
fi
|
||||
fi
|
||||
}
|
||||
|
||||
# Set up cleanup trap
|
||||
trap 'cleanup error' ERR
|
||||
|
||||
# Check if container is running
|
||||
check_container_running() {
|
||||
local container="$1"
|
||||
if docker ps --format "table {{.Names}}" | grep -q "^${container}$"; then
|
||||
return 0
|
||||
else
|
||||
return 1
|
||||
fi
|
||||
}
|
||||
|
||||
# Stop container safely
|
||||
stop_container() {
|
||||
local container="$1"
|
||||
|
||||
log "Stopping container: $container"
|
||||
if [[ "$METRICS_ENABLED" == "true" ]]; then
|
||||
metrics_status_update "stopping_service" "Stopping container: $container"
|
||||
fi
|
||||
|
||||
if ! docker stop "$container" >/dev/null 2>&1; then
|
||||
log "Warning: Failed to stop container $container or container not running"
|
||||
return 1
|
||||
fi
|
||||
|
||||
# Wait for container to fully stop
|
||||
local max_wait=30
|
||||
local wait_count=0
|
||||
while [ $wait_count -lt $max_wait ]; do
|
||||
if ! docker ps -q --filter "name=$container" | grep -q .; then
|
||||
log "Container $container stopped successfully"
|
||||
return 0
|
||||
fi
|
||||
wait_count=$((wait_count + 1))
|
||||
sleep 1
|
||||
done
|
||||
|
||||
log "Warning: Container $container may not have stopped completely"
|
||||
return 1
|
||||
}
|
||||
|
||||
# Start container safely
|
||||
start_container() {
|
||||
local container="$1"
|
||||
|
||||
log "Starting container: $container"
|
||||
if [[ "$METRICS_ENABLED" == "true" ]]; then
|
||||
metrics_status_update "starting_service" "Starting container: $container"
|
||||
fi
|
||||
|
||||
if ! docker start "$container" >/dev/null 2>&1; then
|
||||
log "Error: Failed to start container $container"
|
||||
return 1
|
||||
fi
|
||||
|
||||
# Wait for container to be running
|
||||
local max_wait=30
|
||||
local wait_count=0
|
||||
while [ $wait_count -lt $max_wait ]; do
|
||||
if docker ps -q --filter "name=$container" | grep -q .; then
|
||||
log "Container $container started successfully"
|
||||
return 0
|
||||
fi
|
||||
wait_count=$((wait_count + 1))
|
||||
sleep 1
|
||||
done
|
||||
|
||||
log "Warning: Container $container may not have started properly"
|
||||
return 1
|
||||
}
|
||||
|
||||
# Backup container volume
|
||||
backup_container_volume() {
|
||||
local container="$1"
|
||||
local volume_path="$2"
|
||||
local description="$3"
|
||||
local backup_file="$BACKUP_ROOT/${container}-data-bk-$(date +%Y%m%d).tar.gz"
|
||||
|
||||
log "Starting backup for $container ($description)"
|
||||
|
||||
# Check if volume path exists
|
||||
if [ ! -d "$volume_path" ]; then
|
||||
log "Error: Volume path does not exist: $volume_path"
|
||||
return 1
|
||||
fi
|
||||
|
||||
# Check if container was running
|
||||
local was_running=false
|
||||
if check_container_running "$container"; then
|
||||
was_running=true
|
||||
if ! stop_container "$container"; then
|
||||
log "Error: Failed to stop container $container"
|
||||
return 1
|
||||
fi
|
||||
else
|
||||
log "Container $container is not running, proceeding with backup"
|
||||
fi
|
||||
|
||||
# Create backup
|
||||
log "Creating backup archive: $(basename "$backup_file")"
|
||||
if [[ "$METRICS_ENABLED" == "true" ]]; then
|
||||
metrics_status_update "backing_up" "Creating archive for $container"
|
||||
fi
|
||||
|
||||
if tar -czf "$backup_file" -C "$(dirname "$volume_path")" "$(basename "$volume_path")" 2>/dev/null; then
|
||||
local backup_size
|
||||
backup_size=$(du -h "$backup_file" | cut -f1)
|
||||
log "Backup completed successfully: $(basename "$backup_file") ($backup_size)"
|
||||
|
||||
# Track file completion in metrics
|
||||
if [[ "$METRICS_ENABLED" == "true" ]]; then
|
||||
local file_size_bytes
|
||||
file_size_bytes=$(stat -c%s "$backup_file" 2>/dev/null || echo "0")
|
||||
metrics_file_backup_complete "$(basename "$backup_file")" "$file_size_bytes" "created"
|
||||
fi
|
||||
else
|
||||
log "Error: Failed to create backup for $container"
|
||||
# Try to restart container even if backup failed
|
||||
if [ "$was_running" = true ]; then
|
||||
start_container "$container" || true
|
||||
fi
|
||||
return 1
|
||||
fi
|
||||
|
||||
# Restart container if it was running
|
||||
if [ "$was_running" = true ]; then
|
||||
if ! start_container "$container"; then
|
||||
log "Error: Failed to restart container $container after backup"
|
||||
return 1
|
||||
fi
|
||||
fi
|
||||
|
||||
return 0
|
||||
}
|
||||
|
||||
# Send notification
|
||||
send_notification() {
|
||||
local status="$1"
|
||||
local message="$2"
|
||||
local failed_containers="$3"
|
||||
|
||||
local tags="backup,docker,${HOSTNAME}"
|
||||
local priority="default"
|
||||
|
||||
if [ "$status" = "failed" ]; then
|
||||
priority="high"
|
||||
tags="${tags},error"
|
||||
fi
|
||||
|
||||
# Add successful container names to tags
|
||||
for container in "${!CONTAINERS[@]}"; do
|
||||
if [[ ! " $failed_containers " =~ " $container " ]]; then
|
||||
tags="${tags},$container"
|
||||
fi
|
||||
done
|
||||
|
||||
curl -s \
|
||||
-H "priority:$priority" \
|
||||
-H "tags:$tags" \
|
||||
-d "$message" \
|
||||
"$NOTIFICATION_URL" || log "Warning: Failed to send notification"
|
||||
}
|
||||
|
||||
# Check dependencies
|
||||
check_dependencies() {
|
||||
local missing_deps=()
|
||||
|
||||
if ! command -v docker >/dev/null 2>&1; then
|
||||
missing_deps+=("docker")
|
||||
fi
|
||||
|
||||
if ! command -v tar >/dev/null 2>&1; then
|
||||
missing_deps+=("tar")
|
||||
fi
|
||||
|
||||
if ! command -v curl >/dev/null 2>&1; then
|
||||
missing_deps+=("curl")
|
||||
fi
|
||||
|
||||
if [ ${#missing_deps[@]} -ne 0 ]; then
|
||||
log "Error: Missing required dependencies: ${missing_deps[*]}"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# Check if Docker daemon is running
|
||||
if ! docker info >/dev/null 2>&1; then
|
||||
log "Error: Docker daemon is not running or not accessible"
|
||||
exit 1
|
||||
fi
|
||||
}
|
||||
|
||||
# Main backup function
|
||||
main() {
|
||||
log "=== Docker Volumes Backup Started ==="
|
||||
|
||||
# Initialize metrics if enabled
|
||||
if [[ "$METRICS_ENABLED" == "true" ]]; then
|
||||
metrics_backup_start "docker-volumes" "Docker container volumes backup" "$BACKUP_ROOT"
|
||||
metrics_status_update "initializing" "Preparing Docker volumes backup"
|
||||
fi
|
||||
|
||||
# Check dependencies
|
||||
check_dependencies
|
||||
|
||||
# Check backup directory space
|
||||
local available_space_gb
|
||||
available_space_gb=$(df -BG "$BACKUP_ROOT" | awk 'NR==2 {print $4}' | sed 's/G//')
|
||||
if [ "$available_space_gb" -lt 5 ]; then
|
||||
log "Warning: Low disk space in backup directory: ${available_space_gb}GB available"
|
||||
fi
|
||||
|
||||
local successful_backups=0
|
||||
local failed_backups=0
|
||||
local failed_containers=()
|
||||
|
||||
# Update metrics for backup phase
|
||||
if [[ "$METRICS_ENABLED" == "true" ]]; then
|
||||
metrics_status_update "backing_up" "Backing up Docker container volumes"
|
||||
fi
|
||||
|
||||
# Backup each container
|
||||
for container in "${!CONTAINERS[@]}"; do
|
||||
local volume_info="${CONTAINERS[$container]}"
|
||||
local volume_path="${volume_info%%:*}"
|
||||
local description="${volume_info##*:}"
|
||||
|
||||
if backup_container_volume "$container" "$volume_path" "$description"; then
|
||||
((successful_backups++))
|
||||
else
|
||||
((failed_backups++))
|
||||
failed_containers+=("$container")
|
||||
fi
|
||||
done
|
||||
|
||||
# Update metrics for completion
|
||||
if [[ "$METRICS_ENABLED" == "true" ]]; then
|
||||
if [ $failed_backups -eq 0 ]; then
|
||||
metrics_status_update "completed" "All Docker backups completed successfully"
|
||||
else
|
||||
metrics_status_update "completed_with_errors" "Docker backup completed with $failed_backups failures"
|
||||
fi
|
||||
fi
|
||||
|
||||
# Summary
|
||||
log "=== Docker Volumes Backup Summary ==="
|
||||
log "Successful backups: $successful_backups"
|
||||
log "Failed backups: $failed_backups"
|
||||
|
||||
if [ ${#failed_containers[@]} -gt 0 ]; then
|
||||
log "Failed containers: ${failed_containers[*]}"
|
||||
fi
|
||||
|
||||
# Send notification
|
||||
if [ $failed_backups -eq 0 ]; then
|
||||
log "All backups completed successfully!"
|
||||
send_notification "success" "Completed backup of all Docker containers ($successful_backups services)" ""
|
||||
else
|
||||
log "Some backups failed!"
|
||||
send_notification "failed" "Docker backup completed with errors: $failed_backups failed, $successful_backups succeeded" "${failed_containers[*]}"
|
||||
fi
|
||||
|
||||
# Finalize metrics
|
||||
if [[ "$METRICS_ENABLED" == "true" ]]; then
|
||||
cleanup
|
||||
fi
|
||||
|
||||
log "=== Docker Volumes Backup Finished ==="
|
||||
|
||||
# Exit with error code if any backups failed
|
||||
exit $failed_backups
|
||||
}
|
||||
|
||||
# Run main function
|
||||
main "$@"
|
||||
|
||||
Reference in New Issue
Block a user