mirror of
https://github.com/acedanger/shell.git
synced 2025-12-06 03:20:12 -08:00
58b5dea8b4
- Changed inline variable assignments to separate declaration and assignment for clarity. - Updated condition checks and log messages for better readability and consistency. - Added a backup of validate-plex-recovery.sh for safety. - Introduced a new script run-docker-tests.sh for testing setup in Docker containers. - Enhanced ssh-login.sh to improve condition checks and logging functionality.
397 lines
12 KiB
Bash
Executable File
397 lines
12 KiB
Bash
Executable File
#!/bin/bash
|
|
|
|
# Enhanced Backup Log Monitor
|
|
# Provides real-time monitoring and analysis of backup operations
|
|
|
|
set -e
|
|
|
|
# Color codes for output
|
|
RED='\033[0;31m'
|
|
GREEN='\033[0;32m'
|
|
YELLOW='\033[1;33m'
|
|
BLUE='\033[0;34m'
|
|
CYAN='\033[0;36m'
|
|
PURPLE='\033[0;35m'
|
|
NC='\033[0m' # No Color
|
|
|
|
SCRIPT_DIR="$(dirname "$(readlink -f "$0")")"
|
|
LOG_DIR="$SCRIPT_DIR/logs"
|
|
REPORT_DIR="$LOG_DIR/reports"
|
|
HOSTNAME=$(hostname)
|
|
|
|
# Ensure directories exist
|
|
mkdir -p "$LOG_DIR" "$REPORT_DIR"
|
|
|
|
# Backup service tags for monitoring
|
|
BACKUP_TAGS=("plex-backup" "backup-move" "plex-validation" "immich-backup" "plex-report" "crontab-backup")
|
|
|
|
log_message() {
|
|
echo -e "$(date '+%H:%M:%S') $1"
|
|
}
|
|
|
|
log_error() {
|
|
log_message "${RED}ERROR: $1${NC}"
|
|
}
|
|
|
|
log_success() {
|
|
log_message "${GREEN}SUCCESS: $1${NC}"
|
|
}
|
|
|
|
log_warning() {
|
|
log_message "${YELLOW}WARNING: $1${NC}"
|
|
}
|
|
|
|
log_info() {
|
|
log_message "${BLUE}INFO: $1${NC}"
|
|
}
|
|
|
|
monitor_realtime() {
|
|
local tags_filter=""
|
|
for tag in "${BACKUP_TAGS[@]}"; do
|
|
tags_filter="$tags_filter -t $tag"
|
|
done
|
|
|
|
log_info "Starting real-time monitoring of backup logs"
|
|
log_info "Press Ctrl+C to stop monitoring"
|
|
echo
|
|
|
|
sudo journalctl -f "$tags_filter" --no-hostname --output=short-iso | while read -r line; do
|
|
# Color code different log levels and services
|
|
if [[ "$line" =~ ERROR ]]; then
|
|
echo -e "${RED}$line${NC}"
|
|
elif [[ "$line" =~ SUCCESS ]]; then
|
|
echo -e "${GREEN}$line${NC}"
|
|
elif [[ "$line" =~ WARNING ]]; then
|
|
echo -e "${YELLOW}$line${NC}"
|
|
elif [[ "$line" =~ plex-backup ]]; then
|
|
echo -e "${BLUE}$line${NC}"
|
|
elif [[ "$line" =~ backup-move ]]; then
|
|
echo -e "${CYAN}$line${NC}"
|
|
elif [[ "$line" =~ plex-validation ]]; then
|
|
echo -e "${PURPLE}$line${NC}"
|
|
else
|
|
echo "$line"
|
|
fi
|
|
done
|
|
}
|
|
|
|
show_recent_logs() {
|
|
local hours="${1:-24}"
|
|
local service="${2:-all}"
|
|
|
|
log_info "Showing logs from the last $hours hours"
|
|
|
|
local tags_filter=""
|
|
if [ "$service" = "all" ]; then
|
|
for tag in "${BACKUP_TAGS[@]}"; do
|
|
tags_filter="$tags_filter -t $tag"
|
|
done
|
|
else
|
|
tags_filter="-t $service"
|
|
fi
|
|
|
|
echo
|
|
sudo journalctl --since "${hours} hours ago" "$tags_filter" --no-hostname --output=short-iso | \
|
|
while read -r line; do
|
|
# Color code the output
|
|
if [[ "$line" =~ ERROR ]]; then
|
|
echo -e "${RED}$line${NC}"
|
|
elif [[ "$line" =~ SUCCESS ]]; then
|
|
echo -e "${GREEN}$line${NC}"
|
|
elif [[ "$line" =~ WARNING ]]; then
|
|
echo -e "${YELLOW}$line${NC}"
|
|
else
|
|
echo "$line"
|
|
fi
|
|
done
|
|
}
|
|
|
|
show_error_summary() {
|
|
local days="${1:-7}"
|
|
|
|
log_info "Error summary for the last $days days"
|
|
echo
|
|
|
|
for tag in "${BACKUP_TAGS[@]}"; do
|
|
local error_count
|
|
error_count=$(sudo journalctl --since "${days} days ago" -t "$tag" --grep="ERROR" --output=cat | wc -l)
|
|
if [ "$error_count" -gt 0 ]; then
|
|
echo -e "${RED}$tag: $error_count errors${NC}"
|
|
sudo journalctl --since "${days} days ago" -t "$tag" --grep="ERROR" --output=short-iso | head -5
|
|
echo
|
|
else
|
|
echo -e "${GREEN}$tag: No errors${NC}"
|
|
fi
|
|
done
|
|
}
|
|
|
|
generate_backup_report() {
|
|
local days="${1:-7}"
|
|
local report_file
|
|
report_file="$REPORT_DIR/backup-report-$(date +%Y%m%d_%H%M%S).txt"
|
|
|
|
log_info "Generating comprehensive backup report for the last $days days"
|
|
log_info "Report will be saved to: $report_file"
|
|
|
|
{
|
|
echo "=== BACKUP SYSTEM REPORT ==="
|
|
echo "Generated: $(date)"
|
|
echo "Period: Last $days days"
|
|
echo "System: $(uname -n)"
|
|
echo
|
|
|
|
for tag in "${BACKUP_TAGS[@]}"; do
|
|
echo "=== $tag ==="
|
|
|
|
# Count entries
|
|
local total_entries
|
|
local error_count
|
|
local success_count
|
|
total_entries=$(sudo journalctl --since "${days} days ago" -t "$tag" --output=cat | wc -l)
|
|
error_count=$(sudo journalctl --since "${days} days ago" -t "$tag" --grep="ERROR" --output=cat | wc -l)
|
|
success_count=$(sudo journalctl --since "${days} days ago" -t "$tag" --grep="SUCCESS" --output=cat | wc -l)
|
|
|
|
echo "Total log entries: $total_entries"
|
|
echo "Errors: $error_count"
|
|
echo "Successes: $success_count"
|
|
|
|
if [ "$error_count" -gt 0 ]; then
|
|
echo
|
|
echo "Recent errors:"
|
|
sudo journalctl --since "${days} days ago" -t "$tag" --grep="ERROR" --output=short-iso | head -10
|
|
fi
|
|
|
|
echo
|
|
echo "Recent activity:"
|
|
sudo journalctl --since "${days} days ago" -t "$tag" --output=short-iso | tail -5
|
|
echo
|
|
echo "----------------------------------------"
|
|
echo
|
|
done
|
|
|
|
# System resource usage during backups
|
|
echo "=== SYSTEM ANALYSIS ==="
|
|
echo "Disk usage in backup directories:"
|
|
if [ -d "/mnt/share/media/backups" ]; then
|
|
du -sh /mnt/share/media/backups/* 2>/dev/null || echo "No backup directories found"
|
|
fi
|
|
echo
|
|
|
|
# Cron job status
|
|
echo "Active cron jobs related to backups:"
|
|
sudo crontab -l 2>/dev/null | grep -E "(backup|plex|immich)" || echo "No backup-related cron jobs found"
|
|
echo
|
|
|
|
} > "$report_file"
|
|
|
|
log_success "Report generated: $report_file"
|
|
|
|
# Show summary
|
|
echo
|
|
log_info "Report Summary:"
|
|
grep -E "(Total log entries|Errors|Successes):" "$report_file" | while read -r line; do
|
|
if [[ "$line" =~ Errors:.*[1-9] ]]; then
|
|
echo -e "${RED}$line${NC}"
|
|
else
|
|
echo -e "${GREEN}$line${NC}"
|
|
fi
|
|
done
|
|
}
|
|
|
|
check_backup_health() {
|
|
log_info "Checking backup system health for $HOSTNAME"
|
|
echo
|
|
|
|
local health_score=100
|
|
local issues=()
|
|
|
|
# Check if backup scripts exist
|
|
local backup_scripts=(
|
|
"/home/acedanger/shell/backup-plex.sh"
|
|
"/home/acedanger/shell/move-backups.sh"
|
|
"/home/acedanger/shell/validate-plex-backups.sh"
|
|
"/home/acedanger/shell/crontab/crontab-backup-system.sh"
|
|
)
|
|
|
|
for script in "${backup_scripts[@]}"; do
|
|
if [ ! -f "$script" ]; then
|
|
issues+=("Missing script: $script")
|
|
((health_score -= 20))
|
|
elif [ ! -x "$script" ]; then
|
|
issues+=("Script not executable: $script")
|
|
((health_score -= 10))
|
|
fi
|
|
done
|
|
|
|
# Check if backup directories exist
|
|
local backup_dirs=(
|
|
"/mnt/share/media/backups/plex"
|
|
"/mnt/share/media/backups/docker-data"
|
|
"/mnt/share/media/backups/immich"
|
|
)
|
|
|
|
for dir in "${backup_dirs[@]}"; do
|
|
if [ ! -d "$dir" ]; then
|
|
issues+=("Missing backup directory: $dir")
|
|
((health_score -= 15))
|
|
fi
|
|
done
|
|
|
|
# Check crontab backup system structure
|
|
local crontab_backup_dir="$SCRIPT_DIR/crontab-backups/$HOSTNAME"
|
|
if [ ! -d "$crontab_backup_dir" ]; then
|
|
issues+=("Missing crontab backup directory for $HOSTNAME: $crontab_backup_dir")
|
|
((health_score -= 10))
|
|
fi
|
|
|
|
# Check recent backup activity
|
|
local recent_activity=false
|
|
for tag in "${BACKUP_TAGS[@]}"; do
|
|
if sudo journalctl --since "24 hours ago" -t "$tag" --output=cat | grep -q .; then
|
|
recent_activity=true
|
|
break
|
|
fi
|
|
done
|
|
|
|
if [ "$recent_activity" = false ]; then
|
|
issues+=("No backup activity in the last 24 hours")
|
|
((health_score -= 25))
|
|
fi
|
|
|
|
# Check for recent errors
|
|
local recent_errors=0
|
|
for tag in "${BACKUP_TAGS[@]}"; do
|
|
local error_count
|
|
error_count=$(sudo journalctl --since "24 hours ago" -t "$tag" --grep="ERROR" --output=cat | wc -l)
|
|
((recent_errors += error_count))
|
|
done
|
|
|
|
if [ "$recent_errors" -gt 0 ]; then
|
|
issues+=("$recent_errors errors in the last 24 hours")
|
|
((health_score -= $((recent_errors * 5))))
|
|
fi
|
|
|
|
# Ensure health score doesn't go below 0
|
|
if [ "$health_score" -lt 0 ]; then
|
|
health_score=0
|
|
fi
|
|
|
|
# Display results
|
|
if [ "$health_score" -ge 90 ]; then
|
|
echo -e "${GREEN}Backup System Health ($HOSTNAME): ${health_score}% - EXCELLENT${NC}"
|
|
elif [ "$health_score" -ge 70 ]; then
|
|
echo -e "${YELLOW}Backup System Health ($HOSTNAME): ${health_score}% - GOOD${NC}"
|
|
elif [ "$health_score" -ge 50 ]; then
|
|
echo -e "${YELLOW}Backup System Health ($HOSTNAME): ${health_score}% - FAIR${NC}"
|
|
else
|
|
echo -e "${RED}Backup System Health ($HOSTNAME): ${health_score}% - POOR${NC}"
|
|
fi
|
|
|
|
if [ ${#issues[@]} -gt 0 ]; then
|
|
echo
|
|
log_warning "Issues found:"
|
|
for issue in "${issues[@]}"; do
|
|
echo -e " ${RED}• $issue${NC}"
|
|
done
|
|
|
|
echo
|
|
log_info "Recommended actions:"
|
|
echo " • Run: ./manage-enhanced-crontab.sh verify"
|
|
echo " • Check system logs: sudo journalctl -xe"
|
|
echo " • Verify backup directories are mounted and accessible"
|
|
echo " • Run: ./crontab-backup-system.sh status"
|
|
fi
|
|
}
|
|
|
|
show_service_status() {
|
|
log_info "Backup Service Status Overview"
|
|
echo
|
|
|
|
printf "%-20s %-15s %-20s %-30s\n" "Service" "Status" "Last Activity" "Last Message"
|
|
printf "%-20s %-15s %-20s %-30s\n" "-------" "------" "-------------" "------------"
|
|
|
|
for tag in "${BACKUP_TAGS[@]}"; do
|
|
local last_entry
|
|
last_entry=$(sudo journalctl -t "$tag" --output=short-iso -n 1 2>/dev/null | tail -1)
|
|
|
|
if [ -n "$last_entry" ]; then
|
|
local timestamp
|
|
local message
|
|
local entry_time
|
|
local current_time
|
|
timestamp=$(echo "$last_entry" | cut -d' ' -f1-2)
|
|
message=$(echo "$last_entry" | cut -d' ' -f4- | cut -c1-30)
|
|
|
|
# Check if it's recent (within 48 hours)
|
|
entry_time=$(date -d "$timestamp" +%s 2>/dev/null || echo "0")
|
|
current_time=$(date +%s)
|
|
local hours_diff=$(( (current_time - entry_time) / 3600 ))
|
|
|
|
local status
|
|
if [ "$hours_diff" -le 24 ]; then
|
|
status="${GREEN}Active${NC}"
|
|
elif [ "$hours_diff" -le 48 ]; then
|
|
status="${YELLOW}Recent${NC}"
|
|
else
|
|
status="${RED}Stale${NC}"
|
|
fi
|
|
|
|
printf "%-20s %-25s %-20s %-30s\n" "$tag" "$status" "$timestamp" "$message"
|
|
else
|
|
printf "%-20s %-25s %-20s %-30s\n" "$tag" "${RED}No logs${NC}" "Never" "No activity found"
|
|
fi
|
|
done
|
|
}
|
|
|
|
show_usage() {
|
|
echo -e "${PURPLE}Enhanced Backup Log Monitor${NC}"
|
|
echo
|
|
echo "Usage: $0 [COMMAND] [OPTIONS]"
|
|
echo
|
|
echo "Commands:"
|
|
echo " monitor Real-time monitoring of all backup logs"
|
|
echo " recent [HOURS] [SERVICE] Show recent logs (default: 24 hours, all services)"
|
|
echo " errors [DAYS] Show error summary (default: 7 days)"
|
|
echo " report [DAYS] Generate comprehensive report (default: 7 days)"
|
|
echo " health Check backup system health"
|
|
echo " status Show service status overview"
|
|
echo " help Show this help message"
|
|
echo
|
|
echo "Services:"
|
|
for tag in "${BACKUP_TAGS[@]}"; do
|
|
echo " - $tag"
|
|
done
|
|
echo
|
|
echo "Examples:"
|
|
echo " $0 monitor"
|
|
echo " $0 recent 12 plex-backup"
|
|
echo " $0 errors 3"
|
|
echo " $0 report 14"
|
|
echo
|
|
}
|
|
|
|
# Main command handling
|
|
case "${1:-help}" in
|
|
monitor)
|
|
monitor_realtime
|
|
;;
|
|
recent)
|
|
show_recent_logs "$2" "$3"
|
|
;;
|
|
errors)
|
|
show_error_summary "$2"
|
|
;;
|
|
report)
|
|
generate_backup_report "$2"
|
|
;;
|
|
health)
|
|
check_backup_health
|
|
;;
|
|
status)
|
|
show_service_status
|
|
;;
|
|
help|*)
|
|
show_usage
|
|
;;
|
|
esac
|