#!/bin/bash # Enhanced Backup Log Monitor # Provides real-time monitoring and analysis of backup operations set -e # Color codes for output RED='\033[0;31m' GREEN='\033[0;32m' YELLOW='\033[1;33m' BLUE='\033[0;34m' CYAN='\033[0;36m' PURPLE='\033[0;35m' NC='\033[0m' # No Color SCRIPT_DIR="$(dirname "$(readlink -f "$0")")" LOG_DIR="$SCRIPT_DIR/logs" REPORT_DIR="$LOG_DIR/reports" HOSTNAME=$(hostname) # Ensure directories exist mkdir -p "$LOG_DIR" "$REPORT_DIR" # Backup service tags for monitoring BACKUP_TAGS=("plex-backup" "backup-move" "plex-validation" "immich-backup" "plex-report" "crontab-backup") log_message() { echo -e "$(date '+%H:%M:%S') $1" } log_error() { log_message "${RED}ERROR: $1${NC}" } log_success() { log_message "${GREEN}SUCCESS: $1${NC}" } log_warning() { log_message "${YELLOW}WARNING: $1${NC}" } log_info() { log_message "${BLUE}INFO: $1${NC}" } monitor_realtime() { local tags_filter="" for tag in "${BACKUP_TAGS[@]}"; do tags_filter="$tags_filter -t $tag" done log_info "Starting real-time monitoring of backup logs" log_info "Press Ctrl+C to stop monitoring" echo sudo journalctl -f $tags_filter --no-hostname --output=short-iso | while read -r line; do # Color code different log levels and services if [[ "$line" =~ ERROR ]]; then echo -e "${RED}$line${NC}" elif [[ "$line" =~ SUCCESS ]]; then echo -e "${GREEN}$line${NC}" elif [[ "$line" =~ WARNING ]]; then echo -e "${YELLOW}$line${NC}" elif [[ "$line" =~ plex-backup ]]; then echo -e "${BLUE}$line${NC}" elif [[ "$line" =~ backup-move ]]; then echo -e "${CYAN}$line${NC}" elif [[ "$line" =~ plex-validation ]]; then echo -e "${PURPLE}$line${NC}" else echo "$line" fi done } show_recent_logs() { local hours="${1:-24}" local service="${2:-all}" log_info "Showing logs from the last $hours hours" local tags_filter="" if [ "$service" = "all" ]; then for tag in "${BACKUP_TAGS[@]}"; do tags_filter="$tags_filter -t $tag" done else tags_filter="-t $service" fi echo sudo journalctl --since "${hours} hours ago" $tags_filter --no-hostname --output=short-iso | \ while read -r line; do # Color code the output if [[ "$line" =~ ERROR ]]; then echo -e "${RED}$line${NC}" elif [[ "$line" =~ SUCCESS ]]; then echo -e "${GREEN}$line${NC}" elif [[ "$line" =~ WARNING ]]; then echo -e "${YELLOW}$line${NC}" else echo "$line" fi done } show_error_summary() { local days="${1:-7}" log_info "Error summary for the last $days days" echo local error_file="/tmp/backup_errors_$$.tmp" for tag in "${BACKUP_TAGS[@]}"; do local error_count=$(sudo journalctl --since "${days} days ago" -t "$tag" --grep="ERROR" --output=cat | wc -l) if [ "$error_count" -gt 0 ]; then echo -e "${RED}$tag: $error_count errors${NC}" sudo journalctl --since "${days} days ago" -t "$tag" --grep="ERROR" --output=short-iso | head -5 echo else echo -e "${GREEN}$tag: No errors${NC}" fi done } generate_backup_report() { local days="${1:-7}" local report_file="$REPORT_DIR/backup-report-$(date +%Y%m%d_%H%M%S).txt" log_info "Generating comprehensive backup report for the last $days days" log_info "Report will be saved to: $report_file" { echo "=== BACKUP SYSTEM REPORT ===" echo "Generated: $(date)" echo "Period: Last $days days" echo "System: $(uname -n)" echo for tag in "${BACKUP_TAGS[@]}"; do echo "=== $tag ===" # Count entries local total_entries=$(sudo journalctl --since "${days} days ago" -t "$tag" --output=cat | wc -l) local error_count=$(sudo journalctl --since "${days} days ago" -t "$tag" --grep="ERROR" --output=cat | wc -l) local success_count=$(sudo journalctl --since "${days} days ago" -t "$tag" --grep="SUCCESS" --output=cat | wc -l) echo "Total log entries: $total_entries" echo "Errors: $error_count" echo "Successes: $success_count" if [ "$error_count" -gt 0 ]; then echo echo "Recent errors:" sudo journalctl --since "${days} days ago" -t "$tag" --grep="ERROR" --output=short-iso | head -10 fi echo echo "Recent activity:" sudo journalctl --since "${days} days ago" -t "$tag" --output=short-iso | tail -5 echo echo "----------------------------------------" echo done # System resource usage during backups echo "=== SYSTEM ANALYSIS ===" echo "Disk usage in backup directories:" if [ -d "/mnt/share/media/backups" ]; then du -sh /mnt/share/media/backups/* 2>/dev/null || echo "No backup directories found" fi echo # Cron job status echo "Active cron jobs related to backups:" sudo crontab -l 2>/dev/null | grep -E "(backup|plex|immich)" || echo "No backup-related cron jobs found" echo } > "$report_file" log_success "Report generated: $report_file" # Show summary echo log_info "Report Summary:" grep -E "(Total log entries|Errors|Successes):" "$report_file" | while read -r line; do if [[ "$line" =~ Errors:.*[1-9] ]]; then echo -e "${RED}$line${NC}" else echo -e "${GREEN}$line${NC}" fi done } check_backup_health() { log_info "Checking backup system health for $HOSTNAME" echo local health_score=100 local issues=() # Check if backup scripts exist local backup_scripts=( "/home/acedanger/shell/backup-plex.sh" "/home/acedanger/shell/move-backups.sh" "/home/acedanger/shell/validate-plex-backups.sh" "/home/acedanger/shell/crontab/crontab-backup-system.sh" ) for script in "${backup_scripts[@]}"; do if [ ! -f "$script" ]; then issues+=("Missing script: $script") ((health_score -= 20)) elif [ ! -x "$script" ]; then issues+=("Script not executable: $script") ((health_score -= 10)) fi done # Check if backup directories exist local backup_dirs=( "/mnt/share/media/backups/plex" "/mnt/share/media/backups/docker-data" "/mnt/share/media/backups/immich" ) for dir in "${backup_dirs[@]}"; do if [ ! -d "$dir" ]; then issues+=("Missing backup directory: $dir") ((health_score -= 15)) fi done # Check crontab backup system structure local crontab_backup_dir="$SCRIPT_DIR/crontab-backups/$HOSTNAME" if [ ! -d "$crontab_backup_dir" ]; then issues+=("Missing crontab backup directory for $HOSTNAME: $crontab_backup_dir") ((health_score -= 10)) fi # Check recent backup activity local recent_activity=false for tag in "${BACKUP_TAGS[@]}"; do if sudo journalctl --since "24 hours ago" -t "$tag" --output=cat | grep -q .; then recent_activity=true break fi done if [ "$recent_activity" = false ]; then issues+=("No backup activity in the last 24 hours") ((health_score -= 25)) fi # Check for recent errors local recent_errors=0 for tag in "${BACKUP_TAGS[@]}"; do local error_count=$(sudo journalctl --since "24 hours ago" -t "$tag" --grep="ERROR" --output=cat | wc -l) ((recent_errors += error_count)) done if [ "$recent_errors" -gt 0 ]; then issues+=("$recent_errors errors in the last 24 hours") ((health_score -= $((recent_errors * 5)))) fi # Ensure health score doesn't go below 0 if [ "$health_score" -lt 0 ]; then health_score=0 fi # Display results if [ "$health_score" -ge 90 ]; then echo -e "${GREEN}Backup System Health ($HOSTNAME): ${health_score}% - EXCELLENT${NC}" elif [ "$health_score" -ge 70 ]; then echo -e "${YELLOW}Backup System Health ($HOSTNAME): ${health_score}% - GOOD${NC}" elif [ "$health_score" -ge 50 ]; then echo -e "${YELLOW}Backup System Health ($HOSTNAME): ${health_score}% - FAIR${NC}" else echo -e "${RED}Backup System Health ($HOSTNAME): ${health_score}% - POOR${NC}" fi if [ ${#issues[@]} -gt 0 ]; then echo log_warning "Issues found:" for issue in "${issues[@]}"; do echo -e " ${RED}• $issue${NC}" done echo log_info "Recommended actions:" echo " • Run: ./manage-enhanced-crontab.sh verify" echo " • Check system logs: sudo journalctl -xe" echo " • Verify backup directories are mounted and accessible" echo " • Run: ./crontab-backup-system.sh status" fi } show_service_status() { log_info "Backup Service Status Overview" echo printf "%-20s %-15s %-20s %-30s\n" "Service" "Status" "Last Activity" "Last Message" printf "%-20s %-15s %-20s %-30s\n" "-------" "------" "-------------" "------------" for tag in "${BACKUP_TAGS[@]}"; do local last_entry=$(sudo journalctl -t "$tag" --output=short-iso -n 1 2>/dev/null | tail -1) if [ -n "$last_entry" ]; then local timestamp=$(echo "$last_entry" | cut -d' ' -f1-2) local message=$(echo "$last_entry" | cut -d' ' -f4- | cut -c1-30) # Check if it's recent (within 48 hours) local entry_time=$(date -d "$timestamp" +%s 2>/dev/null || echo "0") local current_time=$(date +%s) local hours_diff=$(( (current_time - entry_time) / 3600 )) local status if [ "$hours_diff" -le 24 ]; then status="${GREEN}Active${NC}" elif [ "$hours_diff" -le 48 ]; then status="${YELLOW}Recent${NC}" else status="${RED}Stale${NC}" fi printf "%-20s %-25s %-20s %-30s\n" "$tag" "$status" "$timestamp" "$message" else printf "%-20s %-25s %-20s %-30s\n" "$tag" "${RED}No logs${NC}" "Never" "No activity found" fi done } show_usage() { echo -e "${PURPLE}Enhanced Backup Log Monitor${NC}" echo echo "Usage: $0 [COMMAND] [OPTIONS]" echo echo "Commands:" echo " monitor Real-time monitoring of all backup logs" echo " recent [HOURS] [SERVICE] Show recent logs (default: 24 hours, all services)" echo " errors [DAYS] Show error summary (default: 7 days)" echo " report [DAYS] Generate comprehensive report (default: 7 days)" echo " health Check backup system health" echo " status Show service status overview" echo " help Show this help message" echo echo "Services:" for tag in "${BACKUP_TAGS[@]}"; do echo " - $tag" done echo echo "Examples:" echo " $0 monitor" echo " $0 recent 12 plex-backup" echo " $0 errors 3" echo " $0 report 14" echo } # Main command handling case "${1:-help}" in monitor) monitor_realtime ;; recent) show_recent_logs "$2" "$3" ;; errors) show_error_summary "$2" ;; report) generate_backup_report "$2" ;; health) check_backup_health ;; status) show_service_status ;; help|*) show_usage ;; esac