mirror of
https://github.com/acedanger/shell.git
synced 2025-12-05 22:50:18 -08:00
Commit local changes before merging with remote
This commit is contained in:
Peter Wood
387
backup-log-monitor.sh
Executable file
387
backup-log-monitor.sh
Executable file
@@ -0,0 +1,387 @@
|
||||
#!/bin/bash
|
||||
|
||||
# Enhanced Backup Log Monitor
|
||||
# Provides real-time monitoring and analysis of backup operations
|
||||
|
||||
set -e
|
||||
|
||||
# Color codes for output
|
||||
RED='\033[0;31m'
|
||||
GREEN='\033[0;32m'
|
||||
YELLOW='\033[1;33m'
|
||||
BLUE='\033[0;34m'
|
||||
CYAN='\033[0;36m'
|
||||
PURPLE='\033[0;35m'
|
||||
NC='\033[0m' # No Color
|
||||
|
||||
SCRIPT_DIR="$(dirname "$(readlink -f "$0")")"
|
||||
LOG_DIR="$SCRIPT_DIR/logs"
|
||||
REPORT_DIR="$LOG_DIR/reports"
|
||||
HOSTNAME=$(hostname)
|
||||
|
||||
# Ensure directories exist
|
||||
mkdir -p "$LOG_DIR" "$REPORT_DIR"
|
||||
|
||||
# Backup service tags for monitoring
|
||||
BACKUP_TAGS=("plex-backup" "backup-move" "plex-validation" "immich-backup" "plex-report" "crontab-backup")
|
||||
|
||||
log_message() {
|
||||
echo -e "$(date '+%H:%M:%S') $1"
|
||||
}
|
||||
|
||||
log_error() {
|
||||
log_message "${RED}ERROR: $1${NC}"
|
||||
}
|
||||
|
||||
log_success() {
|
||||
log_message "${GREEN}SUCCESS: $1${NC}"
|
||||
}
|
||||
|
||||
log_warning() {
|
||||
log_message "${YELLOW}WARNING: $1${NC}"
|
||||
}
|
||||
|
||||
log_info() {
|
||||
log_message "${BLUE}INFO: $1${NC}"
|
||||
}
|
||||
|
||||
monitor_realtime() {
|
||||
local tags_filter=""
|
||||
for tag in "${BACKUP_TAGS[@]}"; do
|
||||
tags_filter="$tags_filter -t $tag"
|
||||
done
|
||||
|
||||
log_info "Starting real-time monitoring of backup logs"
|
||||
log_info "Press Ctrl+C to stop monitoring"
|
||||
echo
|
||||
|
||||
sudo journalctl -f $tags_filter --no-hostname --output=short-iso | while read -r line; do
|
||||
# Color code different log levels and services
|
||||
if [[ "$line" =~ ERROR ]]; then
|
||||
echo -e "${RED}$line${NC}"
|
||||
elif [[ "$line" =~ SUCCESS ]]; then
|
||||
echo -e "${GREEN}$line${NC}"
|
||||
elif [[ "$line" =~ WARNING ]]; then
|
||||
echo -e "${YELLOW}$line${NC}"
|
||||
elif [[ "$line" =~ plex-backup ]]; then
|
||||
echo -e "${BLUE}$line${NC}"
|
||||
elif [[ "$line" =~ backup-move ]]; then
|
||||
echo -e "${CYAN}$line${NC}"
|
||||
elif [[ "$line" =~ plex-validation ]]; then
|
||||
echo -e "${PURPLE}$line${NC}"
|
||||
else
|
||||
echo "$line"
|
||||
fi
|
||||
done
|
||||
}
|
||||
|
||||
show_recent_logs() {
|
||||
local hours="${1:-24}"
|
||||
local service="${2:-all}"
|
||||
|
||||
log_info "Showing logs from the last $hours hours"
|
||||
|
||||
local tags_filter=""
|
||||
if [ "$service" = "all" ]; then
|
||||
for tag in "${BACKUP_TAGS[@]}"; do
|
||||
tags_filter="$tags_filter -t $tag"
|
||||
done
|
||||
else
|
||||
tags_filter="-t $service"
|
||||
fi
|
||||
|
||||
echo
|
||||
sudo journalctl --since "${hours} hours ago" $tags_filter --no-hostname --output=short-iso | \
|
||||
while read -r line; do
|
||||
# Color code the output
|
||||
if [[ "$line" =~ ERROR ]]; then
|
||||
echo -e "${RED}$line${NC}"
|
||||
elif [[ "$line" =~ SUCCESS ]]; then
|
||||
echo -e "${GREEN}$line${NC}"
|
||||
elif [[ "$line" =~ WARNING ]]; then
|
||||
echo -e "${YELLOW}$line${NC}"
|
||||
else
|
||||
echo "$line"
|
||||
fi
|
||||
done
|
||||
}
|
||||
|
||||
show_error_summary() {
|
||||
local days="${1:-7}"
|
||||
|
||||
log_info "Error summary for the last $days days"
|
||||
echo
|
||||
|
||||
local error_file="/tmp/backup_errors_$$.tmp"
|
||||
|
||||
for tag in "${BACKUP_TAGS[@]}"; do
|
||||
local error_count=$(sudo journalctl --since "${days} days ago" -t "$tag" --grep="ERROR" --output=cat | wc -l)
|
||||
if [ "$error_count" -gt 0 ]; then
|
||||
echo -e "${RED}$tag: $error_count errors${NC}"
|
||||
sudo journalctl --since "${days} days ago" -t "$tag" --grep="ERROR" --output=short-iso | head -5
|
||||
echo
|
||||
else
|
||||
echo -e "${GREEN}$tag: No errors${NC}"
|
||||
fi
|
||||
done
|
||||
}
|
||||
|
||||
generate_backup_report() {
|
||||
local days="${1:-7}"
|
||||
local report_file="$REPORT_DIR/backup-report-$(date +%Y%m%d_%H%M%S).txt"
|
||||
|
||||
log_info "Generating comprehensive backup report for the last $days days"
|
||||
log_info "Report will be saved to: $report_file"
|
||||
|
||||
{
|
||||
echo "=== BACKUP SYSTEM REPORT ==="
|
||||
echo "Generated: $(date)"
|
||||
echo "Period: Last $days days"
|
||||
echo "System: $(uname -n)"
|
||||
echo
|
||||
|
||||
for tag in "${BACKUP_TAGS[@]}"; do
|
||||
echo "=== $tag ==="
|
||||
|
||||
# Count entries
|
||||
local total_entries=$(sudo journalctl --since "${days} days ago" -t "$tag" --output=cat | wc -l)
|
||||
local error_count=$(sudo journalctl --since "${days} days ago" -t "$tag" --grep="ERROR" --output=cat | wc -l)
|
||||
local success_count=$(sudo journalctl --since "${days} days ago" -t "$tag" --grep="SUCCESS" --output=cat | wc -l)
|
||||
|
||||
echo "Total log entries: $total_entries"
|
||||
echo "Errors: $error_count"
|
||||
echo "Successes: $success_count"
|
||||
|
||||
if [ "$error_count" -gt 0 ]; then
|
||||
echo
|
||||
echo "Recent errors:"
|
||||
sudo journalctl --since "${days} days ago" -t "$tag" --grep="ERROR" --output=short-iso | head -10
|
||||
fi
|
||||
|
||||
echo
|
||||
echo "Recent activity:"
|
||||
sudo journalctl --since "${days} days ago" -t "$tag" --output=short-iso | tail -5
|
||||
echo
|
||||
echo "----------------------------------------"
|
||||
echo
|
||||
done
|
||||
|
||||
# System resource usage during backups
|
||||
echo "=== SYSTEM ANALYSIS ==="
|
||||
echo "Disk usage in backup directories:"
|
||||
if [ -d "/mnt/share/media/backups" ]; then
|
||||
du -sh /mnt/share/media/backups/* 2>/dev/null || echo "No backup directories found"
|
||||
fi
|
||||
echo
|
||||
|
||||
# Cron job status
|
||||
echo "Active cron jobs related to backups:"
|
||||
sudo crontab -l 2>/dev/null | grep -E "(backup|plex|immich)" || echo "No backup-related cron jobs found"
|
||||
echo
|
||||
|
||||
} > "$report_file"
|
||||
|
||||
log_success "Report generated: $report_file"
|
||||
|
||||
# Show summary
|
||||
echo
|
||||
log_info "Report Summary:"
|
||||
grep -E "(Total log entries|Errors|Successes):" "$report_file" | while read -r line; do
|
||||
if [[ "$line" =~ Errors:.*[1-9] ]]; then
|
||||
echo -e "${RED}$line${NC}"
|
||||
else
|
||||
echo -e "${GREEN}$line${NC}"
|
||||
fi
|
||||
done
|
||||
}
|
||||
|
||||
check_backup_health() {
|
||||
log_info "Checking backup system health for $HOSTNAME"
|
||||
echo
|
||||
|
||||
local health_score=100
|
||||
local issues=()
|
||||
|
||||
# Check if backup scripts exist
|
||||
local backup_scripts=(
|
||||
"/home/acedanger/shell/backup-plex.sh"
|
||||
"/home/acedanger/shell/move-backups.sh"
|
||||
"/home/acedanger/shell/validate-plex-backups.sh"
|
||||
"/home/acedanger/shell/crontab/crontab-backup-system.sh"
|
||||
)
|
||||
|
||||
for script in "${backup_scripts[@]}"; do
|
||||
if [ ! -f "$script" ]; then
|
||||
issues+=("Missing script: $script")
|
||||
((health_score -= 20))
|
||||
elif [ ! -x "$script" ]; then
|
||||
issues+=("Script not executable: $script")
|
||||
((health_score -= 10))
|
||||
fi
|
||||
done
|
||||
|
||||
# Check if backup directories exist
|
||||
local backup_dirs=(
|
||||
"/mnt/share/media/backups/plex"
|
||||
"/mnt/share/media/backups/docker-data"
|
||||
"/mnt/share/media/backups/immich"
|
||||
)
|
||||
|
||||
for dir in "${backup_dirs[@]}"; do
|
||||
if [ ! -d "$dir" ]; then
|
||||
issues+=("Missing backup directory: $dir")
|
||||
((health_score -= 15))
|
||||
fi
|
||||
done
|
||||
|
||||
# Check crontab backup system structure
|
||||
local crontab_backup_dir="$SCRIPT_DIR/crontab-backups/$HOSTNAME"
|
||||
if [ ! -d "$crontab_backup_dir" ]; then
|
||||
issues+=("Missing crontab backup directory for $HOSTNAME: $crontab_backup_dir")
|
||||
((health_score -= 10))
|
||||
fi
|
||||
|
||||
# Check recent backup activity
|
||||
local recent_activity=false
|
||||
for tag in "${BACKUP_TAGS[@]}"; do
|
||||
if sudo journalctl --since "24 hours ago" -t "$tag" --output=cat | grep -q .; then
|
||||
recent_activity=true
|
||||
break
|
||||
fi
|
||||
done
|
||||
|
||||
if [ "$recent_activity" = false ]; then
|
||||
issues+=("No backup activity in the last 24 hours")
|
||||
((health_score -= 25))
|
||||
fi
|
||||
|
||||
# Check for recent errors
|
||||
local recent_errors=0
|
||||
for tag in "${BACKUP_TAGS[@]}"; do
|
||||
local error_count=$(sudo journalctl --since "24 hours ago" -t "$tag" --grep="ERROR" --output=cat | wc -l)
|
||||
((recent_errors += error_count))
|
||||
done
|
||||
|
||||
if [ "$recent_errors" -gt 0 ]; then
|
||||
issues+=("$recent_errors errors in the last 24 hours")
|
||||
((health_score -= $((recent_errors * 5))))
|
||||
fi
|
||||
|
||||
# Ensure health score doesn't go below 0
|
||||
if [ "$health_score" -lt 0 ]; then
|
||||
health_score=0
|
||||
fi
|
||||
|
||||
# Display results
|
||||
if [ "$health_score" -ge 90 ]; then
|
||||
echo -e "${GREEN}Backup System Health ($HOSTNAME): ${health_score}% - EXCELLENT${NC}"
|
||||
elif [ "$health_score" -ge 70 ]; then
|
||||
echo -e "${YELLOW}Backup System Health ($HOSTNAME): ${health_score}% - GOOD${NC}"
|
||||
elif [ "$health_score" -ge 50 ]; then
|
||||
echo -e "${YELLOW}Backup System Health ($HOSTNAME): ${health_score}% - FAIR${NC}"
|
||||
else
|
||||
echo -e "${RED}Backup System Health ($HOSTNAME): ${health_score}% - POOR${NC}"
|
||||
fi
|
||||
|
||||
if [ ${#issues[@]} -gt 0 ]; then
|
||||
echo
|
||||
log_warning "Issues found:"
|
||||
for issue in "${issues[@]}"; do
|
||||
echo -e " ${RED}• $issue${NC}"
|
||||
done
|
||||
|
||||
echo
|
||||
log_info "Recommended actions:"
|
||||
echo " • Run: ./manage-enhanced-crontab.sh verify"
|
||||
echo " • Check system logs: sudo journalctl -xe"
|
||||
echo " • Verify backup directories are mounted and accessible"
|
||||
echo " • Run: ./crontab-backup-system.sh status"
|
||||
fi
|
||||
}
|
||||
|
||||
show_service_status() {
|
||||
log_info "Backup Service Status Overview"
|
||||
echo
|
||||
|
||||
printf "%-20s %-15s %-20s %-30s\n" "Service" "Status" "Last Activity" "Last Message"
|
||||
printf "%-20s %-15s %-20s %-30s\n" "-------" "------" "-------------" "------------"
|
||||
|
||||
for tag in "${BACKUP_TAGS[@]}"; do
|
||||
local last_entry=$(sudo journalctl -t "$tag" --output=short-iso -n 1 2>/dev/null | tail -1)
|
||||
|
||||
if [ -n "$last_entry" ]; then
|
||||
local timestamp=$(echo "$last_entry" | cut -d' ' -f1-2)
|
||||
local message=$(echo "$last_entry" | cut -d' ' -f4- | cut -c1-30)
|
||||
|
||||
# Check if it's recent (within 48 hours)
|
||||
local entry_time=$(date -d "$timestamp" +%s 2>/dev/null || echo "0")
|
||||
local current_time=$(date +%s)
|
||||
local hours_diff=$(( (current_time - entry_time) / 3600 ))
|
||||
|
||||
local status
|
||||
if [ "$hours_diff" -le 24 ]; then
|
||||
status="${GREEN}Active${NC}"
|
||||
elif [ "$hours_diff" -le 48 ]; then
|
||||
status="${YELLOW}Recent${NC}"
|
||||
else
|
||||
status="${RED}Stale${NC}"
|
||||
fi
|
||||
|
||||
printf "%-20s %-25s %-20s %-30s\n" "$tag" "$status" "$timestamp" "$message"
|
||||
else
|
||||
printf "%-20s %-25s %-20s %-30s\n" "$tag" "${RED}No logs${NC}" "Never" "No activity found"
|
||||
fi
|
||||
done
|
||||
}
|
||||
|
||||
show_usage() {
|
||||
echo -e "${PURPLE}Enhanced Backup Log Monitor${NC}"
|
||||
echo
|
||||
echo "Usage: $0 [COMMAND] [OPTIONS]"
|
||||
echo
|
||||
echo "Commands:"
|
||||
echo " monitor Real-time monitoring of all backup logs"
|
||||
echo " recent [HOURS] [SERVICE] Show recent logs (default: 24 hours, all services)"
|
||||
echo " errors [DAYS] Show error summary (default: 7 days)"
|
||||
echo " report [DAYS] Generate comprehensive report (default: 7 days)"
|
||||
echo " health Check backup system health"
|
||||
echo " status Show service status overview"
|
||||
echo " help Show this help message"
|
||||
echo
|
||||
echo "Services:"
|
||||
for tag in "${BACKUP_TAGS[@]}"; do
|
||||
echo " - $tag"
|
||||
done
|
||||
echo
|
||||
echo "Examples:"
|
||||
echo " $0 monitor"
|
||||
echo " $0 recent 12 plex-backup"
|
||||
echo " $0 errors 3"
|
||||
echo " $0 report 14"
|
||||
echo
|
||||
}
|
||||
|
||||
# Main command handling
|
||||
case "${1:-help}" in
|
||||
monitor)
|
||||
monitor_realtime
|
||||
;;
|
||||
recent)
|
||||
show_recent_logs "$2" "$3"
|
||||
;;
|
||||
errors)
|
||||
show_error_summary "$2"
|
||||
;;
|
||||
report)
|
||||
generate_backup_report "$2"
|
||||
;;
|
||||
health)
|
||||
check_backup_health
|
||||
;;
|
||||
status)
|
||||
show_service_status
|
||||
;;
|
||||
help|*)
|
||||
show_usage
|
||||
;;
|
||||
esac
|
||||
Reference in New Issue
Block a user