diff --git a/backup-log-monitor.sh b/backup-log-monitor.sh new file mode 100755 index 0000000..17dfcc2 --- /dev/null +++ b/backup-log-monitor.sh @@ -0,0 +1,387 @@ +#!/bin/bash + +# Enhanced Backup Log Monitor +# Provides real-time monitoring and analysis of backup operations + +set -e + +# Color codes for output +RED='\033[0;31m' +GREEN='\033[0;32m' +YELLOW='\033[1;33m' +BLUE='\033[0;34m' +CYAN='\033[0;36m' +PURPLE='\033[0;35m' +NC='\033[0m' # No Color + +SCRIPT_DIR="$(dirname "$(readlink -f "$0")")" +LOG_DIR="$SCRIPT_DIR/logs" +REPORT_DIR="$LOG_DIR/reports" +HOSTNAME=$(hostname) + +# Ensure directories exist +mkdir -p "$LOG_DIR" "$REPORT_DIR" + +# Backup service tags for monitoring +BACKUP_TAGS=("plex-backup" "backup-move" "plex-validation" "immich-backup" "plex-report" "crontab-backup") + +log_message() { + echo -e "$(date '+%H:%M:%S') $1" +} + +log_error() { + log_message "${RED}ERROR: $1${NC}" +} + +log_success() { + log_message "${GREEN}SUCCESS: $1${NC}" +} + +log_warning() { + log_message "${YELLOW}WARNING: $1${NC}" +} + +log_info() { + log_message "${BLUE}INFO: $1${NC}" +} + +monitor_realtime() { + local tags_filter="" + for tag in "${BACKUP_TAGS[@]}"; do + tags_filter="$tags_filter -t $tag" + done + + log_info "Starting real-time monitoring of backup logs" + log_info "Press Ctrl+C to stop monitoring" + echo + + sudo journalctl -f $tags_filter --no-hostname --output=short-iso | while read -r line; do + # Color code different log levels and services + if [[ "$line" =~ ERROR ]]; then + echo -e "${RED}$line${NC}" + elif [[ "$line" =~ SUCCESS ]]; then + echo -e "${GREEN}$line${NC}" + elif [[ "$line" =~ WARNING ]]; then + echo -e "${YELLOW}$line${NC}" + elif [[ "$line" =~ plex-backup ]]; then + echo -e "${BLUE}$line${NC}" + elif [[ "$line" =~ backup-move ]]; then + echo -e "${CYAN}$line${NC}" + elif [[ "$line" =~ plex-validation ]]; then + echo -e "${PURPLE}$line${NC}" + else + echo "$line" + fi + done +} + +show_recent_logs() { + local hours="${1:-24}" + local service="${2:-all}" + + log_info "Showing logs from the last $hours hours" + + local tags_filter="" + if [ "$service" = "all" ]; then + for tag in "${BACKUP_TAGS[@]}"; do + tags_filter="$tags_filter -t $tag" + done + else + tags_filter="-t $service" + fi + + echo + sudo journalctl --since "${hours} hours ago" $tags_filter --no-hostname --output=short-iso | \ + while read -r line; do + # Color code the output + if [[ "$line" =~ ERROR ]]; then + echo -e "${RED}$line${NC}" + elif [[ "$line" =~ SUCCESS ]]; then + echo -e "${GREEN}$line${NC}" + elif [[ "$line" =~ WARNING ]]; then + echo -e "${YELLOW}$line${NC}" + else + echo "$line" + fi + done +} + +show_error_summary() { + local days="${1:-7}" + + log_info "Error summary for the last $days days" + echo + + local error_file="/tmp/backup_errors_$$.tmp" + + for tag in "${BACKUP_TAGS[@]}"; do + local error_count=$(sudo journalctl --since "${days} days ago" -t "$tag" --grep="ERROR" --output=cat | wc -l) + if [ "$error_count" -gt 0 ]; then + echo -e "${RED}$tag: $error_count errors${NC}" + sudo journalctl --since "${days} days ago" -t "$tag" --grep="ERROR" --output=short-iso | head -5 + echo + else + echo -e "${GREEN}$tag: No errors${NC}" + fi + done +} + +generate_backup_report() { + local days="${1:-7}" + local report_file="$REPORT_DIR/backup-report-$(date +%Y%m%d_%H%M%S).txt" + + log_info "Generating comprehensive backup report for the last $days days" + log_info "Report will be saved to: $report_file" + + { + echo "=== BACKUP SYSTEM REPORT ===" + echo "Generated: $(date)" + echo "Period: Last $days days" + echo "System: $(uname -n)" + echo + + for tag in "${BACKUP_TAGS[@]}"; do + echo "=== $tag ===" + + # Count entries + local total_entries=$(sudo journalctl --since "${days} days ago" -t "$tag" --output=cat | wc -l) + local error_count=$(sudo journalctl --since "${days} days ago" -t "$tag" --grep="ERROR" --output=cat | wc -l) + local success_count=$(sudo journalctl --since "${days} days ago" -t "$tag" --grep="SUCCESS" --output=cat | wc -l) + + echo "Total log entries: $total_entries" + echo "Errors: $error_count" + echo "Successes: $success_count" + + if [ "$error_count" -gt 0 ]; then + echo + echo "Recent errors:" + sudo journalctl --since "${days} days ago" -t "$tag" --grep="ERROR" --output=short-iso | head -10 + fi + + echo + echo "Recent activity:" + sudo journalctl --since "${days} days ago" -t "$tag" --output=short-iso | tail -5 + echo + echo "----------------------------------------" + echo + done + + # System resource usage during backups + echo "=== SYSTEM ANALYSIS ===" + echo "Disk usage in backup directories:" + if [ -d "/mnt/share/media/backups" ]; then + du -sh /mnt/share/media/backups/* 2>/dev/null || echo "No backup directories found" + fi + echo + + # Cron job status + echo "Active cron jobs related to backups:" + sudo crontab -l 2>/dev/null | grep -E "(backup|plex|immich)" || echo "No backup-related cron jobs found" + echo + + } > "$report_file" + + log_success "Report generated: $report_file" + + # Show summary + echo + log_info "Report Summary:" + grep -E "(Total log entries|Errors|Successes):" "$report_file" | while read -r line; do + if [[ "$line" =~ Errors:.*[1-9] ]]; then + echo -e "${RED}$line${NC}" + else + echo -e "${GREEN}$line${NC}" + fi + done +} + +check_backup_health() { + log_info "Checking backup system health for $HOSTNAME" + echo + + local health_score=100 + local issues=() + + # Check if backup scripts exist + local backup_scripts=( + "/home/acedanger/shell/backup-plex.sh" + "/home/acedanger/shell/move-backups.sh" + "/home/acedanger/shell/validate-plex-backups.sh" + "/home/acedanger/shell/crontab-backup-system.sh" + ) + + for script in "${backup_scripts[@]}"; do + if [ ! -f "$script" ]; then + issues+=("Missing script: $script") + ((health_score -= 20)) + elif [ ! -x "$script" ]; then + issues+=("Script not executable: $script") + ((health_score -= 10)) + fi + done + + # Check if backup directories exist + local backup_dirs=( + "/mnt/share/media/backups/plex" + "/mnt/share/media/backups/docker-data" + "/mnt/share/media/backups/immich" + ) + + for dir in "${backup_dirs[@]}"; do + if [ ! -d "$dir" ]; then + issues+=("Missing backup directory: $dir") + ((health_score -= 15)) + fi + done + + # Check crontab backup system structure + local crontab_backup_dir="$SCRIPT_DIR/crontab-backups/$HOSTNAME" + if [ ! -d "$crontab_backup_dir" ]; then + issues+=("Missing crontab backup directory for $HOSTNAME: $crontab_backup_dir") + ((health_score -= 10)) + fi + + # Check recent backup activity + local recent_activity=false + for tag in "${BACKUP_TAGS[@]}"; do + if sudo journalctl --since "24 hours ago" -t "$tag" --output=cat | grep -q .; then + recent_activity=true + break + fi + done + + if [ "$recent_activity" = false ]; then + issues+=("No backup activity in the last 24 hours") + ((health_score -= 25)) + fi + + # Check for recent errors + local recent_errors=0 + for tag in "${BACKUP_TAGS[@]}"; do + local error_count=$(sudo journalctl --since "24 hours ago" -t "$tag" --grep="ERROR" --output=cat | wc -l) + ((recent_errors += error_count)) + done + + if [ "$recent_errors" -gt 0 ]; then + issues+=("$recent_errors errors in the last 24 hours") + ((health_score -= $((recent_errors * 5)))) + fi + + # Ensure health score doesn't go below 0 + if [ "$health_score" -lt 0 ]; then + health_score=0 + fi + + # Display results + if [ "$health_score" -ge 90 ]; then + echo -e "${GREEN}Backup System Health ($HOSTNAME): ${health_score}% - EXCELLENT${NC}" + elif [ "$health_score" -ge 70 ]; then + echo -e "${YELLOW}Backup System Health ($HOSTNAME): ${health_score}% - GOOD${NC}" + elif [ "$health_score" -ge 50 ]; then + echo -e "${YELLOW}Backup System Health ($HOSTNAME): ${health_score}% - FAIR${NC}" + else + echo -e "${RED}Backup System Health ($HOSTNAME): ${health_score}% - POOR${NC}" + fi + + if [ ${#issues[@]} -gt 0 ]; then + echo + log_warning "Issues found:" + for issue in "${issues[@]}"; do + echo -e " ${RED}• $issue${NC}" + done + + echo + log_info "Recommended actions:" + echo " • Run: ./manage-enhanced-crontab.sh verify" + echo " • Check system logs: sudo journalctl -xe" + echo " • Verify backup directories are mounted and accessible" + echo " • Run: ./crontab-backup-system.sh status" + fi +} + +show_service_status() { + log_info "Backup Service Status Overview" + echo + + printf "%-20s %-15s %-20s %-30s\n" "Service" "Status" "Last Activity" "Last Message" + printf "%-20s %-15s %-20s %-30s\n" "-------" "------" "-------------" "------------" + + for tag in "${BACKUP_TAGS[@]}"; do + local last_entry=$(sudo journalctl -t "$tag" --output=short-iso -n 1 2>/dev/null | tail -1) + + if [ -n "$last_entry" ]; then + local timestamp=$(echo "$last_entry" | cut -d' ' -f1-2) + local message=$(echo "$last_entry" | cut -d' ' -f4- | cut -c1-30) + + # Check if it's recent (within 48 hours) + local entry_time=$(date -d "$timestamp" +%s 2>/dev/null || echo "0") + local current_time=$(date +%s) + local hours_diff=$(( (current_time - entry_time) / 3600 )) + + local status + if [ "$hours_diff" -le 24 ]; then + status="${GREEN}Active${NC}" + elif [ "$hours_diff" -le 48 ]; then + status="${YELLOW}Recent${NC}" + else + status="${RED}Stale${NC}" + fi + + printf "%-20s %-25s %-20s %-30s\n" "$tag" "$status" "$timestamp" "$message" + else + printf "%-20s %-25s %-20s %-30s\n" "$tag" "${RED}No logs${NC}" "Never" "No activity found" + fi + done +} + +show_usage() { + echo -e "${PURPLE}Enhanced Backup Log Monitor${NC}" + echo + echo "Usage: $0 [COMMAND] [OPTIONS]" + echo + echo "Commands:" + echo " monitor Real-time monitoring of all backup logs" + echo " recent [HOURS] [SERVICE] Show recent logs (default: 24 hours, all services)" + echo " errors [DAYS] Show error summary (default: 7 days)" + echo " report [DAYS] Generate comprehensive report (default: 7 days)" + echo " health Check backup system health" + echo " status Show service status overview" + echo " help Show this help message" + echo + echo "Services:" + for tag in "${BACKUP_TAGS[@]}"; do + echo " - $tag" + done + echo + echo "Examples:" + echo " $0 monitor" + echo " $0 recent 12 plex-backup" + echo " $0 errors 3" + echo " $0 report 14" + echo +} + +# Main command handling +case "${1:-help}" in + monitor) + monitor_realtime + ;; + recent) + show_recent_logs "$2" "$3" + ;; + errors) + show_error_summary "$2" + ;; + report) + generate_backup_report "$2" + ;; + health) + check_backup_health + ;; + status) + show_service_status + ;; + help|*) + show_usage + ;; +esac diff --git a/crontab-backup-system.sh b/crontab-backup-system.sh new file mode 100755 index 0000000..da6557b --- /dev/null +++ b/crontab-backup-system.sh @@ -0,0 +1,732 @@ +#!/bin/bash + +# Crontab Backup and Recovery System +# This script provides comprehensive backup management for crontab entries + +set -e + +# Color codes for output +RED='\033[0;31m' +GREEN='\033[0;32m' +YELLOW='\033[1;33m' +BLUE='\033[0;34m' +CYAN='\033[0;36m' +PURPLE='\033[0;35m' +NC='\033[0m' # No Color + +SCRIPT_DIR="$(dirname "$(readlink -f "$0")")" +HOSTNAME=$(hostname) +BACKUP_ROOT="$SCRIPT_DIR/crontab-backups" +BACKUP_DIR="$BACKUP_ROOT/$HOSTNAME" +LOG_DIR="$SCRIPT_DIR/logs" +CURRENT_BACKUP="$BACKUP_DIR/current-crontab.backup" +ARCHIVE_DIR="$BACKUP_DIR/archive" + +# Ensure directories exist +mkdir -p "$BACKUP_DIR" "$ARCHIVE_DIR" "$LOG_DIR" + +log_message() { + local message="$1" + local log_file="$LOG_DIR/crontab-management.log" + echo -e "$(date '+%Y-%m-%d %H:%M:%S') $message" + echo "$(date '+%Y-%m-%d %H:%M:%S') $message" | sed 's/\x1b\[[0-9;]*m//g' >> "$log_file" +} + +log_error() { + log_message "${RED}ERROR: $1${NC}" +} + +log_success() { + log_message "${GREEN}SUCCESS: $1${NC}" +} + +log_warning() { + log_message "${YELLOW}WARNING: $1${NC}" +} + +log_info() { + log_message "${BLUE}INFO: $1${NC}" +} + +create_timestamped_backup() { + local backup_type="${1:-manual}" + local timestamp=$(date +%Y%m%d_%H%M%S) + local backup_file="$ARCHIVE_DIR/${HOSTNAME}-crontab-${backup_type}-${timestamp}.backup" + + log_info "Creating timestamped backup for $HOSTNAME: $backup_file" + + if sudo crontab -l > "$backup_file" 2>/dev/null; then + log_success "Backup created: $backup_file" + + # Also update the current backup + cp "$backup_file" "$CURRENT_BACKUP" + + # Add metadata + echo "# Backup created: $(date)" >> "$backup_file" + echo "# Backup type: $backup_type" >> "$backup_file" + echo "# System: $HOSTNAME" >> "$backup_file" + echo "# User: root" >> "$backup_file" + echo "# Full system info: $(uname -a)" >> "$backup_file" + + return 0 + else + log_error "Failed to create backup or no crontab exists" + return 1 + fi +} + +list_backups() { + local target_hostname="${1:-$HOSTNAME}" + local target_dir="$BACKUP_ROOT/$target_hostname/archive" + + log_info "Available crontab backups for $target_hostname:" + echo + + if [ -d "$target_dir" ] && [ "$(ls -A "$target_dir" 2>/dev/null)" ]; then + printf "%-40s %-20s %-15s\n" "Filename" "Date Created" "Size" + printf "%-40s %-20s %-15s\n" "--------" "------------" "----" + + for backup in "$target_dir"/*.backup; do + if [ -f "$backup" ]; then + local filename=$(basename "$backup") + local date_created=$(stat -c %y "$backup" | cut -d' ' -f1,2 | cut -d'.' -f1) + local size=$(stat -c %s "$backup") + printf "%-40s %-20s %-15s bytes\n" "$filename" "$date_created" "$size" + fi + done + else + log_warning "No backups found in $target_dir" + fi + echo + + # Show all available systems if current system has no backups or if showing all + if [ "$target_hostname" = "$HOSTNAME" ] && [ ! -d "$target_dir" ]; then + log_info "Available systems with backups:" + for system_dir in "$BACKUP_ROOT"/*; do + if [ -d "$system_dir/archive" ] && [ "$(ls -A "$system_dir/archive" 2>/dev/null)" ]; then + local system_name=$(basename "$system_dir") + local backup_count=$(ls -1 "$system_dir/archive"/*.backup 2>/dev/null | wc -l) + echo " - $system_name ($backup_count backups)" + fi + done + echo + echo "Use: $0 list [hostname] to view backups for a specific system" + fi +} + +restore_from_backup() { + local backup_file="$1" + local source_hostname="" + + if [ -z "$backup_file" ]; then + log_error "No backup file specified" + list_backups + return 1 + fi + + # Handle different backup file formats and paths + if [[ "$backup_file" == *"/"* ]]; then + # Full or relative path provided + if [[ ! "$backup_file" = /* ]]; then + backup_file="$ARCHIVE_DIR/$backup_file" + fi + else + # Just filename provided - check current system first, then others + if [ -f "$ARCHIVE_DIR/$backup_file" ]; then + backup_file="$ARCHIVE_DIR/$backup_file" + else + # Search in other system directories + local found_file="" + for system_dir in "$BACKUP_ROOT"/*; do + if [ -f "$system_dir/archive/$backup_file" ]; then + found_file="$system_dir/archive/$backup_file" + source_hostname=$(basename "$system_dir") + break + fi + done + + if [ -n "$found_file" ]; then + backup_file="$found_file" + log_warning "Backup file found in $source_hostname system directory" + fi + fi + fi + + if [ ! -f "$backup_file" ]; then + log_error "Backup file not found: $backup_file" + echo + log_info "Available backups:" + list_backups + return 1 + fi + + # Extract source hostname from backup metadata if available + if [ -z "$source_hostname" ]; then + source_hostname=$(grep "^# System:" "$backup_file" 2>/dev/null | cut -d' ' -f3 || echo "unknown") + fi + + log_info "Restoring crontab from: $backup_file" + if [ "$source_hostname" != "unknown" ] && [ "$source_hostname" != "$HOSTNAME" ]; then + log_warning "Restoring backup from different system: $source_hostname -> $HOSTNAME" + echo -n "Continue? [y/N]: " + read -r response + if [[ ! "$response" =~ ^[Yy]$ ]]; then + log_info "Restore cancelled" + return 1 + fi + fi + + # Create a safety backup before restoring + create_timestamped_backup "pre-restore" + + # Remove metadata lines before restoring + grep -v "^# Backup" "$backup_file" > /tmp/crontab_restore.tmp + + if sudo crontab /tmp/crontab_restore.tmp; then + log_success "Crontab restored successfully from $backup_file" + rm -f /tmp/crontab_restore.tmp + else + log_error "Failed to restore crontab" + rm -f /tmp/crontab_restore.tmp + return 1 + fi +} + +compare_crontabs() { + local file1="${1:-current}" + local file2="$2" + + if [ "$file1" = "current" ]; then + sudo crontab -l > /tmp/current_crontab.tmp 2>/dev/null || echo "# No current crontab" > /tmp/current_crontab.tmp + file1="/tmp/current_crontab.tmp" + fi + + if [ -z "$file2" ]; then + file2="$CURRENT_BACKUP" + fi + + # Handle relative paths and cross-system backups + if [[ ! "$file2" = /* ]]; then + # Check current system first + if [ -f "$ARCHIVE_DIR/$file2" ]; then + file2="$ARCHIVE_DIR/$file2" + else + # Search in other system directories + local found_file="" + for system_dir in "$BACKUP_ROOT"/*; do + if [ -f "$system_dir/archive/$file2" ]; then + found_file="$system_dir/archive/$file2" + local source_hostname=$(basename "$system_dir") + log_info "Found backup in $source_hostname system directory" + break + fi + done + + if [ -n "$found_file" ]; then + file2="$found_file" + else + file2="$ARCHIVE_DIR/$file2" # Default back to current system + fi + fi + fi + + if [ ! -f "$file2" ]; then + log_error "Comparison file not found: $file2" + return 1 + fi + + log_info "Comparing current crontab ($HOSTNAME) with: $(basename "$file2")" + echo + + # Clean comparison files (remove metadata) + grep -v "^# Backup" "$file1" > /tmp/clean_file1.tmp 2>/dev/null || touch /tmp/clean_file1.tmp + grep -v "^# Backup" "$file2" > /tmp/clean_file2.tmp 2>/dev/null || touch /tmp/clean_file2.tmp + + if diff -u /tmp/clean_file1.tmp /tmp/clean_file2.tmp; then + log_success "Crontabs are identical" + else + log_warning "Crontabs differ (see above)" + fi + + # Cleanup + rm -f /tmp/current_crontab.tmp /tmp/clean_file1.tmp /tmp/clean_file2.tmp +} + +validate_crontab_syntax() { + local crontab_file="${1:-current}" + + if [ "$crontab_file" = "current" ]; then + sudo crontab -l > /tmp/validate_crontab.tmp 2>/dev/null || echo "# No current crontab" > /tmp/validate_crontab.tmp + crontab_file="/tmp/validate_crontab.tmp" + fi + + if [ ! -f "$crontab_file" ]; then + log_error "Crontab file not found: $crontab_file" + return 1 + fi + + log_info "Validating crontab syntax: $crontab_file" + + local line_num=0 + local errors=0 + + while IFS= read -r line; do + ((line_num++)) + + # Skip comments and empty lines + if [[ "$line" =~ ^[[:space:]]*# ]] || [[ "$line" =~ ^[[:space:]]*$ ]]; then + continue + fi + + # Basic cron format validation + if [[ ! "$line" =~ ^[[:space:]]*([0-9*,-]+[[:space:]]+){4}[0-9*,-]+[[:space:]].+ ]]; then + log_error "Line $line_num: Invalid cron format: $line" + ((errors++)) + fi + + # Check for common issues + if [[ "$line" =~ \$\? ]] && [[ ! "$line" =~ \{ ]]; then + log_warning "Line $line_num: \$? outside of command group may not work as expected" + fi + + done < "$crontab_file" + + if [ $errors -eq 0 ]; then + log_success "Crontab syntax validation passed" + else + log_error "Found $errors syntax errors" + fi + + rm -f /tmp/validate_crontab.tmp + return $errors +} + +cleanup_old_backups() { + local keep_days="${1:-30}" + local target_hostname="${2:-$HOSTNAME}" + local deleted_count=0 + + if [ "$target_hostname" = "all" ]; then + log_info "Cleaning up backups older than $keep_days days for all systems" + + for system_dir in "$BACKUP_ROOT"/*; do + if [ -d "$system_dir/archive" ]; then + local system_name=$(basename "$system_dir") + log_info "Cleaning backups for $system_name" + + while IFS= read -r -d '' backup; do + if [ -f "$backup" ]; then + rm "$backup" + ((deleted_count++)) + log_info "Deleted old backup: $(basename "$backup") from $system_name" + fi + done < <(find "$system_dir/archive" -name "*.backup" -mtime +$keep_days -print0 2>/dev/null) + fi + done + else + local target_dir="$BACKUP_ROOT/$target_hostname/archive" + log_info "Cleaning up backups older than $keep_days days for $target_hostname" + + if [ -d "$target_dir" ]; then + while IFS= read -r -d '' backup; do + if [ -f "$backup" ]; then + rm "$backup" + ((deleted_count++)) + log_info "Deleted old backup: $(basename "$backup")" + fi + done < <(find "$target_dir" -name "*.backup" -mtime +$keep_days -print0 2>/dev/null) + else + log_warning "Backup directory not found: $target_dir" + fi + fi + + if [ $deleted_count -eq 0 ]; then + log_info "No old backups found to clean up" + else + log_success "Cleaned up $deleted_count old backup(s)" + fi +} + +setup_automated_backup() { + log_info "Setting up automated daily crontab backup" + + local backup_script="$SCRIPT_DIR/crontab-backup-system.sh" + local backup_entry="0 0 * * * $backup_script backup auto --auto-cleanup 2>&1 | logger -t crontab-backup -p user.info" + + # Check if backup entry already exists + if sudo crontab -l 2>/dev/null | grep -q "crontab-backup-system.sh"; then + log_warning "Automated backup entry already exists" + return 0 + fi + + # Add the backup entry to current crontab + (sudo crontab -l 2>/dev/null; echo "$backup_entry") | sudo crontab - + + log_success "Automated daily backup configured for $HOSTNAME" + log_info "Backups will run daily at midnight and be logged to syslog" +} + +migrate_legacy_backups() { + local legacy_dir="$SCRIPT_DIR/crontab-backups" + local legacy_archive="$legacy_dir/archive" + + # Check if legacy structure exists (without hostname subdirectory) + if [ -d "$legacy_archive" ] && [ "$legacy_dir" != "$BACKUP_DIR" ]; then + log_info "Found legacy backup structure, migrating to hostname-based structure" + + # Create new structure + mkdir -p "$BACKUP_DIR" "$ARCHIVE_DIR" + + # Move backups and rename them to include hostname + local migrated_count=0 + for backup in "$legacy_archive"/*.backup; do + if [ -f "$backup" ]; then + local filename=$(basename "$backup") + local new_filename="${HOSTNAME}-${filename}" + + if cp "$backup" "$ARCHIVE_DIR/$new_filename"; then + log_success "Migrated: $filename -> $new_filename" + ((migrated_count++)) + else + log_error "Failed to migrate: $filename" + fi + fi + done + + # Move current backup if it exists + if [ -f "$legacy_dir/current-crontab.backup" ]; then + cp "$legacy_dir/current-crontab.backup" "$CURRENT_BACKUP" + log_success "Migrated current backup" + fi + + if [ $migrated_count -gt 0 ]; then + log_success "Migrated $migrated_count backup(s) to new structure" + echo + log_warning "Legacy backups remain in $legacy_archive" + log_info "You can safely remove the legacy directory after verifying the migration" + echo " rm -rf '$legacy_dir'" + fi + fi +} + +list_all_systems() { + log_info "All systems with crontab backups:" + echo + + if [ ! -d "$BACKUP_ROOT" ]; then + log_warning "No backup root directory found: $BACKUP_ROOT" + return 1 + fi + + printf "%-15s %-10s %-20s %-30s\n" "System" "Backups" "Latest Backup" "Status" + printf "%-15s %-10s %-20s %-30s\n" "------" "-------" "-------------" "------" + + local found_systems=false + for system_dir in "$BACKUP_ROOT"/*; do + if [ -d "$system_dir" ]; then + local system_name=$(basename "$system_dir") + + # Skip legacy archive directory - it's not a system + if [ "$system_name" = "archive" ]; then + continue + fi + + found_systems=true + local backup_count=$(ls -1 "$system_dir/archive"/*.backup 2>/dev/null | wc -l || echo "0") + + local latest_backup="None" + local status="Inactive" + + if [ -f "$system_dir/current-crontab.backup" ]; then + latest_backup=$(stat -c %y "$system_dir/current-crontab.backup" | cut -d' ' -f1) + + # Check if backup is recent (within 7 days) + local backup_age=$(( ($(date +%s) - $(stat -c %Y "$system_dir/current-crontab.backup")) / 86400 )) + if [ $backup_age -le 7 ]; then + status="Active" + elif [ $backup_age -le 30 ]; then + status="Recent" + else + status="Stale" + fi + fi + + # Use printf with color formatting + if [ "$status" = "Active" ]; then + printf "%-15s %-10s %-20s ${GREEN}%-30s${NC}\n" "$system_name" "$backup_count" "$latest_backup" "$status" + elif [ "$status" = "Recent" ]; then + printf "%-15s %-10s %-20s ${YELLOW}%-30s${NC}\n" "$system_name" "$backup_count" "$latest_backup" "$status" + elif [ "$status" = "Stale" ]; then + printf "%-15s %-10s %-20s ${RED}%-30s${NC}\n" "$system_name" "$backup_count" "$latest_backup" "$status" + else + printf "%-15s %-10s %-20s %-30s\n" "$system_name" "$backup_count" "$latest_backup" "$status" + fi + fi + done + + if [ "$found_systems" = false ]; then + log_warning "No systems found with backups" + fi + echo +} + +show_status() { + local target_hostname="${1:-$HOSTNAME}" + + if [ "$target_hostname" = "all" ]; then + log_info "Crontab Backup System Status - All Systems" + echo + + for system_dir in "$BACKUP_ROOT"/*; do + if [ -d "$system_dir" ]; then + local system_name=$(basename "$system_dir") + + # Skip legacy archive directory - it's not a system + if [ "$system_name" = "archive" ]; then + continue + fi + + echo -e "${CYAN}=== $system_name ===${NC}" + + local backup_count=$(ls -1 "$system_dir/archive"/*.backup 2>/dev/null | wc -l || echo "0") + echo " - Total backups: $backup_count" + echo " - Backup directory: $system_dir" + + if [ -f "$system_dir/current-crontab.backup" ]; then + echo " - Latest backup: $(stat -c %y "$system_dir/current-crontab.backup" | cut -d'.' -f1)" + else + echo " - Latest backup: None" + fi + echo + fi + done + else + log_info "Crontab Backup System Status - $target_hostname" + echo + + # Current crontab info + if [ "$target_hostname" = "$HOSTNAME" ]; then + local cron_count=$(sudo crontab -l 2>/dev/null | grep -c "^[^#]" || echo "0") + echo -e "${CYAN}Current Crontab ($HOSTNAME):${NC}" + echo " - Active entries: $cron_count" + echo " - Last modified: $(stat -c %y /var/spool/cron/crontabs/root 2>/dev/null | cut -d'.' -f1 || echo "Unknown")" + echo + fi + + # Backup info for specified system + local target_dir="$BACKUP_ROOT/$target_hostname" + local backup_count=$(ls -1 "$target_dir/archive"/*.backup 2>/dev/null | wc -l || echo "0") + echo -e "${CYAN}Backups ($target_hostname):${NC}" + echo " - Total backups: $backup_count" + echo " - Backup directory: $target_dir" + if [ -f "$target_dir/current-crontab.backup" ]; then + echo " - Latest backup: $(stat -c %y "$target_dir/current-crontab.backup" | cut -d'.' -f1)" + else + echo " - Latest backup: None" + fi + echo + fi + + # Log monitoring + echo -e "${CYAN}Log Monitoring:${NC}" + echo " - Management log: $LOG_DIR/crontab-management.log" + echo " - System logs: journalctl -t crontab-backup" + echo +} + +show_usage() { + echo -e "${PURPLE}Crontab Backup and Recovery System (Multi-System)${NC}" + echo + echo "Usage: $0 [COMMAND] [OPTIONS]" + echo + echo "Commands:" + echo " backup [TYPE] Create a timestamped backup (default: manual)" + echo " list [HOSTNAME] List backups for hostname (default: current system)" + echo " list-systems Show all systems with backups" + echo " restore FILE Restore crontab from backup file" + echo " compare [FILE1] [FILE2] Compare current crontab with backup" + echo " validate [FILE] Validate crontab syntax" + echo " cleanup [DAYS] [HOSTNAME] Clean up old backups (default: 30 days, current system)" + echo " status [HOSTNAME|all] Show system status (default: current system)" + echo " setup-auto Setup automated daily backups" + echo " migrate Migrate legacy backups to hostname structure" + echo " import FILE SYSTEM [TYPE] Import backup from external source" + echo " create-test-systems Create test systems for demonstration" + echo " help Show this help message" + echo + echo "Multi-System Examples:" + echo " $0 backup pre-upgrade # Backup current system" + echo " $0 list io # List backups for 'io' system" + echo " $0 restore io-crontab-manual-20250526_120000.backup" + echo " $0 compare current europa-crontab-manual-20250526_120000.backup" + echo " $0 cleanup 7 all # Clean up all systems" + echo " $0 status all # Show status for all systems" + echo " $0 list-systems # Show all available systems" + echo " $0 import /path/to/crontab.backup io manual # Import backup for io system" + echo + echo "Current System: $HOSTNAME" + echo "Backup Root: $BACKUP_ROOT" + echo +} + +create_test_systems() { + log_info "Creating test backup structure for io, europa, and racknerd systems" + + # Create sample systems directories + local test_systems=("io" "racknerd") + + for system in "${test_systems[@]}"; do + local system_dir="$BACKUP_ROOT/$system" + local system_archive="$system_dir/archive" + + mkdir -p "$system_archive" + + # Create a sample backup for each system + local timestamp=$(date +%Y%m%d_%H%M%S) + local sample_backup="$system_archive/${system}-crontab-sample-${timestamp}.backup" + + # Create sample crontab content for each system + case "$system" in + "io") + cat > "$sample_backup" << EOF +# Sample crontab for io system +0 2 * * * /home/user/backup-docker.sh 2>&1 | logger -t docker-backup -p user.info +30 3 * * * /home/user/backup-media.sh 2>&1 | logger -t media-backup -p user.info +0 4 * * * /home/user/validate-backups.sh 2>&1 | logger -t backup-validation -p user.info +# Backup created: $(date) +# Backup type: sample +# System: $system +# User: root +# Full system info: Linux $system 5.15.0-generic #1 SMP x86_64 GNU/Linux +EOF + ;; + "racknerd") + cat > "$sample_backup" << EOF +# Sample crontab for racknerd system +0 1 * * * /home/user/backup-plex.sh 2>&1 | logger -t plex-backup -p user.info +15 1 * * * /home/user/move-backups.sh 2>&1 | logger -t backup-move -p user.info +0 5 * * 0 /home/user/cleanup-old-backups.sh 2>&1 | logger -t backup-cleanup -p user.info +# Backup created: $(date) +# Backup type: sample +# System: $system +# User: root +# Full system info: Linux $system 5.4.0-generic #1 SMP x86_64 GNU/Linux +EOF + ;; + esac + + # Create current backup file + cp "$sample_backup" "$system_dir/current-crontab.backup" + + log_success "Created test system: $system with sample backup" + done + + log_success "Test systems created successfully" +} + +import_backup() { + local source_file="$1" + local source_system="$2" + local backup_type="${3:-imported}" + + if [ -z "$source_file" ] || [ -z "$source_system" ]; then + log_error "Usage: import_backup [backup_type]" + return 1 + fi + + if [ ! -f "$source_file" ]; then + log_error "Source file not found: $source_file" + return 1 + fi + + local target_dir="$BACKUP_ROOT/$source_system" + local target_archive="$target_dir/archive" + + mkdir -p "$target_archive" + + local timestamp=$(date +%Y%m%d_%H%M%S) + local target_file="$target_archive/${source_system}-crontab-${backup_type}-${timestamp}.backup" + + # Copy and add metadata + cp "$source_file" "$target_file" + + # Add metadata if not present + if ! grep -q "^# Backup created:" "$target_file"; then + cat >> "$target_file" << EOF +# Backup created: $(date) +# Backup type: $backup_type +# System: $source_system +# User: root +# Imported from: $source_file +EOF + fi + + # Update current backup + cp "$target_file" "$target_dir/current-crontab.backup" + + log_success "Imported backup for $source_system: $target_file" +} + +# Main command handling +case "${1:-help}" in + backup) + # Check for auto-cleanup flag + if [[ "${@}" == *"--auto-cleanup"* ]]; then + create_timestamped_backup "${2:-auto}" + cleanup_old_backups 30 + else + create_timestamped_backup "${2:-manual}" + fi + ;; + list) + list_backups "$2" + ;; + list-systems) + list_all_systems + ;; + restore) + if [ -z "$2" ]; then + log_error "Please specify a backup file to restore" + list_backups + exit 1 + fi + restore_from_backup "$2" + ;; + compare) + compare_crontabs "$2" "$3" + ;; + validate) + validate_crontab_syntax "$2" + ;; + cleanup) + cleanup_old_backups "${2:-30}" "${3:-$HOSTNAME}" + ;; + status) + show_status "${2:-$HOSTNAME}" + ;; + setup-auto) + setup_automated_backup + ;; + migrate) + migrate_legacy_backups + ;; + import) + if [ -z "$2" ] || [ -z "$3" ]; then + log_error "Please specify source file and target system" + echo "Usage: $0 import [backup_type]" + exit 1 + fi + import_backup "$2" "$3" "$4" + ;; + create-test-systems) + create_test_systems + ;; + help|*) + show_usage + ;; +esac + +# Auto-migrate legacy backups on first run +if [ ! -d "$BACKUP_DIR" ] && [ "$1" != "help" ] && [ "$1" != "migrate" ]; then + migrate_legacy_backups +fi diff --git a/crontab-backups/archive/crontab-pre-install-20250526_100622.backup b/crontab-backups/archive/crontab-pre-install-20250526_100622.backup new file mode 100644 index 0000000..f70c350 --- /dev/null +++ b/crontab-backups/archive/crontab-pre-install-20250526_100622.backup @@ -0,0 +1,40 @@ +# Edit this file to introduce tasks to be run by cron. +# +# Each task to run has to be defined through a single line +# indicating with different fields when the task will be run +# and what command to run for the task +# +# To define the time you can provide concrete values for +# minute (m), hour (h), day of month (dom), month (mon), +# and day of week (dow) or use '*' in these fields (for 'any'). +# +# Notice that tasks will be started based on the cron's system +# daemon's notion of time and timezones. +# +# Output of the crontab jobs (including errors) is sent through +# email to the user the crontab file belongs to (unless redirected). +# +# For example, you can run a backup of all your user accounts +# at 5 a.m every week with: +# 0 5 * * 1 tar -zcf /var/backups/home.tgz /home/ +# +# For more information see the manual pages of crontab(5) and cron(8) +# +# m h dom mon dow command +# 2023-06-11 I can't figure out how to get the nas to mount automagically so here is a band-aid +# @reboot sleep 30 && mount -a +# +# move the files previously backed up at 0100 +0 1 * * * /home/acedanger/shell/move-backups.sh +# daily Plex backup at 0415 +15 4 * * * /home/acedanger/shell/backup-plex.sh +# daily validation at 0700 +0 7 * * * /home/acedanger/shell/validate-plex-backups.sh --fix +# backup immich database +0 5 * * 1 mv /mnt/share/media/immich/uploads/backups/immich-db-backup* /mnt/share/media/backups/immich +# Generate detailed weekly report (Sundays at 0800) +0 8 * * 0 /home/acedanger/shell/validate-plex-backups.sh --report +# Backup created: Mon May 26 10:06:22 AM EDT 2025 +# Backup type: pre-install +# System: europa +# User: root diff --git a/crontab-backups/europa/archive/europa-crontab-initial-20250526_101354.backup b/crontab-backups/europa/archive/europa-crontab-initial-20250526_101354.backup new file mode 100644 index 0000000..6e4fe5c --- /dev/null +++ b/crontab-backups/europa/archive/europa-crontab-initial-20250526_101354.backup @@ -0,0 +1,41 @@ +# Edit this file to introduce tasks to be run by cron. +# +# Each task to run has to be defined through a single line +# indicating with different fields when the task will be run +# and what command to run for the task +# +# To define the time you can provide concrete values for +# minute (m), hour (h), day of month (dom), month (mon), +# and day of week (dow) or use '*' in these fields (for 'any'). +# +# Notice that tasks will be started based on the cron's system +# daemon's notion of time and timezones. +# +# Output of the crontab jobs (including errors) is sent through +# email to the user the crontab file belongs to (unless redirected). +# +# For example, you can run a backup of all your user accounts +# at 5 a.m every week with: +# 0 5 * * 1 tar -zcf /var/backups/home.tgz /home/ +# +# For more information see the manual pages of crontab(5) and cron(8) +# +# m h dom mon dow command +# 2023-06-11 I can't figure out how to get the nas to mount automagically so here is a band-aid +# @reboot sleep 30 && mount -a +# +# move the files previously backed up at 0100 +0 1 * * * /home/acedanger/shell/move-backups.sh +# daily Plex backup at 0415 +15 4 * * * /home/acedanger/shell/backup-plex.sh +# daily validation at 0700 +0 7 * * * /home/acedanger/shell/validate-plex-backups.sh --fix +# backup immich database +0 5 * * 1 mv /mnt/share/media/immich/uploads/backups/immich-db-backup* /mnt/share/media/backups/immich +# Generate detailed weekly report (Sundays at 0800) +0 8 * * 0 /home/acedanger/shell/validate-plex-backups.sh --report +# Backup created: Mon May 26 10:13:54 AM EDT 2025 +# Backup type: initial +# System: europa +# User: root +# Full system info: Linux europa 6.1.0-33-amd64 #1 SMP PREEMPT_DYNAMIC Debian 6.1.133-1 (2025-04-10) x86_64 GNU/Linux diff --git a/crontab-backups/europa/archive/europa-crontab-pre-install-20250526_100622.backup b/crontab-backups/europa/archive/europa-crontab-pre-install-20250526_100622.backup new file mode 100644 index 0000000..f70c350 --- /dev/null +++ b/crontab-backups/europa/archive/europa-crontab-pre-install-20250526_100622.backup @@ -0,0 +1,40 @@ +# Edit this file to introduce tasks to be run by cron. +# +# Each task to run has to be defined through a single line +# indicating with different fields when the task will be run +# and what command to run for the task +# +# To define the time you can provide concrete values for +# minute (m), hour (h), day of month (dom), month (mon), +# and day of week (dow) or use '*' in these fields (for 'any'). +# +# Notice that tasks will be started based on the cron's system +# daemon's notion of time and timezones. +# +# Output of the crontab jobs (including errors) is sent through +# email to the user the crontab file belongs to (unless redirected). +# +# For example, you can run a backup of all your user accounts +# at 5 a.m every week with: +# 0 5 * * 1 tar -zcf /var/backups/home.tgz /home/ +# +# For more information see the manual pages of crontab(5) and cron(8) +# +# m h dom mon dow command +# 2023-06-11 I can't figure out how to get the nas to mount automagically so here is a band-aid +# @reboot sleep 30 && mount -a +# +# move the files previously backed up at 0100 +0 1 * * * /home/acedanger/shell/move-backups.sh +# daily Plex backup at 0415 +15 4 * * * /home/acedanger/shell/backup-plex.sh +# daily validation at 0700 +0 7 * * * /home/acedanger/shell/validate-plex-backups.sh --fix +# backup immich database +0 5 * * 1 mv /mnt/share/media/immich/uploads/backups/immich-db-backup* /mnt/share/media/backups/immich +# Generate detailed weekly report (Sundays at 0800) +0 8 * * 0 /home/acedanger/shell/validate-plex-backups.sh --report +# Backup created: Mon May 26 10:06:22 AM EDT 2025 +# Backup type: pre-install +# System: europa +# User: root diff --git a/crontab-backups/europa/archive/europa-crontab-test-colors-20250526_103005.backup b/crontab-backups/europa/archive/europa-crontab-test-colors-20250526_103005.backup new file mode 100644 index 0000000..8a89404 --- /dev/null +++ b/crontab-backups/europa/archive/europa-crontab-test-colors-20250526_103005.backup @@ -0,0 +1,41 @@ +# Edit this file to introduce tasks to be run by cron. +# +# Each task to run has to be defined through a single line +# indicating with different fields when the task will be run +# and what command to run for the task +# +# To define the time you can provide concrete values for +# minute (m), hour (h), day of month (dom), month (mon), +# and day of week (dow) or use '*' in these fields (for 'any'). +# +# Notice that tasks will be started based on the cron's system +# daemon's notion of time and timezones. +# +# Output of the crontab jobs (including errors) is sent through +# email to the user the crontab file belongs to (unless redirected). +# +# For example, you can run a backup of all your user accounts +# at 5 a.m every week with: +# 0 5 * * 1 tar -zcf /var/backups/home.tgz /home/ +# +# For more information see the manual pages of crontab(5) and cron(8) +# +# m h dom mon dow command +# 2023-06-11 I can't figure out how to get the nas to mount automagically so here is a band-aid +# @reboot sleep 30 && mount -a +# +# move the files previously backed up at 0100 +0 1 * * * /home/acedanger/shell/move-backups.sh +# daily Plex backup at 0415 +15 4 * * * /home/acedanger/shell/backup-plex.sh +# daily validation at 0700 +0 7 * * * /home/acedanger/shell/validate-plex-backups.sh --fix +# backup immich database +0 5 * * 1 mv /mnt/share/media/immich/uploads/backups/immich-db-backup* /mnt/share/media/backups/immich +# Generate detailed weekly report (Sundays at 0800) +0 8 * * 0 /home/acedanger/shell/validate-plex-backups.sh --report +# Backup created: Mon May 26 10:30:05 AM EDT 2025 +# Backup type: test-colors +# System: europa +# User: root +# Full system info: Linux europa 6.1.0-33-amd64 #1 SMP PREEMPT_DYNAMIC Debian 6.1.133-1 (2025-04-10) x86_64 GNU/Linux diff --git a/crontab-backups/europa/current-crontab.backup b/crontab-backups/europa/current-crontab.backup new file mode 100644 index 0000000..7d47a76 --- /dev/null +++ b/crontab-backups/europa/current-crontab.backup @@ -0,0 +1,36 @@ +# Edit this file to introduce tasks to be run by cron. +# +# Each task to run has to be defined through a single line +# indicating with different fields when the task will be run +# and what command to run for the task +# +# To define the time you can provide concrete values for +# minute (m), hour (h), day of month (dom), month (mon), +# and day of week (dow) or use '*' in these fields (for 'any'). +# +# Notice that tasks will be started based on the cron's system +# daemon's notion of time and timezones. +# +# Output of the crontab jobs (including errors) is sent through +# email to the user the crontab file belongs to (unless redirected). +# +# For example, you can run a backup of all your user accounts +# at 5 a.m every week with: +# 0 5 * * 1 tar -zcf /var/backups/home.tgz /home/ +# +# For more information see the manual pages of crontab(5) and cron(8) +# +# m h dom mon dow command +# 2023-06-11 I can't figure out how to get the nas to mount automagically so here is a band-aid +# @reboot sleep 30 && mount -a +# +# move the files previously backed up at 0100 +0 1 * * * /home/acedanger/shell/move-backups.sh +# daily Plex backup at 0415 +15 4 * * * /home/acedanger/shell/backup-plex.sh +# daily validation at 0700 +0 7 * * * /home/acedanger/shell/validate-plex-backups.sh --fix +# backup immich database +0 5 * * 1 mv /mnt/share/media/immich/uploads/backups/immich-db-backup* /mnt/share/media/backups/immich +# Generate detailed weekly report (Sundays at 0800) +0 8 * * 0 /home/acedanger/shell/validate-plex-backups.sh --report diff --git a/docs/enhanced-crontab-system.md b/docs/enhanced-crontab-system.md new file mode 100644 index 0000000..c11cbea --- /dev/null +++ b/docs/enhanced-crontab-system.md @@ -0,0 +1,226 @@ +# Enhanced Crontab and Backup Monitoring System + +## Overview + +This enhanced system provides comprehensive crontab management with automatic backups, system logging, and advanced monitoring capabilities for your backup operations. + +## Components + +### 1. Enhanced Crontab Entries (`enhanced-crontab.txt`) + +- **Move backups** (01:00): Transfers Docker backups with logging +- **Plex backup** (04:15): Daily Plex database backup with auto-repair +- **Plex validation** (07:00): Validates and fixes backup integrity +- **Immich backup** (05:00 Mon): Weekly Immich database backup move +- **Weekly report** (08:00 Sun): Comprehensive backup status report + +### 2. Crontab Management (`manage-enhanced-crontab.sh`) + +- Install enhanced crontab entries with validation +- Verify all backup scripts exist and are executable +- Setup automated backups and log rotation +- Integration with backup and monitoring systems + +### 3. Crontab Backup System (`crontab-backup-system.sh`) + +- Automated timestamped crontab backups +- Backup comparison and restoration +- Syntax validation +- Automated cleanup of old backups +- Daily automated backup scheduling + +### 4. Backup Log Monitor (`backup-log-monitor.sh`) + +- Real-time log monitoring with color coding +- Error analysis and reporting +- System health checks +- Comprehensive backup reports +- Service status overview + +## Quick Start + +### 1. Install the Enhanced System + +```bash +# Make scripts executable +chmod +x manage-enhanced-crontab.sh crontab-backup-system.sh backup-log-monitor.sh + +# Install enhanced crontab with all features +sudo ./manage-enhanced-crontab.sh install +``` + +### 2. Monitor Your Backups + +```bash +# Real-time monitoring +./backup-log-monitor.sh monitor + +# Check system health +./backup-log-monitor.sh health + +# View recent activity +./backup-log-monitor.sh recent 24 + +# Generate weekly report +./backup-log-monitor.sh report 7 +``` + +### 3. Manage Crontab Backups + +```bash +# Create manual backup +./crontab-backup-system.sh backup manual + +# List all backups +./crontab-backup-system.sh list + +# Compare current with backup +./crontab-backup-system.sh compare current + +# System status +./crontab-backup-system.sh status +``` + +## Features + +### Enhanced Logging + +All backup operations now log to syslog with specific tags: + +- `plex-backup`: Plex database backup operations +- `backup-move`: Docker backup file transfers +- `plex-validation`: Backup integrity checks +- `immich-backup`: Immich database operations +- `plex-report`: Weekly reporting +- `crontab-backup`: Crontab backup operations + +### Automatic Backups + +- **Crontab backups**: Daily automated backups at midnight +- **Cleanup**: Automatic removal of backups older than 30 days +- **Validation**: Syntax checking before applying changes +- **Recovery**: Easy restoration from any backup point + +### Health Monitoring + +- Script existence and permissions +- Backup directory availability +- Recent activity tracking +- Error rate monitoring +- Overall system health scoring + +### Error Handling + +- Graceful failure handling in cron jobs +- Detailed error logging and reporting +- Exit code tracking for debugging +- Comprehensive error summaries + +## Log Analysis + +### View Real-time Logs + +```bash +# All backup services +sudo journalctl -f -t plex-backup -t backup-move -t plex-validation -t immich-backup -t plex-report + +# Specific service +sudo journalctl -f -t plex-backup + +# With our monitor (recommended) +./backup-log-monitor.sh monitor +``` + +### Historical Analysis + +```bash +# Last 24 hours +sudo journalctl --since '24 hours ago' -t plex-backup + +# Last week with errors only +sudo journalctl --since '1 week ago' --priority=err -t plex-backup + +# Using our tools +./backup-log-monitor.sh recent 24 plex-backup +./backup-log-monitor.sh errors 7 +``` + +## Backup Recovery + +### Restore Crontab from Backup + +```bash +# List available backups +./crontab-backup-system.sh list + +# Restore specific backup +./crontab-backup-system.sh restore crontab-manual-20250526_120000.backup + +# Compare before restoring +./crontab-backup-system.sh compare current crontab-manual-20250526_120000.backup +``` + +### Emergency Recovery + +If you need to quickly restore the original crontab: + +```bash +# The system automatically creates pre-install backups +./crontab-backup-system.sh list | grep pre-install +./crontab-backup-system.sh restore [backup-filename] +``` + +## Maintenance + +### Regular Tasks + +```bash +# Weekly health check +./backup-log-monitor.sh health + +# Monthly backup cleanup +./crontab-backup-system.sh cleanup 30 + +# Quarterly comprehensive report +./backup-log-monitor.sh report 90 +``` + +### Troubleshooting + +```bash +# Verify all components +./manage-enhanced-crontab.sh verify + +# Check system status +./manage-enhanced-crontab.sh status + +# View configuration +./manage-enhanced-crontab.sh show + +# Monitor for issues +./backup-log-monitor.sh monitor +``` + +## Integration Notes + +- All scripts follow the established shell repository coding standards +- Logging uses consistent tags and priorities +- Error handling preserves backup integrity +- Color-coded output for better readability +- Comprehensive documentation and help systems + +## Security Considerations + +- Scripts validate input and handle errors gracefully +- Backup files include metadata for tracking +- Permissions are properly managed +- Sensitive operations require sudo confirmation +- All operations are logged for audit trails + +## Next Steps + +1. **Install the system**: Run `sudo ./manage-enhanced-crontab.sh install` +2. **Test monitoring**: Use `./backup-log-monitor.sh monitor` during next backup +3. **Review reports**: Generate weekly reports to establish baseline +4. **Set up alerts**: Consider integrating with your notification system +5. **Document customizations**: Add any local modifications to this guide diff --git a/docs/multi-system-crontab-management.md b/docs/multi-system-crontab-management.md new file mode 100644 index 0000000..1459c40 --- /dev/null +++ b/docs/multi-system-crontab-management.md @@ -0,0 +1,280 @@ +# Multi-System Crontab Management Guide + +## Overview + +The enhanced crontab backup system now supports managing crontab backups across multiple systems using a hostname-based directory structure. This enables centralized backup management for distributed environments. + +## System Architecture + +### Directory Structure + +``` +crontab-backups/ +├── europa/ # Current system (example) +│ ├── current-crontab.backup +│ └── archive/ +│ ├── europa-crontab-initial-20250526_101354.backup +│ └── europa-crontab-pre-install-20250526_100622.backup +├── io/ # Remote system backups +│ ├── current-crontab.backup +│ └── archive/ +│ └── io-crontab-sample-20250526_101558.backup +└── racknerd/ # Another remote system + ├── current-crontab.backup + └── archive/ + └── racknerd-crontab-sample-20250526_101558.backup +``` + +### File Naming Convention + +- Format: `{hostname}-crontab-{type}-{timestamp}.backup` +- Examples: + - `europa-crontab-manual-20250526_101354.backup` + - `io-crontab-pre-upgrade-20250526_120000.backup` + - `racknerd-crontab-auto-20250526_000001.backup` + +## Features + +### 🔄 Multi-System Support + +- **Hostname-based organization**: Each system gets its own directory +- **Cross-system operations**: View, compare, and restore backups from any system +- **Centralized management**: Manage all systems from a single location + +### 📊 System Status Monitoring + +```bash +# View status for current system +./crontab-backup-system.sh status + +# View status for specific system +./crontab-backup-system.sh status io + +# View status for all systems +./crontab-backup-system.sh status all +``` + +### 🗂️ Backup Operations + +```bash +# Create backup on current system +./crontab-backup-system.sh backup pre-upgrade + +# List backups for specific system +./crontab-backup-system.sh list io + +# List all systems with backups +./crontab-backup-system.sh list-systems + +# Import backup from external source +./crontab-backup-system.sh import /path/to/backup.txt io manual +``` + +### 🔍 Cross-System Comparison + +```bash +# Compare current crontab with backup from another system +./crontab-backup-system.sh compare current io-crontab-sample-20250526_101558.backup + +# Compare two backups from different systems +./crontab-backup-system.sh compare europa-crontab-manual-20250526_101354.backup racknerd-crontab-sample-20250526_101558.backup +``` + +### 🧹 Cleanup Management + +```bash +# Clean up backups older than 30 days for current system +./crontab-backup-system.sh cleanup 30 + +# Clean up backups for specific system +./crontab-backup-system.sh cleanup 7 io + +# Clean up backups for all systems +./crontab-backup-system.sh cleanup 30 all +``` + +## Enhanced Logging Integration + +All backup operations now integrate with system logging: + +### Syslog Integration + +- **Tag-based logging**: Each operation uses specific syslog tags +- **Priority levels**: Different priorities for info, warnings, and errors +- **Performance monitoring**: Execution time tracking for all operations + +### Example Enhanced Crontab Entries + +```bash +# Plex backup with comprehensive logging +15 4 * * * /home/acedanger/shell/backup-plex.sh 2>&1 | logger -t plex-backup -p user.info + +# Backup move operation with error handling +0 1 * * * /home/acedanger/shell/move-backups.sh 2>&1 | logger -t backup-move -p user.info + +# Validation with performance tracking +0 7 * * * /home/acedanger/shell/validate-plex-backups.sh --fix 2>&1 | logger -t plex-validation -p user.info +``` + +### Log Monitoring + +```bash +# View all backup-related logs +journalctl -t plex-backup -t backup-move -t plex-validation -f + +# View logs for specific operation +journalctl -t plex-backup --since "1 hour ago" + +# Monitor backup performance +./backup-log-monitor.sh --real-time +``` + +## Migration from Legacy Structure + +The system automatically detects and migrates legacy backup structures: + +### Automatic Migration + +- **Legacy detection**: Automatically detects old `crontab-backups/archive/` structure +- **Hostname prefix**: Adds hostname prefix to existing backup files +- **Backward compatibility**: Preserves all existing backup data +- **Safe migration**: Original files remain untouched until manual cleanup + +### Manual Migration + +```bash +# Force migration of legacy backups +./crontab-backup-system.sh migrate +``` + +## Production Deployment + +### System Setup + +1. **Deploy script**: Copy `crontab-backup-system.sh` to each system +2. **Configure permissions**: Ensure proper read/write access to backup directories +3. **Setup automation**: Configure automated daily backups + +### Automated Backup Setup + +```bash +# Setup automated daily backups on each system +./crontab-backup-system.sh setup-auto +``` + +This adds the following entry to the system crontab: + +```bash +0 0 * * * /path/to/crontab-backup-system.sh backup auto --auto-cleanup 2>&1 | logger -t crontab-backup -p user.info +``` + +### Cross-System Synchronization + +For distributed environments, consider setting up backup synchronization: + +```bash +# Example rsync command to sync backups from remote systems +rsync -avz europa:/home/acedanger/shell/crontab-backups/europa/ /home/acedanger/shell/crontab-backups/europa/ +rsync -avz io:/home/acedanger/shell/crontab-backups/io/ /home/acedanger/shell/crontab-backups/io/ +``` + +## Security Considerations + +### File Permissions + +- **Backup directories**: Restrict access to authorized users only +- **Log files**: Ensure proper log rotation and access controls +- **Remote access**: Use secure methods (SSH, rsync) for cross-system operations + +### Backup Integrity + +- **Validation**: Regular syntax validation of backup files +- **Checksums**: Consider adding checksum verification for critical backups +- **Retention**: Implement appropriate backup retention policies + +## Advanced Use Cases + +### Disaster Recovery + +```bash +# Restore from specific system backup during emergency +./crontab-backup-system.sh restore io-crontab-pre-incident-20250526_101354.backup + +# Compare pre-incident and post-incident configurations +./crontab-backup-system.sh compare io-crontab-pre-incident-20250526_101354.backup current +``` + +### Configuration Management + +```bash +# Standardize crontab across multiple systems +./crontab-backup-system.sh compare europa-crontab-standard-20250526_101354.backup io-crontab-current-20250526_120000.backup + +# Validate configurations before deployment +./crontab-backup-system.sh validate new-crontab-config.txt +``` + +### Compliance and Auditing + +- **Change tracking**: Complete history of all crontab changes across systems +- **Audit trails**: System logs provide comprehensive audit information +- **Compliance reporting**: Generate reports showing backup frequency and success rates + +## Monitoring and Alerting + +### Health Checks + +```bash +# Check backup system health +./crontab-backup-system.sh status all + +# Monitor recent backup activity +./backup-log-monitor.sh --health-check +``` + +### Alert Integration + +Consider integrating with monitoring systems: + +- **Backup failures**: Alert when backups fail or are missing +- **Old backups**: Alert when systems haven't been backed up recently +- **Disk space**: Monitor backup directory disk usage + +## Best Practices + +1. **Regular Testing**: Periodically test backup restoration procedures +2. **Documentation**: Keep records of system configurations and backup schedules +3. **Automation**: Use automated cleanup to prevent disk space issues +4. **Monitoring**: Implement comprehensive monitoring and alerting +5. **Security**: Regularly review and update access controls + +## Support and Troubleshooting + +### Common Issues + +- **Permission errors**: Ensure proper file permissions on backup directories +- **Missing backups**: Check automated backup cron entries +- **Syntax errors**: Use validation feature before deploying new crontabs + +### Debug Mode + +Enable verbose logging for troubleshooting: + +```bash +# Add debug logging to any command +./crontab-backup-system.sh status all 2>&1 | tee debug.log +``` + +### Log Analysis + +```bash +# Analyze backup patterns +grep "SUCCESS" logs/crontab-management.log | tail -20 + +# Check for errors +grep "ERROR" logs/crontab-management.log | tail -10 +``` + +--- + +*This multi-system crontab management solution provides robust, scalable backup management for distributed environments while maintaining simplicity and reliability.* diff --git a/enhanced-crontab.txt b/enhanced-crontab.txt new file mode 100644 index 0000000..db1c565 --- /dev/null +++ b/enhanced-crontab.txt @@ -0,0 +1,28 @@ +# Enhanced Crontab Entries with System Logging +# +# These entries include comprehensive logging to syslog with proper tags +# and error handling for better monitoring and troubleshooting + +# Move the files previously backed up at 0100 +# Logs both stdout and stderr with backup-move tag +0 1 * * * /home/acedanger/shell/move-backups.sh 2>&1 | logger -t backup-move -p user.info + +# Daily Plex backup at 0415 with enhanced logging +# Includes execution status and performance metrics +15 4 * * * { echo "Starting Plex backup"; /home/acedanger/shell/backup-plex.sh --non-interactive --auto-repair; echo "Plex backup completed with exit code: $?"; } 2>&1 | logger -t plex-backup -p user.info + +# Daily validation at 0700 with detailed logging +# Logs validation results and any auto-fixes performed +0 7 * * * { echo "Starting Plex backup validation"; /home/acedanger/shell/validate-plex-backups.sh --fix; echo "Validation completed with exit code: $?"; } 2>&1 | logger -t plex-validation -p user.info + +# Backup Immich database weekly (Mondays at 0500) +# Enhanced with proper logging and error handling +0 5 * * 1 { echo "Starting Immich database backup move"; if mv /mnt/share/media/immich/uploads/backups/immich-db-backup* /mnt/share/media/backups/immich 2>/dev/null; then echo "Immich backup move completed successfully"; else echo "No Immich backup files found or move failed"; fi; } 2>&1 | logger -t immich-backup -p user.info + +# Generate detailed weekly report (Sundays at 0800) +# Comprehensive reporting with system logging +0 8 * * 0 { echo "Starting weekly Plex backup report generation"; /home/acedanger/shell/validate-plex-backups.sh --report; echo "Weekly report generation completed with exit code: $?"; } 2>&1 | logger -t plex-report -p user.info + +# Optional: Add a health check entry to monitor cron jobs (every 6 hours) +# This can help detect if any of the backup processes are failing +# 0 */6 * * * { echo "Cron health check - all backup jobs scheduled"; ps aux | grep -E "(backup-plex|validate-plex|move-backups)" | grep -v grep | wc -l; } 2>&1 | logger -t cron-health -p user.info diff --git a/manage-enhanced-crontab.sh b/manage-enhanced-crontab.sh new file mode 100755 index 0000000..9f9b796 --- /dev/null +++ b/manage-enhanced-crontab.sh @@ -0,0 +1,246 @@ +#!/bin/bash + +# Enhanced Crontab Management Script +# This script helps install and manage the enhanced crontab entries with system logging + +set -e + +# Color codes for output +RED='\033[0;31m' +GREEN='\033[0;32m' +YELLOW='\033[1;33m' +BLUE='\033[0;34m' +CYAN='\033[0;36m' +NC='\033[0m' # No Color + +SCRIPT_DIR="$(dirname "$(readlink -f "$0")")" +ENHANCED_CRONTAB_FILE="$SCRIPT_DIR/enhanced-crontab.txt" +BACKUP_CRONTAB_FILE="/tmp/crontab-backup-$(date +%Y%m%d_%H%M%S)" + +log_message() { + echo -e "$(date '+%H:%M:%S') $1" +} + +log_error() { + log_message "${RED}ERROR: $1${NC}" +} + +log_success() { + log_message "${GREEN}SUCCESS: $1${NC}" +} + +log_warning() { + log_message "${YELLOW}WARNING: $1${NC}" +} + +log_info() { + log_message "${BLUE}INFO: $1${NC}" +} + +backup_current_crontab() { + log_info "Backing up current root crontab to $BACKUP_CRONTAB_FILE" + if sudo crontab -l > "$BACKUP_CRONTAB_FILE" 2>/dev/null; then + log_success "Current crontab backed up successfully" + else + log_warning "No existing crontab found or backup failed" + fi +} + +install_enhanced_crontab() { + log_info "Installing enhanced crontab entries" + + if [ ! -f "$ENHANCED_CRONTAB_FILE" ]; then + log_error "Enhanced crontab file not found: $ENHANCED_CRONTAB_FILE" + return 1 + fi + + # Create a backup before making changes + if [ -f "$SCRIPT_DIR/crontab-backup-system.sh" ]; then + log_info "Creating pre-install backup" + "$SCRIPT_DIR/crontab-backup-system.sh" backup pre-install + fi + + # Extract just the cron entries (skip comments and empty lines) + grep -E '^[0-9]' "$ENHANCED_CRONTAB_FILE" > /tmp/cron_entries_only.txt + + # Validate the crontab syntax before installing + log_info "Validating crontab syntax" + if [ -f "$SCRIPT_DIR/crontab-backup-system.sh" ]; then + if ! "$SCRIPT_DIR/crontab-backup-system.sh" validate /tmp/cron_entries_only.txt; then + log_error "Crontab syntax validation failed" + rm -f /tmp/cron_entries_only.txt + return 1 + fi + fi + + if sudo crontab /tmp/cron_entries_only.txt; then + log_success "Enhanced crontab entries installed successfully" + rm -f /tmp/cron_entries_only.txt + + # Create a post-install backup + if [ -f "$SCRIPT_DIR/crontab-backup-system.sh" ]; then + "$SCRIPT_DIR/crontab-backup-system.sh" backup post-install + fi + else + log_error "Failed to install enhanced crontab entries" + rm -f /tmp/cron_entries_only.txt + return 1 + fi +} + +show_current_crontab() { + log_info "Current root crontab entries:" + echo + sudo crontab -l 2>/dev/null || log_warning "No crontab entries found" + echo +} + +show_log_monitoring_commands() { + log_info "Commands to monitor backup logs:" + echo + echo -e "${CYAN}# Use the enhanced backup log monitor:${NC}" + echo "./backup-log-monitor.sh monitor # Real-time monitoring" + echo "./backup-log-monitor.sh recent 24 # Last 24 hours" + echo "./backup-log-monitor.sh health # System health check" + echo "./backup-log-monitor.sh report 7 # Weekly report" + echo + echo -e "${CYAN}# Direct journalctl commands:${NC}" + echo "sudo journalctl -f -t plex-backup -t backup-move -t plex-validation -t immich-backup -t plex-report" + echo + echo -e "${CYAN}# View logs from the last 24 hours:${NC}" + echo "sudo journalctl --since '24 hours ago' -t plex-backup -t backup-move -t plex-validation -t immich-backup -t plex-report" + echo + echo -e "${CYAN}# View only error logs:${NC}" + echo "sudo journalctl --priority=err -t plex-backup -t backup-move -t plex-validation -t immich-backup -t plex-report" + echo + echo -e "${CYAN}# View logs for a specific backup type (e.g., plex-backup):${NC}" + echo "sudo journalctl -t plex-backup --since '1 week ago'" + echo +} + +setup_logrotate() { + log_info "Setting up logrotate for backup logs" + + cat > /tmp/backup-logs-logrotate << 'EOF' +# Logrotate configuration for backup logs +# This ensures syslog doesn't grow too large with backup logs + +/var/log/syslog { + daily + missingok + rotate 7 + compress + delaycompress + notifempty + postrotate + /usr/lib/rsyslog/rsyslog-rotate + endscript +} +EOF + + if sudo cp /tmp/backup-logs-logrotate /etc/logrotate.d/backup-logs; then + log_success "Logrotate configuration installed" + else + log_warning "Failed to install logrotate configuration" + fi + + rm -f /tmp/backup-logs-logrotate +} + +verify_scripts_exist() { + log_info "Verifying all backup scripts exist and are executable" + + local scripts=( + "/home/acedanger/shell/move-backups.sh" + "/home/acedanger/shell/backup-plex.sh" + "/home/acedanger/shell/validate-plex-backups.sh" + ) + + local all_good=true + + for script in "${scripts[@]}"; do + if [ -f "$script" ]; then + if [ -x "$script" ]; then + log_success "✓ $script exists and is executable" + else + log_warning "! $script exists but is not executable" + sudo chmod +x "$script" + log_success "✓ Made $script executable" + fi + else + log_error "✗ $script not found" + all_good=false + fi + done + + if $all_good; then + log_success "All backup scripts are ready" + else + log_error "Some backup scripts are missing" + return 1 + fi +} + +show_usage() { + echo "Enhanced Crontab Management Script" + echo + echo "Usage: $0 [OPTION]" + echo + echo "Options:" + echo " install Install the enhanced crontab entries with backup system" + echo " show Show current crontab entries" + echo " backup Backup current crontab only" + echo " verify Verify all scripts exist and are executable" + echo " monitor Show log monitoring commands" + echo " logrotate Setup logrotate for backup logs" + echo " status Show backup system health status" + echo " help Show this help message" + echo + echo "Additional Tools:" + echo " ./crontab-backup-system.sh Comprehensive crontab backup management" + echo " ./backup-log-monitor.sh Advanced backup log monitoring" + echo +} + +case "${1:-help}" in + install) + verify_scripts_exist + backup_current_crontab + install_enhanced_crontab + show_current_crontab + setup_logrotate + show_log_monitoring_commands + + # Setup automated backup system + if [ -f "$SCRIPT_DIR/crontab-backup-system.sh" ]; then + log_info "Setting up automated crontab backup system" + "$SCRIPT_DIR/crontab-backup-system.sh" setup-auto + fi + ;; + show) + show_current_crontab + ;; + backup) + backup_current_crontab + ;; + verify) + verify_scripts_exist + ;; + monitor) + show_log_monitoring_commands + ;; + logrotate) + setup_logrotate + ;; + status) + if [ -f "$SCRIPT_DIR/backup-log-monitor.sh" ]; then + "$SCRIPT_DIR/backup-log-monitor.sh" health + else + log_warning "Backup log monitor not found, showing basic status" + show_current_crontab + fi + ;; + help|*) + show_usage + ;; +esac