feat: Implement crontab recovery and management scripts with enhanced logging and system-specific entries

crontab-backups/racknerd/archive/racknerd-crontab-post-fix-20250526_111637.backup

@@ -0,0 +1,6 @@
+0 0 * * * { echo "Starting Docker backup"; /home/acedanger/shell/backup-docker.sh; echo "Docker backup completed with exit code: $?"; } 2>&1 | logger -t docker-backup -p user.info
+# Backup created: Mon May 26 11:16:37 AM EDT 2025
+# Backup type: post-fix
+# System: racknerd
+# User: root
+# Full system info: Linux racknerd 6.1.0-9-amd64 #1 SMP PREEMPT_DYNAMIC Debian 6.1.27-1 (2023-05-08) x86_64 GNU/Linux

crontab-backups/racknerd/archive/racknerd-crontab-pre-fix-20250526_111637.backup

@@ -0,0 +1,11 @@
+0 1 * * * /home/acedanger/shell/move-backups.sh 2>&1 | logger -t backup-move -p user.info
+15 4 * * * { echo "Starting Plex backup"; /home/acedanger/shell/backup-plex.sh --non-interactive --auto-repair; echo "Plex backup completed with exit code: $?"; } 2>&1 | logger -t plex-backup -p user.info
+0 7 * * * { echo "Starting Plex backup validation"; /home/acedanger/shell/validate-plex-backups.sh --fix; echo "Validation completed with exit code: $?"; } 2>&1 | logger -t plex-validation -p user.info
+0 5 * * 1 { echo "Starting Immich database backup move"; if mv /mnt/share/media/immich/uploads/backups/immich-db-backup* /mnt/share/media/backups/immich 2>/dev/null; then echo "Immich backup move completed successfully"; else echo "No Immich backup files found or move failed"; fi; } 2>&1 | logger -t immich-backup -p user.info
+0 8 * * 0 { echo "Starting weekly Plex backup report generation"; /home/acedanger/shell/validate-plex-backups.sh --report; echo "Weekly report generation completed with exit code: $?"; } 2>&1 | logger -t plex-report -p user.info
+0 0 * * * /home/acedanger/shell/crontab-backup-system.sh backup auto --auto-cleanup 2>&1 | logger -t crontab-backup -p user.info
+# Backup created: Mon May 26 11:16:37 AM EDT 2025
+# Backup type: pre-fix
+# System: racknerd
+# User: root
+# Full system info: Linux racknerd 6.1.0-9-amd64 #1 SMP PREEMPT_DYNAMIC Debian 6.1.27-1 (2023-05-08) x86_64 GNU/Linux

crontab-backups/racknerd/current-crontab.backup

@@ -1,5 +1 @@
-0 1 * * * /home/acedanger/shell/move-backups.sh 2>&1 | logger -t backup-move -p user.info
+0 0 * * * { echo "Starting Docker backup"; /home/acedanger/shell/backup-docker.sh; echo "Docker backup completed with exit code: $?"; } 2>&1 | logger -t docker-backup -p user.info
-15 4 * * * { echo "Starting Plex backup"; /home/acedanger/shell/backup-plex.sh --non-interactive --auto-repair; echo "Plex backup completed with exit code: $?"; } 2>&1 | logger -t plex-backup -p user.info
-0 7 * * * { echo "Starting Plex backup validation"; /home/acedanger/shell/validate-plex-backups.sh --fix; echo "Validation completed with exit code: $?"; } 2>&1 | logger -t plex-validation -p user.info
-0 5 * * 1 { echo "Starting Immich database backup move"; if mv /mnt/share/media/immich/uploads/backups/immich-db-backup* /mnt/share/media/backups/immich 2>/dev/null; then echo "Immich backup move completed successfully"; else echo "No Immich backup files found or move failed"; fi; } 2>&1 | logger -t immich-backup -p user.info
-0 8 * * 0 { echo "Starting weekly Plex backup report generation"; /home/acedanger/shell/validate-plex-backups.sh --report; echo "Weekly report generation completed with exit code: $?"; } 2>&1 | logger -t plex-report -p user.info

enhanced-crontab-europa.txt

@@ -0,0 +1,31 @@
+# Enhanced Crontab Entries for Europa (Media Server)
+# 
+# These entries include comprehensive logging to syslog with proper tags
+# and error handling for better monitoring and troubleshooting
+
+# Move the files previously backed up at 0100 (FROM racknerd TO europa)
+# Logs both stdout and stderr with backup-move tag
+0 1 * * * /home/acedanger/shell/move-backups.sh 2>&1 | logger -t backup-move -p user.info
+
+# Daily Plex backup at 0415 with enhanced logging
+# Includes execution status and performance metrics
+15 4 * * * { echo "Starting Plex backup"; /home/acedanger/shell/backup-plex.sh --non-interactive --auto-repair; echo "Plex backup completed with exit code: $?"; } 2>&1 | logger -t plex-backup -p user.info
+
+# Daily validation at 0700 with detailed logging
+# Logs validation results and any auto-fixes performed
+0 7 * * * { echo "Starting Plex backup validation"; /home/acedanger/shell/validate-plex-backups.sh --fix; echo "Validation completed with exit code: $?"; } 2>&1 | logger -t plex-validation -p user.info
+
+# Backup Immich database weekly (Mondays at 0500)
+# Enhanced with proper logging and error handling
+0 5 * * 1 { echo "Starting Immich database backup move"; if mv /mnt/share/media/immich/uploads/backups/immich-db-backup* /mnt/share/media/backups/immich 2>/dev/null; then echo "Immich backup move completed successfully"; else echo "No Immich backup files found or move failed"; fi; } 2>&1 | logger -t immich-backup -p user.info
+
+# Generate detailed weekly report (Sundays at 0800)
+# Comprehensive reporting with system logging
+0 8 * * 0 { echo "Starting weekly Plex backup report generation"; /home/acedanger/shell/validate-plex-backups.sh --report; echo "Weekly report generation completed with exit code: $?"; } 2>&1 | logger -t plex-report -p user.info
+
+# Mount check and recovery (legacy from original crontab)
+# @reboot sleep 30 && mount -a
+
+# Optional: Add a health check entry to monitor cron jobs (every 6 hours)
+# This can help detect if any of the backup processes are failing
+# 0 */6 * * * { echo "Cron health check - all backup jobs scheduled"; ps aux | grep -E "(backup-plex|validate-plex|move-backups)" | grep -v grep | wc -l; } 2>&1 | logger -t cron-health -p user.info

enhanced-crontab-racknerd.txt

@@ -0,0 +1,12 @@
+# Enhanced Crontab Entries for Racknerd (Backup Server)
+# 
+# These entries include comprehensive logging to syslog with proper tags
+# and error handling for better monitoring and troubleshooting
+
+# Daily Docker backup at midnight (original racknerd job)
+# Enhanced with proper logging and error handling
+0 0 * * * { echo "Starting Docker backup"; /home/acedanger/shell/backup-docker.sh; echo "Docker backup completed with exit code: $?"; } 2>&1 | logger -t docker-backup -p user.info
+
+# Optional: Add a health check entry to monitor backup jobs (every 6 hours)
+# This can help detect if the backup process is failing
+# 0 */6 * * * { echo "Cron health check - Docker backup job scheduled"; ps aux | grep "backup-docker" | grep -v grep | wc -l; } 2>&1 | logger -t cron-health -p user.info

fix-crontab-merging.sh

@@ -0,0 +1,141 @@
+#!/bin/bash
+
+# Crontab Recovery Script
+# This script fixes the crontab merging issue by restoring system-specific entries
+
+set -e
+
+# Color codes for output
+RED='\033[0;31m'
+GREEN='\033[0;32m'
+YELLOW='\033[1;33m'
+BLUE='\033[0;34m'
+CYAN='\033[0;36m'
+NC='\033[0m' # No Color
+
+SCRIPT_DIR="$(dirname "$(readlink -f "$0")")"
+HOSTNAME=$(hostname)
+
+log_message() {
+    echo -e "$(date '+%H:%M:%S') $1"
+}
+
+log_error() {
+    log_message "${RED}ERROR: $1${NC}"
+}
+
+log_success() {
+    log_message "${GREEN}SUCCESS: $1${NC}"
+}
+
+log_warning() {
+    log_message "${YELLOW}WARNING: $1${NC}"
+}
+
+log_info() {
+    log_message "${BLUE}INFO: $1${NC}"
+}
+
+show_current_problem() {
+    log_info "Current crontab merging issue analysis:"
+    echo
+    
+    log_info "Current root crontab on $HOSTNAME:"
+    sudo crontab -l 2>/dev/null || log_warning "No crontab found"
+    echo
+    
+    case "$HOSTNAME" in
+        "europa")
+            log_info "Europa should have:"
+            echo "  - move-backups.sh (pulls files FROM racknerd)"
+            echo "  - backup-plex.sh (backs up Plex)"
+            echo "  - validate-plex-backups.sh (validates backups)"
+            echo "  - Immich database backup move"
+            echo "  - Weekly Plex reports"
+            ;;
+        "racknerd")
+            log_info "Racknerd should have:"
+            echo "  - backup-docker.sh (backs up Docker containers)"
+            echo "  - NO move-backups.sh (that's Europa's job)"
+            echo "  - NO Plex-related jobs (Plex runs on Europa)"
+            ;;
+        *)
+            log_warning "Unknown hostname: $HOSTNAME"
+            ;;
+    esac
+    echo
+}
+
+fix_crontab() {
+    local system_name="$1"
+    local crontab_file="$SCRIPT_DIR/enhanced-crontab-${system_name}.txt"
+    
+    if [ ! -f "$crontab_file" ]; then
+        log_error "System-specific crontab file not found: $crontab_file"
+        return 1
+    fi
+    
+    log_info "Creating backup before fixing crontab"
+    if [ -f "$SCRIPT_DIR/crontab-backup-system.sh" ]; then
+        "$SCRIPT_DIR/crontab-backup-system.sh" backup "pre-fix"
+    fi
+    
+    log_info "Installing correct crontab for $system_name"
+    
+    # Extract just the cron entries (skip comments and empty lines)
+    grep -E '^[0-9]' "$crontab_file" > /tmp/cron_entries_fix.txt
+    
+    if sudo crontab /tmp/cron_entries_fix.txt; then
+        log_success "Correct crontab installed for $system_name"
+        rm -f /tmp/cron_entries_fix.txt
+        
+        # Create a post-fix backup
+        if [ -f "$SCRIPT_DIR/crontab-backup-system.sh" ]; then
+            "$SCRIPT_DIR/crontab-backup-system.sh" backup "post-fix"
+        fi
+        
+        log_info "New crontab contents:"
+        sudo crontab -l
+        
+        return 0
+    else
+        log_error "Failed to install correct crontab"
+        rm -f /tmp/cron_entries_fix.txt
+        return 1
+    fi
+}
+
+main() {
+    log_info "=== Crontab Recovery Script ==="
+    log_info "System: $HOSTNAME"
+    echo
+    
+    show_current_problem
+    
+    case "$HOSTNAME" in
+        "europa")
+            log_info "Fixing crontab for Europa (media server)"
+            fix_crontab "europa"
+            ;;
+        "racknerd")
+            log_info "Fixing crontab for Racknerd (backup server)"
+            fix_crontab "racknerd"
+            ;;
+        *)
+            log_error "Unknown system: $HOSTNAME"
+            log_info "This script supports: europa, racknerd"
+            exit 1
+            ;;
+    esac
+    
+    echo
+    log_success "Crontab recovery completed for $HOSTNAME"
+    log_info "The enhanced management script now uses system-specific files:"
+    log_info "  - enhanced-crontab-europa.txt"
+    log_info "  - enhanced-crontab-racknerd.txt"
+    echo
+    log_info "To manage crontabs going forward, use:"
+    log_info "  ./manage-enhanced-crontab.sh install"
+}
+
+main "$@"

manage-enhanced-crontab.sh

@@ -14,7 +14,8 @@ CYAN='\033[0;36m'
 NC='\033[0m' # No Color
 
 SCRIPT_DIR="$(dirname "$(readlink -f "$0")")"
-ENHANCED_CRONTAB_FILE="$SCRIPT_DIR/enhanced-crontab.txt"
+HOSTNAME=$(hostname)
+ENHANCED_CRONTAB_FILE="$SCRIPT_DIR/enhanced-crontab-${HOSTNAME}.txt"
 BACKUP_CRONTAB_FILE="/tmp/crontab-backup-$(date +%Y%m%d_%H%M%S)"
 
 log_message() {
@@ -47,10 +48,12 @@ backup_current_crontab() {
 }
 
 install_enhanced_crontab() {
-    log_info "Installing enhanced crontab entries"
+    log_info "Installing enhanced crontab entries for system: $HOSTNAME"
     
     if [ ! -f "$ENHANCED_CRONTAB_FILE" ]; then
         log_error "Enhanced crontab file not found: $ENHANCED_CRONTAB_FILE"
+        log_info "Available crontab files:"
+        ls -la "$SCRIPT_DIR"/enhanced-crontab-*.txt 2>/dev/null || log_warning "No system-specific crontab files found"
         return 1
     fi