diff --git a/crontab-backups/racknerd/archive/racknerd-crontab-post-fix-20250526_111637.backup b/crontab-backups/racknerd/archive/racknerd-crontab-post-fix-20250526_111637.backup new file mode 100644 index 0000000..a5f68c8 --- /dev/null +++ b/crontab-backups/racknerd/archive/racknerd-crontab-post-fix-20250526_111637.backup @@ -0,0 +1,6 @@ +0 0 * * * { echo "Starting Docker backup"; /home/acedanger/shell/backup-docker.sh; echo "Docker backup completed with exit code: $?"; } 2>&1 | logger -t docker-backup -p user.info +# Backup created: Mon May 26 11:16:37 AM EDT 2025 +# Backup type: post-fix +# System: racknerd +# User: root +# Full system info: Linux racknerd 6.1.0-9-amd64 #1 SMP PREEMPT_DYNAMIC Debian 6.1.27-1 (2023-05-08) x86_64 GNU/Linux diff --git a/crontab-backups/racknerd/archive/racknerd-crontab-pre-fix-20250526_111637.backup b/crontab-backups/racknerd/archive/racknerd-crontab-pre-fix-20250526_111637.backup new file mode 100644 index 0000000..3972cbc --- /dev/null +++ b/crontab-backups/racknerd/archive/racknerd-crontab-pre-fix-20250526_111637.backup @@ -0,0 +1,11 @@ +0 1 * * * /home/acedanger/shell/move-backups.sh 2>&1 | logger -t backup-move -p user.info +15 4 * * * { echo "Starting Plex backup"; /home/acedanger/shell/backup-plex.sh --non-interactive --auto-repair; echo "Plex backup completed with exit code: $?"; } 2>&1 | logger -t plex-backup -p user.info +0 7 * * * { echo "Starting Plex backup validation"; /home/acedanger/shell/validate-plex-backups.sh --fix; echo "Validation completed with exit code: $?"; } 2>&1 | logger -t plex-validation -p user.info +0 5 * * 1 { echo "Starting Immich database backup move"; if mv /mnt/share/media/immich/uploads/backups/immich-db-backup* /mnt/share/media/backups/immich 2>/dev/null; then echo "Immich backup move completed successfully"; else echo "No Immich backup files found or move failed"; fi; } 2>&1 | logger -t immich-backup -p user.info +0 8 * * 0 { echo "Starting weekly Plex backup report generation"; /home/acedanger/shell/validate-plex-backups.sh --report; echo "Weekly report generation completed with exit code: $?"; } 2>&1 | logger -t plex-report -p user.info +0 0 * * * /home/acedanger/shell/crontab-backup-system.sh backup auto --auto-cleanup 2>&1 | logger -t crontab-backup -p user.info +# Backup created: Mon May 26 11:16:37 AM EDT 2025 +# Backup type: pre-fix +# System: racknerd +# User: root +# Full system info: Linux racknerd 6.1.0-9-amd64 #1 SMP PREEMPT_DYNAMIC Debian 6.1.27-1 (2023-05-08) x86_64 GNU/Linux diff --git a/crontab-backups/racknerd/current-crontab.backup b/crontab-backups/racknerd/current-crontab.backup index 2fd2558..75c69b3 100644 --- a/crontab-backups/racknerd/current-crontab.backup +++ b/crontab-backups/racknerd/current-crontab.backup @@ -1,5 +1 @@ -0 1 * * * /home/acedanger/shell/move-backups.sh 2>&1 | logger -t backup-move -p user.info -15 4 * * * { echo "Starting Plex backup"; /home/acedanger/shell/backup-plex.sh --non-interactive --auto-repair; echo "Plex backup completed with exit code: $?"; } 2>&1 | logger -t plex-backup -p user.info -0 7 * * * { echo "Starting Plex backup validation"; /home/acedanger/shell/validate-plex-backups.sh --fix; echo "Validation completed with exit code: $?"; } 2>&1 | logger -t plex-validation -p user.info -0 5 * * 1 { echo "Starting Immich database backup move"; if mv /mnt/share/media/immich/uploads/backups/immich-db-backup* /mnt/share/media/backups/immich 2>/dev/null; then echo "Immich backup move completed successfully"; else echo "No Immich backup files found or move failed"; fi; } 2>&1 | logger -t immich-backup -p user.info -0 8 * * 0 { echo "Starting weekly Plex backup report generation"; /home/acedanger/shell/validate-plex-backups.sh --report; echo "Weekly report generation completed with exit code: $?"; } 2>&1 | logger -t plex-report -p user.info +0 0 * * * { echo "Starting Docker backup"; /home/acedanger/shell/backup-docker.sh; echo "Docker backup completed with exit code: $?"; } 2>&1 | logger -t docker-backup -p user.info diff --git a/enhanced-crontab-europa.txt b/enhanced-crontab-europa.txt new file mode 100644 index 0000000..6c4baaa --- /dev/null +++ b/enhanced-crontab-europa.txt @@ -0,0 +1,31 @@ +# Enhanced Crontab Entries for Europa (Media Server) +# +# These entries include comprehensive logging to syslog with proper tags +# and error handling for better monitoring and troubleshooting + +# Move the files previously backed up at 0100 (FROM racknerd TO europa) +# Logs both stdout and stderr with backup-move tag +0 1 * * * /home/acedanger/shell/move-backups.sh 2>&1 | logger -t backup-move -p user.info + +# Daily Plex backup at 0415 with enhanced logging +# Includes execution status and performance metrics +15 4 * * * { echo "Starting Plex backup"; /home/acedanger/shell/backup-plex.sh --non-interactive --auto-repair; echo "Plex backup completed with exit code: $?"; } 2>&1 | logger -t plex-backup -p user.info + +# Daily validation at 0700 with detailed logging +# Logs validation results and any auto-fixes performed +0 7 * * * { echo "Starting Plex backup validation"; /home/acedanger/shell/validate-plex-backups.sh --fix; echo "Validation completed with exit code: $?"; } 2>&1 | logger -t plex-validation -p user.info + +# Backup Immich database weekly (Mondays at 0500) +# Enhanced with proper logging and error handling +0 5 * * 1 { echo "Starting Immich database backup move"; if mv /mnt/share/media/immich/uploads/backups/immich-db-backup* /mnt/share/media/backups/immich 2>/dev/null; then echo "Immich backup move completed successfully"; else echo "No Immich backup files found or move failed"; fi; } 2>&1 | logger -t immich-backup -p user.info + +# Generate detailed weekly report (Sundays at 0800) +# Comprehensive reporting with system logging +0 8 * * 0 { echo "Starting weekly Plex backup report generation"; /home/acedanger/shell/validate-plex-backups.sh --report; echo "Weekly report generation completed with exit code: $?"; } 2>&1 | logger -t plex-report -p user.info + +# Mount check and recovery (legacy from original crontab) +# @reboot sleep 30 && mount -a + +# Optional: Add a health check entry to monitor cron jobs (every 6 hours) +# This can help detect if any of the backup processes are failing +# 0 */6 * * * { echo "Cron health check - all backup jobs scheduled"; ps aux | grep -E "(backup-plex|validate-plex|move-backups)" | grep -v grep | wc -l; } 2>&1 | logger -t cron-health -p user.info diff --git a/enhanced-crontab-racknerd.txt b/enhanced-crontab-racknerd.txt new file mode 100644 index 0000000..ab5f8e1 --- /dev/null +++ b/enhanced-crontab-racknerd.txt @@ -0,0 +1,12 @@ +# Enhanced Crontab Entries for Racknerd (Backup Server) +# +# These entries include comprehensive logging to syslog with proper tags +# and error handling for better monitoring and troubleshooting + +# Daily Docker backup at midnight (original racknerd job) +# Enhanced with proper logging and error handling +0 0 * * * { echo "Starting Docker backup"; /home/acedanger/shell/backup-docker.sh; echo "Docker backup completed with exit code: $?"; } 2>&1 | logger -t docker-backup -p user.info + +# Optional: Add a health check entry to monitor backup jobs (every 6 hours) +# This can help detect if the backup process is failing +# 0 */6 * * * { echo "Cron health check - Docker backup job scheduled"; ps aux | grep "backup-docker" | grep -v grep | wc -l; } 2>&1 | logger -t cron-health -p user.info diff --git a/fix-crontab-merging.sh b/fix-crontab-merging.sh new file mode 100755 index 0000000..09be617 --- /dev/null +++ b/fix-crontab-merging.sh @@ -0,0 +1,141 @@ +#!/bin/bash + +# Crontab Recovery Script +# This script fixes the crontab merging issue by restoring system-specific entries + +set -e + +# Color codes for output +RED='\033[0;31m' +GREEN='\033[0;32m' +YELLOW='\033[1;33m' +BLUE='\033[0;34m' +CYAN='\033[0;36m' +NC='\033[0m' # No Color + +SCRIPT_DIR="$(dirname "$(readlink -f "$0")")" +HOSTNAME=$(hostname) + +log_message() { + echo -e "$(date '+%H:%M:%S') $1" +} + +log_error() { + log_message "${RED}ERROR: $1${NC}" +} + +log_success() { + log_message "${GREEN}SUCCESS: $1${NC}" +} + +log_warning() { + log_message "${YELLOW}WARNING: $1${NC}" +} + +log_info() { + log_message "${BLUE}INFO: $1${NC}" +} + +show_current_problem() { + log_info "Current crontab merging issue analysis:" + echo + + log_info "Current root crontab on $HOSTNAME:" + sudo crontab -l 2>/dev/null || log_warning "No crontab found" + echo + + case "$HOSTNAME" in + "europa") + log_info "Europa should have:" + echo " - move-backups.sh (pulls files FROM racknerd)" + echo " - backup-plex.sh (backs up Plex)" + echo " - validate-plex-backups.sh (validates backups)" + echo " - Immich database backup move" + echo " - Weekly Plex reports" + ;; + "racknerd") + log_info "Racknerd should have:" + echo " - backup-docker.sh (backs up Docker containers)" + echo " - NO move-backups.sh (that's Europa's job)" + echo " - NO Plex-related jobs (Plex runs on Europa)" + ;; + *) + log_warning "Unknown hostname: $HOSTNAME" + ;; + esac + echo +} + +fix_crontab() { + local system_name="$1" + local crontab_file="$SCRIPT_DIR/enhanced-crontab-${system_name}.txt" + + if [ ! -f "$crontab_file" ]; then + log_error "System-specific crontab file not found: $crontab_file" + return 1 + fi + + log_info "Creating backup before fixing crontab" + if [ -f "$SCRIPT_DIR/crontab-backup-system.sh" ]; then + "$SCRIPT_DIR/crontab-backup-system.sh" backup "pre-fix" + fi + + log_info "Installing correct crontab for $system_name" + + # Extract just the cron entries (skip comments and empty lines) + grep -E '^[0-9]' "$crontab_file" > /tmp/cron_entries_fix.txt + + if sudo crontab /tmp/cron_entries_fix.txt; then + log_success "Correct crontab installed for $system_name" + rm -f /tmp/cron_entries_fix.txt + + # Create a post-fix backup + if [ -f "$SCRIPT_DIR/crontab-backup-system.sh" ]; then + "$SCRIPT_DIR/crontab-backup-system.sh" backup "post-fix" + fi + + log_info "New crontab contents:" + sudo crontab -l + + return 0 + else + log_error "Failed to install correct crontab" + rm -f /tmp/cron_entries_fix.txt + return 1 + fi +} + +main() { + log_info "=== Crontab Recovery Script ===" + log_info "System: $HOSTNAME" + echo + + show_current_problem + + case "$HOSTNAME" in + "europa") + log_info "Fixing crontab for Europa (media server)" + fix_crontab "europa" + ;; + "racknerd") + log_info "Fixing crontab for Racknerd (backup server)" + fix_crontab "racknerd" + ;; + *) + log_error "Unknown system: $HOSTNAME" + log_info "This script supports: europa, racknerd" + exit 1 + ;; + esac + + echo + log_success "Crontab recovery completed for $HOSTNAME" + log_info "The enhanced management script now uses system-specific files:" + log_info " - enhanced-crontab-europa.txt" + log_info " - enhanced-crontab-racknerd.txt" + echo + log_info "To manage crontabs going forward, use:" + log_info " ./manage-enhanced-crontab.sh install" +} + +main "$@" diff --git a/manage-enhanced-crontab.sh b/manage-enhanced-crontab.sh index 37c6bc7..0f630fc 100755 --- a/manage-enhanced-crontab.sh +++ b/manage-enhanced-crontab.sh @@ -14,7 +14,8 @@ CYAN='\033[0;36m' NC='\033[0m' # No Color SCRIPT_DIR="$(dirname "$(readlink -f "$0")")" -ENHANCED_CRONTAB_FILE="$SCRIPT_DIR/enhanced-crontab.txt" +HOSTNAME=$(hostname) +ENHANCED_CRONTAB_FILE="$SCRIPT_DIR/enhanced-crontab-${HOSTNAME}.txt" BACKUP_CRONTAB_FILE="/tmp/crontab-backup-$(date +%Y%m%d_%H%M%S)" log_message() { @@ -47,10 +48,12 @@ backup_current_crontab() { } install_enhanced_crontab() { - log_info "Installing enhanced crontab entries" + log_info "Installing enhanced crontab entries for system: $HOSTNAME" if [ ! -f "$ENHANCED_CRONTAB_FILE" ]; then log_error "Enhanced crontab file not found: $ENHANCED_CRONTAB_FILE" + log_info "Available crontab files:" + ls -la "$SCRIPT_DIR"/enhanced-crontab-*.txt 2>/dev/null || log_warning "No system-specific crontab files found" return 1 fi