#!/bin/bash # Crontab Backup and Recovery System # This script provides comprehensive backup management for crontab entries set -e # Color codes for output RED='\033[0;31m' GREEN='\033[0;32m' YELLOW='\033[1;33m' BLUE='\033[0;34m' CYAN='\033[0;36m' PURPLE='\033[0;35m' NC='\033[0m' # No Color SCRIPT_DIR="$(dirname "$(readlink -f "$0")")" HOSTNAME=$(hostname) BACKUP_ROOT="$SCRIPT_DIR/crontab-backups" BACKUP_DIR="$BACKUP_ROOT/$HOSTNAME" LOG_DIR="$SCRIPT_DIR/logs" CURRENT_BACKUP="$BACKUP_DIR/current-crontab.backup" ARCHIVE_DIR="$BACKUP_DIR/archive" # Ensure directories exist mkdir -p "$BACKUP_DIR" "$ARCHIVE_DIR" "$LOG_DIR" log_message() { local message="$1" local log_file="$LOG_DIR/crontab-management.log" echo -e "$(date '+%Y-%m-%d %H:%M:%S') $message" echo "$(date '+%Y-%m-%d %H:%M:%S') $message" | sed 's/\\033\[[0-9;]*m//g' >> "$log_file" } log_error() { log_message "${RED}ERROR: $1${NC}" } log_success() { log_message "${GREEN}SUCCESS: $1${NC}" } log_warning() { log_message "${YELLOW}WARNING: $1${NC}" } log_info() { log_message "${BLUE}INFO: $1${NC}" } create_timestamped_backup() { local backup_type="${1:-manual}" local timestamp=$(date +%Y%m%d_%H%M%S) local backup_file="$ARCHIVE_DIR/${HOSTNAME}-crontab-${backup_type}-${timestamp}.backup" log_info "Creating timestamped backup for $HOSTNAME: $backup_file" if sudo crontab -l > "$backup_file" 2>/dev/null; then log_success "Backup created: $backup_file" # Also update the current backup cp "$backup_file" "$CURRENT_BACKUP" # Add metadata echo "# Backup created: $(date)" >> "$backup_file" echo "# Backup type: $backup_type" >> "$backup_file" echo "# System: $HOSTNAME" >> "$backup_file" echo "# User: root" >> "$backup_file" echo "# Full system info: $(uname -a)" >> "$backup_file" return 0 else log_error "Failed to create backup or no crontab exists" return 1 fi } list_backups() { local target_hostname="${1:-$HOSTNAME}" local target_dir="$BACKUP_ROOT/$target_hostname/archive" log_info "Available crontab backups for $target_hostname:" echo if [ -d "$target_dir" ] && [ "$(ls -A "$target_dir" 2>/dev/null)" ]; then printf "%-40s %-20s %-15s\n" "Filename" "Date Created" "Size" printf "%-40s %-20s %-15s\n" "--------" "------------" "----" for backup in "$target_dir"/*.backup; do if [ -f "$backup" ]; then local filename=$(basename "$backup") local date_created=$(stat -c %y "$backup" | cut -d' ' -f1,2 | cut -d'.' -f1) local size=$(stat -c %s "$backup") printf "%-40s %-20s %-15s bytes\n" "$filename" "$date_created" "$size" fi done else log_warning "No backups found in $target_dir" fi echo # Show all available systems if current system has no backups or if showing all if [ "$target_hostname" = "$HOSTNAME" ] && [ ! -d "$target_dir" ]; then log_info "Available systems with backups:" for system_dir in "$BACKUP_ROOT"/*; do if [ -d "$system_dir/archive" ] && [ "$(ls -A "$system_dir/archive" 2>/dev/null)" ]; then local system_name=$(basename "$system_dir") local backup_count=$(ls -1 "$system_dir/archive"/*.backup 2>/dev/null | wc -l) echo " - $system_name ($backup_count backups)" fi done echo echo "Use: $0 list [hostname] to view backups for a specific system" fi } restore_from_backup() { local backup_file="$1" local source_hostname="" if [ -z "$backup_file" ]; then log_error "No backup file specified" list_backups return 1 fi # Handle different backup file formats and paths if [[ "$backup_file" == *"/"* ]]; then # Full or relative path provided if [[ ! "$backup_file" = /* ]]; then backup_file="$ARCHIVE_DIR/$backup_file" fi else # Just filename provided - check current system first, then others if [ -f "$ARCHIVE_DIR/$backup_file" ]; then backup_file="$ARCHIVE_DIR/$backup_file" else # Search in other system directories local found_file="" for system_dir in "$BACKUP_ROOT"/*; do if [ -f "$system_dir/archive/$backup_file" ]; then found_file="$system_dir/archive/$backup_file" source_hostname=$(basename "$system_dir") break fi done if [ -n "$found_file" ]; then backup_file="$found_file" log_warning "Backup file found in $source_hostname system directory" fi fi fi if [ ! -f "$backup_file" ]; then log_error "Backup file not found: $backup_file" echo log_info "Available backups:" list_backups return 1 fi # Extract source hostname from backup metadata if available if [ -z "$source_hostname" ]; then source_hostname=$(grep "^# System:" "$backup_file" 2>/dev/null | cut -d' ' -f3 || echo "unknown") fi log_info "Restoring crontab from: $backup_file" if [ "$source_hostname" != "unknown" ] && [ "$source_hostname" != "$HOSTNAME" ]; then log_warning "Restoring backup from different system: $source_hostname -> $HOSTNAME" echo -n "Continue? [y/N]: " read -r response if [[ ! "$response" =~ ^[Yy]$ ]]; then log_info "Restore cancelled" return 1 fi fi # Create a safety backup before restoring create_timestamped_backup "pre-restore" # Remove metadata lines before restoring grep -v "^# Backup" "$backup_file" > /tmp/crontab_restore.tmp if sudo crontab /tmp/crontab_restore.tmp; then log_success "Crontab restored successfully from $backup_file" rm -f /tmp/crontab_restore.tmp else log_error "Failed to restore crontab" rm -f /tmp/crontab_restore.tmp return 1 fi } compare_crontabs() { local file1="${1:-current}" local file2="$2" if [ "$file1" = "current" ]; then sudo crontab -l > /tmp/current_crontab.tmp 2>/dev/null || echo "# No current crontab" > /tmp/current_crontab.tmp file1="/tmp/current_crontab.tmp" fi if [ -z "$file2" ]; then file2="$CURRENT_BACKUP" fi # Handle relative paths and cross-system backups if [[ ! "$file2" = /* ]]; then # Check current system first if [ -f "$ARCHIVE_DIR/$file2" ]; then file2="$ARCHIVE_DIR/$file2" else # Search in other system directories local found_file="" for system_dir in "$BACKUP_ROOT"/*; do if [ -f "$system_dir/archive/$file2" ]; then found_file="$system_dir/archive/$file2" local source_hostname=$(basename "$system_dir") log_info "Found backup in $source_hostname system directory" break fi done if [ -n "$found_file" ]; then file2="$found_file" else file2="$ARCHIVE_DIR/$file2" # Default back to current system fi fi fi if [ ! -f "$file2" ]; then log_error "Comparison file not found: $file2" return 1 fi log_info "Comparing current crontab ($HOSTNAME) with: $(basename "$file2")" echo # Clean comparison files (remove metadata) grep -v "^# Backup" "$file1" > /tmp/clean_file1.tmp 2>/dev/null || touch /tmp/clean_file1.tmp grep -v "^# Backup" "$file2" > /tmp/clean_file2.tmp 2>/dev/null || touch /tmp/clean_file2.tmp if diff -u /tmp/clean_file1.tmp /tmp/clean_file2.tmp; then log_success "Crontabs are identical" else log_warning "Crontabs differ (see above)" fi # Cleanup rm -f /tmp/current_crontab.tmp /tmp/clean_file1.tmp /tmp/clean_file2.tmp } validate_crontab_syntax() { local crontab_file="${1:-current}" if [ "$crontab_file" = "current" ]; then sudo crontab -l > /tmp/validate_crontab.tmp 2>/dev/null || echo "# No current crontab" > /tmp/validate_crontab.tmp crontab_file="/tmp/validate_crontab.tmp" fi if [ ! -f "$crontab_file" ]; then log_error "Crontab file not found: $crontab_file" return 1 fi log_info "Validating crontab syntax: $crontab_file" local line_num=0 local errors=0 while IFS= read -r line; do line_num=$((line_num + 1)) # Skip comments and empty lines if [[ "$line" =~ ^[[:space:]]*# ]] || [[ "$line" =~ ^[[:space:]]*$ ]]; then continue fi # Basic cron format validation if ! [[ "$line" =~ ^[[:space:]]*([0-9*,-]+[[:space:]]+){4}[0-9*,-]+[[:space:]].+ ]]; then log_error "Line $line_num: Invalid cron format: $line" errors=$((errors + 1)) fi # Check for common issues if [[ "$line" =~ \$\? ]] && [[ ! "$line" =~ \{ ]]; then log_warning "Line $line_num: \$? outside of command group may not work as expected" fi done < "$crontab_file" if [ $errors -eq 0 ]; then log_success "Crontab syntax validation passed" else log_error "Found $errors syntax errors" fi rm -f /tmp/validate_crontab.tmp return $errors } cleanup_old_backups() { local keep_days="${1:-30}" local target_hostname="${2:-$HOSTNAME}" local deleted_count=0 if [ "$target_hostname" = "all" ]; then log_info "Cleaning up backups older than $keep_days days for all systems" for system_dir in "$BACKUP_ROOT"/*; do if [ -d "$system_dir/archive" ]; then local system_name=$(basename "$system_dir") log_info "Cleaning backups for $system_name" while IFS= read -r -d '' backup; do if [ -f "$backup" ]; then rm "$backup" ((deleted_count++)) log_info "Deleted old backup: $(basename "$backup") from $system_name" fi done < <(find "$system_dir/archive" -name "*.backup" -mtime +$keep_days -print0 2>/dev/null) fi done else local target_dir="$BACKUP_ROOT/$target_hostname/archive" log_info "Cleaning up backups older than $keep_days days for $target_hostname" if [ -d "$target_dir" ]; then while IFS= read -r -d '' backup; do if [ -f "$backup" ]; then rm "$backup" ((deleted_count++)) log_info "Deleted old backup: $(basename "$backup")" fi done < <(find "$target_dir" -name "*.backup" -mtime +$keep_days -print0 2>/dev/null) else log_warning "Backup directory not found: $target_dir" fi fi if [ $deleted_count -eq 0 ]; then log_info "No old backups found to clean up" else log_success "Cleaned up $deleted_count old backup(s)" fi } setup_automated_backup() { log_info "Setting up automated daily crontab backup" local backup_script="$SCRIPT_DIR/crontab-backup-system.sh" local backup_entry="0 0 * * * $backup_script backup auto --auto-cleanup 2>&1 | logger -t crontab-backup -p user.info" # Check if backup entry already exists if sudo crontab -l 2>/dev/null | grep -q "crontab-backup-system.sh"; then log_warning "Automated backup entry already exists" return 0 fi # Add the backup entry to current crontab (sudo crontab -l 2>/dev/null; echo "$backup_entry") | sudo crontab - log_success "Automated daily backup configured for $HOSTNAME" log_info "Backups will run daily at midnight and be logged to syslog" } migrate_legacy_backups() { local legacy_dir="$SCRIPT_DIR/crontab-backups" local legacy_archive="$legacy_dir/archive" # Check if legacy structure exists (without hostname subdirectory) if [ -d "$legacy_archive" ] && [ "$legacy_dir" != "$BACKUP_DIR" ]; then log_info "Found legacy backup structure, migrating to hostname-based structure" # Create new structure mkdir -p "$BACKUP_DIR" "$ARCHIVE_DIR" # Move backups and rename them to include hostname local migrated_count=0 for backup in "$legacy_archive"/*.backup; do if [ -f "$backup" ]; then local filename=$(basename "$backup") local new_filename="${HOSTNAME}-${filename}" if cp "$backup" "$ARCHIVE_DIR/$new_filename"; then log_success "Migrated: $filename -> $new_filename" ((migrated_count++)) else log_error "Failed to migrate: $filename" fi fi done # Move current backup if it exists if [ -f "$legacy_dir/current-crontab.backup" ]; then cp "$legacy_dir/current-crontab.backup" "$CURRENT_BACKUP" log_success "Migrated current backup" fi if [ $migrated_count -gt 0 ]; then log_success "Migrated $migrated_count backup(s) to new structure" echo log_warning "Legacy backups remain in $legacy_archive" log_info "You can safely remove the legacy directory after verifying the migration" echo " rm -rf '$legacy_dir'" fi fi } list_all_systems() { log_info "All systems with crontab backups:" echo if [ ! -d "$BACKUP_ROOT" ]; then log_warning "No backup root directory found: $BACKUP_ROOT" return 1 fi printf "%-15s %-10s %-20s %-30s\n" "System" "Backups" "Latest Backup" "Status" printf "%-15s %-10s %-20s %-30s\n" "------" "-------" "-------------" "------" local found_systems=false for system_dir in "$BACKUP_ROOT"/*; do if [ -d "$system_dir" ]; then local system_name=$(basename "$system_dir") # Skip legacy archive directory - it's not a system if [ "$system_name" = "archive" ]; then continue fi found_systems=true local backup_count=$(ls -1 "$system_dir/archive"/*.backup 2>/dev/null | wc -l || echo "0") local latest_backup="None" local status="Inactive" if [ -f "$system_dir/current-crontab.backup" ]; then latest_backup=$(stat -c %y "$system_dir/current-crontab.backup" | cut -d' ' -f1) # Check if backup is recent (within 7 days) local backup_age=$(( ($(date +%s) - $(stat -c %Y "$system_dir/current-crontab.backup")) / 86400 )) if [ $backup_age -le 7 ]; then status="Active" elif [ $backup_age -le 30 ]; then status="Recent" else status="Stale" fi fi # Use printf with color formatting if [ "$status" = "Active" ]; then printf "%-15s %-10s %-20s ${GREEN}%-30s${NC}\n" "$system_name" "$backup_count" "$latest_backup" "$status" elif [ "$status" = "Recent" ]; then printf "%-15s %-10s %-20s ${YELLOW}%-30s${NC}\n" "$system_name" "$backup_count" "$latest_backup" "$status" elif [ "$status" = "Stale" ]; then printf "%-15s %-10s %-20s ${RED}%-30s${NC}\n" "$system_name" "$backup_count" "$latest_backup" "$status" else printf "%-15s %-10s %-20s %-30s\n" "$system_name" "$backup_count" "$latest_backup" "$status" fi fi done if [ "$found_systems" = false ]; then log_warning "No systems found with backups" fi echo } show_status() { local target_hostname="${1:-$HOSTNAME}" if [ "$target_hostname" = "all" ]; then log_info "Crontab Backup System Status - All Systems" echo for system_dir in "$BACKUP_ROOT"/*; do if [ -d "$system_dir" ]; then local system_name=$(basename "$system_dir") # Skip legacy archive directory - it's not a system if [ "$system_name" = "archive" ]; then continue fi echo -e "${CYAN}=== $system_name ===${NC}" local backup_count=$(ls -1 "$system_dir/archive"/*.backup 2>/dev/null | wc -l || echo "0") echo " - Total backups: $backup_count" echo " - Backup directory: $system_dir" if [ -f "$system_dir/current-crontab.backup" ]; then echo " - Latest backup: $(stat -c %y "$system_dir/current-crontab.backup" | cut -d'.' -f1)" else echo " - Latest backup: None" fi echo fi done else log_info "Crontab Backup System Status - $target_hostname" echo # Current crontab info if [ "$target_hostname" = "$HOSTNAME" ]; then local cron_count=$(sudo crontab -l 2>/dev/null | grep -c "^[^#]" || echo "0") echo -e "${CYAN}Current Crontab ($HOSTNAME):${NC}" echo " - Active entries: $cron_count" echo " - Last modified: $(stat -c %y /var/spool/cron/crontabs/root 2>/dev/null | cut -d'.' -f1 || echo "Unknown")" echo fi # Backup info for specified system local target_dir="$BACKUP_ROOT/$target_hostname" local backup_count=$(ls -1 "$target_dir/archive"/*.backup 2>/dev/null | wc -l || echo "0") echo -e "${CYAN}Backups ($target_hostname):${NC}" echo " - Total backups: $backup_count" echo " - Backup directory: $target_dir" if [ -f "$target_dir/current-crontab.backup" ]; then echo " - Latest backup: $(stat -c %y "$target_dir/current-crontab.backup" | cut -d'.' -f1)" else echo " - Latest backup: None" fi echo fi # Log monitoring echo -e "${CYAN}Log Monitoring:${NC}" echo " - Management log: $LOG_DIR/crontab-management.log" echo " - System logs: journalctl -t crontab-backup" echo } show_usage() { echo -e "${PURPLE}Crontab Backup and Recovery System (Multi-System)${NC}" echo echo "Usage: $0 [COMMAND] [OPTIONS]" echo echo "Commands:" echo " backup [TYPE] Create a timestamped backup (default: manual)" echo " list [HOSTNAME] List backups for hostname (default: current system)" echo " list-systems Show all systems with backups" echo " restore FILE Restore crontab from backup file" echo " compare [FILE1] [FILE2] Compare current crontab with backup" echo " validate [FILE] Validate crontab syntax" echo " cleanup [DAYS] [HOSTNAME] Clean up old backups (default: 30 days, current system)" echo " status [HOSTNAME|all] Show system status (default: current system)" echo " setup-auto Setup automated daily backups" echo " migrate Migrate legacy backups to hostname structure" echo " import FILE SYSTEM [TYPE] Import backup from external source" echo " create-test-systems Create test systems for demonstration" echo " help Show this help message" echo echo "Multi-System Examples:" echo " $0 backup pre-upgrade # Backup current system" echo " $0 list io # List backups for 'io' system" echo " $0 restore io-crontab-manual-20250526_120000.backup" echo " $0 compare current europa-crontab-manual-20250526_120000.backup" echo " $0 cleanup 7 all # Clean up all systems" echo " $0 status all # Show status for all systems" echo " $0 list-systems # Show all available systems" echo " $0 import /path/to/crontab.backup io manual # Import backup for io system" echo echo "Current System: $HOSTNAME" echo "Backup Root: $BACKUP_ROOT" echo } create_test_systems() { log_info "Creating test backup structure for io, europa, and racknerd systems" # Create sample systems directories local test_systems=("io" "racknerd") for system in "${test_systems[@]}"; do local system_dir="$BACKUP_ROOT/$system" local system_archive="$system_dir/archive" mkdir -p "$system_archive" # Create a sample backup for each system local timestamp=$(date +%Y%m%d_%H%M%S) local sample_backup="$system_archive/${system}-crontab-sample-${timestamp}.backup" # Create sample crontab content for each system case "$system" in "io") cat > "$sample_backup" << EOF # Sample crontab for io system 0 2 * * * /home/user/backup-docker.sh 2>&1 | logger -t docker-backup -p user.info 30 3 * * * /home/user/backup-media.sh 2>&1 | logger -t media-backup -p user.info 0 4 * * * /home/user/validate-backups.sh 2>&1 | logger -t backup-validation -p user.info # Backup created: $(date) # Backup type: sample # System: $system # User: root # Full system info: Linux $system 5.15.0-generic #1 SMP x86_64 GNU/Linux EOF ;; "racknerd") cat > "$sample_backup" << EOF # Sample crontab for racknerd system 0 1 * * * /home/user/backup-plex.sh 2>&1 | logger -t plex-backup -p user.info 15 1 * * * /home/user/move-backups.sh 2>&1 | logger -t backup-move -p user.info 0 5 * * 0 /home/user/cleanup-old-backups.sh 2>&1 | logger -t backup-cleanup -p user.info # Backup created: $(date) # Backup type: sample # System: $system # User: root # Full system info: Linux $system 5.4.0-generic #1 SMP x86_64 GNU/Linux EOF ;; esac # Create current backup file cp "$sample_backup" "$system_dir/current-crontab.backup" log_success "Created test system: $system with sample backup" done log_success "Test systems created successfully" } import_backup() { local source_file="$1" local source_system="$2" local backup_type="${3:-imported}" if [ -z "$source_file" ] || [ -z "$source_system" ]; then log_error "Usage: import_backup [backup_type]" return 1 fi if [ ! -f "$source_file" ]; then log_error "Source file not found: $source_file" return 1 fi local target_dir="$BACKUP_ROOT/$source_system" local target_archive="$target_dir/archive" mkdir -p "$target_archive" local timestamp=$(date +%Y%m%d_%H%M%S) local target_file="$target_archive/${source_system}-crontab-${backup_type}-${timestamp}.backup" # Copy and add metadata cp "$source_file" "$target_file" # Add metadata if not present if ! grep -q "^# Backup created:" "$target_file"; then cat >> "$target_file" << EOF # Backup created: $(date) # Backup type: $backup_type # System: $source_system # User: root # Imported from: $source_file EOF fi # Update current backup cp "$target_file" "$target_dir/current-crontab.backup" log_success "Imported backup for $source_system: $target_file" } # Main command handling case "${1:-help}" in backup) # Check for auto-cleanup flag if [[ "${@}" == *"--auto-cleanup"* ]]; then create_timestamped_backup "${2:-auto}" cleanup_old_backups 30 else create_timestamped_backup "${2:-manual}" fi ;; list) list_backups "$2" ;; list-systems) list_all_systems ;; restore) if [ -z "$2" ]; then log_error "Please specify a backup file to restore" list_backups exit 1 fi restore_from_backup "$2" ;; compare) compare_crontabs "$2" "$3" ;; validate) validate_crontab_syntax "$2" ;; cleanup) cleanup_old_backups "${2:-30}" "${3:-$HOSTNAME}" ;; status) show_status "${2:-$HOSTNAME}" ;; setup-auto) setup_automated_backup ;; migrate) migrate_legacy_backups ;; import) if [ -z "$2" ] || [ -z "$3" ]; then log_error "Please specify source file and target system" echo "Usage: $0 import [backup_type]" exit 1 fi import_backup "$2" "$3" "$4" ;; create-test-systems) create_test_systems ;; help|*) show_usage ;; esac # Auto-migrate legacy backups on first run if [ ! -d "$BACKUP_DIR" ] && [ "$1" != "help" ] && [ "$1" != "migrate" ]; then migrate_legacy_backups fi