Files
shell/crontab/crontab-backup-system.sh
Peter Wood c9b69ea789 Add comprehensive Plex backup management scripts
- Introduced `restore-plex.sh` for restoring Plex backups with logging and validation.
- Created `test-plex-backup.sh` for automated testing of backup functionalities.
- Developed `validate-plex-backups.sh` for validating backup integrity and monitoring.
- Updated `update.sh` to reference the correct path for Plex service management.
2025-05-26 13:20:12 -04:00

733 lines
25 KiB
Bash
Executable File

#!/bin/bash
# Crontab Backup and Recovery System
# This script provides comprehensive backup management for crontab entries
set -e
# Color codes for output
RED='\033[0;31m'
GREEN='\033[0;32m'
YELLOW='\033[1;33m'
BLUE='\033[0;34m'
CYAN='\033[0;36m'
PURPLE='\033[0;35m'
NC='\033[0m' # No Color
SCRIPT_DIR="$(dirname "$(readlink -f "$0")")"
HOSTNAME=$(hostname)
BACKUP_ROOT="$SCRIPT_DIR/crontab-backups"
BACKUP_DIR="$BACKUP_ROOT/$HOSTNAME"
LOG_DIR="$SCRIPT_DIR/logs"
CURRENT_BACKUP="$BACKUP_DIR/current-crontab.backup"
ARCHIVE_DIR="$BACKUP_DIR/archive"
# Ensure directories exist
mkdir -p "$BACKUP_DIR" "$ARCHIVE_DIR" "$LOG_DIR"
log_message() {
local message="$1"
local log_file="$LOG_DIR/crontab-management.log"
echo -e "$(date '+%Y-%m-%d %H:%M:%S') $message"
echo "$(date '+%Y-%m-%d %H:%M:%S') $message" | sed 's/\x1b\[[0-9;]*m//g' >> "$log_file"
}
log_error() {
log_message "${RED}ERROR: $1${NC}"
}
log_success() {
log_message "${GREEN}SUCCESS: $1${NC}"
}
log_warning() {
log_message "${YELLOW}WARNING: $1${NC}"
}
log_info() {
log_message "${BLUE}INFO: $1${NC}"
}
create_timestamped_backup() {
local backup_type="${1:-manual}"
local timestamp=$(date +%Y%m%d_%H%M%S)
local backup_file="$ARCHIVE_DIR/${HOSTNAME}-crontab-${backup_type}-${timestamp}.backup"
log_info "Creating timestamped backup for $HOSTNAME: $backup_file"
if sudo crontab -l > "$backup_file" 2>/dev/null; then
log_success "Backup created: $backup_file"
# Also update the current backup
cp "$backup_file" "$CURRENT_BACKUP"
# Add metadata
echo "# Backup created: $(date)" >> "$backup_file"
echo "# Backup type: $backup_type" >> "$backup_file"
echo "# System: $HOSTNAME" >> "$backup_file"
echo "# User: root" >> "$backup_file"
echo "# Full system info: $(uname -a)" >> "$backup_file"
return 0
else
log_error "Failed to create backup or no crontab exists"
return 1
fi
}
list_backups() {
local target_hostname="${1:-$HOSTNAME}"
local target_dir="$BACKUP_ROOT/$target_hostname/archive"
log_info "Available crontab backups for $target_hostname:"
echo
if [ -d "$target_dir" ] && [ "$(ls -A "$target_dir" 2>/dev/null)" ]; then
printf "%-40s %-20s %-15s\n" "Filename" "Date Created" "Size"
printf "%-40s %-20s %-15s\n" "--------" "------------" "----"
for backup in "$target_dir"/*.backup; do
if [ -f "$backup" ]; then
local filename=$(basename "$backup")
local date_created=$(stat -c %y "$backup" | cut -d' ' -f1,2 | cut -d'.' -f1)
local size=$(stat -c %s "$backup")
printf "%-40s %-20s %-15s bytes\n" "$filename" "$date_created" "$size"
fi
done
else
log_warning "No backups found in $target_dir"
fi
echo
# Show all available systems if current system has no backups or if showing all
if [ "$target_hostname" = "$HOSTNAME" ] && [ ! -d "$target_dir" ]; then
log_info "Available systems with backups:"
for system_dir in "$BACKUP_ROOT"/*; do
if [ -d "$system_dir/archive" ] && [ "$(ls -A "$system_dir/archive" 2>/dev/null)" ]; then
local system_name=$(basename "$system_dir")
local backup_count=$(ls -1 "$system_dir/archive"/*.backup 2>/dev/null | wc -l)
echo " - $system_name ($backup_count backups)"
fi
done
echo
echo "Use: $0 list [hostname] to view backups for a specific system"
fi
}
restore_from_backup() {
local backup_file="$1"
local source_hostname=""
if [ -z "$backup_file" ]; then
log_error "No backup file specified"
list_backups
return 1
fi
# Handle different backup file formats and paths
if [[ "$backup_file" == *"/"* ]]; then
# Full or relative path provided
if [[ ! "$backup_file" = /* ]]; then
backup_file="$ARCHIVE_DIR/$backup_file"
fi
else
# Just filename provided - check current system first, then others
if [ -f "$ARCHIVE_DIR/$backup_file" ]; then
backup_file="$ARCHIVE_DIR/$backup_file"
else
# Search in other system directories
local found_file=""
for system_dir in "$BACKUP_ROOT"/*; do
if [ -f "$system_dir/archive/$backup_file" ]; then
found_file="$system_dir/archive/$backup_file"
source_hostname=$(basename "$system_dir")
break
fi
done
if [ -n "$found_file" ]; then
backup_file="$found_file"
log_warning "Backup file found in $source_hostname system directory"
fi
fi
fi
if [ ! -f "$backup_file" ]; then
log_error "Backup file not found: $backup_file"
echo
log_info "Available backups:"
list_backups
return 1
fi
# Extract source hostname from backup metadata if available
if [ -z "$source_hostname" ]; then
source_hostname=$(grep "^# System:" "$backup_file" 2>/dev/null | cut -d' ' -f3 || echo "unknown")
fi
log_info "Restoring crontab from: $backup_file"
if [ "$source_hostname" != "unknown" ] && [ "$source_hostname" != "$HOSTNAME" ]; then
log_warning "Restoring backup from different system: $source_hostname -> $HOSTNAME"
echo -n "Continue? [y/N]: "
read -r response
if [[ ! "$response" =~ ^[Yy]$ ]]; then
log_info "Restore cancelled"
return 1
fi
fi
# Create a safety backup before restoring
create_timestamped_backup "pre-restore"
# Remove metadata lines before restoring
grep -v "^# Backup" "$backup_file" > /tmp/crontab_restore.tmp
if sudo crontab /tmp/crontab_restore.tmp; then
log_success "Crontab restored successfully from $backup_file"
rm -f /tmp/crontab_restore.tmp
else
log_error "Failed to restore crontab"
rm -f /tmp/crontab_restore.tmp
return 1
fi
}
compare_crontabs() {
local file1="${1:-current}"
local file2="$2"
if [ "$file1" = "current" ]; then
sudo crontab -l > /tmp/current_crontab.tmp 2>/dev/null || echo "# No current crontab" > /tmp/current_crontab.tmp
file1="/tmp/current_crontab.tmp"
fi
if [ -z "$file2" ]; then
file2="$CURRENT_BACKUP"
fi
# Handle relative paths and cross-system backups
if [[ ! "$file2" = /* ]]; then
# Check current system first
if [ -f "$ARCHIVE_DIR/$file2" ]; then
file2="$ARCHIVE_DIR/$file2"
else
# Search in other system directories
local found_file=""
for system_dir in "$BACKUP_ROOT"/*; do
if [ -f "$system_dir/archive/$file2" ]; then
found_file="$system_dir/archive/$file2"
local source_hostname=$(basename "$system_dir")
log_info "Found backup in $source_hostname system directory"
break
fi
done
if [ -n "$found_file" ]; then
file2="$found_file"
else
file2="$ARCHIVE_DIR/$file2" # Default back to current system
fi
fi
fi
if [ ! -f "$file2" ]; then
log_error "Comparison file not found: $file2"
return 1
fi
log_info "Comparing current crontab ($HOSTNAME) with: $(basename "$file2")"
echo
# Clean comparison files (remove metadata)
grep -v "^# Backup" "$file1" > /tmp/clean_file1.tmp 2>/dev/null || touch /tmp/clean_file1.tmp
grep -v "^# Backup" "$file2" > /tmp/clean_file2.tmp 2>/dev/null || touch /tmp/clean_file2.tmp
if diff -u /tmp/clean_file1.tmp /tmp/clean_file2.tmp; then
log_success "Crontabs are identical"
else
log_warning "Crontabs differ (see above)"
fi
# Cleanup
rm -f /tmp/current_crontab.tmp /tmp/clean_file1.tmp /tmp/clean_file2.tmp
}
validate_crontab_syntax() {
local crontab_file="${1:-current}"
if [ "$crontab_file" = "current" ]; then
sudo crontab -l > /tmp/validate_crontab.tmp 2>/dev/null || echo "# No current crontab" > /tmp/validate_crontab.tmp
crontab_file="/tmp/validate_crontab.tmp"
fi
if [ ! -f "$crontab_file" ]; then
log_error "Crontab file not found: $crontab_file"
return 1
fi
log_info "Validating crontab syntax: $crontab_file"
local line_num=0
local errors=0
while IFS= read -r line; do
line_num=$((line_num + 1))
# Skip comments and empty lines
if [[ "$line" =~ ^[[:space:]]*# ]] || [[ "$line" =~ ^[[:space:]]*$ ]]; then
continue
fi
# Basic cron format validation
if ! [[ "$line" =~ ^[[:space:]]*([0-9*,-]+[[:space:]]+){4}[0-9*,-]+[[:space:]].+ ]]; then
log_error "Line $line_num: Invalid cron format: $line"
errors=$((errors + 1))
fi
# Check for common issues
if [[ "$line" =~ \$\? ]] && [[ ! "$line" =~ \{ ]]; then
log_warning "Line $line_num: \$? outside of command group may not work as expected"
fi
done < "$crontab_file"
if [ $errors -eq 0 ]; then
log_success "Crontab syntax validation passed"
else
log_error "Found $errors syntax errors"
fi
rm -f /tmp/validate_crontab.tmp
return $errors
}
cleanup_old_backups() {
local keep_days="${1:-30}"
local target_hostname="${2:-$HOSTNAME}"
local deleted_count=0
if [ "$target_hostname" = "all" ]; then
log_info "Cleaning up backups older than $keep_days days for all systems"
for system_dir in "$BACKUP_ROOT"/*; do
if [ -d "$system_dir/archive" ]; then
local system_name=$(basename "$system_dir")
log_info "Cleaning backups for $system_name"
while IFS= read -r -d '' backup; do
if [ -f "$backup" ]; then
rm "$backup"
((deleted_count++))
log_info "Deleted old backup: $(basename "$backup") from $system_name"
fi
done < <(find "$system_dir/archive" -name "*.backup" -mtime +$keep_days -print0 2>/dev/null)
fi
done
else
local target_dir="$BACKUP_ROOT/$target_hostname/archive"
log_info "Cleaning up backups older than $keep_days days for $target_hostname"
if [ -d "$target_dir" ]; then
while IFS= read -r -d '' backup; do
if [ -f "$backup" ]; then
rm "$backup"
((deleted_count++))
log_info "Deleted old backup: $(basename "$backup")"
fi
done < <(find "$target_dir" -name "*.backup" -mtime +$keep_days -print0 2>/dev/null)
else
log_warning "Backup directory not found: $target_dir"
fi
fi
if [ $deleted_count -eq 0 ]; then
log_info "No old backups found to clean up"
else
log_success "Cleaned up $deleted_count old backup(s)"
fi
}
setup_automated_backup() {
log_info "Setting up automated daily crontab backup"
local backup_script="$SCRIPT_DIR/crontab-backup-system.sh"
local backup_entry="0 0 * * * $backup_script backup auto --auto-cleanup 2>&1 | logger -t crontab-backup -p user.info"
# Check if backup entry already exists
if sudo crontab -l 2>/dev/null | grep -q "crontab-backup-system.sh"; then
log_warning "Automated backup entry already exists"
return 0
fi
# Add the backup entry to current crontab
(sudo crontab -l 2>/dev/null; echo "$backup_entry") | sudo crontab -
log_success "Automated daily backup configured for $HOSTNAME"
log_info "Backups will run daily at midnight and be logged to syslog"
}
migrate_legacy_backups() {
local legacy_dir="$SCRIPT_DIR/crontab-backups"
local legacy_archive="$legacy_dir/archive"
# Check if legacy structure exists (without hostname subdirectory)
if [ -d "$legacy_archive" ] && [ "$legacy_dir" != "$BACKUP_DIR" ]; then
log_info "Found legacy backup structure, migrating to hostname-based structure"
# Create new structure
mkdir -p "$BACKUP_DIR" "$ARCHIVE_DIR"
# Move backups and rename them to include hostname
local migrated_count=0
for backup in "$legacy_archive"/*.backup; do
if [ -f "$backup" ]; then
local filename=$(basename "$backup")
local new_filename="${HOSTNAME}-${filename}"
if cp "$backup" "$ARCHIVE_DIR/$new_filename"; then
log_success "Migrated: $filename -> $new_filename"
((migrated_count++))
else
log_error "Failed to migrate: $filename"
fi
fi
done
# Move current backup if it exists
if [ -f "$legacy_dir/current-crontab.backup" ]; then
cp "$legacy_dir/current-crontab.backup" "$CURRENT_BACKUP"
log_success "Migrated current backup"
fi
if [ $migrated_count -gt 0 ]; then
log_success "Migrated $migrated_count backup(s) to new structure"
echo
log_warning "Legacy backups remain in $legacy_archive"
log_info "You can safely remove the legacy directory after verifying the migration"
echo " rm -rf '$legacy_dir'"
fi
fi
}
list_all_systems() {
log_info "All systems with crontab backups:"
echo
if [ ! -d "$BACKUP_ROOT" ]; then
log_warning "No backup root directory found: $BACKUP_ROOT"
return 1
fi
printf "%-15s %-10s %-20s %-30s\n" "System" "Backups" "Latest Backup" "Status"
printf "%-15s %-10s %-20s %-30s\n" "------" "-------" "-------------" "------"
local found_systems=false
for system_dir in "$BACKUP_ROOT"/*; do
if [ -d "$system_dir" ]; then
local system_name=$(basename "$system_dir")
# Skip legacy archive directory - it's not a system
if [ "$system_name" = "archive" ]; then
continue
fi
found_systems=true
local backup_count=$(ls -1 "$system_dir/archive"/*.backup 2>/dev/null | wc -l || echo "0")
local latest_backup="None"
local status="Inactive"
if [ -f "$system_dir/current-crontab.backup" ]; then
latest_backup=$(stat -c %y "$system_dir/current-crontab.backup" | cut -d' ' -f1)
# Check if backup is recent (within 7 days)
local backup_age=$(( ($(date +%s) - $(stat -c %Y "$system_dir/current-crontab.backup")) / 86400 ))
if [ $backup_age -le 7 ]; then
status="Active"
elif [ $backup_age -le 30 ]; then
status="Recent"
else
status="Stale"
fi
fi
# Use printf with color formatting
if [ "$status" = "Active" ]; then
printf "%-15s %-10s %-20s ${GREEN}%-30s${NC}\n" "$system_name" "$backup_count" "$latest_backup" "$status"
elif [ "$status" = "Recent" ]; then
printf "%-15s %-10s %-20s ${YELLOW}%-30s${NC}\n" "$system_name" "$backup_count" "$latest_backup" "$status"
elif [ "$status" = "Stale" ]; then
printf "%-15s %-10s %-20s ${RED}%-30s${NC}\n" "$system_name" "$backup_count" "$latest_backup" "$status"
else
printf "%-15s %-10s %-20s %-30s\n" "$system_name" "$backup_count" "$latest_backup" "$status"
fi
fi
done
if [ "$found_systems" = false ]; then
log_warning "No systems found with backups"
fi
echo
}
show_status() {
local target_hostname="${1:-$HOSTNAME}"
if [ "$target_hostname" = "all" ]; then
log_info "Crontab Backup System Status - All Systems"
echo
for system_dir in "$BACKUP_ROOT"/*; do
if [ -d "$system_dir" ]; then
local system_name=$(basename "$system_dir")
# Skip legacy archive directory - it's not a system
if [ "$system_name" = "archive" ]; then
continue
fi
echo -e "${CYAN}=== $system_name ===${NC}"
local backup_count=$(ls -1 "$system_dir/archive"/*.backup 2>/dev/null | wc -l || echo "0")
echo " - Total backups: $backup_count"
echo " - Backup directory: $system_dir"
if [ -f "$system_dir/current-crontab.backup" ]; then
echo " - Latest backup: $(stat -c %y "$system_dir/current-crontab.backup" | cut -d'.' -f1)"
else
echo " - Latest backup: None"
fi
echo
fi
done
else
log_info "Crontab Backup System Status - $target_hostname"
echo
# Current crontab info
if [ "$target_hostname" = "$HOSTNAME" ]; then
local cron_count=$(sudo crontab -l 2>/dev/null | grep -c "^[^#]" || echo "0")
echo -e "${CYAN}Current Crontab ($HOSTNAME):${NC}"
echo " - Active entries: $cron_count"
echo " - Last modified: $(stat -c %y /var/spool/cron/crontabs/root 2>/dev/null | cut -d'.' -f1 || echo "Unknown")"
echo
fi
# Backup info for specified system
local target_dir="$BACKUP_ROOT/$target_hostname"
local backup_count=$(ls -1 "$target_dir/archive"/*.backup 2>/dev/null | wc -l || echo "0")
echo -e "${CYAN}Backups ($target_hostname):${NC}"
echo " - Total backups: $backup_count"
echo " - Backup directory: $target_dir"
if [ -f "$target_dir/current-crontab.backup" ]; then
echo " - Latest backup: $(stat -c %y "$target_dir/current-crontab.backup" | cut -d'.' -f1)"
else
echo " - Latest backup: None"
fi
echo
fi
# Log monitoring
echo -e "${CYAN}Log Monitoring:${NC}"
echo " - Management log: $LOG_DIR/crontab-management.log"
echo " - System logs: journalctl -t crontab-backup"
echo
}
show_usage() {
echo -e "${PURPLE}Crontab Backup and Recovery System (Multi-System)${NC}"
echo
echo "Usage: $0 [COMMAND] [OPTIONS]"
echo
echo "Commands:"
echo " backup [TYPE] Create a timestamped backup (default: manual)"
echo " list [HOSTNAME] List backups for hostname (default: current system)"
echo " list-systems Show all systems with backups"
echo " restore FILE Restore crontab from backup file"
echo " compare [FILE1] [FILE2] Compare current crontab with backup"
echo " validate [FILE] Validate crontab syntax"
echo " cleanup [DAYS] [HOSTNAME] Clean up old backups (default: 30 days, current system)"
echo " status [HOSTNAME|all] Show system status (default: current system)"
echo " setup-auto Setup automated daily backups"
echo " migrate Migrate legacy backups to hostname structure"
echo " import FILE SYSTEM [TYPE] Import backup from external source"
echo " create-test-systems Create test systems for demonstration"
echo " help Show this help message"
echo
echo "Multi-System Examples:"
echo " $0 backup pre-upgrade # Backup current system"
echo " $0 list io # List backups for 'io' system"
echo " $0 restore io-crontab-manual-20250526_120000.backup"
echo " $0 compare current europa-crontab-manual-20250526_120000.backup"
echo " $0 cleanup 7 all # Clean up all systems"
echo " $0 status all # Show status for all systems"
echo " $0 list-systems # Show all available systems"
echo " $0 import /path/to/crontab.backup io manual # Import backup for io system"
echo
echo "Current System: $HOSTNAME"
echo "Backup Root: $BACKUP_ROOT"
echo
}
create_test_systems() {
log_info "Creating test backup structure for io, europa, and racknerd systems"
# Create sample systems directories
local test_systems=("io" "racknerd")
for system in "${test_systems[@]}"; do
local system_dir="$BACKUP_ROOT/$system"
local system_archive="$system_dir/archive"
mkdir -p "$system_archive"
# Create a sample backup for each system
local timestamp=$(date +%Y%m%d_%H%M%S)
local sample_backup="$system_archive/${system}-crontab-sample-${timestamp}.backup"
# Create sample crontab content for each system
case "$system" in
"io")
cat > "$sample_backup" << EOF
# Sample crontab for io system
0 2 * * * /home/user/backup-docker.sh 2>&1 | logger -t docker-backup -p user.info
30 3 * * * /home/user/backup-media.sh 2>&1 | logger -t media-backup -p user.info
0 4 * * * /home/user/validate-backups.sh 2>&1 | logger -t backup-validation -p user.info
# Backup created: $(date)
# Backup type: sample
# System: $system
# User: root
# Full system info: Linux $system 5.15.0-generic #1 SMP x86_64 GNU/Linux
EOF
;;
"racknerd")
cat > "$sample_backup" << EOF
# Sample crontab for racknerd system
0 1 * * * /home/user/backup-plex.sh 2>&1 | logger -t plex-backup -p user.info
15 1 * * * /home/user/move-backups.sh 2>&1 | logger -t backup-move -p user.info
0 5 * * 0 /home/user/cleanup-old-backups.sh 2>&1 | logger -t backup-cleanup -p user.info
# Backup created: $(date)
# Backup type: sample
# System: $system
# User: root
# Full system info: Linux $system 5.4.0-generic #1 SMP x86_64 GNU/Linux
EOF
;;
esac
# Create current backup file
cp "$sample_backup" "$system_dir/current-crontab.backup"
log_success "Created test system: $system with sample backup"
done
log_success "Test systems created successfully"
}
import_backup() {
local source_file="$1"
local source_system="$2"
local backup_type="${3:-imported}"
if [ -z "$source_file" ] || [ -z "$source_system" ]; then
log_error "Usage: import_backup <source_file> <source_system> [backup_type]"
return 1
fi
if [ ! -f "$source_file" ]; then
log_error "Source file not found: $source_file"
return 1
fi
local target_dir="$BACKUP_ROOT/$source_system"
local target_archive="$target_dir/archive"
mkdir -p "$target_archive"
local timestamp=$(date +%Y%m%d_%H%M%S)
local target_file="$target_archive/${source_system}-crontab-${backup_type}-${timestamp}.backup"
# Copy and add metadata
cp "$source_file" "$target_file"
# Add metadata if not present
if ! grep -q "^# Backup created:" "$target_file"; then
cat >> "$target_file" << EOF
# Backup created: $(date)
# Backup type: $backup_type
# System: $source_system
# User: root
# Imported from: $source_file
EOF
fi
# Update current backup
cp "$target_file" "$target_dir/current-crontab.backup"
log_success "Imported backup for $source_system: $target_file"
}
# Main command handling
case "${1:-help}" in
backup)
# Check for auto-cleanup flag
if [[ "${@}" == *"--auto-cleanup"* ]]; then
create_timestamped_backup "${2:-auto}"
cleanup_old_backups 30
else
create_timestamped_backup "${2:-manual}"
fi
;;
list)
list_backups "$2"
;;
list-systems)
list_all_systems
;;
restore)
if [ -z "$2" ]; then
log_error "Please specify a backup file to restore"
list_backups
exit 1
fi
restore_from_backup "$2"
;;
compare)
compare_crontabs "$2" "$3"
;;
validate)
validate_crontab_syntax "$2"
;;
cleanup)
cleanup_old_backups "${2:-30}" "${3:-$HOSTNAME}"
;;
status)
show_status "${2:-$HOSTNAME}"
;;
setup-auto)
setup_automated_backup
;;
migrate)
migrate_legacy_backups
;;
import)
if [ -z "$2" ] || [ -z "$3" ]; then
log_error "Please specify source file and target system"
echo "Usage: $0 import <source_file> <target_system> [backup_type]"
exit 1
fi
import_backup "$2" "$3" "$4"
;;
create-test-systems)
create_test_systems
;;
help|*)
show_usage
;;
esac
# Auto-migrate legacy backups on first run
if [ ! -d "$BACKUP_DIR" ] && [ "$1" != "help" ] && [ "$1" != "migrate" ]; then
migrate_legacy_backups
fi