mirror of
https://github.com/acedanger/shell.git
synced 2025-12-06 01:10:12 -08:00
Implement enhanced crontab management system with automated backups and logging
- Added enhanced crontab entries with comprehensive logging to syslog. - Created a management script (`manage-enhanced-crontab.sh`) for installing and verifying crontab entries. - Introduced backup scripts for crontab management, including automated timestamped backups and cleanup. - Developed documentation for enhanced crontab and multi-system management. - Established a directory structure for managing crontab backups across multiple systems. - Implemented error handling and logging for backup operations. - Added health monitoring and reporting features for backup processes.
This commit is contained in:
Peter Wood
387
backup-log-monitor.sh
Executable file
387
backup-log-monitor.sh
Executable file
@@ -0,0 +1,387 @@
|
||||
#!/bin/bash
|
||||
|
||||
# Enhanced Backup Log Monitor
|
||||
# Provides real-time monitoring and analysis of backup operations
|
||||
|
||||
set -e
|
||||
|
||||
# Color codes for output
|
||||
RED='\033[0;31m'
|
||||
GREEN='\033[0;32m'
|
||||
YELLOW='\033[1;33m'
|
||||
BLUE='\033[0;34m'
|
||||
CYAN='\033[0;36m'
|
||||
PURPLE='\033[0;35m'
|
||||
NC='\033[0m' # No Color
|
||||
|
||||
SCRIPT_DIR="$(dirname "$(readlink -f "$0")")"
|
||||
LOG_DIR="$SCRIPT_DIR/logs"
|
||||
REPORT_DIR="$LOG_DIR/reports"
|
||||
HOSTNAME=$(hostname)
|
||||
|
||||
# Ensure directories exist
|
||||
mkdir -p "$LOG_DIR" "$REPORT_DIR"
|
||||
|
||||
# Backup service tags for monitoring
|
||||
BACKUP_TAGS=("plex-backup" "backup-move" "plex-validation" "immich-backup" "plex-report" "crontab-backup")
|
||||
|
||||
log_message() {
|
||||
echo -e "$(date '+%H:%M:%S') $1"
|
||||
}
|
||||
|
||||
log_error() {
|
||||
log_message "${RED}ERROR: $1${NC}"
|
||||
}
|
||||
|
||||
log_success() {
|
||||
log_message "${GREEN}SUCCESS: $1${NC}"
|
||||
}
|
||||
|
||||
log_warning() {
|
||||
log_message "${YELLOW}WARNING: $1${NC}"
|
||||
}
|
||||
|
||||
log_info() {
|
||||
log_message "${BLUE}INFO: $1${NC}"
|
||||
}
|
||||
|
||||
monitor_realtime() {
|
||||
local tags_filter=""
|
||||
for tag in "${BACKUP_TAGS[@]}"; do
|
||||
tags_filter="$tags_filter -t $tag"
|
||||
done
|
||||
|
||||
log_info "Starting real-time monitoring of backup logs"
|
||||
log_info "Press Ctrl+C to stop monitoring"
|
||||
echo
|
||||
|
||||
sudo journalctl -f $tags_filter --no-hostname --output=short-iso | while read -r line; do
|
||||
# Color code different log levels and services
|
||||
if [[ "$line" =~ ERROR ]]; then
|
||||
echo -e "${RED}$line${NC}"
|
||||
elif [[ "$line" =~ SUCCESS ]]; then
|
||||
echo -e "${GREEN}$line${NC}"
|
||||
elif [[ "$line" =~ WARNING ]]; then
|
||||
echo -e "${YELLOW}$line${NC}"
|
||||
elif [[ "$line" =~ plex-backup ]]; then
|
||||
echo -e "${BLUE}$line${NC}"
|
||||
elif [[ "$line" =~ backup-move ]]; then
|
||||
echo -e "${CYAN}$line${NC}"
|
||||
elif [[ "$line" =~ plex-validation ]]; then
|
||||
echo -e "${PURPLE}$line${NC}"
|
||||
else
|
||||
echo "$line"
|
||||
fi
|
||||
done
|
||||
}
|
||||
|
||||
show_recent_logs() {
|
||||
local hours="${1:-24}"
|
||||
local service="${2:-all}"
|
||||
|
||||
log_info "Showing logs from the last $hours hours"
|
||||
|
||||
local tags_filter=""
|
||||
if [ "$service" = "all" ]; then
|
||||
for tag in "${BACKUP_TAGS[@]}"; do
|
||||
tags_filter="$tags_filter -t $tag"
|
||||
done
|
||||
else
|
||||
tags_filter="-t $service"
|
||||
fi
|
||||
|
||||
echo
|
||||
sudo journalctl --since "${hours} hours ago" $tags_filter --no-hostname --output=short-iso | \
|
||||
while read -r line; do
|
||||
# Color code the output
|
||||
if [[ "$line" =~ ERROR ]]; then
|
||||
echo -e "${RED}$line${NC}"
|
||||
elif [[ "$line" =~ SUCCESS ]]; then
|
||||
echo -e "${GREEN}$line${NC}"
|
||||
elif [[ "$line" =~ WARNING ]]; then
|
||||
echo -e "${YELLOW}$line${NC}"
|
||||
else
|
||||
echo "$line"
|
||||
fi
|
||||
done
|
||||
}
|
||||
|
||||
show_error_summary() {
|
||||
local days="${1:-7}"
|
||||
|
||||
log_info "Error summary for the last $days days"
|
||||
echo
|
||||
|
||||
local error_file="/tmp/backup_errors_$$.tmp"
|
||||
|
||||
for tag in "${BACKUP_TAGS[@]}"; do
|
||||
local error_count=$(sudo journalctl --since "${days} days ago" -t "$tag" --grep="ERROR" --output=cat | wc -l)
|
||||
if [ "$error_count" -gt 0 ]; then
|
||||
echo -e "${RED}$tag: $error_count errors${NC}"
|
||||
sudo journalctl --since "${days} days ago" -t "$tag" --grep="ERROR" --output=short-iso | head -5
|
||||
echo
|
||||
else
|
||||
echo -e "${GREEN}$tag: No errors${NC}"
|
||||
fi
|
||||
done
|
||||
}
|
||||
|
||||
generate_backup_report() {
|
||||
local days="${1:-7}"
|
||||
local report_file="$REPORT_DIR/backup-report-$(date +%Y%m%d_%H%M%S).txt"
|
||||
|
||||
log_info "Generating comprehensive backup report for the last $days days"
|
||||
log_info "Report will be saved to: $report_file"
|
||||
|
||||
{
|
||||
echo "=== BACKUP SYSTEM REPORT ==="
|
||||
echo "Generated: $(date)"
|
||||
echo "Period: Last $days days"
|
||||
echo "System: $(uname -n)"
|
||||
echo
|
||||
|
||||
for tag in "${BACKUP_TAGS[@]}"; do
|
||||
echo "=== $tag ==="
|
||||
|
||||
# Count entries
|
||||
local total_entries=$(sudo journalctl --since "${days} days ago" -t "$tag" --output=cat | wc -l)
|
||||
local error_count=$(sudo journalctl --since "${days} days ago" -t "$tag" --grep="ERROR" --output=cat | wc -l)
|
||||
local success_count=$(sudo journalctl --since "${days} days ago" -t "$tag" --grep="SUCCESS" --output=cat | wc -l)
|
||||
|
||||
echo "Total log entries: $total_entries"
|
||||
echo "Errors: $error_count"
|
||||
echo "Successes: $success_count"
|
||||
|
||||
if [ "$error_count" -gt 0 ]; then
|
||||
echo
|
||||
echo "Recent errors:"
|
||||
sudo journalctl --since "${days} days ago" -t "$tag" --grep="ERROR" --output=short-iso | head -10
|
||||
fi
|
||||
|
||||
echo
|
||||
echo "Recent activity:"
|
||||
sudo journalctl --since "${days} days ago" -t "$tag" --output=short-iso | tail -5
|
||||
echo
|
||||
echo "----------------------------------------"
|
||||
echo
|
||||
done
|
||||
|
||||
# System resource usage during backups
|
||||
echo "=== SYSTEM ANALYSIS ==="
|
||||
echo "Disk usage in backup directories:"
|
||||
if [ -d "/mnt/share/media/backups" ]; then
|
||||
du -sh /mnt/share/media/backups/* 2>/dev/null || echo "No backup directories found"
|
||||
fi
|
||||
echo
|
||||
|
||||
# Cron job status
|
||||
echo "Active cron jobs related to backups:"
|
||||
sudo crontab -l 2>/dev/null | grep -E "(backup|plex|immich)" || echo "No backup-related cron jobs found"
|
||||
echo
|
||||
|
||||
} > "$report_file"
|
||||
|
||||
log_success "Report generated: $report_file"
|
||||
|
||||
# Show summary
|
||||
echo
|
||||
log_info "Report Summary:"
|
||||
grep -E "(Total log entries|Errors|Successes):" "$report_file" | while read -r line; do
|
||||
if [[ "$line" =~ Errors:.*[1-9] ]]; then
|
||||
echo -e "${RED}$line${NC}"
|
||||
else
|
||||
echo -e "${GREEN}$line${NC}"
|
||||
fi
|
||||
done
|
||||
}
|
||||
|
||||
check_backup_health() {
|
||||
log_info "Checking backup system health for $HOSTNAME"
|
||||
echo
|
||||
|
||||
local health_score=100
|
||||
local issues=()
|
||||
|
||||
# Check if backup scripts exist
|
||||
local backup_scripts=(
|
||||
"/home/acedanger/shell/backup-plex.sh"
|
||||
"/home/acedanger/shell/move-backups.sh"
|
||||
"/home/acedanger/shell/validate-plex-backups.sh"
|
||||
"/home/acedanger/shell/crontab-backup-system.sh"
|
||||
)
|
||||
|
||||
for script in "${backup_scripts[@]}"; do
|
||||
if [ ! -f "$script" ]; then
|
||||
issues+=("Missing script: $script")
|
||||
((health_score -= 20))
|
||||
elif [ ! -x "$script" ]; then
|
||||
issues+=("Script not executable: $script")
|
||||
((health_score -= 10))
|
||||
fi
|
||||
done
|
||||
|
||||
# Check if backup directories exist
|
||||
local backup_dirs=(
|
||||
"/mnt/share/media/backups/plex"
|
||||
"/mnt/share/media/backups/docker-data"
|
||||
"/mnt/share/media/backups/immich"
|
||||
)
|
||||
|
||||
for dir in "${backup_dirs[@]}"; do
|
||||
if [ ! -d "$dir" ]; then
|
||||
issues+=("Missing backup directory: $dir")
|
||||
((health_score -= 15))
|
||||
fi
|
||||
done
|
||||
|
||||
# Check crontab backup system structure
|
||||
local crontab_backup_dir="$SCRIPT_DIR/crontab-backups/$HOSTNAME"
|
||||
if [ ! -d "$crontab_backup_dir" ]; then
|
||||
issues+=("Missing crontab backup directory for $HOSTNAME: $crontab_backup_dir")
|
||||
((health_score -= 10))
|
||||
fi
|
||||
|
||||
# Check recent backup activity
|
||||
local recent_activity=false
|
||||
for tag in "${BACKUP_TAGS[@]}"; do
|
||||
if sudo journalctl --since "24 hours ago" -t "$tag" --output=cat | grep -q .; then
|
||||
recent_activity=true
|
||||
break
|
||||
fi
|
||||
done
|
||||
|
||||
if [ "$recent_activity" = false ]; then
|
||||
issues+=("No backup activity in the last 24 hours")
|
||||
((health_score -= 25))
|
||||
fi
|
||||
|
||||
# Check for recent errors
|
||||
local recent_errors=0
|
||||
for tag in "${BACKUP_TAGS[@]}"; do
|
||||
local error_count=$(sudo journalctl --since "24 hours ago" -t "$tag" --grep="ERROR" --output=cat | wc -l)
|
||||
((recent_errors += error_count))
|
||||
done
|
||||
|
||||
if [ "$recent_errors" -gt 0 ]; then
|
||||
issues+=("$recent_errors errors in the last 24 hours")
|
||||
((health_score -= $((recent_errors * 5))))
|
||||
fi
|
||||
|
||||
# Ensure health score doesn't go below 0
|
||||
if [ "$health_score" -lt 0 ]; then
|
||||
health_score=0
|
||||
fi
|
||||
|
||||
# Display results
|
||||
if [ "$health_score" -ge 90 ]; then
|
||||
echo -e "${GREEN}Backup System Health ($HOSTNAME): ${health_score}% - EXCELLENT${NC}"
|
||||
elif [ "$health_score" -ge 70 ]; then
|
||||
echo -e "${YELLOW}Backup System Health ($HOSTNAME): ${health_score}% - GOOD${NC}"
|
||||
elif [ "$health_score" -ge 50 ]; then
|
||||
echo -e "${YELLOW}Backup System Health ($HOSTNAME): ${health_score}% - FAIR${NC}"
|
||||
else
|
||||
echo -e "${RED}Backup System Health ($HOSTNAME): ${health_score}% - POOR${NC}"
|
||||
fi
|
||||
|
||||
if [ ${#issues[@]} -gt 0 ]; then
|
||||
echo
|
||||
log_warning "Issues found:"
|
||||
for issue in "${issues[@]}"; do
|
||||
echo -e " ${RED}• $issue${NC}"
|
||||
done
|
||||
|
||||
echo
|
||||
log_info "Recommended actions:"
|
||||
echo " • Run: ./manage-enhanced-crontab.sh verify"
|
||||
echo " • Check system logs: sudo journalctl -xe"
|
||||
echo " • Verify backup directories are mounted and accessible"
|
||||
echo " • Run: ./crontab-backup-system.sh status"
|
||||
fi
|
||||
}
|
||||
|
||||
show_service_status() {
|
||||
log_info "Backup Service Status Overview"
|
||||
echo
|
||||
|
||||
printf "%-20s %-15s %-20s %-30s\n" "Service" "Status" "Last Activity" "Last Message"
|
||||
printf "%-20s %-15s %-20s %-30s\n" "-------" "------" "-------------" "------------"
|
||||
|
||||
for tag in "${BACKUP_TAGS[@]}"; do
|
||||
local last_entry=$(sudo journalctl -t "$tag" --output=short-iso -n 1 2>/dev/null | tail -1)
|
||||
|
||||
if [ -n "$last_entry" ]; then
|
||||
local timestamp=$(echo "$last_entry" | cut -d' ' -f1-2)
|
||||
local message=$(echo "$last_entry" | cut -d' ' -f4- | cut -c1-30)
|
||||
|
||||
# Check if it's recent (within 48 hours)
|
||||
local entry_time=$(date -d "$timestamp" +%s 2>/dev/null || echo "0")
|
||||
local current_time=$(date +%s)
|
||||
local hours_diff=$(( (current_time - entry_time) / 3600 ))
|
||||
|
||||
local status
|
||||
if [ "$hours_diff" -le 24 ]; then
|
||||
status="${GREEN}Active${NC}"
|
||||
elif [ "$hours_diff" -le 48 ]; then
|
||||
status="${YELLOW}Recent${NC}"
|
||||
else
|
||||
status="${RED}Stale${NC}"
|
||||
fi
|
||||
|
||||
printf "%-20s %-25s %-20s %-30s\n" "$tag" "$status" "$timestamp" "$message"
|
||||
else
|
||||
printf "%-20s %-25s %-20s %-30s\n" "$tag" "${RED}No logs${NC}" "Never" "No activity found"
|
||||
fi
|
||||
done
|
||||
}
|
||||
|
||||
show_usage() {
|
||||
echo -e "${PURPLE}Enhanced Backup Log Monitor${NC}"
|
||||
echo
|
||||
echo "Usage: $0 [COMMAND] [OPTIONS]"
|
||||
echo
|
||||
echo "Commands:"
|
||||
echo " monitor Real-time monitoring of all backup logs"
|
||||
echo " recent [HOURS] [SERVICE] Show recent logs (default: 24 hours, all services)"
|
||||
echo " errors [DAYS] Show error summary (default: 7 days)"
|
||||
echo " report [DAYS] Generate comprehensive report (default: 7 days)"
|
||||
echo " health Check backup system health"
|
||||
echo " status Show service status overview"
|
||||
echo " help Show this help message"
|
||||
echo
|
||||
echo "Services:"
|
||||
for tag in "${BACKUP_TAGS[@]}"; do
|
||||
echo " - $tag"
|
||||
done
|
||||
echo
|
||||
echo "Examples:"
|
||||
echo " $0 monitor"
|
||||
echo " $0 recent 12 plex-backup"
|
||||
echo " $0 errors 3"
|
||||
echo " $0 report 14"
|
||||
echo
|
||||
}
|
||||
|
||||
# Main command handling
|
||||
case "${1:-help}" in
|
||||
monitor)
|
||||
monitor_realtime
|
||||
;;
|
||||
recent)
|
||||
show_recent_logs "$2" "$3"
|
||||
;;
|
||||
errors)
|
||||
show_error_summary "$2"
|
||||
;;
|
||||
report)
|
||||
generate_backup_report "$2"
|
||||
;;
|
||||
health)
|
||||
check_backup_health
|
||||
;;
|
||||
status)
|
||||
show_service_status
|
||||
;;
|
||||
help|*)
|
||||
show_usage
|
||||
;;
|
||||
esac
|
||||
732
crontab-backup-system.sh
Executable file
732
crontab-backup-system.sh
Executable file
@@ -0,0 +1,732 @@
|
||||
#!/bin/bash
|
||||
|
||||
# Crontab Backup and Recovery System
|
||||
# This script provides comprehensive backup management for crontab entries
|
||||
|
||||
set -e
|
||||
|
||||
# Color codes for output
|
||||
RED='\033[0;31m'
|
||||
GREEN='\033[0;32m'
|
||||
YELLOW='\033[1;33m'
|
||||
BLUE='\033[0;34m'
|
||||
CYAN='\033[0;36m'
|
||||
PURPLE='\033[0;35m'
|
||||
NC='\033[0m' # No Color
|
||||
|
||||
SCRIPT_DIR="$(dirname "$(readlink -f "$0")")"
|
||||
HOSTNAME=$(hostname)
|
||||
BACKUP_ROOT="$SCRIPT_DIR/crontab-backups"
|
||||
BACKUP_DIR="$BACKUP_ROOT/$HOSTNAME"
|
||||
LOG_DIR="$SCRIPT_DIR/logs"
|
||||
CURRENT_BACKUP="$BACKUP_DIR/current-crontab.backup"
|
||||
ARCHIVE_DIR="$BACKUP_DIR/archive"
|
||||
|
||||
# Ensure directories exist
|
||||
mkdir -p "$BACKUP_DIR" "$ARCHIVE_DIR" "$LOG_DIR"
|
||||
|
||||
log_message() {
|
||||
local message="$1"
|
||||
local log_file="$LOG_DIR/crontab-management.log"
|
||||
echo -e "$(date '+%Y-%m-%d %H:%M:%S') $message"
|
||||
echo "$(date '+%Y-%m-%d %H:%M:%S') $message" | sed 's/\x1b\[[0-9;]*m//g' >> "$log_file"
|
||||
}
|
||||
|
||||
log_error() {
|
||||
log_message "${RED}ERROR: $1${NC}"
|
||||
}
|
||||
|
||||
log_success() {
|
||||
log_message "${GREEN}SUCCESS: $1${NC}"
|
||||
}
|
||||
|
||||
log_warning() {
|
||||
log_message "${YELLOW}WARNING: $1${NC}"
|
||||
}
|
||||
|
||||
log_info() {
|
||||
log_message "${BLUE}INFO: $1${NC}"
|
||||
}
|
||||
|
||||
create_timestamped_backup() {
|
||||
local backup_type="${1:-manual}"
|
||||
local timestamp=$(date +%Y%m%d_%H%M%S)
|
||||
local backup_file="$ARCHIVE_DIR/${HOSTNAME}-crontab-${backup_type}-${timestamp}.backup"
|
||||
|
||||
log_info "Creating timestamped backup for $HOSTNAME: $backup_file"
|
||||
|
||||
if sudo crontab -l > "$backup_file" 2>/dev/null; then
|
||||
log_success "Backup created: $backup_file"
|
||||
|
||||
# Also update the current backup
|
||||
cp "$backup_file" "$CURRENT_BACKUP"
|
||||
|
||||
# Add metadata
|
||||
echo "# Backup created: $(date)" >> "$backup_file"
|
||||
echo "# Backup type: $backup_type" >> "$backup_file"
|
||||
echo "# System: $HOSTNAME" >> "$backup_file"
|
||||
echo "# User: root" >> "$backup_file"
|
||||
echo "# Full system info: $(uname -a)" >> "$backup_file"
|
||||
|
||||
return 0
|
||||
else
|
||||
log_error "Failed to create backup or no crontab exists"
|
||||
return 1
|
||||
fi
|
||||
}
|
||||
|
||||
list_backups() {
|
||||
local target_hostname="${1:-$HOSTNAME}"
|
||||
local target_dir="$BACKUP_ROOT/$target_hostname/archive"
|
||||
|
||||
log_info "Available crontab backups for $target_hostname:"
|
||||
echo
|
||||
|
||||
if [ -d "$target_dir" ] && [ "$(ls -A "$target_dir" 2>/dev/null)" ]; then
|
||||
printf "%-40s %-20s %-15s\n" "Filename" "Date Created" "Size"
|
||||
printf "%-40s %-20s %-15s\n" "--------" "------------" "----"
|
||||
|
||||
for backup in "$target_dir"/*.backup; do
|
||||
if [ -f "$backup" ]; then
|
||||
local filename=$(basename "$backup")
|
||||
local date_created=$(stat -c %y "$backup" | cut -d' ' -f1,2 | cut -d'.' -f1)
|
||||
local size=$(stat -c %s "$backup")
|
||||
printf "%-40s %-20s %-15s bytes\n" "$filename" "$date_created" "$size"
|
||||
fi
|
||||
done
|
||||
else
|
||||
log_warning "No backups found in $target_dir"
|
||||
fi
|
||||
echo
|
||||
|
||||
# Show all available systems if current system has no backups or if showing all
|
||||
if [ "$target_hostname" = "$HOSTNAME" ] && [ ! -d "$target_dir" ]; then
|
||||
log_info "Available systems with backups:"
|
||||
for system_dir in "$BACKUP_ROOT"/*; do
|
||||
if [ -d "$system_dir/archive" ] && [ "$(ls -A "$system_dir/archive" 2>/dev/null)" ]; then
|
||||
local system_name=$(basename "$system_dir")
|
||||
local backup_count=$(ls -1 "$system_dir/archive"/*.backup 2>/dev/null | wc -l)
|
||||
echo " - $system_name ($backup_count backups)"
|
||||
fi
|
||||
done
|
||||
echo
|
||||
echo "Use: $0 list [hostname] to view backups for a specific system"
|
||||
fi
|
||||
}
|
||||
|
||||
restore_from_backup() {
|
||||
local backup_file="$1"
|
||||
local source_hostname=""
|
||||
|
||||
if [ -z "$backup_file" ]; then
|
||||
log_error "No backup file specified"
|
||||
list_backups
|
||||
return 1
|
||||
fi
|
||||
|
||||
# Handle different backup file formats and paths
|
||||
if [[ "$backup_file" == *"/"* ]]; then
|
||||
# Full or relative path provided
|
||||
if [[ ! "$backup_file" = /* ]]; then
|
||||
backup_file="$ARCHIVE_DIR/$backup_file"
|
||||
fi
|
||||
else
|
||||
# Just filename provided - check current system first, then others
|
||||
if [ -f "$ARCHIVE_DIR/$backup_file" ]; then
|
||||
backup_file="$ARCHIVE_DIR/$backup_file"
|
||||
else
|
||||
# Search in other system directories
|
||||
local found_file=""
|
||||
for system_dir in "$BACKUP_ROOT"/*; do
|
||||
if [ -f "$system_dir/archive/$backup_file" ]; then
|
||||
found_file="$system_dir/archive/$backup_file"
|
||||
source_hostname=$(basename "$system_dir")
|
||||
break
|
||||
fi
|
||||
done
|
||||
|
||||
if [ -n "$found_file" ]; then
|
||||
backup_file="$found_file"
|
||||
log_warning "Backup file found in $source_hostname system directory"
|
||||
fi
|
||||
fi
|
||||
fi
|
||||
|
||||
if [ ! -f "$backup_file" ]; then
|
||||
log_error "Backup file not found: $backup_file"
|
||||
echo
|
||||
log_info "Available backups:"
|
||||
list_backups
|
||||
return 1
|
||||
fi
|
||||
|
||||
# Extract source hostname from backup metadata if available
|
||||
if [ -z "$source_hostname" ]; then
|
||||
source_hostname=$(grep "^# System:" "$backup_file" 2>/dev/null | cut -d' ' -f3 || echo "unknown")
|
||||
fi
|
||||
|
||||
log_info "Restoring crontab from: $backup_file"
|
||||
if [ "$source_hostname" != "unknown" ] && [ "$source_hostname" != "$HOSTNAME" ]; then
|
||||
log_warning "Restoring backup from different system: $source_hostname -> $HOSTNAME"
|
||||
echo -n "Continue? [y/N]: "
|
||||
read -r response
|
||||
if [[ ! "$response" =~ ^[Yy]$ ]]; then
|
||||
log_info "Restore cancelled"
|
||||
return 1
|
||||
fi
|
||||
fi
|
||||
|
||||
# Create a safety backup before restoring
|
||||
create_timestamped_backup "pre-restore"
|
||||
|
||||
# Remove metadata lines before restoring
|
||||
grep -v "^# Backup" "$backup_file" > /tmp/crontab_restore.tmp
|
||||
|
||||
if sudo crontab /tmp/crontab_restore.tmp; then
|
||||
log_success "Crontab restored successfully from $backup_file"
|
||||
rm -f /tmp/crontab_restore.tmp
|
||||
else
|
||||
log_error "Failed to restore crontab"
|
||||
rm -f /tmp/crontab_restore.tmp
|
||||
return 1
|
||||
fi
|
||||
}
|
||||
|
||||
compare_crontabs() {
|
||||
local file1="${1:-current}"
|
||||
local file2="$2"
|
||||
|
||||
if [ "$file1" = "current" ]; then
|
||||
sudo crontab -l > /tmp/current_crontab.tmp 2>/dev/null || echo "# No current crontab" > /tmp/current_crontab.tmp
|
||||
file1="/tmp/current_crontab.tmp"
|
||||
fi
|
||||
|
||||
if [ -z "$file2" ]; then
|
||||
file2="$CURRENT_BACKUP"
|
||||
fi
|
||||
|
||||
# Handle relative paths and cross-system backups
|
||||
if [[ ! "$file2" = /* ]]; then
|
||||
# Check current system first
|
||||
if [ -f "$ARCHIVE_DIR/$file2" ]; then
|
||||
file2="$ARCHIVE_DIR/$file2"
|
||||
else
|
||||
# Search in other system directories
|
||||
local found_file=""
|
||||
for system_dir in "$BACKUP_ROOT"/*; do
|
||||
if [ -f "$system_dir/archive/$file2" ]; then
|
||||
found_file="$system_dir/archive/$file2"
|
||||
local source_hostname=$(basename "$system_dir")
|
||||
log_info "Found backup in $source_hostname system directory"
|
||||
break
|
||||
fi
|
||||
done
|
||||
|
||||
if [ -n "$found_file" ]; then
|
||||
file2="$found_file"
|
||||
else
|
||||
file2="$ARCHIVE_DIR/$file2" # Default back to current system
|
||||
fi
|
||||
fi
|
||||
fi
|
||||
|
||||
if [ ! -f "$file2" ]; then
|
||||
log_error "Comparison file not found: $file2"
|
||||
return 1
|
||||
fi
|
||||
|
||||
log_info "Comparing current crontab ($HOSTNAME) with: $(basename "$file2")"
|
||||
echo
|
||||
|
||||
# Clean comparison files (remove metadata)
|
||||
grep -v "^# Backup" "$file1" > /tmp/clean_file1.tmp 2>/dev/null || touch /tmp/clean_file1.tmp
|
||||
grep -v "^# Backup" "$file2" > /tmp/clean_file2.tmp 2>/dev/null || touch /tmp/clean_file2.tmp
|
||||
|
||||
if diff -u /tmp/clean_file1.tmp /tmp/clean_file2.tmp; then
|
||||
log_success "Crontabs are identical"
|
||||
else
|
||||
log_warning "Crontabs differ (see above)"
|
||||
fi
|
||||
|
||||
# Cleanup
|
||||
rm -f /tmp/current_crontab.tmp /tmp/clean_file1.tmp /tmp/clean_file2.tmp
|
||||
}
|
||||
|
||||
validate_crontab_syntax() {
|
||||
local crontab_file="${1:-current}"
|
||||
|
||||
if [ "$crontab_file" = "current" ]; then
|
||||
sudo crontab -l > /tmp/validate_crontab.tmp 2>/dev/null || echo "# No current crontab" > /tmp/validate_crontab.tmp
|
||||
crontab_file="/tmp/validate_crontab.tmp"
|
||||
fi
|
||||
|
||||
if [ ! -f "$crontab_file" ]; then
|
||||
log_error "Crontab file not found: $crontab_file"
|
||||
return 1
|
||||
fi
|
||||
|
||||
log_info "Validating crontab syntax: $crontab_file"
|
||||
|
||||
local line_num=0
|
||||
local errors=0
|
||||
|
||||
while IFS= read -r line; do
|
||||
((line_num++))
|
||||
|
||||
# Skip comments and empty lines
|
||||
if [[ "$line" =~ ^[[:space:]]*# ]] || [[ "$line" =~ ^[[:space:]]*$ ]]; then
|
||||
continue
|
||||
fi
|
||||
|
||||
# Basic cron format validation
|
||||
if [[ ! "$line" =~ ^[[:space:]]*([0-9*,-]+[[:space:]]+){4}[0-9*,-]+[[:space:]].+ ]]; then
|
||||
log_error "Line $line_num: Invalid cron format: $line"
|
||||
((errors++))
|
||||
fi
|
||||
|
||||
# Check for common issues
|
||||
if [[ "$line" =~ \$\? ]] && [[ ! "$line" =~ \{ ]]; then
|
||||
log_warning "Line $line_num: \$? outside of command group may not work as expected"
|
||||
fi
|
||||
|
||||
done < "$crontab_file"
|
||||
|
||||
if [ $errors -eq 0 ]; then
|
||||
log_success "Crontab syntax validation passed"
|
||||
else
|
||||
log_error "Found $errors syntax errors"
|
||||
fi
|
||||
|
||||
rm -f /tmp/validate_crontab.tmp
|
||||
return $errors
|
||||
}
|
||||
|
||||
cleanup_old_backups() {
|
||||
local keep_days="${1:-30}"
|
||||
local target_hostname="${2:-$HOSTNAME}"
|
||||
local deleted_count=0
|
||||
|
||||
if [ "$target_hostname" = "all" ]; then
|
||||
log_info "Cleaning up backups older than $keep_days days for all systems"
|
||||
|
||||
for system_dir in "$BACKUP_ROOT"/*; do
|
||||
if [ -d "$system_dir/archive" ]; then
|
||||
local system_name=$(basename "$system_dir")
|
||||
log_info "Cleaning backups for $system_name"
|
||||
|
||||
while IFS= read -r -d '' backup; do
|
||||
if [ -f "$backup" ]; then
|
||||
rm "$backup"
|
||||
((deleted_count++))
|
||||
log_info "Deleted old backup: $(basename "$backup") from $system_name"
|
||||
fi
|
||||
done < <(find "$system_dir/archive" -name "*.backup" -mtime +$keep_days -print0 2>/dev/null)
|
||||
fi
|
||||
done
|
||||
else
|
||||
local target_dir="$BACKUP_ROOT/$target_hostname/archive"
|
||||
log_info "Cleaning up backups older than $keep_days days for $target_hostname"
|
||||
|
||||
if [ -d "$target_dir" ]; then
|
||||
while IFS= read -r -d '' backup; do
|
||||
if [ -f "$backup" ]; then
|
||||
rm "$backup"
|
||||
((deleted_count++))
|
||||
log_info "Deleted old backup: $(basename "$backup")"
|
||||
fi
|
||||
done < <(find "$target_dir" -name "*.backup" -mtime +$keep_days -print0 2>/dev/null)
|
||||
else
|
||||
log_warning "Backup directory not found: $target_dir"
|
||||
fi
|
||||
fi
|
||||
|
||||
if [ $deleted_count -eq 0 ]; then
|
||||
log_info "No old backups found to clean up"
|
||||
else
|
||||
log_success "Cleaned up $deleted_count old backup(s)"
|
||||
fi
|
||||
}
|
||||
|
||||
setup_automated_backup() {
|
||||
log_info "Setting up automated daily crontab backup"
|
||||
|
||||
local backup_script="$SCRIPT_DIR/crontab-backup-system.sh"
|
||||
local backup_entry="0 0 * * * $backup_script backup auto --auto-cleanup 2>&1 | logger -t crontab-backup -p user.info"
|
||||
|
||||
# Check if backup entry already exists
|
||||
if sudo crontab -l 2>/dev/null | grep -q "crontab-backup-system.sh"; then
|
||||
log_warning "Automated backup entry already exists"
|
||||
return 0
|
||||
fi
|
||||
|
||||
# Add the backup entry to current crontab
|
||||
(sudo crontab -l 2>/dev/null; echo "$backup_entry") | sudo crontab -
|
||||
|
||||
log_success "Automated daily backup configured for $HOSTNAME"
|
||||
log_info "Backups will run daily at midnight and be logged to syslog"
|
||||
}
|
||||
|
||||
migrate_legacy_backups() {
|
||||
local legacy_dir="$SCRIPT_DIR/crontab-backups"
|
||||
local legacy_archive="$legacy_dir/archive"
|
||||
|
||||
# Check if legacy structure exists (without hostname subdirectory)
|
||||
if [ -d "$legacy_archive" ] && [ "$legacy_dir" != "$BACKUP_DIR" ]; then
|
||||
log_info "Found legacy backup structure, migrating to hostname-based structure"
|
||||
|
||||
# Create new structure
|
||||
mkdir -p "$BACKUP_DIR" "$ARCHIVE_DIR"
|
||||
|
||||
# Move backups and rename them to include hostname
|
||||
local migrated_count=0
|
||||
for backup in "$legacy_archive"/*.backup; do
|
||||
if [ -f "$backup" ]; then
|
||||
local filename=$(basename "$backup")
|
||||
local new_filename="${HOSTNAME}-${filename}"
|
||||
|
||||
if cp "$backup" "$ARCHIVE_DIR/$new_filename"; then
|
||||
log_success "Migrated: $filename -> $new_filename"
|
||||
((migrated_count++))
|
||||
else
|
||||
log_error "Failed to migrate: $filename"
|
||||
fi
|
||||
fi
|
||||
done
|
||||
|
||||
# Move current backup if it exists
|
||||
if [ -f "$legacy_dir/current-crontab.backup" ]; then
|
||||
cp "$legacy_dir/current-crontab.backup" "$CURRENT_BACKUP"
|
||||
log_success "Migrated current backup"
|
||||
fi
|
||||
|
||||
if [ $migrated_count -gt 0 ]; then
|
||||
log_success "Migrated $migrated_count backup(s) to new structure"
|
||||
echo
|
||||
log_warning "Legacy backups remain in $legacy_archive"
|
||||
log_info "You can safely remove the legacy directory after verifying the migration"
|
||||
echo " rm -rf '$legacy_dir'"
|
||||
fi
|
||||
fi
|
||||
}
|
||||
|
||||
list_all_systems() {
|
||||
log_info "All systems with crontab backups:"
|
||||
echo
|
||||
|
||||
if [ ! -d "$BACKUP_ROOT" ]; then
|
||||
log_warning "No backup root directory found: $BACKUP_ROOT"
|
||||
return 1
|
||||
fi
|
||||
|
||||
printf "%-15s %-10s %-20s %-30s\n" "System" "Backups" "Latest Backup" "Status"
|
||||
printf "%-15s %-10s %-20s %-30s\n" "------" "-------" "-------------" "------"
|
||||
|
||||
local found_systems=false
|
||||
for system_dir in "$BACKUP_ROOT"/*; do
|
||||
if [ -d "$system_dir" ]; then
|
||||
local system_name=$(basename "$system_dir")
|
||||
|
||||
# Skip legacy archive directory - it's not a system
|
||||
if [ "$system_name" = "archive" ]; then
|
||||
continue
|
||||
fi
|
||||
|
||||
found_systems=true
|
||||
local backup_count=$(ls -1 "$system_dir/archive"/*.backup 2>/dev/null | wc -l || echo "0")
|
||||
|
||||
local latest_backup="None"
|
||||
local status="Inactive"
|
||||
|
||||
if [ -f "$system_dir/current-crontab.backup" ]; then
|
||||
latest_backup=$(stat -c %y "$system_dir/current-crontab.backup" | cut -d' ' -f1)
|
||||
|
||||
# Check if backup is recent (within 7 days)
|
||||
local backup_age=$(( ($(date +%s) - $(stat -c %Y "$system_dir/current-crontab.backup")) / 86400 ))
|
||||
if [ $backup_age -le 7 ]; then
|
||||
status="Active"
|
||||
elif [ $backup_age -le 30 ]; then
|
||||
status="Recent"
|
||||
else
|
||||
status="Stale"
|
||||
fi
|
||||
fi
|
||||
|
||||
# Use printf with color formatting
|
||||
if [ "$status" = "Active" ]; then
|
||||
printf "%-15s %-10s %-20s ${GREEN}%-30s${NC}\n" "$system_name" "$backup_count" "$latest_backup" "$status"
|
||||
elif [ "$status" = "Recent" ]; then
|
||||
printf "%-15s %-10s %-20s ${YELLOW}%-30s${NC}\n" "$system_name" "$backup_count" "$latest_backup" "$status"
|
||||
elif [ "$status" = "Stale" ]; then
|
||||
printf "%-15s %-10s %-20s ${RED}%-30s${NC}\n" "$system_name" "$backup_count" "$latest_backup" "$status"
|
||||
else
|
||||
printf "%-15s %-10s %-20s %-30s\n" "$system_name" "$backup_count" "$latest_backup" "$status"
|
||||
fi
|
||||
fi
|
||||
done
|
||||
|
||||
if [ "$found_systems" = false ]; then
|
||||
log_warning "No systems found with backups"
|
||||
fi
|
||||
echo
|
||||
}
|
||||
|
||||
show_status() {
|
||||
local target_hostname="${1:-$HOSTNAME}"
|
||||
|
||||
if [ "$target_hostname" = "all" ]; then
|
||||
log_info "Crontab Backup System Status - All Systems"
|
||||
echo
|
||||
|
||||
for system_dir in "$BACKUP_ROOT"/*; do
|
||||
if [ -d "$system_dir" ]; then
|
||||
local system_name=$(basename "$system_dir")
|
||||
|
||||
# Skip legacy archive directory - it's not a system
|
||||
if [ "$system_name" = "archive" ]; then
|
||||
continue
|
||||
fi
|
||||
|
||||
echo -e "${CYAN}=== $system_name ===${NC}"
|
||||
|
||||
local backup_count=$(ls -1 "$system_dir/archive"/*.backup 2>/dev/null | wc -l || echo "0")
|
||||
echo " - Total backups: $backup_count"
|
||||
echo " - Backup directory: $system_dir"
|
||||
|
||||
if [ -f "$system_dir/current-crontab.backup" ]; then
|
||||
echo " - Latest backup: $(stat -c %y "$system_dir/current-crontab.backup" | cut -d'.' -f1)"
|
||||
else
|
||||
echo " - Latest backup: None"
|
||||
fi
|
||||
echo
|
||||
fi
|
||||
done
|
||||
else
|
||||
log_info "Crontab Backup System Status - $target_hostname"
|
||||
echo
|
||||
|
||||
# Current crontab info
|
||||
if [ "$target_hostname" = "$HOSTNAME" ]; then
|
||||
local cron_count=$(sudo crontab -l 2>/dev/null | grep -c "^[^#]" || echo "0")
|
||||
echo -e "${CYAN}Current Crontab ($HOSTNAME):${NC}"
|
||||
echo " - Active entries: $cron_count"
|
||||
echo " - Last modified: $(stat -c %y /var/spool/cron/crontabs/root 2>/dev/null | cut -d'.' -f1 || echo "Unknown")"
|
||||
echo
|
||||
fi
|
||||
|
||||
# Backup info for specified system
|
||||
local target_dir="$BACKUP_ROOT/$target_hostname"
|
||||
local backup_count=$(ls -1 "$target_dir/archive"/*.backup 2>/dev/null | wc -l || echo "0")
|
||||
echo -e "${CYAN}Backups ($target_hostname):${NC}"
|
||||
echo " - Total backups: $backup_count"
|
||||
echo " - Backup directory: $target_dir"
|
||||
if [ -f "$target_dir/current-crontab.backup" ]; then
|
||||
echo " - Latest backup: $(stat -c %y "$target_dir/current-crontab.backup" | cut -d'.' -f1)"
|
||||
else
|
||||
echo " - Latest backup: None"
|
||||
fi
|
||||
echo
|
||||
fi
|
||||
|
||||
# Log monitoring
|
||||
echo -e "${CYAN}Log Monitoring:${NC}"
|
||||
echo " - Management log: $LOG_DIR/crontab-management.log"
|
||||
echo " - System logs: journalctl -t crontab-backup"
|
||||
echo
|
||||
}
|
||||
|
||||
show_usage() {
|
||||
echo -e "${PURPLE}Crontab Backup and Recovery System (Multi-System)${NC}"
|
||||
echo
|
||||
echo "Usage: $0 [COMMAND] [OPTIONS]"
|
||||
echo
|
||||
echo "Commands:"
|
||||
echo " backup [TYPE] Create a timestamped backup (default: manual)"
|
||||
echo " list [HOSTNAME] List backups for hostname (default: current system)"
|
||||
echo " list-systems Show all systems with backups"
|
||||
echo " restore FILE Restore crontab from backup file"
|
||||
echo " compare [FILE1] [FILE2] Compare current crontab with backup"
|
||||
echo " validate [FILE] Validate crontab syntax"
|
||||
echo " cleanup [DAYS] [HOSTNAME] Clean up old backups (default: 30 days, current system)"
|
||||
echo " status [HOSTNAME|all] Show system status (default: current system)"
|
||||
echo " setup-auto Setup automated daily backups"
|
||||
echo " migrate Migrate legacy backups to hostname structure"
|
||||
echo " import FILE SYSTEM [TYPE] Import backup from external source"
|
||||
echo " create-test-systems Create test systems for demonstration"
|
||||
echo " help Show this help message"
|
||||
echo
|
||||
echo "Multi-System Examples:"
|
||||
echo " $0 backup pre-upgrade # Backup current system"
|
||||
echo " $0 list io # List backups for 'io' system"
|
||||
echo " $0 restore io-crontab-manual-20250526_120000.backup"
|
||||
echo " $0 compare current europa-crontab-manual-20250526_120000.backup"
|
||||
echo " $0 cleanup 7 all # Clean up all systems"
|
||||
echo " $0 status all # Show status for all systems"
|
||||
echo " $0 list-systems # Show all available systems"
|
||||
echo " $0 import /path/to/crontab.backup io manual # Import backup for io system"
|
||||
echo
|
||||
echo "Current System: $HOSTNAME"
|
||||
echo "Backup Root: $BACKUP_ROOT"
|
||||
echo
|
||||
}
|
||||
|
||||
create_test_systems() {
|
||||
log_info "Creating test backup structure for io, europa, and racknerd systems"
|
||||
|
||||
# Create sample systems directories
|
||||
local test_systems=("io" "racknerd")
|
||||
|
||||
for system in "${test_systems[@]}"; do
|
||||
local system_dir="$BACKUP_ROOT/$system"
|
||||
local system_archive="$system_dir/archive"
|
||||
|
||||
mkdir -p "$system_archive"
|
||||
|
||||
# Create a sample backup for each system
|
||||
local timestamp=$(date +%Y%m%d_%H%M%S)
|
||||
local sample_backup="$system_archive/${system}-crontab-sample-${timestamp}.backup"
|
||||
|
||||
# Create sample crontab content for each system
|
||||
case "$system" in
|
||||
"io")
|
||||
cat > "$sample_backup" << EOF
|
||||
# Sample crontab for io system
|
||||
0 2 * * * /home/user/backup-docker.sh 2>&1 | logger -t docker-backup -p user.info
|
||||
30 3 * * * /home/user/backup-media.sh 2>&1 | logger -t media-backup -p user.info
|
||||
0 4 * * * /home/user/validate-backups.sh 2>&1 | logger -t backup-validation -p user.info
|
||||
# Backup created: $(date)
|
||||
# Backup type: sample
|
||||
# System: $system
|
||||
# User: root
|
||||
# Full system info: Linux $system 5.15.0-generic #1 SMP x86_64 GNU/Linux
|
||||
EOF
|
||||
;;
|
||||
"racknerd")
|
||||
cat > "$sample_backup" << EOF
|
||||
# Sample crontab for racknerd system
|
||||
0 1 * * * /home/user/backup-plex.sh 2>&1 | logger -t plex-backup -p user.info
|
||||
15 1 * * * /home/user/move-backups.sh 2>&1 | logger -t backup-move -p user.info
|
||||
0 5 * * 0 /home/user/cleanup-old-backups.sh 2>&1 | logger -t backup-cleanup -p user.info
|
||||
# Backup created: $(date)
|
||||
# Backup type: sample
|
||||
# System: $system
|
||||
# User: root
|
||||
# Full system info: Linux $system 5.4.0-generic #1 SMP x86_64 GNU/Linux
|
||||
EOF
|
||||
;;
|
||||
esac
|
||||
|
||||
# Create current backup file
|
||||
cp "$sample_backup" "$system_dir/current-crontab.backup"
|
||||
|
||||
log_success "Created test system: $system with sample backup"
|
||||
done
|
||||
|
||||
log_success "Test systems created successfully"
|
||||
}
|
||||
|
||||
import_backup() {
|
||||
local source_file="$1"
|
||||
local source_system="$2"
|
||||
local backup_type="${3:-imported}"
|
||||
|
||||
if [ -z "$source_file" ] || [ -z "$source_system" ]; then
|
||||
log_error "Usage: import_backup <source_file> <source_system> [backup_type]"
|
||||
return 1
|
||||
fi
|
||||
|
||||
if [ ! -f "$source_file" ]; then
|
||||
log_error "Source file not found: $source_file"
|
||||
return 1
|
||||
fi
|
||||
|
||||
local target_dir="$BACKUP_ROOT/$source_system"
|
||||
local target_archive="$target_dir/archive"
|
||||
|
||||
mkdir -p "$target_archive"
|
||||
|
||||
local timestamp=$(date +%Y%m%d_%H%M%S)
|
||||
local target_file="$target_archive/${source_system}-crontab-${backup_type}-${timestamp}.backup"
|
||||
|
||||
# Copy and add metadata
|
||||
cp "$source_file" "$target_file"
|
||||
|
||||
# Add metadata if not present
|
||||
if ! grep -q "^# Backup created:" "$target_file"; then
|
||||
cat >> "$target_file" << EOF
|
||||
# Backup created: $(date)
|
||||
# Backup type: $backup_type
|
||||
# System: $source_system
|
||||
# User: root
|
||||
# Imported from: $source_file
|
||||
EOF
|
||||
fi
|
||||
|
||||
# Update current backup
|
||||
cp "$target_file" "$target_dir/current-crontab.backup"
|
||||
|
||||
log_success "Imported backup for $source_system: $target_file"
|
||||
}
|
||||
|
||||
# Main command handling
|
||||
case "${1:-help}" in
|
||||
backup)
|
||||
# Check for auto-cleanup flag
|
||||
if [[ "${@}" == *"--auto-cleanup"* ]]; then
|
||||
create_timestamped_backup "${2:-auto}"
|
||||
cleanup_old_backups 30
|
||||
else
|
||||
create_timestamped_backup "${2:-manual}"
|
||||
fi
|
||||
;;
|
||||
list)
|
||||
list_backups "$2"
|
||||
;;
|
||||
list-systems)
|
||||
list_all_systems
|
||||
;;
|
||||
restore)
|
||||
if [ -z "$2" ]; then
|
||||
log_error "Please specify a backup file to restore"
|
||||
list_backups
|
||||
exit 1
|
||||
fi
|
||||
restore_from_backup "$2"
|
||||
;;
|
||||
compare)
|
||||
compare_crontabs "$2" "$3"
|
||||
;;
|
||||
validate)
|
||||
validate_crontab_syntax "$2"
|
||||
;;
|
||||
cleanup)
|
||||
cleanup_old_backups "${2:-30}" "${3:-$HOSTNAME}"
|
||||
;;
|
||||
status)
|
||||
show_status "${2:-$HOSTNAME}"
|
||||
;;
|
||||
setup-auto)
|
||||
setup_automated_backup
|
||||
;;
|
||||
migrate)
|
||||
migrate_legacy_backups
|
||||
;;
|
||||
import)
|
||||
if [ -z "$2" ] || [ -z "$3" ]; then
|
||||
log_error "Please specify source file and target system"
|
||||
echo "Usage: $0 import <source_file> <target_system> [backup_type]"
|
||||
exit 1
|
||||
fi
|
||||
import_backup "$2" "$3" "$4"
|
||||
;;
|
||||
create-test-systems)
|
||||
create_test_systems
|
||||
;;
|
||||
help|*)
|
||||
show_usage
|
||||
;;
|
||||
esac
|
||||
|
||||
# Auto-migrate legacy backups on first run
|
||||
if [ ! -d "$BACKUP_DIR" ] && [ "$1" != "help" ] && [ "$1" != "migrate" ]; then
|
||||
migrate_legacy_backups
|
||||
fi
|
||||
@@ -0,0 +1,40 @@
|
||||
# Edit this file to introduce tasks to be run by cron.
|
||||
#
|
||||
# Each task to run has to be defined through a single line
|
||||
# indicating with different fields when the task will be run
|
||||
# and what command to run for the task
|
||||
#
|
||||
# To define the time you can provide concrete values for
|
||||
# minute (m), hour (h), day of month (dom), month (mon),
|
||||
# and day of week (dow) or use '*' in these fields (for 'any').
|
||||
#
|
||||
# Notice that tasks will be started based on the cron's system
|
||||
# daemon's notion of time and timezones.
|
||||
#
|
||||
# Output of the crontab jobs (including errors) is sent through
|
||||
# email to the user the crontab file belongs to (unless redirected).
|
||||
#
|
||||
# For example, you can run a backup of all your user accounts
|
||||
# at 5 a.m every week with:
|
||||
# 0 5 * * 1 tar -zcf /var/backups/home.tgz /home/
|
||||
#
|
||||
# For more information see the manual pages of crontab(5) and cron(8)
|
||||
#
|
||||
# m h dom mon dow command
|
||||
# 2023-06-11 I can't figure out how to get the nas to mount automagically so here is a band-aid
|
||||
# @reboot sleep 30 && mount -a
|
||||
#
|
||||
# move the files previously backed up at 0100
|
||||
0 1 * * * /home/acedanger/shell/move-backups.sh
|
||||
# daily Plex backup at 0415
|
||||
15 4 * * * /home/acedanger/shell/backup-plex.sh
|
||||
# daily validation at 0700
|
||||
0 7 * * * /home/acedanger/shell/validate-plex-backups.sh --fix
|
||||
# backup immich database
|
||||
0 5 * * 1 mv /mnt/share/media/immich/uploads/backups/immich-db-backup* /mnt/share/media/backups/immich
|
||||
# Generate detailed weekly report (Sundays at 0800)
|
||||
0 8 * * 0 /home/acedanger/shell/validate-plex-backups.sh --report
|
||||
# Backup created: Mon May 26 10:06:22 AM EDT 2025
|
||||
# Backup type: pre-install
|
||||
# System: europa
|
||||
# User: root
|
||||
@@ -0,0 +1,41 @@
|
||||
# Edit this file to introduce tasks to be run by cron.
|
||||
#
|
||||
# Each task to run has to be defined through a single line
|
||||
# indicating with different fields when the task will be run
|
||||
# and what command to run for the task
|
||||
#
|
||||
# To define the time you can provide concrete values for
|
||||
# minute (m), hour (h), day of month (dom), month (mon),
|
||||
# and day of week (dow) or use '*' in these fields (for 'any').
|
||||
#
|
||||
# Notice that tasks will be started based on the cron's system
|
||||
# daemon's notion of time and timezones.
|
||||
#
|
||||
# Output of the crontab jobs (including errors) is sent through
|
||||
# email to the user the crontab file belongs to (unless redirected).
|
||||
#
|
||||
# For example, you can run a backup of all your user accounts
|
||||
# at 5 a.m every week with:
|
||||
# 0 5 * * 1 tar -zcf /var/backups/home.tgz /home/
|
||||
#
|
||||
# For more information see the manual pages of crontab(5) and cron(8)
|
||||
#
|
||||
# m h dom mon dow command
|
||||
# 2023-06-11 I can't figure out how to get the nas to mount automagically so here is a band-aid
|
||||
# @reboot sleep 30 && mount -a
|
||||
#
|
||||
# move the files previously backed up at 0100
|
||||
0 1 * * * /home/acedanger/shell/move-backups.sh
|
||||
# daily Plex backup at 0415
|
||||
15 4 * * * /home/acedanger/shell/backup-plex.sh
|
||||
# daily validation at 0700
|
||||
0 7 * * * /home/acedanger/shell/validate-plex-backups.sh --fix
|
||||
# backup immich database
|
||||
0 5 * * 1 mv /mnt/share/media/immich/uploads/backups/immich-db-backup* /mnt/share/media/backups/immich
|
||||
# Generate detailed weekly report (Sundays at 0800)
|
||||
0 8 * * 0 /home/acedanger/shell/validate-plex-backups.sh --report
|
||||
# Backup created: Mon May 26 10:13:54 AM EDT 2025
|
||||
# Backup type: initial
|
||||
# System: europa
|
||||
# User: root
|
||||
# Full system info: Linux europa 6.1.0-33-amd64 #1 SMP PREEMPT_DYNAMIC Debian 6.1.133-1 (2025-04-10) x86_64 GNU/Linux
|
||||
@@ -0,0 +1,40 @@
|
||||
# Edit this file to introduce tasks to be run by cron.
|
||||
#
|
||||
# Each task to run has to be defined through a single line
|
||||
# indicating with different fields when the task will be run
|
||||
# and what command to run for the task
|
||||
#
|
||||
# To define the time you can provide concrete values for
|
||||
# minute (m), hour (h), day of month (dom), month (mon),
|
||||
# and day of week (dow) or use '*' in these fields (for 'any').
|
||||
#
|
||||
# Notice that tasks will be started based on the cron's system
|
||||
# daemon's notion of time and timezones.
|
||||
#
|
||||
# Output of the crontab jobs (including errors) is sent through
|
||||
# email to the user the crontab file belongs to (unless redirected).
|
||||
#
|
||||
# For example, you can run a backup of all your user accounts
|
||||
# at 5 a.m every week with:
|
||||
# 0 5 * * 1 tar -zcf /var/backups/home.tgz /home/
|
||||
#
|
||||
# For more information see the manual pages of crontab(5) and cron(8)
|
||||
#
|
||||
# m h dom mon dow command
|
||||
# 2023-06-11 I can't figure out how to get the nas to mount automagically so here is a band-aid
|
||||
# @reboot sleep 30 && mount -a
|
||||
#
|
||||
# move the files previously backed up at 0100
|
||||
0 1 * * * /home/acedanger/shell/move-backups.sh
|
||||
# daily Plex backup at 0415
|
||||
15 4 * * * /home/acedanger/shell/backup-plex.sh
|
||||
# daily validation at 0700
|
||||
0 7 * * * /home/acedanger/shell/validate-plex-backups.sh --fix
|
||||
# backup immich database
|
||||
0 5 * * 1 mv /mnt/share/media/immich/uploads/backups/immich-db-backup* /mnt/share/media/backups/immich
|
||||
# Generate detailed weekly report (Sundays at 0800)
|
||||
0 8 * * 0 /home/acedanger/shell/validate-plex-backups.sh --report
|
||||
# Backup created: Mon May 26 10:06:22 AM EDT 2025
|
||||
# Backup type: pre-install
|
||||
# System: europa
|
||||
# User: root
|
||||
@@ -0,0 +1,41 @@
|
||||
# Edit this file to introduce tasks to be run by cron.
|
||||
#
|
||||
# Each task to run has to be defined through a single line
|
||||
# indicating with different fields when the task will be run
|
||||
# and what command to run for the task
|
||||
#
|
||||
# To define the time you can provide concrete values for
|
||||
# minute (m), hour (h), day of month (dom), month (mon),
|
||||
# and day of week (dow) or use '*' in these fields (for 'any').
|
||||
#
|
||||
# Notice that tasks will be started based on the cron's system
|
||||
# daemon's notion of time and timezones.
|
||||
#
|
||||
# Output of the crontab jobs (including errors) is sent through
|
||||
# email to the user the crontab file belongs to (unless redirected).
|
||||
#
|
||||
# For example, you can run a backup of all your user accounts
|
||||
# at 5 a.m every week with:
|
||||
# 0 5 * * 1 tar -zcf /var/backups/home.tgz /home/
|
||||
#
|
||||
# For more information see the manual pages of crontab(5) and cron(8)
|
||||
#
|
||||
# m h dom mon dow command
|
||||
# 2023-06-11 I can't figure out how to get the nas to mount automagically so here is a band-aid
|
||||
# @reboot sleep 30 && mount -a
|
||||
#
|
||||
# move the files previously backed up at 0100
|
||||
0 1 * * * /home/acedanger/shell/move-backups.sh
|
||||
# daily Plex backup at 0415
|
||||
15 4 * * * /home/acedanger/shell/backup-plex.sh
|
||||
# daily validation at 0700
|
||||
0 7 * * * /home/acedanger/shell/validate-plex-backups.sh --fix
|
||||
# backup immich database
|
||||
0 5 * * 1 mv /mnt/share/media/immich/uploads/backups/immich-db-backup* /mnt/share/media/backups/immich
|
||||
# Generate detailed weekly report (Sundays at 0800)
|
||||
0 8 * * 0 /home/acedanger/shell/validate-plex-backups.sh --report
|
||||
# Backup created: Mon May 26 10:30:05 AM EDT 2025
|
||||
# Backup type: test-colors
|
||||
# System: europa
|
||||
# User: root
|
||||
# Full system info: Linux europa 6.1.0-33-amd64 #1 SMP PREEMPT_DYNAMIC Debian 6.1.133-1 (2025-04-10) x86_64 GNU/Linux
|
||||
36
crontab-backups/europa/current-crontab.backup
Normal file
36
crontab-backups/europa/current-crontab.backup
Normal file
@@ -0,0 +1,36 @@
|
||||
# Edit this file to introduce tasks to be run by cron.
|
||||
#
|
||||
# Each task to run has to be defined through a single line
|
||||
# indicating with different fields when the task will be run
|
||||
# and what command to run for the task
|
||||
#
|
||||
# To define the time you can provide concrete values for
|
||||
# minute (m), hour (h), day of month (dom), month (mon),
|
||||
# and day of week (dow) or use '*' in these fields (for 'any').
|
||||
#
|
||||
# Notice that tasks will be started based on the cron's system
|
||||
# daemon's notion of time and timezones.
|
||||
#
|
||||
# Output of the crontab jobs (including errors) is sent through
|
||||
# email to the user the crontab file belongs to (unless redirected).
|
||||
#
|
||||
# For example, you can run a backup of all your user accounts
|
||||
# at 5 a.m every week with:
|
||||
# 0 5 * * 1 tar -zcf /var/backups/home.tgz /home/
|
||||
#
|
||||
# For more information see the manual pages of crontab(5) and cron(8)
|
||||
#
|
||||
# m h dom mon dow command
|
||||
# 2023-06-11 I can't figure out how to get the nas to mount automagically so here is a band-aid
|
||||
# @reboot sleep 30 && mount -a
|
||||
#
|
||||
# move the files previously backed up at 0100
|
||||
0 1 * * * /home/acedanger/shell/move-backups.sh
|
||||
# daily Plex backup at 0415
|
||||
15 4 * * * /home/acedanger/shell/backup-plex.sh
|
||||
# daily validation at 0700
|
||||
0 7 * * * /home/acedanger/shell/validate-plex-backups.sh --fix
|
||||
# backup immich database
|
||||
0 5 * * 1 mv /mnt/share/media/immich/uploads/backups/immich-db-backup* /mnt/share/media/backups/immich
|
||||
# Generate detailed weekly report (Sundays at 0800)
|
||||
0 8 * * 0 /home/acedanger/shell/validate-plex-backups.sh --report
|
||||
226
docs/enhanced-crontab-system.md
Normal file
226
docs/enhanced-crontab-system.md
Normal file
@@ -0,0 +1,226 @@
|
||||
# Enhanced Crontab and Backup Monitoring System
|
||||
|
||||
## Overview
|
||||
|
||||
This enhanced system provides comprehensive crontab management with automatic backups, system logging, and advanced monitoring capabilities for your backup operations.
|
||||
|
||||
## Components
|
||||
|
||||
### 1. Enhanced Crontab Entries (`enhanced-crontab.txt`)
|
||||
|
||||
- **Move backups** (01:00): Transfers Docker backups with logging
|
||||
- **Plex backup** (04:15): Daily Plex database backup with auto-repair
|
||||
- **Plex validation** (07:00): Validates and fixes backup integrity
|
||||
- **Immich backup** (05:00 Mon): Weekly Immich database backup move
|
||||
- **Weekly report** (08:00 Sun): Comprehensive backup status report
|
||||
|
||||
### 2. Crontab Management (`manage-enhanced-crontab.sh`)
|
||||
|
||||
- Install enhanced crontab entries with validation
|
||||
- Verify all backup scripts exist and are executable
|
||||
- Setup automated backups and log rotation
|
||||
- Integration with backup and monitoring systems
|
||||
|
||||
### 3. Crontab Backup System (`crontab-backup-system.sh`)
|
||||
|
||||
- Automated timestamped crontab backups
|
||||
- Backup comparison and restoration
|
||||
- Syntax validation
|
||||
- Automated cleanup of old backups
|
||||
- Daily automated backup scheduling
|
||||
|
||||
### 4. Backup Log Monitor (`backup-log-monitor.sh`)
|
||||
|
||||
- Real-time log monitoring with color coding
|
||||
- Error analysis and reporting
|
||||
- System health checks
|
||||
- Comprehensive backup reports
|
||||
- Service status overview
|
||||
|
||||
## Quick Start
|
||||
|
||||
### 1. Install the Enhanced System
|
||||
|
||||
```bash
|
||||
# Make scripts executable
|
||||
chmod +x manage-enhanced-crontab.sh crontab-backup-system.sh backup-log-monitor.sh
|
||||
|
||||
# Install enhanced crontab with all features
|
||||
sudo ./manage-enhanced-crontab.sh install
|
||||
```
|
||||
|
||||
### 2. Monitor Your Backups
|
||||
|
||||
```bash
|
||||
# Real-time monitoring
|
||||
./backup-log-monitor.sh monitor
|
||||
|
||||
# Check system health
|
||||
./backup-log-monitor.sh health
|
||||
|
||||
# View recent activity
|
||||
./backup-log-monitor.sh recent 24
|
||||
|
||||
# Generate weekly report
|
||||
./backup-log-monitor.sh report 7
|
||||
```
|
||||
|
||||
### 3. Manage Crontab Backups
|
||||
|
||||
```bash
|
||||
# Create manual backup
|
||||
./crontab-backup-system.sh backup manual
|
||||
|
||||
# List all backups
|
||||
./crontab-backup-system.sh list
|
||||
|
||||
# Compare current with backup
|
||||
./crontab-backup-system.sh compare current
|
||||
|
||||
# System status
|
||||
./crontab-backup-system.sh status
|
||||
```
|
||||
|
||||
## Features
|
||||
|
||||
### Enhanced Logging
|
||||
|
||||
All backup operations now log to syslog with specific tags:
|
||||
|
||||
- `plex-backup`: Plex database backup operations
|
||||
- `backup-move`: Docker backup file transfers
|
||||
- `plex-validation`: Backup integrity checks
|
||||
- `immich-backup`: Immich database operations
|
||||
- `plex-report`: Weekly reporting
|
||||
- `crontab-backup`: Crontab backup operations
|
||||
|
||||
### Automatic Backups
|
||||
|
||||
- **Crontab backups**: Daily automated backups at midnight
|
||||
- **Cleanup**: Automatic removal of backups older than 30 days
|
||||
- **Validation**: Syntax checking before applying changes
|
||||
- **Recovery**: Easy restoration from any backup point
|
||||
|
||||
### Health Monitoring
|
||||
|
||||
- Script existence and permissions
|
||||
- Backup directory availability
|
||||
- Recent activity tracking
|
||||
- Error rate monitoring
|
||||
- Overall system health scoring
|
||||
|
||||
### Error Handling
|
||||
|
||||
- Graceful failure handling in cron jobs
|
||||
- Detailed error logging and reporting
|
||||
- Exit code tracking for debugging
|
||||
- Comprehensive error summaries
|
||||
|
||||
## Log Analysis
|
||||
|
||||
### View Real-time Logs
|
||||
|
||||
```bash
|
||||
# All backup services
|
||||
sudo journalctl -f -t plex-backup -t backup-move -t plex-validation -t immich-backup -t plex-report
|
||||
|
||||
# Specific service
|
||||
sudo journalctl -f -t plex-backup
|
||||
|
||||
# With our monitor (recommended)
|
||||
./backup-log-monitor.sh monitor
|
||||
```
|
||||
|
||||
### Historical Analysis
|
||||
|
||||
```bash
|
||||
# Last 24 hours
|
||||
sudo journalctl --since '24 hours ago' -t plex-backup
|
||||
|
||||
# Last week with errors only
|
||||
sudo journalctl --since '1 week ago' --priority=err -t plex-backup
|
||||
|
||||
# Using our tools
|
||||
./backup-log-monitor.sh recent 24 plex-backup
|
||||
./backup-log-monitor.sh errors 7
|
||||
```
|
||||
|
||||
## Backup Recovery
|
||||
|
||||
### Restore Crontab from Backup
|
||||
|
||||
```bash
|
||||
# List available backups
|
||||
./crontab-backup-system.sh list
|
||||
|
||||
# Restore specific backup
|
||||
./crontab-backup-system.sh restore crontab-manual-20250526_120000.backup
|
||||
|
||||
# Compare before restoring
|
||||
./crontab-backup-system.sh compare current crontab-manual-20250526_120000.backup
|
||||
```
|
||||
|
||||
### Emergency Recovery
|
||||
|
||||
If you need to quickly restore the original crontab:
|
||||
|
||||
```bash
|
||||
# The system automatically creates pre-install backups
|
||||
./crontab-backup-system.sh list | grep pre-install
|
||||
./crontab-backup-system.sh restore [backup-filename]
|
||||
```
|
||||
|
||||
## Maintenance
|
||||
|
||||
### Regular Tasks
|
||||
|
||||
```bash
|
||||
# Weekly health check
|
||||
./backup-log-monitor.sh health
|
||||
|
||||
# Monthly backup cleanup
|
||||
./crontab-backup-system.sh cleanup 30
|
||||
|
||||
# Quarterly comprehensive report
|
||||
./backup-log-monitor.sh report 90
|
||||
```
|
||||
|
||||
### Troubleshooting
|
||||
|
||||
```bash
|
||||
# Verify all components
|
||||
./manage-enhanced-crontab.sh verify
|
||||
|
||||
# Check system status
|
||||
./manage-enhanced-crontab.sh status
|
||||
|
||||
# View configuration
|
||||
./manage-enhanced-crontab.sh show
|
||||
|
||||
# Monitor for issues
|
||||
./backup-log-monitor.sh monitor
|
||||
```
|
||||
|
||||
## Integration Notes
|
||||
|
||||
- All scripts follow the established shell repository coding standards
|
||||
- Logging uses consistent tags and priorities
|
||||
- Error handling preserves backup integrity
|
||||
- Color-coded output for better readability
|
||||
- Comprehensive documentation and help systems
|
||||
|
||||
## Security Considerations
|
||||
|
||||
- Scripts validate input and handle errors gracefully
|
||||
- Backup files include metadata for tracking
|
||||
- Permissions are properly managed
|
||||
- Sensitive operations require sudo confirmation
|
||||
- All operations are logged for audit trails
|
||||
|
||||
## Next Steps
|
||||
|
||||
1. **Install the system**: Run `sudo ./manage-enhanced-crontab.sh install`
|
||||
2. **Test monitoring**: Use `./backup-log-monitor.sh monitor` during next backup
|
||||
3. **Review reports**: Generate weekly reports to establish baseline
|
||||
4. **Set up alerts**: Consider integrating with your notification system
|
||||
5. **Document customizations**: Add any local modifications to this guide
|
||||
280
docs/multi-system-crontab-management.md
Normal file
280
docs/multi-system-crontab-management.md
Normal file
@@ -0,0 +1,280 @@
|
||||
# Multi-System Crontab Management Guide
|
||||
|
||||
## Overview
|
||||
|
||||
The enhanced crontab backup system now supports managing crontab backups across multiple systems using a hostname-based directory structure. This enables centralized backup management for distributed environments.
|
||||
|
||||
## System Architecture
|
||||
|
||||
### Directory Structure
|
||||
|
||||
```
|
||||
crontab-backups/
|
||||
├── europa/ # Current system (example)
|
||||
│ ├── current-crontab.backup
|
||||
│ └── archive/
|
||||
│ ├── europa-crontab-initial-20250526_101354.backup
|
||||
│ └── europa-crontab-pre-install-20250526_100622.backup
|
||||
├── io/ # Remote system backups
|
||||
│ ├── current-crontab.backup
|
||||
│ └── archive/
|
||||
│ └── io-crontab-sample-20250526_101558.backup
|
||||
└── racknerd/ # Another remote system
|
||||
├── current-crontab.backup
|
||||
└── archive/
|
||||
└── racknerd-crontab-sample-20250526_101558.backup
|
||||
```
|
||||
|
||||
### File Naming Convention
|
||||
|
||||
- Format: `{hostname}-crontab-{type}-{timestamp}.backup`
|
||||
- Examples:
|
||||
- `europa-crontab-manual-20250526_101354.backup`
|
||||
- `io-crontab-pre-upgrade-20250526_120000.backup`
|
||||
- `racknerd-crontab-auto-20250526_000001.backup`
|
||||
|
||||
## Features
|
||||
|
||||
### 🔄 Multi-System Support
|
||||
|
||||
- **Hostname-based organization**: Each system gets its own directory
|
||||
- **Cross-system operations**: View, compare, and restore backups from any system
|
||||
- **Centralized management**: Manage all systems from a single location
|
||||
|
||||
### 📊 System Status Monitoring
|
||||
|
||||
```bash
|
||||
# View status for current system
|
||||
./crontab-backup-system.sh status
|
||||
|
||||
# View status for specific system
|
||||
./crontab-backup-system.sh status io
|
||||
|
||||
# View status for all systems
|
||||
./crontab-backup-system.sh status all
|
||||
```
|
||||
|
||||
### 🗂️ Backup Operations
|
||||
|
||||
```bash
|
||||
# Create backup on current system
|
||||
./crontab-backup-system.sh backup pre-upgrade
|
||||
|
||||
# List backups for specific system
|
||||
./crontab-backup-system.sh list io
|
||||
|
||||
# List all systems with backups
|
||||
./crontab-backup-system.sh list-systems
|
||||
|
||||
# Import backup from external source
|
||||
./crontab-backup-system.sh import /path/to/backup.txt io manual
|
||||
```
|
||||
|
||||
### 🔍 Cross-System Comparison
|
||||
|
||||
```bash
|
||||
# Compare current crontab with backup from another system
|
||||
./crontab-backup-system.sh compare current io-crontab-sample-20250526_101558.backup
|
||||
|
||||
# Compare two backups from different systems
|
||||
./crontab-backup-system.sh compare europa-crontab-manual-20250526_101354.backup racknerd-crontab-sample-20250526_101558.backup
|
||||
```
|
||||
|
||||
### 🧹 Cleanup Management
|
||||
|
||||
```bash
|
||||
# Clean up backups older than 30 days for current system
|
||||
./crontab-backup-system.sh cleanup 30
|
||||
|
||||
# Clean up backups for specific system
|
||||
./crontab-backup-system.sh cleanup 7 io
|
||||
|
||||
# Clean up backups for all systems
|
||||
./crontab-backup-system.sh cleanup 30 all
|
||||
```
|
||||
|
||||
## Enhanced Logging Integration
|
||||
|
||||
All backup operations now integrate with system logging:
|
||||
|
||||
### Syslog Integration
|
||||
|
||||
- **Tag-based logging**: Each operation uses specific syslog tags
|
||||
- **Priority levels**: Different priorities for info, warnings, and errors
|
||||
- **Performance monitoring**: Execution time tracking for all operations
|
||||
|
||||
### Example Enhanced Crontab Entries
|
||||
|
||||
```bash
|
||||
# Plex backup with comprehensive logging
|
||||
15 4 * * * /home/acedanger/shell/backup-plex.sh 2>&1 | logger -t plex-backup -p user.info
|
||||
|
||||
# Backup move operation with error handling
|
||||
0 1 * * * /home/acedanger/shell/move-backups.sh 2>&1 | logger -t backup-move -p user.info
|
||||
|
||||
# Validation with performance tracking
|
||||
0 7 * * * /home/acedanger/shell/validate-plex-backups.sh --fix 2>&1 | logger -t plex-validation -p user.info
|
||||
```
|
||||
|
||||
### Log Monitoring
|
||||
|
||||
```bash
|
||||
# View all backup-related logs
|
||||
journalctl -t plex-backup -t backup-move -t plex-validation -f
|
||||
|
||||
# View logs for specific operation
|
||||
journalctl -t plex-backup --since "1 hour ago"
|
||||
|
||||
# Monitor backup performance
|
||||
./backup-log-monitor.sh --real-time
|
||||
```
|
||||
|
||||
## Migration from Legacy Structure
|
||||
|
||||
The system automatically detects and migrates legacy backup structures:
|
||||
|
||||
### Automatic Migration
|
||||
|
||||
- **Legacy detection**: Automatically detects old `crontab-backups/archive/` structure
|
||||
- **Hostname prefix**: Adds hostname prefix to existing backup files
|
||||
- **Backward compatibility**: Preserves all existing backup data
|
||||
- **Safe migration**: Original files remain untouched until manual cleanup
|
||||
|
||||
### Manual Migration
|
||||
|
||||
```bash
|
||||
# Force migration of legacy backups
|
||||
./crontab-backup-system.sh migrate
|
||||
```
|
||||
|
||||
## Production Deployment
|
||||
|
||||
### System Setup
|
||||
|
||||
1. **Deploy script**: Copy `crontab-backup-system.sh` to each system
|
||||
2. **Configure permissions**: Ensure proper read/write access to backup directories
|
||||
3. **Setup automation**: Configure automated daily backups
|
||||
|
||||
### Automated Backup Setup
|
||||
|
||||
```bash
|
||||
# Setup automated daily backups on each system
|
||||
./crontab-backup-system.sh setup-auto
|
||||
```
|
||||
|
||||
This adds the following entry to the system crontab:
|
||||
|
||||
```bash
|
||||
0 0 * * * /path/to/crontab-backup-system.sh backup auto --auto-cleanup 2>&1 | logger -t crontab-backup -p user.info
|
||||
```
|
||||
|
||||
### Cross-System Synchronization
|
||||
|
||||
For distributed environments, consider setting up backup synchronization:
|
||||
|
||||
```bash
|
||||
# Example rsync command to sync backups from remote systems
|
||||
rsync -avz europa:/home/acedanger/shell/crontab-backups/europa/ /home/acedanger/shell/crontab-backups/europa/
|
||||
rsync -avz io:/home/acedanger/shell/crontab-backups/io/ /home/acedanger/shell/crontab-backups/io/
|
||||
```
|
||||
|
||||
## Security Considerations
|
||||
|
||||
### File Permissions
|
||||
|
||||
- **Backup directories**: Restrict access to authorized users only
|
||||
- **Log files**: Ensure proper log rotation and access controls
|
||||
- **Remote access**: Use secure methods (SSH, rsync) for cross-system operations
|
||||
|
||||
### Backup Integrity
|
||||
|
||||
- **Validation**: Regular syntax validation of backup files
|
||||
- **Checksums**: Consider adding checksum verification for critical backups
|
||||
- **Retention**: Implement appropriate backup retention policies
|
||||
|
||||
## Advanced Use Cases
|
||||
|
||||
### Disaster Recovery
|
||||
|
||||
```bash
|
||||
# Restore from specific system backup during emergency
|
||||
./crontab-backup-system.sh restore io-crontab-pre-incident-20250526_101354.backup
|
||||
|
||||
# Compare pre-incident and post-incident configurations
|
||||
./crontab-backup-system.sh compare io-crontab-pre-incident-20250526_101354.backup current
|
||||
```
|
||||
|
||||
### Configuration Management
|
||||
|
||||
```bash
|
||||
# Standardize crontab across multiple systems
|
||||
./crontab-backup-system.sh compare europa-crontab-standard-20250526_101354.backup io-crontab-current-20250526_120000.backup
|
||||
|
||||
# Validate configurations before deployment
|
||||
./crontab-backup-system.sh validate new-crontab-config.txt
|
||||
```
|
||||
|
||||
### Compliance and Auditing
|
||||
|
||||
- **Change tracking**: Complete history of all crontab changes across systems
|
||||
- **Audit trails**: System logs provide comprehensive audit information
|
||||
- **Compliance reporting**: Generate reports showing backup frequency and success rates
|
||||
|
||||
## Monitoring and Alerting
|
||||
|
||||
### Health Checks
|
||||
|
||||
```bash
|
||||
# Check backup system health
|
||||
./crontab-backup-system.sh status all
|
||||
|
||||
# Monitor recent backup activity
|
||||
./backup-log-monitor.sh --health-check
|
||||
```
|
||||
|
||||
### Alert Integration
|
||||
|
||||
Consider integrating with monitoring systems:
|
||||
|
||||
- **Backup failures**: Alert when backups fail or are missing
|
||||
- **Old backups**: Alert when systems haven't been backed up recently
|
||||
- **Disk space**: Monitor backup directory disk usage
|
||||
|
||||
## Best Practices
|
||||
|
||||
1. **Regular Testing**: Periodically test backup restoration procedures
|
||||
2. **Documentation**: Keep records of system configurations and backup schedules
|
||||
3. **Automation**: Use automated cleanup to prevent disk space issues
|
||||
4. **Monitoring**: Implement comprehensive monitoring and alerting
|
||||
5. **Security**: Regularly review and update access controls
|
||||
|
||||
## Support and Troubleshooting
|
||||
|
||||
### Common Issues
|
||||
|
||||
- **Permission errors**: Ensure proper file permissions on backup directories
|
||||
- **Missing backups**: Check automated backup cron entries
|
||||
- **Syntax errors**: Use validation feature before deploying new crontabs
|
||||
|
||||
### Debug Mode
|
||||
|
||||
Enable verbose logging for troubleshooting:
|
||||
|
||||
```bash
|
||||
# Add debug logging to any command
|
||||
./crontab-backup-system.sh status all 2>&1 | tee debug.log
|
||||
```
|
||||
|
||||
### Log Analysis
|
||||
|
||||
```bash
|
||||
# Analyze backup patterns
|
||||
grep "SUCCESS" logs/crontab-management.log | tail -20
|
||||
|
||||
# Check for errors
|
||||
grep "ERROR" logs/crontab-management.log | tail -10
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
*This multi-system crontab management solution provides robust, scalable backup management for distributed environments while maintaining simplicity and reliability.*
|
||||
28
enhanced-crontab.txt
Normal file
28
enhanced-crontab.txt
Normal file
@@ -0,0 +1,28 @@
|
||||
# Enhanced Crontab Entries with System Logging
|
||||
#
|
||||
# These entries include comprehensive logging to syslog with proper tags
|
||||
# and error handling for better monitoring and troubleshooting
|
||||
|
||||
# Move the files previously backed up at 0100
|
||||
# Logs both stdout and stderr with backup-move tag
|
||||
0 1 * * * /home/acedanger/shell/move-backups.sh 2>&1 | logger -t backup-move -p user.info
|
||||
|
||||
# Daily Plex backup at 0415 with enhanced logging
|
||||
# Includes execution status and performance metrics
|
||||
15 4 * * * { echo "Starting Plex backup"; /home/acedanger/shell/backup-plex.sh --non-interactive --auto-repair; echo "Plex backup completed with exit code: $?"; } 2>&1 | logger -t plex-backup -p user.info
|
||||
|
||||
# Daily validation at 0700 with detailed logging
|
||||
# Logs validation results and any auto-fixes performed
|
||||
0 7 * * * { echo "Starting Plex backup validation"; /home/acedanger/shell/validate-plex-backups.sh --fix; echo "Validation completed with exit code: $?"; } 2>&1 | logger -t plex-validation -p user.info
|
||||
|
||||
# Backup Immich database weekly (Mondays at 0500)
|
||||
# Enhanced with proper logging and error handling
|
||||
0 5 * * 1 { echo "Starting Immich database backup move"; if mv /mnt/share/media/immich/uploads/backups/immich-db-backup* /mnt/share/media/backups/immich 2>/dev/null; then echo "Immich backup move completed successfully"; else echo "No Immich backup files found or move failed"; fi; } 2>&1 | logger -t immich-backup -p user.info
|
||||
|
||||
# Generate detailed weekly report (Sundays at 0800)
|
||||
# Comprehensive reporting with system logging
|
||||
0 8 * * 0 { echo "Starting weekly Plex backup report generation"; /home/acedanger/shell/validate-plex-backups.sh --report; echo "Weekly report generation completed with exit code: $?"; } 2>&1 | logger -t plex-report -p user.info
|
||||
|
||||
# Optional: Add a health check entry to monitor cron jobs (every 6 hours)
|
||||
# This can help detect if any of the backup processes are failing
|
||||
# 0 */6 * * * { echo "Cron health check - all backup jobs scheduled"; ps aux | grep -E "(backup-plex|validate-plex|move-backups)" | grep -v grep | wc -l; } 2>&1 | logger -t cron-health -p user.info
|
||||
246
manage-enhanced-crontab.sh
Executable file
246
manage-enhanced-crontab.sh
Executable file
@@ -0,0 +1,246 @@
|
||||
#!/bin/bash
|
||||
|
||||
# Enhanced Crontab Management Script
|
||||
# This script helps install and manage the enhanced crontab entries with system logging
|
||||
|
||||
set -e
|
||||
|
||||
# Color codes for output
|
||||
RED='\033[0;31m'
|
||||
GREEN='\033[0;32m'
|
||||
YELLOW='\033[1;33m'
|
||||
BLUE='\033[0;34m'
|
||||
CYAN='\033[0;36m'
|
||||
NC='\033[0m' # No Color
|
||||
|
||||
SCRIPT_DIR="$(dirname "$(readlink -f "$0")")"
|
||||
ENHANCED_CRONTAB_FILE="$SCRIPT_DIR/enhanced-crontab.txt"
|
||||
BACKUP_CRONTAB_FILE="/tmp/crontab-backup-$(date +%Y%m%d_%H%M%S)"
|
||||
|
||||
log_message() {
|
||||
echo -e "$(date '+%H:%M:%S') $1"
|
||||
}
|
||||
|
||||
log_error() {
|
||||
log_message "${RED}ERROR: $1${NC}"
|
||||
}
|
||||
|
||||
log_success() {
|
||||
log_message "${GREEN}SUCCESS: $1${NC}"
|
||||
}
|
||||
|
||||
log_warning() {
|
||||
log_message "${YELLOW}WARNING: $1${NC}"
|
||||
}
|
||||
|
||||
log_info() {
|
||||
log_message "${BLUE}INFO: $1${NC}"
|
||||
}
|
||||
|
||||
backup_current_crontab() {
|
||||
log_info "Backing up current root crontab to $BACKUP_CRONTAB_FILE"
|
||||
if sudo crontab -l > "$BACKUP_CRONTAB_FILE" 2>/dev/null; then
|
||||
log_success "Current crontab backed up successfully"
|
||||
else
|
||||
log_warning "No existing crontab found or backup failed"
|
||||
fi
|
||||
}
|
||||
|
||||
install_enhanced_crontab() {
|
||||
log_info "Installing enhanced crontab entries"
|
||||
|
||||
if [ ! -f "$ENHANCED_CRONTAB_FILE" ]; then
|
||||
log_error "Enhanced crontab file not found: $ENHANCED_CRONTAB_FILE"
|
||||
return 1
|
||||
fi
|
||||
|
||||
# Create a backup before making changes
|
||||
if [ -f "$SCRIPT_DIR/crontab-backup-system.sh" ]; then
|
||||
log_info "Creating pre-install backup"
|
||||
"$SCRIPT_DIR/crontab-backup-system.sh" backup pre-install
|
||||
fi
|
||||
|
||||
# Extract just the cron entries (skip comments and empty lines)
|
||||
grep -E '^[0-9]' "$ENHANCED_CRONTAB_FILE" > /tmp/cron_entries_only.txt
|
||||
|
||||
# Validate the crontab syntax before installing
|
||||
log_info "Validating crontab syntax"
|
||||
if [ -f "$SCRIPT_DIR/crontab-backup-system.sh" ]; then
|
||||
if ! "$SCRIPT_DIR/crontab-backup-system.sh" validate /tmp/cron_entries_only.txt; then
|
||||
log_error "Crontab syntax validation failed"
|
||||
rm -f /tmp/cron_entries_only.txt
|
||||
return 1
|
||||
fi
|
||||
fi
|
||||
|
||||
if sudo crontab /tmp/cron_entries_only.txt; then
|
||||
log_success "Enhanced crontab entries installed successfully"
|
||||
rm -f /tmp/cron_entries_only.txt
|
||||
|
||||
# Create a post-install backup
|
||||
if [ -f "$SCRIPT_DIR/crontab-backup-system.sh" ]; then
|
||||
"$SCRIPT_DIR/crontab-backup-system.sh" backup post-install
|
||||
fi
|
||||
else
|
||||
log_error "Failed to install enhanced crontab entries"
|
||||
rm -f /tmp/cron_entries_only.txt
|
||||
return 1
|
||||
fi
|
||||
}
|
||||
|
||||
show_current_crontab() {
|
||||
log_info "Current root crontab entries:"
|
||||
echo
|
||||
sudo crontab -l 2>/dev/null || log_warning "No crontab entries found"
|
||||
echo
|
||||
}
|
||||
|
||||
show_log_monitoring_commands() {
|
||||
log_info "Commands to monitor backup logs:"
|
||||
echo
|
||||
echo -e "${CYAN}# Use the enhanced backup log monitor:${NC}"
|
||||
echo "./backup-log-monitor.sh monitor # Real-time monitoring"
|
||||
echo "./backup-log-monitor.sh recent 24 # Last 24 hours"
|
||||
echo "./backup-log-monitor.sh health # System health check"
|
||||
echo "./backup-log-monitor.sh report 7 # Weekly report"
|
||||
echo
|
||||
echo -e "${CYAN}# Direct journalctl commands:${NC}"
|
||||
echo "sudo journalctl -f -t plex-backup -t backup-move -t plex-validation -t immich-backup -t plex-report"
|
||||
echo
|
||||
echo -e "${CYAN}# View logs from the last 24 hours:${NC}"
|
||||
echo "sudo journalctl --since '24 hours ago' -t plex-backup -t backup-move -t plex-validation -t immich-backup -t plex-report"
|
||||
echo
|
||||
echo -e "${CYAN}# View only error logs:${NC}"
|
||||
echo "sudo journalctl --priority=err -t plex-backup -t backup-move -t plex-validation -t immich-backup -t plex-report"
|
||||
echo
|
||||
echo -e "${CYAN}# View logs for a specific backup type (e.g., plex-backup):${NC}"
|
||||
echo "sudo journalctl -t plex-backup --since '1 week ago'"
|
||||
echo
|
||||
}
|
||||
|
||||
setup_logrotate() {
|
||||
log_info "Setting up logrotate for backup logs"
|
||||
|
||||
cat > /tmp/backup-logs-logrotate << 'EOF'
|
||||
# Logrotate configuration for backup logs
|
||||
# This ensures syslog doesn't grow too large with backup logs
|
||||
|
||||
/var/log/syslog {
|
||||
daily
|
||||
missingok
|
||||
rotate 7
|
||||
compress
|
||||
delaycompress
|
||||
notifempty
|
||||
postrotate
|
||||
/usr/lib/rsyslog/rsyslog-rotate
|
||||
endscript
|
||||
}
|
||||
EOF
|
||||
|
||||
if sudo cp /tmp/backup-logs-logrotate /etc/logrotate.d/backup-logs; then
|
||||
log_success "Logrotate configuration installed"
|
||||
else
|
||||
log_warning "Failed to install logrotate configuration"
|
||||
fi
|
||||
|
||||
rm -f /tmp/backup-logs-logrotate
|
||||
}
|
||||
|
||||
verify_scripts_exist() {
|
||||
log_info "Verifying all backup scripts exist and are executable"
|
||||
|
||||
local scripts=(
|
||||
"/home/acedanger/shell/move-backups.sh"
|
||||
"/home/acedanger/shell/backup-plex.sh"
|
||||
"/home/acedanger/shell/validate-plex-backups.sh"
|
||||
)
|
||||
|
||||
local all_good=true
|
||||
|
||||
for script in "${scripts[@]}"; do
|
||||
if [ -f "$script" ]; then
|
||||
if [ -x "$script" ]; then
|
||||
log_success "✓ $script exists and is executable"
|
||||
else
|
||||
log_warning "! $script exists but is not executable"
|
||||
sudo chmod +x "$script"
|
||||
log_success "✓ Made $script executable"
|
||||
fi
|
||||
else
|
||||
log_error "✗ $script not found"
|
||||
all_good=false
|
||||
fi
|
||||
done
|
||||
|
||||
if $all_good; then
|
||||
log_success "All backup scripts are ready"
|
||||
else
|
||||
log_error "Some backup scripts are missing"
|
||||
return 1
|
||||
fi
|
||||
}
|
||||
|
||||
show_usage() {
|
||||
echo "Enhanced Crontab Management Script"
|
||||
echo
|
||||
echo "Usage: $0 [OPTION]"
|
||||
echo
|
||||
echo "Options:"
|
||||
echo " install Install the enhanced crontab entries with backup system"
|
||||
echo " show Show current crontab entries"
|
||||
echo " backup Backup current crontab only"
|
||||
echo " verify Verify all scripts exist and are executable"
|
||||
echo " monitor Show log monitoring commands"
|
||||
echo " logrotate Setup logrotate for backup logs"
|
||||
echo " status Show backup system health status"
|
||||
echo " help Show this help message"
|
||||
echo
|
||||
echo "Additional Tools:"
|
||||
echo " ./crontab-backup-system.sh Comprehensive crontab backup management"
|
||||
echo " ./backup-log-monitor.sh Advanced backup log monitoring"
|
||||
echo
|
||||
}
|
||||
|
||||
case "${1:-help}" in
|
||||
install)
|
||||
verify_scripts_exist
|
||||
backup_current_crontab
|
||||
install_enhanced_crontab
|
||||
show_current_crontab
|
||||
setup_logrotate
|
||||
show_log_monitoring_commands
|
||||
|
||||
# Setup automated backup system
|
||||
if [ -f "$SCRIPT_DIR/crontab-backup-system.sh" ]; then
|
||||
log_info "Setting up automated crontab backup system"
|
||||
"$SCRIPT_DIR/crontab-backup-system.sh" setup-auto
|
||||
fi
|
||||
;;
|
||||
show)
|
||||
show_current_crontab
|
||||
;;
|
||||
backup)
|
||||
backup_current_crontab
|
||||
;;
|
||||
verify)
|
||||
verify_scripts_exist
|
||||
;;
|
||||
monitor)
|
||||
show_log_monitoring_commands
|
||||
;;
|
||||
logrotate)
|
||||
setup_logrotate
|
||||
;;
|
||||
status)
|
||||
if [ -f "$SCRIPT_DIR/backup-log-monitor.sh" ]; then
|
||||
"$SCRIPT_DIR/backup-log-monitor.sh" health
|
||||
else
|
||||
log_warning "Backup log monitor not found, showing basic status"
|
||||
show_current_crontab
|
||||
fi
|
||||
;;
|
||||
help|*)
|
||||
show_usage
|
||||
;;
|
||||
esac
|
||||
Reference in New Issue
Block a user