mirror of
https://github.com/acedanger/shell.git
synced 2025-12-06 01:10:12 -08:00
Implement enhanced crontab management system with automated backups and logging
- Added enhanced crontab entries with comprehensive logging to syslog. - Created a management script (`manage-enhanced-crontab.sh`) for installing and verifying crontab entries. - Introduced backup scripts for crontab management, including automated timestamped backups and cleanup. - Developed documentation for enhanced crontab and multi-system management. - Established a directory structure for managing crontab backups across multiple systems. - Implemented error handling and logging for backup operations. - Added health monitoring and reporting features for backup processes.
This commit is contained in:
Peter Wood
387
backup-log-monitor.sh
Executable file
387
backup-log-monitor.sh
Executable file
@@ -0,0 +1,387 @@
|
|||||||
|
#!/bin/bash
|
||||||
|
|
||||||
|
# Enhanced Backup Log Monitor
|
||||||
|
# Provides real-time monitoring and analysis of backup operations
|
||||||
|
|
||||||
|
set -e
|
||||||
|
|
||||||
|
# Color codes for output
|
||||||
|
RED='\033[0;31m'
|
||||||
|
GREEN='\033[0;32m'
|
||||||
|
YELLOW='\033[1;33m'
|
||||||
|
BLUE='\033[0;34m'
|
||||||
|
CYAN='\033[0;36m'
|
||||||
|
PURPLE='\033[0;35m'
|
||||||
|
NC='\033[0m' # No Color
|
||||||
|
|
||||||
|
SCRIPT_DIR="$(dirname "$(readlink -f "$0")")"
|
||||||
|
LOG_DIR="$SCRIPT_DIR/logs"
|
||||||
|
REPORT_DIR="$LOG_DIR/reports"
|
||||||
|
HOSTNAME=$(hostname)
|
||||||
|
|
||||||
|
# Ensure directories exist
|
||||||
|
mkdir -p "$LOG_DIR" "$REPORT_DIR"
|
||||||
|
|
||||||
|
# Backup service tags for monitoring
|
||||||
|
BACKUP_TAGS=("plex-backup" "backup-move" "plex-validation" "immich-backup" "plex-report" "crontab-backup")
|
||||||
|
|
||||||
|
log_message() {
|
||||||
|
echo -e "$(date '+%H:%M:%S') $1"
|
||||||
|
}
|
||||||
|
|
||||||
|
log_error() {
|
||||||
|
log_message "${RED}ERROR: $1${NC}"
|
||||||
|
}
|
||||||
|
|
||||||
|
log_success() {
|
||||||
|
log_message "${GREEN}SUCCESS: $1${NC}"
|
||||||
|
}
|
||||||
|
|
||||||
|
log_warning() {
|
||||||
|
log_message "${YELLOW}WARNING: $1${NC}"
|
||||||
|
}
|
||||||
|
|
||||||
|
log_info() {
|
||||||
|
log_message "${BLUE}INFO: $1${NC}"
|
||||||
|
}
|
||||||
|
|
||||||
|
monitor_realtime() {
|
||||||
|
local tags_filter=""
|
||||||
|
for tag in "${BACKUP_TAGS[@]}"; do
|
||||||
|
tags_filter="$tags_filter -t $tag"
|
||||||
|
done
|
||||||
|
|
||||||
|
log_info "Starting real-time monitoring of backup logs"
|
||||||
|
log_info "Press Ctrl+C to stop monitoring"
|
||||||
|
echo
|
||||||
|
|
||||||
|
sudo journalctl -f $tags_filter --no-hostname --output=short-iso | while read -r line; do
|
||||||
|
# Color code different log levels and services
|
||||||
|
if [[ "$line" =~ ERROR ]]; then
|
||||||
|
echo -e "${RED}$line${NC}"
|
||||||
|
elif [[ "$line" =~ SUCCESS ]]; then
|
||||||
|
echo -e "${GREEN}$line${NC}"
|
||||||
|
elif [[ "$line" =~ WARNING ]]; then
|
||||||
|
echo -e "${YELLOW}$line${NC}"
|
||||||
|
elif [[ "$line" =~ plex-backup ]]; then
|
||||||
|
echo -e "${BLUE}$line${NC}"
|
||||||
|
elif [[ "$line" =~ backup-move ]]; then
|
||||||
|
echo -e "${CYAN}$line${NC}"
|
||||||
|
elif [[ "$line" =~ plex-validation ]]; then
|
||||||
|
echo -e "${PURPLE}$line${NC}"
|
||||||
|
else
|
||||||
|
echo "$line"
|
||||||
|
fi
|
||||||
|
done
|
||||||
|
}
|
||||||
|
|
||||||
|
show_recent_logs() {
|
||||||
|
local hours="${1:-24}"
|
||||||
|
local service="${2:-all}"
|
||||||
|
|
||||||
|
log_info "Showing logs from the last $hours hours"
|
||||||
|
|
||||||
|
local tags_filter=""
|
||||||
|
if [ "$service" = "all" ]; then
|
||||||
|
for tag in "${BACKUP_TAGS[@]}"; do
|
||||||
|
tags_filter="$tags_filter -t $tag"
|
||||||
|
done
|
||||||
|
else
|
||||||
|
tags_filter="-t $service"
|
||||||
|
fi
|
||||||
|
|
||||||
|
echo
|
||||||
|
sudo journalctl --since "${hours} hours ago" $tags_filter --no-hostname --output=short-iso | \
|
||||||
|
while read -r line; do
|
||||||
|
# Color code the output
|
||||||
|
if [[ "$line" =~ ERROR ]]; then
|
||||||
|
echo -e "${RED}$line${NC}"
|
||||||
|
elif [[ "$line" =~ SUCCESS ]]; then
|
||||||
|
echo -e "${GREEN}$line${NC}"
|
||||||
|
elif [[ "$line" =~ WARNING ]]; then
|
||||||
|
echo -e "${YELLOW}$line${NC}"
|
||||||
|
else
|
||||||
|
echo "$line"
|
||||||
|
fi
|
||||||
|
done
|
||||||
|
}
|
||||||
|
|
||||||
|
show_error_summary() {
|
||||||
|
local days="${1:-7}"
|
||||||
|
|
||||||
|
log_info "Error summary for the last $days days"
|
||||||
|
echo
|
||||||
|
|
||||||
|
local error_file="/tmp/backup_errors_$$.tmp"
|
||||||
|
|
||||||
|
for tag in "${BACKUP_TAGS[@]}"; do
|
||||||
|
local error_count=$(sudo journalctl --since "${days} days ago" -t "$tag" --grep="ERROR" --output=cat | wc -l)
|
||||||
|
if [ "$error_count" -gt 0 ]; then
|
||||||
|
echo -e "${RED}$tag: $error_count errors${NC}"
|
||||||
|
sudo journalctl --since "${days} days ago" -t "$tag" --grep="ERROR" --output=short-iso | head -5
|
||||||
|
echo
|
||||||
|
else
|
||||||
|
echo -e "${GREEN}$tag: No errors${NC}"
|
||||||
|
fi
|
||||||
|
done
|
||||||
|
}
|
||||||
|
|
||||||
|
generate_backup_report() {
|
||||||
|
local days="${1:-7}"
|
||||||
|
local report_file="$REPORT_DIR/backup-report-$(date +%Y%m%d_%H%M%S).txt"
|
||||||
|
|
||||||
|
log_info "Generating comprehensive backup report for the last $days days"
|
||||||
|
log_info "Report will be saved to: $report_file"
|
||||||
|
|
||||||
|
{
|
||||||
|
echo "=== BACKUP SYSTEM REPORT ==="
|
||||||
|
echo "Generated: $(date)"
|
||||||
|
echo "Period: Last $days days"
|
||||||
|
echo "System: $(uname -n)"
|
||||||
|
echo
|
||||||
|
|
||||||
|
for tag in "${BACKUP_TAGS[@]}"; do
|
||||||
|
echo "=== $tag ==="
|
||||||
|
|
||||||
|
# Count entries
|
||||||
|
local total_entries=$(sudo journalctl --since "${days} days ago" -t "$tag" --output=cat | wc -l)
|
||||||
|
local error_count=$(sudo journalctl --since "${days} days ago" -t "$tag" --grep="ERROR" --output=cat | wc -l)
|
||||||
|
local success_count=$(sudo journalctl --since "${days} days ago" -t "$tag" --grep="SUCCESS" --output=cat | wc -l)
|
||||||
|
|
||||||
|
echo "Total log entries: $total_entries"
|
||||||
|
echo "Errors: $error_count"
|
||||||
|
echo "Successes: $success_count"
|
||||||
|
|
||||||
|
if [ "$error_count" -gt 0 ]; then
|
||||||
|
echo
|
||||||
|
echo "Recent errors:"
|
||||||
|
sudo journalctl --since "${days} days ago" -t "$tag" --grep="ERROR" --output=short-iso | head -10
|
||||||
|
fi
|
||||||
|
|
||||||
|
echo
|
||||||
|
echo "Recent activity:"
|
||||||
|
sudo journalctl --since "${days} days ago" -t "$tag" --output=short-iso | tail -5
|
||||||
|
echo
|
||||||
|
echo "----------------------------------------"
|
||||||
|
echo
|
||||||
|
done
|
||||||
|
|
||||||
|
# System resource usage during backups
|
||||||
|
echo "=== SYSTEM ANALYSIS ==="
|
||||||
|
echo "Disk usage in backup directories:"
|
||||||
|
if [ -d "/mnt/share/media/backups" ]; then
|
||||||
|
du -sh /mnt/share/media/backups/* 2>/dev/null || echo "No backup directories found"
|
||||||
|
fi
|
||||||
|
echo
|
||||||
|
|
||||||
|
# Cron job status
|
||||||
|
echo "Active cron jobs related to backups:"
|
||||||
|
sudo crontab -l 2>/dev/null | grep -E "(backup|plex|immich)" || echo "No backup-related cron jobs found"
|
||||||
|
echo
|
||||||
|
|
||||||
|
} > "$report_file"
|
||||||
|
|
||||||
|
log_success "Report generated: $report_file"
|
||||||
|
|
||||||
|
# Show summary
|
||||||
|
echo
|
||||||
|
log_info "Report Summary:"
|
||||||
|
grep -E "(Total log entries|Errors|Successes):" "$report_file" | while read -r line; do
|
||||||
|
if [[ "$line" =~ Errors:.*[1-9] ]]; then
|
||||||
|
echo -e "${RED}$line${NC}"
|
||||||
|
else
|
||||||
|
echo -e "${GREEN}$line${NC}"
|
||||||
|
fi
|
||||||
|
done
|
||||||
|
}
|
||||||
|
|
||||||
|
check_backup_health() {
|
||||||
|
log_info "Checking backup system health for $HOSTNAME"
|
||||||
|
echo
|
||||||
|
|
||||||
|
local health_score=100
|
||||||
|
local issues=()
|
||||||
|
|
||||||
|
# Check if backup scripts exist
|
||||||
|
local backup_scripts=(
|
||||||
|
"/home/acedanger/shell/backup-plex.sh"
|
||||||
|
"/home/acedanger/shell/move-backups.sh"
|
||||||
|
"/home/acedanger/shell/validate-plex-backups.sh"
|
||||||
|
"/home/acedanger/shell/crontab-backup-system.sh"
|
||||||
|
)
|
||||||
|
|
||||||
|
for script in "${backup_scripts[@]}"; do
|
||||||
|
if [ ! -f "$script" ]; then
|
||||||
|
issues+=("Missing script: $script")
|
||||||
|
((health_score -= 20))
|
||||||
|
elif [ ! -x "$script" ]; then
|
||||||
|
issues+=("Script not executable: $script")
|
||||||
|
((health_score -= 10))
|
||||||
|
fi
|
||||||
|
done
|
||||||
|
|
||||||
|
# Check if backup directories exist
|
||||||
|
local backup_dirs=(
|
||||||
|
"/mnt/share/media/backups/plex"
|
||||||
|
"/mnt/share/media/backups/docker-data"
|
||||||
|
"/mnt/share/media/backups/immich"
|
||||||
|
)
|
||||||
|
|
||||||
|
for dir in "${backup_dirs[@]}"; do
|
||||||
|
if [ ! -d "$dir" ]; then
|
||||||
|
issues+=("Missing backup directory: $dir")
|
||||||
|
((health_score -= 15))
|
||||||
|
fi
|
||||||
|
done
|
||||||
|
|
||||||
|
# Check crontab backup system structure
|
||||||
|
local crontab_backup_dir="$SCRIPT_DIR/crontab-backups/$HOSTNAME"
|
||||||
|
if [ ! -d "$crontab_backup_dir" ]; then
|
||||||
|
issues+=("Missing crontab backup directory for $HOSTNAME: $crontab_backup_dir")
|
||||||
|
((health_score -= 10))
|
||||||
|
fi
|
||||||
|
|
||||||
|
# Check recent backup activity
|
||||||
|
local recent_activity=false
|
||||||
|
for tag in "${BACKUP_TAGS[@]}"; do
|
||||||
|
if sudo journalctl --since "24 hours ago" -t "$tag" --output=cat | grep -q .; then
|
||||||
|
recent_activity=true
|
||||||
|
break
|
||||||
|
fi
|
||||||
|
done
|
||||||
|
|
||||||
|
if [ "$recent_activity" = false ]; then
|
||||||
|
issues+=("No backup activity in the last 24 hours")
|
||||||
|
((health_score -= 25))
|
||||||
|
fi
|
||||||
|
|
||||||
|
# Check for recent errors
|
||||||
|
local recent_errors=0
|
||||||
|
for tag in "${BACKUP_TAGS[@]}"; do
|
||||||
|
local error_count=$(sudo journalctl --since "24 hours ago" -t "$tag" --grep="ERROR" --output=cat | wc -l)
|
||||||
|
((recent_errors += error_count))
|
||||||
|
done
|
||||||
|
|
||||||
|
if [ "$recent_errors" -gt 0 ]; then
|
||||||
|
issues+=("$recent_errors errors in the last 24 hours")
|
||||||
|
((health_score -= $((recent_errors * 5))))
|
||||||
|
fi
|
||||||
|
|
||||||
|
# Ensure health score doesn't go below 0
|
||||||
|
if [ "$health_score" -lt 0 ]; then
|
||||||
|
health_score=0
|
||||||
|
fi
|
||||||
|
|
||||||
|
# Display results
|
||||||
|
if [ "$health_score" -ge 90 ]; then
|
||||||
|
echo -e "${GREEN}Backup System Health ($HOSTNAME): ${health_score}% - EXCELLENT${NC}"
|
||||||
|
elif [ "$health_score" -ge 70 ]; then
|
||||||
|
echo -e "${YELLOW}Backup System Health ($HOSTNAME): ${health_score}% - GOOD${NC}"
|
||||||
|
elif [ "$health_score" -ge 50 ]; then
|
||||||
|
echo -e "${YELLOW}Backup System Health ($HOSTNAME): ${health_score}% - FAIR${NC}"
|
||||||
|
else
|
||||||
|
echo -e "${RED}Backup System Health ($HOSTNAME): ${health_score}% - POOR${NC}"
|
||||||
|
fi
|
||||||
|
|
||||||
|
if [ ${#issues[@]} -gt 0 ]; then
|
||||||
|
echo
|
||||||
|
log_warning "Issues found:"
|
||||||
|
for issue in "${issues[@]}"; do
|
||||||
|
echo -e " ${RED}• $issue${NC}"
|
||||||
|
done
|
||||||
|
|
||||||
|
echo
|
||||||
|
log_info "Recommended actions:"
|
||||||
|
echo " • Run: ./manage-enhanced-crontab.sh verify"
|
||||||
|
echo " • Check system logs: sudo journalctl -xe"
|
||||||
|
echo " • Verify backup directories are mounted and accessible"
|
||||||
|
echo " • Run: ./crontab-backup-system.sh status"
|
||||||
|
fi
|
||||||
|
}
|
||||||
|
|
||||||
|
show_service_status() {
|
||||||
|
log_info "Backup Service Status Overview"
|
||||||
|
echo
|
||||||
|
|
||||||
|
printf "%-20s %-15s %-20s %-30s\n" "Service" "Status" "Last Activity" "Last Message"
|
||||||
|
printf "%-20s %-15s %-20s %-30s\n" "-------" "------" "-------------" "------------"
|
||||||
|
|
||||||
|
for tag in "${BACKUP_TAGS[@]}"; do
|
||||||
|
local last_entry=$(sudo journalctl -t "$tag" --output=short-iso -n 1 2>/dev/null | tail -1)
|
||||||
|
|
||||||
|
if [ -n "$last_entry" ]; then
|
||||||
|
local timestamp=$(echo "$last_entry" | cut -d' ' -f1-2)
|
||||||
|
local message=$(echo "$last_entry" | cut -d' ' -f4- | cut -c1-30)
|
||||||
|
|
||||||
|
# Check if it's recent (within 48 hours)
|
||||||
|
local entry_time=$(date -d "$timestamp" +%s 2>/dev/null || echo "0")
|
||||||
|
local current_time=$(date +%s)
|
||||||
|
local hours_diff=$(( (current_time - entry_time) / 3600 ))
|
||||||
|
|
||||||
|
local status
|
||||||
|
if [ "$hours_diff" -le 24 ]; then
|
||||||
|
status="${GREEN}Active${NC}"
|
||||||
|
elif [ "$hours_diff" -le 48 ]; then
|
||||||
|
status="${YELLOW}Recent${NC}"
|
||||||
|
else
|
||||||
|
status="${RED}Stale${NC}"
|
||||||
|
fi
|
||||||
|
|
||||||
|
printf "%-20s %-25s %-20s %-30s\n" "$tag" "$status" "$timestamp" "$message"
|
||||||
|
else
|
||||||
|
printf "%-20s %-25s %-20s %-30s\n" "$tag" "${RED}No logs${NC}" "Never" "No activity found"
|
||||||
|
fi
|
||||||
|
done
|
||||||
|
}
|
||||||
|
|
||||||
|
show_usage() {
|
||||||
|
echo -e "${PURPLE}Enhanced Backup Log Monitor${NC}"
|
||||||
|
echo
|
||||||
|
echo "Usage: $0 [COMMAND] [OPTIONS]"
|
||||||
|
echo
|
||||||
|
echo "Commands:"
|
||||||
|
echo " monitor Real-time monitoring of all backup logs"
|
||||||
|
echo " recent [HOURS] [SERVICE] Show recent logs (default: 24 hours, all services)"
|
||||||
|
echo " errors [DAYS] Show error summary (default: 7 days)"
|
||||||
|
echo " report [DAYS] Generate comprehensive report (default: 7 days)"
|
||||||
|
echo " health Check backup system health"
|
||||||
|
echo " status Show service status overview"
|
||||||
|
echo " help Show this help message"
|
||||||
|
echo
|
||||||
|
echo "Services:"
|
||||||
|
for tag in "${BACKUP_TAGS[@]}"; do
|
||||||
|
echo " - $tag"
|
||||||
|
done
|
||||||
|
echo
|
||||||
|
echo "Examples:"
|
||||||
|
echo " $0 monitor"
|
||||||
|
echo " $0 recent 12 plex-backup"
|
||||||
|
echo " $0 errors 3"
|
||||||
|
echo " $0 report 14"
|
||||||
|
echo
|
||||||
|
}
|
||||||
|
|
||||||
|
# Main command handling
|
||||||
|
case "${1:-help}" in
|
||||||
|
monitor)
|
||||||
|
monitor_realtime
|
||||||
|
;;
|
||||||
|
recent)
|
||||||
|
show_recent_logs "$2" "$3"
|
||||||
|
;;
|
||||||
|
errors)
|
||||||
|
show_error_summary "$2"
|
||||||
|
;;
|
||||||
|
report)
|
||||||
|
generate_backup_report "$2"
|
||||||
|
;;
|
||||||
|
health)
|
||||||
|
check_backup_health
|
||||||
|
;;
|
||||||
|
status)
|
||||||
|
show_service_status
|
||||||
|
;;
|
||||||
|
help|*)
|
||||||
|
show_usage
|
||||||
|
;;
|
||||||
|
esac
|
||||||
732
crontab-backup-system.sh
Executable file
732
crontab-backup-system.sh
Executable file
@@ -0,0 +1,732 @@
|
|||||||
|
#!/bin/bash
|
||||||
|
|
||||||
|
# Crontab Backup and Recovery System
|
||||||
|
# This script provides comprehensive backup management for crontab entries
|
||||||
|
|
||||||
|
set -e
|
||||||
|
|
||||||
|
# Color codes for output
|
||||||
|
RED='\033[0;31m'
|
||||||
|
GREEN='\033[0;32m'
|
||||||
|
YELLOW='\033[1;33m'
|
||||||
|
BLUE='\033[0;34m'
|
||||||
|
CYAN='\033[0;36m'
|
||||||
|
PURPLE='\033[0;35m'
|
||||||
|
NC='\033[0m' # No Color
|
||||||
|
|
||||||
|
SCRIPT_DIR="$(dirname "$(readlink -f "$0")")"
|
||||||
|
HOSTNAME=$(hostname)
|
||||||
|
BACKUP_ROOT="$SCRIPT_DIR/crontab-backups"
|
||||||
|
BACKUP_DIR="$BACKUP_ROOT/$HOSTNAME"
|
||||||
|
LOG_DIR="$SCRIPT_DIR/logs"
|
||||||
|
CURRENT_BACKUP="$BACKUP_DIR/current-crontab.backup"
|
||||||
|
ARCHIVE_DIR="$BACKUP_DIR/archive"
|
||||||
|
|
||||||
|
# Ensure directories exist
|
||||||
|
mkdir -p "$BACKUP_DIR" "$ARCHIVE_DIR" "$LOG_DIR"
|
||||||
|
|
||||||
|
log_message() {
|
||||||
|
local message="$1"
|
||||||
|
local log_file="$LOG_DIR/crontab-management.log"
|
||||||
|
echo -e "$(date '+%Y-%m-%d %H:%M:%S') $message"
|
||||||
|
echo "$(date '+%Y-%m-%d %H:%M:%S') $message" | sed 's/\x1b\[[0-9;]*m//g' >> "$log_file"
|
||||||
|
}
|
||||||
|
|
||||||
|
log_error() {
|
||||||
|
log_message "${RED}ERROR: $1${NC}"
|
||||||
|
}
|
||||||
|
|
||||||
|
log_success() {
|
||||||
|
log_message "${GREEN}SUCCESS: $1${NC}"
|
||||||
|
}
|
||||||
|
|
||||||
|
log_warning() {
|
||||||
|
log_message "${YELLOW}WARNING: $1${NC}"
|
||||||
|
}
|
||||||
|
|
||||||
|
log_info() {
|
||||||
|
log_message "${BLUE}INFO: $1${NC}"
|
||||||
|
}
|
||||||
|
|
||||||
|
create_timestamped_backup() {
|
||||||
|
local backup_type="${1:-manual}"
|
||||||
|
local timestamp=$(date +%Y%m%d_%H%M%S)
|
||||||
|
local backup_file="$ARCHIVE_DIR/${HOSTNAME}-crontab-${backup_type}-${timestamp}.backup"
|
||||||
|
|
||||||
|
log_info "Creating timestamped backup for $HOSTNAME: $backup_file"
|
||||||
|
|
||||||
|
if sudo crontab -l > "$backup_file" 2>/dev/null; then
|
||||||
|
log_success "Backup created: $backup_file"
|
||||||
|
|
||||||
|
# Also update the current backup
|
||||||
|
cp "$backup_file" "$CURRENT_BACKUP"
|
||||||
|
|
||||||
|
# Add metadata
|
||||||
|
echo "# Backup created: $(date)" >> "$backup_file"
|
||||||
|
echo "# Backup type: $backup_type" >> "$backup_file"
|
||||||
|
echo "# System: $HOSTNAME" >> "$backup_file"
|
||||||
|
echo "# User: root" >> "$backup_file"
|
||||||
|
echo "# Full system info: $(uname -a)" >> "$backup_file"
|
||||||
|
|
||||||
|
return 0
|
||||||
|
else
|
||||||
|
log_error "Failed to create backup or no crontab exists"
|
||||||
|
return 1
|
||||||
|
fi
|
||||||
|
}
|
||||||
|
|
||||||
|
list_backups() {
|
||||||
|
local target_hostname="${1:-$HOSTNAME}"
|
||||||
|
local target_dir="$BACKUP_ROOT/$target_hostname/archive"
|
||||||
|
|
||||||
|
log_info "Available crontab backups for $target_hostname:"
|
||||||
|
echo
|
||||||
|
|
||||||
|
if [ -d "$target_dir" ] && [ "$(ls -A "$target_dir" 2>/dev/null)" ]; then
|
||||||
|
printf "%-40s %-20s %-15s\n" "Filename" "Date Created" "Size"
|
||||||
|
printf "%-40s %-20s %-15s\n" "--------" "------------" "----"
|
||||||
|
|
||||||
|
for backup in "$target_dir"/*.backup; do
|
||||||
|
if [ -f "$backup" ]; then
|
||||||
|
local filename=$(basename "$backup")
|
||||||
|
local date_created=$(stat -c %y "$backup" | cut -d' ' -f1,2 | cut -d'.' -f1)
|
||||||
|
local size=$(stat -c %s "$backup")
|
||||||
|
printf "%-40s %-20s %-15s bytes\n" "$filename" "$date_created" "$size"
|
||||||
|
fi
|
||||||
|
done
|
||||||
|
else
|
||||||
|
log_warning "No backups found in $target_dir"
|
||||||
|
fi
|
||||||
|
echo
|
||||||
|
|
||||||
|
# Show all available systems if current system has no backups or if showing all
|
||||||
|
if [ "$target_hostname" = "$HOSTNAME" ] && [ ! -d "$target_dir" ]; then
|
||||||
|
log_info "Available systems with backups:"
|
||||||
|
for system_dir in "$BACKUP_ROOT"/*; do
|
||||||
|
if [ -d "$system_dir/archive" ] && [ "$(ls -A "$system_dir/archive" 2>/dev/null)" ]; then
|
||||||
|
local system_name=$(basename "$system_dir")
|
||||||
|
local backup_count=$(ls -1 "$system_dir/archive"/*.backup 2>/dev/null | wc -l)
|
||||||
|
echo " - $system_name ($backup_count backups)"
|
||||||
|
fi
|
||||||
|
done
|
||||||
|
echo
|
||||||
|
echo "Use: $0 list [hostname] to view backups for a specific system"
|
||||||
|
fi
|
||||||
|
}
|
||||||
|
|
||||||
|
restore_from_backup() {
|
||||||
|
local backup_file="$1"
|
||||||
|
local source_hostname=""
|
||||||
|
|
||||||
|
if [ -z "$backup_file" ]; then
|
||||||
|
log_error "No backup file specified"
|
||||||
|
list_backups
|
||||||
|
return 1
|
||||||
|
fi
|
||||||
|
|
||||||
|
# Handle different backup file formats and paths
|
||||||
|
if [[ "$backup_file" == *"/"* ]]; then
|
||||||
|
# Full or relative path provided
|
||||||
|
if [[ ! "$backup_file" = /* ]]; then
|
||||||
|
backup_file="$ARCHIVE_DIR/$backup_file"
|
||||||
|
fi
|
||||||
|
else
|
||||||
|
# Just filename provided - check current system first, then others
|
||||||
|
if [ -f "$ARCHIVE_DIR/$backup_file" ]; then
|
||||||
|
backup_file="$ARCHIVE_DIR/$backup_file"
|
||||||
|
else
|
||||||
|
# Search in other system directories
|
||||||
|
local found_file=""
|
||||||
|
for system_dir in "$BACKUP_ROOT"/*; do
|
||||||
|
if [ -f "$system_dir/archive/$backup_file" ]; then
|
||||||
|
found_file="$system_dir/archive/$backup_file"
|
||||||
|
source_hostname=$(basename "$system_dir")
|
||||||
|
break
|
||||||
|
fi
|
||||||
|
done
|
||||||
|
|
||||||
|
if [ -n "$found_file" ]; then
|
||||||
|
backup_file="$found_file"
|
||||||
|
log_warning "Backup file found in $source_hostname system directory"
|
||||||
|
fi
|
||||||
|
fi
|
||||||
|
fi
|
||||||
|
|
||||||
|
if [ ! -f "$backup_file" ]; then
|
||||||
|
log_error "Backup file not found: $backup_file"
|
||||||
|
echo
|
||||||
|
log_info "Available backups:"
|
||||||
|
list_backups
|
||||||
|
return 1
|
||||||
|
fi
|
||||||
|
|
||||||
|
# Extract source hostname from backup metadata if available
|
||||||
|
if [ -z "$source_hostname" ]; then
|
||||||
|
source_hostname=$(grep "^# System:" "$backup_file" 2>/dev/null | cut -d' ' -f3 || echo "unknown")
|
||||||
|
fi
|
||||||
|
|
||||||
|
log_info "Restoring crontab from: $backup_file"
|
||||||
|
if [ "$source_hostname" != "unknown" ] && [ "$source_hostname" != "$HOSTNAME" ]; then
|
||||||
|
log_warning "Restoring backup from different system: $source_hostname -> $HOSTNAME"
|
||||||
|
echo -n "Continue? [y/N]: "
|
||||||
|
read -r response
|
||||||
|
if [[ ! "$response" =~ ^[Yy]$ ]]; then
|
||||||
|
log_info "Restore cancelled"
|
||||||
|
return 1
|
||||||
|
fi
|
||||||
|
fi
|
||||||
|
|
||||||
|
# Create a safety backup before restoring
|
||||||
|
create_timestamped_backup "pre-restore"
|
||||||
|
|
||||||
|
# Remove metadata lines before restoring
|
||||||
|
grep -v "^# Backup" "$backup_file" > /tmp/crontab_restore.tmp
|
||||||
|
|
||||||
|
if sudo crontab /tmp/crontab_restore.tmp; then
|
||||||
|
log_success "Crontab restored successfully from $backup_file"
|
||||||
|
rm -f /tmp/crontab_restore.tmp
|
||||||
|
else
|
||||||
|
log_error "Failed to restore crontab"
|
||||||
|
rm -f /tmp/crontab_restore.tmp
|
||||||
|
return 1
|
||||||
|
fi
|
||||||
|
}
|
||||||
|
|
||||||
|
compare_crontabs() {
|
||||||
|
local file1="${1:-current}"
|
||||||
|
local file2="$2"
|
||||||
|
|
||||||
|
if [ "$file1" = "current" ]; then
|
||||||
|
sudo crontab -l > /tmp/current_crontab.tmp 2>/dev/null || echo "# No current crontab" > /tmp/current_crontab.tmp
|
||||||
|
file1="/tmp/current_crontab.tmp"
|
||||||
|
fi
|
||||||
|
|
||||||
|
if [ -z "$file2" ]; then
|
||||||
|
file2="$CURRENT_BACKUP"
|
||||||
|
fi
|
||||||
|
|
||||||
|
# Handle relative paths and cross-system backups
|
||||||
|
if [[ ! "$file2" = /* ]]; then
|
||||||
|
# Check current system first
|
||||||
|
if [ -f "$ARCHIVE_DIR/$file2" ]; then
|
||||||
|
file2="$ARCHIVE_DIR/$file2"
|
||||||
|
else
|
||||||
|
# Search in other system directories
|
||||||
|
local found_file=""
|
||||||
|
for system_dir in "$BACKUP_ROOT"/*; do
|
||||||
|
if [ -f "$system_dir/archive/$file2" ]; then
|
||||||
|
found_file="$system_dir/archive/$file2"
|
||||||
|
local source_hostname=$(basename "$system_dir")
|
||||||
|
log_info "Found backup in $source_hostname system directory"
|
||||||
|
break
|
||||||
|
fi
|
||||||
|
done
|
||||||
|
|
||||||
|
if [ -n "$found_file" ]; then
|
||||||
|
file2="$found_file"
|
||||||
|
else
|
||||||
|
file2="$ARCHIVE_DIR/$file2" # Default back to current system
|
||||||
|
fi
|
||||||
|
fi
|
||||||
|
fi
|
||||||
|
|
||||||
|
if [ ! -f "$file2" ]; then
|
||||||
|
log_error "Comparison file not found: $file2"
|
||||||
|
return 1
|
||||||
|
fi
|
||||||
|
|
||||||
|
log_info "Comparing current crontab ($HOSTNAME) with: $(basename "$file2")"
|
||||||
|
echo
|
||||||
|
|
||||||
|
# Clean comparison files (remove metadata)
|
||||||
|
grep -v "^# Backup" "$file1" > /tmp/clean_file1.tmp 2>/dev/null || touch /tmp/clean_file1.tmp
|
||||||
|
grep -v "^# Backup" "$file2" > /tmp/clean_file2.tmp 2>/dev/null || touch /tmp/clean_file2.tmp
|
||||||
|
|
||||||
|
if diff -u /tmp/clean_file1.tmp /tmp/clean_file2.tmp; then
|
||||||
|
log_success "Crontabs are identical"
|
||||||
|
else
|
||||||
|
log_warning "Crontabs differ (see above)"
|
||||||
|
fi
|
||||||
|
|
||||||
|
# Cleanup
|
||||||
|
rm -f /tmp/current_crontab.tmp /tmp/clean_file1.tmp /tmp/clean_file2.tmp
|
||||||
|
}
|
||||||
|
|
||||||
|
validate_crontab_syntax() {
|
||||||
|
local crontab_file="${1:-current}"
|
||||||
|
|
||||||
|
if [ "$crontab_file" = "current" ]; then
|
||||||
|
sudo crontab -l > /tmp/validate_crontab.tmp 2>/dev/null || echo "# No current crontab" > /tmp/validate_crontab.tmp
|
||||||
|
crontab_file="/tmp/validate_crontab.tmp"
|
||||||
|
fi
|
||||||
|
|
||||||
|
if [ ! -f "$crontab_file" ]; then
|
||||||
|
log_error "Crontab file not found: $crontab_file"
|
||||||
|
return 1
|
||||||
|
fi
|
||||||
|
|
||||||
|
log_info "Validating crontab syntax: $crontab_file"
|
||||||
|
|
||||||
|
local line_num=0
|
||||||
|
local errors=0
|
||||||
|
|
||||||
|
while IFS= read -r line; do
|
||||||
|
((line_num++))
|
||||||
|
|
||||||
|
# Skip comments and empty lines
|
||||||
|
if [[ "$line" =~ ^[[:space:]]*# ]] || [[ "$line" =~ ^[[:space:]]*$ ]]; then
|
||||||
|
continue
|
||||||
|
fi
|
||||||
|
|
||||||
|
# Basic cron format validation
|
||||||
|
if [[ ! "$line" =~ ^[[:space:]]*([0-9*,-]+[[:space:]]+){4}[0-9*,-]+[[:space:]].+ ]]; then
|
||||||
|
log_error "Line $line_num: Invalid cron format: $line"
|
||||||
|
((errors++))
|
||||||
|
fi
|
||||||
|
|
||||||
|
# Check for common issues
|
||||||
|
if [[ "$line" =~ \$\? ]] && [[ ! "$line" =~ \{ ]]; then
|
||||||
|
log_warning "Line $line_num: \$? outside of command group may not work as expected"
|
||||||
|
fi
|
||||||
|
|
||||||
|
done < "$crontab_file"
|
||||||
|
|
||||||
|
if [ $errors -eq 0 ]; then
|
||||||
|
log_success "Crontab syntax validation passed"
|
||||||
|
else
|
||||||
|
log_error "Found $errors syntax errors"
|
||||||
|
fi
|
||||||
|
|
||||||
|
rm -f /tmp/validate_crontab.tmp
|
||||||
|
return $errors
|
||||||
|
}
|
||||||
|
|
||||||
|
cleanup_old_backups() {
|
||||||
|
local keep_days="${1:-30}"
|
||||||
|
local target_hostname="${2:-$HOSTNAME}"
|
||||||
|
local deleted_count=0
|
||||||
|
|
||||||
|
if [ "$target_hostname" = "all" ]; then
|
||||||
|
log_info "Cleaning up backups older than $keep_days days for all systems"
|
||||||
|
|
||||||
|
for system_dir in "$BACKUP_ROOT"/*; do
|
||||||
|
if [ -d "$system_dir/archive" ]; then
|
||||||
|
local system_name=$(basename "$system_dir")
|
||||||
|
log_info "Cleaning backups for $system_name"
|
||||||
|
|
||||||
|
while IFS= read -r -d '' backup; do
|
||||||
|
if [ -f "$backup" ]; then
|
||||||
|
rm "$backup"
|
||||||
|
((deleted_count++))
|
||||||
|
log_info "Deleted old backup: $(basename "$backup") from $system_name"
|
||||||
|
fi
|
||||||
|
done < <(find "$system_dir/archive" -name "*.backup" -mtime +$keep_days -print0 2>/dev/null)
|
||||||
|
fi
|
||||||
|
done
|
||||||
|
else
|
||||||
|
local target_dir="$BACKUP_ROOT/$target_hostname/archive"
|
||||||
|
log_info "Cleaning up backups older than $keep_days days for $target_hostname"
|
||||||
|
|
||||||
|
if [ -d "$target_dir" ]; then
|
||||||
|
while IFS= read -r -d '' backup; do
|
||||||
|
if [ -f "$backup" ]; then
|
||||||
|
rm "$backup"
|
||||||
|
((deleted_count++))
|
||||||
|
log_info "Deleted old backup: $(basename "$backup")"
|
||||||
|
fi
|
||||||
|
done < <(find "$target_dir" -name "*.backup" -mtime +$keep_days -print0 2>/dev/null)
|
||||||
|
else
|
||||||
|
log_warning "Backup directory not found: $target_dir"
|
||||||
|
fi
|
||||||
|
fi
|
||||||
|
|
||||||
|
if [ $deleted_count -eq 0 ]; then
|
||||||
|
log_info "No old backups found to clean up"
|
||||||
|
else
|
||||||
|
log_success "Cleaned up $deleted_count old backup(s)"
|
||||||
|
fi
|
||||||
|
}
|
||||||
|
|
||||||
|
setup_automated_backup() {
|
||||||
|
log_info "Setting up automated daily crontab backup"
|
||||||
|
|
||||||
|
local backup_script="$SCRIPT_DIR/crontab-backup-system.sh"
|
||||||
|
local backup_entry="0 0 * * * $backup_script backup auto --auto-cleanup 2>&1 | logger -t crontab-backup -p user.info"
|
||||||
|
|
||||||
|
# Check if backup entry already exists
|
||||||
|
if sudo crontab -l 2>/dev/null | grep -q "crontab-backup-system.sh"; then
|
||||||
|
log_warning "Automated backup entry already exists"
|
||||||
|
return 0
|
||||||
|
fi
|
||||||
|
|
||||||
|
# Add the backup entry to current crontab
|
||||||
|
(sudo crontab -l 2>/dev/null; echo "$backup_entry") | sudo crontab -
|
||||||
|
|
||||||
|
log_success "Automated daily backup configured for $HOSTNAME"
|
||||||
|
log_info "Backups will run daily at midnight and be logged to syslog"
|
||||||
|
}
|
||||||
|
|
||||||
|
migrate_legacy_backups() {
|
||||||
|
local legacy_dir="$SCRIPT_DIR/crontab-backups"
|
||||||
|
local legacy_archive="$legacy_dir/archive"
|
||||||
|
|
||||||
|
# Check if legacy structure exists (without hostname subdirectory)
|
||||||
|
if [ -d "$legacy_archive" ] && [ "$legacy_dir" != "$BACKUP_DIR" ]; then
|
||||||
|
log_info "Found legacy backup structure, migrating to hostname-based structure"
|
||||||
|
|
||||||
|
# Create new structure
|
||||||
|
mkdir -p "$BACKUP_DIR" "$ARCHIVE_DIR"
|
||||||
|
|
||||||
|
# Move backups and rename them to include hostname
|
||||||
|
local migrated_count=0
|
||||||
|
for backup in "$legacy_archive"/*.backup; do
|
||||||
|
if [ -f "$backup" ]; then
|
||||||
|
local filename=$(basename "$backup")
|
||||||
|
local new_filename="${HOSTNAME}-${filename}"
|
||||||
|
|
||||||
|
if cp "$backup" "$ARCHIVE_DIR/$new_filename"; then
|
||||||
|
log_success "Migrated: $filename -> $new_filename"
|
||||||
|
((migrated_count++))
|
||||||
|
else
|
||||||
|
log_error "Failed to migrate: $filename"
|
||||||
|
fi
|
||||||
|
fi
|
||||||
|
done
|
||||||
|
|
||||||
|
# Move current backup if it exists
|
||||||
|
if [ -f "$legacy_dir/current-crontab.backup" ]; then
|
||||||
|
cp "$legacy_dir/current-crontab.backup" "$CURRENT_BACKUP"
|
||||||
|
log_success "Migrated current backup"
|
||||||
|
fi
|
||||||
|
|
||||||
|
if [ $migrated_count -gt 0 ]; then
|
||||||
|
log_success "Migrated $migrated_count backup(s) to new structure"
|
||||||
|
echo
|
||||||
|
log_warning "Legacy backups remain in $legacy_archive"
|
||||||
|
log_info "You can safely remove the legacy directory after verifying the migration"
|
||||||
|
echo " rm -rf '$legacy_dir'"
|
||||||
|
fi
|
||||||
|
fi
|
||||||
|
}
|
||||||
|
|
||||||
|
list_all_systems() {
|
||||||
|
log_info "All systems with crontab backups:"
|
||||||
|
echo
|
||||||
|
|
||||||
|
if [ ! -d "$BACKUP_ROOT" ]; then
|
||||||
|
log_warning "No backup root directory found: $BACKUP_ROOT"
|
||||||
|
return 1
|
||||||
|
fi
|
||||||
|
|
||||||
|
printf "%-15s %-10s %-20s %-30s\n" "System" "Backups" "Latest Backup" "Status"
|
||||||
|
printf "%-15s %-10s %-20s %-30s\n" "------" "-------" "-------------" "------"
|
||||||
|
|
||||||
|
local found_systems=false
|
||||||
|
for system_dir in "$BACKUP_ROOT"/*; do
|
||||||
|
if [ -d "$system_dir" ]; then
|
||||||
|
local system_name=$(basename "$system_dir")
|
||||||
|
|
||||||
|
# Skip legacy archive directory - it's not a system
|
||||||
|
if [ "$system_name" = "archive" ]; then
|
||||||
|
continue
|
||||||
|
fi
|
||||||
|
|
||||||
|
found_systems=true
|
||||||
|
local backup_count=$(ls -1 "$system_dir/archive"/*.backup 2>/dev/null | wc -l || echo "0")
|
||||||
|
|
||||||
|
local latest_backup="None"
|
||||||
|
local status="Inactive"
|
||||||
|
|
||||||
|
if [ -f "$system_dir/current-crontab.backup" ]; then
|
||||||
|
latest_backup=$(stat -c %y "$system_dir/current-crontab.backup" | cut -d' ' -f1)
|
||||||
|
|
||||||
|
# Check if backup is recent (within 7 days)
|
||||||
|
local backup_age=$(( ($(date +%s) - $(stat -c %Y "$system_dir/current-crontab.backup")) / 86400 ))
|
||||||
|
if [ $backup_age -le 7 ]; then
|
||||||
|
status="Active"
|
||||||
|
elif [ $backup_age -le 30 ]; then
|
||||||
|
status="Recent"
|
||||||
|
else
|
||||||
|
status="Stale"
|
||||||
|
fi
|
||||||
|
fi
|
||||||
|
|
||||||
|
# Use printf with color formatting
|
||||||
|
if [ "$status" = "Active" ]; then
|
||||||
|
printf "%-15s %-10s %-20s ${GREEN}%-30s${NC}\n" "$system_name" "$backup_count" "$latest_backup" "$status"
|
||||||
|
elif [ "$status" = "Recent" ]; then
|
||||||
|
printf "%-15s %-10s %-20s ${YELLOW}%-30s${NC}\n" "$system_name" "$backup_count" "$latest_backup" "$status"
|
||||||
|
elif [ "$status" = "Stale" ]; then
|
||||||
|
printf "%-15s %-10s %-20s ${RED}%-30s${NC}\n" "$system_name" "$backup_count" "$latest_backup" "$status"
|
||||||
|
else
|
||||||
|
printf "%-15s %-10s %-20s %-30s\n" "$system_name" "$backup_count" "$latest_backup" "$status"
|
||||||
|
fi
|
||||||
|
fi
|
||||||
|
done
|
||||||
|
|
||||||
|
if [ "$found_systems" = false ]; then
|
||||||
|
log_warning "No systems found with backups"
|
||||||
|
fi
|
||||||
|
echo
|
||||||
|
}
|
||||||
|
|
||||||
|
show_status() {
|
||||||
|
local target_hostname="${1:-$HOSTNAME}"
|
||||||
|
|
||||||
|
if [ "$target_hostname" = "all" ]; then
|
||||||
|
log_info "Crontab Backup System Status - All Systems"
|
||||||
|
echo
|
||||||
|
|
||||||
|
for system_dir in "$BACKUP_ROOT"/*; do
|
||||||
|
if [ -d "$system_dir" ]; then
|
||||||
|
local system_name=$(basename "$system_dir")
|
||||||
|
|
||||||
|
# Skip legacy archive directory - it's not a system
|
||||||
|
if [ "$system_name" = "archive" ]; then
|
||||||
|
continue
|
||||||
|
fi
|
||||||
|
|
||||||
|
echo -e "${CYAN}=== $system_name ===${NC}"
|
||||||
|
|
||||||
|
local backup_count=$(ls -1 "$system_dir/archive"/*.backup 2>/dev/null | wc -l || echo "0")
|
||||||
|
echo " - Total backups: $backup_count"
|
||||||
|
echo " - Backup directory: $system_dir"
|
||||||
|
|
||||||
|
if [ -f "$system_dir/current-crontab.backup" ]; then
|
||||||
|
echo " - Latest backup: $(stat -c %y "$system_dir/current-crontab.backup" | cut -d'.' -f1)"
|
||||||
|
else
|
||||||
|
echo " - Latest backup: None"
|
||||||
|
fi
|
||||||
|
echo
|
||||||
|
fi
|
||||||
|
done
|
||||||
|
else
|
||||||
|
log_info "Crontab Backup System Status - $target_hostname"
|
||||||
|
echo
|
||||||
|
|
||||||
|
# Current crontab info
|
||||||
|
if [ "$target_hostname" = "$HOSTNAME" ]; then
|
||||||
|
local cron_count=$(sudo crontab -l 2>/dev/null | grep -c "^[^#]" || echo "0")
|
||||||
|
echo -e "${CYAN}Current Crontab ($HOSTNAME):${NC}"
|
||||||
|
echo " - Active entries: $cron_count"
|
||||||
|
echo " - Last modified: $(stat -c %y /var/spool/cron/crontabs/root 2>/dev/null | cut -d'.' -f1 || echo "Unknown")"
|
||||||
|
echo
|
||||||
|
fi
|
||||||
|
|
||||||
|
# Backup info for specified system
|
||||||
|
local target_dir="$BACKUP_ROOT/$target_hostname"
|
||||||
|
local backup_count=$(ls -1 "$target_dir/archive"/*.backup 2>/dev/null | wc -l || echo "0")
|
||||||
|
echo -e "${CYAN}Backups ($target_hostname):${NC}"
|
||||||
|
echo " - Total backups: $backup_count"
|
||||||
|
echo " - Backup directory: $target_dir"
|
||||||
|
if [ -f "$target_dir/current-crontab.backup" ]; then
|
||||||
|
echo " - Latest backup: $(stat -c %y "$target_dir/current-crontab.backup" | cut -d'.' -f1)"
|
||||||
|
else
|
||||||
|
echo " - Latest backup: None"
|
||||||
|
fi
|
||||||
|
echo
|
||||||
|
fi
|
||||||
|
|
||||||
|
# Log monitoring
|
||||||
|
echo -e "${CYAN}Log Monitoring:${NC}"
|
||||||
|
echo " - Management log: $LOG_DIR/crontab-management.log"
|
||||||
|
echo " - System logs: journalctl -t crontab-backup"
|
||||||
|
echo
|
||||||
|
}
|
||||||
|
|
||||||
|
show_usage() {
|
||||||
|
echo -e "${PURPLE}Crontab Backup and Recovery System (Multi-System)${NC}"
|
||||||
|
echo
|
||||||
|
echo "Usage: $0 [COMMAND] [OPTIONS]"
|
||||||
|
echo
|
||||||
|
echo "Commands:"
|
||||||
|
echo " backup [TYPE] Create a timestamped backup (default: manual)"
|
||||||
|
echo " list [HOSTNAME] List backups for hostname (default: current system)"
|
||||||
|
echo " list-systems Show all systems with backups"
|
||||||
|
echo " restore FILE Restore crontab from backup file"
|
||||||
|
echo " compare [FILE1] [FILE2] Compare current crontab with backup"
|
||||||
|
echo " validate [FILE] Validate crontab syntax"
|
||||||
|
echo " cleanup [DAYS] [HOSTNAME] Clean up old backups (default: 30 days, current system)"
|
||||||
|
echo " status [HOSTNAME|all] Show system status (default: current system)"
|
||||||
|
echo " setup-auto Setup automated daily backups"
|
||||||
|
echo " migrate Migrate legacy backups to hostname structure"
|
||||||
|
echo " import FILE SYSTEM [TYPE] Import backup from external source"
|
||||||
|
echo " create-test-systems Create test systems for demonstration"
|
||||||
|
echo " help Show this help message"
|
||||||
|
echo
|
||||||
|
echo "Multi-System Examples:"
|
||||||
|
echo " $0 backup pre-upgrade # Backup current system"
|
||||||
|
echo " $0 list io # List backups for 'io' system"
|
||||||
|
echo " $0 restore io-crontab-manual-20250526_120000.backup"
|
||||||
|
echo " $0 compare current europa-crontab-manual-20250526_120000.backup"
|
||||||
|
echo " $0 cleanup 7 all # Clean up all systems"
|
||||||
|
echo " $0 status all # Show status for all systems"
|
||||||
|
echo " $0 list-systems # Show all available systems"
|
||||||
|
echo " $0 import /path/to/crontab.backup io manual # Import backup for io system"
|
||||||
|
echo
|
||||||
|
echo "Current System: $HOSTNAME"
|
||||||
|
echo "Backup Root: $BACKUP_ROOT"
|
||||||
|
echo
|
||||||
|
}
|
||||||
|
|
||||||
|
create_test_systems() {
|
||||||
|
log_info "Creating test backup structure for io, europa, and racknerd systems"
|
||||||
|
|
||||||
|
# Create sample systems directories
|
||||||
|
local test_systems=("io" "racknerd")
|
||||||
|
|
||||||
|
for system in "${test_systems[@]}"; do
|
||||||
|
local system_dir="$BACKUP_ROOT/$system"
|
||||||
|
local system_archive="$system_dir/archive"
|
||||||
|
|
||||||
|
mkdir -p "$system_archive"
|
||||||
|
|
||||||
|
# Create a sample backup for each system
|
||||||
|
local timestamp=$(date +%Y%m%d_%H%M%S)
|
||||||
|
local sample_backup="$system_archive/${system}-crontab-sample-${timestamp}.backup"
|
||||||
|
|
||||||
|
# Create sample crontab content for each system
|
||||||
|
case "$system" in
|
||||||
|
"io")
|
||||||
|
cat > "$sample_backup" << EOF
|
||||||
|
# Sample crontab for io system
|
||||||
|
0 2 * * * /home/user/backup-docker.sh 2>&1 | logger -t docker-backup -p user.info
|
||||||
|
30 3 * * * /home/user/backup-media.sh 2>&1 | logger -t media-backup -p user.info
|
||||||
|
0 4 * * * /home/user/validate-backups.sh 2>&1 | logger -t backup-validation -p user.info
|
||||||
|
# Backup created: $(date)
|
||||||
|
# Backup type: sample
|
||||||
|
# System: $system
|
||||||
|
# User: root
|
||||||
|
# Full system info: Linux $system 5.15.0-generic #1 SMP x86_64 GNU/Linux
|
||||||
|
EOF
|
||||||
|
;;
|
||||||
|
"racknerd")
|
||||||
|
cat > "$sample_backup" << EOF
|
||||||
|
# Sample crontab for racknerd system
|
||||||
|
0 1 * * * /home/user/backup-plex.sh 2>&1 | logger -t plex-backup -p user.info
|
||||||
|
15 1 * * * /home/user/move-backups.sh 2>&1 | logger -t backup-move -p user.info
|
||||||
|
0 5 * * 0 /home/user/cleanup-old-backups.sh 2>&1 | logger -t backup-cleanup -p user.info
|
||||||
|
# Backup created: $(date)
|
||||||
|
# Backup type: sample
|
||||||
|
# System: $system
|
||||||
|
# User: root
|
||||||
|
# Full system info: Linux $system 5.4.0-generic #1 SMP x86_64 GNU/Linux
|
||||||
|
EOF
|
||||||
|
;;
|
||||||
|
esac
|
||||||
|
|
||||||
|
# Create current backup file
|
||||||
|
cp "$sample_backup" "$system_dir/current-crontab.backup"
|
||||||
|
|
||||||
|
log_success "Created test system: $system with sample backup"
|
||||||
|
done
|
||||||
|
|
||||||
|
log_success "Test systems created successfully"
|
||||||
|
}
|
||||||
|
|
||||||
|
import_backup() {
|
||||||
|
local source_file="$1"
|
||||||
|
local source_system="$2"
|
||||||
|
local backup_type="${3:-imported}"
|
||||||
|
|
||||||
|
if [ -z "$source_file" ] || [ -z "$source_system" ]; then
|
||||||
|
log_error "Usage: import_backup <source_file> <source_system> [backup_type]"
|
||||||
|
return 1
|
||||||
|
fi
|
||||||
|
|
||||||
|
if [ ! -f "$source_file" ]; then
|
||||||
|
log_error "Source file not found: $source_file"
|
||||||
|
return 1
|
||||||
|
fi
|
||||||
|
|
||||||
|
local target_dir="$BACKUP_ROOT/$source_system"
|
||||||
|
local target_archive="$target_dir/archive"
|
||||||
|
|
||||||
|
mkdir -p "$target_archive"
|
||||||
|
|
||||||
|
local timestamp=$(date +%Y%m%d_%H%M%S)
|
||||||
|
local target_file="$target_archive/${source_system}-crontab-${backup_type}-${timestamp}.backup"
|
||||||
|
|
||||||
|
# Copy and add metadata
|
||||||
|
cp "$source_file" "$target_file"
|
||||||
|
|
||||||
|
# Add metadata if not present
|
||||||
|
if ! grep -q "^# Backup created:" "$target_file"; then
|
||||||
|
cat >> "$target_file" << EOF
|
||||||
|
# Backup created: $(date)
|
||||||
|
# Backup type: $backup_type
|
||||||
|
# System: $source_system
|
||||||
|
# User: root
|
||||||
|
# Imported from: $source_file
|
||||||
|
EOF
|
||||||
|
fi
|
||||||
|
|
||||||
|
# Update current backup
|
||||||
|
cp "$target_file" "$target_dir/current-crontab.backup"
|
||||||
|
|
||||||
|
log_success "Imported backup for $source_system: $target_file"
|
||||||
|
}
|
||||||
|
|
||||||
|
# Main command handling
|
||||||
|
case "${1:-help}" in
|
||||||
|
backup)
|
||||||
|
# Check for auto-cleanup flag
|
||||||
|
if [[ "${@}" == *"--auto-cleanup"* ]]; then
|
||||||
|
create_timestamped_backup "${2:-auto}"
|
||||||
|
cleanup_old_backups 30
|
||||||
|
else
|
||||||
|
create_timestamped_backup "${2:-manual}"
|
||||||
|
fi
|
||||||
|
;;
|
||||||
|
list)
|
||||||
|
list_backups "$2"
|
||||||
|
;;
|
||||||
|
list-systems)
|
||||||
|
list_all_systems
|
||||||
|
;;
|
||||||
|
restore)
|
||||||
|
if [ -z "$2" ]; then
|
||||||
|
log_error "Please specify a backup file to restore"
|
||||||
|
list_backups
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
restore_from_backup "$2"
|
||||||
|
;;
|
||||||
|
compare)
|
||||||
|
compare_crontabs "$2" "$3"
|
||||||
|
;;
|
||||||
|
validate)
|
||||||
|
validate_crontab_syntax "$2"
|
||||||
|
;;
|
||||||
|
cleanup)
|
||||||
|
cleanup_old_backups "${2:-30}" "${3:-$HOSTNAME}"
|
||||||
|
;;
|
||||||
|
status)
|
||||||
|
show_status "${2:-$HOSTNAME}"
|
||||||
|
;;
|
||||||
|
setup-auto)
|
||||||
|
setup_automated_backup
|
||||||
|
;;
|
||||||
|
migrate)
|
||||||
|
migrate_legacy_backups
|
||||||
|
;;
|
||||||
|
import)
|
||||||
|
if [ -z "$2" ] || [ -z "$3" ]; then
|
||||||
|
log_error "Please specify source file and target system"
|
||||||
|
echo "Usage: $0 import <source_file> <target_system> [backup_type]"
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
import_backup "$2" "$3" "$4"
|
||||||
|
;;
|
||||||
|
create-test-systems)
|
||||||
|
create_test_systems
|
||||||
|
;;
|
||||||
|
help|*)
|
||||||
|
show_usage
|
||||||
|
;;
|
||||||
|
esac
|
||||||
|
|
||||||
|
# Auto-migrate legacy backups on first run
|
||||||
|
if [ ! -d "$BACKUP_DIR" ] && [ "$1" != "help" ] && [ "$1" != "migrate" ]; then
|
||||||
|
migrate_legacy_backups
|
||||||
|
fi
|
||||||
@@ -0,0 +1,40 @@
|
|||||||
|
# Edit this file to introduce tasks to be run by cron.
|
||||||
|
#
|
||||||
|
# Each task to run has to be defined through a single line
|
||||||
|
# indicating with different fields when the task will be run
|
||||||
|
# and what command to run for the task
|
||||||
|
#
|
||||||
|
# To define the time you can provide concrete values for
|
||||||
|
# minute (m), hour (h), day of month (dom), month (mon),
|
||||||
|
# and day of week (dow) or use '*' in these fields (for 'any').
|
||||||
|
#
|
||||||
|
# Notice that tasks will be started based on the cron's system
|
||||||
|
# daemon's notion of time and timezones.
|
||||||
|
#
|
||||||
|
# Output of the crontab jobs (including errors) is sent through
|
||||||
|
# email to the user the crontab file belongs to (unless redirected).
|
||||||
|
#
|
||||||
|
# For example, you can run a backup of all your user accounts
|
||||||
|
# at 5 a.m every week with:
|
||||||
|
# 0 5 * * 1 tar -zcf /var/backups/home.tgz /home/
|
||||||
|
#
|
||||||
|
# For more information see the manual pages of crontab(5) and cron(8)
|
||||||
|
#
|
||||||
|
# m h dom mon dow command
|
||||||
|
# 2023-06-11 I can't figure out how to get the nas to mount automagically so here is a band-aid
|
||||||
|
# @reboot sleep 30 && mount -a
|
||||||
|
#
|
||||||
|
# move the files previously backed up at 0100
|
||||||
|
0 1 * * * /home/acedanger/shell/move-backups.sh
|
||||||
|
# daily Plex backup at 0415
|
||||||
|
15 4 * * * /home/acedanger/shell/backup-plex.sh
|
||||||
|
# daily validation at 0700
|
||||||
|
0 7 * * * /home/acedanger/shell/validate-plex-backups.sh --fix
|
||||||
|
# backup immich database
|
||||||
|
0 5 * * 1 mv /mnt/share/media/immich/uploads/backups/immich-db-backup* /mnt/share/media/backups/immich
|
||||||
|
# Generate detailed weekly report (Sundays at 0800)
|
||||||
|
0 8 * * 0 /home/acedanger/shell/validate-plex-backups.sh --report
|
||||||
|
# Backup created: Mon May 26 10:06:22 AM EDT 2025
|
||||||
|
# Backup type: pre-install
|
||||||
|
# System: europa
|
||||||
|
# User: root
|
||||||
@@ -0,0 +1,41 @@
|
|||||||
|
# Edit this file to introduce tasks to be run by cron.
|
||||||
|
#
|
||||||
|
# Each task to run has to be defined through a single line
|
||||||
|
# indicating with different fields when the task will be run
|
||||||
|
# and what command to run for the task
|
||||||
|
#
|
||||||
|
# To define the time you can provide concrete values for
|
||||||
|
# minute (m), hour (h), day of month (dom), month (mon),
|
||||||
|
# and day of week (dow) or use '*' in these fields (for 'any').
|
||||||
|
#
|
||||||
|
# Notice that tasks will be started based on the cron's system
|
||||||
|
# daemon's notion of time and timezones.
|
||||||
|
#
|
||||||
|
# Output of the crontab jobs (including errors) is sent through
|
||||||
|
# email to the user the crontab file belongs to (unless redirected).
|
||||||
|
#
|
||||||
|
# For example, you can run a backup of all your user accounts
|
||||||
|
# at 5 a.m every week with:
|
||||||
|
# 0 5 * * 1 tar -zcf /var/backups/home.tgz /home/
|
||||||
|
#
|
||||||
|
# For more information see the manual pages of crontab(5) and cron(8)
|
||||||
|
#
|
||||||
|
# m h dom mon dow command
|
||||||
|
# 2023-06-11 I can't figure out how to get the nas to mount automagically so here is a band-aid
|
||||||
|
# @reboot sleep 30 && mount -a
|
||||||
|
#
|
||||||
|
# move the files previously backed up at 0100
|
||||||
|
0 1 * * * /home/acedanger/shell/move-backups.sh
|
||||||
|
# daily Plex backup at 0415
|
||||||
|
15 4 * * * /home/acedanger/shell/backup-plex.sh
|
||||||
|
# daily validation at 0700
|
||||||
|
0 7 * * * /home/acedanger/shell/validate-plex-backups.sh --fix
|
||||||
|
# backup immich database
|
||||||
|
0 5 * * 1 mv /mnt/share/media/immich/uploads/backups/immich-db-backup* /mnt/share/media/backups/immich
|
||||||
|
# Generate detailed weekly report (Sundays at 0800)
|
||||||
|
0 8 * * 0 /home/acedanger/shell/validate-plex-backups.sh --report
|
||||||
|
# Backup created: Mon May 26 10:13:54 AM EDT 2025
|
||||||
|
# Backup type: initial
|
||||||
|
# System: europa
|
||||||
|
# User: root
|
||||||
|
# Full system info: Linux europa 6.1.0-33-amd64 #1 SMP PREEMPT_DYNAMIC Debian 6.1.133-1 (2025-04-10) x86_64 GNU/Linux
|
||||||
@@ -0,0 +1,40 @@
|
|||||||
|
# Edit this file to introduce tasks to be run by cron.
|
||||||
|
#
|
||||||
|
# Each task to run has to be defined through a single line
|
||||||
|
# indicating with different fields when the task will be run
|
||||||
|
# and what command to run for the task
|
||||||
|
#
|
||||||
|
# To define the time you can provide concrete values for
|
||||||
|
# minute (m), hour (h), day of month (dom), month (mon),
|
||||||
|
# and day of week (dow) or use '*' in these fields (for 'any').
|
||||||
|
#
|
||||||
|
# Notice that tasks will be started based on the cron's system
|
||||||
|
# daemon's notion of time and timezones.
|
||||||
|
#
|
||||||
|
# Output of the crontab jobs (including errors) is sent through
|
||||||
|
# email to the user the crontab file belongs to (unless redirected).
|
||||||
|
#
|
||||||
|
# For example, you can run a backup of all your user accounts
|
||||||
|
# at 5 a.m every week with:
|
||||||
|
# 0 5 * * 1 tar -zcf /var/backups/home.tgz /home/
|
||||||
|
#
|
||||||
|
# For more information see the manual pages of crontab(5) and cron(8)
|
||||||
|
#
|
||||||
|
# m h dom mon dow command
|
||||||
|
# 2023-06-11 I can't figure out how to get the nas to mount automagically so here is a band-aid
|
||||||
|
# @reboot sleep 30 && mount -a
|
||||||
|
#
|
||||||
|
# move the files previously backed up at 0100
|
||||||
|
0 1 * * * /home/acedanger/shell/move-backups.sh
|
||||||
|
# daily Plex backup at 0415
|
||||||
|
15 4 * * * /home/acedanger/shell/backup-plex.sh
|
||||||
|
# daily validation at 0700
|
||||||
|
0 7 * * * /home/acedanger/shell/validate-plex-backups.sh --fix
|
||||||
|
# backup immich database
|
||||||
|
0 5 * * 1 mv /mnt/share/media/immich/uploads/backups/immich-db-backup* /mnt/share/media/backups/immich
|
||||||
|
# Generate detailed weekly report (Sundays at 0800)
|
||||||
|
0 8 * * 0 /home/acedanger/shell/validate-plex-backups.sh --report
|
||||||
|
# Backup created: Mon May 26 10:06:22 AM EDT 2025
|
||||||
|
# Backup type: pre-install
|
||||||
|
# System: europa
|
||||||
|
# User: root
|
||||||
@@ -0,0 +1,41 @@
|
|||||||
|
# Edit this file to introduce tasks to be run by cron.
|
||||||
|
#
|
||||||
|
# Each task to run has to be defined through a single line
|
||||||
|
# indicating with different fields when the task will be run
|
||||||
|
# and what command to run for the task
|
||||||
|
#
|
||||||
|
# To define the time you can provide concrete values for
|
||||||
|
# minute (m), hour (h), day of month (dom), month (mon),
|
||||||
|
# and day of week (dow) or use '*' in these fields (for 'any').
|
||||||
|
#
|
||||||
|
# Notice that tasks will be started based on the cron's system
|
||||||
|
# daemon's notion of time and timezones.
|
||||||
|
#
|
||||||
|
# Output of the crontab jobs (including errors) is sent through
|
||||||
|
# email to the user the crontab file belongs to (unless redirected).
|
||||||
|
#
|
||||||
|
# For example, you can run a backup of all your user accounts
|
||||||
|
# at 5 a.m every week with:
|
||||||
|
# 0 5 * * 1 tar -zcf /var/backups/home.tgz /home/
|
||||||
|
#
|
||||||
|
# For more information see the manual pages of crontab(5) and cron(8)
|
||||||
|
#
|
||||||
|
# m h dom mon dow command
|
||||||
|
# 2023-06-11 I can't figure out how to get the nas to mount automagically so here is a band-aid
|
||||||
|
# @reboot sleep 30 && mount -a
|
||||||
|
#
|
||||||
|
# move the files previously backed up at 0100
|
||||||
|
0 1 * * * /home/acedanger/shell/move-backups.sh
|
||||||
|
# daily Plex backup at 0415
|
||||||
|
15 4 * * * /home/acedanger/shell/backup-plex.sh
|
||||||
|
# daily validation at 0700
|
||||||
|
0 7 * * * /home/acedanger/shell/validate-plex-backups.sh --fix
|
||||||
|
# backup immich database
|
||||||
|
0 5 * * 1 mv /mnt/share/media/immich/uploads/backups/immich-db-backup* /mnt/share/media/backups/immich
|
||||||
|
# Generate detailed weekly report (Sundays at 0800)
|
||||||
|
0 8 * * 0 /home/acedanger/shell/validate-plex-backups.sh --report
|
||||||
|
# Backup created: Mon May 26 10:30:05 AM EDT 2025
|
||||||
|
# Backup type: test-colors
|
||||||
|
# System: europa
|
||||||
|
# User: root
|
||||||
|
# Full system info: Linux europa 6.1.0-33-amd64 #1 SMP PREEMPT_DYNAMIC Debian 6.1.133-1 (2025-04-10) x86_64 GNU/Linux
|
||||||
36
crontab-backups/europa/current-crontab.backup
Normal file
36
crontab-backups/europa/current-crontab.backup
Normal file
@@ -0,0 +1,36 @@
|
|||||||
|
# Edit this file to introduce tasks to be run by cron.
|
||||||
|
#
|
||||||
|
# Each task to run has to be defined through a single line
|
||||||
|
# indicating with different fields when the task will be run
|
||||||
|
# and what command to run for the task
|
||||||
|
#
|
||||||
|
# To define the time you can provide concrete values for
|
||||||
|
# minute (m), hour (h), day of month (dom), month (mon),
|
||||||
|
# and day of week (dow) or use '*' in these fields (for 'any').
|
||||||
|
#
|
||||||
|
# Notice that tasks will be started based on the cron's system
|
||||||
|
# daemon's notion of time and timezones.
|
||||||
|
#
|
||||||
|
# Output of the crontab jobs (including errors) is sent through
|
||||||
|
# email to the user the crontab file belongs to (unless redirected).
|
||||||
|
#
|
||||||
|
# For example, you can run a backup of all your user accounts
|
||||||
|
# at 5 a.m every week with:
|
||||||
|
# 0 5 * * 1 tar -zcf /var/backups/home.tgz /home/
|
||||||
|
#
|
||||||
|
# For more information see the manual pages of crontab(5) and cron(8)
|
||||||
|
#
|
||||||
|
# m h dom mon dow command
|
||||||
|
# 2023-06-11 I can't figure out how to get the nas to mount automagically so here is a band-aid
|
||||||
|
# @reboot sleep 30 && mount -a
|
||||||
|
#
|
||||||
|
# move the files previously backed up at 0100
|
||||||
|
0 1 * * * /home/acedanger/shell/move-backups.sh
|
||||||
|
# daily Plex backup at 0415
|
||||||
|
15 4 * * * /home/acedanger/shell/backup-plex.sh
|
||||||
|
# daily validation at 0700
|
||||||
|
0 7 * * * /home/acedanger/shell/validate-plex-backups.sh --fix
|
||||||
|
# backup immich database
|
||||||
|
0 5 * * 1 mv /mnt/share/media/immich/uploads/backups/immich-db-backup* /mnt/share/media/backups/immich
|
||||||
|
# Generate detailed weekly report (Sundays at 0800)
|
||||||
|
0 8 * * 0 /home/acedanger/shell/validate-plex-backups.sh --report
|
||||||
226
docs/enhanced-crontab-system.md
Normal file
226
docs/enhanced-crontab-system.md
Normal file
@@ -0,0 +1,226 @@
|
|||||||
|
# Enhanced Crontab and Backup Monitoring System
|
||||||
|
|
||||||
|
## Overview
|
||||||
|
|
||||||
|
This enhanced system provides comprehensive crontab management with automatic backups, system logging, and advanced monitoring capabilities for your backup operations.
|
||||||
|
|
||||||
|
## Components
|
||||||
|
|
||||||
|
### 1. Enhanced Crontab Entries (`enhanced-crontab.txt`)
|
||||||
|
|
||||||
|
- **Move backups** (01:00): Transfers Docker backups with logging
|
||||||
|
- **Plex backup** (04:15): Daily Plex database backup with auto-repair
|
||||||
|
- **Plex validation** (07:00): Validates and fixes backup integrity
|
||||||
|
- **Immich backup** (05:00 Mon): Weekly Immich database backup move
|
||||||
|
- **Weekly report** (08:00 Sun): Comprehensive backup status report
|
||||||
|
|
||||||
|
### 2. Crontab Management (`manage-enhanced-crontab.sh`)
|
||||||
|
|
||||||
|
- Install enhanced crontab entries with validation
|
||||||
|
- Verify all backup scripts exist and are executable
|
||||||
|
- Setup automated backups and log rotation
|
||||||
|
- Integration with backup and monitoring systems
|
||||||
|
|
||||||
|
### 3. Crontab Backup System (`crontab-backup-system.sh`)
|
||||||
|
|
||||||
|
- Automated timestamped crontab backups
|
||||||
|
- Backup comparison and restoration
|
||||||
|
- Syntax validation
|
||||||
|
- Automated cleanup of old backups
|
||||||
|
- Daily automated backup scheduling
|
||||||
|
|
||||||
|
### 4. Backup Log Monitor (`backup-log-monitor.sh`)
|
||||||
|
|
||||||
|
- Real-time log monitoring with color coding
|
||||||
|
- Error analysis and reporting
|
||||||
|
- System health checks
|
||||||
|
- Comprehensive backup reports
|
||||||
|
- Service status overview
|
||||||
|
|
||||||
|
## Quick Start
|
||||||
|
|
||||||
|
### 1. Install the Enhanced System
|
||||||
|
|
||||||
|
```bash
|
||||||
|
# Make scripts executable
|
||||||
|
chmod +x manage-enhanced-crontab.sh crontab-backup-system.sh backup-log-monitor.sh
|
||||||
|
|
||||||
|
# Install enhanced crontab with all features
|
||||||
|
sudo ./manage-enhanced-crontab.sh install
|
||||||
|
```
|
||||||
|
|
||||||
|
### 2. Monitor Your Backups
|
||||||
|
|
||||||
|
```bash
|
||||||
|
# Real-time monitoring
|
||||||
|
./backup-log-monitor.sh monitor
|
||||||
|
|
||||||
|
# Check system health
|
||||||
|
./backup-log-monitor.sh health
|
||||||
|
|
||||||
|
# View recent activity
|
||||||
|
./backup-log-monitor.sh recent 24
|
||||||
|
|
||||||
|
# Generate weekly report
|
||||||
|
./backup-log-monitor.sh report 7
|
||||||
|
```
|
||||||
|
|
||||||
|
### 3. Manage Crontab Backups
|
||||||
|
|
||||||
|
```bash
|
||||||
|
# Create manual backup
|
||||||
|
./crontab-backup-system.sh backup manual
|
||||||
|
|
||||||
|
# List all backups
|
||||||
|
./crontab-backup-system.sh list
|
||||||
|
|
||||||
|
# Compare current with backup
|
||||||
|
./crontab-backup-system.sh compare current
|
||||||
|
|
||||||
|
# System status
|
||||||
|
./crontab-backup-system.sh status
|
||||||
|
```
|
||||||
|
|
||||||
|
## Features
|
||||||
|
|
||||||
|
### Enhanced Logging
|
||||||
|
|
||||||
|
All backup operations now log to syslog with specific tags:
|
||||||
|
|
||||||
|
- `plex-backup`: Plex database backup operations
|
||||||
|
- `backup-move`: Docker backup file transfers
|
||||||
|
- `plex-validation`: Backup integrity checks
|
||||||
|
- `immich-backup`: Immich database operations
|
||||||
|
- `plex-report`: Weekly reporting
|
||||||
|
- `crontab-backup`: Crontab backup operations
|
||||||
|
|
||||||
|
### Automatic Backups
|
||||||
|
|
||||||
|
- **Crontab backups**: Daily automated backups at midnight
|
||||||
|
- **Cleanup**: Automatic removal of backups older than 30 days
|
||||||
|
- **Validation**: Syntax checking before applying changes
|
||||||
|
- **Recovery**: Easy restoration from any backup point
|
||||||
|
|
||||||
|
### Health Monitoring
|
||||||
|
|
||||||
|
- Script existence and permissions
|
||||||
|
- Backup directory availability
|
||||||
|
- Recent activity tracking
|
||||||
|
- Error rate monitoring
|
||||||
|
- Overall system health scoring
|
||||||
|
|
||||||
|
### Error Handling
|
||||||
|
|
||||||
|
- Graceful failure handling in cron jobs
|
||||||
|
- Detailed error logging and reporting
|
||||||
|
- Exit code tracking for debugging
|
||||||
|
- Comprehensive error summaries
|
||||||
|
|
||||||
|
## Log Analysis
|
||||||
|
|
||||||
|
### View Real-time Logs
|
||||||
|
|
||||||
|
```bash
|
||||||
|
# All backup services
|
||||||
|
sudo journalctl -f -t plex-backup -t backup-move -t plex-validation -t immich-backup -t plex-report
|
||||||
|
|
||||||
|
# Specific service
|
||||||
|
sudo journalctl -f -t plex-backup
|
||||||
|
|
||||||
|
# With our monitor (recommended)
|
||||||
|
./backup-log-monitor.sh monitor
|
||||||
|
```
|
||||||
|
|
||||||
|
### Historical Analysis
|
||||||
|
|
||||||
|
```bash
|
||||||
|
# Last 24 hours
|
||||||
|
sudo journalctl --since '24 hours ago' -t plex-backup
|
||||||
|
|
||||||
|
# Last week with errors only
|
||||||
|
sudo journalctl --since '1 week ago' --priority=err -t plex-backup
|
||||||
|
|
||||||
|
# Using our tools
|
||||||
|
./backup-log-monitor.sh recent 24 plex-backup
|
||||||
|
./backup-log-monitor.sh errors 7
|
||||||
|
```
|
||||||
|
|
||||||
|
## Backup Recovery
|
||||||
|
|
||||||
|
### Restore Crontab from Backup
|
||||||
|
|
||||||
|
```bash
|
||||||
|
# List available backups
|
||||||
|
./crontab-backup-system.sh list
|
||||||
|
|
||||||
|
# Restore specific backup
|
||||||
|
./crontab-backup-system.sh restore crontab-manual-20250526_120000.backup
|
||||||
|
|
||||||
|
# Compare before restoring
|
||||||
|
./crontab-backup-system.sh compare current crontab-manual-20250526_120000.backup
|
||||||
|
```
|
||||||
|
|
||||||
|
### Emergency Recovery
|
||||||
|
|
||||||
|
If you need to quickly restore the original crontab:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
# The system automatically creates pre-install backups
|
||||||
|
./crontab-backup-system.sh list | grep pre-install
|
||||||
|
./crontab-backup-system.sh restore [backup-filename]
|
||||||
|
```
|
||||||
|
|
||||||
|
## Maintenance
|
||||||
|
|
||||||
|
### Regular Tasks
|
||||||
|
|
||||||
|
```bash
|
||||||
|
# Weekly health check
|
||||||
|
./backup-log-monitor.sh health
|
||||||
|
|
||||||
|
# Monthly backup cleanup
|
||||||
|
./crontab-backup-system.sh cleanup 30
|
||||||
|
|
||||||
|
# Quarterly comprehensive report
|
||||||
|
./backup-log-monitor.sh report 90
|
||||||
|
```
|
||||||
|
|
||||||
|
### Troubleshooting
|
||||||
|
|
||||||
|
```bash
|
||||||
|
# Verify all components
|
||||||
|
./manage-enhanced-crontab.sh verify
|
||||||
|
|
||||||
|
# Check system status
|
||||||
|
./manage-enhanced-crontab.sh status
|
||||||
|
|
||||||
|
# View configuration
|
||||||
|
./manage-enhanced-crontab.sh show
|
||||||
|
|
||||||
|
# Monitor for issues
|
||||||
|
./backup-log-monitor.sh monitor
|
||||||
|
```
|
||||||
|
|
||||||
|
## Integration Notes
|
||||||
|
|
||||||
|
- All scripts follow the established shell repository coding standards
|
||||||
|
- Logging uses consistent tags and priorities
|
||||||
|
- Error handling preserves backup integrity
|
||||||
|
- Color-coded output for better readability
|
||||||
|
- Comprehensive documentation and help systems
|
||||||
|
|
||||||
|
## Security Considerations
|
||||||
|
|
||||||
|
- Scripts validate input and handle errors gracefully
|
||||||
|
- Backup files include metadata for tracking
|
||||||
|
- Permissions are properly managed
|
||||||
|
- Sensitive operations require sudo confirmation
|
||||||
|
- All operations are logged for audit trails
|
||||||
|
|
||||||
|
## Next Steps
|
||||||
|
|
||||||
|
1. **Install the system**: Run `sudo ./manage-enhanced-crontab.sh install`
|
||||||
|
2. **Test monitoring**: Use `./backup-log-monitor.sh monitor` during next backup
|
||||||
|
3. **Review reports**: Generate weekly reports to establish baseline
|
||||||
|
4. **Set up alerts**: Consider integrating with your notification system
|
||||||
|
5. **Document customizations**: Add any local modifications to this guide
|
||||||
280
docs/multi-system-crontab-management.md
Normal file
280
docs/multi-system-crontab-management.md
Normal file
@@ -0,0 +1,280 @@
|
|||||||
|
# Multi-System Crontab Management Guide
|
||||||
|
|
||||||
|
## Overview
|
||||||
|
|
||||||
|
The enhanced crontab backup system now supports managing crontab backups across multiple systems using a hostname-based directory structure. This enables centralized backup management for distributed environments.
|
||||||
|
|
||||||
|
## System Architecture
|
||||||
|
|
||||||
|
### Directory Structure
|
||||||
|
|
||||||
|
```
|
||||||
|
crontab-backups/
|
||||||
|
├── europa/ # Current system (example)
|
||||||
|
│ ├── current-crontab.backup
|
||||||
|
│ └── archive/
|
||||||
|
│ ├── europa-crontab-initial-20250526_101354.backup
|
||||||
|
│ └── europa-crontab-pre-install-20250526_100622.backup
|
||||||
|
├── io/ # Remote system backups
|
||||||
|
│ ├── current-crontab.backup
|
||||||
|
│ └── archive/
|
||||||
|
│ └── io-crontab-sample-20250526_101558.backup
|
||||||
|
└── racknerd/ # Another remote system
|
||||||
|
├── current-crontab.backup
|
||||||
|
└── archive/
|
||||||
|
└── racknerd-crontab-sample-20250526_101558.backup
|
||||||
|
```
|
||||||
|
|
||||||
|
### File Naming Convention
|
||||||
|
|
||||||
|
- Format: `{hostname}-crontab-{type}-{timestamp}.backup`
|
||||||
|
- Examples:
|
||||||
|
- `europa-crontab-manual-20250526_101354.backup`
|
||||||
|
- `io-crontab-pre-upgrade-20250526_120000.backup`
|
||||||
|
- `racknerd-crontab-auto-20250526_000001.backup`
|
||||||
|
|
||||||
|
## Features
|
||||||
|
|
||||||
|
### 🔄 Multi-System Support
|
||||||
|
|
||||||
|
- **Hostname-based organization**: Each system gets its own directory
|
||||||
|
- **Cross-system operations**: View, compare, and restore backups from any system
|
||||||
|
- **Centralized management**: Manage all systems from a single location
|
||||||
|
|
||||||
|
### 📊 System Status Monitoring
|
||||||
|
|
||||||
|
```bash
|
||||||
|
# View status for current system
|
||||||
|
./crontab-backup-system.sh status
|
||||||
|
|
||||||
|
# View status for specific system
|
||||||
|
./crontab-backup-system.sh status io
|
||||||
|
|
||||||
|
# View status for all systems
|
||||||
|
./crontab-backup-system.sh status all
|
||||||
|
```
|
||||||
|
|
||||||
|
### 🗂️ Backup Operations
|
||||||
|
|
||||||
|
```bash
|
||||||
|
# Create backup on current system
|
||||||
|
./crontab-backup-system.sh backup pre-upgrade
|
||||||
|
|
||||||
|
# List backups for specific system
|
||||||
|
./crontab-backup-system.sh list io
|
||||||
|
|
||||||
|
# List all systems with backups
|
||||||
|
./crontab-backup-system.sh list-systems
|
||||||
|
|
||||||
|
# Import backup from external source
|
||||||
|
./crontab-backup-system.sh import /path/to/backup.txt io manual
|
||||||
|
```
|
||||||
|
|
||||||
|
### 🔍 Cross-System Comparison
|
||||||
|
|
||||||
|
```bash
|
||||||
|
# Compare current crontab with backup from another system
|
||||||
|
./crontab-backup-system.sh compare current io-crontab-sample-20250526_101558.backup
|
||||||
|
|
||||||
|
# Compare two backups from different systems
|
||||||
|
./crontab-backup-system.sh compare europa-crontab-manual-20250526_101354.backup racknerd-crontab-sample-20250526_101558.backup
|
||||||
|
```
|
||||||
|
|
||||||
|
### 🧹 Cleanup Management
|
||||||
|
|
||||||
|
```bash
|
||||||
|
# Clean up backups older than 30 days for current system
|
||||||
|
./crontab-backup-system.sh cleanup 30
|
||||||
|
|
||||||
|
# Clean up backups for specific system
|
||||||
|
./crontab-backup-system.sh cleanup 7 io
|
||||||
|
|
||||||
|
# Clean up backups for all systems
|
||||||
|
./crontab-backup-system.sh cleanup 30 all
|
||||||
|
```
|
||||||
|
|
||||||
|
## Enhanced Logging Integration
|
||||||
|
|
||||||
|
All backup operations now integrate with system logging:
|
||||||
|
|
||||||
|
### Syslog Integration
|
||||||
|
|
||||||
|
- **Tag-based logging**: Each operation uses specific syslog tags
|
||||||
|
- **Priority levels**: Different priorities for info, warnings, and errors
|
||||||
|
- **Performance monitoring**: Execution time tracking for all operations
|
||||||
|
|
||||||
|
### Example Enhanced Crontab Entries
|
||||||
|
|
||||||
|
```bash
|
||||||
|
# Plex backup with comprehensive logging
|
||||||
|
15 4 * * * /home/acedanger/shell/backup-plex.sh 2>&1 | logger -t plex-backup -p user.info
|
||||||
|
|
||||||
|
# Backup move operation with error handling
|
||||||
|
0 1 * * * /home/acedanger/shell/move-backups.sh 2>&1 | logger -t backup-move -p user.info
|
||||||
|
|
||||||
|
# Validation with performance tracking
|
||||||
|
0 7 * * * /home/acedanger/shell/validate-plex-backups.sh --fix 2>&1 | logger -t plex-validation -p user.info
|
||||||
|
```
|
||||||
|
|
||||||
|
### Log Monitoring
|
||||||
|
|
||||||
|
```bash
|
||||||
|
# View all backup-related logs
|
||||||
|
journalctl -t plex-backup -t backup-move -t plex-validation -f
|
||||||
|
|
||||||
|
# View logs for specific operation
|
||||||
|
journalctl -t plex-backup --since "1 hour ago"
|
||||||
|
|
||||||
|
# Monitor backup performance
|
||||||
|
./backup-log-monitor.sh --real-time
|
||||||
|
```
|
||||||
|
|
||||||
|
## Migration from Legacy Structure
|
||||||
|
|
||||||
|
The system automatically detects and migrates legacy backup structures:
|
||||||
|
|
||||||
|
### Automatic Migration
|
||||||
|
|
||||||
|
- **Legacy detection**: Automatically detects old `crontab-backups/archive/` structure
|
||||||
|
- **Hostname prefix**: Adds hostname prefix to existing backup files
|
||||||
|
- **Backward compatibility**: Preserves all existing backup data
|
||||||
|
- **Safe migration**: Original files remain untouched until manual cleanup
|
||||||
|
|
||||||
|
### Manual Migration
|
||||||
|
|
||||||
|
```bash
|
||||||
|
# Force migration of legacy backups
|
||||||
|
./crontab-backup-system.sh migrate
|
||||||
|
```
|
||||||
|
|
||||||
|
## Production Deployment
|
||||||
|
|
||||||
|
### System Setup
|
||||||
|
|
||||||
|
1. **Deploy script**: Copy `crontab-backup-system.sh` to each system
|
||||||
|
2. **Configure permissions**: Ensure proper read/write access to backup directories
|
||||||
|
3. **Setup automation**: Configure automated daily backups
|
||||||
|
|
||||||
|
### Automated Backup Setup
|
||||||
|
|
||||||
|
```bash
|
||||||
|
# Setup automated daily backups on each system
|
||||||
|
./crontab-backup-system.sh setup-auto
|
||||||
|
```
|
||||||
|
|
||||||
|
This adds the following entry to the system crontab:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
0 0 * * * /path/to/crontab-backup-system.sh backup auto --auto-cleanup 2>&1 | logger -t crontab-backup -p user.info
|
||||||
|
```
|
||||||
|
|
||||||
|
### Cross-System Synchronization
|
||||||
|
|
||||||
|
For distributed environments, consider setting up backup synchronization:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
# Example rsync command to sync backups from remote systems
|
||||||
|
rsync -avz europa:/home/acedanger/shell/crontab-backups/europa/ /home/acedanger/shell/crontab-backups/europa/
|
||||||
|
rsync -avz io:/home/acedanger/shell/crontab-backups/io/ /home/acedanger/shell/crontab-backups/io/
|
||||||
|
```
|
||||||
|
|
||||||
|
## Security Considerations
|
||||||
|
|
||||||
|
### File Permissions
|
||||||
|
|
||||||
|
- **Backup directories**: Restrict access to authorized users only
|
||||||
|
- **Log files**: Ensure proper log rotation and access controls
|
||||||
|
- **Remote access**: Use secure methods (SSH, rsync) for cross-system operations
|
||||||
|
|
||||||
|
### Backup Integrity
|
||||||
|
|
||||||
|
- **Validation**: Regular syntax validation of backup files
|
||||||
|
- **Checksums**: Consider adding checksum verification for critical backups
|
||||||
|
- **Retention**: Implement appropriate backup retention policies
|
||||||
|
|
||||||
|
## Advanced Use Cases
|
||||||
|
|
||||||
|
### Disaster Recovery
|
||||||
|
|
||||||
|
```bash
|
||||||
|
# Restore from specific system backup during emergency
|
||||||
|
./crontab-backup-system.sh restore io-crontab-pre-incident-20250526_101354.backup
|
||||||
|
|
||||||
|
# Compare pre-incident and post-incident configurations
|
||||||
|
./crontab-backup-system.sh compare io-crontab-pre-incident-20250526_101354.backup current
|
||||||
|
```
|
||||||
|
|
||||||
|
### Configuration Management
|
||||||
|
|
||||||
|
```bash
|
||||||
|
# Standardize crontab across multiple systems
|
||||||
|
./crontab-backup-system.sh compare europa-crontab-standard-20250526_101354.backup io-crontab-current-20250526_120000.backup
|
||||||
|
|
||||||
|
# Validate configurations before deployment
|
||||||
|
./crontab-backup-system.sh validate new-crontab-config.txt
|
||||||
|
```
|
||||||
|
|
||||||
|
### Compliance and Auditing
|
||||||
|
|
||||||
|
- **Change tracking**: Complete history of all crontab changes across systems
|
||||||
|
- **Audit trails**: System logs provide comprehensive audit information
|
||||||
|
- **Compliance reporting**: Generate reports showing backup frequency and success rates
|
||||||
|
|
||||||
|
## Monitoring and Alerting
|
||||||
|
|
||||||
|
### Health Checks
|
||||||
|
|
||||||
|
```bash
|
||||||
|
# Check backup system health
|
||||||
|
./crontab-backup-system.sh status all
|
||||||
|
|
||||||
|
# Monitor recent backup activity
|
||||||
|
./backup-log-monitor.sh --health-check
|
||||||
|
```
|
||||||
|
|
||||||
|
### Alert Integration
|
||||||
|
|
||||||
|
Consider integrating with monitoring systems:
|
||||||
|
|
||||||
|
- **Backup failures**: Alert when backups fail or are missing
|
||||||
|
- **Old backups**: Alert when systems haven't been backed up recently
|
||||||
|
- **Disk space**: Monitor backup directory disk usage
|
||||||
|
|
||||||
|
## Best Practices
|
||||||
|
|
||||||
|
1. **Regular Testing**: Periodically test backup restoration procedures
|
||||||
|
2. **Documentation**: Keep records of system configurations and backup schedules
|
||||||
|
3. **Automation**: Use automated cleanup to prevent disk space issues
|
||||||
|
4. **Monitoring**: Implement comprehensive monitoring and alerting
|
||||||
|
5. **Security**: Regularly review and update access controls
|
||||||
|
|
||||||
|
## Support and Troubleshooting
|
||||||
|
|
||||||
|
### Common Issues
|
||||||
|
|
||||||
|
- **Permission errors**: Ensure proper file permissions on backup directories
|
||||||
|
- **Missing backups**: Check automated backup cron entries
|
||||||
|
- **Syntax errors**: Use validation feature before deploying new crontabs
|
||||||
|
|
||||||
|
### Debug Mode
|
||||||
|
|
||||||
|
Enable verbose logging for troubleshooting:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
# Add debug logging to any command
|
||||||
|
./crontab-backup-system.sh status all 2>&1 | tee debug.log
|
||||||
|
```
|
||||||
|
|
||||||
|
### Log Analysis
|
||||||
|
|
||||||
|
```bash
|
||||||
|
# Analyze backup patterns
|
||||||
|
grep "SUCCESS" logs/crontab-management.log | tail -20
|
||||||
|
|
||||||
|
# Check for errors
|
||||||
|
grep "ERROR" logs/crontab-management.log | tail -10
|
||||||
|
```
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
*This multi-system crontab management solution provides robust, scalable backup management for distributed environments while maintaining simplicity and reliability.*
|
||||||
28
enhanced-crontab.txt
Normal file
28
enhanced-crontab.txt
Normal file
@@ -0,0 +1,28 @@
|
|||||||
|
# Enhanced Crontab Entries with System Logging
|
||||||
|
#
|
||||||
|
# These entries include comprehensive logging to syslog with proper tags
|
||||||
|
# and error handling for better monitoring and troubleshooting
|
||||||
|
|
||||||
|
# Move the files previously backed up at 0100
|
||||||
|
# Logs both stdout and stderr with backup-move tag
|
||||||
|
0 1 * * * /home/acedanger/shell/move-backups.sh 2>&1 | logger -t backup-move -p user.info
|
||||||
|
|
||||||
|
# Daily Plex backup at 0415 with enhanced logging
|
||||||
|
# Includes execution status and performance metrics
|
||||||
|
15 4 * * * { echo "Starting Plex backup"; /home/acedanger/shell/backup-plex.sh --non-interactive --auto-repair; echo "Plex backup completed with exit code: $?"; } 2>&1 | logger -t plex-backup -p user.info
|
||||||
|
|
||||||
|
# Daily validation at 0700 with detailed logging
|
||||||
|
# Logs validation results and any auto-fixes performed
|
||||||
|
0 7 * * * { echo "Starting Plex backup validation"; /home/acedanger/shell/validate-plex-backups.sh --fix; echo "Validation completed with exit code: $?"; } 2>&1 | logger -t plex-validation -p user.info
|
||||||
|
|
||||||
|
# Backup Immich database weekly (Mondays at 0500)
|
||||||
|
# Enhanced with proper logging and error handling
|
||||||
|
0 5 * * 1 { echo "Starting Immich database backup move"; if mv /mnt/share/media/immich/uploads/backups/immich-db-backup* /mnt/share/media/backups/immich 2>/dev/null; then echo "Immich backup move completed successfully"; else echo "No Immich backup files found or move failed"; fi; } 2>&1 | logger -t immich-backup -p user.info
|
||||||
|
|
||||||
|
# Generate detailed weekly report (Sundays at 0800)
|
||||||
|
# Comprehensive reporting with system logging
|
||||||
|
0 8 * * 0 { echo "Starting weekly Plex backup report generation"; /home/acedanger/shell/validate-plex-backups.sh --report; echo "Weekly report generation completed with exit code: $?"; } 2>&1 | logger -t plex-report -p user.info
|
||||||
|
|
||||||
|
# Optional: Add a health check entry to monitor cron jobs (every 6 hours)
|
||||||
|
# This can help detect if any of the backup processes are failing
|
||||||
|
# 0 */6 * * * { echo "Cron health check - all backup jobs scheduled"; ps aux | grep -E "(backup-plex|validate-plex|move-backups)" | grep -v grep | wc -l; } 2>&1 | logger -t cron-health -p user.info
|
||||||
246
manage-enhanced-crontab.sh
Executable file
246
manage-enhanced-crontab.sh
Executable file
@@ -0,0 +1,246 @@
|
|||||||
|
#!/bin/bash
|
||||||
|
|
||||||
|
# Enhanced Crontab Management Script
|
||||||
|
# This script helps install and manage the enhanced crontab entries with system logging
|
||||||
|
|
||||||
|
set -e
|
||||||
|
|
||||||
|
# Color codes for output
|
||||||
|
RED='\033[0;31m'
|
||||||
|
GREEN='\033[0;32m'
|
||||||
|
YELLOW='\033[1;33m'
|
||||||
|
BLUE='\033[0;34m'
|
||||||
|
CYAN='\033[0;36m'
|
||||||
|
NC='\033[0m' # No Color
|
||||||
|
|
||||||
|
SCRIPT_DIR="$(dirname "$(readlink -f "$0")")"
|
||||||
|
ENHANCED_CRONTAB_FILE="$SCRIPT_DIR/enhanced-crontab.txt"
|
||||||
|
BACKUP_CRONTAB_FILE="/tmp/crontab-backup-$(date +%Y%m%d_%H%M%S)"
|
||||||
|
|
||||||
|
log_message() {
|
||||||
|
echo -e "$(date '+%H:%M:%S') $1"
|
||||||
|
}
|
||||||
|
|
||||||
|
log_error() {
|
||||||
|
log_message "${RED}ERROR: $1${NC}"
|
||||||
|
}
|
||||||
|
|
||||||
|
log_success() {
|
||||||
|
log_message "${GREEN}SUCCESS: $1${NC}"
|
||||||
|
}
|
||||||
|
|
||||||
|
log_warning() {
|
||||||
|
log_message "${YELLOW}WARNING: $1${NC}"
|
||||||
|
}
|
||||||
|
|
||||||
|
log_info() {
|
||||||
|
log_message "${BLUE}INFO: $1${NC}"
|
||||||
|
}
|
||||||
|
|
||||||
|
backup_current_crontab() {
|
||||||
|
log_info "Backing up current root crontab to $BACKUP_CRONTAB_FILE"
|
||||||
|
if sudo crontab -l > "$BACKUP_CRONTAB_FILE" 2>/dev/null; then
|
||||||
|
log_success "Current crontab backed up successfully"
|
||||||
|
else
|
||||||
|
log_warning "No existing crontab found or backup failed"
|
||||||
|
fi
|
||||||
|
}
|
||||||
|
|
||||||
|
install_enhanced_crontab() {
|
||||||
|
log_info "Installing enhanced crontab entries"
|
||||||
|
|
||||||
|
if [ ! -f "$ENHANCED_CRONTAB_FILE" ]; then
|
||||||
|
log_error "Enhanced crontab file not found: $ENHANCED_CRONTAB_FILE"
|
||||||
|
return 1
|
||||||
|
fi
|
||||||
|
|
||||||
|
# Create a backup before making changes
|
||||||
|
if [ -f "$SCRIPT_DIR/crontab-backup-system.sh" ]; then
|
||||||
|
log_info "Creating pre-install backup"
|
||||||
|
"$SCRIPT_DIR/crontab-backup-system.sh" backup pre-install
|
||||||
|
fi
|
||||||
|
|
||||||
|
# Extract just the cron entries (skip comments and empty lines)
|
||||||
|
grep -E '^[0-9]' "$ENHANCED_CRONTAB_FILE" > /tmp/cron_entries_only.txt
|
||||||
|
|
||||||
|
# Validate the crontab syntax before installing
|
||||||
|
log_info "Validating crontab syntax"
|
||||||
|
if [ -f "$SCRIPT_DIR/crontab-backup-system.sh" ]; then
|
||||||
|
if ! "$SCRIPT_DIR/crontab-backup-system.sh" validate /tmp/cron_entries_only.txt; then
|
||||||
|
log_error "Crontab syntax validation failed"
|
||||||
|
rm -f /tmp/cron_entries_only.txt
|
||||||
|
return 1
|
||||||
|
fi
|
||||||
|
fi
|
||||||
|
|
||||||
|
if sudo crontab /tmp/cron_entries_only.txt; then
|
||||||
|
log_success "Enhanced crontab entries installed successfully"
|
||||||
|
rm -f /tmp/cron_entries_only.txt
|
||||||
|
|
||||||
|
# Create a post-install backup
|
||||||
|
if [ -f "$SCRIPT_DIR/crontab-backup-system.sh" ]; then
|
||||||
|
"$SCRIPT_DIR/crontab-backup-system.sh" backup post-install
|
||||||
|
fi
|
||||||
|
else
|
||||||
|
log_error "Failed to install enhanced crontab entries"
|
||||||
|
rm -f /tmp/cron_entries_only.txt
|
||||||
|
return 1
|
||||||
|
fi
|
||||||
|
}
|
||||||
|
|
||||||
|
show_current_crontab() {
|
||||||
|
log_info "Current root crontab entries:"
|
||||||
|
echo
|
||||||
|
sudo crontab -l 2>/dev/null || log_warning "No crontab entries found"
|
||||||
|
echo
|
||||||
|
}
|
||||||
|
|
||||||
|
show_log_monitoring_commands() {
|
||||||
|
log_info "Commands to monitor backup logs:"
|
||||||
|
echo
|
||||||
|
echo -e "${CYAN}# Use the enhanced backup log monitor:${NC}"
|
||||||
|
echo "./backup-log-monitor.sh monitor # Real-time monitoring"
|
||||||
|
echo "./backup-log-monitor.sh recent 24 # Last 24 hours"
|
||||||
|
echo "./backup-log-monitor.sh health # System health check"
|
||||||
|
echo "./backup-log-monitor.sh report 7 # Weekly report"
|
||||||
|
echo
|
||||||
|
echo -e "${CYAN}# Direct journalctl commands:${NC}"
|
||||||
|
echo "sudo journalctl -f -t plex-backup -t backup-move -t plex-validation -t immich-backup -t plex-report"
|
||||||
|
echo
|
||||||
|
echo -e "${CYAN}# View logs from the last 24 hours:${NC}"
|
||||||
|
echo "sudo journalctl --since '24 hours ago' -t plex-backup -t backup-move -t plex-validation -t immich-backup -t plex-report"
|
||||||
|
echo
|
||||||
|
echo -e "${CYAN}# View only error logs:${NC}"
|
||||||
|
echo "sudo journalctl --priority=err -t plex-backup -t backup-move -t plex-validation -t immich-backup -t plex-report"
|
||||||
|
echo
|
||||||
|
echo -e "${CYAN}# View logs for a specific backup type (e.g., plex-backup):${NC}"
|
||||||
|
echo "sudo journalctl -t plex-backup --since '1 week ago'"
|
||||||
|
echo
|
||||||
|
}
|
||||||
|
|
||||||
|
setup_logrotate() {
|
||||||
|
log_info "Setting up logrotate for backup logs"
|
||||||
|
|
||||||
|
cat > /tmp/backup-logs-logrotate << 'EOF'
|
||||||
|
# Logrotate configuration for backup logs
|
||||||
|
# This ensures syslog doesn't grow too large with backup logs
|
||||||
|
|
||||||
|
/var/log/syslog {
|
||||||
|
daily
|
||||||
|
missingok
|
||||||
|
rotate 7
|
||||||
|
compress
|
||||||
|
delaycompress
|
||||||
|
notifempty
|
||||||
|
postrotate
|
||||||
|
/usr/lib/rsyslog/rsyslog-rotate
|
||||||
|
endscript
|
||||||
|
}
|
||||||
|
EOF
|
||||||
|
|
||||||
|
if sudo cp /tmp/backup-logs-logrotate /etc/logrotate.d/backup-logs; then
|
||||||
|
log_success "Logrotate configuration installed"
|
||||||
|
else
|
||||||
|
log_warning "Failed to install logrotate configuration"
|
||||||
|
fi
|
||||||
|
|
||||||
|
rm -f /tmp/backup-logs-logrotate
|
||||||
|
}
|
||||||
|
|
||||||
|
verify_scripts_exist() {
|
||||||
|
log_info "Verifying all backup scripts exist and are executable"
|
||||||
|
|
||||||
|
local scripts=(
|
||||||
|
"/home/acedanger/shell/move-backups.sh"
|
||||||
|
"/home/acedanger/shell/backup-plex.sh"
|
||||||
|
"/home/acedanger/shell/validate-plex-backups.sh"
|
||||||
|
)
|
||||||
|
|
||||||
|
local all_good=true
|
||||||
|
|
||||||
|
for script in "${scripts[@]}"; do
|
||||||
|
if [ -f "$script" ]; then
|
||||||
|
if [ -x "$script" ]; then
|
||||||
|
log_success "✓ $script exists and is executable"
|
||||||
|
else
|
||||||
|
log_warning "! $script exists but is not executable"
|
||||||
|
sudo chmod +x "$script"
|
||||||
|
log_success "✓ Made $script executable"
|
||||||
|
fi
|
||||||
|
else
|
||||||
|
log_error "✗ $script not found"
|
||||||
|
all_good=false
|
||||||
|
fi
|
||||||
|
done
|
||||||
|
|
||||||
|
if $all_good; then
|
||||||
|
log_success "All backup scripts are ready"
|
||||||
|
else
|
||||||
|
log_error "Some backup scripts are missing"
|
||||||
|
return 1
|
||||||
|
fi
|
||||||
|
}
|
||||||
|
|
||||||
|
show_usage() {
|
||||||
|
echo "Enhanced Crontab Management Script"
|
||||||
|
echo
|
||||||
|
echo "Usage: $0 [OPTION]"
|
||||||
|
echo
|
||||||
|
echo "Options:"
|
||||||
|
echo " install Install the enhanced crontab entries with backup system"
|
||||||
|
echo " show Show current crontab entries"
|
||||||
|
echo " backup Backup current crontab only"
|
||||||
|
echo " verify Verify all scripts exist and are executable"
|
||||||
|
echo " monitor Show log monitoring commands"
|
||||||
|
echo " logrotate Setup logrotate for backup logs"
|
||||||
|
echo " status Show backup system health status"
|
||||||
|
echo " help Show this help message"
|
||||||
|
echo
|
||||||
|
echo "Additional Tools:"
|
||||||
|
echo " ./crontab-backup-system.sh Comprehensive crontab backup management"
|
||||||
|
echo " ./backup-log-monitor.sh Advanced backup log monitoring"
|
||||||
|
echo
|
||||||
|
}
|
||||||
|
|
||||||
|
case "${1:-help}" in
|
||||||
|
install)
|
||||||
|
verify_scripts_exist
|
||||||
|
backup_current_crontab
|
||||||
|
install_enhanced_crontab
|
||||||
|
show_current_crontab
|
||||||
|
setup_logrotate
|
||||||
|
show_log_monitoring_commands
|
||||||
|
|
||||||
|
# Setup automated backup system
|
||||||
|
if [ -f "$SCRIPT_DIR/crontab-backup-system.sh" ]; then
|
||||||
|
log_info "Setting up automated crontab backup system"
|
||||||
|
"$SCRIPT_DIR/crontab-backup-system.sh" setup-auto
|
||||||
|
fi
|
||||||
|
;;
|
||||||
|
show)
|
||||||
|
show_current_crontab
|
||||||
|
;;
|
||||||
|
backup)
|
||||||
|
backup_current_crontab
|
||||||
|
;;
|
||||||
|
verify)
|
||||||
|
verify_scripts_exist
|
||||||
|
;;
|
||||||
|
monitor)
|
||||||
|
show_log_monitoring_commands
|
||||||
|
;;
|
||||||
|
logrotate)
|
||||||
|
setup_logrotate
|
||||||
|
;;
|
||||||
|
status)
|
||||||
|
if [ -f "$SCRIPT_DIR/backup-log-monitor.sh" ]; then
|
||||||
|
"$SCRIPT_DIR/backup-log-monitor.sh" health
|
||||||
|
else
|
||||||
|
log_warning "Backup log monitor not found, showing basic status"
|
||||||
|
show_current_crontab
|
||||||
|
fi
|
||||||
|
;;
|
||||||
|
help|*)
|
||||||
|
show_usage
|
||||||
|
;;
|
||||||
|
esac
|
||||||
Reference in New Issue
Block a user