From 94dc0e6c5bcaa668ab65efafb9c21f3b60272150 Mon Sep 17 00:00:00 2001 From: Peter Wood Date: Tue, 3 Jun 2025 08:59:11 -0400 Subject: [PATCH] feat: Add Plex scripts documentation and enhance monitoring script with detailed checks and recommendations --- .github/copilot-instructions.md | 229 ++++++++++++++++++++++++++++++++ plex/monitor-plex-backup.sh | 97 +++++++------- 2 files changed, 278 insertions(+), 48 deletions(-) diff --git a/.github/copilot-instructions.md b/.github/copilot-instructions.md index 1a56ddc..eae204b 100644 --- a/.github/copilot-instructions.md +++ b/.github/copilot-instructions.md @@ -42,6 +42,17 @@ This repository contains: - **docs/docker-bootstrap-testing-framework.md**: Detailed documentation for the Docker-based bootstrap validation framework - **dotfiles/README.md**: Documentation for dotfiles setup and usage +### Plex Scripts + +- **plex.sh**: Main Plex Media Server management script +- **backup-plex.sh**: Automated backup system for Plex databases and configurations +- **monitor-plex-backup.sh**: Monitoring and reporting for backup operations +- **restore-plex.sh**: Database and configuration restoration utilities +- **validate-plex-backups.sh**: Backup integrity verification system +- **test-plex-backup.sh**: Testing framework for backup operations +- **integration-test-plex.sh**: End-to-end integration testing for Plex services +- **plex-recent-additions.sh**: Recent media additions reporting + ## Style Guidelines When suggesting code or modifications: @@ -159,3 +170,221 @@ For contributors and Copilot suggestions: - Run tests before submitting changes - Document what was changed and why - Consider both Ubuntu, Debian, and Fedora compatibility + +## Automatic Code Validation and Critical Error Checking + +When generating or modifying shell scripts, GitHub Copilot must automatically validate code and check for critical errors before suggesting changes. + +### Mandatory Validation Checks + +1. **Syntax Validation**: + - Always verify proper `if`/`fi`, `for`/`done`, `while`/`done` matching + - Check for balanced brackets: `[`, `]`, `[[`, `]]` + - Ensure proper function definition syntax: `function_name() { ... }` + - Validate case statement structure: `case`/`esac` matching + - Check for missing quotes around variables in conditions + +2. **Control Flow Validation**: + - Verify `else` statements have corresponding `if` statements + - Check that nested conditionals are properly indented and closed + - Ensure `break` and `continue` are only used within loops + - Validate that functions return appropriate exit codes + +3. **Variable and Quote Validation**: + - Always quote variables to prevent word splitting: `"$VARIABLE"` + - Check for undefined variables being used + - Validate array syntax and indexing + - Ensure proper escaping in strings containing special characters + +### Critical Logic Error Patterns to Check + +Based on analysis of plex scripts, watch for these common issues: + +1. **Service Management Logic**: + ```bash + # CORRECT: Check service status before operations + if sudo systemctl is-active --quiet plexmediaserver; then + sudo systemctl stop plexmediaserver + fi + + # INCORRECT: Assuming service state without checking + sudo systemctl stop plexmediaserver # May fail if already stopped + ``` + +2. **File Operation Safety**: + ```bash + # CORRECT: Check file existence and permissions + if [[ -f "$BACKUP_FILE" && -r "$BACKUP_FILE" ]]; then + # Process file + else + echo "Error: Backup file not found or not readable" + exit 1 + fi + + # INCORRECT: Operating on files without validation + tar -xzf "$BACKUP_FILE" # May fail silently + ``` + +3. **Directory Operations**: + ```bash + # CORRECT: Verify directory creation and permissions + if ! mkdir -p "$BACKUP_DIR"; then + echo "Error: Failed to create backup directory" + exit 1 + fi + + # Set proper permissions + chmod 755 "$BACKUP_DIR" + + # INCORRECT: Assuming directory operations succeed + mkdir -p "$BACKUP_DIR" + cd "$BACKUP_DIR" # May fail if mkdir failed + ``` + +4. **Database Operations**: + ```bash + # CORRECT: Validate database integrity before and after operations + if ! sqlite3 "$DB_FILE" "PRAGMA integrity_check;" | grep -q "ok"; then + echo "Error: Database integrity check failed" + exit 1 + fi + + # INCORRECT: Operating on database without validation + sqlite3 "$DB_FILE" ".backup backup.db" # May corrupt if DB is damaged + ``` + +5. **Parallel Processing Safety**: + ```bash + # CORRECT: Proper parallel job management with wait + for file in "${files[@]}"; do + process_file "$file" & + ((++job_count)) + + # Limit concurrent jobs + if (( job_count >= MAX_JOBS )); then + wait + job_count=0 + fi + done + wait # Wait for remaining jobs + + # INCORRECT: Uncontrolled parallel execution + for file in "${files[@]}"; do + process_file "$file" & # May overwhelm system + done + ``` + +### Error Handling Patterns + +1. **Function Error Handling**: + ```bash + # CORRECT: Proper function with error handling + backup_database() { + local db_file="$1" + local backup_file="$2" + + if [[ -z "$db_file" || -z "$backup_file" ]]; then + echo "Error: Missing required parameters" + return 1 + fi + + if [[ ! -f "$db_file" ]]; then + echo "Error: Database file not found: $db_file" + return 1 + fi + + if ! sqlite3 "$db_file" ".backup $backup_file"; then + echo "Error: Database backup failed" + return 1 + fi + + return 0 + } + ``` + +2. **Cleanup on Error**: + ```bash + # CORRECT: Cleanup trap for error handling + cleanup() { + if [[ -n "$TEMP_DIR" && -d "$TEMP_DIR" ]]; then + rm -rf "$TEMP_DIR" + fi + + if [[ "$SERVICE_STOPPED" == "true" ]]; then + sudo systemctl start plexmediaserver + fi + } + + trap cleanup EXIT ERR + ``` + +### Validation Checklist for Code Generation + +Before suggesting any shell script code, verify: + +- [ ] All `if` statements have matching `fi` +- [ ] All `for`/`while` loops have matching `done` +- [ ] All `case` statements have matching `esac` +- [ ] Functions are properly defined with `()` and `{}` +- [ ] Variables are quoted in conditions and expansions +- [ ] File operations check for existence and permissions +- [ ] Service operations verify current state before changes +- [ ] Database operations include integrity checks +- [ ] Parallel operations are properly managed with job limits +- [ ] Error handling includes cleanup and restoration +- [ ] Exit codes are properly set and checked +- [ ] Temporary files and directories are cleaned up + +### Testing Integration + +When modifying scripts: + +1. **Syntax Check**: Always run `bash -n script.sh` validation +2. **Logic Testing**: Test with various input conditions +3. **Error Scenarios**: Test failure modes and recovery +4. **Integration Testing**: Verify interaction with system services +5. **Permission Testing**: Test with different user privileges + +### Common Anti-Patterns to Avoid + +1. **Unmatched Control Structures**: + ```bash + # WRONG: Missing fi + if [[ condition ]]; then + do_something + # Missing fi here + ``` + +2. **Unsafe Variable Expansion**: + ```bash + # WRONG: Unquoted variables + if [ $VAR = "value" ]; then # Will break if VAR contains spaces + + # CORRECT: + if [[ "$VAR" = "value" ]]; then + ``` + +3. **Ignoring Command Failures**: + ```bash + # WRONG: Not checking critical command results + sudo systemctl stop plexmediaserver + # Continue without knowing if stop succeeded + + # CORRECT: + if ! sudo systemctl stop plexmediaserver; then + echo "Error: Failed to stop Plex service" + exit 1 + fi + ``` + +4. **Race Conditions in Service Management**: + ```bash + # WRONG: Race condition + sudo systemctl stop plexmediaserver + sudo systemctl start plexmediaserver # May start before stop completes + + # CORRECT: + sudo systemctl stop plexmediaserver + sleep 2 # Allow time for clean shutdown + sudo systemctl start plexmediaserver + ``` diff --git a/plex/monitor-plex-backup.sh b/plex/monitor-plex-backup.sh index 6502106..a7e410b 100755 --- a/plex/monitor-plex-backup.sh +++ b/plex/monitor-plex-backup.sh @@ -75,17 +75,17 @@ clear_screen() { find_most_recent_log() { local log_pattern="$1" local recent_log="" - + # Check local logs first (preferred) if [ -d "$LOCAL_LOG_ROOT" ]; then recent_log=$(find "$LOCAL_LOG_ROOT" -name "$log_pattern" -type f 2>/dev/null | sort | tail -1) fi - + # If no local log found, check shared location as fallback if [ -z "$recent_log" ] && [ -d "$SHARED_LOG_ROOT" ]; then recent_log=$(find "$SHARED_LOG_ROOT" -name "$log_pattern" -type f 2>/dev/null | sort | tail -1) fi - + echo "$recent_log" } @@ -102,41 +102,41 @@ show_header() { check_system_status() { echo -e "${BLUE}📊 SYSTEM STATUS${NC}" echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" - + # Check Plex service if systemctl is-active --quiet plexmediaserver; then log_status "OK" "Plex Media Server is running" else log_status "ERROR" "Plex Media Server is not running" fi - + # Check backup script if [ -f "$SCRIPT_DIR/backup-plex.sh" ]; then log_status "OK" "Backup script is present" else log_status "ERROR" "Backup script not found" fi - + # Check directories if [ -d "$BACKUP_ROOT" ]; then log_status "OK" "Backup directory exists" else log_status "ERROR" "Backup directory missing: $BACKUP_ROOT" fi - + # Check log directories (prioritize local, show shared as secondary) if [ -d "$LOCAL_LOG_ROOT" ]; then log_status "OK" "Local log directory exists" else log_status "WARN" "Local log directory missing: $LOCAL_LOG_ROOT" fi - + if [ -d "$SHARED_LOG_ROOT" ]; then log_status "INFO" "Shared log directory accessible" else log_status "WARN" "Shared log directory missing: $SHARED_LOG_ROOT" fi - + # Check dependencies for cmd in jq sqlite3 curl; do if command -v "$cmd" >/dev/null 2>&1; then @@ -145,7 +145,7 @@ check_system_status() { log_status "WARN" "$cmd is not installed" fi done - + echo } @@ -153,16 +153,16 @@ check_system_status() { check_backup_status() { echo -e "${BLUE}💾 BACKUP STATUS${NC}" echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" - + # Count total backups local backup_count=0 if [ -d "$BACKUP_ROOT" ]; then backup_count=$(find "$BACKUP_ROOT" -maxdepth 1 -type f -name "plex-backup-*.tar.gz" 2>/dev/null | wc -l) fi - + if [ "$backup_count" -gt 0 ]; then log_status "OK" "Total backups: $backup_count" - + # Find latest backup local latest_backup=$(find "$BACKUP_ROOT" -maxdepth 1 -type f -name "plex-backup-*.tar.gz" 2>/dev/null | sort | tail -1) if [ -n "$latest_backup" ]; then @@ -171,7 +171,7 @@ check_backup_status() { local backup_date=$(echo "$backup_filename" | sed 's/plex-backup-//' | sed 's/_.*$//') local readable_date=$(date -d "${backup_date:0:4}-${backup_date:4:2}-${backup_date:6:2}" '+%B %d, %Y' 2>/dev/null || echo "Invalid date") local backup_age_days=$(( ($(date +%s) - $(date -d "${backup_date:0:4}-${backup_date:4:2}-${backup_date:6:2}" +%s 2>/dev/null || echo "0")) / 86400 )) - + if [ "$backup_age_days" -le 1 ]; then log_status "OK" "Latest backup: $readable_date ($backup_age_days days ago)" elif [ "$backup_age_days" -le 7 ]; then @@ -179,11 +179,11 @@ check_backup_status() { else log_status "ERROR" "Latest backup: $readable_date ($backup_age_days days ago)" fi - + # Check backup size local backup_size=$(du -sh "$latest_backup" 2>/dev/null | cut -f1) log_status "INFO" "Latest backup size: $backup_size" - + # Check backup contents (via tar listing) local file_count=$(tar -tzf "$latest_backup" 2>/dev/null | wc -l) log_status "INFO" "Files in latest backup: $file_count" @@ -191,16 +191,16 @@ check_backup_status() { else log_status "WARN" "No backups found" fi - + # Disk usage if [ -d "$BACKUP_ROOT" ]; then local total_backup_size=$(du -sh "$BACKUP_ROOT" 2>/dev/null | cut -f1) local available_space=$(df -h "$BACKUP_ROOT" 2>/dev/null | awk 'NR==2 {print $4}') local used_percentage=$(df "$BACKUP_ROOT" 2>/dev/null | awk 'NR==2 {print $5}' | sed 's/%//') - + log_status "INFO" "Total backup storage: $total_backup_size" log_status "INFO" "Available space: $available_space" - + if [ -n "$used_percentage" ]; then if [ "$used_percentage" -lt 80 ]; then log_status "OK" "Disk usage: $used_percentage%" @@ -211,7 +211,7 @@ check_backup_status() { fi fi fi - + echo } @@ -219,21 +219,21 @@ check_backup_status() { show_performance_metrics() { echo -e "${BLUE}⚡ PERFORMANCE METRICS${NC}" echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" - + if [ -f "$PERFORMANCE_LOG_FILE" ]; then log_status "OK" "Performance log found" - + # Recent operations local recent_count=$(jq length "$PERFORMANCE_LOG_FILE" 2>/dev/null || echo "0") log_status "INFO" "Total logged operations: $recent_count" - + if [ "$recent_count" -gt 0 ]; then # Average times for different operations local avg_backup=$(jq '[.[] | select(.operation == "full_backup") | .duration_seconds] | if length > 0 then add/length else 0 end' "$PERFORMANCE_LOG_FILE" 2>/dev/null || echo "0") local avg_verification=$(jq '[.[] | select(.operation == "verification") | .duration_seconds] | if length > 0 then add/length else 0 end' "$PERFORMANCE_LOG_FILE" 2>/dev/null || echo "0") local avg_service_stop=$(jq '[.[] | select(.operation == "service_stop") | .duration_seconds] | if length > 0 then add/length else 0 end' "$PERFORMANCE_LOG_FILE" 2>/dev/null || echo "0") local avg_service_start=$(jq '[.[] | select(.operation == "service_start") | .duration_seconds] | if length > 0 then add/length else 0 end' "$PERFORMANCE_LOG_FILE" 2>/dev/null || echo "0") - + if [ "$avg_backup" != "0" ] && [ "$avg_backup" != "null" ]; then log_status "INFO" "Average backup time: ${avg_backup}s" fi @@ -246,7 +246,7 @@ show_performance_metrics() { if [ "$avg_service_start" != "0" ] && [ "$avg_service_start" != "null" ]; then log_status "INFO" "Average service start time: ${avg_service_start}s" fi - + # Last 3 operations echo -e "${YELLOW}Recent Operations:${NC}" jq -r '.[-3:] | .[] | " \(.timestamp): \(.operation) (\(.duration_seconds)s)"' "$PERFORMANCE_LOG_FILE" 2>/dev/null | sed 's/T/ /' | sed 's/+.*$//' || echo " No recent operations" @@ -254,7 +254,7 @@ show_performance_metrics() { else log_status "WARN" "Performance log not found (no backups run yet)" fi - + echo } @@ -262,7 +262,7 @@ show_performance_metrics() { show_recent_activity() { echo -e "${BLUE}📋 RECENT ACTIVITY${NC}" echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" - + # Check recent log files local recent_log=$(find_most_recent_log "plex-backup-*.log") if [ -n "$recent_log" ]; then @@ -274,21 +274,22 @@ show_recent_activity() { log_location=" (shared)" fi log_status "INFO" "Most recent log: $log_date$log_location" - + # Check for errors in recent log local error_count=$(grep -c "ERROR:" "$recent_log" 2>/dev/null || echo "0") local warning_count=$(grep -c "WARNING:" "$recent_log" 2>/dev/null || echo "0") - - if [ "$error_count" -eq 0 ] && [ "$warning_count" -eq 0 ]; then - log_status "OK" "No errors or warnings in recent log" - elif [ "$error_count" -eq 0 ]; then - log_status "WARN" "$warning_count warnings in recent log" - else - log_status "ERROR" "$error_count errors, $warning_count warnings in recent log" - fi + + if [ "$error_count" -eq 0 ] && [ "$warning_count" -eq 0 ]; then + log_status "OK" "No errors or warnings in recent log" + elif [ "$error_count" -eq 0 ]; then + log_status "WARN" "$warning_count warnings in recent log" + else + log_status "ERROR" "$error_count errors, $warning_count warnings in recent log" fi + else + log_status "WARN" "No recent logs found" fi - + echo } @@ -296,7 +297,7 @@ show_recent_activity() { show_scheduling_status() { echo -e "${BLUE}⏰ SCHEDULING STATUS${NC}" echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" - + # Check cron jobs local cron_jobs=0 if crontab -l 2>/dev/null | grep -q "backup-plex"; then @@ -309,7 +310,7 @@ show_scheduling_status() { else log_status "WARN" "No cron jobs found for backup-plex" fi - + # Check systemd timers if systemctl list-timers --all 2>/dev/null | grep -q "plex-backup"; then log_status "OK" "Systemd timer configured" @@ -326,7 +327,7 @@ show_scheduling_status() { else log_status "INFO" "No systemd timer configured" fi - + echo } @@ -334,9 +335,9 @@ show_scheduling_status() { show_recommendations() { echo -e "${BLUE}💡 RECOMMENDATIONS${NC}" echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" - + local recommendations=() - + # Check backup age if [ -d "$BACKUP_ROOT" ]; then local latest_backup=$(find "$BACKUP_ROOT" -maxdepth 1 -type f -name "plex-backup-*.tar.gz" 2>/dev/null | sort | tail -1) @@ -352,7 +353,7 @@ show_recommendations() { recommendations+=("No backups found - run initial backup with: sudo ./backup-plex.sh") fi fi - + # Check scheduling local cron_jobs=0 if crontab -l 2>/dev/null | grep -q "backup-plex"; then @@ -361,7 +362,7 @@ show_recommendations() { if [ "$cron_jobs" -eq 0 ] && ! systemctl list-timers --all 2>/dev/null | grep -q "plex-backup"; then recommendations+=("Set up automated backup scheduling with cron or systemd timer") fi - + # Check disk space if [ -d "$BACKUP_ROOT" ]; then local used_percentage=$(df "$BACKUP_ROOT" 2>/dev/null | awk 'NR==2 {print $5}' | sed 's/%//') @@ -369,12 +370,12 @@ show_recommendations() { recommendations+=("Backup disk usage is high ($used_percentage%) - consider cleaning old backups") fi fi - + # Check dependencies if ! command -v jq >/dev/null 2>&1; then recommendations+=("Install jq for enhanced performance monitoring: sudo apt install jq") fi - + # Show recommendations if [ ${#recommendations[@]} -eq 0 ]; then log_status "OK" "No immediate recommendations - system looks healthy!" @@ -383,7 +384,7 @@ show_recommendations() { log_status "INFO" "$rec" done fi - + echo } @@ -419,7 +420,7 @@ main() { echo "Error: Invalid refresh interval. Must be a positive integer." exit 1 fi - + # Continuous monitoring while true; do show_dashboard