clean up log files with ANSI color codes

cleanup-log-ansi.sh

@@ -0,0 +1,347 @@
+#!/bin/bash
+
+# Log ANSI Color Code Cleanup Utility
+# This script removes ANSI color codes from log files to improve readability
+# and reduce file sizes when viewing logs in editors or processing them with tools
+
+set -e
+
+# Color codes for output
+RED='\033[0;31m'
+GREEN='\033[0;32m'
+YELLOW='\033[1;33m'
+BLUE='\033[0;34m'
+CYAN='\033[0;36m'
+NC='\033[0m' # No Color
+
+SCRIPT_DIR="$(dirname "$(readlink -f "$0")")"
+BACKUP_SUFFIX=".ansi-backup"
+
+# Function to display usage
+show_usage() {
+    echo -e "${CYAN}Log ANSI Cleanup Utility${NC}"
+    echo
+    echo "Usage: $0 [OPTIONS] [FILE|DIRECTORY]..."
+    echo
+    echo "Options:"
+    echo "  -h, --help           Show this help message"
+    echo "  -r, --recursive      Process directories recursively"
+    echo "  -b, --backup         Create backup files before cleaning (recommended)"
+    echo "  -n, --no-backup      Don't create backup files (use with caution)"
+    echo "  -v, --verbose        Show detailed output"
+    echo "  -d, --dry-run        Show what would be done without making changes"
+    echo "  -f, --filter PATTERN Only process files matching pattern (e.g., '*.log')"
+    echo "  -a, --auto-discover  Find common log files automatically"
+    echo
+    echo "Examples:"
+    echo "  $0 --backup /var/log/app.log"
+    echo "  $0 --recursive --backup /var/log/"
+    echo "  $0 --auto-discover --backup"
+    echo "  $0 --filter '*.log' --recursive /home/user/logs/"
+    echo "  $0 --dry-run --auto-discover"
+    echo
+    echo "Common log locations checked:"
+    echo "  - ~/shell/logs/"
+    echo "  - ~/shell/crontab/logs/"
+    echo "  - ~/shell/plex/logs/"
+    echo "  - ~/shell/jellyfin/logs/"
+    echo "  - ~/shell/*/logs/ (any subdirectory with logs folder, excluding .git)"
+}
+
+# Function to log messages
+log_message() {
+    local level="$1"
+    local message="$2"
+    local color=""
+    
+    case "$level" in
+        "INFO")  color="$BLUE" ;;
+        "SUCCESS") color="$GREEN" ;;
+        "WARNING") color="$YELLOW" ;;
+        "ERROR") color="$RED" ;;
+        *) color="$NC" ;;
+    esac
+    
+    if [[ "$VERBOSE" == "true" || "$level" != "INFO" ]]; then
+        echo -e "${color}${level}: ${message}${NC}"
+    fi
+}
+
+# Function to check if file contains ANSI codes
+has_ansi_codes() {
+    local file="$1"
+    # Check for both literal \033 and actual escape sequences
+    grep -q '\\033\[[0-9;]*m\|\x1b\[[0-9;]*m' "$file" 2>/dev/null
+}
+
+# Function to get file size in human readable format
+get_file_size() {
+    local file="$1"
+    if [[ -f "$file" ]]; then
+        du -h "$file" | cut -f1
+    else
+        echo "0B"
+    fi
+}
+
+# Function to clean ANSI codes from a file
+clean_file() {
+    local file="$1"
+    local backup_file="${file}${BACKUP_SUFFIX}"
+    local temp_file
+    temp_file=$(mktemp)
+    
+    if [[ ! -f "$file" ]]; then
+        log_message "ERROR" "File not found: $file"
+        return 1
+    fi
+    
+    if [[ ! -r "$file" ]]; then
+        log_message "ERROR" "Cannot read file: $file"
+        return 1
+    fi
+    
+    # Check if file has ANSI codes
+    if ! has_ansi_codes "$file"; then
+        log_message "INFO" "No ANSI codes found in: $file"
+        return 0
+    fi
+    
+    local original_size
+    original_size=$(get_file_size "$file")
+    
+    if [[ "$DRY_RUN" == "true" ]]; then
+        log_message "INFO" "Would clean ANSI codes from: $file (${original_size})"
+        return 0
+    fi
+    
+    # Create backup if requested
+    if [[ "$CREATE_BACKUP" == "true" ]]; then
+        if ! cp "$file" "$backup_file"; then
+            log_message "ERROR" "Failed to create backup: $backup_file"
+            return 1
+        fi
+        log_message "INFO" "Backup created: $backup_file"
+    fi
+    
+    # Clean ANSI codes using multiple patterns to be thorough
+    if sed -e 's/\\033\[[0-9;]*m//g' \
+           -e 's/\x1b\[[0-9;]*m//g' \
+           -e 's/\033\[[0-9;]*m//g' \
+           "$file" > "$temp_file"; then
+        
+        # Verify the temp file was created successfully
+        if [[ -s "$temp_file" || ! -s "$file" ]]; then
+            if mv "$temp_file" "$file"; then
+                local new_size
+                new_size=$(get_file_size "$file")
+                log_message "SUCCESS" "Cleaned: $file (${original_size} → ${new_size})"
+                ((FILES_CLEANED++))
+            else
+                log_message "ERROR" "Failed to replace original file: $file"
+                rm -f "$temp_file"
+                return 1
+            fi
+        else
+            log_message "ERROR" "Cleaning resulted in empty file: $file"
+            rm -f "$temp_file"
+            return 1
+        fi
+    else
+        log_message "ERROR" "Failed to clean ANSI codes from: $file"
+        rm -f "$temp_file"
+        return 1
+    fi
+}
+
+# Function to find common log files automatically
+auto_discover_logs() {
+    local shell_dir="$HOME/shell"
+    local search_paths=(
+        "$shell_dir/logs"
+        "$shell_dir/crontab/logs"
+        "$shell_dir/plex/logs"
+        "$shell_dir/jellyfin/logs"
+    )
+    
+    # Add any other log directories found in ~/shell (excluding .git/logs)
+    while IFS= read -r -d '' dir; do
+        # Skip .git/logs and already included directories
+        if [[ "$dir" != *"/.git/logs" ]] && 
+           [[ "$dir" != "$shell_dir/logs" ]] && 
+           [[ "$dir" != "$shell_dir/crontab/logs" ]] && 
+           [[ "$dir" != "$shell_dir/plex/logs" ]] && 
+           [[ "$dir" != "$shell_dir/jellyfin/logs" ]]; then
+            search_paths+=("$dir")
+        fi
+    done < <(find "$shell_dir" -type d -name "logs" -print0 2>/dev/null)
+    
+    local found_files=()
+    
+    for path in "${search_paths[@]}"; do
+        if [[ -d "$path" && -r "$path" ]]; then
+            # Find log files with common extensions
+            while IFS= read -r -d '' file; do
+                if [[ -f "$file" && -r "$file" ]]; then
+                    found_files+=("$file")
+                fi
+            done < <(find "$path" -maxdepth 2 \( -name "*.log" -o -name "*.out" -o -name "*.err" \) -type f -print0 2>/dev/null)
+        fi
+    done
+    
+    if [[ ${#found_files[@]} -eq 0 ]]; then
+        return 1
+    fi
+    
+    for file in "${found_files[@]}"; do
+        echo "$file"
+    done
+}
+
+# Function to process a directory
+process_directory() {
+    local dir="$1"
+    local files_found=0
+    
+    if [[ ! -d "$dir" ]]; then
+        log_message "ERROR" "Directory not found: $dir"
+        return 1
+    fi
+    
+    local find_args=("$dir")
+    
+    if [[ "$RECURSIVE" != "true" ]]; then
+        find_args+=("-maxdepth" "1")
+    fi
+    
+    find_args+=("-type" "f")
+    
+    if [[ -n "$FILE_PATTERN" ]]; then
+        find_args+=("-name" "$FILE_PATTERN")
+    fi
+    
+    while IFS= read -r -d '' file; do
+        clean_file "$file"
+        ((files_found++))
+    done < <(find "${find_args[@]}" -print0 2>/dev/null)
+    
+    if [[ $files_found -eq 0 ]]; then
+        log_message "WARNING" "No files found in directory: $dir"
+    fi
+}
+
+# Initialize variables
+RECURSIVE=false
+CREATE_BACKUP=true
+VERBOSE=false
+DRY_RUN=false
+FILE_PATTERN=""
+AUTO_DISCOVER=false
+FILES_CLEANED=0
+
+# Parse command line arguments
+while [[ $# -gt 0 ]]; do
+    case $1 in
+        -h|--help)
+            show_usage
+            exit 0
+            ;;
+        -r|--recursive)
+            RECURSIVE=true
+            shift
+            ;;
+        -b|--backup)
+            CREATE_BACKUP=true
+            shift
+            ;;
+        -n|--no-backup)
+            CREATE_BACKUP=false
+            shift
+            ;;
+        -v|--verbose)
+            VERBOSE=true
+            shift
+            ;;
+        -d|--dry-run)
+            DRY_RUN=true
+            VERBOSE=true
+            shift
+            ;;
+        -f|--filter)
+            FILE_PATTERN="$2"
+            shift 2
+            ;;
+        -a|--auto-discover)
+            AUTO_DISCOVER=true
+            shift
+            ;;
+        -*)
+            echo "Unknown option: $1"
+            show_usage
+            exit 1
+            ;;
+        *)
+            break
+            ;;
+    esac
+done
+
+# Main execution
+log_message "INFO" "Starting ANSI cleanup utility"
+
+if [[ "$DRY_RUN" == "true" ]]; then
+    log_message "WARNING" "DRY RUN MODE - No files will be modified"
+fi
+
+if [[ "$AUTO_DISCOVER" == "true" ]]; then
+    log_message "INFO" "Auto-discovering log files..."
+    
+    discovered_files=()
+    while IFS= read -r file; do
+        if [[ -f "$file" ]]; then
+            discovered_files+=("$file")
+        fi
+    done < <(auto_discover_logs)
+    
+    if [[ ${#discovered_files[@]} -eq 0 ]]; then
+        log_message "WARNING" "No log files discovered in common locations"
+        log_message "INFO" "Try specifying files or directories manually"
+        exit 0
+    fi
+    
+    log_message "INFO" "Found ${#discovered_files[@]} log files"
+    for file in "${discovered_files[@]}"; do
+        clean_file "$file"
+    done
+elif [[ $# -eq 0 ]]; then
+    log_message "ERROR" "No files or directories specified"
+    show_usage
+    exit 1
+else
+    # Process specified files/directories
+    for target in "$@"; do
+        if [[ -f "$target" ]]; then
+            clean_file "$target"
+        elif [[ -d "$target" ]]; then
+            process_directory "$target"
+        else
+            log_message "ERROR" "File or directory not found: $target"
+        fi
+    done
+fi
+
+# Summary
+if [[ "$DRY_RUN" != "true" ]]; then
+    if [[ $FILES_CLEANED -gt 0 ]]; then
+        log_message "SUCCESS" "Cleanup completed. $FILES_CLEANED files processed."
+        
+        if [[ "$CREATE_BACKUP" == "true" ]]; then
+            log_message "INFO" "Backup files created with suffix: $BACKUP_SUFFIX"
+            log_message "INFO" "Remove backup files when satisfied: rm -f *$BACKUP_SUFFIX"
+        fi
+    else
+        log_message "INFO" "No files needed cleaning."
+    fi
+else
+    log_message "INFO" "Dry run completed. Use without --dry-run to make changes."
+fi

crontab/crontab-backup-system.sh

@@ -29,7 +29,7 @@ log_message() {
     local message="$1"
     local log_file="$LOG_DIR/crontab-management.log"
     echo -e "$(date '+%Y-%m-%d %H:%M:%S') $message"
-    echo "$(date '+%Y-%m-%d %H:%M:%S') $message" | sed 's/\x1b\[[0-9;]*m//g' >> "$log_file"
+    echo "$(date '+%Y-%m-%d %H:%M:%S') $message" | sed 's/\\033\[[0-9;]*m//g' >> "$log_file"
 }
 
 log_error() {

crontab/crontab-europa.txt

@@ -39,6 +39,10 @@
 # Validates integrity of .env backup repository
 30 8 * * 0 { echo "Starting .env backup validation"; /home/acedanger/shell/validate-env-backups.sh; echo ".env validation completed with exit code: $?"; } 2>&1 | logger -t env-validation -p user.info
 
+# Weekly log files ANSI cleanup (Sundays at 0900)
+# Removes ANSI color codes from log files to improve readability and reduce size
+0 9 * * 0 { echo "Starting log ANSI cleanup"; /home/acedanger/shell/cleanup-log-ansi.sh --auto-discover --backup >/dev/null; echo "Log cleanup completed with exit code: $?"; } 2>&1 | logger -t log-cleanup -p user.info
+
 # Optional: Add a health check entry to monitor cron jobs (every 6 hours)
 # This can help detect if any of the backup processes are failing
 # 0 */6 * * * { echo "Cron health check - all backup jobs scheduled"; ps aux | grep -E "(backup-plex|validate-plex|move-backups)" | grep -v grep | wc -l; } 2>&1 | logger -t cron-health -p user.info

docs/cleanup-log-ansi.md

@@ -0,0 +1,221 @@
+# Log ANSI Cleanup Utility
+
+This utility script removes ANSI color codes from log files to improve readability and reduce file sizes when viewing logs in editors or processing them with other tools.
+
+## Features
+
+- **Auto-discovery**: Automatically finds log files in common locations
+- **Backup creation**: Creates backup files before cleaning (recommended)
+- **Pattern filtering**: Process only files matching specific patterns
+- **Recursive processing**: Can process directories recursively
+- **Dry run mode**: Preview changes without making them
+- **Comprehensive detection**: Detects various ANSI escape sequence formats
+
+## Installation
+
+```bash
+# Make the script executable
+chmod +x cleanup-log-ansi.sh
+
+# Optional: Create a symbolic link for system-wide access
+sudo ln -s "$(pwd)/cleanup-log-ansi.sh" /usr/local/bin/cleanup-log-ansi
+```
+
+## Usage
+
+### Basic Usage
+
+```bash
+# Clean a single log file with backup
+./cleanup-log-ansi.sh --backup /path/to/logfile.log
+
+# Clean all log files in a directory
+./cleanup-log-ansi.sh --recursive --backup /path/to/logs/
+
+# Auto-discover and clean all common log files
+./cleanup-log-ansi.sh --auto-discover --backup
+```
+
+### Advanced Usage
+
+```bash
+# Dry run to see what would be cleaned
+./cleanup-log-ansi.sh --dry-run --auto-discover
+
+# Clean only .log files in a directory tree
+./cleanup-log-ansi.sh --recursive --filter "*.log" --backup /var/log/
+
+# Clean without creating backups (use with caution)
+./cleanup-log-ansi.sh --no-backup /path/to/logfile.log
+
+# Verbose output
+./cleanup-log-ansi.sh --verbose --auto-discover --backup
+```
+
+## Options
+
+| Option | Description |
+|--------|-------------|
+| `-h, --help` | Show help message |
+| `-r, --recursive` | Process directories recursively |
+| `-b, --backup` | Create backup files before cleaning (recommended) |
+| `-n, --no-backup` | Don't create backup files |
+| `-v, --verbose` | Show detailed output |
+| `-d, --dry-run` | Show what would be done without making changes |
+| `-f, --filter PATTERN` | Only process files matching pattern (e.g., '*.log') |
+| `-a, --auto-discover` | Find common log files automatically |
+
+## Auto-Discovery Locations
+
+The script automatically searches for log files in these locations:
+
+- `./logs/`
+- `./crontab/logs/`
+- `./plex/logs/`
+- `../logs/`
+- `~/.local/share/logs/`
+- `/var/log/` (if readable)
+
+## File Extensions
+
+Auto-discovery looks for files with these extensions:
+- `.log`
+- `.out`
+- `.err`
+
+## ANSI Code Detection
+
+The script detects and removes these ANSI escape sequence patterns:
+- `\033[0;31m` (literal backslash sequences)
+- `\x1b[0;31m` (hexadecimal escape sequences)
+- `\033[0;31m` (octal escape sequences)
+
+## Examples
+
+### Clean Crontab Logs
+
+```bash
+# Clean all crontab management logs
+./cleanup-log-ansi.sh --backup crontab/logs/
+
+# Example output:
+# INFO: Starting ANSI cleanup utility
+# INFO: Backup created: crontab/logs/crontab-management.log.ansi-backup
+# SUCCESS: Cleaned: crontab/logs/crontab-management.log (24K → 21K)
+```
+
+### Clean Plex Logs
+
+```bash
+# Clean specific Plex database recovery logs
+./cleanup-log-ansi.sh --backup plex/logs/database-recovery-*.log
+
+# Clean all Plex logs recursively
+./cleanup-log-ansi.sh --recursive --backup plex/logs/
+```
+
+### System-wide Cleanup
+
+```bash
+# Find and clean all log files (with dry run first)
+./cleanup-log-ansi.sh --dry-run --auto-discover
+
+# If satisfied with the preview, run for real
+./cleanup-log-ansi.sh --auto-discover --backup
+```
+
+## Backup Management
+
+When using `--backup`, backup files are created with the `.ansi-backup` suffix:
+
+```bash
+# Original file: app.log
+# Backup file:   app.log.ansi-backup
+
+# Remove all backup files when satisfied
+rm -f *.ansi-backup **/*.ansi-backup
+```
+
+## Integration with Crontab
+
+You can add this script to your crontab for regular cleanup:
+
+```bash
+# Clean logs weekly (Sundays at 3 AM)
+0 3 * * 0 /path/to/cleanup-log-ansi.sh --auto-discover --backup --quiet 2>&1 | logger -t log-cleanup
+
+# Clean specific directory daily
+0 2 * * * /path/to/cleanup-log-ansi.sh --recursive --no-backup /var/log/myapp/ 2>&1 | logger -t log-cleanup
+```
+
+## Safety Features
+
+1. **Backup Creation**: Always create backups by default (use `--backup`)
+2. **File Validation**: Checks file existence and readability before processing
+3. **Integrity Verification**: Ensures cleaned files aren't corrupted
+4. **Dry Run Mode**: Preview changes before applying them
+5. **Error Handling**: Graceful error handling with meaningful messages
+
+## Common Use Cases
+
+### After Log Rotation
+```bash
+# Clean old log files after rotation
+./cleanup-log-ansi.sh --filter "*.log.*" --backup /var/log/
+```
+
+### CI/CD Integration
+```bash
+# Clean build logs in CI pipeline
+./cleanup-log-ansi.sh --no-backup --recursive build/logs/
+```
+
+### Development Environment
+```bash
+# Clean development logs regularly
+./cleanup-log-ansi.sh --auto-discover --backup --verbose
+```
+
+## Troubleshooting
+
+### Permission Denied
+```bash
+# If you get permission errors for system logs
+sudo ./cleanup-log-ansi.sh --backup /var/log/
+```
+
+### No Files Found
+```bash
+# Check what files would be processed
+./cleanup-log-ansi.sh --dry-run --verbose --auto-discover
+
+# Specify paths manually if auto-discovery fails
+./cleanup-log-ansi.sh --backup /specific/path/to/logs/
+```
+
+### Large Files
+For very large log files, the script processes them efficiently in a single pass, but you may want to:
+
+1. Use `--dry-run` first to estimate processing time
+2. Process files individually for better control
+3. Ensure sufficient disk space for backups
+
+## Performance
+
+- **Memory Efficient**: Uses streaming processing via `sed`
+- **Fast**: Single-pass processing of files
+- **Safe**: Creates backups before modification
+- **Scalable**: Can handle hundreds of log files
+
+## Related Scripts
+
+- `crontab-backup-system.sh`: Now has improved logging (ANSI codes properly stripped)
+- `backup-*.sh`: Various backup scripts that generate logs
+- `validate-*.sh`: Validation scripts with colored output
+
+## Version History
+
+- **v1.0**: Initial release with basic ANSI cleanup functionality
+- **v1.1**: Added auto-discovery and recursive processing
+- **v1.2**: Improved pattern matching and backup handling
+- **v1.3**: Added comprehensive safety features and verbose output