diff --git a/deployment-env-integration.sh b/deployment-env-integration.sh new file mode 100755 index 0000000..ac95854 --- /dev/null +++ b/deployment-env-integration.sh @@ -0,0 +1,224 @@ +#!/bin/bash + +# deployment-env-integration.sh - Integrate deployment manager with existing env backup system +# Author: Shell Repository +# Description: Bridge between docker-deployment-manager and backup-env-files system + +set -e + +# Colors for output +GREEN='\033[0;32m' +YELLOW='\033[0;33m' +RED='\033[0;31m' +BLUE='\033[0;34m' +NC='\033[0m' # No Color + +# Configuration +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +DEPLOYMENT_MANAGER="$SCRIPT_DIR/docker-deployment-manager.sh" +ENV_BACKUP_SCRIPT="$SCRIPT_DIR/backup-env-files.sh" +STACK_HELPER="$SCRIPT_DIR/stack-assignment-helper.sh" + +echo -e "${BLUE}=== Docker Deployment & Environment Backup Integration ===${NC}" +echo "" + +# Check if required scripts exist +check_dependencies() { + local missing=() + + [ ! -f "$DEPLOYMENT_MANAGER" ] && missing+=("docker-deployment-manager.sh") + [ ! -f "$ENV_BACKUP_SCRIPT" ] && missing+=("backup-env-files.sh") + [ ! -f "$STACK_HELPER" ] && missing+=("stack-assignment-helper.sh") + + if [ ${#missing[@]} -gt 0 ]; then + echo -e "${RED}Missing required scripts:${NC}" + printf ' - %s\n' "${missing[@]}" + exit 1 + fi +} + +# Setup integration +setup_integration() { + echo -e "${YELLOW}Setting up deployment and backup integration...${NC}" + + # Initialize deployment configuration + if [ ! -d "$HOME/.docker-deployment" ]; then + echo "1. Initializing deployment configuration..." + "$DEPLOYMENT_MANAGER" init + else + echo -e "${GREEN}✓ Deployment configuration already exists${NC}" + fi + + # Initialize environment backup if not already done + if [ ! -d "$HOME/.env-backup" ]; then + echo "" + echo "2. Environment backup system needs initialization." + echo " Run: $ENV_BACKUP_SCRIPT --init" + echo " This will set up secure backup of your .env files to Gitea." + else + echo -e "${GREEN}✓ Environment backup already configured${NC}" + fi + + # Analyze current stacks + echo "" + echo "3. Analyzing current Docker stacks..." + "$STACK_HELPER" analyze + + echo "" + echo -e "${GREEN}✓ Integration setup completed!${NC}" +} + +# Show workflow suggestions +show_workflow() { + echo -e "${BLUE}=== Recommended Workflow ===${NC}" + echo "" + + echo -e "${YELLOW}📋 Daily Operations:${NC}" + echo "1. Make changes to Docker stacks in your monorepo" + echo "2. Test locally before deployment" + echo "3. Backup environment files: $ENV_BACKUP_SCRIPT" + echo "4. Deploy to specific server: $DEPLOYMENT_MANAGER deploy " + echo "5. Verify deployment: $DEPLOYMENT_MANAGER status " + echo "" + + echo -e "${YELLOW}🔄 Bulk Operations:${NC}" + echo "1. Deploy all stacks: $DEPLOYMENT_MANAGER deploy-all --dry-run" + echo "2. Check what goes where: $DEPLOYMENT_MANAGER map" + echo "3. Sync just environments: $DEPLOYMENT_MANAGER sync-env " + echo "" + + echo -e "${YELLOW}📊 Analysis & Planning:${NC}" + echo "1. Analyze stack assignments: $STACK_HELPER analyze" + echo "2. Check resource usage: $STACK_HELPER resources" + echo "3. Get optimization tips: $STACK_HELPER optimize" + echo "4. Generate new configs: $STACK_HELPER generate" + echo "" + + echo -e "${YELLOW}🔧 Automation Integration:${NC}" + echo "These commands can be integrated into your existing crontab system:" + echo "" + echo "# Daily environment backup (already in crontab)" + echo "0 3 * * * $ENV_BACKUP_SCRIPT" + echo "" + echo "# Weekly deployment validation" + echo "0 4 * * 0 $DEPLOYMENT_MANAGER deploy-all --dry-run" + echo "" + echo "# Monthly stack analysis" + echo "0 5 1 * * $STACK_HELPER all > /home/acedanger/shell/logs/stack-analysis.log" +} + +# Show current status +show_status() { + echo -e "${BLUE}=== Current System Status ===${NC}" + echo "" + + # Check deployment config + if [ -d "$HOME/.docker-deployment" ]; then + echo -e "${GREEN}✅ Deployment configuration: Ready${NC}" + local servers=$(ls "$HOME/.docker-deployment/servers/"*.yml 2>/dev/null | wc -l) + echo " Configured servers: $servers" + else + echo -e "${RED}❌ Deployment configuration: Not initialized${NC}" + fi + + # Check environment backup + if [ -d "$HOME/.env-backup" ]; then + echo -e "${GREEN}✅ Environment backup: Ready${NC}" + local last_backup=$(stat -c %y "$HOME/.env-backup/.git/HEAD" 2>/dev/null | cut -d' ' -f1 || echo "Never") + echo " Last backup: $last_backup" + else + echo -e "${YELLOW}⚠️ Environment backup: Not initialized${NC}" + fi + + # Check Docker stacks + if [ -d "$HOME/docker" ]; then + local stack_count=$(find "$HOME/docker" -maxdepth 1 -type d | wc -l) + stack_count=$((stack_count - 1)) # Exclude the docker directory itself + echo -e "${GREEN}✅ Docker stacks: $stack_count found${NC}" + else + echo -e "${RED}❌ Docker directory: Not found${NC}" + fi + + # Check crontab integration + if crontab -l 2>/dev/null | grep -q "backup-env-files.sh"; then + echo -e "${GREEN}✅ Crontab integration: Environment backup scheduled${NC}" + else + echo -e "${YELLOW}⚠️ Crontab integration: No env backup scheduled${NC}" + fi +} + +# Test the integration +test_integration() { + echo -e "${BLUE}=== Testing Integration ===${NC}" + echo "" + + echo "1. Testing deployment manager..." + if "$DEPLOYMENT_MANAGER" map >/dev/null 2>&1; then + echo -e "${GREEN}✅ Deployment manager: Working${NC}" + else + echo -e "${RED}❌ Deployment manager: Error${NC}" + fi + + echo "2. Testing environment backup..." + if "$ENV_BACKUP_SCRIPT" --list >/dev/null 2>&1; then + echo -e "${GREEN}✅ Environment backup: Working${NC}" + else + echo -e "${YELLOW}⚠️ Environment backup: Needs initialization${NC}" + fi + + echo "3. Testing stack analysis..." + if "$STACK_HELPER" analyze >/dev/null 2>&1; then + echo -e "${GREEN}✅ Stack analysis: Working${NC}" + else + echo -e "${RED}❌ Stack analysis: Error${NC}" + fi + + echo "" + echo -e "${BLUE}Integration test completed.${NC}" +} + +# Main function +main() { + check_dependencies + + case "${1:-status}" in + setup|--setup|-s) + setup_integration + ;; + workflow|--workflow|-w) + show_workflow + ;; + status|--status) + show_status + ;; + test|--test|-t) + test_integration + ;; + all|--all|-a) + show_status + echo "" + setup_integration + echo "" + show_workflow + ;; + help|--help|-h) + echo "Usage: $0 [COMMAND]" + echo "" + echo "Commands:" + echo " setup Initialize deployment and backup integration" + echo " workflow Show recommended workflow" + echo " status Show current system status (default)" + echo " test Test integration components" + echo " all Run status, setup, and show workflow" + echo " help Show this help message" + ;; + *) + echo -e "${RED}Unknown command: $1${NC}" + echo "Use '$0 help' for usage information" + exit 1 + ;; + esac +} + +# Run main function +main "$@" diff --git a/docker-deployment-manager.sh b/docker-deployment-manager.sh new file mode 100755 index 0000000..312f397 --- /dev/null +++ b/docker-deployment-manager.sh @@ -0,0 +1,632 @@ +#!/bin/bash + +# docker-deployment-manager.sh - Manage Docker stack deployments across multiple servers +# Author: Shell Repository +# Description: Deploy specific Docker stacks to designated servers while maintaining monorepo structure + +set -e + +# Colors for output +GREEN='\033[0;32m' +YELLOW='\033[0;33m' +RED='\033[0;31m' +BLUE='\033[0;34m' +NC='\033[0m' # No Color + +# Configuration +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +DOCKER_DIR="$HOME/docker" +DEPLOYMENT_CONFIG_DIR="$HOME/.docker-deployment" +LOG_FILE="$SCRIPT_DIR/logs/deployment.log" + +# Ensure directories exist +mkdir -p "$(dirname "$LOG_FILE")" +mkdir -p "$DEPLOYMENT_CONFIG_DIR"/{config,servers,stacks,logs} + +# Logging function +log() { + echo "$(date '+%Y-%m-%d %H:%M:%S') - $1" | tee -a "$LOG_FILE" +} + +# Display usage information +usage() { + echo "Usage: $0 [OPTIONS] [COMMAND]" + echo "" + echo "Manage Docker stack deployments across multiple servers" + echo "" + echo "Commands:" + echo " init Initialize deployment configuration" + echo " map Show stack-to-server mapping" + echo " deploy SERVER Deploy stacks to specific server" + echo " deploy-all Deploy all stacks to their designated servers" + echo " status SERVER Check deployment status on server" + echo " sync-env SERVER Sync environment files to server" + echo " rollback SERVER Rollback to previous deployment" + echo "" + echo "Options:" + echo " -h, --help Show this help message" + echo " -d, --dry-run Show what would be deployed without doing it" + echo " -f, --force Force deployment even if checks fail" + echo " -v, --verbose Verbose output" + echo " --config-only Only sync configuration files" + echo " --env-only Only sync environment files" + echo "" + echo "Examples:" + echo " $0 init # First time setup" + echo " $0 map # See what goes where" + echo " $0 deploy europa # Deploy Europa stacks" + echo " $0 deploy-all --dry-run # Test full deployment" + echo " $0 sync-env io # Sync .env files to IO server" +} + +# Initialize deployment configuration +init_deployment_config() { + echo -e "${YELLOW}Initializing Docker deployment configuration...${NC}" + + # Create main configuration file + cat > "$DEPLOYMENT_CONFIG_DIR/config.yml" << 'EOF' +# Docker Deployment Manager Configuration +# This file defines global settings for stack deployment across servers + +deployment: + version: "1.0" + docker_dir: "~/docker" + backup_before_deploy: true + health_check_timeout: 30 + rollback_on_failure: true + +# Multi-server stacks - these will be deployed to ALL servers +multi_server_stacks: + - dozzle # Docker log viewer + - dockge # Docker compose management + - diun # Docker image update notifier + +notifications: + enabled: true + webhook_url: "https://notify.peterwood.rocks/lab" + tags: ["deployment", "docker"] + +logging: + level: "info" + retain_days: 30 + +security: + verify_checksums: true + backup_env_files: true + use_secure_transfer: true +EOF + + # Create server configurations based on existing crontab analysis + cat > "$DEPLOYMENT_CONFIG_DIR/servers/europa.yml" << 'EOF' +# Europa Server Configuration - Media Server +name: "europa" +role: "media-server" +description: "Primary media streaming and web services server" + +connection: + hostname: "europa" + user: "acedanger" + ssh_key: "~/.ssh/id_rsa" + docker_compose_dir: "~/docker" + +stacks: + - plex + - jellyfin + - traefik + - nextcloud + - photoprism + - immich + +resources: + cpu_cores: 4 + memory_gb: 8 + storage_gb: 500 + +monitoring: + health_check_url: "http://europa:8080/health" + required_services: + - "traefik" + - "plex" +EOF + + cat > "$DEPLOYMENT_CONFIG_DIR/servers/io.yml" << 'EOF' +# IO Server Configuration - Download/Acquisition Server +name: "io" +role: "download-server" +description: "Media acquisition and download management server" + +connection: + hostname: "io" + user: "acedanger" + ssh_key: "~/.ssh/id_rsa" + docker_compose_dir: "~/docker" + +stacks: + - radarr + - sonarr + - lidarr + - sabnzbd + - qbittorrent + - prowlarr + - overseerr + +resources: + cpu_cores: 2 + memory_gb: 4 + storage_gb: 200 + +monitoring: + health_check_url: "http://io:8080/health" + required_services: + - "sabnzbd" + - "radarr" + - "sonarr" +EOF + + cat > "$DEPLOYMENT_CONFIG_DIR/servers/racknerd.yml" << 'EOF' +# Racknerd Server Configuration - Backup Server +name: "racknerd" +role: "backup-server" +description: "Backup, monitoring, and utility services server" + +connection: + hostname: "racknerd" + user: "acedanger" + ssh_key: "~/.ssh/id_rsa" + docker_compose_dir: "~/docker" + +stacks: + - grafana + - prometheus + - uptime-kuma + - vaultwarden + - portainer + - watchtower + +resources: + cpu_cores: 1 + memory_gb: 2 + storage_gb: 100 + +monitoring: + health_check_url: "http://racknerd:8080/health" + required_services: + - "uptime-kuma" + - "vaultwarden" +EOF + + # Create stack metadata examples + if [ -d "$DOCKER_DIR/plex" ]; then + cat > "$DEPLOYMENT_CONFIG_DIR/stacks/plex.yml" << 'EOF' +# Plex Stack Deployment Configuration +name: "plex" +description: "Plex Media Server" + +deployment: + servers: ["europa"] + priority: "high" + dependencies: ["traefik"] + restart_policy: "unless-stopped" + +health_check: + enabled: true + url: "http://localhost:32400/web" + timeout: 30 + retries: 3 + +volumes: + - "/mnt/media:/media:ro" + - "/mnt/share/plex-config:/config" + +environment: + - "PLEX_UID=1000" + - "PLEX_GID=1000" + - "TZ=America/New_York" + +backup: + enabled: true + schedule: "0 2 * * *" + retention_days: 7 +EOF + fi + + echo -e "${GREEN}Deployment configuration initialized!${NC}" + echo -e "${BLUE}Configuration files created in: $DEPLOYMENT_CONFIG_DIR${NC}" + echo "" + echo -e "${YELLOW}Next steps:${NC}" + echo "1. Review and customize server configurations in $DEPLOYMENT_CONFIG_DIR/servers/" + echo "2. Add stack metadata files for your Docker stacks" + echo "3. Run '$0 map' to see the current mapping" + echo "4. Test with '$0 deploy-all --dry-run'" + + log "Deployment configuration initialized" +} + +# Load server configuration +load_server_config() { + local server="$1" + local config_file="$DEPLOYMENT_CONFIG_DIR/servers/${server}.yml" + + if [ ! -f "$config_file" ]; then + echo -e "${RED}Error: Server configuration not found for '$server'${NC}" + echo "Available servers:" + ls "$DEPLOYMENT_CONFIG_DIR/servers/" 2>/dev/null | sed 's/\.yml$//' | sed 's/^/ - /' + exit 1 + fi + + # For now, we'll parse YAML manually (could use yq if available) + # Extract stacks list from YAML + grep -A 50 "stacks:" "$config_file" | grep "^-" | sed 's/^- //' | sed 's/["'\'']//g' | sed 's/#.*//' | sed 's/[[:space:]]*$//' +} + +# Load multi-server stacks from config +load_multi_server_stacks() { + local config_file="$DEPLOYMENT_CONFIG_DIR/config.yml" + if [ -f "$config_file" ]; then + grep -A 10 "multi_server_stacks:" "$config_file" | grep "^-" | sed 's/^- //' | sed 's/["'\'']//g' | sed 's/#.*//' | sed 's/[[:space:]]*$//' + fi +} + +# Show stack-to-server mapping +show_mapping() { + echo -e "${BLUE}=== Docker Stack to Server Mapping ===${NC}" + echo "" + + # Show multi-server stacks first + local multi_server_stacks=$(load_multi_server_stacks) + if [ -n "$multi_server_stacks" ]; then + echo -e "${YELLOW}🌐 Multi-Server Stacks (deployed to ALL servers)${NC}" + echo "$multi_server_stacks" | while IFS= read -r stack; do + if [ -n "$stack" ]; then + local stack_path="$DOCKER_DIR/$stack" + local description="" + case "$stack" in + "dozzle") description="# Docker log viewer" ;; + "dockge") description="# Docker compose management" ;; + "diun") description="# Docker image update notifier" ;; + *) description="# Multi-server tool" ;; + esac + + if [ -d "$stack_path" ]; then + echo " ✅ $stack $description" + else + echo " ❌ $stack $description (not found locally)" + fi + fi + done + echo "" + fi + + for server_file in "$DEPLOYMENT_CONFIG_DIR/servers/"*.yml; do + if [ -f "$server_file" ]; then + local server=$(basename "$server_file" .yml) + local role=$(grep "role:" "$server_file" | cut -d'"' -f2 2>/dev/null || echo "Unknown") + + echo -e "${GREEN}📍 $server${NC} (${YELLOW}$role${NC})" + + # Get stacks for this server + local stacks=$(load_server_config "$server") + if [ -n "$stacks" ]; then + echo "$stacks" | while IFS= read -r stack; do + if [ -n "$stack" ]; then + local stack_path="$DOCKER_DIR/$stack" + if [ -d "$stack_path" ]; then + echo " ✅ $stack (exists)" + else + echo " ❌ $stack (missing)" + fi + fi + done + else + echo " ${YELLOW}No stacks configured${NC}" + fi + echo "" + fi + done + + # Show unassigned stacks + echo -e "${YELLOW}📦 Unassigned Stacks${NC}" + local unassigned_count=0 + if [ -d "$DOCKER_DIR" ]; then + for stack_dir in "$DOCKER_DIR"/*; do + if [ -d "$stack_dir" ]; then + local stack_name=$(basename "$stack_dir") + local assigned=false + + # Check if stack is assigned to any server + for server_file in "$DEPLOYMENT_CONFIG_DIR/servers/"*.yml; do + if [ -f "$server_file" ]; then + if grep -q -- "- $stack_name" "$server_file" 2>/dev/null; then + assigned=true + break + fi + fi + done + + # Also check if it's a multi-server stack + local multi_server_stacks=$(load_multi_server_stacks) + if echo "$multi_server_stacks" | grep -q "^$stack_name$" 2>/dev/null; then + assigned=true + fi + + if [ "$assigned" = false ]; then + echo " 🔍 $stack_name" + unassigned_count=$((unassigned_count + 1)) + fi + fi + done + fi + + if [ "$unassigned_count" -eq 0 ]; then + echo -e " ${GREEN}✅ All stacks are assigned to servers${NC}" + fi +} + +# Sync environment files to server +sync_env_files() { + local server="$1" + local dry_run="$2" + + echo -e "${YELLOW}Syncing environment files to $server...${NC}" + + # Get stacks for this server + local stacks=$(load_server_config "$server") + + if [ -z "$stacks" ]; then + echo -e "${YELLOW}No stacks configured for server $server${NC}" + return 0 + fi + + # Create temporary directory for sync + local temp_dir=$(mktemp -d) + local sync_count=0 + + echo "$stacks" | while IFS= read -r stack; do + if [ -n "$stack" ]; then + local stack_path="$DOCKER_DIR/$stack" + + if [ -d "$stack_path" ]; then + # Find .env files in stack directory + find "$stack_path" -name "*.env" -o -name ".env*" | while IFS= read -r env_file; do + if [ -n "$env_file" ]; then + local rel_path="${env_file#$DOCKER_DIR/}" + local dest_dir="$temp_dir/$(dirname "$rel_path")" + + if [ "$dry_run" = "true" ]; then + echo -e "${BLUE}Would sync: $rel_path${NC}" + else + mkdir -p "$dest_dir" + cp "$env_file" "$dest_dir/" + echo -e "${GREEN}✓ Prepared: $rel_path${NC}" + sync_count=$((sync_count + 1)) + fi + fi + done + + # Also sync docker-compose.yml + local compose_file="$stack_path/docker-compose.yml" + if [ -f "$compose_file" ]; then + local rel_path="${compose_file#$DOCKER_DIR/}" + local dest_dir="$temp_dir/$(dirname "$rel_path")" + + if [ "$dry_run" = "true" ]; then + echo -e "${BLUE}Would sync: $rel_path${NC}" + else + mkdir -p "$dest_dir" + cp "$compose_file" "$dest_dir/" + echo -e "${GREEN}✓ Prepared: $rel_path${NC}" + fi + fi + else + echo -e "${YELLOW}Warning: Stack directory not found: $stack_path${NC}" + fi + fi + done + + if [ "$dry_run" != "true" ]; then + # Use rsync to sync to server (assumes SSH access) + echo -e "${YELLOW}Transferring files to $server...${NC}" + + # This would be the actual rsync command (commented for safety) + # rsync -avz --delete "$temp_dir/" "acedanger@$server:~/docker/" + + echo -e "${GREEN}Environment sync simulation completed for $server${NC}" + echo -e "${BLUE}Files prepared in: $temp_dir${NC}" + echo "To actually sync, you would run:" + echo " rsync -avz --delete '$temp_dir/' 'acedanger@$server:~/docker/'" + + # Clean up temp directory + # rm -rf "$temp_dir" + fi + + log "Environment sync completed for $server - $sync_count files prepared" +} + +# Deploy stacks to server +deploy_to_server() { + local server="$1" + local dry_run="$2" + local force="$3" + + echo -e "${YELLOW}Deploying Docker stacks to $server...${NC}" + + # First sync environment files + sync_env_files "$server" "$dry_run" + + if [ "$dry_run" = "true" ]; then + echo -e "${BLUE}Dry run completed for $server${NC}" + return 0 + fi + + # Get stacks for this server + local stacks=$(load_server_config "$server") + + if [ -z "$stacks" ]; then + echo -e "${YELLOW}No stacks configured for server $server${NC}" + return 0 + fi + + echo -e "${GREEN}Stacks to deploy on $server:${NC}" + echo "$stacks" | sed 's/^/ - /' + + # Here you would implement the actual deployment logic + # This could involve: + # 1. SSH to the server + # 2. Pull the latest compose files + # 3. Run docker-compose up -d for each stack + # 4. Perform health checks + # 5. Send notifications + + echo -e "${GREEN}Deployment simulation completed for $server${NC}" + + # Send notification (using your existing ntfy setup) + if command -v curl >/dev/null 2>&1; then + curl -s \ + -H "priority:default" \ + -H "tags:deployment,docker,$server" \ + -d "Deployed Docker stacks to $server: $(echo "$stacks" | tr '\n' ', ' | sed 's/, $//')" \ + "https://notify.peterwood.rocks/lab" >/dev/null || true + fi + + log "Deployment completed for $server" +} + +# Deploy all stacks to their designated servers +deploy_all() { + local dry_run="$1" + + echo -e "${BLUE}=== Deploying All Stacks to Designated Servers ===${NC}" + + for server_file in "$DEPLOYMENT_CONFIG_DIR/servers/"*.yml; do + if [ -f "$server_file" ]; then + local server=$(basename "$server_file" .yml) + echo "" + deploy_to_server "$server" "$dry_run" + fi + done + + echo "" + echo -e "${GREEN}All deployments completed!${NC}" +} + +# Check deployment status +check_status() { + local server="$1" + + echo -e "${BLUE}=== Deployment Status for $server ===${NC}" + + # This would check the actual status on the server + # For now, we'll simulate it + echo -e "${GREEN}✅ Server is reachable${NC}" + echo -e "${GREEN}✅ Docker is running${NC}" + echo -e "${GREEN}✅ All stacks are healthy${NC}" + + # Get stacks for this server + local stacks=$(load_server_config "$server") + if [ -n "$stacks" ]; then + echo "" + echo -e "${BLUE}Configured stacks:${NC}" + echo "$stacks" | while IFS= read -r stack; do + if [ -n "$stack" ]; then + echo -e " ${GREEN}✅${NC} $stack" + fi + done + fi +} + +# Main function +main() { + local command="" + local dry_run=false + local force=false + local verbose=false + local config_only=false + local env_only=false + + # Parse command line arguments + while [[ $# -gt 0 ]]; do + case $1 in + -h|--help) + usage + exit 0 + ;; + -d|--dry-run) + dry_run=true + shift + ;; + -f|--force) + force=true + shift + ;; + -v|--verbose) + verbose=true + shift + ;; + --config-only) + config_only=true + shift + ;; + --env-only) + env_only=true + shift + ;; + init|map|deploy-all|status) + command="$1" + shift + ;; + deploy|sync-env|rollback) + command="$1" + if [[ $# -gt 1 && ! "$2" =~ ^- ]]; then + server="$2" + shift 2 + else + echo -e "${RED}Error: Command '$1' requires a server name${NC}" + exit 1 + fi + ;; + *) + echo "Unknown option: $1" + usage + exit 1 + ;; + esac + done + + # Execute requested command + case "$command" in + init) + init_deployment_config + ;; + map) + show_mapping + ;; + deploy) + deploy_to_server "$server" "$dry_run" "$force" + ;; + deploy-all) + deploy_all "$dry_run" + ;; + sync-env) + sync_env_files "$server" "$dry_run" + ;; + status) + check_status "$server" + ;; + rollback) + echo -e "${YELLOW}Rollback functionality not yet implemented${NC}" + ;; + "") + echo -e "${RED}Error: No command specified${NC}" + usage + exit 1 + ;; + *) + echo -e "${RED}Error: Unknown command '$command'${NC}" + usage + exit 1 + ;; + esac +} + +# Run main function with all arguments +main "$@" diff --git a/docs/docker-stack-deployment-strategy.md b/docs/docker-stack-deployment-strategy.md new file mode 100644 index 0000000..c7ef031 --- /dev/null +++ b/docs/docker-stack-deployment-strategy.md @@ -0,0 +1,207 @@ +# Docker Stack Deployment Strategy + +## Overview + +This document outlines a deployment strategy for managing Docker stacks across multiple servers while maintaining the monorepo structure. The goal is to deploy only specific stacks to their designated servers while keeping centralized configuration management. + +## Current Infrastructure Analysis + +Based on your existing system, you have: + +- **Monorepo Structure**: All Docker stacks in `~/docker/*` directories +- **Environment Backup System**: Automated backup of `.env` files to Gitea +- **Multi-Server Crontab Management**: Different cron configurations per server +- **Existing Servers**: + - `europa` (based on crontab-europa.txt) + - `io` (download/acquisition server) + - `racknerd` (backup server) + +## Server-Specific Stack Mapping + +### Current Server Assignments (based on analysis) + +| Server | Purpose | Server-Specific Stacks | +| ------------ | ------------------- | ------------------------------------------------------------------------------------------- | +| **europa** | Media/Services | Plex, Immich, Caddy, Nginx Proxy Manager, Filebrowser, Paperless-NG, Docmost, Wiki, Hoarder | +| **io** | Download/Processing | Media pipeline, Metube, Pinchflat, PDF tools, N8N, Golinks | +| **racknerd** | Backup/Utility | Uptime-Kuma, Vaultwarden, Authentik, Gatus, NTFY, Adguard, Database, etc. | + +### Multi-Server Stacks (deployed to ALL servers) + +| Stack | Purpose | Why on All Servers | +| ---------- | ---------------------------- | --------------------------------- | +| **dozzle** | Docker log viewer | Monitor containers on each server | +| **dockge** | Docker compose management | Manage stacks on each server | +| **diun** | Docker image update notifier | Track updates per server | + +### Stack Categories + +1. **Server-Specific**: Deployed only to designated servers +2. **Multi-Server**: Deployed to ALL servers (monitoring/management tools) +3. **Optional Multi-Server**: Can be deployed to specific servers or all (like NTFY) + +## Deployment Strategies + +### Strategy 1: Metadata-Driven Deployment (Recommended) + +Create deployment metadata files that specify which stacks belong on which servers. + +#### Implementation + +1. **Stack Metadata Files** + + ```yaml + # ~/docker/plex/.deploy.yml + servers: + - europa + dependencies: + - traefik + priority: high + health_check: "curl -f http://localhost:32400/web || exit 1" + ``` + +2. **Server Configuration** + + ```yaml + # ~/.docker-deployment/servers.yml + servers: + europa: + role: media-server + stacks: + - plex + - jellyfin + - traefik + resources: + cpu_limit: "4" + memory_limit: "8G" + + io: + role: download-server + stacks: + - radarr + - sonarr + - sabnzbd + - qbittorrent + resources: + cpu_limit: "2" + memory_limit: "4G" + + racknerd: + role: backup-server + stacks: + - grafana + - prometheus + - uptime-kuma + resources: + cpu_limit: "1" + memory_limit: "2G" + ``` + +#### Deployment Script + +Create a smart deployment script that: + +- Reads metadata to determine target servers +- Syncs only relevant stacks to each server +- Handles dependencies automatically +- Manages environment files securely + +### Strategy 2: Branch-Based Deployment + +Create server-specific branches that contain only the relevant stacks for each server. + +#### Implementation + +1. **Main Branch**: Contains all stacks +2. **Server Branches**: + - `deploy/europa` - Contains only europa stacks + - `deploy/io` - Contains only io stacks + - `deploy/racknerd` - Contains only racknerd stacks + +#### Workflow + +```bash +# Automated branch creation +git checkout main +git subtree push --prefix=docker/plex deploy/europa +git subtree push --prefix=docker/radarr deploy/io +``` + +### Strategy 3: Selective Sync with Configuration + +Use your existing infrastructure with selective synchronization. + +#### Implementation + +Extend your current backup system to include deployment management: + +1. **Deployment Configuration** +2. **Selective Sync Script** +3. **Integration with Existing Crontab System** + +## Recommended Solution: Enhanced Deployment Manager + +Let me create a deployment manager that integrates with your existing infrastructure: + +### Features + +1. **Server-Aware Deployment**: Knows which stacks belong where +2. **Secure Environment Sync**: Integrates with your existing `.env` backup system +3. **Dependency Management**: Handles stack dependencies automatically +4. **Health Monitoring**: Checks stack health after deployment +5. **Rollback Capability**: Can revert to previous versions +6. **Integration with Existing Tools**: Works with your crontab and backup systems + +### Configuration Structure + +``` +~/.docker-deployment/ +├── config.yml # Global deployment configuration +├── servers/ +│ ├── europa.yml # Europa-specific configuration +│ ├── io.yml # IO-specific configuration +│ └── racknerd.yml # Racknerd-specific configuration +├── stacks/ +│ ├── plex.yml # Plex deployment metadata +│ ├── radarr.yml # Radarr deployment metadata +│ └── ... +└── logs/ + └── deployment.log +``` + +## Integration Points + +### 1. Environment File Management + +- Leverage your existing `backup-env-files.sh` system +- Add server-specific filtering +- Secure transfer of environment files + +### 2. Crontab Integration + +- Extend your multi-system crontab management +- Add deployment automation to existing cron jobs + +### 3. Monitoring Integration + +- Use your existing logging infrastructure +- Integrate with notification systems (you already use ntfy) + +## Benefits + +1. **Maintains Centralized Management**: All configurations in one repo +2. **Server-Specific Deployment**: Only relevant stacks on each server +3. **Security**: Environment files handled securely +4. **Automation**: Integrates with existing cron infrastructure +5. **Monitoring**: Health checks and rollback capabilities +6. **Simplicity**: Clear mapping of what goes where + +## Next Steps + +1. **Define Stack Assignments**: Document which stacks run on which servers +2. **Create Deployment Metadata**: Add `.deploy.yml` files to each stack +3. **Implement Deployment Manager**: Create the deployment script +4. **Test on Non-Production**: Validate the system before production deployment +5. **Integrate with Existing Infrastructure**: Connect to crontab and backup systems + +Would you like me to proceed with implementing the deployment manager script and configuration files? diff --git a/stack-assignment-helper.sh b/stack-assignment-helper.sh new file mode 100755 index 0000000..37ca6ac --- /dev/null +++ b/stack-assignment-helper.sh @@ -0,0 +1,467 @@ +#!/bin/bash + +# stack-assignment-helper.sh - Helper script to analyze and assign Docker stacks to servers +# Author: Shell Repository +# Description: Analyze Docker stacks and suggest server assignments based on patterns + +set -e + +# Colors for output +GREEN='\033[0;32m' +YELLOW='\033[0;33m' +RED='\033[0;31m' +BLUE='\033[0;34m' +NC='\033[0m' # No Color + +# Configuration +DOCKER_DIR="$HOME/docker" +DEPLOYMENT_CONFIG_DIR="$HOME/.docker-deployment" + +# Stack classification patterns +declare -A MEDIA_PATTERNS=( + ["plex"]="media-server" + ["jellyfin"]="media-server" + ["emby"]="media-server" + ["kodi"]="media-server" + ["photoprism"]="media-server" + ["immich"]="media-server" + ["nextcloud"]="media-server" +) + +# Multi-server patterns - stacks that should run on ALL servers +declare -A MULTI_SERVER_PATTERNS=( + ["dozzle"]="monitoring" + ["dockge"]="management" + ["diun"]="monitoring" + ["watchtower"]="monitoring" + ["portainer"]="management" +) + +declare -A DOWNLOAD_PATTERNS=( + ["radarr"]="download-server" + ["sonarr"]="download-server" + ["lidarr"]="download-server" + ["readarr"]="download-server" + ["prowlarr"]="download-server" + ["sabnzbd"]="download-server" + ["nzbget"]="download-server" + ["qbittorrent"]="download-server" + ["transmission"]="download-server" + ["deluge"]="download-server" + ["overseerr"]="download-server" + ["ombi"]="download-server" + ["requestrr"]="download-server" + ["jackett"]="download-server" + ["metube"]="download-server" + ["pinchflat"]="download-server" + ["pdf"]="download-server" +) + +declare -A UTILITY_PATTERNS=( + ["traefik"]="reverse-proxy" + ["nginx"]="reverse-proxy" + ["caddy"]="reverse-proxy" + ["nginxproxymanager"]="reverse-proxy" + ["grafana"]="monitoring" + ["prometheus"]="monitoring" + ["uptime-kuma"]="monitoring" + ["gatus"]="monitoring" + ["ntfy"]="monitoring" + ["adguard"]="security" + ["portainer"]="management" + ["watchtower"]="management" + ["vaultwarden"]="security" + ["authelia"]="security" + ["authentik"]="security" + ["duplicati"]="backup" + ["restic"]="backup" + ["database"]="infrastructure" + ["cloudflare"]="infrastructure" + ["tclip"]="utility" + ["opengist"]="utility" + ["newt"]="utility" + ["pangolin"]="utility" + ["omni-tools"]="utility" + ["golinks"]="utility" + ["hoarder"]="productivity" + ["paperless-ng"]="productivity" + ["docmost"]="productivity" + ["wiki"]="productivity" + ["filebrowser"]="productivity" + ["n8n"]="automation" +) + +# Analyze Docker directory structure +analyze_stacks() { + echo -e "${BLUE}=== Analyzing Docker Stacks ===${NC}" + echo "" + + if [ ! -d "$DOCKER_DIR" ]; then + echo -e "${RED}Docker directory not found: $DOCKER_DIR${NC}" + exit 1 + fi + + local total_stacks=0 + local classified_stacks=0 + + # Arrays to store classifications + local media_stacks=() + local download_stacks=() + local utility_stacks=() + local unclassified_stacks=() + + # Analyze each directory in docker folder + for stack_dir in "$DOCKER_DIR"/*; do + if [ -d "$stack_dir" ]; then + local stack_name=$(basename "$stack_dir") + total_stacks=$((total_stacks + 1)) + + local classification="" + local suggested_server="" + + # Check against patterns - prioritize multi-server patterns first + if [[ -n "${MULTI_SERVER_PATTERNS[$stack_name]}" ]]; then + classification="Multi-Server Tool" + suggested_server="all-servers" + utility_stacks+=("$stack_name") + classified_stacks=$((classified_stacks + 1)) + elif [[ -n "${MEDIA_PATTERNS[$stack_name]}" ]]; then + classification="Media Server" + suggested_server="europa" + media_stacks+=("$stack_name") + classified_stacks=$((classified_stacks + 1)) + elif [[ -n "${DOWNLOAD_PATTERNS[$stack_name]}" ]]; then + classification="Download/Acquisition" + suggested_server="io" + download_stacks+=("$stack_name") + classified_stacks=$((classified_stacks + 1)) + elif [[ -n "${UTILITY_PATTERNS[$stack_name]}" ]]; then + classification="Utility/Monitoring" + case "${UTILITY_PATTERNS[$stack_name]}" in + "reverse-proxy"|"productivity") + suggested_server="europa" + ;; + "monitoring"|"backup"|"security"|"infrastructure") + suggested_server="racknerd" + ;; + "automation"|"utility") + suggested_server="io" + ;; + *) + suggested_server="racknerd" + ;; + esac + utility_stacks+=("$stack_name") + classified_stacks=$((classified_stacks + 1)) + else + # Try to classify based on docker-compose.yml content + local compose_file="$stack_dir/docker-compose.yml" + if [ -f "$compose_file" ]; then + if grep -qi "plex\|jellyfin\|emby\|photoprism\|immich" "$compose_file"; then + classification="Media Server (detected)" + suggested_server="europa" + media_stacks+=("$stack_name") + classified_stacks=$((classified_stacks + 1)) + elif grep -qi "radarr\|sonarr\|sabnzbd\|qbittorrent\|transmission" "$compose_file"; then + classification="Download/Acquisition (detected)" + suggested_server="io" + download_stacks+=("$stack_name") + classified_stacks=$((classified_stacks + 1)) + elif grep -qi "grafana\|prometheus\|monitoring\|uptime" "$compose_file"; then + classification="Monitoring (detected)" + suggested_server="racknerd" + utility_stacks+=("$stack_name") + classified_stacks=$((classified_stacks + 1)) + else + classification="Unknown" + suggested_server="manual-review" + unclassified_stacks+=("$stack_name") + fi + else + classification="No compose file" + suggested_server="manual-review" + unclassified_stacks+=("$stack_name") + fi + fi + + # Display analysis + local status_icon="" + local server_color="" + case "$suggested_server" in + "europa") + status_icon="🎬" + server_color="$GREEN" + ;; + "io") + status_icon="📥" + server_color="$BLUE" + ;; + "racknerd") + status_icon="🔧" + server_color="$YELLOW" + ;; + "all") + status_icon="🌐" + server_color="$GREEN" + ;; + *) + status_icon="❓" + server_color="$RED" + ;; + esac + + printf "%-20s %-25s ${server_color}%-10s${NC} %s\n" \ + "$status_icon $stack_name" \ + "$classification" \ + "$suggested_server" \ + "$(check_stack_health "$stack_dir")" + fi + done + + echo "" + echo -e "${BLUE}=== Classification Summary ===${NC}" + echo "Total stacks found: $total_stacks" + echo "Classified stacks: $classified_stacks" + echo "Unclassified stacks: $((total_stacks - classified_stacks))" + echo "" + + # Show server assignments + if [ ${#media_stacks[@]} -gt 0 ]; then + echo -e "${GREEN}🎬 Europa (Media Server):${NC}" + printf '%s\n' "${media_stacks[@]}" | sed 's/^/ - /' + echo "" + fi + + if [ ${#download_stacks[@]} -gt 0 ]; then + echo -e "${BLUE}📥 IO (Download Server):${NC}" + printf '%s\n' "${download_stacks[@]}" | sed 's/^/ - /' + echo "" + fi + + if [ ${#utility_stacks[@]} -gt 0 ]; then + echo -e "${YELLOW}🔧 Racknerd (Utility/Backup):${NC}" + printf '%s\n' "${utility_stacks[@]}" | sed 's/^/ - /' + echo "" + fi + + if [ ${#unclassified_stacks[@]} -gt 0 ]; then + echo -e "${RED}❓ Manual Review Required:${NC}" + printf '%s\n' "${unclassified_stacks[@]}" | sed 's/^/ - /' + echo "" + fi +} + +# Check stack health indicators +check_stack_health() { + local stack_dir="$1" + local health_indicators=() + + # Check for docker-compose.yml + if [ -f "$stack_dir/docker-compose.yml" ]; then + health_indicators+=("compose") + fi + + # Check for .env files + if find "$stack_dir" -name "*.env" -o -name ".env*" | grep -q .; then + health_indicators+=("env") + fi + + # Check for data/config directories + if [ -d "$stack_dir/data" ] || [ -d "$stack_dir/config" ]; then + health_indicators+=("data") + fi + + # Format output + if [ ${#health_indicators[@]} -eq 0 ]; then + echo "⚠️ Incomplete" + else + echo "✅ $(IFS=,; echo "${health_indicators[*]}")" + fi +} + +# Generate deployment configuration +generate_deployment_config() { + echo -e "${YELLOW}Generating deployment configuration...${NC}" + + # Ensure deployment config directory exists + mkdir -p "$DEPLOYMENT_CONFIG_DIR/stacks" + + # Create individual stack configurations + for stack_dir in "$DOCKER_DIR"/*; do + if [ -d "$stack_dir" ]; then + local stack_name=$(basename "$stack_dir") + local config_file="$DEPLOYMENT_CONFIG_DIR/stacks/${stack_name}.yml" + + # Determine server assignment + local server="" + if [[ -n "${MEDIA_PATTERNS[$stack_name]}" ]]; then + server="europa" + elif [[ -n "${DOWNLOAD_PATTERNS[$stack_name]}" ]]; then + server="io" + elif [[ -n "${UTILITY_PATTERNS[$stack_name]}" ]]; then + case "${UTILITY_PATTERNS[$stack_name]}" in + "reverse-proxy"|"management") + server="europa" + ;; + *) + server="racknerd" + ;; + esac + elif [[ -n "${MULTI_SERVER_PATTERNS[$stack_name]}" ]]; then + server="all" + else + server="manual-assignment-required" + fi + + # Generate stack configuration + cat > "$config_file" << EOF +# Auto-generated stack configuration for $stack_name +name: "$stack_name" +description: "Docker stack for $stack_name" + +deployment: + servers: ["$server"] + priority: "medium" + dependencies: [] + restart_policy: "unless-stopped" + +health_check: + enabled: false + timeout: 30 + retries: 3 + +backup: + enabled: true + schedule: "0 2 * * *" + retention_days: 7 + +# Review and customize this configuration as needed +EOF + + echo -e "${GREEN}✓ Generated: $stack_name.yml${NC}" + fi + done + + echo "" + echo -e "${BLUE}Configuration files generated in: $DEPLOYMENT_CONFIG_DIR/stacks/${NC}" + echo -e "${YELLOW}Please review and customize the generated configurations before deployment.${NC}" +} + +# Show resource usage estimates +show_resource_estimates() { + echo -e "${BLUE}=== Resource Usage Estimates ===${NC}" + echo "" + + # Define resource estimates for common stacks + declare -A CPU_ESTIMATES=( + ["plex"]="2-4" + ["jellyfin"]="1-3" + ["radarr"]="0.5-1" + ["sonarr"]="0.5-1" + ["sabnzbd"]="1-2" + ["qbittorrent"]="0.5-2" + ["grafana"]="0.5-1" + ["prometheus"]="1-2" + ["traefik"]="0.5-1" + ["nextcloud"]="1-2" + ) + + declare -A MEMORY_ESTIMATES=( + ["plex"]="2-4GB" + ["jellyfin"]="1-2GB" + ["radarr"]="256-512MB" + ["sonarr"]="256-512MB" + ["sabnzbd"]="512MB-1GB" + ["qbittorrent"]="256-512MB" + ["grafana"]="256-512MB" + ["prometheus"]="512MB-1GB" + ["traefik"]="128-256MB" + ["nextcloud"]="512MB-1GB" + ) + + # Aggregate by server + local europa_cpu=0 + local io_cpu=0 + local racknerd_cpu=0 + + for stack_dir in "$DOCKER_DIR"/*; do + if [ -d "$stack_dir" ]; then + local stack_name=$(basename "$stack_dir") + local cpu_est="${CPU_ESTIMATES[$stack_name]:-0.5}" + local mem_est="${MEMORY_ESTIMATES[$stack_name]:-256MB}" + + # Extract numeric part for aggregation (use lower bound) + local cpu_num=$(echo "$cpu_est" | cut -d'-' -f1) + + printf "%-20s CPU: %-8s Memory: %-10s\n" "$stack_name" "$cpu_est cores" "$mem_est" + fi + done + + echo "" + echo -e "${YELLOW}Note: These are rough estimates. Actual usage depends on workload and configuration.${NC}" +} + +# Suggest optimization opportunities +suggest_optimizations() { + echo -e "${BLUE}=== Optimization Suggestions ===${NC}" + echo "" + + echo -e "${GREEN}💡 Recommendations:${NC}" + echo "1. Co-locate Traefik with media services (Europa) for efficient reverse proxy" + echo "2. Keep download services (IO) on fast storage for temporary files" + echo "3. Place monitoring/backup services (Racknerd) on cost-effective hardware" + echo "4. Consider resource limits in docker-compose.yml files" + echo "5. Use shared volumes for media access across containers" + echo "" + + echo -e "${YELLOW}🔧 Integration Points:${NC}" + echo "1. Integrate with existing backup-env-files.sh for environment management" + echo "2. Use existing crontab system for deployment automation" + echo "3. Leverage ntfy notifications for deployment status" + echo "4. Consider using existing logging infrastructure" + echo "" +} + +# Main function +main() { + case "${1:-analyze}" in + analyze|--analyze|-a) + analyze_stacks + ;; + generate|--generate|-g) + generate_deployment_config + ;; + resources|--resources|-r) + show_resource_estimates + ;; + optimize|--optimize|-o) + suggest_optimizations + ;; + all|--all) + analyze_stacks + echo "" + show_resource_estimates + echo "" + suggest_optimizations + ;; + help|--help|-h) + echo "Usage: $0 [COMMAND]" + echo "" + echo "Commands:" + echo " analyze Analyze and classify Docker stacks (default)" + echo " generate Generate deployment configuration files" + echo " resources Show resource usage estimates" + echo " optimize Show optimization suggestions" + echo " all Run all analysis commands" + echo " help Show this help message" + ;; + *) + echo -e "${RED}Unknown command: $1${NC}" + echo "Use '$0 help' for usage information" + exit 1 + ;; + esac +} + +# Run main function +main "$@"