peterwood/shell
1
0
Fork 0
mirror of https://github.com/acedanger/shell.git synced 2025-12-05 22:50:18 -08:00
Code Issues 19 Packages Projects Releases Wiki Activity

Add Docker Deployment Manager and supporting scripts

Browse Source 
- Introduced `docker-deployment-manager.sh` for managing Docker stack deployments across multiple servers, including initialization, deployment, and status checking.
- Created `docker-stack-deployment-strategy.md` to outline deployment strategies and server-specific stack mappings.
- Added `stack-assignment-helper.sh` to analyze Docker stacks, suggest server assignments, and generate deployment configurations based on predefined patterns.
This commit is contained in:
Peter Wood
2025-05-29 15:05:20 -04:00
parent 9d09181085
commit f022215ac1
4 changed files with 1530 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

224
deployment-env-integration.sh Executable file
View File

@@ -0,0 +1,224 @@
#!/bin/bash
# deployment-env-integration.sh - Integrate deployment manager with existing env backup system
# Author: Shell Repository
# Description: Bridge between docker-deployment-manager and backup-env-files system
set -e
# Colors for output
GREEN='\033[0;32m'
YELLOW='\033[0;33m'
RED='\033[0;31m'
BLUE='\033[0;34m'
NC='\033[0m' # No Color
# Configuration
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
DEPLOYMENT_MANAGER="$SCRIPT_DIR/docker-deployment-manager.sh"
ENV_BACKUP_SCRIPT="$SCRIPT_DIR/backup-env-files.sh"
STACK_HELPER="$SCRIPT_DIR/stack-assignment-helper.sh"
echo -e "${BLUE}=== Docker Deployment & Environment Backup Integration ===${NC}"
echo ""
# Check if required scripts exist
check_dependencies() {
local missing=()
[ ! -f "$DEPLOYMENT_MANAGER" ] && missing+=("docker-deployment-manager.sh")
[ ! -f "$ENV_BACKUP_SCRIPT" ] && missing+=("backup-env-files.sh")
[ ! -f "$STACK_HELPER" ] && missing+=("stack-assignment-helper.sh")
if [ ${#missing[@]} -gt 0 ]; then
echo -e "${RED}Missing required scripts:${NC}"
printf ' - %s\n' "${missing[@]}"
exit 1
fi
}
# Setup integration
setup_integration() {
echo -e "${YELLOW}Setting up deployment and backup integration...${NC}"
# Initialize deployment configuration
if [ ! -d "$HOME/.docker-deployment" ]; then
echo "1. Initializing deployment configuration..."
"$DEPLOYMENT_MANAGER" init
else
echo -e "${GREEN}✓ Deployment configuration already exists${NC}"
fi
# Initialize environment backup if not already done
if [ ! -d "$HOME/.env-backup" ]; then
echo ""
echo "2. Environment backup system needs initialization."
echo " Run: $ENV_BACKUP_SCRIPT --init"
echo " This will set up secure backup of your .env files to Gitea."
else
echo -e "${GREEN}✓ Environment backup already configured${NC}"
fi
# Analyze current stacks
echo ""
echo "3. Analyzing current Docker stacks..."
"$STACK_HELPER" analyze
echo ""
echo -e "${GREEN}✓ Integration setup completed!${NC}"
}
# Show workflow suggestions
show_workflow() {
echo -e "${BLUE}=== Recommended Workflow ===${NC}"
echo ""
echo -e "${YELLOW}📋 Daily Operations:${NC}"
echo "1. Make changes to Docker stacks in your monorepo"
echo "2. Test locally before deployment"
echo "3. Backup environment files: $ENV_BACKUP_SCRIPT"
echo "4. Deploy to specific server: $DEPLOYMENT_MANAGER deploy <server>"
echo "5. Verify deployment: $DEPLOYMENT_MANAGER status <server>"
echo ""
echo -e "${YELLOW}🔄 Bulk Operations:${NC}"
echo "1. Deploy all stacks: $DEPLOYMENT_MANAGER deploy-all --dry-run"
echo "2. Check what goes where: $DEPLOYMENT_MANAGER map"
echo "3. Sync just environments: $DEPLOYMENT_MANAGER sync-env <server>"
echo ""
echo -e "${YELLOW}📊 Analysis & Planning:${NC}"
echo "1. Analyze stack assignments: $STACK_HELPER analyze"
echo "2. Check resource usage: $STACK_HELPER resources"
echo "3. Get optimization tips: $STACK_HELPER optimize"
echo "4. Generate new configs: $STACK_HELPER generate"
echo ""
echo -e "${YELLOW}🔧 Automation Integration:${NC}"
echo "These commands can be integrated into your existing crontab system:"
echo ""
echo "# Daily environment backup (already in crontab)"
echo "0 3 * * * $ENV_BACKUP_SCRIPT"
echo ""
echo "# Weekly deployment validation"
echo "0 4 * * 0 $DEPLOYMENT_MANAGER deploy-all --dry-run"
echo ""
echo "# Monthly stack analysis"
echo "0 5 1 * * $STACK_HELPER all > /home/acedanger/shell/logs/stack-analysis.log"
}
# Show current status
show_status() {
echo -e "${BLUE}=== Current System Status ===${NC}"
echo ""
# Check deployment config
if [ -d "$HOME/.docker-deployment" ]; then
echo -e "${GREEN}✅ Deployment configuration: Ready${NC}"
local servers=$(ls "$HOME/.docker-deployment/servers/"*.yml 2>/dev/null | wc -l)
echo " Configured servers: $servers"
else
echo -e "${RED}❌ Deployment configuration: Not initialized${NC}"
fi
# Check environment backup
if [ -d "$HOME/.env-backup" ]; then
echo -e "${GREEN}✅ Environment backup: Ready${NC}"
local last_backup=$(stat -c %y "$HOME/.env-backup/.git/HEAD" 2>/dev/null | cut -d' ' -f1 || echo "Never")
echo " Last backup: $last_backup"
else
echo -e "${YELLOW}⚠️ Environment backup: Not initialized${NC}"
fi
# Check Docker stacks
if [ -d "$HOME/docker" ]; then
local stack_count=$(find "$HOME/docker" -maxdepth 1 -type d | wc -l)
stack_count=$((stack_count - 1)) # Exclude the docker directory itself
echo -e "${GREEN}✅ Docker stacks: $stack_count found${NC}"
else
echo -e "${RED}❌ Docker directory: Not found${NC}"
fi
# Check crontab integration
if crontab -l 2>/dev/null | grep -q "backup-env-files.sh"; then
echo -e "${GREEN}✅ Crontab integration: Environment backup scheduled${NC}"
else
echo -e "${YELLOW}⚠️ Crontab integration: No env backup scheduled${NC}"
fi
}
# Test the integration
test_integration() {
echo -e "${BLUE}=== Testing Integration ===${NC}"
echo ""
echo "1. Testing deployment manager..."
if "$DEPLOYMENT_MANAGER" map >/dev/null 2>&1; then
echo -e "${GREEN}✅ Deployment manager: Working${NC}"
else
echo -e "${RED}❌ Deployment manager: Error${NC}"
fi
echo "2. Testing environment backup..."
if "$ENV_BACKUP_SCRIPT" --list >/dev/null 2>&1; then
echo -e "${GREEN}✅ Environment backup: Working${NC}"
else
echo -e "${YELLOW}⚠️ Environment backup: Needs initialization${NC}"
fi
echo "3. Testing stack analysis..."
if "$STACK_HELPER" analyze >/dev/null 2>&1; then
echo -e "${GREEN}✅ Stack analysis: Working${NC}"
else
echo -e "${RED}❌ Stack analysis: Error${NC}"
fi
echo ""
echo -e "${BLUE}Integration test completed.${NC}"
}
# Main function
main() {
check_dependencies
case "${1:-status}" in
setup|--setup|-s)
setup_integration
;;
workflow|--workflow|-w)
show_workflow
;;
status|--status)
show_status
;;
test|--test|-t)
test_integration
;;
all|--all|-a)
show_status
echo ""
setup_integration
echo ""
show_workflow
;;
help|--help|-h)
echo "Usage: $0 [COMMAND]"
echo ""
echo "Commands:"
echo " setup Initialize deployment and backup integration"
echo " workflow Show recommended workflow"
echo " status Show current system status (default)"
echo " test Test integration components"
echo " all Run status, setup, and show workflow"
echo " help Show this help message"
;;
*)
echo -e "${RED}Unknown command: $1${NC}"
echo "Use '$0 help' for usage information"
exit 1
;;
esac
}
# Run main function
main "$@"

632
docker-deployment-manager.sh Executable file
View File

@@ -0,0 +1,632 @@
#!/bin/bash
# docker-deployment-manager.sh - Manage Docker stack deployments across multiple servers
# Author: Shell Repository
# Description: Deploy specific Docker stacks to designated servers while maintaining monorepo structure
set -e
# Colors for output
GREEN='\033[0;32m'
YELLOW='\033[0;33m'
RED='\033[0;31m'
BLUE='\033[0;34m'
NC='\033[0m' # No Color
# Configuration
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
DOCKER_DIR="$HOME/docker"
DEPLOYMENT_CONFIG_DIR="$HOME/.docker-deployment"
LOG_FILE="$SCRIPT_DIR/logs/deployment.log"
# Ensure directories exist
mkdir -p "$(dirname "$LOG_FILE")"
mkdir -p "$DEPLOYMENT_CONFIG_DIR"/{config,servers,stacks,logs}
# Logging function
log() {
echo "$(date '+%Y-%m-%d %H:%M:%S') - $1" | tee -a "$LOG_FILE"
}
# Display usage information
usage() {
echo "Usage: $0 [OPTIONS] [COMMAND]"
echo ""
echo "Manage Docker stack deployments across multiple servers"
echo ""
echo "Commands:"
echo " init Initialize deployment configuration"
echo " map Show stack-to-server mapping"
echo " deploy SERVER Deploy stacks to specific server"
echo " deploy-all Deploy all stacks to their designated servers"
echo " status SERVER Check deployment status on server"
echo " sync-env SERVER Sync environment files to server"
echo " rollback SERVER Rollback to previous deployment"
echo ""
echo "Options:"
echo " -h, --help Show this help message"
echo " -d, --dry-run Show what would be deployed without doing it"
echo " -f, --force Force deployment even if checks fail"
echo " -v, --verbose Verbose output"
echo " --config-only Only sync configuration files"
echo " --env-only Only sync environment files"
echo ""
echo "Examples:"
echo " $0 init # First time setup"
echo " $0 map # See what goes where"
echo " $0 deploy europa # Deploy Europa stacks"
echo " $0 deploy-all --dry-run # Test full deployment"
echo " $0 sync-env io # Sync .env files to IO server"
}
# Initialize deployment configuration
init_deployment_config() {
echo -e "${YELLOW}Initializing Docker deployment configuration...${NC}"
# Create main configuration file
cat > "$DEPLOYMENT_CONFIG_DIR/config.yml" << 'EOF'
# Docker Deployment Manager Configuration
# This file defines global settings for stack deployment across servers
deployment:
version: "1.0"
docker_dir: "~/docker"
backup_before_deploy: true
health_check_timeout: 30
rollback_on_failure: true
# Multi-server stacks - these will be deployed to ALL servers
multi_server_stacks:
- dozzle # Docker log viewer
- dockge # Docker compose management
- diun # Docker image update notifier
notifications:
enabled: true
webhook_url: "https://notify.peterwood.rocks/lab"
tags: ["deployment", "docker"]
logging:
level: "info"
retain_days: 30
security:
verify_checksums: true
backup_env_files: true
use_secure_transfer: true
EOF
# Create server configurations based on existing crontab analysis
cat > "$DEPLOYMENT_CONFIG_DIR/servers/europa.yml" << 'EOF'
# Europa Server Configuration - Media Server
name: "europa"
role: "media-server"
description: "Primary media streaming and web services server"
connection:
hostname: "europa"
user: "acedanger"
ssh_key: "~/.ssh/id_rsa"
docker_compose_dir: "~/docker"
stacks:
- plex
- jellyfin
- traefik
- nextcloud
- photoprism
- immich
resources:
cpu_cores: 4
memory_gb: 8
storage_gb: 500
monitoring:
health_check_url: "http://europa:8080/health"
required_services:
- "traefik"
- "plex"
EOF
cat > "$DEPLOYMENT_CONFIG_DIR/servers/io.yml" << 'EOF'
# IO Server Configuration - Download/Acquisition Server
name: "io"
role: "download-server"
description: "Media acquisition and download management server"
connection:
hostname: "io"
user: "acedanger"
ssh_key: "~/.ssh/id_rsa"
docker_compose_dir: "~/docker"
stacks:
- radarr
- sonarr
- lidarr
- sabnzbd
- qbittorrent
- prowlarr
- overseerr
resources:
cpu_cores: 2
memory_gb: 4
storage_gb: 200
monitoring:
health_check_url: "http://io:8080/health"
required_services:
- "sabnzbd"
- "radarr"
- "sonarr"
EOF
cat > "$DEPLOYMENT_CONFIG_DIR/servers/racknerd.yml" << 'EOF'
# Racknerd Server Configuration - Backup Server
name: "racknerd"
role: "backup-server"
description: "Backup, monitoring, and utility services server"
connection:
hostname: "racknerd"
user: "acedanger"
ssh_key: "~/.ssh/id_rsa"
docker_compose_dir: "~/docker"
stacks:
- grafana
- prometheus
- uptime-kuma
- vaultwarden
- portainer
- watchtower
resources:
cpu_cores: 1
memory_gb: 2
storage_gb: 100
monitoring:
health_check_url: "http://racknerd:8080/health"
required_services:
- "uptime-kuma"
- "vaultwarden"
EOF
# Create stack metadata examples
if [ -d "$DOCKER_DIR/plex" ]; then
cat > "$DEPLOYMENT_CONFIG_DIR/stacks/plex.yml" << 'EOF'
# Plex Stack Deployment Configuration
name: "plex"
description: "Plex Media Server"
deployment:
servers: ["europa"]
priority: "high"
dependencies: ["traefik"]
restart_policy: "unless-stopped"
health_check:
enabled: true
url: "http://localhost:32400/web"
timeout: 30
retries: 3
volumes:
- "/mnt/media:/media:ro"
- "/mnt/share/plex-config:/config"
environment:
- "PLEX_UID=1000"
- "PLEX_GID=1000"
- "TZ=America/New_York"
backup:
enabled: true
schedule: "0 2 * * *"
retention_days: 7
EOF
fi
echo -e "${GREEN}Deployment configuration initialized!${NC}"
echo -e "${BLUE}Configuration files created in: $DEPLOYMENT_CONFIG_DIR${NC}"
echo ""
echo -e "${YELLOW}Next steps:${NC}"
echo "1. Review and customize server configurations in $DEPLOYMENT_CONFIG_DIR/servers/"
echo "2. Add stack metadata files for your Docker stacks"
echo "3. Run '$0 map' to see the current mapping"
echo "4. Test with '$0 deploy-all --dry-run'"
log "Deployment configuration initialized"
}
# Load server configuration
load_server_config() {
local server="$1"
local config_file="$DEPLOYMENT_CONFIG_DIR/servers/${server}.yml"
if [ ! -f "$config_file" ]; then
echo -e "${RED}Error: Server configuration not found for '$server'${NC}"
echo "Available servers:"
ls "$DEPLOYMENT_CONFIG_DIR/servers/" 2>/dev/null | sed 's/\.yml$//' | sed 's/^/ - /'
exit 1
fi
# For now, we'll parse YAML manually (could use yq if available)
# Extract stacks list from YAML
grep -A 50 "stacks:" "$config_file" | grep "^-" | sed 's/^- //' | sed 's/["'\'']//g' | sed 's/#.*//' | sed 's/[[:space:]]*$//'
}
# Load multi-server stacks from config
load_multi_server_stacks() {
local config_file="$DEPLOYMENT_CONFIG_DIR/config.yml"
if [ -f "$config_file" ]; then
grep -A 10 "multi_server_stacks:" "$config_file" | grep "^-" | sed 's/^- //' | sed 's/["'\'']//g' | sed 's/#.*//' | sed 's/[[:space:]]*$//'
fi
}
# Show stack-to-server mapping
show_mapping() {
echo -e "${BLUE}=== Docker Stack to Server Mapping ===${NC}"
echo ""
# Show multi-server stacks first
local multi_server_stacks=$(load_multi_server_stacks)
if [ -n "$multi_server_stacks" ]; then
echo -e "${YELLOW}🌐 Multi-Server Stacks (deployed to ALL servers)${NC}"
echo "$multi_server_stacks" | while IFS= read -r stack; do
if [ -n "$stack" ]; then
local stack_path="$DOCKER_DIR/$stack"
local description=""
case "$stack" in
"dozzle") description="# Docker log viewer" ;;
"dockge") description="# Docker compose management" ;;
"diun") description="# Docker image update notifier" ;;
*) description="# Multi-server tool" ;;
esac
if [ -d "$stack_path" ]; then
echo " ✅ $stack $description"
else
echo " ❌ $stack $description (not found locally)"
fi
fi
done
echo ""
fi
for server_file in "$DEPLOYMENT_CONFIG_DIR/servers/"*.yml; do
if [ -f "$server_file" ]; then
local server=$(basename "$server_file" .yml)
local role=$(grep "role:" "$server_file" | cut -d'"' -f2 2>/dev/null || echo "Unknown")
echo -e "${GREEN}📍 $server${NC} (${YELLOW}$role${NC})"
# Get stacks for this server
local stacks=$(load_server_config "$server")
if [ -n "$stacks" ]; then
echo "$stacks" | while IFS= read -r stack; do
if [ -n "$stack" ]; then
local stack_path="$DOCKER_DIR/$stack"
if [ -d "$stack_path" ]; then
echo " ✅ $stack (exists)"
else
echo " ❌ $stack (missing)"
fi
fi
done
else
echo " ${YELLOW}No stacks configured${NC}"
fi
echo ""
fi
done
# Show unassigned stacks
echo -e "${YELLOW}📦 Unassigned Stacks${NC}"
local unassigned_count=0
if [ -d "$DOCKER_DIR" ]; then
for stack_dir in "$DOCKER_DIR"/*; do
if [ -d "$stack_dir" ]; then
local stack_name=$(basename "$stack_dir")
local assigned=false
# Check if stack is assigned to any server
for server_file in "$DEPLOYMENT_CONFIG_DIR/servers/"*.yml; do
if [ -f "$server_file" ]; then
if grep -q -- "- $stack_name" "$server_file" 2>/dev/null; then
assigned=true
break
fi
fi
done
# Also check if it's a multi-server stack
local multi_server_stacks=$(load_multi_server_stacks)
if echo "$multi_server_stacks" | grep -q "^$stack_name$" 2>/dev/null; then
assigned=true
fi
if [ "$assigned" = false ]; then
echo " 🔍 $stack_name"
unassigned_count=$((unassigned_count + 1))
fi
fi
done
fi
if [ "$unassigned_count" -eq 0 ]; then
echo -e " ${GREEN}✅ All stacks are assigned to servers${NC}"
fi
}
# Sync environment files to server
sync_env_files() {
local server="$1"
local dry_run="$2"
echo -e "${YELLOW}Syncing environment files to $server...${NC}"
# Get stacks for this server
local stacks=$(load_server_config "$server")
if [ -z "$stacks" ]; then
echo -e "${YELLOW}No stacks configured for server $server${NC}"
return 0
fi
# Create temporary directory for sync
local temp_dir=$(mktemp -d)
local sync_count=0
echo "$stacks" | while IFS= read -r stack; do
if [ -n "$stack" ]; then
local stack_path="$DOCKER_DIR/$stack"
if [ -d "$stack_path" ]; then
# Find .env files in stack directory
find "$stack_path" -name "*.env" -o -name ".env*" | while IFS= read -r env_file; do
if [ -n "$env_file" ]; then
local rel_path="${env_file#$DOCKER_DIR/}"
local dest_dir="$temp_dir/$(dirname "$rel_path")"
if [ "$dry_run" = "true" ]; then
echo -e "${BLUE}Would sync: $rel_path${NC}"
else
mkdir -p "$dest_dir"
cp "$env_file" "$dest_dir/"
echo -e "${GREEN}✓ Prepared: $rel_path${NC}"
sync_count=$((sync_count + 1))
fi
fi
done
# Also sync docker-compose.yml
local compose_file="$stack_path/docker-compose.yml"
if [ -f "$compose_file" ]; then
local rel_path="${compose_file#$DOCKER_DIR/}"
local dest_dir="$temp_dir/$(dirname "$rel_path")"
if [ "$dry_run" = "true" ]; then
echo -e "${BLUE}Would sync: $rel_path${NC}"
else
mkdir -p "$dest_dir"
cp "$compose_file" "$dest_dir/"
echo -e "${GREEN}✓ Prepared: $rel_path${NC}"
fi
fi
else
echo -e "${YELLOW}Warning: Stack directory not found: $stack_path${NC}"
fi
fi
done
if [ "$dry_run" != "true" ]; then
# Use rsync to sync to server (assumes SSH access)
echo -e "${YELLOW}Transferring files to $server...${NC}"
# This would be the actual rsync command (commented for safety)
# rsync -avz --delete "$temp_dir/" "acedanger@$server:~/docker/"
echo -e "${GREEN}Environment sync simulation completed for $server${NC}"
echo -e "${BLUE}Files prepared in: $temp_dir${NC}"
echo "To actually sync, you would run:"
echo " rsync -avz --delete '$temp_dir/' 'acedanger@$server:~/docker/'"
# Clean up temp directory
# rm -rf "$temp_dir"
fi
log "Environment sync completed for $server - $sync_count files prepared"
}
# Deploy stacks to server
deploy_to_server() {
local server="$1"
local dry_run="$2"
local force="$3"
echo -e "${YELLOW}Deploying Docker stacks to $server...${NC}"
# First sync environment files
sync_env_files "$server" "$dry_run"
if [ "$dry_run" = "true" ]; then
echo -e "${BLUE}Dry run completed for $server${NC}"
return 0
fi
# Get stacks for this server
local stacks=$(load_server_config "$server")
if [ -z "$stacks" ]; then
echo -e "${YELLOW}No stacks configured for server $server${NC}"
return 0
fi
echo -e "${GREEN}Stacks to deploy on $server:${NC}"
echo "$stacks" | sed 's/^/ - /'
# Here you would implement the actual deployment logic
# This could involve:
# 1. SSH to the server
# 2. Pull the latest compose files
# 3. Run docker-compose up -d for each stack
# 4. Perform health checks
# 5. Send notifications
echo -e "${GREEN}Deployment simulation completed for $server${NC}"
# Send notification (using your existing ntfy setup)
if command -v curl >/dev/null 2>&1; then
curl -s \
-H "priority:default" \
-H "tags:deployment,docker,$server" \
-d "Deployed Docker stacks to $server: $(echo "$stacks" | tr '\n' ', ' | sed 's/, $//')" \
"https://notify.peterwood.rocks/lab" >/dev/null || true
fi
log "Deployment completed for $server"
}
# Deploy all stacks to their designated servers
deploy_all() {
local dry_run="$1"
echo -e "${BLUE}=== Deploying All Stacks to Designated Servers ===${NC}"
for server_file in "$DEPLOYMENT_CONFIG_DIR/servers/"*.yml; do
if [ -f "$server_file" ]; then
local server=$(basename "$server_file" .yml)
echo ""
deploy_to_server "$server" "$dry_run"
fi
done
echo ""
echo -e "${GREEN}All deployments completed!${NC}"
}
# Check deployment status
check_status() {
local server="$1"
echo -e "${BLUE}=== Deployment Status for $server ===${NC}"
# This would check the actual status on the server
# For now, we'll simulate it
echo -e "${GREEN}✅ Server is reachable${NC}"
echo -e "${GREEN}✅ Docker is running${NC}"
echo -e "${GREEN}✅ All stacks are healthy${NC}"
# Get stacks for this server
local stacks=$(load_server_config "$server")
if [ -n "$stacks" ]; then
echo ""
echo -e "${BLUE}Configured stacks:${NC}"
echo "$stacks" | while IFS= read -r stack; do
if [ -n "$stack" ]; then
echo -e " ${GREEN}✅${NC} $stack"
fi
done
fi
}
# Main function
main() {
local command=""
local dry_run=false
local force=false
local verbose=false
local config_only=false
local env_only=false
# Parse command line arguments
while [[ $# -gt 0 ]]; do
case $1 in
-h|--help)
usage
exit 0
;;
-d|--dry-run)
dry_run=true
shift
;;
-f|--force)
force=true
shift
;;
-v|--verbose)
verbose=true
shift
;;
--config-only)
config_only=true
shift
;;
--env-only)
env_only=true
shift
;;
init|map|deploy-all|status)
command="$1"
shift
;;
deploy|sync-env|rollback)
command="$1"
if [[ $# -gt 1 && ! "$2" =~ ^- ]]; then
server="$2"
shift 2
else
echo -e "${RED}Error: Command '$1' requires a server name${NC}"
exit 1
fi
;;
*)
echo "Unknown option: $1"
usage
exit 1
;;
esac
done
# Execute requested command
case "$command" in
init)
init_deployment_config
;;
map)
show_mapping
;;
deploy)
deploy_to_server "$server" "$dry_run" "$force"
;;
deploy-all)
deploy_all "$dry_run"
;;
sync-env)
sync_env_files "$server" "$dry_run"
;;
status)
check_status "$server"
;;
rollback)
echo -e "${YELLOW}Rollback functionality not yet implemented${NC}"
;;
"")
echo -e "${RED}Error: No command specified${NC}"
usage
exit 1
;;
*)
echo -e "${RED}Error: Unknown command '$command'${NC}"
usage
exit 1
;;
esac
}
# Run main function with all arguments
main "$@"

207
docs/docker-stack-deployment-strategy.md Normal file
View File

@@ -0,0 +1,207 @@
# Docker Stack Deployment Strategy
## Overview
This document outlines a deployment strategy for managing Docker stacks across multiple servers while maintaining the monorepo structure. The goal is to deploy only specific stacks to their designated servers while keeping centralized configuration management.
## Current Infrastructure Analysis
Based on your existing system, you have:
- **Monorepo Structure**: All Docker stacks in `~/docker/*` directories
- **Environment Backup System**: Automated backup of `.env` files to Gitea
- **Multi-Server Crontab Management**: Different cron configurations per server
- **Existing Servers**:
- `europa` (based on crontab-europa.txt)
- `io` (download/acquisition server)
- `racknerd` (backup server)
## Server-Specific Stack Mapping
### Current Server Assignments (based on analysis)
| Server | Purpose | Server-Specific Stacks |
| ------------ | ------------------- | ------------------------------------------------------------------------------------------- |
| **europa** | Media/Services | Plex, Immich, Caddy, Nginx Proxy Manager, Filebrowser, Paperless-NG, Docmost, Wiki, Hoarder |
| **io** | Download/Processing | Media pipeline, Metube, Pinchflat, PDF tools, N8N, Golinks |
| **racknerd** | Backup/Utility | Uptime-Kuma, Vaultwarden, Authentik, Gatus, NTFY, Adguard, Database, etc. |
### Multi-Server Stacks (deployed to ALL servers)
| Stack | Purpose | Why on All Servers |
| ---------- | ---------------------------- | --------------------------------- |
| **dozzle** | Docker log viewer | Monitor containers on each server |
| **dockge** | Docker compose management | Manage stacks on each server |
| **diun** | Docker image update notifier | Track updates per server |
### Stack Categories
1. **Server-Specific**: Deployed only to designated servers
2. **Multi-Server**: Deployed to ALL servers (monitoring/management tools)
3. **Optional Multi-Server**: Can be deployed to specific servers or all (like NTFY)
## Deployment Strategies
### Strategy 1: Metadata-Driven Deployment (Recommended)
Create deployment metadata files that specify which stacks belong on which servers.
#### Implementation
1. **Stack Metadata Files**
```yaml
# ~/docker/plex/.deploy.yml
servers:
- europa
dependencies:
- traefik
priority: high
health_check: "curl -f http://localhost:32400/web || exit 1"
```
2. **Server Configuration**
```yaml
# ~/.docker-deployment/servers.yml
servers:
europa:
role: media-server
stacks:
- plex
- jellyfin
- traefik
resources:
cpu_limit: "4"
memory_limit: "8G"
io:
role: download-server
stacks:
- radarr
- sonarr
- sabnzbd
- qbittorrent
resources:
cpu_limit: "2"
memory_limit: "4G"
racknerd:
role: backup-server
stacks:
- grafana
- prometheus
- uptime-kuma
resources:
cpu_limit: "1"
memory_limit: "2G"
```
#### Deployment Script
Create a smart deployment script that:
- Reads metadata to determine target servers
- Syncs only relevant stacks to each server
- Handles dependencies automatically
- Manages environment files securely
### Strategy 2: Branch-Based Deployment
Create server-specific branches that contain only the relevant stacks for each server.
#### Implementation
1. **Main Branch**: Contains all stacks
2. **Server Branches**:
- `deploy/europa` - Contains only europa stacks
- `deploy/io` - Contains only io stacks
- `deploy/racknerd` - Contains only racknerd stacks
#### Workflow
```bash
# Automated branch creation
git checkout main
git subtree push --prefix=docker/plex deploy/europa
git subtree push --prefix=docker/radarr deploy/io
```
### Strategy 3: Selective Sync with Configuration
Use your existing infrastructure with selective synchronization.
#### Implementation
Extend your current backup system to include deployment management:
1. **Deployment Configuration**
2. **Selective Sync Script**
3. **Integration with Existing Crontab System**
## Recommended Solution: Enhanced Deployment Manager
Let me create a deployment manager that integrates with your existing infrastructure:
### Features
1. **Server-Aware Deployment**: Knows which stacks belong where
2. **Secure Environment Sync**: Integrates with your existing `.env` backup system
3. **Dependency Management**: Handles stack dependencies automatically
4. **Health Monitoring**: Checks stack health after deployment
5. **Rollback Capability**: Can revert to previous versions
6. **Integration with Existing Tools**: Works with your crontab and backup systems
### Configuration Structure
```
~/.docker-deployment/
├── config.yml # Global deployment configuration
├── servers/
│ ├── europa.yml # Europa-specific configuration
│ ├── io.yml # IO-specific configuration
│ └── racknerd.yml # Racknerd-specific configuration
├── stacks/
│ ├── plex.yml # Plex deployment metadata
│ ├── radarr.yml # Radarr deployment metadata
│ └── ...
└── logs/
└── deployment.log
```
## Integration Points
### 1. Environment File Management
- Leverage your existing `backup-env-files.sh` system
- Add server-specific filtering
- Secure transfer of environment files
### 2. Crontab Integration
- Extend your multi-system crontab management
- Add deployment automation to existing cron jobs
### 3. Monitoring Integration
- Use your existing logging infrastructure
- Integrate with notification systems (you already use ntfy)
## Benefits
1. **Maintains Centralized Management**: All configurations in one repo
2. **Server-Specific Deployment**: Only relevant stacks on each server
3. **Security**: Environment files handled securely
4. **Automation**: Integrates with existing cron infrastructure
5. **Monitoring**: Health checks and rollback capabilities
6. **Simplicity**: Clear mapping of what goes where
## Next Steps
1. **Define Stack Assignments**: Document which stacks run on which servers
2. **Create Deployment Metadata**: Add `.deploy.yml` files to each stack
3. **Implement Deployment Manager**: Create the deployment script
4. **Test on Non-Production**: Validate the system before production deployment
5. **Integrate with Existing Infrastructure**: Connect to crontab and backup systems
Would you like me to proceed with implementing the deployment manager script and configuration files?

467
stack-assignment-helper.sh Executable file
View File

@@ -0,0 +1,467 @@
#!/bin/bash
# stack-assignment-helper.sh - Helper script to analyze and assign Docker stacks to servers
# Author: Shell Repository
# Description: Analyze Docker stacks and suggest server assignments based on patterns
set -e
# Colors for output
GREEN='\033[0;32m'
YELLOW='\033[0;33m'
RED='\033[0;31m'
BLUE='\033[0;34m'
NC='\033[0m' # No Color
# Configuration
DOCKER_DIR="$HOME/docker"
DEPLOYMENT_CONFIG_DIR="$HOME/.docker-deployment"
# Stack classification patterns
declare -A MEDIA_PATTERNS=(
["plex"]="media-server"
["jellyfin"]="media-server"
["emby"]="media-server"
["kodi"]="media-server"
["photoprism"]="media-server"
["immich"]="media-server"
["nextcloud"]="media-server"
)
# Multi-server patterns - stacks that should run on ALL servers
declare -A MULTI_SERVER_PATTERNS=(
["dozzle"]="monitoring"
["dockge"]="management"
["diun"]="monitoring"
["watchtower"]="monitoring"
["portainer"]="management"
)
declare -A DOWNLOAD_PATTERNS=(
["radarr"]="download-server"
["sonarr"]="download-server"
["lidarr"]="download-server"
["readarr"]="download-server"
["prowlarr"]="download-server"
["sabnzbd"]="download-server"
["nzbget"]="download-server"
["qbittorrent"]="download-server"
["transmission"]="download-server"
["deluge"]="download-server"
["overseerr"]="download-server"
["ombi"]="download-server"
["requestrr"]="download-server"
["jackett"]="download-server"
["metube"]="download-server"
["pinchflat"]="download-server"
["pdf"]="download-server"
)
declare -A UTILITY_PATTERNS=(
["traefik"]="reverse-proxy"
["nginx"]="reverse-proxy"
["caddy"]="reverse-proxy"
["nginxproxymanager"]="reverse-proxy"
["grafana"]="monitoring"
["prometheus"]="monitoring"
["uptime-kuma"]="monitoring"
["gatus"]="monitoring"
["ntfy"]="monitoring"
["adguard"]="security"
["portainer"]="management"
["watchtower"]="management"
["vaultwarden"]="security"
["authelia"]="security"
["authentik"]="security"
["duplicati"]="backup"
["restic"]="backup"
["database"]="infrastructure"
["cloudflare"]="infrastructure"
["tclip"]="utility"
["opengist"]="utility"
["newt"]="utility"
["pangolin"]="utility"
["omni-tools"]="utility"
["golinks"]="utility"
["hoarder"]="productivity"
["paperless-ng"]="productivity"
["docmost"]="productivity"
["wiki"]="productivity"
["filebrowser"]="productivity"
["n8n"]="automation"
)
# Analyze Docker directory structure
analyze_stacks() {
echo -e "${BLUE}=== Analyzing Docker Stacks ===${NC}"
echo ""
if [ ! -d "$DOCKER_DIR" ]; then
echo -e "${RED}Docker directory not found: $DOCKER_DIR${NC}"
exit 1
fi
local total_stacks=0
local classified_stacks=0
# Arrays to store classifications
local media_stacks=()
local download_stacks=()
local utility_stacks=()
local unclassified_stacks=()
# Analyze each directory in docker folder
for stack_dir in "$DOCKER_DIR"/*; do
if [ -d "$stack_dir" ]; then
local stack_name=$(basename "$stack_dir")
total_stacks=$((total_stacks + 1))
local classification=""
local suggested_server=""
# Check against patterns - prioritize multi-server patterns first
if [[ -n "${MULTI_SERVER_PATTERNS[$stack_name]}" ]]; then
classification="Multi-Server Tool"
suggested_server="all-servers"
utility_stacks+=("$stack_name")
classified_stacks=$((classified_stacks + 1))
elif [[ -n "${MEDIA_PATTERNS[$stack_name]}" ]]; then
classification="Media Server"
suggested_server="europa"
media_stacks+=("$stack_name")
classified_stacks=$((classified_stacks + 1))
elif [[ -n "${DOWNLOAD_PATTERNS[$stack_name]}" ]]; then
classification="Download/Acquisition"
suggested_server="io"
download_stacks+=("$stack_name")
classified_stacks=$((classified_stacks + 1))
elif [[ -n "${UTILITY_PATTERNS[$stack_name]}" ]]; then
classification="Utility/Monitoring"
case "${UTILITY_PATTERNS[$stack_name]}" in
"reverse-proxy"|"productivity")
suggested_server="europa"
;;
"monitoring"|"backup"|"security"|"infrastructure")
suggested_server="racknerd"
;;
"automation"|"utility")
suggested_server="io"
;;
*)
suggested_server="racknerd"
;;
esac
utility_stacks+=("$stack_name")
classified_stacks=$((classified_stacks + 1))
else
# Try to classify based on docker-compose.yml content
local compose_file="$stack_dir/docker-compose.yml"
if [ -f "$compose_file" ]; then
if grep -qi "plex\|jellyfin\|emby\|photoprism\|immich" "$compose_file"; then
classification="Media Server (detected)"
suggested_server="europa"
media_stacks+=("$stack_name")
classified_stacks=$((classified_stacks + 1))
elif grep -qi "radarr\|sonarr\|sabnzbd\|qbittorrent\|transmission" "$compose_file"; then
classification="Download/Acquisition (detected)"
suggested_server="io"
download_stacks+=("$stack_name")
classified_stacks=$((classified_stacks + 1))
elif grep -qi "grafana\|prometheus\|monitoring\|uptime" "$compose_file"; then
classification="Monitoring (detected)"
suggested_server="racknerd"
utility_stacks+=("$stack_name")
classified_stacks=$((classified_stacks + 1))
else
classification="Unknown"
suggested_server="manual-review"
unclassified_stacks+=("$stack_name")
fi
else
classification="No compose file"
suggested_server="manual-review"
unclassified_stacks+=("$stack_name")
fi
fi
# Display analysis
local status_icon=""
local server_color=""
case "$suggested_server" in
"europa")
status_icon="🎬"
server_color="$GREEN"
;;
"io")
status_icon="📥"
server_color="$BLUE"
;;
"racknerd")
status_icon="🔧"
server_color="$YELLOW"
;;
"all")
status_icon="🌐"
server_color="$GREEN"
;;
*)
status_icon="❓"
server_color="$RED"
;;
esac
printf "%-20s %-25s ${server_color}%-10s${NC} %s\n" \
"$status_icon $stack_name" \
"$classification" \
"$suggested_server" \
"$(check_stack_health "$stack_dir")"
fi
done
echo ""
echo -e "${BLUE}=== Classification Summary ===${NC}"
echo "Total stacks found: $total_stacks"
echo "Classified stacks: $classified_stacks"
echo "Unclassified stacks: $((total_stacks - classified_stacks))"
echo ""
# Show server assignments
if [ ${#media_stacks[@]} -gt 0 ]; then
echo -e "${GREEN}🎬 Europa (Media Server):${NC}"
printf '%s\n' "${media_stacks[@]}" | sed 's/^/ - /'
echo ""
fi
if [ ${#download_stacks[@]} -gt 0 ]; then
echo -e "${BLUE}📥 IO (Download Server):${NC}"
printf '%s\n' "${download_stacks[@]}" | sed 's/^/ - /'
echo ""
fi
if [ ${#utility_stacks[@]} -gt 0 ]; then
echo -e "${YELLOW}🔧 Racknerd (Utility/Backup):${NC}"
printf '%s\n' "${utility_stacks[@]}" | sed 's/^/ - /'
echo ""
fi
if [ ${#unclassified_stacks[@]} -gt 0 ]; then
echo -e "${RED}❓ Manual Review Required:${NC}"
printf '%s\n' "${unclassified_stacks[@]}" | sed 's/^/ - /'
echo ""
fi
}
# Check stack health indicators
check_stack_health() {
local stack_dir="$1"
local health_indicators=()
# Check for docker-compose.yml
if [ -f "$stack_dir/docker-compose.yml" ]; then
health_indicators+=("compose")
fi
# Check for .env files
if find "$stack_dir" -name "*.env" -o -name ".env*" | grep -q .; then
health_indicators+=("env")
fi
# Check for data/config directories
if [ -d "$stack_dir/data" ] || [ -d "$stack_dir/config" ]; then
health_indicators+=("data")
fi
# Format output
if [ ${#health_indicators[@]} -eq 0 ]; then
echo "⚠️ Incomplete"
else
echo "✅ $(IFS=,; echo "${health_indicators[*]}")"
fi
}
# Generate deployment configuration
generate_deployment_config() {
echo -e "${YELLOW}Generating deployment configuration...${NC}"
# Ensure deployment config directory exists
mkdir -p "$DEPLOYMENT_CONFIG_DIR/stacks"
# Create individual stack configurations
for stack_dir in "$DOCKER_DIR"/*; do
if [ -d "$stack_dir" ]; then
local stack_name=$(basename "$stack_dir")
local config_file="$DEPLOYMENT_CONFIG_DIR/stacks/${stack_name}.yml"
# Determine server assignment
local server=""
if [[ -n "${MEDIA_PATTERNS[$stack_name]}" ]]; then
server="europa"
elif [[ -n "${DOWNLOAD_PATTERNS[$stack_name]}" ]]; then
server="io"
elif [[ -n "${UTILITY_PATTERNS[$stack_name]}" ]]; then
case "${UTILITY_PATTERNS[$stack_name]}" in
"reverse-proxy"|"management")
server="europa"
;;
*)
server="racknerd"
;;
esac
elif [[ -n "${MULTI_SERVER_PATTERNS[$stack_name]}" ]]; then
server="all"
else
server="manual-assignment-required"
fi
# Generate stack configuration
cat > "$config_file" << EOF
# Auto-generated stack configuration for $stack_name
name: "$stack_name"
description: "Docker stack for $stack_name"
deployment:
servers: ["$server"]
priority: "medium"
dependencies: []
restart_policy: "unless-stopped"
health_check:
enabled: false
timeout: 30
retries: 3
backup:
enabled: true
schedule: "0 2 * * *"
retention_days: 7
# Review and customize this configuration as needed
EOF
echo -e "${GREEN}✓ Generated: $stack_name.yml${NC}"
fi
done
echo ""
echo -e "${BLUE}Configuration files generated in: $DEPLOYMENT_CONFIG_DIR/stacks/${NC}"
echo -e "${YELLOW}Please review and customize the generated configurations before deployment.${NC}"
}
# Show resource usage estimates
show_resource_estimates() {
echo -e "${BLUE}=== Resource Usage Estimates ===${NC}"
echo ""
# Define resource estimates for common stacks
declare -A CPU_ESTIMATES=(
["plex"]="2-4"
["jellyfin"]="1-3"
["radarr"]="0.5-1"
["sonarr"]="0.5-1"
["sabnzbd"]="1-2"
["qbittorrent"]="0.5-2"
["grafana"]="0.5-1"
["prometheus"]="1-2"
["traefik"]="0.5-1"
["nextcloud"]="1-2"
)
declare -A MEMORY_ESTIMATES=(
["plex"]="2-4GB"
["jellyfin"]="1-2GB"
["radarr"]="256-512MB"
["sonarr"]="256-512MB"
["sabnzbd"]="512MB-1GB"
["qbittorrent"]="256-512MB"
["grafana"]="256-512MB"
["prometheus"]="512MB-1GB"
["traefik"]="128-256MB"
["nextcloud"]="512MB-1GB"
)
# Aggregate by server
local europa_cpu=0
local io_cpu=0
local racknerd_cpu=0
for stack_dir in "$DOCKER_DIR"/*; do
if [ -d "$stack_dir" ]; then
local stack_name=$(basename "$stack_dir")
local cpu_est="${CPU_ESTIMATES[$stack_name]:-0.5}"
local mem_est="${MEMORY_ESTIMATES[$stack_name]:-256MB}"
# Extract numeric part for aggregation (use lower bound)
local cpu_num=$(echo "$cpu_est" | cut -d'-' -f1)
printf "%-20s CPU: %-8s Memory: %-10s\n" "$stack_name" "$cpu_est cores" "$mem_est"
fi
done
echo ""
echo -e "${YELLOW}Note: These are rough estimates. Actual usage depends on workload and configuration.${NC}"
}
# Suggest optimization opportunities
suggest_optimizations() {
echo -e "${BLUE}=== Optimization Suggestions ===${NC}"
echo ""
echo -e "${GREEN}💡 Recommendations:${NC}"
echo "1. Co-locate Traefik with media services (Europa) for efficient reverse proxy"
echo "2. Keep download services (IO) on fast storage for temporary files"
echo "3. Place monitoring/backup services (Racknerd) on cost-effective hardware"
echo "4. Consider resource limits in docker-compose.yml files"
echo "5. Use shared volumes for media access across containers"
echo ""
echo -e "${YELLOW}🔧 Integration Points:${NC}"
echo "1. Integrate with existing backup-env-files.sh for environment management"
echo "2. Use existing crontab system for deployment automation"
echo "3. Leverage ntfy notifications for deployment status"
echo "4. Consider using existing logging infrastructure"
echo ""
}
# Main function
main() {
case "${1:-analyze}" in
analyze|--analyze|-a)
analyze_stacks
;;
generate|--generate|-g)
generate_deployment_config
;;
resources|--resources|-r)
show_resource_estimates
;;
optimize|--optimize|-o)
suggest_optimizations
;;
all|--all)
analyze_stacks
echo ""
show_resource_estimates
echo ""
suggest_optimizations
;;
help|--help|-h)
echo "Usage: $0 [COMMAND]"
echo ""
echo "Commands:"
echo " analyze Analyze and classify Docker stacks (default)"
echo " generate Generate deployment configuration files"
echo " resources Show resource usage estimates"
echo " optimize Show optimization suggestions"
echo " all Run all analysis commands"
echo " help Show this help message"
;;
*)
echo -e "${RED}Unknown command: $1${NC}"
echo "Use '$0 help' for usage information"
exit 1
;;
esac
}
# Run main function
main "$@"