#!/bin/bash # Site11 Pipeline K8s Local Deployment Script # =========================================== # Deploys pipeline workers to K8s with external infrastructure (Docker Compose) set -e echo "πŸš€ Site11 Pipeline K8s Local Deployment (AWS-like Environment)" echo "==============================================================" echo "" echo "This deployment simulates AWS architecture:" echo " - Infrastructure: External (Docker Compose) - simulates AWS managed services" echo " - Workers: K8s (local cluster) - simulates EKS workloads" echo "" # Colors for output RED='\033[0;31m' GREEN='\033[0;32m' YELLOW='\033[1;33m' BLUE='\033[0;34m' NC='\033[0m' # No Color # Check prerequisites echo -e "${BLUE}Checking prerequisites...${NC}" # Check if kubectl is available if ! command -v kubectl &> /dev/null; then echo -e "${RED}❌ kubectl is not installed${NC}" exit 1 fi # Check K8s cluster connection echo -n " K8s cluster connection... " if kubectl cluster-info &> /dev/null; then echo -e "${GREEN}βœ“${NC}" else echo -e "${RED}βœ— Cannot connect to K8s cluster${NC}" exit 1 fi # Check if Docker infrastructure is running echo -n " Docker infrastructure services... " if docker ps | grep -q "site11_mongodb" && docker ps | grep -q "site11_redis"; then echo -e "${GREEN}βœ“${NC}" else echo -e "${YELLOW}⚠️ Infrastructure not running. Start with: docker-compose -f docker-compose-hybrid.yml up -d${NC}" exit 1 fi # Check local registry echo -n " Local registry (port 5555)... " if docker ps | grep -q "site11_registry"; then echo -e "${GREEN}βœ“${NC}" else echo -e "${YELLOW}⚠️ Registry not running. Start with: docker-compose -f docker-compose-hybrid.yml up -d registry${NC}" exit 1 fi # Step 1: Create namespace echo "" echo -e "${BLUE}1. Creating K8s namespace...${NC}" kubectl apply -f namespace.yaml # Step 2: Create ConfigMap and Secrets for external services echo "" echo -e "${BLUE}2. Configuring external service connections...${NC}" cat > configmap-local.yaml << 'EOF' apiVersion: v1 kind: ConfigMap metadata: name: pipeline-config namespace: site11-pipeline data: # External Redis (Docker host) - simulates AWS ElastiCache REDIS_URL: "redis://host.docker.internal:6379" # External MongoDB (Docker host) - simulates AWS DocumentDB MONGODB_URL: "mongodb://host.docker.internal:27017" DB_NAME: "ai_writer_db" # Logging LOG_LEVEL: "INFO" # Worker settings WORKER_COUNT: "2" BATCH_SIZE: "10" # Queue delays RSS_ENQUEUE_DELAY: "1.0" GOOGLE_SEARCH_DELAY: "2.0" TRANSLATION_DELAY: "1.0" --- apiVersion: v1 kind: Secret metadata: name: pipeline-secrets namespace: site11-pipeline type: Opaque stringData: DEEPL_API_KEY: "3abbc796-2515-44a8-972d-22dcf27ab54a" CLAUDE_API_KEY: "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" OPENAI_API_KEY: "sk-openai-api-key-here" # Replace with actual key SERP_API_KEY: "serp-api-key-here" # Replace with actual key EOF kubectl apply -f configmap-local.yaml # Step 3: Update deployment YAMLs to use local registry echo "" echo -e "${BLUE}3. Updating deployments for local registry...${NC}" services=("rss-collector" "google-search" "translator" "ai-article-generator" "image-generator") for service in "${services[@]}"; do # Update image references in deployment files sed -i.bak "s|image: site11/pipeline-$service:latest|image: localhost:5555/pipeline-$service:latest|g" $service.yaml 2>/dev/null || \ sed -i '' "s|image: site11/pipeline-$service:latest|image: localhost:5555/pipeline-$service:latest|g" $service.yaml done # Step 4: Push images to local registry echo "" echo -e "${BLUE}4. Pushing images to local registry...${NC}" for service in "${services[@]}"; do echo -n " Pushing pipeline-$service... " docker tag site11-pipeline-$service:latest localhost:5555/pipeline-$service:latest 2>/dev/null docker push localhost:5555/pipeline-$service:latest 2>/dev/null && echo -e "${GREEN}βœ“${NC}" || echo -e "${YELLOW}already exists${NC}" done # Step 5: Deploy services to K8s echo "" echo -e "${BLUE}5. Deploying workers to K8s...${NC}" for service in "${services[@]}"; do echo -n " Deploying $service... " kubectl apply -f $service.yaml && echo -e "${GREEN}βœ“${NC}" done # Step 6: Check deployment status echo "" echo -e "${BLUE}6. Verifying deployments...${NC}" kubectl -n site11-pipeline get deployments echo "" echo -e "${BLUE}7. Waiting for pods to be ready...${NC}" kubectl -n site11-pipeline wait --for=condition=Ready pods --all --timeout=60s 2>/dev/null || { echo -e "${YELLOW}⚠️ Some pods are still initializing...${NC}" } # Step 7: Show final status echo "" echo -e "${GREEN}βœ… Deployment Complete!${NC}" echo "" echo -e "${BLUE}Current pod status:${NC}" kubectl -n site11-pipeline get pods echo "" echo -e "${BLUE}External infrastructure status:${NC}" docker ps --format "table {{.Names}}\t{{.Status}}" | grep -E "site11_(mongodb|redis|kafka|zookeeper|registry)" || echo "No infrastructure services found" echo "" echo -e "${BLUE}Useful commands:${NC}" echo " View logs: kubectl -n site11-pipeline logs -f deployment/pipeline-translator" echo " Scale workers: kubectl -n site11-pipeline scale deployment pipeline-translator --replicas=5" echo " Check HPA: kubectl -n site11-pipeline get hpa" echo " Monitor queues: docker-compose -f docker-compose-hybrid.yml logs -f pipeline-monitor" echo " Delete K8s: kubectl delete namespace site11-pipeline" echo "" echo -e "${BLUE}Architecture Overview:${NC}" echo " πŸ“¦ Infrastructure (Docker): MongoDB, Redis, Kafka, Registry" echo " ☸️ Workers (K8s): RSS, Search, Translation, AI Generation, Image Generation" echo " πŸŽ›οΈ Control (Docker): Scheduler, Monitor, Language Sync"