#!/bin/bash # Site11 Pipeline Kind Deployment Script # ======================================= # Deploys pipeline workers to Kind cluster with external infrastructure set -e echo "πŸš€ Site11 Pipeline Kind Deployment" echo "===================================" echo "" echo "This deployment uses:" echo " - Infrastructure: External (Docker Compose)" echo " - Workers: Kind K8s cluster" echo "" # Colors for output RED='\033[0;31m' GREEN='\033[0;32m' YELLOW='\033[1;33m' BLUE='\033[0;34m' NC='\033[0m' # No Color # Check prerequisites echo -e "${BLUE}Checking prerequisites...${NC}" # Check if kind is available if ! command -v kind &> /dev/null; then echo -e "${RED}❌ kind is not installed${NC}" echo "Install with: brew install kind" exit 1 fi # Check if Docker infrastructure is running echo -n " Docker infrastructure services... " if docker ps | grep -q "site11_mongodb" && docker ps | grep -q "site11_redis"; then echo -e "${GREEN}βœ“${NC}" else echo -e "${YELLOW}⚠️ Infrastructure not running. Start with: docker-compose -f docker-compose-hybrid.yml up -d${NC}" exit 1 fi # Step 1: Create or use existing Kind cluster echo "" echo -e "${BLUE}1. Setting up Kind cluster...${NC}" if kind get clusters | grep -q "site11-cluster"; then echo " Using existing site11-cluster" kubectl config use-context kind-site11-cluster else echo " Creating new Kind cluster..." kind create cluster --config kind-config.yaml fi # Step 2: Load Docker images to Kind echo "" echo -e "${BLUE}2. Loading Docker images to Kind cluster...${NC}" services=("rss-collector" "google-search" "translator" "ai-article-generator" "image-generator") for service in "${services[@]}"; do echo -n " Loading pipeline-$service... " kind load docker-image site11-pipeline-$service:latest --name site11-cluster && echo -e "${GREEN}βœ“${NC}" done # Step 3: Create namespace echo "" echo -e "${BLUE}3. Creating K8s namespace...${NC}" kubectl apply -f namespace.yaml # Step 4: Create ConfigMap and Secrets for external services echo "" echo -e "${BLUE}4. Configuring external service connections...${NC}" cat > configmap-kind.yaml << 'EOF' apiVersion: v1 kind: ConfigMap metadata: name: pipeline-config namespace: site11-pipeline data: # External Redis (host network) - Docker services REDIS_URL: "redis://host.docker.internal:6379" # External MongoDB (host network) - Docker services MONGODB_URL: "mongodb://host.docker.internal:27017" DB_NAME: "ai_writer_db" # Logging LOG_LEVEL: "INFO" # Worker settings WORKER_COUNT: "2" BATCH_SIZE: "10" # Queue delays RSS_ENQUEUE_DELAY: "1.0" GOOGLE_SEARCH_DELAY: "2.0" TRANSLATION_DELAY: "1.0" --- apiVersion: v1 kind: Secret metadata: name: pipeline-secrets namespace: site11-pipeline type: Opaque stringData: DEEPL_API_KEY: "3abbc796-2515-44a8-972d-22dcf27ab54a" CLAUDE_API_KEY: "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" OPENAI_API_KEY: "sk-openai-api-key-here" # Replace with actual key SERP_API_KEY: "serp-api-key-here" # Replace with actual key EOF kubectl apply -f configmap-kind.yaml # Step 5: Create deployments for Kind echo "" echo -e "${BLUE}5. Creating deployments for Kind...${NC}" for service in "${services[@]}"; do cat > ${service}-kind.yaml << EOF apiVersion: apps/v1 kind: Deployment metadata: name: pipeline-$service namespace: site11-pipeline labels: app: pipeline-$service spec: replicas: $([ "$service" = "translator" ] && echo "3" || echo "2") selector: matchLabels: app: pipeline-$service template: metadata: labels: app: pipeline-$service spec: containers: - name: $service image: site11-pipeline-$service:latest imagePullPolicy: Never # Use loaded image envFrom: - configMapRef: name: pipeline-config - secretRef: name: pipeline-secrets resources: requests: memory: "256Mi" cpu: "100m" limits: memory: "512Mi" cpu: "500m" readinessProbe: exec: command: - python - -c - "import sys; sys.exit(0)" initialDelaySeconds: 10 periodSeconds: 5 livenessProbe: exec: command: - python - -c - "import sys; sys.exit(0)" initialDelaySeconds: 30 periodSeconds: 10 --- apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler metadata: name: pipeline-$service-hpa namespace: site11-pipeline spec: scaleTargetRef: apiVersion: apps/v1 kind: Deployment name: pipeline-$service minReplicas: $([ "$service" = "translator" ] && echo "3" || echo "2") maxReplicas: 10 metrics: - type: Resource resource: name: cpu target: type: Utilization averageUtilization: 70 - type: Resource resource: name: memory target: type: Utilization averageUtilization: 80 EOF done # Step 6: Deploy services to K8s echo "" echo -e "${BLUE}6. Deploying workers to Kind cluster...${NC}" for service in "${services[@]}"; do echo -n " Deploying $service... " kubectl apply -f ${service}-kind.yaml && echo -e "${GREEN}βœ“${NC}" done # Step 7: Check deployment status echo "" echo -e "${BLUE}7. Verifying deployments...${NC}" kubectl -n site11-pipeline get deployments echo "" echo -e "${BLUE}8. Waiting for pods to be ready...${NC}" kubectl -n site11-pipeline wait --for=condition=Ready pods --all --timeout=120s 2>/dev/null || { echo -e "${YELLOW}⚠️ Some pods are still initializing...${NC}" } # Step 8: Show final status echo "" echo -e "${GREEN}βœ… Deployment Complete!${NC}" echo "" echo -e "${BLUE}Current pod status:${NC}" kubectl -n site11-pipeline get pods echo "" echo -e "${BLUE}External infrastructure status:${NC}" docker ps --format "table {{.Names}}\t{{.Status}}" | grep -E "site11_(mongodb|redis|kafka|zookeeper)" || echo "No infrastructure services found" echo "" echo -e "${BLUE}Useful commands:${NC}" echo " View logs: kubectl -n site11-pipeline logs -f deployment/pipeline-translator" echo " Scale workers: kubectl -n site11-pipeline scale deployment pipeline-translator --replicas=5" echo " Check HPA: kubectl -n site11-pipeline get hpa" echo " Monitor queues: docker-compose -f docker-compose-hybrid.yml logs -f pipeline-monitor" echo " Delete cluster: kind delete cluster --name site11-cluster" echo "" echo -e "${BLUE}Architecture Overview:${NC}" echo " πŸ“¦ Infrastructure (Docker): MongoDB, Redis, Kafka" echo " ☸️ Workers (Kind K8s): RSS, Search, Translation, AI Generation, Image Generation" echo " πŸŽ›οΈ Control (Docker): Scheduler, Monitor, Language Sync" echo "" echo -e "${YELLOW}Note: Kind uses 'host.docker.internal' to access host services${NC}"