feat: Add hybrid deployment with Docker and Kubernetes
- Docker Compose for infrastructure (MongoDB, Redis, Kafka, Zookeeper) - Docker for central control (Scheduler, Monitor, Language Sync) - K8s for scalable workers (RSS, Google Search, Translator, AI Generator, Image Generator) - Automatic scaling with HPA (Horizontal Pod Autoscaler) - Comprehensive deployment scripts and documentation - Updated README with hybrid deployment guide
This commit is contained in:
jungwoo choi
78
k8s/pipeline/ai-article-generator.yaml
Normal file
78
k8s/pipeline/ai-article-generator.yaml
Normal file
@ -0,0 +1,78 @@
|
||||
apiVersion: apps/v1
|
||||
kind: Deployment
|
||||
metadata:
|
||||
name: pipeline-ai-article-generator
|
||||
namespace: site11-pipeline
|
||||
labels:
|
||||
app: pipeline-ai-article-generator
|
||||
component: processor
|
||||
spec:
|
||||
replicas: 2
|
||||
selector:
|
||||
matchLabels:
|
||||
app: pipeline-ai-article-generator
|
||||
template:
|
||||
metadata:
|
||||
labels:
|
||||
app: pipeline-ai-article-generator
|
||||
component: processor
|
||||
spec:
|
||||
containers:
|
||||
- name: ai-article-generator
|
||||
image: site11/pipeline-ai-article-generator:latest
|
||||
imagePullPolicy: Always
|
||||
envFrom:
|
||||
- configMapRef:
|
||||
name: pipeline-config
|
||||
- secretRef:
|
||||
name: pipeline-secrets
|
||||
resources:
|
||||
requests:
|
||||
memory: "512Mi"
|
||||
cpu: "200m"
|
||||
limits:
|
||||
memory: "1Gi"
|
||||
cpu: "1000m"
|
||||
livenessProbe:
|
||||
exec:
|
||||
command:
|
||||
- python
|
||||
- -c
|
||||
- "import redis; r=redis.from_url('redis://host.docker.internal:6379'); r.ping()"
|
||||
initialDelaySeconds: 30
|
||||
periodSeconds: 30
|
||||
readinessProbe:
|
||||
exec:
|
||||
command:
|
||||
- python
|
||||
- -c
|
||||
- "import redis; r=redis.from_url('redis://host.docker.internal:6379'); r.ping()"
|
||||
initialDelaySeconds: 10
|
||||
periodSeconds: 10
|
||||
|
||||
---
|
||||
apiVersion: autoscaling/v2
|
||||
kind: HorizontalPodAutoscaler
|
||||
metadata:
|
||||
name: pipeline-ai-article-generator-hpa
|
||||
namespace: site11-pipeline
|
||||
spec:
|
||||
scaleTargetRef:
|
||||
apiVersion: apps/v1
|
||||
kind: Deployment
|
||||
name: pipeline-ai-article-generator
|
||||
minReplicas: 1
|
||||
maxReplicas: 8
|
||||
metrics:
|
||||
- type: Resource
|
||||
resource:
|
||||
name: cpu
|
||||
target:
|
||||
type: Utilization
|
||||
averageUtilization: 70
|
||||
- type: Resource
|
||||
resource:
|
||||
name: memory
|
||||
target:
|
||||
type: Utilization
|
||||
averageUtilization: 80
|
||||
38
k8s/pipeline/configmap.yaml
Normal file
38
k8s/pipeline/configmap.yaml
Normal file
@ -0,0 +1,38 @@
|
||||
apiVersion: v1
|
||||
kind: ConfigMap
|
||||
metadata:
|
||||
name: pipeline-config
|
||||
namespace: site11-pipeline
|
||||
data:
|
||||
# Redis 연결 (Docker 호스트)
|
||||
REDIS_URL: "redis://host.docker.internal:6379"
|
||||
|
||||
# MongoDB 연결 (Docker 호스트)
|
||||
MONGODB_URL: "mongodb://host.docker.internal:27017"
|
||||
DB_NAME: "ai_writer_db"
|
||||
|
||||
# 로깅
|
||||
LOG_LEVEL: "INFO"
|
||||
|
||||
# 워커 설정
|
||||
WORKER_COUNT: "2"
|
||||
BATCH_SIZE: "10"
|
||||
|
||||
# 큐 설정
|
||||
RSS_ENQUEUE_DELAY: "1.0"
|
||||
GOOGLE_SEARCH_DELAY: "2.0"
|
||||
TRANSLATION_DELAY: "1.0"
|
||||
|
||||
---
|
||||
apiVersion: v1
|
||||
kind: Secret
|
||||
metadata:
|
||||
name: pipeline-secrets
|
||||
namespace: site11-pipeline
|
||||
type: Opaque
|
||||
stringData:
|
||||
# API Keys (실제 값)
|
||||
DEEPL_API_KEY: "3abbc796-2515-44a8-972d-22dcf27ab54a"
|
||||
OPENAI_API_KEY: "sk-openai-api-key-here" # OpenAI API 키 필요
|
||||
CLAUDE_API_KEY: "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"
|
||||
SERP_API_KEY: "serp-api-key-here" # SERP API 키 필요
|
||||
96
k8s/pipeline/deploy.sh
Executable file
96
k8s/pipeline/deploy.sh
Executable file
@ -0,0 +1,96 @@
|
||||
#!/bin/bash
|
||||
|
||||
# Site11 Pipeline K8s Deployment Script
|
||||
# ======================================
|
||||
|
||||
set -e
|
||||
|
||||
echo "🚀 Site11 Pipeline K8s Deployment"
|
||||
echo "================================="
|
||||
|
||||
# Colors for output
|
||||
RED='\033[0;31m'
|
||||
GREEN='\033[0;32m'
|
||||
YELLOW='\033[1;33m'
|
||||
NC='\033[0m' # No Color
|
||||
|
||||
# Check if kubectl is available
|
||||
if ! command -v kubectl &> /dev/null; then
|
||||
echo -e "${RED}❌ kubectl is not installed${NC}"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# Check K8s cluster connection
|
||||
echo -n "Checking K8s cluster connection... "
|
||||
if kubectl cluster-info &> /dev/null; then
|
||||
echo -e "${GREEN}✓${NC}"
|
||||
else
|
||||
echo -e "${RED}✗ Cannot connect to K8s cluster${NC}"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# Step 1: Create namespace
|
||||
echo ""
|
||||
echo "1. Creating namespace..."
|
||||
kubectl apply -f namespace.yaml
|
||||
|
||||
# Step 2: Create ConfigMap and Secrets
|
||||
echo ""
|
||||
echo "2. Creating ConfigMap and Secrets..."
|
||||
echo -e "${YELLOW}⚠️ Remember to update API keys in configmap.yaml${NC}"
|
||||
kubectl apply -f configmap.yaml
|
||||
|
||||
# Step 3: Build and push Docker images
|
||||
echo ""
|
||||
echo "3. Building Docker images..."
|
||||
echo -e "${YELLOW}Note: This script assumes images are already built${NC}"
|
||||
echo "To build images, run from project root:"
|
||||
echo " docker-compose build pipeline-rss-collector"
|
||||
echo " docker-compose build pipeline-google-search"
|
||||
echo " docker-compose build pipeline-translator"
|
||||
echo " docker-compose build pipeline-ai-article-generator"
|
||||
echo " docker-compose build pipeline-image-generator"
|
||||
|
||||
# Step 4: Tag and push images (if using local registry)
|
||||
echo ""
|
||||
echo "4. Tagging images for K8s..."
|
||||
services=("rss-collector" "google-search" "translator" "ai-article-generator" "image-generator")
|
||||
|
||||
for service in "${services[@]}"; do
|
||||
echo "Tagging pipeline-$service..."
|
||||
docker tag site11_pipeline-$service:latest site11/pipeline-$service:latest
|
||||
done
|
||||
|
||||
# Step 5: Deploy services
|
||||
echo ""
|
||||
echo "5. Deploying services to K8s..."
|
||||
|
||||
for service in "${services[@]}"; do
|
||||
echo "Deploying $service..."
|
||||
kubectl apply -f $service.yaml
|
||||
done
|
||||
|
||||
# Step 6: Check deployment status
|
||||
echo ""
|
||||
echo "6. Checking deployment status..."
|
||||
kubectl -n site11-pipeline get deployments
|
||||
|
||||
echo ""
|
||||
echo "7. Waiting for pods to be ready..."
|
||||
kubectl -n site11-pipeline wait --for=condition=Ready pods --all --timeout=300s || true
|
||||
|
||||
# Step 7: Show final status
|
||||
echo ""
|
||||
echo "✅ Deployment Complete!"
|
||||
echo ""
|
||||
echo "Current status:"
|
||||
kubectl -n site11-pipeline get pods
|
||||
echo ""
|
||||
echo "To view logs:"
|
||||
echo " kubectl -n site11-pipeline logs -f deployment/pipeline-translator"
|
||||
echo ""
|
||||
echo "To scale deployments:"
|
||||
echo " kubectl -n site11-pipeline scale deployment pipeline-translator --replicas=5"
|
||||
echo ""
|
||||
echo "To delete all resources:"
|
||||
echo " kubectl delete namespace site11-pipeline"
|
||||
78
k8s/pipeline/google-search.yaml
Normal file
78
k8s/pipeline/google-search.yaml
Normal file
@ -0,0 +1,78 @@
|
||||
apiVersion: apps/v1
|
||||
kind: Deployment
|
||||
metadata:
|
||||
name: pipeline-google-search
|
||||
namespace: site11-pipeline
|
||||
labels:
|
||||
app: pipeline-google-search
|
||||
component: data-collector
|
||||
spec:
|
||||
replicas: 2
|
||||
selector:
|
||||
matchLabels:
|
||||
app: pipeline-google-search
|
||||
template:
|
||||
metadata:
|
||||
labels:
|
||||
app: pipeline-google-search
|
||||
component: data-collector
|
||||
spec:
|
||||
containers:
|
||||
- name: google-search
|
||||
image: site11/pipeline-google-search:latest
|
||||
imagePullPolicy: Always
|
||||
envFrom:
|
||||
- configMapRef:
|
||||
name: pipeline-config
|
||||
- secretRef:
|
||||
name: pipeline-secrets
|
||||
resources:
|
||||
requests:
|
||||
memory: "256Mi"
|
||||
cpu: "100m"
|
||||
limits:
|
||||
memory: "512Mi"
|
||||
cpu: "500m"
|
||||
livenessProbe:
|
||||
exec:
|
||||
command:
|
||||
- python
|
||||
- -c
|
||||
- "import redis; r=redis.from_url('redis://host.docker.internal:6379'); r.ping()"
|
||||
initialDelaySeconds: 30
|
||||
periodSeconds: 30
|
||||
readinessProbe:
|
||||
exec:
|
||||
command:
|
||||
- python
|
||||
- -c
|
||||
- "import redis; r=redis.from_url('redis://host.docker.internal:6379'); r.ping()"
|
||||
initialDelaySeconds: 10
|
||||
periodSeconds: 10
|
||||
|
||||
---
|
||||
apiVersion: autoscaling/v2
|
||||
kind: HorizontalPodAutoscaler
|
||||
metadata:
|
||||
name: pipeline-google-search-hpa
|
||||
namespace: site11-pipeline
|
||||
spec:
|
||||
scaleTargetRef:
|
||||
apiVersion: apps/v1
|
||||
kind: Deployment
|
||||
name: pipeline-google-search
|
||||
minReplicas: 1
|
||||
maxReplicas: 5
|
||||
metrics:
|
||||
- type: Resource
|
||||
resource:
|
||||
name: cpu
|
||||
target:
|
||||
type: Utilization
|
||||
averageUtilization: 70
|
||||
- type: Resource
|
||||
resource:
|
||||
name: memory
|
||||
target:
|
||||
type: Utilization
|
||||
averageUtilization: 80
|
||||
78
k8s/pipeline/image-generator.yaml
Normal file
78
k8s/pipeline/image-generator.yaml
Normal file
@ -0,0 +1,78 @@
|
||||
apiVersion: apps/v1
|
||||
kind: Deployment
|
||||
metadata:
|
||||
name: pipeline-image-generator
|
||||
namespace: site11-pipeline
|
||||
labels:
|
||||
app: pipeline-image-generator
|
||||
component: processor
|
||||
spec:
|
||||
replicas: 2
|
||||
selector:
|
||||
matchLabels:
|
||||
app: pipeline-image-generator
|
||||
template:
|
||||
metadata:
|
||||
labels:
|
||||
app: pipeline-image-generator
|
||||
component: processor
|
||||
spec:
|
||||
containers:
|
||||
- name: image-generator
|
||||
image: site11/pipeline-image-generator:latest
|
||||
imagePullPolicy: Always
|
||||
envFrom:
|
||||
- configMapRef:
|
||||
name: pipeline-config
|
||||
- secretRef:
|
||||
name: pipeline-secrets
|
||||
resources:
|
||||
requests:
|
||||
memory: "512Mi"
|
||||
cpu: "200m"
|
||||
limits:
|
||||
memory: "1Gi"
|
||||
cpu: "1000m"
|
||||
livenessProbe:
|
||||
exec:
|
||||
command:
|
||||
- python
|
||||
- -c
|
||||
- "import redis; r=redis.from_url('redis://host.docker.internal:6379'); r.ping()"
|
||||
initialDelaySeconds: 30
|
||||
periodSeconds: 30
|
||||
readinessProbe:
|
||||
exec:
|
||||
command:
|
||||
- python
|
||||
- -c
|
||||
- "import redis; r=redis.from_url('redis://host.docker.internal:6379'); r.ping()"
|
||||
initialDelaySeconds: 10
|
||||
periodSeconds: 10
|
||||
|
||||
---
|
||||
apiVersion: autoscaling/v2
|
||||
kind: HorizontalPodAutoscaler
|
||||
metadata:
|
||||
name: pipeline-image-generator-hpa
|
||||
namespace: site11-pipeline
|
||||
spec:
|
||||
scaleTargetRef:
|
||||
apiVersion: apps/v1
|
||||
kind: Deployment
|
||||
name: pipeline-image-generator
|
||||
minReplicas: 1
|
||||
maxReplicas: 6
|
||||
metrics:
|
||||
- type: Resource
|
||||
resource:
|
||||
name: cpu
|
||||
target:
|
||||
type: Utilization
|
||||
averageUtilization: 70
|
||||
- type: Resource
|
||||
resource:
|
||||
name: memory
|
||||
target:
|
||||
type: Utilization
|
||||
averageUtilization: 80
|
||||
7
k8s/pipeline/namespace.yaml
Normal file
7
k8s/pipeline/namespace.yaml
Normal file
@ -0,0 +1,7 @@
|
||||
apiVersion: v1
|
||||
kind: Namespace
|
||||
metadata:
|
||||
name: site11-pipeline
|
||||
labels:
|
||||
name: site11-pipeline
|
||||
environment: production
|
||||
78
k8s/pipeline/rss-collector.yaml
Normal file
78
k8s/pipeline/rss-collector.yaml
Normal file
@ -0,0 +1,78 @@
|
||||
apiVersion: apps/v1
|
||||
kind: Deployment
|
||||
metadata:
|
||||
name: pipeline-rss-collector
|
||||
namespace: site11-pipeline
|
||||
labels:
|
||||
app: pipeline-rss-collector
|
||||
component: data-collector
|
||||
spec:
|
||||
replicas: 2
|
||||
selector:
|
||||
matchLabels:
|
||||
app: pipeline-rss-collector
|
||||
template:
|
||||
metadata:
|
||||
labels:
|
||||
app: pipeline-rss-collector
|
||||
component: data-collector
|
||||
spec:
|
||||
containers:
|
||||
- name: rss-collector
|
||||
image: site11/pipeline-rss-collector:latest
|
||||
imagePullPolicy: Always
|
||||
envFrom:
|
||||
- configMapRef:
|
||||
name: pipeline-config
|
||||
- secretRef:
|
||||
name: pipeline-secrets
|
||||
resources:
|
||||
requests:
|
||||
memory: "256Mi"
|
||||
cpu: "100m"
|
||||
limits:
|
||||
memory: "512Mi"
|
||||
cpu: "500m"
|
||||
livenessProbe:
|
||||
exec:
|
||||
command:
|
||||
- python
|
||||
- -c
|
||||
- "import redis; r=redis.from_url('redis://host.docker.internal:6379'); r.ping()"
|
||||
initialDelaySeconds: 30
|
||||
periodSeconds: 30
|
||||
readinessProbe:
|
||||
exec:
|
||||
command:
|
||||
- python
|
||||
- -c
|
||||
- "import redis; r=redis.from_url('redis://host.docker.internal:6379'); r.ping()"
|
||||
initialDelaySeconds: 10
|
||||
periodSeconds: 10
|
||||
|
||||
---
|
||||
apiVersion: autoscaling/v2
|
||||
kind: HorizontalPodAutoscaler
|
||||
metadata:
|
||||
name: pipeline-rss-collector-hpa
|
||||
namespace: site11-pipeline
|
||||
spec:
|
||||
scaleTargetRef:
|
||||
apiVersion: apps/v1
|
||||
kind: Deployment
|
||||
name: pipeline-rss-collector
|
||||
minReplicas: 1
|
||||
maxReplicas: 5
|
||||
metrics:
|
||||
- type: Resource
|
||||
resource:
|
||||
name: cpu
|
||||
target:
|
||||
type: Utilization
|
||||
averageUtilization: 70
|
||||
- type: Resource
|
||||
resource:
|
||||
name: memory
|
||||
target:
|
||||
type: Utilization
|
||||
averageUtilization: 80
|
||||
78
k8s/pipeline/translator.yaml
Normal file
78
k8s/pipeline/translator.yaml
Normal file
@ -0,0 +1,78 @@
|
||||
apiVersion: apps/v1
|
||||
kind: Deployment
|
||||
metadata:
|
||||
name: pipeline-translator
|
||||
namespace: site11-pipeline
|
||||
labels:
|
||||
app: pipeline-translator
|
||||
component: processor
|
||||
spec:
|
||||
replicas: 3
|
||||
selector:
|
||||
matchLabels:
|
||||
app: pipeline-translator
|
||||
template:
|
||||
metadata:
|
||||
labels:
|
||||
app: pipeline-translator
|
||||
component: processor
|
||||
spec:
|
||||
containers:
|
||||
- name: translator
|
||||
image: site11/pipeline-translator:latest
|
||||
imagePullPolicy: Always
|
||||
envFrom:
|
||||
- configMapRef:
|
||||
name: pipeline-config
|
||||
- secretRef:
|
||||
name: pipeline-secrets
|
||||
resources:
|
||||
requests:
|
||||
memory: "512Mi"
|
||||
cpu: "200m"
|
||||
limits:
|
||||
memory: "1Gi"
|
||||
cpu: "1000m"
|
||||
livenessProbe:
|
||||
exec:
|
||||
command:
|
||||
- python
|
||||
- -c
|
||||
- "import redis; r=redis.from_url('redis://host.docker.internal:6379'); r.ping()"
|
||||
initialDelaySeconds: 30
|
||||
periodSeconds: 30
|
||||
readinessProbe:
|
||||
exec:
|
||||
command:
|
||||
- python
|
||||
- -c
|
||||
- "import redis; r=redis.from_url('redis://host.docker.internal:6379'); r.ping()"
|
||||
initialDelaySeconds: 10
|
||||
periodSeconds: 10
|
||||
|
||||
---
|
||||
apiVersion: autoscaling/v2
|
||||
kind: HorizontalPodAutoscaler
|
||||
metadata:
|
||||
name: pipeline-translator-hpa
|
||||
namespace: site11-pipeline
|
||||
spec:
|
||||
scaleTargetRef:
|
||||
apiVersion: apps/v1
|
||||
kind: Deployment
|
||||
name: pipeline-translator
|
||||
minReplicas: 2
|
||||
maxReplicas: 10
|
||||
metrics:
|
||||
- type: Resource
|
||||
resource:
|
||||
name: cpu
|
||||
target:
|
||||
type: Utilization
|
||||
averageUtilization: 70
|
||||
- type: Resource
|
||||
resource:
|
||||
name: memory
|
||||
target:
|
||||
type: Utilization
|
||||
averageUtilization: 80
|
||||
Reference in New Issue
Block a user