feat: Add hybrid deployment with Docker and Kubernetes
- Docker Compose for infrastructure (MongoDB, Redis, Kafka, Zookeeper) - Docker for central control (Scheduler, Monitor, Language Sync) - K8s for scalable workers (RSS, Google Search, Translator, AI Generator, Image Generator) - Automatic scaling with HPA (Horizontal Pod Autoscaler) - Comprehensive deployment scripts and documentation - Updated README with hybrid deployment guide
This commit is contained in:
78
k8s/pipeline/google-search.yaml
Normal file
78
k8s/pipeline/google-search.yaml
Normal file
@ -0,0 +1,78 @@
|
||||
apiVersion: apps/v1
|
||||
kind: Deployment
|
||||
metadata:
|
||||
name: pipeline-google-search
|
||||
namespace: site11-pipeline
|
||||
labels:
|
||||
app: pipeline-google-search
|
||||
component: data-collector
|
||||
spec:
|
||||
replicas: 2
|
||||
selector:
|
||||
matchLabels:
|
||||
app: pipeline-google-search
|
||||
template:
|
||||
metadata:
|
||||
labels:
|
||||
app: pipeline-google-search
|
||||
component: data-collector
|
||||
spec:
|
||||
containers:
|
||||
- name: google-search
|
||||
image: site11/pipeline-google-search:latest
|
||||
imagePullPolicy: Always
|
||||
envFrom:
|
||||
- configMapRef:
|
||||
name: pipeline-config
|
||||
- secretRef:
|
||||
name: pipeline-secrets
|
||||
resources:
|
||||
requests:
|
||||
memory: "256Mi"
|
||||
cpu: "100m"
|
||||
limits:
|
||||
memory: "512Mi"
|
||||
cpu: "500m"
|
||||
livenessProbe:
|
||||
exec:
|
||||
command:
|
||||
- python
|
||||
- -c
|
||||
- "import redis; r=redis.from_url('redis://host.docker.internal:6379'); r.ping()"
|
||||
initialDelaySeconds: 30
|
||||
periodSeconds: 30
|
||||
readinessProbe:
|
||||
exec:
|
||||
command:
|
||||
- python
|
||||
- -c
|
||||
- "import redis; r=redis.from_url('redis://host.docker.internal:6379'); r.ping()"
|
||||
initialDelaySeconds: 10
|
||||
periodSeconds: 10
|
||||
|
||||
---
|
||||
apiVersion: autoscaling/v2
|
||||
kind: HorizontalPodAutoscaler
|
||||
metadata:
|
||||
name: pipeline-google-search-hpa
|
||||
namespace: site11-pipeline
|
||||
spec:
|
||||
scaleTargetRef:
|
||||
apiVersion: apps/v1
|
||||
kind: Deployment
|
||||
name: pipeline-google-search
|
||||
minReplicas: 1
|
||||
maxReplicas: 5
|
||||
metrics:
|
||||
- type: Resource
|
||||
resource:
|
||||
name: cpu
|
||||
target:
|
||||
type: Utilization
|
||||
averageUtilization: 70
|
||||
- type: Resource
|
||||
resource:
|
||||
name: memory
|
||||
target:
|
||||
type: Utilization
|
||||
averageUtilization: 80
|
||||
Reference in New Issue
Block a user