46b5135f45
- Docker Compose for infrastructure (MongoDB, Redis, Kafka, Zookeeper) - Docker for central control (Scheduler, Monitor, Language Sync) - K8s for scalable workers (RSS, Google Search, Translator, AI Generator, Image Generator) - Automatic scaling with HPA (Horizontal Pod Autoscaler) - Comprehensive deployment scripts and documentation - Updated README with hybrid deployment guide
78 lines
1.8 KiB
YAML
78 lines
1.8 KiB
YAML
apiVersion: apps/v1
|
|
kind: Deployment
|
|
metadata:
|
|
name: pipeline-google-search
|
|
namespace: site11-pipeline
|
|
labels:
|
|
app: pipeline-google-search
|
|
component: data-collector
|
|
spec:
|
|
replicas: 2
|
|
selector:
|
|
matchLabels:
|
|
app: pipeline-google-search
|
|
template:
|
|
metadata:
|
|
labels:
|
|
app: pipeline-google-search
|
|
component: data-collector
|
|
spec:
|
|
containers:
|
|
- name: google-search
|
|
image: site11/pipeline-google-search:latest
|
|
imagePullPolicy: Always
|
|
envFrom:
|
|
- configMapRef:
|
|
name: pipeline-config
|
|
- secretRef:
|
|
name: pipeline-secrets
|
|
resources:
|
|
requests:
|
|
memory: "256Mi"
|
|
cpu: "100m"
|
|
limits:
|
|
memory: "512Mi"
|
|
cpu: "500m"
|
|
livenessProbe:
|
|
exec:
|
|
command:
|
|
- python
|
|
- -c
|
|
- "import redis; r=redis.from_url('redis://host.docker.internal:6379'); r.ping()"
|
|
initialDelaySeconds: 30
|
|
periodSeconds: 30
|
|
readinessProbe:
|
|
exec:
|
|
command:
|
|
- python
|
|
- -c
|
|
- "import redis; r=redis.from_url('redis://host.docker.internal:6379'); r.ping()"
|
|
initialDelaySeconds: 10
|
|
periodSeconds: 10
|
|
|
|
---
|
|
apiVersion: autoscaling/v2
|
|
kind: HorizontalPodAutoscaler
|
|
metadata:
|
|
name: pipeline-google-search-hpa
|
|
namespace: site11-pipeline
|
|
spec:
|
|
scaleTargetRef:
|
|
apiVersion: apps/v1
|
|
kind: Deployment
|
|
name: pipeline-google-search
|
|
minReplicas: 1
|
|
maxReplicas: 5
|
|
metrics:
|
|
- type: Resource
|
|
resource:
|
|
name: cpu
|
|
target:
|
|
type: Utilization
|
|
averageUtilization: 70
|
|
- type: Resource
|
|
resource:
|
|
name: memory
|
|
target:
|
|
type: Utilization
|
|
averageUtilization: 80 |