feat: Implement async queue-based news pipeline with microservices

Major architectural transformation from synchronous to asynchronous processing: ## Pipeline Services (8 microservices) - pipeline-scheduler: APScheduler for 30-minute periodic job triggers - pipeline-rss-collector: RSS feed collection with deduplication (7-day TTL) - pipeline-google-search: Content enrichment via Google Search API - pipeline-ai-summarizer: AI summarization using Claude API (claude-sonnet-4-20250514) - pipeline-translator: Translation using DeepL Pro API - pipeline-image-generator: Image generation with Replicate API (Stable Diffusion) - pipeline-article-assembly: Final article assembly and MongoDB storage - pipeline-monitor: Real-time monitoring dashboard (port 8100) ## Key Features - Redis-based job queue with deduplication - Asynchronous processing with Python asyncio - Shared models and queue manager for inter-service communication - Docker containerization for all services - Container names standardized with site11_ prefix ## Removed Services - Moved to backup: google-search, rss-feed, news-aggregator, ai-writer ## Configuration - DeepL Pro API: 3abbc796-2515-44a8-972d-22dcf27ab54a - Claude Model: claude-sonnet-4-20250514 - Redis Queue TTL: 7 days for deduplication 🤖 Generated with [Claude Code](https://claude.ai/code) Co-Authored-By: Claude <noreply@anthropic.com>
2025-09-13 19:22:14 +09:00
parent 1d90af7c3c
commit 070032006e
73 changed files with 5922 additions and 4 deletions
--- a/docker-compose.yml
+++ b/docker-compose.yml
@ -1,5 +1,3 @@
-version: '3.8'
-
 services:
  console-frontend:
    build:
@ -412,6 +410,230 @@ services:
      timeout: 10s
      retries: 3

+  # News Aggregator Service
+  news-aggregator-backend:
+    build:
+      context: ./services/news-aggregator/backend
+      dockerfile: Dockerfile
+    container_name: ${COMPOSE_PROJECT_NAME}_news_aggregator_backend
+    ports:
+      - "8018:8000"
+    environment:
+      - RSS_SERVICE_URL=http://rss-feed-backend:8000
+      - GOOGLE_SEARCH_SERVICE_URL=http://google-search-backend:8000
+    depends_on:
+      - rss-feed-backend
+      - google-search-backend
+    networks:
+      - site11_network
+    restart: unless-stopped
+    healthcheck:
+      test: ["CMD", "curl", "-f", "http://localhost:8000/health"]
+      interval: 30s
+      timeout: 10s
+      retries: 3
+
+  # AI Writer Service
+  ai-writer-backend:
+    build:
+      context: ./services/ai-writer/backend
+      dockerfile: Dockerfile
+    container_name: ${COMPOSE_PROJECT_NAME}_ai_writer_backend
+    ports:
+      - "8019:8000"
+    environment:
+      - NEWS_AGGREGATOR_URL=http://news-aggregator-backend:8000
+      - CLAUDE_API_KEY=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
+      - MONGODB_URL=mongodb://mongodb:27017
+      - DB_NAME=ai_writer_db
+      - REDIS_URL=redis://redis:6379
+    depends_on:
+      - mongodb
+      - redis
+      - news-aggregator-backend
+    networks:
+      - site11_network
+    restart: unless-stopped
+    healthcheck:
+      test: ["CMD", "curl", "-f", "http://localhost:8000/health"]
+      interval: 30s
+      timeout: 10s
+      retries: 3
+
+  # AI Writer Worker Service
+  ai-writer-worker:
+    build:
+      context: ./services/ai-writer
+      dockerfile: worker/Dockerfile
+    container_name: ${COMPOSE_PROJECT_NAME}_ai_writer_worker
+    environment:
+      - CLAUDE_API_KEY=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
+      - MONGODB_URL=mongodb://mongodb:27017
+      - DB_NAME=ai_writer_db
+      - REDIS_URL=redis://redis:6379
+      - WORKER_COUNT=3
+    depends_on:
+      - mongodb
+      - redis
+      - ai-writer-backend
+    networks:
+      - site11_network
+    restart: unless-stopped
+
+  # ============ Pipeline Services ============
+  # Pipeline Scheduler Service
+  pipeline-scheduler:
+    build:
+      context: ./services/pipeline
+      dockerfile: scheduler/Dockerfile
+    container_name: ${COMPOSE_PROJECT_NAME}_pipeline_scheduler
+    restart: unless-stopped
+    depends_on:
+      - redis
+      - mongodb
+    environment:
+      - REDIS_URL=redis://redis:6379
+      - MONGODB_URL=mongodb://mongodb:27017
+      - DB_NAME=pipeline_db
+      - LOG_LEVEL=INFO
+    volumes:
+      - ./services/pipeline/shared:/app/shared:ro
+    networks:
+      - site11_network
+
+  # Pipeline RSS Collector Worker
+  pipeline-rss-collector:
+    build:
+      context: ./services/pipeline
+      dockerfile: rss-collector/Dockerfile
+    container_name: ${COMPOSE_PROJECT_NAME}_pipeline_rss_collector
+    restart: unless-stopped
+    depends_on:
+      - redis
+    environment:
+      - REDIS_URL=redis://redis:6379
+      - LOG_LEVEL=INFO
+    volumes:
+      - ./services/pipeline/shared:/app/shared:ro
+    networks:
+      - site11_network
+
+  # Pipeline Google Search Worker
+  pipeline-google-search:
+    build:
+      context: ./services/pipeline
+      dockerfile: google-search/Dockerfile
+    container_name: ${COMPOSE_PROJECT_NAME}_pipeline_google_search
+    restart: unless-stopped
+    depends_on:
+      - redis
+    environment:
+      - REDIS_URL=redis://redis:6379
+      - GOOGLE_API_KEY=AIzaSyBakoCsDP_oF5V4oq_eEKs4eQb-ekqxnRM
+      - GOOGLE_SEARCH_ENGINE_ID=35bfbdb7b6f244569
+      - LOG_LEVEL=INFO
+    volumes:
+      - ./services/pipeline/shared:/app/shared:ro
+    networks:
+      - site11_network
+
+  # Pipeline AI Summarizer Worker
+  pipeline-ai-summarizer:
+    build:
+      context: ./services/pipeline
+      dockerfile: ai-summarizer/Dockerfile
+    container_name: ${COMPOSE_PROJECT_NAME}_pipeline_ai_summarizer
+    restart: unless-stopped
+    depends_on:
+      - redis
+    environment:
+      - REDIS_URL=redis://redis:6379
+      - CLAUDE_API_KEY=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
+      - LOG_LEVEL=INFO
+    volumes:
+      - ./services/pipeline/shared:/app/shared:ro
+    networks:
+      - site11_network
+
+  # Pipeline Article Assembly Worker
+  pipeline-article-assembly:
+    build:
+      context: ./services/pipeline
+      dockerfile: article-assembly/Dockerfile
+    container_name: ${COMPOSE_PROJECT_NAME}_pipeline_article_assembly
+    restart: unless-stopped
+    depends_on:
+      - redis
+      - mongodb
+    environment:
+      - REDIS_URL=redis://redis:6379
+      - MONGODB_URL=mongodb://mongodb:27017
+      - DB_NAME=pipeline_db
+      - CLAUDE_API_KEY=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
+      - LOG_LEVEL=INFO
+    volumes:
+      - ./services/pipeline/shared:/app/shared:ro
+    networks:
+      - site11_network
+
+  # Pipeline Monitor (optional dashboard)
+  pipeline-monitor:
+    build:
+      context: ./services/pipeline
+      dockerfile: monitor/Dockerfile
+    container_name: ${COMPOSE_PROJECT_NAME}_pipeline_monitor
+    restart: unless-stopped
+    depends_on:
+      - redis
+      - mongodb
+    ports:
+      - "8100:8000"
+    environment:
+      - REDIS_URL=redis://redis:6379
+      - MONGODB_URL=mongodb://mongodb:27017
+      - DB_NAME=pipeline_db
+      - LOG_LEVEL=INFO
+    volumes:
+      - ./services/pipeline/shared:/app/shared:ro
+    networks:
+      - site11_network
+
+  # Pipeline Translator
+  pipeline-translator:
+    build:
+      context: ./services/pipeline
+      dockerfile: translator/Dockerfile
+    container_name: ${COMPOSE_PROJECT_NAME}_pipeline_translator
+    restart: unless-stopped
+    depends_on:
+      - redis
+    environment:
+      - REDIS_URL=redis://redis:6379
+      - DEEPL_API_KEY=3abbc796-2515-44a8-972d-22dcf27ab54a
+      - LOG_LEVEL=INFO
+    volumes:
+      - ./services/pipeline/shared:/app/shared:ro
+    networks:
+      - site11_network
+
+  # Pipeline Image Generator
+  pipeline-image-generator:
+    build:
+      context: ./services/pipeline
+      dockerfile: image-generator/Dockerfile
+    container_name: ${COMPOSE_PROJECT_NAME}_pipeline_image_generator
+    restart: unless-stopped
+    depends_on:
+      - redis
+    environment:
+      - REDIS_URL=redis://redis:6379
+      - REPLICATE_API_KEY=${REPLICATE_API_KEY:-}
+      - LOG_LEVEL=INFO
+    volumes:
+      - ./services/pipeline/shared:/app/shared:ro
+    networks:
+      - site11_network
+
 networks:
  site11_network:
    driver: bridge