070032006e
Major architectural transformation from synchronous to asynchronous processing: ## Pipeline Services (8 microservices) - pipeline-scheduler: APScheduler for 30-minute periodic job triggers - pipeline-rss-collector: RSS feed collection with deduplication (7-day TTL) - pipeline-google-search: Content enrichment via Google Search API - pipeline-ai-summarizer: AI summarization using Claude API (claude-sonnet-4-20250514) - pipeline-translator: Translation using DeepL Pro API - pipeline-image-generator: Image generation with Replicate API (Stable Diffusion) - pipeline-article-assembly: Final article assembly and MongoDB storage - pipeline-monitor: Real-time monitoring dashboard (port 8100) ## Key Features - Redis-based job queue with deduplication - Asynchronous processing with Python asyncio - Shared models and queue manager for inter-service communication - Docker containerization for all services - Container names standardized with site11_ prefix ## Removed Services - Moved to backup: google-search, rss-feed, news-aggregator, ai-writer ## Configuration - DeepL Pro API: 3abbc796-2515-44a8-972d-22dcf27ab54a - Claude Model: claude-sonnet-4-20250514 - Redis Queue TTL: 7 days for deduplication 🤖 Generated with [Claude Code](https://claude.ai/code) Co-Authored-By: Claude <noreply@anthropic.com>
90 lines
2.2 KiB
Makefile
90 lines
2.2 KiB
Makefile
# Pipeline Makefile
|
|
|
|
.PHONY: help build up down restart logs clean test monitor
|
|
|
|
help:
|
|
@echo "Pipeline Management Commands:"
|
|
@echo " make build - Build all Docker images"
|
|
@echo " make up - Start all services"
|
|
@echo " make down - Stop all services"
|
|
@echo " make restart - Restart all services"
|
|
@echo " make logs - View logs for all services"
|
|
@echo " make clean - Clean up containers and volumes"
|
|
@echo " make monitor - Open monitor dashboard"
|
|
@echo " make test - Test pipeline with sample keyword"
|
|
|
|
build:
|
|
docker-compose build
|
|
|
|
up:
|
|
docker-compose up -d
|
|
|
|
down:
|
|
docker-compose down
|
|
|
|
restart:
|
|
docker-compose restart
|
|
|
|
logs:
|
|
docker-compose logs -f
|
|
|
|
clean:
|
|
docker-compose down -v
|
|
docker system prune -f
|
|
|
|
monitor:
|
|
@echo "Opening monitor dashboard..."
|
|
@echo "Dashboard: http://localhost:8100"
|
|
@echo "API Docs: http://localhost:8100/docs"
|
|
|
|
test:
|
|
@echo "Testing pipeline with sample keyword..."
|
|
curl -X POST http://localhost:8100/api/keywords \
|
|
-H "Content-Type: application/json" \
|
|
-d '{"keyword": "테스트", "schedule": "30min"}'
|
|
@echo "\nTriggering immediate processing..."
|
|
curl -X POST http://localhost:8100/api/trigger/테스트
|
|
|
|
# Service-specific commands
|
|
scheduler-logs:
|
|
docker-compose logs -f scheduler
|
|
|
|
rss-logs:
|
|
docker-compose logs -f rss-collector
|
|
|
|
search-logs:
|
|
docker-compose logs -f google-search
|
|
|
|
summarizer-logs:
|
|
docker-compose logs -f ai-summarizer
|
|
|
|
assembly-logs:
|
|
docker-compose logs -f article-assembly
|
|
|
|
monitor-logs:
|
|
docker-compose logs -f monitor
|
|
|
|
# Database commands
|
|
redis-cli:
|
|
docker-compose exec redis redis-cli
|
|
|
|
mongo-shell:
|
|
docker-compose exec mongodb mongosh -u admin -p password123
|
|
|
|
# Queue management
|
|
queue-status:
|
|
@echo "Checking queue status..."
|
|
docker-compose exec redis redis-cli --raw LLEN queue:keyword
|
|
docker-compose exec redis redis-cli --raw LLEN queue:rss
|
|
docker-compose exec redis redis-cli --raw LLEN queue:search
|
|
docker-compose exec redis redis-cli --raw LLEN queue:summarize
|
|
docker-compose exec redis redis-cli --raw LLEN queue:assembly
|
|
|
|
queue-clear:
|
|
@echo "Clearing all queues..."
|
|
docker-compose exec redis redis-cli FLUSHDB
|
|
|
|
# Health check
|
|
health:
|
|
@echo "Checking service health..."
|
|
curl -s http://localhost:8100/api/health | python3 -m json.tool
|