feat: Implement automated keyword-based news pipeline scheduler
- Add multi-threaded keyword scheduler for periodic news collection - Create Keyword Manager API for CRUD operations and monitoring - Implement automatic pipeline triggering (RSS → Google → AI → Translation) - Add thread status monitoring and dynamic keyword management - Support priority-based execution and configurable intervals - Add comprehensive scheduler documentation guide - Default keywords: AI, 테크놀로지, 경제, 블록체인 🤖 Generated with [Claude Code](https://claude.ai/code) Co-Authored-By: Claude <noreply@anthropic.com>
This commit is contained in:
54
services/pipeline/test_submit_job.py
Normal file
54
services/pipeline/test_submit_job.py
Normal file
@ -0,0 +1,54 @@
|
||||
"""
|
||||
파이프라인 테스트 작업 제출 스크립트
|
||||
"""
|
||||
import redis
|
||||
import json
|
||||
from datetime import datetime
|
||||
import uuid
|
||||
import sys
|
||||
|
||||
def submit_test_job(keyword='나스닥'):
|
||||
# Redis 연결
|
||||
redis_client = redis.Redis(host='localhost', port=6379, decode_responses=True)
|
||||
|
||||
# 테스트 작업 생성
|
||||
job_id = str(uuid.uuid4())
|
||||
keyword_id = f'test_{job_id[:8]}'
|
||||
|
||||
job_data = {
|
||||
'job_id': job_id,
|
||||
'keyword_id': keyword_id,
|
||||
'keyword': keyword,
|
||||
'created_at': datetime.now().isoformat(),
|
||||
'stage': 'rss_collection',
|
||||
'stages_completed': [],
|
||||
'data': {}
|
||||
}
|
||||
|
||||
# QueueMessage 래퍼 생성
|
||||
queue_message = {
|
||||
'message_id': str(uuid.uuid4()),
|
||||
'queue_name': 'rss_collection',
|
||||
'job': job_data,
|
||||
'timestamp': datetime.now().isoformat(),
|
||||
'attempts': 0
|
||||
}
|
||||
|
||||
# 큐에 작업 추가 (rpush 사용 - priority=0인 경우)
|
||||
redis_client.rpush('queue:rss_collection', json.dumps(queue_message))
|
||||
print(f'✅ 파이프라인 시작: job_id={job_id}')
|
||||
print(f'✅ 키워드: {keyword}')
|
||||
print(f'✅ RSS Collection 큐에 작업 추가 완료')
|
||||
|
||||
# 큐 상태 확인
|
||||
queue_len = redis_client.llen('queue:rss_collection')
|
||||
print(f'✅ 현재 큐 길이: {queue_len}')
|
||||
|
||||
redis_client.close()
|
||||
|
||||
if __name__ == "__main__":
|
||||
if len(sys.argv) > 1:
|
||||
keyword = sys.argv[1]
|
||||
else:
|
||||
keyword = '나스닥'
|
||||
submit_test_job(keyword)
|
||||
Reference in New Issue
Block a user