Files
site11/backup-services/google-search/backend/app/main.py
jungwoo choi 070032006e feat: Implement async queue-based news pipeline with microservices
Major architectural transformation from synchronous to asynchronous processing:

## Pipeline Services (8 microservices)
- pipeline-scheduler: APScheduler for 30-minute periodic job triggers
- pipeline-rss-collector: RSS feed collection with deduplication (7-day TTL)
- pipeline-google-search: Content enrichment via Google Search API
- pipeline-ai-summarizer: AI summarization using Claude API (claude-sonnet-4-20250514)
- pipeline-translator: Translation using DeepL Pro API
- pipeline-image-generator: Image generation with Replicate API (Stable Diffusion)
- pipeline-article-assembly: Final article assembly and MongoDB storage
- pipeline-monitor: Real-time monitoring dashboard (port 8100)

## Key Features
- Redis-based job queue with deduplication
- Asynchronous processing with Python asyncio
- Shared models and queue manager for inter-service communication
- Docker containerization for all services
- Container names standardized with site11_ prefix

## Removed Services
- Moved to backup: google-search, rss-feed, news-aggregator, ai-writer

## Configuration
- DeepL Pro API: 3abbc796-2515-44a8-972d-22dcf27ab54a
- Claude Model: claude-sonnet-4-20250514
- Redis Queue TTL: 7 days for deduplication

🤖 Generated with [Claude Code](https://claude.ai/code)

Co-Authored-By: Claude <noreply@anthropic.com>
2025-09-13 19:22:14 +09:00

188 lines
6.7 KiB
Python

from fastapi import FastAPI, Query, HTTPException
from fastapi.middleware.cors import CORSMiddleware
from typing import Optional
from datetime import datetime
from contextlib import asynccontextmanager
from .search_service import GoogleSearchService
from .config import settings
@asynccontextmanager
async def lifespan(app: FastAPI):
# 시작 시
print("Google Search Service starting...")
yield
# 종료 시
print("Google Search Service stopping...")
app = FastAPI(
title="Google Search Service",
description="구글 검색 결과를 수신하는 서비스",
version="1.0.0",
lifespan=lifespan
)
# CORS 설정
app.add_middleware(
CORSMiddleware,
allow_origins=["*"],
allow_credentials=True,
allow_methods=["*"],
allow_headers=["*"],
)
# 검색 서비스 초기화
search_service = GoogleSearchService()
@app.get("/")
async def root():
return {
"service": "Google Search Service",
"version": "1.0.0",
"timestamp": datetime.now().isoformat(),
"endpoints": {
"search": "/api/search?q=keyword",
"custom_search": "/api/search/custom?q=keyword",
"serpapi_search": "/api/search/serpapi?q=keyword",
"scraping_search": "/api/search/scraping?q=keyword",
"trending": "/api/trending",
"health": "/health"
}
}
@app.get("/health")
async def health_check():
return {
"status": "healthy",
"service": "google-search",
"timestamp": datetime.now().isoformat()
}
@app.get("/api/search")
async def search(
q: str = Query(..., description="검색 키워드"),
num: int = Query(10, description="결과 개수", ge=1, le=20),
lang: Optional[str] = Query(None, description="언어 코드 (ko, en 등)"),
country: Optional[str] = Query(None, description="국가 코드 (kr, us 등)"),
date_restrict: Optional[str] = Query(None, description="날짜 제한 (d7=일주일, m1=한달, m3=3개월, y1=1년)"),
sort_by_date: bool = Query(False, description="최신순 정렬")
):
"""
자동으로 최적의 방법을 선택하여 구글 검색
1. Google Custom Search API (설정된 경우)
2. SerpAPI (설정된 경우)
3. 웹 스크래핑 (폴백)
"""
# Google Custom Search API 시도
if settings.google_api_key and settings.google_search_engine_id:
result = await search_service.search_with_custom_api(q, num, lang, country, date_restrict, sort_by_date)
if "error" not in result or not result["error"]:
result["method"] = "google_custom_search"
return result
# SerpAPI 시도
if settings.serpapi_key:
result = await search_service.search_with_serpapi(q, num, lang, country)
if "error" not in result or not result["error"]:
result["method"] = "serpapi"
return result
# 웹 스크래핑 폴백
result = await search_service.search_with_scraping(q, num, lang)
result["method"] = "web_scraping"
result["warning"] = "API 키가 설정되지 않아 웹 스크래핑을 사용합니다. 제한적이고 불안정할 수 있습니다."
return result
@app.get("/api/search/custom")
async def search_custom(
q: str = Query(..., description="검색 키워드"),
num: int = Query(10, description="결과 개수", ge=1, le=10),
lang: Optional[str] = Query(None, description="언어 코드"),
country: Optional[str] = Query(None, description="국가 코드")
):
"""Google Custom Search API를 사용한 검색"""
if not settings.google_api_key or not settings.google_search_engine_id:
raise HTTPException(
status_code=503,
detail="Google Custom Search API credentials not configured"
)
result = await search_service.search_with_custom_api(q, num, lang, country)
if "error" in result and result["error"]:
raise HTTPException(status_code=500, detail=result["error"])
return result
@app.get("/api/search/serpapi")
async def search_serpapi(
q: str = Query(..., description="검색 키워드"),
num: int = Query(10, description="결과 개수", ge=1, le=50),
lang: Optional[str] = Query(None, description="언어 코드"),
country: Optional[str] = Query(None, description="국가 코드")
):
"""SerpAPI를 사용한 검색"""
if not settings.serpapi_key:
raise HTTPException(
status_code=503,
detail="SerpAPI key not configured"
)
result = await search_service.search_with_serpapi(q, num, lang, country)
if "error" in result and result["error"]:
raise HTTPException(status_code=500, detail=result["error"])
return result
@app.get("/api/search/scraping")
async def search_scraping(
q: str = Query(..., description="검색 키워드"),
num: int = Query(10, description="결과 개수", ge=1, le=20),
lang: Optional[str] = Query(None, description="언어 코드")
):
"""웹 스크래핑을 사용한 검색 (제한적)"""
result = await search_service.search_with_scraping(q, num, lang)
if "error" in result and result["error"]:
raise HTTPException(status_code=500, detail=result["error"])
result["warning"] = "웹 스크래핑은 제한적이고 불안정할 수 있습니다"
return result
@app.get("/api/search/full")
async def search_with_full_content(
q: str = Query(..., description="검색 키워드"),
num: int = Query(5, description="결과 개수", ge=1, le=10),
lang: Optional[str] = Query(None, description="언어 코드 (ko, en 등)"),
country: Optional[str] = Query(None, description="국가 코드 (kr, us 등)")
):
"""
Google 검색 후 각 결과 페이지의 전체 내용을 가져오기
주의: 시간이 오래 걸릴 수 있음
"""
result = await search_service.search_with_full_content(q, num, lang, country)
if "error" in result and result["error"]:
raise HTTPException(status_code=500, detail=result["error"])
return result
@app.get("/api/trending")
async def get_trending(
country: Optional[str] = Query(None, description="국가 코드 (kr, us 등)")
):
"""실시간 트렌딩 검색어 조회"""
result = await search_service.get_trending_searches(country)
if "error" in result and result["error"]:
raise HTTPException(status_code=500, detail=result["error"])
return result
@app.post("/api/clear-cache")
async def clear_cache():
"""캐시 초기화"""
try:
search_service.redis_client.flushdb()
return {
"status": "success",
"message": "캐시가 초기화되었습니다"
}
except Exception as e:
raise HTTPException(status_code=500, detail=str(e))