feat: Integrate News Engine Console with Pipeline Monitor service
Backend Integration:
- Created PipelineClient for communicating with Pipeline Monitor (port 8100)
- Added proxy endpoints in monitoring.py:
* /api/v1/monitoring/pipeline/stats - Queue status and article counts
* /api/v1/monitoring/pipeline/health - Pipeline service health
* /api/v1/monitoring/pipeline/queues/{name} - Queue details
* /api/v1/monitoring/pipeline/workers - Worker status
Frontend Integration:
- Added Pipeline Monitor API functions to monitoring.ts
- Updated Monitoring page to display:
* Redis queue status (keyword, rss, search, summarize, assembly)
* Article statistics (today, total, active keywords)
* Pipeline health status
* Worker status for each pipeline type
Architecture:
- Console acts as API Gateway, proxying requests to Pipeline Monitor
- Pipeline Monitor (services/pipeline/monitor) manages:
* RSS Collector, Google Search, AI Summarizer, Article Assembly workers
* Redis queues for async job processing
* MongoDB for article and keyword storage
This integration allows the News Engine Console to monitor and display
real-time pipeline activity, queue status, and worker health.
🤖 Generated with [Claude Code](https://claude.com/claude-code)
Co-Authored-By: Claude <noreply@anthropic.com>
This commit is contained in:
jungwoo choi
@ -65,3 +65,47 @@ export const getPipelineActivity = async (params?: {
|
||||
)
|
||||
return response.data
|
||||
}
|
||||
|
||||
// =============================================================================
|
||||
// Pipeline Monitor Proxy Endpoints
|
||||
// =============================================================================
|
||||
|
||||
export const getPipelineStats = async (): Promise<{
|
||||
queues: Record<string, number>
|
||||
articles_today: number
|
||||
active_keywords: number
|
||||
total_articles: number
|
||||
timestamp: string
|
||||
}> => {
|
||||
const response = await apiClient.get('/api/v1/monitoring/pipeline/stats')
|
||||
return response.data
|
||||
}
|
||||
|
||||
export const getPipelineHealth = async (): Promise<{
|
||||
status: string
|
||||
redis: string
|
||||
mongodb: string
|
||||
timestamp: string
|
||||
}> => {
|
||||
const response = await apiClient.get('/api/v1/monitoring/pipeline/health')
|
||||
return response.data
|
||||
}
|
||||
|
||||
export const getQueueDetails = async (queueName: string): Promise<{
|
||||
queue: string
|
||||
length: number
|
||||
processing_count: number
|
||||
failed_count: number
|
||||
preview: any[]
|
||||
timestamp: string
|
||||
}> => {
|
||||
const response = await apiClient.get(`/api/v1/monitoring/pipeline/queues/${queueName}`)
|
||||
return response.data
|
||||
}
|
||||
|
||||
export const getPipelineWorkers = async (): Promise<
|
||||
Record<string, { active: number; worker_ids: string[] }>
|
||||
> => {
|
||||
const response = await apiClient.get('/api/v1/monitoring/pipeline/workers')
|
||||
return response.data
|
||||
}
|
||||
|
||||
@ -29,6 +29,9 @@ import {
|
||||
getHealthCheck,
|
||||
getDatabaseStats,
|
||||
getRecentLogs,
|
||||
getPipelineStats,
|
||||
getPipelineHealth,
|
||||
getPipelineWorkers,
|
||||
} from '@/api/monitoring'
|
||||
import type { MonitoringOverview, SystemStatus, DatabaseStats, LogEntry } from '@/types'
|
||||
|
||||
@ -41,6 +44,11 @@ const Monitoring = () => {
|
||||
const [error, setError] = useState<string | null>(null)
|
||||
const [autoRefresh, setAutoRefresh] = useState(true)
|
||||
|
||||
// Pipeline Monitor stats
|
||||
const [pipelineStats, setPipelineStats] = useState<any>(null)
|
||||
const [pipelineHealth, setPipelineHealth] = useState<any>(null)
|
||||
const [pipelineWorkers, setPipelineWorkers] = useState<any>(null)
|
||||
|
||||
useEffect(() => {
|
||||
fetchData()
|
||||
const interval = setInterval(() => {
|
||||
@ -56,16 +64,22 @@ const Monitoring = () => {
|
||||
setLoading(true)
|
||||
setError(null)
|
||||
try {
|
||||
const [overviewData, healthData, dbData, logsData] = await Promise.all([
|
||||
const [overviewData, healthData, dbData, logsData, pipelineStatsData, pipelineHealthData, pipelineWorkersData] = await Promise.all([
|
||||
getMonitoringOverview().catch(() => null),
|
||||
getHealthCheck().catch(() => null),
|
||||
getDatabaseStats().catch(() => null),
|
||||
getRecentLogs({ limit: 50 }).catch(() => []),
|
||||
getPipelineStats().catch(() => null),
|
||||
getPipelineHealth().catch(() => null),
|
||||
getPipelineWorkers().catch(() => null),
|
||||
])
|
||||
setOverview(overviewData)
|
||||
setHealth(healthData)
|
||||
setDbStats(dbData)
|
||||
setLogs(logsData)
|
||||
setPipelineStats(pipelineStatsData)
|
||||
setPipelineHealth(pipelineHealthData)
|
||||
setPipelineWorkers(pipelineWorkersData)
|
||||
} catch (err: any) {
|
||||
setError(err.message || 'Failed to fetch monitoring data')
|
||||
} finally {
|
||||
@ -350,6 +364,95 @@ const Monitoring = () => {
|
||||
</Paper>
|
||||
)}
|
||||
|
||||
{/* Pipeline Monitor Stats */}
|
||||
{(pipelineStats || pipelineWorkers) && (
|
||||
<Paper sx={{ p: 2, mb: 3 }}>
|
||||
<Typography variant="h6" gutterBottom>
|
||||
Pipeline Monitor
|
||||
</Typography>
|
||||
|
||||
{pipelineStats && (
|
||||
<Box sx={{ mb: 3 }}>
|
||||
<Typography variant="subtitle2" color="text.secondary" gutterBottom>
|
||||
Queue Status
|
||||
</Typography>
|
||||
<Grid container spacing={2}>
|
||||
{Object.entries(pipelineStats.queues || {}).map(([queueName, count]) => (
|
||||
<Grid item xs={6} sm={4} md={2} key={queueName}>
|
||||
<Box sx={{ p: 2, border: 1, borderColor: 'divider', borderRadius: 1 }}>
|
||||
<Typography variant="caption" color="text.secondary">
|
||||
{queueName.replace('queue:', '')}
|
||||
</Typography>
|
||||
<Typography variant="h5">{count as number}</Typography>
|
||||
</Box>
|
||||
</Grid>
|
||||
))}
|
||||
</Grid>
|
||||
|
||||
<Grid container spacing={2} sx={{ mt: 2 }}>
|
||||
<Grid item xs={6} md={3}>
|
||||
<Typography variant="body2" color="text.secondary">
|
||||
Articles Today
|
||||
</Typography>
|
||||
<Typography variant="h5">{pipelineStats.articles_today || 0}</Typography>
|
||||
</Grid>
|
||||
<Grid item xs={6} md={3}>
|
||||
<Typography variant="body2" color="text.secondary">
|
||||
Active Keywords
|
||||
</Typography>
|
||||
<Typography variant="h5">{pipelineStats.active_keywords || 0}</Typography>
|
||||
</Grid>
|
||||
<Grid item xs={6} md={3}>
|
||||
<Typography variant="body2" color="text.secondary">
|
||||
Total Articles
|
||||
</Typography>
|
||||
<Typography variant="h5">{pipelineStats.total_articles || 0}</Typography>
|
||||
</Grid>
|
||||
<Grid item xs={6} md={3}>
|
||||
<Typography variant="body2" color="text.secondary">
|
||||
Pipeline Health
|
||||
</Typography>
|
||||
<Chip
|
||||
label={pipelineHealth?.status || 'Unknown'}
|
||||
color={
|
||||
pipelineHealth?.status === 'healthy' ? 'success' : 'error'
|
||||
}
|
||||
size="small"
|
||||
/>
|
||||
</Grid>
|
||||
</Grid>
|
||||
</Box>
|
||||
)}
|
||||
|
||||
{pipelineWorkers && (
|
||||
<Box>
|
||||
<Typography variant="subtitle2" color="text.secondary" gutterBottom>
|
||||
Pipeline Workers
|
||||
</Typography>
|
||||
<Grid container spacing={2}>
|
||||
{Object.entries(pipelineWorkers).map(([workerType, workerInfo]: [string, any]) => (
|
||||
<Grid item xs={6} sm={4} md={3} key={workerType}>
|
||||
<Box
|
||||
sx={{
|
||||
p: 2,
|
||||
border: 1,
|
||||
borderColor: 'divider',
|
||||
borderRadius: 1,
|
||||
}}
|
||||
>
|
||||
<Typography variant="body2">{workerType.replace('_', ' ')}</Typography>
|
||||
<Typography variant="h6" color={workerInfo.active > 0 ? 'success.main' : 'text.secondary'}>
|
||||
{workerInfo.active} active
|
||||
</Typography>
|
||||
</Box>
|
||||
</Grid>
|
||||
))}
|
||||
</Grid>
|
||||
</Box>
|
||||
)}
|
||||
</Paper>
|
||||
)}
|
||||
|
||||
{/* Recent Logs */}
|
||||
<Paper sx={{ p: 2 }}>
|
||||
<Typography variant="h6" gutterBottom>
|
||||
|
||||
Reference in New Issue
Block a user