Initial commit - cleaned repository

2025-09-28 20:41:57 +09:00
commit e3c28f796a
188 changed files with 28102 additions and 0 deletions
--- a/services/statistics/backend/main.py
+++ b/services/statistics/backend/main.py
@ -0,0 +1,396 @@
+"""
+Statistics Service - Real-time Analytics and Metrics
+"""
+from fastapi import FastAPI, HTTPException, Depends, Query
+from fastapi.middleware.cors import CORSMiddleware
+from fastapi.responses import StreamingResponse
+import uvicorn
+from datetime import datetime, timedelta
+from typing import Optional, List, Dict, Any
+import asyncio
+import json
+import os
+from contextlib import asynccontextmanager
+import logging
+
+# Import custom modules
+from models import Metric, AggregatedMetric, TimeSeriesData, DashboardConfig
+from metrics_collector import MetricsCollector
+from aggregator import DataAggregator
+from websocket_manager import WebSocketManager
+from time_series_db import TimeSeriesDB
+from cache_manager import CacheManager
+
+logging.basicConfig(level=logging.INFO)
+logger = logging.getLogger(__name__)
+
+# Global instances
+metrics_collector = None
+data_aggregator = None
+ws_manager = None
+ts_db = None
+cache_manager = None
+
+@asynccontextmanager
+async def lifespan(app: FastAPI):
+    # Startup
+    global metrics_collector, data_aggregator, ws_manager, ts_db, cache_manager
+    
+    try:
+        # Initialize TimeSeriesDB (using InfluxDB)
+        ts_db = TimeSeriesDB(
+            host=os.getenv("INFLUXDB_HOST", "influxdb"),
+            port=int(os.getenv("INFLUXDB_PORT", 8086)),
+            database=os.getenv("INFLUXDB_DATABASE", "statistics")
+        )
+        await ts_db.connect()
+        logger.info("Connected to InfluxDB")
+        
+        # Initialize Cache Manager
+        cache_manager = CacheManager(
+            redis_url=os.getenv("REDIS_URL", "redis://redis:6379")
+        )
+        await cache_manager.connect()
+        logger.info("Connected to Redis cache")
+        
+        # Initialize Metrics Collector (optional Kafka connection)
+        try:
+            metrics_collector = MetricsCollector(
+                kafka_bootstrap_servers=os.getenv("KAFKA_BOOTSTRAP_SERVERS", "kafka:9092"),
+                ts_db=ts_db,
+                cache=cache_manager
+            )
+            await metrics_collector.start()
+            logger.info("Metrics collector started")
+        except Exception as e:
+            logger.warning(f"Metrics collector failed to start (Kafka not available): {e}")
+            metrics_collector = None
+        
+        # Initialize Data Aggregator
+        data_aggregator = DataAggregator(
+            ts_db=ts_db,
+            cache=cache_manager
+        )
+        asyncio.create_task(data_aggregator.start_aggregation_jobs())
+        logger.info("Data aggregator started")
+        
+        # Initialize WebSocket Manager
+        ws_manager = WebSocketManager()
+        logger.info("WebSocket manager initialized")
+        
+    except Exception as e:
+        logger.error(f"Failed to start Statistics service: {e}")
+        raise
+    
+    yield
+    
+    # Shutdown
+    if metrics_collector:
+        await metrics_collector.stop()
+    if data_aggregator:
+        await data_aggregator.stop()
+    if ts_db:
+        await ts_db.close()
+    if cache_manager:
+        await cache_manager.close()
+    
+    logger.info("Statistics service shutdown complete")
+
+app = FastAPI(
+    title="Statistics Service",
+    description="Real-time Analytics and Metrics Service",
+    version="1.0.0",
+    lifespan=lifespan
+)
+
+# CORS middleware
+app.add_middleware(
+    CORSMiddleware,
+    allow_origins=["*"],
+    allow_credentials=True,
+    allow_methods=["*"],
+    allow_headers=["*"],
+)
+
+@app.get("/")
+async def root():
+    return {
+        "service": "Statistics Service",
+        "status": "running",
+        "timestamp": datetime.now().isoformat()
+    }
+
+@app.get("/health")
+async def health_check():
+    return {
+        "status": "healthy",
+        "service": "statistics",
+        "components": {
+            "influxdb": "connected" if ts_db and ts_db.is_connected else "disconnected",
+            "redis": "connected" if cache_manager and cache_manager.is_connected else "disconnected",
+            "metrics_collector": "running" if metrics_collector and metrics_collector.is_running else "stopped",
+            "aggregator": "running" if data_aggregator and data_aggregator.is_running else "stopped"
+        },
+        "timestamp": datetime.now().isoformat()
+    }
+
+# Metrics Endpoints
+@app.post("/api/metrics")
+async def record_metric(metric: Metric):
+    """Record a single metric"""
+    try:
+        await metrics_collector.record_metric(metric)
+        return {"status": "recorded", "metric_id": metric.id}
+    except Exception as e:
+        raise HTTPException(status_code=500, detail=str(e))
+
+@app.post("/api/metrics/batch")
+async def record_metrics_batch(metrics: List[Metric]):
+    """Record multiple metrics in batch"""
+    try:
+        await metrics_collector.record_metrics_batch(metrics)
+        return {"status": "recorded", "count": len(metrics)}
+    except Exception as e:
+        raise HTTPException(status_code=500, detail=str(e))
+
+@app.get("/api/metrics/realtime/{metric_type}")
+async def get_realtime_metrics(
+    metric_type: str,
+    duration: int = Query(60, description="Duration in seconds")
+):
+    """Get real-time metrics for the specified type"""
+    try:
+        end_time = datetime.now()
+        start_time = end_time - timedelta(seconds=duration)
+        
+        metrics = await ts_db.query_metrics(
+            metric_type=metric_type,
+            start_time=start_time,
+            end_time=end_time
+        )
+        
+        return {
+            "metric_type": metric_type,
+            "duration": duration,
+            "data": metrics,
+            "timestamp": datetime.now().isoformat()
+        }
+    except Exception as e:
+        raise HTTPException(status_code=500, detail=str(e))
+
+# Analytics Endpoints
+@app.get("/api/analytics/overview")
+async def get_analytics_overview():
+    """Get overall analytics overview"""
+    try:
+        # Try to get from cache first
+        cached = await cache_manager.get("analytics:overview")
+        if cached:
+            return json.loads(cached)
+        
+        # Calculate analytics
+        overview = await data_aggregator.get_overview()
+        
+        # Cache for 1 minute
+        await cache_manager.set(
+            "analytics:overview",
+            json.dumps(overview),
+            expire=60
+        )
+        
+        return overview
+    except Exception as e:
+        raise HTTPException(status_code=500, detail=str(e))
+
+@app.get("/api/analytics/users")
+async def get_user_analytics(
+    start_date: Optional[datetime] = None,
+    end_date: Optional[datetime] = None,
+    granularity: str = Query("hour", regex="^(minute|hour|day|week|month)$")
+):
+    """Get user analytics"""
+    try:
+        if not start_date:
+            start_date = datetime.now() - timedelta(days=7)
+        if not end_date:
+            end_date = datetime.now()
+        
+        analytics = await data_aggregator.get_user_analytics(
+            start_date=start_date,
+            end_date=end_date,
+            granularity=granularity
+        )
+        
+        return analytics
+    except Exception as e:
+        raise HTTPException(status_code=500, detail=str(e))
+
+@app.get("/api/analytics/system")
+async def get_system_analytics():
+    """Get system performance analytics"""
+    try:
+        analytics = await data_aggregator.get_system_analytics()
+        return analytics
+    except Exception as e:
+        raise HTTPException(status_code=500, detail=str(e))
+
+@app.get("/api/analytics/events")
+async def get_event_analytics(
+    event_type: Optional[str] = None,
+    limit: int = Query(100, le=1000)
+):
+    """Get event analytics"""
+    try:
+        analytics = await data_aggregator.get_event_analytics(
+            event_type=event_type,
+            limit=limit
+        )
+        return analytics
+    except Exception as e:
+        raise HTTPException(status_code=500, detail=str(e))
+
+# Time Series Endpoints
+@app.get("/api/timeseries/{metric_name}")
+async def get_time_series(
+    metric_name: str,
+    start_time: datetime,
+    end_time: datetime,
+    interval: str = Query("1m", regex="^\\d+[smhd]$")
+):
+    """Get time series data for a specific metric"""
+    try:
+        data = await ts_db.get_time_series(
+            metric_name=metric_name,
+            start_time=start_time,
+            end_time=end_time,
+            interval=interval
+        )
+        
+        return TimeSeriesData(
+            metric_name=metric_name,
+            start_time=start_time,
+            end_time=end_time,
+            interval=interval,
+            data=data
+        )
+    except Exception as e:
+        raise HTTPException(status_code=500, detail=str(e))
+
+# Aggregation Endpoints
+@app.get("/api/aggregates/{metric_type}")
+async def get_aggregated_metrics(
+    metric_type: str,
+    aggregation: str = Query("avg", regex="^(avg|sum|min|max|count)$"),
+    group_by: Optional[str] = None,
+    start_time: Optional[datetime] = None,
+    end_time: Optional[datetime] = None
+):
+    """Get aggregated metrics"""
+    try:
+        if not start_time:
+            start_time = datetime.now() - timedelta(hours=24)
+        if not end_time:
+            end_time = datetime.now()
+        
+        result = await data_aggregator.aggregate_metrics(
+            metric_type=metric_type,
+            aggregation=aggregation,
+            group_by=group_by,
+            start_time=start_time,
+            end_time=end_time
+        )
+        
+        return result
+    except Exception as e:
+        raise HTTPException(status_code=500, detail=str(e))
+
+# Dashboard Endpoints
+@app.get("/api/dashboard/configs")
+async def get_dashboard_configs():
+    """Get available dashboard configurations"""
+    configs = await data_aggregator.get_dashboard_configs()
+    return {"configs": configs}
+
+@app.get("/api/dashboard/{dashboard_id}")
+async def get_dashboard_data(dashboard_id: str):
+    """Get data for a specific dashboard"""
+    try:
+        data = await data_aggregator.get_dashboard_data(dashboard_id)
+        return data
+    except Exception as e:
+        raise HTTPException(status_code=500, detail=str(e))
+
+# WebSocket Endpoint for Real-time Updates
+from fastapi import WebSocket, WebSocketDisconnect
+
+@app.websocket("/ws/metrics")
+async def websocket_metrics(websocket: WebSocket):
+    """WebSocket endpoint for real-time metrics streaming"""
+    await ws_manager.connect(websocket)
+    try:
+        while True:
+            # Send metrics updates every second
+            metrics = await metrics_collector.get_latest_metrics()
+            await websocket.send_json({
+                "type": "metrics_update",
+                "data": metrics,
+                "timestamp": datetime.now().isoformat()
+            })
+            await asyncio.sleep(1)
+    except WebSocketDisconnect:
+        ws_manager.disconnect(websocket)
+    except Exception as e:
+        logger.error(f"WebSocket error: {e}")
+        ws_manager.disconnect(websocket)
+
+# Alert Management Endpoints
+@app.post("/api/alerts/rules")
+async def create_alert_rule(rule: Dict[str, Any]):
+    """Create a new alert rule"""
+    try:
+        rule_id = await data_aggregator.create_alert_rule(rule)
+        return {"rule_id": rule_id, "status": "created"}
+    except Exception as e:
+        raise HTTPException(status_code=500, detail=str(e))
+
+@app.get("/api/alerts/active")
+async def get_active_alerts():
+    """Get currently active alerts"""
+    try:
+        alerts = await data_aggregator.get_active_alerts()
+        return {"alerts": alerts, "count": len(alerts)}
+    except Exception as e:
+        raise HTTPException(status_code=500, detail=str(e))
+
+# Export Endpoints
+@app.get("/api/export/csv")
+async def export_metrics_csv(
+    metric_type: str,
+    start_time: datetime,
+    end_time: datetime
+):
+    """Export metrics as CSV"""
+    try:
+        csv_data = await data_aggregator.export_to_csv(
+            metric_type=metric_type,
+            start_time=start_time,
+            end_time=end_time
+        )
+        
+        return StreamingResponse(
+            csv_data,
+            media_type="text/csv",
+            headers={
+                "Content-Disposition": f"attachment; filename=metrics_{metric_type}_{datetime.now().strftime('%Y%m%d_%H%M%S')}.csv"
+            }
+        )
+    except Exception as e:
+        raise HTTPException(status_code=500, detail=str(e))
+
+if __name__ == "__main__":
+    uvicorn.run(
+        "main:app",
+        host="0.0.0.0",
+        port=8000,
+        reload=True
+    )