Initial commit - cleaned repository

2025-09-28 20:41:57 +09:00
commit e3c28f796a
188 changed files with 28102 additions and 0 deletions
--- a/services/files/backend/main.py
+++ b/services/files/backend/main.py
@ -0,0 +1,541 @@
+"""
+File Management Service - S3-compatible Object Storage with MinIO
+"""
+from fastapi import FastAPI, File, UploadFile, HTTPException, Depends, Query, Form
+from fastapi.middleware.cors import CORSMiddleware
+from fastapi.responses import StreamingResponse, FileResponse
+import uvicorn
+from datetime import datetime, timedelta
+from typing import Optional, List, Dict, Any
+import asyncio
+import os
+import hashlib
+import magic
+import io
+from contextlib import asynccontextmanager
+import logging
+from pathlib import Path
+import json
+
+# Import custom modules
+from models import FileMetadata, FileUploadResponse, FileListResponse, StorageStats
+from minio_client import MinIOManager
+from thumbnail_generator import ThumbnailGenerator
+from metadata_manager import MetadataManager
+from file_processor import FileProcessor
+
+logging.basicConfig(level=logging.INFO)
+logger = logging.getLogger(__name__)
+
+# Global instances
+minio_manager = None
+thumbnail_generator = None
+metadata_manager = None
+file_processor = None
+
+@asynccontextmanager
+async def lifespan(app: FastAPI):
+    # Startup
+    global minio_manager, thumbnail_generator, metadata_manager, file_processor
+    
+    try:
+        # Initialize MinIO client
+        minio_manager = MinIOManager(
+            endpoint=os.getenv("MINIO_ENDPOINT", "minio:9000"),
+            access_key=os.getenv("MINIO_ACCESS_KEY", "minioadmin"),
+            secret_key=os.getenv("MINIO_SECRET_KEY", "minioadmin"),
+            secure=os.getenv("MINIO_SECURE", "false").lower() == "true"
+        )
+        await minio_manager.initialize()
+        logger.info("MinIO client initialized")
+        
+        # Initialize Metadata Manager (MongoDB)
+        metadata_manager = MetadataManager(
+            mongodb_url=os.getenv("MONGODB_URL", "mongodb://mongodb:27017"),
+            database=os.getenv("FILES_DB_NAME", "files_db")
+        )
+        await metadata_manager.connect()
+        logger.info("Metadata manager connected to MongoDB")
+        
+        # Initialize Thumbnail Generator
+        thumbnail_generator = ThumbnailGenerator(
+            minio_client=minio_manager,
+            cache_dir="/tmp/thumbnails"
+        )
+        logger.info("Thumbnail generator initialized")
+        
+        # Initialize File Processor
+        file_processor = FileProcessor(
+            minio_client=minio_manager,
+            metadata_manager=metadata_manager,
+            thumbnail_generator=thumbnail_generator
+        )
+        logger.info("File processor initialized")
+        
+    except Exception as e:
+        logger.error(f"Failed to start File service: {e}")
+        raise
+    
+    yield
+    
+    # Shutdown
+    if metadata_manager:
+        await metadata_manager.close()
+    
+    logger.info("File service shutdown complete")
+
+app = FastAPI(
+    title="File Management Service",
+    description="S3-compatible object storage with MinIO",
+    version="1.0.0",
+    lifespan=lifespan
+)
+
+# CORS middleware
+app.add_middleware(
+    CORSMiddleware,
+    allow_origins=["*"],
+    allow_credentials=True,
+    allow_methods=["*"],
+    allow_headers=["*"],
+)
+
+@app.get("/")
+async def root():
+    return {
+        "service": "File Management Service",
+        "status": "running",
+        "timestamp": datetime.now().isoformat()
+    }
+
+@app.get("/health")
+async def health_check():
+    return {
+        "status": "healthy",
+        "service": "files",
+        "components": {
+            "minio": "connected" if minio_manager and minio_manager.is_connected else "disconnected",
+            "mongodb": "connected" if metadata_manager and metadata_manager.is_connected else "disconnected",
+            "thumbnail_generator": "ready" if thumbnail_generator else "not_initialized"
+        },
+        "timestamp": datetime.now().isoformat()
+    }
+
+# File Upload Endpoints
+@app.post("/api/files/upload")
+async def upload_file(
+    file: UploadFile = File(...),
+    user_id: str = Form(...),
+    bucket: str = Form("default"),
+    public: bool = Form(False),
+    generate_thumbnail: bool = Form(True),
+    tags: Optional[str] = Form(None)
+):
+    """Upload a file to object storage"""
+    try:
+        # Validate file
+        if not file.filename:
+            raise HTTPException(status_code=400, detail="No file provided")
+        
+        # Process file upload
+        result = await file_processor.process_upload(
+            file=file,
+            user_id=user_id,
+            bucket=bucket,
+            public=public,
+            generate_thumbnail=generate_thumbnail,
+            tags=json.loads(tags) if tags else {}
+        )
+        
+        return FileUploadResponse(**result)
+    except Exception as e:
+        logger.error(f"File upload error: {e}")
+        raise HTTPException(status_code=500, detail=str(e))
+
+@app.post("/api/files/upload-multiple")
+async def upload_multiple_files(
+    files: List[UploadFile] = File(...),
+    user_id: str = Form(...),
+    bucket: str = Form("default"),
+    public: bool = Form(False)
+):
+    """Upload multiple files"""
+    try:
+        results = []
+        for file in files:
+            result = await file_processor.process_upload(
+                file=file,
+                user_id=user_id,
+                bucket=bucket,
+                public=public,
+                generate_thumbnail=True
+            )
+            results.append(result)
+        
+        return {
+            "uploaded": len(results),
+            "files": results
+        }
+    except Exception as e:
+        logger.error(f"Multiple file upload error: {e}")
+        raise HTTPException(status_code=500, detail=str(e))
+
+# File Retrieval Endpoints
+@app.get("/api/files/{file_id}")
+async def get_file(file_id: str):
+    """Get file by ID"""
+    try:
+        # Get metadata
+        metadata = await metadata_manager.get_file_metadata(file_id)
+        if not metadata:
+            raise HTTPException(status_code=404, detail="File not found")
+        
+        # Get file from MinIO
+        file_stream = await minio_manager.get_file(
+            bucket=metadata["bucket"],
+            object_name=metadata["object_name"]
+        )
+        
+        return StreamingResponse(
+            file_stream,
+            media_type=metadata.get("content_type", "application/octet-stream"),
+            headers={
+                "Content-Disposition": f'attachment; filename="{metadata["filename"]}"'
+            }
+        )
+    except HTTPException:
+        raise
+    except Exception as e:
+        logger.error(f"File retrieval error: {e}")
+        raise HTTPException(status_code=500, detail=str(e))
+
+@app.get("/api/files/{file_id}/metadata")
+async def get_file_metadata(file_id: str):
+    """Get file metadata"""
+    try:
+        metadata = await metadata_manager.get_file_metadata(file_id)
+        if not metadata:
+            raise HTTPException(status_code=404, detail="File not found")
+        
+        return FileMetadata(**metadata)
+    except HTTPException:
+        raise
+    except Exception as e:
+        logger.error(f"Metadata retrieval error: {e}")
+        raise HTTPException(status_code=500, detail=str(e))
+
+@app.get("/api/files/{file_id}/thumbnail")
+async def get_thumbnail(
+    file_id: str,
+    width: int = Query(200, ge=50, le=1000),
+    height: int = Query(200, ge=50, le=1000)
+):
+    """Get file thumbnail"""
+    try:
+        # Get metadata
+        metadata = await metadata_manager.get_file_metadata(file_id)
+        if not metadata:
+            raise HTTPException(status_code=404, detail="File not found")
+        
+        # Check if file has thumbnail
+        if not metadata.get("has_thumbnail"):
+            raise HTTPException(status_code=404, detail="No thumbnail available")
+        
+        # Get or generate thumbnail
+        thumbnail = await thumbnail_generator.get_thumbnail(
+            file_id=file_id,
+            bucket=metadata["bucket"],
+            object_name=metadata["object_name"],
+            width=width,
+            height=height
+        )
+        
+        return StreamingResponse(
+            io.BytesIO(thumbnail),
+            media_type="image/jpeg"
+        )
+    except HTTPException:
+        raise
+    except Exception as e:
+        logger.error(f"Thumbnail retrieval error: {e}")
+        raise HTTPException(status_code=500, detail=str(e))
+
+@app.get("/api/files/{file_id}/download")
+async def download_file(file_id: str):
+    """Download file with proper headers"""
+    try:
+        # Get metadata
+        metadata = await metadata_manager.get_file_metadata(file_id)
+        if not metadata:
+            raise HTTPException(status_code=404, detail="File not found")
+        
+        # Update download count
+        await metadata_manager.increment_download_count(file_id)
+        
+        # Get file from MinIO
+        file_stream = await minio_manager.get_file(
+            bucket=metadata["bucket"],
+            object_name=metadata["object_name"]
+        )
+        
+        return StreamingResponse(
+            file_stream,
+            media_type=metadata.get("content_type", "application/octet-stream"),
+            headers={
+                "Content-Disposition": f'attachment; filename="{metadata["filename"]}"',
+                "Content-Length": str(metadata["size"])
+            }
+        )
+    except HTTPException:
+        raise
+    except Exception as e:
+        logger.error(f"File download error: {e}")
+        raise HTTPException(status_code=500, detail=str(e))
+
+# File Management Endpoints
+@app.delete("/api/files/{file_id}")
+async def delete_file(file_id: str, user_id: str):
+    """Delete a file"""
+    try:
+        # Get metadata
+        metadata = await metadata_manager.get_file_metadata(file_id)
+        if not metadata:
+            raise HTTPException(status_code=404, detail="File not found")
+        
+        # Check ownership
+        if metadata["user_id"] != user_id:
+            raise HTTPException(status_code=403, detail="Permission denied")
+        
+        # Delete from MinIO
+        await minio_manager.delete_file(
+            bucket=metadata["bucket"],
+            object_name=metadata["object_name"]
+        )
+        
+        # Delete thumbnail if exists
+        if metadata.get("has_thumbnail"):
+            await thumbnail_generator.delete_thumbnail(file_id)
+        
+        # Delete metadata
+        await metadata_manager.delete_file_metadata(file_id)
+        
+        return {"status": "deleted", "file_id": file_id}
+    except HTTPException:
+        raise
+    except Exception as e:
+        logger.error(f"File deletion error: {e}")
+        raise HTTPException(status_code=500, detail=str(e))
+
+@app.patch("/api/files/{file_id}")
+async def update_file_metadata(
+    file_id: str,
+    user_id: str,
+    updates: Dict[str, Any]
+):
+    """Update file metadata"""
+    try:
+        # Get existing metadata
+        metadata = await metadata_manager.get_file_metadata(file_id)
+        if not metadata:
+            raise HTTPException(status_code=404, detail="File not found")
+        
+        # Check ownership
+        if metadata["user_id"] != user_id:
+            raise HTTPException(status_code=403, detail="Permission denied")
+        
+        # Update metadata
+        updated = await metadata_manager.update_file_metadata(file_id, updates)
+        
+        return {"status": "updated", "file_id": file_id, "metadata": updated}
+    except HTTPException:
+        raise
+    except Exception as e:
+        logger.error(f"Metadata update error: {e}")
+        raise HTTPException(status_code=500, detail=str(e))
+
+# File Listing Endpoints
+@app.get("/api/files")
+async def list_files(
+    user_id: Optional[str] = None,
+    bucket: str = Query("default"),
+    limit: int = Query(20, le=100),
+    offset: int = Query(0),
+    search: Optional[str] = None,
+    file_type: Optional[str] = None,
+    sort_by: str = Query("created_at", pattern="^(created_at|filename|size)$"),
+    order: str = Query("desc", pattern="^(asc|desc)$")
+):
+    """List files with filtering and pagination"""
+    try:
+        files = await metadata_manager.list_files(
+            user_id=user_id,
+            bucket=bucket,
+            limit=limit,
+            offset=offset,
+            search=search,
+            file_type=file_type,
+            sort_by=sort_by,
+            order=order
+        )
+        
+        return FileListResponse(**files)
+    except Exception as e:
+        logger.error(f"File listing error: {e}")
+        raise HTTPException(status_code=500, detail=str(e))
+
+@app.get("/api/files/user/{user_id}")
+async def get_user_files(
+    user_id: str,
+    limit: int = Query(20, le=100),
+    offset: int = Query(0)
+):
+    """Get all files for a specific user"""
+    try:
+        files = await metadata_manager.list_files(
+            user_id=user_id,
+            limit=limit,
+            offset=offset
+        )
+        
+        return FileListResponse(**files)
+    except Exception as e:
+        logger.error(f"User files listing error: {e}")
+        raise HTTPException(status_code=500, detail=str(e))
+
+# Storage Management Endpoints
+@app.get("/api/storage/stats")
+async def get_storage_stats():
+    """Get storage statistics"""
+    try:
+        stats = await minio_manager.get_storage_stats()
+        db_stats = await metadata_manager.get_storage_stats()
+        
+        return StorageStats(
+            total_files=db_stats["total_files"],
+            total_size=db_stats["total_size"],
+            buckets=stats["buckets"],
+            users_count=db_stats["users_count"],
+            file_types=db_stats["file_types"]
+        )
+    except Exception as e:
+        logger.error(f"Storage stats error: {e}")
+        raise HTTPException(status_code=500, detail=str(e))
+
+@app.post("/api/storage/buckets")
+async def create_bucket(bucket_name: str, public: bool = False):
+    """Create a new storage bucket"""
+    try:
+        await minio_manager.create_bucket(bucket_name, public=public)
+        return {"status": "created", "bucket": bucket_name}
+    except Exception as e:
+        logger.error(f"Bucket creation error: {e}")
+        raise HTTPException(status_code=500, detail=str(e))
+
+@app.get("/api/storage/buckets")
+async def list_buckets():
+    """List all storage buckets"""
+    try:
+        buckets = await minio_manager.list_buckets()
+        return {"buckets": buckets}
+    except Exception as e:
+        logger.error(f"Bucket listing error: {e}")
+        raise HTTPException(status_code=500, detail=str(e))
+
+# Presigned URL Endpoints
+@app.post("/api/files/presigned-upload")
+async def generate_presigned_upload_url(
+    filename: str,
+    content_type: str,
+    bucket: str = "default",
+    expires_in: int = Query(3600, ge=60, le=86400)
+):
+    """Generate presigned URL for direct upload to MinIO"""
+    try:
+        url = await minio_manager.generate_presigned_upload_url(
+            bucket=bucket,
+            object_name=f"{datetime.now().strftime('%Y%m%d')}/{filename}",
+            expires_in=expires_in
+        )
+        
+        return {
+            "upload_url": url,
+            "expires_in": expires_in,
+            "method": "PUT"
+        }
+    except Exception as e:
+        logger.error(f"Presigned URL generation error: {e}")
+        raise HTTPException(status_code=500, detail=str(e))
+
+@app.get("/api/files/{file_id}/share")
+async def generate_share_link(
+    file_id: str,
+    expires_in: int = Query(86400, ge=60, le=604800)  # 1 day default, max 7 days
+):
+    """Generate a shareable link for a file"""
+    try:
+        # Get metadata
+        metadata = await metadata_manager.get_file_metadata(file_id)
+        if not metadata:
+            raise HTTPException(status_code=404, detail="File not found")
+        
+        # Generate presigned URL
+        url = await minio_manager.generate_presigned_download_url(
+            bucket=metadata["bucket"],
+            object_name=metadata["object_name"],
+            expires_in=expires_in
+        )
+        
+        return {
+            "share_url": url,
+            "expires_in": expires_in,
+            "file_id": file_id,
+            "filename": metadata["filename"]
+        }
+    except HTTPException:
+        raise
+    except Exception as e:
+        logger.error(f"Share link generation error: {e}")
+        raise HTTPException(status_code=500, detail=str(e))
+
+# Batch Operations
+@app.post("/api/files/batch-delete")
+async def batch_delete_files(file_ids: List[str], user_id: str):
+    """Delete multiple files at once"""
+    try:
+        deleted = []
+        errors = []
+        
+        for file_id in file_ids:
+            try:
+                # Get metadata
+                metadata = await metadata_manager.get_file_metadata(file_id)
+                if metadata and metadata["user_id"] == user_id:
+                    # Delete from MinIO
+                    await minio_manager.delete_file(
+                        bucket=metadata["bucket"],
+                        object_name=metadata["object_name"]
+                    )
+                    # Delete metadata
+                    await metadata_manager.delete_file_metadata(file_id)
+                    deleted.append(file_id)
+                else:
+                    errors.append({"file_id": file_id, "error": "Not found or permission denied"})
+            except Exception as e:
+                errors.append({"file_id": file_id, "error": str(e)})
+        
+        return {
+            "deleted": deleted,
+            "errors": errors,
+            "total_deleted": len(deleted)
+        }
+    except Exception as e:
+        logger.error(f"Batch delete error: {e}")
+        raise HTTPException(status_code=500, detail=str(e))
+
+if __name__ == "__main__":
+    uvicorn.run(
+        "main:app",
+        host="0.0.0.0",
+        port=8000,
+        reload=True
+    )