""" MinIO Client for S3-compatible object storage """ from minio import Minio from minio.error import S3Error import asyncio import io from typing import Optional, Dict, Any, List import logging from datetime import timedelta logger = logging.getLogger(__name__) class MinIOManager: def __init__(self, endpoint: str, access_key: str, secret_key: str, secure: bool = False): self.endpoint = endpoint self.access_key = access_key self.secret_key = secret_key self.secure = secure self.client = None self.is_connected = False async def initialize(self): """Initialize MinIO client and create default buckets""" try: self.client = Minio( self.endpoint, access_key=self.access_key, secret_key=self.secret_key, secure=self.secure ) # Create default buckets default_buckets = ["default", "public", "thumbnails", "temp"] for bucket in default_buckets: await self.create_bucket(bucket, public=(bucket == "public")) self.is_connected = True logger.info(f"Connected to MinIO at {self.endpoint}") except Exception as e: logger.error(f"Failed to initialize MinIO: {e}") self.is_connected = False raise async def create_bucket(self, bucket_name: str, public: bool = False): """Create a new bucket""" try: # Run in executor to avoid blocking loop = asyncio.get_event_loop() # Check if bucket exists exists = await loop.run_in_executor( None, self.client.bucket_exists, bucket_name ) if not exists: await loop.run_in_executor( None, self.client.make_bucket, bucket_name ) logger.info(f"Created bucket: {bucket_name}") # Set bucket policy if public if public: policy = { "Version": "2012-10-17", "Statement": [ { "Effect": "Allow", "Principal": {"AWS": ["*"]}, "Action": ["s3:GetObject"], "Resource": [f"arn:aws:s3:::{bucket_name}/*"] } ] } import json await loop.run_in_executor( None, self.client.set_bucket_policy, bucket_name, json.dumps(policy) ) logger.info(f"Set public policy for bucket: {bucket_name}") except Exception as e: logger.error(f"Failed to create bucket {bucket_name}: {e}") raise async def upload_file(self, bucket: str, object_name: str, file_data: bytes, content_type: str = "application/octet-stream", metadata: Optional[Dict[str, str]] = None): """Upload a file to MinIO""" try: loop = asyncio.get_event_loop() # Convert bytes to BytesIO file_stream = io.BytesIO(file_data) length = len(file_data) # Upload file result = await loop.run_in_executor( None, self.client.put_object, bucket, object_name, file_stream, length, content_type, metadata ) logger.info(f"Uploaded {object_name} to {bucket}") return { "bucket": bucket, "object_name": object_name, "etag": result.etag, "version_id": result.version_id } except Exception as e: logger.error(f"Failed to upload file: {e}") raise async def get_file(self, bucket: str, object_name: str) -> io.BytesIO: """Get a file from MinIO""" try: loop = asyncio.get_event_loop() # Get object response = await loop.run_in_executor( None, self.client.get_object, bucket, object_name ) # Read data data = response.read() response.close() response.release_conn() return io.BytesIO(data) except Exception as e: logger.error(f"Failed to get file: {e}") raise async def delete_file(self, bucket: str, object_name: str): """Delete a file from MinIO""" try: loop = asyncio.get_event_loop() await loop.run_in_executor( None, self.client.remove_object, bucket, object_name ) logger.info(f"Deleted {object_name} from {bucket}") except Exception as e: logger.error(f"Failed to delete file: {e}") raise async def list_files(self, bucket: str, prefix: Optional[str] = None, recursive: bool = True) -> List[Dict[str, Any]]: """List files in a bucket""" try: loop = asyncio.get_event_loop() objects = await loop.run_in_executor( None, lambda: list(self.client.list_objects( bucket, prefix=prefix, recursive=recursive )) ) files = [] for obj in objects: files.append({ "name": obj.object_name, "size": obj.size, "last_modified": obj.last_modified, "etag": obj.etag, "content_type": obj.content_type }) return files except Exception as e: logger.error(f"Failed to list files: {e}") raise async def get_file_info(self, bucket: str, object_name: str) -> Dict[str, Any]: """Get file information""" try: loop = asyncio.get_event_loop() stat = await loop.run_in_executor( None, self.client.stat_object, bucket, object_name ) return { "size": stat.size, "etag": stat.etag, "content_type": stat.content_type, "last_modified": stat.last_modified, "metadata": stat.metadata } except Exception as e: logger.error(f"Failed to get file info: {e}") raise async def generate_presigned_download_url(self, bucket: str, object_name: str, expires_in: int = 3600) -> str: """Generate a presigned URL for downloading""" try: loop = asyncio.get_event_loop() url = await loop.run_in_executor( None, self.client.presigned_get_object, bucket, object_name, timedelta(seconds=expires_in) ) return url except Exception as e: logger.error(f"Failed to generate presigned URL: {e}") raise async def generate_presigned_upload_url(self, bucket: str, object_name: str, expires_in: int = 3600) -> str: """Generate a presigned URL for uploading""" try: loop = asyncio.get_event_loop() url = await loop.run_in_executor( None, self.client.presigned_put_object, bucket, object_name, timedelta(seconds=expires_in) ) return url except Exception as e: logger.error(f"Failed to generate presigned upload URL: {e}") raise async def copy_file(self, source_bucket: str, source_object: str, dest_bucket: str, dest_object: str): """Copy a file within MinIO""" try: loop = asyncio.get_event_loop() await loop.run_in_executor( None, self.client.copy_object, dest_bucket, dest_object, f"/{source_bucket}/{source_object}" ) logger.info(f"Copied {source_object} to {dest_object}") except Exception as e: logger.error(f"Failed to copy file: {e}") raise async def list_buckets(self) -> List[str]: """List all buckets""" try: loop = asyncio.get_event_loop() buckets = await loop.run_in_executor( None, self.client.list_buckets ) return [bucket.name for bucket in buckets] except Exception as e: logger.error(f"Failed to list buckets: {e}") raise async def get_storage_stats(self) -> Dict[str, Any]: """Get storage statistics""" try: buckets = await self.list_buckets() stats = { "buckets": buckets, "bucket_count": len(buckets), "bucket_stats": {} } # Get stats for each bucket for bucket in buckets: files = await self.list_files(bucket) total_size = sum(f["size"] for f in files) stats["bucket_stats"][bucket] = { "file_count": len(files), "total_size": total_size } return stats except Exception as e: logger.error(f"Failed to get storage stats: {e}") raise async def check_file_exists(self, bucket: str, object_name: str) -> bool: """Check if a file exists""" try: await self.get_file_info(bucket, object_name) return True except: return False