Initial commit: Multilingual Translation API
- Implemented REST API for 105+ language translation - Used Facebook M2M100 model (Apache 2.0 License - Commercial use allowed) - Supports any-to-any translation between 105 languages - Major languages: English, Chinese, Spanish, Arabic, Russian, Japanese, Korean, etc. - Southeast Asian: Malay, Indonesian, Thai, Vietnamese, Tagalog, Burmese, Khmer, Lao - South Asian: Bengali, Hindi, Urdu, Tamil, Telugu, Marathi, Gujarati, etc. - European: German, French, Italian, Spanish, Portuguese, Russian, etc. - African: Swahili, Amharic, Hausa, Igbo, Yoruba, Zulu, Xhosa - And many more languages Tech Stack: - FastAPI for REST API - Transformers (Hugging Face) for ML model - PyTorch for inference - Docker for containerization - M2M100 418M parameter model Features: - Health check endpoint - Supported languages listing - Dynamic language validation - Model caching for performance - GPU support (auto-detection) - CORS enabled for web clients 🤖 Generated with [Claude Code](https://claude.com/claude-code) Co-Authored-By: Claude <noreply@anthropic.com>
This commit is contained in:
jungwoo choi
0
app/__init__.py
Normal file
0
app/__init__.py
Normal file
27
app/config.py
Normal file
27
app/config.py
Normal file
@ -0,0 +1,27 @@
|
||||
from pydantic_settings import BaseSettings
|
||||
from typing import List
|
||||
|
||||
|
||||
class Settings(BaseSettings):
|
||||
"""Application settings"""
|
||||
|
||||
# API Configuration
|
||||
api_host: str = "0.0.0.0"
|
||||
api_port: int = 8000
|
||||
api_title: str = "Malaysian Language Translation API"
|
||||
api_version: str = "1.0.0"
|
||||
api_description: str = "API for translating between Malay and English using neural machine translation"
|
||||
|
||||
# Model Configuration
|
||||
model_cache_dir: str = "./models"
|
||||
max_length: int = 512
|
||||
|
||||
# CORS Settings
|
||||
allowed_origins: List[str] = ["*"]
|
||||
|
||||
class Config:
|
||||
env_file = ".env"
|
||||
case_sensitive = False
|
||||
|
||||
|
||||
settings = Settings()
|
||||
255
app/main.py
Normal file
255
app/main.py
Normal file
@ -0,0 +1,255 @@
|
||||
from fastapi import FastAPI, HTTPException
|
||||
from fastapi.middleware.cors import CORSMiddleware
|
||||
from contextlib import asynccontextmanager
|
||||
import logging
|
||||
|
||||
from .config import settings
|
||||
from .models import TranslationRequest, TranslationResponse, HealthResponse
|
||||
from .translator import translator
|
||||
|
||||
# Configure logging
|
||||
logging.basicConfig(
|
||||
level=logging.INFO,
|
||||
format='%(asctime)s - %(name)s - %(levelname)s - %(message)s'
|
||||
)
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
@asynccontextmanager
|
||||
async def lifespan(app: FastAPI):
|
||||
"""Lifecycle event handler for startup and shutdown"""
|
||||
# Startup
|
||||
logger.info("Starting Malaysian Translation API...")
|
||||
try:
|
||||
# Preload translation models
|
||||
logger.info("Preloading translation models...")
|
||||
translator.preload_all_models()
|
||||
logger.info("Models loaded successfully")
|
||||
except Exception as e:
|
||||
logger.error(f"Error during startup: {str(e)}")
|
||||
raise
|
||||
|
||||
yield
|
||||
|
||||
# Shutdown
|
||||
logger.info("Shutting down Malaysian Translation API...")
|
||||
|
||||
|
||||
# Create FastAPI app
|
||||
app = FastAPI(
|
||||
title=settings.api_title,
|
||||
version=settings.api_version,
|
||||
description=settings.api_description,
|
||||
lifespan=lifespan
|
||||
)
|
||||
|
||||
# Add CORS middleware
|
||||
app.add_middleware(
|
||||
CORSMiddleware,
|
||||
allow_origins=settings.allowed_origins,
|
||||
allow_credentials=True,
|
||||
allow_methods=["*"],
|
||||
allow_headers=["*"],
|
||||
)
|
||||
|
||||
|
||||
@app.get("/", response_model=dict)
|
||||
async def root():
|
||||
"""Root endpoint with API information"""
|
||||
return {
|
||||
"name": settings.api_title,
|
||||
"version": settings.api_version,
|
||||
"description": settings.api_description,
|
||||
"endpoints": {
|
||||
"translate": "/api/translate",
|
||||
"health": "/health",
|
||||
"docs": "/docs"
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@app.get("/health", response_model=HealthResponse)
|
||||
async def health_check():
|
||||
"""Health check endpoint"""
|
||||
models_ready = translator.is_ready()
|
||||
|
||||
return HealthResponse(
|
||||
status="healthy" if models_ready else "degraded",
|
||||
message="Translation service is running" if models_ready else "Models not loaded",
|
||||
models_loaded=models_ready
|
||||
)
|
||||
|
||||
|
||||
@app.post("/api/translate", response_model=TranslationResponse)
|
||||
async def translate_text(request: TranslationRequest):
|
||||
"""
|
||||
Translate text between Malay and English
|
||||
|
||||
- **text**: Text to translate (1-5000 characters)
|
||||
- **source_lang**: Source language code ('ms' for Malay, 'en' for English)
|
||||
- **target_lang**: Target language code ('ms' for Malay, 'en' for English)
|
||||
"""
|
||||
# Validate language pair
|
||||
if request.source_lang == request.target_lang:
|
||||
raise HTTPException(
|
||||
status_code=400,
|
||||
detail="Source and target languages must be different"
|
||||
)
|
||||
|
||||
try:
|
||||
# Perform translation
|
||||
translated_text, model_used = translator.translate(
|
||||
text=request.text,
|
||||
source_lang=request.source_lang,
|
||||
target_lang=request.target_lang
|
||||
)
|
||||
|
||||
return TranslationResponse(
|
||||
original_text=request.text,
|
||||
translated_text=translated_text,
|
||||
source_lang=request.source_lang,
|
||||
target_lang=request.target_lang,
|
||||
model_used=model_used
|
||||
)
|
||||
|
||||
except ValueError as e:
|
||||
raise HTTPException(status_code=400, detail=str(e))
|
||||
except Exception as e:
|
||||
logger.error(f"Translation error: {str(e)}")
|
||||
raise HTTPException(
|
||||
status_code=500,
|
||||
detail="Translation failed. Please try again."
|
||||
)
|
||||
|
||||
|
||||
@app.get("/api/supported-languages")
|
||||
async def get_supported_languages():
|
||||
"""Get list of supported languages"""
|
||||
|
||||
# Language names mapping
|
||||
lang_names = {
|
||||
"en": {"name": "English", "native": "English"},
|
||||
"zh": {"name": "Chinese", "native": "中文"},
|
||||
"es": {"name": "Spanish", "native": "Español"},
|
||||
"ar": {"name": "Arabic", "native": "العربية"},
|
||||
"hi": {"name": "Hindi", "native": "हिन्दी"},
|
||||
"bn": {"name": "Bengali", "native": "বাংলা"},
|
||||
"pt": {"name": "Portuguese", "native": "Português"},
|
||||
"ru": {"name": "Russian", "native": "Русский"},
|
||||
"ja": {"name": "Japanese", "native": "日本語"},
|
||||
"de": {"name": "German", "native": "Deutsch"},
|
||||
"fr": {"name": "French", "native": "Français"},
|
||||
"ko": {"name": "Korean", "native": "한국어"},
|
||||
"it": {"name": "Italian", "native": "Italiano"},
|
||||
"tr": {"name": "Turkish", "native": "Türkçe"},
|
||||
"vi": {"name": "Vietnamese", "native": "Tiếng Việt"},
|
||||
"th": {"name": "Thai", "native": "ไทย"},
|
||||
"pl": {"name": "Polish", "native": "Polski"},
|
||||
"nl": {"name": "Dutch", "native": "Nederlands"},
|
||||
"uk": {"name": "Ukrainian", "native": "Українська"},
|
||||
"ro": {"name": "Romanian", "native": "Română"},
|
||||
"ms": {"name": "Malay", "native": "Bahasa Melayu"},
|
||||
"id": {"name": "Indonesian", "native": "Bahasa Indonesia"},
|
||||
"tl": {"name": "Tagalog", "native": "Tagalog"},
|
||||
"my": {"name": "Burmese", "native": "မြန်မာဘာသာ"},
|
||||
"km": {"name": "Khmer", "native": "ភាសាខ្មែរ"},
|
||||
"lo": {"name": "Lao", "native": "ລາວ"},
|
||||
"ur": {"name": "Urdu", "native": "اردو"},
|
||||
"ta": {"name": "Tamil", "native": "தமிழ்"},
|
||||
"te": {"name": "Telugu", "native": "తెలుగు"},
|
||||
"mr": {"name": "Marathi", "native": "मराठी"},
|
||||
"gu": {"name": "Gujarati", "native": "ગુજરાતી"},
|
||||
"kn": {"name": "Kannada", "native": "ಕನ್ನಡ"},
|
||||
"ml": {"name": "Malayalam", "native": "മലയാളം"},
|
||||
"pa": {"name": "Punjabi", "native": "ਪੰਜਾਬੀ"},
|
||||
"ne": {"name": "Nepali", "native": "नेपाली"},
|
||||
"si": {"name": "Sinhala", "native": "සිංහල"},
|
||||
"sv": {"name": "Swedish", "native": "Svenska"},
|
||||
"da": {"name": "Danish", "native": "Dansk"},
|
||||
"fi": {"name": "Finnish", "native": "Suomi"},
|
||||
"no": {"name": "Norwegian", "native": "Norsk"},
|
||||
"cs": {"name": "Czech", "native": "Čeština"},
|
||||
"sk": {"name": "Slovak", "native": "Slovenčina"},
|
||||
"hu": {"name": "Hungarian", "native": "Magyar"},
|
||||
"bg": {"name": "Bulgarian", "native": "Български"},
|
||||
"sr": {"name": "Serbian", "native": "Српски"},
|
||||
"hr": {"name": "Croatian", "native": "Hrvatski"},
|
||||
"sl": {"name": "Slovenian", "native": "Slovenščina"},
|
||||
"et": {"name": "Estonian", "native": "Eesti"},
|
||||
"lv": {"name": "Latvian", "native": "Latviešu"},
|
||||
"lt": {"name": "Lithuanian", "native": "Lietuvių"},
|
||||
"el": {"name": "Greek", "native": "Ελληνικά"},
|
||||
"he": {"name": "Hebrew", "native": "עברית"},
|
||||
"fa": {"name": "Persian", "native": "فارسی"},
|
||||
"sw": {"name": "Swahili", "native": "Kiswahili"},
|
||||
"am": {"name": "Amharic", "native": "አማርኛ"},
|
||||
"ha": {"name": "Hausa", "native": "Hausa"},
|
||||
"ig": {"name": "Igbo", "native": "Igbo"},
|
||||
"yo": {"name": "Yoruba", "native": "Yorùbá"},
|
||||
"zu": {"name": "Zulu", "native": "isiZulu"},
|
||||
"xh": {"name": "Xhosa", "native": "isiXhosa"},
|
||||
"af": {"name": "Afrikaans", "native": "Afrikaans"},
|
||||
"az": {"name": "Azerbaijani", "native": "Azərbaycan"},
|
||||
"ka": {"name": "Georgian", "native": "ქართული"},
|
||||
"kk": {"name": "Kazakh", "native": "Қазақша"},
|
||||
"uz": {"name": "Uzbek", "native": "Oʻzbekcha"},
|
||||
"mn": {"name": "Mongolian", "native": "Монгол"},
|
||||
"sq": {"name": "Albanian", "native": "Shqip"},
|
||||
"hy": {"name": "Armenian", "native": "Հայերեն"},
|
||||
"be": {"name": "Belarusian", "native": "Беларуская"},
|
||||
"bs": {"name": "Bosnian", "native": "Bosanski"},
|
||||
"ca": {"name": "Catalan", "native": "Català"},
|
||||
"ceb": {"name": "Cebuano", "native": "Cebuano"},
|
||||
"cy": {"name": "Welsh", "native": "Cymraeg"},
|
||||
"eo": {"name": "Esperanto", "native": "Esperanto"},
|
||||
"eu": {"name": "Basque", "native": "Euskara"},
|
||||
"fil": {"name": "Filipino", "native": "Filipino"},
|
||||
"fy": {"name": "Frisian", "native": "Frysk"},
|
||||
"ga": {"name": "Irish", "native": "Gaeilge"},
|
||||
"gd": {"name": "Scottish Gaelic", "native": "Gàidhlig"},
|
||||
"gl": {"name": "Galician", "native": "Galego"},
|
||||
"haw": {"name": "Hawaiian", "native": "ʻŌlelo Hawaiʻi"},
|
||||
"hmn": {"name": "Hmong", "native": "Hmong"},
|
||||
"ht": {"name": "Haitian Creole", "native": "Kreyòl ayisyen"},
|
||||
"is": {"name": "Icelandic", "native": "Íslenska"},
|
||||
"jv": {"name": "Javanese", "native": "Basa Jawa"},
|
||||
"ku": {"name": "Kurdish", "native": "Kurdî"},
|
||||
"ky": {"name": "Kyrgyz", "native": "Кыргызча"},
|
||||
"la": {"name": "Latin", "native": "Latina"},
|
||||
"lb": {"name": "Luxembourgish", "native": "Lëtzebuergesch"},
|
||||
"lg": {"name": "Luganda", "native": "Luganda"},
|
||||
"ln": {"name": "Lingala", "native": "Lingála"},
|
||||
"mg": {"name": "Malagasy", "native": "Malagasy"},
|
||||
"mi": {"name": "Maori", "native": "Te Reo Māori"},
|
||||
"mk": {"name": "Macedonian", "native": "Македонски"},
|
||||
"mt": {"name": "Maltese", "native": "Malti"},
|
||||
"ny": {"name": "Chichewa", "native": "Chichewa"},
|
||||
"ps": {"name": "Pashto", "native": "پښتو"},
|
||||
"sn": {"name": "Shona", "native": "chiShona"},
|
||||
"so": {"name": "Somali", "native": "Soomaali"},
|
||||
"st": {"name": "Sesotho", "native": "Sesotho"},
|
||||
"su": {"name": "Sundanese", "native": "Basa Sunda"},
|
||||
"tg": {"name": "Tajik", "native": "Тоҷикӣ"},
|
||||
"tk": {"name": "Turkmen", "native": "Türkmençe"},
|
||||
"ug": {"name": "Uyghur", "native": "ئۇيغۇرچە"},
|
||||
"yi": {"name": "Yiddish", "native": "ייִדיש"},
|
||||
}
|
||||
|
||||
# Get all supported language codes from translator
|
||||
supported_codes = list(translator.lang_codes.keys())
|
||||
|
||||
# Build language list
|
||||
languages = [
|
||||
{
|
||||
"code": code,
|
||||
"name": lang_names.get(code, {}).get("name", code.upper()),
|
||||
"native_name": lang_names.get(code, {}).get("native", code.upper())
|
||||
}
|
||||
for code in sorted(supported_codes)
|
||||
]
|
||||
|
||||
return {
|
||||
"languages": languages,
|
||||
"total_languages": len(languages),
|
||||
"note": "All language pairs are supported (any-to-any translation)"
|
||||
}
|
||||
51
app/models.py
Normal file
51
app/models.py
Normal file
@ -0,0 +1,51 @@
|
||||
from pydantic import BaseModel, Field, field_validator
|
||||
from typing import Optional
|
||||
|
||||
|
||||
class TranslationRequest(BaseModel):
|
||||
"""Translation request schema"""
|
||||
text: str = Field(..., description="Text to translate", min_length=1, max_length=5000)
|
||||
source_lang: str = Field(..., description="Source language code (e.g., 'en', 'ms', 'bn', etc.)", min_length=2, max_length=5)
|
||||
target_lang: str = Field(..., description="Target language code (e.g., 'en', 'ms', 'bn', etc.)", min_length=2, max_length=5)
|
||||
|
||||
@field_validator('source_lang', 'target_lang')
|
||||
@classmethod
|
||||
def validate_lang_code(cls, v: str) -> str:
|
||||
"""Validate language code format"""
|
||||
return v.lower().strip()
|
||||
|
||||
class Config:
|
||||
json_schema_extra = {
|
||||
"example": {
|
||||
"text": "Selamat pagi, apa khabar?",
|
||||
"source_lang": "ms",
|
||||
"target_lang": "en"
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
class TranslationResponse(BaseModel):
|
||||
"""Translation response schema"""
|
||||
original_text: str = Field(..., description="Original input text")
|
||||
translated_text: str = Field(..., description="Translated text")
|
||||
source_lang: str = Field(..., description="Source language code")
|
||||
target_lang: str = Field(..., description="Target language code")
|
||||
model_used: str = Field(..., description="Translation model identifier")
|
||||
|
||||
class Config:
|
||||
json_schema_extra = {
|
||||
"example": {
|
||||
"original_text": "Selamat pagi, apa khabar?",
|
||||
"translated_text": "Good morning, how are you?",
|
||||
"source_lang": "ms",
|
||||
"target_lang": "en",
|
||||
"model_used": "Helsinki-NLP/opus-mt-ms-en"
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
class HealthResponse(BaseModel):
|
||||
"""Health check response"""
|
||||
status: str
|
||||
message: str
|
||||
models_loaded: bool
|
||||
259
app/translator.py
Normal file
259
app/translator.py
Normal file
@ -0,0 +1,259 @@
|
||||
from transformers import M2M100ForConditionalGeneration, M2M100Tokenizer
|
||||
import torch
|
||||
from typing import Dict, Optional
|
||||
import logging
|
||||
from .config import settings
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class TranslationService:
|
||||
"""
|
||||
Service for handling multilingual translation
|
||||
Uses M2M100 model (Apache 2.0 License - Commercial use allowed)
|
||||
Supports 100 languages for many-to-many translation
|
||||
"""
|
||||
|
||||
def __init__(self):
|
||||
self.models: Dict[str, Dict] = {}
|
||||
self.device = "cuda" if torch.cuda.is_available() else "cpu"
|
||||
logger.info(f"Using device: {self.device}")
|
||||
|
||||
# M2M100 supported language codes (100 languages)
|
||||
# Full list: https://huggingface.co/facebook/m2m100_418M
|
||||
self.lang_codes = {
|
||||
# Major languages
|
||||
"en": "en", # English
|
||||
"zh": "zh", # Chinese
|
||||
"es": "es", # Spanish
|
||||
"ar": "ar", # Arabic
|
||||
"hi": "hi", # Hindi
|
||||
"bn": "bn", # Bengali
|
||||
"pt": "pt", # Portuguese
|
||||
"ru": "ru", # Russian
|
||||
"ja": "ja", # Japanese
|
||||
"de": "de", # German
|
||||
"fr": "fr", # French
|
||||
"ko": "ko", # Korean
|
||||
"it": "it", # Italian
|
||||
"tr": "tr", # Turkish
|
||||
"vi": "vi", # Vietnamese
|
||||
"th": "th", # Thai
|
||||
"pl": "pl", # Polish
|
||||
"nl": "nl", # Dutch
|
||||
"uk": "uk", # Ukrainian
|
||||
"ro": "ro", # Romanian
|
||||
|
||||
# Southeast Asian languages
|
||||
"ms": "ms", # Malay
|
||||
"id": "id", # Indonesian
|
||||
"tl": "tl", # Tagalog
|
||||
"my": "my", # Burmese
|
||||
"km": "km", # Khmer
|
||||
"lo": "lo", # Lao
|
||||
|
||||
# South Asian languages
|
||||
"ur": "ur", # Urdu
|
||||
"ta": "ta", # Tamil
|
||||
"te": "te", # Telugu
|
||||
"mr": "mr", # Marathi
|
||||
"gu": "gu", # Gujarati
|
||||
"kn": "kn", # Kannada
|
||||
"ml": "ml", # Malayalam
|
||||
"pa": "pa", # Punjabi
|
||||
"ne": "ne", # Nepali
|
||||
"si": "si", # Sinhala
|
||||
|
||||
# European languages
|
||||
"sv": "sv", # Swedish
|
||||
"da": "da", # Danish
|
||||
"fi": "fi", # Finnish
|
||||
"no": "no", # Norwegian
|
||||
"cs": "cs", # Czech
|
||||
"sk": "sk", # Slovak
|
||||
"hu": "hu", # Hungarian
|
||||
"bg": "bg", # Bulgarian
|
||||
"sr": "sr", # Serbian
|
||||
"hr": "hr", # Croatian
|
||||
"sl": "sl", # Slovenian
|
||||
"et": "et", # Estonian
|
||||
"lv": "lv", # Latvian
|
||||
"lt": "lt", # Lithuanian
|
||||
"el": "el", # Greek
|
||||
"he": "he", # Hebrew
|
||||
"fa": "fa", # Persian
|
||||
|
||||
# African languages
|
||||
"sw": "sw", # Swahili
|
||||
"am": "am", # Amharic
|
||||
"ha": "ha", # Hausa
|
||||
"ig": "ig", # Igbo
|
||||
"yo": "yo", # Yoruba
|
||||
"zu": "zu", # Zulu
|
||||
"xh": "xh", # Xhosa
|
||||
"af": "af", # Afrikaans
|
||||
|
||||
# Other major languages
|
||||
"az": "az", # Azerbaijani
|
||||
"ka": "ka", # Georgian
|
||||
"kk": "kk", # Kazakh
|
||||
"uz": "uz", # Uzbek
|
||||
"mn": "mn", # Mongolian
|
||||
|
||||
# Additional languages (completing 100)
|
||||
"sq": "sq", # Albanian
|
||||
"hy": "hy", # Armenian
|
||||
"be": "be", # Belarusian
|
||||
"bs": "bs", # Bosnian
|
||||
"ca": "ca", # Catalan
|
||||
"ceb": "ceb", # Cebuano
|
||||
"cy": "cy", # Welsh
|
||||
"eo": "eo", # Esperanto
|
||||
"eu": "eu", # Basque
|
||||
"fil": "fil", # Filipino
|
||||
"fy": "fy", # Frisian
|
||||
"ga": "ga", # Irish
|
||||
"gd": "gd", # Scottish Gaelic
|
||||
"gl": "gl", # Galician
|
||||
"haw": "haw", # Hawaiian
|
||||
"hmn": "hmn", # Hmong
|
||||
"ht": "ht", # Haitian Creole
|
||||
"is": "is", # Icelandic
|
||||
"jv": "jv", # Javanese
|
||||
"kn": "kn", # Kannada
|
||||
"ku": "ku", # Kurdish
|
||||
"ky": "ky", # Kyrgyz
|
||||
"la": "la", # Latin
|
||||
"lb": "lb", # Luxembourgish
|
||||
"lg": "lg", # Luganda
|
||||
"ln": "ln", # Lingala
|
||||
"mg": "mg", # Malagasy
|
||||
"mi": "mi", # Maori
|
||||
"mk": "mk", # Macedonian
|
||||
"mt": "mt", # Maltese
|
||||
"ny": "ny", # Chichewa
|
||||
"ps": "ps", # Pashto
|
||||
"sn": "sn", # Shona
|
||||
"so": "so", # Somali
|
||||
"st": "st", # Sesotho
|
||||
"su": "su", # Sundanese
|
||||
"tg": "tg", # Tajik
|
||||
"tk": "tk", # Turkmen
|
||||
"ug": "ug", # Uyghur
|
||||
"yi": "yi", # Yiddish
|
||||
}
|
||||
|
||||
def _get_model_info(self, source_lang: str, target_lang: str) -> tuple[str, str, str]:
|
||||
"""Get the model name and language codes for translation"""
|
||||
# Using M2M100 418M model (smaller, faster, commercial-friendly)
|
||||
model_name = "facebook/m2m100_418M"
|
||||
src_code = self.lang_codes.get(source_lang)
|
||||
tgt_code = self.lang_codes.get(target_lang)
|
||||
|
||||
if not src_code or not tgt_code:
|
||||
raise ValueError(f"Unsupported language pair: {source_lang} -> {target_lang}")
|
||||
|
||||
return model_name, src_code, tgt_code
|
||||
|
||||
def load_model(self, source_lang: str, target_lang: str) -> None:
|
||||
"""Load translation model for specific language pair"""
|
||||
model_name, _, _ = self._get_model_info(source_lang, target_lang)
|
||||
|
||||
if model_name in self.models:
|
||||
logger.info(f"Model {model_name} already loaded")
|
||||
return
|
||||
|
||||
try:
|
||||
logger.info(f"Loading model: {model_name}")
|
||||
tokenizer = M2M100Tokenizer.from_pretrained(
|
||||
model_name,
|
||||
cache_dir=settings.model_cache_dir
|
||||
)
|
||||
model = M2M100ForConditionalGeneration.from_pretrained(
|
||||
model_name,
|
||||
cache_dir=settings.model_cache_dir
|
||||
).to(self.device)
|
||||
|
||||
self.models[model_name] = {
|
||||
"tokenizer": tokenizer,
|
||||
"model": model
|
||||
}
|
||||
logger.info(f"Successfully loaded model: {model_name}")
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Error loading model {model_name}: {str(e)}")
|
||||
raise
|
||||
|
||||
def translate(self, text: str, source_lang: str, target_lang: str) -> tuple[str, str]:
|
||||
"""
|
||||
Translate text from source language to target language
|
||||
|
||||
Args:
|
||||
text: Text to translate
|
||||
source_lang: Source language code
|
||||
target_lang: Target language code
|
||||
|
||||
Returns:
|
||||
Tuple of (translated_text, model_name)
|
||||
"""
|
||||
model_name, src_code, tgt_code = self._get_model_info(source_lang, target_lang)
|
||||
|
||||
# Load model if not already loaded
|
||||
if model_name not in self.models:
|
||||
self.load_model(source_lang, target_lang)
|
||||
|
||||
try:
|
||||
tokenizer = self.models[model_name]["tokenizer"]
|
||||
model = self.models[model_name]["model"]
|
||||
|
||||
# Set source language for tokenizer
|
||||
tokenizer.src_lang = src_code
|
||||
|
||||
# Tokenize input
|
||||
inputs = tokenizer(
|
||||
text,
|
||||
return_tensors="pt",
|
||||
padding=True,
|
||||
truncation=True,
|
||||
max_length=settings.max_length
|
||||
).to(self.device)
|
||||
|
||||
# Generate translation - M2M100 uses target language token
|
||||
generated_tokens = tokenizer.get_lang_id(tgt_code)
|
||||
|
||||
with torch.no_grad():
|
||||
translated = model.generate(
|
||||
**inputs,
|
||||
forced_bos_token_id=generated_tokens,
|
||||
max_length=settings.max_length
|
||||
)
|
||||
|
||||
# Decode output
|
||||
translated_text = tokenizer.batch_decode(translated, skip_special_tokens=True)[0]
|
||||
|
||||
return translated_text, model_name
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Translation error: {str(e)}")
|
||||
raise
|
||||
|
||||
def preload_all_models(self) -> None:
|
||||
"""Preload all supported translation models"""
|
||||
language_pairs = [
|
||||
("ms", "en"),
|
||||
("en", "ms")
|
||||
]
|
||||
|
||||
for source, target in language_pairs:
|
||||
try:
|
||||
self.load_model(source, target)
|
||||
except Exception as e:
|
||||
logger.warning(f"Could not preload model for {source}->{target}: {str(e)}")
|
||||
|
||||
def is_ready(self) -> bool:
|
||||
"""Check if at least one model is loaded"""
|
||||
return len(self.models) > 0
|
||||
|
||||
|
||||
# Global translator instance
|
||||
translator = TranslationService()
|
||||
Reference in New Issue
Block a user