aimond/site11
1
1
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

test: Fix Pydantic v2 compatibility and comprehensive API testing

Browse Source 
This commit migrates all models to Pydantic v2 and adds comprehensive
testing infrastructure for the news-engine-console backend.

Model Changes (Pydantic v2 Migration):
- Removed PyObjectId custom validators (v1 pattern incompatible with v2)
- Changed all model id fields from Optional[PyObjectId] to Optional[str]
- Replaced class Config with model_config = ConfigDict(populate_by_name=True)
- Updated User, Keyword, Pipeline, and Application models

Service Changes (ObjectId Handling):
- Added ObjectId to string conversion in all service methods before creating model instances
- Updated UserService: get_users(), get_user_by_id(), get_user_by_username()
- Updated KeywordService: 6 methods with ObjectId conversions
- Updated PipelineService: 8 methods with ObjectId conversions
- Updated ApplicationService: 6 methods with ObjectId conversions

Testing Infrastructure:
- Created comprehensive test_api.py (700+ lines) with 8 test suites:
  * Health check, Authentication, Users API, Keywords API, Pipelines API,
    Applications API, Monitoring API
- Created test_motor.py for debugging Motor async MongoDB connection
- Added Dockerfile for containerized deployment
- Created fix_objectid.py helper script for automated ObjectId conversion

Configuration Updates:
- Changed backend port from 8100 to 8101 (avoid conflict with pipeline_monitor)
- Made get_database() async for proper FastAPI dependency injection
- Updated DB_NAME from ai_writer_db to news_engine_console_db

Bug Fixes:
- Fixed environment variable override issue (system env > .env file)
- Fixed Pydantic v2 validator incompatibility causing TypeError
- Fixed list comprehension in bulk_create_keywords to properly convert ObjectIds

Test Results:
- All 8 test suites passing (100% success rate)
- Tested 37 API endpoints across all services
- No validation errors or ObjectId conversion issues

🤖 Generated with [Claude Code](https://claude.com/claude-code)

Co-Authored-By: Claude <noreply@anthropic.com>
This commit is contained in:
jungwoo choi
2025-11-04 17:17:54 +09:00
parent 52c857fced
commit 1d461a7ded
15 changed files with 757 additions and 149 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
services/news-engine-console/backend/app/core/config.py
View File

@ -4,7 +4,7 @@ from typing import List
class Settings(BaseSettings):
# MongoDB
MONGODB_URL: str = "mongodb://localhost:27017"
DB_NAME: str = "ai_writer_db"
DB_NAME: str = "news_engine_console_db"
# Redis
REDIS_URL: str = "redis://localhost:6379"
@ -17,7 +17,7 @@ class Settings(BaseSettings):
# Service
SERVICE_NAME: str = "news-engine-console"
API_V1_STR: str = "/api/v1"
PORT: int = 8100
PORT: int = 8101
# CORS
ALLOWED_ORIGINS: List[str] = [

4
services/news-engine-console/backend/app/core/database.py
View File

@ -19,6 +19,6 @@ async def close_mongo_connection():
db_instance.client.close()
print("Closed MongoDB connection")
def get_database():
"""Get database instance"""
async def get_database():
"""Get database instance (async for FastAPI dependency)"""
return db_instance.db

39
services/news-engine-console/backend/app/core/object_id.py Normal file
View File

@ -0,0 +1,39 @@
"""Custom ObjectId handler for Pydantic v2"""
from typing import Any
from bson import ObjectId
from pydantic import field_validator
from pydantic_core import core_schema
class PyObjectId(str):
"""Custom ObjectId type for Pydantic v2"""
@classmethod
def __get_pydantic_core_schema__(
cls,
_source_type: Any,
_handler: Any,
) -> core_schema.CoreSchema:
"""Pydantic v2 core schema"""
return core_schema.json_or_python_schema(
json_schema=core_schema.str_schema(),
python_schema=core_schema.union_schema([
core_schema.is_instance_schema(ObjectId),
core_schema.chain_schema([
core_schema.str_schema(),
core_schema.no_info_plain_validator_function(cls.validate),
])
]),
serialization=core_schema.plain_serializer_function_ser_schema(
lambda x: str(x)
),
)
@classmethod
def validate(cls, v: Any) -> ObjectId:
"""Validate ObjectId"""
if isinstance(v, ObjectId):
return v
if isinstance(v, str) and ObjectId.is_valid(v):
return ObjectId(v)
raise ValueError(f"Invalid ObjectId: {v}")

57
services/news-engine-console/backend/app/models/application.py
View File

@ -1,30 +1,29 @@
from datetime import datetime
from typing import Optional, List
from pydantic import BaseModel, Field
from bson import ObjectId
class PyObjectId(ObjectId):
"""Custom ObjectId type for Pydantic"""
@classmethod
def __get_validators__(cls):
yield cls.validate
@classmethod
def validate(cls, v):
if not ObjectId.is_valid(v):
raise ValueError("Invalid ObjectId")
return ObjectId(v)
@classmethod
def __get_pydantic_json_schema__(cls, field_schema):
field_schema.update(type="string")
from pydantic import BaseModel, Field, ConfigDict
class Application(BaseModel):
"""OAuth2 Application data model"""
id: Optional[PyObjectId] = Field(default=None, alias="_id")
model_config = ConfigDict(
populate_by_name=True,
json_schema_extra={
"example": {
"name": "News Frontend App",
"client_id": "news_app_12345",
"redirect_uris": [
"http://localhost:3000/auth/callback",
"https://news.example.com/auth/callback"
],
"grant_types": ["authorization_code", "refresh_token"],
"scopes": ["read", "write"],
"owner_id": "507f1f77bcf86cd799439011"
}
}
)
id: Optional[str] = Field(default=None, alias="_id")
name: str = Field(..., min_length=1, max_length=100)
client_id: str = Field(..., description="OAuth2 Client ID (unique)")
client_secret: str = Field(..., description="Hashed client secret")
@ -38,21 +37,3 @@ class Application(BaseModel):
owner_id: str = Field(..., description="User ID who owns this application")
created_at: datetime = Field(default_factory=datetime.utcnow)
updated_at: datetime = Field(default_factory=datetime.utcnow)
class Config:
populate_by_name = True
arbitrary_types_allowed = True
json_encoders = {ObjectId: str}
json_schema_extra = {
"example": {
"name": "News Frontend App",
"client_id": "news_app_12345",
"redirect_uris": [
"http://localhost:3000/auth/callback",
"https://news.example.com/auth/callback"
],
"grant_types": ["authorization_code", "refresh_token"],
"scopes": ["read", "write"],
"owner_id": "507f1f77bcf86cd799439011"
}
}

51
services/news-engine-console/backend/app/models/keyword.py
View File

@ -1,45 +1,14 @@
from datetime import datetime
from typing import Optional, Dict, Any
from pydantic import BaseModel, Field
from bson import ObjectId
class PyObjectId(ObjectId):
"""Custom ObjectId type for Pydantic"""
@classmethod
def __get_validators__(cls):
yield cls.validate
@classmethod
def validate(cls, v):
if not ObjectId.is_valid(v):
raise ValueError("Invalid ObjectId")
return ObjectId(v)
@classmethod
def __get_pydantic_json_schema__(cls, field_schema):
field_schema.update(type="string")
from pydantic import BaseModel, Field, ConfigDict
class Keyword(BaseModel):
"""Keyword data model for pipeline management"""
id: Optional[PyObjectId] = Field(default=None, alias="_id")
keyword: str = Field(..., min_length=1, max_length=200)
category: str = Field(..., description="Category: people, topics, companies")
status: str = Field(default="active", description="Status: active, inactive")
pipeline_type: str = Field(default="all", description="Pipeline type: rss, translation, all")
priority: int = Field(default=5, ge=1, le=10, description="Priority level 1-10")
metadata: Dict[str, Any] = Field(default_factory=dict)
created_at: datetime = Field(default_factory=datetime.utcnow)
updated_at: datetime = Field(default_factory=datetime.utcnow)
created_by: Optional[str] = Field(default=None, description="User ID who created this keyword")
class Config:
populate_by_name = True
arbitrary_types_allowed = True
json_encoders = {ObjectId: str}
json_schema_extra = {
model_config = ConfigDict(
populate_by_name=True,
json_schema_extra={
"example": {
"keyword": "도널드 트럼프",
"category": "people",
@ -53,3 +22,15 @@ class Keyword(BaseModel):
"created_by": "admin"
}
}
)
id: Optional[str] = Field(default=None, alias="_id")
keyword: str = Field(..., min_length=1, max_length=200)
category: str = Field(..., description="Category: people, topics, companies")
status: str = Field(default="active", description="Status: active, inactive")
pipeline_type: str = Field(default="all", description="Pipeline type: rss, translation, all")
priority: int = Field(default=5, ge=1, le=10, description="Priority level 1-10")
metadata: Dict[str, Any] = Field(default_factory=dict)
created_at: datetime = Field(default_factory=datetime.utcnow)
updated_at: datetime = Field(default_factory=datetime.utcnow)
created_by: Optional[str] = Field(default=None, description="User ID who created this keyword")

53
services/news-engine-console/backend/app/models/pipeline.py
View File

@ -1,24 +1,6 @@
from datetime import datetime
from typing import Optional, Dict, Any
from pydantic import BaseModel, Field
from bson import ObjectId
class PyObjectId(ObjectId):
"""Custom ObjectId type for Pydantic"""
@classmethod
def __get_validators__(cls):
yield cls.validate
@classmethod
def validate(cls, v):
if not ObjectId.is_valid(v):
raise ValueError("Invalid ObjectId")
return ObjectId(v)
@classmethod
def __get_pydantic_json_schema__(cls, field_schema):
field_schema.update(type="string")
from pydantic import BaseModel, Field, ConfigDict
class PipelineStats(BaseModel):
@ -33,23 +15,9 @@ class PipelineStats(BaseModel):
class Pipeline(BaseModel):
"""Pipeline data model for process management"""
id: Optional[PyObjectId] = Field(default=None, alias="_id")
name: str = Field(..., min_length=1, max_length=100)
type: str = Field(..., description="Type: rss_collector, translator, image_generator")
status: str = Field(default="stopped", description="Status: running, stopped, error")
config: Dict[str, Any] = Field(default_factory=dict)
schedule: Optional[str] = Field(default=None, description="Cron expression for scheduling")
stats: PipelineStats = Field(default_factory=PipelineStats)
last_run: Optional[datetime] = None
next_run: Optional[datetime] = None
created_at: datetime = Field(default_factory=datetime.utcnow)
updated_at: datetime = Field(default_factory=datetime.utcnow)
class Config:
populate_by_name = True
arbitrary_types_allowed = True
json_encoders = {ObjectId: str}
json_schema_extra = {
model_config = ConfigDict(
populate_by_name=True,
json_schema_extra={
"example": {
"name": "RSS Collector - Politics",
"type": "rss_collector",
@ -68,3 +36,16 @@ class Pipeline(BaseModel):
}
}
}
)
id: Optional[str] = Field(default=None, alias="_id")
name: str = Field(..., min_length=1, max_length=100)
type: str = Field(..., description="Type: rss_collector, translator, image_generator")
status: str = Field(default="stopped", description="Status: running, stopped, error")
config: Dict[str, Any] = Field(default_factory=dict)
schedule: Optional[str] = Field(default=None, description="Cron expression for scheduling")
stats: PipelineStats = Field(default_factory=PipelineStats)
last_run: Optional[datetime] = None
next_run: Optional[datetime] = None
created_at: datetime = Field(default_factory=datetime.utcnow)
updated_at: datetime = Field(default_factory=datetime.utcnow)

49
services/news-engine-console/backend/app/models/user.py
View File

@ -1,44 +1,14 @@
from datetime import datetime
from typing import Optional
from pydantic import BaseModel, Field, EmailStr
from bson import ObjectId
class PyObjectId(ObjectId):
"""Custom ObjectId type for Pydantic"""
@classmethod
def __get_validators__(cls):
yield cls.validate
@classmethod
def validate(cls, v):
if not ObjectId.is_valid(v):
raise ValueError("Invalid ObjectId")
return ObjectId(v)
@classmethod
def __get_pydantic_json_schema__(cls, field_schema):
field_schema.update(type="string")
from pydantic import BaseModel, Field, EmailStr, ConfigDict
class User(BaseModel):
"""User data model for authentication and authorization"""
id: Optional[PyObjectId] = Field(default=None, alias="_id")
username: str = Field(..., min_length=3, max_length=50)
email: EmailStr = Field(...)
hashed_password: str = Field(...)
full_name: str = Field(..., min_length=1, max_length=100)
role: str = Field(default="viewer", description="Role: admin, editor, viewer")
disabled: bool = Field(default=False)
created_at: datetime = Field(default_factory=datetime.utcnow)
last_login: Optional[datetime] = None
class Config:
populate_by_name = True
arbitrary_types_allowed = True
json_encoders = {ObjectId: str}
json_schema_extra = {
model_config = ConfigDict(
populate_by_name=True,
json_schema_extra={
"example": {
"username": "johndoe",
"email": "johndoe@example.com",
@ -47,3 +17,14 @@ class User(BaseModel):
"disabled": False
}
}
)
id: Optional[str] = Field(default=None, alias="_id")
username: str = Field(..., min_length=3, max_length=50)
email: EmailStr = Field(...)
hashed_password: str = Field(...)
full_name: str = Field(..., min_length=1, max_length=100)
role: str = Field(default="viewer", description="Role: admin, editor, viewer")
disabled: bool = Field(default=False)
created_at: datetime = Field(default_factory=datetime.utcnow)
last_login: Optional[datetime] = None

6
services/news-engine-console/backend/app/services/application_service.py
View File

@ -45,6 +45,7 @@ class ApplicationService:
applications = []
async for doc in cursor:
doc["_id"] = str(doc["_id"]) # Convert ObjectId to string
applications.append(Application(**doc))
return applications
@ -56,6 +57,7 @@ class ApplicationService:
doc = await self.collection.find_one({"_id": ObjectId(app_id)})
if doc:
doc["_id"] = str(doc["_id"]) # Convert ObjectId to string
return Application(**doc)
return None
@ -63,6 +65,7 @@ class ApplicationService:
"""Get an application by client ID"""
doc = await self.collection.find_one({"client_id": client_id})
if doc:
doc["_id"] = str(doc["_id"]) # Convert ObjectId to string
return Application(**doc)
return None
@ -101,6 +104,7 @@ class ApplicationService:
result = await self.collection.insert_one(app_dict)
app_dict["_id"] = result.inserted_id
app_dict["_id"] = str(app_dict["_id"]) # Convert ObjectId to string
application = Application(**app_dict)
# Return both application and plain text secret (only shown once)
@ -141,6 +145,7 @@ class ApplicationService:
)
if result:
result["_id"] = str(result["_id"]) # Convert ObjectId to string
return Application(**result)
return None
@ -208,6 +213,7 @@ class ApplicationService:
)
if result:
result["_id"] = str(result["_id"]) # Convert ObjectId to string
application = Application(**result)
return application, new_secret
return None

9
services/news-engine-console/backend/app/services/keyword_service.py
View File

@ -57,6 +57,7 @@ class KeywordService:
keywords = []
async for doc in cursor:
doc["_id"] = str(doc["_id"]) # Convert ObjectId to string
keywords.append(Keyword(**doc))
return keywords, total
@ -68,6 +69,7 @@ class KeywordService:
doc = await self.collection.find_one({"_id": ObjectId(keyword_id)})
if doc:
doc["_id"] = str(doc["_id"]) # Convert ObjectId to string
return Keyword(**doc)
return None
@ -88,6 +90,7 @@ class KeywordService:
result = await self.collection.insert_one(keyword_dict)
keyword_dict["_id"] = result.inserted_id
keyword_dict["_id"] = str(keyword_dict["_id"]) # Convert ObjectId to string
return Keyword(**keyword_dict)
async def update_keyword(
@ -118,6 +121,7 @@ class KeywordService:
)
if result:
result["_id"] = str(result["_id"]) # Convert ObjectId to string
return Keyword(**result)
return None
@ -152,6 +156,7 @@ class KeywordService:
)
if result:
result["_id"] = str(result["_id"]) # Convert ObjectId to string
return Keyword(**result)
return None
@ -227,8 +232,8 @@ class KeywordService:
result = await self.collection.insert_many(keywords_dicts)
# Update with inserted IDs
# Update with inserted IDs and convert ObjectId to string
for i, inserted_id in enumerate(result.inserted_ids):
keywords_dicts[i]["_id"] = inserted_id
keywords_dicts[i]["_id"] = str(inserted_id) # Convert ObjectId to string
return [Keyword(**kw) for kw in keywords_dicts]

8
services/news-engine-console/backend/app/services/pipeline_service.py
View File

@ -40,6 +40,7 @@ class PipelineService:
pipelines = []
async for doc in cursor:
doc["_id"] = str(doc["_id"]) # Convert ObjectId to string
pipelines.append(Pipeline(**doc))
return pipelines
@ -51,6 +52,7 @@ class PipelineService:
doc = await self.collection.find_one({"_id": ObjectId(pipeline_id)})
if doc:
doc["_id"] = str(doc["_id"]) # Convert ObjectId to string
return Pipeline(**doc)
return None
@ -67,6 +69,7 @@ class PipelineService:
result = await self.collection.insert_one(pipeline_dict)
pipeline_dict["_id"] = result.inserted_id
pipeline_dict["_id"] = str(pipeline_dict["_id"]) # Convert ObjectId to string
return Pipeline(**pipeline_dict)
async def update_pipeline(
@ -95,6 +98,7 @@ class PipelineService:
)
if result:
result["_id"] = str(result["_id"]) # Convert ObjectId to string
return Pipeline(**result)
return None
@ -140,6 +144,7 @@ class PipelineService:
"INFO",
f"Pipeline {pipeline.name} started"
)
result["_id"] = str(result["_id"]) # Convert ObjectId to string
return Pipeline(**result)
return None
@ -175,6 +180,7 @@ class PipelineService:
"INFO",
f"Pipeline {pipeline.name} stopped"
)
result["_id"] = str(result["_id"]) # Convert ObjectId to string
return Pipeline(**result)
return None
@ -244,6 +250,7 @@ class PipelineService:
)
if result:
result["_id"] = str(result["_id"]) # Convert ObjectId to string
return Pipeline(**result)
return None
@ -328,5 +335,6 @@ class PipelineService:
"INFO",
"Pipeline configuration updated"
)
result["_id"] = str(result["_id"]) # Convert ObjectId to string
return Pipeline(**result)
return None

3
services/news-engine-console/backend/app/services/user_service.py
View File

@ -49,6 +49,7 @@ class UserService:
users = []
async for doc in cursor:
doc["_id"] = str(doc["_id"]) # Convert ObjectId to string
users.append(User(**doc))
return users
@ -60,6 +61,7 @@ class UserService:
doc = await self.collection.find_one({"_id": ObjectId(user_id)})
if doc:
doc["_id"] = str(doc["_id"]) # Convert ObjectId to string
return User(**doc)
return None
@ -67,6 +69,7 @@ class UserService:
"""Get a user by username"""
doc = await self.collection.find_one({"username": username})
if doc:
doc["_id"] = str(doc["_id"]) # Convert ObjectId to string
return User(**doc)
return None