diff --git a/src/dataprocessor/domain/sqlite_datasaver.py b/src/dataprocessor/domain/sqlite_datasaver.py index 5fdac38..f7c03aa 100644 --- a/src/dataprocessor/domain/sqlite_datasaver.py +++ b/src/dataprocessor/domain/sqlite_datasaver.py @@ -4,7 +4,9 @@ import pandas as pd import logging from dataclasses import dataclass +from pathlib import Path from sqlalchemy.ext.asyncio import create_async_engine, AsyncEngine +from sqlalchemy.pool import NullPool from src.dataprocessor.domain.base_datasaver import BaseDataSaver @@ -21,9 +23,15 @@ class SQLiteDataSaver(BaseDataSaver): @classmethod async def create(cls, db_path: str) -> "SQLiteDataSaver": """Create a new instance of DataSaver.""" - # Build the full db url - db_url = f"sqlite+aiosqlite:///{db_path}" - engine = create_async_engine(db_url, future=True) + # Ensure the directory exists + db_file = Path(db_path) + db_file.parent.mkdir(parents=True, exist_ok=True) + + # Build the full db url with absolute path (4 slashes total) + db_url = f"sqlite+aiosqlite:///{db_file.as_posix()}" + + # Use NullPool to avoid connection pooling issues with SQLite + engine = create_async_engine(db_url, poolclass=NullPool, future=True) return cls(db_url=db_url, engine=engine) async def save_table(