From b3193ee56d85e0a53e5d620f45e7b3d5d102dccb Mon Sep 17 00:00:00 2001 From: Christopher Gondek Date: Wed, 15 Oct 2025 11:42:44 +0200 Subject: [PATCH] fix: datasaver for Azure --- src/dataprocessor/domain/sqlite_datasaver.py | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/src/dataprocessor/domain/sqlite_datasaver.py b/src/dataprocessor/domain/sqlite_datasaver.py index 5fdac38..f7c03aa 100644 --- a/src/dataprocessor/domain/sqlite_datasaver.py +++ b/src/dataprocessor/domain/sqlite_datasaver.py @@ -4,7 +4,9 @@ import pandas as pd import logging from dataclasses import dataclass +from pathlib import Path from sqlalchemy.ext.asyncio import create_async_engine, AsyncEngine +from sqlalchemy.pool import NullPool from src.dataprocessor.domain.base_datasaver import BaseDataSaver @@ -21,9 +23,15 @@ class SQLiteDataSaver(BaseDataSaver): @classmethod async def create(cls, db_path: str) -> "SQLiteDataSaver": """Create a new instance of DataSaver.""" - # Build the full db url - db_url = f"sqlite+aiosqlite:///{db_path}" - engine = create_async_engine(db_url, future=True) + # Ensure the directory exists + db_file = Path(db_path) + db_file.parent.mkdir(parents=True, exist_ok=True) + + # Build the full db url with absolute path (4 slashes total) + db_url = f"sqlite+aiosqlite:///{db_file.as_posix()}" + + # Use NullPool to avoid connection pooling issues with SQLite + engine = create_async_engine(db_url, poolclass=NullPool, future=True) return cls(db_url=db_url, engine=engine) async def save_table(