fix: datasaver for Azure
This commit is contained in:
parent
845636a424
commit
b3193ee56d
1 changed files with 11 additions and 3 deletions
|
|
@ -4,7 +4,9 @@ import pandas as pd
|
|||
import logging
|
||||
|
||||
from dataclasses import dataclass
|
||||
from pathlib import Path
|
||||
from sqlalchemy.ext.asyncio import create_async_engine, AsyncEngine
|
||||
from sqlalchemy.pool import NullPool
|
||||
|
||||
from src.dataprocessor.domain.base_datasaver import BaseDataSaver
|
||||
|
||||
|
|
@ -21,9 +23,15 @@ class SQLiteDataSaver(BaseDataSaver):
|
|||
@classmethod
|
||||
async def create(cls, db_path: str) -> "SQLiteDataSaver":
|
||||
"""Create a new instance of DataSaver."""
|
||||
# Build the full db url
|
||||
db_url = f"sqlite+aiosqlite:///{db_path}"
|
||||
engine = create_async_engine(db_url, future=True)
|
||||
# Ensure the directory exists
|
||||
db_file = Path(db_path)
|
||||
db_file.parent.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
# Build the full db url with absolute path (4 slashes total)
|
||||
db_url = f"sqlite+aiosqlite:///{db_file.as_posix()}"
|
||||
|
||||
# Use NullPool to avoid connection pooling issues with SQLite
|
||||
engine = create_async_engine(db_url, poolclass=NullPool, future=True)
|
||||
return cls(db_url=db_url, engine=engine)
|
||||
|
||||
async def save_table(
|
||||
|
|
|
|||
Loading…
Reference in a new issue