gateway/app.py

import os
import sys
from urllib.parse import quote_plus

os.environ["NUMEXPR_MAX_THREADS"] = "12"

from fastapi import FastAPI
from fastapi.middleware.cors import CORSMiddleware
from fastapi.security import HTTPBearer
from contextlib import asynccontextmanager

import logging
from logging.handlers import RotatingFileHandler
from datetime import datetime

from modules.shared.configuration import APP_CONFIG
from modules.shared.eventManagement import eventManager
from modules.features import featuresLifecycle as featuresLifecycle

class DailyRotatingFileHandler(RotatingFileHandler):
    """
    A rotating file handler that automatically switches to a new file when the date changes.
    The log file name includes the current date and switches at midnight.
    """

    def __init__(
        self, logDir, filenamePrefix, maxBytes=10485760, backupCount=5, **kwargs
    ):
        self.logDir = logDir
        self.filenamePrefix = filenamePrefix
        self.currentDate = None
        self.currentFile = None

        # Initialize with today's file
        self._updateFileIfNeeded()

        # Call parent constructor with current file
        super().__init__(
            self.currentFile, maxBytes=maxBytes, backupCount=backupCount, **kwargs
        )

    def _updateFileIfNeeded(self):
        """Update the log file if the date has changed"""
        today = datetime.now().strftime("%Y%m%d")

        if self.currentDate != today:
            self.currentDate = today
            newFile = os.path.join(self.logDir, f"{self.filenamePrefix}_{today}.log")

            if self.currentFile != newFile:
                self.currentFile = newFile
                return True
        return False

    def emit(self, record):
        """Emit a log record, switching files if date has changed"""
        # Check if we need to switch to a new file
        if self._updateFileIfNeeded():
            # Close current file and open new one
            if self.stream:
                self.stream.close()
                self.stream = None

            # Update the baseFilename for the parent class
            self.baseFilename = self.currentFile
            # Reopen the stream
            if not self.delay:
                self.stream = self._open()

        # Call parent emit method
        super().emit(record)


def initLogging():
    """Initialize logging with configuration from APP_CONFIG"""
    # Get log level from config (default to INFO if not found)
    logLevelName = APP_CONFIG.get("APP_LOGGING_LOG_LEVEL", "WARNING")
    logLevel = getattr(logging, logLevelName)

    # Get log directory from config
    logDir = APP_CONFIG.get("APP_LOGGING_LOG_DIR", "./")
    if not os.path.isabs(logDir):
        # If relative path, make it relative to the gateway directory
        gatewayDir = os.path.dirname(os.path.abspath(__file__))
        logDir = os.path.join(gatewayDir, logDir)

    # Ensure log directory exists
    os.makedirs(logDir, exist_ok=True)

    # Create formatters - using single line format
    consoleFormatter = logging.Formatter(
        fmt="%(asctime)s - %(levelname)s - %(name)s - %(message)s",
        datefmt=APP_CONFIG.get("APP_LOGGING_DATE_FORMAT", "%Y-%m-%d %H:%M:%S"),
    )

    # File formatter with more detailed error information but still single line
    fileFormatter = logging.Formatter(
        fmt="%(asctime)s - %(levelname)s - %(name)s - %(message)s - %(pathname)s:%(lineno)d - %(funcName)s",
        datefmt=APP_CONFIG.get("APP_LOGGING_DATE_FORMAT", "%Y-%m-%d %H:%M:%S"),
    )

    # Add filter to exclude Chrome DevTools requests
    class ChromeDevToolsFilter(logging.Filter):
        def filter(self, record):
            return not (
                isinstance(record.msg, str)
                and (
                    ".well-known/appspecific/com.chrome.devtools.json" in record.msg
                    or "Request: /index.html" in record.msg
                )
            )

    # Add filter to exclude all httpcore loggers (including sub-loggers)
    class HttpcoreStarFilter(logging.Filter):
        def filter(self, record):
            return not (
                record.name == "httpcore" or record.name.startswith("httpcore.")
            )

    # Add filter to exclude HTTP debug messages
    class HTTPDebugFilter(logging.Filter):
        def filter(self, record):
            if isinstance(record.msg, str):
                # Filter out HTTP debug messages
                http_debug_patterns = [
                    "receive_response_body.started",
                    "receive_response_body.complete",
                    "response_closed.started",
                    "_send_single_request",
                    "httpcore.http11",
                    "httpx._client",
                    "HTTP Request",
                    "multipart.multipart",
                ]
                return not any(pattern in record.msg for pattern in http_debug_patterns)
            return True

    # Add filter to remove emojis from log messages to prevent Unicode encoding errors
    class EmojiFilter(logging.Filter):
        def filter(self, record):
            if isinstance(record.msg, str):
                # Remove only emojis, preserve other Unicode characters like quotes
                import re
                import unicodedata

                # Remove emoji characters specifically
                record.msg = "".join(
                    char
                    for char in record.msg
                    if unicodedata.category(char) != "So"
                    or not (
                        0x1F600 <= ord(char) <= 0x1F64F
                        or 0x1F300 <= ord(char) <= 0x1F5FF
                        or 0x1F680 <= ord(char) <= 0x1F6FF
                        or 0x1F1E0 <= ord(char) <= 0x1F1FF
                        or 0x2600 <= ord(char) <= 0x26FF
                        or 0x2700 <= ord(char) <= 0x27BF
                    )
                )
            return True

    # Add filter to normalize problematic unicode (e.g., arrows) to ASCII for terminals like cp1252
    class UnicodeArrowFilter(logging.Filter):
        def filter(self, record):
            if isinstance(record.msg, str):
                translation_map = {
                    "\u2192": "->",  # rightwards arrow
                    "\u2190": "<-",  # leftwards arrow
                    "\u2194": "<->", # left right arrow
                    "\u21D2": "=>",  # rightwards double arrow
                    "\u21D0": "<=",  # leftwards double arrow
                    "\u21D4": "<=>", # left right double arrow
                    "\u00AB": "<<",  # left-pointing double angle quotation mark
                    "\u00BB": ">>",  # right-pointing double angle quotation mark
                }
                for u, ascii_eq in translation_map.items():
                    record.msg = record.msg.replace(u, ascii_eq)
            return True

    # Configure handlers based on config
    handlers = []

    # Add console handler if enabled
    if APP_CONFIG.get("APP_LOGGING_CONSOLE_ENABLED", True):
        consoleHandler = logging.StreamHandler()
        consoleHandler.setFormatter(consoleFormatter)
        consoleHandler.addFilter(ChromeDevToolsFilter())
        consoleHandler.addFilter(HttpcoreStarFilter())
        consoleHandler.addFilter(HTTPDebugFilter())
        consoleHandler.addFilter(EmojiFilter())
        consoleHandler.addFilter(UnicodeArrowFilter())
        handlers.append(consoleHandler)

    # Add file handler if enabled
    if APP_CONFIG.get("APP_LOGGING_FILE_ENABLED", True):
        # Create daily application log file with automatic date switching
        rotationSize = int(
            APP_CONFIG.get("APP_LOGGING_ROTATION_SIZE", 10485760)
        )  # Default: 10MB
        backupCount = int(APP_CONFIG.get("APP_LOGGING_BACKUP_COUNT", 5))

        fileHandler = DailyRotatingFileHandler(
            logDir=logDir,
            filenamePrefix="log_app",
            maxBytes=rotationSize,
            backupCount=backupCount,
            encoding="utf-8",
        )
        fileHandler.setFormatter(fileFormatter)
        fileHandler.addFilter(ChromeDevToolsFilter())
        fileHandler.addFilter(HttpcoreStarFilter())
        fileHandler.addFilter(HTTPDebugFilter())
        fileHandler.addFilter(EmojiFilter())
        fileHandler.addFilter(UnicodeArrowFilter())
        handlers.append(fileHandler)

    # Configure the root logger
    logging.basicConfig(
        level=logLevel,
        format="%(asctime)s - %(levelname)s - %(name)s - %(message)s",
        datefmt=APP_CONFIG.get("APP_LOGGING_DATE_FORMAT", "%Y-%m-%d %H:%M:%S"),
        handlers=handlers,
        force=True,  # Force reconfiguration of the root logger
    )

    # Silence noisy third-party libraries - use the same level as the root logger
    noisyLoggers = [
        "httpx",
        "httpcore",
        "urllib3",
        "asyncio",
        "fastapi.security.oauth2",
        "msal",
    ]
    for loggerName in noisyLoggers:
        logging.getLogger(loggerName).setLevel(logging.WARNING)

    # Log the current logging configuration
    logger = logging.getLogger(__name__)
    logger.info(f"Logging initialized with level {logLevelName}")
    logger.info(f"Log directory: {logDir}")

    if APP_CONFIG.get("APP_LOGGING_FILE_ENABLED", True):
        today = datetime.now().strftime("%Y%m%d")
        appLogFile = os.path.join(logDir, f"log_app_{today}.log")
        logger.info(f"Application log file: {appLogFile} (auto-switches daily)")
    else:
        logger.info("Application log file: disabled")

    logger.info(
        f"Console logging: {'enabled' if APP_CONFIG.get('APP_LOGGING_CONSOLE_ENABLED', True) else 'disabled'}"
    )


def makeSqlalchemyDbUrl() -> str:
    host = APP_CONFIG.get("SQLALCHEMY_DB_HOST", "localhost")
    port = APP_CONFIG.get("SQLALCHEMY_DB_PORT", "5432")
    db = APP_CONFIG.get("SQLALCHEMY_DB_DATABASE", "project_gateway")
    user = APP_CONFIG.get("SQLALCHEMY_DB_USER", "postgres")
    pwd = quote_plus(APP_CONFIG.get("SQLALCHEMY_DB_PASSWORD_SECRET", ""))
    # On Windows, prefer asyncpg to avoid psycopg + ProactorEventLoop incompatibility
    if sys.platform == "win32":
        return f"postgresql+asyncpg://{user}:{pwd}@{host}:{port}/{db}"
    return f"postgresql+psycopg://{user}:{pwd}@{host}:{port}/{db}"


# Initialize logging
initLogging()
logger = logging.getLogger(__name__)
instanceLabel = APP_CONFIG.get("APP_ENV_LABEL")


# Define lifespan context manager for application startup/shutdown events
@asynccontextmanager
async def lifespan(app: FastAPI):
    logger.info("Application is starting up")

    # --- Init Managers ---
    await featuresLifecycle.start()
    eventManager.start()

    yield

    # ---  Stop Managers ---
    eventManager.stop()
    await featuresLifecycle.stop()
    logger.info("Application has been shut down")


# START APP
app = FastAPI(
    title="PowerOn | Data Platform API",
    description=f"Backend API for the Multi-Agent Platform by ValueOn AG ({instanceLabel})",
    lifespan=lifespan,
    swagger_ui_init_oauth={
        "usePkceWithAuthorizationCodeGrant": True,
    },
)

# Configure OpenAPI security scheme for Swagger UI
# This adds the "Authorize" button to the /docs page
securityScheme = HTTPBearer()
app.openapi_schema = None  # Reset schema to regenerate with security


def customOpenapi():
    if app.openapi_schema:
        return app.openapi_schema

    from fastapi.openapi.utils import get_openapi

    openapiSchema = get_openapi(
        title=app.title,
        version="1.0.0",
        description=app.description,
        routes=app.routes,
    )

    # Add security scheme definition
    openapiSchema["components"]["securitySchemes"] = {
        "BearerAuth": {
            "type": "http",
            "scheme": "bearer",
            "bearerFormat": "JWT",
            "description": "Enter your JWT token (obtained from login endpoint or browser cookies)",
        }
    }

    # Apply security globally to all endpoints
    # Individual endpoints can override this if needed
    openapiSchema["security"] = [{"BearerAuth": []}]

    app.openapi_schema = openapiSchema
    return app.openapi_schema


app.openapi = customOpenapi


# Parse CORS origins from environment variable
def getAllowedOrigins():
    originsStr = APP_CONFIG.get("APP_ALLOWED_ORIGINS", "http://localhost:8080")
    # Split by comma and strip whitespace
    origins = [origin.strip() for origin in originsStr.split(",")]
    logger.info(f"CORS allowed origins: {origins}")
    return origins


# CORS configuration using environment variables
app.add_middleware(
    CORSMiddleware,
    allow_origins=getAllowedOrigins(),
    allow_credentials=True,
    allow_methods=["GET", "POST", "PUT", "DELETE", "OPTIONS"],
    allow_headers=["*"],
    expose_headers=["*"],
    max_age=86400,  # Increased caching for preflight requests
)

# CSRF protection middleware
from modules.security.csrf import CSRFMiddleware
from modules.security.tokenRefreshMiddleware import (
    TokenRefreshMiddleware,
    ProactiveTokenRefreshMiddleware,
)

app.add_middleware(CSRFMiddleware)

# Token refresh middleware (silent refresh for expired OAuth tokens)
app.add_middleware(TokenRefreshMiddleware, enabled=True)

# Proactive token refresh middleware (refresh tokens before they expire)
app.add_middleware(
    ProactiveTokenRefreshMiddleware, enabled=True, check_interval_minutes=5
)

# Include all routers

from modules.routes.routeAdmin import router as generalRouter
app.include_router(generalRouter)

from modules.routes.routeAttributes import router as attributesRouter
app.include_router(attributesRouter)

from modules.routes.routeDataMandates import router as mandateRouter
app.include_router(mandateRouter)

from modules.routes.routeDataUsers import router as userRouter
app.include_router(userRouter)

from modules.routes.routeDataFiles import router as fileRouter
app.include_router(fileRouter)

from modules.routes.routeDataNeutralization import router as neutralizationRouter
app.include_router(neutralizationRouter)

from modules.routes.routeDataPrompts import router as promptRouter
app.include_router(promptRouter)

from modules.routes.routeDataConnections import router as connectionsRouter
app.include_router(connectionsRouter)

from modules.routes.routeWorkflows import router as workflowRouter
app.include_router(workflowRouter)

from modules.routes.routeChatPlayground import router as chatPlaygroundRouter
app.include_router(chatPlaygroundRouter)

from modules.routes.routeSecurityLocal import router as localRouter
app.include_router(localRouter)

from modules.routes.routeSecurityMsft import router as msftRouter
app.include_router(msftRouter)

from modules.routes.routeSecurityGoogle import router as googleRouter
app.include_router(googleRouter)

from modules.routes.routeVoiceGoogle import router as voiceGoogleRouter
app.include_router(voiceGoogleRouter)

from modules.routes.routeSecurityAdmin import router as adminSecurityRouter
app.include_router(adminSecurityRouter)

from modules.routes.routeSharepoint import router as sharepointRouter
app.include_router(sharepointRouter)