compliance view
This commit is contained in:
parent
591e890527
commit
1230a953bd
8 changed files with 640 additions and 43 deletions
3
app.py
3
app.py
|
|
@ -573,6 +573,9 @@ app.include_router(voiceUserRouter)
|
||||||
from modules.routes.routeSharepoint import router as sharepointRouter
|
from modules.routes.routeSharepoint import router as sharepointRouter
|
||||||
app.include_router(sharepointRouter)
|
app.include_router(sharepointRouter)
|
||||||
|
|
||||||
|
from modules.routes.routeAudit import router as auditRouter
|
||||||
|
app.include_router(auditRouter)
|
||||||
|
|
||||||
from modules.routes.routeAdminLogs import router as adminLogsRouter
|
from modules.routes.routeAdminLogs import router as adminLogsRouter
|
||||||
app.include_router(adminLogsRouter)
|
app.include_router(adminLogsRouter)
|
||||||
|
|
||||||
|
|
|
||||||
150
modules/datamodels/datamodelAiAudit.py
Normal file
150
modules/datamodels/datamodelAiAudit.py
Normal file
|
|
@ -0,0 +1,150 @@
|
||||||
|
# Copyright (c) 2025 Patrick Motsch
|
||||||
|
# All rights reserved.
|
||||||
|
"""AI Audit Log data model for Compliance & AI-Datenfluss tracking.
|
||||||
|
|
||||||
|
Records metadata (and optionally content) of every AI provider call
|
||||||
|
for compliance, audit, and data-protection reporting.
|
||||||
|
"""
|
||||||
|
|
||||||
|
import uuid
|
||||||
|
from typing import Optional
|
||||||
|
|
||||||
|
from pydantic import BaseModel, Field
|
||||||
|
|
||||||
|
from modules.shared.i18nRegistry import i18nModel
|
||||||
|
from modules.shared.timeUtils import getUtcTimestamp
|
||||||
|
|
||||||
|
|
||||||
|
@i18nModel("AI-Audit-Eintrag")
|
||||||
|
class AiAuditLogEntry(BaseModel):
|
||||||
|
id: str = Field(
|
||||||
|
default_factory=lambda: str(uuid.uuid4()),
|
||||||
|
description="Primary key",
|
||||||
|
json_schema_extra={"label": "ID"},
|
||||||
|
)
|
||||||
|
timestamp: float = Field(
|
||||||
|
default_factory=getUtcTimestamp,
|
||||||
|
description="Event timestamp (UTC epoch seconds)",
|
||||||
|
json_schema_extra={
|
||||||
|
"label": "Zeitpunkt",
|
||||||
|
"frontend_type": "timestamp",
|
||||||
|
"frontend_readonly": True,
|
||||||
|
},
|
||||||
|
)
|
||||||
|
|
||||||
|
userId: str = Field(
|
||||||
|
description="ID of the user who triggered the AI call",
|
||||||
|
json_schema_extra={"label": "Benutzer-ID"},
|
||||||
|
)
|
||||||
|
username: Optional[str] = Field(
|
||||||
|
default=None,
|
||||||
|
description="Username at time of call (denormalized for display)",
|
||||||
|
json_schema_extra={"label": "Benutzername"},
|
||||||
|
)
|
||||||
|
mandateId: str = Field(
|
||||||
|
description="Mandate context of the call",
|
||||||
|
json_schema_extra={"label": "Mandanten-ID"},
|
||||||
|
)
|
||||||
|
featureInstanceId: Optional[str] = Field(
|
||||||
|
default=None,
|
||||||
|
description="Feature instance context",
|
||||||
|
json_schema_extra={"label": "Feature-Instanz-ID"},
|
||||||
|
)
|
||||||
|
featureCode: Optional[str] = Field(
|
||||||
|
default=None,
|
||||||
|
description="Feature code (e.g. workspace, trustee)",
|
||||||
|
json_schema_extra={"label": "Feature"},
|
||||||
|
)
|
||||||
|
instanceLabel: Optional[str] = Field(
|
||||||
|
default=None,
|
||||||
|
description="Human-readable instance label at time of call",
|
||||||
|
json_schema_extra={"label": "Instanz"},
|
||||||
|
)
|
||||||
|
|
||||||
|
aiProvider: str = Field(
|
||||||
|
description="AI provider key (e.g. azure-openai, anthropic)",
|
||||||
|
json_schema_extra={"label": "AI-Provider"},
|
||||||
|
)
|
||||||
|
aiModel: str = Field(
|
||||||
|
description="Model name used (e.g. gpt-4o, claude-3.5-sonnet)",
|
||||||
|
json_schema_extra={"label": "AI-Modell"},
|
||||||
|
)
|
||||||
|
operationType: Optional[str] = Field(
|
||||||
|
default=None,
|
||||||
|
description="Operation type (chat, embedding, image, tts, …)",
|
||||||
|
json_schema_extra={"label": "Typ"},
|
||||||
|
)
|
||||||
|
|
||||||
|
tokensInput: Optional[int] = Field(
|
||||||
|
default=None,
|
||||||
|
description="Input tokens consumed",
|
||||||
|
json_schema_extra={"label": "Tokens (Input)"},
|
||||||
|
)
|
||||||
|
tokensOutput: Optional[int] = Field(
|
||||||
|
default=None,
|
||||||
|
description="Output tokens consumed",
|
||||||
|
json_schema_extra={"label": "Tokens (Output)"},
|
||||||
|
)
|
||||||
|
processingTimeMs: Optional[int] = Field(
|
||||||
|
default=None,
|
||||||
|
description="Processing time in milliseconds",
|
||||||
|
json_schema_extra={"label": "Verarbeitungszeit (ms)"},
|
||||||
|
)
|
||||||
|
priceCHF: Optional[float] = Field(
|
||||||
|
default=None,
|
||||||
|
description="Cost in CHF (base price, before markup)",
|
||||||
|
json_schema_extra={"label": "Kosten (CHF)"},
|
||||||
|
)
|
||||||
|
|
||||||
|
neutralizationActive: bool = Field(
|
||||||
|
default=False,
|
||||||
|
description="Whether neutralization was active for this call",
|
||||||
|
json_schema_extra={"label": "Neutralisierung"},
|
||||||
|
)
|
||||||
|
neutralizationMappingsCount: Optional[int] = Field(
|
||||||
|
default=None,
|
||||||
|
description="Number of neutralization mappings applied",
|
||||||
|
json_schema_extra={"label": "Neutralisierungs-Mappings"},
|
||||||
|
)
|
||||||
|
|
||||||
|
contentStored: bool = Field(
|
||||||
|
default=False,
|
||||||
|
description="Whether full content was persisted (mandate opt-in)",
|
||||||
|
json_schema_extra={"label": "Inhalt gespeichert"},
|
||||||
|
)
|
||||||
|
contentInputHash: Optional[str] = Field(
|
||||||
|
default=None,
|
||||||
|
description="SHA-256 hash of the input content",
|
||||||
|
json_schema_extra={"label": "Input-Hash", "frontend_visible": False},
|
||||||
|
)
|
||||||
|
contentInputPreview: Optional[str] = Field(
|
||||||
|
default=None,
|
||||||
|
description="First ~200 chars of input (always stored)",
|
||||||
|
json_schema_extra={"label": "Input-Vorschau"},
|
||||||
|
)
|
||||||
|
contentOutputPreview: Optional[str] = Field(
|
||||||
|
default=None,
|
||||||
|
description="First ~200 chars of output (always stored)",
|
||||||
|
json_schema_extra={"label": "Output-Vorschau"},
|
||||||
|
)
|
||||||
|
contentInputFull: Optional[str] = Field(
|
||||||
|
default=None,
|
||||||
|
description="Full input content (only if mandate opted in)",
|
||||||
|
json_schema_extra={"label": "Vollständiger Input", "frontend_visible": False},
|
||||||
|
)
|
||||||
|
contentOutputFull: Optional[str] = Field(
|
||||||
|
default=None,
|
||||||
|
description="Full output content (only if mandate opted in)",
|
||||||
|
json_schema_extra={"label": "Vollständiger Output", "frontend_visible": False},
|
||||||
|
)
|
||||||
|
|
||||||
|
success: bool = Field(
|
||||||
|
default=True,
|
||||||
|
description="Whether the AI call succeeded",
|
||||||
|
json_schema_extra={"label": "Erfolgreich"},
|
||||||
|
)
|
||||||
|
errorMessage: Optional[str] = Field(
|
||||||
|
default=None,
|
||||||
|
description="Error message if the call failed",
|
||||||
|
json_schema_extra={"label": "Fehlermeldung"},
|
||||||
|
)
|
||||||
149
modules/routes/routeAudit.py
Normal file
149
modules/routes/routeAudit.py
Normal file
|
|
@ -0,0 +1,149 @@
|
||||||
|
# Copyright (c) 2025 Patrick Motsch
|
||||||
|
# All rights reserved.
|
||||||
|
"""Compliance & Audit API endpoints.
|
||||||
|
|
||||||
|
Provides three views:
|
||||||
|
- AI Data-Flow Log (Tab A)
|
||||||
|
- Security/GDPR Audit Log (Tab B)
|
||||||
|
- Aggregated Audit Statistics (Tab C)
|
||||||
|
|
||||||
|
RBAC: mandate-admin or compliance-viewer role required.
|
||||||
|
"""
|
||||||
|
|
||||||
|
import logging
|
||||||
|
from typing import Optional
|
||||||
|
|
||||||
|
from fastapi import APIRouter, Depends, HTTPException, Query, Path, status
|
||||||
|
from starlette.requests import Request
|
||||||
|
|
||||||
|
from modules.auth import limiter, getCurrentUser, getRequestContext, RequestContext
|
||||||
|
from modules.datamodels.datamodelUam import User
|
||||||
|
from modules.shared.i18nRegistry import apiRouteContext
|
||||||
|
|
||||||
|
logger = logging.getLogger(__name__)
|
||||||
|
routeApiMsg = apiRouteContext("routeAudit")
|
||||||
|
|
||||||
|
router = APIRouter(prefix="/api/audit", tags=["Audit"])
|
||||||
|
|
||||||
|
|
||||||
|
def _requireAuditAccess(context: RequestContext):
|
||||||
|
"""Raise 403 unless user has mandate-admin or compliance-viewer access."""
|
||||||
|
from modules.auth.authentication import _hasSysAdminRole
|
||||||
|
if _hasSysAdminRole(str(context.user.id)):
|
||||||
|
return
|
||||||
|
|
||||||
|
from modules.interfaces.interfaceDbApp import getInterface
|
||||||
|
appIf = getInterface(context.user, mandateId=str(context.mandateId) if context.mandateId else None)
|
||||||
|
mandateId = str(context.mandateId) if context.mandateId else None
|
||||||
|
if not mandateId:
|
||||||
|
raise HTTPException(status_code=403, detail=routeApiMsg("Mandanten-Kontext erforderlich"))
|
||||||
|
|
||||||
|
from modules.datamodels.datamodelRbac import AccessRuleContext
|
||||||
|
permissions = appIf.rbac.getUserPermissions(
|
||||||
|
context.user, AccessRuleContext.UI, "ui.system.complianceAudit",
|
||||||
|
mandateId=mandateId,
|
||||||
|
)
|
||||||
|
if not permissions or not permissions.view:
|
||||||
|
raise HTTPException(status_code=403, detail=routeApiMsg("Kein Zugriff auf Audit-Daten"))
|
||||||
|
|
||||||
|
|
||||||
|
# ── Tab A: AI Data-Flow Log ──
|
||||||
|
|
||||||
|
@router.get("/ai-log")
|
||||||
|
@limiter.limit("120/minute")
|
||||||
|
async def getAiAuditLog(
|
||||||
|
request: Request,
|
||||||
|
context: RequestContext = Depends(getRequestContext),
|
||||||
|
userId: Optional[str] = Query(None),
|
||||||
|
featureInstanceId: Optional[str] = Query(None),
|
||||||
|
aiModel: Optional[str] = Query(None),
|
||||||
|
dateFrom: Optional[float] = Query(None, description="UTC epoch seconds"),
|
||||||
|
dateTo: Optional[float] = Query(None, description="UTC epoch seconds"),
|
||||||
|
limit: int = Query(50, ge=1, le=500),
|
||||||
|
offset: int = Query(0, ge=0),
|
||||||
|
):
|
||||||
|
_requireAuditAccess(context)
|
||||||
|
mandateId = str(context.mandateId) if context.mandateId else ""
|
||||||
|
if not mandateId:
|
||||||
|
raise HTTPException(status_code=400, detail=routeApiMsg("Mandanten-ID erforderlich"))
|
||||||
|
|
||||||
|
from modules.shared.aiAuditLogger import aiAuditLogger
|
||||||
|
return aiAuditLogger.getAiAuditLogs(
|
||||||
|
mandateId,
|
||||||
|
userId=userId,
|
||||||
|
featureInstanceId=featureInstanceId,
|
||||||
|
aiModel=aiModel,
|
||||||
|
fromTimestamp=dateFrom,
|
||||||
|
toTimestamp=dateTo,
|
||||||
|
limit=limit,
|
||||||
|
offset=offset,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
@router.get("/ai-log/{entryId}/content")
|
||||||
|
@limiter.limit("60/minute")
|
||||||
|
async def getAiAuditEntryContent(
|
||||||
|
request: Request,
|
||||||
|
entryId: str = Path(...),
|
||||||
|
context: RequestContext = Depends(getRequestContext),
|
||||||
|
):
|
||||||
|
_requireAuditAccess(context)
|
||||||
|
mandateId = str(context.mandateId) if context.mandateId else ""
|
||||||
|
|
||||||
|
from modules.shared.aiAuditLogger import aiAuditLogger
|
||||||
|
result = aiAuditLogger.getAiAuditEntryContent(entryId, mandateId)
|
||||||
|
if not result:
|
||||||
|
raise HTTPException(status_code=404, detail=routeApiMsg("Audit-Eintrag nicht gefunden"))
|
||||||
|
return result
|
||||||
|
|
||||||
|
|
||||||
|
# ── Tab B: Security / GDPR Audit Log ──
|
||||||
|
|
||||||
|
@router.get("/log")
|
||||||
|
@limiter.limit("120/minute")
|
||||||
|
async def getAuditLog(
|
||||||
|
request: Request,
|
||||||
|
context: RequestContext = Depends(getRequestContext),
|
||||||
|
userId: Optional[str] = Query(None),
|
||||||
|
category: Optional[str] = Query(None),
|
||||||
|
action: Optional[str] = Query(None),
|
||||||
|
dateFrom: Optional[float] = Query(None),
|
||||||
|
dateTo: Optional[float] = Query(None),
|
||||||
|
limit: int = Query(100, ge=1, le=500),
|
||||||
|
offset: int = Query(0, ge=0),
|
||||||
|
):
|
||||||
|
_requireAuditAccess(context)
|
||||||
|
mandateId = str(context.mandateId) if context.mandateId else None
|
||||||
|
|
||||||
|
from modules.shared.auditLogger import audit_logger
|
||||||
|
records = audit_logger.getAuditLogs(
|
||||||
|
userId=userId,
|
||||||
|
mandateId=mandateId,
|
||||||
|
category=category,
|
||||||
|
action=action,
|
||||||
|
fromTimestamp=dateFrom,
|
||||||
|
toTimestamp=dateTo,
|
||||||
|
limit=limit + offset + 1,
|
||||||
|
)
|
||||||
|
totalItems = len(records)
|
||||||
|
page = records[offset: offset + limit]
|
||||||
|
return {"items": page, "totalItems": totalItems}
|
||||||
|
|
||||||
|
|
||||||
|
# ── Tab C: Audit Statistics ──
|
||||||
|
|
||||||
|
@router.get("/stats")
|
||||||
|
@limiter.limit("60/minute")
|
||||||
|
async def getAuditStats(
|
||||||
|
request: Request,
|
||||||
|
context: RequestContext = Depends(getRequestContext),
|
||||||
|
timeRange: int = Query(30, ge=1, le=365, description="Days to aggregate"),
|
||||||
|
groupBy: str = Query("model", description="Grouping: model, user, feature, day"),
|
||||||
|
):
|
||||||
|
_requireAuditAccess(context)
|
||||||
|
mandateId = str(context.mandateId) if context.mandateId else ""
|
||||||
|
if not mandateId:
|
||||||
|
raise HTTPException(status_code=400, detail=routeApiMsg("Mandanten-ID erforderlich"))
|
||||||
|
|
||||||
|
from modules.shared.aiAuditLogger import aiAuditLogger
|
||||||
|
return aiAuditLogger.getAiAuditStats(mandateId, timeRangeDays=timeRange, groupBy=groupBy)
|
||||||
|
|
@ -148,13 +148,11 @@ def _registerDefaultToolboxes() -> None:
|
||||||
ToolboxDefinition(
|
ToolboxDefinition(
|
||||||
id="email",
|
id="email",
|
||||||
label="Email",
|
label="Email",
|
||||||
description="Read and send emails via Outlook/Gmail",
|
description="Send emails or save as draft via Outlook (supports HTML body and file attachments). Use sendMail with draft=true for drafts.",
|
||||||
requiresConnection="microsoft",
|
requiresConnection="microsoft",
|
||||||
isDefault=False,
|
isDefault=False,
|
||||||
tools=[
|
tools=[
|
||||||
"sendMail",
|
"sendMail",
|
||||||
"outlook_readEmails", "outlook_searchEmails",
|
|
||||||
"outlook_composeAndDraftReply", "outlook_sendDraft",
|
|
||||||
],
|
],
|
||||||
),
|
),
|
||||||
ToolboxDefinition(
|
ToolboxDefinition(
|
||||||
|
|
|
||||||
|
|
@ -1038,6 +1038,7 @@ detectedIntent-Werte:
|
||||||
|
|
||||||
Returns a function that records one billing transaction per individual model call.
|
Returns a function that records one billing transaction per individual model call.
|
||||||
Each transaction contains the exact provider name AND model name.
|
Each transaction contains the exact provider name AND model name.
|
||||||
|
Also writes an AI audit log entry for compliance tracking.
|
||||||
|
|
||||||
For a 200 MB document processed with N parallel AI calls (possibly different models),
|
For a 200 MB document processed with N parallel AI calls (possibly different models),
|
||||||
this creates N separate billing transactions - one per model call.
|
this creates N separate billing transactions - one per model call.
|
||||||
|
|
@ -1047,7 +1048,6 @@ detectedIntent-Werte:
|
||||||
featureInstanceId = getattr(self.services, 'featureInstanceId', None)
|
featureInstanceId = getattr(self.services, 'featureInstanceId', None)
|
||||||
featureCode = getattr(self.services, 'featureCode', None)
|
featureCode = getattr(self.services, 'featureCode', None)
|
||||||
|
|
||||||
# Get workflow ID if available
|
|
||||||
workflowId = None
|
workflowId = None
|
||||||
workflow = getattr(self.services, 'workflow', None)
|
workflow = getattr(self.services, 'workflow', None)
|
||||||
if workflow and hasattr(workflow, 'id'):
|
if workflow and hasattr(workflow, 'id'):
|
||||||
|
|
@ -1056,17 +1056,19 @@ detectedIntent-Werte:
|
||||||
billingService = getBillingService(user, mandateId, featureInstanceId, featureCode)
|
billingService = getBillingService(user, mandateId, featureInstanceId, featureCode)
|
||||||
|
|
||||||
def _billingCallback(response) -> None:
|
def _billingCallback(response) -> None:
|
||||||
"""Record billing transaction with full AI call metadata."""
|
"""Record billing transaction + AI audit entry."""
|
||||||
if not response or getattr(response, 'errorCount', 0) > 0:
|
if not response:
|
||||||
return
|
|
||||||
|
|
||||||
basePriceCHF = getattr(response, 'priceCHF', 0.0)
|
|
||||||
if not basePriceCHF or basePriceCHF <= 0:
|
|
||||||
return
|
return
|
||||||
|
|
||||||
provider = getattr(response, 'provider', None) or 'unknown'
|
provider = getattr(response, 'provider', None) or 'unknown'
|
||||||
modelName = getattr(response, 'modelName', None) or 'unknown'
|
modelName = getattr(response, 'modelName', None) or 'unknown'
|
||||||
|
basePriceCHF = getattr(response, 'priceCHF', 0.0)
|
||||||
|
hasError = getattr(response, 'errorCount', 0) > 0
|
||||||
|
processingTime = getattr(response, 'processingTime', None)
|
||||||
|
bytesSent = getattr(response, 'bytesSent', None)
|
||||||
|
bytesReceived = getattr(response, 'bytesReceived', None)
|
||||||
|
|
||||||
|
if not hasError and basePriceCHF and basePriceCHF > 0:
|
||||||
try:
|
try:
|
||||||
billingService.recordUsage(
|
billingService.recordUsage(
|
||||||
priceCHF=basePriceCHF,
|
priceCHF=basePriceCHF,
|
||||||
|
|
@ -1074,9 +1076,9 @@ detectedIntent-Werte:
|
||||||
aicoreProvider=provider,
|
aicoreProvider=provider,
|
||||||
aicoreModel=modelName,
|
aicoreModel=modelName,
|
||||||
description=f"AI: {modelName}",
|
description=f"AI: {modelName}",
|
||||||
processingTime=getattr(response, 'processingTime', None),
|
processingTime=processingTime,
|
||||||
bytesSent=getattr(response, 'bytesSent', None),
|
bytesSent=bytesSent,
|
||||||
bytesReceived=getattr(response, 'bytesReceived', None),
|
bytesReceived=bytesReceived,
|
||||||
errorCount=getattr(response, 'errorCount', None)
|
errorCount=getattr(response, 'errorCount', None)
|
||||||
)
|
)
|
||||||
logger.debug(
|
logger.debug(
|
||||||
|
|
@ -1090,6 +1092,33 @@ detectedIntent-Werte:
|
||||||
f"provider={provider}, model={modelName}, error={e}"
|
f"provider={provider}, model={modelName}, error={e}"
|
||||||
)
|
)
|
||||||
|
|
||||||
|
try:
|
||||||
|
from modules.shared.aiAuditLogger import aiAuditLogger
|
||||||
|
|
||||||
|
contentOut = getattr(response, 'content', None)
|
||||||
|
metadata = getattr(response, 'metadata', None) or {}
|
||||||
|
tokensUsed = metadata.get('tokensUsed') if isinstance(metadata, dict) else None
|
||||||
|
|
||||||
|
aiAuditLogger.logAiCall(
|
||||||
|
userId=user.id,
|
||||||
|
mandateId=mandateId or "",
|
||||||
|
aiProvider=provider,
|
||||||
|
aiModel=modelName,
|
||||||
|
username=getattr(user, 'username', None),
|
||||||
|
featureInstanceId=featureInstanceId,
|
||||||
|
featureCode=featureCode,
|
||||||
|
operationType=metadata.get('operationType') if isinstance(metadata, dict) else None,
|
||||||
|
tokensInput=tokensUsed.get('input') if isinstance(tokensUsed, dict) else None,
|
||||||
|
tokensOutput=tokensUsed.get('output') if isinstance(tokensUsed, dict) else None,
|
||||||
|
processingTimeMs=int(processingTime * 1000) if processingTime else None,
|
||||||
|
priceCHF=basePriceCHF if basePriceCHF else None,
|
||||||
|
contentOutput=str(contentOut)[:500] if contentOut else None,
|
||||||
|
success=not hasError,
|
||||||
|
errorMessage=str(getattr(response, 'errorMessage', None)) if hasError else None,
|
||||||
|
)
|
||||||
|
except Exception as e:
|
||||||
|
logger.warning(f"AI audit log failed (non-critical): {e}")
|
||||||
|
|
||||||
return _billingCallback
|
return _billingCallback
|
||||||
|
|
||||||
def _calculateEffectiveProviders(self) -> Optional[List[str]]:
|
def _calculateEffectiveProviders(self) -> Optional[List[str]]:
|
||||||
|
|
|
||||||
253
modules/shared/aiAuditLogger.py
Normal file
253
modules/shared/aiAuditLogger.py
Normal file
|
|
@ -0,0 +1,253 @@
|
||||||
|
# Copyright (c) 2025 Patrick Motsch
|
||||||
|
# All rights reserved.
|
||||||
|
"""AI Audit Logger — records every AI provider call for compliance reporting.
|
||||||
|
|
||||||
|
Usage:
|
||||||
|
from modules.shared.aiAuditLogger import aiAuditLogger
|
||||||
|
aiAuditLogger.logAiCall(userId=..., mandateId=..., ...)
|
||||||
|
"""
|
||||||
|
|
||||||
|
import hashlib
|
||||||
|
import logging
|
||||||
|
from collections import defaultdict
|
||||||
|
from datetime import datetime, timezone, timedelta
|
||||||
|
from typing import Any, Dict, List, Optional
|
||||||
|
|
||||||
|
from modules.shared.timeUtils import getUtcTimestamp
|
||||||
|
|
||||||
|
logger = logging.getLogger(__name__)
|
||||||
|
|
||||||
|
_PREVIEW_LENGTH = 200
|
||||||
|
|
||||||
|
|
||||||
|
class AiAuditLogger:
|
||||||
|
"""Persists AI audit entries to the poweron_app database."""
|
||||||
|
|
||||||
|
def __init__(self):
|
||||||
|
self._db = None
|
||||||
|
self._initialized = False
|
||||||
|
|
||||||
|
def _ensureInitialized(self):
|
||||||
|
if self._initialized:
|
||||||
|
return
|
||||||
|
try:
|
||||||
|
from modules.connectors.connectorDbPostgre import _get_cached_connector
|
||||||
|
from modules.shared.configuration import APP_CONFIG
|
||||||
|
from modules.datamodels.datamodelAiAudit import AiAuditLogEntry
|
||||||
|
|
||||||
|
dbHost = APP_CONFIG.get("DB_HOST", "_no_config_default_data")
|
||||||
|
dbUser = APP_CONFIG.get("DB_USER")
|
||||||
|
dbPassword = APP_CONFIG.get("DB_PASSWORD_SECRET")
|
||||||
|
dbPort = int(APP_CONFIG.get("DB_PORT", 5432))
|
||||||
|
|
||||||
|
self._db = _get_cached_connector(
|
||||||
|
dbHost=dbHost,
|
||||||
|
dbDatabase="poweron_app",
|
||||||
|
dbUser=dbUser,
|
||||||
|
dbPassword=dbPassword,
|
||||||
|
dbPort=dbPort,
|
||||||
|
userId="system",
|
||||||
|
)
|
||||||
|
self._db._ensureTableExists(AiAuditLogEntry)
|
||||||
|
self._initialized = True
|
||||||
|
except Exception as e:
|
||||||
|
logger.error(f"AI audit logger init failed: {e}")
|
||||||
|
|
||||||
|
def logAiCall(
|
||||||
|
self,
|
||||||
|
userId: str,
|
||||||
|
mandateId: str,
|
||||||
|
aiProvider: str,
|
||||||
|
aiModel: str,
|
||||||
|
*,
|
||||||
|
username: Optional[str] = None,
|
||||||
|
featureInstanceId: Optional[str] = None,
|
||||||
|
featureCode: Optional[str] = None,
|
||||||
|
instanceLabel: Optional[str] = None,
|
||||||
|
operationType: Optional[str] = None,
|
||||||
|
tokensInput: Optional[int] = None,
|
||||||
|
tokensOutput: Optional[int] = None,
|
||||||
|
processingTimeMs: Optional[int] = None,
|
||||||
|
priceCHF: Optional[float] = None,
|
||||||
|
neutralizationActive: bool = False,
|
||||||
|
neutralizationMappingsCount: Optional[int] = None,
|
||||||
|
contentInput: Optional[str] = None,
|
||||||
|
contentOutput: Optional[str] = None,
|
||||||
|
storeFullContent: bool = False,
|
||||||
|
success: bool = True,
|
||||||
|
errorMessage: Optional[str] = None,
|
||||||
|
) -> Optional[str]:
|
||||||
|
"""Write one AI audit entry. Returns the entry id or None on failure."""
|
||||||
|
self._ensureInitialized()
|
||||||
|
if not self._db:
|
||||||
|
return None
|
||||||
|
|
||||||
|
try:
|
||||||
|
from modules.datamodels.datamodelAiAudit import AiAuditLogEntry
|
||||||
|
|
||||||
|
inputPreview = (contentInput or "")[:_PREVIEW_LENGTH] or None
|
||||||
|
outputPreview = (contentOutput or "")[:_PREVIEW_LENGTH] or None
|
||||||
|
inputHash = hashlib.sha256(contentInput.encode("utf-8")).hexdigest() if contentInput else None
|
||||||
|
|
||||||
|
entry = AiAuditLogEntry(
|
||||||
|
userId=userId,
|
||||||
|
username=username,
|
||||||
|
mandateId=mandateId,
|
||||||
|
featureInstanceId=featureInstanceId or "",
|
||||||
|
featureCode=featureCode,
|
||||||
|
instanceLabel=instanceLabel,
|
||||||
|
aiProvider=aiProvider,
|
||||||
|
aiModel=aiModel,
|
||||||
|
operationType=operationType,
|
||||||
|
tokensInput=tokensInput,
|
||||||
|
tokensOutput=tokensOutput,
|
||||||
|
processingTimeMs=processingTimeMs,
|
||||||
|
priceCHF=priceCHF,
|
||||||
|
neutralizationActive=neutralizationActive,
|
||||||
|
neutralizationMappingsCount=neutralizationMappingsCount,
|
||||||
|
contentStored=storeFullContent and bool(contentInput),
|
||||||
|
contentInputHash=inputHash,
|
||||||
|
contentInputPreview=inputPreview,
|
||||||
|
contentOutputPreview=outputPreview,
|
||||||
|
contentInputFull=contentInput if storeFullContent else None,
|
||||||
|
contentOutputFull=contentOutput if storeFullContent else None,
|
||||||
|
success=success,
|
||||||
|
errorMessage=errorMessage,
|
||||||
|
)
|
||||||
|
self._db.recordCreate(AiAuditLogEntry, entry.model_dump())
|
||||||
|
return entry.id
|
||||||
|
except Exception as e:
|
||||||
|
logger.error(f"Failed to write AI audit entry: {e}", exc_info=True)
|
||||||
|
return None
|
||||||
|
|
||||||
|
# ── Read helpers (used by route) ──
|
||||||
|
|
||||||
|
def getAiAuditLogs(
|
||||||
|
self,
|
||||||
|
mandateId: str,
|
||||||
|
*,
|
||||||
|
userId: Optional[str] = None,
|
||||||
|
featureInstanceId: Optional[str] = None,
|
||||||
|
aiModel: Optional[str] = None,
|
||||||
|
fromTimestamp: Optional[float] = None,
|
||||||
|
toTimestamp: Optional[float] = None,
|
||||||
|
limit: int = 100,
|
||||||
|
offset: int = 0,
|
||||||
|
) -> Dict[str, Any]:
|
||||||
|
"""Return paginated AI audit entries for a mandate."""
|
||||||
|
self._ensureInitialized()
|
||||||
|
if not self._db:
|
||||||
|
return {"items": [], "totalItems": 0}
|
||||||
|
|
||||||
|
from modules.datamodels.datamodelAiAudit import AiAuditLogEntry
|
||||||
|
|
||||||
|
recordFilter: Dict[str, Any] = {"mandateId": mandateId}
|
||||||
|
if userId:
|
||||||
|
recordFilter["userId"] = userId
|
||||||
|
if featureInstanceId:
|
||||||
|
recordFilter["featureInstanceId"] = featureInstanceId
|
||||||
|
if aiModel:
|
||||||
|
recordFilter["aiModel"] = aiModel
|
||||||
|
|
||||||
|
allRecords = self._db.getRecordset(
|
||||||
|
AiAuditLogEntry,
|
||||||
|
recordFilter=recordFilter,
|
||||||
|
)
|
||||||
|
|
||||||
|
if fromTimestamp is not None:
|
||||||
|
allRecords = [r for r in allRecords if (r.get("timestamp") or 0) >= fromTimestamp]
|
||||||
|
if toTimestamp is not None:
|
||||||
|
allRecords = [r for r in allRecords if (r.get("timestamp") or 0) <= toTimestamp]
|
||||||
|
|
||||||
|
allRecords.sort(key=lambda r: r.get("timestamp") or 0, reverse=True)
|
||||||
|
totalItems = len(allRecords)
|
||||||
|
page = allRecords[offset: offset + limit]
|
||||||
|
|
||||||
|
for item in page:
|
||||||
|
item.pop("contentInputFull", None)
|
||||||
|
item.pop("contentOutputFull", None)
|
||||||
|
|
||||||
|
return {"items": page, "totalItems": totalItems}
|
||||||
|
|
||||||
|
def getAiAuditEntryContent(self, entryId: str, mandateId: str) -> Optional[Dict[str, Any]]:
|
||||||
|
"""Return full content for a single entry (RBAC checked by route)."""
|
||||||
|
self._ensureInitialized()
|
||||||
|
if not self._db:
|
||||||
|
return None
|
||||||
|
|
||||||
|
from modules.datamodels.datamodelAiAudit import AiAuditLogEntry
|
||||||
|
|
||||||
|
records = self._db.getRecordset(
|
||||||
|
AiAuditLogEntry, recordFilter={"id": entryId, "mandateId": mandateId}
|
||||||
|
)
|
||||||
|
if not records:
|
||||||
|
return None
|
||||||
|
rec = records[0]
|
||||||
|
return {
|
||||||
|
"id": rec.get("id"),
|
||||||
|
"contentStored": rec.get("contentStored", False),
|
||||||
|
"contentInputFull": rec.get("contentInputFull"),
|
||||||
|
"contentOutputFull": rec.get("contentOutputFull"),
|
||||||
|
"contentInputPreview": rec.get("contentInputPreview"),
|
||||||
|
"contentOutputPreview": rec.get("contentOutputPreview"),
|
||||||
|
}
|
||||||
|
|
||||||
|
def getAiAuditStats(
|
||||||
|
self,
|
||||||
|
mandateId: str,
|
||||||
|
*,
|
||||||
|
timeRangeDays: int = 30,
|
||||||
|
groupBy: str = "model",
|
||||||
|
) -> Dict[str, Any]:
|
||||||
|
"""Aggregate statistics for Tab C."""
|
||||||
|
self._ensureInitialized()
|
||||||
|
if not self._db:
|
||||||
|
return {}
|
||||||
|
|
||||||
|
from modules.datamodels.datamodelAiAudit import AiAuditLogEntry
|
||||||
|
|
||||||
|
cutoff = (datetime.now(timezone.utc) - timedelta(days=max(1, timeRangeDays))).timestamp()
|
||||||
|
|
||||||
|
allRecords = self._db.getRecordset(
|
||||||
|
AiAuditLogEntry, recordFilter={"mandateId": mandateId}
|
||||||
|
)
|
||||||
|
records = [r for r in allRecords if (r.get("timestamp") or 0) >= cutoff]
|
||||||
|
|
||||||
|
callsByDay: Dict[str, int] = defaultdict(int)
|
||||||
|
callsByModel: Dict[str, int] = defaultdict(int)
|
||||||
|
callsByFeature: Dict[str, int] = defaultdict(int)
|
||||||
|
costByDay: Dict[str, float] = defaultdict(float)
|
||||||
|
callsByUser: Dict[str, int] = defaultdict(int)
|
||||||
|
neutralizationCount = 0
|
||||||
|
totalCalls = len(records)
|
||||||
|
|
||||||
|
for r in records:
|
||||||
|
ts = r.get("timestamp") or 0
|
||||||
|
try:
|
||||||
|
day = datetime.fromtimestamp(ts, tz=timezone.utc).strftime("%Y-%m-%d")
|
||||||
|
except (ValueError, OSError):
|
||||||
|
day = "unknown"
|
||||||
|
callsByDay[day] += 1
|
||||||
|
callsByModel[r.get("aiModel") or "unknown"] += 1
|
||||||
|
callsByFeature[r.get("featureCode") or "system"] += 1
|
||||||
|
costByDay[day] += r.get("priceCHF") or 0.0
|
||||||
|
callsByUser[r.get("username") or r.get("userId") or "unknown"] += 1
|
||||||
|
if r.get("neutralizationActive"):
|
||||||
|
neutralizationCount += 1
|
||||||
|
|
||||||
|
sortedDays = sorted(callsByDay.keys())
|
||||||
|
neutralizationPercent = round(100.0 * neutralizationCount / totalCalls, 1) if totalCalls else 0.0
|
||||||
|
|
||||||
|
return {
|
||||||
|
"totalCalls": totalCalls,
|
||||||
|
"timeRangeDays": timeRangeDays,
|
||||||
|
"callsPerDay": [{"date": d, "calls": callsByDay[d]} for d in sortedDays],
|
||||||
|
"costPerDay": [{"date": d, "cost": round(costByDay[d], 4)} for d in sortedDays],
|
||||||
|
"callsByModel": dict(sorted(callsByModel.items(), key=lambda x: -x[1])),
|
||||||
|
"callsByFeature": dict(sorted(callsByFeature.items(), key=lambda x: -x[1])),
|
||||||
|
"topUsers": dict(sorted(callsByUser.items(), key=lambda x: -x[1])[:10]),
|
||||||
|
"neutralizationPercent": neutralizationPercent,
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
aiAuditLogger = AiAuditLogger()
|
||||||
|
|
@ -352,7 +352,6 @@ class AuditLogger:
|
||||||
records = self._db.getRecordset(
|
records = self._db.getRecordset(
|
||||||
AuditLogEntry,
|
AuditLogEntry,
|
||||||
recordFilter=recordFilter if recordFilter else None,
|
recordFilter=recordFilter if recordFilter else None,
|
||||||
orderBy="timestamp DESC"
|
|
||||||
)
|
)
|
||||||
|
|
||||||
# Apply timestamp filtering in Python (PostgreSQL connector may not support $gt/$lt)
|
# Apply timestamp filtering in Python (PostgreSQL connector may not support $gt/$lt)
|
||||||
|
|
@ -367,7 +366,7 @@ class AuditLogger:
|
||||||
filteredRecords.append(record)
|
filteredRecords.append(record)
|
||||||
records = filteredRecords
|
records = filteredRecords
|
||||||
|
|
||||||
# Apply limit
|
records.sort(key=lambda r: r.get("timestamp", 0), reverse=True)
|
||||||
return records[:limit]
|
return records[:limit]
|
||||||
|
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
|
|
|
||||||
|
|
@ -46,23 +46,39 @@ NAVIGATION_SECTIONS = [
|
||||||
{
|
{
|
||||||
"id": "home",
|
"id": "home",
|
||||||
"objectKey": "ui.system.home",
|
"objectKey": "ui.system.home",
|
||||||
"label": t("Übersicht"),
|
"label": t("Start"),
|
||||||
"icon": "FaHome",
|
"icon": "FaHome",
|
||||||
"path": "/",
|
"path": "/",
|
||||||
"order": 10,
|
"order": 10,
|
||||||
"public": True,
|
"public": True,
|
||||||
},
|
},
|
||||||
|
],
|
||||||
|
"subgroups": [
|
||||||
|
# ── Übersichten ──
|
||||||
|
{
|
||||||
|
"id": "system-overviews",
|
||||||
|
"title": t("Übersichten"),
|
||||||
|
"order": 15,
|
||||||
|
"items": [
|
||||||
{
|
{
|
||||||
"id": "integrations",
|
"id": "integrations",
|
||||||
"objectKey": "ui.system.integrations",
|
"objectKey": "ui.system.integrations",
|
||||||
"label": t("Integrationen"),
|
"label": t("Integrationen"),
|
||||||
"icon": "FaProjectDiagram",
|
"icon": "FaProjectDiagram",
|
||||||
"path": "/integrations",
|
"path": "/integrations",
|
||||||
"order": 15,
|
"order": 10,
|
||||||
"public": True,
|
"public": True,
|
||||||
},
|
},
|
||||||
|
{
|
||||||
|
"id": "compliance-audit",
|
||||||
|
"objectKey": "ui.system.complianceAudit",
|
||||||
|
"label": t("Compliance & Audit"),
|
||||||
|
"icon": "FaShieldAlt",
|
||||||
|
"path": "/compliance-audit",
|
||||||
|
"order": 20,
|
||||||
|
},
|
||||||
],
|
],
|
||||||
"subgroups": [
|
},
|
||||||
# ── Basisdaten ──
|
# ── Basisdaten ──
|
||||||
{
|
{
|
||||||
"id": "system-basedata",
|
"id": "system-basedata",
|
||||||
|
|
|
||||||
Loading…
Reference in a new issue