404 lines
15 KiB
Python
404 lines
15 KiB
Python
# Copyright (c) 2025 Patrick Motsch
|
|
# All rights reserved.
|
|
"""Compliance & Audit API endpoints.
|
|
|
|
Provides three views:
|
|
- AI Data-Flow Log (Tab A)
|
|
- Security/GDPR Audit Log (Tab B)
|
|
- Aggregated Audit Statistics (Tab C)
|
|
|
|
RBAC: mandate-admin or compliance-viewer role required.
|
|
"""
|
|
|
|
import json
|
|
import logging
|
|
import re
|
|
from typing import Any, Dict, List, Optional
|
|
|
|
from fastapi import APIRouter, Depends, HTTPException, Query, Path, status
|
|
from starlette.requests import Request
|
|
|
|
from modules.auth import limiter, getCurrentUser, getRequestContext, RequestContext
|
|
from modules.datamodels.datamodelUam import User
|
|
from modules.shared.i18nRegistry import apiRouteContext
|
|
|
|
logger = logging.getLogger(__name__)
|
|
routeApiMsg = apiRouteContext("routeAudit")
|
|
|
|
router = APIRouter(prefix="/api/audit", tags=["Audit"])
|
|
|
|
|
|
def _applySortFilterSearch(
|
|
items: List[Dict[str, Any]],
|
|
*,
|
|
sortJson: Optional[str] = None,
|
|
filtersJson: Optional[str] = None,
|
|
search: Optional[str] = None,
|
|
searchableKeys: Optional[List[str]] = None,
|
|
) -> List[Dict[str, Any]]:
|
|
"""Apply sort, filter and search to a list of dicts in-memory."""
|
|
if filtersJson:
|
|
try:
|
|
filters = json.loads(filtersJson) if isinstance(filtersJson, str) else filtersJson
|
|
if isinstance(filters, dict):
|
|
for key, val in filters.items():
|
|
if val is None or val == "":
|
|
continue
|
|
if isinstance(val, list):
|
|
items = [r for r in items if str(r.get(key, "")) in [str(v) for v in val]]
|
|
else:
|
|
items = [r for r in items if str(r.get(key, "")).lower() == str(val).lower()]
|
|
except (json.JSONDecodeError, TypeError):
|
|
pass
|
|
|
|
if search and searchableKeys:
|
|
needle = search.lower()
|
|
items = [r for r in items if any(needle in str(r.get(k, "")).lower() for k in searchableKeys)]
|
|
|
|
if sortJson:
|
|
try:
|
|
sortList = json.loads(sortJson) if isinstance(sortJson, str) else sortJson
|
|
if isinstance(sortList, list):
|
|
for sortDef in reversed(sortList):
|
|
field = sortDef.get("field", "")
|
|
desc = sortDef.get("direction", "asc") == "desc"
|
|
items.sort(key=lambda r, f=field: (r.get(f) is None, r.get(f, "")), reverse=desc)
|
|
except (json.JSONDecodeError, TypeError):
|
|
pass
|
|
|
|
return items
|
|
|
|
|
|
def _distinctColumnValues(items: List[Dict[str, Any]], column: str) -> List[str]:
|
|
"""Extract sorted distinct non-empty string values for a column."""
|
|
vals = set()
|
|
for r in items:
|
|
v = r.get(column)
|
|
if v is not None and v != "":
|
|
vals.add(str(v))
|
|
return sorted(vals)
|
|
|
|
|
|
def _enrichUserAndInstanceLabels(
|
|
items: List[Dict[str, Any]],
|
|
context: "RequestContext",
|
|
userKey: str = "userId",
|
|
usernameKey: str = "username",
|
|
instanceKey: str = "featureInstanceId",
|
|
instanceLabelKey: str = "instanceLabel",
|
|
) -> None:
|
|
"""Resolve userId → username and featureInstanceId → label in-place."""
|
|
userIds = set()
|
|
instanceIds = set()
|
|
for r in items:
|
|
uid = r.get(userKey)
|
|
if uid and not r.get(usernameKey):
|
|
userIds.add(uid)
|
|
iid = r.get(instanceKey)
|
|
if iid:
|
|
instanceIds.add(iid)
|
|
|
|
userMap: Dict[str, str] = {}
|
|
instanceMap: Dict[str, str] = {}
|
|
|
|
try:
|
|
from modules.interfaces.interfaceDbApp import getInterface
|
|
appIf = getInterface(
|
|
context.user,
|
|
mandateId=str(context.mandateId) if context.mandateId else None,
|
|
)
|
|
if userIds:
|
|
users = appIf.getUsersByIds(list(userIds))
|
|
for uid, u in users.items():
|
|
name = getattr(u, "displayName", None) or getattr(u, "email", None) or uid
|
|
userMap[uid] = name
|
|
if instanceIds:
|
|
for iid in instanceIds:
|
|
fi = appIf.getFeatureInstance(iid)
|
|
if fi:
|
|
instanceMap[iid] = getattr(fi, "label", None) or getattr(fi, "featureCode", None) or iid
|
|
except Exception as e:
|
|
logger.debug("_enrichUserAndInstanceLabels: %s", e)
|
|
|
|
for r in items:
|
|
uid = r.get(userKey)
|
|
if uid and not r.get(usernameKey) and uid in userMap:
|
|
r[usernameKey] = userMap[uid]
|
|
iid = r.get(instanceKey)
|
|
if iid and iid in instanceMap:
|
|
r[instanceLabelKey] = instanceMap[iid]
|
|
|
|
|
|
def _requireAuditAccess(context: RequestContext):
|
|
"""Raise 403 unless user has mandate-admin or compliance-viewer access."""
|
|
if context.isPlatformAdmin:
|
|
return
|
|
|
|
from modules.interfaces.interfaceDbApp import getInterface
|
|
appIf = getInterface(context.user, mandateId=str(context.mandateId) if context.mandateId else None)
|
|
mandateId = str(context.mandateId) if context.mandateId else None
|
|
if not mandateId:
|
|
raise HTTPException(status_code=403, detail=routeApiMsg("Mandanten-Kontext erforderlich"))
|
|
|
|
from modules.datamodels.datamodelRbac import AccessRuleContext
|
|
permissions = appIf.rbac.getUserPermissions(
|
|
context.user, AccessRuleContext.UI, "ui.system.complianceAudit",
|
|
mandateId=mandateId,
|
|
)
|
|
if not permissions or not permissions.view:
|
|
raise HTTPException(status_code=403, detail=routeApiMsg("Kein Zugriff auf Audit-Daten"))
|
|
|
|
|
|
# ── Tab A: AI Data-Flow Log ──
|
|
|
|
@router.get("/ai-log")
|
|
@limiter.limit("120/minute")
|
|
async def getAiAuditLog(
|
|
request: Request,
|
|
context: RequestContext = Depends(getRequestContext),
|
|
userId: Optional[str] = Query(None),
|
|
featureInstanceId: Optional[str] = Query(None),
|
|
aiModel: Optional[str] = Query(None),
|
|
dateFrom: Optional[float] = Query(None, description="UTC epoch seconds"),
|
|
dateTo: Optional[float] = Query(None, description="UTC epoch seconds"),
|
|
limit: int = Query(50, ge=1, le=500),
|
|
offset: int = Query(0, ge=0),
|
|
sort: Optional[str] = Query(None, description='JSON array, e.g. [{"field":"timestamp","direction":"desc"}]'),
|
|
filters: Optional[str] = Query(None, description='JSON object, e.g. {"aiModel":"gpt-4o"}'),
|
|
search: Optional[str] = Query(None),
|
|
mode: Optional[str] = Query(None, description="'filterValues' to get distinct values for a column"),
|
|
column: Optional[str] = Query(None, description="Column key (required when mode=filterValues)"),
|
|
):
|
|
_requireAuditAccess(context)
|
|
mandateId = str(context.mandateId) if context.mandateId else ""
|
|
if not mandateId:
|
|
raise HTTPException(status_code=400, detail=routeApiMsg("Mandanten-ID erforderlich"))
|
|
|
|
from modules.shared.aiAuditLogger import aiAuditLogger
|
|
result = aiAuditLogger.getAiAuditLogs(
|
|
mandateId,
|
|
userId=userId,
|
|
featureInstanceId=featureInstanceId,
|
|
aiModel=aiModel,
|
|
fromTimestamp=dateFrom,
|
|
toTimestamp=dateTo,
|
|
limit=9999,
|
|
offset=0,
|
|
)
|
|
items = result.get("items", [])
|
|
|
|
_enrichUserAndInstanceLabels(items, context)
|
|
|
|
if mode == "filterValues" and column:
|
|
items = _applySortFilterSearch(items, filtersJson=filters)
|
|
return _distinctColumnValues(items, column)
|
|
|
|
items = _applySortFilterSearch(
|
|
items,
|
|
sortJson=sort,
|
|
filtersJson=filters,
|
|
search=search,
|
|
searchableKeys=["username", "aiModel", "instanceLabel", "aiProvider", "operationType"],
|
|
)
|
|
|
|
totalItems = len(items)
|
|
page = items[offset: offset + limit]
|
|
return {"items": page, "totalItems": totalItems}
|
|
|
|
|
|
@router.get("/ai-log/{entryId}/content")
|
|
@limiter.limit("60/minute")
|
|
async def getAiAuditEntryContent(
|
|
request: Request,
|
|
entryId: str = Path(...),
|
|
context: RequestContext = Depends(getRequestContext),
|
|
):
|
|
_requireAuditAccess(context)
|
|
mandateId = str(context.mandateId) if context.mandateId else ""
|
|
|
|
from modules.shared.aiAuditLogger import aiAuditLogger
|
|
result = aiAuditLogger.getAiAuditEntryContent(entryId, mandateId)
|
|
if not result:
|
|
raise HTTPException(status_code=404, detail=routeApiMsg("Audit-Eintrag nicht gefunden"))
|
|
|
|
_phRx = re.compile(r"\[([a-z]+)\.([a-f0-9-]{36})\]")
|
|
combinedText = (result.get("contentInputFull") or "") + (result.get("contentOutputFull") or "")
|
|
phIds = set(m.group(2) for m in _phRx.finditer(combinedText))
|
|
mappings = []
|
|
|
|
if phIds:
|
|
try:
|
|
from modules.features.neutralization.interfaceFeatureNeutralizer import getInterface as getNeutIf
|
|
neutIf = getNeutIf(context.user, mandateId=mandateId, featureInstanceId=None)
|
|
for phId in phIds:
|
|
attr = neutIf.getAttributeById(phId)
|
|
if attr:
|
|
mappings.append({
|
|
"id": attr.get("id", ""),
|
|
"originalText": attr.get("originalText", ""),
|
|
"patternType": attr.get("patternType", ""),
|
|
})
|
|
except Exception as mapErr:
|
|
logger.warning(f"Could not resolve neutralization mappings for audit entry {entryId}: {mapErr}")
|
|
|
|
result["neutralizationMappings"] = mappings
|
|
return result
|
|
|
|
|
|
# ── Tab B: Security / GDPR Audit Log ──
|
|
|
|
@router.get("/log")
|
|
@limiter.limit("120/minute")
|
|
async def getAuditLog(
|
|
request: Request,
|
|
context: RequestContext = Depends(getRequestContext),
|
|
userId: Optional[str] = Query(None),
|
|
category: Optional[str] = Query(None),
|
|
action: Optional[str] = Query(None),
|
|
dateFrom: Optional[float] = Query(None),
|
|
dateTo: Optional[float] = Query(None),
|
|
limit: int = Query(100, ge=1, le=500),
|
|
offset: int = Query(0, ge=0),
|
|
sort: Optional[str] = Query(None),
|
|
filters: Optional[str] = Query(None),
|
|
search: Optional[str] = Query(None),
|
|
mode: Optional[str] = Query(None),
|
|
column: Optional[str] = Query(None),
|
|
):
|
|
_requireAuditAccess(context)
|
|
mandateId = str(context.mandateId) if context.mandateId else None
|
|
|
|
from modules.shared.auditLogger import audit_logger
|
|
records = audit_logger.getAuditLogs(
|
|
userId=userId,
|
|
mandateId=mandateId,
|
|
category=category,
|
|
action=action,
|
|
fromTimestamp=dateFrom,
|
|
toTimestamp=dateTo,
|
|
limit=9999,
|
|
)
|
|
|
|
_enrichUserAndInstanceLabels(records, context)
|
|
|
|
if mode == "filterValues" and column:
|
|
records = _applySortFilterSearch(records, filtersJson=filters)
|
|
return _distinctColumnValues(records, column)
|
|
|
|
records = _applySortFilterSearch(
|
|
records,
|
|
sortJson=sort,
|
|
filtersJson=filters,
|
|
search=search,
|
|
searchableKeys=["username", "action", "resourceType", "category"],
|
|
)
|
|
|
|
totalItems = len(records)
|
|
page = records[offset: offset + limit]
|
|
return {"items": page, "totalItems": totalItems}
|
|
|
|
|
|
# ── Tab C: Audit Statistics ──
|
|
|
|
@router.get("/stats")
|
|
@limiter.limit("60/minute")
|
|
async def getAuditStats(
|
|
request: Request,
|
|
context: RequestContext = Depends(getRequestContext),
|
|
dateFrom: str = Query(..., description="ISO YYYY-MM-DD (inclusive)"),
|
|
dateTo: str = Query(..., description="ISO YYYY-MM-DD (inclusive)"),
|
|
groupBy: str = Query("model", pattern="^(model|user|feature|day)$",
|
|
description="Grouping: model, user, feature, day"),
|
|
):
|
|
_requireAuditAccess(context)
|
|
mandateId = str(context.mandateId) if context.mandateId else ""
|
|
if not mandateId:
|
|
raise HTTPException(status_code=400, detail=routeApiMsg("Mandanten-ID erforderlich"))
|
|
|
|
from modules.shared.aiAuditLogger import aiAuditLogger
|
|
from modules.shared.dateRange import isoDateRangeToLocalEpoch
|
|
|
|
fromTs, toTs = isoDateRangeToLocalEpoch(dateFrom, dateTo)
|
|
return aiAuditLogger.getAiAuditStats(
|
|
mandateId, fromTs=fromTs, toTs=toTs, groupBy=groupBy,
|
|
)
|
|
|
|
|
|
# ── Tab D: Neutralization Mappings ──
|
|
|
|
@router.get("/neutralization-mappings")
|
|
@limiter.limit("120/minute")
|
|
async def getNeutralizationMappings(
|
|
request: Request,
|
|
context: RequestContext = Depends(getRequestContext),
|
|
limit: int = Query(200, ge=1, le=2000),
|
|
offset: int = Query(0, ge=0),
|
|
sort: Optional[str] = Query(None),
|
|
filters: Optional[str] = Query(None),
|
|
search: Optional[str] = Query(None),
|
|
mode: Optional[str] = Query(None),
|
|
column: Optional[str] = Query(None),
|
|
):
|
|
_requireAuditAccess(context)
|
|
mandateId = str(context.mandateId) if context.mandateId else ""
|
|
if not mandateId:
|
|
raise HTTPException(status_code=400, detail=routeApiMsg("Mandanten-ID erforderlich"))
|
|
|
|
try:
|
|
from modules.features.neutralization.interfaceFeatureNeutralizer import getInterface as getNeutIf
|
|
neutIf = getNeutIf(context.user, mandateId=mandateId, featureInstanceId=None)
|
|
attrs = neutIf.getNeutralizationAttributes()
|
|
items = [a.model_dump() if hasattr(a, "model_dump") else dict(a) for a in attrs]
|
|
for item in items:
|
|
pType = item.get("patternType", "")
|
|
uid = item.get("id", "")
|
|
item["placeholder"] = f"[{pType}.{uid}]" if pType and uid else uid
|
|
|
|
_enrichUserAndInstanceLabels(items, context)
|
|
|
|
if mode == "filterValues" and column:
|
|
items = _applySortFilterSearch(items, filtersJson=filters)
|
|
return _distinctColumnValues(items, column)
|
|
|
|
items = _applySortFilterSearch(
|
|
items,
|
|
sortJson=sort,
|
|
filtersJson=filters,
|
|
search=search,
|
|
searchableKeys=["placeholder", "originalText", "patternType"],
|
|
)
|
|
if not sort:
|
|
items.sort(key=lambda r: (r.get("patternType", ""), r.get("originalText", "")))
|
|
|
|
totalItems = len(items)
|
|
page = items[offset: offset + limit]
|
|
return {"items": page, "totalItems": totalItems}
|
|
except Exception as e:
|
|
logger.error(f"Failed to load neutralization mappings: {e}")
|
|
raise HTTPException(status_code=500, detail=routeApiMsg("Fehler beim Laden der Neutralisierungs-Zuordnungen"))
|
|
|
|
|
|
@router.delete("/neutralization-mappings/{mappingId}")
|
|
@limiter.limit("60/minute")
|
|
async def deleteNeutralizationMapping(
|
|
request: Request,
|
|
mappingId: str = Path(...),
|
|
context: RequestContext = Depends(getRequestContext),
|
|
):
|
|
_requireAuditAccess(context)
|
|
mandateId = str(context.mandateId) if context.mandateId else ""
|
|
if not mandateId:
|
|
raise HTTPException(status_code=400, detail=routeApiMsg("Mandanten-ID erforderlich"))
|
|
|
|
try:
|
|
from modules.features.neutralization.interfaceFeatureNeutralizer import getInterface as getNeutIf
|
|
neutIf = getNeutIf(context.user, mandateId=mandateId, featureInstanceId=None)
|
|
deleted = neutIf.deleteAttributeById(mappingId)
|
|
if not deleted:
|
|
raise HTTPException(status_code=404, detail=routeApiMsg("Zuordnung nicht gefunden"))
|
|
return {"success": True, "id": mappingId}
|
|
except HTTPException:
|
|
raise
|
|
except Exception as e:
|
|
logger.error(f"Failed to delete neutralization mapping {mappingId}: {e}")
|
|
raise HTTPException(status_code=500, detail=routeApiMsg("Fehler beim Löschen der Zuordnung"))
|