gpdr compliancy implemented
This commit is contained in:
ValueOn AG
parent
bc2877bcc1
commit
e737bf5cdb
23 changed files with 2386 additions and 376 deletions
7
app.py
7
app.py
|
|
@ -409,7 +409,7 @@ app.add_middleware(
|
|||
CORSMiddleware,
|
||||
allow_origins=getAllowedOrigins(),
|
||||
allow_credentials=True,
|
||||
allow_methods=["GET", "POST", "PUT", "DELETE", "OPTIONS"],
|
||||
allow_methods=["GET", "POST", "PUT", "PATCH", "DELETE", "OPTIONS"],
|
||||
allow_headers=["*"],
|
||||
expose_headers=["*"],
|
||||
max_age=86400, # Increased caching for preflight requests
|
||||
|
|
@ -495,6 +495,9 @@ app.include_router(invitationsRouter)
|
|||
from modules.routes.routeAdminRbacExport import router as rbacAdminExportRouter
|
||||
app.include_router(rbacAdminExportRouter)
|
||||
|
||||
from modules.routes.routeAdminUserAccessOverview import router as userAccessOverviewRouter
|
||||
app.include_router(userAccessOverviewRouter)
|
||||
|
||||
from modules.routes.routeGdpr import router as gdprRouter
|
||||
app.include_router(gdprRouter)
|
||||
|
||||
|
|
@ -504,7 +507,7 @@ app.include_router(chatRouter)
|
|||
# ============================================================================
|
||||
# SYSTEM ROUTES (Navigation, etc.)
|
||||
# ============================================================================
|
||||
from modules.system.routeSystem import router as systemRouter, navigationRouter
|
||||
from modules.routes.routeSystem import router as systemRouter, navigationRouter
|
||||
app.include_router(systemRouter)
|
||||
app.include_router(navigationRouter)
|
||||
|
||||
|
|
|
|||
|
|
@ -19,7 +19,7 @@ class AutomationDefinition(BaseModel):
|
|||
{"value": "0 10 * * 1", "label": {"en": "Weekly Monday 10:00", "fr": "Hebdomadaire lundi 10:00"}}
|
||||
]})
|
||||
template: str = Field(description="JSON template with placeholders (format: {{KEY:PLACEHOLDER_NAME}})", json_schema_extra={"frontend_type": "textarea", "frontend_required": True})
|
||||
placeholders: Dict[str, str] = Field(default_factory=dict, description="Dictionary of placeholder key/value pairs (e.g., {'connectionName': 'MyConnection', 'sharepointFolderNameSource': '/folder/path', 'webResearchUrl': 'https://...', 'webResearchPrompt': '...', 'documentPrompt': '...'})", json_schema_extra={"frontend_type": "text"})
|
||||
placeholders: Dict[str, str] = Field(default_factory=dict, description="Dictionary of placeholder key/value pairs (e.g., {'connectionName': 'MyConnection', 'sharepointFolderNameSource': '/folder/path', 'webResearchUrl': 'https://...', 'webResearchPrompt': '...', 'documentPrompt': '...'})", json_schema_extra={"frontend_type": "textarea"})
|
||||
active: bool = Field(default=False, description="Whether automation should be launched in event handler", json_schema_extra={"frontend_type": "checkbox", "frontend_required": False})
|
||||
eventId: Optional[str] = Field(None, description="Event ID from event management (None if not registered)", json_schema_extra={"frontend_type": "text", "frontend_readonly": True, "frontend_required": False})
|
||||
status: Optional[str] = Field(None, description="Status: 'active' if event is registered, 'inactive' if not (computed, readonly)", json_schema_extra={"frontend_type": "text", "frontend_readonly": True, "frontend_required": False})
|
||||
|
|
|
|||
|
|
@ -14,7 +14,7 @@ import json
|
|||
|
||||
# Import interfaces and models
|
||||
from modules.interfaces.interfaceDbChat import getInterface as getChatInterface
|
||||
from modules.auth import getCurrentUser, limiter, getRequestContext, RequestContext
|
||||
from modules.auth import limiter, getRequestContext, RequestContext
|
||||
from modules.features.automation.datamodelFeatureAutomation import AutomationDefinition
|
||||
from modules.datamodels.datamodelChat import ChatWorkflow
|
||||
from modules.datamodels.datamodelPagination import PaginationParams, PaginatedResponse, PaginationMetadata, normalize_pagination_dict
|
||||
|
|
@ -210,6 +210,47 @@ async def update_automation(
|
|||
detail=f"Error updating automation: {str(e)}"
|
||||
)
|
||||
|
||||
@router.patch("/{automationId}/status")
|
||||
@limiter.limit("30/minute")
|
||||
async def update_automation_status(
|
||||
request: Request,
|
||||
automationId: str = Path(..., description="Automation ID"),
|
||||
active: bool = Body(..., embed=True),
|
||||
context: RequestContext = Depends(getRequestContext)
|
||||
) -> AutomationDefinition:
|
||||
"""Update only the active status of an automation definition"""
|
||||
try:
|
||||
chatInterface = getChatInterface(context.user, mandateId=str(context.mandateId) if context.mandateId else None, featureInstanceId=str(context.featureInstanceId) if context.featureInstanceId else None)
|
||||
|
||||
# Get existing automation
|
||||
automation = chatInterface.getAutomationDefinition(automationId)
|
||||
if not automation:
|
||||
raise HTTPException(
|
||||
status_code=404,
|
||||
detail=f"Automation {automationId} not found"
|
||||
)
|
||||
|
||||
# Update only the active field
|
||||
automationData = automation if isinstance(automation, dict) else automation.model_dump()
|
||||
automationData['active'] = active
|
||||
|
||||
updated = chatInterface.updateAutomationDefinition(automationId, automationData)
|
||||
return updated
|
||||
except HTTPException:
|
||||
raise
|
||||
except PermissionError as e:
|
||||
raise HTTPException(
|
||||
status_code=403,
|
||||
detail=str(e)
|
||||
)
|
||||
except Exception as e:
|
||||
logger.error(f"Error updating automation status: {str(e)}")
|
||||
raise HTTPException(
|
||||
status_code=500,
|
||||
detail=f"Error updating automation status: {str(e)}"
|
||||
)
|
||||
|
||||
|
||||
@router.delete("/{automationId}")
|
||||
@limiter.limit("10/minute")
|
||||
async def delete_automation(
|
||||
|
|
|
|||
|
|
@ -369,6 +369,38 @@ AUTOMATION_TEMPLATES: Dict[str, Any] = {
|
|||
"jiraIssueType": "Task",
|
||||
"taskSyncDefinition": "{\"ID\":[\"get\",[\"key\"]],\"Module Category\":[\"get\",[\"fields\",\"customfield_10058\",\"value\"]],\"Summary\":[\"get\",[\"fields\",\"summary\"]],\"Description\":[\"get\",[\"fields\",\"description\"]],\"References\":[\"get\",[\"fields\",\"customfield_10066\"]],\"Priority\":[\"get\",[\"fields\",\"priority\",\"name\"]],\"Issue Status\":[\"get\",[\"fields\",\"status\",\"name\"]],\"Assignee\":[\"get\",[\"fields\",\"assignee\",\"displayName\"]],\"Issue Created\":[\"get\",[\"fields\",\"created\"]],\"Due Date\":[\"get\",[\"fields\",\"duedate\"]],\"DELTA Comments\":[\"get\",[\"fields\",\"customfield_10167\"]],\"SELISE Ticket References\":[\"put\",[\"fields\",\"customfield_10067\"]],\"SELISE Status Values\":[\"put\",[\"fields\",\"customfield_10065\"]],\"SELISE Comments\":[\"put\",[\"fields\",\"customfield_10168\"]]}"
|
||||
}
|
||||
},
|
||||
{
|
||||
"template": {
|
||||
"overview": "Expenses PDF to Trustee Position",
|
||||
"tasks": [
|
||||
{
|
||||
"id": "Task01",
|
||||
"title": "Extract Expenses from SharePoint PDFs",
|
||||
"description": "Reads PDF expense documents from SharePoint folder, extracts data via AI, and saves to TrusteePosition",
|
||||
"objective": "Extract expense data from PDF documents and store in Trustee database with automatic file organization",
|
||||
"actionList": [
|
||||
{
|
||||
"execMethod": "sharepoint",
|
||||
"execAction": "getExpensesFromPdf",
|
||||
"execParameters": {
|
||||
"connectionReference": "{{KEY:connectionName}}",
|
||||
"sharepointFolder": "{{KEY:sharepointFolder}}",
|
||||
"featureInstanceId": "{{KEY:featureInstanceId}}",
|
||||
"prompt": "{{KEY:extractionPrompt}}"
|
||||
},
|
||||
"execResultLabel": "expense_extraction_result"
|
||||
}
|
||||
]
|
||||
}
|
||||
]
|
||||
},
|
||||
"parameters": {
|
||||
"connectionName": "",
|
||||
"sharepointFolder": "",
|
||||
"featureInstanceId": "",
|
||||
"extractionPrompt": "Du bist ein Spezialist für die Extraktion von Spesendaten aus PDF-Dokumenten.\n\nAUFGABE:\nExtrahiere alle Speseneinträge aus dem bereitgestellten PDF-Dokument und gib sie im CSV-Format zurück.\n\nWICHTIGE REGELN:\n1. Pro MwSt-Prozentsatz einen separaten Datensatz erstellen\n2. Alle Datensätze zusammen müssen den Gesamtbetrag des Dokuments ergeben\n3. Der gesamte extrahierte Text des Dokuments muss im Feld \"desc\" erfasst werden\n4. Feld \"company\" enthält den Lieferanten/Verkäufer der Buchung\n5. Tags müssen aus dieser Liste gewählt werden: customer, meeting, license, subscription, fuel, food, material\n - Mehrere zutreffende Tags mit Komma trennen\n\nCSV-SPALTEN (in dieser Reihenfolge):\nvaluta,transactionDateTime,company,desc,tags,bookingCurrency,bookingAmount,originalCurrency,originalAmount,vatPercentage,vatAmount\n\nDATENFORMAT:\n- valuta: YYYY-MM-DD (Valutadatum)\n- transactionDateTime: Unix-Timestamp in Sekunden (Transaktionszeitpunkt)\n- company: Lieferant/Verkäufer Name\n- desc: Vollständiger extrahierter Text des Dokuments\n- tags: Komma-getrennte Tags aus der erlaubten Liste\n- bookingCurrency: Währungscode (CHF, EUR, USD, GBP)\n- bookingAmount: Buchungsbetrag als Dezimalzahl\n- originalCurrency: Original-Währungscode\n- originalAmount: Original-Betrag als Dezimalzahl\n- vatPercentage: MwSt-Prozentsatz (z.B. 8.1 für 8.1%)\n- vatAmount: MwSt-Betrag als Dezimalzahl\n\nBEISPIEL OUTPUT:\nvaluta,transactionDateTime,company,desc,tags,bookingCurrency,bookingAmount,originalCurrency,originalAmount,vatPercentage,vatAmount\n2026-01-15,1736953200,Migros AG,\"Einkauf Migros Zürich...\",food,CHF,45.50,CHF,45.50,2.6,1.15\n2026-01-15,1736953200,Migros AG,\"Einkauf Migros Zürich...\",material,CHF,12.30,CHF,12.30,8.1,0.92\n\nHINWEISE:\n- Wenn nur ein MwSt-Satz vorhanden ist, einen Datensatz erstellen\n- Wenn mehrere MwSt-Sätze vorhanden sind (z.B. Lebensmittel 2.6% und Non-Food 8.1%), separate Datensätze erstellen\n- Bei fehlenden Informationen: leeres Feld oder Standardwert\n- Keine Anführungszeichen um numerische Werte"
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
|
|
|
|||
|
|
@ -39,6 +39,10 @@ class RealEstateObjects:
|
|||
Handles CRUD operations on Real Estate entities.
|
||||
"""
|
||||
|
||||
# Feature code for RBAC objectKey construction
|
||||
# Used to build: data.feature.realestate.{TableName}
|
||||
FEATURE_CODE = "realestate"
|
||||
|
||||
def __init__(self, currentUser: Optional[User] = None, mandateId: Optional[str] = None, featureInstanceId: Optional[str] = None):
|
||||
"""Initializes the Real Estate Interface.
|
||||
|
||||
|
|
@ -167,7 +171,8 @@ class RealEstateObjects:
|
|||
self.db,
|
||||
Projekt,
|
||||
self.currentUser,
|
||||
recordFilter={"id": projektId}
|
||||
recordFilter={"id": projektId},
|
||||
featureCode=self.FEATURE_CODE
|
||||
)
|
||||
|
||||
if not records:
|
||||
|
|
@ -181,7 +186,8 @@ class RealEstateObjects:
|
|||
self.db,
|
||||
Projekt,
|
||||
self.currentUser,
|
||||
recordFilter=recordFilter or {}
|
||||
recordFilter=recordFilter or {},
|
||||
featureCode=self.FEATURE_CODE
|
||||
)
|
||||
|
||||
return [Projekt(**r) for r in records]
|
||||
|
|
@ -255,7 +261,8 @@ class RealEstateObjects:
|
|||
self.db,
|
||||
Parzelle,
|
||||
self.currentUser,
|
||||
recordFilter={"id": parzelleId}
|
||||
recordFilter={"id": parzelleId},
|
||||
featureCode=self.FEATURE_CODE
|
||||
)
|
||||
|
||||
if not records:
|
||||
|
|
@ -464,7 +471,8 @@ class RealEstateObjects:
|
|||
self.db,
|
||||
Dokument,
|
||||
self.currentUser,
|
||||
recordFilter={"id": dokumentId}
|
||||
recordFilter={"id": dokumentId},
|
||||
featureCode=self.FEATURE_CODE
|
||||
)
|
||||
|
||||
if not records:
|
||||
|
|
@ -478,7 +486,8 @@ class RealEstateObjects:
|
|||
self.db,
|
||||
Dokument,
|
||||
self.currentUser,
|
||||
recordFilter=recordFilter or {}
|
||||
recordFilter=recordFilter or {},
|
||||
featureCode=self.FEATURE_CODE
|
||||
)
|
||||
return [Dokument(**r) for r in records]
|
||||
|
||||
|
|
@ -533,7 +542,8 @@ class RealEstateObjects:
|
|||
self.db,
|
||||
Gemeinde,
|
||||
self.currentUser,
|
||||
recordFilter={"id": gemeindeId}
|
||||
recordFilter={"id": gemeindeId},
|
||||
featureCode=self.FEATURE_CODE
|
||||
)
|
||||
|
||||
if not records:
|
||||
|
|
@ -547,7 +557,8 @@ class RealEstateObjects:
|
|||
self.db,
|
||||
Gemeinde,
|
||||
self.currentUser,
|
||||
recordFilter=recordFilter or {}
|
||||
recordFilter=recordFilter or {},
|
||||
featureCode=self.FEATURE_CODE
|
||||
)
|
||||
return [Gemeinde(**r) for r in records]
|
||||
|
||||
|
|
@ -602,7 +613,8 @@ class RealEstateObjects:
|
|||
self.db,
|
||||
Kanton,
|
||||
self.currentUser,
|
||||
recordFilter={"id": kantonId}
|
||||
recordFilter={"id": kantonId},
|
||||
featureCode=self.FEATURE_CODE
|
||||
)
|
||||
|
||||
if not records:
|
||||
|
|
@ -616,7 +628,8 @@ class RealEstateObjects:
|
|||
self.db,
|
||||
Kanton,
|
||||
self.currentUser,
|
||||
recordFilter=recordFilter or {}
|
||||
recordFilter=recordFilter or {},
|
||||
featureCode=self.FEATURE_CODE
|
||||
)
|
||||
return [Kanton(**r) for r in records]
|
||||
|
||||
|
|
@ -671,7 +684,8 @@ class RealEstateObjects:
|
|||
self.db,
|
||||
Land,
|
||||
self.currentUser,
|
||||
recordFilter={"id": landId}
|
||||
recordFilter={"id": landId},
|
||||
featureCode=self.FEATURE_CODE
|
||||
)
|
||||
|
||||
if not records:
|
||||
|
|
@ -685,7 +699,8 @@ class RealEstateObjects:
|
|||
self.db,
|
||||
Land,
|
||||
self.currentUser,
|
||||
recordFilter=recordFilter or {}
|
||||
recordFilter=recordFilter or {},
|
||||
featureCode=self.FEATURE_CODE
|
||||
)
|
||||
return [Land(**r) for r in records]
|
||||
|
||||
|
|
|
|||
|
|
@ -66,6 +66,10 @@ class TrusteeObjects:
|
|||
Interface to Trustee database.
|
||||
Manages trustee organisations, roles, access, contracts, documents, and positions.
|
||||
"""
|
||||
|
||||
# Feature code for RBAC objectKey construction
|
||||
# Used to build: data.feature.trustee.{TableName}
|
||||
FEATURE_CODE = "trustee"
|
||||
|
||||
def __init__(self, currentUser: Optional[User] = None, mandateId: Optional[str] = None, featureInstanceId: Optional[str] = None):
|
||||
"""Initializes the Trustee Interface.
|
||||
|
|
@ -173,7 +177,8 @@ class TrusteeObjects:
|
|||
AccessRuleContext.DATA,
|
||||
tableName,
|
||||
mandateId=self.mandateId,
|
||||
featureInstanceId=self.featureInstanceId
|
||||
featureInstanceId=self.featureInstanceId,
|
||||
featureCode=self.FEATURE_CODE
|
||||
)
|
||||
|
||||
if not permissions.view:
|
||||
|
|
@ -200,7 +205,8 @@ class TrusteeObjects:
|
|||
AccessRuleContext.DATA,
|
||||
tableName,
|
||||
mandateId=self.mandateId,
|
||||
featureInstanceId=self.featureInstanceId
|
||||
featureInstanceId=self.featureInstanceId,
|
||||
featureCode=self.FEATURE_CODE
|
||||
)
|
||||
|
||||
if not permissions.view:
|
||||
|
|
@ -264,7 +270,8 @@ class TrusteeObjects:
|
|||
recordFilter=None,
|
||||
orderBy="id",
|
||||
mandateId=self.mandateId,
|
||||
featureInstanceId=self.featureInstanceId
|
||||
featureInstanceId=self.featureInstanceId,
|
||||
featureCode=self.FEATURE_CODE
|
||||
)
|
||||
logger.debug(f"getAllOrganisations: getRecordsetWithRBAC returned {len(records)} records")
|
||||
|
||||
|
|
@ -357,7 +364,8 @@ class TrusteeObjects:
|
|||
recordFilter=None,
|
||||
orderBy="id",
|
||||
mandateId=self.mandateId,
|
||||
featureInstanceId=self.featureInstanceId
|
||||
featureInstanceId=self.featureInstanceId,
|
||||
featureCode=self.FEATURE_CODE
|
||||
)
|
||||
|
||||
# Users with ALL access level (from system RBAC) see all roles
|
||||
|
|
@ -467,7 +475,8 @@ class TrusteeObjects:
|
|||
recordFilter=None,
|
||||
orderBy="id",
|
||||
mandateId=self.mandateId,
|
||||
featureInstanceId=self.featureInstanceId
|
||||
featureInstanceId=self.featureInstanceId,
|
||||
featureCode=self.FEATURE_CODE
|
||||
)
|
||||
|
||||
# Users with ALL access level (from system RBAC) see all records
|
||||
|
|
@ -526,7 +535,8 @@ class TrusteeObjects:
|
|||
recordFilter={"organisationId": organisationId},
|
||||
orderBy="id",
|
||||
mandateId=self.mandateId,
|
||||
featureInstanceId=self.featureInstanceId
|
||||
featureInstanceId=self.featureInstanceId,
|
||||
featureCode=self.FEATURE_CODE
|
||||
)
|
||||
return [TrusteeAccess(**{k: v for k, v in r.items() if not k.startswith("_")}) for r in records]
|
||||
|
||||
|
|
@ -543,7 +553,8 @@ class TrusteeObjects:
|
|||
recordFilter={"userId": userId},
|
||||
orderBy="id",
|
||||
mandateId=self.mandateId,
|
||||
featureInstanceId=self.featureInstanceId
|
||||
featureInstanceId=self.featureInstanceId,
|
||||
featureCode=self.FEATURE_CODE
|
||||
)
|
||||
|
||||
# Users with ALL access level (from system RBAC) see all records
|
||||
|
|
@ -660,7 +671,8 @@ class TrusteeObjects:
|
|||
recordFilter=None,
|
||||
orderBy="id",
|
||||
mandateId=self.mandateId,
|
||||
featureInstanceId=self.featureInstanceId
|
||||
featureInstanceId=self.featureInstanceId,
|
||||
featureCode=self.FEATURE_CODE
|
||||
)
|
||||
|
||||
totalItems = len(records)
|
||||
|
|
@ -693,7 +705,8 @@ class TrusteeObjects:
|
|||
recordFilter={"organisationId": organisationId},
|
||||
orderBy="label",
|
||||
mandateId=self.mandateId,
|
||||
featureInstanceId=self.featureInstanceId
|
||||
featureInstanceId=self.featureInstanceId,
|
||||
featureCode=self.FEATURE_CODE
|
||||
)
|
||||
return [TrusteeContract(**{k: v for k, v in r.items() if not k.startswith("_")}) for r in records]
|
||||
|
||||
|
|
@ -799,7 +812,8 @@ class TrusteeObjects:
|
|||
recordFilter=None,
|
||||
orderBy="documentName",
|
||||
mandateId=self.mandateId,
|
||||
featureInstanceId=self.featureInstanceId
|
||||
featureInstanceId=self.featureInstanceId,
|
||||
featureCode=self.FEATURE_CODE
|
||||
)
|
||||
|
||||
# Convert dicts to Pydantic objects (remove binary data and internal fields)
|
||||
|
|
@ -838,7 +852,8 @@ class TrusteeObjects:
|
|||
recordFilter={"contractId": contractId},
|
||||
orderBy="documentName",
|
||||
mandateId=self.mandateId,
|
||||
featureInstanceId=self.featureInstanceId
|
||||
featureInstanceId=self.featureInstanceId,
|
||||
featureCode=self.FEATURE_CODE
|
||||
)
|
||||
|
||||
result = []
|
||||
|
|
@ -945,7 +960,8 @@ class TrusteeObjects:
|
|||
recordFilter=None,
|
||||
orderBy="valuta",
|
||||
mandateId=self.mandateId,
|
||||
featureInstanceId=self.featureInstanceId
|
||||
featureInstanceId=self.featureInstanceId,
|
||||
featureCode=self.FEATURE_CODE
|
||||
)
|
||||
|
||||
# Convert dicts to Pydantic objects (remove internal fields)
|
||||
|
|
@ -984,7 +1000,8 @@ class TrusteeObjects:
|
|||
recordFilter={"contractId": contractId},
|
||||
orderBy="valuta",
|
||||
mandateId=self.mandateId,
|
||||
featureInstanceId=self.featureInstanceId
|
||||
featureInstanceId=self.featureInstanceId,
|
||||
featureCode=self.FEATURE_CODE
|
||||
)
|
||||
return [TrusteePosition(**{k: v for k, v in r.items() if not k.startswith("_")}) for r in records]
|
||||
|
||||
|
|
@ -998,7 +1015,8 @@ class TrusteeObjects:
|
|||
recordFilter={"organisationId": organisationId},
|
||||
orderBy="valuta",
|
||||
mandateId=self.mandateId,
|
||||
featureInstanceId=self.featureInstanceId
|
||||
featureInstanceId=self.featureInstanceId,
|
||||
featureCode=self.FEATURE_CODE
|
||||
)
|
||||
return [TrusteePosition(**{k: v for k, v in r.items() if not k.startswith("_")}) for r in records]
|
||||
|
||||
|
|
@ -1155,7 +1173,8 @@ class TrusteeObjects:
|
|||
recordFilter={"positionId": positionId},
|
||||
orderBy="id",
|
||||
mandateId=self.mandateId,
|
||||
featureInstanceId=self.featureInstanceId
|
||||
featureInstanceId=self.featureInstanceId,
|
||||
featureCode=self.FEATURE_CODE
|
||||
)
|
||||
return [TrusteePositionDocument(**{k: v for k, v in r.items() if not k.startswith("_")}) for r in links]
|
||||
|
||||
|
|
@ -1169,7 +1188,8 @@ class TrusteeObjects:
|
|||
recordFilter={"documentId": documentId},
|
||||
orderBy="id",
|
||||
mandateId=self.mandateId,
|
||||
featureInstanceId=self.featureInstanceId
|
||||
featureInstanceId=self.featureInstanceId,
|
||||
featureCode=self.FEATURE_CODE
|
||||
)
|
||||
return [TrusteePositionDocument(**{k: v for k, v in r.items() if not k.startswith("_")}) for r in links]
|
||||
|
||||
|
|
|
|||
|
|
@ -38,6 +38,11 @@ UI_OBJECTS = [
|
|||
"label": {"en": "Position Documents", "de": "Positions-Dokumente", "fr": "Documents de position"},
|
||||
"meta": {"area": "position-documents"}
|
||||
},
|
||||
{
|
||||
"objectKey": "ui.feature.trustee.expense-import",
|
||||
"label": {"en": "Expense Import", "de": "Spesen Import", "fr": "Import de dépenses"},
|
||||
"meta": {"area": "expense-import"}
|
||||
},
|
||||
{
|
||||
"objectKey": "ui.feature.trustee.instance-roles",
|
||||
"label": {"en": "Instance Roles & Permissions", "de": "Instanz-Rollen & Berechtigungen", "fr": "Rôles et permissions d'instance"},
|
||||
|
|
@ -144,6 +149,7 @@ TEMPLATE_ROLES = [
|
|||
{"context": "UI", "item": "ui.feature.trustee.positions", "view": True},
|
||||
{"context": "UI", "item": "ui.feature.trustee.documents", "view": True},
|
||||
{"context": "UI", "item": "ui.feature.trustee.position-documents", "view": True},
|
||||
{"context": "UI", "item": "ui.feature.trustee.expense-import", "view": True},
|
||||
# Group-level DATA access
|
||||
{"context": "DATA", "item": None, "view": True, "read": "g", "create": "g", "update": "g", "delete": "g"},
|
||||
]
|
||||
|
|
|
|||
|
|
@ -376,13 +376,21 @@ def _createTableSpecificRules(db: DatabaseConnector) -> None:
|
|||
userId = _getRoleId(db, "user")
|
||||
viewerId = _getRoleId(db, "viewer")
|
||||
|
||||
# ==========================================================================
|
||||
# SYSTEM TABLE RULES - Using standardized dot format: data.system.{TableName}
|
||||
# ==========================================================================
|
||||
# All DATA context items MUST use the full objectKey format for consistency.
|
||||
# This matches the DATA_OBJECTS registration in mainSystem.py.
|
||||
# Feature tables use: data.feature.{featureCode}.{TableName}
|
||||
# ==========================================================================
|
||||
|
||||
# Mandate table - Only SysAdmin (flag) can access, not roles
|
||||
# Regular roles have no access to Mandate table
|
||||
if adminId:
|
||||
tableRules.append(AccessRule(
|
||||
roleId=adminId,
|
||||
context=AccessRuleContext.DATA,
|
||||
item="Mandate",
|
||||
item="data.system.Mandate",
|
||||
view=False,
|
||||
read=AccessLevel.NONE,
|
||||
create=AccessLevel.NONE,
|
||||
|
|
@ -393,7 +401,7 @@ def _createTableSpecificRules(db: DatabaseConnector) -> None:
|
|||
tableRules.append(AccessRule(
|
||||
roleId=userId,
|
||||
context=AccessRuleContext.DATA,
|
||||
item="Mandate",
|
||||
item="data.system.Mandate",
|
||||
view=False,
|
||||
read=AccessLevel.NONE,
|
||||
create=AccessLevel.NONE,
|
||||
|
|
@ -404,7 +412,7 @@ def _createTableSpecificRules(db: DatabaseConnector) -> None:
|
|||
tableRules.append(AccessRule(
|
||||
roleId=viewerId,
|
||||
context=AccessRuleContext.DATA,
|
||||
item="Mandate",
|
||||
item="data.system.Mandate",
|
||||
view=False,
|
||||
read=AccessLevel.NONE,
|
||||
create=AccessLevel.NONE,
|
||||
|
|
@ -417,7 +425,7 @@ def _createTableSpecificRules(db: DatabaseConnector) -> None:
|
|||
tableRules.append(AccessRule(
|
||||
roleId=adminId,
|
||||
context=AccessRuleContext.DATA,
|
||||
item="UserInDB",
|
||||
item="data.system.UserInDB",
|
||||
view=True,
|
||||
read=AccessLevel.GROUP,
|
||||
create=AccessLevel.GROUP,
|
||||
|
|
@ -428,7 +436,7 @@ def _createTableSpecificRules(db: DatabaseConnector) -> None:
|
|||
tableRules.append(AccessRule(
|
||||
roleId=userId,
|
||||
context=AccessRuleContext.DATA,
|
||||
item="UserInDB",
|
||||
item="data.system.UserInDB",
|
||||
view=True,
|
||||
read=AccessLevel.MY,
|
||||
create=AccessLevel.NONE,
|
||||
|
|
@ -439,7 +447,7 @@ def _createTableSpecificRules(db: DatabaseConnector) -> None:
|
|||
tableRules.append(AccessRule(
|
||||
roleId=viewerId,
|
||||
context=AccessRuleContext.DATA,
|
||||
item="UserInDB",
|
||||
item="data.system.UserInDB",
|
||||
view=True,
|
||||
read=AccessLevel.MY,
|
||||
create=AccessLevel.NONE,
|
||||
|
|
@ -447,26 +455,19 @@ def _createTableSpecificRules(db: DatabaseConnector) -> None:
|
|||
delete=AccessLevel.NONE,
|
||||
))
|
||||
|
||||
# System tables only - NOT feature-specific tables!
|
||||
# Feature tables (TrusteeXXX, Projekt, etc.) are handled by FEATURE-TEMPLATE roles.
|
||||
# NOTE: No sysadmin rules - SysAdmin users bypass RBAC via isSysAdmin flag
|
||||
#
|
||||
# Proper format: Just table names for DATA context (item="TableName")
|
||||
# The full data.system.TableName format is for catalog registration only.
|
||||
|
||||
# FileItem and UserConnection: All users (user, admin, viewer) only MY-level CRUD
|
||||
restrictedTables = [
|
||||
"UserConnection", # User connections/sessions - only own records
|
||||
"FileItem", # Uploaded files - only own files
|
||||
"data.system.UserConnection", # User connections/sessions - only own records
|
||||
"data.system.FileItem", # Uploaded files - only own files
|
||||
]
|
||||
|
||||
for table in restrictedTables:
|
||||
for objectKey in restrictedTables:
|
||||
# Admin: Only MY-level access (not group-level!)
|
||||
if adminId:
|
||||
tableRules.append(AccessRule(
|
||||
roleId=adminId,
|
||||
context=AccessRuleContext.DATA,
|
||||
item=table,
|
||||
item=objectKey,
|
||||
view=True,
|
||||
read=AccessLevel.MY,
|
||||
create=AccessLevel.MY,
|
||||
|
|
@ -478,7 +479,7 @@ def _createTableSpecificRules(db: DatabaseConnector) -> None:
|
|||
tableRules.append(AccessRule(
|
||||
roleId=userId,
|
||||
context=AccessRuleContext.DATA,
|
||||
item=table,
|
||||
item=objectKey,
|
||||
view=True,
|
||||
read=AccessLevel.MY,
|
||||
create=AccessLevel.MY,
|
||||
|
|
@ -490,7 +491,7 @@ def _createTableSpecificRules(db: DatabaseConnector) -> None:
|
|||
tableRules.append(AccessRule(
|
||||
roleId=viewerId,
|
||||
context=AccessRuleContext.DATA,
|
||||
item=table,
|
||||
item=objectKey,
|
||||
view=True,
|
||||
read=AccessLevel.MY,
|
||||
create=AccessLevel.NONE,
|
||||
|
|
@ -501,37 +502,34 @@ def _createTableSpecificRules(db: DatabaseConnector) -> None:
|
|||
# Prompt: Special rule - CRUD for MY + Read for GROUP
|
||||
# Each user can manage own prompts (m) but can read group prompts (g)
|
||||
if adminId:
|
||||
# Admin: MY-level CRUD + GROUP-level read
|
||||
tableRules.append(AccessRule(
|
||||
roleId=adminId,
|
||||
context=AccessRuleContext.DATA,
|
||||
item="Prompt",
|
||||
item="data.system.Prompt",
|
||||
view=True,
|
||||
read=AccessLevel.GROUP, # Can read group prompts
|
||||
create=AccessLevel.MY, # Can create own prompts
|
||||
update=AccessLevel.MY, # Can update own prompts
|
||||
delete=AccessLevel.MY, # Can delete own prompts
|
||||
read=AccessLevel.GROUP,
|
||||
create=AccessLevel.MY,
|
||||
update=AccessLevel.MY,
|
||||
delete=AccessLevel.MY,
|
||||
))
|
||||
if userId:
|
||||
# User: MY-level CRUD + GROUP-level read
|
||||
tableRules.append(AccessRule(
|
||||
roleId=userId,
|
||||
context=AccessRuleContext.DATA,
|
||||
item="Prompt",
|
||||
item="data.system.Prompt",
|
||||
view=True,
|
||||
read=AccessLevel.GROUP, # Can read group prompts
|
||||
create=AccessLevel.MY, # Can create own prompts
|
||||
update=AccessLevel.MY, # Can update own prompts
|
||||
delete=AccessLevel.MY, # Can delete own prompts
|
||||
read=AccessLevel.GROUP,
|
||||
create=AccessLevel.MY,
|
||||
update=AccessLevel.MY,
|
||||
delete=AccessLevel.MY,
|
||||
))
|
||||
if viewerId:
|
||||
# Viewer: MY-level read + GROUP-level read
|
||||
tableRules.append(AccessRule(
|
||||
roleId=viewerId,
|
||||
context=AccessRuleContext.DATA,
|
||||
item="Prompt",
|
||||
item="data.system.Prompt",
|
||||
view=True,
|
||||
read=AccessLevel.GROUP, # Can read group prompts
|
||||
read=AccessLevel.GROUP,
|
||||
create=AccessLevel.NONE,
|
||||
update=AccessLevel.NONE,
|
||||
delete=AccessLevel.NONE,
|
||||
|
|
@ -542,7 +540,7 @@ def _createTableSpecificRules(db: DatabaseConnector) -> None:
|
|||
tableRules.append(AccessRule(
|
||||
roleId=adminId,
|
||||
context=AccessRuleContext.DATA,
|
||||
item="Invitation",
|
||||
item="data.system.Invitation",
|
||||
view=True,
|
||||
read=AccessLevel.GROUP,
|
||||
create=AccessLevel.GROUP,
|
||||
|
|
@ -553,7 +551,7 @@ def _createTableSpecificRules(db: DatabaseConnector) -> None:
|
|||
tableRules.append(AccessRule(
|
||||
roleId=userId,
|
||||
context=AccessRuleContext.DATA,
|
||||
item="Invitation",
|
||||
item="data.system.Invitation",
|
||||
view=True,
|
||||
read=AccessLevel.MY,
|
||||
create=AccessLevel.MY,
|
||||
|
|
@ -564,7 +562,7 @@ def _createTableSpecificRules(db: DatabaseConnector) -> None:
|
|||
tableRules.append(AccessRule(
|
||||
roleId=viewerId,
|
||||
context=AccessRuleContext.DATA,
|
||||
item="Invitation",
|
||||
item="data.system.Invitation",
|
||||
view=True,
|
||||
read=AccessLevel.MY,
|
||||
create=AccessLevel.NONE,
|
||||
|
|
@ -578,20 +576,20 @@ def _createTableSpecificRules(db: DatabaseConnector) -> None:
|
|||
tableRules.append(AccessRule(
|
||||
roleId=adminId,
|
||||
context=AccessRuleContext.DATA,
|
||||
item="AuthEvent",
|
||||
item="data.system.AuthEvent",
|
||||
view=True,
|
||||
read=AccessLevel.ALL, # Admin can see all auth events for security monitoring
|
||||
create=AccessLevel.NONE, # Events are system-generated
|
||||
update=AccessLevel.NONE, # Audit logs are immutable
|
||||
delete=AccessLevel.NONE, # NO delete - audit integrity!
|
||||
read=AccessLevel.ALL,
|
||||
create=AccessLevel.NONE,
|
||||
update=AccessLevel.NONE,
|
||||
delete=AccessLevel.NONE,
|
||||
))
|
||||
if userId:
|
||||
tableRules.append(AccessRule(
|
||||
roleId=userId,
|
||||
context=AccessRuleContext.DATA,
|
||||
item="AuthEvent",
|
||||
item="data.system.AuthEvent",
|
||||
view=True,
|
||||
read=AccessLevel.MY, # Users can see their own auth events
|
||||
read=AccessLevel.MY,
|
||||
create=AccessLevel.NONE,
|
||||
update=AccessLevel.NONE,
|
||||
delete=AccessLevel.NONE,
|
||||
|
|
@ -600,7 +598,7 @@ def _createTableSpecificRules(db: DatabaseConnector) -> None:
|
|||
tableRules.append(AccessRule(
|
||||
roleId=viewerId,
|
||||
context=AccessRuleContext.DATA,
|
||||
item="AuthEvent",
|
||||
item="data.system.AuthEvent",
|
||||
view=True,
|
||||
read=AccessLevel.MY,
|
||||
create=AccessLevel.NONE,
|
||||
|
|
|
|||
|
|
@ -21,6 +21,27 @@ from modules.security.rootAccess import getRootDbAppConnector
|
|||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def buildDataObjectKey(tableName: str, featureCode: Optional[str] = None) -> str:
|
||||
"""
|
||||
Build the standardized objectKey for a DATA context item.
|
||||
|
||||
Format:
|
||||
- System tables: data.system.{TableName}
|
||||
- Feature tables: data.feature.{featureCode}.{TableName}
|
||||
|
||||
Args:
|
||||
tableName: The database table name (e.g., "UserInDB", "TrusteePosition")
|
||||
featureCode: Optional feature code (e.g., "trustee", "realestate")
|
||||
If None, assumes system table.
|
||||
|
||||
Returns:
|
||||
Full objectKey string (e.g., "data.system.UserInDB" or "data.feature.trustee.TrusteePosition")
|
||||
"""
|
||||
if featureCode:
|
||||
return f"data.feature.{featureCode}.{tableName}"
|
||||
return f"data.system.{tableName}"
|
||||
|
||||
|
||||
def getRecordsetWithRBAC(
|
||||
connector, # DatabaseConnector instance
|
||||
modelClass: Type[BaseModel],
|
||||
|
|
@ -31,6 +52,7 @@ def getRecordsetWithRBAC(
|
|||
mandateId: Optional[str] = None,
|
||||
featureInstanceId: Optional[str] = None,
|
||||
enrichPermissions: bool = False,
|
||||
featureCode: Optional[str] = None,
|
||||
) -> List[Dict[str, Any]]:
|
||||
"""
|
||||
Get records with RBAC filtering applied at database level.
|
||||
|
|
@ -50,11 +72,15 @@ def getRecordsetWithRBAC(
|
|||
featureInstanceId: Explicit feature instance context
|
||||
enrichPermissions: If True, adds _permissions field to each record with row-level
|
||||
permissions { canUpdate, canDelete } based on RBAC rules and _createdBy
|
||||
featureCode: Optional feature code for feature-specific tables (e.g., "trustee").
|
||||
If None, table is treated as a system table.
|
||||
|
||||
Returns:
|
||||
List of filtered records (with _permissions if enrichPermissions=True)
|
||||
"""
|
||||
table = modelClass.__name__
|
||||
# Build full objectKey for RBAC lookup
|
||||
objectKey = buildDataObjectKey(table, featureCode)
|
||||
|
||||
effectiveMandateId = mandateId
|
||||
|
||||
|
|
@ -74,21 +100,21 @@ def getRecordsetWithRBAC(
|
|||
record["_permissions"] = {"canUpdate": True, "canDelete": True}
|
||||
return records
|
||||
|
||||
# Get RBAC permissions for this table
|
||||
# Get RBAC permissions for this table using full objectKey
|
||||
# AccessRule table is always in DbApp database
|
||||
dbApp = getRootDbAppConnector()
|
||||
rbacInstance = RbacClass(connector, dbApp=dbApp)
|
||||
permissions = rbacInstance.getUserPermissions(
|
||||
currentUser,
|
||||
AccessRuleContext.DATA,
|
||||
table,
|
||||
objectKey, # Use full objectKey (e.g., "data.system.UserInDB")
|
||||
mandateId=effectiveMandateId,
|
||||
featureInstanceId=featureInstanceId
|
||||
)
|
||||
|
||||
# Check view permission first
|
||||
if not permissions.view:
|
||||
logger.debug(f"User {currentUser.id} has no view permission for table {table}")
|
||||
logger.debug(f"User {currentUser.id} has no view permission for {objectKey}")
|
||||
return []
|
||||
|
||||
# Build WHERE clause with RBAC filtering
|
||||
|
|
|
|||
|
|
@ -207,7 +207,7 @@ async def import_global_rbac(
|
|||
existingRole = existingRoles[0]
|
||||
roleId = existingRole.get("id")
|
||||
|
||||
rootInterface.db.recordUpdate(
|
||||
rootInterface.db.recordModify(
|
||||
Role,
|
||||
roleId,
|
||||
{
|
||||
|
|
@ -469,7 +469,7 @@ async def import_mandate_rbac(
|
|||
existingRole = existingRoles[0]
|
||||
roleId = existingRole.get("id")
|
||||
|
||||
rootInterface.db.recordUpdate(
|
||||
rootInterface.db.recordModify(
|
||||
Role,
|
||||
roleId,
|
||||
{"description": roleData.get("description", {})}
|
||||
|
|
|
|||
503
modules/routes/routeAdminUserAccessOverview.py
Normal file
503
modules/routes/routeAdminUserAccessOverview.py
Normal file
|
|
@ -0,0 +1,503 @@
|
|||
# Copyright (c) 2025 Patrick Motsch
|
||||
# All rights reserved.
|
||||
"""
|
||||
Admin User Access Overview routes.
|
||||
Provides endpoints for viewing complete user access permissions.
|
||||
|
||||
MULTI-TENANT: These are SYSTEM-LEVEL operations requiring isSysAdmin=true.
|
||||
Shows comprehensive view of what a user can see and access.
|
||||
"""
|
||||
|
||||
from fastapi import APIRouter, HTTPException, Depends, Query, Path, Request
|
||||
from typing import List, Dict, Any, Optional, Set
|
||||
import logging
|
||||
|
||||
from modules.auth import limiter, requireSysAdmin
|
||||
from modules.datamodels.datamodelUam import User, UserInDB
|
||||
from modules.datamodels.datamodelRbac import Role, AccessRule, AccessRuleContext
|
||||
from modules.datamodels.datamodelMembership import (
|
||||
UserMandate,
|
||||
UserMandateRole,
|
||||
FeatureAccess,
|
||||
FeatureAccessRole,
|
||||
)
|
||||
from modules.datamodels.datamodelFeatures import FeatureInstance, Feature
|
||||
from modules.interfaces.interfaceDbApp import getRootInterface
|
||||
|
||||
# Configure logger
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
router = APIRouter(
|
||||
prefix="/api/admin/user-access-overview",
|
||||
tags=["Admin User Access Overview"],
|
||||
responses={404: {"description": "Not found"}}
|
||||
)
|
||||
|
||||
|
||||
def _getAccessLevelLabel(level: Optional[str]) -> str:
|
||||
"""Convert access level code to human-readable label."""
|
||||
labels = {
|
||||
"a": "ALL",
|
||||
"m": "MY",
|
||||
"g": "GROUP",
|
||||
"n": "NONE",
|
||||
None: "-"
|
||||
}
|
||||
return labels.get(level, "-")
|
||||
|
||||
|
||||
def _getRoleScope(role: Dict[str, Any]) -> str:
|
||||
"""Determine the scope of a role."""
|
||||
if role.get("featureInstanceId"):
|
||||
return "instance"
|
||||
elif role.get("mandateId"):
|
||||
return "mandate"
|
||||
else:
|
||||
return "global"
|
||||
|
||||
|
||||
def _getRoleScopePriority(scope: str) -> int:
|
||||
"""Get priority for role scope (higher = more specific)."""
|
||||
priorities = {"global": 1, "mandate": 2, "instance": 3}
|
||||
return priorities.get(scope, 0)
|
||||
|
||||
|
||||
@router.get("/users", response_model=List[Dict[str, Any]])
|
||||
@limiter.limit("60/minute")
|
||||
async def listUsersForOverview(
|
||||
request: Request,
|
||||
currentUser: User = Depends(requireSysAdmin)
|
||||
) -> List[Dict[str, Any]]:
|
||||
"""
|
||||
Get list of all users for selection in the overview.
|
||||
MULTI-TENANT: SysAdmin-only.
|
||||
|
||||
Returns:
|
||||
- List of user dictionaries with basic info
|
||||
"""
|
||||
try:
|
||||
interface = getRootInterface()
|
||||
|
||||
# Get all users
|
||||
allUsersData = interface.db.getRecordset(UserInDB)
|
||||
|
||||
result = []
|
||||
for u in allUsersData:
|
||||
result.append({
|
||||
"id": u.get("id"),
|
||||
"username": u.get("username"),
|
||||
"email": u.get("email"),
|
||||
"fullName": u.get("fullName"),
|
||||
"isSysAdmin": u.get("isSysAdmin", False),
|
||||
"enabled": u.get("enabled", True),
|
||||
})
|
||||
|
||||
# Sort by username
|
||||
result.sort(key=lambda x: (x.get("username") or "").lower())
|
||||
|
||||
return result
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Error listing users for overview: {str(e)}")
|
||||
raise HTTPException(
|
||||
status_code=500,
|
||||
detail=f"Failed to list users: {str(e)}"
|
||||
)
|
||||
|
||||
|
||||
@router.get("/{userId}", response_model=Dict[str, Any])
|
||||
@limiter.limit("60/minute")
|
||||
async def getUserAccessOverview(
|
||||
request: Request,
|
||||
userId: str = Path(..., description="User ID to get access overview for"),
|
||||
mandateId: Optional[str] = Query(None, description="Filter by mandate ID"),
|
||||
featureInstanceId: Optional[str] = Query(None, description="Filter by feature instance ID"),
|
||||
currentUser: User = Depends(requireSysAdmin)
|
||||
) -> Dict[str, Any]:
|
||||
"""
|
||||
Get comprehensive access overview for a specific user.
|
||||
MULTI-TENANT: SysAdmin-only.
|
||||
|
||||
Path Parameters:
|
||||
- userId: User ID
|
||||
|
||||
Query Parameters:
|
||||
- mandateId: Optional filter by mandate ID
|
||||
- featureInstanceId: Optional filter by feature instance ID
|
||||
|
||||
Returns:
|
||||
- Comprehensive access overview including:
|
||||
- User info
|
||||
- All assigned roles with scope
|
||||
- UI access (what pages/views the user can see)
|
||||
- Data access (what tables/fields the user can access)
|
||||
- Resource access (what resources the user can use)
|
||||
"""
|
||||
try:
|
||||
interface = getRootInterface()
|
||||
|
||||
# Get user
|
||||
user = interface.getUser(userId)
|
||||
if not user:
|
||||
raise HTTPException(
|
||||
status_code=404,
|
||||
detail=f"User {userId} not found"
|
||||
)
|
||||
|
||||
# Build user info
|
||||
userInfo = {
|
||||
"id": user.id,
|
||||
"username": user.username,
|
||||
"email": user.email,
|
||||
"fullName": user.fullName,
|
||||
"isSysAdmin": user.isSysAdmin,
|
||||
"enabled": user.enabled,
|
||||
}
|
||||
|
||||
# If user is SysAdmin, they have full access to everything
|
||||
if user.isSysAdmin:
|
||||
return {
|
||||
"user": userInfo,
|
||||
"isSysAdmin": True,
|
||||
"sysAdminNote": "SysAdmin users have full access to all system-level resources without mandate context.",
|
||||
"roles": [],
|
||||
"mandates": [],
|
||||
"uiAccess": [],
|
||||
"dataAccess": [],
|
||||
"resourceAccess": [],
|
||||
}
|
||||
|
||||
# Collect all roles for the user
|
||||
allRoles = []
|
||||
roleIdToInfo = {} # Map roleId to role info for later reference
|
||||
|
||||
# Get mandates for this user
|
||||
mandateFilter = {"userId": userId, "enabled": True}
|
||||
if mandateId:
|
||||
mandateFilter["mandateId"] = mandateId
|
||||
|
||||
userMandates = interface.db.getRecordset(UserMandate, recordFilter=mandateFilter)
|
||||
|
||||
mandatesInfo = []
|
||||
for um in userMandates:
|
||||
umId = um.get("id")
|
||||
umMandateId = um.get("mandateId")
|
||||
|
||||
# Get mandate name
|
||||
mandate = interface.getMandate(umMandateId)
|
||||
mandateName = mandate.name if mandate else umMandateId
|
||||
|
||||
# Get roles for this UserMandate
|
||||
umRoles = interface.db.getRecordset(
|
||||
UserMandateRole,
|
||||
recordFilter={"userMandateId": umId}
|
||||
)
|
||||
|
||||
mandateRoleIds = []
|
||||
for umr in umRoles:
|
||||
roleId = umr.get("roleId")
|
||||
if roleId:
|
||||
mandateRoleIds.append(roleId)
|
||||
|
||||
# Get role details
|
||||
roleRecords = interface.db.getRecordset(Role, recordFilter={"id": roleId})
|
||||
if roleRecords:
|
||||
role = roleRecords[0]
|
||||
scope = _getRoleScope(role)
|
||||
roleInfo = {
|
||||
"id": roleId,
|
||||
"roleLabel": role.get("roleLabel"),
|
||||
"description": role.get("description", {}),
|
||||
"scope": scope,
|
||||
"scopePriority": _getRoleScopePriority(scope),
|
||||
"mandateId": role.get("mandateId"),
|
||||
"featureInstanceId": role.get("featureInstanceId"),
|
||||
"source": "mandate",
|
||||
"sourceMandateId": umMandateId,
|
||||
"sourceMandateName": mandateName,
|
||||
}
|
||||
allRoles.append(roleInfo)
|
||||
roleIdToInfo[roleId] = roleInfo
|
||||
|
||||
# Get feature instances for this mandate
|
||||
featureInstanceFilter = {"userId": userId, "enabled": True}
|
||||
featureAccesses = interface.db.getRecordset(FeatureAccess, recordFilter=featureInstanceFilter)
|
||||
|
||||
featureInstancesInfo = []
|
||||
for fa in featureAccesses:
|
||||
faId = fa.get("id")
|
||||
faInstanceId = fa.get("featureInstanceId")
|
||||
|
||||
# Check if instance belongs to this mandate
|
||||
instance = interface.db.getRecordset(FeatureInstance, recordFilter={"id": faInstanceId})
|
||||
if not instance:
|
||||
continue
|
||||
instance = instance[0]
|
||||
|
||||
if instance.get("mandateId") != umMandateId:
|
||||
continue
|
||||
|
||||
# Filter by featureInstanceId if specified
|
||||
if featureInstanceId and faInstanceId != featureInstanceId:
|
||||
continue
|
||||
|
||||
# Get feature info
|
||||
featureCode = instance.get("featureCode")
|
||||
featureRecords = interface.db.getRecordset(Feature, recordFilter={"code": featureCode})
|
||||
featureLabel = featureRecords[0].get("label", {}) if featureRecords else {}
|
||||
|
||||
# Get roles for this FeatureAccess
|
||||
faRoles = interface.db.getRecordset(
|
||||
FeatureAccessRole,
|
||||
recordFilter={"featureAccessId": faId}
|
||||
)
|
||||
|
||||
instanceRoleIds = []
|
||||
for far in faRoles:
|
||||
roleId = far.get("roleId")
|
||||
if roleId:
|
||||
instanceRoleIds.append(roleId)
|
||||
|
||||
# Get role details (if not already added)
|
||||
if roleId not in roleIdToInfo:
|
||||
roleRecords = interface.db.getRecordset(Role, recordFilter={"id": roleId})
|
||||
if roleRecords:
|
||||
role = roleRecords[0]
|
||||
scope = _getRoleScope(role)
|
||||
roleInfo = {
|
||||
"id": roleId,
|
||||
"roleLabel": role.get("roleLabel"),
|
||||
"description": role.get("description", {}),
|
||||
"scope": scope,
|
||||
"scopePriority": _getRoleScopePriority(scope),
|
||||
"mandateId": role.get("mandateId"),
|
||||
"featureInstanceId": role.get("featureInstanceId"),
|
||||
"source": "featureInstance",
|
||||
"sourceInstanceId": faInstanceId,
|
||||
"sourceInstanceLabel": instance.get("label"),
|
||||
}
|
||||
allRoles.append(roleInfo)
|
||||
roleIdToInfo[roleId] = roleInfo
|
||||
|
||||
featureInstancesInfo.append({
|
||||
"id": faInstanceId,
|
||||
"label": instance.get("label"),
|
||||
"featureCode": featureCode,
|
||||
"featureLabel": featureLabel,
|
||||
"roleIds": instanceRoleIds,
|
||||
})
|
||||
|
||||
mandatesInfo.append({
|
||||
"id": umMandateId,
|
||||
"name": mandateName,
|
||||
"roleIds": mandateRoleIds,
|
||||
"featureInstances": featureInstancesInfo,
|
||||
})
|
||||
|
||||
# Remove duplicate roles (keep most specific)
|
||||
uniqueRoles = {}
|
||||
for role in allRoles:
|
||||
roleId = role["id"]
|
||||
if roleId not in uniqueRoles or role["scopePriority"] > uniqueRoles[roleId]["scopePriority"]:
|
||||
uniqueRoles[roleId] = role
|
||||
|
||||
allRoles = list(uniqueRoles.values())
|
||||
|
||||
# Get all AccessRules for all role IDs
|
||||
allRoleIds = list(roleIdToInfo.keys())
|
||||
|
||||
# Collect access by context
|
||||
uiAccess = []
|
||||
dataAccess = []
|
||||
resourceAccess = []
|
||||
|
||||
for roleId in allRoleIds:
|
||||
roleInfo = roleIdToInfo.get(roleId, {})
|
||||
roleLabel = roleInfo.get("roleLabel", "unknown")
|
||||
roleScope = roleInfo.get("scope", "unknown")
|
||||
|
||||
# Get all rules for this role
|
||||
rules = interface.db.getRecordset(AccessRule, recordFilter={"roleId": roleId})
|
||||
|
||||
for rule in rules:
|
||||
context = rule.get("context")
|
||||
item = rule.get("item")
|
||||
|
||||
accessEntry = {
|
||||
"item": item or "(all)",
|
||||
"grantedByRoleId": roleId,
|
||||
"grantedByRoleLabel": roleLabel,
|
||||
"roleScope": roleScope,
|
||||
"scopePriority": roleInfo.get("scopePriority", 0),
|
||||
}
|
||||
|
||||
if context == "UI":
|
||||
accessEntry["view"] = rule.get("view", False)
|
||||
if accessEntry["view"]:
|
||||
uiAccess.append(accessEntry)
|
||||
|
||||
elif context == "DATA":
|
||||
accessEntry["view"] = rule.get("view", False)
|
||||
accessEntry["read"] = _getAccessLevelLabel(rule.get("read"))
|
||||
accessEntry["create"] = _getAccessLevelLabel(rule.get("create"))
|
||||
accessEntry["update"] = _getAccessLevelLabel(rule.get("update"))
|
||||
accessEntry["delete"] = _getAccessLevelLabel(rule.get("delete"))
|
||||
dataAccess.append(accessEntry)
|
||||
|
||||
elif context == "RESOURCE":
|
||||
accessEntry["view"] = rule.get("view", False)
|
||||
if accessEntry["view"]:
|
||||
resourceAccess.append(accessEntry)
|
||||
|
||||
# Merge and deduplicate access entries (keep highest priority)
|
||||
def _mergeAccessEntries(entries: List[Dict], isDataContext: bool = False) -> List[Dict]:
|
||||
"""Merge entries for same item, keeping highest priority."""
|
||||
merged = {}
|
||||
for entry in entries:
|
||||
item = entry["item"]
|
||||
priority = entry.get("scopePriority", 0)
|
||||
|
||||
if item not in merged or priority > merged[item].get("scopePriority", 0):
|
||||
merged[item] = entry
|
||||
elif item in merged and priority == merged[item].get("scopePriority", 0):
|
||||
# Same priority - merge grantedBy info
|
||||
existingRoles = merged[item].get("grantedByRoleLabels", [merged[item].get("grantedByRoleLabel")])
|
||||
if entry["grantedByRoleLabel"] not in existingRoles:
|
||||
existingRoles.append(entry["grantedByRoleLabel"])
|
||||
merged[item]["grantedByRoleLabels"] = existingRoles
|
||||
|
||||
# For DATA context, merge to most permissive
|
||||
if isDataContext:
|
||||
levelOrder = {"NONE": 0, "-": 0, "MY": 1, "GROUP": 2, "ALL": 3}
|
||||
for field in ["read", "create", "update", "delete"]:
|
||||
existingLevel = merged[item].get(field, "-")
|
||||
newLevel = entry.get(field, "-")
|
||||
if levelOrder.get(newLevel, 0) > levelOrder.get(existingLevel, 0):
|
||||
merged[item][field] = newLevel
|
||||
|
||||
# Clean up and sort
|
||||
result = list(merged.values())
|
||||
for entry in result:
|
||||
if "grantedByRoleLabels" not in entry:
|
||||
entry["grantedByRoleLabels"] = [entry.get("grantedByRoleLabel")]
|
||||
# Remove internal priority field from response
|
||||
entry.pop("scopePriority", None)
|
||||
|
||||
result.sort(key=lambda x: x.get("item", ""))
|
||||
return result
|
||||
|
||||
uiAccess = _mergeAccessEntries(uiAccess)
|
||||
dataAccess = _mergeAccessEntries(dataAccess, isDataContext=True)
|
||||
resourceAccess = _mergeAccessEntries(resourceAccess)
|
||||
|
||||
# Clean up roles for response
|
||||
for role in allRoles:
|
||||
role.pop("scopePriority", None)
|
||||
|
||||
# Sort roles by scope (instance > mandate > global) then by label
|
||||
allRoles.sort(key=lambda r: (-_getRoleScopePriority(r.get("scope", "")), r.get("roleLabel", "").lower()))
|
||||
|
||||
return {
|
||||
"user": userInfo,
|
||||
"isSysAdmin": False,
|
||||
"roles": allRoles,
|
||||
"mandates": mandatesInfo,
|
||||
"uiAccess": uiAccess,
|
||||
"dataAccess": dataAccess,
|
||||
"resourceAccess": resourceAccess,
|
||||
}
|
||||
|
||||
except HTTPException:
|
||||
raise
|
||||
except Exception as e:
|
||||
logger.error(f"Error getting user access overview: {str(e)}")
|
||||
raise HTTPException(
|
||||
status_code=500,
|
||||
detail=f"Failed to get user access overview: {str(e)}"
|
||||
)
|
||||
|
||||
|
||||
@router.get("/{userId}/effective-permissions", response_model=Dict[str, Any])
|
||||
@limiter.limit("60/minute")
|
||||
async def getEffectivePermissions(
|
||||
request: Request,
|
||||
userId: str = Path(..., description="User ID"),
|
||||
mandateId: str = Query(..., description="Mandate ID context"),
|
||||
featureInstanceId: Optional[str] = Query(None, description="Feature instance ID context"),
|
||||
context: str = Query("DATA", description="Context type: DATA, UI, or RESOURCE"),
|
||||
item: Optional[str] = Query(None, description="Specific item to check permissions for"),
|
||||
currentUser: User = Depends(requireSysAdmin)
|
||||
) -> Dict[str, Any]:
|
||||
"""
|
||||
Get effective (resolved) permissions for a user in a specific context.
|
||||
This uses the RBAC resolution logic to show what permissions actually apply.
|
||||
MULTI-TENANT: SysAdmin-only.
|
||||
|
||||
Path Parameters:
|
||||
- userId: User ID
|
||||
|
||||
Query Parameters:
|
||||
- mandateId: Required mandate context
|
||||
- featureInstanceId: Optional feature instance context
|
||||
- context: Permission context (DATA, UI, RESOURCE)
|
||||
- item: Optional specific item to check
|
||||
|
||||
Returns:
|
||||
- Effective permissions after RBAC resolution
|
||||
"""
|
||||
try:
|
||||
interface = getRootInterface()
|
||||
|
||||
# Get user
|
||||
user = interface.getUser(userId)
|
||||
if not user:
|
||||
raise HTTPException(
|
||||
status_code=404,
|
||||
detail=f"User {userId} not found"
|
||||
)
|
||||
|
||||
# Convert context string to enum
|
||||
try:
|
||||
contextEnum = AccessRuleContext(context)
|
||||
except ValueError:
|
||||
raise HTTPException(
|
||||
status_code=400,
|
||||
detail=f"Invalid context: {context}. Must be DATA, UI, or RESOURCE."
|
||||
)
|
||||
|
||||
# Use RBAC interface to get actual permissions
|
||||
from modules.security.rbac import RbacClass
|
||||
rbac = RbacClass(interface.db, dbApp=interface.db)
|
||||
|
||||
permissions = rbac.getUserPermissions(
|
||||
user=user,
|
||||
context=contextEnum,
|
||||
item=item or "",
|
||||
mandateId=mandateId,
|
||||
featureInstanceId=featureInstanceId
|
||||
)
|
||||
|
||||
return {
|
||||
"userId": userId,
|
||||
"mandateId": mandateId,
|
||||
"featureInstanceId": featureInstanceId,
|
||||
"context": context,
|
||||
"item": item,
|
||||
"effectivePermissions": {
|
||||
"view": permissions.view,
|
||||
"read": _getAccessLevelLabel(permissions.read.value if permissions.read else None),
|
||||
"create": _getAccessLevelLabel(permissions.create.value if permissions.create else None),
|
||||
"update": _getAccessLevelLabel(permissions.update.value if permissions.update else None),
|
||||
"delete": _getAccessLevelLabel(permissions.delete.value if permissions.delete else None),
|
||||
}
|
||||
}
|
||||
|
||||
except HTTPException:
|
||||
raise
|
||||
except Exception as e:
|
||||
logger.error(f"Error getting effective permissions: {str(e)}")
|
||||
raise HTTPException(
|
||||
status_code=500,
|
||||
detail=f"Failed to get effective permissions: {str(e)}"
|
||||
)
|
||||
|
|
@ -20,10 +20,11 @@ import json
|
|||
from pydantic import BaseModel, Field
|
||||
|
||||
from modules.auth import limiter, getCurrentUser
|
||||
from modules.datamodels.datamodelUam import User
|
||||
from modules.datamodels.datamodelUam import User, UserInDB, Mandate, UserConnection
|
||||
from modules.interfaces.interfaceDbApp import getRootInterface
|
||||
from modules.shared.timeUtils import getUtcTimestamp
|
||||
from modules.shared.auditLogger import audit_logger
|
||||
from modules.shared.gdprDeletion import deleteUserDataAcrossAllDatabases, buildDeletionSummary
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
|
@ -98,8 +99,7 @@ async def export_user_data(
|
|||
"id": str(currentUser.id),
|
||||
"username": currentUser.username,
|
||||
"email": currentUser.email,
|
||||
"firstname": currentUser.firstname,
|
||||
"lastname": currentUser.lastname,
|
||||
"fullName": getattr(currentUser, "fullName", None),
|
||||
"enabled": currentUser.enabled,
|
||||
"isSysAdmin": getattr(currentUser, "isSysAdmin", False),
|
||||
"createdAt": getattr(currentUser, "createdAt", None),
|
||||
|
|
@ -257,17 +257,11 @@ async def export_portable_data(
|
|||
"@context": "https://schema.org",
|
||||
"@type": "Person",
|
||||
"identifier": str(currentUser.id),
|
||||
"name": f"{currentUser.firstname or ''} {currentUser.lastname or ''}".strip() or currentUser.username,
|
||||
"name": getattr(currentUser, "fullName", None) or currentUser.username,
|
||||
"email": currentUser.email,
|
||||
"additionalProperty": []
|
||||
}
|
||||
|
||||
# Add profile properties
|
||||
if currentUser.firstname:
|
||||
portableData["givenName"] = currentUser.firstname
|
||||
if currentUser.lastname:
|
||||
portableData["familyName"] = currentUser.lastname
|
||||
|
||||
# Add mandate memberships as organization affiliations
|
||||
from modules.datamodels.datamodelMembership import UserMandate
|
||||
userMandates = rootInterface.db.getRecordset(
|
||||
|
|
@ -364,10 +358,17 @@ async def delete_account(
|
|||
)
|
||||
|
||||
try:
|
||||
rootInterface = getRootInterface()
|
||||
deletedData = []
|
||||
# Step 1: Audit log BEFORE deletion (audit needs userId)
|
||||
audit_logger.logGdprEvent(
|
||||
userId=str(currentUser.id),
|
||||
mandateId="system",
|
||||
action="gdpr_account_deletion_started",
|
||||
details=f"User initiated account deletion (GDPR Article 17 - Right to Erasure)",
|
||||
ipAddress=request.client.host if request.client else None
|
||||
)
|
||||
|
||||
# 1. Revoke all invitations created by user
|
||||
# Step 2: Revoke invitations BEFORE generic deletion (business logic)
|
||||
rootInterface = getRootInterface()
|
||||
from modules.datamodels.datamodelInvitation import Invitation
|
||||
userInvitations = rootInterface.db.getRecordset(
|
||||
Invitation,
|
||||
|
|
@ -375,78 +376,37 @@ async def delete_account(
|
|||
)
|
||||
|
||||
for inv in userInvitations:
|
||||
rootInterface.db.recordUpdate(
|
||||
rootInterface.db.recordModify(
|
||||
Invitation,
|
||||
inv.get("id"),
|
||||
{"revokedAt": getUtcTimestamp()}
|
||||
)
|
||||
deletedData.append(f"Invitations revoked: {len(userInvitations)}")
|
||||
|
||||
# 2. Delete feature accesses (CASCADE will delete FeatureAccessRoles)
|
||||
from modules.datamodels.datamodelMembership import FeatureAccess
|
||||
featureAccesses = rootInterface.db.getRecordset(
|
||||
FeatureAccess,
|
||||
recordFilter={"userId": str(currentUser.id)}
|
||||
)
|
||||
logger.info(f"Revoked {len(userInvitations)} invitations for user {currentUser.id}")
|
||||
|
||||
for fa in featureAccesses:
|
||||
rootInterface.db.recordDelete(FeatureAccess, fa.get("id"))
|
||||
deletedData.append(f"Feature accesses deleted: {len(featureAccesses)}")
|
||||
# Step 3: Generic deletion across ALL databases
|
||||
deletionStats = deleteUserDataAcrossAllDatabases(str(currentUser.id), currentUser)
|
||||
|
||||
# 3. Delete mandate memberships (CASCADE will delete UserMandateRoles)
|
||||
from modules.datamodels.datamodelMembership import UserMandate
|
||||
userMandates = rootInterface.db.getRecordset(
|
||||
UserMandate,
|
||||
recordFilter={"userId": str(currentUser.id)}
|
||||
)
|
||||
|
||||
for um in userMandates:
|
||||
rootInterface.db.recordDelete(UserMandate, um.get("id"))
|
||||
deletedData.append(f"Mandate memberships deleted: {len(userMandates)}")
|
||||
|
||||
# 4. Delete active tokens
|
||||
from modules.datamodels.datamodelSecurity import Token
|
||||
userTokens = rootInterface.db.getRecordset(
|
||||
Token,
|
||||
recordFilter={"userId": str(currentUser.id)}
|
||||
)
|
||||
|
||||
for token in userTokens:
|
||||
rootInterface.db.recordDelete(Token, token.get("id"))
|
||||
deletedData.append(f"Tokens deleted: {len(userTokens)}")
|
||||
|
||||
# 5. Delete user connections (OAuth)
|
||||
userConnections = rootInterface.db.getRecordset(
|
||||
UserConnection,
|
||||
recordFilter={"userId": str(currentUser.id)}
|
||||
)
|
||||
|
||||
for conn in userConnections:
|
||||
rootInterface.db.recordDelete(UserConnection, conn.get("id"))
|
||||
deletedData.append(f"Connections deleted: {len(userConnections)}")
|
||||
|
||||
# 6. Finally, delete the user
|
||||
# Step 4: Delete the user account from UserInDB (authentication table)
|
||||
# This must be done AFTER all other deletions to maintain audit trail
|
||||
deletedAt = getUtcTimestamp()
|
||||
rootInterface.db.recordDelete(User, str(currentUser.id))
|
||||
deletedData.append("User account deleted")
|
||||
rootInterface.db.recordDelete(UserInDB, str(currentUser.id))
|
||||
|
||||
# Audit log (before user is deleted) - GDPR Article 17 account deletion
|
||||
audit_logger.logGdprEvent(
|
||||
userId=str(currentUser.id),
|
||||
mandateId="system",
|
||||
action="gdpr_account_deletion",
|
||||
details=f"User deleted own account (GDPR Article 17 - Right to Erasure). Data: {', '.join(deletedData)}",
|
||||
ipAddress=request.client.host if request.client else None
|
||||
)
|
||||
# Build summary for response
|
||||
deletedData = buildDeletionSummary(deletionStats)
|
||||
deletedData.insert(0, f"Invitations revoked: {len(userInvitations)}")
|
||||
deletedData.append("User account deleted from authentication system")
|
||||
|
||||
logger.info(f"User {currentUser.id} deleted own account (GDPR Art. 17)")
|
||||
logger.info(f"User {currentUser.id} deleted own account (GDPR Art. 17). "
|
||||
f"Stats: {deletionStats['totalRecordsDeleted']} deleted, "
|
||||
f"{deletionStats['totalRecordsAnonymized']} anonymized")
|
||||
|
||||
return DeletionResult(
|
||||
success=True,
|
||||
userId=str(currentUser.id),
|
||||
deletedAt=deletedAt,
|
||||
deletedData=deletedData,
|
||||
message="Account and all associated data have been permanently deleted."
|
||||
message="Account and all associated data have been permanently deleted or anonymized."
|
||||
)
|
||||
|
||||
except HTTPException:
|
||||
|
|
|
|||
|
|
@ -82,26 +82,6 @@ class InvitationValidation(BaseModel):
|
|||
roleIds: List[str]
|
||||
|
||||
|
||||
class RegisterAndAcceptRequest(BaseModel):
|
||||
"""Request model for combined registration + invitation acceptance"""
|
||||
token: str = Field(..., description="Invitation token")
|
||||
username: str = Field(..., min_length=3, max_length=50, description="Username for the new account")
|
||||
email: str = Field(..., description="Email address")
|
||||
password: str = Field(..., min_length=8, description="Password (min 8 characters)")
|
||||
firstname: Optional[str] = Field(None, description="First name")
|
||||
lastname: Optional[str] = Field(None, description="Last name")
|
||||
|
||||
|
||||
class RegisterAndAcceptResponse(BaseModel):
|
||||
"""Response model for combined registration + invitation acceptance"""
|
||||
message: str
|
||||
userId: str
|
||||
mandateId: str
|
||||
userMandateId: str
|
||||
featureAccessId: Optional[str]
|
||||
roleIds: List[str]
|
||||
|
||||
|
||||
# =============================================================================
|
||||
# Invitation CRUD Endpoints
|
||||
# =============================================================================
|
||||
|
|
@ -371,7 +351,7 @@ async def revoke_invitation(
|
|||
)
|
||||
|
||||
# Revoke invitation
|
||||
rootInterface.db.recordUpdate(
|
||||
rootInterface.db.recordModify(
|
||||
Invitation,
|
||||
invitationId,
|
||||
{"revokedAt": getUtcTimestamp()}
|
||||
|
|
@ -575,7 +555,7 @@ async def accept_invitation(
|
|||
featureAccessId = str(featureAccess.id)
|
||||
|
||||
# Update invitation usage
|
||||
rootInterface.db.recordUpdate(
|
||||
rootInterface.db.recordModify(
|
||||
Invitation,
|
||||
invitation.get("id"),
|
||||
{
|
||||
|
|
@ -608,161 +588,6 @@ async def accept_invitation(
|
|||
)
|
||||
|
||||
|
||||
# =============================================================================
|
||||
# Combined Registration + Accept Invitation
|
||||
# =============================================================================
|
||||
|
||||
@router.post("/register-and-accept", response_model=RegisterAndAcceptResponse)
|
||||
@limiter.limit("10/minute") # Stricter rate limit for registration
|
||||
async def register_and_accept_invitation(
|
||||
request: Request,
|
||||
data: RegisterAndAcceptRequest
|
||||
) -> RegisterAndAcceptResponse:
|
||||
"""
|
||||
Combined endpoint: Register a new user AND accept an invitation in one step.
|
||||
|
||||
This is a PUBLIC endpoint - no authentication required.
|
||||
|
||||
Flow:
|
||||
1. Validate invitation token
|
||||
2. Check email matches (if invitation has email restriction)
|
||||
3. Create new user account
|
||||
4. Create UserMandate membership with roles
|
||||
5. Optionally grant FeatureAccess
|
||||
6. Update invitation usage
|
||||
|
||||
The user can then login with their new credentials.
|
||||
"""
|
||||
try:
|
||||
rootInterface = getRootInterface()
|
||||
|
||||
# 1. Validate invitation
|
||||
invitation = rootInterface.getInvitationByToken(data.token)
|
||||
if not invitation:
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_404_NOT_FOUND,
|
||||
detail="Invalid invitation token"
|
||||
)
|
||||
|
||||
if invitation.get("revokedAt"):
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_400_BAD_REQUEST,
|
||||
detail="Invitation has been revoked"
|
||||
)
|
||||
|
||||
currentTime = getUtcTimestamp()
|
||||
if invitation.get("expiresAt", 0) < currentTime:
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_400_BAD_REQUEST,
|
||||
detail="Invitation has expired"
|
||||
)
|
||||
|
||||
if invitation.get("currentUses", 0) >= invitation.get("maxUses", 1):
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_400_BAD_REQUEST,
|
||||
detail="Invitation has reached maximum uses"
|
||||
)
|
||||
|
||||
# 2. Check email restriction
|
||||
invitationEmail = invitation.get("email")
|
||||
if invitationEmail and invitationEmail.lower() != data.email.lower():
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_400_BAD_REQUEST,
|
||||
detail="Email does not match the invitation"
|
||||
)
|
||||
|
||||
# 3. Check if username or email already exists
|
||||
existingUsername = rootInterface.getUserByUsername(data.username)
|
||||
if existingUsername:
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_409_CONFLICT,
|
||||
detail="Username already exists"
|
||||
)
|
||||
|
||||
existingEmail = rootInterface.getUserByEmail(data.email)
|
||||
if existingEmail:
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_409_CONFLICT,
|
||||
detail="Email already registered. Please login and accept the invitation."
|
||||
)
|
||||
|
||||
# 4. Create new user
|
||||
from modules.security.passwordUtils import hashPassword
|
||||
hashedPassword = hashPassword(data.password)
|
||||
|
||||
newUser = rootInterface.createUser(
|
||||
username=data.username,
|
||||
email=data.email,
|
||||
passwordHash=hashedPassword,
|
||||
firstname=data.firstname,
|
||||
lastname=data.lastname
|
||||
)
|
||||
|
||||
if not newUser:
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,
|
||||
detail="Failed to create user account"
|
||||
)
|
||||
|
||||
userId = str(newUser.id)
|
||||
mandateId = invitation.get("mandateId")
|
||||
roleIds = invitation.get("roleIds", [])
|
||||
featureInstanceId = invitation.get("featureInstanceId")
|
||||
|
||||
# 5. Create UserMandate membership
|
||||
userMandate = rootInterface.createUserMandate(
|
||||
userId=userId,
|
||||
mandateId=mandateId,
|
||||
roleIds=roleIds
|
||||
)
|
||||
userMandateId = str(userMandate.id)
|
||||
|
||||
# 6. Grant feature access if specified
|
||||
featureAccessId = None
|
||||
if featureInstanceId:
|
||||
instanceRoleIds = [r for r in roleIds if _isInstanceRole(rootInterface, r, featureInstanceId)]
|
||||
featureAccess = rootInterface.createFeatureAccess(
|
||||
userId=userId,
|
||||
featureInstanceId=featureInstanceId,
|
||||
roleIds=instanceRoleIds
|
||||
)
|
||||
featureAccessId = str(featureAccess.id)
|
||||
|
||||
# 7. Update invitation usage
|
||||
rootInterface.db.recordUpdate(
|
||||
Invitation,
|
||||
invitation.get("id"),
|
||||
{
|
||||
"currentUses": invitation.get("currentUses", 0) + 1,
|
||||
"usedBy": userId,
|
||||
"usedAt": currentTime
|
||||
}
|
||||
)
|
||||
|
||||
logger.info(
|
||||
f"New user {userId} registered and accepted invitation {invitation.get('id')} "
|
||||
f"for mandate {mandateId}"
|
||||
)
|
||||
|
||||
return RegisterAndAcceptResponse(
|
||||
message="Account created and invitation accepted successfully",
|
||||
userId=userId,
|
||||
mandateId=mandateId,
|
||||
userMandateId=userMandateId,
|
||||
featureAccessId=featureAccessId,
|
||||
roleIds=roleIds
|
||||
)
|
||||
|
||||
except HTTPException:
|
||||
raise
|
||||
except Exception as e:
|
||||
logger.error(f"Error in register-and-accept: {e}")
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,
|
||||
detail=f"Failed to complete registration: {str(e)}"
|
||||
)
|
||||
|
||||
|
||||
# =============================================================================
|
||||
# Helper Functions
|
||||
# =============================================================================
|
||||
|
|
|
|||
|
|
@ -146,3 +146,108 @@ async def list_sharepoint_folders(
|
|||
detail=f"Error listing SharePoint folders: {str(e)}"
|
||||
)
|
||||
|
||||
|
||||
@router.get("/{connectionId}/folder-options", response_model=List[Dict[str, Any]])
|
||||
@limiter.limit("30/minute")
|
||||
async def getSharepointFolderOptions(
|
||||
request: Request,
|
||||
connectionId: str = Path(..., description="Microsoft connection ID"),
|
||||
siteId: Optional[str] = Query(None, description="Specific site ID to browse (if omitted, returns sites only)"),
|
||||
path: Optional[str] = Query(None, description="Folder path within site to browse"),
|
||||
currentUser: User = Depends(getCurrentUser)
|
||||
) -> List[Dict[str, Any]]:
|
||||
"""
|
||||
Get SharePoint folders formatted as dropdown options.
|
||||
|
||||
Two modes:
|
||||
1. If siteId is not provided: Returns list of sites (for site selection)
|
||||
2. If siteId is provided: Returns folders within that site (optionally at specific path)
|
||||
|
||||
This avoids expensive iteration through all sites and folders.
|
||||
"""
|
||||
try:
|
||||
interface = getInterface(currentUser)
|
||||
|
||||
# Get the connection and verify it belongs to the user
|
||||
connection = _getUserConnection(interface, connectionId, currentUser.id)
|
||||
if not connection:
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_404_NOT_FOUND,
|
||||
detail=f"Connection {connectionId} not found or does not belong to user"
|
||||
)
|
||||
|
||||
# Verify it's a Microsoft connection
|
||||
authority = connection.authority.value if hasattr(connection.authority, 'value') else str(connection.authority)
|
||||
if authority.lower() != 'msft':
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_400_BAD_REQUEST,
|
||||
detail=f"Connection {connectionId} is not a Microsoft connection"
|
||||
)
|
||||
|
||||
# Initialize services
|
||||
services = getServices(currentUser, None)
|
||||
|
||||
# Set access token on SharePoint service
|
||||
if not services.sharepoint.setAccessTokenFromConnection(connection):
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_401_UNAUTHORIZED,
|
||||
detail="Failed to set SharePoint access token. Connection may be expired or invalid."
|
||||
)
|
||||
|
||||
# Mode 1: Return sites list if no siteId specified
|
||||
if not siteId:
|
||||
sites = await services.sharepoint.discoverSites()
|
||||
return [
|
||||
{
|
||||
"type": "site",
|
||||
"value": site.get("id"),
|
||||
"label": site.get("displayName", "Unknown Site"),
|
||||
"siteId": site.get("id"),
|
||||
"siteName": site.get("displayName", "Unknown Site"),
|
||||
"webUrl": site.get("webUrl", ""),
|
||||
"path": _extractSitePath(site.get("webUrl", ""))
|
||||
}
|
||||
for site in sites
|
||||
]
|
||||
|
||||
# Mode 2: Return folders within specific site
|
||||
folderPath = path or ""
|
||||
items = await services.sharepoint.listFolderContents(siteId, folderPath)
|
||||
|
||||
if not items:
|
||||
return []
|
||||
|
||||
folderOptions = []
|
||||
for item in items:
|
||||
if item.get("type") == "folder":
|
||||
folderName = item.get("name", "")
|
||||
itemPath = f"{folderPath}/{folderName}" if folderPath else folderName
|
||||
|
||||
folderOptions.append({
|
||||
"type": "folder",
|
||||
"value": itemPath,
|
||||
"label": folderName,
|
||||
"siteId": siteId,
|
||||
"folderName": folderName,
|
||||
"path": itemPath,
|
||||
"hasChildren": True # Assume folders may have children
|
||||
})
|
||||
|
||||
return folderOptions
|
||||
|
||||
except HTTPException:
|
||||
raise
|
||||
except Exception as e:
|
||||
logger.error(f"Error getting SharePoint folder options: {str(e)}")
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,
|
||||
detail=f"Error getting SharePoint folder options: {str(e)}"
|
||||
)
|
||||
|
||||
|
||||
def _extractSitePath(webUrl: str) -> str:
|
||||
"""Extract site path from webUrl (e.g., https://company.sharepoint.com/sites/MySite -> /sites/MySite)"""
|
||||
if "/sites/" in webUrl:
|
||||
return "/sites/" + webUrl.split("/sites/")[1].split("/")[0]
|
||||
return ""
|
||||
|
||||
|
|
|
|||
|
|
@ -381,10 +381,20 @@ class RbacClass:
|
|||
"""
|
||||
Check if a rule matches the given item.
|
||||
|
||||
Matching rules (in order of specificity):
|
||||
1. Generic rule (item=None) matches everything
|
||||
2. Exact match (rule.item == item)
|
||||
3. Prefix match (item starts with rule.item + ".")
|
||||
Example: rule "data.feature.trustee" matches item "data.feature.trustee.TrusteePosition"
|
||||
|
||||
All items MUST use the full objectKey format:
|
||||
- System: data.system.{TableName} (e.g., "data.system.UserInDB")
|
||||
- Feature: data.feature.{featureCode}.{TableName} (e.g., "data.feature.trustee.TrusteePosition")
|
||||
- UI: ui.{area}.{page} (e.g., "ui.admin.users")
|
||||
|
||||
Args:
|
||||
rule: Access rule to check
|
||||
item: Item to match against (can be short name like "TrusteePosition" or
|
||||
fully qualified like "data.feature.trustee.TrusteePosition")
|
||||
item: Full objectKey to match against
|
||||
|
||||
Returns:
|
||||
True if rule matches item
|
||||
|
|
@ -401,15 +411,10 @@ class RbacClass:
|
|||
if rule.item == item:
|
||||
return True
|
||||
|
||||
# Prefix match (e.g., "trustee" matches "trustee.contract")
|
||||
# Prefix match (e.g., "data.feature.trustee" matches "data.feature.trustee.TrusteePosition")
|
||||
if item.startswith(rule.item + "."):
|
||||
return True
|
||||
|
||||
# Suffix match: rule.item ends with ".{item}" (e.g., "data.feature.trustee.TrusteePosition" matches "TrusteePosition")
|
||||
# This allows short table names to match fully qualified objectKeys
|
||||
if rule.item.endswith("." + item):
|
||||
return True
|
||||
|
||||
return False
|
||||
|
||||
def findMostSpecificRule(self, rules: List[AccessRule], item: str) -> Optional[AccessRule]:
|
||||
|
|
|
|||
679
modules/shared/gdprDeletion.py
Normal file
679
modules/shared/gdprDeletion.py
Normal file
|
|
@ -0,0 +1,679 @@
|
|||
# Copyright (c) 2025 Patrick Motsch
|
||||
# All rights reserved.
|
||||
"""
|
||||
Generic GDPR data deletion engine.
|
||||
Automatically discovers and deletes user data across all databases and tables.
|
||||
|
||||
Design:
|
||||
- Schema-based: Inspects database schemas to find user-related columns
|
||||
- Generic: Works with any new features/tables without code changes
|
||||
- Safe: Anonymizes audit logs instead of deleting them
|
||||
- Comprehensive: Covers all databases (App, Management, Chat, Feature-DBs)
|
||||
"""
|
||||
|
||||
import logging
|
||||
from typing import List, Dict, Any, Set, Tuple
|
||||
from modules.shared.timeUtils import getUtcTimestamp
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
# Tables to SKIP (never delete from these)
|
||||
PROTECTED_TABLES = {
|
||||
"_system", # System metadata table
|
||||
"UserInDB", # User account table (deleted separately at the end)
|
||||
}
|
||||
|
||||
# Tables to ANONYMIZE instead of DELETE (for compliance)
|
||||
ANONYMIZE_TABLES = {
|
||||
"AuditEvent", # Audit logs must be retained for compliance
|
||||
"AuthEvent", # Authentication logs must be retained for compliance
|
||||
}
|
||||
|
||||
# User reference column patterns to search for
|
||||
USER_COLUMNS = [
|
||||
"userId",
|
||||
"createdBy",
|
||||
"usedBy",
|
||||
"revokedBy",
|
||||
"_createdBy",
|
||||
"_modifiedBy",
|
||||
]
|
||||
|
||||
|
||||
def _getTableColumns(dbConnector, tableName: str) -> List[str]:
|
||||
"""
|
||||
Get all column names for a table by inspecting the schema.
|
||||
|
||||
Args:
|
||||
dbConnector: DatabaseConnector instance
|
||||
tableName: Name of the table
|
||||
|
||||
Returns:
|
||||
List of column names
|
||||
"""
|
||||
try:
|
||||
query = """
|
||||
SELECT column_name
|
||||
FROM information_schema.columns
|
||||
WHERE table_name = %s
|
||||
AND table_schema = 'public'
|
||||
ORDER BY ordinal_position
|
||||
"""
|
||||
|
||||
cursor = dbConnector.connection.cursor()
|
||||
cursor.execute(query, (tableName,))
|
||||
columns = [row[0] for row in cursor.fetchall()]
|
||||
cursor.close()
|
||||
|
||||
return columns
|
||||
except Exception as e:
|
||||
logger.error(f"Error getting columns for table {tableName}: {e}")
|
||||
return []
|
||||
|
||||
|
||||
def _getAllTables(dbConnector) -> List[str]:
|
||||
"""
|
||||
Get all table names from a database, sorted by dependency order.
|
||||
Child tables (with foreign keys) come before parent tables.
|
||||
|
||||
Args:
|
||||
dbConnector: DatabaseConnector instance
|
||||
|
||||
Returns:
|
||||
List of table names in deletion order
|
||||
"""
|
||||
try:
|
||||
# Get all tables
|
||||
query = """
|
||||
SELECT table_name
|
||||
FROM information_schema.tables
|
||||
WHERE table_schema = 'public'
|
||||
AND table_type = 'BASE TABLE'
|
||||
ORDER BY table_name
|
||||
"""
|
||||
|
||||
cursor = dbConnector.connection.cursor()
|
||||
cursor.execute(query)
|
||||
allTables = [row[0] for row in cursor.fetchall()]
|
||||
|
||||
# Get foreign key relationships to determine dependency order
|
||||
fkQuery = """
|
||||
SELECT
|
||||
tc.table_name,
|
||||
ccu.table_name AS foreign_table_name
|
||||
FROM information_schema.table_constraints AS tc
|
||||
JOIN information_schema.key_column_usage AS kcu
|
||||
ON tc.constraint_name = kcu.constraint_name
|
||||
AND tc.table_schema = kcu.table_schema
|
||||
JOIN information_schema.constraint_column_usage AS ccu
|
||||
ON ccu.constraint_name = tc.constraint_name
|
||||
AND ccu.table_schema = tc.table_schema
|
||||
WHERE tc.constraint_type = 'FOREIGN KEY'
|
||||
AND tc.table_schema = 'public'
|
||||
"""
|
||||
|
||||
cursor.execute(fkQuery)
|
||||
foreignKeys = cursor.fetchall()
|
||||
cursor.close()
|
||||
|
||||
# Build dependency graph (child -> parent mapping)
|
||||
dependencies = {}
|
||||
for childTable, parentTable in foreignKeys:
|
||||
if childTable not in dependencies:
|
||||
dependencies[childTable] = []
|
||||
dependencies[childTable].append(parentTable)
|
||||
|
||||
# Sort tables by dependency (topological sort)
|
||||
sortedTables = []
|
||||
visited = set()
|
||||
|
||||
def visit(table):
|
||||
if table in visited or table not in allTables:
|
||||
return
|
||||
visited.add(table)
|
||||
|
||||
# Visit dependencies first (parents)
|
||||
if table in dependencies:
|
||||
for parent in dependencies[table]:
|
||||
visit(parent)
|
||||
|
||||
sortedTables.append(table)
|
||||
|
||||
# Visit all tables
|
||||
for table in allTables:
|
||||
visit(table)
|
||||
|
||||
# Reverse to get deletion order (children before parents)
|
||||
sortedTables.reverse()
|
||||
|
||||
# Filter out protected tables
|
||||
return [t for t in sortedTables if t not in PROTECTED_TABLES]
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Error getting tables from database: {e}")
|
||||
# Fallback: return simple list without ordering
|
||||
try:
|
||||
query = "SELECT table_name FROM information_schema.tables WHERE table_schema = 'public' AND table_type = 'BASE TABLE'"
|
||||
cursor = dbConnector.connection.cursor()
|
||||
cursor.execute(query)
|
||||
tables = [row[0] for row in cursor.fetchall()]
|
||||
cursor.close()
|
||||
return [t for t in tables if t not in PROTECTED_TABLES]
|
||||
except Exception:
|
||||
return []
|
||||
|
||||
|
||||
def _getPrimaryKeyColumns(dbConnector, tableName: str) -> List[str]:
|
||||
"""
|
||||
Get primary key column(s) for a table.
|
||||
|
||||
Args:
|
||||
dbConnector: DatabaseConnector instance
|
||||
tableName: Name of the table
|
||||
|
||||
Returns:
|
||||
List of primary key column names
|
||||
"""
|
||||
try:
|
||||
query = """
|
||||
SELECT a.attname
|
||||
FROM pg_index i
|
||||
JOIN pg_attribute a ON a.attrelid = i.indrelid
|
||||
AND a.attnum = ANY(i.indkey)
|
||||
WHERE i.indrelid = %s::regclass
|
||||
AND i.indisprimary
|
||||
"""
|
||||
|
||||
cursor = dbConnector.connection.cursor()
|
||||
cursor.execute(query, (tableName,))
|
||||
pkColumns = [row[0] for row in cursor.fetchall()]
|
||||
cursor.close()
|
||||
|
||||
return pkColumns
|
||||
except Exception as e:
|
||||
logger.debug(f"Could not get primary key for {tableName}: {e}")
|
||||
return ["id"] # Fallback to 'id'
|
||||
|
||||
|
||||
def _findUserReferencesInTable(
|
||||
dbConnector,
|
||||
tableName: str,
|
||||
userId: str
|
||||
) -> Dict[str, List[Tuple]]:
|
||||
"""
|
||||
Find all records in a table that reference a user.
|
||||
|
||||
Args:
|
||||
dbConnector: DatabaseConnector instance
|
||||
tableName: Name of the table
|
||||
userId: User ID to search for
|
||||
|
||||
Returns:
|
||||
Dict mapping column names to lists of primary key tuples
|
||||
"""
|
||||
try:
|
||||
# Get all columns for this table
|
||||
columns = _getTableColumns(dbConnector, tableName)
|
||||
|
||||
# Find user-related columns in this table
|
||||
userColumns = [col for col in columns if col in USER_COLUMNS]
|
||||
|
||||
if not userColumns:
|
||||
return {}
|
||||
|
||||
# Get primary key columns
|
||||
pkColumns = _getPrimaryKeyColumns(dbConnector, tableName)
|
||||
|
||||
if not pkColumns:
|
||||
logger.warning(f"Table {tableName} has no primary key, skipping")
|
||||
return {}
|
||||
|
||||
references = {}
|
||||
cursor = dbConnector.connection.cursor()
|
||||
|
||||
for userColumn in userColumns:
|
||||
# Build SELECT for primary key columns
|
||||
pkSelect = ", ".join([f'"{pk}"' for pk in pkColumns])
|
||||
query = f'SELECT {pkSelect} FROM "{tableName}" WHERE "{userColumn}" = %s'
|
||||
|
||||
cursor.execute(query, (userId,))
|
||||
recordKeys = cursor.fetchall()
|
||||
|
||||
if recordKeys:
|
||||
references[userColumn] = recordKeys
|
||||
logger.debug(f"Found {len(recordKeys)} records in {tableName}.{userColumn} for user {userId}")
|
||||
|
||||
cursor.close()
|
||||
return references
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Error finding user references in {tableName}: {e}")
|
||||
return {}
|
||||
|
||||
|
||||
def _anonymizeRecords(
|
||||
dbConnector,
|
||||
tableName: str,
|
||||
columnName: str,
|
||||
recordKeys: List[Tuple],
|
||||
pkColumns: List[str],
|
||||
anonymousValue: str = "deleted_user"
|
||||
) -> int:
|
||||
"""
|
||||
Anonymize user references in records (set to 'deleted_user').
|
||||
|
||||
Args:
|
||||
dbConnector: DatabaseConnector instance
|
||||
tableName: Name of the table
|
||||
columnName: Name of the column to anonymize
|
||||
recordKeys: List of primary key tuples
|
||||
pkColumns: List of primary key column names
|
||||
anonymousValue: Value to set (default: "deleted_user")
|
||||
|
||||
Returns:
|
||||
Number of records anonymized
|
||||
"""
|
||||
if not recordKeys:
|
||||
return 0
|
||||
|
||||
try:
|
||||
cursor = dbConnector.connection.cursor()
|
||||
count = 0
|
||||
|
||||
for recordKey in recordKeys:
|
||||
# Build WHERE clause for primary key
|
||||
whereClause = " AND ".join([f'"{pk}" = %s' for pk in pkColumns])
|
||||
|
||||
# Check if table has _modifiedAt column
|
||||
columns = _getTableColumns(dbConnector, tableName)
|
||||
hasModifiedAt = "_modifiedAt" in columns
|
||||
|
||||
if hasModifiedAt:
|
||||
query = f'UPDATE "{tableName}" SET "{columnName}" = %s, "_modifiedAt" = %s WHERE {whereClause}'
|
||||
params = [anonymousValue, getUtcTimestamp()]
|
||||
else:
|
||||
query = f'UPDATE "{tableName}" SET "{columnName}" = %s WHERE {whereClause}'
|
||||
params = [anonymousValue]
|
||||
|
||||
# Add primary key values to params
|
||||
if isinstance(recordKey, tuple):
|
||||
params.extend(recordKey)
|
||||
else:
|
||||
params.append(recordKey)
|
||||
|
||||
cursor.execute(query, params)
|
||||
count += cursor.rowcount
|
||||
|
||||
dbConnector.connection.commit()
|
||||
cursor.close()
|
||||
|
||||
logger.info(f"Anonymized {count} records in {tableName}.{columnName}")
|
||||
return count
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Error anonymizing records in {tableName}.{columnName}: {e}")
|
||||
dbConnector.connection.rollback()
|
||||
return 0
|
||||
|
||||
|
||||
def _deleteRecords(
|
||||
dbConnector,
|
||||
tableName: str,
|
||||
recordKeys: List[Tuple],
|
||||
pkColumns: List[str]
|
||||
) -> int:
|
||||
"""
|
||||
Delete records from a table.
|
||||
|
||||
Args:
|
||||
dbConnector: DatabaseConnector instance
|
||||
tableName: Name of the table
|
||||
recordKeys: List of primary key tuples
|
||||
pkColumns: List of primary key column names
|
||||
|
||||
Returns:
|
||||
Number of records deleted
|
||||
"""
|
||||
if not recordKeys:
|
||||
return 0
|
||||
|
||||
try:
|
||||
cursor = dbConnector.connection.cursor()
|
||||
count = 0
|
||||
|
||||
for recordKey in recordKeys:
|
||||
# Build WHERE clause for primary key
|
||||
whereClause = " AND ".join([f'"{pk}" = %s' for pk in pkColumns])
|
||||
query = f'DELETE FROM "{tableName}" WHERE {whereClause}'
|
||||
|
||||
# Prepare params
|
||||
if isinstance(recordKey, tuple):
|
||||
params = list(recordKey)
|
||||
else:
|
||||
params = [recordKey]
|
||||
|
||||
cursor.execute(query, params)
|
||||
count += cursor.rowcount
|
||||
|
||||
dbConnector.connection.commit()
|
||||
cursor.close()
|
||||
|
||||
logger.info(f"Deleted {count} records from {tableName}")
|
||||
return count
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Error deleting records from {tableName}: {e}")
|
||||
dbConnector.connection.rollback()
|
||||
return 0
|
||||
|
||||
|
||||
def deleteUserDataFromDatabase(
|
||||
dbConnector,
|
||||
userId: str,
|
||||
databaseName: str
|
||||
) -> Dict[str, Any]:
|
||||
"""
|
||||
Delete or anonymize all user data from a single database.
|
||||
|
||||
Args:
|
||||
dbConnector: DatabaseConnector instance
|
||||
userId: User ID to delete
|
||||
databaseName: Name of the database (for logging)
|
||||
|
||||
Returns:
|
||||
Dict with deletion statistics
|
||||
"""
|
||||
stats = {
|
||||
"database": databaseName,
|
||||
"tablesProcessed": 0,
|
||||
"recordsDeleted": 0,
|
||||
"recordsAnonymized": 0,
|
||||
"errors": []
|
||||
}
|
||||
|
||||
try:
|
||||
# Get all tables in this database
|
||||
tables = _getAllTables(dbConnector)
|
||||
|
||||
logger.info(f"Processing {len(tables)} tables in {databaseName} for user {userId}")
|
||||
|
||||
for tableName in tables:
|
||||
try:
|
||||
# Get primary key columns for this table
|
||||
pkColumns = _getPrimaryKeyColumns(dbConnector, tableName)
|
||||
|
||||
if not pkColumns:
|
||||
logger.debug(f"Skipping {tableName} - no primary key")
|
||||
continue
|
||||
|
||||
# Find user references in this table
|
||||
references = _findUserReferencesInTable(dbConnector, tableName, userId)
|
||||
|
||||
if not references:
|
||||
continue
|
||||
|
||||
stats["tablesProcessed"] += 1
|
||||
|
||||
# Decide: Anonymize or Delete?
|
||||
shouldAnonymize = tableName in ANONYMIZE_TABLES
|
||||
|
||||
for columnName, recordKeys in references.items():
|
||||
if shouldAnonymize:
|
||||
# Anonymize audit/log tables
|
||||
count = _anonymizeRecords(
|
||||
dbConnector, tableName, columnName, recordKeys, pkColumns
|
||||
)
|
||||
stats["recordsAnonymized"] += count
|
||||
else:
|
||||
# Delete from regular tables
|
||||
count = _deleteRecords(dbConnector, tableName, recordKeys, pkColumns)
|
||||
stats["recordsDeleted"] += count
|
||||
|
||||
except Exception as tableErr:
|
||||
errorMsg = f"Error processing table {tableName}: {tableErr}"
|
||||
logger.error(errorMsg)
|
||||
stats["errors"].append(errorMsg)
|
||||
|
||||
logger.info(f"Completed deletion in {databaseName}: {stats}")
|
||||
return stats
|
||||
|
||||
except Exception as e:
|
||||
errorMsg = f"Error processing database {databaseName}: {e}"
|
||||
logger.error(errorMsg)
|
||||
stats["errors"].append(errorMsg)
|
||||
return stats
|
||||
|
||||
|
||||
def deleteUserDataAcrossAllDatabases(userId: str, currentUser) -> Dict[str, Any]:
|
||||
"""
|
||||
Delete or anonymize all user data across ALL databases.
|
||||
|
||||
This is the main entry point for GDPR Article 17 (Right to Erasure).
|
||||
|
||||
Features:
|
||||
- Automatically discovers all databases and tables
|
||||
- Schema-based: No hardcoded table lists
|
||||
- Safe: Anonymizes audit logs instead of deleting them
|
||||
- Comprehensive: Covers App, Management, Chat, and all Feature DBs
|
||||
|
||||
Args:
|
||||
userId: User ID to delete
|
||||
currentUser: User object (for interface access)
|
||||
|
||||
Returns:
|
||||
Dict with comprehensive deletion statistics
|
||||
"""
|
||||
allStats = {
|
||||
"userId": userId,
|
||||
"deletedAt": getUtcTimestamp(),
|
||||
"databases": [],
|
||||
"totalTablesProcessed": 0,
|
||||
"totalRecordsDeleted": 0,
|
||||
"totalRecordsAnonymized": 0,
|
||||
"errors": []
|
||||
}
|
||||
|
||||
try:
|
||||
# Import all database interfaces
|
||||
from modules.interfaces.interfaceDbApp import getRootInterface as getAppInterface
|
||||
from modules.interfaces.interfaceDbManagement import getInterface as getMgmtInterface
|
||||
from modules.interfaces.interfaceDbChat import getInterface as getChatInterface
|
||||
|
||||
# 1. Process App DB (poweron_app)
|
||||
try:
|
||||
appInterface = getAppInterface()
|
||||
appStats = deleteUserDataFromDatabase(appInterface.db, userId, "poweron_app")
|
||||
allStats["databases"].append(appStats)
|
||||
allStats["totalTablesProcessed"] += appStats["tablesProcessed"]
|
||||
allStats["totalRecordsDeleted"] += appStats["recordsDeleted"]
|
||||
allStats["totalRecordsAnonymized"] += appStats["recordsAnonymized"]
|
||||
allStats["errors"].extend(appStats["errors"])
|
||||
except Exception as appErr:
|
||||
errorMsg = f"Error processing App DB: {appErr}"
|
||||
logger.error(errorMsg)
|
||||
allStats["errors"].append(errorMsg)
|
||||
|
||||
# 2. Process Management DB (poweron_management)
|
||||
try:
|
||||
mgmtInterface = getMgmtInterface(currentUser)
|
||||
mgmtStats = deleteUserDataFromDatabase(mgmtInterface.db, userId, "poweron_management")
|
||||
allStats["databases"].append(mgmtStats)
|
||||
allStats["totalTablesProcessed"] += mgmtStats["tablesProcessed"]
|
||||
allStats["totalRecordsDeleted"] += mgmtStats["recordsDeleted"]
|
||||
allStats["totalRecordsAnonymized"] += mgmtStats["recordsAnonymized"]
|
||||
allStats["errors"].extend(mgmtStats["errors"])
|
||||
except Exception as mgmtErr:
|
||||
errorMsg = f"Error processing Management DB: {mgmtErr}"
|
||||
logger.error(errorMsg)
|
||||
allStats["errors"].append(errorMsg)
|
||||
|
||||
# 3. Process Chat DB (poweron_chat)
|
||||
try:
|
||||
chatInterface = getChatInterface(currentUser)
|
||||
chatStats = deleteUserDataFromDatabase(chatInterface.db, userId, "poweron_chat")
|
||||
allStats["databases"].append(chatStats)
|
||||
allStats["totalTablesProcessed"] += chatStats["tablesProcessed"]
|
||||
allStats["totalRecordsDeleted"] += chatStats["recordsDeleted"]
|
||||
allStats["totalRecordsAnonymized"] += chatStats["recordsAnonymized"]
|
||||
allStats["errors"].extend(chatStats["errors"])
|
||||
except Exception as chatErr:
|
||||
errorMsg = f"Error processing Chat DB: {chatErr}"
|
||||
logger.error(errorMsg)
|
||||
allStats["errors"].append(errorMsg)
|
||||
|
||||
# 4. Process Feature DBs (discover dynamically)
|
||||
try:
|
||||
featureStats = _deleteUserDataFromFeatureDatabases(userId, currentUser)
|
||||
allStats["databases"].extend(featureStats["databases"])
|
||||
allStats["totalTablesProcessed"] += featureStats["totalTablesProcessed"]
|
||||
allStats["totalRecordsDeleted"] += featureStats["totalRecordsDeleted"]
|
||||
allStats["totalRecordsAnonymized"] += featureStats["totalRecordsAnonymized"]
|
||||
allStats["errors"].extend(featureStats["errors"])
|
||||
except Exception as featureErr:
|
||||
errorMsg = f"Error processing Feature DBs: {featureErr}"
|
||||
logger.error(errorMsg)
|
||||
allStats["errors"].append(errorMsg)
|
||||
|
||||
# Log summary
|
||||
logger.info(f"GDPR deletion completed for user {userId}: "
|
||||
f"{allStats['totalRecordsDeleted']} deleted, "
|
||||
f"{allStats['totalRecordsAnonymized']} anonymized across "
|
||||
f"{len(allStats['databases'])} databases")
|
||||
|
||||
return allStats
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Fatal error in deleteUserDataAcrossAllDatabases: {e}")
|
||||
allStats["errors"].append(f"Fatal error: {e}")
|
||||
return allStats
|
||||
|
||||
|
||||
def _deleteUserDataFromFeatureDatabases(userId: str, currentUser) -> Dict[str, Any]:
|
||||
"""
|
||||
Delete user data from all feature-specific databases.
|
||||
Discovers feature interfaces dynamically.
|
||||
|
||||
Args:
|
||||
userId: User ID to delete
|
||||
currentUser: User object
|
||||
|
||||
Returns:
|
||||
Dict with deletion statistics
|
||||
"""
|
||||
stats = {
|
||||
"databases": [],
|
||||
"totalTablesProcessed": 0,
|
||||
"totalRecordsDeleted": 0,
|
||||
"totalRecordsAnonymized": 0,
|
||||
"errors": []
|
||||
}
|
||||
|
||||
try:
|
||||
# Get all feature instances for this user to determine which feature DBs to check
|
||||
from modules.interfaces.interfaceDbApp import getRootInterface
|
||||
from modules.datamodels.datamodelMembership import FeatureAccess
|
||||
from modules.datamodels.datamodelFeatures import FeatureInstance
|
||||
|
||||
rootInterface = getRootInterface()
|
||||
|
||||
# Get all feature accesses for this user
|
||||
featureAccesses = rootInterface.db.getRecordset(
|
||||
FeatureAccess,
|
||||
recordFilter={"userId": str(userId)}
|
||||
)
|
||||
|
||||
# Collect unique feature codes
|
||||
featureCodes: Set[str] = set()
|
||||
for fa in featureAccesses:
|
||||
instanceId = fa.get("featureInstanceId")
|
||||
instanceRecords = rootInterface.db.getRecordset(
|
||||
FeatureInstance,
|
||||
recordFilter={"id": instanceId}
|
||||
)
|
||||
if instanceRecords:
|
||||
featureCode = instanceRecords[0].get("featureCode")
|
||||
if featureCode:
|
||||
featureCodes.add(featureCode)
|
||||
|
||||
logger.info(f"Found {len(featureCodes)} feature types to process: {featureCodes}")
|
||||
|
||||
# Process each feature type
|
||||
for featureCode in featureCodes:
|
||||
try:
|
||||
dbName = f"poweron_{featureCode}"
|
||||
|
||||
# Try to get feature interface
|
||||
featureInterface = None
|
||||
|
||||
if featureCode == "trustee":
|
||||
from modules.features.trustee.interfaceFeatureTrustee import getInterface as getTrusteeInterface
|
||||
featureInterface = getTrusteeInterface(currentUser)
|
||||
elif featureCode == "realestate":
|
||||
from modules.features.realestate.interfaceFeatureRealEstate import getInterface as getRealEstateInterface
|
||||
featureInterface = getRealEstateInterface(currentUser)
|
||||
elif featureCode == "chatbot":
|
||||
from modules.features.chatbot.interfaceFeatureChatbot import getInterface as getChatbotInterface
|
||||
featureInterface = getChatbotInterface(currentUser)
|
||||
elif featureCode == "neutralization":
|
||||
from modules.features.neutralization.interfaceFeatureNeutralizer import getInterface as getNeutralizerInterface
|
||||
featureInterface = getNeutralizerInterface(currentUser)
|
||||
else:
|
||||
logger.warning(f"No interface found for feature code: {featureCode}")
|
||||
continue
|
||||
|
||||
if featureInterface and hasattr(featureInterface, 'db'):
|
||||
featureStats = deleteUserDataFromDatabase(
|
||||
featureInterface.db,
|
||||
userId,
|
||||
dbName
|
||||
)
|
||||
stats["databases"].append(featureStats)
|
||||
stats["totalTablesProcessed"] += featureStats["tablesProcessed"]
|
||||
stats["totalRecordsDeleted"] += featureStats["recordsDeleted"]
|
||||
stats["totalRecordsAnonymized"] += featureStats["recordsAnonymized"]
|
||||
stats["errors"].extend(featureStats["errors"])
|
||||
|
||||
except Exception as featureErr:
|
||||
errorMsg = f"Error processing feature {featureCode}: {featureErr}"
|
||||
logger.warning(errorMsg)
|
||||
stats["errors"].append(errorMsg)
|
||||
|
||||
return stats
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Error in _deleteUserDataFromFeatureDatabases: {e}")
|
||||
stats["errors"].append(f"Feature DB error: {e}")
|
||||
return stats
|
||||
|
||||
|
||||
def buildDeletionSummary(stats: Dict[str, Any]) -> List[str]:
|
||||
"""
|
||||
Build a human-readable summary of the deletion operation.
|
||||
|
||||
Args:
|
||||
stats: Statistics dict from deleteUserDataAcrossAllDatabases
|
||||
|
||||
Returns:
|
||||
List of summary strings
|
||||
"""
|
||||
summary = []
|
||||
|
||||
for dbStats in stats.get("databases", []):
|
||||
dbName = dbStats.get("database", "unknown")
|
||||
deleted = dbStats.get("recordsDeleted", 0)
|
||||
anonymized = dbStats.get("recordsAnonymized", 0)
|
||||
|
||||
if deleted > 0 or anonymized > 0:
|
||||
parts = []
|
||||
if deleted > 0:
|
||||
parts.append(f"{deleted} deleted")
|
||||
if anonymized > 0:
|
||||
parts.append(f"{anonymized} anonymized")
|
||||
summary.append(f"{dbName}: {', '.join(parts)}")
|
||||
|
||||
# Add totals
|
||||
totalDeleted = stats.get("totalRecordsDeleted", 0)
|
||||
totalAnonymized = stats.get("totalRecordsAnonymized", 0)
|
||||
summary.append(f"Total: {totalDeleted} deleted, {totalAnonymized} anonymized")
|
||||
|
||||
return summary
|
||||
|
|
@ -244,6 +244,15 @@ NAVIGATION_SECTIONS = [
|
|||
"order": 90,
|
||||
"adminOnly": True,
|
||||
},
|
||||
{
|
||||
"id": "admin-user-access-overview",
|
||||
"objectKey": "ui.admin.user-access-overview",
|
||||
"label": {"en": "User Access Overview", "de": "Benutzer-Zugriffsübersicht", "fr": "Aperçu des accès utilisateur"},
|
||||
"icon": "FaUserShield",
|
||||
"path": "/admin/user-access-overview",
|
||||
"order": 100,
|
||||
"adminOnly": True,
|
||||
},
|
||||
],
|
||||
},
|
||||
]
|
||||
|
|
@ -290,16 +299,39 @@ UI_OBJECTS = _buildUiObjectsFromNavigation()
|
|||
# =============================================================================
|
||||
|
||||
DATA_OBJECTS = [
|
||||
# User/Auth tables
|
||||
{
|
||||
"objectKey": "data.system.User",
|
||||
"objectKey": "data.system.UserInDB",
|
||||
"label": {"en": "User", "de": "Benutzer", "fr": "Utilisateur"},
|
||||
"meta": {"table": "UserInDB"}
|
||||
},
|
||||
{
|
||||
"objectKey": "data.system.AuthEvent",
|
||||
"label": {"en": "Auth Event", "de": "Auth-Ereignis", "fr": "Événement d'auth"},
|
||||
"meta": {"table": "AuthEvent"}
|
||||
},
|
||||
{
|
||||
"objectKey": "data.system.UserConnection",
|
||||
"label": {"en": "Connection", "de": "Verbindung", "fr": "Connexion"},
|
||||
"meta": {"table": "UserConnection"}
|
||||
},
|
||||
# Mandate/Membership tables
|
||||
{
|
||||
"objectKey": "data.system.Mandate",
|
||||
"label": {"en": "Mandate", "de": "Mandant", "fr": "Mandat"},
|
||||
"meta": {"table": "Mandate"}
|
||||
},
|
||||
{
|
||||
"objectKey": "data.system.UserMandate",
|
||||
"label": {"en": "User Mandate", "de": "Benutzer-Mandant", "fr": "Mandat utilisateur"},
|
||||
"meta": {"table": "UserMandate"}
|
||||
},
|
||||
{
|
||||
"objectKey": "data.system.Invitation",
|
||||
"label": {"en": "Invitation", "de": "Einladung", "fr": "Invitation"},
|
||||
"meta": {"table": "Invitation"}
|
||||
},
|
||||
# RBAC tables
|
||||
{
|
||||
"objectKey": "data.system.Role",
|
||||
"label": {"en": "Role", "de": "Rolle", "fr": "Rôle"},
|
||||
|
|
@ -310,11 +342,13 @@ DATA_OBJECTS = [
|
|||
"label": {"en": "Access Rule", "de": "Zugriffsregel", "fr": "Règle d'accès"},
|
||||
"meta": {"table": "AccessRule"}
|
||||
},
|
||||
# Feature tables
|
||||
{
|
||||
"objectKey": "data.system.UserMandate",
|
||||
"label": {"en": "User Mandate", "de": "Benutzer-Mandant", "fr": "Mandat utilisateur"},
|
||||
"meta": {"table": "UserMandate"}
|
||||
"objectKey": "data.system.FeatureInstance",
|
||||
"label": {"en": "Feature Instance", "de": "Feature-Instanz", "fr": "Instance de feature"},
|
||||
"meta": {"table": "FeatureInstance"}
|
||||
},
|
||||
# Content tables
|
||||
{
|
||||
"objectKey": "data.system.Prompt",
|
||||
"label": {"en": "Prompt", "de": "Prompt", "fr": "Prompt"},
|
||||
|
|
@ -330,16 +364,6 @@ DATA_OBJECTS = [
|
|||
"label": {"en": "File", "de": "Datei", "fr": "Fichier"},
|
||||
"meta": {"table": "FileItem"}
|
||||
},
|
||||
{
|
||||
"objectKey": "data.system.UserConnection",
|
||||
"label": {"en": "Connection", "de": "Verbindung", "fr": "Connexion"},
|
||||
"meta": {"table": "UserConnection"}
|
||||
},
|
||||
{
|
||||
"objectKey": "data.system.FeatureInstance",
|
||||
"label": {"en": "Feature Instance", "de": "Feature-Instanz", "fr": "Instance de feature"},
|
||||
"meta": {"table": "FeatureInstance"}
|
||||
},
|
||||
]
|
||||
|
||||
# =============================================================================
|
||||
|
|
|
|||
|
|
@ -13,6 +13,7 @@ from .findSiteByUrl import findSiteByUrl
|
|||
from .downloadFileByPath import downloadFileByPath
|
||||
from .copyFile import copyFile
|
||||
from .uploadFile import uploadFile
|
||||
from .getExpensesFromPdf import getExpensesFromPdf
|
||||
|
||||
__all__ = [
|
||||
'findDocumentPath',
|
||||
|
|
@ -24,5 +25,6 @@ __all__ = [
|
|||
'downloadFileByPath',
|
||||
'copyFile',
|
||||
'uploadFile',
|
||||
'getExpensesFromPdf',
|
||||
]
|
||||
|
||||
|
|
|
|||
|
|
@ -0,0 +1,733 @@
|
|||
# Copyright (c) 2025 Patrick Motsch
|
||||
# All rights reserved.
|
||||
|
||||
"""
|
||||
Action to extract expenses from PDF documents in SharePoint and save to TrusteePosition.
|
||||
|
||||
Process:
|
||||
1. Read PDF files from SharePoint folder (max 50 files per execution)
|
||||
2. FOR EACH PDF document:
|
||||
a. AI call to extract expense data in CSV format
|
||||
b. If 0 records: move to "error" folder
|
||||
c. Validate/calculate VAT, complete valuta/transactionDateTime
|
||||
d. Save all records to TrusteePosition
|
||||
e. Move document to "processed" subfolder with timestamp prefix
|
||||
"""
|
||||
|
||||
import logging
|
||||
import time
|
||||
import json
|
||||
import csv
|
||||
import io
|
||||
import asyncio
|
||||
from datetime import datetime, UTC
|
||||
from typing import Dict, Any, List, Optional
|
||||
from modules.datamodels.datamodelChat import ActionResult, ActionDocument
|
||||
from modules.datamodels.datamodelAi import AiCallOptions, OperationTypeEnum
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
# Configuration
|
||||
MAX_FILES_PER_EXECUTION = 50
|
||||
ALLOWED_TAGS = ["customer", "meeting", "license", "subscription", "fuel", "food", "material"]
|
||||
RATE_LIMIT_WAIT_SECONDS = 60
|
||||
|
||||
|
||||
async def getExpensesFromPdf(self, parameters: Dict[str, Any]) -> ActionResult:
|
||||
"""
|
||||
Extract expenses from PDF documents in SharePoint and save to TrusteePosition.
|
||||
|
||||
Parameters:
|
||||
- connectionReference (str): Microsoft connection label
|
||||
- sharepointFolder (str): SharePoint folder path (e.g., /sites/MySite/Documents/Expenses)
|
||||
- featureInstanceId (str): Feature instance ID for TrusteePosition
|
||||
- prompt (str): AI prompt for content extraction
|
||||
|
||||
Returns:
|
||||
ActionResult with success status and processing summary
|
||||
"""
|
||||
operationId = None
|
||||
processedDocuments = []
|
||||
skippedDocuments = []
|
||||
errorDocuments = []
|
||||
totalPositions = 0
|
||||
|
||||
try:
|
||||
# Initialize progress tracking
|
||||
workflowId = self.services.workflow.id if self.services.workflow else f"no-workflow-{int(time.time())}"
|
||||
operationId = f"sharepoint_expenses_{workflowId}_{int(time.time())}"
|
||||
|
||||
parentOperationId = parameters.get('parentOperationId')
|
||||
self.services.chat.progressLogStart(
|
||||
operationId,
|
||||
"Extract Expenses from PDF",
|
||||
"SharePoint PDF Processing",
|
||||
"Initializing expense extraction",
|
||||
parentOperationId=parentOperationId
|
||||
)
|
||||
|
||||
# Extract and validate parameters
|
||||
connectionReference = parameters.get("connectionReference")
|
||||
sharepointFolder = parameters.get("sharepointFolder")
|
||||
featureInstanceId = parameters.get("featureInstanceId")
|
||||
prompt = parameters.get("prompt")
|
||||
|
||||
if not connectionReference:
|
||||
self.services.chat.progressLogFinish(operationId, False)
|
||||
return ActionResult.isFailure(error="connectionReference is required")
|
||||
if not sharepointFolder:
|
||||
self.services.chat.progressLogFinish(operationId, False)
|
||||
return ActionResult.isFailure(error="sharepointFolder is required")
|
||||
if not featureInstanceId:
|
||||
self.services.chat.progressLogFinish(operationId, False)
|
||||
return ActionResult.isFailure(error="featureInstanceId is required")
|
||||
if not prompt:
|
||||
self.services.chat.progressLogFinish(operationId, False)
|
||||
return ActionResult.isFailure(error="prompt is required")
|
||||
|
||||
# Get Microsoft connection
|
||||
self.services.chat.progressLogUpdate(operationId, 0.05, "Getting Microsoft connection")
|
||||
connection = self.connection.getMicrosoftConnection(connectionReference)
|
||||
if not connection:
|
||||
self.services.chat.progressLogFinish(operationId, False)
|
||||
return ActionResult.isFailure(error="No valid Microsoft connection found")
|
||||
|
||||
# Find site and folder info
|
||||
self.services.chat.progressLogUpdate(operationId, 0.1, "Resolving SharePoint site")
|
||||
siteInfo, folderPath = await _resolveSiteAndFolder(self, sharepointFolder)
|
||||
if not siteInfo:
|
||||
self.services.chat.progressLogFinish(operationId, False)
|
||||
return ActionResult.isFailure(error=f"Could not resolve SharePoint site from path: {sharepointFolder}")
|
||||
|
||||
siteId = siteInfo.get("id")
|
||||
|
||||
# List PDF files in folder
|
||||
self.services.chat.progressLogUpdate(operationId, 0.15, "Finding PDF files in folder")
|
||||
pdfFiles = await _listPdfFilesInFolder(self, siteId, folderPath)
|
||||
|
||||
if not pdfFiles:
|
||||
self.services.chat.progressLogFinish(operationId, True)
|
||||
return ActionResult.isSuccess(
|
||||
documents=[ActionDocument(
|
||||
documentName="expense_extraction_result.json",
|
||||
documentData=json.dumps({
|
||||
"status": "no_documents",
|
||||
"message": "No PDF files found in the specified folder",
|
||||
"folder": sharepointFolder
|
||||
}, indent=2),
|
||||
mimeType="application/json",
|
||||
validationMetadata={"actionType": "sharepoint.getExpensesFromPdf"}
|
||||
)]
|
||||
)
|
||||
|
||||
# Limit files
|
||||
originalFileCount = len(pdfFiles)
|
||||
if originalFileCount > MAX_FILES_PER_EXECUTION:
|
||||
logger.warning(f"Found {originalFileCount} PDFs, limiting to {MAX_FILES_PER_EXECUTION}")
|
||||
pdfFiles = pdfFiles[:MAX_FILES_PER_EXECUTION]
|
||||
|
||||
totalFiles = len(pdfFiles)
|
||||
progressPerFile = 0.7 / totalFiles
|
||||
|
||||
# Get Trustee interface
|
||||
from modules.features.trustee.interfaceFeatureTrustee import getInterface as getTrusteeInterface
|
||||
trusteeInterface = getTrusteeInterface(
|
||||
self.services.user,
|
||||
mandateId=self.services.mandateId,
|
||||
featureInstanceId=featureInstanceId
|
||||
)
|
||||
|
||||
# Process each PDF
|
||||
for idx, pdfFile in enumerate(pdfFiles):
|
||||
currentProgress = 0.2 + (idx * progressPerFile)
|
||||
fileName = pdfFile.get("name", f"file_{idx}")
|
||||
fileId = pdfFile.get("id")
|
||||
|
||||
self.services.chat.progressLogUpdate(
|
||||
operationId,
|
||||
currentProgress,
|
||||
f"Processing {idx + 1}/{totalFiles}: {fileName}"
|
||||
)
|
||||
|
||||
try:
|
||||
# Download PDF content
|
||||
fileContent = await self.services.sharepoint.downloadFile(siteId, fileId)
|
||||
if not fileContent:
|
||||
await _moveToErrorFolder(self, siteId, folderPath, fileName)
|
||||
errorDocuments.append({
|
||||
"file": fileName,
|
||||
"error": "Failed to download",
|
||||
"movedTo": "error/"
|
||||
})
|
||||
continue
|
||||
|
||||
# AI call to extract expense data
|
||||
aiResult = await _extractExpensesWithAi(self.services, fileContent, fileName, prompt, featureInstanceId)
|
||||
|
||||
if not aiResult.get("success"):
|
||||
await _moveToErrorFolder(self, siteId, folderPath, fileName)
|
||||
errorDocuments.append({
|
||||
"file": fileName,
|
||||
"error": aiResult.get("error", "AI extraction failed"),
|
||||
"movedTo": "error/"
|
||||
})
|
||||
continue
|
||||
|
||||
records = aiResult.get("records", [])
|
||||
|
||||
# Check for empty records
|
||||
if not records:
|
||||
logger.warning(f"Document {fileName}: No records extracted, moving to error folder")
|
||||
await _moveToErrorFolder(self, siteId, folderPath, fileName)
|
||||
skippedDocuments.append({
|
||||
"file": fileName,
|
||||
"reason": "No expense records extracted",
|
||||
"movedTo": "error/"
|
||||
})
|
||||
continue
|
||||
|
||||
# Validate and enrich records
|
||||
validatedRecords = _validateAndEnrichRecords(records, fileName)
|
||||
|
||||
# Save to TrusteePosition
|
||||
savedCount = _saveToTrusteePosition(trusteeInterface, validatedRecords, featureInstanceId, self.services.mandateId)
|
||||
totalPositions += savedCount
|
||||
|
||||
# Move document to "processed" subfolder
|
||||
timestamp = datetime.now(UTC).strftime("%Y%m%d-%H%M%S")
|
||||
newFileName = f"{timestamp}_{fileName}"
|
||||
|
||||
moveSuccess = await _moveToProcessedFolder(self, siteId, folderPath, fileName, newFileName)
|
||||
|
||||
processedDocuments.append({
|
||||
"file": fileName,
|
||||
"newLocation": f"processed/{newFileName}" if moveSuccess else "move_failed",
|
||||
"recordsExtracted": len(validatedRecords),
|
||||
"recordsSaved": savedCount
|
||||
})
|
||||
|
||||
except Exception as e:
|
||||
errorMsg = str(e)
|
||||
logger.error(f"Error processing {fileName}: {errorMsg}")
|
||||
|
||||
# Handle rate limit
|
||||
if "429" in errorMsg or "throttl" in errorMsg.lower():
|
||||
logger.warning(f"Rate limit hit, waiting {RATE_LIMIT_WAIT_SECONDS} seconds")
|
||||
await asyncio.sleep(RATE_LIMIT_WAIT_SECONDS)
|
||||
|
||||
await _moveToErrorFolder(self, siteId, folderPath, fileName)
|
||||
errorDocuments.append({
|
||||
"file": fileName,
|
||||
"error": errorMsg,
|
||||
"movedTo": "error/"
|
||||
})
|
||||
|
||||
# Create result summary
|
||||
self.services.chat.progressLogUpdate(operationId, 0.95, "Creating result summary")
|
||||
|
||||
remainingFiles = max(0, originalFileCount - MAX_FILES_PER_EXECUTION)
|
||||
|
||||
resultSummary = {
|
||||
"status": "completed",
|
||||
"folder": sharepointFolder,
|
||||
"featureInstanceId": featureInstanceId,
|
||||
"summary": {
|
||||
"totalFilesFound": originalFileCount,
|
||||
"filesProcessedThisRun": totalFiles,
|
||||
"remainingFiles": remainingFiles,
|
||||
"successfulDocuments": len(processedDocuments),
|
||||
"skippedDocuments": len(skippedDocuments),
|
||||
"errorDocuments": len(errorDocuments),
|
||||
"totalPositionsSaved": totalPositions
|
||||
},
|
||||
"processedDocuments": processedDocuments,
|
||||
"skippedDocuments": skippedDocuments,
|
||||
"errorDocuments": errorDocuments
|
||||
}
|
||||
|
||||
if remainingFiles > 0:
|
||||
resultSummary["note"] = f"{remainingFiles} files remaining for next execution"
|
||||
|
||||
self.services.chat.progressLogFinish(operationId, True)
|
||||
|
||||
return ActionResult.isSuccess(
|
||||
documents=[ActionDocument(
|
||||
documentName="expense_extraction_result.json",
|
||||
documentData=json.dumps(resultSummary, indent=2),
|
||||
mimeType="application/json",
|
||||
validationMetadata={
|
||||
"actionType": "sharepoint.getExpensesFromPdf",
|
||||
"sharepointFolder": sharepointFolder,
|
||||
"featureInstanceId": featureInstanceId,
|
||||
"totalPositions": totalPositions
|
||||
}
|
||||
)]
|
||||
)
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Error in getExpensesFromPdf: {str(e)}")
|
||||
if operationId:
|
||||
self.services.chat.progressLogFinish(operationId, False)
|
||||
return ActionResult.isFailure(error=str(e))
|
||||
|
||||
|
||||
async def _resolveSiteAndFolder(self, sharepointFolder: str) -> tuple:
|
||||
"""Resolve SharePoint site and folder path from the given path."""
|
||||
try:
|
||||
# Parse path format: /sites/SiteName/FolderPath
|
||||
if sharepointFolder.startswith('/sites/'):
|
||||
parts = sharepointFolder[7:].split('/', 1) # Remove '/sites/' prefix
|
||||
if len(parts) >= 1:
|
||||
siteName = parts[0]
|
||||
folderPath = parts[1] if len(parts) > 1 else ""
|
||||
|
||||
# Try to find site by name
|
||||
sites, _ = await self.siteDiscovery.resolveSitesFromPathQuery(sharepointFolder)
|
||||
if sites:
|
||||
return sites[0], folderPath
|
||||
|
||||
# Fallback: try to resolve via siteDiscovery
|
||||
sites, _ = await self.siteDiscovery.resolveSitesFromPathQuery(sharepointFolder)
|
||||
if sites:
|
||||
return sites[0], ""
|
||||
|
||||
return None, None
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Error resolving site and folder: {str(e)}")
|
||||
return None, None
|
||||
|
||||
|
||||
async def _listPdfFilesInFolder(self, siteId: str, folderPath: str) -> List[Dict[str, Any]]:
|
||||
"""List PDF files in the given folder."""
|
||||
try:
|
||||
import urllib.parse
|
||||
|
||||
# Build endpoint
|
||||
if not folderPath or folderPath == "/":
|
||||
endpoint = f"sites/{siteId}/drive/root/children"
|
||||
else:
|
||||
cleanPath = folderPath.strip('/')
|
||||
encodedPath = urllib.parse.quote(cleanPath, safe='/')
|
||||
endpoint = f"sites/{siteId}/drive/root:/{encodedPath}:/children"
|
||||
|
||||
result = await self.apiClient.makeGraphApiCall(endpoint)
|
||||
|
||||
if "error" in result:
|
||||
logger.error(f"Error listing folder: {result['error']}")
|
||||
return []
|
||||
|
||||
items = result.get("value", [])
|
||||
|
||||
# Filter for PDF files only
|
||||
pdfFiles = []
|
||||
for item in items:
|
||||
name = item.get("name", "")
|
||||
if name.lower().endswith('.pdf') and "file" in item:
|
||||
pdfFiles.append({
|
||||
"id": item.get("id"),
|
||||
"name": name,
|
||||
"size": item.get("size", 0),
|
||||
"webUrl": item.get("webUrl"),
|
||||
"lastModifiedDateTime": item.get("lastModifiedDateTime")
|
||||
})
|
||||
|
||||
logger.info(f"Found {len(pdfFiles)} PDF files in folder")
|
||||
return pdfFiles
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Error listing PDF files: {str(e)}")
|
||||
return []
|
||||
|
||||
|
||||
async def _extractExpensesWithAi(services, fileContent: bytes, fileName: str, prompt: str, featureInstanceId: str) -> Dict[str, Any]:
|
||||
"""
|
||||
Call AI service to extract expense data from PDF content.
|
||||
Uses the full AI service pipeline which handles:
|
||||
- Document extraction (text + images)
|
||||
- Intent analysis
|
||||
- Chunking for large documents
|
||||
- Vision processing for images
|
||||
"""
|
||||
try:
|
||||
import uuid
|
||||
|
||||
# Ensure AI is initialized
|
||||
await services.ai.ensureAiObjectsInitialized()
|
||||
|
||||
# Step 1: Store file temporarily in database so AI service can access it
|
||||
from modules.interfaces.interfaceDbManagement import getInterface as getDbInterface
|
||||
from modules.datamodels.datamodelChat import ChatDocument
|
||||
from modules.datamodels.datamodelDocref import DocumentReferenceList
|
||||
|
||||
dbInterface = getDbInterface()
|
||||
|
||||
# Create file record
|
||||
fileItem = dbInterface.createFile(
|
||||
name=fileName,
|
||||
mimeType="application/pdf",
|
||||
content=fileContent
|
||||
)
|
||||
|
||||
# Store file data
|
||||
dbInterface.createFileData(fileItem.id, fileContent)
|
||||
|
||||
logger.info(f"Stored PDF {fileName} ({len(fileContent)} bytes) with fileId: {fileItem.id}")
|
||||
|
||||
# Step 2: Create ChatDocument referencing the file
|
||||
# Use workflow context if available
|
||||
workflowId = services.workflow.id if services.workflow else str(uuid.uuid4())
|
||||
messageId = f"expense_import_{workflowId}_{str(uuid.uuid4())[:8]}"
|
||||
|
||||
chatDocument = ChatDocument(
|
||||
id=str(uuid.uuid4()),
|
||||
mandateId=services.mandateId or "",
|
||||
featureInstanceId=featureInstanceId or "",
|
||||
messageId=messageId,
|
||||
fileId=fileItem.id,
|
||||
fileName=fileName,
|
||||
fileSize=len(fileContent),
|
||||
mimeType="application/pdf"
|
||||
)
|
||||
|
||||
# Step 3: Create DocumentReferenceList for AI service
|
||||
from modules.datamodels.datamodelDocref import DocumentItemReference
|
||||
documentList = DocumentReferenceList(
|
||||
references=[
|
||||
DocumentItemReference(
|
||||
documentId=chatDocument.id,
|
||||
fileName=fileName
|
||||
)
|
||||
]
|
||||
)
|
||||
|
||||
# Step 4: Store the ChatDocument so AI service can retrieve it
|
||||
# The AI service uses getChatDocumentsFromDocumentList which queries the database
|
||||
from modules.interfaces.interfaceDbChat import getInterface as getChatInterface
|
||||
chatInterface = getChatInterface(services.user, mandateId=services.mandateId, featureInstanceId=featureInstanceId)
|
||||
chatInterface.createDocument(chatDocument.model_dump())
|
||||
|
||||
logger.info(f"Created ChatDocument {chatDocument.id} for AI processing")
|
||||
|
||||
# Step 5: Call AI with documentList - let AI service handle everything
|
||||
# (extraction, intent analysis, chunking, image processing)
|
||||
options = AiCallOptions(
|
||||
resultFormat="csv",
|
||||
operationType=OperationTypeEnum.DATA_EXTRACT
|
||||
)
|
||||
|
||||
aiResponse = await services.ai.callAiContent(
|
||||
prompt=prompt,
|
||||
options=options,
|
||||
documentList=documentList,
|
||||
contentParts=None, # Let AI service extract from documents
|
||||
outputFormat="csv"
|
||||
)
|
||||
|
||||
if not aiResponse or not aiResponse.content:
|
||||
return {"success": False, "error": "AI returned empty response"}
|
||||
|
||||
# Parse CSV response
|
||||
csvContent = aiResponse.content
|
||||
records = _parseCsvToRecords(csvContent)
|
||||
|
||||
return {"success": True, "records": records}
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"AI extraction error for {fileName}: {str(e)}")
|
||||
return {"success": False, "error": str(e)}
|
||||
|
||||
|
||||
def _parseCsvToRecords(csvContent: str) -> List[Dict[str, Any]]:
|
||||
"""Parse CSV content to list of expense records."""
|
||||
records = []
|
||||
try:
|
||||
# Clean up CSV content - remove markdown code blocks if present
|
||||
content = csvContent.strip()
|
||||
if content.startswith("```"):
|
||||
lines = content.split('\n')
|
||||
# Remove first and last line if they're code block markers
|
||||
if lines[0].startswith("```"):
|
||||
lines = lines[1:]
|
||||
if lines and lines[-1].strip() == "```":
|
||||
lines = lines[:-1]
|
||||
content = '\n'.join(lines)
|
||||
|
||||
reader = csv.DictReader(io.StringIO(content))
|
||||
for row in reader:
|
||||
# Clean up keys (remove whitespace)
|
||||
cleanedRow = {k.strip(): v.strip() if isinstance(v, str) else v for k, v in row.items()}
|
||||
records.append(cleanedRow)
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Error parsing CSV: {str(e)}")
|
||||
|
||||
return records
|
||||
|
||||
|
||||
def _validateAndEnrichRecords(records: List[Dict[str, Any]], sourceFileName: str) -> List[Dict[str, Any]]:
|
||||
"""
|
||||
Validate and enrich expense records:
|
||||
1. Calculate/correct VAT amount
|
||||
2. Complete valuta/transactionDateTime if one is missing
|
||||
3. Validate tags
|
||||
"""
|
||||
enrichedRecords = []
|
||||
|
||||
for record in records:
|
||||
enriched = record.copy()
|
||||
|
||||
# VAT calculation/validation
|
||||
vatPercentage = _parseFloat(record.get("vatPercentage", 0))
|
||||
vatAmount = _parseFloat(record.get("vatAmount", 0))
|
||||
bookingAmount = _parseFloat(record.get("bookingAmount", 0))
|
||||
|
||||
if vatPercentage > 0 and bookingAmount > 0:
|
||||
# Calculate expected VAT amount (VAT is included in bookingAmount)
|
||||
expectedVat = bookingAmount * vatPercentage / (100 + vatPercentage)
|
||||
|
||||
# If vatAmount is missing or significantly different, recalculate
|
||||
if vatAmount == 0 or abs(vatAmount - expectedVat) > 0.01:
|
||||
enriched["vatAmount"] = round(expectedVat, 2)
|
||||
logger.debug(f"VAT amount corrected: {vatAmount} -> {enriched['vatAmount']}")
|
||||
|
||||
# Valuta / transactionDateTime completion
|
||||
valuta = record.get("valuta")
|
||||
transactionDateTime = record.get("transactionDateTime")
|
||||
|
||||
if valuta and not transactionDateTime:
|
||||
try:
|
||||
dt = datetime.strptime(str(valuta).strip(), "%Y-%m-%d")
|
||||
enriched["transactionDateTime"] = dt.replace(hour=12).timestamp()
|
||||
except:
|
||||
pass
|
||||
elif transactionDateTime and not valuta:
|
||||
try:
|
||||
ts = float(transactionDateTime)
|
||||
dt = datetime.fromtimestamp(ts, UTC)
|
||||
enriched["valuta"] = dt.strftime("%Y-%m-%d")
|
||||
except:
|
||||
pass
|
||||
|
||||
# Validate tags
|
||||
tags = record.get("tags", "")
|
||||
if tags:
|
||||
tagList = [t.strip().lower() for t in str(tags).split(",")]
|
||||
validTags = [t for t in tagList if t in ALLOWED_TAGS]
|
||||
enriched["tags"] = ",".join(validTags)
|
||||
|
||||
# Store source file info in description
|
||||
existingDesc = record.get("desc", "")
|
||||
if sourceFileName and sourceFileName not in str(existingDesc):
|
||||
enriched["desc"] = f"[Source: {sourceFileName}]\n{existingDesc}"
|
||||
|
||||
enrichedRecords.append(enriched)
|
||||
|
||||
return enrichedRecords
|
||||
|
||||
|
||||
def _parseFloat(value) -> float:
|
||||
"""Safely parse float value."""
|
||||
try:
|
||||
if value is None or value == "":
|
||||
return 0.0
|
||||
return float(value)
|
||||
except (ValueError, TypeError):
|
||||
return 0.0
|
||||
|
||||
|
||||
def _saveToTrusteePosition(trusteeInterface, records: List[Dict[str, Any]], featureInstanceId: str, mandateId: str) -> int:
|
||||
"""Save validated records to TrusteePosition table."""
|
||||
savedCount = 0
|
||||
|
||||
for record in records:
|
||||
try:
|
||||
position = {
|
||||
"valuta": record.get("valuta"),
|
||||
"transactionDateTime": record.get("transactionDateTime"),
|
||||
"company": record.get("company", ""),
|
||||
"desc": record.get("desc", ""),
|
||||
"tags": record.get("tags", ""),
|
||||
"bookingCurrency": record.get("bookingCurrency", "CHF"),
|
||||
"bookingAmount": _parseFloat(record.get("bookingAmount", 0)),
|
||||
"originalCurrency": record.get("originalCurrency") or record.get("bookingCurrency", "CHF"),
|
||||
"originalAmount": _parseFloat(record.get("originalAmount", 0)) or _parseFloat(record.get("bookingAmount", 0)),
|
||||
"vatPercentage": _parseFloat(record.get("vatPercentage", 0)),
|
||||
"vatAmount": _parseFloat(record.get("vatAmount", 0)),
|
||||
"featureInstanceId": featureInstanceId,
|
||||
"mandateId": mandateId
|
||||
}
|
||||
|
||||
result = trusteeInterface.createPosition(position)
|
||||
if result:
|
||||
savedCount += 1
|
||||
logger.debug(f"Saved position: {position.get('company')} - {position.get('bookingAmount')}")
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Failed to save position: {str(e)}")
|
||||
|
||||
return savedCount
|
||||
|
||||
|
||||
async def _ensureFolderExists(self, siteId: str, folderPath: str) -> bool:
|
||||
"""Create folder if it doesn't exist."""
|
||||
try:
|
||||
import urllib.parse
|
||||
|
||||
# Check if folder exists
|
||||
cleanPath = folderPath.strip('/')
|
||||
encodedPath = urllib.parse.quote(cleanPath, safe='/')
|
||||
checkEndpoint = f"sites/{siteId}/drive/root:/{encodedPath}"
|
||||
|
||||
result = await self.apiClient.makeGraphApiCall(checkEndpoint)
|
||||
|
||||
if "error" not in result:
|
||||
return True # Folder exists
|
||||
|
||||
# Create folder - need to create parent first if nested
|
||||
pathParts = cleanPath.split('/')
|
||||
currentPath = ""
|
||||
|
||||
for part in pathParts:
|
||||
parentPath = currentPath if currentPath else "root"
|
||||
currentPath = f"{currentPath}/{part}" if currentPath else part
|
||||
|
||||
# Check if this level exists
|
||||
checkPath = urllib.parse.quote(currentPath, safe='/')
|
||||
checkResult = await self.apiClient.makeGraphApiCall(f"sites/{siteId}/drive/root:/{checkPath}")
|
||||
|
||||
if "error" in checkResult:
|
||||
# Create this folder
|
||||
if parentPath == "root":
|
||||
createEndpoint = f"sites/{siteId}/drive/root/children"
|
||||
else:
|
||||
encodedParent = urllib.parse.quote(parentPath, safe='/')
|
||||
createEndpoint = f"sites/{siteId}/drive/root:/{encodedParent}:/children"
|
||||
|
||||
createData = json.dumps({
|
||||
"name": part,
|
||||
"folder": {},
|
||||
"@microsoft.graph.conflictBehavior": "fail"
|
||||
}).encode('utf-8')
|
||||
|
||||
createResult = await self.apiClient.makeGraphApiCall(createEndpoint, method="POST", data=createData)
|
||||
|
||||
if "error" in createResult:
|
||||
logger.warning(f"Failed to create folder {part}: {createResult['error']}")
|
||||
return False
|
||||
|
||||
logger.info(f"Created folder: {currentPath}")
|
||||
|
||||
return True
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Failed to ensure folder exists: {str(e)}")
|
||||
return False
|
||||
|
||||
|
||||
async def _moveToProcessedFolder(self, siteId: str, sourceFolderPath: str, sourceFileName: str, destFileName: str) -> bool:
|
||||
"""Move processed PDF to 'processed' subfolder."""
|
||||
try:
|
||||
# Build processed folder path
|
||||
cleanSource = sourceFolderPath.strip('/')
|
||||
processedFolder = f"{cleanSource}/processed" if cleanSource else "processed"
|
||||
|
||||
# Ensure processed folder exists
|
||||
await _ensureFolderExists(self, siteId, processedFolder)
|
||||
|
||||
# Copy file to new location
|
||||
await self.services.sharepoint.copyFileAsync(
|
||||
siteId=siteId,
|
||||
sourceFolder=cleanSource if cleanSource else "/",
|
||||
sourceFile=sourceFileName,
|
||||
destFolder=processedFolder,
|
||||
destFile=destFileName
|
||||
)
|
||||
|
||||
# Delete original file
|
||||
await _deleteFile(self, siteId, sourceFolderPath, sourceFileName)
|
||||
|
||||
logger.info(f"Moved {sourceFileName} to processed/{destFileName}")
|
||||
return True
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Failed to move file to processed: {str(e)}")
|
||||
return False
|
||||
|
||||
|
||||
async def _moveToErrorFolder(self, siteId: str, sourceFolderPath: str, sourceFileName: str) -> bool:
|
||||
"""Move failed PDF to 'error' subfolder (filename unchanged)."""
|
||||
try:
|
||||
# Build error folder path
|
||||
cleanSource = sourceFolderPath.strip('/')
|
||||
errorFolder = f"{cleanSource}/error" if cleanSource else "error"
|
||||
|
||||
# Ensure error folder exists
|
||||
await _ensureFolderExists(self, siteId, errorFolder)
|
||||
|
||||
# Copy file to error folder (keep original name)
|
||||
await self.services.sharepoint.copyFileAsync(
|
||||
siteId=siteId,
|
||||
sourceFolder=cleanSource if cleanSource else "/",
|
||||
sourceFile=sourceFileName,
|
||||
destFolder=errorFolder,
|
||||
destFile=sourceFileName # Same filename
|
||||
)
|
||||
|
||||
# Delete original file
|
||||
await _deleteFile(self, siteId, sourceFolderPath, sourceFileName)
|
||||
|
||||
logger.info(f"Moved {sourceFileName} to error/")
|
||||
return True
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Failed to move file to error folder: {str(e)}")
|
||||
return False
|
||||
|
||||
|
||||
async def _deleteFile(self, siteId: str, folderPath: str, fileName: str) -> bool:
|
||||
"""Delete file from SharePoint."""
|
||||
try:
|
||||
import urllib.parse
|
||||
|
||||
cleanPath = folderPath.strip('/')
|
||||
filePath = f"{cleanPath}/{fileName}" if cleanPath else fileName
|
||||
encodedPath = urllib.parse.quote(filePath, safe='/')
|
||||
|
||||
endpoint = f"sites/{siteId}/drive/root:/{encodedPath}"
|
||||
|
||||
# Get file ID first
|
||||
fileInfo = await self.apiClient.makeGraphApiCall(endpoint)
|
||||
if "error" in fileInfo:
|
||||
logger.warning(f"File not found for deletion: {filePath}")
|
||||
return False
|
||||
|
||||
fileId = fileInfo.get("id")
|
||||
if not fileId:
|
||||
return False
|
||||
|
||||
# Delete by ID
|
||||
deleteEndpoint = f"sites/{siteId}/drive/items/{fileId}"
|
||||
|
||||
# Make DELETE request
|
||||
if self.services.sharepoint.accessToken is None:
|
||||
logger.error("Access token not set for delete")
|
||||
return False
|
||||
|
||||
import aiohttp
|
||||
headers = {"Authorization": f"Bearer {self.services.sharepoint.accessToken}"}
|
||||
url = f"https://graph.microsoft.com/v1.0/{deleteEndpoint}"
|
||||
|
||||
async with aiohttp.ClientSession() as session:
|
||||
async with session.delete(url, headers=headers) as response:
|
||||
if response.status in [200, 204]:
|
||||
logger.debug(f"Deleted file: {filePath}")
|
||||
return True
|
||||
else:
|
||||
errorText = await response.text()
|
||||
logger.warning(f"Delete failed: {response.status} - {errorText}")
|
||||
return False
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Failed to delete file: {str(e)}")
|
||||
return False
|
||||
|
|
@ -28,6 +28,7 @@ from .actions.findSiteByUrl import findSiteByUrl
|
|||
from .actions.downloadFileByPath import downloadFileByPath
|
||||
from .actions.copyFile import copyFile
|
||||
from .actions.uploadFile import uploadFile
|
||||
from .actions.getExpensesFromPdf import getExpensesFromPdf
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
|
@ -377,6 +378,42 @@ class MethodSharepoint(MethodBase):
|
|||
)
|
||||
},
|
||||
execute=uploadFile.__get__(self, self.__class__)
|
||||
),
|
||||
"getExpensesFromPdf": WorkflowActionDefinition(
|
||||
actionId="sharepoint.getExpensesFromPdf",
|
||||
description="Extract expenses from PDF documents in SharePoint folder and save to TrusteePosition",
|
||||
dynamicMode=False, # Not for dynamic workflow
|
||||
parameters={
|
||||
"connectionReference": WorkflowActionParameter(
|
||||
name="connectionReference",
|
||||
type="str",
|
||||
frontendType=FrontendType.USER_CONNECTION,
|
||||
required=True,
|
||||
description="Microsoft connection label for SharePoint access"
|
||||
),
|
||||
"sharepointFolder": WorkflowActionParameter(
|
||||
name="sharepointFolder",
|
||||
type="str",
|
||||
frontendType=FrontendType.TEXT,
|
||||
required=True,
|
||||
description="SharePoint folder path containing PDF expense documents (e.g., /sites/MySite/Documents/Expenses)"
|
||||
),
|
||||
"featureInstanceId": WorkflowActionParameter(
|
||||
name="featureInstanceId",
|
||||
type="str",
|
||||
frontendType=FrontendType.TEXT,
|
||||
required=True,
|
||||
description="Feature Instance ID for the Trustee feature where positions will be stored"
|
||||
),
|
||||
"prompt": WorkflowActionParameter(
|
||||
name="prompt",
|
||||
type="str",
|
||||
frontendType=FrontendType.TEXTAREA,
|
||||
required=True,
|
||||
description="AI prompt for extracting expense data from PDF content"
|
||||
)
|
||||
},
|
||||
execute=getExpensesFromPdf.__get__(self, self.__class__)
|
||||
)
|
||||
}
|
||||
|
||||
|
|
@ -393,4 +430,5 @@ class MethodSharepoint(MethodBase):
|
|||
self.downloadFileByPath = downloadFileByPath.__get__(self, self.__class__)
|
||||
self.copyFile = copyFile.__get__(self, self.__class__)
|
||||
self.uploadFile = uploadFile.__get__(self, self.__class__)
|
||||
self.getExpensesFromPdf = getExpensesFromPdf.__get__(self, self.__class__)
|
||||
|
||||
|
|
|
|||
|
|
@ -119,33 +119,27 @@ class TestRbacBootstrap:
|
|||
# Should create multiple rules for different tables
|
||||
assert db.recordCreate.call_count > 0
|
||||
|
||||
# Check that Mandate table rules are created
|
||||
# Check that Mandate table rules are created with full objectKey
|
||||
mandateCalls = [call for call in db.recordCreate.call_args_list
|
||||
if call[0][1].item == "Mandate"]
|
||||
if call[0][1].item == "data.system.Mandate"]
|
||||
assert len(mandateCalls) > 0
|
||||
|
||||
# Check sysadmin rule for Mandate
|
||||
sysadminMandateCall = [call for call in mandateCalls
|
||||
if call[0][1].roleLabel == "sysadmin"][0]
|
||||
sysadminRule = sysadminMandateCall[0][1]
|
||||
assert sysadminRule.view == True
|
||||
assert sysadminRule.read == AccessLevel.ALL
|
||||
|
||||
# Check that other roles have view=False for Mandate
|
||||
otherMandateCalls = [call for call in mandateCalls
|
||||
if call[0][1].roleLabel != "sysadmin"]
|
||||
for call in otherMandateCalls:
|
||||
# Check that all roles have view=False and no access for Mandate
|
||||
# (SysAdmin bypasses RBAC via isSysAdmin flag, not via roles)
|
||||
for call in mandateCalls:
|
||||
rule = call[0][1]
|
||||
assert rule.view == False
|
||||
assert rule.read == AccessLevel.NONE
|
||||
|
||||
def testInitRbacRulesSkipsIfExists(self):
|
||||
"""Test that initRbacRules skips default rule creation if rules already exist, but adds missing table-specific rules."""
|
||||
db = Mock()
|
||||
# Mock existing rules - include rules for ChatWorkflow and Prompt to prevent adding missing rules
|
||||
# Need rules for all required roles to fully prevent creation
|
||||
# Using full objectKey format: data.system.{TableName}
|
||||
existingRules = []
|
||||
for table in ["ChatWorkflow", "Prompt"]:
|
||||
for role in ["sysadmin", "admin", "user", "viewer"]:
|
||||
for table in ["data.system.ChatWorkflow", "data.system.Prompt"]:
|
||||
for role in ["admin", "user", "viewer"]:
|
||||
existingRules.append({
|
||||
"id": f"rule_{table}_{role}",
|
||||
"item": table,
|
||||
|
|
|
|||
|
|
@ -94,7 +94,7 @@ class TestRbacPermissionResolution:
|
|||
AccessRule(
|
||||
roleLabel="user",
|
||||
context=AccessRuleContext.DATA,
|
||||
item="UserInDB", # Specific rule
|
||||
item="data.system.UserInDB", # Specific rule with full objectKey
|
||||
view=True,
|
||||
read=AccessLevel.MY,
|
||||
create=AccessLevel.NONE,
|
||||
|
|
@ -107,10 +107,11 @@ class TestRbacPermissionResolution:
|
|||
rbac._getRulesForRole = mockGetRulesForRole
|
||||
|
||||
# Get permissions for UserInDB table - should use specific rule
|
||||
# Using full objectKey format: data.system.UserInDB
|
||||
permissions = rbac.getUserPermissions(
|
||||
user,
|
||||
AccessRuleContext.DATA,
|
||||
"UserInDB"
|
||||
"data.system.UserInDB"
|
||||
)
|
||||
|
||||
# Most specific rule should win
|
||||
|
|
@ -252,29 +253,29 @@ class TestRbacPermissionResolution:
|
|||
AccessRule(
|
||||
roleLabel="user",
|
||||
context=AccessRuleContext.DATA,
|
||||
item="UserInDB", # Table-level
|
||||
item="data.system.UserInDB", # Table-level with full objectKey
|
||||
view=True,
|
||||
read=AccessLevel.MY
|
||||
),
|
||||
AccessRule(
|
||||
roleLabel="user",
|
||||
context=AccessRuleContext.DATA,
|
||||
item="UserInDB.email", # Field-level - most specific
|
||||
item="data.system.UserInDB.email", # Field-level - most specific
|
||||
view=True,
|
||||
read=AccessLevel.NONE
|
||||
)
|
||||
]
|
||||
|
||||
# Test exact match
|
||||
rule = rbac.findMostSpecificRule(rules, "UserInDB.email")
|
||||
rule = rbac.findMostSpecificRule(rules, "data.system.UserInDB.email")
|
||||
assert rule is not None
|
||||
assert rule.item == "UserInDB.email"
|
||||
assert rule.item == "data.system.UserInDB.email"
|
||||
assert rule.read == AccessLevel.NONE
|
||||
|
||||
# Test table-level match
|
||||
rule = rbac.findMostSpecificRule(rules, "UserInDB")
|
||||
rule = rbac.findMostSpecificRule(rules, "data.system.UserInDB")
|
||||
assert rule is not None
|
||||
assert rule.item == "UserInDB"
|
||||
assert rule.item == "data.system.UserInDB"
|
||||
assert rule.read == AccessLevel.MY
|
||||
|
||||
# Test generic fallback
|
||||
|
|
@ -293,7 +294,7 @@ class TestRbacPermissionResolution:
|
|||
rule1 = AccessRule(
|
||||
roleLabel="user",
|
||||
context=AccessRuleContext.DATA,
|
||||
item="UserInDB",
|
||||
item="data.system.UserInDB",
|
||||
view=True,
|
||||
read=AccessLevel.MY,
|
||||
create=AccessLevel.MY,
|
||||
|
|
@ -306,7 +307,7 @@ class TestRbacPermissionResolution:
|
|||
rule2 = AccessRule(
|
||||
roleLabel="user",
|
||||
context=AccessRuleContext.DATA,
|
||||
item="UserInDB",
|
||||
item="data.system.UserInDB",
|
||||
view=True,
|
||||
read=AccessLevel.MY,
|
||||
create=AccessLevel.GROUP, # Not allowed
|
||||
|
|
@ -319,7 +320,7 @@ class TestRbacPermissionResolution:
|
|||
rule3 = AccessRule(
|
||||
roleLabel="admin",
|
||||
context=AccessRuleContext.DATA,
|
||||
item="UserInDB",
|
||||
item="data.system.UserInDB",
|
||||
view=True,
|
||||
read=AccessLevel.GROUP,
|
||||
create=AccessLevel.GROUP,
|
||||
|
|
@ -332,7 +333,7 @@ class TestRbacPermissionResolution:
|
|||
rule4 = AccessRule(
|
||||
roleLabel="user",
|
||||
context=AccessRuleContext.DATA,
|
||||
item="UserInDB",
|
||||
item="data.system.UserInDB",
|
||||
view=True,
|
||||
read=AccessLevel.NONE,
|
||||
create=AccessLevel.MY, # Not allowed without read
|
||||
|
|
|
|||
Loading…
Reference in a new issue