From 2e7a0a73c790f26beba755caaa1880244c5b37b3 Mon Sep 17 00:00:00 2001 From: ValueOn AG Date: Sun, 22 Mar 2026 21:34:54 +0100 Subject: [PATCH] streamlined formgeneratortable and sort/filter globally --- modules/connectors/connectorDbPostgre.py | 270 +- modules/datamodels/datamodelPagination.py | 6 + .../automation/interfaceFeatureAutomation.py | 2 +- .../automation/routeFeatureAutomation.py | 44 + .../chatbot/interfaceFeatureChatbot.py | 6 +- .../realEstate/interfaceFeatureRealEstate.py | 86 +- .../realEstate/routeFeatureRealEstate.py | 50 + .../trustee/interfaceFeatureTrustee.py | 453 ++- .../features/trustee/routeFeatureTrustee.py | 128 +- modules/interfaces/interfaceDbApp.py | 148 +- modules/interfaces/interfaceDbBilling.py | 78 +- modules/interfaces/interfaceDbChat.py | 6 +- modules/interfaces/interfaceDbManagement.py | 56 +- modules/interfaces/interfaceRbac.py | 307 +- modules/routes/routeAdminAutomationEvents.py | 278 +- modules/routes/routeAdminFeatures.py | 176 +- modules/routes/routeAdminRbacRules.py | 81 +- modules/routes/routeBilling.py | 245 +- modules/routes/routeDataConnections.py | 41 + modules/routes/routeDataFiles.py | 50 + modules/routes/routeDataMandates.py | 117 + modules/routes/routeDataPrompts.py | 23 + modules/routes/routeDataUsers.py | 221 +- modules/routes/routeInvitations.py | 55 + modules/routes/routeMessaging.py | 10 +- modules/routes/routeRealEstate.py | 70 +- modules/routes/routeSubscription.py | 95 +- modules/system/mainSystem.py | 9 + scripts/.$import_diagram.drawio.bkp | 2723 ----------------- .../.$import_diagram_containers.drawio.bkp | 317 -- 30 files changed, 2402 insertions(+), 3749 deletions(-) delete mode 100644 scripts/.$import_diagram.drawio.bkp delete mode 100644 scripts/.$import_diagram_containers.drawio.bkp diff --git a/modules/connectors/connectorDbPostgre.py b/modules/connectors/connectorDbPostgre.py index 83517f31..9675ffca 100644 --- a/modules/connectors/connectorDbPostgre.py +++ b/modules/connectors/connectorDbPostgre.py @@ -1,6 +1,7 @@ # Copyright (c) 2025 Patrick Motsch # All rights reserved. import contextvars +import re import psycopg2 import psycopg2.extras import logging @@ -92,19 +93,27 @@ def _get_model_fields(model_class) -> Dict[str, str]: fields[field_name] = extra["db_type"] continue - # Check for JSONB fields (Dict, List, or complex types) + # Unwrap Optional[X] → X (handles both typing.Union and types.UnionType) + origin = get_origin(field_type) + if origin is Union: + args = [a for a in get_args(field_type) if a is not type(None)] + if len(args) == 1: + field_type = args[0] + elif hasattr(field_type, '__args__') and type(None) in getattr(field_type, '__args__', ()): + args = [a for a in field_type.__args__ if a is not type(None)] + if len(args) == 1: + field_type = args[0] + if _isJsonbType(field_type): fields[field_name] = "JSONB" - elif field_type in (str, type(None)) or ( - get_origin(field_type) is Union and type(None) in get_args(field_type) - ): - fields[field_name] = "TEXT" - elif field_type == int: - fields[field_name] = "INTEGER" - elif field_type == float: - fields[field_name] = "DOUBLE PRECISION" - elif field_type == bool: + elif field_type is bool: fields[field_name] = "BOOLEAN" + elif field_type is int: + fields[field_name] = "INTEGER" + elif field_type is float: + fields[field_name] = "DOUBLE PRECISION" + elif field_type in (str, type(None)): + fields[field_name] = "TEXT" else: fields[field_name] = "TEXT" @@ -135,6 +144,9 @@ def _parseRecordFields(record: Dict[str, Any], fields: Dict[str, str], context: elif isinstance(value, list): pass # already a list + elif fieldType == "BOOLEAN": + record[fieldName] = bool(value) if value is not None else False + elif fieldType == "JSONB" and value is not None: try: if isinstance(value, str): @@ -969,6 +981,244 @@ class DatabaseConnector: logger.error(f"Error loading records from table {table}: {e}") return [] + def _buildPaginationClauses( + self, + model_class: type, + pagination, + recordFilter: Dict[str, Any] = None, + ): + """ + Translate PaginationParams + recordFilter into SQL clauses. + Returns (where_clause, order_clause, limit_clause, values, count_values). + """ + fields = _get_model_fields(model_class) + validColumns = set(fields.keys()) + where_parts: List[str] = [] + values: List[Any] = [] + + if recordFilter: + for field, value in recordFilter.items(): + if value is None: + where_parts.append(f'"{field}" IS NULL') + elif isinstance(value, list): + where_parts.append(f'"{field}" = ANY(%s)') + values.append(value) + else: + where_parts.append(f'"{field}" = %s') + values.append(value) + + if pagination and pagination.filters: + for key, val in pagination.filters.items(): + if key == "search" and isinstance(val, str) and val.strip(): + term = f"%{val.strip()}%" + textCols = [c for c, t in fields.items() if t == "TEXT"] + if textCols: + orParts = [f'COALESCE("{c}"::TEXT, \'\') ILIKE %s' for c in textCols] + where_parts.append(f"({' OR '.join(orParts)})") + values.extend([term] * len(textCols)) + continue + if key not in validColumns: + logger.debug(f"_buildPaginationClauses: key '{key}' NOT in validColumns {list(validColumns)[:10]}") + continue + colType = fields.get(key, "TEXT") + logger.debug(f"_buildPaginationClauses: filter key='{key}' val={val!r} type(val)={type(val).__name__} colType={colType}") + if isinstance(val, dict): + op = val.get("operator", "equals") + v = val.get("value", "") + if op in ("equals", "eq"): + if colType == "BOOLEAN": + where_parts.append(f'COALESCE("{key}", FALSE) = %s') + values.append(str(v).lower() == "true") + else: + where_parts.append(f'"{key}"::TEXT = %s') + values.append(str(v)) + elif op == "contains": + where_parts.append(f'"{key}"::TEXT ILIKE %s') + values.append(f"%{v}%") + elif op == "startsWith": + where_parts.append(f'"{key}"::TEXT ILIKE %s') + values.append(f"{v}%") + elif op == "endsWith": + where_parts.append(f'"{key}"::TEXT ILIKE %s') + values.append(f"%{v}") + elif op in ("gt", "gte", "lt", "lte"): + sqlOp = {"gt": ">", "gte": ">=", "lt": "<", "lte": "<="}[op] + where_parts.append(f'"{key}"::TEXT {sqlOp} %s') + values.append(str(v)) + elif op == "between": + fromVal = v.get("from", "") if isinstance(v, dict) else "" + toVal = v.get("to", "") if isinstance(v, dict) else "" + if not fromVal and not toVal: + continue + colType = fields.get(key, "TEXT") + isNumericCol = colType in ("INTEGER", "DOUBLE PRECISION") + isDateVal = bool(fromVal and re.match(r'^\d{4}-\d{2}-\d{2}$', str(fromVal))) or \ + bool(toVal and re.match(r'^\d{4}-\d{2}-\d{2}$', str(toVal))) + if isNumericCol and isDateVal: + from datetime import datetime as _dt, timezone as _tz + if fromVal and toVal: + fromTs = _dt.strptime(str(fromVal), '%Y-%m-%d').replace(tzinfo=_tz.utc).timestamp() + toTs = _dt.strptime(str(toVal), '%Y-%m-%d').replace(hour=23, minute=59, second=59, tzinfo=_tz.utc).timestamp() + where_parts.append(f'"{key}" >= %s AND "{key}" <= %s') + values.extend([fromTs, toTs]) + elif fromVal: + fromTs = _dt.strptime(str(fromVal), '%Y-%m-%d').replace(tzinfo=_tz.utc).timestamp() + where_parts.append(f'"{key}" >= %s') + values.append(fromTs) + else: + toTs = _dt.strptime(str(toVal), '%Y-%m-%d').replace(hour=23, minute=59, second=59, tzinfo=_tz.utc).timestamp() + where_parts.append(f'"{key}" <= %s') + values.append(toTs) + else: + if fromVal and toVal: + where_parts.append(f'"{key}"::TEXT >= %s AND "{key}"::TEXT <= %s') + values.extend([str(fromVal), str(toVal)]) + elif fromVal: + where_parts.append(f'"{key}"::TEXT >= %s') + values.append(str(fromVal)) + elif toVal: + where_parts.append(f'"{key}"::TEXT <= %s') + values.append(str(toVal)) + else: + if colType == "BOOLEAN": + where_parts.append(f'COALESCE("{key}", FALSE) = %s') + values.append(str(val).lower() == "true") + else: + where_parts.append(f'"{key}"::TEXT ILIKE %s') + values.append(str(val)) + + where_clause = " WHERE " + " AND ".join(where_parts) if where_parts else "" + count_values = list(values) + + orderParts: List[str] = [] + if pagination and pagination.sort: + for sf in pagination.sort: + if sf.field in validColumns: + direction = "DESC" if sf.direction.lower() == "desc" else "ASC" + colType = fields.get(sf.field, "TEXT") + if colType == "BOOLEAN": + orderParts.append(f'COALESCE("{sf.field}", FALSE) {direction}') + else: + orderParts.append(f'"{sf.field}" {direction} NULLS LAST') + if not orderParts: + orderParts.append('"id"') + order_clause = " ORDER BY " + ", ".join(orderParts) + + limit_clause = "" + if pagination: + offset = (pagination.page - 1) * pagination.pageSize + limit_clause = f" LIMIT {pagination.pageSize} OFFSET {offset}" + + return where_clause, order_clause, limit_clause, values, count_values + + def getRecordsetPaginated( + self, + model_class: type, + pagination=None, + recordFilter: Dict[str, Any] = None, + fieldFilter: List[str] = None, + ) -> Dict[str, Any]: + """ + Returns paginated records with filtering + sorting at the SQL level. + Returns { "items": [...], "totalItems": int, "totalPages": int }. + If pagination is None, returns all records (no LIMIT/OFFSET). + """ + from modules.datamodels.datamodelPagination import PaginationParams + import math + + table = model_class.__name__ + + try: + if not self._ensureTableExists(model_class): + return {"items": [], "totalItems": 0, "totalPages": 0} + + where_clause, order_clause, limit_clause, values, count_values = \ + self._buildPaginationClauses(model_class, pagination, recordFilter) + + with self.connection.cursor() as cursor: + countSql = f'SELECT COUNT(*) FROM "{table}"{where_clause}' + cursor.execute(countSql, count_values) + totalItems = cursor.fetchone()["count"] + + dataSql = f'SELECT * FROM "{table}"{where_clause}{order_clause}{limit_clause}' + cursor.execute(dataSql, values) + records = [dict(row) for row in cursor.fetchall()] + + fields = _get_model_fields(model_class) + modelFields = model_class.model_fields + for record in records: + _parseRecordFields(record, fields, f"table {table}") + for fieldName, fieldType in fields.items(): + if fieldType == "JSONB" and fieldName in record and record[fieldName] is None: + fieldInfo = modelFields.get(fieldName) + if fieldInfo: + fieldAnnotation = fieldInfo.annotation + if (fieldAnnotation == list or + (hasattr(fieldAnnotation, "__origin__") and + fieldAnnotation.__origin__ is list)): + record[fieldName] = [] + elif (fieldAnnotation == dict or + (hasattr(fieldAnnotation, "__origin__") and + fieldAnnotation.__origin__ is dict)): + record[fieldName] = {} + + if fieldFilter and isinstance(fieldFilter, list): + records = [{f: r[f] for f in fieldFilter if f in r} for r in records] + + pageSize = pagination.pageSize if pagination else max(totalItems, 1) + totalPages = math.ceil(totalItems / pageSize) if totalItems > 0 else 0 + + return {"items": records, "totalItems": totalItems, "totalPages": totalPages} + except Exception as e: + logger.error(f"Error in getRecordsetPaginated for table {table}: {e}") + return {"items": [], "totalItems": 0, "totalPages": 0} + + def getDistinctColumnValues( + self, + model_class: type, + column: str, + pagination=None, + recordFilter: Dict[str, Any] = None, + ) -> List[str]: + """ + Returns sorted distinct non-null values for a column using SQL DISTINCT. + Applies cross-filtering (all filters except the requested column). + """ + table = model_class.__name__ + fields = _get_model_fields(model_class) + + if column not in fields: + return [] + + try: + if not self._ensureTableExists(model_class): + return [] + + if pagination: + if pagination.filters and column in pagination.filters: + import copy + pagination = copy.deepcopy(pagination) + pagination.filters.pop(column, None) + + where_clause, _, _, values, _ = \ + self._buildPaginationClauses(model_class, pagination, recordFilter) + + sql = ( + f'SELECT DISTINCT "{column}"::TEXT AS val FROM "{table}"{where_clause} ' + f'WHERE "{column}" IS NOT NULL AND "{column}"::TEXT != \'\' ' + if not where_clause else + f'SELECT DISTINCT "{column}"::TEXT AS val FROM "{table}"{where_clause} ' + f'AND "{column}" IS NOT NULL AND "{column}"::TEXT != \'\' ' + ) + sql += 'ORDER BY val' + + with self.connection.cursor() as cursor: + cursor.execute(sql, values) + return [row["val"] for row in cursor.fetchall()] + except Exception as e: + logger.error(f"Error in getDistinctColumnValues for {table}.{column}: {e}") + return [] + def recordCreate( self, model_class: type, record: Union[Dict[str, Any], BaseModel] ) -> Dict[str, Any]: diff --git a/modules/datamodels/datamodelPagination.py b/modules/datamodels/datamodelPagination.py index 9815027f..2719327b 100644 --- a/modules/datamodels/datamodelPagination.py +++ b/modules/datamodels/datamodelPagination.py @@ -98,6 +98,12 @@ def normalize_pagination_dict(pagination_dict: Dict[str, Any]) -> Dict[str, Any] # Create a copy to avoid modifying the original normalized = dict(pagination_dict) + # Ensure required fields have sensible defaults + if "page" not in normalized: + normalized["page"] = 1 + if "pageSize" not in normalized: + normalized["pageSize"] = 25 + # Move top-level "search" into filters if present if "search" in normalized: if "filters" not in normalized or normalized["filters"] is None: diff --git a/modules/features/automation/interfaceFeatureAutomation.py b/modules/features/automation/interfaceFeatureAutomation.py index 3b20ca3d..3fc2420b 100644 --- a/modules/features/automation/interfaceFeatureAutomation.py +++ b/modules/features/automation/interfaceFeatureAutomation.py @@ -270,7 +270,7 @@ class AutomationObjects: if value.lower() not in itemValue.lower(): match = False break - elif itemValue != value: + elif str(itemValue).lower() != str(value).lower(): match = False break if match: diff --git a/modules/features/automation/routeFeatureAutomation.py b/modules/features/automation/routeFeatureAutomation.py index 6cdc4d44..81f0852b 100644 --- a/modules/features/automation/routeFeatureAutomation.py +++ b/modules/features/automation/routeFeatureAutomation.py @@ -109,6 +109,26 @@ def get_automations( detail=f"Error getting automations: {str(e)}" ) +@router.get("/filter-values") +@limiter.limit("60/minute") +def get_automation_filter_values( + request: Request, + column: str = Query(..., description="Column key"), + pagination: Optional[str] = Query(None, description="JSON-encoded current filters"), + context: RequestContext = Depends(getRequestContext) +) -> list: + """Return distinct filter values for a column in automations.""" + try: + from modules.routes.routeDataUsers import _handleFilterValuesRequest + chatInterface = getAutomationInterface(context.user, mandateId=str(context.mandateId) if context.mandateId else None) + result = chatInterface.getAllAutomationDefinitions(pagination=None) + items = result if isinstance(result, list) else [r if isinstance(r, dict) else r.model_dump() if hasattr(r, 'model_dump') else r for r in result] + return _handleFilterValuesRequest(items, column, pagination) + except Exception as e: + logger.error(f"Error getting filter values for automations: {str(e)}") + raise HTTPException(status_code=500, detail=str(e)) + + @router.post("", response_model=AutomationDefinition) @limiter.limit("10/minute") def create_automation( @@ -1017,6 +1037,30 @@ def get_db_templates( ) +@templateRouter.get("/filter-values") +@limiter.limit("60/minute") +def get_template_filter_values( + request: Request, + column: str = Query(..., description="Column key"), + pagination: Optional[str] = Query(None, description="JSON-encoded current filters"), + context: RequestContext = Depends(getRequestContext) +) -> list: + """Return distinct filter values for a column in automation templates.""" + try: + from modules.routes.routeDataUsers import _handleFilterValuesRequest + chatInterface = getAutomationInterface( + context.user, + mandateId=str(context.mandateId) if context.mandateId else None, + featureInstanceId=str(context.featureInstanceId) if context.featureInstanceId else None + ) + result = chatInterface.getAllAutomationTemplates(pagination=None) + items = [r if isinstance(r, dict) else r.model_dump() if hasattr(r, 'model_dump') else r for r in result] + return _handleFilterValuesRequest(items, column, pagination) + except Exception as e: + logger.error(f"Error getting filter values for automation templates: {str(e)}") + raise HTTPException(status_code=500, detail=str(e)) + + @templateRouter.get("/attributes", response_model=Dict[str, Any]) def get_template_attributes( request: Request diff --git a/modules/features/chatbot/interfaceFeatureChatbot.py b/modules/features/chatbot/interfaceFeatureChatbot.py index 559a9187..4a03bec9 100644 --- a/modules/features/chatbot/interfaceFeatureChatbot.py +++ b/modules/features/chatbot/interfaceFeatureChatbot.py @@ -514,7 +514,7 @@ class ChatObjects: # Handle simple value (equals operator) if not isinstance(filter_value, dict): - if record_value != filter_value: + if str(record_value).lower() != str(filter_value).lower(): matches = False break continue @@ -524,7 +524,7 @@ class ChatObjects: filter_val = filter_value.get("value") if operator in ["equals", "eq"]: - if record_value != filter_val: + if str(record_value).lower() != str(filter_val).lower(): matches = False break @@ -609,7 +609,7 @@ class ChatObjects: else: # Unknown operator - default to equals - if record_value != filter_val: + if str(record_value).lower() != str(filter_val).lower(): matches = False break diff --git a/modules/features/realEstate/interfaceFeatureRealEstate.py b/modules/features/realEstate/interfaceFeatureRealEstate.py index 65601d9a..f7ed52b6 100644 --- a/modules/features/realEstate/interfaceFeatureRealEstate.py +++ b/modules/features/realEstate/interfaceFeatureRealEstate.py @@ -24,7 +24,8 @@ from modules.shared.configuration import APP_CONFIG from modules.security.rbac import RbacClass from modules.datamodels.datamodelRbac import AccessRuleContext from modules.datamodels.datamodelUam import AccessLevel -from modules.interfaces.interfaceRbac import getRecordsetWithRBAC +from modules.interfaces.interfaceRbac import getRecordsetWithRBAC, getRecordsetPaginatedWithRBAC, getDistinctColumnValuesWithRBAC +from modules.datamodels.datamodelPagination import PaginationParams, PaginatedResult logger = logging.getLogger(__name__) @@ -175,17 +176,21 @@ class RealEstateObjects: return Projekt(**records[0]) - def getProjekte(self, recordFilter: Optional[Dict[str, Any]] = None) -> List[Projekt]: - """Get all projects matching the filter.""" - records = getRecordsetWithRBAC( + def getProjekte(self, recordFilter: Optional[Dict[str, Any]] = None, pagination: Optional[PaginationParams] = None) -> Union[List[Projekt], PaginatedResult]: + """Get all projects matching the filter with optional DB-level pagination.""" + result = getRecordsetPaginatedWithRBAC( self.db, Projekt, self.currentUser, + pagination=pagination, recordFilter=recordFilter or {}, featureCode=self.FEATURE_CODE ) - - return [Projekt(**r) for r in records] + + if isinstance(result, PaginatedResult): + result.items = [Projekt(**r) for r in result.items] + return result + return [Projekt(**r) for r in result] def updateProjekt(self, projektId_or_projekt: Union[str, Projekt], updateData: Optional[Dict[str, Any]] = None) -> Optional[Projekt]: """Update a project. @@ -265,20 +270,23 @@ class RealEstateObjects: return Parzelle(**records[0]) - def getParzellen(self, recordFilter: Optional[Dict[str, Any]] = None) -> List[Parzelle]: - """Get all plots matching the filter.""" - # Resolve location names to IDs if needed + def getParzellen(self, recordFilter: Optional[Dict[str, Any]] = None, pagination: Optional[PaginationParams] = None) -> Union[List[Parzelle], PaginatedResult]: + """Get all plots matching the filter with optional DB-level pagination.""" if recordFilter: recordFilter = self._resolveLocationFilters(recordFilter) - - records = getRecordsetWithRBAC( + + result = getRecordsetPaginatedWithRBAC( self.db, Parzelle, self.currentUser, + pagination=pagination, recordFilter=recordFilter or {} ) - - return [Parzelle(**r) for r in records] + + if isinstance(result, PaginatedResult): + result.items = [Parzelle(**r) for r in result.items] + return result + return [Parzelle(**r) for r in result] def _resolveLocationFilters(self, recordFilter: Dict[str, Any]) -> Dict[str, Any]: """ @@ -477,18 +485,23 @@ class RealEstateObjects: return Dokument(**records[0]) - def getDokumente(self, recordFilter: Optional[Dict[str, Any]] = None) -> List[Dokument]: - """Get all documents matching the filter.""" - records = getRecordsetWithRBAC( + def getDokumente(self, recordFilter: Optional[Dict[str, Any]] = None, pagination: Optional[PaginationParams] = None) -> Union[List[Dokument], PaginatedResult]: + """Get all documents matching the filter with optional DB-level pagination.""" + result = getRecordsetPaginatedWithRBAC( self.db, Dokument, self.currentUser, + pagination=pagination, recordFilter=recordFilter or {}, mandateId=self.mandateId, featureInstanceId=self.featureInstanceId, featureCode=self.FEATURE_CODE ) - return [Dokument(**r) for r in records] + + if isinstance(result, PaginatedResult): + result.items = [Dokument(**r) for r in result.items] + return result + return [Dokument(**r) for r in result] def updateDokument(self, dokumentId: str, updateData: Dict[str, Any]) -> Optional[Dokument]: """Update a document.""" @@ -552,18 +565,23 @@ class RealEstateObjects: return Gemeinde(**records[0]) - def getGemeinden(self, recordFilter: Optional[Dict[str, Any]] = None) -> List[Gemeinde]: - """Get all municipalities matching the filter.""" - records = getRecordsetWithRBAC( + def getGemeinden(self, recordFilter: Optional[Dict[str, Any]] = None, pagination: Optional[PaginationParams] = None) -> Union[List[Gemeinde], PaginatedResult]: + """Get all municipalities matching the filter with optional DB-level pagination.""" + result = getRecordsetPaginatedWithRBAC( self.db, Gemeinde, self.currentUser, + pagination=pagination, recordFilter=recordFilter or {}, mandateId=self.mandateId, featureInstanceId=self.featureInstanceId, featureCode=self.FEATURE_CODE ) - return [Gemeinde(**r) for r in records] + + if isinstance(result, PaginatedResult): + result.items = [Gemeinde(**r) for r in result.items] + return result + return [Gemeinde(**r) for r in result] def updateGemeinde(self, gemeindeId: str, updateData: Dict[str, Any]) -> Optional[Gemeinde]: """Update a municipality.""" @@ -627,18 +645,23 @@ class RealEstateObjects: return Kanton(**records[0]) - def getKantone(self, recordFilter: Optional[Dict[str, Any]] = None) -> List[Kanton]: - """Get all cantons matching the filter.""" - records = getRecordsetWithRBAC( + def getKantone(self, recordFilter: Optional[Dict[str, Any]] = None, pagination: Optional[PaginationParams] = None) -> Union[List[Kanton], PaginatedResult]: + """Get all cantons matching the filter with optional DB-level pagination.""" + result = getRecordsetPaginatedWithRBAC( self.db, Kanton, self.currentUser, + pagination=pagination, recordFilter=recordFilter or {}, mandateId=self.mandateId, featureInstanceId=self.featureInstanceId, featureCode=self.FEATURE_CODE ) - return [Kanton(**r) for r in records] + + if isinstance(result, PaginatedResult): + result.items = [Kanton(**r) for r in result.items] + return result + return [Kanton(**r) for r in result] def updateKanton(self, kantonId: str, updateData: Dict[str, Any]) -> Optional[Kanton]: """Update a canton.""" @@ -700,16 +723,21 @@ class RealEstateObjects: return Land(**records[0]) - def getLaender(self, recordFilter: Optional[Dict[str, Any]] = None) -> List[Land]: - """Get all countries matching the filter.""" - records = getRecordsetWithRBAC( + def getLaender(self, recordFilter: Optional[Dict[str, Any]] = None, pagination: Optional[PaginationParams] = None) -> Union[List[Land], PaginatedResult]: + """Get all countries matching the filter with optional DB-level pagination.""" + result = getRecordsetPaginatedWithRBAC( self.db, Land, self.currentUser, + pagination=pagination, recordFilter=recordFilter or {}, featureCode=self.FEATURE_CODE ) - return [Land(**r) for r in records] + + if isinstance(result, PaginatedResult): + result.items = [Land(**r) for r in result.items] + return result + return [Land(**r) for r in result] def updateLand(self, landId: str, updateData: Dict[str, Any]) -> Optional[Land]: """Update a country.""" diff --git a/modules/features/realEstate/routeFeatureRealEstate.py b/modules/features/realEstate/routeFeatureRealEstate.py index b4df86dd..82fa55ba 100644 --- a/modules/features/realEstate/routeFeatureRealEstate.py +++ b/modules/features/realEstate/routeFeatureRealEstate.py @@ -252,6 +252,31 @@ def get_projects( return PaginatedResponse(items=items, pagination=None) +@router.get("/{instanceId}/projects/filter-values") +@limiter.limit("60/minute") +def get_project_filter_values( + request: Request, + instanceId: str = Path(..., description="Feature Instance ID"), + column: str = Query(..., description="Column key"), + pagination: Optional[str] = Query(None, description="JSON-encoded current filters"), + context: RequestContext = Depends(getRequestContext) +) -> list: + """Return distinct filter values for a column in real estate projects.""" + mandateId = _validateInstanceAccess(instanceId, context) + try: + from modules.routes.routeDataUsers import _handleFilterValuesRequest + interface = getRealEstateInterface( + context.user, mandateId=mandateId, featureInstanceId=instanceId + ) + recordFilter = {"featureInstanceId": instanceId} + items = interface.getProjekte(recordFilter=recordFilter) + itemDicts = [i.model_dump() if hasattr(i, 'model_dump') else i for i in items] + return _handleFilterValuesRequest(itemDicts, column, pagination) + except Exception as e: + logger.error(f"Error getting filter values for projects: {str(e)}") + raise HTTPException(status_code=500, detail=str(e)) + + @router.get("/{instanceId}/projects/{projectId}", response_model=Projekt) @limiter.limit("30/minute") def get_project_by_id( @@ -384,6 +409,31 @@ def get_parcels( return PaginatedResponse(items=items, pagination=None) +@router.get("/{instanceId}/parcels/filter-values") +@limiter.limit("60/minute") +def get_parcel_filter_values( + request: Request, + instanceId: str = Path(..., description="Feature Instance ID"), + column: str = Query(..., description="Column key"), + pagination: Optional[str] = Query(None, description="JSON-encoded current filters"), + context: RequestContext = Depends(getRequestContext) +) -> list: + """Return distinct filter values for a column in real estate parcels.""" + mandateId = _validateInstanceAccess(instanceId, context) + try: + from modules.routes.routeDataUsers import _handleFilterValuesRequest + interface = getRealEstateInterface( + context.user, mandateId=mandateId, featureInstanceId=instanceId + ) + recordFilter = {"featureInstanceId": instanceId} + items = interface.getParzellen(recordFilter=recordFilter) + itemDicts = [i.model_dump() if hasattr(i, 'model_dump') else i for i in items] + return _handleFilterValuesRequest(itemDicts, column, pagination) + except Exception as e: + logger.error(f"Error getting filter values for parcels: {str(e)}") + raise HTTPException(status_code=500, detail=str(e)) + + @router.get("/{instanceId}/parcels/{parcelId}", response_model=Parzelle) @limiter.limit("30/minute") def get_parcel_by_id( diff --git a/modules/features/trustee/interfaceFeatureTrustee.py b/modules/features/trustee/interfaceFeatureTrustee.py index a4b13c27..b9a95005 100644 --- a/modules/features/trustee/interfaceFeatureTrustee.py +++ b/modules/features/trustee/interfaceFeatureTrustee.py @@ -14,7 +14,7 @@ from pydantic import ValidationError from modules.connectors.connectorDbPostgre import DatabaseConnector from modules.shared.configuration import APP_CONFIG -from modules.interfaces.interfaceRbac import getRecordsetWithRBAC +from modules.interfaces.interfaceRbac import getRecordsetWithRBAC, getRecordsetPaginatedWithRBAC, getDistinctColumnValuesWithRBAC from modules.security.rbac import RbacClass from modules.datamodels.datamodelUam import User, AccessLevel from modules.datamodels.datamodelRbac import AccessRuleContext @@ -414,7 +414,7 @@ class TrusteeObjects: # Handle simple value (equals operator) if not isinstance(filterValue, dict): - if recordValue == filterValue: + if str(recordValue).lower() == str(filterValue).lower(): fieldFiltered.append(record) continue @@ -424,7 +424,7 @@ class TrusteeObjects: matches = False if operator in ["equals", "eq"]: - matches = recordValue == filterVal + matches = str(recordValue).lower() == str(filterVal).lower() elif operator == "contains": recordStr = str(recordValue).lower() if recordValue is not None else "" @@ -577,8 +577,8 @@ class TrusteeObjects: return None return TrusteeOrganisation(**{k: v for k, v in records[0].items() if not k.startswith("_")}) - def getAllOrganisations(self, params: Optional[PaginationParams] = None) -> PaginatedResult: - """Get all organisations with RBAC filtering. + def getAllOrganisations(self, params: Optional[PaginationParams] = None) -> Union[List[Dict], PaginatedResult]: + """Get all organisations with RBAC filtering and optional DB-level pagination. Note: Organisations are managed at system level (by mandate). Feature-level filtering (trustee.access) is NOT applied here because: @@ -586,42 +586,18 @@ class TrusteeObjects: - trustee.access grants access to specific orgs for other users - New organisations wouldn't be visible without an access record """ - # Debug: Log user info and permissions logger.debug(f"getAllOrganisations called for user {self.userId}, mandateId: {self.mandateId}") - # System RBAC filtering (filters by mandate for GROUP access level) - records = getRecordsetWithRBAC( + return getRecordsetPaginatedWithRBAC( connector=self.db, modelClass=TrusteeOrganisation, currentUser=self.currentUser, + pagination=params, recordFilter=None, - orderBy="id", mandateId=self.mandateId, featureInstanceId=self.featureInstanceId, featureCode=self.FEATURE_CODE ) - logger.debug(f"getAllOrganisations: getRecordsetWithRBAC returned {len(records)} records") - - # Apply pagination - totalItems = len(records) - if params: - pageSize = params.pageSize or 20 - page = params.page or 1 - startIdx = (page - 1) * pageSize - endIdx = startIdx + pageSize - items = records[startIdx:endIdx] - totalPages = math.ceil(totalItems / pageSize) if pageSize > 0 else 1 - else: - items = records - totalPages = 1 - page = 1 - pageSize = totalItems - - return PaginatedResult( - items=items, - totalItems=totalItems, - totalPages=totalPages - ) def updateOrganisation(self, orgId: str, data: Dict[str, Any]) -> Optional[TrusteeOrganisation]: """Update an organisation.""" @@ -677,51 +653,40 @@ class TrusteeObjects: return None return TrusteeRole(**{k: v for k, v in records[0].items() if not k.startswith("_")}) - def getAllRoles(self, params: Optional[PaginationParams] = None) -> PaginatedResult: - """Get all roles with RBAC filtering. + def getAllRoles(self, params: Optional[PaginationParams] = None) -> Union[List[Dict], PaginatedResult]: + """Get all roles with RBAC filtering and optional DB-level pagination. Note: Roles are available to all users with trustee access. They are not filtered by organisation since they define the role types. Users with ALL access level see all roles; others need trustee.access records. + + NOTE(post-filter): Feature-level access check runs after the paginated query. + When pagination is active the totals may overcount if the user lacks any + trustee.access record (in that case the entire result is emptied). """ - records = getRecordsetWithRBAC( + result = getRecordsetPaginatedWithRBAC( connector=self.db, modelClass=TrusteeRole, currentUser=self.currentUser, + pagination=params, recordFilter=None, - orderBy="id", mandateId=self.mandateId, featureInstanceId=self.featureInstanceId, featureCode=self.FEATURE_CODE ) - # Users with ALL access level (from system RBAC) see all roles - # Others need at least one trustee.access record accessLevel = self.getRbacAccessLevel(TrusteeRole, "read") if accessLevel != AccessLevel.ALL: userAccess = self.getAllUserAccess(self.userId) if not userAccess: - records = [] # No trustee access at all + if isinstance(result, PaginatedResult): + result.items = [] + result.totalItems = 0 + result.totalPages = 0 + return result + return [] - totalItems = len(records) - if params: - pageSize = params.pageSize or 20 - page = params.page or 1 - startIdx = (page - 1) * pageSize - endIdx = startIdx + pageSize - items = records[startIdx:endIdx] - totalPages = math.ceil(totalItems / pageSize) if pageSize > 0 else 1 - else: - items = records - totalPages = 1 - page = 1 - pageSize = totalItems - - return PaginatedResult( - items=items, - totalItems=totalItems, - totalPages=totalPages - ) + return result def updateRole(self, roleId: str, data: Dict[str, Any]) -> Optional[TrusteeRole]: """Update a role (sysadmin only).""" @@ -788,116 +753,113 @@ class TrusteeObjects: return None return TrusteeAccess(**{k: v for k, v in records[0].items() if not k.startswith("_")}) - def getAllAccess(self, params: Optional[PaginationParams] = None) -> PaginatedResult: + def getAllAccess(self, params: Optional[PaginationParams] = None) -> Union[List[Dict], PaginatedResult]: """Get all access records with RBAC filtering + feature-level filtering. Users with ALL access level see all access records. Others can only see access records for organisations they have admin access to. + + NOTE(post-filter): Feature-level admin-org filtering runs after the paginated + query, so totals may overcount when the user has restricted org access. """ - # Step 1: System RBAC filtering - records = getRecordsetWithRBAC( + result = getRecordsetPaginatedWithRBAC( connector=self.db, modelClass=TrusteeAccess, currentUser=self.currentUser, + pagination=params, recordFilter=None, - orderBy="id", mandateId=self.mandateId, featureInstanceId=self.featureInstanceId, featureCode=self.FEATURE_CODE ) - # Users with ALL access level (from system RBAC) see all records accessLevel = self.getRbacAccessLevel(TrusteeAccess, "read") - + if accessLevel != AccessLevel.ALL: - # Step 2: Feature-level filtering - only see access for organisations where user is admin userAccess = self.getAllUserAccess(self.userId) - - # Get organisations where user has admin role adminOrgs = set() for access in userAccess: if access.get("roleId") == "admin": adminOrgs.add(access.get("organisationId")) - - # Filter records to only show those in admin organisations + + if isinstance(result, PaginatedResult): + if adminOrgs: + result.items = [r for r in result.items if r.get("organisationId") in adminOrgs] + else: + result.items = [] + return result + if adminOrgs: - records = [r for r in records if r.get("organisationId") in adminOrgs] + result = [r for r in result if r.get("organisationId") in adminOrgs] else: - records = [] + result = [] - totalItems = len(records) - if params: - pageSize = params.pageSize or 20 - page = params.page or 1 - startIdx = (page - 1) * pageSize - endIdx = startIdx + pageSize - items = records[startIdx:endIdx] - totalPages = math.ceil(totalItems / pageSize) if pageSize > 0 else 1 - else: - items = records - totalPages = 1 - page = 1 - pageSize = totalItems + return result - return PaginatedResult( - items=items, - totalItems=totalItems, - totalPages=totalPages - ) - - def getAccessByOrganisation(self, organisationId: str) -> List[TrusteeAccess]: + def getAccessByOrganisation(self, organisationId: str, pagination: Optional[PaginationParams] = None) -> Union[List[TrusteeAccess], PaginatedResult]: """Get all access records for a specific organisation. Requires admin role for the organisation. """ - # Check if user has admin access for this organisation if not self.checkUserTrusteePermission(self.userId, organisationId, "admin"): logger.warning(f"User {self.userId} lacks admin role for organisation {organisationId}") return [] - - records = getRecordsetWithRBAC( + + result = getRecordsetPaginatedWithRBAC( connector=self.db, modelClass=TrusteeAccess, currentUser=self.currentUser, + pagination=pagination, recordFilter={"organisationId": organisationId}, - orderBy="id", mandateId=self.mandateId, featureInstanceId=self.featureInstanceId, featureCode=self.FEATURE_CODE ) - return [TrusteeAccess(**{k: v for k, v in r.items() if not k.startswith("_")}) for r in records] - def getAccessByUser(self, userId: str) -> List[TrusteeAccess]: + if isinstance(result, PaginatedResult): + result.items = [TrusteeAccess(**{k: v for k, v in r.items() if not k.startswith("_")}) for r in result.items] + return result + return [TrusteeAccess(**{k: v for k, v in r.items() if not k.startswith("_")}) for r in result] + + def getAccessByUser(self, userId: str, pagination: Optional[PaginationParams] = None) -> Union[List[TrusteeAccess], PaginatedResult]: """Get all access records for a specific user. Users with ALL access level see all access records. Others can only see access records for organisations where they have admin role. + + NOTE(post-filter): Admin-org filtering runs after the paginated query, + so totals may overcount when the user has restricted org access. """ - records = getRecordsetWithRBAC( + result = getRecordsetPaginatedWithRBAC( connector=self.db, modelClass=TrusteeAccess, currentUser=self.currentUser, + pagination=pagination, recordFilter={"userId": userId}, - orderBy="id", mandateId=self.mandateId, featureInstanceId=self.featureInstanceId, featureCode=self.FEATURE_CODE ) - - # Users with ALL access level (from system RBAC) see all records + accessLevel = self.getRbacAccessLevel(TrusteeAccess, "read") - if accessLevel == AccessLevel.ALL: - return [TrusteeAccess(**{k: v for k, v in r.items() if not k.startswith("_")}) for r in records] - - # Filter to only organisations where current user has admin role - userAccess = self.getAllUserAccess(self.userId) - adminOrgs = set() - for access in userAccess: - if access.get("roleId") == "admin": - adminOrgs.add(access.get("organisationId")) - - filtered = [r for r in records if r.get("organisationId") in adminOrgs] - return [TrusteeAccess(**{k: v for k, v in r.items() if not k.startswith("_")}) for r in filtered] + + if accessLevel != AccessLevel.ALL: + userAccess = self.getAllUserAccess(self.userId) + adminOrgs = set() + for access in userAccess: + if access.get("roleId") == "admin": + adminOrgs.add(access.get("organisationId")) + + if isinstance(result, PaginatedResult): + result.items = [r for r in result.items if r.get("organisationId") in adminOrgs] + result.items = [TrusteeAccess(**{k: v for k, v in r.items() if not k.startswith("_")}) for r in result.items] + return result + result = [r for r in result if r.get("organisationId") in adminOrgs] + + if isinstance(result, PaginatedResult): + result.items = [TrusteeAccess(**{k: v for k, v in r.items() if not k.startswith("_")}) for r in result.items] + return result + return [TrusteeAccess(**{k: v for k, v in r.items() if not k.startswith("_")}) for r in result] def updateAccess(self, accessId: str, data: Dict[str, Any]) -> Optional[TrusteeAccess]: """Update an access record. Requires admin role for the organisation or ALL access level.""" @@ -988,54 +950,36 @@ class TrusteeObjects: return None return TrusteeContract(**{k: v for k, v in records[0].items() if not k.startswith("_")}) - def getAllContracts(self, params: Optional[PaginationParams] = None) -> PaginatedResult: - """Get all contracts with RBAC filtering + feature-level access filtering.""" - # Step 1: System RBAC filtering - records = getRecordsetWithRBAC( + def getAllContracts(self, params: Optional[PaginationParams] = None) -> Union[List[Dict], PaginatedResult]: + """Get all contracts with RBAC filtering and optional DB-level pagination.""" + return getRecordsetPaginatedWithRBAC( connector=self.db, modelClass=TrusteeContract, currentUser=self.currentUser, + pagination=params, recordFilter=None, - orderBy="id", mandateId=self.mandateId, featureInstanceId=self.featureInstanceId, featureCode=self.FEATURE_CODE ) - totalItems = len(records) - if params: - pageSize = params.pageSize or 20 - page = params.page or 1 - startIdx = (page - 1) * pageSize - endIdx = startIdx + pageSize - items = records[startIdx:endIdx] - totalPages = math.ceil(totalItems / pageSize) if pageSize > 0 else 1 - else: - items = records - totalPages = 1 - page = 1 - pageSize = totalItems - - return PaginatedResult( - items=items, - totalItems=totalItems, - totalPages=totalPages - ) - - def getContractsByOrganisation(self, organisationId: str) -> List[TrusteeContract]: + def getContractsByOrganisation(self, organisationId: str, pagination: Optional[PaginationParams] = None) -> Union[List[TrusteeContract], PaginatedResult]: """Get all contracts for a specific organisation.""" - # Step 1: System RBAC filtering - records = getRecordsetWithRBAC( + result = getRecordsetPaginatedWithRBAC( connector=self.db, modelClass=TrusteeContract, currentUser=self.currentUser, + pagination=pagination, recordFilter={"organisationId": organisationId}, - orderBy="label", mandateId=self.mandateId, featureInstanceId=self.featureInstanceId, featureCode=self.FEATURE_CODE ) - return [TrusteeContract(**{k: v for k, v in r.items() if not k.startswith("_")}) for r in records] + + if isinstance(result, PaginatedResult): + result.items = [TrusteeContract(**{k: v for k, v in r.items() if not k.startswith("_")}) for r in result.items] + return result + return [TrusteeContract(**{k: v for k, v in r.items() if not k.startswith("_")}) for r in result] def updateContract(self, contractId: str, data: Dict[str, Any]) -> Optional[TrusteeContract]: """Update a contract (organisationId is immutable).""" @@ -1142,75 +1086,58 @@ class TrusteeObjects: # Legacy fallback: documentData was stored directly (for migration) return record.get("documentData") - def getAllDocuments(self, params: Optional[PaginationParams] = None) -> PaginatedResult: - """Get all documents with RBAC filtering + feature-level access filtering (metadata only).""" - # Step 1: System RBAC filtering - records = getRecordsetWithRBAC( + def getAllDocuments(self, params: Optional[PaginationParams] = None) -> Union[List[Dict], PaginatedResult]: + """Get all documents with RBAC filtering and optional DB-level pagination (metadata only). + + Filtering, sorting, and pagination are handled at the SQL level by + getRecordsetPaginatedWithRBAC. Binary documentData is stripped from + the returned items. + """ + result = getRecordsetPaginatedWithRBAC( connector=self.db, modelClass=TrusteeDocument, currentUser=self.currentUser, + pagination=params, recordFilter=None, - orderBy="documentName", mandateId=self.mandateId, featureInstanceId=self.featureInstanceId, featureCode=self.FEATURE_CODE ) - # Clean records (remove binary data and internal fields) - keep as dicts for filtering/sorting - cleanedRecords = [] - for record in records: - cleanedRecord = {k: v for k, v in record.items() if not k.startswith("_") and k != "documentData"} - cleanedRecords.append(cleanedRecord) + def _cleanDocumentRecords(records): + return [ + TrusteeDocument(**{k: v for k, v in r.items() if not k.startswith("_") and k != "documentData"}) + for r in records + ] - # Step 2: Apply filters (search and field filters) - filteredRecords = self._applyFilters(cleanedRecords, params) - - # Step 3: Apply sorting - sortedRecords = self._applySorting(filteredRecords, params) - - # Step 4: Convert to Pydantic objects - pydanticItems = [TrusteeDocument(**r) for r in sortedRecords] + if isinstance(result, PaginatedResult): + result.items = _cleanDocumentRecords(result.items) + return result + return _cleanDocumentRecords(result) - # Step 5: Apply pagination - totalItems = len(pydanticItems) - if params: - pageSize = params.pageSize or 20 - page = params.page or 1 - startIdx = (page - 1) * pageSize - endIdx = startIdx + pageSize - items = pydanticItems[startIdx:endIdx] - totalPages = math.ceil(totalItems / pageSize) if pageSize > 0 else 1 - else: - items = pydanticItems - totalPages = 1 - page = 1 - pageSize = totalItems - - return PaginatedResult( - items=items, - totalItems=totalItems, - totalPages=totalPages - ) - - def getDocumentsByContract(self, contractId: str) -> List[TrusteeDocument]: + def getDocumentsByContract(self, contractId: str, pagination: Optional[PaginationParams] = None) -> Union[List[TrusteeDocument], PaginatedResult]: """Get all documents for a specific contract.""" - # Step 1: System RBAC filtering - records = getRecordsetWithRBAC( + result = getRecordsetPaginatedWithRBAC( connector=self.db, modelClass=TrusteeDocument, currentUser=self.currentUser, + pagination=pagination, recordFilter={"contractId": contractId}, - orderBy="documentName", mandateId=self.mandateId, featureInstanceId=self.featureInstanceId, featureCode=self.FEATURE_CODE ) - - result = [] - for record in records: - cleanedRecord = {k: v for k, v in record.items() if not k.startswith("_") and k != "documentData"} - result.append(TrusteeDocument(**cleanedRecord)) - return result + + def _cleanDocumentRecords(records): + return [ + TrusteeDocument(**{k: v for k, v in r.items() if not k.startswith("_") and k != "documentData"}) + for r in records + ] + + if isinstance(result, PaginatedResult): + result.items = _cleanDocumentRecords(result.items) + return result + return _cleanDocumentRecords(result) def updateDocument(self, documentId: str, data: Dict[str, Any]) -> Optional[TrusteeDocument]: """Update a document. @@ -1340,101 +1267,94 @@ class TrusteeObjects: return None return self._toTrusteePositionOrDelete(records[0], deleteCorrupt=True) - def getAllPositions(self, params: Optional[PaginationParams] = None) -> PaginatedResult: - """Get all positions with RBAC filtering + feature-level access filtering.""" - # Step 1: System RBAC filtering - records = getRecordsetWithRBAC( + def getAllPositions(self, params: Optional[PaginationParams] = None) -> Union[List[Dict], PaginatedResult]: + """Get all positions with RBAC filtering and optional DB-level pagination. + + Filtering, sorting, and pagination are handled at the SQL level. + Post-processing cleans internal fields (keeps _createdAt) and validates + each record via _toTrusteePositionOrDelete (corrupt rows are deleted). + + NOTE(post-process): totalItems may slightly overcount when corrupt legacy + records are removed from the current page. + """ + result = getRecordsetPaginatedWithRBAC( connector=self.db, modelClass=TrusteePosition, currentUser=self.currentUser, + pagination=params, recordFilter=None, - orderBy="valuta", mandateId=self.mandateId, featureInstanceId=self.featureInstanceId, featureCode=self.FEATURE_CODE ) - # Clean records (remove internal fields except _createdAt) - keep as dicts for filtering/sorting - # Keep _createdAt for display in frontend keepFields = {'_createdAt'} - cleanedRecords = [] - for record in records: - cleanedRecord = {k: v for k, v in record.items() if not k.startswith("_") or k in keepFields} - cleanedRecords.append(cleanedRecord) - # Step 2: Apply filters (search and field filters) - filteredRecords = self._applyFilters(cleanedRecords, params) - - # Step 3: Apply sorting - sortedRecords = self._applySorting(filteredRecords, params) - - # Step 4: Convert to Pydantic objects and cleanup corrupt legacy records. - pydanticItems = [] - for record in sortedRecords: - position = self._toTrusteePositionOrDelete(record, deleteCorrupt=True) - if position is not None: - pydanticItems.append(position) + def _cleanAndValidate(records): + items = [] + for record in records: + cleanedRecord = {k: v for k, v in record.items() if not k.startswith("_") or k in keepFields} + position = self._toTrusteePositionOrDelete(cleanedRecord, deleteCorrupt=True) + if position is not None: + items.append(position) + return items - # Step 5: Apply pagination - totalItems = len(pydanticItems) - if params: - pageSize = params.pageSize or 20 - page = params.page or 1 - startIdx = (page - 1) * pageSize - endIdx = startIdx + pageSize - items = pydanticItems[startIdx:endIdx] - totalPages = math.ceil(totalItems / pageSize) if pageSize > 0 else 1 - else: - items = pydanticItems - totalPages = 1 - page = 1 - pageSize = totalItems + if isinstance(result, PaginatedResult): + result.items = _cleanAndValidate(result.items) + return result + return _cleanAndValidate(result) - return PaginatedResult( - items=items, - totalItems=totalItems, - totalPages=totalPages - ) - - def getPositionsByContract(self, contractId: str) -> List[TrusteePosition]: + def getPositionsByContract(self, contractId: str, pagination: Optional[PaginationParams] = None) -> Union[List[TrusteePosition], PaginatedResult]: """Get all positions for a specific contract.""" - # Step 1: System RBAC filtering - records = getRecordsetWithRBAC( + result = getRecordsetPaginatedWithRBAC( connector=self.db, modelClass=TrusteePosition, currentUser=self.currentUser, + pagination=pagination, recordFilter={"contractId": contractId}, - orderBy="valuta", mandateId=self.mandateId, featureInstanceId=self.featureInstanceId, featureCode=self.FEATURE_CODE ) - safeItems = [] - for record in records: - position = self._toTrusteePositionOrDelete(record, deleteCorrupt=True) - if position is not None: - safeItems.append(position) - return safeItems - def getPositionsByOrganisation(self, organisationId: str) -> List[TrusteePosition]: + def _validatePositions(records): + items = [] + for record in records: + position = self._toTrusteePositionOrDelete(record, deleteCorrupt=True) + if position is not None: + items.append(position) + return items + + if isinstance(result, PaginatedResult): + result.items = _validatePositions(result.items) + return result + return _validatePositions(result) + + def getPositionsByOrganisation(self, organisationId: str, pagination: Optional[PaginationParams] = None) -> Union[List[TrusteePosition], PaginatedResult]: """Get all positions for a specific organisation.""" - # Step 1: System RBAC filtering - records = getRecordsetWithRBAC( + result = getRecordsetPaginatedWithRBAC( connector=self.db, modelClass=TrusteePosition, currentUser=self.currentUser, + pagination=pagination, recordFilter={"organisationId": organisationId}, - orderBy="valuta", mandateId=self.mandateId, featureInstanceId=self.featureInstanceId, featureCode=self.FEATURE_CODE ) - safeItems = [] - for record in records: - position = self._toTrusteePositionOrDelete(record, deleteCorrupt=True) - if position is not None: - safeItems.append(position) - return safeItems + + def _validatePositions(records): + items = [] + for record in records: + position = self._toTrusteePositionOrDelete(record, deleteCorrupt=True) + if position is not None: + items.append(position) + return items + + if isinstance(result, PaginatedResult): + result.items = _validatePositions(result.items) + return result + return _validatePositions(result) def updatePosition(self, positionId: str, data: Dict[str, Any]) -> Optional[TrusteePosition]: """Update a position. @@ -1481,24 +1401,31 @@ class TrusteeObjects: # ===== Position-Document Queries ===== - def getPositionsByDocument(self, documentId: str) -> List[TrusteePosition]: + def getPositionsByDocument(self, documentId: str, pagination: Optional[PaginationParams] = None) -> Union[List[TrusteePosition], PaginatedResult]: """Get all positions that reference a specific document (1:N via documentId FK).""" - records = getRecordsetWithRBAC( + result = getRecordsetPaginatedWithRBAC( connector=self.db, modelClass=TrusteePosition, currentUser=self.currentUser, + pagination=pagination, recordFilter={"documentId": documentId}, - orderBy="valuta", mandateId=self.mandateId, featureInstanceId=self.featureInstanceId, featureCode=self.FEATURE_CODE ) - safeItems = [] - for record in records: - position = self._toTrusteePositionOrDelete(record, deleteCorrupt=True) - if position is not None: - safeItems.append(position) - return safeItems + + def _validatePositions(records): + items = [] + for record in records: + position = self._toTrusteePositionOrDelete(record, deleteCorrupt=True) + if position is not None: + items.append(position) + return items + + if isinstance(result, PaginatedResult): + result.items = _validatePositions(result.items) + return result + return _validatePositions(result) # ===== Trustee-specific Access Check ===== diff --git a/modules/features/trustee/routeFeatureTrustee.py b/modules/features/trustee/routeFeatureTrustee.py index feb873ae..13b28b07 100644 --- a/modules/features/trustee/routeFeatureTrustee.py +++ b/modules/features/trustee/routeFeatureTrustee.py @@ -338,7 +338,7 @@ def get_organisations( interface = getInterface(context.user, mandateId=mandateId, featureInstanceId=instanceId) result = interface.getAllOrganisations(paginationParams) - if paginationParams: + if paginationParams and hasattr(result, 'items'): return PaginatedResponse( items=result.items, pagination=PaginationMetadata( @@ -350,7 +350,7 @@ def get_organisations( filters=paginationParams.filters if paginationParams else None ) ) - return PaginatedResponse(items=result.items, pagination=None) + return PaginatedResponse(items=result if isinstance(result, list) else result.items, pagination=None) @router.get("/{instanceId}/organisations/{orgId}", response_model=TrusteeOrganisation) @@ -451,7 +451,7 @@ def get_roles( interface = getInterface(context.user, mandateId=mandateId, featureInstanceId=instanceId) result = interface.getAllRoles(paginationParams) - if paginationParams: + if paginationParams and hasattr(result, 'items'): return PaginatedResponse( items=result.items, pagination=PaginationMetadata( @@ -463,7 +463,7 @@ def get_roles( filters=paginationParams.filters if paginationParams else None ) ) - return PaginatedResponse(items=result.items, pagination=None) + return PaginatedResponse(items=result if isinstance(result, list) else result.items, pagination=None) @router.get("/{instanceId}/roles/{roleId}", response_model=TrusteeRole) @@ -564,7 +564,7 @@ def get_all_access( interface = getInterface(context.user, mandateId=mandateId, featureInstanceId=instanceId) result = interface.getAllAccess(paginationParams) - if paginationParams: + if paginationParams and hasattr(result, 'items'): return PaginatedResponse( items=result.items, pagination=PaginationMetadata( @@ -576,7 +576,7 @@ def get_all_access( filters=paginationParams.filters if paginationParams else None ) ) - return PaginatedResponse(items=result.items, pagination=None) + return PaginatedResponse(items=result if isinstance(result, list) else result.items, pagination=None) @router.get("/{instanceId}/access/{accessId}", response_model=TrusteeAccess) @@ -707,7 +707,7 @@ def get_contracts( interface = getInterface(context.user, mandateId=mandateId, featureInstanceId=instanceId) result = interface.getAllContracts(paginationParams) - if paginationParams: + if paginationParams and hasattr(result, 'items'): return PaginatedResponse( items=result.items, pagination=PaginationMetadata( @@ -719,7 +719,7 @@ def get_contracts( filters=paginationParams.filters if paginationParams else None ) ) - return PaginatedResponse(items=result.items, pagination=None) + return PaginatedResponse(items=result if isinstance(result, list) else result.items, pagination=None) @router.get("/{instanceId}/contracts/{contractId}", response_model=TrusteeContract) @@ -835,7 +835,7 @@ def get_documents( interface = getInterface(context.user, mandateId=mandateId, featureInstanceId=instanceId) result = interface.getAllDocuments(paginationParams) - if paginationParams: + if paginationParams and hasattr(result, 'items'): return PaginatedResponse( items=result.items, pagination=PaginationMetadata( @@ -847,7 +847,59 @@ def get_documents( filters=paginationParams.filters if paginationParams else None ) ) - return PaginatedResponse(items=result.items, pagination=None) + return PaginatedResponse(items=result if isinstance(result, list) else result.items, pagination=None) + + +@router.get("/{instanceId}/documents/filter-values") +@limiter.limit("60/minute") +def get_document_filter_values( + request: Request, + instanceId: str = Path(..., description="Feature Instance ID"), + column: str = Query(..., description="Column key"), + pagination: Optional[str] = Query(None, description="JSON-encoded current filters"), + context: RequestContext = Depends(getRequestContext) +) -> list: + """Return distinct filter values for a column in trustee documents.""" + mandateId = _validateInstanceAccess(instanceId, context) + try: + from modules.interfaces.interfaceRbac import getDistinctColumnValuesWithRBAC + interface = getInterface(context.user, mandateId=mandateId, featureInstanceId=instanceId) + + crossFilterPagination = None + if pagination: + try: + paginationDict = json.loads(pagination) + if paginationDict: + paginationDict = normalize_pagination_dict(paginationDict) + filters = paginationDict.get("filters", {}) + filters.pop(column, None) + paginationDict["filters"] = filters + paginationDict.pop("sort", None) + crossFilterPagination = PaginationParams(**paginationDict) + except (json.JSONDecodeError, ValueError): + pass + + try: + values = getDistinctColumnValuesWithRBAC( + connector=interface.db, + modelClass=TrusteeDocument, + column=column, + currentUser=interface.currentUser, + pagination=crossFilterPagination, + recordFilter=None, + mandateId=interface.mandateId, + featureInstanceId=interface.featureInstanceId, + featureCode=interface.FEATURE_CODE + ) + return sorted(values, key=lambda v: str(v).lower()) + except Exception: + from modules.routes.routeDataUsers import _handleFilterValuesRequest + result = interface.getAllDocuments(None) + items = [r.model_dump() if hasattr(r, 'model_dump') else r for r in (result.items if hasattr(result, 'items') else result)] + return _handleFilterValuesRequest(items, column, pagination) + except Exception as e: + logger.error(f"Error getting filter values for trustee documents: {str(e)}") + raise HTTPException(status_code=500, detail=str(e)) @router.get("/{instanceId}/documents/{documentId}", response_model=TrusteeDocument) @@ -1039,7 +1091,7 @@ def get_positions( interface = getInterface(context.user, mandateId=mandateId, featureInstanceId=instanceId) result = interface.getAllPositions(paginationParams) - if paginationParams: + if paginationParams and hasattr(result, 'items'): return PaginatedResponse( items=result.items, pagination=PaginationMetadata( @@ -1051,7 +1103,59 @@ def get_positions( filters=paginationParams.filters if paginationParams else None ) ) - return PaginatedResponse(items=result.items, pagination=None) + return PaginatedResponse(items=result if isinstance(result, list) else result.items, pagination=None) + + +@router.get("/{instanceId}/positions/filter-values") +@limiter.limit("60/minute") +def get_position_filter_values( + request: Request, + instanceId: str = Path(..., description="Feature Instance ID"), + column: str = Query(..., description="Column key"), + pagination: Optional[str] = Query(None, description="JSON-encoded current filters"), + context: RequestContext = Depends(getRequestContext) +) -> list: + """Return distinct filter values for a column in trustee positions.""" + mandateId = _validateInstanceAccess(instanceId, context) + try: + from modules.interfaces.interfaceRbac import getDistinctColumnValuesWithRBAC + interface = getInterface(context.user, mandateId=mandateId, featureInstanceId=instanceId) + + crossFilterPagination = None + if pagination: + try: + paginationDict = json.loads(pagination) + if paginationDict: + paginationDict = normalize_pagination_dict(paginationDict) + filters = paginationDict.get("filters", {}) + filters.pop(column, None) + paginationDict["filters"] = filters + paginationDict.pop("sort", None) + crossFilterPagination = PaginationParams(**paginationDict) + except (json.JSONDecodeError, ValueError): + pass + + try: + values = getDistinctColumnValuesWithRBAC( + connector=interface.db, + modelClass=TrusteePosition, + column=column, + currentUser=interface.currentUser, + pagination=crossFilterPagination, + recordFilter=None, + mandateId=interface.mandateId, + featureInstanceId=interface.featureInstanceId, + featureCode=interface.FEATURE_CODE + ) + return sorted(values, key=lambda v: str(v).lower()) + except Exception: + from modules.routes.routeDataUsers import _handleFilterValuesRequest + result = interface.getAllPositions(None) + items = [r.model_dump() if hasattr(r, 'model_dump') else r for r in (result.items if hasattr(result, 'items') else result)] + return _handleFilterValuesRequest(items, column, pagination) + except Exception as e: + logger.error(f"Error getting filter values for trustee positions: {str(e)}") + raise HTTPException(status_code=500, detail=str(e)) @router.get("/{instanceId}/positions/{positionId}", response_model=TrusteePosition) diff --git a/modules/interfaces/interfaceDbApp.py b/modules/interfaces/interfaceDbApp.py index 092fd589..12eb935b 100644 --- a/modules/interfaces/interfaceDbApp.py +++ b/modules/interfaces/interfaceDbApp.py @@ -423,7 +423,7 @@ class AppObjects: else: # Unknown operator - default to equals - if record_value != filter_val: + if str(record_value).lower() != str(filter_val).lower(): matches = False break @@ -512,53 +512,34 @@ class AppObjects: If pagination is None: List[User] If pagination is provided: PaginatedResult with items and metadata """ - # Get user IDs via UserMandate junction table (UserInDB has no mandateId column) userMandates = self.db.getRecordset(UserMandate, recordFilter={"mandateId": mandateId}) userIds = [um.get("userId") for um in userMandates if um.get("userId")] - - # Fetch each user by ID - filteredUsers = [] - for userId in userIds: - userRecords = self.db.getRecordset(UserInDB, recordFilter={"id": userId}) - if userRecords: - cleanedUser = {k: v for k, v in userRecords[0].items() if not k.startswith("_")} - if cleanedUser.get("roleLabels") is None: - cleanedUser["roleLabels"] = [] - filteredUsers.append(cleanedUser) - # If no pagination requested, return all items + if not userIds: + if pagination is None: + return [] + return PaginatedResult(items=[], totalItems=0, totalPages=0) + + result = self.db.getRecordsetPaginated( + UserInDB, + pagination=pagination, + recordFilter={"id": userIds} + ) + + items = [] + for record in result["items"]: + cleanedUser = {k: v for k, v in record.items() if not k.startswith("_")} + if cleanedUser.get("roleLabels") is None: + cleanedUser["roleLabels"] = [] + items.append(User(**cleanedUser)) + if pagination is None: - return [User(**user) for user in filteredUsers] - - # Apply filtering (if filters provided) - if pagination.filters: - filteredUsers = self._applyFilters(filteredUsers, pagination.filters) - - # Apply sorting (in order of sortFields) - if pagination.sort: - filteredUsers = self._applySorting(filteredUsers, pagination.sort) - - # Count total items after filters - totalItems = len(filteredUsers) - totalPages = math.ceil(totalItems / pagination.pageSize) if totalItems > 0 else 0 - - # Apply pagination (skip/limit) - startIdx = (pagination.page - 1) * pagination.pageSize - endIdx = startIdx + pagination.pageSize - pagedUsers = filteredUsers[startIdx:endIdx] - - # Ensure roleLabels is always a list for paginated results too - for user in pagedUsers: - if user.get("roleLabels") is None: - user["roleLabels"] = [] - - # Convert to model objects - items = [User(**user) for user in pagedUsers] - + return items + return PaginatedResult( items=items, - totalItems=totalItems, - totalPages=totalPages + totalItems=result["totalItems"], + totalPages=result["totalPages"] ) def getUserByUsername(self, username: str) -> Optional[User]: @@ -2312,30 +2293,43 @@ class AppObjects: # Additional Helper Methods # ============================================ - def getAllUsers(self) -> List[User]: + def getAllUsers(self, pagination: Optional[PaginationParams] = None) -> Union[List[User], PaginatedResult]: """ Get all users (for SysAdmin only). + Args: + pagination: Optional pagination parameters. If None, returns all items. + Returns: - List of User objects (without sensitive fields) + If pagination is None: List[User] (without sensitive fields) + If pagination is provided: PaginatedResult with items and metadata """ try: - records = self.db.getRecordset(UserInDB) - result = [] - for record in records: - # Filter out sensitive and internal fields + result = self.db.getRecordsetPaginated(UserInDB, pagination=pagination) + + items = [] + for record in result["items"]: cleanedRecord = { - k: v for k, v in record.items() + k: v for k, v in record.items() if not k.startswith("_") and k not in ["hashedPassword", "resetToken", "resetTokenExpires"] } - # Ensure roleLabels is a list if cleanedRecord.get("roleLabels") is None: cleanedRecord["roleLabels"] = [] - result.append(User(**cleanedRecord)) - return result + items.append(User(**cleanedRecord)) + + if pagination is None: + return items + + return PaginatedResult( + items=items, + totalItems=result["totalItems"], + totalPages=result["totalPages"] + ) except Exception as e: logger.error(f"Error getting all users: {e}") - return [] + if pagination is None: + return [] + return PaginatedResult(items=[], totalItems=0, totalPages=0) def getUserMandateById(self, userMandateId: str) -> Optional[UserMandate]: """ @@ -3321,50 +3315,26 @@ class AppObjects: If pagination is provided: PaginatedResult with items and metadata """ try: - # Get all roles from database - roles = self.db.getRecordset(Role) - - # Filter out database-specific fields - filteredRoles = [] - for role in roles: - cleanedRole = {k: v for k, v in role.items() if not k.startswith("_")} - filteredRoles.append(cleanedRole) - - # If no pagination requested, return all items + result = self.db.getRecordsetPaginated(Role, pagination=pagination) + + items = [] + for record in result["items"]: + cleanedRole = {k: v for k, v in record.items() if not k.startswith("_")} + items.append(Role(**cleanedRole)) + if pagination is None: - return [Role(**role) for role in filteredRoles] - - # Apply filtering (if filters provided) - if pagination.filters: - filteredRoles = self._applyFilters(filteredRoles, pagination.filters) - - # Apply sorting (in order of sortFields) - if pagination.sort: - filteredRoles = self._applySorting(filteredRoles, pagination.sort) - - # Count total items after filters - totalItems = len(filteredRoles) - totalPages = math.ceil(totalItems / pagination.pageSize) if totalItems > 0 else 0 - - # Apply pagination (skip/limit) - startIdx = (pagination.page - 1) * pagination.pageSize - endIdx = startIdx + pagination.pageSize - pagedRoles = filteredRoles[startIdx:endIdx] - - # Convert to model objects - items = [Role(**role) for role in pagedRoles] - + return items + return PaginatedResult( items=items, - totalItems=totalItems, - totalPages=totalPages + totalItems=result["totalItems"], + totalPages=result["totalPages"] ) except Exception as e: logger.error(f"Error getting all roles: {str(e)}") if pagination is None: return [] - else: - return PaginatedResult(items=[], totalItems=0, totalPages=0) + return PaginatedResult(items=[], totalItems=0, totalPages=0) def countRoleAssignments(self) -> Dict[str, int]: """ diff --git a/modules/interfaces/interfaceDbBilling.py b/modules/interfaces/interfaceDbBilling.py index 58d56895..2db71bb4 100644 --- a/modules/interfaces/interfaceDbBilling.py +++ b/modules/interfaces/interfaceDbBilling.py @@ -8,7 +8,7 @@ All billing data is stored in the poweron_billing database. """ import logging -from typing import Dict, Any, List, Optional +from typing import Dict, Any, List, Optional, Union from datetime import date, datetime, timedelta import uuid @@ -17,6 +17,7 @@ from modules.shared.configuration import APP_CONFIG from modules.shared.timeUtils import getUtcTimestamp from modules.datamodels.datamodelUam import User, Mandate from modules.datamodels.datamodelMembership import UserMandate +from modules.datamodels.datamodelPagination import PaginationParams, PaginatedResult from modules.datamodels.datamodelBilling import ( BillingAccount, BillingTransaction, @@ -633,26 +634,43 @@ class BillingObjects: limit: int = 100, offset: int = 0, startDate: date = None, - endDate: date = None - ) -> List[Dict[str, Any]]: + endDate: date = None, + pagination: PaginationParams = None + ) -> Union[List[Dict[str, Any]], PaginatedResult]: """ Get transactions for an account. + When pagination is provided, uses database-level pagination and returns + PaginatedResult. Otherwise falls back to in-memory filtering/sorting/slicing. + Args: accountId: Account ID - limit: Maximum number of results - offset: Offset for pagination - startDate: Filter by start date - endDate: Filter by end date + limit: Maximum number of results (legacy path) + offset: Offset for pagination (legacy path) + startDate: Filter by start date (legacy path) + endDate: Filter by end date (legacy path) + pagination: PaginationParams for DB-level pagination Returns: - List of transaction dicts + PaginatedResult when pagination is provided, List of dicts otherwise """ try: + if pagination: + recordFilter = {"accountId": accountId} + result = self.db.getRecordsetPaginated( + BillingTransaction, + pagination=pagination, + recordFilter=recordFilter + ) + return PaginatedResult( + items=result["items"], + totalItems=result["totalItems"], + totalPages=result["totalPages"] + ) + filterDict = {"accountId": accountId} results = self.db.getRecordset(BillingTransaction, recordFilter=filterDict) - # Apply date filters if provided if startDate or endDate: filtered = [] for t in results: @@ -666,35 +684,61 @@ class BillingObjects: filtered.append(t) results = filtered - # Sort by creation date descending results.sort(key=lambda x: x.get("_createdAt", ""), reverse=True) - # Apply pagination return results[offset:offset + limit] except Exception as e: logger.error(f"Error getting transactions: {e}") + if pagination: + return PaginatedResult(items=[], totalItems=0, totalPages=0) return [] - def getTransactionsByMandate(self, mandateId: str, limit: int = 100) -> List[Dict[str, Any]]: + def getTransactionsByMandate( + self, + mandateId: str, + limit: int = 100, + pagination: PaginationParams = None + ) -> Union[List[Dict[str, Any]], PaginatedResult]: """ Get all transactions for a mandate (across all accounts). + When pagination is provided, collects all accountIds for the mandate and + issues a single DB query with SQL-level filtering, sorting, and pagination. + Otherwise falls back to per-account querying and in-memory merging. + Args: mandateId: Mandate ID - limit: Maximum number of results + limit: Maximum number of results (legacy path) + pagination: PaginationParams for DB-level pagination Returns: - List of transaction dicts + PaginatedResult when pagination is provided, List of dicts otherwise """ - # Get all accounts for mandate accounts = self.db.getRecordset(BillingAccount, recordFilter={"mandateId": mandateId}) - + accountIds = [acc["id"] for acc in accounts if acc.get("id")] + + if not accountIds: + if pagination: + return PaginatedResult(items=[], totalItems=0, totalPages=0) + return [] + + if pagination: + result = self.db.getRecordsetPaginated( + BillingTransaction, + pagination=pagination, + recordFilter={"accountId": accountIds} + ) + return PaginatedResult( + items=result["items"], + totalItems=result["totalItems"], + totalPages=result["totalPages"] + ) + allTransactions = [] for account in accounts: transactions = self.getTransactions(account["id"], limit=limit) allTransactions.extend(transactions) - # Sort by creation date descending and limit allTransactions.sort(key=lambda x: x.get("_createdAt", ""), reverse=True) return allTransactions[:limit] diff --git a/modules/interfaces/interfaceDbChat.py b/modules/interfaces/interfaceDbChat.py index f2a3d4c6..b0d4aff3 100644 --- a/modules/interfaces/interfaceDbChat.py +++ b/modules/interfaces/interfaceDbChat.py @@ -450,7 +450,7 @@ class ChatObjects: # Handle simple value (equals operator) if not isinstance(filter_value, dict): - if record_value != filter_value: + if str(record_value).lower() != str(filter_value).lower(): matches = False break continue @@ -460,7 +460,7 @@ class ChatObjects: filter_val = filter_value.get("value") if operator in ["equals", "eq"]: - if record_value != filter_val: + if str(record_value).lower() != str(filter_val).lower(): matches = False break @@ -545,7 +545,7 @@ class ChatObjects: else: # Unknown operator - default to equals - if record_value != filter_val: + if str(record_value).lower() != str(filter_val).lower(): matches = False break diff --git a/modules/interfaces/interfaceDbManagement.py b/modules/interfaces/interfaceDbManagement.py index 9c266aac..ae0b14b0 100644 --- a/modules/interfaces/interfaceDbManagement.py +++ b/modules/interfaces/interfaceDbManagement.py @@ -390,7 +390,7 @@ class ComponentObjects: # Handle simple value (equals operator) if not isinstance(filter_value, dict): - if record_value != filter_value: + if str(record_value).lower() != str(filter_value).lower(): matches = False break continue @@ -400,7 +400,7 @@ class ComponentObjects: filter_val = filter_value.get("value") if operator in ["equals", "eq"]: - if record_value != filter_val: + if str(record_value).lower() != str(filter_val).lower(): matches = False break @@ -650,6 +650,10 @@ class ComponentObjects: - Regular user: sees own prompts + system prompts (isSystem=True), can only CRUD own - Row-level _permissions control edit/delete buttons in the UI + NOTE: Cannot use db.getRecordsetPaginated() because visibility rules + (_getPromptsForUser: own + system for regular, all for SysAdmin) and + per-row _permissions enrichment require loading all records first. + Args: pagination: Optional pagination parameters. If None, returns all items. @@ -914,7 +918,7 @@ class ComponentObjects: """ Returns files owned by the current user (user-scoped, not RBAC-based). Every user (including SysAdmin) only sees their own files. - Supports optional pagination, sorting, and filtering. + Supports optional pagination, sorting, and filtering via database-level queries. Args: pagination: Optional pagination parameters. If None, returns all items. @@ -923,24 +927,21 @@ class ComponentObjects: If pagination is None: List[FileItem] If pagination is provided: PaginatedResult with items and metadata """ - # Files are always user-scoped: filter by _createdBy (bypasses RBAC SysAdmin override) - filteredFiles = self._getFilesByCurrentUser() - - # Convert database records to FileItem instances (extra='allow' preserves system fields like _createdBy) - def convertFileItems(files): + # User-scoping filter: every user only sees their own files (bypasses RBAC SysAdmin override) + recordFilter = {"_createdBy": self.userId} + + def _convertFileItems(files): fileItems = [] for file in files: try: - # Ensure proper values, use defaults for invalid data creationDate = file.get("creationDate") if creationDate is None or not isinstance(creationDate, (int, float)) or creationDate <= 0: file["creationDate"] = getUtcTimestamp() fileName = file.get("fileName") if not fileName or fileName == "None": - continue # Skip records with invalid fileName + continue - # Use **file to pass all fields including system fields (_createdBy, etc.) fileItem = FileItem(**file) fileItems.append(fileItem) except Exception as e: @@ -948,34 +949,23 @@ class ComponentObjects: continue return fileItems - # If no pagination requested, return all items if pagination is None: - return convertFileItems(filteredFiles) + allFiles = self._getFilesByCurrentUser() + return _convertFileItems(allFiles) - # Apply filtering (if filters provided) - if pagination.filters: - filteredFiles = self._applyFilters(filteredFiles, pagination.filters) + # Database-level pagination: filtering, sorting, and LIMIT/OFFSET happen in SQL + result = self.db.getRecordsetPaginated( + FileItem, + pagination=pagination, + recordFilter=recordFilter + ) - # Apply sorting (in order of sortFields) - if pagination.sort: - filteredFiles = self._applySorting(filteredFiles, pagination.sort) - - # Count total items after filters - totalItems = len(filteredFiles) - totalPages = math.ceil(totalItems / pagination.pageSize) if totalItems > 0 else 0 - - # Apply pagination (skip/limit) - startIdx = (pagination.page - 1) * pagination.pageSize - endIdx = startIdx + pagination.pageSize - pagedFiles = filteredFiles[startIdx:endIdx] - - # Convert to model objects (extra='allow' on FileItem preserves system fields) - items = convertFileItems(pagedFiles) + items = _convertFileItems(result["items"]) return PaginatedResult( items=items, - totalItems=totalItems, - totalPages=totalPages + totalItems=result["totalItems"], + totalPages=result["totalPages"] ) def getFile(self, fileId: str) -> Optional[FileItem]: diff --git a/modules/interfaces/interfaceRbac.py b/modules/interfaces/interfaceRbac.py index b4c9a3b4..b9a4ac9b 100644 --- a/modules/interfaces/interfaceRbac.py +++ b/modules/interfaces/interfaceRbac.py @@ -23,10 +23,12 @@ GROUP-Berechtigung: import logging import json -from typing import List, Dict, Any, Optional, Type +import math +from typing import List, Dict, Any, Optional, Type, Union from pydantic import BaseModel from modules.datamodels.datamodelRbac import AccessRuleContext from modules.datamodels.datamodelUam import User, UserPermissions, AccessLevel +from modules.datamodels.datamodelPagination import PaginationParams, PaginatedResult from modules.security.rbac import RbacClass from modules.security.rootAccess import getRootDbAppConnector @@ -297,6 +299,309 @@ def getRecordsetWithRBAC( return [] +def getRecordsetPaginatedWithRBAC( + connector, + modelClass: Type[BaseModel], + currentUser: User, + pagination: Optional[PaginationParams] = None, + recordFilter: Dict[str, Any] = None, + mandateId: Optional[str] = None, + featureInstanceId: Optional[str] = None, + enrichPermissions: bool = False, + featureCode: Optional[str] = None, +) -> Union[List[Dict[str, Any]], PaginatedResult]: + """ + Get records with RBAC filtering and SQL-level pagination. + When pagination is None, returns a plain list (backward compatible). + When pagination is provided, returns PaginatedResult with COUNT + LIMIT/OFFSET at SQL level. + """ + table = modelClass.__name__ + objectKey = buildDataObjectKey(table, featureCode) + effectiveMandateId = mandateId + + try: + if not connector._ensureTableExists(modelClass): + return PaginatedResult(items=[], totalItems=0, totalPages=0) if pagination else [] + + dbApp = getRootDbAppConnector() + rbacInstance = RbacClass(connector, dbApp=dbApp) + permissions = rbacInstance.getUserPermissions( + currentUser, + AccessRuleContext.DATA, + objectKey, + mandateId=effectiveMandateId, + featureInstanceId=featureInstanceId + ) + + if not permissions.view: + return PaginatedResult(items=[], totalItems=0, totalPages=0) if pagination else [] + + whereConditions = [] + whereValues = [] + + featureInstanceIdForQuery = featureInstanceId + if featureInstanceId and hasattr(modelClass, 'model_fields') and "featureInstanceId" not in modelClass.model_fields: + featureInstanceIdForQuery = None + + rbacWhereClause = buildRbacWhereClause( + permissions, currentUser, table, connector, + mandateId=effectiveMandateId, + featureInstanceId=featureInstanceIdForQuery + ) + if rbacWhereClause: + whereConditions.append(rbacWhereClause["condition"]) + whereValues.extend(rbacWhereClause["values"]) + + if recordFilter: + for field, value in recordFilter.items(): + if isinstance(value, (list, tuple)): + if len(value) == 0: + whereConditions.append("1 = 0") + else: + whereConditions.append(f'"{field}" = ANY(%s)') + whereValues.append(list(value)) + elif value is None: + whereConditions.append(f'"{field}" IS NULL') + else: + whereConditions.append(f'"{field}" = %s') + whereValues.append(value) + + if pagination and pagination.filters: + from modules.connectors.connectorDbPostgre import _get_model_fields + fields = _get_model_fields(modelClass) + validColumns = set(fields.keys()) + for key, val in pagination.filters.items(): + if key == "search" and isinstance(val, str) and val.strip(): + term = f"%{val.strip()}%" + textCols = [c for c, t in fields.items() if t == "TEXT"] + if textCols: + orParts = [f'COALESCE("{c}"::TEXT, \'\') ILIKE %s' for c in textCols] + whereConditions.append(f"({' OR '.join(orParts)})") + whereValues.extend([term] * len(textCols)) + continue + if key not in validColumns: + continue + if isinstance(val, dict): + op = val.get("operator", "equals") + v = val.get("value", "") + if op in ("equals", "eq"): + whereConditions.append(f'"{key}"::TEXT = %s') + whereValues.append(str(v)) + elif op == "contains": + whereConditions.append(f'"{key}"::TEXT ILIKE %s') + whereValues.append(f"%{v}%") + elif op == "startsWith": + whereConditions.append(f'"{key}"::TEXT ILIKE %s') + whereValues.append(f"{v}%") + elif op == "endsWith": + whereConditions.append(f'"{key}"::TEXT ILIKE %s') + whereValues.append(f"%{v}") + elif op in ("gt", "gte", "lt", "lte"): + sqlOp = {"gt": ">", "gte": ">=", "lt": "<", "lte": "<="}[op] + whereConditions.append(f'"{key}"::TEXT {sqlOp} %s') + whereValues.append(str(v)) + elif op == "between": + fromVal = v.get("from", "") if isinstance(v, dict) else "" + toVal = v.get("to", "") if isinstance(v, dict) else "" + if fromVal and toVal: + whereConditions.append(f'"{key}"::TEXT >= %s AND "{key}"::TEXT <= %s') + whereValues.extend([str(fromVal), str(toVal)]) + elif fromVal: + whereConditions.append(f'"{key}"::TEXT >= %s') + whereValues.append(str(fromVal)) + elif toVal: + whereConditions.append(f'"{key}"::TEXT <= %s') + whereValues.append(str(toVal)) + else: + whereConditions.append(f'"{key}"::TEXT ILIKE %s') + whereValues.append(str(val)) + + whereClause = " WHERE " + " AND ".join(whereConditions) if whereConditions else "" + countValues = list(whereValues) + + orderParts: List[str] = [] + if pagination and pagination.sort: + from modules.connectors.connectorDbPostgre import _get_model_fields + validColumns = set(_get_model_fields(modelClass).keys()) + for sf in pagination.sort: + if sf.field in validColumns: + direction = "DESC" if sf.direction.lower() == "desc" else "ASC" + orderParts.append(f'"{sf.field}" {direction}') + if not orderParts: + orderParts.append('"id"') + orderByClause = " ORDER BY " + ", ".join(orderParts) + + limitClause = "" + if pagination: + offset = (pagination.page - 1) * pagination.pageSize + limitClause = f" LIMIT {pagination.pageSize} OFFSET {offset}" + + with connector.connection.cursor() as cursor: + countSql = f'SELECT COUNT(*) FROM "{table}"{whereClause}' + cursor.execute(countSql, countValues) + totalItems = cursor.fetchone()["count"] + + dataSql = f'SELECT * FROM "{table}"{whereClause}{orderByClause}{limitClause}' + cursor.execute(dataSql, whereValues) + records = [dict(row) for row in cursor.fetchall()] + + from modules.connectors.connectorDbPostgre import _get_model_fields, _parseRecordFields + fields = _get_model_fields(modelClass) + for record in records: + _parseRecordFields(record, fields, f"table {table}") + for fieldName, fieldType in fields.items(): + if fieldType == "JSONB" and fieldName in record and record[fieldName] is None: + modelFields = modelClass.model_fields + fieldInfo = modelFields.get(fieldName) + if fieldInfo: + fieldAnnotation = fieldInfo.annotation + if (fieldAnnotation == list or + (hasattr(fieldAnnotation, "__origin__") and fieldAnnotation.__origin__ is list)): + record[fieldName] = [] + elif (fieldAnnotation == dict or + (hasattr(fieldAnnotation, "__origin__") and fieldAnnotation.__origin__ is dict)): + record[fieldName] = {} + + if enrichPermissions: + records = _enrichRecordsWithPermissions(records, permissions, currentUser) + + if pagination: + pageSize = pagination.pageSize + totalPages = math.ceil(totalItems / pageSize) if totalItems > 0 else 0 + return PaginatedResult(items=records, totalItems=totalItems, totalPages=totalPages) + + return records + except Exception as e: + logger.error(f"Error in getRecordsetPaginatedWithRBAC for table {table}: {e}") + return PaginatedResult(items=[], totalItems=0, totalPages=0) if pagination else [] + + +def getDistinctColumnValuesWithRBAC( + connector, + modelClass: Type[BaseModel], + currentUser: User, + column: str, + pagination: Optional[PaginationParams] = None, + recordFilter: Dict[str, Any] = None, + mandateId: Optional[str] = None, + featureInstanceId: Optional[str] = None, + featureCode: Optional[str] = None, +) -> List[str]: + """ + Get sorted distinct values for a column with RBAC filtering at SQL level. + Cross-filtering: removes the requested column from active filters. + """ + import copy + table = modelClass.__name__ + objectKey = buildDataObjectKey(table, featureCode) + + try: + if not connector._ensureTableExists(modelClass): + return [] + + from modules.connectors.connectorDbPostgre import _get_model_fields + fields = _get_model_fields(modelClass) + if column not in fields: + return [] + + dbApp = getRootDbAppConnector() + rbacInstance = RbacClass(connector, dbApp=dbApp) + permissions = rbacInstance.getUserPermissions( + currentUser, AccessRuleContext.DATA, objectKey, + mandateId=mandateId, featureInstanceId=featureInstanceId + ) + if not permissions.view: + return [] + + whereConditions = [] + whereValues = [] + + featureInstanceIdForQuery = featureInstanceId + if featureInstanceId and hasattr(modelClass, 'model_fields') and "featureInstanceId" not in modelClass.model_fields: + featureInstanceIdForQuery = None + + rbacWhereClause = buildRbacWhereClause( + permissions, currentUser, table, connector, + mandateId=mandateId, featureInstanceId=featureInstanceIdForQuery + ) + if rbacWhereClause: + whereConditions.append(rbacWhereClause["condition"]) + whereValues.extend(rbacWhereClause["values"]) + + if recordFilter: + for field, value in recordFilter.items(): + if isinstance(value, (list, tuple)): + if not value: + whereConditions.append("1 = 0") + else: + whereConditions.append(f'"{field}" = ANY(%s)') + whereValues.append(list(value)) + elif value is None: + whereConditions.append(f'"{field}" IS NULL') + else: + whereConditions.append(f'"{field}" = %s') + whereValues.append(value) + + crossPagination = copy.deepcopy(pagination) if pagination else None + if crossPagination and crossPagination.filters: + crossPagination.filters.pop(column, None) + validColumns = set(fields.keys()) + for key, val in crossPagination.filters.items(): + if key == "search" and isinstance(val, str) and val.strip(): + term = f"%{val.strip()}%" + textCols = [c for c, t in fields.items() if t == "TEXT"] + if textCols: + orParts = [f'COALESCE("{c}"::TEXT, \'\') ILIKE %s' for c in textCols] + whereConditions.append(f"({' OR '.join(orParts)})") + whereValues.extend([term] * len(textCols)) + continue + if key not in validColumns: + continue + if isinstance(val, dict): + op = val.get("operator", "equals") + v = val.get("value", "") + if op in ("equals", "eq"): + whereConditions.append(f'"{key}"::TEXT = %s') + whereValues.append(str(v)) + elif op == "contains": + whereConditions.append(f'"{key}"::TEXT ILIKE %s') + whereValues.append(f"%{v}%") + elif op == "between": + fromVal = v.get("from", "") if isinstance(v, dict) else "" + toVal = v.get("to", "") if isinstance(v, dict) else "" + if fromVal and toVal: + whereConditions.append(f'"{key}"::TEXT >= %s AND "{key}"::TEXT <= %s') + whereValues.extend([str(fromVal), str(toVal)]) + elif fromVal: + whereConditions.append(f'"{key}"::TEXT >= %s') + whereValues.append(str(fromVal)) + elif toVal: + whereConditions.append(f'"{key}"::TEXT <= %s') + whereValues.append(str(toVal)) + else: + whereConditions.append(f'"{key}"::TEXT ILIKE %s') + whereValues.append(str(v) if isinstance(v, str) else str(val)) + else: + whereConditions.append(f'"{key}"::TEXT ILIKE %s') + whereValues.append(str(val)) + + whereClause = " WHERE " + " AND ".join(whereConditions) if whereConditions else "" + notNullCond = f'"{column}" IS NOT NULL AND "{column}"::TEXT != \'\'' + if whereClause: + whereClause += f" AND {notNullCond}" + else: + whereClause = f" WHERE {notNullCond}" + + sql = f'SELECT DISTINCT "{column}"::TEXT AS val FROM "{table}"{whereClause} ORDER BY val' + + with connector.connection.cursor() as cursor: + cursor.execute(sql, whereValues) + return [row["val"] for row in cursor.fetchall()] + except Exception as e: + logger.error(f"Error in getDistinctColumnValuesWithRBAC for {table}.{column}: {e}") + return [] + + def buildRbacWhereClause( permissions: UserPermissions, currentUser: User, diff --git a/modules/routes/routeAdminAutomationEvents.py b/modules/routes/routeAdminAutomationEvents.py index d184ae76..47d3ac9c 100644 --- a/modules/routes/routeAdminAutomationEvents.py +++ b/modules/routes/routeAdminAutomationEvents.py @@ -5,15 +5,19 @@ Admin automation events routes for the backend API. Sysadmin-only endpoints for viewing and controlling scheduler events. """ -from fastapi import APIRouter, HTTPException, Depends, Path, Request, Response -from typing import List, Dict, Any +from fastapi import APIRouter, HTTPException, Depends, Path, Request, Response, Query +from typing import List, Dict, Any, Optional from fastapi import status import logging +import json +import math # Import interfaces and models from feature containers import modules.features.automation.interfaceFeatureAutomation as interfaceAutomation from modules.auth import limiter, getRequestContext, requireSysAdminRole, RequestContext from modules.datamodels.datamodelUam import User +from modules.datamodels.datamodelPagination import PaginationParams, PaginationMetadata, normalize_pagination_dict +from modules.routes.routeDataUsers import _applyFiltersAndSort, _extractDistinctValues # Configure logger logger = logging.getLogger(__name__) @@ -31,122 +35,176 @@ router = APIRouter( } ) +def _buildEnrichedAutomationEvents(currentUser: User) -> List[Dict[str, Any]]: + """Build the full enriched automation events list.""" + from modules.shared.eventManagement import eventManager + from modules.interfaces.interfaceDbApp import getRootInterface + from modules.features.automation.mainAutomation import getAutomationServices + + if not eventManager.scheduler: + return [] + + jobs = [] + for job in eventManager.scheduler.get_jobs(): + if job.id.startswith("automation."): + automationId = job.id.replace("automation.", "") + jobs.append({ + "eventId": job.id, + "id": job.id, + "automationId": automationId, + "nextRunTime": str(job.next_run_time) if job.next_run_time else None, + "trigger": str(job.trigger) if job.trigger else None, + "name": "", + "createdBy": "", + "mandate": "", + "featureInstance": "" + }) + + if jobs: + try: + rootInterface = getRootInterface() + eventUser = rootInterface.getUserByUsername("event") + if eventUser: + services = getAutomationServices(currentUser, mandateId=None, featureInstanceId=None) + allAutomations = services.interfaceDbAutomation.getAllAutomationDefinitionsWithRBAC(eventUser) + + automationLookup = {} + for a in allAutomations: + aId = a.get("id", "") if isinstance(a, dict) else getattr(a, "id", "") + automationLookup[aId] = a + + _userCache: Dict[str, str] = {} + _mandateCache: Dict[str, str] = {} + _featureCache: Dict[str, str] = {} + + def _resolveUsername(userId): + if not userId: return "" + if userId not in _userCache: + try: + user = rootInterface.getUser(userId) + _userCache[userId] = user.username if user else userId[:8] + except Exception: + _userCache[userId] = userId[:8] + return _userCache[userId] + + def _resolveMandateLabel(mandateId): + if not mandateId: return "" + if mandateId not in _mandateCache: + try: + mandate = rootInterface.getMandate(mandateId) + _mandateCache[mandateId] = getattr(mandate, "label", None) or mandateId[:8] + except Exception: + _mandateCache[mandateId] = mandateId[:8] + return _mandateCache[mandateId] + + def _resolveFeatureLabel(featureInstanceId): + if not featureInstanceId: return "" + if featureInstanceId not in _featureCache: + try: + instance = rootInterface.getFeatureInstance(featureInstanceId) + _featureCache[featureInstanceId] = getattr(instance, "label", None) or getattr(instance, "featureCode", None) or featureInstanceId[:8] + except Exception: + _featureCache[featureInstanceId] = featureInstanceId[:8] + return _featureCache[featureInstanceId] + + for job in jobs: + automation = automationLookup.get(job["automationId"]) + if automation: + if isinstance(automation, dict): + job["name"] = automation.get("label", "") + job["createdBy"] = _resolveUsername(automation.get("_createdBy", "")) + job["mandate"] = _resolveMandateLabel(automation.get("mandateId", "")) + job["featureInstance"] = _resolveFeatureLabel(automation.get("featureInstanceId", "")) + else: + job["name"] = getattr(automation, "label", "") + job["createdBy"] = _resolveUsername(getattr(automation, "_createdBy", "")) + job["mandate"] = _resolveMandateLabel(getattr(automation, "mandateId", "")) + job["featureInstance"] = _resolveFeatureLabel(getattr(automation, "featureInstanceId", "")) + else: + job["name"] = "(orphaned)" + except Exception as e: + logger.warning(f"Could not enrich automation events with context: {e}") + + return jobs + + @router.get("") @limiter.limit("30/minute") def get_all_automation_events( request: Request, - currentUser: User = Depends(requireSysAdminRole) -) -> List[Dict[str, Any]]: - """ - Get all active scheduler jobs (sysadmin only). - Each job is enriched with context from its automation definition - (name, mandate, feature instance, creator) for readability. - """ + pagination: Optional[str] = Query(None, description="JSON-encoded PaginationParams"), + currentUser: User = Depends(requireSysAdminRole), +): + """Get all active scheduler jobs with pagination support (sysadmin only).""" try: - from modules.shared.eventManagement import eventManager - from modules.interfaces.interfaceDbApp import getRootInterface - from modules.features.automation.mainAutomation import getAutomationServices - - if not eventManager.scheduler: - return [] - - # 1. Collect all scheduler jobs - jobs = [] - automationIds = [] - for job in eventManager.scheduler.get_jobs(): - if job.id.startswith("automation."): - automationId = job.id.replace("automation.", "") - automationIds.append(automationId) - jobs.append({ - "eventId": job.id, - "automationId": automationId, - "nextRunTime": str(job.next_run_time) if job.next_run_time else None, - "trigger": str(job.trigger) if job.trigger else None, - "name": "", - "createdBy": "", - "mandate": "", - "featureInstance": "" - }) - - # 2. Enrich with context from automation definitions - if jobs: + paginationParams: Optional[PaginationParams] = None + if pagination: try: - rootInterface = getRootInterface() - eventUser = rootInterface.getUserByUsername("event") - if eventUser: - services = getAutomationServices(currentUser, mandateId=None, featureInstanceId=None) - allAutomations = services.interfaceDbAutomation.getAllAutomationDefinitionsWithRBAC(eventUser) - - # Build lookup by automation ID - automationLookup = {} - for a in allAutomations: - aId = a.get("id", "") if isinstance(a, dict) else getattr(a, "id", "") - automationLookup[aId] = a - - # Caches for resolving UUIDs to names - _userCache = {} - _mandateCache = {} - _featureCache = {} - - def _resolveUsername(userId): - if not userId: - return "" - if userId not in _userCache: - try: - user = rootInterface.getUser(userId) - _userCache[userId] = user.username if user else userId[:8] - except Exception: - _userCache[userId] = userId[:8] - return _userCache[userId] - - def _resolveMandateLabel(mandateId): - if not mandateId: - return "" - if mandateId not in _mandateCache: - try: - mandate = rootInterface.getMandate(mandateId) - _mandateCache[mandateId] = getattr(mandate, "label", None) or mandateId[:8] - except Exception: - _mandateCache[mandateId] = mandateId[:8] - return _mandateCache[mandateId] - - def _resolveFeatureLabel(featureInstanceId): - if not featureInstanceId: - return "" - if featureInstanceId not in _featureCache: - try: - instance = rootInterface.getFeatureInstance(featureInstanceId) - _featureCache[featureInstanceId] = getattr(instance, "label", None) or getattr(instance, "featureCode", None) or featureInstanceId[:8] - except Exception: - _featureCache[featureInstanceId] = featureInstanceId[:8] - return _featureCache[featureInstanceId] - - # Enrich each job - for job in jobs: - automation = automationLookup.get(job["automationId"]) - if automation: - if isinstance(automation, dict): - job["name"] = automation.get("label", "") - job["createdBy"] = _resolveUsername(automation.get("_createdBy", "")) - job["mandate"] = _resolveMandateLabel(automation.get("mandateId", "")) - job["featureInstance"] = _resolveFeatureLabel(automation.get("featureInstanceId", "")) - else: - job["name"] = getattr(automation, "label", "") - job["createdBy"] = _resolveUsername(getattr(automation, "_createdBy", "")) - job["mandate"] = _resolveMandateLabel(getattr(automation, "mandateId", "")) - job["featureInstance"] = _resolveFeatureLabel(getattr(automation, "featureInstanceId", "")) - else: - job["name"] = "(orphaned)" - except Exception as e: - logger.warning(f"Could not enrich automation events with context: {e}") - - return jobs + paginationDict = json.loads(pagination) + if paginationDict: + paginationDict = normalize_pagination_dict(paginationDict) + paginationParams = PaginationParams(**paginationDict) + except (json.JSONDecodeError, ValueError) as e: + raise HTTPException(status_code=400, detail=f"Invalid pagination parameter: {str(e)}") + + enriched = _buildEnrichedAutomationEvents(currentUser) + filtered = _applyFiltersAndSort(enriched, paginationParams) + + if paginationParams: + totalItems = len(filtered) + totalPages = math.ceil(totalItems / paginationParams.pageSize) if totalItems > 0 else 0 + startIdx = (paginationParams.page - 1) * paginationParams.pageSize + endIdx = startIdx + paginationParams.pageSize + return { + "items": filtered[startIdx:endIdx], + "pagination": PaginationMetadata( + currentPage=paginationParams.page, + pageSize=paginationParams.pageSize, + totalItems=totalItems, + totalPages=totalPages, + sort=paginationParams.sort, + filters=paginationParams.filters, + ).model_dump(), + } + + return {"items": enriched, "pagination": None} + except HTTPException: + raise except Exception as e: logger.error(f"Error getting automation events: {str(e)}") - raise HTTPException( - status_code=500, - detail=f"Error getting automation events: {str(e)}" - ) + raise HTTPException(status_code=500, detail=f"Error getting automation events: {str(e)}") + + +@router.get("/filter-values") +@limiter.limit("60/minute") +def get_automation_event_filter_values( + request: Request, + column: str = Query(..., description="Column key"), + pagination: Optional[str] = Query(None, description="JSON-encoded current filters"), + currentUser: User = Depends(requireSysAdminRole), +): + """Return distinct filter values for a column in automation events.""" + try: + crossFilterParams: Optional[PaginationParams] = None + if pagination: + try: + paginationDict = json.loads(pagination) + if paginationDict: + paginationDict = normalize_pagination_dict(paginationDict) + filters = paginationDict.get("filters", {}) + filters.pop(column, None) + paginationDict["filters"] = filters + paginationDict.pop("sort", None) + crossFilterParams = PaginationParams(**paginationDict) + except (json.JSONDecodeError, ValueError): + pass + + enriched = _buildEnrichedAutomationEvents(currentUser) + crossFiltered = _applyFiltersAndSort(enriched, crossFilterParams) + return _extractDistinctValues(crossFiltered, column) + except Exception as e: + logger.error(f"Error getting filter values: {str(e)}") + raise HTTPException(status_code=500, detail=str(e)) @router.post("/sync") @limiter.limit("5/minute") diff --git a/modules/routes/routeAdminFeatures.py b/modules/routes/routeAdminFeatures.py index b31b5e4c..3e701548 100644 --- a/modules/routes/routeAdminFeatures.py +++ b/modules/routes/routeAdminFeatures.py @@ -14,7 +14,11 @@ from fastapi import APIRouter, HTTPException, Depends, Request, Query from typing import List, Dict, Any, Optional from fastapi import status import logging +import json +import math from pydantic import BaseModel, Field +from modules.datamodels.datamodelPagination import PaginationParams, PaginationMetadata, normalize_pagination_dict +from modules.routes.routeDataUsers import _applyFiltersAndSort, _extractDistinctValues from modules.auth import limiter, getRequestContext, RequestContext, requireSysAdminRole from modules.datamodels.datamodelUam import User, UserInDB @@ -433,6 +437,35 @@ def list_feature_instances( ) +@router.get("/instances/filter-values") +@limiter.limit("60/minute") +def get_feature_instance_filter_values( + request: Request, + column: str = Query(..., description="Column key"), + pagination: Optional[str] = Query(None, description="JSON-encoded current filters"), + featureCode: Optional[str] = Query(None, description="Filter by feature code"), + context: RequestContext = Depends(getRequestContext) +) -> list: + """Return distinct filter values for a column in feature instances.""" + if not context.mandateId: + raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail="X-Mandate-Id header is required") + try: + from modules.routes.routeDataUsers import _handleFilterValuesRequest + rootInterface = getRootInterface() + featureInterface = getFeatureInterface(rootInterface.db) + instances = featureInterface.getFeatureInstancesForMandate( + mandateId=str(context.mandateId), + featureCode=featureCode + ) + items = [inst.model_dump() for inst in instances] + return _handleFilterValuesRequest(items, column, pagination) + except HTTPException: + raise + except Exception as e: + logger.error(f"Error getting filter values for feature instances: {e}") + raise HTTPException(status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, detail=str(e)) + + @router.get("/instances/{instanceId}", response_model=Dict[str, Any]) @limiter.limit("60/minute") def get_feature_instance( @@ -783,27 +816,55 @@ def sync_instance_roles( # Template Role Endpoints (SysAdmin only) # ============================================================================= -@router.get("/templates/roles", response_model=List[Dict[str, Any]]) +def _buildTemplateRolesList(featureCode: Optional[str] = None) -> List[Dict[str, Any]]: + """Build the full template roles list.""" + rootInterface = getRootInterface() + featureInterface = getFeatureInterface(rootInterface.db) + roles = featureInterface.getTemplateRoles(featureCode) + return [r.model_dump() for r in roles] + + +@router.get("/templates/roles") @limiter.limit("60/minute") def list_template_roles( request: Request, featureCode: Optional[str] = Query(None, description="Filter by feature code"), - sysAdmin: User = Depends(requireSysAdminRole) -) -> List[Dict[str, Any]]: - """ - List global template roles. - - SysAdmin only - returns template roles that are copied to new feature instances. - - Args: - featureCode: Optional filter by feature code - """ + pagination: Optional[str] = Query(None, description="JSON-encoded PaginationParams"), + sysAdmin: User = Depends(requireSysAdminRole), +): + """List global template roles with pagination support.""" try: - rootInterface = getRootInterface() - featureInterface = getFeatureInterface(rootInterface.db) - - roles = featureInterface.getTemplateRoles(featureCode) - return [r.model_dump() for r in roles] + paginationParams: Optional[PaginationParams] = None + if pagination: + try: + paginationDict = json.loads(pagination) + if paginationDict: + paginationDict = normalize_pagination_dict(paginationDict) + paginationParams = PaginationParams(**paginationDict) + except (json.JSONDecodeError, ValueError) as e: + raise HTTPException(status_code=400, detail=f"Invalid pagination parameter: {str(e)}") + + enriched = _buildTemplateRolesList(featureCode) + filtered = _applyFiltersAndSort(enriched, paginationParams) + + if paginationParams: + totalItems = len(filtered) + totalPages = math.ceil(totalItems / paginationParams.pageSize) if totalItems > 0 else 0 + startIdx = (paginationParams.page - 1) * paginationParams.pageSize + endIdx = startIdx + paginationParams.pageSize + return { + "items": filtered[startIdx:endIdx], + "pagination": PaginationMetadata( + currentPage=paginationParams.page, + pageSize=paginationParams.pageSize, + totalItems=totalItems, + totalPages=totalPages, + sort=paginationParams.sort, + filters=paginationParams.filters, + ).model_dump(), + } + + return {"items": enriched, "pagination": None} except Exception as e: logger.error(f"Error listing template roles: {e}") @@ -813,6 +874,39 @@ def list_template_roles( ) +@router.get("/templates/roles/filter-values") +@limiter.limit("60/minute") +def get_template_role_filter_values( + request: Request, + column: str = Query(..., description="Column key"), + featureCode: Optional[str] = Query(None, description="Filter by feature code"), + pagination: Optional[str] = Query(None, description="JSON-encoded current filters"), + sysAdmin: User = Depends(requireSysAdminRole), +): + """Return distinct filter values for a column in template roles.""" + try: + crossFilterParams: Optional[PaginationParams] = None + if pagination: + try: + paginationDict = json.loads(pagination) + if paginationDict: + paginationDict = normalize_pagination_dict(paginationDict) + filters = paginationDict.get("filters", {}) + filters.pop(column, None) + paginationDict["filters"] = filters + paginationDict.pop("sort", None) + crossFilterParams = PaginationParams(**paginationDict) + except (json.JSONDecodeError, ValueError): + pass + + enriched = _buildTemplateRolesList(featureCode) + crossFiltered = _applyFiltersAndSort(enriched, crossFilterParams) + return _extractDistinctValues(crossFiltered, column) + except Exception as e: + logger.error(f"Error getting filter values: {e}") + raise HTTPException(status_code=500, detail=str(e)) + + @router.post("/templates/roles", response_model=Dict[str, Any]) @limiter.limit("10/minute") def create_template_role( @@ -976,6 +1070,56 @@ def list_feature_instance_users( ) +@router.get("/instances/{instanceId}/users/filter-values") +@limiter.limit("60/minute") +def get_feature_instance_users_filter_values( + request: Request, + instanceId: str, + column: str = Query(..., description="Column key"), + pagination: Optional[str] = Query(None, description="JSON-encoded current filters"), + context: RequestContext = Depends(getRequestContext) +) -> list: + """Return distinct filter values for a column in feature instance users.""" + try: + from modules.routes.routeDataUsers import _handleFilterValuesRequest + rootInterface = getRootInterface() + featureInterface = getFeatureInterface(rootInterface.db) + instance = featureInterface.getFeatureInstance(instanceId) + if not instance: + raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail=f"Feature instance '{instanceId}' not found") + if context.mandateId and str(instance.mandateId) != str(context.mandateId): + if not context.hasSysAdminRole: + raise HTTPException(status_code=status.HTTP_403_FORBIDDEN, detail="Access denied to this feature instance") + featureAccesses = rootInterface.getFeatureAccessesByInstance(instanceId) + result = [] + for fa in featureAccesses: + user = rootInterface.getUser(str(fa.userId)) + if not user: + continue + roleIds = rootInterface.getRoleIdsForFeatureAccess(str(fa.id)) + roleLabels = [] + for roleId in roleIds: + role = rootInterface.getRole(roleId) + if role: + roleLabels.append(role.roleLabel) + result.append({ + "id": str(fa.id), + "userId": str(fa.userId), + "username": user.username, + "email": user.email, + "fullName": user.fullName, + "roleIds": roleIds, + "roleLabels": roleLabels, + "enabled": fa.enabled + }) + return _handleFilterValuesRequest(result, column, pagination) + except HTTPException: + raise + except Exception as e: + logger.error(f"Error getting filter values for feature instance users: {e}") + raise HTTPException(status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, detail=str(e)) + + @router.post("/instances/{instanceId}/users", response_model=Dict[str, Any]) @limiter.limit("30/minute") def add_user_to_feature_instance( diff --git a/modules/routes/routeAdminRbacRules.py b/modules/routes/routeAdminRbacRules.py index a7d4637e..3778d227 100644 --- a/modules/routes/routeAdminRbacRules.py +++ b/modules/routes/routeAdminRbacRules.py @@ -394,7 +394,10 @@ def get_access_rules( ) # Get rules with optional pagination - # MandateAdmin: fetch all then filter by admin's mandates + # MandateAdmin: fetch all then filter by admin's mandates. + # NOTE: Cannot use DB-level pagination for MandateAdmin because + # _isRoleInAdminMandates requires joining Role → mandateId which + # isn't expressible via getRecordsetPaginated's recordFilter. if not isSysAdmin: allRules = interface.getAccessRules( roleLabel=roleLabel, @@ -814,6 +817,11 @@ def list_roles( By default, only returns true global roles (mandateId=None, featureInstanceId=None, featureCode=None). Feature template roles are managed via /api/features/templates/roles. + NOTE: Base query (getAllRoles) already uses db.getRecordsetPaginated() internally. + However pagination=None is passed here because post-processing adds computed fields + (userCount, scopeType) and applies scope/mandate/template filtering that cannot run + at the DB level. In-memory pagination is applied after all transformations. + Args: pagination: Optional pagination parameters (includes search, filters, sort) includeTemplates: If True, also include feature template roles (featureCode != None) @@ -983,6 +991,77 @@ def list_roles( ) +@router.get("/roles/filter-values") +@limiter.limit("60/minute") +def get_roles_filter_values( + request: Request, + column: str = Query(..., description="Column key"), + pagination: Optional[str] = Query(None, description="JSON-encoded current filters"), + includeTemplates: bool = Query(False, description="Include feature template roles"), + mandateId: Optional[str] = Query(None, description="Include mandate-specific roles for this mandate"), + scopeFilter: Optional[str] = Query(None, description="Filter by scope: 'all', 'mandate', 'global', 'system'"), + reqContext: RequestContext = Depends(getRequestContext) +) -> list: + """Return distinct filter values for a column in roles.""" + try: + from modules.routes.routeDataUsers import _handleFilterValuesRequest + isSysAdmin = reqContext.hasSysAdminRole + adminMandateIds = [] if isSysAdmin else _getAdminMandateIds(reqContext) + if not isSysAdmin and not adminMandateIds: + raise HTTPException(status_code=403, detail="Admin role required") + + interface = getRootInterface() + dbRoles = interface.getAllRoles(pagination=None) + roleCounts = interface.countRoleAssignments() + + def _computeScopeType(role) -> str: + if role.mandateId: + return "mandate" + if role.isSystemRole: + return "system" + return "global" + + result = [] + for role in dbRoles: + if role.featureInstanceId is not None: + continue + if mandateId: + if role.mandateId != mandateId: + continue + else: + if role.mandateId is not None: + continue + if not includeTemplates and role.featureCode is not None: + continue + scopeType = _computeScopeType(role) + if scopeFilter and scopeFilter != 'all': + if scopeFilter == 'mandate' and scopeType != 'mandate': + continue + if scopeFilter == 'global' and scopeType not in ('global', 'system'): + continue + if scopeFilter == 'system' and scopeType != 'system': + continue + result.append({ + "id": role.id, + "roleLabel": role.roleLabel, + "description": role.description, + "mandateId": role.mandateId, + "featureInstanceId": role.featureInstanceId, + "featureCode": role.featureCode, + "userCount": roleCounts.get(str(role.id), 0), + "isSystemRole": role.isSystemRole, + "scopeType": scopeType + }) + if not isSysAdmin: + result = [r for r in result if r.get("mandateId") and str(r["mandateId"]) in adminMandateIds] + return _handleFilterValuesRequest(result, column, pagination) + except HTTPException: + raise + except Exception as e: + logger.error(f"Error getting filter values for roles: {str(e)}") + raise HTTPException(status_code=500, detail=str(e)) + + @router.post("/roles", response_model=Dict[str, Any]) @limiter.limit("30/minute") def create_role( diff --git a/modules/routes/routeBilling.py b/modules/routes/routeBilling.py index e1c23b48..04412752 100644 --- a/modules/routes/routeBilling.py +++ b/modules/routes/routeBilling.py @@ -22,8 +22,10 @@ from modules.auth import limiter, requireSysAdminRole, getRequestContext, Reques # Import billing components from modules.interfaces.interfaceDbBilling import getInterface as getBillingInterface, _getRootInterface from modules.serviceCenter.services.serviceBilling.mainServiceBilling import getService as getBillingService +import json +import math from modules.datamodels.datamodelPagination import PaginationParams, PaginatedResponse, PaginationMetadata, normalize_pagination_dict -from modules.routes.routeDataUsers import _applyFiltersAndSort +from modules.routes.routeDataUsers import _applyFiltersAndSort, _extractDistinctValues, _handleFilterValuesRequest from modules.datamodels.datamodelBilling import ( BillingAccount, BillingTransaction, @@ -1402,49 +1404,187 @@ def getUsersForMandate( raise HTTPException(status_code=500, detail=str(e)) -@router.get("/admin/transactions/{targetMandateId}", response_model=List[TransactionResponse]) +def _enrichTransactionRows(transactions) -> List[Dict[str, Any]]: + """Convert raw transaction dicts to enriched TransactionResponse rows with resolved usernames.""" + result = [] + for t in transactions: + row = TransactionResponse( + id=t.get("id"), + accountId=t.get("accountId"), + transactionType=TransactionTypeEnum(t.get("transactionType", "DEBIT")), + amount=t.get("amount", 0.0), + description=t.get("description", ""), + referenceType=ReferenceTypeEnum(t["referenceType"]) if t.get("referenceType") else None, + workflowId=t.get("workflowId"), + featureCode=t.get("featureCode"), + featureInstanceId=t.get("featureInstanceId"), + aicoreProvider=t.get("aicoreProvider"), + aicoreModel=t.get("aicoreModel"), + createdByUserId=t.get("createdByUserId"), + createdAt=t.get("_createdAt") + ) + result.append(row.model_dump()) + + try: + from modules.interfaces.interfaceDbUam import _getRootInterface as getUamRoot + uamInterface = getUamRoot() + userNames: Dict[str, str] = {} + for row in result: + uid = row.get("createdByUserId") + if uid and uid not in userNames: + try: + user = uamInterface.getUser(uid) + userNames[uid] = user.get("username", uid[:8]) if user else uid[:8] + except Exception: + userNames[uid] = uid[:8] + row["userName"] = userNames.get(uid, "") if uid else "" + except Exception: + for row in result: + row["userName"] = row.get("createdByUserId", "")[:8] if row.get("createdByUserId") else "" + + return result + + +def _buildTransactionsList(ctx: RequestContext, targetMandateId: str) -> List[Dict[str, Any]]: + """Build the full enriched transactions list for a mandate.""" + billingInterface = getBillingInterface(ctx.user, targetMandateId) + transactions = billingInterface.getTransactionsByMandate(targetMandateId, limit=5000) + + result = [] + for t in transactions: + row = TransactionResponse( + id=t.get("id"), + accountId=t.get("accountId"), + transactionType=TransactionTypeEnum(t.get("transactionType", "DEBIT")), + amount=t.get("amount", 0.0), + description=t.get("description", ""), + referenceType=ReferenceTypeEnum(t["referenceType"]) if t.get("referenceType") else None, + workflowId=t.get("workflowId"), + featureCode=t.get("featureCode"), + featureInstanceId=t.get("featureInstanceId"), + aicoreProvider=t.get("aicoreProvider"), + aicoreModel=t.get("aicoreModel"), + createdByUserId=t.get("createdByUserId"), + createdAt=t.get("_createdAt") + ) + result.append(row.model_dump()) + + # Resolve user names + try: + from modules.interfaces.interfaceDbUam import _getRootInterface as getUamRoot + uamInterface = getUamRoot() + userNames: Dict[str, str] = {} + for row in result: + uid = row.get("createdByUserId") + if uid and uid not in userNames: + try: + user = uamInterface.getUser(uid) + userNames[uid] = user.get("username", uid[:8]) if user else uid[:8] + except Exception: + userNames[uid] = uid[:8] + row["userName"] = userNames.get(uid, "") if uid else "" + except Exception: + for row in result: + row["userName"] = row.get("createdByUserId", "")[:8] if row.get("createdByUserId") else "" + + return result + + +@router.get("/admin/transactions/{targetMandateId}") @limiter.limit("30/minute") def getTransactionsAdmin( request: Request, targetMandateId: str = Path(..., description="Mandate ID"), + pagination: Optional[str] = Query(None, description="JSON-encoded PaginationParams"), limit: int = Query(default=100, ge=1, le=1000), ctx: RequestContext = Depends(getRequestContext), ): - """ - Get all transactions for a mandate. - Access: SysAdmin (any mandate) or MandateAdmin (own mandate). - """ + """Get all transactions for a mandate with pagination support.""" if not _isAdminOfMandate(ctx, targetMandateId): raise HTTPException(status_code=status.HTTP_403_FORBIDDEN, detail="Admin role required for this mandate") try: - billingInterface = getBillingInterface(ctx.user, targetMandateId) - transactions = billingInterface.getTransactionsByMandate(targetMandateId, limit=limit) - - result = [] - for t in transactions: - result.append(TransactionResponse( - id=t.get("id"), - accountId=t.get("accountId"), - transactionType=TransactionTypeEnum(t.get("transactionType", "DEBIT")), - amount=t.get("amount", 0.0), - description=t.get("description", ""), - referenceType=ReferenceTypeEnum(t["referenceType"]) if t.get("referenceType") else None, - workflowId=t.get("workflowId"), - featureCode=t.get("featureCode"), - featureInstanceId=t.get("featureInstanceId"), - aicoreProvider=t.get("aicoreProvider"), - aicoreModel=t.get("aicoreModel"), - createdByUserId=t.get("createdByUserId"), - createdAt=t.get("_createdAt") - )) - - return result - + paginationParams: Optional[PaginationParams] = None + if pagination: + try: + paginationDict = json.loads(pagination) + if paginationDict: + paginationDict = normalize_pagination_dict(paginationDict) + paginationParams = PaginationParams(**paginationDict) + except (json.JSONDecodeError, ValueError) as e: + raise HTTPException(status_code=400, detail=f"Invalid pagination parameter: {str(e)}") + + if paginationParams: + # DB-level pagination — enrich only the returned page + billingInterface = getBillingInterface(ctx.user, targetMandateId) + result = billingInterface.getTransactionsByMandate(targetMandateId, pagination=paginationParams) + transactions = result.items if hasattr(result, 'items') else result + enrichedItems = _enrichTransactionRows(transactions) + return { + "items": enrichedItems, + "pagination": PaginationMetadata( + currentPage=paginationParams.page, + pageSize=paginationParams.pageSize, + totalItems=result.totalItems if hasattr(result, 'totalItems') else len(enrichedItems), + totalPages=result.totalPages if hasattr(result, 'totalPages') else 0, + sort=paginationParams.sort, + filters=paginationParams.filters, + ).model_dump(), + } + + enriched = _buildTransactionsList(ctx, targetMandateId) + return {"items": enriched, "pagination": None} + + except HTTPException: + raise except Exception as e: logger.error(f"Error getting billing transactions for mandate {targetMandateId}: {e}") raise HTTPException(status_code=500, detail=str(e)) +@router.get("/admin/transactions/{targetMandateId}/filter-values") +@limiter.limit("60/minute") +def getTransactionFilterValues( + request: Request, + targetMandateId: str = Path(..., description="Mandate ID"), + column: str = Query(..., description="Column key"), + pagination: Optional[str] = Query(None, description="JSON-encoded current filters"), + ctx: RequestContext = Depends(getRequestContext), +): + """Return distinct filter values for a column in mandate transactions.""" + if not _isAdminOfMandate(ctx, targetMandateId): + raise HTTPException(status_code=status.HTTP_403_FORBIDDEN, detail="Admin role required for this mandate") + try: + crossFilterParams: Optional[PaginationParams] = None + if pagination: + try: + paginationDict = json.loads(pagination) + if paginationDict: + paginationDict = normalize_pagination_dict(paginationDict) + filters = paginationDict.get("filters", {}) + filters.pop(column, None) + paginationDict["filters"] = filters + paginationDict.pop("sort", None) + crossFilterParams = PaginationParams(**paginationDict) + except (json.JSONDecodeError, ValueError): + pass + + # Try SQL DISTINCT for native DB columns; fallback to in-memory for enriched columns (e.g. userName) + try: + rootBillingInterface = _getRootInterface() + recordFilter = {"mandateId": targetMandateId} + values = rootBillingInterface.db.getDistinctColumnValues( + BillingTransaction, column, crossFilterParams, recordFilter + ) + return sorted(values, key=lambda v: str(v).lower()) + except Exception: + enriched = _buildTransactionsList(ctx, targetMandateId) + crossFiltered = _applyFiltersAndSort(enriched, crossFilterParams) + return _extractDistinctValues(crossFiltered, column) + except Exception as e: + logger.error(f"Error getting filter values for transactions: {e}") + raise HTTPException(status_code=500, detail=str(e)) + + # ============================================================================= # Mandate View Endpoints (for Admins) # ============================================================================= @@ -1892,3 +2032,50 @@ def getUserViewTransactions( except Exception as e: logger.error(f"Error getting user view transactions: {e}") raise HTTPException(status_code=500, detail=str(e)) + + +@router.get("/view/users/transactions/filter-values") +@limiter.limit("60/minute") +def getUserViewTransactionsFilterValues( + request: Request, + column: str = Query(..., description="Column key"), + pagination: Optional[str] = Query(None, description="JSON-encoded current filters"), + ctx: RequestContext = Depends(getRequestContext) +): + """Return distinct filter values for a column in user transactions.""" + try: + billingInterface = getBillingInterface(ctx.user, ctx.mandateId) + scope = _getBillingDataScope(ctx.user) + if scope.isGlobalAdmin: + mandateIds = None + else: + mandateIds = scope.adminMandateIds + scope.memberMandateIds + if not mandateIds: + return [] + allTransactions = billingInterface.getUserTransactionsForMandates(mandateIds, limit=10000) + allTransactions = _filterTransactionsByScope(allTransactions, scope) + transactionDicts = [] + for t in allTransactions: + transactionDicts.append({ + "id": t.get("id"), + "accountId": t.get("accountId"), + "transactionType": t.get("transactionType", "DEBIT"), + "amount": t.get("amount", 0.0), + "description": t.get("description", ""), + "referenceType": t.get("referenceType"), + "workflowId": t.get("workflowId"), + "featureCode": t.get("featureCode"), + "featureInstanceId": t.get("featureInstanceId"), + "aicoreProvider": t.get("aicoreProvider"), + "aicoreModel": t.get("aicoreModel"), + "createdByUserId": t.get("createdByUserId"), + "createdAt": t.get("_createdAt"), + "mandateId": t.get("mandateId"), + "mandateName": t.get("mandateName"), + "userId": t.get("userId"), + "userName": t.get("userName"), + }) + return _handleFilterValuesRequest(transactionDicts, column, pagination) + except Exception as e: + logger.error(f"Error getting filter values for user transactions: {e}") + raise HTTPException(status_code=500, detail=str(e)) diff --git a/modules/routes/routeDataConnections.py b/modules/routes/routeDataConnections.py index 2a6be738..17ef0115 100644 --- a/modules/routes/routeDataConnections.py +++ b/modules/routes/routeDataConnections.py @@ -147,6 +147,11 @@ async def get_connections( try: interface = getInterface(currentUser) + # NOTE: Cannot use db.getRecordsetPaginated() here because each connection + # is enriched with computed tokenStatus/tokenExpiresAt (requires per-row DB lookup). + # Token refresh also may trigger re-fetch. Connections per user are typically < 10, + # so in-memory pagination is acceptable. + # Parse pagination parameter paginationParams = None if pagination: @@ -287,6 +292,42 @@ async def get_connections( detail=f"Failed to get connections: {str(e)}" ) +@router.get("/filter-values") +@limiter.limit("60/minute") +def get_connection_filter_values( + request: Request, + column: str = Query(..., description="Column key"), + pagination: Optional[str] = Query(None, description="JSON-encoded current filters"), + currentUser: User = Depends(getCurrentUser) +) -> List[str]: + """Return distinct filter values for a column in connections.""" + try: + from modules.routes.routeDataUsers import _handleFilterValuesRequest + interface = getInterface(currentUser) + connections = interface.getUserConnections(currentUser.id) + items = [] + for connection in connections: + tokenStatus, tokenExpiresAt = getTokenStatusForConnection(interface, connection.id) + items.append({ + "id": connection.id, + "userId": connection.userId, + "authority": connection.authority.value if hasattr(connection.authority, 'value') else str(connection.authority), + "externalId": connection.externalId, + "externalUsername": connection.externalUsername or "", + "externalEmail": connection.externalEmail, + "status": connection.status.value if hasattr(connection.status, 'value') else str(connection.status), + "connectedAt": connection.connectedAt, + "lastChecked": connection.lastChecked, + "expiresAt": connection.expiresAt, + "tokenStatus": tokenStatus, + "tokenExpiresAt": tokenExpiresAt + }) + return _handleFilterValuesRequest(items, column, pagination) + except Exception as e: + logger.error(f"Error getting filter values for connections: {str(e)}") + raise HTTPException(status_code=500, detail=str(e)) + + @router.post("/", response_model=UserConnection) @limiter.limit("10/minute") def create_connection( diff --git a/modules/routes/routeDataFiles.py b/modules/routes/routeDataFiles.py index 470793bb..bb77ffc5 100644 --- a/modules/routes/routeDataFiles.py +++ b/modules/routes/routeDataFiles.py @@ -214,6 +214,56 @@ def get_files( detail=f"Failed to get files: {str(e)}" ) +@router.get("/list/filter-values") +@limiter.limit("60/minute") +def get_file_filter_values( + request: Request, + column: str = Query(..., description="Column key"), + pagination: Optional[str] = Query(None, description="JSON-encoded current filters"), + currentUser: User = Depends(getCurrentUser), + context: RequestContext = Depends(getRequestContext) +) -> list: + """Return distinct filter values for a column in files.""" + try: + managementInterface = interfaceDbManagement.getInterface( + currentUser, + mandateId=str(context.mandateId) if context.mandateId else None, + featureInstanceId=str(context.featureInstanceId) if context.featureInstanceId else None + ) + + crossFilterPagination = None + if pagination: + try: + paginationDict = json.loads(pagination) + if paginationDict: + paginationDict = normalize_pagination_dict(paginationDict) + filters = paginationDict.get("filters", {}) + filters.pop(column, None) + paginationDict["filters"] = filters + paginationDict.pop("sort", None) + crossFilterPagination = PaginationParams(**paginationDict) + except (json.JSONDecodeError, ValueError): + pass + + try: + recordFilter = {"_createdBy": managementInterface.userId} + values = managementInterface.db.getDistinctColumnValues( + FileItem, column, crossFilterPagination, recordFilter + ) + return sorted(values, key=lambda v: str(v).lower()) + except Exception: + from modules.routes.routeDataUsers import _handleFilterValuesRequest + result = managementInterface.getAllFiles(pagination=None) + items = [r.model_dump() if hasattr(r, 'model_dump') else r for r in result] + return _handleFilterValuesRequest(items, column, pagination) + except Exception as e: + logger.error(f"Error getting filter values for files: {str(e)}") + raise HTTPException( + status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, + detail=str(e) + ) + + @router.post("/upload", status_code=status.HTTP_201_CREATED) @limiter.limit("10/minute") async def upload_file( diff --git a/modules/routes/routeDataMandates.py b/modules/routes/routeDataMandates.py index 16b7166d..2c2bd31c 100644 --- a/modules/routes/routeDataMandates.py +++ b/modules/routes/routeDataMandates.py @@ -164,6 +164,66 @@ def get_mandates( detail=f"Failed to get mandates: {str(e)}" ) +@router.get("/filter-values") +@limiter.limit("60/minute") +def get_mandate_filter_values( + request: Request, + column: str = Query(..., description="Column key"), + pagination: Optional[str] = Query(None, description="JSON-encoded current filters"), + context: RequestContext = Depends(getRequestContext) +) -> list: + """Return distinct filter values for a column in mandates.""" + try: + from modules.routes.routeDataUsers import _handleFilterValuesRequest + isSysAdmin = context.hasSysAdminRole + if not isSysAdmin: + adminMandateIds = _getAdminMandateIds(context) + if not adminMandateIds: + raise HTTPException(status_code=status.HTTP_403_FORBIDDEN, detail="Admin role required") + + appInterface = interfaceDbApp.getRootInterface() + + if isSysAdmin: + # SysAdmin: try SQL DISTINCT for DB columns + crossFilterPagination = None + if pagination: + try: + paginationDict = json.loads(pagination) + if paginationDict: + paginationDict = normalize_pagination_dict(paginationDict) + filters = paginationDict.get("filters", {}) + filters.pop(column, None) + paginationDict["filters"] = filters + paginationDict.pop("sort", None) + crossFilterPagination = PaginationParams(**paginationDict) + except (json.JSONDecodeError, ValueError): + pass + try: + values = appInterface.db.getDistinctColumnValues( + Mandate, column, crossFilterPagination + ) + return sorted(values, key=lambda v: str(v).lower()) + except Exception: + result = appInterface.getAllMandates(pagination=None) + items = result if isinstance(result, list) else (result.items if hasattr(result, 'items') else result) + items = [i.model_dump() if hasattr(i, 'model_dump') else i for i in items] + return _handleFilterValuesRequest(items, column, pagination) + else: + # MandateAdmin: in-memory (small set of individual mandate lookups) + result = [] + for mid in adminMandateIds: + mandate = appInterface.getMandate(mid) + if mandate: + result.append(mandate if isinstance(mandate, dict) else mandate.model_dump() if hasattr(mandate, 'model_dump') else vars(mandate)) + items = [i.model_dump() if hasattr(i, 'model_dump') else i for i in result] + return _handleFilterValuesRequest(items, column, pagination) + except HTTPException: + raise + except Exception as e: + logger.error(f"Error getting filter values for mandates: {str(e)}") + raise HTTPException(status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, detail=str(e)) + + @router.get("/{targetMandateId}", response_model=Mandate) @limiter.limit("30/minute") def get_mandate( @@ -540,6 +600,63 @@ def list_mandate_users( ) +@router.get("/{targetMandateId}/users/filter-values") +@limiter.limit("60/minute") +def get_mandate_users_filter_values( + request: Request, + targetMandateId: str = Path(..., description="ID of the mandate"), + column: str = Query(..., description="Column key"), + pagination: Optional[str] = Query(None, description="JSON-encoded current filters"), + context: RequestContext = Depends(getRequestContext) +) -> list: + """Return distinct filter values for a column in mandate users.""" + if not _hasMandateAdminRole(context, targetMandateId) and not context.hasSysAdminRole: + raise HTTPException(status_code=status.HTTP_403_FORBIDDEN, detail="Mandate-Admin role required") + + try: + from modules.routes.routeDataUsers import _handleFilterValuesRequest + rootInterface = interfaceDbApp.getRootInterface() + mandate = rootInterface.getMandate(targetMandateId) + if not mandate: + raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail=f"Mandate {targetMandateId} not found") + + userMandates = rootInterface.getUserMandatesByMandate(targetMandateId) + result = [] + for um in userMandates: + user = rootInterface.getUser(str(um.userId)) + if not user: + continue + roleIds = rootInterface.getRoleIdsForUserMandate(str(um.id)) + roleLabels = [] + filteredRoleIds = [] + seenLabels = set() + for roleId in roleIds: + role = rootInterface.getRole(roleId) + if role: + if role.featureInstanceId: + continue + filteredRoleIds.append(roleId) + if role.roleLabel not in seenLabels: + roleLabels.append(role.roleLabel) + seenLabels.add(role.roleLabel) + result.append({ + "id": str(um.id), + "userId": str(user.id), + "username": user.username, + "email": user.email, + "fullName": user.fullName, + "roleIds": filteredRoleIds, + "roleLabels": roleLabels, + "enabled": um.enabled + }) + return _handleFilterValuesRequest(result, column, pagination) + except HTTPException: + raise + except Exception as e: + logger.error(f"Error getting filter values for mandate users: {str(e)}") + raise HTTPException(status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, detail=str(e)) + + @router.post("/{targetMandateId}/users", response_model=UserMandateResponse) @limiter.limit("30/minute") def add_user_to_mandate( diff --git a/modules/routes/routeDataPrompts.py b/modules/routes/routeDataPrompts.py index faf692fe..f9246ab6 100644 --- a/modules/routes/routeDataPrompts.py +++ b/modules/routes/routeDataPrompts.py @@ -81,6 +81,29 @@ def get_prompts( pagination=None ) +@router.get("/filter-values") +@limiter.limit("60/minute") +def get_prompt_filter_values( + request: Request, + column: str = Query(..., description="Column key"), + pagination: Optional[str] = Query(None, description="JSON-encoded current filters"), + currentUser: User = Depends(getCurrentUser) +) -> list: + """Return distinct filter values for a column in prompts. + + NOTE: Cannot use db.getDistinctColumnValues() because visibility rules + (own + system for regular users) require pre-filtering the recordset. + """ + try: + from modules.routes.routeDataUsers import _handleFilterValuesRequest + managementInterface = interfaceDbManagement.getInterface(currentUser) + result = managementInterface.getAllPrompts(pagination=None) + items = [r.model_dump() if hasattr(r, 'model_dump') else r for r in result] + return _handleFilterValuesRequest(items, column, pagination) + except Exception as e: + raise HTTPException(status_code=500, detail=str(e)) + + @router.post("", response_model=Prompt) @limiter.limit("10/minute") def create_prompt( diff --git a/modules/routes/routeDataUsers.py b/modules/routes/routeDataUsers.py index 16742388..7e903466 100644 --- a/modules/routes/routeDataUsers.py +++ b/modules/routes/routeDataUsers.py @@ -69,6 +69,55 @@ def _isAdminForUser(context: RequestContext, targetUserId: str) -> bool: return False +def _extractDistinctValues(items: List[Dict[str, Any]], columnKey: str) -> List[str]: + """Extract sorted distinct display values for a column from enriched items.""" + values = set() + for item in items: + val = item.get(columnKey) + if val is None or val == "": + continue + if isinstance(val, bool): + values.add("true" if val else "false") + elif isinstance(val, (int, float)): + values.add(str(val)) + elif isinstance(val, dict): + text = val.get("en") or next((v for v in val.values() if isinstance(v, str) and v), None) + if text: + values.add(str(text)) + else: + values.add(str(val)) + return sorted(values, key=lambda v: v.lower()) + + +def _handleFilterValuesRequest( + items: List[Dict[str, Any]], + column: str, + paginationJson: Optional[str] = None, +) -> List[str]: + """ + Generic handler for /filter-values endpoints. + Applies all active filters EXCEPT the one for the requested column (cross-filtering), + then extracts distinct values for that column. + """ + crossFilterParams: Optional[PaginationParams] = None + if paginationJson: + try: + import json + paginationDict = json.loads(paginationJson) + if paginationDict: + paginationDict = normalize_pagination_dict(paginationDict) + filters = paginationDict.get("filters", {}) + filters.pop(column, None) + paginationDict["filters"] = filters + paginationDict.pop("sort", None) + crossFilterParams = PaginationParams(**paginationDict) + except (json.JSONDecodeError, ValueError): + pass + + crossFiltered = _applyFiltersAndSort(items, crossFilterParams) + return _extractDistinctValues(crossFiltered, column) + + def _applyFiltersAndSort(items: List[Dict[str, Any]], paginationParams: Optional[PaginationParams]) -> List[Dict[str, Any]]: """ Apply filters and sorting to a list of items. @@ -121,7 +170,6 @@ def _applyFiltersAndSort(items: List[Dict[str, Any]], paginationParams: Optional if itemValue is None: return False - # Convert to string for comparison if needed itemStr = str(itemValue).lower() valueStr = str(v).lower() @@ -147,6 +195,42 @@ def _applyFiltersAndSort(items: List[Dict[str, Any]], paginationParams: Optional return itemNum <= valueNum except (ValueError, TypeError): return False + elif op == 'between': + if isinstance(v, dict): + fromVal = v.get('from', '') + toVal = v.get('to', '') + if not fromVal and not toVal: + return True + # Date range: from/to are YYYY-MM-DD strings, itemValue may be Unix timestamp + try: + from datetime import datetime, timezone + fromTs = None + toTs = None + if fromVal: + fromTs = datetime.strptime(str(fromVal), '%Y-%m-%d').replace(tzinfo=timezone.utc).timestamp() + if toVal: + toTs = datetime.strptime(str(toVal), '%Y-%m-%d').replace(hour=23, minute=59, second=59, tzinfo=timezone.utc).timestamp() + itemNum = float(itemValue) if not isinstance(itemValue, (int, float)) else itemValue + # Normalize: if item looks like a millisecond timestamp, convert to seconds + if itemNum > 10000000000: + itemNum = itemNum / 1000 + if fromTs is not None and toTs is not None: + return fromTs <= itemNum <= toTs + elif fromTs is not None: + return itemNum >= fromTs + elif toTs is not None: + return itemNum <= toTs + except (ValueError, TypeError): + # Fallback: string comparison (for non-numeric date fields) + fromStr = str(fromVal).lower() if fromVal else '' + toStr = str(toVal).lower() if toVal else '' + if fromStr and toStr: + return fromStr <= itemStr <= toStr + elif fromStr: + return itemStr >= fromStr + elif toStr: + return itemStr <= toStr + return True elif op == 'in': if isinstance(v, list): return itemStr in [str(x).lower() for x in v] @@ -159,23 +243,25 @@ def _applyFiltersAndSort(items: List[Dict[str, Any]], paginationParams: Optional result = [item for item in result if matchesFilter(item, field, operator, value)] - # Apply sorting + # Apply sorting — None values always last if paginationParams.sort: for sortField in reversed(paginationParams.sort): fieldName = sortField.field ascending = sortField.direction == 'asc' - def getSortKey(item: Dict[str, Any]): - value = item.get(fieldName) - if value is None: - return (1, '') # Nulls last - if isinstance(value, bool): - return (0, not value if ascending else value) - if isinstance(value, (int, float)): - return (0, value) - return (0, str(value).lower()) + noneItems = [item for item in result if item.get(fieldName) is None] + nonNoneItems = [item for item in result if item.get(fieldName) is not None] - result = sorted(result, key=getSortKey, reverse=not ascending) + def getSortKey(item: Dict[str, Any], _fn=fieldName): + value = item.get(_fn) + if isinstance(value, bool): + return (0, int(value), '') + if isinstance(value, (int, float)): + return (0, value, '') + return (1, 0, str(value).lower()) + + nonNoneItems = sorted(nonNoneItems, key=getSortKey, reverse=not ascending) + result = nonNoneItems + noneItems return result @@ -291,38 +377,23 @@ def get_users( pagination=None ) elif context.hasSysAdminRole: - # SysAdmin without mandateId sees all users - # Get all users via interface method (returns Pydantic User models) - allUserModels = appInterface.getAllUsers() - # Convert to dictionaries for filtering/sorting - cleanedUsers = [u.model_dump() for u in allUserModels] + # SysAdmin without mandateId — DB-level pagination via interface + result = appInterface.getAllUsers(paginationParams) - # Apply server-side filtering and sorting - filteredUsers = _applyFiltersAndSort(cleanedUsers, paginationParams) - - # Convert to User objects - users = [User(**u) for u in filteredUsers] - - if paginationParams: - import math - totalItems = len(users) - totalPages = math.ceil(totalItems / paginationParams.pageSize) if totalItems > 0 else 0 - startIdx = (paginationParams.page - 1) * paginationParams.pageSize - endIdx = startIdx + paginationParams.pageSize - paginatedUsers = users[startIdx:endIdx] - + if paginationParams and hasattr(result, 'items'): return PaginatedResponse( - items=paginatedUsers, + items=result.items, pagination=PaginationMetadata( currentPage=paginationParams.page, pageSize=paginationParams.pageSize, - totalItems=totalItems, - totalPages=totalPages, + totalItems=result.totalItems, + totalPages=result.totalPages, sort=paginationParams.sort, filters=paginationParams.filters ) ) else: + users = result if isinstance(result, list) else (result.items if hasattr(result, 'items') else []) return PaginatedResponse( items=users, pagination=None @@ -407,6 +478,88 @@ def get_users( detail=f"Failed to get users: {str(e)}" ) +@router.get("/filter-values") +@limiter.limit("60/minute") +def get_user_filter_values( + request: Request, + column: str = Query(..., description="Column key"), + pagination: Optional[str] = Query(None, description="JSON-encoded current filters"), + context: RequestContext = Depends(getRequestContext) +) -> list: + """Return distinct filter values for a column in users.""" + try: + appInterface = interfaceDbApp.getInterface(context.user, mandateId=context.mandateId) + + # Build cross-filter pagination (all filters except the requested column) + crossFilterPagination = None + if pagination: + try: + paginationDict = json.loads(pagination) + if paginationDict: + paginationDict = normalize_pagination_dict(paginationDict) + filters = paginationDict.get("filters", {}) + filters.pop(column, None) + paginationDict["filters"] = filters + paginationDict.pop("sort", None) + crossFilterPagination = PaginationParams(**paginationDict) + except (json.JSONDecodeError, ValueError): + pass + + if context.mandateId: + # Mandate-scoped: in-memory (users require UserMandate join) + result = appInterface.getUsersByMandate(str(context.mandateId), None) + users = result if isinstance(result, list) else (result.items if hasattr(result, 'items') else []) + items = [u.model_dump() if hasattr(u, 'model_dump') else u for u in users] + return _handleFilterValuesRequest(items, column, pagination) + elif context.hasSysAdminRole: + # SysAdmin: use SQL DISTINCT for DB columns + try: + rootInterface = getRootInterface() + values = rootInterface.db.getDistinctColumnValues( + UserInDB, column, crossFilterPagination + ) + return sorted(values, key=lambda v: v.lower()) + except Exception: + users = appInterface.getAllUsers() + items = [u.model_dump() if hasattr(u, 'model_dump') else u for u in users] + return _handleFilterValuesRequest(items, column, pagination) + else: + # Non-admin multi-mandate: aggregate across admin mandates (in-memory) + rootInterface = getRootInterface() + userMandates = rootInterface.getUserMandates(str(context.user.id)) + adminMandateIds = [] + for um in userMandates: + umId = getattr(um, 'id', None) + mandateId = getattr(um, 'mandateId', None) + if not umId or not mandateId: + continue + roleIds = rootInterface.getRoleIdsForUserMandate(str(umId)) + for roleId in roleIds: + role = rootInterface.getRole(roleId) + if role and role.roleLabel == "admin" and not role.featureInstanceId: + adminMandateIds.append(str(mandateId)) + break + if not adminMandateIds: + return [] + seenUserIds = set() + users = [] + for mid in adminMandateIds: + mandateUsers = rootInterface.getUsersByMandate(mid) + uList = mandateUsers if isinstance(mandateUsers, list) else (mandateUsers.items if hasattr(mandateUsers, 'items') else []) + for u in uList: + uid = u.get("id") if isinstance(u, dict) else getattr(u, "id", None) + if uid and uid not in seenUserIds: + seenUserIds.add(uid) + users.append(u) + items = [u.model_dump() if hasattr(u, 'model_dump') else u for u in users] + return _handleFilterValuesRequest(items, column, pagination) + except HTTPException: + raise + except Exception as e: + logger.error(f"Error getting filter values for users: {str(e)}") + raise HTTPException(status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, detail=str(e)) + + @router.get("/{userId}", response_model=User) @limiter.limit("30/minute") def get_user( diff --git a/modules/routes/routeInvitations.py b/modules/routes/routeInvitations.py index 906b11e7..cb913137 100644 --- a/modules/routes/routeInvitations.py +++ b/modules/routes/routeInvitations.py @@ -420,6 +420,13 @@ def list_invitations( Requires Mandate-Admin role. Returns all invitations created for this mandate. + NOTE: Cannot use db.getRecordsetPaginated() because: + - Computed status fields (isExpired, isUsedUp) are derived in-memory + - Filtering by revoked/used/expired requires post-fetch logic + - Invitation volume per mandate is typically low (< 100) + When this endpoint needs FormGeneratorTable pagination, add PaginatedResponse + support with in-memory slicing (similar to routeDataConnections). + Args: includeUsed: Include invitations that have reached maxUses includeExpired: Include expired invitations @@ -485,6 +492,54 @@ def list_invitations( ) +@router.get("/filter-values") +@limiter.limit("60/minute") +def get_invitation_filter_values( + request: Request, + column: str = Query(..., description="Column key"), + pagination: Optional[str] = Query(None, description="JSON-encoded current filters"), + frontendUrl: str = Query("", description="Frontend URL for building invite links"), + includeUsed: bool = Query(False, description="Include already used invitations"), + includeExpired: bool = Query(False, description="Include expired invitations"), + context: RequestContext = Depends(getRequestContext) +) -> list: + """Return distinct filter values for a column in invitations.""" + if not context.mandateId: + raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail="X-Mandate-Id header is required") + if not _hasMandateAdminRole(context): + raise HTTPException(status_code=status.HTTP_403_FORBIDDEN, detail="Mandate-Admin role required") + try: + from modules.routes.routeDataUsers import _handleFilterValuesRequest + rootInterface = getRootInterface() + allInvitations = rootInterface.getInvitationsByMandate(str(context.mandateId)) + currentTime = getUtcTimestamp() + result = [] + for inv in allInvitations: + if inv.revokedAt: + continue + currentUses = inv.currentUses or 0 + maxUses = inv.maxUses or 1 + if not includeUsed and currentUses >= maxUses: + continue + expiresAt = inv.expiresAt or 0 + if not includeExpired and expiresAt < currentTime: + continue + baseUrl = frontendUrl.rstrip("/") if frontendUrl else "" + inviteUrl = f"{baseUrl}/invite/{inv.token}" if baseUrl else "" + result.append({ + **inv.model_dump(), + "inviteUrl": inviteUrl, + "isExpired": expiresAt < currentTime, + "isUsedUp": currentUses >= maxUses + }) + return _handleFilterValuesRequest(result, column, pagination) + except HTTPException: + raise + except Exception as e: + logger.error(f"Error getting filter values for invitations: {e}") + raise HTTPException(status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, detail=str(e)) + + @router.delete("/{invitationId}", response_model=Dict[str, str]) @limiter.limit("30/minute") def revoke_invitation( diff --git a/modules/routes/routeMessaging.py b/modules/routes/routeMessaging.py index 0b4784ff..42e15f0e 100644 --- a/modules/routes/routeMessaging.py +++ b/modules/routes/routeMessaging.py @@ -21,7 +21,7 @@ from modules.datamodels.datamodelMessaging import ( MessagingSubscriptionExecutionResult ) from modules.datamodels.datamodelUam import User -from modules.datamodels.datamodelPagination import PaginationParams, PaginatedResponse, PaginationMetadata +from modules.datamodels.datamodelPagination import PaginationParams, PaginatedResponse, PaginationMetadata, normalize_pagination_dict # Configure logger logger = logging.getLogger(__name__) @@ -48,6 +48,8 @@ def get_subscriptions( if pagination: try: paginationDict = json.loads(pagination) + if paginationDict: + paginationDict = normalize_pagination_dict(paginationDict) paginationParams = PaginationParams(**paginationDict) if paginationDict else None except (json.JSONDecodeError, ValueError) as e: raise HTTPException( @@ -185,6 +187,8 @@ def get_subscription_registrations( if pagination: try: paginationDict = json.loads(pagination) + if paginationDict: + paginationDict = normalize_pagination_dict(paginationDict) paginationParams = PaginationParams(**paginationDict) if paginationDict else None except (json.JSONDecodeError, ValueError) as e: raise HTTPException( @@ -277,6 +281,8 @@ def get_my_registrations( if pagination: try: paginationDict = json.loads(pagination) + if paginationDict: + paginationDict = normalize_pagination_dict(paginationDict) paginationParams = PaginationParams(**paginationDict) if paginationDict else None except (json.JSONDecodeError, ValueError) as e: raise HTTPException( @@ -450,6 +456,8 @@ def get_deliveries( if pagination: try: paginationDict = json.loads(pagination) + if paginationDict: + paginationDict = normalize_pagination_dict(paginationDict) paginationParams = PaginationParams(**paginationDict) if paginationDict else None except (json.JSONDecodeError, ValueError) as e: raise HTTPException( diff --git a/modules/routes/routeRealEstate.py b/modules/routes/routeRealEstate.py index 881a77dc..a3466aca 100644 --- a/modules/routes/routeRealEstate.py +++ b/modules/routes/routeRealEstate.py @@ -227,33 +227,22 @@ async def get_projects( context.user, mandateId=mandateId, featureInstanceId=instanceId ) recordFilter = {"featureInstanceId": instanceId} - items = interface.getProjekte(recordFilter=recordFilter) paginationParams = _parsePagination(pagination) if paginationParams: - if paginationParams.sort: - for sort_field in reversed(paginationParams.sort): - field_name = sort_field.field - direction = sort_field.direction.lower() - items.sort( - key=lambda x: getattr(x, field_name, None), - reverse=(direction == "desc") + result = interface.getProjekte(pagination=paginationParams, recordFilter=recordFilter) + if hasattr(result, 'items'): + return PaginatedResponse( + items=result.items, + pagination=PaginationMetadata( + currentPage=paginationParams.page, + pageSize=paginationParams.pageSize, + totalItems=result.totalItems, + totalPages=result.totalPages, + sort=paginationParams.sort or [], + filters=paginationParams.filters ) - total_items = len(items) - total_pages = (total_items + paginationParams.pageSize - 1) // paginationParams.pageSize - start_idx = (paginationParams.page - 1) * paginationParams.pageSize - end_idx = start_idx + paginationParams.pageSize - paginated_items = items[start_idx:end_idx] - return PaginatedResponse( - items=paginated_items, - pagination=PaginationMetadata( - currentPage=paginationParams.page, - pageSize=paginationParams.pageSize, - totalItems=total_items, - totalPages=total_pages, - sort=paginationParams.sort or [], - filters=paginationParams.filters ) - ) + items = interface.getProjekte(recordFilter=recordFilter) return PaginatedResponse(items=items, pagination=None) @@ -359,33 +348,22 @@ async def get_parcels( context.user, mandateId=mandateId, featureInstanceId=instanceId ) recordFilter = {"featureInstanceId": instanceId} - items = interface.getParzellen(recordFilter=recordFilter) paginationParams = _parsePagination(pagination) if paginationParams: - if paginationParams.sort: - for sort_field in reversed(paginationParams.sort): - field_name = sort_field.field - direction = sort_field.direction.lower() - items.sort( - key=lambda x: getattr(x, field_name, None), - reverse=(direction == "desc") + result = interface.getParzellen(pagination=paginationParams, recordFilter=recordFilter) + if hasattr(result, 'items'): + return PaginatedResponse( + items=result.items, + pagination=PaginationMetadata( + currentPage=paginationParams.page, + pageSize=paginationParams.pageSize, + totalItems=result.totalItems, + totalPages=result.totalPages, + sort=paginationParams.sort or [], + filters=paginationParams.filters ) - total_items = len(items) - total_pages = (total_items + paginationParams.pageSize - 1) // paginationParams.pageSize - start_idx = (paginationParams.page - 1) * paginationParams.pageSize - end_idx = start_idx + paginationParams.pageSize - paginated_items = items[start_idx:end_idx] - return PaginatedResponse( - items=paginated_items, - pagination=PaginationMetadata( - currentPage=paginationParams.page, - pageSize=paginationParams.pageSize, - totalItems=total_items, - totalPages=total_pages, - sort=paginationParams.sort or [], - filters=paginationParams.filters ) - ) + items = interface.getParzellen(recordFilter=recordFilter) return PaginatedResponse(items=items, pagination=None) diff --git a/modules/routes/routeSubscription.py b/modules/routes/routeSubscription.py index 28ad7aa9..8334a8c0 100644 --- a/modules/routes/routeSubscription.py +++ b/modules/routes/routeSubscription.py @@ -12,13 +12,17 @@ Endpoints: - POST /api/subscription/force-cancel — sysadmin immediate cancel (by ID) """ -from fastapi import APIRouter, HTTPException, Depends, Request +from fastapi import APIRouter, HTTPException, Depends, Request, Query from fastapi import status from typing import Dict, Any, List, Optional import logging +import json +import math from pydantic import BaseModel, Field from modules.auth import limiter, getRequestContext, RequestContext +from modules.datamodels.datamodelPagination import PaginationParams, PaginatedResponse, PaginationMetadata, normalize_pagination_dict +from modules.routes.routeDataUsers import _applyFiltersAndSort, _extractDistinctValues logger = logging.getLogger(__name__) @@ -303,16 +307,8 @@ def verifyCheckout( # SysAdmin: global subscription overview # ============================================================================= -@router.get("/admin/all", response_model=List[Dict[str, Any]]) -@limiter.limit("30/minute") -def getAllSubscriptions( - request: Request, - context: RequestContext = Depends(getRequestContext), -): - """SysAdmin: list ALL subscriptions across all mandates with enriched metadata.""" - if not context.hasSysAdminRole: - raise HTTPException(status_code=status.HTTP_403_FORBIDDEN, detail="Sysadmin role required") - +def _buildEnrichedSubscriptions() -> List[Dict[str, Any]]: + """Build the full enriched subscription list (shared by list + filter-values endpoints).""" from modules.interfaces.interfaceDbSubscription import _getRootInterface as getSubRootInterface from modules.datamodels.datamodelSubscription import BUILTIN_PLANS, OPERATIVE_STATUSES @@ -362,3 +358,80 @@ def getAllSubscriptions( enriched.append(sub) return enriched + + +@router.get("/admin/all") +@limiter.limit("30/minute") +def getAllSubscriptions( + request: Request, + pagination: Optional[str] = Query(None, description="JSON-encoded PaginationParams object"), + context: RequestContext = Depends(getRequestContext), +): + """SysAdmin: list ALL subscriptions across all mandates with enriched metadata.""" + if not context.hasSysAdminRole: + raise HTTPException(status_code=status.HTTP_403_FORBIDDEN, detail="Sysadmin role required") + + paginationParams: Optional[PaginationParams] = None + if pagination: + try: + paginationDict = json.loads(pagination) + if paginationDict: + paginationDict = normalize_pagination_dict(paginationDict) + paginationParams = PaginationParams(**paginationDict) + except (json.JSONDecodeError, ValueError) as e: + raise HTTPException(status_code=400, detail=f"Invalid pagination parameter: {str(e)}") + + enriched = _buildEnrichedSubscriptions() + filtered = _applyFiltersAndSort(enriched, paginationParams) + + if paginationParams: + totalItems = len(filtered) + totalPages = math.ceil(totalItems / paginationParams.pageSize) if totalItems > 0 else 0 + startIdx = (paginationParams.page - 1) * paginationParams.pageSize + endIdx = startIdx + paginationParams.pageSize + pageItems = filtered[startIdx:endIdx] + return { + "items": pageItems, + "pagination": PaginationMetadata( + currentPage=paginationParams.page, + pageSize=paginationParams.pageSize, + totalItems=totalItems, + totalPages=totalPages, + sort=paginationParams.sort, + filters=paginationParams.filters, + ).model_dump(), + } + + return {"items": enriched, "pagination": None} + + +@router.get("/admin/all/filter-values") +@limiter.limit("60/minute") +def getFilterValues( + request: Request, + column: str = Query(..., description="Column key to extract distinct values for"), + pagination: Optional[str] = Query(None, description="JSON-encoded current filters (applied except for the requested column)"), + context: RequestContext = Depends(getRequestContext), +): + """Return distinct values for a column, respecting all active filters except the requested one.""" + if not context.hasSysAdminRole: + raise HTTPException(status_code=status.HTTP_403_FORBIDDEN, detail="Sysadmin role required") + + crossFilterParams: Optional[PaginationParams] = None + if pagination: + try: + paginationDict = json.loads(pagination) + if paginationDict: + paginationDict = normalize_pagination_dict(paginationDict) + filters = paginationDict.get("filters", {}) + filters.pop(column, None) + paginationDict["filters"] = filters + paginationDict.pop("sort", None) + crossFilterParams = PaginationParams(**paginationDict) + except (json.JSONDecodeError, ValueError) as e: + raise HTTPException(status_code=400, detail=f"Invalid pagination parameter: {str(e)}") + + enriched = _buildEnrichedSubscriptions() + crossFiltered = _applyFiltersAndSort(enriched, crossFilterParams) + + return _extractDistinctValues(crossFiltered, column) diff --git a/modules/system/mainSystem.py b/modules/system/mainSystem.py index 58667645..bfa3e23f 100644 --- a/modules/system/mainSystem.py +++ b/modules/system/mainSystem.py @@ -191,6 +191,15 @@ NAVIGATION_SECTIONS = [ "order": 40, "adminOnly": True, }, + { + "id": "admin-subscriptions", + "objectKey": "ui.admin.subscriptions", + "label": {"en": "Subscriptions", "de": "Abonnements", "fr": "Abonnements"}, + "icon": "FaFileContract", + "path": "/admin/billing/subscriptions", + "order": 50, + "adminOnly": True, + }, ], }, # ── System ── diff --git a/scripts/.$import_diagram.drawio.bkp b/scripts/.$import_diagram.drawio.bkp deleted file mode 100644 index 588285b0..00000000 --- a/scripts/.$import_diagram.drawio.bkp +++ /dev/null @@ -1,2723 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - \ No newline at end of file diff --git a/scripts/.$import_diagram_containers.drawio.bkp b/scripts/.$import_diagram_containers.drawio.bkp deleted file mode 100644 index cad95eef..00000000 --- a/scripts/.$import_diagram_containers.drawio.bkp +++ /dev/null @@ -1,317 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - \ No newline at end of file