fixed data source

This commit is contained in:
ValueOn AG 2026-03-31 23:40:59 +02:00
parent 413dcd9b6c
commit 5a40b54524
8 changed files with 282 additions and 30 deletions

View file

@ -6,7 +6,7 @@ A FeatureDataSource links a FeatureInstance table (DATA_OBJECT) to a workspace
so the agent can query structured feature data (e.g. TrusteePosition rows).
"""
from typing import Optional
from typing import Dict, Optional
from pydantic import BaseModel, Field
from modules.datamodels.datamodelBase import PowerOnModel
from modules.shared.attributeUtils import registerModelLabels
@ -39,6 +39,10 @@ class FeatureDataSource(PowerOnModel):
description="Whether this data source should be neutralized before AI processing",
json_schema_extra={"frontend_type": "checkbox", "frontend_readonly": False, "frontend_required": False}
)
recordFilter: Optional[Dict[str, str]] = Field(
default=None,
description="Record-level filter applied when querying this table, e.g. {'sessionId': 'abc-123'}",
)
registerModelLabels(

View file

@ -36,12 +36,22 @@ DATA_OBJECTS = [
{
"objectKey": "data.feature.commcoach.CoachingContext",
"label": {"en": "Coaching Context", "de": "Coaching-Kontext", "fr": "Contexte coaching"},
"meta": {"table": "CoachingContext", "fields": ["id", "title", "category", "status"]}
"meta": {
"table": "CoachingContext",
"fields": ["id", "title", "category", "status"],
"isParent": True,
"displayFields": ["title", "category", "status"],
}
},
{
"objectKey": "data.feature.commcoach.CoachingSession",
"label": {"en": "Coaching Session", "de": "Coaching-Session", "fr": "Session coaching"},
"meta": {"table": "CoachingSession", "fields": ["id", "contextId", "status", "summary"]}
"meta": {
"table": "CoachingSession",
"fields": ["id", "contextId", "status", "summary"],
"parentTable": "CoachingContext",
"parentKey": "contextId",
}
},
{
"objectKey": "data.feature.commcoach.CoachingMessage",
@ -51,7 +61,12 @@ DATA_OBJECTS = [
{
"objectKey": "data.feature.commcoach.CoachingTask",
"label": {"en": "Coaching Task", "de": "Coaching-Aufgabe", "fr": "Tache coaching"},
"meta": {"table": "CoachingTask", "fields": ["id", "contextId", "title", "status"]}
"meta": {
"table": "CoachingTask",
"fields": ["id", "contextId", "title", "status"],
"parentTable": "CoachingContext",
"parentKey": "contextId",
}
},
{
"objectKey": "data.feature.commcoach.CoachingScore",

View file

@ -39,17 +39,32 @@ DATA_OBJECTS = [
{
"objectKey": "data.feature.teamsbot.TeamsbotSession",
"label": {"en": "Session", "de": "Sitzung", "fr": "Session"},
"meta": {"table": "TeamsbotSession", "fields": ["id", "meetingLink", "botName", "status", "startedAt", "endedAt"]}
"meta": {
"table": "TeamsbotSession",
"fields": ["id", "meetingLink", "botName", "status", "startedAt", "endedAt"],
"isParent": True,
"displayFields": ["botName", "status", "startedAt"],
}
},
{
"objectKey": "data.feature.teamsbot.TeamsbotTranscript",
"label": {"en": "Transcript", "de": "Transkript", "fr": "Transcription"},
"meta": {"table": "TeamsbotTranscript", "fields": ["id", "sessionId", "speaker", "text", "timestamp"]}
"meta": {
"table": "TeamsbotTranscript",
"fields": ["id", "sessionId", "speaker", "text", "timestamp"],
"parentTable": "TeamsbotSession",
"parentKey": "sessionId",
}
},
{
"objectKey": "data.feature.teamsbot.TeamsbotBotResponse",
"label": {"en": "Bot Response", "de": "Bot-Antwort", "fr": "Réponse du bot"},
"meta": {"table": "TeamsbotBotResponse", "fields": ["id", "sessionId", "responseText", "detectedIntent"]}
"meta": {
"table": "TeamsbotBotResponse",
"fields": ["id", "sessionId", "responseText", "detectedIntent"],
"parentTable": "TeamsbotSession",
"parentKey": "sessionId",
}
},
{
"objectKey": "data.feature.teamsbot.*",

View file

@ -58,10 +58,25 @@ UI_OBJECTS = [
# DATA Objects for RBAC catalog (tables/entities)
# Used for AccessRules on data-level permissions
DATA_OBJECTS = [
{
"objectKey": "data.feature.trustee.TrusteeOrganisation",
"label": {"en": "Organisation", "de": "Organisation", "fr": "Organisation"},
"meta": {
"table": "TrusteeOrganisation",
"fields": ["id", "label", "enabled"],
"isParent": True,
"displayFields": ["label"],
}
},
{
"objectKey": "data.feature.trustee.TrusteePosition",
"label": {"en": "Position", "de": "Position", "fr": "Position"},
"meta": {"table": "TrusteePosition", "fields": ["id", "label", "description", "organisationId"]}
"meta": {
"table": "TrusteePosition",
"fields": ["id", "label", "description", "organisationId"],
"parentTable": "TrusteeOrganisation",
"parentKey": "organisationId",
}
},
{
"objectKey": "data.feature.trustee.TrusteeDocument",
@ -71,7 +86,12 @@ DATA_OBJECTS = [
{
"objectKey": "data.feature.trustee.TrusteeAccountingConfig",
"label": {"en": "Accounting Config", "de": "Buchhaltungs-Konfiguration", "fr": "Config. comptable"},
"meta": {"table": "TrusteeAccountingConfig", "fields": ["id", "connectorType", "displayLabel", "encryptedConfig", "isActive"]}
"meta": {
"table": "TrusteeAccountingConfig",
"fields": ["id", "connectorType", "displayLabel", "encryptedConfig", "isActive"],
"parentTable": "TrusteeOrganisation",
"parentKey": "organisationId",
}
},
{
"objectKey": "data.feature.trustee.TrusteeAccountingSync",

View file

@ -270,12 +270,19 @@ def _buildFeatureDataSourceContext(featureDataSourceIds: List[str]) -> str:
tableFields = obj.get("meta", {}).get("fields", [])
break
recordFilter = fds.get("recordFilter")
filterLine = ""
if recordFilter and isinstance(recordFilter, dict):
filterParts = [f"{k} = {v}" for k, v in recordFilter.items()]
filterLine = f"\n recordFilter: {', '.join(filterParts)} (data is scoped to this record)"
parts.append(
f"- featureInstanceId: {fiId}\n"
f" feature: {featureCode}\n"
f" instance: \"{instanceLabel}\"\n"
f" table: {tableName} ({label})\n"
f" fields: {', '.join(tableFields) if tableFields else 'all'}"
f"{filterLine}"
)
except Exception as e:
logger.warning(f"Error loading FeatureDataSource {fdsId}: {e}")
@ -1336,8 +1343,8 @@ async def listFeatureConnections(
instanceId: str = Path(...),
context: RequestContext = Depends(getRequestContext),
):
"""List feature instances the user has access to across ALL mandates."""
_validateInstanceAccess(instanceId, context)
"""List feature instances the user has access to, scoped to the workspace mandate."""
wsMandateId, _ = _validateInstanceAccess(instanceId, context)
from modules.interfaces.interfaceDbApp import getRootInterface
from modules.security.rbacCatalog import getCatalogService
from modules.datamodels.datamodelUam import Mandate
@ -1352,8 +1359,14 @@ async def listFeatureConnections(
if not userMandates:
return JSONResponse({"featureConnectionsByMandate": []})
allowedMandateIds = {um.mandateId for um in userMandates}
if wsMandateId and wsMandateId in allowedMandateIds:
allowedMandateIds = {wsMandateId}
mandateLabels: dict = {}
for um in userMandates:
if um.mandateId not in allowedMandateIds:
continue
try:
rows = rootIf.db.getRecordset(Mandate, recordFilter={"id": um.mandateId})
if rows:
@ -1365,6 +1378,8 @@ async def listFeatureConnections(
byMandate: dict = {}
seenIds: set = set()
for um in userMandates:
if um.mandateId not in allowedMandateIds:
continue
allInstances = rootIf.getFeatureInstancesByMandate(um.mandateId)
for inst in allInstances:
if inst.id in seenIds:
@ -1418,7 +1433,7 @@ async def listFeatureConnectionTables(
context: RequestContext = Depends(getRequestContext),
):
"""List data tables (DATA_OBJECTS) for a feature instance, filtered by RBAC."""
_validateInstanceAccess(instanceId, context)
wsMandateId, _ = _validateInstanceAccess(instanceId, context)
from modules.interfaces.interfaceDbApp import getRootInterface
from modules.security.rbacCatalog import getCatalogService
@ -1428,6 +1443,8 @@ async def listFeatureConnectionTables(
raise HTTPException(status_code=404, detail="Feature instance not found")
mandateId = str(inst.mandateId) if inst.mandateId else None
if wsMandateId and mandateId and mandateId != wsMandateId:
raise HTTPException(status_code=403, detail="Feature instance does not belong to workspace mandate")
catalog = getCatalogService()
try:
@ -1448,16 +1465,132 @@ async def listFeatureConnectionTables(
tables = []
for obj in accessible:
meta = obj.get("meta", {})
tables.append({
node = {
"objectKey": obj.get("objectKey", ""),
"tableName": meta.get("table", ""),
"label": obj.get("label", {}),
"fields": meta.get("fields", []),
})
}
if meta.get("isParent"):
node["isParent"] = True
node["displayFields"] = meta.get("displayFields", [])
if meta.get("parentTable"):
node["parentTable"] = meta["parentTable"]
node["parentKey"] = meta.get("parentKey", "")
tables.append(node)
return JSONResponse({"tables": tables})
@router.get("/{instanceId}/feature-connections/{fiId}/parent-objects/{tableName}")
@limiter.limit("120/minute")
async def listParentObjects(
request: Request,
instanceId: str = Path(...),
fiId: str = Path(..., description="Feature instance ID"),
tableName: str = Path(..., description="Parent table name from DATA_OBJECTS"),
context: RequestContext = Depends(getRequestContext),
):
"""List records from a parent table so the user can pick a specific record to scope data."""
wsMandateId, _ = _validateInstanceAccess(instanceId, context)
from modules.interfaces.interfaceDbApp import getRootInterface
from modules.security.rbacCatalog import getCatalogService
rootIf = getRootInterface()
inst = rootIf.getFeatureInstance(fiId)
if not inst:
raise HTTPException(status_code=404, detail="Feature instance not found")
featureCode = inst.featureCode
mandateId = str(inst.mandateId) if inst.mandateId else ""
if wsMandateId and mandateId and mandateId != wsMandateId:
raise HTTPException(status_code=403, detail="Feature instance does not belong to workspace mandate")
catalog = getCatalogService()
parentObj = None
for obj in catalog.getDataObjects(featureCode):
meta = obj.get("meta", {})
if meta.get("table") == tableName and meta.get("isParent"):
parentObj = obj
break
if not parentObj:
raise HTTPException(status_code=400, detail=f"Table '{tableName}' is not a registered parent table")
displayFields = parentObj["meta"].get("displayFields", [])
selectCols = ', '.join(f'"{f}"' for f in (["id"] + displayFields)) if displayFields else "*"
from modules.connectors.connectorDbPostgre import DatabaseConnector
from modules.shared.configuration import APP_CONFIG
featureDbName = f"poweron_{featureCode.lower()}"
featureDbConn = None
try:
featureDbConn = DatabaseConnector(
dbHost=APP_CONFIG.get("DB_HOST", "localhost"),
dbDatabase=featureDbName,
dbUser=APP_CONFIG.get("DB_USER"),
dbPassword=APP_CONFIG.get("DB_PASSWORD_SECRET"),
dbPort=int(APP_CONFIG.get("DB_PORT", 5432)),
userId=str(context.user.id),
)
conn = featureDbConn.connection
with conn.cursor() as cur:
cur.execute(
"SELECT column_name FROM information_schema.columns "
"WHERE table_schema = 'public' AND LOWER(table_name) = LOWER(%s) "
"AND column_name IN ('featureInstanceId', 'instanceId')",
[tableName],
)
instanceCols = [row["column_name"] for row in cur.fetchall()]
instanceCol = "featureInstanceId" if "featureInstanceId" in instanceCols else "instanceId"
cur.execute(
"SELECT column_name FROM information_schema.columns "
"WHERE table_schema = 'public' AND LOWER(table_name) = LOWER(%s) "
"AND column_name = 'userId'",
[tableName],
)
hasUserId = cur.rowcount > 0
sql = (
f'SELECT {selectCols} FROM "{tableName}" '
f'WHERE "{instanceCol}" = %s'
)
params = [fiId]
if mandateId:
sql += ' AND "mandateId" = %s'
params.append(mandateId)
if hasUserId:
sql += ' AND "userId" = %s'
params.append(str(context.user.id))
sql += ' ORDER BY "id" DESC LIMIT 100'
cur.execute(sql, params)
rows = []
for row in cur.fetchall():
r = dict(row)
for k, v in r.items():
if hasattr(v, "isoformat"):
r[k] = v.isoformat()
elif isinstance(v, (bytes, bytearray)):
r[k] = f"<binary {len(v)} bytes>"
displayParts = [str(r.get(f, "")) for f in displayFields if r.get(f) is not None]
rows.append({
"id": r.get("id", ""),
"displayLabel": " | ".join(displayParts) if displayParts else r.get("id", ""),
"fields": {f: r.get(f) for f in displayFields},
})
except Exception as e:
logger.error(f"listParentObjects({tableName}) failed: {e}", exc_info=True)
raise HTTPException(status_code=500, detail=f"Failed to list parent objects: {e}")
finally:
if featureDbConn:
try:
featureDbConn.close()
except Exception:
pass
return JSONResponse({"parentObjects": rows})
class CreateFeatureDataSourceRequest(BaseModel):
"""Request body for adding a feature table as data source."""
featureInstanceId: str = Field(description="Feature instance ID")
@ -1465,6 +1598,7 @@ class CreateFeatureDataSourceRequest(BaseModel):
tableName: str = Field(description="Table name from DATA_OBJECTS")
objectKey: str = Field(description="RBAC object key")
label: str = Field(description="User-visible label")
recordFilter: Optional[dict] = Field(default=None, description="Record-level filter for scoping")
@router.post("/{instanceId}/feature-datasources")
@ -1476,13 +1610,15 @@ async def createFeatureDataSource(
context: RequestContext = Depends(getRequestContext),
):
"""Create a FeatureDataSource for this workspace instance."""
_validateInstanceAccess(instanceId, context)
wsMandateId, _ = _validateInstanceAccess(instanceId, context)
from modules.interfaces.interfaceDbApp import getRootInterface
from modules.datamodels.datamodelFeatureDataSource import FeatureDataSource
rootIf = getRootInterface()
inst = rootIf.getFeatureInstance(body.featureInstanceId)
mandateId = str(inst.mandateId) if inst else (str(context.mandateId) if context.mandateId else "")
if wsMandateId and mandateId and mandateId != wsMandateId:
raise HTTPException(status_code=403, detail="Feature instance does not belong to workspace mandate")
fds = FeatureDataSource(
featureInstanceId=body.featureInstanceId,
@ -1493,6 +1629,7 @@ async def createFeatureDataSource(
mandateId=mandateId,
userId=str(context.user.id),
workspaceInstanceId=instanceId,
recordFilter=body.recordFilter,
)
created = rootIf.db.recordCreate(FeatureDataSource, fds.model_dump())
return JSONResponse(created if isinstance(created, dict) else fds.model_dump())

View file

@ -38,6 +38,7 @@ async def runFeatureDataAgent(
aiCallFn: Callable[[AiCallRequest], Awaitable[AiCallResponse]],
dbConnector,
instanceLabel: str = "",
tableFilters: Optional[Dict[str, Dict[str, str]]] = None,
) -> str:
"""Run the feature data sub-agent and return the textual result.
@ -51,13 +52,14 @@ async def runFeatureDataAgent(
aiCallFn: AI call function (with billing).
dbConnector: DatabaseConnector for queries.
instanceLabel: Human-readable instance name for context.
tableFilters: Per-table record filters from FeatureDataSource.recordFilter.
Returns:
Plain-text answer produced by the sub-agent.
"""
provider = FeatureDataProvider(dbConnector)
registry = _buildSubAgentTools(provider, featureInstanceId, mandateId)
registry = _buildSubAgentTools(provider, featureInstanceId, mandateId, tableFilters or {})
for tbl in selectedTables:
meta = tbl.get("meta", {})
@ -103,9 +105,18 @@ def _buildSubAgentTools(
provider: FeatureDataProvider,
featureInstanceId: str,
mandateId: str,
tableFilters: Dict[str, Dict[str, str]] = None,
) -> ToolRegistry:
"""Register browseTable and queryTable as sub-agent tools."""
registry = ToolRegistry()
_tableFilters = tableFilters or {}
def _recordFilterToList(tableName: str) -> Optional[List[Dict[str, Any]]]:
"""Convert a recordFilter dict to a list of {field, op, value} filter dicts."""
rf = _tableFilters.get(tableName)
if not rf:
return None
return [{"field": k, "op": "=", "value": v} for k, v in rf.items()]
async def _browseTable(args: Dict[str, Any], context: Dict[str, Any]):
tableName = args.get("tableName", "")
@ -121,6 +132,7 @@ def _buildSubAgentTools(
fields=fields,
limit=min(limit, 200),
offset=offset,
extraFilters=_recordFilterToList(tableName),
)
return ToolResult(
toolCallId="", toolName="browseTable",
@ -147,6 +159,7 @@ def _buildSubAgentTools(
orderBy=orderBy,
limit=min(limit, 200),
offset=offset,
extraFilters=_recordFilterToList(tableName),
)
return ToolResult(
toolCallId="", toolName="queryTable",

View file

@ -69,28 +69,36 @@ class FeatureDataProvider:
fields: List[str] = None,
limit: int = 50,
offset: int = 0,
extraFilters: Optional[List[Dict[str, Any]]] = None,
) -> Dict[str, Any]:
"""List rows from a feature table with pagination.
Returns ``{"rows": [...], "total": N, "limit": L, "offset": O}``.
"""
_validateTableName(tableName)
scopeFilter = _buildScopeFilter(tableName, featureInstanceId, mandateId)
conn = self._db.connection
scopeFilter = _buildScopeFilter(tableName, featureInstanceId, mandateId, dbConnection=conn)
extraWhere, extraParams = _buildFilterClauses(extraFilters)
fullWhere = scopeFilter["where"]
allParams = list(scopeFilter["params"])
if extraWhere:
fullWhere += " AND " + extraWhere
allParams.extend(extraParams)
try:
conn = self._db.connection
with conn.cursor() as cur:
countSql = f'SELECT COUNT(*) FROM "{tableName}" WHERE {scopeFilter["where"]}'
cur.execute(countSql, scopeFilter["params"])
countSql = f'SELECT COUNT(*) FROM "{tableName}" WHERE {fullWhere}'
cur.execute(countSql, allParams)
total = cur.fetchone()["count"] if cur.rowcount else 0
selectCols = ", ".join(f'"{f}"' for f in fields) if fields else "*"
dataSql = (
f'SELECT {selectCols} FROM "{tableName}" '
f'WHERE {scopeFilter["where"]} '
f'WHERE {fullWhere} '
f'ORDER BY "id" LIMIT %s OFFSET %s'
)
cur.execute(dataSql, scopeFilter["params"] + [limit, offset])
cur.execute(dataSql, allParams + [limit, offset])
rows = [_serializeRow(dict(r)) for r in cur.fetchall()]
return {"rows": rows, "total": total, "limit": limit, "offset": offset}
@ -108,14 +116,19 @@ class FeatureDataProvider:
orderBy: str = None,
limit: int = 50,
offset: int = 0,
extraFilters: Optional[List[Dict[str, Any]]] = None,
) -> Dict[str, Any]:
"""Query a feature table with optional filters.
``filters`` is a list of ``{"field": "x", "op": "=", "value": "y"}``.
``extraFilters`` are mandatory record-level scoping filters injected by the pipeline.
"""
_validateTableName(tableName)
scopeFilter = _buildScopeFilter(tableName, featureInstanceId, mandateId)
extraWhere, extraParams = _buildFilterClauses(filters)
conn = self._db.connection
scopeFilter = _buildScopeFilter(tableName, featureInstanceId, mandateId, dbConnection=conn)
combinedFilters = list(filters or []) + list(extraFilters or [])
extraWhere, extraParams = _buildFilterClauses(combinedFilters if combinedFilters else None)
fullWhere = scopeFilter["where"]
allParams = list(scopeFilter["params"])
@ -124,7 +137,6 @@ class FeatureDataProvider:
allParams.extend(extraParams)
try:
conn = self._db.connection
with conn.cursor() as cur:
countSql = f'SELECT COUNT(*) FROM "{tableName}" WHERE {fullWhere}'
cur.execute(countSql, allParams)
@ -149,6 +161,34 @@ class FeatureDataProvider:
# helpers
# ------------------------------------------------------------------
_instanceColCache: Dict[str, str] = {}
def _resolveInstanceColumn(tableName: str, dbConnection=None) -> str:
"""Detect whether the table uses ``instanceId`` or ``featureInstanceId``."""
if tableName in _instanceColCache:
return _instanceColCache[tableName]
if dbConnection:
try:
with dbConnection.cursor() as cur:
cur.execute(
"SELECT column_name FROM information_schema.columns "
"WHERE table_schema = 'public' AND LOWER(table_name) = LOWER(%s) "
"AND column_name IN ('featureInstanceId', 'instanceId')",
[tableName],
)
cols = [row["column_name"] for row in cur.fetchall()]
if "featureInstanceId" in cols:
_instanceColCache[tableName] = "featureInstanceId"
return "featureInstanceId"
if "instanceId" in cols:
_instanceColCache[tableName] = "instanceId"
return "instanceId"
except Exception:
pass
return "instanceId"
def _validateTableName(tableName: str):
if not tableName or not _isValidIdentifier(tableName):
raise ValueError(f"Invalid table name: {tableName}")
@ -159,17 +199,19 @@ def _isValidIdentifier(name: str) -> bool:
return name.isidentifier()
def _buildScopeFilter(tableName: str, featureInstanceId: str, mandateId: str) -> Dict[str, Any]:
def _buildScopeFilter(tableName: str, featureInstanceId: str, mandateId: str, dbConnection=None) -> Dict[str, Any]:
"""Build the mandatory WHERE clause that scopes rows to the feature instance.
Feature tables usually have either ``featureInstanceId`` or a combination
of ``mandateId`` + an org/context FK. We try ``featureInstanceId`` first,
then fall back to ``mandateId``.
Feature tables use either ``instanceId`` (commcoach, teamsbot) or
``featureInstanceId`` (trustee) as the FK. We detect the actual column
from ``information_schema`` when a DB connection is provided.
"""
instanceCol = _resolveInstanceColumn(tableName, dbConnection)
conditions = []
params = []
conditions.append('"featureInstanceId" = %s')
conditions.append(f'"{instanceCol}" = %s')
params.append(featureInstanceId)
if mandateId:

View file

@ -3192,11 +3192,16 @@ def _registerCoreTools(registry: ToolRegistry, services):
from modules.security.rbacCatalog import getCatalogService
catalog = getCatalogService()
tableFilters = {}
if not featureDataSources:
selectedTables = catalog.getDataObjects(featureCode)
else:
allObjs = {o["meta"]["table"]: o for o in catalog.getDataObjects(featureCode) if "meta" in o and "table" in o.get("meta", {})}
selectedTables = [allObjs[ds["tableName"]] for ds in featureDataSources if ds.get("tableName") in allObjs]
for ds in featureDataSources:
rf = ds.get("recordFilter")
if rf and isinstance(rf, dict) and ds.get("tableName"):
tableFilters[ds["tableName"]] = rf
if not selectedTables:
return ToolResult(
@ -3239,6 +3244,7 @@ def _registerCoreTools(registry: ToolRegistry, services):
aiCallFn=_subAgentAiCall,
dbConnector=featureDbConn,
instanceLabel=instanceLabel,
tableFilters=tableFilters,
)
finally:
try: