cleanup intra referencings in codebase
This commit is contained in:
parent
4f8473bd70
commit
26dd8f6f3f
78 changed files with 1048 additions and 781 deletions
14
app.py
14
app.py
|
|
@ -318,9 +318,23 @@ async def lifespan(app: FastAPI):
|
||||||
onMandateDelete as _waOnMandateDelete,
|
onMandateDelete as _waOnMandateDelete,
|
||||||
onInstanceCreate as _waOnInstanceCreate,
|
onInstanceCreate as _waOnInstanceCreate,
|
||||||
)
|
)
|
||||||
|
from modules.interfaces.interfaceDbBilling import (
|
||||||
|
onMandateDelete as _billingOnMandateDelete,
|
||||||
|
onMandateProvision as _billingOnMandateProvision,
|
||||||
|
onStorageChanged as _billingOnStorageChanged,
|
||||||
|
onUserMandateCreate as _billingOnUserMandateCreate,
|
||||||
|
onUserMandateDelete as _billingOnUserMandateDelete,
|
||||||
|
onUserBudgetAdjust as _billingOnUserBudgetAdjust,
|
||||||
|
)
|
||||||
registerLifecycleHook("onBootstrap", _waOnBootstrap)
|
registerLifecycleHook("onBootstrap", _waOnBootstrap)
|
||||||
registerLifecycleHook("onMandateDelete", _waOnMandateDelete)
|
registerLifecycleHook("onMandateDelete", _waOnMandateDelete)
|
||||||
|
registerLifecycleHook("onMandateDelete", _billingOnMandateDelete)
|
||||||
|
registerLifecycleHook("onMandateProvision", _billingOnMandateProvision)
|
||||||
|
registerLifecycleHook("onStorageChanged", _billingOnStorageChanged)
|
||||||
registerLifecycleHook("onInstanceCreate", _waOnInstanceCreate)
|
registerLifecycleHook("onInstanceCreate", _waOnInstanceCreate)
|
||||||
|
registerLifecycleHook("onUserMandateCreate", _billingOnUserMandateCreate)
|
||||||
|
registerLifecycleHook("onUserMandateDelete", _billingOnUserMandateDelete)
|
||||||
|
registerLifecycleHook("onUserBudgetAdjust", _billingOnUserBudgetAdjust)
|
||||||
|
|
||||||
# Bootstrap database if needed (creates initial users, mandates, roles, etc.)
|
# Bootstrap database if needed (creates initial users, mandates, roles, etc.)
|
||||||
# This must happen before getting root interface
|
# This must happen before getting root interface
|
||||||
|
|
|
||||||
|
|
@ -10,16 +10,13 @@ import importlib
|
||||||
import os
|
import os
|
||||||
import time
|
import time
|
||||||
import threading
|
import threading
|
||||||
from typing import Dict, List, Optional, Any, Tuple, TYPE_CHECKING
|
from typing import Dict, List, Optional, Any, Tuple
|
||||||
from modules.datamodels.datamodelAi import AiModel
|
from modules.datamodels.datamodelAi import AiModel
|
||||||
from modules.datamodels.datamodelRbac import AccessRuleContext
|
from modules.datamodels.datamodelRbac import AccessRuleContext, RbacProtocol
|
||||||
from .aicoreBase import BaseConnectorAi
|
from .aicoreBase import BaseConnectorAi
|
||||||
from modules.datamodels.datamodelUam import User
|
from modules.datamodels.datamodelUam import User
|
||||||
from modules.connectors.connectorDbPostgre import DatabaseConnector
|
from modules.connectors.connectorDbPostgre import DatabaseConnector
|
||||||
|
|
||||||
if TYPE_CHECKING:
|
|
||||||
from modules.security.rbac import RbacClass
|
|
||||||
|
|
||||||
logger = logging.getLogger(__name__)
|
logger = logging.getLogger(__name__)
|
||||||
|
|
||||||
# TODO TESTING: Override maxTokens for all models during testing
|
# TODO TESTING: Override maxTokens for all models during testing
|
||||||
|
|
@ -188,7 +185,7 @@ class ModelRegistry:
|
||||||
def getAvailableModels(
|
def getAvailableModels(
|
||||||
self,
|
self,
|
||||||
currentUser: Optional[User] = None,
|
currentUser: Optional[User] = None,
|
||||||
rbacInstance: Optional["RbacClass"] = None,
|
rbacInstance: Optional[RbacProtocol] = None,
|
||||||
mandateId: Optional[str] = None,
|
mandateId: Optional[str] = None,
|
||||||
featureInstanceId: Optional[str] = None
|
featureInstanceId: Optional[str] = None
|
||||||
) -> List[AiModel]:
|
) -> List[AiModel]:
|
||||||
|
|
@ -239,7 +236,7 @@ class ModelRegistry:
|
||||||
self,
|
self,
|
||||||
models: List[AiModel],
|
models: List[AiModel],
|
||||||
currentUser: User,
|
currentUser: User,
|
||||||
rbacInstance: "RbacClass",
|
rbacInstance: RbacProtocol,
|
||||||
mandateId: Optional[str] = None,
|
mandateId: Optional[str] = None,
|
||||||
featureInstanceId: Optional[str] = None
|
featureInstanceId: Optional[str] = None
|
||||||
) -> List[AiModel]:
|
) -> List[AiModel]:
|
||||||
|
|
@ -264,7 +261,7 @@ class ModelRegistry:
|
||||||
logger.debug(f"User {currentUser.username} does not have access to model {model.displayName} (connector: {model.connectorType})")
|
logger.debug(f"User {currentUser.username} does not have access to model {model.displayName} (connector: {model.connectorType})")
|
||||||
return filteredModels
|
return filteredModels
|
||||||
|
|
||||||
def getModel(self, displayName: str, currentUser: Optional[User] = None, rbacInstance: Optional["RbacClass"] = None) -> Optional[AiModel]:
|
def getModel(self, displayName: str, currentUser: Optional[User] = None, rbacInstance: Optional[RbacProtocol] = None) -> Optional[AiModel]:
|
||||||
"""Get a specific model by displayName, optionally checking RBAC permissions.
|
"""Get a specific model by displayName, optionally checking RBAC permissions.
|
||||||
|
|
||||||
Args:
|
Args:
|
||||||
|
|
|
||||||
|
|
@ -10,7 +10,7 @@ Multi-Tenant Design:
|
||||||
"""
|
"""
|
||||||
|
|
||||||
import uuid
|
import uuid
|
||||||
from typing import Optional
|
from typing import Optional, Dict, List, Protocol, runtime_checkable
|
||||||
from enum import Enum
|
from enum import Enum
|
||||||
from pydantic import BaseModel, Field
|
from pydantic import BaseModel, Field
|
||||||
from modules.datamodels.datamodelBase import PowerOnModel
|
from modules.datamodels.datamodelBase import PowerOnModel
|
||||||
|
|
@ -174,6 +174,20 @@ class AccessRule(PowerOnModel):
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
|
@runtime_checkable
|
||||||
|
class RbacProtocol(Protocol):
|
||||||
|
"""Structural type for RBAC checkers — allows aicore (L3) to reference
|
||||||
|
the RBAC contract without importing from security (L4)."""
|
||||||
|
|
||||||
|
def checkResourceAccessBulk(
|
||||||
|
self,
|
||||||
|
user: "User",
|
||||||
|
resourcePaths: List[str],
|
||||||
|
mandateId: Optional[str] = None,
|
||||||
|
featureInstanceId: Optional[str] = None,
|
||||||
|
) -> Dict[str, bool]: ...
|
||||||
|
|
||||||
|
|
||||||
# IMMUTABLE Fields Definition - für Enforcement auf Application-Level
|
# IMMUTABLE Fields Definition - für Enforcement auf Application-Level
|
||||||
IMMUTABLE_FIELDS = {
|
IMMUTABLE_FIELDS = {
|
||||||
"Role": ["mandateId", "featureInstanceId", "featureCode"],
|
"Role": ["mandateId", "featureInstanceId", "featureCode"],
|
||||||
|
|
|
||||||
|
|
@ -169,7 +169,7 @@ class InvestorDemo2026(BaseDemoConfig):
|
||||||
|
|
||||||
def _ensureMandate(self, db, mandateDef: Dict, summary: Dict) -> Optional[str]:
|
def _ensureMandate(self, db, mandateDef: Dict, summary: Dict) -> Optional[str]:
|
||||||
from modules.datamodels.datamodelUam import Mandate
|
from modules.datamodels.datamodelUam import Mandate
|
||||||
from modules.interfaces.interfaceBootstrap import copySystemRolesToMandate
|
from modules.interfaces.interfaceRbac import copySystemRolesToMandate
|
||||||
|
|
||||||
existing = db.getRecordset(Mandate, recordFilter={"name": mandateDef["name"]})
|
existing = db.getRecordset(Mandate, recordFilter={"name": mandateDef["name"]})
|
||||||
if existing:
|
if existing:
|
||||||
|
|
|
||||||
|
|
@ -154,7 +154,7 @@ class PwgDemo2026(BaseDemoConfig):
|
||||||
|
|
||||||
def _ensureMandate(self, db, mandateDef: Dict, summary: Dict) -> Optional[str]:
|
def _ensureMandate(self, db, mandateDef: Dict, summary: Dict) -> Optional[str]:
|
||||||
from modules.datamodels.datamodelUam import Mandate
|
from modules.datamodels.datamodelUam import Mandate
|
||||||
from modules.interfaces.interfaceBootstrap import copySystemRolesToMandate
|
from modules.interfaces.interfaceRbac import copySystemRolesToMandate
|
||||||
|
|
||||||
existing = db.getRecordset(Mandate, recordFilter={"name": mandateDef["name"]})
|
existing = db.getRecordset(Mandate, recordFilter={"name": mandateDef["name"]})
|
||||||
if existing:
|
if existing:
|
||||||
|
|
|
||||||
|
|
@ -597,8 +597,8 @@ def _createCommcoachRagFn(
|
||||||
from modules.serviceCenter.context import ServiceCenterContext
|
from modules.serviceCenter.context import ServiceCenterContext
|
||||||
serviceContext = ServiceCenterContext(
|
serviceContext = ServiceCenterContext(
|
||||||
user=currentUser,
|
user=currentUser,
|
||||||
mandate_id=mandateId,
|
mandateId=mandateId,
|
||||||
feature_instance_id=featureInstanceId,
|
featureInstanceId=featureInstanceId,
|
||||||
)
|
)
|
||||||
knowledgeService = getService("knowledge", serviceContext)
|
knowledgeService = getService("knowledge", serviceContext)
|
||||||
ragContext = await knowledgeService.buildAgentContext(
|
ragContext = await knowledgeService.buildAgentContext(
|
||||||
|
|
@ -902,8 +902,8 @@ class CommcoachService:
|
||||||
|
|
||||||
serviceContext = ServiceCenterContext(
|
serviceContext = ServiceCenterContext(
|
||||||
user=self.currentUser,
|
user=self.currentUser,
|
||||||
mandate_id=self.mandateId,
|
mandateId=self.mandateId,
|
||||||
feature_instance_id=self.instanceId,
|
featureInstanceId=self.instanceId,
|
||||||
)
|
)
|
||||||
agentService = getService("agent", serviceContext)
|
agentService = getService("agent", serviceContext)
|
||||||
|
|
||||||
|
|
@ -1240,8 +1240,8 @@ class CommcoachService:
|
||||||
|
|
||||||
serviceContext = ServiceCenterContext(
|
serviceContext = ServiceCenterContext(
|
||||||
user=self.currentUser,
|
user=self.currentUser,
|
||||||
mandate_id=self.mandateId,
|
mandateId=self.mandateId,
|
||||||
feature_instance_id=self.instanceId,
|
featureInstanceId=self.instanceId,
|
||||||
)
|
)
|
||||||
knowledgeService = getService("knowledge", serviceContext)
|
knowledgeService = getService("knowledge", serviceContext)
|
||||||
parsedGoals = aiPrompts._parseJsonField(context.get("goals") if context else None, [])
|
parsedGoals = aiPrompts._parseJsonField(context.get("goals") if context else None, [])
|
||||||
|
|
@ -1535,8 +1535,8 @@ class CommcoachService:
|
||||||
|
|
||||||
serviceContext = ServiceCenterContext(
|
serviceContext = ServiceCenterContext(
|
||||||
user=self.currentUser,
|
user=self.currentUser,
|
||||||
mandate_id=self.mandateId,
|
mandateId=self.mandateId,
|
||||||
feature_instance_id=self.instanceId,
|
featureInstanceId=self.instanceId,
|
||||||
)
|
)
|
||||||
aiService = getService("ai", serviceContext)
|
aiService = getService("ai", serviceContext)
|
||||||
await aiService.ensureAiObjectsInitialized()
|
await aiService.ensureAiObjectsInitialized()
|
||||||
|
|
@ -1561,8 +1561,8 @@ class CommcoachService:
|
||||||
|
|
||||||
serviceContext = ServiceCenterContext(
|
serviceContext = ServiceCenterContext(
|
||||||
user=self.currentUser,
|
user=self.currentUser,
|
||||||
mandate_id=self.mandateId,
|
mandateId=self.mandateId,
|
||||||
feature_instance_id=self.instanceId,
|
featureInstanceId=self.instanceId,
|
||||||
)
|
)
|
||||||
aiService = getService("ai", serviceContext)
|
aiService = getService("ai", serviceContext)
|
||||||
await aiService.ensureAiObjectsInitialized()
|
await aiService.ensureAiObjectsInitialized()
|
||||||
|
|
|
||||||
|
|
@ -309,8 +309,8 @@ class InterfaceFeatureNeutralizer:
|
||||||
) -> Optional[DataNeutralizerAttributes]:
|
) -> Optional[DataNeutralizerAttributes]:
|
||||||
"""Create a neutralization attribute for placeholder resolution."""
|
"""Create a neutralization attribute for placeholder resolution."""
|
||||||
try:
|
try:
|
||||||
mandate_id = self.mandateId or ""
|
mandateId = self.mandateId or ""
|
||||||
feature_instance_id = self.featureInstanceId or ""
|
featureInstanceId = self.featureInstanceId or ""
|
||||||
if not self.userId:
|
if not self.userId:
|
||||||
logger.warning("Cannot create attribute: missing userId")
|
logger.warning("Cannot create attribute: missing userId")
|
||||||
return None
|
return None
|
||||||
|
|
|
||||||
|
|
@ -22,7 +22,7 @@ class NeutralizationPlayground:
|
||||||
self.currentUser = currentUser
|
self.currentUser = currentUser
|
||||||
self.mandateId = mandateId
|
self.mandateId = mandateId
|
||||||
self.featureInstanceId = featureInstanceId
|
self.featureInstanceId = featureInstanceId
|
||||||
self._ctx = ServiceCenterContext(user=currentUser, mandate_id=mandateId, feature_instance_id=featureInstanceId)
|
self._ctx = ServiceCenterContext(user=currentUser, mandateId=mandateId, featureInstanceId=featureInstanceId)
|
||||||
|
|
||||||
def _getService(self, name: str):
|
def _getService(self, name: str):
|
||||||
return getService(name, self._ctx)
|
return getService(name, self._ctx)
|
||||||
|
|
@ -258,7 +258,7 @@ class SharepointProcessor:
|
||||||
self._sharepoint = getService("sharepoint", ctx)
|
self._sharepoint = getService("sharepoint", ctx)
|
||||||
self._neutralization = getService("neutralization", ctx)
|
self._neutralization = getService("neutralization", ctx)
|
||||||
from modules.interfaces.interfaceDbApp import getInterface as _getAppInterface
|
from modules.interfaces.interfaceDbApp import getInterface as _getAppInterface
|
||||||
self._interfaceDbApp = _getAppInterface(currentUser, mandateId=ctx.mandate_id)
|
self._interfaceDbApp = _getAppInterface(currentUser, mandateId=ctx.mandateId)
|
||||||
|
|
||||||
async def processSharepointFiles(self, sourcePath: str, targetPath: str) -> Dict[str, Any]:
|
async def processSharepointFiles(self, sourcePath: str, targetPath: str) -> Dict[str, Any]:
|
||||||
try:
|
try:
|
||||||
|
|
|
||||||
|
|
@ -58,18 +58,17 @@ def get_neutralization_config(
|
||||||
) -> DataNeutraliserConfig:
|
) -> DataNeutraliserConfig:
|
||||||
"""Get data neutralization configuration"""
|
"""Get data neutralization configuration"""
|
||||||
try:
|
try:
|
||||||
mandate_id = str(context.mandateId) if context.mandateId else ""
|
mandateId = str(context.mandateId) if context.mandateId else ""
|
||||||
feature_instance_id = str(context.featureInstanceId) if context.featureInstanceId else ""
|
featureInstanceId = str(context.featureInstanceId) if context.featureInstanceId else ""
|
||||||
service = NeutralizationPlayground(
|
service = NeutralizationPlayground(
|
||||||
context.user, mandate_id, featureInstanceId=feature_instance_id or None
|
context.user, mandateId, featureInstanceId=featureInstanceId or None
|
||||||
)
|
)
|
||||||
config = service.getConfig()
|
config = service.getConfig()
|
||||||
|
|
||||||
if not config:
|
if not config:
|
||||||
# Return default config instead of 404 (requires mandateId and featureInstanceId for instance-scoped config)
|
|
||||||
return DataNeutraliserConfig(
|
return DataNeutraliserConfig(
|
||||||
mandateId=mandate_id,
|
mandateId=mandateId,
|
||||||
featureInstanceId=feature_instance_id,
|
featureInstanceId=featureInstanceId,
|
||||||
userId=context.user.id,
|
userId=context.user.id,
|
||||||
enabled=True,
|
enabled=True,
|
||||||
namesToParse="",
|
namesToParse="",
|
||||||
|
|
|
||||||
|
|
@ -64,8 +64,8 @@ class NeutralizationService:
|
||||||
elif serviceCenter and getattr(serviceCenter, "user", None):
|
elif serviceCenter and getattr(serviceCenter, "user", None):
|
||||||
self.interfaceNeutralizer = getNeutralizerInterface(
|
self.interfaceNeutralizer = getNeutralizerInterface(
|
||||||
currentUser=serviceCenter.user,
|
currentUser=serviceCenter.user,
|
||||||
mandateId=getattr(serviceCenter, 'mandateId', None) or getattr(serviceCenter, 'mandate_id', None),
|
mandateId=getattr(serviceCenter, 'mandateId', None),
|
||||||
featureInstanceId=getattr(serviceCenter, 'featureInstanceId', None) or getattr(serviceCenter, 'feature_instance_id', None),
|
featureInstanceId=getattr(serviceCenter, 'featureInstanceId', None),
|
||||||
)
|
)
|
||||||
|
|
||||||
namesList = NamesToParse if isinstance(NamesToParse, list) else []
|
namesList = NamesToParse if isinstance(NamesToParse, list) else []
|
||||||
|
|
|
||||||
|
|
@ -232,7 +232,7 @@ async def processNaturalLanguageCommand(
|
||||||
logger.info(f"Processing natural language command for user {currentUser.id} (mandate: {mandateId})")
|
logger.info(f"Processing natural language command for user {currentUser.id} (mandate: {mandateId})")
|
||||||
logger.debug(f"User input: {userInput}")
|
logger.debug(f"User input: {userInput}")
|
||||||
|
|
||||||
ctx = ServiceCenterContext(user=currentUser, mandate_id=mandateId)
|
ctx = ServiceCenterContext(user=currentUser, mandateId=mandateId)
|
||||||
aiService = getService("ai", ctx)
|
aiService = getService("ai", ctx)
|
||||||
|
|
||||||
intentAnalysis = await analyzeUserIntent(aiService, userInput)
|
intentAnalysis = await analyzeUserIntent(aiService, userInput)
|
||||||
|
|
|
||||||
|
|
@ -234,7 +234,7 @@ async def extract_bzo_information(
|
||||||
|
|
||||||
bzo_params_result = None
|
bzo_params_result = None
|
||||||
try:
|
try:
|
||||||
ctx = ServiceCenterContext(user=currentUser, mandate_id=_mandateId, feature_instance_id=featureInstanceId)
|
ctx = ServiceCenterContext(user=currentUser, mandateId=_mandateId, featureInstanceId=featureInstanceId)
|
||||||
ai_service = getService("ai", ctx)
|
ai_service = getService("ai", ctx)
|
||||||
bzo_params_result = await run_bzo_params_extraction(
|
bzo_params_result = await run_bzo_params_extraction(
|
||||||
extracted_content=all_extracted_content,
|
extracted_content=all_extracted_content,
|
||||||
|
|
@ -520,7 +520,7 @@ async def generate_bauzone_ai_summary(
|
||||||
AI-generated summary string
|
AI-generated summary string
|
||||||
"""
|
"""
|
||||||
try:
|
try:
|
||||||
ctx = ServiceCenterContext(user=currentUser, mandate_id=mandateId, feature_instance_id=featureInstanceId)
|
ctx = ServiceCenterContext(user=currentUser, mandateId=mandateId, featureInstanceId=featureInstanceId)
|
||||||
aiService = getService("ai", ctx)
|
aiService = getService("ai", ctx)
|
||||||
|
|
||||||
context_parts = []
|
context_parts = []
|
||||||
|
|
|
||||||
|
|
@ -13,7 +13,7 @@ from __future__ import annotations
|
||||||
|
|
||||||
import logging
|
import logging
|
||||||
import time
|
import time
|
||||||
from typing import Any, Dict, Optional
|
from typing import Any, Dict, List, Optional, Tuple
|
||||||
|
|
||||||
from modules.connectors.connectorDbPostgre import DatabaseConnector
|
from modules.connectors.connectorDbPostgre import DatabaseConnector
|
||||||
from modules.connectors.connectorTicketsRedmine import ConnectorTicketsRedmine
|
from modules.connectors.connectorTicketsRedmine import ConnectorTicketsRedmine
|
||||||
|
|
@ -21,6 +21,9 @@ from modules.datamodels.datamodelUam import User
|
||||||
from modules.features.redmine.datamodelRedmine import (
|
from modules.features.redmine.datamodelRedmine import (
|
||||||
RedmineConfigDto,
|
RedmineConfigDto,
|
||||||
RedmineConfigUpdateRequest,
|
RedmineConfigUpdateRequest,
|
||||||
|
RedmineCustomFieldSchemaDto,
|
||||||
|
RedmineFieldChoiceDto,
|
||||||
|
RedmineFieldSchemaDto,
|
||||||
RedmineInstanceConfig,
|
RedmineInstanceConfig,
|
||||||
RedmineRelationMirror,
|
RedmineRelationMirror,
|
||||||
RedmineTicketMirror,
|
RedmineTicketMirror,
|
||||||
|
|
@ -447,3 +450,135 @@ def getInterface(
|
||||||
featureInstanceId=effectiveFeatureInstanceId,
|
featureInstanceId=effectiveFeatureInstanceId,
|
||||||
)
|
)
|
||||||
return _redmineInterfaces[contextKey]
|
return _redmineInterfaces[contextKey]
|
||||||
|
|
||||||
|
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
# Project meta -- with TTL cache stored on the config record
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
class RedmineNotConfiguredError(RuntimeError):
|
||||||
|
"""The given feature instance has no usable Redmine config."""
|
||||||
|
|
||||||
|
|
||||||
|
def _resolveRootTrackerId(
|
||||||
|
rootTrackerName: str, trackers: List[Dict[str, Any]]
|
||||||
|
) -> Optional[int]:
|
||||||
|
"""Resolve the configured root tracker name to a tracker id.
|
||||||
|
|
||||||
|
Strict: case-insensitive exact match. Returns ``None`` if not found
|
||||||
|
(the UI must surface this as a config error).
|
||||||
|
"""
|
||||||
|
target = (rootTrackerName or "").strip().lower()
|
||||||
|
if not target:
|
||||||
|
return None
|
||||||
|
for t in trackers:
|
||||||
|
if str(t.get("name") or "").strip().lower() == target:
|
||||||
|
tid = t.get("id")
|
||||||
|
return int(tid) if tid is not None else None
|
||||||
|
return None
|
||||||
|
|
||||||
|
|
||||||
|
def _schemaFromCache(
|
||||||
|
projectId: str, cache: Optional[Dict[str, Any]], rootTrackerName: str
|
||||||
|
) -> Optional[RedmineFieldSchemaDto]:
|
||||||
|
if not cache:
|
||||||
|
return None
|
||||||
|
trackers = cache.get("trackers") or []
|
||||||
|
return RedmineFieldSchemaDto(
|
||||||
|
projectId=projectId,
|
||||||
|
projectName=str(cache.get("projectName") or ""),
|
||||||
|
trackers=[RedmineFieldChoiceDto(**t) for t in trackers],
|
||||||
|
statuses=[RedmineFieldChoiceDto(**s) for s in cache.get("statuses") or []],
|
||||||
|
priorities=[RedmineFieldChoiceDto(**p) for p in cache.get("priorities") or []],
|
||||||
|
users=[RedmineFieldChoiceDto(**u) for u in cache.get("users") or []],
|
||||||
|
categories=[RedmineFieldChoiceDto(**c) for c in cache.get("categories") or []],
|
||||||
|
customFields=[
|
||||||
|
RedmineCustomFieldSchemaDto(
|
||||||
|
id=cf.get("id"),
|
||||||
|
name=cf.get("name", ""),
|
||||||
|
fieldFormat=cf.get("fieldFormat", "string"),
|
||||||
|
isRequired=bool(cf.get("isRequired")),
|
||||||
|
possibleValues=list(cf.get("possibleValues") or []),
|
||||||
|
multiple=bool(cf.get("multiple")),
|
||||||
|
defaultValue=cf.get("defaultValue"),
|
||||||
|
)
|
||||||
|
for cf in cache.get("customFields") or []
|
||||||
|
if cf.get("id") is not None
|
||||||
|
],
|
||||||
|
rootTrackerName=rootTrackerName,
|
||||||
|
rootTrackerId=_resolveRootTrackerId(rootTrackerName, trackers),
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
async def getProjectMeta(
|
||||||
|
currentUser: User,
|
||||||
|
mandateId: Optional[str],
|
||||||
|
featureInstanceId: str,
|
||||||
|
*,
|
||||||
|
forceRefresh: bool = False,
|
||||||
|
) -> RedmineFieldSchemaDto:
|
||||||
|
"""Fetch (or return cached) project metadata: trackers, statuses, priorities, etc."""
|
||||||
|
iface = getInterface(currentUser, mandateId=mandateId, featureInstanceId=featureInstanceId)
|
||||||
|
connector = iface.resolveConnector(featureInstanceId)
|
||||||
|
if not connector:
|
||||||
|
raise RedmineNotConfiguredError(
|
||||||
|
f"Redmine instance {featureInstanceId} is not configured or inactive"
|
||||||
|
)
|
||||||
|
cfg = iface.getConfig(featureInstanceId)
|
||||||
|
if cfg is None:
|
||||||
|
raise RedmineNotConfiguredError("Config row vanished after connector resolve")
|
||||||
|
|
||||||
|
ttl = cfg.schemaCacheTtlSeconds if cfg.schemaCacheTtlSeconds is not None else 24 * 60 * 60
|
||||||
|
fresh_enough = (
|
||||||
|
cfg.schemaCache
|
||||||
|
and cfg.schemaCachedAt
|
||||||
|
and (time.time() - cfg.schemaCachedAt) < ttl
|
||||||
|
)
|
||||||
|
if fresh_enough and not forceRefresh:
|
||||||
|
schema = _schemaFromCache(cfg.projectId, cfg.schemaCache, cfg.rootTrackerName)
|
||||||
|
if schema is not None:
|
||||||
|
return schema
|
||||||
|
|
||||||
|
project_info = await connector.getProjectInfo()
|
||||||
|
trackers_raw = await connector.getTrackers()
|
||||||
|
statuses_raw = await connector.getStatuses()
|
||||||
|
priorities_raw = await connector.getPriorities()
|
||||||
|
custom_fields_raw = await connector.getCustomFields()
|
||||||
|
users_raw = await connector.getProjectUsers()
|
||||||
|
categories_raw = await connector.getIssueCategories()
|
||||||
|
|
||||||
|
schema_cache: Dict[str, Any] = {
|
||||||
|
"projectName": project_info.get("name", ""),
|
||||||
|
"trackers": [{"id": t.get("id"), "name": t.get("name")} for t in trackers_raw],
|
||||||
|
"statuses": [
|
||||||
|
{
|
||||||
|
"id": s.get("id"),
|
||||||
|
"name": s.get("name"),
|
||||||
|
"isClosed": bool(s.get("is_closed")),
|
||||||
|
}
|
||||||
|
for s in statuses_raw
|
||||||
|
],
|
||||||
|
"priorities": [{"id": p.get("id"), "name": p.get("name")} for p in priorities_raw],
|
||||||
|
"users": [{"id": u.get("id"), "name": u.get("name")} for u in users_raw],
|
||||||
|
"categories": [{"id": c.get("id"), "name": c.get("name")} for c in categories_raw if c.get("id") is not None],
|
||||||
|
"customFields": [
|
||||||
|
{
|
||||||
|
"id": cf.get("id"),
|
||||||
|
"name": cf.get("name"),
|
||||||
|
"fieldFormat": cf.get("field_format", "string"),
|
||||||
|
"isRequired": bool(cf.get("is_required")),
|
||||||
|
"possibleValues": [pv.get("value") for pv in (cf.get("possible_values") or []) if pv.get("value") is not None],
|
||||||
|
"multiple": bool(cf.get("multiple")),
|
||||||
|
"defaultValue": cf.get("default_value"),
|
||||||
|
}
|
||||||
|
for cf in custom_fields_raw
|
||||||
|
],
|
||||||
|
}
|
||||||
|
iface.updateSchemaCache(featureInstanceId, schema_cache)
|
||||||
|
iface.markConfigConnected(featureInstanceId)
|
||||||
|
|
||||||
|
return _schemaFromCache(cfg.projectId, schema_cache, cfg.rootTrackerName) or RedmineFieldSchemaDto(
|
||||||
|
projectId=cfg.projectId,
|
||||||
|
projectName=schema_cache["projectName"],
|
||||||
|
rootTrackerName=cfg.rootTrackerName,
|
||||||
|
)
|
||||||
|
|
|
||||||
|
|
@ -32,7 +32,7 @@ from modules.features.redmine.datamodelRedmine import (
|
||||||
RedmineTicketDto,
|
RedmineTicketDto,
|
||||||
RedmineTicketUpdateRequest,
|
RedmineTicketUpdateRequest,
|
||||||
)
|
)
|
||||||
from modules.features.redmine.serviceRedmine import RedmineNotConfiguredError
|
from modules.features.redmine.interfaceFeatureRedmine import RedmineNotConfiguredError
|
||||||
from modules.connectors.connectorTicketsRedmine import RedmineApiError
|
from modules.connectors.connectorTicketsRedmine import RedmineApiError
|
||||||
from modules.interfaces.interfaceDbApp import getRootInterface
|
from modules.interfaces.interfaceDbApp import getRootInterface
|
||||||
from modules.interfaces.interfaceFeatures import getFeatureInterface
|
from modules.interfaces.interfaceFeatures import getFeatureInterface
|
||||||
|
|
|
||||||
|
|
@ -25,7 +25,6 @@ workflow engine without context-magic.
|
||||||
from __future__ import annotations
|
from __future__ import annotations
|
||||||
|
|
||||||
import logging
|
import logging
|
||||||
import time
|
|
||||||
from datetime import datetime
|
from datetime import datetime
|
||||||
from typing import Any, Dict, List, Optional, Tuple
|
from typing import Any, Dict, List, Optional, Tuple
|
||||||
|
|
||||||
|
|
@ -35,9 +34,7 @@ from modules.connectors.connectorTicketsRedmine import (
|
||||||
)
|
)
|
||||||
from modules.datamodels.datamodelUam import User
|
from modules.datamodels.datamodelUam import User
|
||||||
from modules.features.redmine.datamodelRedmine import (
|
from modules.features.redmine.datamodelRedmine import (
|
||||||
RedmineCustomFieldSchemaDto,
|
|
||||||
RedmineCustomFieldValueDto,
|
RedmineCustomFieldValueDto,
|
||||||
RedmineFieldChoiceDto,
|
|
||||||
RedmineFieldSchemaDto,
|
RedmineFieldSchemaDto,
|
||||||
RedmineRelationCreateRequest,
|
RedmineRelationCreateRequest,
|
||||||
RedmineRelationDto,
|
RedmineRelationDto,
|
||||||
|
|
@ -46,8 +43,10 @@ from modules.features.redmine.datamodelRedmine import (
|
||||||
RedmineTicketUpdateRequest,
|
RedmineTicketUpdateRequest,
|
||||||
)
|
)
|
||||||
from modules.features.redmine.interfaceFeatureRedmine import (
|
from modules.features.redmine.interfaceFeatureRedmine import (
|
||||||
|
RedmineNotConfiguredError,
|
||||||
RedmineObjects,
|
RedmineObjects,
|
||||||
getInterface,
|
getInterface,
|
||||||
|
getProjectMeta,
|
||||||
)
|
)
|
||||||
from modules.features.redmine.serviceRedmineStatsCache import getStatsCache
|
from modules.features.redmine.serviceRedmineStatsCache import getStatsCache
|
||||||
|
|
||||||
|
|
@ -58,9 +57,6 @@ logger = logging.getLogger(__name__)
|
||||||
# Resolution helpers
|
# Resolution helpers
|
||||||
# ---------------------------------------------------------------------------
|
# ---------------------------------------------------------------------------
|
||||||
|
|
||||||
class RedmineNotConfiguredError(RuntimeError):
|
|
||||||
"""The given feature instance has no usable Redmine config."""
|
|
||||||
|
|
||||||
|
|
||||||
def _resolveContext(
|
def _resolveContext(
|
||||||
currentUser: User, mandateId: Optional[str], featureInstanceId: str
|
currentUser: User, mandateId: Optional[str], featureInstanceId: str
|
||||||
|
|
@ -74,127 +70,6 @@ def _resolveContext(
|
||||||
return iface, connector
|
return iface, connector
|
||||||
|
|
||||||
|
|
||||||
# ---------------------------------------------------------------------------
|
|
||||||
# Project meta -- with TTL cache stored on the config record
|
|
||||||
# ---------------------------------------------------------------------------
|
|
||||||
|
|
||||||
async def getProjectMeta(
|
|
||||||
currentUser: User,
|
|
||||||
mandateId: Optional[str],
|
|
||||||
featureInstanceId: str,
|
|
||||||
*,
|
|
||||||
forceRefresh: bool = False,
|
|
||||||
) -> RedmineFieldSchemaDto:
|
|
||||||
iface, connector = _resolveContext(currentUser, mandateId, featureInstanceId)
|
|
||||||
cfg = iface.getConfig(featureInstanceId)
|
|
||||||
if cfg is None:
|
|
||||||
raise RedmineNotConfiguredError("Config row vanished after connector resolve")
|
|
||||||
|
|
||||||
ttl = cfg.schemaCacheTtlSeconds if cfg.schemaCacheTtlSeconds is not None else 24 * 60 * 60
|
|
||||||
fresh_enough = (
|
|
||||||
cfg.schemaCache
|
|
||||||
and cfg.schemaCachedAt
|
|
||||||
and (time.time() - cfg.schemaCachedAt) < ttl
|
|
||||||
)
|
|
||||||
if fresh_enough and not forceRefresh:
|
|
||||||
schema = _schemaFromCache(cfg.projectId, cfg.schemaCache, cfg.rootTrackerName)
|
|
||||||
if schema is not None:
|
|
||||||
return schema
|
|
||||||
|
|
||||||
project_info = await connector.getProjectInfo()
|
|
||||||
trackers_raw = await connector.getTrackers()
|
|
||||||
statuses_raw = await connector.getStatuses()
|
|
||||||
priorities_raw = await connector.getPriorities()
|
|
||||||
custom_fields_raw = await connector.getCustomFields()
|
|
||||||
users_raw = await connector.getProjectUsers()
|
|
||||||
categories_raw = await connector.getIssueCategories()
|
|
||||||
|
|
||||||
schema_cache: Dict[str, Any] = {
|
|
||||||
"projectName": project_info.get("name", ""),
|
|
||||||
"trackers": [{"id": t.get("id"), "name": t.get("name")} for t in trackers_raw],
|
|
||||||
"statuses": [
|
|
||||||
{
|
|
||||||
"id": s.get("id"),
|
|
||||||
"name": s.get("name"),
|
|
||||||
"isClosed": bool(s.get("is_closed")),
|
|
||||||
}
|
|
||||||
for s in statuses_raw
|
|
||||||
],
|
|
||||||
"priorities": [{"id": p.get("id"), "name": p.get("name")} for p in priorities_raw],
|
|
||||||
"users": [{"id": u.get("id"), "name": u.get("name")} for u in users_raw],
|
|
||||||
"categories": [{"id": c.get("id"), "name": c.get("name")} for c in categories_raw if c.get("id") is not None],
|
|
||||||
"customFields": [
|
|
||||||
{
|
|
||||||
"id": cf.get("id"),
|
|
||||||
"name": cf.get("name"),
|
|
||||||
"fieldFormat": cf.get("field_format", "string"),
|
|
||||||
"isRequired": bool(cf.get("is_required")),
|
|
||||||
"possibleValues": [pv.get("value") for pv in (cf.get("possible_values") or []) if pv.get("value") is not None],
|
|
||||||
"multiple": bool(cf.get("multiple")),
|
|
||||||
"defaultValue": cf.get("default_value"),
|
|
||||||
}
|
|
||||||
for cf in custom_fields_raw
|
|
||||||
],
|
|
||||||
}
|
|
||||||
iface.updateSchemaCache(featureInstanceId, schema_cache)
|
|
||||||
iface.markConfigConnected(featureInstanceId)
|
|
||||||
|
|
||||||
return _schemaFromCache(cfg.projectId, schema_cache, cfg.rootTrackerName) or RedmineFieldSchemaDto(
|
|
||||||
projectId=cfg.projectId,
|
|
||||||
projectName=schema_cache["projectName"],
|
|
||||||
rootTrackerName=cfg.rootTrackerName,
|
|
||||||
)
|
|
||||||
|
|
||||||
|
|
||||||
def _resolveRootTrackerId(
|
|
||||||
rootTrackerName: str, trackers: List[Dict[str, Any]]
|
|
||||||
) -> Optional[int]:
|
|
||||||
"""Resolve the configured root tracker name to a tracker id.
|
|
||||||
|
|
||||||
Strict: case-insensitive exact match. Returns ``None`` if not found
|
|
||||||
(the UI must surface this as a config error).
|
|
||||||
"""
|
|
||||||
target = (rootTrackerName or "").strip().lower()
|
|
||||||
if not target:
|
|
||||||
return None
|
|
||||||
for t in trackers:
|
|
||||||
if str(t.get("name") or "").strip().lower() == target:
|
|
||||||
tid = t.get("id")
|
|
||||||
return int(tid) if tid is not None else None
|
|
||||||
return None
|
|
||||||
|
|
||||||
|
|
||||||
def _schemaFromCache(
|
|
||||||
projectId: str, cache: Optional[Dict[str, Any]], rootTrackerName: str
|
|
||||||
) -> Optional[RedmineFieldSchemaDto]:
|
|
||||||
if not cache:
|
|
||||||
return None
|
|
||||||
trackers = cache.get("trackers") or []
|
|
||||||
return RedmineFieldSchemaDto(
|
|
||||||
projectId=projectId,
|
|
||||||
projectName=str(cache.get("projectName") or ""),
|
|
||||||
trackers=[RedmineFieldChoiceDto(**t) for t in trackers],
|
|
||||||
statuses=[RedmineFieldChoiceDto(**s) for s in cache.get("statuses") or []],
|
|
||||||
priorities=[RedmineFieldChoiceDto(**p) for p in cache.get("priorities") or []],
|
|
||||||
users=[RedmineFieldChoiceDto(**u) for u in cache.get("users") or []],
|
|
||||||
categories=[RedmineFieldChoiceDto(**c) for c in cache.get("categories") or []],
|
|
||||||
customFields=[
|
|
||||||
RedmineCustomFieldSchemaDto(
|
|
||||||
id=cf.get("id"),
|
|
||||||
name=cf.get("name", ""),
|
|
||||||
fieldFormat=cf.get("fieldFormat", "string"),
|
|
||||||
isRequired=bool(cf.get("isRequired")),
|
|
||||||
possibleValues=list(cf.get("possibleValues") or []),
|
|
||||||
multiple=bool(cf.get("multiple")),
|
|
||||||
defaultValue=cf.get("defaultValue"),
|
|
||||||
)
|
|
||||||
for cf in cache.get("customFields") or []
|
|
||||||
if cf.get("id") is not None
|
|
||||||
],
|
|
||||||
rootTrackerName=rootTrackerName,
|
|
||||||
rootTrackerId=_resolveRootTrackerId(rootTrackerName, trackers),
|
|
||||||
)
|
|
||||||
|
|
||||||
|
|
||||||
# ---------------------------------------------------------------------------
|
# ---------------------------------------------------------------------------
|
||||||
# Mirror -> RedmineTicketDto
|
# Mirror -> RedmineTicketDto
|
||||||
|
|
|
||||||
|
|
@ -83,10 +83,8 @@ async def getStats(
|
||||||
|
|
||||||
# Lazy import: keeps the pure aggregation helpers below importable
|
# Lazy import: keeps the pure aggregation helpers below importable
|
||||||
# without dragging in aiohttp / DB connector at module load.
|
# without dragging in aiohttp / DB connector at module load.
|
||||||
from modules.features.redmine.serviceRedmine import (
|
from modules.features.redmine.interfaceFeatureRedmine import getProjectMeta
|
||||||
getProjectMeta,
|
from modules.features.redmine.serviceRedmine import listTickets
|
||||||
listTickets,
|
|
||||||
)
|
|
||||||
|
|
||||||
schema = await getProjectMeta(currentUser, mandateId, featureInstanceId)
|
schema = await getProjectMeta(currentUser, mandateId, featureInstanceId)
|
||||||
root_tracker_id = schema.rootTrackerId
|
root_tracker_id = schema.rootTrackerId
|
||||||
|
|
|
||||||
|
|
@ -38,7 +38,7 @@ from modules.features.redmine.datamodelRedmine import (
|
||||||
RedmineSyncStatusDto,
|
RedmineSyncStatusDto,
|
||||||
RedmineTicketMirror,
|
RedmineTicketMirror,
|
||||||
)
|
)
|
||||||
from modules.features.redmine.interfaceFeatureRedmine import getInterface
|
from modules.features.redmine.interfaceFeatureRedmine import getInterface, getProjectMeta
|
||||||
from modules.features.redmine.serviceRedmineStatsCache import getStatsCache
|
from modules.features.redmine.serviceRedmineStatsCache import getStatsCache
|
||||||
|
|
||||||
logger = logging.getLogger(__name__)
|
logger = logging.getLogger(__name__)
|
||||||
|
|
@ -281,8 +281,6 @@ async def _ensureSchemaWarm(
|
||||||
statuses = (cfg.schemaCache or {}).get("statuses") or []
|
statuses = (cfg.schemaCache or {}).get("statuses") or []
|
||||||
if statuses:
|
if statuses:
|
||||||
return
|
return
|
||||||
# Lazy import to avoid a circular dependency at module load.
|
|
||||||
from modules.features.redmine.serviceRedmine import getProjectMeta
|
|
||||||
try:
|
try:
|
||||||
await getProjectMeta(currentUser, mandateId, featureInstanceId, forceRefresh=True)
|
await getProjectMeta(currentUser, mandateId, featureInstanceId, forceRefresh=True)
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
|
|
|
||||||
|
|
@ -405,9 +405,9 @@ def createAiService(user, mandateId, featureInstanceId=None):
|
||||||
"""Create a properly wired AiService via the service center."""
|
"""Create a properly wired AiService via the service center."""
|
||||||
ctx = ServiceCenterContext(
|
ctx = ServiceCenterContext(
|
||||||
user=user,
|
user=user,
|
||||||
mandate_id=mandateId,
|
mandateId=mandateId,
|
||||||
feature_instance_id=featureInstanceId,
|
featureInstanceId=featureInstanceId,
|
||||||
feature_code="teamsbot",
|
featureCode="teamsbot",
|
||||||
)
|
)
|
||||||
return _getServiceCenterService("ai", ctx)
|
return _getServiceCenterService("ai", ctx)
|
||||||
|
|
||||||
|
|
@ -1320,9 +1320,9 @@ class TeamsbotService:
|
||||||
|
|
||||||
ctx = ServiceCenterContext(
|
ctx = ServiceCenterContext(
|
||||||
user=self.currentUser,
|
user=self.currentUser,
|
||||||
mandate_id=self.mandateId,
|
mandateId=self.mandateId,
|
||||||
feature_instance_id=self.instanceId,
|
featureInstanceId=self.instanceId,
|
||||||
feature_code="teamsbot",
|
featureCode="teamsbot",
|
||||||
)
|
)
|
||||||
agentService = _getServiceCenterService("agent", ctx)
|
agentService = _getServiceCenterService("agent", ctx)
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -247,8 +247,8 @@ async def _cmdSendMail(service, sessionId: str, params: dict):
|
||||||
from modules.serviceCenter import ServiceCenterContext, getService
|
from modules.serviceCenter import ServiceCenterContext, getService
|
||||||
ctx = ServiceCenterContext(
|
ctx = ServiceCenterContext(
|
||||||
user=service.currentUser,
|
user=service.currentUser,
|
||||||
mandate_id=service.mandateId,
|
mandateId=service.mandateId,
|
||||||
feature_instance_id=service.instanceId,
|
featureInstanceId=service.instanceId,
|
||||||
)
|
)
|
||||||
messaging = getService("messaging", ctx)
|
messaging = getService("messaging", ctx)
|
||||||
success = messaging.sendEmailDirect(
|
success = messaging.sendEmailDirect(
|
||||||
|
|
@ -280,8 +280,8 @@ async def _cmdStoreDocument(service, sessionId: str, params: dict):
|
||||||
from modules.serviceCenter import ServiceCenterContext, getService
|
from modules.serviceCenter import ServiceCenterContext, getService
|
||||||
ctx = ServiceCenterContext(
|
ctx = ServiceCenterContext(
|
||||||
user=service.currentUser,
|
user=service.currentUser,
|
||||||
mandate_id=service.mandateId,
|
mandateId=service.mandateId,
|
||||||
feature_instance_id=service.instanceId,
|
featureInstanceId=service.instanceId,
|
||||||
)
|
)
|
||||||
sharepoint = getService("sharepoint", ctx)
|
sharepoint = getService("sharepoint", ctx)
|
||||||
if not sharepoint.setAccessTokenFromConnection(service.currentUser):
|
if not sharepoint.setAccessTokenFromConnection(service.currentUser):
|
||||||
|
|
|
||||||
|
|
@ -566,10 +566,10 @@ async def streamWorkspaceStart(
|
||||||
wsBillingFeatureCode = _workspaceBillingFeatureCode(context.user, mandateId or "", instanceId)
|
wsBillingFeatureCode = _workspaceBillingFeatureCode(context.user, mandateId or "", instanceId)
|
||||||
svcCtx = ServiceCenterContext(
|
svcCtx = ServiceCenterContext(
|
||||||
user=context.user,
|
user=context.user,
|
||||||
mandate_id=mandateId or "",
|
mandateId=mandateId or "",
|
||||||
feature_instance_id=instanceId,
|
featureInstanceId=instanceId,
|
||||||
workflow_id=workflowId,
|
workflowId=workflowId,
|
||||||
feature_code=wsBillingFeatureCode,
|
featureCode=wsBillingFeatureCode,
|
||||||
)
|
)
|
||||||
chatSvc = getService("chat", svcCtx)
|
chatSvc = getService("chat", svcCtx)
|
||||||
attachmentLabel = _buildWorkspaceAttachmentLabel(
|
attachmentLabel = _buildWorkspaceAttachmentLabel(
|
||||||
|
|
@ -687,10 +687,10 @@ async def _runWorkspaceAgent(
|
||||||
from modules.serviceCenter.context import ServiceCenterContext
|
from modules.serviceCenter.context import ServiceCenterContext
|
||||||
ctx = ServiceCenterContext(
|
ctx = ServiceCenterContext(
|
||||||
user=user,
|
user=user,
|
||||||
mandate_id=mandateId,
|
mandateId=mandateId,
|
||||||
feature_instance_id=instanceId,
|
featureInstanceId=instanceId,
|
||||||
workflow_id=workflowId,
|
workflowId=workflowId,
|
||||||
feature_code=billingFeatureCode,
|
featureCode=billingFeatureCode,
|
||||||
)
|
)
|
||||||
agentService = getService("agent", ctx)
|
agentService = getService("agent", ctx)
|
||||||
chatService = getService("chat", ctx)
|
chatService = getService("chat", ctx)
|
||||||
|
|
@ -1299,7 +1299,7 @@ async def listWorkspaceDataSources(
|
||||||
try:
|
try:
|
||||||
from modules.datamodels.datamodelDataSource import DataSource
|
from modules.datamodels.datamodelDataSource import DataSource
|
||||||
from modules.interfaces.interfaceDbApp import getRootInterface
|
from modules.interfaces.interfaceDbApp import getRootInterface
|
||||||
from modules.serviceCenter.services.serviceKnowledge._inheritFlags import buildEffectiveByConnection
|
from modules.serviceCenter.core.flagResolution import buildEffectiveByConnection
|
||||||
rootIf = getRootInterface()
|
rootIf = getRootInterface()
|
||||||
recordFilter: dict = {"featureInstanceId": instanceId}
|
recordFilter: dict = {"featureInstanceId": instanceId}
|
||||||
if wsMandateId:
|
if wsMandateId:
|
||||||
|
|
@ -1352,8 +1352,8 @@ async def createWorkspaceDataSource(
|
||||||
from modules.serviceCenter.context import ServiceCenterContext
|
from modules.serviceCenter.context import ServiceCenterContext
|
||||||
ctx = ServiceCenterContext(
|
ctx = ServiceCenterContext(
|
||||||
user=context.user,
|
user=context.user,
|
||||||
mandate_id=_mandateId or "",
|
mandateId=_mandateId or "",
|
||||||
feature_instance_id=instanceId,
|
featureInstanceId=instanceId,
|
||||||
)
|
)
|
||||||
chatService = getService("chat", ctx)
|
chatService = getService("chat", ctx)
|
||||||
dataSource = chatService.createDataSource(
|
dataSource = chatService.createDataSource(
|
||||||
|
|
@ -1381,8 +1381,8 @@ async def deleteWorkspaceDataSource(
|
||||||
from modules.serviceCenter.context import ServiceCenterContext
|
from modules.serviceCenter.context import ServiceCenterContext
|
||||||
ctx = ServiceCenterContext(
|
ctx = ServiceCenterContext(
|
||||||
user=context.user,
|
user=context.user,
|
||||||
mandate_id=_mandateId or "",
|
mandateId=_mandateId or "",
|
||||||
feature_instance_id=instanceId,
|
featureInstanceId=instanceId,
|
||||||
)
|
)
|
||||||
chatService = getService("chat", ctx)
|
chatService = getService("chat", ctx)
|
||||||
chatService.deleteDataSource(dataSourceId)
|
chatService.deleteDataSource(dataSourceId)
|
||||||
|
|
@ -1464,7 +1464,7 @@ async def listFeatureDataSources(
|
||||||
wsMandateId, _ = _validateInstanceAccess(instanceId, context)
|
wsMandateId, _ = _validateInstanceAccess(instanceId, context)
|
||||||
from modules.interfaces.interfaceDbApp import getRootInterface
|
from modules.interfaces.interfaceDbApp import getRootInterface
|
||||||
from modules.datamodels.datamodelFeatures import FeatureDataSource
|
from modules.datamodels.datamodelFeatures import FeatureDataSource
|
||||||
from modules.serviceCenter.services.serviceKnowledge._inheritFlags import buildEffectiveByWorkspaceFds
|
from modules.serviceCenter.core.flagResolution import buildEffectiveByWorkspaceFds
|
||||||
|
|
||||||
rootIf = getRootInterface()
|
rootIf = getRootInterface()
|
||||||
recordFilter: dict = {}
|
recordFilter: dict = {}
|
||||||
|
|
|
||||||
|
|
@ -11,7 +11,6 @@ Multi-Tenant Design:
|
||||||
"""
|
"""
|
||||||
|
|
||||||
import logging
|
import logging
|
||||||
import uuid
|
|
||||||
from typing import Optional, Dict
|
from typing import Optional, Dict
|
||||||
from passlib.context import CryptContext
|
from passlib.context import CryptContext
|
||||||
from modules.connectors.connectorDbPostgre import DatabaseConnector
|
from modules.connectors.connectorDbPostgre import DatabaseConnector
|
||||||
|
|
@ -521,6 +520,8 @@ def _ensureAllMandatesHaveSystemRoles(db: DatabaseConnector) -> None:
|
||||||
Ensure all existing mandates have system-instance roles.
|
Ensure all existing mandates have system-instance roles.
|
||||||
Serves as both initial setup and migration for existing mandates.
|
Serves as both initial setup and migration for existing mandates.
|
||||||
"""
|
"""
|
||||||
|
from modules.interfaces.interfaceRbac import copySystemRolesToMandate
|
||||||
|
|
||||||
allMandates = db.getRecordset(Mandate)
|
allMandates = db.getRecordset(Mandate)
|
||||||
if not allMandates:
|
if not allMandates:
|
||||||
logger.info("No mandates found, skipping system role copy")
|
logger.info("No mandates found, skipping system role copy")
|
||||||
|
|
@ -534,94 +535,6 @@ def _ensureAllMandatesHaveSystemRoles(db: DatabaseConnector) -> None:
|
||||||
logger.info(f"Copied {copiedCount} system roles to mandate {mandateId}")
|
logger.info(f"Copied {copiedCount} system roles to mandate {mandateId}")
|
||||||
|
|
||||||
|
|
||||||
def copySystemRolesToMandate(db: DatabaseConnector, mandateId: str) -> int:
|
|
||||||
"""
|
|
||||||
Copy system template roles (mandateId=None, isSystemRole=True) to a mandate
|
|
||||||
as mandate-instance roles. Also copies all AccessRules for each role.
|
|
||||||
|
|
||||||
This is analogous to how feature template roles are copied to feature instances.
|
|
||||||
Each mandate gets its own instances of admin/user/viewer with their AccessRules.
|
|
||||||
|
|
||||||
Args:
|
|
||||||
db: Database connector instance
|
|
||||||
mandateId: Target mandate ID
|
|
||||||
|
|
||||||
Returns:
|
|
||||||
Number of roles copied
|
|
||||||
"""
|
|
||||||
# Find system template roles (global: mandateId=NULL, isSystemRole=True)
|
|
||||||
templateRoles = db.getRecordset(
|
|
||||||
Role,
|
|
||||||
recordFilter={"isSystemRole": True, "mandateId": None}
|
|
||||||
)
|
|
||||||
|
|
||||||
if not templateRoles:
|
|
||||||
logger.warning(f"No system template roles found (mandateId IS NULL, isSystemRole=True)")
|
|
||||||
return 0
|
|
||||||
|
|
||||||
# Check which mandate-level roles already exist for this mandate
|
|
||||||
existingMandateRoles = db.getRecordset(
|
|
||||||
Role,
|
|
||||||
recordFilter={"mandateId": mandateId, "featureInstanceId": None}
|
|
||||||
)
|
|
||||||
existingLabels = {r.get("roleLabel") for r in existingMandateRoles}
|
|
||||||
logger.info(f"copySystemRolesToMandate: mandate={mandateId}, templates={len(templateRoles)}, existing={len(existingMandateRoles)}, labels={existingLabels}")
|
|
||||||
|
|
||||||
# Load all AccessRules for template roles
|
|
||||||
templateRoleIds = [r.get("id") for r in templateRoles]
|
|
||||||
rulesByRoleId = {}
|
|
||||||
for roleId in templateRoleIds:
|
|
||||||
rules = db.getRecordset(AccessRule, recordFilter={"roleId": roleId})
|
|
||||||
rulesByRoleId[roleId] = rules
|
|
||||||
|
|
||||||
copiedCount = 0
|
|
||||||
for templateRole in templateRoles:
|
|
||||||
roleLabel = templateRole.get("roleLabel")
|
|
||||||
|
|
||||||
# Skip if mandate already has this role
|
|
||||||
if roleLabel in existingLabels:
|
|
||||||
logger.debug(f"Mandate {mandateId} already has role '{roleLabel}', skipping")
|
|
||||||
continue
|
|
||||||
|
|
||||||
newRoleId = str(uuid.uuid4())
|
|
||||||
|
|
||||||
# Create mandate-instance role
|
|
||||||
newRole = Role(
|
|
||||||
id=newRoleId,
|
|
||||||
roleLabel=roleLabel,
|
|
||||||
description=coerce_text_multilingual(templateRole.get("description", {})),
|
|
||||||
mandateId=mandateId,
|
|
||||||
featureInstanceId=None,
|
|
||||||
featureCode=None,
|
|
||||||
isSystemRole=False # Mandate-level role, not a system template
|
|
||||||
)
|
|
||||||
db.recordCreate(Role, newRole.model_dump())
|
|
||||||
|
|
||||||
# Copy AccessRules
|
|
||||||
templateRules = rulesByRoleId.get(templateRole.get("id"), [])
|
|
||||||
for rule in templateRules:
|
|
||||||
newRule = AccessRule(
|
|
||||||
id=str(uuid.uuid4()),
|
|
||||||
roleId=newRoleId,
|
|
||||||
context=rule.get("context"),
|
|
||||||
item=rule.get("item"),
|
|
||||||
view=rule.get("view", False),
|
|
||||||
read=rule.get("read"),
|
|
||||||
create=rule.get("create"),
|
|
||||||
update=rule.get("update"),
|
|
||||||
delete=rule.get("delete")
|
|
||||||
)
|
|
||||||
db.recordCreate(AccessRule, newRule.model_dump())
|
|
||||||
|
|
||||||
copiedCount += 1
|
|
||||||
logger.info(f"Copied system role '{roleLabel}' to mandate {mandateId} with {len(templateRules)} AccessRules")
|
|
||||||
|
|
||||||
if copiedCount > 0:
|
|
||||||
logger.info(f"Copied {copiedCount} system roles to mandate {mandateId}")
|
|
||||||
|
|
||||||
return copiedCount
|
|
||||||
|
|
||||||
|
|
||||||
def _getRoleId(db: DatabaseConnector, roleLabel: str) -> Optional[str]:
|
def _getRoleId(db: DatabaseConnector, roleLabel: str) -> Optional[str]:
|
||||||
"""
|
"""
|
||||||
Get role ID by label, using cache or database lookup.
|
Get role ID by label, using cache or database lookup.
|
||||||
|
|
|
||||||
|
|
@ -1560,7 +1560,7 @@ class AppObjects:
|
||||||
|
|
||||||
# Copy system template roles to new mandate (admin, user, viewer + AccessRules)
|
# Copy system template roles to new mandate (admin, user, viewer + AccessRules)
|
||||||
try:
|
try:
|
||||||
from modules.interfaces.interfaceBootstrap import copySystemRolesToMandate
|
from modules.interfaces.interfaceRbac import copySystemRolesToMandate
|
||||||
copiedCount = copySystemRolesToMandate(self.db, mandateId)
|
copiedCount = copySystemRolesToMandate(self.db, mandateId)
|
||||||
logger.info(f"Copied {copiedCount} system roles to new mandate {mandateId}")
|
logger.info(f"Copied {copiedCount} system roles to new mandate {mandateId}")
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
|
|
@ -1577,7 +1577,7 @@ class AppObjects:
|
||||||
"""
|
"""
|
||||||
from modules.datamodels.datamodelSubscription import MandateSubscription, SubscriptionStatusEnum, BUILTIN_PLANS
|
from modules.datamodels.datamodelSubscription import MandateSubscription, SubscriptionStatusEnum, BUILTIN_PLANS
|
||||||
from modules.datamodels.datamodelFeatures import FeatureInstance
|
from modules.datamodels.datamodelFeatures import FeatureInstance
|
||||||
from modules.interfaces.interfaceBootstrap import copySystemRolesToMandate
|
from modules.interfaces.interfaceRbac import copySystemRolesToMandate
|
||||||
from modules.interfaces.interfaceFeatures import getFeatureInterface
|
from modules.interfaces.interfaceFeatures import getFeatureInterface
|
||||||
from modules.shared.featureDiscovery import loadFeatureMainModules
|
from modules.shared.featureDiscovery import loadFeatureMainModules
|
||||||
plan = BUILTIN_PLANS.get(planKey)
|
plan = BUILTIN_PLANS.get(planKey)
|
||||||
|
|
@ -1615,7 +1615,7 @@ class AppObjects:
|
||||||
raise ValueError(f"No admin role found for mandate {mandateId} — cannot assign user without role")
|
raise ValueError(f"No admin role found for mandate {mandateId} — cannot assign user without role")
|
||||||
|
|
||||||
from modules.interfaces.interfaceDbSubscription import getRootInterface as _getSubRoot
|
from modules.interfaces.interfaceDbSubscription import getRootInterface as _getSubRoot
|
||||||
from modules.interfaces.interfaceDbBilling import getRootInterface as _getBillingRoot
|
from modules.shared.systemComponentRegistry import getLifecycleHooks as _getHooks
|
||||||
|
|
||||||
now = datetime.now(timezone.utc)
|
now = datetime.now(timezone.utc)
|
||||||
nowTs = now.timestamp()
|
nowTs = now.timestamp()
|
||||||
|
|
@ -1635,17 +1635,11 @@ class AppObjects:
|
||||||
subInterface = _getSubRoot()
|
subInterface = _getSubRoot()
|
||||||
subInterface.createSubscription(subscription)
|
subInterface.createSubscription(subscription)
|
||||||
|
|
||||||
|
for _hook in _getHooks("onMandateProvision"):
|
||||||
try:
|
try:
|
||||||
billingRoot = _getBillingRoot()
|
_hook(mandateId, planKey)
|
||||||
billingRoot.getOrCreateSettings(mandateId)
|
except Exception as _hookErr:
|
||||||
billingRoot.ensureActivationBudget(mandateId, planKey)
|
logger.error("onMandateProvision hook failed: %s", _hookErr)
|
||||||
except Exception as billingEx:
|
|
||||||
logger.error(
|
|
||||||
"Initial billing setup failed for mandate %s (plan=%s): %s",
|
|
||||||
mandateId,
|
|
||||||
planKey,
|
|
||||||
billingEx,
|
|
||||||
)
|
|
||||||
|
|
||||||
self.createUserMandate(userId, mandateId, roleIds=[adminRoleId], skipCapacityCheck=True)
|
self.createUserMandate(userId, mandateId, roleIds=[adminRoleId], skipCapacityCheck=True)
|
||||||
|
|
||||||
|
|
@ -1865,7 +1859,6 @@ class AppObjects:
|
||||||
from modules.datamodels.datamodelDataSource import DataSource
|
from modules.datamodels.datamodelDataSource import DataSource
|
||||||
from modules.datamodels.datamodelKnowledge import FileContentIndex, ContentChunk
|
from modules.datamodels.datamodelKnowledge import FileContentIndex, ContentChunk
|
||||||
from modules.datamodels.datamodelFeatures import FeatureDataSource
|
from modules.datamodels.datamodelFeatures import FeatureDataSource
|
||||||
from modules.datamodels.datamodelBilling import BillingSettings, BillingAccount, BillingTransaction
|
|
||||||
from modules.datamodels.datamodelFeatures import DataNeutralizerAttributes
|
from modules.datamodels.datamodelFeatures import DataNeutralizerAttributes
|
||||||
|
|
||||||
instances = self.db.getRecordset(FeatureInstance, recordFilter={"mandateId": mandateId})
|
instances = self.db.getRecordset(FeatureInstance, recordFilter={"mandateId": mandateId})
|
||||||
|
|
@ -1987,20 +1980,7 @@ class AppObjects:
|
||||||
subInterface.db.recordDelete(MandateSubscription, subId)
|
subInterface.db.recordDelete(MandateSubscription, subId)
|
||||||
logger.info(f"Cascade: deleted {len(subs)} subscriptions for mandate {mandateId}")
|
logger.info(f"Cascade: deleted {len(subs)} subscriptions for mandate {mandateId}")
|
||||||
|
|
||||||
# 3b. Delete Billing data (poweron_billing)
|
# 3b. Billing data cascade handled by onMandateDelete lifecycle hook (interfaceDbBilling)
|
||||||
from modules.interfaces.interfaceDbBilling import getRootInterface as _getBillingRoot
|
|
||||||
billingDb = _getBillingRoot().db
|
|
||||||
billingAccounts = billingDb.getRecordset(BillingAccount, recordFilter={"mandateId": mandateId})
|
|
||||||
for acc in billingAccounts:
|
|
||||||
accTxs = billingDb.getRecordset(BillingTransaction, recordFilter={"accountId": acc.get("id")})
|
|
||||||
for tx in accTxs:
|
|
||||||
billingDb.recordDelete(BillingTransaction, tx.get("id"))
|
|
||||||
billingDb.recordDelete(BillingAccount, acc.get("id"))
|
|
||||||
billingSettings = billingDb.getRecordset(BillingSettings, recordFilter={"mandateId": mandateId})
|
|
||||||
for bs in billingSettings:
|
|
||||||
billingDb.recordDelete(BillingSettings, bs.get("id"))
|
|
||||||
if billingAccounts or billingSettings:
|
|
||||||
logger.info(f"Cascade: deleted billing data for mandate {mandateId}")
|
|
||||||
|
|
||||||
# 3c. Delete Invitations for this mandate
|
# 3c. Delete Invitations for this mandate
|
||||||
from modules.datamodels.datamodelInvitation import Invitation
|
from modules.datamodels.datamodelInvitation import Invitation
|
||||||
|
|
@ -2155,10 +2135,20 @@ class AppObjects:
|
||||||
)
|
)
|
||||||
self.db.recordCreate(UserMandateRole, userMandateRole.model_dump())
|
self.db.recordCreate(UserMandateRole, userMandateRole.model_dump())
|
||||||
|
|
||||||
self._ensureUserBillingAccount(userId, mandateId)
|
from modules.shared.systemComponentRegistry import getLifecycleHooks
|
||||||
|
for _hook in getLifecycleHooks("onUserMandateCreate"):
|
||||||
|
try:
|
||||||
|
_hook(userId, mandateId)
|
||||||
|
except Exception as _hookErr:
|
||||||
|
logger.warning("onUserMandateCreate hook failed: %s", _hookErr)
|
||||||
|
|
||||||
self._syncSubscriptionQuantity(mandateId)
|
self._syncSubscriptionQuantity(mandateId)
|
||||||
if not skipCapacityCheck:
|
if not skipCapacityCheck:
|
||||||
self._adjustAiBudgetForUserChange(mandateId, delta=+1)
|
for _hook in getLifecycleHooks("onUserBudgetAdjust"):
|
||||||
|
try:
|
||||||
|
_hook(mandateId, +1)
|
||||||
|
except Exception as _hookErr:
|
||||||
|
logger.warning("onUserBudgetAdjust hook failed: %s", _hookErr)
|
||||||
|
|
||||||
cleanedRecord = dict(createdRecord)
|
cleanedRecord = dict(createdRecord)
|
||||||
return UserMandate(**cleanedRecord)
|
return UserMandate(**cleanedRecord)
|
||||||
|
|
@ -2168,26 +2158,6 @@ class AppObjects:
|
||||||
logger.error(f"Error creating UserMandate: {e}")
|
logger.error(f"Error creating UserMandate: {e}")
|
||||||
raise ValueError(f"Failed to create UserMandate: {e}") from e
|
raise ValueError(f"Failed to create UserMandate: {e}") from e
|
||||||
|
|
||||||
def _ensureUserBillingAccount(self, userId: str, mandateId: str) -> None:
|
|
||||||
"""
|
|
||||||
Ensure a user has a billing audit account for the mandate.
|
|
||||||
Balance is always on the mandate pool (PREPAY_MANDATE). User accounts are for audit trail only.
|
|
||||||
"""
|
|
||||||
try:
|
|
||||||
from modules.interfaces.interfaceDbBilling import getRootInterface as getBillingRootInterface
|
|
||||||
|
|
||||||
billingInterface = getBillingRootInterface()
|
|
||||||
settings = billingInterface.getSettings(mandateId)
|
|
||||||
|
|
||||||
if not settings:
|
|
||||||
return
|
|
||||||
|
|
||||||
billingInterface.getOrCreateUserAccount(mandateId, userId, initialBalance=0.0)
|
|
||||||
logger.info(f"Ensured billing audit account for user {userId} in mandate {mandateId}")
|
|
||||||
|
|
||||||
except Exception as e:
|
|
||||||
logger.warning(f"Failed to create billing account for user {userId} (non-critical): {e}")
|
|
||||||
|
|
||||||
def _checkSubscriptionCapacity(self, mandateId: str, resourceType: str, delta: int = 1) -> None:
|
def _checkSubscriptionCapacity(self, mandateId: str, resourceType: str, delta: int = 1) -> None:
|
||||||
"""Check subscription capacity before creating a resource. Raises on cap violation."""
|
"""Check subscription capacity before creating a resource. Raises on cap violation."""
|
||||||
try:
|
try:
|
||||||
|
|
@ -2222,23 +2192,6 @@ class AppObjects:
|
||||||
raise
|
raise
|
||||||
logger.debug(f"Subscription quantity sync skipped: {e}")
|
logger.debug(f"Subscription quantity sync skipped: {e}")
|
||||||
|
|
||||||
def _adjustAiBudgetForUserChange(self, mandateId: str, delta: int) -> None:
|
|
||||||
"""Pro-rata AI budget credit/debit when a user is added or removed mid-cycle."""
|
|
||||||
try:
|
|
||||||
from modules.interfaces.interfaceDbSubscription import getInterface as getSubInterface
|
|
||||||
from modules.interfaces.interfaceDbBilling import getInterface as getBillingInterface
|
|
||||||
from modules.security.rootAccess import getRootUser
|
|
||||||
rootUser = getRootUser()
|
|
||||||
subIf = getSubInterface(rootUser, mandateId)
|
|
||||||
operative = subIf.getOperativeForMandate(mandateId)
|
|
||||||
if not operative:
|
|
||||||
return
|
|
||||||
planKey = operative.get("planKey", "")
|
|
||||||
billingIf = getBillingInterface(rootUser)
|
|
||||||
billingIf.adjustAiBudgetForUserChange(mandateId, planKey, delta)
|
|
||||||
except Exception as e:
|
|
||||||
logger.debug(f"AI budget adjustment skipped: {e}")
|
|
||||||
|
|
||||||
def deleteUserMandate(self, userId: str, mandateId: str) -> bool:
|
def deleteUserMandate(self, userId: str, mandateId: str) -> bool:
|
||||||
"""
|
"""
|
||||||
Delete a UserMandate record (remove user from mandate).
|
Delete a UserMandate record (remove user from mandate).
|
||||||
|
|
@ -2278,7 +2231,14 @@ class AppObjects:
|
||||||
|
|
||||||
result = self.db.recordDelete(UserMandate, existing.id)
|
result = self.db.recordDelete(UserMandate, existing.id)
|
||||||
self._syncSubscriptionQuantity(mandateId)
|
self._syncSubscriptionQuantity(mandateId)
|
||||||
self._adjustAiBudgetForUserChange(mandateId, delta=-1)
|
|
||||||
|
from modules.shared.systemComponentRegistry import getLifecycleHooks
|
||||||
|
for _hook in getLifecycleHooks("onUserMandateDelete"):
|
||||||
|
try:
|
||||||
|
_hook(userId, mandateId)
|
||||||
|
except Exception as _hookErr:
|
||||||
|
logger.warning("onUserMandateDelete hook failed: %s", _hookErr)
|
||||||
|
|
||||||
return result
|
return result
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
logger.error(f"Error deleting UserMandate: {e}")
|
logger.error(f"Error deleting UserMandate: {e}")
|
||||||
|
|
|
||||||
|
|
@ -2144,3 +2144,83 @@ class BillingObjects:
|
||||||
# Sort by creation date descending and limit
|
# Sort by creation date descending and limit
|
||||||
_sortBillingTransactionsBySysCreatedAtDesc(allTransactions, "getUserTransactionsForMandates")
|
_sortBillingTransactionsBySysCreatedAtDesc(allTransactions, "getUserTransactionsForMandates")
|
||||||
return allTransactions[:limit]
|
return allTransactions[:limit]
|
||||||
|
|
||||||
|
def deleteMandateData(self, mandateId: str) -> None:
|
||||||
|
"""Delete all billing data for a mandate (accounts, transactions, settings).
|
||||||
|
|
||||||
|
Used as cascade during mandate hard-delete via the onMandateDelete lifecycle hook.
|
||||||
|
"""
|
||||||
|
billingAccounts = self.db.getRecordset(BillingAccount, recordFilter={"mandateId": mandateId})
|
||||||
|
for acc in billingAccounts:
|
||||||
|
accTxs = self.db.getRecordset(BillingTransaction, recordFilter={"accountId": acc.get("id")})
|
||||||
|
for tx in accTxs:
|
||||||
|
self.db.recordDelete(BillingTransaction, tx.get("id"))
|
||||||
|
self.db.recordDelete(BillingAccount, acc.get("id"))
|
||||||
|
billingSettings = self.db.getRecordset(BillingSettings, recordFilter={"mandateId": mandateId})
|
||||||
|
for bs in billingSettings:
|
||||||
|
self.db.recordDelete(BillingSettings, bs.get("id"))
|
||||||
|
if billingAccounts or billingSettings:
|
||||||
|
logger.info("deleteMandateData: deleted billing data for mandate %s", mandateId)
|
||||||
|
|
||||||
|
|
||||||
|
def onMandateDelete(mandateId: str, instances: list) -> None:
|
||||||
|
"""Lifecycle hook: cascade-delete billing data when a mandate is hard-deleted."""
|
||||||
|
getRootInterface().deleteMandateData(mandateId)
|
||||||
|
|
||||||
|
|
||||||
|
def onUserMandateCreate(userId: str, mandateId: str) -> None:
|
||||||
|
"""Lifecycle hook: ensure user has a billing audit account when added to a mandate."""
|
||||||
|
try:
|
||||||
|
billingInterface = getRootInterface()
|
||||||
|
settings = billingInterface.getSettings(mandateId)
|
||||||
|
if not settings:
|
||||||
|
return
|
||||||
|
billingInterface.getOrCreateUserAccount(mandateId, userId, initialBalance=0.0)
|
||||||
|
logger.info("Ensured billing audit account for user %s in mandate %s", userId, mandateId)
|
||||||
|
except Exception as e:
|
||||||
|
logger.warning("Failed to create billing account for user %s (non-critical): %s", userId, e)
|
||||||
|
|
||||||
|
|
||||||
|
def onUserMandateDelete(userId: str, mandateId: str) -> None:
|
||||||
|
"""Lifecycle hook: pro-rata AI budget debit when user is removed from a mandate."""
|
||||||
|
_adjustAiBudgetForUserChange(mandateId, delta=-1)
|
||||||
|
|
||||||
|
|
||||||
|
def onUserBudgetAdjust(mandateId: str, delta: int) -> None:
|
||||||
|
"""Lifecycle hook: pro-rata AI budget credit/debit for user membership changes."""
|
||||||
|
_adjustAiBudgetForUserChange(mandateId, delta)
|
||||||
|
|
||||||
|
|
||||||
|
def onMandateProvision(mandateId: str, planKey: str) -> None:
|
||||||
|
"""Lifecycle hook: create billing settings and activation budget for a new mandate."""
|
||||||
|
try:
|
||||||
|
billingRoot = getRootInterface()
|
||||||
|
billingRoot.getOrCreateSettings(mandateId)
|
||||||
|
billingRoot.ensureActivationBudget(mandateId, planKey)
|
||||||
|
except Exception as e:
|
||||||
|
logger.error("Initial billing setup failed for mandate %s (plan=%s): %s", mandateId, planKey, e)
|
||||||
|
|
||||||
|
|
||||||
|
def onStorageChanged(mandateId: str) -> None:
|
||||||
|
"""Lifecycle hook: reconcile storage billing after knowledge content changes."""
|
||||||
|
try:
|
||||||
|
getRootInterface().reconcileMandateStorageBilling(mandateId)
|
||||||
|
except Exception as e:
|
||||||
|
logger.warning("reconcileMandateStorageBilling failed for mandate %s: %s", mandateId, e)
|
||||||
|
|
||||||
|
|
||||||
|
def _adjustAiBudgetForUserChange(mandateId: str, delta: int) -> None:
|
||||||
|
"""Pro-rata AI budget credit/debit when a user is added or removed mid-cycle."""
|
||||||
|
try:
|
||||||
|
from modules.interfaces.interfaceDbSubscription import getInterface as getSubInterface
|
||||||
|
from modules.security.rootAccess import getRootUser
|
||||||
|
rootUser = getRootUser()
|
||||||
|
subIf = getSubInterface(rootUser, mandateId)
|
||||||
|
operative = subIf.getOperativeForMandate(mandateId)
|
||||||
|
if not operative:
|
||||||
|
return
|
||||||
|
planKey = operative.get("planKey", "")
|
||||||
|
billingIf = getInterface(rootUser)
|
||||||
|
billingIf.adjustAiBudgetForUserChange(mandateId, planKey, delta)
|
||||||
|
except Exception as e:
|
||||||
|
logger.debug("AI budget adjustment skipped: %s", e)
|
||||||
|
|
|
||||||
|
|
@ -123,13 +123,13 @@ class KnowledgeObjects:
|
||||||
if mid:
|
if mid:
|
||||||
mandateIds.add(str(mid))
|
mandateIds.add(str(mid))
|
||||||
|
|
||||||
|
from modules.shared.systemComponentRegistry import getLifecycleHooks
|
||||||
for mid in mandateIds:
|
for mid in mandateIds:
|
||||||
|
for _hook in getLifecycleHooks("onStorageChanged"):
|
||||||
try:
|
try:
|
||||||
from modules.interfaces.interfaceDbBilling import getRootInterface as getBillingRoot
|
_hook(mid)
|
||||||
|
|
||||||
getBillingRoot().reconcileMandateStorageBilling(mid)
|
|
||||||
except Exception as ex:
|
except Exception as ex:
|
||||||
logger.warning("reconcileMandateStorageBilling after connection purge failed: %s", ex)
|
logger.warning("onStorageChanged hook after connection purge failed: %s", ex)
|
||||||
|
|
||||||
return {"indexRows": indexCount, "chunks": chunkCount}
|
return {"indexRows": indexCount, "chunks": chunkCount}
|
||||||
|
|
||||||
|
|
@ -166,12 +166,13 @@ class KnowledgeObjects:
|
||||||
if mid:
|
if mid:
|
||||||
mandateIds.add(str(mid))
|
mandateIds.add(str(mid))
|
||||||
|
|
||||||
|
from modules.shared.systemComponentRegistry import getLifecycleHooks
|
||||||
for mid in mandateIds:
|
for mid in mandateIds:
|
||||||
|
for _hook in getLifecycleHooks("onStorageChanged"):
|
||||||
try:
|
try:
|
||||||
from modules.interfaces.interfaceDbBilling import getRootInterface as getBillingRoot
|
_hook(mid)
|
||||||
getBillingRoot().reconcileMandateStorageBilling(mid)
|
|
||||||
except Exception as ex:
|
except Exception as ex:
|
||||||
logger.warning("reconcileMandateStorageBilling after datasource purge failed: %s", ex)
|
logger.warning("onStorageChanged hook after datasource purge failed: %s", ex)
|
||||||
|
|
||||||
return {"indexRows": indexCount, "chunks": chunkCount}
|
return {"indexRows": indexCount, "chunks": chunkCount}
|
||||||
|
|
||||||
|
|
@ -196,12 +197,12 @@ class KnowledgeObjects:
|
||||||
self.db.recordDelete(ContentChunk, chunk["id"])
|
self.db.recordDelete(ContentChunk, chunk["id"])
|
||||||
ok = self.db.recordDelete(FileContentIndex, fileId)
|
ok = self.db.recordDelete(FileContentIndex, fileId)
|
||||||
if ok and mandateId:
|
if ok and mandateId:
|
||||||
|
from modules.shared.systemComponentRegistry import getLifecycleHooks
|
||||||
|
for _hook in getLifecycleHooks("onStorageChanged"):
|
||||||
try:
|
try:
|
||||||
from modules.interfaces.interfaceDbBilling import getRootInterface
|
_hook(str(mandateId))
|
||||||
|
|
||||||
getRootInterface().reconcileMandateStorageBilling(str(mandateId))
|
|
||||||
except Exception as ex:
|
except Exception as ex:
|
||||||
logger.warning("reconcileMandateStorageBilling after delete failed: %s", ex)
|
logger.warning("onStorageChanged hook after delete failed: %s", ex)
|
||||||
return ok
|
return ok
|
||||||
|
|
||||||
# =========================================================================
|
# =========================================================================
|
||||||
|
|
|
||||||
|
|
@ -27,12 +27,15 @@ import json
|
||||||
import math
|
import math
|
||||||
import re
|
import re
|
||||||
import copy
|
import copy
|
||||||
|
import uuid
|
||||||
from datetime import datetime, timezone
|
from datetime import datetime, timezone
|
||||||
from typing import List, Dict, Any, Optional, Type, Union
|
from typing import List, Dict, Any, Optional, Type, Union
|
||||||
from pydantic import BaseModel
|
from pydantic import BaseModel
|
||||||
from modules.datamodels.datamodelRbac import AccessRuleContext
|
from modules.datamodels.datamodelRbac import AccessRule, AccessRuleContext, Role
|
||||||
from modules.datamodels.datamodelUam import User, UserPermissions, AccessLevel
|
from modules.datamodels.datamodelUam import User, UserPermissions, AccessLevel
|
||||||
from modules.datamodels.datamodelPagination import PaginationParams, PaginatedResult
|
from modules.datamodels.datamodelPagination import PaginationParams, PaginatedResult
|
||||||
|
from modules.datamodels.datamodelUtils import coerce_text_multilingual
|
||||||
|
from modules.connectors.connectorDbPostgre import DatabaseConnector
|
||||||
from modules.security.rbac import RbacClass
|
from modules.security.rbac import RbacClass
|
||||||
from modules.security.rootAccess import getRootDbAppConnector
|
from modules.security.rootAccess import getRootDbAppConnector
|
||||||
|
|
||||||
|
|
@ -1123,3 +1126,96 @@ def _checkRowPermission(
|
||||||
|
|
||||||
# Unknown level - deny by default
|
# Unknown level - deny by default
|
||||||
return False
|
return False
|
||||||
|
|
||||||
|
|
||||||
|
# =============================================================================
|
||||||
|
# System Role Provisioning
|
||||||
|
# =============================================================================
|
||||||
|
|
||||||
|
|
||||||
|
def copySystemRolesToMandate(db: DatabaseConnector, mandateId: str) -> int:
|
||||||
|
"""
|
||||||
|
Copy system template roles (mandateId=None, isSystemRole=True) to a mandate
|
||||||
|
as mandate-instance roles. Also copies all AccessRules for each role.
|
||||||
|
|
||||||
|
This is analogous to how feature template roles are copied to feature instances.
|
||||||
|
Each mandate gets its own instances of admin/user/viewer with their AccessRules.
|
||||||
|
|
||||||
|
Args:
|
||||||
|
db: Database connector instance
|
||||||
|
mandateId: Target mandate ID
|
||||||
|
|
||||||
|
Returns:
|
||||||
|
Number of roles copied
|
||||||
|
"""
|
||||||
|
templateRoles = db.getRecordset(
|
||||||
|
Role,
|
||||||
|
recordFilter={"isSystemRole": True, "mandateId": None}
|
||||||
|
)
|
||||||
|
|
||||||
|
if not templateRoles:
|
||||||
|
logger.warning("No system template roles found (mandateId IS NULL, isSystemRole=True)")
|
||||||
|
return 0
|
||||||
|
|
||||||
|
existingMandateRoles = db.getRecordset(
|
||||||
|
Role,
|
||||||
|
recordFilter={"mandateId": mandateId, "featureInstanceId": None}
|
||||||
|
)
|
||||||
|
existingLabels = {r.get("roleLabel") for r in existingMandateRoles}
|
||||||
|
logger.info(
|
||||||
|
"copySystemRolesToMandate: mandate=%s, templates=%s, existing=%s, labels=%s",
|
||||||
|
mandateId, len(templateRoles), len(existingMandateRoles), existingLabels,
|
||||||
|
)
|
||||||
|
|
||||||
|
templateRoleIds = [r.get("id") for r in templateRoles]
|
||||||
|
rulesByRoleId = {}
|
||||||
|
for roleId in templateRoleIds:
|
||||||
|
rules = db.getRecordset(AccessRule, recordFilter={"roleId": roleId})
|
||||||
|
rulesByRoleId[roleId] = rules
|
||||||
|
|
||||||
|
copiedCount = 0
|
||||||
|
for templateRole in templateRoles:
|
||||||
|
roleLabel = templateRole.get("roleLabel")
|
||||||
|
|
||||||
|
if roleLabel in existingLabels:
|
||||||
|
logger.debug("Mandate %s already has role '%s', skipping", mandateId, roleLabel)
|
||||||
|
continue
|
||||||
|
|
||||||
|
newRoleId = str(uuid.uuid4())
|
||||||
|
|
||||||
|
newRole = Role(
|
||||||
|
id=newRoleId,
|
||||||
|
roleLabel=roleLabel,
|
||||||
|
description=coerce_text_multilingual(templateRole.get("description", {})),
|
||||||
|
mandateId=mandateId,
|
||||||
|
featureInstanceId=None,
|
||||||
|
featureCode=None,
|
||||||
|
isSystemRole=False,
|
||||||
|
)
|
||||||
|
db.recordCreate(Role, newRole.model_dump())
|
||||||
|
|
||||||
|
templateRules = rulesByRoleId.get(templateRole.get("id"), [])
|
||||||
|
for rule in templateRules:
|
||||||
|
newRule = AccessRule(
|
||||||
|
id=str(uuid.uuid4()),
|
||||||
|
roleId=newRoleId,
|
||||||
|
context=rule.get("context"),
|
||||||
|
item=rule.get("item"),
|
||||||
|
view=rule.get("view", False),
|
||||||
|
read=rule.get("read"),
|
||||||
|
create=rule.get("create"),
|
||||||
|
update=rule.get("update"),
|
||||||
|
delete=rule.get("delete"),
|
||||||
|
)
|
||||||
|
db.recordCreate(AccessRule, newRule.model_dump())
|
||||||
|
|
||||||
|
copiedCount += 1
|
||||||
|
logger.info(
|
||||||
|
"Copied system role '%s' to mandate %s with %s AccessRules",
|
||||||
|
roleLabel, mandateId, len(templateRules),
|
||||||
|
)
|
||||||
|
|
||||||
|
if copiedCount > 0:
|
||||||
|
logger.info("Copied %s system roles to mandate %s", copiedCount, mandateId)
|
||||||
|
|
||||||
|
return copiedCount
|
||||||
|
|
|
||||||
|
|
@ -798,7 +798,7 @@ async def _updateKnowledgeConsent(
|
||||||
cancelled = cancelJobsByConnection(connectionId)
|
cancelled = cancelJobsByConnection(connectionId)
|
||||||
else:
|
else:
|
||||||
from modules.datamodels.datamodelDataSource import DataSource
|
from modules.datamodels.datamodelDataSource import DataSource
|
||||||
from modules.serviceCenter.services.serviceKnowledge._inheritFlags import getEffectiveFlag
|
from modules.serviceCenter.core.flagResolution import getEffectiveFlag
|
||||||
allConnDs = rootIf.db.getRecordset(DataSource, recordFilter={"connectionId": connectionId})
|
allConnDs = rootIf.db.getRecordset(DataSource, recordFilter={"connectionId": connectionId})
|
||||||
dataSources = [
|
dataSources = [
|
||||||
ds for ds in (allConnDs or [])
|
ds for ds in (allConnDs or [])
|
||||||
|
|
|
||||||
|
|
@ -98,17 +98,17 @@ async def _autoIndexFile(fileId: str, fileName: str, mimeType: str, user, *, man
|
||||||
return
|
return
|
||||||
|
|
||||||
file_meta = mgmtInterface.getFile(fileId)
|
file_meta = mgmtInterface.getFile(fileId)
|
||||||
feature_instance_id = ""
|
featureInstanceId = ""
|
||||||
mandate_id = ""
|
mandateId = ""
|
||||||
file_scope = "personal"
|
file_scope = "personal"
|
||||||
if file_meta:
|
if file_meta:
|
||||||
if isinstance(file_meta, dict):
|
if isinstance(file_meta, dict):
|
||||||
feature_instance_id = file_meta.get("featureInstanceId") or ""
|
featureInstanceId = file_meta.get("featureInstanceId") or ""
|
||||||
mandate_id = file_meta.get("mandateId") or ""
|
mandateId = file_meta.get("mandateId") or ""
|
||||||
file_scope = file_meta.get("scope") or "personal"
|
file_scope = file_meta.get("scope") or "personal"
|
||||||
else:
|
else:
|
||||||
feature_instance_id = getattr(file_meta, "featureInstanceId", None) or ""
|
featureInstanceId = getattr(file_meta, "featureInstanceId", None) or ""
|
||||||
mandate_id = getattr(file_meta, "mandateId", None) or ""
|
mandateId = getattr(file_meta, "mandateId", None) or ""
|
||||||
file_scope = getattr(file_meta, "scope", None) or "personal"
|
file_scope = getattr(file_meta, "scope", None) or "personal"
|
||||||
|
|
||||||
logger.info(f"Auto-index starting for {fileName} ({len(rawBytes)} bytes, {mimeType})")
|
logger.info(f"Auto-index starting for {fileName} ({len(rawBytes)} bytes, {mimeType})")
|
||||||
|
|
@ -121,8 +121,8 @@ async def _autoIndexFile(fileId: str, fileName: str, mimeType: str, user, *, man
|
||||||
fileId=fileId,
|
fileId=fileId,
|
||||||
fileName=fileName,
|
fileName=fileName,
|
||||||
userId=userId,
|
userId=userId,
|
||||||
featureInstanceId=str(feature_instance_id) if feature_instance_id else "",
|
featureInstanceId=str(featureInstanceId) if featureInstanceId else "",
|
||||||
mandateId=str(mandate_id) if mandate_id else "",
|
mandateId=str(mandateId) if mandateId else "",
|
||||||
scope=file_scope,
|
scope=file_scope,
|
||||||
)
|
)
|
||||||
logger.info(
|
logger.info(
|
||||||
|
|
@ -208,8 +208,8 @@ async def _autoIndexFile(fileId: str, fileName: str, mimeType: str, user, *, man
|
||||||
|
|
||||||
ctx = ServiceCenterContext(
|
ctx = ServiceCenterContext(
|
||||||
user=user,
|
user=user,
|
||||||
mandate_id=str(mandate_id) if mandate_id else "",
|
mandateId=str(mandateId) if mandateId else "",
|
||||||
feature_instance_id=str(feature_instance_id) if feature_instance_id else "",
|
featureInstanceId=str(featureInstanceId) if featureInstanceId else "",
|
||||||
)
|
)
|
||||||
knowledgeService = getService("knowledge", ctx)
|
knowledgeService = getService("knowledge", ctx)
|
||||||
|
|
||||||
|
|
@ -222,8 +222,8 @@ async def _autoIndexFile(fileId: str, fileName: str, mimeType: str, user, *, man
|
||||||
fileName=fileName,
|
fileName=fileName,
|
||||||
mimeType=mimeType,
|
mimeType=mimeType,
|
||||||
userId=userId,
|
userId=userId,
|
||||||
featureInstanceId=str(feature_instance_id) if feature_instance_id else "",
|
featureInstanceId=str(featureInstanceId) if featureInstanceId else "",
|
||||||
mandateId=str(mandate_id) if mandate_id else "",
|
mandateId=str(mandateId) if mandateId else "",
|
||||||
contentObjects=contentObjects,
|
contentObjects=contentObjects,
|
||||||
structure=contentIndex.structure,
|
structure=contentIndex.structure,
|
||||||
provenance={"lane": "upload", "route": "routeDataFiles._autoIndexFile"},
|
provenance={"lane": "upload", "route": "routeDataFiles._autoIndexFile"},
|
||||||
|
|
|
||||||
|
|
@ -86,7 +86,7 @@ def _buildConnectionInventory(connections, rootIf, knowledgeIf, jobService) -> L
|
||||||
"""
|
"""
|
||||||
from modules.datamodels.datamodelDataSource import DataSource
|
from modules.datamodels.datamodelDataSource import DataSource
|
||||||
from modules.datamodels.datamodelKnowledge import FileContentIndex
|
from modules.datamodels.datamodelKnowledge import FileContentIndex
|
||||||
from modules.serviceCenter.services.serviceKnowledge._inheritFlags import getEffectiveFlag
|
from modules.serviceCenter.core.flagResolution import getEffectiveFlag
|
||||||
|
|
||||||
out = []
|
out = []
|
||||||
for conn in connections:
|
for conn in connections:
|
||||||
|
|
@ -236,7 +236,7 @@ def _buildFeatureInstanceInventory(featureInstanceIds, rootIf, knowledgeIf) -> L
|
||||||
from modules.datamodels.datamodelKnowledge import FileContentIndex
|
from modules.datamodels.datamodelKnowledge import FileContentIndex
|
||||||
from modules.datamodels.datamodelFeatures import FeatureDataSource
|
from modules.datamodels.datamodelFeatures import FeatureDataSource
|
||||||
from modules.interfaces.interfaceFeatures import getFeatureInterface
|
from modules.interfaces.interfaceFeatures import getFeatureInterface
|
||||||
from modules.serviceCenter.services.serviceKnowledge._inheritFlags import getEffectiveFlagFds
|
from modules.serviceCenter.core.flagResolution import getEffectiveFlagFds
|
||||||
from modules.serviceCenter.services.serviceBackgroundJobs import mainBackgroundJobService as jobService
|
from modules.serviceCenter.services.serviceBackgroundJobs import mainBackgroundJobService as jobService
|
||||||
from modules.serviceCenter.services.serviceKnowledge.subFeatureBootstrap import FEATURE_BOOTSTRAP_JOB_TYPE
|
from modules.serviceCenter.services.serviceKnowledge.subFeatureBootstrap import FEATURE_BOOTSTRAP_JOB_TYPE
|
||||||
|
|
||||||
|
|
@ -548,7 +548,7 @@ async def _reindexConnection(
|
||||||
if str(conn.userId) != str(currentUser.id):
|
if str(conn.userId) != str(currentUser.id):
|
||||||
raise HTTPException(status_code=403, detail="Not your connection")
|
raise HTTPException(status_code=403, detail="Not your connection")
|
||||||
|
|
||||||
from modules.serviceCenter.services.serviceKnowledge._inheritFlags import getEffectiveFlag
|
from modules.serviceCenter.core.flagResolution import getEffectiveFlag
|
||||||
dataSources = rootIf.db.getRecordset(DataSource, recordFilter={"connectionId": connectionId})
|
dataSources = rootIf.db.getRecordset(DataSource, recordFilter={"connectionId": connectionId})
|
||||||
ragDs = [ds for ds in dataSources if getEffectiveFlag(ds, "ragIndexEnabled", dataSources, mode="walk") is True]
|
ragDs = [ds for ds in dataSources if getEffectiveFlag(ds, "ragIndexEnabled", dataSources, mode="walk") is True]
|
||||||
if not ragDs:
|
if not ragDs:
|
||||||
|
|
|
||||||
|
|
@ -251,7 +251,7 @@ async def _generateTtsSampleTextForLocale(
|
||||||
from modules.serviceCenter.services.serviceSubscription.mainServiceSubscription import SubscriptionInactiveException
|
from modules.serviceCenter.services.serviceSubscription.mainServiceSubscription import SubscriptionInactiveException
|
||||||
|
|
||||||
mandateId = _resolveMandateIdForVoiceTestAi(request, currentUser)
|
mandateId = _resolveMandateIdForVoiceTestAi(request, currentUser)
|
||||||
ctx = ServiceCenterContext(user=currentUser, mandate_id=mandateId, feature_instance_id=None)
|
ctx = ServiceCenterContext(user=currentUser, mandateId=mandateId, featureInstanceId=None)
|
||||||
aiService = getService("ai", ctx)
|
aiService = getService("ai", ctx)
|
||||||
|
|
||||||
systemPrompt = (
|
systemPrompt = (
|
||||||
|
|
|
||||||
|
|
@ -596,8 +596,8 @@ def _buildServiceCenterContext(context: RequestContext, mandateId: str, instance
|
||||||
from modules.serviceCenter.context import ServiceCenterContext
|
from modules.serviceCenter.context import ServiceCenterContext
|
||||||
return ServiceCenterContext(
|
return ServiceCenterContext(
|
||||||
user=context.user,
|
user=context.user,
|
||||||
mandate_id=str(context.mandateId) if context.mandateId else mandateId,
|
mandateId=str(context.mandateId) if context.mandateId else mandateId,
|
||||||
feature_instance_id=instanceId,
|
featureInstanceId=instanceId,
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
|
|
@ -1366,6 +1366,21 @@ def _buildExecuteRunEnvelope(
|
||||||
return env
|
return env
|
||||||
|
|
||||||
|
|
||||||
|
def _startEmailPollerIfNeeded(result: dict) -> None:
|
||||||
|
"""Start the background email poller when a run pauses for email wait."""
|
||||||
|
if not isinstance(result, dict) or result.get("waitReason") != "email":
|
||||||
|
return
|
||||||
|
try:
|
||||||
|
from modules.interfaces.interfaceDbApp import getRootInterface
|
||||||
|
from modules.workflowAutomation.scheduler.emailPoller import ensureRunning
|
||||||
|
root = getRootInterface()
|
||||||
|
eventUser = root.getUserByUsername("event") if root else None
|
||||||
|
if eventUser:
|
||||||
|
ensureRunning(eventUser)
|
||||||
|
except Exception as pollErr:
|
||||||
|
logger.warning("Could not start email poller: %s", pollErr)
|
||||||
|
|
||||||
|
|
||||||
@router.post("/workflows/{workflowId}/execute")
|
@router.post("/workflows/{workflowId}/execute")
|
||||||
@limiter.limit("30/minute")
|
@limiter.limit("30/minute")
|
||||||
async def _executeWorkflow(
|
async def _executeWorkflow(
|
||||||
|
|
@ -1446,6 +1461,7 @@ async def _executeWorkflow(
|
||||||
"workflowAutomation execute result: success=%s error=%s paused=%s",
|
"workflowAutomation execute result: success=%s error=%s paused=%s",
|
||||||
result.get("success"), result.get("error"), result.get("paused"),
|
result.get("success"), result.get("error"), result.get("paused"),
|
||||||
)
|
)
|
||||||
|
_startEmailPollerIfNeeded(result)
|
||||||
return result
|
return result
|
||||||
|
|
||||||
|
|
||||||
|
|
@ -1778,7 +1794,7 @@ async def _completeTask(
|
||||||
|
|
||||||
graph = wfForGraph["graph"]
|
graph = wfForGraph["graph"]
|
||||||
services = _getWorkflowAutomationServices(context.user, mandateId=mandateId, featureInstanceId=instanceId)
|
services = _getWorkflowAutomationServices(context.user, mandateId=mandateId, featureInstanceId=instanceId)
|
||||||
return await executeGraph(
|
result = await executeGraph(
|
||||||
graph=graph,
|
graph=graph,
|
||||||
services=services,
|
services=services,
|
||||||
workflowId=workflowId,
|
workflowId=workflowId,
|
||||||
|
|
@ -1790,6 +1806,8 @@ async def _completeTask(
|
||||||
startAfterNodeId=taskNodeId,
|
startAfterNodeId=taskNodeId,
|
||||||
runId=runId,
|
runId=runId,
|
||||||
)
|
)
|
||||||
|
_startEmailPollerIfNeeded(result)
|
||||||
|
return result
|
||||||
|
|
||||||
|
|
||||||
@router.post("/tasks/{taskId}/cancel")
|
@router.post("/tasks/{taskId}/cancel")
|
||||||
|
|
|
||||||
|
|
@ -33,7 +33,7 @@ def getService(
|
||||||
|
|
||||||
Args:
|
Args:
|
||||||
key: Service key (e.g., "web", "extraction", "utils")
|
key: Service key (e.g., "web", "extraction", "utils")
|
||||||
context: ServiceCenterContext with user, mandate_id, feature_instance_id, workflow
|
context: ServiceCenterContext with user, mandateId, featureInstanceId, workflow
|
||||||
|
|
||||||
Returns:
|
Returns:
|
||||||
Service instance
|
Service instance
|
||||||
|
|
|
||||||
|
|
@ -16,20 +16,10 @@ class ServiceCenterContext:
|
||||||
"""Context for service resolution: user, mandate, feature instance, optional workflow."""
|
"""Context for service resolution: user, mandate, feature instance, optional workflow."""
|
||||||
|
|
||||||
user: User
|
user: User
|
||||||
mandate_id: Optional[str] = None
|
mandateId: Optional[str] = None
|
||||||
feature_instance_id: Optional[str] = None
|
featureInstanceId: Optional[str] = None
|
||||||
workflow_id: Optional[str] = None
|
workflowId: Optional[str] = None
|
||||||
workflow: Any = None
|
workflow: Any = None
|
||||||
requireNeutralization: Optional[bool] = None
|
requireNeutralization: Optional[bool] = None
|
||||||
# When workflow is absent (e.g. workspace agent), billing/UI still need feature code for transactions.
|
# When workflow is absent (e.g. workspace agent), billing/UI still need feature code for transactions.
|
||||||
feature_code: Optional[str] = None
|
featureCode: Optional[str] = None
|
||||||
|
|
||||||
@property
|
|
||||||
def mandateId(self) -> Optional[str]:
|
|
||||||
"""Alias for mandate_id (backward compatibility)."""
|
|
||||||
return self.mandate_id
|
|
||||||
|
|
||||||
@property
|
|
||||||
def featureInstanceId(self) -> Optional[str]:
|
|
||||||
"""Alias for feature_instance_id (backward compatibility)."""
|
|
||||||
return self.feature_instance_id
|
|
||||||
|
|
|
||||||
|
|
@ -20,12 +20,12 @@ class SecurityService:
|
||||||
def __init__(self, context: Any, get_service: Callable[[str], Any]):
|
def __init__(self, context: Any, get_service: Callable[[str], Any]):
|
||||||
"""Initialize with service center context and resolver."""
|
"""Initialize with service center context and resolver."""
|
||||||
self._context = context
|
self._context = context
|
||||||
self._get_service = get_service
|
self._getService = get_service
|
||||||
self._tokenManager = TokenManager()
|
self._tokenManager = TokenManager()
|
||||||
from modules.interfaces.interfaceDbApp import getInterface as getAppInterface
|
from modules.interfaces.interfaceDbApp import getInterface as getAppInterface
|
||||||
self._interfaceDbApp = getAppInterface(
|
self._interfaceDbApp = getAppInterface(
|
||||||
context.user,
|
context.user,
|
||||||
mandateId=context.mandate_id,
|
mandateId=context.mandateId,
|
||||||
)
|
)
|
||||||
|
|
||||||
def getFreshToken(self, connectionId: str, secondsBeforeExpiry: int = 30 * 60) -> Optional[Token]:
|
def getFreshToken(self, connectionId: str, secondsBeforeExpiry: int = 30 * 60) -> Optional[Token]:
|
||||||
|
|
|
||||||
|
|
@ -19,7 +19,7 @@ class StreamingService:
|
||||||
def __init__(self, context: Any, get_service: Callable[[str], Any]):
|
def __init__(self, context: Any, get_service: Callable[[str], Any]):
|
||||||
"""Initialize with service center context and resolver."""
|
"""Initialize with service center context and resolver."""
|
||||||
self._context = context
|
self._context = context
|
||||||
self._get_service = get_service
|
self._getService = get_service
|
||||||
|
|
||||||
def getEventManager(self) -> EventManager:
|
def getEventManager(self) -> EventManager:
|
||||||
"""Get the global event manager instance for SSE streaming."""
|
"""Get the global event manager instance for SSE streaming."""
|
||||||
|
|
|
||||||
|
|
@ -22,7 +22,7 @@ class UtilsService:
|
||||||
def __init__(self, context, get_service: Callable[[str], Any]):
|
def __init__(self, context, get_service: Callable[[str], Any]):
|
||||||
"""Initialize with service center context and resolver."""
|
"""Initialize with service center context and resolver."""
|
||||||
self._context = context
|
self._context = context
|
||||||
self._get_service = get_service
|
self._getService = get_service
|
||||||
|
|
||||||
# ===== Event handling =====
|
# ===== Event handling =====
|
||||||
|
|
||||||
|
|
|
||||||
90
modules/serviceCenter/core/types.py
Normal file
90
modules/serviceCenter/core/types.py
Normal file
|
|
@ -0,0 +1,90 @@
|
||||||
|
# Copyright (c) 2025 Patrick Motsch
|
||||||
|
# All rights reserved.
|
||||||
|
"""Neutral protocol types used across serviceCenter services.
|
||||||
|
|
||||||
|
Protocols defined here break import cycles by providing structural typing
|
||||||
|
contracts that services can depend on without importing concrete classes
|
||||||
|
from sibling services.
|
||||||
|
"""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
from typing import Any, Dict, List, Optional, Protocol, runtime_checkable
|
||||||
|
|
||||||
|
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
# FeatureDataProviderProtocol (used by serviceKnowledge, implemented in serviceAgent)
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
@runtime_checkable
|
||||||
|
class FeatureDataProviderProtocol(Protocol):
|
||||||
|
"""Structural contract for the RBAC-scoped feature-data read layer.
|
||||||
|
|
||||||
|
serviceKnowledge depends on this Protocol for RAG indexing;
|
||||||
|
serviceAgent supplies the concrete FeatureDataProvider implementation.
|
||||||
|
"""
|
||||||
|
|
||||||
|
def browseTable(
|
||||||
|
self,
|
||||||
|
tableName: str,
|
||||||
|
featureInstanceId: str,
|
||||||
|
mandateId: str,
|
||||||
|
fields: Optional[List[str]] = None,
|
||||||
|
limit: int = 50,
|
||||||
|
offset: int = 0,
|
||||||
|
extraFilters: Optional[List[Dict[str, Any]]] = None,
|
||||||
|
) -> Dict[str, Any]: ...
|
||||||
|
|
||||||
|
async def finalizeRowsAsync(
|
||||||
|
self,
|
||||||
|
tableName: str,
|
||||||
|
rows: List[Dict[str, Any]],
|
||||||
|
) -> List[Dict[str, Any]]: ...
|
||||||
|
|
||||||
|
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
# FeatureDataProvider factory registry
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
_featureDataProviderFactory = None
|
||||||
|
|
||||||
|
|
||||||
|
def registerFeatureDataProviderFactory(factory) -> None:
|
||||||
|
"""Register the concrete FeatureDataProvider class (called at composition time)."""
|
||||||
|
global _featureDataProviderFactory
|
||||||
|
_featureDataProviderFactory = factory
|
||||||
|
|
||||||
|
|
||||||
|
def createFeatureDataProvider(
|
||||||
|
dbConnector,
|
||||||
|
neutralizeFields: Optional[Dict[str, List[str]]] = None,
|
||||||
|
neutralizePolicy: Optional[Dict[str, Dict[str, Any]]] = None,
|
||||||
|
neutralizationService: Optional[Any] = None,
|
||||||
|
) -> FeatureDataProviderProtocol:
|
||||||
|
"""Instantiate a FeatureDataProvider without importing serviceAgent."""
|
||||||
|
if _featureDataProviderFactory is None:
|
||||||
|
raise RuntimeError(
|
||||||
|
"FeatureDataProvider factory not registered. "
|
||||||
|
"Ensure serviceAgent is initialized before serviceKnowledge bootstrap runs."
|
||||||
|
)
|
||||||
|
return _featureDataProviderFactory(
|
||||||
|
dbConnector,
|
||||||
|
neutralizeFields=neutralizeFields,
|
||||||
|
neutralizePolicy=neutralizePolicy,
|
||||||
|
neutralizationService=neutralizationService,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
# RendererProtocol (used by serviceExtraction, implemented in serviceGeneration)
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
@runtime_checkable
|
||||||
|
class RendererProtocol(Protocol):
|
||||||
|
"""Structural contract for document renderers.
|
||||||
|
|
||||||
|
serviceExtraction depends on this Protocol for type hints;
|
||||||
|
serviceGeneration supplies BaseRenderer and its subclasses.
|
||||||
|
"""
|
||||||
|
|
||||||
|
def getExtractionGuidelines(self) -> str: ...
|
||||||
|
|
@ -19,7 +19,7 @@ GetServiceFunc = Callable[[str], Any]
|
||||||
|
|
||||||
def _make_context_id(ctx: ServiceCenterContext) -> str:
|
def _make_context_id(ctx: ServiceCenterContext) -> str:
|
||||||
"""Create a stable cache key from context."""
|
"""Create a stable cache key from context."""
|
||||||
return f"{id(ctx.user)}_{ctx.mandate_id or ''}_{ctx.feature_instance_id or ''}"
|
return f"{id(ctx.user)}_{ctx.mandateId or ''}_{ctx.featureInstanceId or ''}"
|
||||||
|
|
||||||
|
|
||||||
def _load_service_class(module_path: str, class_name: str):
|
def _load_service_class(module_path: str, class_name: str):
|
||||||
|
|
|
||||||
|
|
@ -1,3 +1,8 @@
|
||||||
# Copyright (c) 2025 Patrick Motsch
|
# Copyright (c) 2025 Patrick Motsch
|
||||||
# All rights reserved.
|
# All rights reserved.
|
||||||
"""serviceAgent: AI Agent with ReAct loop and native function calling."""
|
"""serviceAgent: AI Agent with ReAct loop and native function calling."""
|
||||||
|
|
||||||
|
from modules.serviceCenter.core.types import registerFeatureDataProviderFactory
|
||||||
|
from modules.serviceCenter.services.serviceAgent.featureDataProvider import FeatureDataProvider
|
||||||
|
|
||||||
|
registerFeatureDataProviderFactory(FeatureDataProvider)
|
||||||
|
|
|
||||||
|
|
@ -151,7 +151,7 @@ def registerDataSourceTools(registry: ToolRegistry, services):
|
||||||
sourceType = ds.get("sourceType", "")
|
sourceType = ds.get("sourceType", "")
|
||||||
path = ds.get("path", "/")
|
path = ds.get("path", "/")
|
||||||
label = ds.get("label", "")
|
label = ds.get("label", "")
|
||||||
from modules.serviceCenter.services.serviceKnowledge._inheritFlags import getEffectiveFlag
|
from modules.serviceCenter.core.flagResolution import getEffectiveFlag
|
||||||
from modules.datamodels.datamodelDataSource import DataSource
|
from modules.datamodels.datamodelDataSource import DataSource
|
||||||
from modules.interfaces.interfaceDbApp import getRootInterface
|
from modules.interfaces.interfaceDbApp import getRootInterface
|
||||||
rootIf = getRootInterface()
|
rootIf = getRootInterface()
|
||||||
|
|
|
||||||
|
|
@ -109,7 +109,7 @@ def registerFeatureSubAgentTools(registry: ToolRegistry, services):
|
||||||
recordFilter={"featureInstanceId": featureInstanceId},
|
recordFilter={"featureInstanceId": featureInstanceId},
|
||||||
)
|
)
|
||||||
|
|
||||||
from modules.serviceCenter.services.serviceKnowledge._inheritFlags import getEffectiveFlagFds
|
from modules.serviceCenter.core.flagResolution import getEffectiveFlagFds
|
||||||
_fdsAll = featureDataSources or []
|
_fdsAll = featureDataSources or []
|
||||||
_anySourceNeutralize = any(
|
_anySourceNeutralize = any(
|
||||||
getEffectiveFlagFds(ds, "neutralize", _fdsAll, mode="walk") is True
|
getEffectiveFlagFds(ds, "neutralize", _fdsAll, mode="walk") is True
|
||||||
|
|
@ -160,7 +160,7 @@ def registerFeatureSubAgentTools(registry: ToolRegistry, services):
|
||||||
# A2: build the per-table type/inheritance-aware neutralization policy.
|
# A2: build the per-table type/inheritance-aware neutralization policy.
|
||||||
# tableActive = effective (own or inherited) table-level neutralize flag;
|
# tableActive = effective (own or inherited) table-level neutralize flag;
|
||||||
# explicitFields = fields whose neutralize flag is set explicitly.
|
# explicitFields = fields whose neutralize flag is set explicitly.
|
||||||
from modules.serviceCenter.services.serviceKnowledge._inheritFlags import resolveEffectiveForFds
|
from modules.serviceCenter.core.flagResolution import resolveEffectiveForFds
|
||||||
neutralizePolicy: Dict[str, Dict[str, Any]] = {}
|
neutralizePolicy: Dict[str, Dict[str, Any]] = {}
|
||||||
for tblObj in selectedTables:
|
for tblObj in selectedTables:
|
||||||
tn = tblObj.get("meta", {}).get("table", "") if isinstance(tblObj, dict) else ""
|
tn = tblObj.get("meta", {}).get("table", "") if isinstance(tblObj, dict) else ""
|
||||||
|
|
|
||||||
|
|
@ -68,8 +68,8 @@ class ServicesBag:
|
||||||
self._context = context
|
self._context = context
|
||||||
self._getService = getService
|
self._getService = getService
|
||||||
self.user = context.user
|
self.user = context.user
|
||||||
self.mandateId = context.mandate_id
|
self.mandateId = context.mandateId
|
||||||
self.featureInstanceId = context.feature_instance_id
|
self.featureInstanceId = context.featureInstanceId
|
||||||
|
|
||||||
@property
|
@property
|
||||||
def workflow(self):
|
def workflow(self):
|
||||||
|
|
|
||||||
|
|
@ -42,10 +42,10 @@ class _ServicesAdapter:
|
||||||
Workflow is read from context dynamically so propagation updates are visible."""
|
Workflow is read from context dynamically so propagation updates are visible."""
|
||||||
def __init__(self, context, get_service: Callable[[str], Any]):
|
def __init__(self, context, get_service: Callable[[str], Any]):
|
||||||
self._context = context
|
self._context = context
|
||||||
self._get_service = get_service
|
self._getService = get_service
|
||||||
self.user = context.user
|
self.user = context.user
|
||||||
self.mandateId = context.mandate_id
|
self.mandateId = context.mandateId
|
||||||
self.featureInstanceId = context.feature_instance_id
|
self.featureInstanceId = context.featureInstanceId
|
||||||
|
|
||||||
@property
|
@property
|
||||||
def workflow(self):
|
def workflow(self):
|
||||||
|
|
@ -57,31 +57,31 @@ class _ServicesAdapter:
|
||||||
|
|
||||||
@property
|
@property
|
||||||
def chat(self):
|
def chat(self):
|
||||||
return self._get_service("chat")
|
return self._getService("chat")
|
||||||
|
|
||||||
@property
|
@property
|
||||||
def extraction(self):
|
def extraction(self):
|
||||||
return self._get_service("extraction")
|
return self._getService("extraction")
|
||||||
|
|
||||||
@property
|
@property
|
||||||
def utils(self):
|
def utils(self):
|
||||||
return self._get_service("utils")
|
return self._getService("utils")
|
||||||
|
|
||||||
@property
|
@property
|
||||||
def ai(self):
|
def ai(self):
|
||||||
return self._get_service("ai")
|
return self._getService("ai")
|
||||||
|
|
||||||
@property
|
@property
|
||||||
def interfaceDbChat(self):
|
def interfaceDbChat(self):
|
||||||
return self._get_service("chat").interfaceDbChat
|
return self._getService("chat").interfaceDbChat
|
||||||
|
|
||||||
@property
|
@property
|
||||||
def interfaceDbComponent(self):
|
def interfaceDbComponent(self):
|
||||||
return self._get_service("chat").interfaceDbComponent
|
return self._getService("chat").interfaceDbComponent
|
||||||
|
|
||||||
@property
|
@property
|
||||||
def featureCode(self) -> Optional[str]:
|
def featureCode(self) -> Optional[str]:
|
||||||
fc = getattr(self._context, "feature_code", None)
|
fc = getattr(self._context, "featureCode", None)
|
||||||
if fc and str(fc).strip():
|
if fc and str(fc).strip():
|
||||||
return str(fc).strip()
|
return str(fc).strip()
|
||||||
w = self.workflow
|
w = self.workflow
|
||||||
|
|
@ -102,11 +102,11 @@ class AiService:
|
||||||
"""Initialize with ServiceCenterContext and service resolver.
|
"""Initialize with ServiceCenterContext and service resolver.
|
||||||
|
|
||||||
Args:
|
Args:
|
||||||
context: ServiceCenterContext with user, mandate_id, feature_instance_id, workflow
|
context: ServiceCenterContext with user, mandateId, featureInstanceId, workflow
|
||||||
get_service: Callable to resolve dependency services by key
|
get_service: Callable to resolve dependency services by key
|
||||||
"""
|
"""
|
||||||
self.services = _ServicesAdapter(context, get_service)
|
self.services = _ServicesAdapter(context, get_service)
|
||||||
self._get_service = get_service
|
self._getService = get_service
|
||||||
self.aiObjects = None
|
self.aiObjects = None
|
||||||
self.extractionService = None
|
self.extractionService = None
|
||||||
|
|
||||||
|
|
@ -117,7 +117,7 @@ class AiService:
|
||||||
|
|
||||||
if self.extractionService is None:
|
if self.extractionService is None:
|
||||||
logger.info("Initializing ExtractionService via service center...")
|
logger.info("Initializing ExtractionService via service center...")
|
||||||
self.extractionService = self._get_service("extraction")
|
self.extractionService = self._getService("extraction")
|
||||||
|
|
||||||
# Initialize new submodules
|
# Initialize new submodules
|
||||||
from .subResponseParsing import ResponseParser
|
from .subResponseParsing import ResponseParser
|
||||||
|
|
@ -673,7 +673,7 @@ detectedIntent-Werte:
|
||||||
_sources = []
|
_sources = []
|
||||||
|
|
||||||
# Source 1: Feature-Instance config
|
# Source 1: Feature-Instance config
|
||||||
_neutralSvc = self._get_service("neutralization")
|
_neutralSvc = self._getService("neutralization")
|
||||||
if _neutralSvc and hasattr(_neutralSvc, 'getConfig'):
|
if _neutralSvc and hasattr(_neutralSvc, 'getConfig'):
|
||||||
_config = _neutralSvc.getConfig()
|
_config = _neutralSvc.getConfig()
|
||||||
if _config and getattr(_config, 'enabled', False):
|
if _config and getattr(_config, 'enabled', False):
|
||||||
|
|
@ -721,7 +721,7 @@ detectedIntent-Werte:
|
||||||
_hardMode = request.requireNeutralization is True
|
_hardMode = request.requireNeutralization is True
|
||||||
excludedDocs: List[str] = []
|
excludedDocs: List[str] = []
|
||||||
|
|
||||||
neutralSvc = self._get_service("neutralization")
|
neutralSvc = self._getService("neutralization")
|
||||||
if not neutralSvc or not hasattr(neutralSvc, 'processTextAsync'):
|
if not neutralSvc or not hasattr(neutralSvc, 'processTextAsync'):
|
||||||
if _hardMode:
|
if _hardMode:
|
||||||
raise RuntimeError("Neutralization explicitly required but service unavailable — AI call BLOCKED")
|
raise RuntimeError("Neutralization explicitly required but service unavailable — AI call BLOCKED")
|
||||||
|
|
@ -1193,7 +1193,7 @@ detectedIntent-Werte:
|
||||||
contentOut = getattr(response, 'content', None)
|
contentOut = getattr(response, 'content', None)
|
||||||
contentOutput = str(contentOut) if contentOut else None
|
contentOutput = str(contentOut) if contentOut else None
|
||||||
|
|
||||||
neutralSvc = self._get_service("neutralization") if wasNeutralized else None
|
neutralSvc = self._getService("neutralization") if wasNeutralized else None
|
||||||
mappingsCount = None
|
mappingsCount = None
|
||||||
if neutralSvc and hasattr(neutralSvc, 'getActiveMappingsCount'):
|
if neutralSvc and hasattr(neutralSvc, 'getActiveMappingsCount'):
|
||||||
try:
|
try:
|
||||||
|
|
@ -1324,8 +1324,8 @@ detectedIntent-Werte:
|
||||||
from modules.serviceCenter.context import ServiceCenterContext
|
from modules.serviceCenter.context import ServiceCenterContext
|
||||||
ctx = ServiceCenterContext(
|
ctx = ServiceCenterContext(
|
||||||
user=servicesHub.user,
|
user=servicesHub.user,
|
||||||
mandate_id=servicesHub.mandateId,
|
mandateId=servicesHub.mandateId,
|
||||||
feature_instance_id=servicesHub.featureInstanceId,
|
featureInstanceId=servicesHub.featureInstanceId,
|
||||||
workflow=getattr(servicesHub, "workflow", None),
|
workflow=getattr(servicesHub, "workflow", None),
|
||||||
)
|
)
|
||||||
return getService("ai", ctx)
|
return getService("ai", ctx)
|
||||||
|
|
@ -1721,7 +1721,7 @@ Respond with ONLY a JSON object in this exact format:
|
||||||
)
|
)
|
||||||
|
|
||||||
try:
|
try:
|
||||||
generationService = self._get_service("generation")
|
generationService = self._getService("generation")
|
||||||
|
|
||||||
# renderReport verarbeitet jetzt jedes Dokument einzeln
|
# renderReport verarbeitet jetzt jedes Dokument einzeln
|
||||||
# und gibt Liste von (documentData, mimeType, filename) zurück
|
# und gibt Liste von (documentData, mimeType, filename) zurück
|
||||||
|
|
|
||||||
|
|
@ -56,9 +56,9 @@ def getService(currentUser: User, mandateId: str, featureInstanceId: str = None,
|
||||||
return _billingServices[cacheKey]
|
return _billingServices[cacheKey]
|
||||||
|
|
||||||
|
|
||||||
def _get_feature_code_from_context(context) -> Optional[str]:
|
def _getFeatureCodeFromContext(context) -> Optional[str]:
|
||||||
"""Extract featureCode from ServiceCenterContext."""
|
"""Extract featureCode from ServiceCenterContext."""
|
||||||
explicit = getattr(context, "feature_code", None)
|
explicit = getattr(context, "featureCode", None)
|
||||||
if explicit and str(explicit).strip():
|
if explicit and str(explicit).strip():
|
||||||
return str(explicit).strip()
|
return str(explicit).strip()
|
||||||
if context.workflow and hasattr(context.workflow, "feature") and context.workflow.feature:
|
if context.workflow and hasattr(context.workflow, "feature") and context.workflow.feature:
|
||||||
|
|
@ -91,15 +91,15 @@ class BillingService:
|
||||||
ctx = context_or_user
|
ctx = context_or_user
|
||||||
get_service = mandateId
|
get_service = mandateId
|
||||||
self.currentUser = ctx.user
|
self.currentUser = ctx.user
|
||||||
self.mandateId = ctx.mandate_id or ""
|
self.mandateId = ctx.mandateId or ""
|
||||||
self.featureInstanceId = ctx.feature_instance_id
|
self.featureInstanceId = ctx.featureInstanceId
|
||||||
self.featureCode = _get_feature_code_from_context(ctx)
|
self.featureCode = _getFeatureCodeFromContext(ctx)
|
||||||
elif get_service is not None and hasattr(context_or_user, "user"):
|
elif get_service is not None and hasattr(context_or_user, "user"):
|
||||||
ctx = context_or_user
|
ctx = context_or_user
|
||||||
self.currentUser = ctx.user
|
self.currentUser = ctx.user
|
||||||
self.mandateId = ctx.mandate_id or ""
|
self.mandateId = ctx.mandateId or ""
|
||||||
self.featureInstanceId = ctx.feature_instance_id
|
self.featureInstanceId = ctx.featureInstanceId
|
||||||
self.featureCode = _get_feature_code_from_context(ctx)
|
self.featureCode = _getFeatureCodeFromContext(ctx)
|
||||||
else:
|
else:
|
||||||
self.currentUser = context_or_user
|
self.currentUser = context_or_user
|
||||||
self.mandateId = mandateId or ""
|
self.mandateId = mandateId or ""
|
||||||
|
|
|
||||||
|
|
@ -18,17 +18,17 @@ class ChatService:
|
||||||
def __init__(self, context, get_service: Callable[[str], Any]):
|
def __init__(self, context, get_service: Callable[[str], Any]):
|
||||||
"""Initialize with ServiceCenterContext and service resolver."""
|
"""Initialize with ServiceCenterContext and service resolver."""
|
||||||
self._context = context
|
self._context = context
|
||||||
self._get_service = get_service
|
self._getService = get_service
|
||||||
self.user = context.user
|
self.user = context.user
|
||||||
from modules.interfaces.interfaceDbApp import getInterface as getAppInterface
|
from modules.interfaces.interfaceDbApp import getInterface as getAppInterface
|
||||||
from modules.interfaces.interfaceDbManagement import getInterface as getComponentInterface
|
from modules.interfaces.interfaceDbManagement import getInterface as getComponentInterface
|
||||||
from modules.interfaces.interfaceDbChat import getInterface as getChatInterface
|
from modules.interfaces.interfaceDbChat import getInterface as getChatInterface
|
||||||
self.interfaceDbApp = getAppInterface(context.user, mandateId=context.mandate_id)
|
self.interfaceDbApp = getAppInterface(context.user, mandateId=context.mandateId)
|
||||||
self.interfaceDbComponent = getComponentInterface(context.user, mandateId=context.mandate_id, featureInstanceId=context.feature_instance_id)
|
self.interfaceDbComponent = getComponentInterface(context.user, mandateId=context.mandateId, featureInstanceId=context.featureInstanceId)
|
||||||
self.interfaceDbChat = getChatInterface(
|
self.interfaceDbChat = getChatInterface(
|
||||||
context.user,
|
context.user,
|
||||||
mandateId=context.mandate_id,
|
mandateId=context.mandateId,
|
||||||
featureInstanceId=context.feature_instance_id,
|
featureInstanceId=context.featureInstanceId,
|
||||||
)
|
)
|
||||||
self._progressLogger = None
|
self._progressLogger = None
|
||||||
|
|
||||||
|
|
@ -374,10 +374,10 @@ class ChatService:
|
||||||
try:
|
try:
|
||||||
# Get a fresh token via security service
|
# Get a fresh token via security service
|
||||||
logger.debug(f"Getting fresh token for connection {connection.id}")
|
logger.debug(f"Getting fresh token for connection {connection.id}")
|
||||||
token = self._get_service("security").getFreshToken(connection.id)
|
token = self._getService("security").getFreshToken(connection.id)
|
||||||
if token:
|
if token:
|
||||||
if hasattr(token, 'expiresAt') and token.expiresAt:
|
if hasattr(token, 'expiresAt') and token.expiresAt:
|
||||||
current_time = self._get_service("utils").timestampGetUtc()
|
current_time = self._getService("utils").timestampGetUtc()
|
||||||
if current_time > token.expiresAt:
|
if current_time > token.expiresAt:
|
||||||
token_status = "expired"
|
token_status = "expired"
|
||||||
else:
|
else:
|
||||||
|
|
@ -462,7 +462,7 @@ class ChatService:
|
||||||
Token object or None if not found/expired
|
Token object or None if not found/expired
|
||||||
"""
|
"""
|
||||||
try:
|
try:
|
||||||
return self._get_service("security").getFreshToken(connectionId)
|
return self._getService("security").getFreshToken(connectionId)
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
logger.error(f"Error getting fresh token for connection {connectionId}: {str(e)}")
|
logger.error(f"Error getting fresh token for connection {connectionId}: {str(e)}")
|
||||||
return None
|
return None
|
||||||
|
|
@ -575,8 +575,8 @@ class ChatService:
|
||||||
path=path,
|
path=path,
|
||||||
label=label,
|
label=label,
|
||||||
displayPath=displayPath,
|
displayPath=displayPath,
|
||||||
featureInstanceId=featureInstanceId or self._context.feature_instance_id or "",
|
featureInstanceId=featureInstanceId or self._context.featureInstanceId or "",
|
||||||
mandateId=self._context.mandate_id or "",
|
mandateId=self._context.mandateId or "",
|
||||||
userId=self.user.id if self.user else "",
|
userId=self.user.id if self.user else "",
|
||||||
)
|
)
|
||||||
return self.interfaceDbApp.db.recordCreate(DataSource, ds)
|
return self.interfaceDbApp.db.recordCreate(DataSource, ds)
|
||||||
|
|
|
||||||
|
|
@ -30,7 +30,7 @@ class ClickupService(ClickupApiClient):
|
||||||
def __init__(self, context, get_service: Callable[[str], Any]):
|
def __init__(self, context, get_service: Callable[[str], Any]):
|
||||||
super().__init__(accessToken="")
|
super().__init__(accessToken="")
|
||||||
self._context = context
|
self._context = context
|
||||||
self._get_service = get_service
|
self._getService = get_service
|
||||||
|
|
||||||
def setAccessTokenFromConnection(self, userConnection) -> bool:
|
def setAccessTokenFromConnection(self, userConnection) -> bool:
|
||||||
"""Load OAuth/personal token from SecurityService for this UserConnection."""
|
"""Load OAuth/personal token from SecurityService for this UserConnection."""
|
||||||
|
|
@ -45,7 +45,7 @@ class ClickupService(ClickupApiClient):
|
||||||
if not connection_id:
|
if not connection_id:
|
||||||
logger.error("UserConnection must have an 'id' field")
|
logger.error("UserConnection must have an 'id' field")
|
||||||
return False
|
return False
|
||||||
security = self._get_service("security")
|
security = self._getService("security")
|
||||||
if not security:
|
if not security:
|
||||||
logger.error("Security service not available for token access")
|
logger.error("Security service not available for token access")
|
||||||
return False
|
return False
|
||||||
|
|
|
||||||
|
|
@ -28,12 +28,12 @@ class ExtractionService:
|
||||||
def __init__(self, context, get_service: Callable[[str], Any]):
|
def __init__(self, context, get_service: Callable[[str], Any]):
|
||||||
"""Initialize with ServiceCenterContext and service resolver."""
|
"""Initialize with ServiceCenterContext and service resolver."""
|
||||||
self._context = context
|
self._context = context
|
||||||
self._get_service = get_service
|
self._getService = get_service
|
||||||
from modules.interfaces.interfaceDbManagement import getInterface as getComponentInterface
|
from modules.interfaces.interfaceDbManagement import getInterface as getComponentInterface
|
||||||
self._interfaceDbComponent = getComponentInterface(
|
self._interfaceDbComponent = getComponentInterface(
|
||||||
context.user,
|
context.user,
|
||||||
mandateId=context.mandate_id,
|
mandateId=context.mandateId,
|
||||||
featureInstanceId=context.feature_instance_id,
|
featureInstanceId=context.featureInstanceId,
|
||||||
)
|
)
|
||||||
self._extractorRegistry = getExtractorRegistry()
|
self._extractorRegistry = getExtractorRegistry()
|
||||||
if ExtractionService._sharedChunkerRegistry is None:
|
if ExtractionService._sharedChunkerRegistry is None:
|
||||||
|
|
@ -117,7 +117,7 @@ class ExtractionService:
|
||||||
docOperationId = f"{operationId}_doc_{i}"
|
docOperationId = f"{operationId}_doc_{i}"
|
||||||
# Use parentOperationId if provided, otherwise use operationId as parent
|
# Use parentOperationId if provided, otherwise use operationId as parent
|
||||||
parentId = parentOperationId if parentOperationId else operationId
|
parentId = parentOperationId if parentOperationId else operationId
|
||||||
self._get_service("chat").progressLogStart(
|
self._getService("chat").progressLogStart(
|
||||||
docOperationId,
|
docOperationId,
|
||||||
"Extracting Document",
|
"Extracting Document",
|
||||||
f"Document {i + 1}/{totalDocs}",
|
f"Document {i + 1}/{totalDocs}",
|
||||||
|
|
@ -130,17 +130,17 @@ class ExtractionService:
|
||||||
|
|
||||||
try:
|
try:
|
||||||
if docOperationId:
|
if docOperationId:
|
||||||
self._get_service("chat").progressLogUpdate(docOperationId, 0.1, "Loading document data")
|
self._getService("chat").progressLogUpdate(docOperationId, 0.1, "Loading document data")
|
||||||
|
|
||||||
# Resolve raw bytes for this document using interface
|
# Resolve raw bytes for this document using interface
|
||||||
documentBytes = dbInterface.getFileData(doc.fileId)
|
documentBytes = dbInterface.getFileData(doc.fileId)
|
||||||
if not documentBytes:
|
if not documentBytes:
|
||||||
if docOperationId:
|
if docOperationId:
|
||||||
self._get_service("chat").progressLogFinish(docOperationId, False)
|
self._getService("chat").progressLogFinish(docOperationId, False)
|
||||||
raise ValueError(f"No file data found for fileId={doc.fileId}")
|
raise ValueError(f"No file data found for fileId={doc.fileId}")
|
||||||
|
|
||||||
if docOperationId:
|
if docOperationId:
|
||||||
self._get_service("chat").progressLogUpdate(docOperationId, 0.2, "Running extraction pipeline")
|
self._getService("chat").progressLogUpdate(docOperationId, 0.2, "Running extraction pipeline")
|
||||||
|
|
||||||
# Convert ChatDocument to the format expected by runExtraction
|
# Convert ChatDocument to the format expected by runExtraction
|
||||||
documentData = {
|
documentData = {
|
||||||
|
|
@ -160,7 +160,7 @@ class ExtractionService:
|
||||||
)
|
)
|
||||||
|
|
||||||
if docOperationId:
|
if docOperationId:
|
||||||
self._get_service("chat").progressLogUpdate(docOperationId, 0.7, f"Extracted {len(ec.parts)} parts")
|
self._getService("chat").progressLogUpdate(docOperationId, 0.7, f"Extracted {len(ec.parts)} parts")
|
||||||
|
|
||||||
# Log content parts metadata
|
# Log content parts metadata
|
||||||
logger.debug(f"Content parts: {len(ec.parts)}")
|
logger.debug(f"Content parts: {len(ec.parts)}")
|
||||||
|
|
@ -223,7 +223,7 @@ class ExtractionService:
|
||||||
# Use document name and part index for filename
|
# Use document name and part index for filename
|
||||||
doc_name_safe = documentData["fileName"].replace(" ", "_").replace("/", "_").replace("\\", "_")[:50]
|
doc_name_safe = documentData["fileName"].replace(" ", "_").replace("/", "_").replace("\\", "_")[:50]
|
||||||
debug_filename = f"extraction_text_part_{j+1}_{doc_name_safe}.txt"
|
debug_filename = f"extraction_text_part_{j+1}_{doc_name_safe}.txt"
|
||||||
self._get_service("utils").writeDebugFile(debug_json, debug_filename)
|
self._getService("utils").writeDebugFile(debug_json, debug_filename)
|
||||||
logger.info(f"Wrote debug file for extracted text part {j+1}/{len(ec.parts)}: {debug_filename}")
|
logger.info(f"Wrote debug file for extracted text part {j+1}/{len(ec.parts)}: {debug_filename}")
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
logger.warning(f"Failed to write debug file for text part {j+1}: {str(e)}")
|
logger.warning(f"Failed to write debug file for text part {j+1}: {str(e)}")
|
||||||
|
|
@ -240,7 +240,7 @@ class ExtractionService:
|
||||||
logger.debug(f"No chunking needed - {len(ec.parts)} parts fit within size limits")
|
logger.debug(f"No chunking needed - {len(ec.parts)} parts fit within size limits")
|
||||||
|
|
||||||
if docOperationId:
|
if docOperationId:
|
||||||
self._get_service("chat").progressLogUpdate(docOperationId, 0.9, f"Processing complete: {len(ec.parts)} parts extracted")
|
self._getService("chat").progressLogUpdate(docOperationId, 0.9, f"Processing complete: {len(ec.parts)} parts extracted")
|
||||||
|
|
||||||
# Calculate timing and emit stats
|
# Calculate timing and emit stats
|
||||||
endTime = time.time()
|
endTime = time.time()
|
||||||
|
|
@ -256,7 +256,7 @@ class ExtractionService:
|
||||||
# Hard fail if model is missing; caller must ensure connectors are registered
|
# Hard fail if model is missing; caller must ensure connectors are registered
|
||||||
if model is None or model.calculatepriceCHF is None:
|
if model is None or model.calculatepriceCHF is None:
|
||||||
if docOperationId:
|
if docOperationId:
|
||||||
self._get_service("chat").progressLogFinish(docOperationId, False)
|
self._getService("chat").progressLogFinish(docOperationId, False)
|
||||||
raise RuntimeError(f"Pricing model not available: {modelDisplayName}")
|
raise RuntimeError(f"Pricing model not available: {modelDisplayName}")
|
||||||
priceCHF = model.calculatepriceCHF(processingTime, bytesSent, bytesReceived)
|
priceCHF = model.calculatepriceCHF(processingTime, bytesSent, bytesReceived)
|
||||||
|
|
||||||
|
|
@ -309,13 +309,13 @@ class ExtractionService:
|
||||||
|
|
||||||
# Finish document operation successfully
|
# Finish document operation successfully
|
||||||
if docOperationId:
|
if docOperationId:
|
||||||
self._get_service("chat").progressLogFinish(docOperationId, True)
|
self._getService("chat").progressLogFinish(docOperationId, True)
|
||||||
|
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
logger.error(f"Error extracting content from document {i + 1}/{totalDocs} ({doc.fileName}): {str(e)}")
|
logger.error(f"Error extracting content from document {i + 1}/{totalDocs} ({doc.fileName}): {str(e)}")
|
||||||
if docOperationId:
|
if docOperationId:
|
||||||
try:
|
try:
|
||||||
self._get_service("chat").progressLogFinish(docOperationId, False)
|
self._getService("chat").progressLogFinish(docOperationId, False)
|
||||||
except:
|
except:
|
||||||
pass # Don't fail on progress logging errors
|
pass # Don't fail on progress logging errors
|
||||||
# Continue with next document instead of failing completely
|
# Continue with next document instead of failing completely
|
||||||
|
|
@ -355,7 +355,7 @@ class ExtractionService:
|
||||||
if not operationId:
|
if not operationId:
|
||||||
workflowId = self._context.workflow.id if self._context.workflow else f"no-workflow-{int(time.time())}"
|
workflowId = self._context.workflow.id if self._context.workflow else f"no-workflow-{int(time.time())}"
|
||||||
operationId = f"ai_text_extract_{workflowId}_{int(time.time())}"
|
operationId = f"ai_text_extract_{workflowId}_{int(time.time())}"
|
||||||
self._get_service("chat").progressLogStart(
|
self._getService("chat").progressLogStart(
|
||||||
operationId,
|
operationId,
|
||||||
"AI Text Extract",
|
"AI Text Extract",
|
||||||
"Document Processing",
|
"Document Processing",
|
||||||
|
|
@ -383,19 +383,19 @@ class ExtractionService:
|
||||||
|
|
||||||
# Extract content WITHOUT chunking
|
# Extract content WITHOUT chunking
|
||||||
if operationId:
|
if operationId:
|
||||||
self._get_service("chat").progressLogUpdate(operationId, 0.1, f"Extracting content from {len(documents)} documents")
|
self._getService("chat").progressLogUpdate(operationId, 0.1, f"Extracting content from {len(documents)} documents")
|
||||||
# Pass operationId as parentOperationId for hierarchical logging
|
# Pass operationId as parentOperationId for hierarchical logging
|
||||||
# Correct hierarchy: parentOperationId -> operationId -> docOperationId
|
# Correct hierarchy: parentOperationId -> operationId -> docOperationId
|
||||||
extractionResult = self.extractContent(documents, extractionOptions, operationId=operationId, parentOperationId=operationId)
|
extractionResult = self.extractContent(documents, extractionOptions, operationId=operationId, parentOperationId=operationId)
|
||||||
|
|
||||||
if not isinstance(extractionResult, list):
|
if not isinstance(extractionResult, list):
|
||||||
if operationId:
|
if operationId:
|
||||||
self._get_service("chat").progressLogFinish(operationId, False)
|
self._getService("chat").progressLogFinish(operationId, False)
|
||||||
return "[Error: No extraction results]"
|
return "[Error: No extraction results]"
|
||||||
|
|
||||||
# Process parts (not chunks) with model-aware AI calls
|
# Process parts (not chunks) with model-aware AI calls
|
||||||
if operationId:
|
if operationId:
|
||||||
self._get_service("chat").progressLogUpdate(operationId, 0.3, f"Processing {len(extractionResult)} extracted content parts")
|
self._getService("chat").progressLogUpdate(operationId, 0.3, f"Processing {len(extractionResult)} extracted content parts")
|
||||||
# Use operationId as parentOperationId for child operations
|
# Use operationId as parentOperationId for child operations
|
||||||
# Correct hierarchy: parentOperationId -> operationId -> partOperationId
|
# Correct hierarchy: parentOperationId -> operationId -> partOperationId
|
||||||
processParentOperationId = operationId
|
processParentOperationId = operationId
|
||||||
|
|
@ -403,20 +403,20 @@ class ExtractionService:
|
||||||
|
|
||||||
# Merge results using existing merging system
|
# Merge results using existing merging system
|
||||||
if operationId:
|
if operationId:
|
||||||
self._get_service("chat").progressLogUpdate(operationId, 0.9, f"Merging {len(partResults)} part results")
|
self._getService("chat").progressLogUpdate(operationId, 0.9, f"Merging {len(partResults)} part results")
|
||||||
mergedContent = self.mergePartResults(partResults, options)
|
mergedContent = self.mergePartResults(partResults, options)
|
||||||
|
|
||||||
# Save merged extraction content to debug
|
# Save merged extraction content to debug
|
||||||
self._get_service("utils").writeDebugFile(mergedContent or '', "extraction_merged_text")
|
self._getService("utils").writeDebugFile(mergedContent or '', "extraction_merged_text")
|
||||||
|
|
||||||
if operationId:
|
if operationId:
|
||||||
self._get_service("chat").progressLogFinish(operationId, True)
|
self._getService("chat").progressLogFinish(operationId, True)
|
||||||
|
|
||||||
return mergedContent
|
return mergedContent
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
logger.error(f"Error in processDocumentsPerChunk: {str(e)}")
|
logger.error(f"Error in processDocumentsPerChunk: {str(e)}")
|
||||||
if operationId:
|
if operationId:
|
||||||
self._get_service("chat").progressLogFinish(operationId, False)
|
self._getService("chat").progressLogFinish(operationId, False)
|
||||||
raise
|
raise
|
||||||
|
|
||||||
async def _processPartsWithMapping(
|
async def _processPartsWithMapping(
|
||||||
|
|
@ -468,7 +468,7 @@ class ExtractionService:
|
||||||
if operationId:
|
if operationId:
|
||||||
workflowId = self._context.workflow.id if self._context.workflow else f"no-workflow-{int(time.time())}"
|
workflowId = self._context.workflow.id if self._context.workflow else f"no-workflow-{int(time.time())}"
|
||||||
partOperationId = f"{operationId}_part_{part_index}"
|
partOperationId = f"{operationId}_part_{part_index}"
|
||||||
self._get_service("chat").progressLogStart(
|
self._getService("chat").progressLogStart(
|
||||||
partOperationId,
|
partOperationId,
|
||||||
"Content Processing",
|
"Content Processing",
|
||||||
f"Part {part_index + 1}",
|
f"Part {part_index + 1}",
|
||||||
|
|
@ -487,15 +487,15 @@ class ExtractionService:
|
||||||
|
|
||||||
# Update progress - initiating
|
# Update progress - initiating
|
||||||
if partOperationId:
|
if partOperationId:
|
||||||
self._get_service("chat").progressLogUpdate(partOperationId, 0.3, "Initiating")
|
self._getService("chat").progressLogUpdate(partOperationId, 0.3, "Initiating")
|
||||||
|
|
||||||
# Call AI with model-aware chunking (no progress callback - handled by parent operation)
|
# Call AI with model-aware chunking (no progress callback - handled by parent operation)
|
||||||
response = await aiObjects.call(request)
|
response = await aiObjects.call(request)
|
||||||
|
|
||||||
# Update progress - completed
|
# Update progress - completed
|
||||||
if partOperationId:
|
if partOperationId:
|
||||||
self._get_service("chat").progressLogUpdate(partOperationId, 0.9, "Completed")
|
self._getService("chat").progressLogUpdate(partOperationId, 0.9, "Completed")
|
||||||
self._get_service("chat").progressLogFinish(partOperationId, True)
|
self._getService("chat").progressLogFinish(partOperationId, True)
|
||||||
|
|
||||||
processing_time = time.time() - start_time
|
processing_time = time.time() - start_time
|
||||||
|
|
||||||
|
|
@ -1133,7 +1133,7 @@ class ExtractionService:
|
||||||
"perPartExtractedData": per_part_extracted_data
|
"perPartExtractedData": per_part_extracted_data
|
||||||
}
|
}
|
||||||
debug_json = json.dumps(debug_content, indent=2, ensure_ascii=False)
|
debug_json = json.dumps(debug_content, indent=2, ensure_ascii=False)
|
||||||
self._get_service("utils").writeDebugFile(debug_json, "content_extraction_per_part")
|
self._getService("utils").writeDebugFile(debug_json, "content_extraction_per_part")
|
||||||
logger.info(f"Wrote per-part extracted data to debug file: {len(per_part_extracted_data)} blocks from {len(content_parts)} content parts")
|
logger.info(f"Wrote per-part extracted data to debug file: {len(per_part_extracted_data)} blocks from {len(content_parts)} content parts")
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
logger.warning(f"Failed to write per-part extracted data to debug file: {str(e)}")
|
logger.warning(f"Failed to write per-part extracted data to debug file: {str(e)}")
|
||||||
|
|
@ -1172,7 +1172,7 @@ class ExtractionService:
|
||||||
extraction_result_format["parts"].append(formatted_part)
|
extraction_result_format["parts"].append(formatted_part)
|
||||||
|
|
||||||
result_json = json.dumps(extraction_result_format, indent=2, ensure_ascii=False)
|
result_json = json.dumps(extraction_result_format, indent=2, ensure_ascii=False)
|
||||||
self._get_service("utils").writeDebugFile(result_json, "content_extraction_original_parts")
|
self._getService("utils").writeDebugFile(result_json, "content_extraction_original_parts")
|
||||||
logger.info(f"Wrote original parts extracted data to debug file: {len(original_parts_extracted_data)} original parts")
|
logger.info(f"Wrote original parts extracted data to debug file: {len(original_parts_extracted_data)} original parts")
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
logger.warning(f"Failed to write original parts extracted data to debug file: {str(e)}")
|
logger.warning(f"Failed to write original parts extracted data to debug file: {str(e)}")
|
||||||
|
|
@ -1764,11 +1764,11 @@ class ExtractionService:
|
||||||
debugPrefix = f"generation_contentPart_{partId}_{partLabelSafe}"
|
debugPrefix = f"generation_contentPart_{partId}_{partLabelSafe}"
|
||||||
|
|
||||||
# Write prompt
|
# Write prompt
|
||||||
self._get_service("utils").writeDebugFile(prompt, f"{debugPrefix}_prompt")
|
self._getService("utils").writeDebugFile(prompt, f"{debugPrefix}_prompt")
|
||||||
|
|
||||||
# Write response
|
# Write response
|
||||||
responseContent = partResult.content if partResult.content else ""
|
responseContent = partResult.content if partResult.content else ""
|
||||||
self._get_service("utils").writeDebugFile(responseContent, f"{debugPrefix}_response")
|
self._getService("utils").writeDebugFile(responseContent, f"{debugPrefix}_response")
|
||||||
|
|
||||||
logger.debug(f"Wrote debug files for contentPart {partId} (generation): {debugPrefix}_prompt, {debugPrefix}_response")
|
logger.debug(f"Wrote debug files for contentPart {partId} (generation): {debugPrefix}_prompt, {debugPrefix}_response")
|
||||||
except Exception as debugError:
|
except Exception as debugError:
|
||||||
|
|
|
||||||
|
|
@ -10,13 +10,7 @@ import logging
|
||||||
from typing import Dict, Any, Optional
|
from typing import Dict, Any, Optional
|
||||||
from modules.datamodels.datamodelAi import AiCallRequest, AiCallOptions, OperationTypeEnum
|
from modules.datamodels.datamodelAi import AiCallRequest, AiCallOptions, OperationTypeEnum
|
||||||
|
|
||||||
# Type hint for renderer parameter
|
from modules.serviceCenter.core.types import RendererProtocol
|
||||||
from typing import TYPE_CHECKING
|
|
||||||
if TYPE_CHECKING:
|
|
||||||
from modules.serviceCenter.services.serviceGeneration.renderers.documentRendererBaseTemplate import BaseRenderer
|
|
||||||
_RendererLike = BaseRenderer
|
|
||||||
else:
|
|
||||||
_RendererLike = Any
|
|
||||||
|
|
||||||
logger = logging.getLogger(__name__)
|
logger = logging.getLogger(__name__)
|
||||||
|
|
||||||
|
|
@ -27,7 +21,7 @@ async def buildExtractionPrompt(
|
||||||
title: str,
|
title: str,
|
||||||
aiService=None,
|
aiService=None,
|
||||||
services=None,
|
services=None,
|
||||||
renderer: _RendererLike = None
|
renderer: Optional[RendererProtocol] = None
|
||||||
) -> str:
|
) -> str:
|
||||||
"""
|
"""
|
||||||
Build unified extraction prompt for extracting content from documents.
|
Build unified extraction prompt for extracting content from documents.
|
||||||
|
|
|
||||||
|
|
@ -26,10 +26,10 @@ class _ServicesAdapter:
|
||||||
Workflow is read from context dynamically so propagation updates are visible."""
|
Workflow is read from context dynamically so propagation updates are visible."""
|
||||||
def __init__(self, context, get_service: Callable[[str], Any]):
|
def __init__(self, context, get_service: Callable[[str], Any]):
|
||||||
self._context = context
|
self._context = context
|
||||||
self._get_service = get_service
|
self._getService = get_service
|
||||||
self.user = context.user
|
self.user = context.user
|
||||||
self.mandateId = context.mandate_id
|
self.mandateId = context.mandateId
|
||||||
self.featureInstanceId = context.feature_instance_id
|
self.featureInstanceId = context.featureInstanceId
|
||||||
chat = get_service("chat")
|
chat = get_service("chat")
|
||||||
self.interfaceDbChat = chat.interfaceDbChat
|
self.interfaceDbChat = chat.interfaceDbChat
|
||||||
|
|
||||||
|
|
@ -39,22 +39,22 @@ class _ServicesAdapter:
|
||||||
|
|
||||||
@property
|
@property
|
||||||
def chat(self):
|
def chat(self):
|
||||||
return self._get_service("chat")
|
return self._getService("chat")
|
||||||
|
|
||||||
@property
|
@property
|
||||||
def utils(self):
|
def utils(self):
|
||||||
return self._get_service("utils")
|
return self._getService("utils")
|
||||||
|
|
||||||
@property
|
@property
|
||||||
def ai(self):
|
def ai(self):
|
||||||
return self._get_service("ai")
|
return self._getService("ai")
|
||||||
|
|
||||||
|
|
||||||
class GenerationService:
|
class GenerationService:
|
||||||
def __init__(self, context, get_service: Callable[[str], Any]):
|
def __init__(self, context, get_service: Callable[[str], Any]):
|
||||||
"""Initialize with ServiceCenterContext and service resolver."""
|
"""Initialize with ServiceCenterContext and service resolver."""
|
||||||
self.services = _ServicesAdapter(context, get_service)
|
self.services = _ServicesAdapter(context, get_service)
|
||||||
self._get_service = get_service
|
self._getService = get_service
|
||||||
self.interfaceDbChat = self.services.interfaceDbChat
|
self.interfaceDbChat = self.services.interfaceDbChat
|
||||||
|
|
||||||
def processActionResultDocuments(self, actionResult, action) -> List[Dict[str, Any]]:
|
def processActionResultDocuments(self, actionResult, action) -> List[Dict[str, Any]]:
|
||||||
|
|
|
||||||
|
|
@ -112,7 +112,7 @@ def _findDsRecord(
|
||||||
sourceType: str,
|
sourceType: str,
|
||||||
path: str,
|
path: str,
|
||||||
) -> Optional[Dict[str, Any]]:
|
) -> Optional[Dict[str, Any]]:
|
||||||
from modules.serviceCenter.services.serviceKnowledge._inheritFlags import normalisePath
|
from modules.serviceCenter.core.flagResolution import normalisePath
|
||||||
norm = normalisePath(path)
|
norm = normalisePath(path)
|
||||||
for ds in allDs:
|
for ds in allDs:
|
||||||
if (
|
if (
|
||||||
|
|
@ -191,8 +191,8 @@ def _personalRootChildrenNodes(
|
||||||
mandateId = getattr(context, "mandateId", "") or ""
|
mandateId = getattr(context, "mandateId", "") or ""
|
||||||
ctx = ServiceCenterContext(
|
ctx = ServiceCenterContext(
|
||||||
user=context.user,
|
user=context.user,
|
||||||
mandate_id=mandateId,
|
mandateId=mandateId,
|
||||||
feature_instance_id="",
|
featureInstanceId="",
|
||||||
)
|
)
|
||||||
chatService = getService("chat", ctx)
|
chatService = getService("chat", ctx)
|
||||||
connections = chatService.getUserConnections() or []
|
connections = chatService.getUserConnections() or []
|
||||||
|
|
@ -295,8 +295,8 @@ async def _connectionServiceNodes(
|
||||||
mandateId = getattr(context, "mandateId", "") or ""
|
mandateId = getattr(context, "mandateId", "") or ""
|
||||||
ctx = ServiceCenterContext(
|
ctx = ServiceCenterContext(
|
||||||
user=context.user,
|
user=context.user,
|
||||||
mandate_id=mandateId,
|
mandateId=mandateId,
|
||||||
feature_instance_id=instanceId,
|
featureInstanceId=instanceId,
|
||||||
)
|
)
|
||||||
chatService = getService("chat", ctx)
|
chatService = getService("chat", ctx)
|
||||||
securityService = getService("security", ctx)
|
securityService = getService("security", ctx)
|
||||||
|
|
@ -347,8 +347,8 @@ async def _browseChildNodes(
|
||||||
mandateId = getattr(context, "mandateId", "") or ""
|
mandateId = getattr(context, "mandateId", "") or ""
|
||||||
ctx = ServiceCenterContext(
|
ctx = ServiceCenterContext(
|
||||||
user=context.user,
|
user=context.user,
|
||||||
mandate_id=mandateId,
|
mandateId=mandateId,
|
||||||
feature_instance_id=instanceId,
|
featureInstanceId=instanceId,
|
||||||
)
|
)
|
||||||
chatService = getService("chat", ctx)
|
chatService = getService("chat", ctx)
|
||||||
securityService = getService("security", ctx)
|
securityService = getService("security", ctx)
|
||||||
|
|
@ -683,9 +683,9 @@ def _callerInstanceId(context: Any) -> str:
|
||||||
"""The UDB is feature-agnostic, but `_browseChildNodes` and
|
"""The UDB is feature-agnostic, but `_browseChildNodes` and
|
||||||
`_connectionServiceNodes` need a feature instance id for the
|
`_connectionServiceNodes` need a feature instance id for the
|
||||||
ServiceCenterContext (the underlying connector resolver wants one).
|
ServiceCenterContext (the underlying connector resolver wants one).
|
||||||
Use the caller's current feature_instance_id (workspace) when
|
Use the caller's current featureInstanceId (workspace) when
|
||||||
available, else an empty string. The id is NOT used for FDS scoping."""
|
available, else an empty string. The id is NOT used for FDS scoping."""
|
||||||
fid = getattr(context, "feature_instance_id", None) or getattr(context, "featureInstanceId", None)
|
fid = getattr(context, "featureInstanceId", None)
|
||||||
return str(fid) if fid else ""
|
return str(fid) if fid else ""
|
||||||
|
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -926,7 +926,7 @@ class KnowledgeService:
|
||||||
contentObjectId=f"page-{pageIdx}",
|
contentObjectId=f"page-{pageIdx}",
|
||||||
fileId=fileId,
|
fileId=fileId,
|
||||||
userId=self._context.user.id if self._context.user else "",
|
userId=self._context.user.id if self._context.user else "",
|
||||||
featureInstanceId=self._context.feature_instance_id or "",
|
featureInstanceId=self._context.featureInstanceId or "",
|
||||||
contentType="text",
|
contentType="text",
|
||||||
data=text,
|
data=text,
|
||||||
contextRef={
|
contextRef={
|
||||||
|
|
|
||||||
|
|
@ -172,7 +172,7 @@ def _loadRagEnabledDataSources(connectionId: str, dataSourceIds: Optional[list]
|
||||||
"""
|
"""
|
||||||
from modules.interfaces.interfaceDbApp import getRootInterface
|
from modules.interfaces.interfaceDbApp import getRootInterface
|
||||||
from modules.datamodels.datamodelDataSource import DataSource
|
from modules.datamodels.datamodelDataSource import DataSource
|
||||||
from modules.serviceCenter.services.serviceKnowledge._inheritFlags import getEffectiveFlag
|
from modules.serviceCenter.core.flagResolution import getEffectiveFlag
|
||||||
|
|
||||||
rootIf = getRootInterface()
|
rootIf = getRootInterface()
|
||||||
allDs = rootIf.db.getRecordset(DataSource, recordFilter={"connectionId": connectionId})
|
allDs = rootIf.db.getRecordset(DataSource, recordFilter={"connectionId": connectionId})
|
||||||
|
|
|
||||||
|
|
@ -314,7 +314,7 @@ async def _resolveDependencies(connectionId: str):
|
||||||
rootUser = getRootUser()
|
rootUser = getRootUser()
|
||||||
ctx = ServiceCenterContext(
|
ctx = ServiceCenterContext(
|
||||||
user=rootUser,
|
user=rootUser,
|
||||||
mandate_id=str(getattr(connection, "mandateId", "") or ""),
|
mandateId=str(getattr(connection, "mandateId", "") or ""),
|
||||||
)
|
)
|
||||||
knowledgeService = getService("knowledge", ctx)
|
knowledgeService = getService("knowledge", ctx)
|
||||||
return adapter, connection, knowledgeService
|
return adapter, connection, knowledgeService
|
||||||
|
|
|
||||||
|
|
@ -244,7 +244,7 @@ async def _resolveDependencies(connectionId: str):
|
||||||
rootUser = getRootUser()
|
rootUser = getRootUser()
|
||||||
ctx = ServiceCenterContext(
|
ctx = ServiceCenterContext(
|
||||||
user=rootUser,
|
user=rootUser,
|
||||||
mandate_id=str(getattr(connection, "mandateId", "") or ""),
|
mandateId=str(getattr(connection, "mandateId", "") or ""),
|
||||||
)
|
)
|
||||||
knowledgeService = getService("knowledge", ctx)
|
knowledgeService = getService("knowledge", ctx)
|
||||||
return adapter, connection, knowledgeService
|
return adapter, connection, knowledgeService
|
||||||
|
|
|
||||||
|
|
@ -297,7 +297,7 @@ async def _resolveDependencies(connectionId: str):
|
||||||
rootUser = getRootUser()
|
rootUser = getRootUser()
|
||||||
ctx = ServiceCenterContext(
|
ctx = ServiceCenterContext(
|
||||||
user=rootUser,
|
user=rootUser,
|
||||||
mandate_id=str(getattr(connection, "mandateId", "") or ""),
|
mandateId=str(getattr(connection, "mandateId", "") or ""),
|
||||||
)
|
)
|
||||||
knowledgeService = getService("knowledge", ctx)
|
knowledgeService = getService("knowledge", ctx)
|
||||||
return adapter, connection, knowledgeService
|
return adapter, connection, knowledgeService
|
||||||
|
|
|
||||||
|
|
@ -211,7 +211,7 @@ async def _resolveDependencies(connectionId: str):
|
||||||
rootUser = getRootUser()
|
rootUser = getRootUser()
|
||||||
ctx = ServiceCenterContext(
|
ctx = ServiceCenterContext(
|
||||||
user=rootUser,
|
user=rootUser,
|
||||||
mandate_id=str(getattr(connection, "mandateId", "") or ""),
|
mandateId=str(getattr(connection, "mandateId", "") or ""),
|
||||||
)
|
)
|
||||||
knowledgeService = getService("knowledge", ctx)
|
knowledgeService = getService("knowledge", ctx)
|
||||||
return adapter, connection, knowledgeService
|
return adapter, connection, knowledgeService
|
||||||
|
|
|
||||||
|
|
@ -256,7 +256,7 @@ async def _resolveDependencies(connectionId: str):
|
||||||
rootUser = getRootUser()
|
rootUser = getRootUser()
|
||||||
ctx = ServiceCenterContext(
|
ctx = ServiceCenterContext(
|
||||||
user=rootUser,
|
user=rootUser,
|
||||||
mandate_id=str(getattr(connection, "mandateId", "") or ""),
|
mandateId=str(getattr(connection, "mandateId", "") or ""),
|
||||||
)
|
)
|
||||||
knowledgeService = getService("knowledge", ctx)
|
knowledgeService = getService("knowledge", ctx)
|
||||||
return adapter, connection, knowledgeService
|
return adapter, connection, knowledgeService
|
||||||
|
|
|
||||||
|
|
@ -245,7 +245,7 @@ async def _resolveDependencies(connectionId: str):
|
||||||
rootUser = getRootUser()
|
rootUser = getRootUser()
|
||||||
ctx = ServiceCenterContext(
|
ctx = ServiceCenterContext(
|
||||||
user=rootUser,
|
user=rootUser,
|
||||||
mandate_id=str(getattr(connection, "mandateId", "") or ""),
|
mandateId=str(getattr(connection, "mandateId", "") or ""),
|
||||||
)
|
)
|
||||||
knowledgeService = getService("knowledge", ctx)
|
knowledgeService = getService("knowledge", ctx)
|
||||||
return adapter, connection, knowledgeService
|
return adapter, connection, knowledgeService
|
||||||
|
|
|
||||||
|
|
@ -30,7 +30,7 @@ def _loadRagEnabledFds(featureInstanceId: str, featureDataSourceIds: Optional[Li
|
||||||
"""
|
"""
|
||||||
from modules.interfaces.interfaceDbApp import getRootInterface
|
from modules.interfaces.interfaceDbApp import getRootInterface
|
||||||
from modules.datamodels.datamodelFeatures import FeatureDataSource
|
from modules.datamodels.datamodelFeatures import FeatureDataSource
|
||||||
from modules.serviceCenter.services.serviceKnowledge._inheritFlags import getEffectiveFlagFds
|
from modules.serviceCenter.core.flagResolution import getEffectiveFlagFds
|
||||||
|
|
||||||
rootIf = getRootInterface()
|
rootIf = getRootInterface()
|
||||||
allFds = rootIf.db.getRecordset(
|
allFds = rootIf.db.getRecordset(
|
||||||
|
|
@ -118,7 +118,7 @@ async def _featureBootstrapHandler(
|
||||||
)
|
)
|
||||||
return {"featureInstanceId": featureInstanceId, "skipped": True, "reason": "no_rag_enabled_fds"}
|
return {"featureInstanceId": featureInstanceId, "skipped": True, "reason": "no_rag_enabled_fds"}
|
||||||
|
|
||||||
from modules.serviceCenter.services.serviceAgent.featureDataProvider import FeatureDataProvider
|
from modules.serviceCenter.core.types import createFeatureDataProvider
|
||||||
from modules.serviceCenter.services.serviceKnowledge.mainServiceKnowledge import IngestionJob
|
from modules.serviceCenter.services.serviceKnowledge.mainServiceKnowledge import IngestionJob
|
||||||
from modules.serviceCenter.context import ServiceCenterContext
|
from modules.serviceCenter.context import ServiceCenterContext
|
||||||
from modules.serviceCenter import getService
|
from modules.serviceCenter import getService
|
||||||
|
|
@ -156,8 +156,8 @@ async def _featureBootstrapHandler(
|
||||||
rootUser = getRootUser()
|
rootUser = getRootUser()
|
||||||
ctx = ServiceCenterContext(
|
ctx = ServiceCenterContext(
|
||||||
user=rootUser,
|
user=rootUser,
|
||||||
mandate_id=mandateId,
|
mandateId=mandateId,
|
||||||
feature_instance_id=fdsFeatureInstanceId,
|
featureInstanceId=fdsFeatureInstanceId,
|
||||||
)
|
)
|
||||||
knowledgeService = getService("knowledge", ctx)
|
knowledgeService = getService("knowledge", ctx)
|
||||||
|
|
||||||
|
|
@ -171,7 +171,7 @@ async def _featureBootstrapHandler(
|
||||||
"explicitFields": set(neutralizeFields),
|
"explicitFields": set(neutralizeFields),
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
provider = FeatureDataProvider(
|
provider = createFeatureDataProvider(
|
||||||
dbConnector,
|
dbConnector,
|
||||||
neutralizePolicy=neutralizePolicy,
|
neutralizePolicy=neutralizePolicy,
|
||||||
neutralizationService=neutralizationService,
|
neutralizationService=neutralizationService,
|
||||||
|
|
|
||||||
|
|
@ -251,7 +251,7 @@ class _DataSourceFamilyNode(UdbNode):
|
||||||
def getEffectiveFlag(self, flag, allDs, allFds, mode="aggregate") -> Any:
|
def getEffectiveFlag(self, flag, allDs, allFds, mode="aggregate") -> Any:
|
||||||
if not self.supportsFlag(flag):
|
if not self.supportsFlag(flag):
|
||||||
return False
|
return False
|
||||||
from modules.serviceCenter.services.serviceKnowledge._inheritFlags import (
|
from modules.serviceCenter.core.flagResolution import (
|
||||||
resolveEffectiveForPath,
|
resolveEffectiveForPath,
|
||||||
)
|
)
|
||||||
out = resolveEffectiveForPath(self.connectionId, self.sourceType, self.path, allDs, mode=mode)
|
out = resolveEffectiveForPath(self.connectionId, self.sourceType, self.path, allDs, mode=mode)
|
||||||
|
|
@ -260,7 +260,7 @@ class _DataSourceFamilyNode(UdbNode):
|
||||||
|
|
||||||
def setFlag(self, flag, value, rootIf) -> List[str]:
|
def setFlag(self, flag, value, rootIf) -> List[str]:
|
||||||
from modules.datamodels.datamodelDataSource import DataSource
|
from modules.datamodels.datamodelDataSource import DataSource
|
||||||
from modules.serviceCenter.services.serviceKnowledge._inheritFlags import (
|
from modules.serviceCenter.core.flagResolution import (
|
||||||
cascadeResetDescendants,
|
cascadeResetDescendants,
|
||||||
)
|
)
|
||||||
if not self.rec:
|
if not self.rec:
|
||||||
|
|
@ -416,7 +416,7 @@ class _FdsFamilyNode(UdbNode):
|
||||||
def getEffectiveFlag(self, flag, allDs, allFds, mode="aggregate") -> Any:
|
def getEffectiveFlag(self, flag, allDs, allFds, mode="aggregate") -> Any:
|
||||||
if not self.supportsFlag(flag):
|
if not self.supportsFlag(flag):
|
||||||
return None
|
return None
|
||||||
from modules.serviceCenter.services.serviceKnowledge._inheritFlags import (
|
from modules.serviceCenter.core.flagResolution import (
|
||||||
resolveEffectiveForFds,
|
resolveEffectiveForFds,
|
||||||
)
|
)
|
||||||
out = resolveEffectiveForFds(self.featureInstanceId, self.tableName,
|
out = resolveEffectiveForFds(self.featureInstanceId, self.tableName,
|
||||||
|
|
@ -428,7 +428,7 @@ class _FdsFamilyNode(UdbNode):
|
||||||
if not self.supportsFlag(flag):
|
if not self.supportsFlag(flag):
|
||||||
raise ValueError(f"FDS does not support flag {flag!r}")
|
raise ValueError(f"FDS does not support flag {flag!r}")
|
||||||
from modules.datamodels.datamodelFeatures import FeatureDataSource
|
from modules.datamodels.datamodelFeatures import FeatureDataSource
|
||||||
from modules.serviceCenter.services.serviceKnowledge._inheritFlags import (
|
from modules.serviceCenter.core.flagResolution import (
|
||||||
cascadeResetDescendantsFds,
|
cascadeResetDescendantsFds,
|
||||||
)
|
)
|
||||||
if not self.rec:
|
if not self.rec:
|
||||||
|
|
@ -669,7 +669,7 @@ class FdsFieldNode(UdbNode):
|
||||||
# Not explicitly overridden -> inherit from the table's effective
|
# Not explicitly overridden -> inherit from the table's effective
|
||||||
# neutralize. Use walk mode so the inherited value is concrete
|
# neutralize. Use walk mode so the inherited value is concrete
|
||||||
# (never 'mixed'); a single field cannot itself be ambiguous.
|
# (never 'mixed'); a single field cannot itself be ambiguous.
|
||||||
from modules.serviceCenter.services.serviceKnowledge._inheritFlags import (
|
from modules.serviceCenter.core.flagResolution import (
|
||||||
resolveEffectiveForFds,
|
resolveEffectiveForFds,
|
||||||
)
|
)
|
||||||
out = resolveEffectiveForFds(
|
out = resolveEffectiveForFds(
|
||||||
|
|
@ -753,7 +753,7 @@ def _findOrCreateDs(rootIf: Any, connectionId: str, sourceType: str,
|
||||||
"""
|
"""
|
||||||
from modules.datamodels.datamodelDataSource import DataSource
|
from modules.datamodels.datamodelDataSource import DataSource
|
||||||
from modules.datamodels.datamodelUam import UserConnection
|
from modules.datamodels.datamodelUam import UserConnection
|
||||||
from modules.serviceCenter.services.serviceKnowledge._inheritFlags import normalisePath
|
from modules.serviceCenter.core.flagResolution import normalisePath
|
||||||
|
|
||||||
normPath = normalisePath(path)
|
normPath = normalisePath(path)
|
||||||
|
|
||||||
|
|
@ -1007,7 +1007,7 @@ def buildNodeForKey(key: str, context: Any, rootIf: Any) -> Optional[UdbNode]:
|
||||||
def _findDsByCoord(rootIf: Any, connectionId: str, sourceType: Optional[str],
|
def _findDsByCoord(rootIf: Any, connectionId: str, sourceType: Optional[str],
|
||||||
path: str) -> Optional[Dict[str, Any]]:
|
path: str) -> Optional[Dict[str, Any]]:
|
||||||
from modules.datamodels.datamodelDataSource import DataSource
|
from modules.datamodels.datamodelDataSource import DataSource
|
||||||
from modules.serviceCenter.services.serviceKnowledge._inheritFlags import normalisePath
|
from modules.serviceCenter.core.flagResolution import normalisePath
|
||||||
rf = {"connectionId": connectionId}
|
rf = {"connectionId": connectionId}
|
||||||
if sourceType is not None:
|
if sourceType is not None:
|
||||||
rf["sourceType"] = sourceType
|
rf["sourceType"] = sourceType
|
||||||
|
|
|
||||||
|
|
@ -33,7 +33,7 @@ class _ServicesAdapter:
|
||||||
from modules.interfaces.interfaceDbManagement import getInterface as getComponentInterface
|
from modules.interfaces.interfaceDbManagement import getInterface as getComponentInterface
|
||||||
self.interfaceDbComponent = getComponentInterface(
|
self.interfaceDbComponent = getComponentInterface(
|
||||||
context.user,
|
context.user,
|
||||||
mandateId=context.mandate_id
|
mandateId=context.mandateId
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -24,13 +24,13 @@ class SharepointService:
|
||||||
"""Initialize SharePoint service without access token.
|
"""Initialize SharePoint service without access token.
|
||||||
|
|
||||||
Args:
|
Args:
|
||||||
context: ServiceCenterContext with user, mandate_id, etc.
|
context: ServiceCenterContext with user, mandateId, etc.
|
||||||
get_service: Service resolver for dependency injection (e.g. security)
|
get_service: Service resolver for dependency injection (e.g. security)
|
||||||
|
|
||||||
Use setAccessTokenFromConnection() method to configure the access token before making API calls.
|
Use setAccessTokenFromConnection() method to configure the access token before making API calls.
|
||||||
"""
|
"""
|
||||||
self._context = context
|
self._context = context
|
||||||
self._get_service = get_service
|
self._getService = get_service
|
||||||
self.accessToken = None
|
self.accessToken = None
|
||||||
self.baseUrl = "https://graph.microsoft.com/v1.0"
|
self.baseUrl = "https://graph.microsoft.com/v1.0"
|
||||||
|
|
||||||
|
|
@ -59,7 +59,7 @@ class SharepointService:
|
||||||
return False
|
return False
|
||||||
|
|
||||||
# Get a fresh token for this specific connection via security service
|
# Get a fresh token for this specific connection via security service
|
||||||
security = self._get_service("security")
|
security = self._getService("security")
|
||||||
if not security:
|
if not security:
|
||||||
logger.error("Security service not available for token access")
|
logger.error("Security service not available for token access")
|
||||||
return False
|
return False
|
||||||
|
|
|
||||||
|
|
@ -55,11 +55,11 @@ class SubscriptionService:
|
||||||
if mandateId is not None and callable(mandateId):
|
if mandateId is not None and callable(mandateId):
|
||||||
ctx = contextOrUser
|
ctx = contextOrUser
|
||||||
self.currentUser = ctx.user
|
self.currentUser = ctx.user
|
||||||
self.mandateId = ctx.mandate_id or ""
|
self.mandateId = ctx.mandateId or ""
|
||||||
elif get_service is not None and hasattr(contextOrUser, "user"):
|
elif get_service is not None and hasattr(contextOrUser, "user"):
|
||||||
ctx = contextOrUser
|
ctx = contextOrUser
|
||||||
self.currentUser = ctx.user
|
self.currentUser = ctx.user
|
||||||
self.mandateId = ctx.mandate_id or ""
|
self.mandateId = ctx.mandateId or ""
|
||||||
else:
|
else:
|
||||||
self.currentUser = contextOrUser
|
self.currentUser = contextOrUser
|
||||||
self.mandateId = mandateId or ""
|
self.mandateId = mandateId or ""
|
||||||
|
|
|
||||||
|
|
@ -15,7 +15,7 @@ class TicketService:
|
||||||
def __init__(self, context, get_service: Callable[[str], Any]):
|
def __init__(self, context, get_service: Callable[[str], Any]):
|
||||||
"""Initialize with context and service resolver."""
|
"""Initialize with context and service resolver."""
|
||||||
self._context = context
|
self._context = context
|
||||||
self._get_service = get_service
|
self._getService = get_service
|
||||||
|
|
||||||
async def connectTicket(
|
async def connectTicket(
|
||||||
self,
|
self,
|
||||||
|
|
|
||||||
|
|
@ -22,14 +22,14 @@ class WebService:
|
||||||
def __init__(self, context, get_service):
|
def __init__(self, context, get_service):
|
||||||
"""Initialize webcrawl service with context and service resolver."""
|
"""Initialize webcrawl service with context and service resolver."""
|
||||||
self._context = context
|
self._context = context
|
||||||
self._get_service = get_service
|
self._getService = get_service
|
||||||
|
|
||||||
def _workflow_id(self):
|
def _workflow_id(self):
|
||||||
"""Get workflow ID for operation IDs."""
|
"""Get workflow ID for operation IDs."""
|
||||||
if self._context.workflow:
|
if self._context.workflow:
|
||||||
return self._context.workflow.id
|
return self._context.workflow.id
|
||||||
if self._context.workflow_id:
|
if self._context.workflowId:
|
||||||
return self._context.workflow_id
|
return self._context.workflowId
|
||||||
return f"no-workflow-{int(time.time())}"
|
return f"no-workflow-{int(time.time())}"
|
||||||
|
|
||||||
async def performWebResearch(
|
async def performWebResearch(
|
||||||
|
|
@ -61,7 +61,7 @@ class WebService:
|
||||||
"""
|
"""
|
||||||
# Start progress tracking if operationId provided
|
# Start progress tracking if operationId provided
|
||||||
if operationId:
|
if operationId:
|
||||||
self._get_service("chat").progressLogStart(
|
self._getService("chat").progressLogStart(
|
||||||
operationId,
|
operationId,
|
||||||
"Web Research",
|
"Web Research",
|
||||||
"Research",
|
"Research",
|
||||||
|
|
@ -71,7 +71,7 @@ class WebService:
|
||||||
try:
|
try:
|
||||||
# Step 1: AI intention analysis - extract URLs and parameters from prompt
|
# Step 1: AI intention analysis - extract URLs and parameters from prompt
|
||||||
if operationId:
|
if operationId:
|
||||||
self._get_service("chat").progressLogUpdate(operationId, 0.1, "Analyzing research intent")
|
self._getService("chat").progressLogUpdate(operationId, 0.1, "Analyzing research intent")
|
||||||
|
|
||||||
analysisResult = await self._analyzeResearchIntent(prompt, urls, country, language, researchDepth)
|
analysisResult = await self._analyzeResearchIntent(prompt, urls, country, language, researchDepth)
|
||||||
|
|
||||||
|
|
@ -99,7 +99,7 @@ class WebService:
|
||||||
searchResultsWithContent = []
|
searchResultsWithContent = []
|
||||||
if needsSearch and (not allUrls or len(allUrls) < maxNumberPages):
|
if needsSearch and (not allUrls or len(allUrls) < maxNumberPages):
|
||||||
if operationId:
|
if operationId:
|
||||||
self._get_service("chat").progressLogUpdate(operationId, 0.3, "Searching for URLs and content")
|
self._getService("chat").progressLogUpdate(operationId, 0.3, "Searching for URLs and content")
|
||||||
|
|
||||||
try:
|
try:
|
||||||
searchUrls, searchResultsWithContent = await self._performWebSearch(
|
searchUrls, searchResultsWithContent = await self._performWebSearch(
|
||||||
|
|
@ -121,7 +121,7 @@ class WebService:
|
||||||
logger.warning("Tavily search returned no URLs, using AI-extracted URLs only")
|
logger.warning("Tavily search returned no URLs, using AI-extracted URLs only")
|
||||||
|
|
||||||
if operationId:
|
if operationId:
|
||||||
self._get_service("chat").progressLogUpdate(operationId, 0.5, f"Found {len(allUrls)} total URLs")
|
self._getService("chat").progressLogUpdate(operationId, 0.5, f"Found {len(allUrls)} total URLs")
|
||||||
|
|
||||||
# If we have search results (even without content), use them directly instead of crawling
|
# If we have search results (even without content), use them directly instead of crawling
|
||||||
# Tavily search results are more relevant than generic AI-extracted URLs
|
# Tavily search results are more relevant than generic AI-extracted URLs
|
||||||
|
|
@ -179,7 +179,7 @@ class WebService:
|
||||||
"total_urls": len(searchUrls),
|
"total_urls": len(searchUrls),
|
||||||
"urls_with_content": urlsWithContent,
|
"urls_with_content": urlsWithContent,
|
||||||
"total_content_length": totalContentLength,
|
"total_content_length": totalContentLength,
|
||||||
"search_date": self._get_service("utils").timestampGetUtc()
|
"search_date": self._getService("utils").timestampGetUtc()
|
||||||
},
|
},
|
||||||
"sections": sections,
|
"sections": sections,
|
||||||
"statistics": {
|
"statistics": {
|
||||||
|
|
@ -201,8 +201,8 @@ class WebService:
|
||||||
result["metadata"]["suggested_filename"] = suggestedFilename
|
result["metadata"]["suggested_filename"] = suggestedFilename
|
||||||
|
|
||||||
if operationId:
|
if operationId:
|
||||||
self._get_service("chat").progressLogUpdate(operationId, 0.9, "Completed")
|
self._getService("chat").progressLogUpdate(operationId, 0.9, "Completed")
|
||||||
self._get_service("chat").progressLogFinish(operationId, True)
|
self._getService("chat").progressLogFinish(operationId, True)
|
||||||
|
|
||||||
return result
|
return result
|
||||||
|
|
||||||
|
|
@ -231,8 +231,8 @@ class WebService:
|
||||||
|
|
||||||
# Step 5: Crawl all URLs with hierarchical logging
|
# Step 5: Crawl all URLs with hierarchical logging
|
||||||
if operationId:
|
if operationId:
|
||||||
self._get_service("chat").progressLogUpdate(operationId, 0.4, "Initiating")
|
self._getService("chat").progressLogUpdate(operationId, 0.4, "Initiating")
|
||||||
self._get_service("chat").progressLogUpdate(operationId, 0.6, f"Crawling {len(validatedUrls)} URLs")
|
self._getService("chat").progressLogUpdate(operationId, 0.6, f"Crawling {len(validatedUrls)} URLs")
|
||||||
|
|
||||||
# Use parent operation ID directly (parentId should be operationId, not log entry ID)
|
# Use parent operation ID directly (parentId should be operationId, not log entry ID)
|
||||||
parentOperationId = operationId # Use the parent's operationId directly
|
parentOperationId = operationId # Use the parent's operationId directly
|
||||||
|
|
@ -246,9 +246,9 @@ class WebService:
|
||||||
)
|
)
|
||||||
|
|
||||||
if operationId:
|
if operationId:
|
||||||
self._get_service("chat").progressLogUpdate(operationId, 0.9, "Consolidating results")
|
self._getService("chat").progressLogUpdate(operationId, 0.9, "Consolidating results")
|
||||||
self._get_service("chat").progressLogUpdate(operationId, 0.95, "Completed")
|
self._getService("chat").progressLogUpdate(operationId, 0.95, "Completed")
|
||||||
self._get_service("chat").progressLogFinish(operationId, True)
|
self._getService("chat").progressLogFinish(operationId, True)
|
||||||
|
|
||||||
# Calculate statistics about crawl results
|
# Calculate statistics about crawl results
|
||||||
totalResults = len(crawlResult) if isinstance(crawlResult, list) else 1
|
totalResults = len(crawlResult) if isinstance(crawlResult, list) else 1
|
||||||
|
|
@ -317,7 +317,7 @@ class WebService:
|
||||||
"total_urls": len(validatedUrls),
|
"total_urls": len(validatedUrls),
|
||||||
"urls_with_content": urlsWithContent,
|
"urls_with_content": urlsWithContent,
|
||||||
"total_content_length": totalContentLength,
|
"total_content_length": totalContentLength,
|
||||||
"crawl_date": self._get_service("utils").timestampGetUtc()
|
"crawl_date": self._getService("utils").timestampGetUtc()
|
||||||
},
|
},
|
||||||
"sections": sections,
|
"sections": sections,
|
||||||
"statistics": {
|
"statistics": {
|
||||||
|
|
@ -345,7 +345,7 @@ class WebService:
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
logger.error(f"Error in web research: {str(e)}")
|
logger.error(f"Error in web research: {str(e)}")
|
||||||
if operationId:
|
if operationId:
|
||||||
self._get_service("chat").progressLogFinish(operationId, False)
|
self._getService("chat").progressLogFinish(operationId, False)
|
||||||
raise
|
raise
|
||||||
|
|
||||||
async def _analyzeResearchIntent(
|
async def _analyzeResearchIntent(
|
||||||
|
|
@ -397,13 +397,13 @@ Return ONLY valid JSON, no additional text:
|
||||||
|
|
||||||
try:
|
try:
|
||||||
# Call AI planning to analyze intent
|
# Call AI planning to analyze intent
|
||||||
analysisJson = await self._get_service("ai").callAiPlanning(
|
analysisJson = await self._getService("ai").callAiPlanning(
|
||||||
analysisPrompt,
|
analysisPrompt,
|
||||||
debugType="webresearchintent"
|
debugType="webresearchintent"
|
||||||
)
|
)
|
||||||
|
|
||||||
# Extract JSON from response (handles markdown code blocks)
|
# Extract JSON from response (handles markdown code blocks)
|
||||||
extractedJson = self._get_service("utils").jsonExtractString(analysisJson)
|
extractedJson = self._getService("utils").jsonExtractString(analysisJson)
|
||||||
if not extractedJson:
|
if not extractedJson:
|
||||||
raise ValueError("No JSON found in AI response")
|
raise ValueError("No JSON found in AI response")
|
||||||
|
|
||||||
|
|
@ -454,7 +454,7 @@ Return ONLY valid JSON, no additional text:
|
||||||
searchPrompt = searchPromptModel.model_dump_json(exclude_none=True, indent=2)
|
searchPrompt = searchPromptModel.model_dump_json(exclude_none=True, indent=2)
|
||||||
|
|
||||||
# Debug: persist search prompt
|
# Debug: persist search prompt
|
||||||
self._get_service("utils").writeDebugFile(searchPrompt, "websearch_prompt")
|
self._getService("utils").writeDebugFile(searchPrompt, "websearch_prompt")
|
||||||
|
|
||||||
# Call AI with WEB_SEARCH_DATA operation
|
# Call AI with WEB_SEARCH_DATA operation
|
||||||
searchOptions = AiCallOptions(
|
searchOptions = AiCallOptions(
|
||||||
|
|
@ -463,7 +463,7 @@ Return ONLY valid JSON, no additional text:
|
||||||
)
|
)
|
||||||
|
|
||||||
# Use unified callAiContent method
|
# Use unified callAiContent method
|
||||||
searchResponse = await self._get_service("ai").callAiContent(
|
searchResponse = await self._getService("ai").callAiContent(
|
||||||
prompt=searchPrompt,
|
prompt=searchPrompt,
|
||||||
options=searchOptions,
|
options=searchOptions,
|
||||||
outputFormat="json"
|
outputFormat="json"
|
||||||
|
|
@ -518,16 +518,16 @@ Return ONLY valid JSON, no additional text:
|
||||||
|
|
||||||
# Debug: persist search response
|
# Debug: persist search response
|
||||||
if isinstance(searchResult, str):
|
if isinstance(searchResult, str):
|
||||||
self._get_service("utils").writeDebugFile(searchResult, "websearch_response")
|
self._getService("utils").writeDebugFile(searchResult, "websearch_response")
|
||||||
logger.debug(f"Search response (first 500 chars): {searchResult[:500]}")
|
logger.debug(f"Search response (first 500 chars): {searchResult[:500]}")
|
||||||
else:
|
else:
|
||||||
self._get_service("utils").writeDebugFile(json.dumps(searchResult, indent=2), "websearch_response")
|
self._getService("utils").writeDebugFile(json.dumps(searchResult, indent=2), "websearch_response")
|
||||||
logger.debug(f"Search response type: {type(searchResult)}, keys: {list(searchResult.keys()) if isinstance(searchResult, dict) else 'N/A'}")
|
logger.debug(f"Search response type: {type(searchResult)}, keys: {list(searchResult.keys()) if isinstance(searchResult, dict) else 'N/A'}")
|
||||||
|
|
||||||
# Parse and extract URLs and content
|
# Parse and extract URLs and content
|
||||||
if isinstance(searchResult, str):
|
if isinstance(searchResult, str):
|
||||||
# Extract JSON from response (handles markdown code blocks)
|
# Extract JSON from response (handles markdown code blocks)
|
||||||
extractedJson = self._get_service("utils").jsonExtractString(searchResult)
|
extractedJson = self._getService("utils").jsonExtractString(searchResult)
|
||||||
if extractedJson:
|
if extractedJson:
|
||||||
try:
|
try:
|
||||||
searchData = json.loads(extractedJson)
|
searchData = json.loads(extractedJson)
|
||||||
|
|
@ -800,7 +800,7 @@ Return ONLY valid JSON, no additional text:
|
||||||
if parentOperationId:
|
if parentOperationId:
|
||||||
workflowId = self._workflow_id()
|
workflowId = self._workflow_id()
|
||||||
urlOperationId = f"web_crawl_url_{workflowId}_{urlIndex}_{int(time.time())}"
|
urlOperationId = f"web_crawl_url_{workflowId}_{urlIndex}_{int(time.time())}"
|
||||||
self._get_service("chat").progressLogStart(
|
self._getService("chat").progressLogStart(
|
||||||
urlOperationId,
|
urlOperationId,
|
||||||
"Web Crawl",
|
"Web Crawl",
|
||||||
f"URL {urlIndex + 1}/{totalUrls}",
|
f"URL {urlIndex + 1}/{totalUrls}",
|
||||||
|
|
@ -813,8 +813,8 @@ Return ONLY valid JSON, no additional text:
|
||||||
|
|
||||||
if urlOperationId:
|
if urlOperationId:
|
||||||
displayUrl = url[:50] + "..." if len(url) > 50 else url
|
displayUrl = url[:50] + "..." if len(url) > 50 else url
|
||||||
self._get_service("chat").progressLogUpdate(urlOperationId, 0.2, f"Crawling: {displayUrl}")
|
self._getService("chat").progressLogUpdate(urlOperationId, 0.2, f"Crawling: {displayUrl}")
|
||||||
self._get_service("chat").progressLogUpdate(urlOperationId, 0.3, "Initiating crawl")
|
self._getService("chat").progressLogUpdate(urlOperationId, 0.3, "Initiating crawl")
|
||||||
|
|
||||||
# Build crawl prompt model for single URL
|
# Build crawl prompt model for single URL
|
||||||
# maxWidth is passed from performWebResearch based on researchDepth
|
# maxWidth is passed from performWebResearch based on researchDepth
|
||||||
|
|
@ -829,7 +829,7 @@ Return ONLY valid JSON, no additional text:
|
||||||
|
|
||||||
# Debug: persist crawl prompt (with URL identifier in content for clarity)
|
# Debug: persist crawl prompt (with URL identifier in content for clarity)
|
||||||
debugPrompt = f"URL: {url}\n\n{crawlPrompt}"
|
debugPrompt = f"URL: {url}\n\n{crawlPrompt}"
|
||||||
self._get_service("utils").writeDebugFile(debugPrompt, "webcrawl_prompt")
|
self._getService("utils").writeDebugFile(debugPrompt, "webcrawl_prompt")
|
||||||
|
|
||||||
# Call AI with WEB_CRAWL operation
|
# Call AI with WEB_CRAWL operation
|
||||||
crawlOptions = AiCallOptions(
|
crawlOptions = AiCallOptions(
|
||||||
|
|
@ -838,10 +838,10 @@ Return ONLY valid JSON, no additional text:
|
||||||
)
|
)
|
||||||
|
|
||||||
if urlOperationId:
|
if urlOperationId:
|
||||||
self._get_service("chat").progressLogUpdate(urlOperationId, 0.4, "Calling crawl connector")
|
self._getService("chat").progressLogUpdate(urlOperationId, 0.4, "Calling crawl connector")
|
||||||
|
|
||||||
# Use unified callAiContent method with parentOperationId for hierarchical logging
|
# Use unified callAiContent method with parentOperationId for hierarchical logging
|
||||||
crawlResponse = await self._get_service("ai").callAiContent(
|
crawlResponse = await self._getService("ai").callAiContent(
|
||||||
prompt=crawlPrompt,
|
prompt=crawlPrompt,
|
||||||
options=crawlOptions,
|
options=crawlOptions,
|
||||||
outputFormat="json",
|
outputFormat="json",
|
||||||
|
|
@ -849,22 +849,22 @@ Return ONLY valid JSON, no additional text:
|
||||||
)
|
)
|
||||||
|
|
||||||
if urlOperationId:
|
if urlOperationId:
|
||||||
self._get_service("chat").progressLogUpdate(urlOperationId, 0.7, "Processing crawl results")
|
self._getService("chat").progressLogUpdate(urlOperationId, 0.7, "Processing crawl results")
|
||||||
|
|
||||||
# Extract content from AiResponse
|
# Extract content from AiResponse
|
||||||
crawlResult = crawlResponse.content
|
crawlResult = crawlResponse.content
|
||||||
|
|
||||||
# Debug: persist crawl response
|
# Debug: persist crawl response
|
||||||
if isinstance(crawlResult, str):
|
if isinstance(crawlResult, str):
|
||||||
self._get_service("utils").writeDebugFile(crawlResult, "webcrawl_response")
|
self._getService("utils").writeDebugFile(crawlResult, "webcrawl_response")
|
||||||
else:
|
else:
|
||||||
self._get_service("utils").writeDebugFile(json.dumps(crawlResult, indent=2), "webcrawl_response")
|
self._getService("utils").writeDebugFile(json.dumps(crawlResult, indent=2), "webcrawl_response")
|
||||||
|
|
||||||
# Parse crawl result
|
# Parse crawl result
|
||||||
if isinstance(crawlResult, str):
|
if isinstance(crawlResult, str):
|
||||||
try:
|
try:
|
||||||
# Extract JSON from response (handles markdown code blocks)
|
# Extract JSON from response (handles markdown code blocks)
|
||||||
extractedJson = self._get_service("utils").jsonExtractString(crawlResult)
|
extractedJson = self._getService("utils").jsonExtractString(crawlResult)
|
||||||
crawlData = json.loads(extractedJson) if extractedJson else json.loads(crawlResult)
|
crawlData = json.loads(extractedJson) if extractedJson else json.loads(crawlResult)
|
||||||
except:
|
except:
|
||||||
crawlData = {"url": url, "content": crawlResult}
|
crawlData = {"url": url, "content": crawlResult}
|
||||||
|
|
@ -873,7 +873,7 @@ Return ONLY valid JSON, no additional text:
|
||||||
|
|
||||||
# Process crawl results and create hierarchical progress logging for sub-URLs
|
# Process crawl results and create hierarchical progress logging for sub-URLs
|
||||||
if urlOperationId:
|
if urlOperationId:
|
||||||
self._get_service("chat").progressLogUpdate(urlOperationId, 0.8, "Processing crawl results")
|
self._getService("chat").progressLogUpdate(urlOperationId, 0.8, "Processing crawl results")
|
||||||
|
|
||||||
# Recursively process crawl results to find nested URLs and create child operations
|
# Recursively process crawl results to find nested URLs and create child operations
|
||||||
processedResults = self._processCrawlResultsWithHierarchy(crawlData, url, urlOperationId, maxDepth, 0)
|
processedResults = self._processCrawlResultsWithHierarchy(crawlData, url, urlOperationId, maxDepth, 0)
|
||||||
|
|
@ -891,17 +891,17 @@ Return ONLY valid JSON, no additional text:
|
||||||
|
|
||||||
if urlOperationId:
|
if urlOperationId:
|
||||||
if totalUrlsCrawled > 1:
|
if totalUrlsCrawled > 1:
|
||||||
self._get_service("chat").progressLogUpdate(urlOperationId, 0.9, f"Crawled {totalUrlsCrawled} URLs (including sub-URLs)")
|
self._getService("chat").progressLogUpdate(urlOperationId, 0.9, f"Crawled {totalUrlsCrawled} URLs (including sub-URLs)")
|
||||||
else:
|
else:
|
||||||
self._get_service("chat").progressLogUpdate(urlOperationId, 0.9, "Crawl completed")
|
self._getService("chat").progressLogUpdate(urlOperationId, 0.9, "Crawl completed")
|
||||||
self._get_service("chat").progressLogFinish(urlOperationId, True)
|
self._getService("chat").progressLogFinish(urlOperationId, True)
|
||||||
|
|
||||||
return results
|
return results
|
||||||
|
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
logger.error(f"Error crawling URL {url}: {str(e)}")
|
logger.error(f"Error crawling URL {url}: {str(e)}")
|
||||||
if urlOperationId:
|
if urlOperationId:
|
||||||
self._get_service("chat").progressLogFinish(urlOperationId, False)
|
self._getService("chat").progressLogFinish(urlOperationId, False)
|
||||||
return [{"url": url, "error": str(e)}]
|
return [{"url": url, "error": str(e)}]
|
||||||
|
|
||||||
def _processCrawlResultsWithHierarchy(
|
def _processCrawlResultsWithHierarchy(
|
||||||
|
|
@ -943,7 +943,7 @@ Return ONLY valid JSON, no additional text:
|
||||||
# This is a sub-URL - create child operation
|
# This is a sub-URL - create child operation
|
||||||
workflowId = self._workflow_id()
|
workflowId = self._workflow_id()
|
||||||
subUrlOperationId = f"{parentOperationId}_sub_{idx}_{int(time.time())}"
|
subUrlOperationId = f"{parentOperationId}_sub_{idx}_{int(time.time())}"
|
||||||
self._get_service("chat").progressLogStart(
|
self._getService("chat").progressLogStart(
|
||||||
subUrlOperationId,
|
subUrlOperationId,
|
||||||
"Crawling Sub-URL",
|
"Crawling Sub-URL",
|
||||||
f"Depth {currentDepth + 1}",
|
f"Depth {currentDepth + 1}",
|
||||||
|
|
@ -969,12 +969,12 @@ Return ONLY valid JSON, no additional text:
|
||||||
)
|
)
|
||||||
item["subUrls"] = nestedResults
|
item["subUrls"] = nestedResults
|
||||||
|
|
||||||
self._get_service("chat").progressLogUpdate(subUrlOperationId, 0.9, "Completed")
|
self._getService("chat").progressLogUpdate(subUrlOperationId, 0.9, "Completed")
|
||||||
self._get_service("chat").progressLogFinish(subUrlOperationId, True)
|
self._getService("chat").progressLogFinish(subUrlOperationId, True)
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
logger.error(f"Error processing sub-URL {itemUrl}: {str(e)}")
|
logger.error(f"Error processing sub-URL {itemUrl}: {str(e)}")
|
||||||
if subUrlOperationId:
|
if subUrlOperationId:
|
||||||
self._get_service("chat").progressLogFinish(subUrlOperationId, False)
|
self._getService("chat").progressLogFinish(subUrlOperationId, False)
|
||||||
|
|
||||||
results.append(item)
|
results.append(item)
|
||||||
else:
|
else:
|
||||||
|
|
|
||||||
|
|
@ -6,7 +6,9 @@ Higher-layer system components (e.g. workflowAutomation) register their
|
||||||
lifecycle hooks here at boot time via ``app.py`` (Composition Root, L7).
|
lifecycle hooks here at boot time via ``app.py`` (Composition Root, L7).
|
||||||
Interface modules read the registry generically — no upward imports needed.
|
Interface modules read the registry generically — no upward imports needed.
|
||||||
|
|
||||||
Supported events: ``onBootstrap``, ``onMandateDelete``, ``onInstanceCreate``.
|
Supported events: ``onBootstrap``, ``onMandateDelete``, ``onMandateProvision``,
|
||||||
|
``onInstanceCreate``, ``onUserMandateCreate``, ``onUserMandateDelete``,
|
||||||
|
``onUserBudgetAdjust``, ``onStorageChanged``.
|
||||||
|
|
||||||
This is the same inversion pattern used by
|
This is the same inversion pattern used by
|
||||||
``serviceAgent/externalToolRegistry.py`` for agent tools.
|
``serviceAgent/externalToolRegistry.py`` for agent tools.
|
||||||
|
|
|
||||||
102
modules/workflowAutomation/editor/_valueKindResolver.py
Normal file
102
modules/workflowAutomation/editor/_valueKindResolver.py
Normal file
|
|
@ -0,0 +1,102 @@
|
||||||
|
# Copyright (c) 2025 Patrick Motsch
|
||||||
|
"""Shared value-kind resolution helpers.
|
||||||
|
|
||||||
|
Extracted from conditionOperators so that upstreamPathsService can resolve
|
||||||
|
value kinds without importing conditionOperators (breaking the bidirectional
|
||||||
|
import cycle).
|
||||||
|
"""
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
from typing import Any, Dict, List
|
||||||
|
|
||||||
|
|
||||||
|
def catalogTypeToValueKind(catalogType: str) -> str:
|
||||||
|
"""Map port-catalog / dataPickOptions type strings to condition valueKind."""
|
||||||
|
ct = (catalogType or "").strip()
|
||||||
|
if not ct or ct == "Any":
|
||||||
|
return "unknown"
|
||||||
|
low = ct.lower()
|
||||||
|
if low in ("str", "string", "email", "url"):
|
||||||
|
return "string"
|
||||||
|
if low in ("int", "float", "number"):
|
||||||
|
return "number"
|
||||||
|
if low == "bool":
|
||||||
|
return "boolean"
|
||||||
|
if low in ("date", "datetime", "timestamp"):
|
||||||
|
return "datetime"
|
||||||
|
if low.startswith("list[") or low == "list":
|
||||||
|
return "array"
|
||||||
|
if low.startswith("dict") or low == "dict":
|
||||||
|
return "object"
|
||||||
|
if low in ("file", "actiondocument", "fileref"):
|
||||||
|
return "file"
|
||||||
|
return "unknown"
|
||||||
|
|
||||||
|
|
||||||
|
def _isContextProducer(nodeType: str) -> bool:
|
||||||
|
return nodeType in ("context.extractContent", "context.mergeContext", "context.setContext")
|
||||||
|
|
||||||
|
|
||||||
|
def _pathSuggestsContext(path: List[Any], producerType: str) -> bool:
|
||||||
|
if not path:
|
||||||
|
return _isContextProducer(producerType)
|
||||||
|
last = str(path[-1])
|
||||||
|
if last in ("data", "files", "merged", "presentation"):
|
||||||
|
return True
|
||||||
|
if "files" in [str(p) for p in path]:
|
||||||
|
return True
|
||||||
|
if _isContextProducer(producerType) and path[0] in ("data", "response", "merged"):
|
||||||
|
return True
|
||||||
|
return False
|
||||||
|
|
||||||
|
|
||||||
|
def _pathSuggestsFile(path: List[Any], producerType: str) -> bool:
|
||||||
|
pathStr = [str(p) for p in path]
|
||||||
|
if producerType == "input.upload":
|
||||||
|
return True
|
||||||
|
if "file" in pathStr or "documents" in pathStr or "mimeType" in pathStr or "fileName" in pathStr:
|
||||||
|
return True
|
||||||
|
if producerType.startswith("sharepoint.") and "file" in pathStr:
|
||||||
|
return True
|
||||||
|
return False
|
||||||
|
|
||||||
|
|
||||||
|
def _pathsEqual(a: List[Any], b: List[Any]) -> bool:
|
||||||
|
if len(a) != len(b):
|
||||||
|
return False
|
||||||
|
return all(str(x) == str(y) for x, y in zip(a, b))
|
||||||
|
|
||||||
|
|
||||||
|
def resolveValueKindFromRef(graph: Dict[str, Any], ref: Dict[str, Any]) -> str:
|
||||||
|
"""Resolve condition valueKind using graph-local heuristics only.
|
||||||
|
|
||||||
|
Unlike ``conditionOperators.resolve_value_kind`` this does NOT call
|
||||||
|
``compute_upstream_paths``, so it is safe to import from
|
||||||
|
upstreamPathsService without creating a cycle.
|
||||||
|
"""
|
||||||
|
if not isinstance(ref, dict):
|
||||||
|
return "unknown"
|
||||||
|
producerId = ref.get("nodeId")
|
||||||
|
path = ref.get("path") or []
|
||||||
|
if not isinstance(path, list):
|
||||||
|
path = []
|
||||||
|
if not producerId:
|
||||||
|
return "unknown"
|
||||||
|
|
||||||
|
nodes = graph.get("nodes") or []
|
||||||
|
nodeById = {n.get("id"): n for n in nodes if n.get("id")}
|
||||||
|
producer = nodeById.get(producerId) or {}
|
||||||
|
producerType = str(producer.get("type") or "")
|
||||||
|
|
||||||
|
if _pathSuggestsContext(path, producerType):
|
||||||
|
return "context"
|
||||||
|
if _pathSuggestsFile(path, producerType):
|
||||||
|
tail = str(path[-1]) if path else ""
|
||||||
|
if tail in ("mimeType", "fileName"):
|
||||||
|
return "string"
|
||||||
|
return "file"
|
||||||
|
|
||||||
|
if producerType in ("trigger.form", "input.form") and path and str(path[0]) == "payload":
|
||||||
|
return "string"
|
||||||
|
|
||||||
|
return "unknown"
|
||||||
|
|
@ -10,6 +10,12 @@ from typing import Any, Dict, List, Optional, Tuple
|
||||||
|
|
||||||
from modules.nodeCatalog.nodeDefinitions import STATIC_NODE_TYPES
|
from modules.nodeCatalog.nodeDefinitions import STATIC_NODE_TYPES
|
||||||
from modules.shared.i18nRegistry import resolveText, t
|
from modules.shared.i18nRegistry import resolveText, t
|
||||||
|
from modules.workflowAutomation.editor._valueKindResolver import (
|
||||||
|
catalogTypeToValueKind as catalog_type_to_value_kind,
|
||||||
|
_pathSuggestsContext as _path_suggests_context,
|
||||||
|
_pathSuggestsFile as _path_suggests_file,
|
||||||
|
_pathsEqual as _paths_equal,
|
||||||
|
)
|
||||||
|
|
||||||
logger = logging.getLogger(__name__)
|
logger = logging.getLogger(__name__)
|
||||||
|
|
||||||
|
|
@ -200,64 +206,7 @@ def localize_operator_catalog(lang: str = "de") -> Dict[str, List[Dict[str, Any]
|
||||||
return out
|
return out
|
||||||
|
|
||||||
|
|
||||||
def catalog_type_to_value_kind(catalog_type: str) -> str:
|
def resolve_value_kind(graph: Dict[str, Any], ref: Dict[str, Any]) -> str:
|
||||||
"""Map port-catalog / dataPickOptions type strings to condition valueKind."""
|
|
||||||
ct = (catalog_type or "").strip()
|
|
||||||
if not ct or ct == "Any":
|
|
||||||
return "unknown"
|
|
||||||
low = ct.lower()
|
|
||||||
if low in ("str", "string", "email", "url"):
|
|
||||||
return "string"
|
|
||||||
if low in ("int", "float", "number"):
|
|
||||||
return "number"
|
|
||||||
if low == "bool":
|
|
||||||
return "boolean"
|
|
||||||
if low in ("date", "datetime", "timestamp"):
|
|
||||||
return "datetime"
|
|
||||||
if low.startswith("list[") or low == "list":
|
|
||||||
return "array"
|
|
||||||
if low.startswith("dict") or low == "dict":
|
|
||||||
return "object"
|
|
||||||
if low in ("file", "actiondocument", "fileref"):
|
|
||||||
return "file"
|
|
||||||
return "unknown"
|
|
||||||
|
|
||||||
|
|
||||||
def _paths_equal(a: List[Any], b: List[Any]) -> bool:
|
|
||||||
if len(a) != len(b):
|
|
||||||
return False
|
|
||||||
return all(str(x) == str(y) for x, y in zip(a, b))
|
|
||||||
|
|
||||||
|
|
||||||
def _is_context_producer(node_type: str) -> bool:
|
|
||||||
return node_type in ("context.extractContent", "context.mergeContext", "context.setContext")
|
|
||||||
|
|
||||||
|
|
||||||
def _path_suggests_context(path: List[Any], producer_type: str) -> bool:
|
|
||||||
if not path:
|
|
||||||
return _is_context_producer(producer_type)
|
|
||||||
last = str(path[-1])
|
|
||||||
if last in ("data", "files", "merged", "presentation"):
|
|
||||||
return True
|
|
||||||
if "files" in [str(p) for p in path]:
|
|
||||||
return True
|
|
||||||
if _is_context_producer(producer_type) and path[0] in ("data", "response", "merged"):
|
|
||||||
return True
|
|
||||||
return False
|
|
||||||
|
|
||||||
|
|
||||||
def _path_suggests_file(path: List[Any], producer_type: str) -> bool:
|
|
||||||
path_str = [str(p) for p in path]
|
|
||||||
if producer_type == "input.upload":
|
|
||||||
return True
|
|
||||||
if "file" in path_str or "documents" in path_str or "mimeType" in path_str or "fileName" in path_str:
|
|
||||||
return True
|
|
||||||
if producer_type.startswith("sharepoint.") and "file" in path_str:
|
|
||||||
return True
|
|
||||||
return False
|
|
||||||
|
|
||||||
|
|
||||||
def resolve_value_kind(graph: Dict[str, Any], ref: Dict[str, Any], *, _skip_upstream: bool = False) -> str:
|
|
||||||
"""Resolve condition valueKind for a DataRef against the workflow graph."""
|
"""Resolve condition valueKind for a DataRef against the workflow graph."""
|
||||||
if not isinstance(ref, dict):
|
if not isinstance(ref, dict):
|
||||||
return "unknown"
|
return "unknown"
|
||||||
|
|
@ -281,7 +230,6 @@ def resolve_value_kind(graph: Dict[str, Any], ref: Dict[str, Any], *, _skip_upst
|
||||||
return "string"
|
return "string"
|
||||||
return "file"
|
return "file"
|
||||||
|
|
||||||
if not _skip_upstream:
|
|
||||||
from modules.workflowAutomation.editor.upstreamPathsService import compute_upstream_paths
|
from modules.workflowAutomation.editor.upstreamPathsService import compute_upstream_paths
|
||||||
|
|
||||||
target_id = graph.get("targetNodeId") or producer_id
|
target_id = graph.get("targetNodeId") or producer_id
|
||||||
|
|
|
||||||
|
|
@ -4,7 +4,10 @@ from __future__ import annotations
|
||||||
|
|
||||||
from typing import Any, Dict, List, Set
|
from typing import Any, Dict, List, Set
|
||||||
|
|
||||||
from modules.workflowAutomation.editor.conditionOperators import catalog_type_to_value_kind, resolve_value_kind
|
from modules.workflowAutomation.editor._valueKindResolver import (
|
||||||
|
catalogTypeToValueKind,
|
||||||
|
resolveValueKindFromRef,
|
||||||
|
)
|
||||||
from modules.nodeCatalog.nodeDefinitions import STATIC_NODE_TYPES
|
from modules.nodeCatalog.nodeDefinitions import STATIC_NODE_TYPES
|
||||||
from modules.nodeCatalog.portTypes import PORT_TYPE_CATALOG, PortSchema, parse_graph_defined_output_schema
|
from modules.nodeCatalog.portTypes import PORT_TYPE_CATALOG, PortSchema, parse_graph_defined_output_schema
|
||||||
from modules.workflowAutomation.engine.graphUtils import buildConnectionMap, getLoopBodyNodeIds, getLoopDoneNodeIds
|
from modules.workflowAutomation.engine.graphUtils import buildConnectionMap, getLoopBodyNodeIds, getLoopDoneNodeIds
|
||||||
|
|
@ -170,14 +173,14 @@ def compute_upstream_paths(graph: Dict[str, Any], target_node_id: str) -> List[D
|
||||||
|
|
||||||
for entry in paths:
|
for entry in paths:
|
||||||
ct = str(entry.get("type") or "Any")
|
ct = str(entry.get("type") or "Any")
|
||||||
vk = catalog_type_to_value_kind(ct)
|
vk = catalogTypeToValueKind(ct)
|
||||||
if vk == "unknown":
|
if vk == "unknown":
|
||||||
ref = {
|
ref = {
|
||||||
"nodeId": entry.get("producerNodeId"),
|
"nodeId": entry.get("producerNodeId"),
|
||||||
"path": entry.get("path") or [],
|
"path": entry.get("path") or [],
|
||||||
}
|
}
|
||||||
graph_with_target = {**graph, "targetNodeId": target_node_id}
|
graph_with_target = {**graph, "targetNodeId": target_node_id}
|
||||||
vk = resolve_value_kind(graph_with_target, ref, _skip_upstream=True)
|
vk = resolveValueKindFromRef(graph_with_target, ref)
|
||||||
entry["valueKind"] = vk
|
entry["valueKind"] = vk
|
||||||
|
|
||||||
return paths
|
return paths
|
||||||
|
|
|
||||||
118
modules/workflowAutomation/engine/_runNotifications.py
Normal file
118
modules/workflowAutomation/engine/_runNotifications.py
Normal file
|
|
@ -0,0 +1,118 @@
|
||||||
|
# Copyright (c) 2025 Patrick Motsch
|
||||||
|
"""Run failure notification helpers.
|
||||||
|
|
||||||
|
Extracted from scheduler/mainScheduler to break the bidirectional import
|
||||||
|
cycle between executionEngine and mainScheduler. The engine calls
|
||||||
|
``notifyRunFailed`` directly (same subfolder, no cycle).
|
||||||
|
"""
|
||||||
|
|
||||||
|
import logging
|
||||||
|
from typing import Optional
|
||||||
|
|
||||||
|
from modules.shared.eventManagement import eventManager
|
||||||
|
|
||||||
|
logger = logging.getLogger(__name__)
|
||||||
|
|
||||||
|
|
||||||
|
def notifyRunFailed(
|
||||||
|
workflowId: str,
|
||||||
|
runId: str,
|
||||||
|
error: str,
|
||||||
|
mandateId: str = None,
|
||||||
|
workflowLabel: str = None,
|
||||||
|
) -> None:
|
||||||
|
"""Notify on workflow run failure: emit event, create in-app notification, trigger email subscription."""
|
||||||
|
try:
|
||||||
|
eventManager.emit("workflowAutomation.run.failed", {
|
||||||
|
"workflowId": workflowId,
|
||||||
|
"runId": runId,
|
||||||
|
"error": error,
|
||||||
|
"mandateId": mandateId,
|
||||||
|
})
|
||||||
|
logger.info("Emitted run.failed event for run %s (workflow %s)", runId, workflowId)
|
||||||
|
except Exception as e:
|
||||||
|
logger.warning("Failed to emit run.failed event: %s", e)
|
||||||
|
|
||||||
|
_createRunFailedNotification(workflowId, runId, error, mandateId, workflowLabel)
|
||||||
|
_triggerRunFailedSubscription(workflowId, runId, error, mandateId, workflowLabel)
|
||||||
|
|
||||||
|
|
||||||
|
def _createRunFailedNotification(
|
||||||
|
workflowId: str,
|
||||||
|
runId: str,
|
||||||
|
error: str,
|
||||||
|
mandateId: str = None,
|
||||||
|
workflowLabel: str = None,
|
||||||
|
) -> None:
|
||||||
|
"""Create in-app notification for the workflow creator."""
|
||||||
|
try:
|
||||||
|
from modules.interfaces.interfaceDbApp import getRootInterface
|
||||||
|
from modules.datamodels.datamodelNotification import UserNotification, NotificationType, NotificationStatus
|
||||||
|
|
||||||
|
rootInterface = getRootInterface()
|
||||||
|
if not rootInterface:
|
||||||
|
return
|
||||||
|
|
||||||
|
from modules.interfaces.interfaceWorkflowAutomation import _getWorkflowAutomationInterface
|
||||||
|
eventUser = rootInterface.getUserByUsername("event")
|
||||||
|
if not eventUser:
|
||||||
|
return
|
||||||
|
|
||||||
|
iface = _getWorkflowAutomationInterface(eventUser, mandateId or "", "")
|
||||||
|
wf = iface.getWorkflow(workflowId)
|
||||||
|
if not wf:
|
||||||
|
return
|
||||||
|
|
||||||
|
creatorId = wf.get("sysCreatedBy") if isinstance(wf, dict) else getattr(wf, "sysCreatedBy", None)
|
||||||
|
if not creatorId:
|
||||||
|
return
|
||||||
|
|
||||||
|
label = workflowLabel or (wf.get("label") if isinstance(wf, dict) else getattr(wf, "label", ""))
|
||||||
|
notification = UserNotification(
|
||||||
|
userId=creatorId,
|
||||||
|
type=NotificationType.SYSTEM,
|
||||||
|
status=NotificationStatus.UNREAD,
|
||||||
|
title="Workflow fehlgeschlagen",
|
||||||
|
message=f"Workflow '{label or workflowId}' ist fehlgeschlagen: {error[:200]}",
|
||||||
|
referenceType="AutoRun",
|
||||||
|
referenceId=runId,
|
||||||
|
icon="alert-triangle",
|
||||||
|
)
|
||||||
|
rootInterface.db.recordCreate(
|
||||||
|
model_class=UserNotification,
|
||||||
|
record=notification.model_dump(),
|
||||||
|
)
|
||||||
|
logger.info("Created in-app notification for user %s (run %s)", creatorId, runId)
|
||||||
|
except Exception as e:
|
||||||
|
logger.warning("Failed to create in-app run.failed notification: %s", e)
|
||||||
|
|
||||||
|
|
||||||
|
_onRunFailedCallback = None
|
||||||
|
|
||||||
|
|
||||||
|
def setOnRunFailedCallback(callback) -> None:
|
||||||
|
"""Set the callback for run failure notifications (injected by app.py)."""
|
||||||
|
global _onRunFailedCallback
|
||||||
|
_onRunFailedCallback = callback
|
||||||
|
|
||||||
|
|
||||||
|
def _triggerRunFailedSubscription(
|
||||||
|
workflowId: str,
|
||||||
|
runId: str,
|
||||||
|
error: str,
|
||||||
|
mandateId: str = None,
|
||||||
|
workflowLabel: str = None,
|
||||||
|
) -> None:
|
||||||
|
"""Trigger the messaging subscription for run failures via injected callback."""
|
||||||
|
if _onRunFailedCallback is None:
|
||||||
|
return
|
||||||
|
try:
|
||||||
|
_onRunFailedCallback(
|
||||||
|
workflowId=workflowId,
|
||||||
|
runId=runId,
|
||||||
|
error=error,
|
||||||
|
mandateId=mandateId,
|
||||||
|
workflowLabel=workflowLabel,
|
||||||
|
)
|
||||||
|
except Exception as e:
|
||||||
|
logger.warning("Failed to trigger run.failed subscription: %s", e)
|
||||||
|
|
@ -1540,15 +1540,6 @@ async def executeGraph(
|
||||||
duration_ms=_emailPauseMs,
|
duration_ms=_emailPauseMs,
|
||||||
)
|
)
|
||||||
logger.info("executeGraph paused for email wait (run %s, node %s)", e.runId, e.nodeId)
|
logger.info("executeGraph paused for email wait (run %s, node %s)", e.runId, e.nodeId)
|
||||||
try:
|
|
||||||
from modules.interfaces.interfaceDbApp import getRootInterface
|
|
||||||
from modules.workflowAutomation.scheduler.emailPoller import ensureRunning
|
|
||||||
root = getRootInterface()
|
|
||||||
event_user = root.getUserByUsername("event") if root else None
|
|
||||||
if event_user:
|
|
||||||
ensureRunning(event_user)
|
|
||||||
except Exception as poll_err:
|
|
||||||
logger.warning("Could not start email poller: %s", poll_err)
|
|
||||||
paused_at = datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ")
|
paused_at = datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ")
|
||||||
run_ctx = {
|
run_ctx = {
|
||||||
"connectionMap": context.get("connectionMap"),
|
"connectionMap": context.get("connectionMap"),
|
||||||
|
|
@ -1612,7 +1603,7 @@ async def executeGraph(
|
||||||
) if _wfObj else {}
|
) if _wfObj else {}
|
||||||
_shouldNotify = _wfDict.get("notifyOnFailure", True) if _wfDict else True
|
_shouldNotify = _wfDict.get("notifyOnFailure", True) if _wfDict else True
|
||||||
if _shouldNotify:
|
if _shouldNotify:
|
||||||
from modules.workflowAutomation.scheduler.mainScheduler import notifyRunFailed
|
from modules.workflowAutomation.engine._runNotifications import notifyRunFailed
|
||||||
notifyRunFailed(
|
notifyRunFailed(
|
||||||
workflowId or "", runId or "", str(e),
|
workflowId or "", runId or "", str(e),
|
||||||
mandateId=mandateId,
|
mandateId=mandateId,
|
||||||
|
|
|
||||||
|
|
@ -383,6 +383,21 @@ def _pathContainsWildcard(path: List[Any]) -> bool:
|
||||||
# (``featureInstanceRefMigration.materializeFeatureInstanceRefs``) writes the
|
# (``featureInstanceRefMigration.materializeFeatureInstanceRefs``) writes the
|
||||||
# envelope, the resolver unwraps it on its way to the action.
|
# envelope, the resolver unwraps it on its way to the action.
|
||||||
|
|
||||||
|
_STALE_FILE_CREATE_CONTEXT_PATHS = frozenset({
|
||||||
|
("responseData",),
|
||||||
|
("response",),
|
||||||
|
("merged",),
|
||||||
|
("documents", 0, "documentData"),
|
||||||
|
})
|
||||||
|
|
||||||
|
|
||||||
|
def remap_stale_presentation_ref_path(path: List[Any]) -> List[Any]:
|
||||||
|
"""Map legacy text-handover paths to unified presentation ``data``."""
|
||||||
|
if tuple(path) in _STALE_FILE_CREATE_CONTEXT_PATHS:
|
||||||
|
return ["data"]
|
||||||
|
return list(path)
|
||||||
|
|
||||||
|
|
||||||
_TYPED_REF_PRIMARY_FIELD = {
|
_TYPED_REF_PRIMARY_FIELD = {
|
||||||
"FeatureInstanceRef": "id",
|
"FeatureInstanceRef": "id",
|
||||||
"ConnectionRef": "id",
|
"ConnectionRef": "id",
|
||||||
|
|
@ -450,9 +465,6 @@ def resolveParameterReferences(
|
||||||
plist = list(path)
|
plist = list(path)
|
||||||
resolved = _get_by_path(data, plist)
|
resolved = _get_by_path(data, plist)
|
||||||
if resolved is None:
|
if resolved is None:
|
||||||
from modules.workflowAutomation.engine.pickNotPushMigration import (
|
|
||||||
remap_stale_presentation_ref_path,
|
|
||||||
)
|
|
||||||
alt_path = remap_stale_presentation_ref_path(plist)
|
alt_path = remap_stale_presentation_ref_path(plist)
|
||||||
if alt_path != plist:
|
if alt_path != plist:
|
||||||
resolved = _get_by_path(data, alt_path)
|
resolved = _get_by_path(data, alt_path)
|
||||||
|
|
|
||||||
|
|
@ -21,7 +21,11 @@ from modules.nodeCatalog.portTypes import (
|
||||||
PRIMARY_TEXT_HANDOVER_REF_PATH,
|
PRIMARY_TEXT_HANDOVER_REF_PATH,
|
||||||
resolve_output_schema_name,
|
resolve_output_schema_name,
|
||||||
)
|
)
|
||||||
from modules.workflowAutomation.engine.graphUtils import buildConnectionMap, getInputSources
|
from modules.workflowAutomation.engine.graphUtils import (
|
||||||
|
buildConnectionMap,
|
||||||
|
getInputSources,
|
||||||
|
remap_stale_presentation_ref_path,
|
||||||
|
)
|
||||||
|
|
||||||
logger = logging.getLogger(__name__)
|
logger = logging.getLogger(__name__)
|
||||||
|
|
||||||
|
|
@ -243,20 +247,6 @@ def materializeRecommendedDataPickRef(graph: Dict[str, Any]) -> Dict[str, Any]:
|
||||||
return g
|
return g
|
||||||
|
|
||||||
|
|
||||||
_STALE_FILE_CREATE_CONTEXT_PATHS = frozenset({
|
|
||||||
("responseData",),
|
|
||||||
("response",),
|
|
||||||
("merged",),
|
|
||||||
("documents", 0, "documentData"),
|
|
||||||
})
|
|
||||||
|
|
||||||
|
|
||||||
def remap_stale_presentation_ref_path(path: List[Any]) -> List[Any]:
|
|
||||||
"""Map legacy text-handover paths to unified presentation ``data``."""
|
|
||||||
if tuple(path) in _STALE_FILE_CREATE_CONTEXT_PATHS:
|
|
||||||
return ["data"]
|
|
||||||
return list(path)
|
|
||||||
|
|
||||||
|
|
||||||
def _normalize_presentation_refs_in_value(val: Any) -> Any:
|
def _normalize_presentation_refs_in_value(val: Any) -> Any:
|
||||||
"""Rewrite stale ref paths inside ``contextBuilder`` lists or bare refs."""
|
"""Rewrite stale ref paths inside ``contextBuilder`` lists or bare refs."""
|
||||||
|
|
|
||||||
|
|
@ -57,8 +57,8 @@ def _getWorkflowAutomationServices(
|
||||||
|
|
||||||
ctx = ServiceCenterContext(
|
ctx = ServiceCenterContext(
|
||||||
user=user,
|
user=user,
|
||||||
mandate_id=mandateId,
|
mandateId=mandateId,
|
||||||
feature_instance_id=featureInstanceId,
|
featureInstanceId=featureInstanceId,
|
||||||
workflow=_workflow,
|
workflow=_workflow,
|
||||||
)
|
)
|
||||||
return ServicesBag(ctx, lambda key: getService(key, ctx))
|
return ServicesBag(ctx, lambda key: getService(key, ctx))
|
||||||
|
|
|
||||||
|
|
@ -6,6 +6,8 @@ from modules.workflowAutomation.scheduler.mainScheduler import (
|
||||||
stop,
|
stop,
|
||||||
syncNow,
|
syncNow,
|
||||||
setMainLoop,
|
setMainLoop,
|
||||||
|
)
|
||||||
|
from modules.workflowAutomation.engine._runNotifications import (
|
||||||
notifyRunFailed,
|
notifyRunFailed,
|
||||||
setOnRunFailedCallback,
|
setOnRunFailedCallback,
|
||||||
)
|
)
|
||||||
|
|
|
||||||
|
|
@ -263,6 +263,12 @@ class WorkflowScheduler:
|
||||||
"WorkflowScheduler: executed workflow %s success=%s paused=%s",
|
"WorkflowScheduler: executed workflow %s success=%s paused=%s",
|
||||||
workflowId, result.get("success"), result.get("paused"),
|
workflowId, result.get("success"), result.get("paused"),
|
||||||
)
|
)
|
||||||
|
if result.get("waitReason") == "email":
|
||||||
|
try:
|
||||||
|
from modules.workflowAutomation.scheduler.emailPoller import ensureRunning
|
||||||
|
ensureRunning(eventUser)
|
||||||
|
except Exception as pollErr:
|
||||||
|
logger.warning("WorkflowScheduler: could not start email poller: %s", pollErr)
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
logger.exception("WorkflowScheduler: failed to execute workflow %s: %s", workflowId, e)
|
logger.exception("WorkflowScheduler: failed to execute workflow %s: %s", workflowId, e)
|
||||||
|
|
||||||
|
|
@ -333,94 +339,10 @@ def _cronToIntervalSeconds(cron: str):
|
||||||
return None
|
return None
|
||||||
|
|
||||||
|
|
||||||
def notifyRunFailed(workflowId: str, runId: str, error: str, mandateId: str = None, workflowLabel: str = None) -> None:
|
from modules.workflowAutomation.engine._runNotifications import ( # noqa: E402 — re-export
|
||||||
"""Notify on workflow run failure: emit event, create in-app notification, trigger email subscription."""
|
notifyRunFailed,
|
||||||
try:
|
setOnRunFailedCallback,
|
||||||
eventManager.emit("workflowAutomation.run.failed", {
|
|
||||||
"workflowId": workflowId,
|
|
||||||
"runId": runId,
|
|
||||||
"error": error,
|
|
||||||
"mandateId": mandateId,
|
|
||||||
})
|
|
||||||
logger.info("Emitted run.failed event for run %s (workflow %s)", runId, workflowId)
|
|
||||||
except Exception as e:
|
|
||||||
logger.warning("Failed to emit run.failed event: %s", e)
|
|
||||||
|
|
||||||
_createRunFailedNotification(workflowId, runId, error, mandateId, workflowLabel)
|
|
||||||
_triggerRunFailedSubscription(workflowId, runId, error, mandateId, workflowLabel)
|
|
||||||
|
|
||||||
|
|
||||||
def _createRunFailedNotification(
|
|
||||||
workflowId: str, runId: str, error: str, mandateId: str = None, workflowLabel: str = None
|
|
||||||
) -> None:
|
|
||||||
"""Create in-app notification for the workflow creator."""
|
|
||||||
try:
|
|
||||||
from modules.interfaces.interfaceDbApp import getRootInterface
|
|
||||||
from modules.datamodels.datamodelNotification import UserNotification, NotificationType, NotificationStatus
|
|
||||||
|
|
||||||
rootInterface = getRootInterface()
|
|
||||||
if not rootInterface:
|
|
||||||
return
|
|
||||||
|
|
||||||
from modules.interfaces.interfaceWorkflowAutomation import _getWorkflowAutomationInterface
|
|
||||||
eventUser = rootInterface.getUserByUsername("event")
|
|
||||||
if not eventUser:
|
|
||||||
return
|
|
||||||
|
|
||||||
iface = _getWorkflowAutomationInterface(eventUser, mandateId or "", "")
|
|
||||||
wf = iface.getWorkflow(workflowId)
|
|
||||||
if not wf:
|
|
||||||
return
|
|
||||||
|
|
||||||
creatorId = wf.get("sysCreatedBy") if isinstance(wf, dict) else getattr(wf, "sysCreatedBy", None)
|
|
||||||
if not creatorId:
|
|
||||||
return
|
|
||||||
|
|
||||||
label = workflowLabel or (wf.get("label") if isinstance(wf, dict) else getattr(wf, "label", ""))
|
|
||||||
notification = UserNotification(
|
|
||||||
userId=creatorId,
|
|
||||||
type=NotificationType.SYSTEM,
|
|
||||||
status=NotificationStatus.UNREAD,
|
|
||||||
title="Workflow fehlgeschlagen",
|
|
||||||
message=f"Workflow '{label or workflowId}' ist fehlgeschlagen: {error[:200]}",
|
|
||||||
referenceType="AutoRun",
|
|
||||||
referenceId=runId,
|
|
||||||
icon="alert-triangle",
|
|
||||||
)
|
)
|
||||||
rootInterface.db.recordCreate(
|
|
||||||
model_class=UserNotification,
|
|
||||||
record=notification.model_dump(),
|
|
||||||
)
|
|
||||||
logger.info("Created in-app notification for user %s (run %s)", creatorId, runId)
|
|
||||||
except Exception as e:
|
|
||||||
logger.warning("Failed to create in-app run.failed notification: %s", e)
|
|
||||||
|
|
||||||
|
|
||||||
_onRunFailedCallback = None
|
|
||||||
|
|
||||||
|
|
||||||
def setOnRunFailedCallback(callback) -> None:
|
|
||||||
"""Set the callback for run failure notifications (injected by app.py)."""
|
|
||||||
global _onRunFailedCallback
|
|
||||||
_onRunFailedCallback = callback
|
|
||||||
|
|
||||||
|
|
||||||
def _triggerRunFailedSubscription(
|
|
||||||
workflowId: str, runId: str, error: str, mandateId: str = None, workflowLabel: str = None
|
|
||||||
) -> None:
|
|
||||||
"""Trigger the messaging subscription for run failures via injected callback."""
|
|
||||||
if _onRunFailedCallback is None:
|
|
||||||
return
|
|
||||||
try:
|
|
||||||
_onRunFailedCallback(
|
|
||||||
workflowId=workflowId,
|
|
||||||
runId=runId,
|
|
||||||
error=error,
|
|
||||||
mandateId=mandateId,
|
|
||||||
workflowLabel=workflowLabel,
|
|
||||||
)
|
|
||||||
except Exception as e:
|
|
||||||
logger.warning("Failed to trigger run.failed subscription: %s", e)
|
|
||||||
|
|
||||||
|
|
||||||
# Module-level singleton
|
# Module-level singleton
|
||||||
|
|
|
||||||
|
|
@ -41,7 +41,7 @@ def generateDynamicPlanSelectionPrompt(services, context: Any, learningEngine=No
|
||||||
# Add adaptive learning context if available
|
# Add adaptive learning context if available
|
||||||
adaptiveContext = {}
|
adaptiveContext = {}
|
||||||
if learningEngine:
|
if learningEngine:
|
||||||
workflowId = getattr(context, 'workflow_id', 'unknown')
|
workflowId = getattr(context, 'workflowId', 'unknown')
|
||||||
userPrompt = extractUserPrompt(context)
|
userPrompt = extractUserPrompt(context)
|
||||||
adaptiveContext = learningEngine.getAdaptiveContextForActionSelection(workflowId, userPrompt)
|
adaptiveContext = learningEngine.getAdaptiveContextForActionSelection(workflowId, userPrompt)
|
||||||
|
|
||||||
|
|
@ -226,7 +226,7 @@ Excludes documents/connections/history entirely.
|
||||||
# Add adaptive learning context if available
|
# Add adaptive learning context if available
|
||||||
adaptiveContext = {}
|
adaptiveContext = {}
|
||||||
if learningEngine:
|
if learningEngine:
|
||||||
workflowId = getattr(context, 'workflow_id', 'unknown')
|
workflowId = getattr(context, 'workflowId', 'unknown')
|
||||||
adaptiveContext = learningEngine.getAdaptiveContextForParameters(workflowId, compoundActionName, parametersContext or "")
|
adaptiveContext = learningEngine.getAdaptiveContextForParameters(workflowId, compoundActionName, parametersContext or "")
|
||||||
|
|
||||||
if adaptiveContext:
|
if adaptiveContext:
|
||||||
|
|
|
||||||
|
|
@ -78,7 +78,7 @@ def _buildInterface(db: _FakeDb) -> AppObjects:
|
||||||
def _stubCopySystemRoles():
|
def _stubCopySystemRoles():
|
||||||
"""Avoid touching the bootstrap module (which would need a real DB)."""
|
"""Avoid touching the bootstrap module (which would need a real DB)."""
|
||||||
with patch(
|
with patch(
|
||||||
"modules.interfaces.interfaceBootstrap.copySystemRolesToMandate",
|
"modules.interfaces.interfaceRbac.copySystemRolesToMandate",
|
||||||
return_value=0,
|
return_value=0,
|
||||||
):
|
):
|
||||||
yield
|
yield
|
||||||
|
|
|
||||||
Loading…
Reference in a new issue