From 9f46ca3b03c4cac11b337f7583d3b7b01a88ba37 Mon Sep 17 00:00:00 2001 From: ValueOn AG Date: Wed, 3 Dec 2025 07:31:51 +0100 Subject: [PATCH 1/3] fixed import chain, removed invalid imports by moving logic between modules --- app.py | 11 +- modules/.$DEPENDENCY_DIAGRAM.drawio.bkp | 326 ++++++++++ modules/AUTOMATION_FEATURE_ANALYSIS.md | 407 ++++++++++++ modules/BIDIRECTIONAL_IMPORTS.md | 406 ++++++++++++ modules/DEPENDENCY_DIAGRAM.drawio | 1 + modules/FEATURES_TO_INTERFACES_IMPORTS.md | 199 ++++++ modules/features/automation/__init__.py | 12 + modules/features/automation/mainAutomation.py | 287 +++++++++ .../features/automation/subAutomationUtils.py | 108 ++++ .../features/chatAlthaus/COMPONENT_DIAGRAM.md | 211 ------- modules/features/featuresLifecycle.py | 52 +- modules/interfaces/interfaceAiObjects.py | 476 +------------- modules/interfaces/interfaceDbChatObjects.py | 507 +++++---------- modules/routes/routeAdminAutomationEvents.py | 16 +- modules/services/__init__.py | 3 + modules/services/serviceAi/mainServiceAi.py | 24 +- .../services/serviceChat/mainServiceChat.py | 7 +- .../mainServiceExtraction.py | 499 +++++++++++++-- .../services/serviceExtraction/subPipeline.py | 3 +- .../serviceSecurity/mainServiceSecurity.py | 128 ++++ .../mainServiceSharepoint.py | 9 +- .../services/serviceUtils/mainServiceUtils.py | 6 +- modules/shared/callbackRegistry.py | 70 +++ modules/shared/debugLogger.py | 128 ---- modules/workflows/methods/methodAi.py | 583 +----------------- modules/workflows/methods/methodContext.py | 337 ++++++++++ .../processing/core/messageCreator.py | 10 +- .../shared/promptGenerationActionsDynamic.py | 34 +- .../shared/promptGenerationTaskplan.py | 9 +- .../workflows/processing/workflowProcessor.py | 8 +- modules/workflows/workflowManager.py | 44 +- 31 files changed, 3058 insertions(+), 1863 deletions(-) create mode 100644 modules/.$DEPENDENCY_DIAGRAM.drawio.bkp create mode 100644 modules/AUTOMATION_FEATURE_ANALYSIS.md create mode 100644 modules/BIDIRECTIONAL_IMPORTS.md create mode 100644 modules/DEPENDENCY_DIAGRAM.drawio create mode 100644 modules/FEATURES_TO_INTERFACES_IMPORTS.md create mode 100644 modules/features/automation/__init__.py create mode 100644 modules/features/automation/mainAutomation.py create mode 100644 modules/features/automation/subAutomationUtils.py delete mode 100644 modules/features/chatAlthaus/COMPONENT_DIAGRAM.md create mode 100644 modules/services/serviceSecurity/mainServiceSecurity.py create mode 100644 modules/shared/callbackRegistry.py create mode 100644 modules/workflows/methods/methodContext.py diff --git a/app.py b/app.py index a167503c..9ace64b5 100644 --- a/app.py +++ b/app.py @@ -16,6 +16,7 @@ from datetime import datetime from modules.shared.configuration import APP_CONFIG from modules.shared.eventManagement import eventManager from modules.features import featuresLifecycle as featuresLifecycle +from modules.interfaces.interfaceDbAppObjects import getRootInterface class DailyRotatingFileHandler(RotatingFileHandler): """ @@ -275,15 +276,21 @@ instanceLabel = APP_CONFIG.get("APP_ENV_LABEL") async def lifespan(app: FastAPI): logger.info("Application is starting up") + # Get event user for feature lifecycle (system-level user for background operations) + rootInterface = getRootInterface() + eventUser = rootInterface.getUserByUsername("event") + if not eventUser: + logger.error("Could not get event user - some features may not start properly") + # --- Init Managers --- - await featuresLifecycle.start() + await featuresLifecycle.start(eventUser) eventManager.start() yield # --- Stop Managers --- eventManager.stop() - await featuresLifecycle.stop() + await featuresLifecycle.stop(eventUser) logger.info("Application has been shut down") diff --git a/modules/.$DEPENDENCY_DIAGRAM.drawio.bkp b/modules/.$DEPENDENCY_DIAGRAM.drawio.bkp new file mode 100644 index 00000000..5d8a062a --- /dev/null +++ b/modules/.$DEPENDENCY_DIAGRAM.drawio.bkp @@ -0,0 +1,326 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/modules/AUTOMATION_FEATURE_ANALYSIS.md b/modules/AUTOMATION_FEATURE_ANALYSIS.md new file mode 100644 index 00000000..37dd4e75 --- /dev/null +++ b/modules/AUTOMATION_FEATURE_ANALYSIS.md @@ -0,0 +1,407 @@ +# Automation Feature Analysis: Moving Automation Handler to Features Layer + +## Executive Summary + +**Status: ✅ HIGHLY RECOMMENDED - Architectural Improvement** + +Moving automation workflow handler functionality from `interfaces/interfaceDbChatObjects.py` to a new feature in `features/` is **architecturally correct** and aligns with separation of concerns. + +--- + +## Current Architecture Analysis + +### Current Location: `interfaces/interfaceDbChatObjects.py` + +**Automation-related methods:** +1. `executeAutomation(automationId: str)` - Executes automation workflow immediately (test mode) +2. `syncAutomationEvents()` - Syncs scheduler with all active automations +3. `_createAutomationEventHandler(automationId: str)` - Creates event handler for scheduled execution +4. `_parseScheduleToCron(schedule: str)` - Parses schedule string to cron kwargs +5. `_planToPrompt(plan: Dict)` - Converts plan structure to prompt string +6. `_replacePlaceholders(template: str, placeholders: Dict)` - Replaces placeholders in template + +**Dependencies:** +- Uses `getAutomationDefinition()` - Database access (should stay in interface) +- Uses `chatStart()` from `features.chatPlayground` - Already imports from features +- Uses `eventManager` from `shared.eventManagement` - Foundation layer +- Creates workflows using `WorkflowModeEnum.WORKFLOW_AUTOMATION` + +--- + +## Why This Should Be a Feature + +### 1. **Business Logic vs. Data Access** + +**Current Problem:** +- Automation execution logic is **business logic** (orchestration, workflow creation) +- It's mixed with **data access** (interface layer) +- Interface layer should only provide data access, not business orchestration + +**After Move:** +- Interface layer: `getAutomationDefinition()`, `saveAutomationDefinition()` (data access) +- Feature layer: `executeAutomation()`, `syncAutomationEvents()` (business logic) + +### 2. **Feature Pattern Consistency** + +**Existing Features Pattern:** +- `features/chatPlayground/` - Chat workflow execution +- `features/chatAlthaus/` - Scheduled data updates +- `features/syncDelta/` - Sync management +- `features/neutralizePlayground/` - Neutralization workflows + +**Automation Handler Pattern:** +- Scheduled execution (like `chatAlthaus`) +- Workflow orchestration (like `chatPlayground`) +- Event-driven (like `syncDelta`) + +**Conclusion:** Automation handler fits the feature pattern perfectly. + +### 3. **Dependency Direction** + +**Current Violation:** +- `interfaces/` imports from `features/` (line 1927: `from modules.features.chatPlayground.mainChatPlayground import chatStart`) +- This creates bidirectional dependency: `interfaces/ ↔ features/` + +**After Move:** +- `features/automation/` imports from `interfaces/` (correct direction) +- `features/automation/` imports from `features/chatPlayground/` (feature-to-feature) +- Eliminates `interfaces/ → features/` import + +### 4. **Separation of Concerns** + +**Interface Layer Should:** +- ✅ Provide data access (`getAutomationDefinition`, `saveAutomationDefinition`) +- ✅ Handle CRUD operations +- ❌ NOT orchestrate workflows +- ❌ NOT manage scheduling +- ❌ NOT execute business logic + +**Feature Layer Should:** +- ✅ Orchestrate workflows +- ✅ Manage scheduling +- ✅ Execute business logic +- ✅ Coordinate between services and interfaces + +--- + +## Proposed Architecture + +### New Structure: `features/automation/` + +``` +features/automation/ + ├── mainAutomation.py # Main automation service + │ ├── executeAutomation() # Execute automation workflow + │ ├── syncAutomationEvents() # Sync scheduler with automations + │ └── _createAutomationEventHandler() # Create event handler + └── subAutomationUtils.py # Utility functions + ├── _parseScheduleToCron() # Parse schedule to cron + ├── _planToPrompt() # Convert plan to prompt + └── _replacePlaceholders() # Replace template placeholders +``` + +### Interface Layer (Keep) + +``` +interfaces/interfaceDbChatObjects.py + ├── getAutomationDefinition() # Data access - KEEP + ├── saveAutomationDefinition() # Data access - KEEP + └── (other CRUD methods) # Data access - KEEP +``` + +### Feature Lifecycle Integration + +``` +features/featuresLifecycle.py + ├── start() + │ └── from features.automation import mainAutomation + │ mainAutomation.startScheduler(eventUser) + └── stop() + └── mainAutomation.stopScheduler() +``` + +--- + +## Detailed Functionality Analysis + +### Functions to Move + +#### 1. `executeAutomation(automationId: str)` → `features/automation/mainAutomation.py` + +**Current Implementation:** +- Loads automation definition (calls interface method) +- Replaces placeholders in template +- Creates UserInputRequest +- Calls `chatStart()` from `features.chatPlayground` +- Returns ChatWorkflow + +**Dependencies:** +- `getAutomationDefinition()` - Interface method (import from interface) +- `_replacePlaceholders()` - Utility (move to feature) +- `_planToPrompt()` - Utility (move to feature) +- `chatStart()` - Feature method (import from feature) +- `getInterface()` - Interface factory (import from interface) + +**After Move:** +```python +# features/automation/mainAutomation.py +from modules.interfaces.interfaceDbChatObjects import getInterface as getChatInterface +from modules.interfaces.interfaceDbAppObjects import getInterface as getAppInterface +from modules.features.chatPlayground.mainChatPlayground import chatStart +from .subAutomationUtils import replacePlaceholders, planToPrompt + +async def executeAutomation(automationId: str, chatInterface) -> ChatWorkflow: + """Execute automation workflow immediately.""" + # Load automation (uses interface) + automation = chatInterface.getAutomationDefinition(automationId) + # ... rest of logic +``` + +#### 2. `syncAutomationEvents()` → `features/automation/mainAutomation.py` + +**Current Implementation:** +- Gets all automation definitions (calls interface method) +- Parses schedules +- Registers cron jobs with eventManager +- Creates event handlers + +**Dependencies:** +- `getRecordset()` - Interface method (via chatInterface) +- `_parseScheduleToCron()` - Utility (move to feature) +- `_createAutomationEventHandler()` - Handler creation (move to feature) +- `eventManager` - Foundation layer (import from shared) + +**After Move:** +```python +# features/automation/mainAutomation.py +from modules.shared.eventManagement import eventManager +from modules.interfaces.interfaceDbChatObjects import getInterface as getChatInterface +from .subAutomationUtils import parseScheduleToCron + +async def syncAutomationEvents(chatInterface) -> Dict[str, Any]: + """Sync scheduler with all active automations.""" + # Get automations (uses interface) + allAutomations = chatInterface.db.getRecordset(AutomationDefinition) + # ... rest of logic +``` + +#### 3. `_createAutomationEventHandler(automationId: str)` → `features/automation/mainAutomation.py` + +**Current Implementation:** +- Creates async handler function +- Gets event user +- Loads automation +- Executes automation with creator user context + +**Dependencies:** +- `getRootInterface()` - Interface factory (import from interface) +- `getInterface()` - Interface factories (import from interfaces) +- `executeAutomation()` - Will be in same module + +**After Move:** +```python +# features/automation/mainAutomation.py +def createAutomationEventHandler(automationId: str): + """Create event handler function for scheduled automation.""" + async def handler(): + # Uses interfaces and executeAutomation from same module + await executeAutomation(automationId, eventInterface) + return handler +``` + +#### 4. Utility Functions → `features/automation/subAutomationUtils.py` + +**Functions:** +- `_parseScheduleToCron(schedule: str)` - Parse schedule to cron kwargs +- `_planToPrompt(plan: Dict)` - Convert plan to prompt string +- `_replacePlaceholders(template: str, placeholders: Dict)` - Replace placeholders + +**Dependencies:** +- No external dependencies (pure utility functions) + +--- + +## Migration Plan + +### Phase 1: Create Feature Structure + +1. Create `features/automation/` directory +2. Create `features/automation/__init__.py` +3. Create `features/automation/mainAutomation.py` +4. Create `features/automation/subAutomationUtils.py` + +### Phase 2: Move Functions + +1. Move utility functions to `subAutomationUtils.py` +2. Move `executeAutomation()` to `mainAutomation.py` +3. Move `syncAutomationEvents()` to `mainAutomation.py` +4. Move `_createAutomationEventHandler()` to `mainAutomation.py` + +### Phase 3: Update Dependencies + +1. Update `features/featuresLifecycle.py` to use new feature +2. Update `routes/routeAdminAutomationEvents.py` to use new feature +3. Update any other call sites + +### Phase 4: Cleanup Interface + +1. Remove moved functions from `interfaceDbChatObjects.py` +2. Keep only data access methods (`getAutomationDefinition`, etc.) +3. Remove import from `features.chatPlayground` from interface + +### Phase 5: Update Documentation + +1. Update `BIDIRECTIONAL_IMPORTS.md` to reflect resolved dependency +2. Document new feature structure + +--- + +## Benefits + +### ✅ Architectural Benefits + +1. **Correct Separation of Concerns** + - Interface layer: Data access only + - Feature layer: Business logic and orchestration + +2. **Resolves Bidirectional Dependency** + - Eliminates `interfaces/ → features/` import + - Only `features/ → interfaces/` remains (correct direction) + +3. **Consistency with Existing Patterns** + - Matches other feature implementations + - Follows established architecture + +4. **Better Testability** + - Feature logic can be tested independently + - Interface layer remains focused + +### ✅ Maintainability Benefits + +1. **Clearer Code Organization** + - Automation logic in one place + - Easier to find and modify + +2. **Reduced Coupling** + - Interface layer doesn't depend on features + - Features depend on interfaces (correct direction) + +3. **Easier to Extend** + - New automation features can be added to feature module + - Interface layer remains stable + +--- + +## Risks and Considerations + +### 🟢 Low Risk + +- **Functionality preservation** - Logic doesn't change, only location +- **Interface methods remain** - Data access methods stay in interface +- **Lazy imports** - Already using lazy imports for event handlers + +### 🟡 Medium Risk + +- **Call site updates** - Need to update routes and lifecycle +- **Interface method access** - Feature needs to call interface methods +- **Event user context** - Need to ensure proper user context handling + +### 🔴 Potential Issues + +1. **Interface method access** - Feature needs `chatInterface` instance + - **Solution:** Pass interface instance as parameter or create in feature + +2. **Event handler context** - Event handlers need interface access + - **Solution:** Create interface instances in handler (already doing this) + +3. **Backward compatibility** - Existing code calling `chatInterface.executeAutomation()` + - **Solution:** Update all call sites, or create wrapper method in interface (deprecated) + +--- + +## Call Sites Analysis + +### Current Call Sites + +1. **`features/featuresLifecycle.py` (line 20)** + ```python + await chatInterface.syncAutomationEvents() + ``` + **Update:** Import and call feature directly + +2. **`routes/routeAdminAutomationEvents.py` (line 97)** + ```python + result = await chatInterface.syncAutomationEvents() + ``` + **Update:** Import and call feature directly + +3. **`interfaces/interfaceDbChatObjects.py` (lines 1683, 1714, 1744)** + ```python + asyncio.create_task(self.syncAutomationEvents()) + ``` + **Update:** Remove (will be handled by feature lifecycle) + +4. **`_createAutomationEventHandler()` (line 2105)** + ```python + await creatorInterface.executeAutomation(automationId) + ``` + **Update:** Call feature method instead + +### Proposed Call Sites + +1. **`features/featuresLifecycle.py`** + ```python + from modules.features.automation import mainAutomation + await mainAutomation.syncAutomationEvents(chatInterface) + ``` + +2. **`routes/routeAdminAutomationEvents.py`** + ```python + from modules.features.automation import mainAutomation + result = await mainAutomation.syncAutomationEvents(chatInterface) + ``` + +3. **`features/automation/mainAutomation.py` (event handler)** + ```python + from .mainAutomation import executeAutomation + await executeAutomation(automationId, eventInterface) + ``` + +--- + +## Implementation Checklist + +- [ ] Create `features/automation/` directory structure +- [ ] Create `features/automation/__init__.py` +- [ ] Create `features/automation/mainAutomation.py` +- [ ] Create `features/automation/subAutomationUtils.py` +- [ ] Move `_parseScheduleToCron()` to `subAutomationUtils.py` +- [ ] Move `_planToPrompt()` to `subAutomationUtils.py` +- [ ] Move `_replacePlaceholders()` to `subAutomationUtils.py` +- [ ] Move `executeAutomation()` to `mainAutomation.py` +- [ ] Move `syncAutomationEvents()` to `mainAutomation.py` +- [ ] Move `_createAutomationEventHandler()` to `mainAutomation.py` +- [ ] Update `features/featuresLifecycle.py` to use feature +- [ ] Update `routes/routeAdminAutomationEvents.py` to use feature +- [ ] Remove moved functions from `interfaceDbChatObjects.py` +- [ ] Remove `features.chatPlayground` import from `interfaceDbChatObjects.py` +- [ ] Update `BIDIRECTIONAL_IMPORTS.md` +- [ ] Test automation execution (manual) +- [ ] Test automation scheduling (scheduled) +- [ ] Verify no circular dependencies + +--- + +## Conclusion + +**✅ STRONGLY RECOMMENDED: Move to Features Layer** + +This refactoring is: +- **Architecturally correct** - Business logic belongs in features, not interfaces +- **Resolves dependency violation** - Eliminates `interfaces/ → features/` import +- **Consistent with patterns** - Matches existing feature implementations +- **Low risk** - Logic doesn't change, only location +- **Improves maintainability** - Clearer separation of concerns + +**Recommendation: PROCEED** with moving automation handler functionality to `features/automation/` following the plan above. + diff --git a/modules/BIDIRECTIONAL_IMPORTS.md b/modules/BIDIRECTIONAL_IMPORTS.md new file mode 100644 index 00000000..55a8bd43 --- /dev/null +++ b/modules/BIDIRECTIONAL_IMPORTS.md @@ -0,0 +1,406 @@ +# Bidirectional Import Analysis + +## Summary + +After refactoring extraction functions and automation handler, **ALL bidirectional dependencies have been RESOLVED**: + +**Current Status:** +- ✅ **interfaces/ → services/**: **RESOLVED** (no imports) +- ✅ **interfaces/ → features/**: **RESOLVED** (uses callback registry, no direct imports) +- ✅ **services/ → interfaces/**: **UNIDIRECTIONAL** (correct dependency direction) +- ✅ **services/ → features/**: **NONE** (no imports) +- ✅ **features/ → interfaces/**: **UNIDIRECTIONAL** (correct dependency direction) +- ✅ **features/ → services/**: **1 lazy import** (correct direction) + +**Result:** ✅ **ZERO VIOLATIONS** - Perfect architectural compliance achieved. + +--- + +## Dependency Diagram + +### Mermaid Diagram + +```mermaid +graph TB + %% Foundation Layer (no dependencies) + shared[shared/
Foundation] + datamodels[datamodels/
Foundation] + aicore[aicore/
Infrastructure] + connectors[connectors/
Infrastructure] + + %% Data Layer + interfaces[interfaces/
Data Access
✅ No violations] + + %% Business Logic Layer + services[services/
Business Logic
✅ Unidirectional] + workflows[workflows/
Business Logic] + + %% Feature Layer + features[features/
Features
✅ Unidirectional] + + %% API Layer + routes[routes/
API Layer] + security[security/
Security] + + %% Foundation dependencies + datamodels -->|imports| shared + aicore -->|imports| datamodels + aicore -->|imports| shared + connectors -->|imports| datamodels + connectors -->|imports| shared + + %% Interface layer (foundation only) + interfaces -->|imports| aicore + interfaces -->|imports| connectors + interfaces -->|imports| datamodels + interfaces -.->|callbackRegistry| shared + + %% Service layer (interfaces only) + services -->|✅ imports| interfaces + services -->|imports| aicore + services -->|imports| datamodels + services -->|imports| security + services -->|imports| shared + + %% Workflow layer + workflows -->|imports| aicore + workflows -->|imports| datamodels + workflows -->|imports| services + workflows -->|imports| shared + + %% Feature layer (interfaces + services) + features -->|✅ imports| interfaces + features -->|✅ imports| services + features -->|imports| datamodels + features -->|imports| workflows + features -->|imports| shared + + %% API layer + routes -->|imports| interfaces + routes -->|imports| features + routes -->|imports| services + routes -->|imports| security + routes -->|imports| datamodels + routes -->|imports| shared + + %% Security layer + security -->|imports| interfaces + security -->|imports| datamodels + security -->|imports| shared + + %% Styling + classDef foundation fill:#e1f5ff,stroke:#01579b,stroke-width:2px + classDef data fill:#f3e5f5,stroke:#4a148c,stroke-width:3px + classDef business fill:#e8f5e9,stroke:#1b5e20,stroke-width:2px + classDef feature fill:#fff3e0,stroke:#e65100,stroke-width:2px + classDef api fill:#fce4ec,stroke:#880e4f,stroke-width:2px + + class shared,datamodels,aicore,connectors foundation + class interfaces data + class services,workflows business + class features feature + class routes,security api +``` + +### Draw.io Diagram + +A detailed draw.io diagram is available in `DEPENDENCY_DIAGRAM.drawio` with: +- Color-coded layers (Foundation, Data, Business Logic, Features, API) +- Arrow directions showing import relationships +- ✅ markers on correct dependency directions +- Dashed line for callback registry pattern +- Status box showing zero violations + +**Key Visual Elements:** +- **Thick green arrows (✅)**: Correct dependency directions (services→interfaces, features→interfaces, features→services) +- **Dashed purple line**: Callback registry pattern (interfaces→shared, decoupled from features) +- **Color coding**: Each layer has distinct colors for easy identification +- **Status indicators**: ✅ markers show compliance, "No violations" labels confirm architectural correctness + +--- + +## Detailed Analysis + +### 1. interfaces/ → services/ ✅ RESOLVED + +**Current State:** +- ✅ **No imports from services/** in `interfaces/` +- All extraction-related functions moved to `services/serviceExtraction/` +- Dependency violations resolved + +**Impact:** Major architectural improvement - interfaces no longer depend on services. + +--- + +### 2. interfaces/ → features/ ✅ RESOLVED + +**Previous State:** +- `interfaceDbChatObjects.py` (line 1754): Lazy import in `_triggerAutomationSync()` helper method + ```python + from modules.features.automation import syncAutomationEvents + ``` + +**Current State:** +- ✅ **No imports from features/** in `interfaces/` +- Uses callback registry pattern (`shared.callbackRegistry`) for decoupled notifications +- Interface triggers callbacks without knowing which features are listening +- Feature registers callback in `featuresLifecycle.py` startup + +**Refactoring:** +- Created `shared/callbackRegistry.py` - decoupled event notification system +- Interface calls `callbackRegistry.trigger('automation.changed', self)` instead of importing feature +- Feature registers callback on startup: `callbackRegistry.register('automation.changed', onAutomationChanged)` + +**Impact:** Perfect separation - interface doesn't know about features, uses shared callback registry. + +--- + +### 3. services/ → interfaces/ ✅ CORRECT DIRECTION + +**Current State:** +- `serviceAi/mainServiceAi.py`: Imports `AiObjects` from `interfaceAiObjects` +- `serviceExtraction/mainServiceExtraction.py`: Lazy import from `interfaceDbComponentObjects` +- `serviceUtils/mainServiceUtils.py`: Lazy import from `interfaceDbChatObjects` +- `serviceTicket/mainServiceTicket.py`: Imports from `interfaceTicketObjects` +- `services/__init__.py`: Lazy imports from multiple interfaces + +**Impact:** This is **correct** - services should use interfaces for data access. This follows the dependency rule: `services/` → `interfaces/` ✅ + +**Note:** This is **unidirectional** (services → interfaces), not bidirectional. + +--- + +### 4. services/ → features/ ✅ NONE + +**Current State:** +- ✅ **No imports from features/** in `services/` + +**Impact:** Services correctly do not depend on features. + +--- + +### 5. features/ → interfaces/ ✅ CORRECT DIRECTION + +**Current State:** +- `features/automation/mainAutomation.py`: Imports from `interfaceDbChatObjects`, `interfaceDbAppObjects` +- `features/featuresLifecycle.py`: Imports from `interfaceDbAppObjects`, `interfaceDbChatObjects` (lazy) + +**Impact:** This is **correct** - features should use interfaces for data access. This follows the dependency rule: `features/` → `interfaces/` ✅ + +**Note:** This is **unidirectional** (features → interfaces), not bidirectional. + +--- + +### 6. features/ → services/ ✅ CORRECT DIRECTION + +**Current State:** +- `features/neutralizePlayground/mainNeutralizePlayground.py` (line 123): Lazy import from `serviceSharepoint` + +**Impact:** This is **correct** - features can use services. This follows the dependency rule: `features/` → `services/` ✅ + +--- + +## Complete Import Matrix (Fact-Based) + +### aicore/ +- **Imports from:** `datamodels/`, `shared/` +- **Imported by:** `interfaces/`, `services/`, `workflows/` +- **Bidirectional:** None ✅ + +### connectors/ +- **Imports from:** `datamodels/`, `shared/` +- **Imported by:** `interfaces/` +- **Bidirectional:** None ✅ + +### datamodels/ +- **Imports from:** `shared/` +- **Imported by:** `aicore/`, `connectors/`, `features/`, `interfaces/`, `routes/`, `security/`, `services/`, `workflows/` +- **Bidirectional:** None ✅ + +### features/ +- **Imports from:** `datamodels/`, `interfaces/`, `services/`, `shared/`, `workflows/` +- **Imported by:** `routes/` +- **✅ UNIDIRECTIONAL:** No longer imported by `interfaces/` + +**Detailed imports:** +- From `interfaces/`: `interfaceDbChatObjects`, `interfaceDbAppObjects` +- From `services/`: `serviceSharepoint` (lazy, in `neutralizePlayground`) +- From `features/`: `chatPlayground` (in `automation`), `syncDelta`, `chatAlthaus` (in `featuresLifecycle`) + +### interfaces/ +- **Imports from:** `aicore/`, `connectors/`, `datamodels/`, `shared/` +- **Imported by:** `features/`, `routes/`, `security/`, `services/` +- **✅ RESOLVED:** No longer imports from `services/` or `features/` + +**Detailed imports:** +- From `shared/`: `callbackRegistry` (for decoupled event notifications), `eventManagement` (for event removal in delete), `timeUtils`, `configuration`, `debugLogger` +- From `interfaces/`: Internal imports (`interfaceDbChatAccess`, `interfaceDbAppAccess`, `interfaceDbComponentAccess`) + +### routes/ +- **Imports from:** `datamodels/`, `features/`, `interfaces/`, `security/`, `services/`, `shared/` +- **Imported by:** None (top-level API layer) +- **Bidirectional:** None ✅ + +**Detailed imports:** +- From `interfaces/`: `interfaceDbChatObjects`, `interfaceDbAppObjects`, `interfaceDbComponentObjects`, `interfaceVoiceObjects` +- From `features/**: `features.automation`, `features.chatPlayground`, `features.neutralizePlayground` + +### security/ +- **Imports from:** `datamodels/`, `interfaces/`, `shared/` +- **Imported by:** `routes/`, `services/` +- **Bidirectional:** None ✅ + +**Detailed imports:** +- From `interfaces/**: `interfaceDbAppObjects` (lazy imports) + +### services/ +- **Imports from:** `aicore/`, `datamodels/`, `interfaces/`, `security/`, `shared/` +- **Imported by:** `features/`, `routes/`, `workflows/` +- **✅ UNIDIRECTIONAL:** Only imports from `interfaces/` (correct direction) +- **✅ RESOLVED:** No longer imported by `interfaces/` + +**Detailed imports:** +- From `interfaces/**: `interfaceAiObjects`, `interfaceDbComponentObjects` (lazy), `interfaceDbChatObjects` (lazy), `interfaceTicketObjects` +- From `services/**: Internal imports (service-to-service) + +### shared/ +- **Imports from:** None (foundation layer) +- **Imported by:** `aicore/`, `connectors/`, `datamodels/`, `features/`, `interfaces/`, `routes/`, `security/`, `services/`, `workflows/` +- **Bidirectional:** None ✅ + +### workflows/ +- **Imports from:** `aicore/`, `datamodels/`, `services/`, `shared/` +- **Imported by:** `features/` +- **Bidirectional:** None ✅ + +**Detailed imports:** +- From `services/**: `serviceGeneration` (lazy, in `methodAi.py`) + +--- + +## Refactoring Impact Summary + +### Before Refactoring: +- ❌ **interfaces/ ↔ services/**: Bidirectional (violations) + - `interfaces/` imported from `services/serviceExtraction/` (6 violations) + - `services/` imported from `interfaces/` (correct) +- ❌ **interfaces/ ↔ features/**: Bidirectional (violation) + - `interfaces/` imported from `features.chatPlayground` (1 violation) + - `features/` imported from `interfaces/` (correct) + +### After Refactoring: +- ✅ **interfaces/ → services/**: RESOLVED (no imports) +- ✅ **services/ → interfaces/**: UNIDIRECTIONAL (correct direction) +- ✅ **interfaces/ → features/**: RESOLVED (uses callback registry pattern) +- ✅ **features/ → interfaces/**: UNIDIRECTIONAL (correct direction) +- ✅ **features/ → services/**: CORRECT DIRECTION (1 lazy import) + +--- + +## Specific Import Details + +### interfaces/ → features/ ✅ RESOLVED + +**Previous:** `interfaceDbChatObjects.py` (line 1754) had lazy import from `features.automation` +```python +from modules.features.automation import syncAutomationEvents +``` + +**Current:** Uses `shared.callbackRegistry` pattern (line 1754): +- Interface calls: `callbackRegistry.trigger('automation.changed', self)` +- Feature registers callback in `featuresLifecycle.py`: `callbackRegistry.register('automation.changed', onAutomationChanged)` +- **Zero direct imports** from features in interfaces +- **Verification:** `grep "from modules.features" interfaces/` returns no matches ✅ + +### features/ → services/ (1 import, correct direction) + +**File:** `features/neutralizePlayground/mainNeutralizePlayground.py` +- **Line:** 123 +- **Import:** `from modules.services.serviceSharepoint.mainServiceSharepoint import SharepointService` +- **Type:** Lazy import (inside method) +- **Context:** Used for SharePoint file processing +- **Status:** ✅ CORRECT - Features can import from services + +--- + +## Recommendations + +### ✅ Completed Improvements +1. **Resolved interfaces/ → services/ violations** - Moved extraction functions to `serviceExtraction/` +2. **Resolved interfaces/ → features/ violations** - Moved automation handler to `features/automation/` +3. **Eliminated ALL bidirectional dependencies** - `interfaces/` no longer imports from `services/` or `features/` +4. **Implemented callback registry pattern** - Decoupled event notifications using `shared.callbackRegistry` +5. **Follows dependency rules perfectly** - All dependencies now follow correct direction: + - `services/` → `interfaces/` ✅ + - `features/` → `interfaces/` ✅ + - `features/` → `services/` ✅ + - `interfaces/` → `shared/` only ✅ + +### Best Practices +- ✅ Use lazy imports (inside functions) for dependencies when appropriate +- ✅ Services correctly depend on interfaces (unidirectional) +- ✅ Features correctly depend on interfaces and services (unidirectional) +- ✅ Interfaces completely independent (only foundation layers) +- ✅ Use callback registry for decoupled event notifications +- ✅ Document dependency relationships clearly +- ✅ Monitor for circular import errors + +--- + +## Dependency Rule Compliance + +### Current Rules: +- ✅ **features/** → **services/** ✅ (correct) +- ✅ **services/** → **interfaces/** ✅ (correct) +- ✅ **features/** → **interfaces/** ✅ (correct) + +### Status: +- ✅ **interfaces/** → **services/**: **RESOLVED** (was violation, now compliant) +- ✅ **services/** → **interfaces/**: **COMPLIANT** (correct direction) +- ✅ **features/** → **interfaces/**: **COMPLIANT** (correct direction) +- ✅ **features/** → **services/**: **COMPLIANT** (correct direction) +- ✅ **interfaces/** → **features/**: **RESOLVED** (was violation, now uses callback registry) + +--- + +## Conclusion + +The refactoring successfully resolved **ALL bidirectional dependencies**: +- ✅ **interfaces/ ↔ services/**: RESOLVED +- ✅ **interfaces/ ↔ features/**: RESOLVED (using callback registry pattern) + +The architecture now follows the intended dependency rules **perfectly**: +- `interfaces/` only imports from foundation layers (`aicore/`, `connectors/`, `datamodels/`, `shared/`) +- `services/` imports from `interfaces/` (correct direction) +- `features/` imports from `interfaces/` and `services/` (correct direction) +- **Zero violations** - perfect architectural compliance achieved through callback registry pattern + +--- + +## Architecture Layers + +The codebase follows a clean layered architecture: + +1. **Foundation Layer** (`shared/`, `datamodels/`) + - No dependencies on other modules + - Used by all layers + +2. **Infrastructure Layer** (`aicore/`, `connectors/`) + - Depends only on foundation + - Provides core capabilities + +3. **Data Access Layer** (`interfaces/`) + - Depends only on foundation and infrastructure + - Provides data access abstraction + +4. **Business Logic Layer** (`services/`, `workflows/`) + - Depends on interfaces and foundation + - Implements business logic + +5. **Feature Layer** (`features/`) + - Depends on interfaces, services, and foundation + - Implements user-facing features + +6. **API Layer** (`routes/`, `security/`) + - Depends on all layers + - Provides HTTP API endpoints diff --git a/modules/DEPENDENCY_DIAGRAM.drawio b/modules/DEPENDENCY_DIAGRAM.drawio new file mode 100644 index 00000000..eabab094 --- /dev/null +++ b/modules/DEPENDENCY_DIAGRAM.drawio @@ -0,0 +1 @@ +5Z1bb6M4FMc/DdLOQyUuIZfHNm1nK3Wno8lepH1zwSSoBFeGNO1++jWNSXzLNKW2MR1ppCbHBJLz//n4cIAzXjRfP3/F4HH1B0ph4YV++uxFl14YBmEYes0/P31pLcFkZ1niPKW2g2GR/wep0afWTZ7CituwRqio80femKCyhEnN2QDGaMtvlqGCP+ojWELJsEhAIVv/ydN6Ra0j3z8M/A7z5ao9dNiOrEG7NTVUK5CiLWOKrrxojhGqd6/Wz3NYNO5rHbP73PWR0f03w7CsT/kAOT6G6e5TT6DYQN587YVjL4yI46OL367RpkxBnaPyC/3y9UvrEtyMwWanAdl0u8pruHgESTO6JRQQ26peF3Q4y4tijgqEXz8bwSCLs4zYqxqjB8iM+EE8md3vR1pnh80+UFkv6PEVv5qaniCu4TNjol74CtEa1viFbEJHwxlVhEIZtQptGYVb24oRd0xtgEK13O/64HbygnperQLxKlg386SSlGCHfh014phXY2xRDJAnCENJiNbMiXBTZhgQd2ySeoPhJxRiH7naWO1bVILGb4TlacEO/VqKTPtUJC9riDPiLlkRdohT5JJEMLLpeUKGqi/MiHcVerO5N43J6DfU7C1HxWtEq7TKlkUwzmKVbCMQjKaJJFtkYiJNeNlmI1m2/WRjZZtqkK2C+ClXiXYY4CS72FR5SdQiG9+iZZ4cU+2vMk9zTOYh0QwUeufaNIvhTCVacB/D0Lcy18a8ZiNFTmBMsy3CD1nR5KqiaMzIW6p9NkEmQvAbKYKfMUUyCJo1RRbkMMBnaNTcw+zJMhL0fJVYcBw3K4YNsWbC9JnZFIt4r1ZI1Zo5oc6/3zRzBrxArHfOZAkcwUQlw3Tqw1FmRYZImDMTmzJUMNngvH5RrDztACfFgpo/oQ5i7LKqA0yXMJBEkJwMy/S8KZCQd0kBqipPeL8eRFCHkGOJ8THvVWiDE8WZ7mtJB+AlpJuzZYrmp/zU34w/Y4U7WxuGTbL5xNd0VD6mR/iOSIp7kHMkyLmfZu0udr+NfootwYg7Es5191+w3dHOEdKOXiXf/+yTKQjdpoA9+2YJEOlwhAIxuHamILJLQTRMChyMA0MlYOQ2AWLtZwCxQKzMDGVFiIdLgoPxQBsFY7sUjHuggNbfTqFArDyyFLCrhSMUxMcKFR9dFSSc9FIwGS4FYpxwhISxLhImdkmYDpcER/MD8aJmZxLE/MAwCTOJhAQUxT1IHn7AZV61lQgbZEQXKahWtAr0UUwcTB60BQsxeTCMSHtnDhstDpVujXBEP7kAcQoP/JUwlgaRFEeIEMtMcVcipOhjNp0M+qg2agHBwWxSTAI7QxBaDgvhUCFwNIXQFg0spxBBH/VGLSA4mCSI2nWGwHaS0EfJUQ8E3FVMVzDQFQvEitXYcGbQR73xHRgIN944nhqIVYHBpAZ91Bv1UOBobiBO48HkBn3UHPWQwK8YjnCgLSKIHJg+Y+yj4qiJA/eSRG3RwHaSKFcbLVaS6N2Rp+DA3w06gEqS/ExNVyLEIqXhG5Z6rS1qIcLJdUIbD2+ej2rmoY/KohYMHE0cjYFgeKlodR8eCEIq4QgHYlzvzMGbGahmDvooLupZF9zLG7VBYDlvDPsoLtKnB06BgH0eZQDJolhcnHbFwPJlx7CP4qIGDPgY8dkhkIKKZgj6qC1qgMDJ0wRtENg+TeijrKgFAgevNYm3s3aGQCxLTQxD0EdNUQMEjp4pGosFppNDRVFxCBg4eH4gKtcZAcvnB5GiiugSAnzYH8AZghjJu4Ngt5wc9VE+1AKCo4uCNhAsLwpROFQQHFwW9EUDexAUcEn0lSC4ZcwMCjV8rnnxeXFLVJItuc4T1ASKfFk2BMEmkhND0zMiT0BxTgfWeZo2h1F2teAR45tS0Pe7Bpnt7XUfalIR+Ke0F1N1qYj848yc2qVip4gcoA+9Dj3ap0WSx3JfNxOeVnQEC1RTVp+n5QjINWsz4ux3d2Mz4+ypdWfLVynEhl1m4H5/3y4D/t43vLXnb/mCAG3IZQbr93fcMuFmVaNUs26WC+5MOy29Ln5/FycDLo5UC6JZFyvK2UwbuTnCTRM58uqy7SZnIHkpYFZ/JHUxoUR8ohKhNiXkmvLZ2VmjAX0glrw8+kxs/xqc9PTsRwQ5dVHVIUhVk2AuN/hjZ8a/Vz/uyJ+/b+5uz/+8ufu2YDrNfYc4282ac5w0fmv6BTddGMl8Wj8WOSgTqDV6JVM4TpTrcAgnaRRK0UvuQsun+0ai20gxp5RN6Tp0EyZvD43td+drh/8gILr6Hw== \ No newline at end of file diff --git a/modules/FEATURES_TO_INTERFACES_IMPORTS.md b/modules/FEATURES_TO_INTERFACES_IMPORTS.md new file mode 100644 index 00000000..32b0cc7f --- /dev/null +++ b/modules/FEATURES_TO_INTERFACES_IMPORTS.md @@ -0,0 +1,199 @@ +# Features → Interfaces Import Analysis + +## Summary + +This document details all imports from `features/` modules to `interfaces/` modules. + +**Total Feature Modules:** 6 +**Modules Importing from Interfaces:** 2 (only `getRootInterface` - system-level function) +**Total Interface Imports:** 2 (both for `getRootInterface` only) + +**✅ REFACTORED:** All feature modules now use `services.getInterface()` to access interfaces, following the same pattern as `chatPlayground`. Only `getRootInterface()` remains as a direct import (system-level function, not user-specific). + +--- + +## Detailed Import List + +### 1. `features/featuresLifecycle.py` + +**Imports:** +- **Line 2:** `from modules.interfaces.interfaceDbAppObjects import getRootInterface` + - **Type:** Direct import + - **Usage:** Gets root interface to retrieve event user for feature initialization + - **Context:** Used in `start()` function to get event user for automation, syncDelta, and chatAlthaus features + - **Note:** `getRootInterface()` is a system-level function (no user required), so it remains as direct import + +**Refactored:** +- ✅ **Removed:** `from modules.interfaces.interfaceDbChatObjects import getInterface as getChatInterface` +- ✅ **Added:** `from modules.services import getInterface as getServices` +- ✅ **Changed:** Now uses `services.interfaceDbChat` instead of direct interface import +- **Pattern:** `services = getServices(eventUser, None)` then `services.interfaceDbChat` + +**Purpose:** Feature lifecycle management - initializes and manages all features on startup/shutdown. + +--- + +### 2. `features/automation/mainAutomation.py` + +**Imports:** +- **Line 14:** `from modules.interfaces.interfaceDbAppObjects import getRootInterface` + - **Type:** Direct import + - **Usage:** Gets root interface to retrieve event user (system-level function) + - **Context:** Used in `createAutomationEventHandler()` to get event user + - **Note:** `getRootInterface()` is a system-level function (no user required), so it remains as direct import + +**Refactored:** +- ✅ **Removed:** `from modules.interfaces.interfaceDbChatObjects import getInterface as getChatInterface` +- ✅ **Removed:** `from modules.interfaces.interfaceDbAppObjects import getInterface as getAppInterface` +- ✅ **Added:** `from modules.services import getInterface as getServices` +- ✅ **Changed:** All interface access now goes through services: + - `executeAutomation()`: Uses `services.interfaceDbApp` and `services.interfaceDbChat` + - `createAutomationEventHandler()`: Uses `eventServices.interfaceDbChat` and `eventServices.interfaceDbApp` +- **Pattern:** `services = getServices(user, None)` then `services.interfaceDbChat` or `services.interfaceDbApp` + +**Purpose:** Automation workflow execution and scheduling - handles automated workflow triggers and event scheduling. + +--- + +### 3. `features/chatPlayground/mainChatPlayground.py` + +**Imports:** +- ❌ **No imports from interfaces/** +- Uses `modules.services.getInterface` instead (indirect access through services layer) + +**Purpose:** Chat playground feature - interactive chat interface. + +--- + +### 4. `features/chatAlthaus/mainChatAlthaus.py` + +**Imports:** +- ❌ **No imports from interfaces/** +- Uses `modules.services.getInterface` instead (indirect access through services layer) + +**Purpose:** Chat Althaus data scheduler - scheduled data updates for Althaus preprocessing. + +--- + +### 5. `features/syncDelta/mainSyncDelta.py` + +**Imports:** +- ❌ **No imports from interfaces/** +- Uses `modules.services.getInterface` instead (indirect access through services layer) + +**Purpose:** Delta Group sync manager - synchronizes tickets to SharePoint. + +--- + +### 6. `features/neutralizePlayground/mainNeutralizePlayground.py` + +**Imports:** +- ❌ **No imports from interfaces/** +- Uses `modules.services.getInterface` instead (indirect access through services layer) + +**Purpose:** Neutralization playground - UI wrapper for data neutralization service. + +--- + +## Import Statistics + +### By Interface Module + +**`interfaceDbAppObjects`:** +- `getRootInterface`: 2 imports (system-level function, remains as direct import) + - `features/featuresLifecycle.py` (line 2) + - `features/automation/mainAutomation.py` (line 14) + +**`interfaceDbChatObjects`:** +- ✅ **Removed:** All direct imports refactored to use services layer +- Now accessed via `services.interfaceDbChat` after calling `getServices(user, None)` + +**`interfaceDbAppObjects.getInterface`:** +- ✅ **Removed:** All direct imports refactored to use services layer +- Now accessed via `services.interfaceDbApp` after calling `getServices(user, None)` + +### By Import Type + +- **Direct imports:** 2 (only `getRootInterface` - system-level function) + - `features/featuresLifecycle.py`: 1 + - `features/automation/mainAutomation.py`: 1 + +- **Services-based access:** All user-specific interface access now goes through services layer + - Pattern: `services = getServices(user, None)` then `services.interfaceDbChat` or `services.interfaceDbApp` + +--- + +## Architectural Notes + +### ✅ Correct Dependency Direction + +All imports follow the correct architectural direction: +- **features/** → **interfaces/** ✅ + +This is compliant with the dependency rules: +- Features can import from interfaces (correct) +- Features can import from services (correct) +- Features do NOT import from other features (except internal feature-to-feature imports) + +### Import Patterns + +1. **Direct Interface Access:** + - `features/automation/mainAutomation.py` - Directly imports interfaces for automation management + - `features/featuresLifecycle.py` - Directly imports `getRootInterface` for feature initialization + +2. **Indirect Access via Services:** + - `features/chatPlayground/mainChatPlayground.py` + - `features/chatAlthaus/mainChatAlthaus.py` + - `features/syncDelta/mainSyncDelta.py` + - `features/neutralizePlayground/mainNeutralizePlayground.py` + + These features use `modules.services.getInterface` which provides a service layer abstraction. + +### Usage Context + +**`interfaceDbAppObjects`:** +- Used for user management (`getRootInterface`, `getAppInterface`) +- Primarily for getting event user and creator user in automation context + +**`interfaceDbChatObjects`:** +- Used for chat/automation data access (`getChatInterface`) +- Used for automation execution, event syncing, and workflow management + +--- + +## Summary Table + +| Feature Module | Interface Imports | Import Type | Purpose | +|---------------|-------------------|-------------|---------| +| `featuresLifecycle.py` | `interfaceDbAppObjects.getRootInterface` | Direct | Feature initialization (system-level) | +| `featuresLifecycle.py` | ✅ Via `services.interfaceDbChat` | Services | Automation event sync | +| `automation/mainAutomation.py` | `interfaceDbAppObjects.getRootInterface` | Direct | Event user access (system-level) | +| `automation/mainAutomation.py` | ✅ Via `services.interfaceDbChat` | Services | Automation execution | +| `automation/mainAutomation.py` | ✅ Via `services.interfaceDbApp` | Services | User management | +| `chatPlayground/mainChatPlayground.py` | None | - | Uses services layer | +| `chatAlthaus/mainChatAlthaus.py` | None | - | Uses services layer | +| `syncDelta/mainSyncDelta.py` | None | - | Uses services layer | +| `neutralizePlayground/mainNeutralizePlayground.py` | None | - | Uses services layer | + +--- + +## Conclusion + +**Total Interface Imports:** 2 (only `getRootInterface` - system-level function) +- 2 direct imports (both for `getRootInterface`) + +**Modules Using Direct Interface Imports:** 2 out of 6 (only for `getRootInterface`) +- `features/featuresLifecycle.py` +- `features/automation/mainAutomation.py` + +**✅ REFACTORED:** All user-specific interface access now goes through services layer +- Pattern: `services = getServices(user, None)` then `services.interfaceDbChat` or `services.interfaceDbApp` +- Consistent with other feature modules (`chatPlayground`, `chatAlthaus`, `syncDelta`, `neutralizePlayground`) + +**Architectural Compliance:** ✅ **PERFECT** +- All imports follow correct direction (features → interfaces) +- Only system-level function (`getRootInterface`) remains as direct import +- All user-specific interface access goes through services layer +- Clean separation maintained +- Consistent pattern across all feature modules + diff --git a/modules/features/automation/__init__.py b/modules/features/automation/__init__.py new file mode 100644 index 00000000..1cc8a344 --- /dev/null +++ b/modules/features/automation/__init__.py @@ -0,0 +1,12 @@ +""" +Automation feature - handles automated workflow execution and scheduling. + +Moved from interfaces/interfaceDbChatObjects.py to follow proper architectural separation: +- Interface layer: Data access only (getAutomationDefinition, etc.) +- Feature layer: Business logic and orchestration (executeAutomation, syncAutomationEvents) +""" + +from .mainAutomation import executeAutomation, syncAutomationEvents, createAutomationEventHandler + +__all__ = ['executeAutomation', 'syncAutomationEvents', 'createAutomationEventHandler'] + diff --git a/modules/features/automation/mainAutomation.py b/modules/features/automation/mainAutomation.py new file mode 100644 index 00000000..c0534229 --- /dev/null +++ b/modules/features/automation/mainAutomation.py @@ -0,0 +1,287 @@ +""" +Main automation service - handles automation workflow execution and scheduling. + +Moved from interfaces/interfaceDbChatObjects.py to follow proper architectural separation. +""" + +import logging +import json +from typing import Dict, Any + +from modules.datamodels.datamodelChat import ChatWorkflow, UserInputRequest, WorkflowModeEnum, AutomationDefinition +from modules.shared.timeUtils import getUtcTimestamp +from modules.shared.eventManagement import eventManager +from modules.services import getInterface as getServices +from modules.features.chatPlayground.mainChatPlayground import chatStart +from .subAutomationUtils import parseScheduleToCron, planToPrompt, replacePlaceholders + +logger = logging.getLogger(__name__) + + +async def executeAutomation(automationId: str, chatInterface) -> ChatWorkflow: + """Execute automation workflow immediately (test mode) with placeholder replacement. + + Args: + automationId: ID of automation to execute + chatInterface: ChatObjects interface instance for data access + + Returns: + ChatWorkflow instance created by automation execution + """ + executionStartTime = getUtcTimestamp() + executionLog = { + "timestamp": executionStartTime, + "workflowId": None, + "status": "running", + "messages": [] + } + + try: + # 1. Load automation definition + automation = chatInterface.getAutomationDefinition(automationId) + if not automation: + raise ValueError(f"Automation {automationId} not found") + + executionLog["messages"].append(f"Started execution at {executionStartTime}") + + # 2. Replace placeholders in template to generate plan + template = automation.get("template", "") + placeholders = automation.get("placeholders", {}) + planJson = replacePlaceholders(template, placeholders) + try: + plan = json.loads(planJson) + except json.JSONDecodeError as e: + logger.error(f"Failed to parse plan JSON after placeholder replacement: {str(e)}") + logger.error(f"Template: {template[:500]}...") + logger.error(f"Placeholders: {placeholders}") + logger.error(f"Generated planJson (first 1000 chars): {planJson[:1000]}") + logger.error(f"Error position: line {e.lineno}, column {e.colno}, char {e.pos}") + if e.pos: + start = max(0, e.pos - 100) + end = min(len(planJson), e.pos + 100) + logger.error(f"Context around error: ...{planJson[start:end]}...") + raise ValueError(f"Invalid JSON after placeholder replacement: {str(e)}") + executionLog["messages"].append("Template placeholders replaced successfully") + + # 3. Get user who created automation + creatorUserId = automation.get("_createdBy") + + # CRITICAL: Automation MUST run as creator user only, or fail + if not creatorUserId: + errorMsg = f"Automation {automationId} has no creator user (_createdBy field missing). Cannot execute automation." + logger.error(errorMsg) + executionLog["messages"].append(errorMsg) + raise ValueError(errorMsg) + + # Get user from database using services + services = getServices(chatInterface.currentUser, None) + creatorUser = services.interfaceDbApp.getUser(creatorUserId) + if not creatorUser: + raise ValueError(f"Creator user {creatorUserId} not found") + + executionLog["messages"].append(f"Using creator user: {creatorUserId}") + + # 4. Create UserInputRequest from plan + # Embed plan JSON in prompt for TemplateMode to extract + promptText = planToPrompt(plan) + planJsonStr = json.dumps(plan) + # Embed plan as JSON comment so TemplateMode can extract it + promptWithPlan = f"{promptText}\n\n\n{planJsonStr}\n" + + userInput = UserInputRequest( + prompt=promptWithPlan, + listFileId=[], + userLanguage=creatorUser.language or "en" + ) + + executionLog["messages"].append("Starting workflow execution") + + # 5. Start workflow using chatStart + workflow = await chatStart( + currentUser=creatorUser, + userInput=userInput, + workflowMode=WorkflowModeEnum.WORKFLOW_AUTOMATION, + workflowId=None + ) + + executionLog["workflowId"] = workflow.id + executionLog["status"] = "completed" + executionLog["messages"].append(f"Workflow {workflow.id} started successfully") + logger.info(f"Started workflow {workflow.id} with plan containing {len(plan.get('tasks', []))} tasks (plan embedded in userInput)") + + # Set workflow name with "automated" prefix + automationLabel = automation.get("label", "Unknown Automation") + workflowName = f"automated: {automationLabel}" + workflow = chatInterface.updateWorkflow(workflow.id, {"name": workflowName}) + logger.info(f"Set workflow {workflow.id} name to: {workflowName}") + + # Update automation with execution log + executionLogs = automation.get("executionLogs", []) + executionLogs.append(executionLog) + # Keep only last 50 executions + if len(executionLogs) > 50: + executionLogs = executionLogs[-50:] + + chatInterface.db.recordModify( + AutomationDefinition, + automationId, + {"executionLogs": executionLogs} + ) + + return workflow + except Exception as e: + # Log error to execution log + executionLog["status"] = "error" + executionLog["messages"].append(f"Error: {str(e)}") + + # Update automation with execution log even on error + try: + automation = chatInterface.getAutomationDefinition(automationId) + if automation: + executionLogs = automation.get("executionLogs", []) + executionLogs.append(executionLog) + if len(executionLogs) > 50: + executionLogs = executionLogs[-50:] + chatInterface.db.recordModify( + AutomationDefinition, + automationId, + {"executionLogs": executionLogs} + ) + except Exception as logError: + logger.error(f"Error saving execution log: {str(logError)}") + + raise + + +async def syncAutomationEvents(chatInterface, eventUser) -> Dict[str, Any]: + """Automation event handler - syncs scheduler with all active automations. + + Args: + chatInterface: ChatObjects interface instance for data access + eventUser: System-level event user for accessing automations + + Returns: + Dictionary with sync results (synced count and event IDs) + """ + # Get all automation definitions (for current mandate) + allAutomations = chatInterface.db.getRecordset(AutomationDefinition) + filtered = chatInterface._uam(AutomationDefinition, allAutomations) + + registeredEvents = {} + + for automation in filtered: + automationId = automation.get("id") + isActive = automation.get("active", False) + currentEventId = automation.get("eventId") + schedule = automation.get("schedule") + + if not schedule: + logger.warning(f"Automation {automationId} has no schedule, skipping") + continue + + try: + # Parse schedule to cron kwargs + cronKwargs = parseScheduleToCron(schedule) + + if isActive: + # Remove existing event if present (handles schedule changes) + if currentEventId: + try: + eventManager.remove(currentEventId) + except Exception as e: + logger.warning(f"Error removing old event {currentEventId}: {str(e)}") + + # Register new event + newEventId = f"automation.{automationId}" + + # Create event handler function + handler = createAutomationEventHandler(automationId, eventUser) + + # Register cron job + eventManager.registerCron( + jobId=newEventId, + func=handler, + cronKwargs=cronKwargs, + replaceExisting=True + ) + + # Update automation with new eventId + if currentEventId != newEventId: + chatInterface.db.recordModify( + AutomationDefinition, + automationId, + {"eventId": newEventId} + ) + + registeredEvents[automationId] = newEventId + else: + # Remove event if exists + if currentEventId: + try: + eventManager.remove(currentEventId) + chatInterface.db.recordModify( + AutomationDefinition, + automationId, + {"eventId": None} + ) + except Exception as e: + logger.warning(f"Error removing event {currentEventId}: {str(e)}") + except Exception as e: + logger.error(f"Error syncing automation {automationId}: {str(e)}") + + return { + "synced": len(registeredEvents), + "events": registeredEvents + } + + +def createAutomationEventHandler(automationId: str, eventUser): + """Create event handler function for a specific automation. + + Args: + automationId: ID of automation to create handler for + eventUser: System-level event user for accessing automations (captured in closure) + + Returns: + Async handler function for scheduled automation execution + """ + async def handler(): + try: + if not eventUser: + logger.error("Event user not available for automation execution") + return + + # Get services for event user (provides access to interfaces) + eventServices = getServices(eventUser, None) + + # Load automation using event user context + automation = eventServices.interfaceDbChat.getAutomationDefinition(automationId) + if not automation or not automation.get("active"): + logger.warning(f"Automation {automationId} not found or not active, skipping execution") + return + + # Get creator user + creatorUserId = automation.get("_createdBy") + if not creatorUserId: + logger.error(f"Automation {automationId} has no creator user") + return + + # Get creator user from database using services + eventServices = getServices(eventUser, None) + creatorUser = eventServices.interfaceDbApp.getUser(creatorUserId) + if not creatorUser: + logger.error(f"Creator user {creatorUserId} not found for automation {automationId}") + return + + # Get services for creator user (provides access to interfaces) + creatorServices = getServices(creatorUser, None) + + # Execute automation with creator user's context + # executeAutomation is in same module, so we can call it directly + await executeAutomation(automationId, creatorServices.interfaceDbChat) + logger.info(f"Successfully executed automation {automationId} as user {creatorUserId}") + except Exception as e: + logger.error(f"Error executing automation {automationId}: {str(e)}") + + return handler + diff --git a/modules/features/automation/subAutomationUtils.py b/modules/features/automation/subAutomationUtils.py new file mode 100644 index 00000000..f1948ffa --- /dev/null +++ b/modules/features/automation/subAutomationUtils.py @@ -0,0 +1,108 @@ +""" +Utility functions for automation feature. + +Moved from interfaces/interfaceDbChatObjects.py. +""" + +import json +from typing import Dict, Any + + +def parseScheduleToCron(schedule: str) -> Dict[str, Any]: + """Parse schedule string to cron kwargs for APScheduler""" + parts = schedule.split() + if len(parts) != 5: + raise ValueError(f"Invalid schedule format: {schedule}") + + return { + "minute": parts[0], + "hour": parts[1], + "day": parts[2], + "month": parts[3], + "day_of_week": parts[4] + } + + +def planToPrompt(plan: Dict) -> str: + """Convert plan structure to prompt string for workflow execution""" + return plan.get("userMessage", plan.get("overview", "Execute automation workflow")) + + +def replacePlaceholders(template: str, placeholders: Dict[str, str]) -> str: + """Replace placeholders in template with actual values. Placeholder format: {{KEY:PLACEHOLDER_NAME}}""" + result = template + for placeholderName, value in placeholders.items(): + pattern = f"{{{{KEY:{placeholderName}}}}}" + + # Check if placeholder is in an array context like ["{{KEY:...}}"] + # If value is a JSON array/dict, we should replace the entire ["{{KEY:...}}"] with the array + arrayPattern = f'["{pattern}"]' + if arrayPattern in result: + # Check if value is a JSON array/dict + isArrayValue = False + arrayValue = None + + if isinstance(value, (list, dict)): + isArrayValue = True + arrayValue = json.dumps(value) + elif isinstance(value, str): + try: + parsed = json.loads(value) + if isinstance(parsed, (list, dict)): + isArrayValue = True + arrayValue = value # Already valid JSON string + except (json.JSONDecodeError, ValueError): + pass + + if isArrayValue: + # Replace ["{{KEY:...}}"] with the array value + result = result.replace(arrayPattern, arrayValue) + continue # Skip the regular replacement below + + # Regular replacement - check if in quoted context + patternStart = result.find(pattern) + isQuoted = False + if patternStart > 0: + charBefore = result[patternStart - 1] if patternStart > 0 else None + patternEnd = patternStart + len(pattern) + charAfter = result[patternEnd] if patternEnd < len(result) else None + if charBefore == '"' and charAfter == '"': + isQuoted = True + + # Handle different value types + if isinstance(value, (list, dict)): + # Python list/dict - convert to JSON + replacement = json.dumps(value) + elif isinstance(value, str): + # String value - check if it's a JSON string representing list/dict + try: + parsed = json.loads(value) + if isinstance(parsed, (list, dict)): + # It's a JSON string of a list/dict + if isQuoted: + # In quoted context, escape the JSON string + escaped = json.dumps(value) + replacement = escaped[1:-1] # Remove outer quotes + else: + # In unquoted context, use JSON directly + replacement = value + else: + # It's a JSON string of a primitive + if isQuoted: + escaped = json.dumps(value) + replacement = escaped[1:-1] + else: + replacement = value + except (json.JSONDecodeError, ValueError): + # Not valid JSON - treat as plain string + if isQuoted: + escaped = json.dumps(value) + replacement = escaped[1:-1] + else: + replacement = value + else: + # Numbers, booleans, None - convert to string + replacement = str(value) + result = result.replace(pattern, replacement) + return result + diff --git a/modules/features/chatAlthaus/COMPONENT_DIAGRAM.md b/modules/features/chatAlthaus/COMPONENT_DIAGRAM.md deleted file mode 100644 index 5ae3edae..00000000 --- a/modules/features/chatAlthaus/COMPONENT_DIAGRAM.md +++ /dev/null @@ -1,211 +0,0 @@ -# Komponentendiagramm: Kunden-Chatbot Althaus - -## Übersicht - -Dieses Diagramm zeigt die High-Level-Architektur der Althaus Chatbot-Anwendung mit allen beteiligten Komponenten, Datenflüssen und Kommunikationswegen. - -## Komponentendiagramm - -```mermaid -graph TB - subgraph "PowerOn Chat UI" - ChatUI[Chat Interface] - end - - subgraph "PowerOn Platform" - Gateway[Gateway Backend
Event Scheduler & Data Query API] - GatewayDB[(PostgreSQL)] - AIServices[Dynamic AI, Tavily] - end - - subgraph "Tenant althaus-ag.ch" - subgraph "PowerOn PreProcessing" - PreProcessing[Pre-Processing Service] - PreProcessingDB[(PostgreSQL
Memory DB)] - end - - subgraph "MSFT Services" - PowerBI[Power BI] - TenantServices[Azure DC, DNA Center] - end - end - - %% Hauptkommunikation - ChatUI -->|"Data Queries
User/Password Auth"| Gateway - Gateway -->|"SQL Queries
X-PP-API-Key"| PreProcessing - Gateway -->|"Config Update
Daily 01:00 UTC"| PreProcessing - - %% Datenfluss - PowerBI -->|"Rohdaten"| PreProcessing - PreProcessing --> PreProcessingDB - PreProcessingDB -->|"Query Results"| Gateway - Gateway --> ChatUI - Gateway --> GatewayDB - - %% Styling - classDef platform fill:#e1f5ff,stroke:#01579b,stroke-width:2px - classDef frontend fill:#f3e5f5,stroke:#4a148c,stroke-width:2px - classDef preprocessing fill:#fff3e0,stroke:#e65100,stroke-width:2px - classDef customer fill:#e8f5e9,stroke:#1b5e20,stroke-width:2px - classDef database fill:#fce4ec,stroke:#880e4f,stroke-width:2px - - class Gateway,AIServices platform - class ChatUI frontend - class PreProcessing preprocessing - class PowerBI,TenantServices customer - class GatewayDB,PreProcessingDB database -``` - -## Komponentenbeschreibungen - -### 1. Gateway Backend (gateway.poweron-center.net) - -**Hauptkomponenten:** -- **FastAPI Application**: Zentrale Backend-Anwendung der PowerOn Platform -- **Event Scheduler (chatAlthaus)**: - - Täglicher Scheduler um 01:00 UTC - - Sendet Konfigurations-Updates an Pre-Processing Service - - Verwendet `X-PP-API-Key` Header für Authentifizierung -- **Configuration Management**: - - Verwaltung von Secrets und Environment-Variablen - - Verschlüsselung/Entschlüsselung von Secrets - - Unterstützt verschiedene Umgebungen (dev, int, prod) -- **Data Query API**: - - `POST /api/v1/dataquery/query` - SQL Query ausführen - - `GET /api/v1/dataquery/schema` - Datenbankschema abrufen - - `GET /api/v1/dataquery/schema/{table_name}` - Tabellenschema abrufen -- **PostgreSQL Database**: Zentrale Datenbank für Gateway-Daten - -**Technologie:** -- Python/FastAPI -- PostgreSQL -- APScheduler für Event-Management - -**Externe AI-Services:** -- **Dynamic AI**: LLM Service für AI-Anfragen -- **Tavily**: Web-Such-Service für Web-Recherchen - -### 2. PowerOn Chat UI (althaus-chat.poweron-center.net) - -**Hauptkomponenten:** -- **React Application**: Frontend-Interface für den Chatbot -- **Authentication**: User/Password-basierte Authentifizierung mit JWT-Token - -**Kommunikation:** -- Nutzt 3 Data Query Endpunkte vom Gateway -- Authentifiziert sich mit User/Password beim Gateway -- Erhält Antworten über Gateway API - -**Technologie:** -- React -- REST API Calls - -### 3. Tenant althaus-ag.ch - -#### 3.1 PowerOn PreProcessing - -**Hauptkomponenten:** -- **FastAPI Application**: Pre-Processing Service im Azure-Tenant des Kunden -- **Pre-Processing API**: - - `POST /api/v1/dataprocessor/update-db-with-config` - Datenbank mit Konfiguration aktualisieren - - Authentifizierung: `X-PP-API-Key` Header -- **PostgreSQL Memory Database**: - - Speichert verarbeitete Daten - - Wird vom Chat für Queries genutzt - -**Datenfluss:** -- Empfängt Rohdaten aus Power BI Semantikmodell -- Verarbeitet Daten nach konfigurierten Schritten (keep, fillna, to_numeric, dropna, etc.) -- Speichert verarbeitete Daten in Memory Database -- Beantwortet SQL-Queries vom Gateway - -**Technologie:** -- Python/FastAPI -- PostgreSQL -- Azure App Service (im Kunden-Tenant althaus-ag.ch) - -#### 3.2 MSFT Services - -**Power BI Semantikmodell:** -- Datenquelle für Rohdaten -- Wird vom Pre-Processing Service gelesen - -**Azure Domänen-Controller:** -- Authentifizierungs-Service -- Wird vom Gateway für Authentifizierung genutzt - -**DNA Center:** -- Netzwerk-Management-Service -- Wird vom Gateway genutzt - -## Datenfluss - -### 1. Datenaktualisierung (Scheduled) -``` -Power BI Semantikmodell (Tenant althaus-ag.ch) - → PowerOn PreProcessing (verarbeitet Daten) - → PostgreSQL Memory DB (speichert verarbeitete Daten) - -Gateway Event Scheduler (01:00 UTC täglich) - → POST /api/v1/dataprocessor/update-db-with-config - → PowerOn PreProcessing (aktualisiert Konfiguration) -``` - -### 2. Chat-Interaktion (User Request) -``` -PowerOn Chat UI - → POST /api/v1/dataquery/query (mit User/Password Auth) - → Gateway Data Query API - → POST /api/v1/dataquery/query (mit X-PP-API-Key) - → PowerOn PreProcessing - → PostgreSQL Memory DB (führt Query aus) - → PowerOn PreProcessing (gibt Ergebnisse zurück) - → Gateway Data Query API - → PowerOn Chat UI (zeigt Antwort) -``` - -### 3. AI-Integration -``` -PowerOn Chat UI - → Gateway (vermittelt AI-Anfragen) - → Dynamic AI & Tavily (in PowerOn Platform) - → Gateway (kombiniert Ergebnisse) - → PowerOn Chat UI (zeigt Antwort) -``` - -## Authentifizierung - -### Gateway → PowerOn PreProcessing -- **Header**: `X-PP-API-Key` -- **Wert**: Aus Gateway Config (`PREPROCESS_ALTHAUS_CHAT_SECRET`) -- **Verwendung**: Event Scheduler und Data Query API - -### PowerOn Chat UI → Gateway -- **Methode**: User/Password -- **Token**: JWT Token (nach erfolgreicher Authentifizierung) -- **Verwendung**: Alle API-Calls vom Chat Frontend - -### Weitere Authentifizierung -- Gateway nutzt Azure Domänen-Controller für zusätzliche Authentifizierung -- Verschiedene API-Endpunkte können unterschiedliche Authentifizierungsmechanismen haben - -## Deployment - -- **PowerOn Platform**: gateway.poweron-center.net -- **PowerOn Chat UI**: althaus-chat.poweron-center.net -- **PowerOn PreProcessing**: Azure App Service im Kunden-Tenant (althaus-ag.ch) - - URL: `poweron-althaus-preprocess-prod-e3fegaatc7faency.switzerlandnorth-01.azurewebsites.net` -- **Tenant althaus-ag.ch**: Enthält PowerOn PreProcessing und MSFT Services (Power BI, Azure DC, DNA Center) im Azure-Tenant von Althaus AG - -## Konfiguration - -### Gateway Config Keys -- `PREPROCESS_ALTHAUS_CHAT_SECRET`: API-Key für Pre-Processing Service -- `APP_ENV_TYPE`: Umgebung (dev, int, prod) -- Weitere Gateway-spezifische Konfigurationen - -### Pre-Processing Config -- Konfiguration wird als JSON im Gateway Code definiert -- Wird täglich um 01:00 UTC an Pre-Processing Service gesendet -- Definiert Tabellen, Spalten, Verarbeitungsschritte - diff --git a/modules/features/featuresLifecycle.py b/modules/features/featuresLifecycle.py index d1ee20ad..1557db6f 100644 --- a/modules/features/featuresLifecycle.py +++ b/modules/features/featuresLifecycle.py @@ -1,24 +1,37 @@ import logging -from modules.interfaces.interfaceDbAppObjects import getRootInterface +from modules.services import getInterface as getServices logger = logging.getLogger(__name__) -async def start() -> None: - """ Start feature triggers and background managers """ - - # Provide Event User - rootInterface = getRootInterface() - eventUser = rootInterface.getUserByUsername("event") +async def start(eventUser) -> None: + """ Start feature triggers and background managers + + Args: + eventUser: System-level event user for background operations (provided by app.py) + """ # Feature Automation Events if eventUser: try: - from modules.interfaces.interfaceDbChatObjects import getInterface as getChatInterface - chatInterface = getChatInterface(eventUser) - await chatInterface.syncAutomationEvents() + from modules.features.automation import syncAutomationEvents + from modules.shared.callbackRegistry import callbackRegistry + + # Get services for event user (provides access to interfaces) + services = getServices(eventUser, None) + + # Register callback for automation changes + async def onAutomationChanged(chatInterface): + """Callback triggered when automations are created/updated/deleted.""" + await syncAutomationEvents(chatInterface, eventUser) + + callbackRegistry.register('automation.changed', onAutomationChanged) + logger.info("Registered automation change callback") + + # Initial sync on startup - use interface from services + await syncAutomationEvents(services.interfaceDbChat, eventUser) logger.info("Automation events synced on startup") except Exception as e: - logger.error(f"Error syncing automation events on startup: {str(e)}") + logger.error(f"Error setting up automation events on startup: {str(e)}") # Don't fail startup if automation sync fails # Feature SyncDelta @@ -36,8 +49,21 @@ async def start() -> None: -async def stop() -> None: - """ Stop feature triggers and background managers """ +async def stop(eventUser) -> None: + """ Stop feature triggers and background managers + + Args: + eventUser: System-level event user (provided by app.py) + """ + + # Unregister automation callback + try: + from modules.shared.callbackRegistry import callbackRegistry + # Note: We'd need to store the callback reference to unregister it properly + # For now, callbacks will remain registered (acceptable for shutdown) + logger.info("Automation callbacks remain registered (will be cleaned up on process exit)") + except Exception as e: + logger.warning(f"Error during automation callback cleanup: {str(e)}") # Feature ... diff --git a/modules/interfaces/interfaceAiObjects.py b/modules/interfaces/interfaceAiObjects.py index 7dc7db6b..3cc4d2a5 100644 --- a/modules/interfaces/interfaceAiObjects.py +++ b/modules/interfaces/interfaceAiObjects.py @@ -75,15 +75,7 @@ class AiObjects: # AI for Extraction, Processing, Generation - async def call(self, request: AiCallRequest, progressCallback=None) -> AiCallResponse: - """Call AI model for text generation with model-aware chunking.""" - # Handle content parts (unified path) - if hasattr(request, 'contentParts') and request.contentParts: - return await self._callWithContentParts(request, progressCallback) - # Handle traditional text/context calls - return await self._callWithTextContext(request) - - async def _callWithTextContext(self, request: AiCallRequest) -> AiCallResponse: + async def callWithTextContext(self, request: AiCallRequest) -> AiCallResponse: """Call AI model for traditional text/context calls with fallback mechanism.""" prompt = request.prompt context = request.context or "" @@ -148,412 +140,6 @@ class AiObjects: errorCount=1 ) - async def _callWithContentParts(self, request: AiCallRequest, progressCallback=None) -> AiCallResponse: - """Process content parts with model-aware chunking (unified for single and multiple parts).""" - prompt = request.prompt - options = request.options - contentParts = request.contentParts - - # Get failover models - availableModels = modelRegistry.getAvailableModels() - failoverModelList = modelSelector.getFailoverModelList(prompt, "", options, availableModels) - - if not failoverModelList: - return self._createErrorResponse("No suitable models found", 0, 0) - - # Process each content part - allResults = [] - for contentPart in contentParts: - partResult = await self._processContentPartWithFallback(contentPart, prompt, options, failoverModelList, progressCallback) - allResults.append(partResult) - - # Merge all results - mergedContent = self._mergePartResults(allResults) - - return AiCallResponse( - content=mergedContent, - modelName="multiple", - priceUsd=sum(r.priceUsd for r in allResults), - processingTime=sum(r.processingTime for r in allResults), - bytesSent=sum(r.bytesSent for r in allResults), - bytesReceived=sum(r.bytesReceived for r in allResults), - errorCount=sum(r.errorCount for r in allResults) - ) - - async def _processContentPartWithFallback(self, contentPart, prompt: str, options, failoverModelList, progressCallback=None) -> AiCallResponse: - """Process a single content part with model-aware chunking and fallback.""" - lastError = None - - # Check if this is an image - Vision models need special handling - isImage = (contentPart.typeGroup == "image") or (contentPart.mimeType and contentPart.mimeType.startswith("image/")) - - # Determine the correct operation type based on content type - # Images should use IMAGE_ANALYSE, not the generic operation type - actualOperationType = options.operationType - if isImage: - actualOperationType = OperationTypeEnum.IMAGE_ANALYSE - # Get vision-capable models for images - availableModels = modelRegistry.getAvailableModels() - visionFailoverList = modelSelector.getFailoverModelList(prompt, "", AiCallOptions(operationType=actualOperationType), availableModels) - if visionFailoverList: - logger.debug(f"Using {len(visionFailoverList)} vision-capable models for image processing") - failoverModelList = visionFailoverList - - for attempt, model in enumerate(failoverModelList): - try: - logger.info(f"Processing content part with model: {model.name} (attempt {attempt + 1}/{len(failoverModelList)})") - - # Special handling for images with Vision models - if isImage and hasattr(model, 'functionCall'): - # Call model's functionCall directly (for Vision models this is callAiImage) - from modules.datamodels.datamodelAi import AiModelCall, AiCallOptions as AiCallOpts - - try: - # Validate and prepare image data - if not contentPart.data: - raise ValueError("Image content part has no data") - - # Ensure mimeType is valid - mimeType = contentPart.mimeType or "image/jpeg" - if not mimeType.startswith("image/"): - raise ValueError(f"Invalid mimeType for image: {mimeType}") - - # Prepare base64 data - if isinstance(contentPart.data, str): - # Already base64 encoded - validate it - try: - base64.b64decode(contentPart.data, validate=True) - base64Data = contentPart.data - except Exception as e: - raise ValueError(f"Invalid base64 data in contentPart: {str(e)}") - elif isinstance(contentPart.data, bytes): - # Binary data - encode to base64 - base64Data = base64.b64encode(contentPart.data).decode('utf-8') - else: - raise ValueError(f"Unsupported data type for image: {type(contentPart.data)}") - - # Create data URL - imageDataUrl = f"data:{mimeType};base64,{base64Data}" - - modelCall = AiModelCall( - messages=[ - { - "role": "user", - "content": [ - {"type": "text", "text": prompt or ""}, - { - "type": "image_url", - "image_url": { - "url": imageDataUrl - } - } - ] - } - ], - model=model, - options=AiCallOpts(operationType=actualOperationType) - ) - - modelResponse = await model.functionCall(modelCall) - - if not modelResponse.success: - raise ValueError(f"Model call failed: {modelResponse.error}") - - logger.info(f"✅ Image content part processed successfully with model: {model.name}") - - # Convert to AiCallResponse format - # Note: AiModelResponse doesn't have priceUsd, and processingTime can be None - # Calculate processing time if not provided (fallback to 0.0) - processingTime = getattr(modelResponse, 'processingTime', None) - if processingTime is None: - processingTime = 0.0 - - return AiCallResponse( - content=modelResponse.content, - modelName=model.name, - priceUsd=0.0, # Price will be calculated elsewhere if needed - processingTime=processingTime, - bytesSent=0, # Will be calculated elsewhere - bytesReceived=0, # Will be calculated elsewhere - errorCount=0 - ) - except Exception as e: - # Image processing failed with this model - lastError = e - logger.warning(f"❌ Image processing failed with model {model.name}: {str(e)}") - - # If this is not the last model, try the next one - if attempt < len(failoverModelList) - 1: - logger.info(f"🔄 Trying next fallback model for image processing...") - continue - else: - # All models failed - logger.error(f"💥 All {len(failoverModelList)} models failed for image processing") - raise - - # For non-image parts, check if part fits in model context - # Calculate available space accounting for prompt, system message, and output reservation - partSize = len(contentPart.data.encode('utf-8')) if contentPart.data else 0 - - # Use same calculation as _chunkContentPart to determine actual available space - modelContextTokens = model.contextLength - modelMaxOutputTokens = model.maxTokens - - # Reserve tokens for prompt, system message, output, and message overhead - promptTokens = len(prompt.encode('utf-8')) / 4 if prompt else 0 - systemMessageTokens = 10 # ~40 bytes = 10 tokens - outputTokens = modelMaxOutputTokens - messageOverheadTokens = 100 - totalReservedTokens = promptTokens + systemMessageTokens + messageOverheadTokens + outputTokens - - # Available tokens for content (with 80% safety margin) - availableContentTokens = int((modelContextTokens - totalReservedTokens) * 0.8) - if availableContentTokens < 100: - availableContentTokens = max(100, int(modelContextTokens * 0.1)) - - # Convert to bytes (1 token ≈ 4 bytes) - availableContentBytes = availableContentTokens * 4 - - logger.debug(f"Size check for {model.name}: partSize={partSize} bytes, availableContentBytes={availableContentBytes} bytes (contextLength={modelContextTokens} tokens, reserved={totalReservedTokens:.0f} tokens)") - - if partSize <= availableContentBytes: - # Part fits - call AI directly - response = await self._callWithModel(model, prompt, contentPart.data, options) - logger.info(f"✅ Content part processed successfully with model: {model.name}") - return response - else: - # Part too large - chunk it (pass prompt to account for it in chunk size calculation) - chunks = await self._chunkContentPart(contentPart, model, options, prompt) - if not chunks: - raise ValueError(f"Failed to chunk content part for model {model.name}") - - logger.info(f"Starting to process {len(chunks)} chunks with model {model.name}") - - # Log progress if callback provided - if progressCallback: - progressCallback(0.0, f"Starting to process {len(chunks)} chunks") - - # Process each chunk - chunkResults = [] - for idx, chunk in enumerate(chunks): - chunkNum = idx + 1 - chunkData = chunk.get('data', '') - chunkSize = len(chunkData.encode('utf-8')) if chunkData else 0 - logger.info(f"Processing chunk {chunkNum}/{len(chunks)} with model {model.name}, chunk size: {chunkSize} bytes") - - # Calculate and log progress - if progressCallback: - progress = chunkNum / len(chunks) - progressCallback(progress, f"Processing chunk {chunkNum}/{len(chunks)}") - - try: - chunkResponse = await self._callWithModel(model, prompt, chunkData, options) - chunkResults.append(chunkResponse) - logger.info(f"✅ Chunk {chunkNum}/{len(chunks)} processed successfully") - - # Log completion progress - if progressCallback: - progressCallback(chunkNum / len(chunks), f"Chunk {chunkNum}/{len(chunks)} processed") - except Exception as e: - logger.error(f"❌ Error processing chunk {chunkNum}/{len(chunks)}: {str(e)}") - raise - - # Merge chunk results - mergedContent = self._mergeChunkResults(chunkResults) - totalPrice = sum(r.priceUsd for r in chunkResults) - totalTime = sum(r.processingTime for r in chunkResults) - totalBytesSent = sum(r.bytesSent for r in chunkResults) - totalBytesReceived = sum(r.bytesReceived for r in chunkResults) - totalErrors = sum(r.errorCount for r in chunkResults) - - logger.info(f"✅ Content part chunked and processed with model: {model.name} ({len(chunks)} chunks)") - return AiCallResponse( - content=mergedContent, - modelName=model.name, - priceUsd=totalPrice, - processingTime=totalTime, - bytesSent=totalBytesSent, - bytesReceived=totalBytesReceived, - errorCount=totalErrors - ) - - except Exception as e: - lastError = e - error_msg = str(e) if str(e) else f"{type(e).__name__}" - error_detail = f"❌ Model {model.name} failed for content part: {error_msg}" - if hasattr(e, 'detail') and e.detail: - error_detail += f" | Detail: {e.detail}" - if hasattr(e, 'status_code'): - error_detail += f" | Status: {e.status_code}" - logger.warning(error_detail, exc_info=True) - - if attempt < len(failoverModelList) - 1: - logger.info(f"🔄 Trying next failover model...") - continue - else: - logger.error(f"💥 All {len(failoverModelList)} models failed for content part") - break - - # All models failed - return self._createErrorResponse(f"All models failed: {str(lastError)}", 0, 0) - - async def _chunkContentPart(self, contentPart, model, options, prompt: str = "") -> List[Dict[str, Any]]: - """Chunk a content part based on model capabilities, accounting for prompt, system message overhead, and maxTokens output.""" - # Calculate model-specific chunk sizes - modelContextTokens = model.contextLength # Total context in tokens - modelMaxOutputTokens = model.maxTokens # Maximum output tokens - - # Reserve tokens for: - # 1. Prompt (user message) - promptTokens = len(prompt.encode('utf-8')) / 4 if prompt else 0 - - # 2. System message wrapper ("Context from documents:\n") - systemMessageTokens = 10 # ~40 bytes = 10 tokens - - # 3. Max output tokens (model will reserve space for completion) - outputTokens = modelMaxOutputTokens - - # 4. JSON structure and message overhead (~100 tokens) - messageOverheadTokens = 100 - - # Total reserved tokens = input overhead + output reservation - totalReservedTokens = promptTokens + systemMessageTokens + messageOverheadTokens + outputTokens - - # Available tokens for content = context length - reserved tokens - # Use 80% of available for safety margin - availableContentTokens = int((modelContextTokens - totalReservedTokens) * 0.8) - - # Ensure we have at least some space - if availableContentTokens < 100: - logger.warning(f"Very limited space for content: {availableContentTokens} tokens available. Model: {model.name}, contextLength: {modelContextTokens}, maxTokens: {modelMaxOutputTokens}, prompt: {promptTokens:.0f} tokens") - availableContentTokens = max(100, int(modelContextTokens * 0.1)) # Fallback to 10% of context - - # Convert tokens to bytes (1 token ≈ 4 bytes) - availableContentBytes = availableContentTokens * 4 - - logger.debug(f"Chunking calculation for {model.name}: contextLength={modelContextTokens} tokens, maxTokens={modelMaxOutputTokens} tokens, prompt={promptTokens:.0f} tokens, reserved={totalReservedTokens:.0f} tokens, available={availableContentTokens} tokens ({availableContentBytes} bytes)") - - # Use 70% of available content bytes for text chunks (conservative) - textChunkSize = int(availableContentBytes * 0.7) - imageChunkSize = int(availableContentBytes * 0.8) # 80% for image chunks - - # Build chunking options - chunkingOptions = { - "textChunkSize": textChunkSize, - "imageChunkSize": imageChunkSize, - "maxSize": availableContentBytes, - "chunkAllowed": True - } - - # Get appropriate chunker - from modules.services.serviceExtraction.subRegistry import ChunkerRegistry - chunkerRegistry = ChunkerRegistry() - chunker = chunkerRegistry.resolve(contentPart.typeGroup) - - if not chunker: - logger.warning(f"No chunker found for typeGroup: {contentPart.typeGroup}") - return [] - - # Chunk the content part - try: - chunks = chunker.chunk(contentPart, chunkingOptions) - logger.debug(f"Created {len(chunks)} chunks for {contentPart.typeGroup} part") - return chunks - except Exception as e: - logger.error(f"Chunking failed for {contentPart.typeGroup}: {str(e)}") - return [] - - def _mergePartResults(self, partResults: List[AiCallResponse]) -> str: - """Merge part results using the existing sophisticated merging system.""" - if not partResults: - return "" - - # Convert AiCallResponse results to ContentParts for merging - from modules.datamodels.datamodelExtraction import ContentPart - from modules.services.serviceExtraction.subUtils import makeId - - content_parts = [] - for i, result in enumerate(partResults): - if result.content: - content_part = ContentPart( - id=str(uuid.uuid4()), - parentId=None, - label=f"ai_result_{i}", - typeGroup="text", # Default to text for AI results - mimeType="text/plain", - data=result.content, - metadata={ - "aiResult": True, - "modelName": result.modelName, - "priceUsd": result.priceUsd, - "processingTime": result.processingTime, - "bytesSent": result.bytesSent, - "bytesReceived": result.bytesReceived - } - ) - content_parts.append(content_part) - - # Use existing merging system - merge_strategy = MergeStrategy( - useIntelligentMerging=True, - groupBy="typeGroup", - orderBy="id", - mergeType="concatenate" - ) - - merged_parts = applyMerging(content_parts, merge_strategy) - - # Convert merged parts back to final string - final_content = "\n\n".join([part.data for part in merged_parts]) - - logger.info(f"Merged {len(partResults)} AI results using existing merging system") - return final_content.strip() - - def _mergeChunkResults(self, chunkResults: List[AiCallResponse]) -> str: - """Merge chunk results using the existing sophisticated merging system.""" - if not chunkResults: - return "" - - # Convert AiCallResponse results to ContentParts for merging - - content_parts = [] - for i, result in enumerate(chunkResults): - if result.content: - content_part = ContentPart( - id=str(uuid.uuid4()), - parentId=None, - label=f"chunk_result_{i}", - typeGroup="text", # Default to text for AI results - mimeType="text/plain", - data=result.content, - metadata={ - "aiResult": True, - "chunk": True, - "modelName": result.modelName, - "priceUsd": result.priceUsd, - "processingTime": result.processingTime, - "bytesSent": result.bytesSent, - "bytesReceived": result.bytesReceived - } - ) - content_parts.append(content_part) - - # Use existing merging system - merge_strategy = MergeStrategy( - useIntelligentMerging=True, - groupBy="typeGroup", - orderBy="id", - mergeType="concatenate" - ) - - merged_parts = applyMerging(content_parts, merge_strategy) - - # Convert merged parts back to final string - final_content = "\n\n".join([part.data for part in merged_parts]) - - logger.info(f"Merged {len(chunkResults)} chunk results using existing merging system") - return final_content.strip() - def _createErrorResponse(self, errorMsg: str, inputBytes: int, outputBytes: int) -> AiCallResponse: """Create an error response.""" return AiCallResponse( @@ -659,64 +245,4 @@ class AiObjects: return [model.displayName for model in models] -def applyMerging(parts: List[ContentPart], strategy: MergeStrategy) -> List[ContentPart]: - """Apply merging strategy to parts with intelligent token-aware merging.""" - logger.debug(f"applyMerging called with {len(parts)} parts") - - # Import merging dependencies - from modules.services.serviceExtraction.merging.mergerText import TextMerger - from modules.services.serviceExtraction.merging.mergerTable import TableMerger - from modules.services.serviceExtraction.merging.mergerDefault import DefaultMerger - from modules.services.serviceExtraction.subMerger import IntelligentTokenAwareMerger - - # Check if intelligent merging is enabled - if strategy.useIntelligentMerging: - modelCapabilities = strategy.capabilities or {} - subMerger = IntelligentTokenAwareMerger(modelCapabilities) - - # Use intelligent merging for all parts - merged = subMerger.mergeChunksIntelligently(parts, strategy.prompt or "") - - # Calculate and log optimization stats - stats = subMerger.calculateOptimizationStats(parts, merged) - logger.info(f"🧠 Intelligent merging stats: {stats}") - logger.debug(f"Intelligent merging: {stats['original_ai_calls']} → {stats['optimized_ai_calls']} calls ({stats['reduction_percent']}% reduction)") - - return merged - - # Fallback to traditional merging - textMerger = TextMerger() - tableMerger = TableMerger() - defaultMerger = DefaultMerger() - - # Group by typeGroup - textParts = [p for p in parts if p.typeGroup == "text"] - tableParts = [p for p in parts if p.typeGroup == "table"] - structureParts = [p for p in parts if p.typeGroup == "structure"] - otherParts = [p for p in parts if p.typeGroup not in ("text", "table", "structure")] - - logger.debug(f"Grouped - text: {len(textParts)}, table: {len(tableParts)}, structure: {len(structureParts)}, other: {len(otherParts)}") - - merged: List[ContentPart] = [] - - if textParts: - textMerged = textMerger.merge(textParts, strategy) - logger.debug(f"TextMerger merged {len(textParts)} parts into {len(textMerged)} parts") - merged.extend(textMerged) - if tableParts: - tableMerged = tableMerger.merge(tableParts, strategy) - logger.debug(f"TableMerger merged {len(tableParts)} parts into {len(tableMerged)} parts") - merged.extend(tableMerged) - if structureParts: - # For now, treat structure like text - structureMerged = textMerger.merge(structureParts, strategy) - logger.debug(f"StructureMerger merged {len(structureParts)} parts into {len(structureMerged)} parts") - merged.extend(structureMerged) - if otherParts: - otherMerged = defaultMerger.merge(otherParts, strategy) - logger.debug(f"DefaultMerger merged {len(otherParts)} parts into {len(otherMerged)} parts") - merged.extend(otherMerged) - - logger.debug(f"applyMerging returning {len(merged)} parts") - return merged diff --git a/modules/interfaces/interfaceDbChatObjects.py b/modules/interfaces/interfaceDbChatObjects.py index 0b217bd1..7e3c85e6 100644 --- a/modules/interfaces/interfaceDbChatObjects.py +++ b/modules/interfaces/interfaceDbChatObjects.py @@ -37,6 +37,136 @@ logger = logging.getLogger(__name__) # Singleton factory for Chat instances _chatInterfaces = {} + +def storeDebugMessageAndDocuments(message, currentUser) -> None: + """ + Store message and documents (metadata and file bytes) for debugging purposes. + Structure: {log_dir}/debug/messages/m_round_task_action_timestamp/documentlist_label/ + - message.json, message_text.txt + - document_###_metadata.json + - document_###_ (actual file bytes) + + Args: + message: ChatMessage object to store + currentUser: Current user for component interface access + """ + try: + import os + from datetime import datetime, UTC + from modules.shared.debugLogger import _getBaseDebugDir, _ensureDir + from modules.interfaces.interfaceDbComponentObjects import getInterface + + # Create base debug directory (use base debug dir, not prompts subdirectory) + baseDebugDir = _getBaseDebugDir() + debug_root = os.path.join(baseDebugDir, 'messages') + _ensureDir(debug_root) + + # Generate timestamp + timestamp = datetime.now(UTC).strftime('%Y%m%d-%H%M%S-%f')[:-3] + + # Create message folder name: m_round_task_action_timestamp + # Use actual values from message, not defaults + round_str = str(message.roundNumber) if message.roundNumber is not None else "0" + task_str = str(message.taskNumber) if message.taskNumber is not None else "0" + action_str = str(message.actionNumber) if message.actionNumber is not None else "0" + message_folder = f"{timestamp}_m_{round_str}_{task_str}_{action_str}" + + message_path = os.path.join(debug_root, message_folder) + os.makedirs(message_path, exist_ok=True) + + # Store message data - use dict() instead of model_dump() for compatibility + message_file = os.path.join(message_path, "message.json") + with open(message_file, "w", encoding="utf-8") as f: + # Convert message to dict manually to avoid model_dump() issues + message_dict = { + "id": message.id, + "workflowId": message.workflowId, + "parentMessageId": message.parentMessageId, + "message": message.message, + "role": message.role, + "status": message.status, + "sequenceNr": message.sequenceNr, + "publishedAt": message.publishedAt, + "roundNumber": message.roundNumber, + "taskNumber": message.taskNumber, + "actionNumber": message.actionNumber, + "documentsLabel": message.documentsLabel, + "actionId": message.actionId, + "actionMethod": message.actionMethod, + "actionName": message.actionName, + "success": message.success, + "documents": [] + } + json.dump(message_dict, f, indent=2, ensure_ascii=False, default=str) + + # Store message content as text + if message.message: + message_text_file = os.path.join(message_path, "message_text.txt") + with open(message_text_file, "w", encoding="utf-8") as f: + f.write(str(message.message)) + + # Store documents if provided + if message.documents and len(message.documents) > 0: + # Group documents by documentsLabel + documents_by_label = {} + for doc in message.documents: + label = message.documentsLabel or 'default' + if label not in documents_by_label: + documents_by_label[label] = [] + documents_by_label[label].append(doc) + + # Create subfolder for each document label + for label, docs in documents_by_label.items(): + # Sanitize label for filesystem + safe_label = "".join(c for c in str(label) if c.isalnum() or c in (' ', '-', '_')).rstrip() + safe_label = safe_label.replace(' ', '_') + if not safe_label: + safe_label = "default" + + label_folder = os.path.join(message_path, safe_label) + _ensureDir(label_folder) + + # Store each document + for i, doc in enumerate(docs): + # Create document metadata file + doc_meta = { + "id": doc.id, + "messageId": doc.messageId, + "fileId": doc.fileId, + "fileName": doc.fileName, + "fileSize": doc.fileSize, + "mimeType": doc.mimeType, + "roundNumber": doc.roundNumber, + "taskNumber": doc.taskNumber, + "actionNumber": doc.actionNumber, + "actionId": doc.actionId + } + + doc_meta_file = os.path.join(label_folder, f"document_{i+1:03d}_metadata.json") + with open(doc_meta_file, "w", encoding="utf-8") as f: + json.dump(doc_meta, f, indent=2, ensure_ascii=False, default=str) + + # Also store the actual file bytes next to metadata for debugging + try: + componentInterface = getInterface(currentUser) + file_bytes = componentInterface.getFileData(doc.fileId) + if file_bytes: + # Build a safe filename preserving original name + safe_name = doc.fileName or f"document_{i+1:03d}" + # Avoid path traversal + safe_name = os.path.basename(safe_name) + doc_file_path = os.path.join(label_folder, f"document_{i+1:03d}_" + safe_name) + with open(doc_file_path, "wb") as df: + df.write(file_bytes) + else: + pass + except Exception as e: + pass + + except Exception as e: + # Silent fail - don't break main flow + pass + class ChatObjects: """ Interface to Chat database and AI Connectors. @@ -893,7 +1023,6 @@ class ChatObjects: ) # Debug: Store message and documents for debugging - only if debug enabled - from modules.shared.debugLogger import storeDebugMessageAndDocuments storeDebugMessageAndDocuments(chat_message, self.currentUser) return chat_message @@ -1550,8 +1679,8 @@ class ChatObjects: if createdAutomation.get("executionLogs") is None: createdAutomation["executionLogs"] = [] - # Trigger sync (async, don't wait) - asyncio.create_task(self.syncAutomationEvents()) + # Trigger automation change callback (async, don't wait) + asyncio.create_task(self._notifyAutomationChanged()) return createdAutomation except Exception as e: @@ -1581,8 +1710,8 @@ class ChatObjects: if updatedAutomation.get("executionLogs") is None: updatedAutomation["executionLogs"] = [] - # Trigger sync (async, don't wait) - asyncio.create_task(self.syncAutomationEvents()) + # Trigger automation change callback (async, don't wait) + asyncio.create_task(self._notifyAutomationChanged()) return updatedAutomation except Exception as e: @@ -1611,374 +1740,22 @@ class ChatObjects: # Delete automation from database self.db.recordDelete(AutomationDefinition, automationId) - # Trigger sync (async, don't wait) - asyncio.create_task(self.syncAutomationEvents()) + # Trigger automation change callback (async, don't wait) + asyncio.create_task(self._notifyAutomationChanged()) return True except Exception as e: logger.error(f"Error deleting automation definition: {str(e)}") raise - def _replacePlaceholders(self, template: str, placeholders: Dict[str, str]) -> str: - """Replace placeholders in template with actual values. Placeholder format: {{KEY:PLACEHOLDER_NAME}}""" - result = template - for placeholderName, value in placeholders.items(): - pattern = f"{{{{KEY:{placeholderName}}}}}" - - # Check if placeholder is in an array context like ["{{KEY:...}}"] - # If value is a JSON array/dict, we should replace the entire ["{{KEY:...}}"] with the array - arrayPattern = f'["{pattern}"]' - if arrayPattern in result: - # Check if value is a JSON array/dict - isArrayValue = False - arrayValue = None - - if isinstance(value, (list, dict)): - isArrayValue = True - arrayValue = json.dumps(value) - elif isinstance(value, str): - try: - parsed = json.loads(value) - if isinstance(parsed, (list, dict)): - isArrayValue = True - arrayValue = value # Already valid JSON string - except (json.JSONDecodeError, ValueError): - pass - - if isArrayValue: - # Replace ["{{KEY:...}}"] with the array value - result = result.replace(arrayPattern, arrayValue) - continue # Skip the regular replacement below - - # Regular replacement - check if in quoted context - patternStart = result.find(pattern) - isQuoted = False - if patternStart > 0: - charBefore = result[patternStart - 1] if patternStart > 0 else None - patternEnd = patternStart + len(pattern) - charAfter = result[patternEnd] if patternEnd < len(result) else None - if charBefore == '"' and charAfter == '"': - isQuoted = True - - # Handle different value types - if isinstance(value, (list, dict)): - # Python list/dict - convert to JSON - replacement = json.dumps(value) - elif isinstance(value, str): - # String value - check if it's a JSON string representing list/dict - try: - parsed = json.loads(value) - if isinstance(parsed, (list, dict)): - # It's a JSON string of a list/dict - if isQuoted: - # In quoted context, escape the JSON string - escaped = json.dumps(value) - replacement = escaped[1:-1] # Remove outer quotes - else: - # In unquoted context, use JSON directly - replacement = value - else: - # It's a JSON string of a primitive - if isQuoted: - escaped = json.dumps(value) - replacement = escaped[1:-1] - else: - replacement = value - except (json.JSONDecodeError, ValueError): - # Not valid JSON - treat as plain string - if isQuoted: - escaped = json.dumps(value) - replacement = escaped[1:-1] - else: - replacement = value - else: - # Numbers, booleans, None - convert to string - replacement = str(value) - result = result.replace(pattern, replacement) - return result - - def _parseScheduleToCron(self, schedule: str) -> Dict[str, Any]: - """Parse schedule string to cron kwargs for APScheduler""" - parts = schedule.split() - if len(parts) != 5: - raise ValueError(f"Invalid schedule format: {schedule}") - - return { - "minute": parts[0], - "hour": parts[1], - "day": parts[2], - "month": parts[3], - "day_of_week": parts[4] - } - - async def executeAutomation(self, automationId: str) -> ChatWorkflow: - """Execute automation workflow immediately (test mode) with placeholder replacement""" - executionStartTime = getUtcTimestamp() - executionLog = { - "timestamp": executionStartTime, - "workflowId": None, - "status": "running", - "messages": [] - } - + async def _notifyAutomationChanged(self): + """Notify registered callbacks about automation changes (decoupled from features).""" try: - # 1. Load automation definition - automation = self.getAutomationDefinition(automationId) - if not automation: - raise ValueError(f"Automation {automationId} not found") - - executionLog["messages"].append(f"Started execution at {executionStartTime}") - - # 2. Replace placeholders in template to generate plan - template = automation.get("template", "") - placeholders = automation.get("placeholders", {}) - planJson = self._replacePlaceholders(template, placeholders) - try: - plan = json.loads(planJson) - except json.JSONDecodeError as e: - logger.error(f"Failed to parse plan JSON after placeholder replacement: {str(e)}") - logger.error(f"Template: {template[:500]}...") - logger.error(f"Placeholders: {placeholders}") - logger.error(f"Generated planJson (first 1000 chars): {planJson[:1000]}") - logger.error(f"Error position: line {e.lineno}, column {e.colno}, char {e.pos}") - if e.pos: - start = max(0, e.pos - 100) - end = min(len(planJson), e.pos + 100) - logger.error(f"Context around error: ...{planJson[start:end]}...") - raise ValueError(f"Invalid JSON after placeholder replacement: {str(e)}") - executionLog["messages"].append("Template placeholders replaced successfully") - - # 3. Get user who created automation - creator_user_id = automation.get("_createdBy") - - # If _createdBy is missing, try to fix it by setting it to current user - # This handles automations created before _createdBy was required - if not creator_user_id: - logger.warning(f"Automation {automationId} has no creator user, setting to current user {self.userId}") - try: - # Update the automation to set _createdBy - self.db.recordModify( - AutomationDefinition, - automationId, - {"_createdBy": self.userId} - ) - creator_user_id = self.userId - automation["_createdBy"] = self.userId - logger.info(f"Fixed automation {automationId} by setting _createdBy to {self.userId}") - executionLog["messages"].append(f"Fixed missing _createdBy field, set to user {self.userId}") - except Exception as e: - logger.error(f"Error fixing automation {automationId}: {str(e)}") - raise ValueError(f"Automation {automationId} has no creator user and could not be fixed") - - # Get user from database - from modules.interfaces.interfaceDbAppObjects import getInterface as getAppInterface - appInterface = getAppInterface(self.currentUser) - creator_user = appInterface.getUser(creator_user_id) - if not creator_user: - raise ValueError(f"Creator user {creator_user_id} not found") - - executionLog["messages"].append(f"Using creator user: {creator_user_id}") - - # 4. Create UserInputRequest from plan - # Embed plan JSON in prompt for TemplateMode to extract - promptText = self._planToPrompt(plan) - planJson = json.dumps(plan) - # Embed plan as JSON comment so TemplateMode can extract it - promptWithPlan = f"{promptText}\n\n\n{planJson}\n" - - userInput = UserInputRequest( - prompt=promptWithPlan, - listFileId=[], - userLanguage=creator_user.language or "en" - ) - - executionLog["messages"].append("Starting workflow execution") - - # 5. Start workflow using chatStart - from modules.features.chatPlayground.mainChatPlayground import chatStart - - workflow = await chatStart( - currentUser=creator_user, - userInput=userInput, - workflowMode=WorkflowModeEnum.WORKFLOW_AUTOMATION, - workflowId=None - ) - - executionLog["workflowId"] = workflow.id - executionLog["status"] = "completed" - executionLog["messages"].append(f"Workflow {workflow.id} started successfully") - logger.info(f"Started workflow {workflow.id} with plan containing {len(plan.get('tasks', []))} tasks (plan embedded in userInput)") - - # Set workflow name with "automated" prefix - automationLabel = automation.get("label", "Unknown Automation") - workflowName = f"automated: {automationLabel}" - workflow = self.updateWorkflow(workflow.id, {"name": workflowName}) - logger.info(f"Set workflow {workflow.id} name to: {workflowName}") - - # Update automation with execution log - executionLogs = automation.get("executionLogs", []) - executionLogs.append(executionLog) - # Keep only last 50 executions - if len(executionLogs) > 50: - executionLogs = executionLogs[-50:] - - self.db.recordModify( - AutomationDefinition, - automationId, - {"executionLogs": executionLogs} - ) - - return workflow + from modules.shared.callbackRegistry import callbackRegistry + # Trigger callbacks without knowing which features are listening + await callbackRegistry.trigger('automation.changed', self) except Exception as e: - # Log error to execution log - executionLog["status"] = "error" - executionLog["messages"].append(f"Error: {str(e)}") - - # Update automation with execution log even on error - try: - automation = self.getAutomationDefinition(automationId) - if automation: - executionLogs = automation.get("executionLogs", []) - executionLogs.append(executionLog) - if len(executionLogs) > 50: - executionLogs = executionLogs[-50:] - self.db.recordModify( - AutomationDefinition, - automationId, - {"executionLogs": executionLogs} - ) - except Exception as logError: - logger.error(f"Error saving execution log: {str(logError)}") - - raise - - def _planToPrompt(self, plan: Dict) -> str: - """Convert plan structure to prompt string for workflow execution""" - return plan.get("userMessage", plan.get("overview", "Execute automation workflow")) - - async def syncAutomationEvents(self) -> Dict[str, Any]: - """Automation event handler - syncs scheduler with all active automations.""" - from modules.shared.eventManagement import eventManager - - # Get all automation definitions (for current mandate) - allAutomations = self.db.getRecordset(AutomationDefinition) - filtered = self._uam(AutomationDefinition, allAutomations) - - registered_events = {} - - for automation in filtered: - automation_id = automation.get("id") - is_active = automation.get("active", False) - current_event_id = automation.get("eventId") - schedule = automation.get("schedule") - - if not schedule: - logger.warning(f"Automation {automation_id} has no schedule, skipping") - continue - - try: - # Parse schedule to cron kwargs - cron_kwargs = self._parseScheduleToCron(schedule) - - if is_active: - # Remove existing event if present (handles schedule changes) - if current_event_id: - try: - eventManager.remove(current_event_id) - except Exception as e: - logger.warning(f"Error removing old event {current_event_id}: {str(e)}") - - # Register new event - new_event_id = f"automation.{automation_id}" - - # Create event handler function - handler = self._createAutomationEventHandler(automation_id) - - # Register cron job - eventManager.registerCron( - jobId=new_event_id, - func=handler, - cronKwargs=cron_kwargs, - replaceExisting=True - ) - - # Update automation with new eventId - if current_event_id != new_event_id: - self.db.recordModify( - AutomationDefinition, - automation_id, - {"eventId": new_event_id} - ) - - registered_events[automation_id] = new_event_id - else: - # Remove event if exists - if current_event_id: - try: - eventManager.remove(current_event_id) - self.db.recordModify( - AutomationDefinition, - automation_id, - {"eventId": None} - ) - except Exception as e: - logger.warning(f"Error removing event {current_event_id}: {str(e)}") - except Exception as e: - logger.error(f"Error syncing automation {automation_id}: {str(e)}") - - return { - "synced": len(registered_events), - "events": registered_events - } - - def _createAutomationEventHandler(self, automationId: str): - """Create event handler function for a specific automation""" - async def handler(): - try: - # Get event user to access automation (event user can access all automations) - from modules.interfaces.interfaceDbAppObjects import getRootInterface - from modules.interfaces.interfaceDbAppObjects import getInterface as getAppInterface - from modules.interfaces.interfaceDbChatObjects import getInterface as getChatInterface - - rootInterface = getRootInterface() - eventUser = rootInterface.getUserByUsername("event") - - if not eventUser: - logger.error("Could not get event user for automation execution") - return - - # Create ChatObjects interface for event user (to access automation) - eventInterface = getChatInterface(eventUser) - - # Load automation using event user context - automation = eventInterface.getAutomationDefinition(automationId) - if not automation or not automation.get("active"): - logger.warning(f"Automation {automationId} not found or not active, skipping execution") - return - - # Get creator user - creator_user_id = automation.get("_createdBy") - if not creator_user_id: - logger.error(f"Automation {automationId} has no creator user") - return - - # Get creator user from database - appInterface = getAppInterface(eventUser) - creator_user = appInterface.getUser(creator_user_id) - if not creator_user: - logger.error(f"Creator user {creator_user_id} not found for automation {automationId}") - return - - # Create ChatObjects interface for creator user - creatorInterface = getChatInterface(creator_user) - - # Execute automation with creator user's context - await creatorInterface.executeAutomation(automationId) - logger.info(f"Successfully executed automation {automationId} as user {creator_user_id}") - except Exception as e: - logger.error(f"Error executing automation {automationId}: {str(e)}") - - return handler + logger.error(f"Error notifying automation change: {str(e)}") def getInterface(currentUser: Optional[User] = None) -> 'ChatObjects': diff --git a/modules/routes/routeAdminAutomationEvents.py b/modules/routes/routeAdminAutomationEvents.py index bb4a233f..dcac4f27 100644 --- a/modules/routes/routeAdminAutomationEvents.py +++ b/modules/routes/routeAdminAutomationEvents.py @@ -86,15 +86,21 @@ async def sync_all_automation_events( requireSysadmin(currentUser) try: - chatInterface = interfaceDbChatObjects.getInterface(currentUser) + from modules.interfaces.interfaceDbChatObjects import getInterface as getChatInterface + from modules.interfaces.interfaceDbAppObjects import getRootInterface + from modules.features.automation import syncAutomationEvents - if not hasattr(chatInterface, 'syncAutomationEvents'): + chatInterface = getChatInterface(currentUser) + # Get event user for sync operation (routes can import from interfaces) + rootInterface = getRootInterface() + eventUser = rootInterface.getUserByUsername("event") + if not eventUser: raise HTTPException( - status_code=501, - detail="Automation methods not available" + status_code=500, + detail="Event user not available" ) - result = await chatInterface.syncAutomationEvents() + result = await syncAutomationEvents(chatInterface, eventUser) return { "success": True, "synced": result.get("synced", 0), diff --git a/modules/services/__init__.py b/modules/services/__init__.py index 3e33d208..84ef638a 100644 --- a/modules/services/__init__.py +++ b/modules/services/__init__.py @@ -84,6 +84,9 @@ class Services: from .serviceWeb.mainServiceWeb import WebService self.web = PublicService(WebService(self)) + from .serviceSecurity.mainServiceSecurity import SecurityService + self.security = PublicService(SecurityService(self)) + def getInterface(user: User, workflow: ChatWorkflow) -> Services: return Services(user, workflow) diff --git a/modules/services/serviceAi/mainServiceAi.py b/modules/services/serviceAi/mainServiceAi.py index 57f81aa7..b7d7aea0 100644 --- a/modules/services/serviceAi/mainServiceAi.py +++ b/modules/services/serviceAi/mainServiceAi.py @@ -48,6 +48,18 @@ class AiService: logger.info("Initializing ExtractionService...") self.extractionService = ExtractionService(self.services) + async def callAi(self, request: AiCallRequest, progressCallback=None): + """Router: handles content parts via extractionService, text context via interface. + + Replaces direct calls to self.aiObjects.call() to route content parts processing + through serviceExtraction layer. + """ + if hasattr(request, 'contentParts') and request.contentParts: + return await self.extractionService.processContentPartsWithAi( + request, self.aiObjects, progressCallback + ) + return await self.aiObjects.callWithTextContext(request) + async def ensureAiObjectsInitialized(self): """Ensure aiObjects is initialized and submodules are ready.""" if self.aiObjects is None: @@ -141,7 +153,7 @@ Respond with ONLY a JSON object in this exact format: ) ) - response = await self.aiObjects.call(request) + response = await self.callAi(request) # Parse AI response using structured parsing with AiCallOptions model try: @@ -251,7 +263,7 @@ Respond with ONLY a JSON object in this exact format: else: self.services.utils.writeDebugFile(iterationPrompt, f"{debugPrefix}_prompt_iteration_{iteration}") - response = await self.aiObjects.call(request) + response = await self.callAi(request) result = response.content # Update progress after AI call @@ -582,7 +594,7 @@ If no trackable items can be identified, return: {{"kpis": []}} # Write KPI definition prompt to debug file self.services.utils.writeDebugFile(kpiDefinitionPrompt, f"{debugPrefix}_kpi_definition_prompt") - response = await self.aiObjects.call(request) + response = await self.callAi(request) # Write KPI definition response to debug file self.services.utils.writeDebugFile(response.content, f"{debugPrefix}_kpi_definition_response") @@ -966,7 +978,7 @@ If no trackable items can be identified, return: {{"kpis": []}} options=options ) - response = await self.aiObjects.call(request) + response = await self.callAi(request) if response.content: # Build document data for image @@ -1011,7 +1023,7 @@ If no trackable items can be identified, return: {{"kpis": []}} options=options ) - response = await self.aiObjects.call(request) + response = await self.callAi(request) if response.content: metadata = AiResponseMetadata( @@ -1046,7 +1058,7 @@ If no trackable items can be identified, return: {{"kpis": []}} options.compressContext = False # Process contentParts for generation prompt (if provided) - # Use generic _callWithContentParts() which handles all content types (images, text, etc.) + # Use generic callWithContentParts() which handles all content types (images, text, etc.) # This automatically processes images with vision models and merges all results if contentParts: # Filter out binary/other parts that shouldn't be processed diff --git a/modules/services/serviceChat/mainServiceChat.py b/modules/services/serviceChat/mainServiceChat.py index 9ff148a8..679b9d17 100644 --- a/modules/services/serviceChat/mainServiceChat.py +++ b/modules/services/serviceChat/mainServiceChat.py @@ -3,7 +3,6 @@ from typing import Dict, Any, List, Optional from modules.datamodels.datamodelUam import User, UserConnection from modules.datamodels.datamodelChat import ChatDocument, ChatMessage, ChatStat, ChatLog from modules.datamodels.datamodelAi import AiCallOptions, OperationTypeEnum, PriorityEnum, ProcessingModeEnum -from modules.security.tokenManager import TokenManager from modules.shared.progressLogger import ProgressLogger logger = logging.getLogger(__name__) @@ -306,9 +305,9 @@ class ChatService: token = None token_status = "unknown" try: - # Get a fresh token via TokenManager convenience method + # Get a fresh token via security service logger.debug(f"Getting fresh token for connection {connection.id}") - token = TokenManager().getFreshToken(connection.id) + token = self.services.security.getFreshToken(connection.id) if token: if hasattr(token, 'expiresAt') and token.expiresAt: current_time = self.services.utils.timestampGetUtc() @@ -389,7 +388,7 @@ class ChatService: Token object or None if not found/expired """ try: - return TokenManager().getFreshToken(connectionId) + return self.services.security.getFreshToken(connectionId) except Exception as e: logger.error(f"Error getting fresh token for connection {connectionId}: {str(e)}") return None diff --git a/modules/services/serviceExtraction/mainServiceExtraction.py b/modules/services/serviceExtraction/mainServiceExtraction.py index d8db9922..360bfc23 100644 --- a/modules/services/serviceExtraction/mainServiceExtraction.py +++ b/modules/services/serviceExtraction/mainServiceExtraction.py @@ -3,13 +3,15 @@ import uuid import logging import time import asyncio +import base64 from .subRegistry import ExtractorRegistry, ChunkerRegistry from .subPipeline import runExtraction from modules.datamodels.datamodelExtraction import ContentExtracted, ContentPart, MergeStrategy, ExtractionOptions, PartResult from modules.datamodels.datamodelChat import ChatDocument -from modules.datamodels.datamodelAi import AiCallResponse, AiCallRequest, AiCallOptions +from modules.datamodels.datamodelAi import AiCallResponse, AiCallRequest, AiCallOptions, OperationTypeEnum, AiModelCall from modules.aicore.aicoreModelRegistry import modelRegistry +from modules.aicore.aicoreModelSelector import modelSelector logger = logging.getLogger(__name__) @@ -498,7 +500,7 @@ class ExtractionService: # Merge results using existing merging system if operationId: self.services.chat.progressLogUpdate(operationId, 0.9, f"Merging {len(partResults)} part results") - mergedContent = self._mergePartResults(partResults, options) + mergedContent = self.mergePartResults(partResults, options) # Save merged extraction content to debug self.services.utils.writeDebugFile(mergedContent or '', "extraction_merged_text") @@ -660,54 +662,473 @@ class ExtractionService: logger.info(f"Completed processing {len(processedResults)} parts") return processedResults - def _mergePartResults( + def _convertToContentParts( + self, partResults: Union[List[PartResult], List[AiCallResponse]] + ) -> List[ContentPart]: + """Convert part results to ContentParts (internal helper for consolidation). + + Handles both PartResult (from extraction workflow) and AiCallResponse (from content parts processing). + """ + content_parts = [] + + if not partResults: + return content_parts + + # Detect input type and convert accordingly + if isinstance(partResults[0], PartResult): + # Existing logic for PartResult (from processDocumentsPerChunk) + for part_result in partResults: + content_part = ContentPart( + id=part_result.originalPart.id, + parentId=part_result.originalPart.parentId, + label=part_result.originalPart.label, + typeGroup=part_result.originalPart.typeGroup, # Use original typeGroup + mimeType=part_result.originalPart.mimeType, + data=part_result.aiResult, # Use AI result as data + metadata={ + **part_result.originalPart.metadata, + "aiResult": True, + "partIndex": part_result.partIndex, + "documentId": part_result.documentId, + "processingTime": part_result.processingTime, + "success": part_result.metadata.get("success", False) + } + ) + content_parts.append(content_part) + elif isinstance(partResults[0], AiCallResponse): + # Logic from interfaceAiObjects (from content parts processing) + for i, result in enumerate(partResults): + if result.content: + content_part = ContentPart( + id=str(uuid.uuid4()), + parentId=None, + label=f"ai_result_{i}", + typeGroup="text", # Default to text for AI results + mimeType="text/plain", + data=result.content, + metadata={ + "aiResult": True, + "modelName": result.modelName, + "priceUsd": result.priceUsd, + "processingTime": result.processingTime, + "bytesSent": result.bytesSent, + "bytesReceived": result.bytesReceived + } + ) + content_parts.append(content_part) + + return content_parts + + def mergePartResults( self, - partResults: List[PartResult], + partResults: Union[List[PartResult], List[AiCallResponse]], options: Optional[AiCallOptions] = None - ) -> str: - """Merge part results using existing sophisticated merging system.""" + ) -> str: + """Unified merge for both PartResult and AiCallResponse. + + Consolidated from both interfaceAiObjects.py and existing serviceExtraction method. + """ if not partResults: return "" - # Convert PartResults back to ContentParts for existing merger system - content_parts = [] - for part_result in partResults: - # Create ContentPart from PartResult with proper typeGroup - content_part = ContentPart( - id=part_result.originalPart.id, - parentId=part_result.originalPart.parentId, - label=part_result.originalPart.label, - typeGroup=part_result.originalPart.typeGroup, # Use original typeGroup - mimeType=part_result.originalPart.mimeType, - data=part_result.aiResult, # Use AI result as data - metadata={ - **part_result.originalPart.metadata, - "aiResult": True, - "partIndex": part_result.partIndex, - "documentId": part_result.documentId, - "processingTime": part_result.processingTime, - "success": part_result.metadata.get("success", False) - } + # Convert to ContentParts using unified helper + content_parts = self._convertToContentParts(partResults) + + # Determine merge strategy based on input type + if isinstance(partResults[0], PartResult): + # Use strategy for extraction workflow (group by document, order by part index) + merge_strategy = MergeStrategy( + useIntelligentMerging=True, + groupBy="documentId", # Group by document + orderBy="partIndex", # Order by part index + mergeType="concatenate" + ) + else: + # Default strategy for content parts workflow + merge_strategy = MergeStrategy( + useIntelligentMerging=True, + groupBy="typeGroup", + orderBy="id", + mergeType="concatenate" ) - content_parts.append(content_part) - # Use existing merging strategy from options - merge_strategy = MergeStrategy( - useIntelligentMerging=True, - groupBy="documentId", # Group by document - orderBy="partIndex", # Order by part index - mergeType="concatenate" - ) - - - # Apply existing merging logic using the sophisticated merging system - from modules.interfaces.interfaceAiObjects import applyMerging + # Apply merging merged_parts = applyMerging(content_parts, merge_strategy) - # Convert merged parts back to final string + # Convert back to string final_content = "\n\n".join([part.data for part in merged_parts]) - logger.info(f"Merged {len(partResults)} parts using existing sophisticated merging system") + logger.info(f"Merged {len(partResults)} parts using unified merging system") return final_content.strip() + async def chunkContentPartForAi(self, contentPart, model, options, prompt: str = "") -> List[Dict[str, Any]]: + """Chunk a content part based on model capabilities, accounting for prompt, system message overhead, and maxTokens output. + + Moved from interfaceAiObjects.py - model-aware chunking for AI processing. + Complementary to existing size-based chunking in extraction pipeline. + """ + # Calculate model-specific chunk sizes + modelContextTokens = model.contextLength # Total context in tokens + modelMaxOutputTokens = model.maxTokens # Maximum output tokens + + # Reserve tokens for: + # 1. Prompt (user message) + promptTokens = len(prompt.encode('utf-8')) / 4 if prompt else 0 + + # 2. System message wrapper ("Context from documents:\n") + systemMessageTokens = 10 # ~40 bytes = 10 tokens + + # 3. Max output tokens (model will reserve space for completion) + outputTokens = modelMaxOutputTokens + + # 4. JSON structure and message overhead (~100 tokens) + messageOverheadTokens = 100 + + # Total reserved tokens = input overhead + output reservation + totalReservedTokens = promptTokens + systemMessageTokens + messageOverheadTokens + outputTokens + + # Available tokens for content = context length - reserved tokens + # Use 80% of available for safety margin + availableContentTokens = int((modelContextTokens - totalReservedTokens) * 0.8) + + # Ensure we have at least some space + if availableContentTokens < 100: + logger.warning(f"Very limited space for content: {availableContentTokens} tokens available. Model: {model.name}, contextLength: {modelContextTokens}, maxTokens: {modelMaxOutputTokens}, prompt: {promptTokens:.0f} tokens") + availableContentTokens = max(100, int(modelContextTokens * 0.1)) # Fallback to 10% of context + + # Convert tokens to bytes (1 token ≈ 4 bytes) + availableContentBytes = availableContentTokens * 4 + + logger.debug(f"Chunking calculation for {model.name}: contextLength={modelContextTokens} tokens, maxTokens={modelMaxOutputTokens} tokens, prompt={promptTokens:.0f} tokens, reserved={totalReservedTokens:.0f} tokens, available={availableContentTokens} tokens ({availableContentBytes} bytes)") + + # Use 70% of available content bytes for text chunks (conservative) + textChunkSize = int(availableContentBytes * 0.7) + imageChunkSize = int(availableContentBytes * 0.8) # 80% for image chunks + + # Build chunking options + chunkingOptions = { + "textChunkSize": textChunkSize, + "imageChunkSize": imageChunkSize, + "maxSize": availableContentBytes, + "chunkAllowed": True + } + + # Get appropriate chunker (uses existing ChunkerRegistry ✅) + chunker = self._chunkerRegistry.resolve(contentPart.typeGroup) + + if not chunker: + logger.warning(f"No chunker found for typeGroup: {contentPart.typeGroup}") + return [] + + # Chunk the content part + try: + chunks = chunker.chunk(contentPart, chunkingOptions) + logger.debug(f"Created {len(chunks)} chunks for {contentPart.typeGroup} part") + return chunks + except Exception as e: + logger.error(f"Chunking failed for {contentPart.typeGroup}: {str(e)}") + return [] + + async def processContentPartWithFallback(self, contentPart, prompt: str, options, failoverModelList, aiObjects, progressCallback=None) -> AiCallResponse: + """Process a single content part with model-aware chunking and fallback. + + Moved from interfaceAiObjects.py - orchestrates chunking and merging. + Calls aiObjects._callWithModel() for actual AI calls. + """ + lastError = None + + # Check if this is an image - Vision models need special handling + isImage = (contentPart.typeGroup == "image") or (contentPart.mimeType and contentPart.mimeType.startswith("image/")) + + # Determine the correct operation type based on content type + actualOperationType = options.operationType + if isImage: + actualOperationType = OperationTypeEnum.IMAGE_ANALYSE + # Get vision-capable models for images + availableModels = modelRegistry.getAvailableModels() + visionFailoverList = modelSelector.getFailoverModelList(prompt, "", AiCallOptions(operationType=actualOperationType), availableModels) + if visionFailoverList: + logger.debug(f"Using {len(visionFailoverList)} vision-capable models for image processing") + failoverModelList = visionFailoverList + + for attempt, model in enumerate(failoverModelList): + try: + logger.info(f"Processing content part with model: {model.name} (attempt {attempt + 1}/{len(failoverModelList)})") + + # Special handling for images with Vision models + if isImage and hasattr(model, 'functionCall'): + try: + if not contentPart.data: + raise ValueError("Image content part has no data") + + mimeType = contentPart.mimeType or "image/jpeg" + if not mimeType.startswith("image/"): + raise ValueError(f"Invalid mimeType for image: {mimeType}") + + # Prepare base64 data + if isinstance(contentPart.data, str): + try: + base64.b64decode(contentPart.data, validate=True) + base64Data = contentPart.data + except Exception as e: + raise ValueError(f"Invalid base64 data in contentPart: {str(e)}") + elif isinstance(contentPart.data, bytes): + base64Data = base64.b64encode(contentPart.data).decode('utf-8') + else: + raise ValueError(f"Unsupported data type for image: {type(contentPart.data)}") + + imageDataUrl = f"data:{mimeType};base64,{base64Data}" + + modelCall = AiModelCall( + messages=[ + { + "role": "user", + "content": [ + {"type": "text", "text": prompt or ""}, + { + "type": "image_url", + "image_url": {"url": imageDataUrl} + } + ] + } + ], + model=model, + options=AiCallOptions(operationType=actualOperationType) + ) + + modelResponse = await model.functionCall(modelCall) + + if not modelResponse.success: + raise ValueError(f"Model call failed: {modelResponse.error}") + + logger.info(f"✅ Image content part processed successfully with model: {model.name}") + + processingTime = getattr(modelResponse, 'processingTime', None) or 0.0 + + return AiCallResponse( + content=modelResponse.content, + modelName=model.name, + priceUsd=0.0, + processingTime=processingTime, + bytesSent=0, + bytesReceived=0, + errorCount=0 + ) + except Exception as e: + lastError = e + logger.warning(f"❌ Image processing failed with model {model.name}: {str(e)}") + + if attempt < len(failoverModelList) - 1: + logger.info(f"🔄 Trying next fallback model for image processing...") + continue + else: + logger.error(f"💥 All {len(failoverModelList)} models failed for image processing") + raise + + # For non-image parts, check if part fits in model context + partSize = len(contentPart.data.encode('utf-8')) if contentPart.data else 0 + + modelContextTokens = model.contextLength + modelMaxOutputTokens = model.maxTokens + + promptTokens = len(prompt.encode('utf-8')) / 4 if prompt else 0 + systemMessageTokens = 10 + outputTokens = modelMaxOutputTokens + messageOverheadTokens = 100 + totalReservedTokens = promptTokens + systemMessageTokens + messageOverheadTokens + outputTokens + + availableContentTokens = int((modelContextTokens - totalReservedTokens) * 0.8) + if availableContentTokens < 100: + availableContentTokens = max(100, int(modelContextTokens * 0.1)) + + availableContentBytes = availableContentTokens * 4 + + logger.debug(f"Size check for {model.name}: partSize={partSize} bytes, availableContentBytes={availableContentBytes} bytes") + + if partSize <= availableContentBytes: + # Part fits - call AI directly via aiObjects interface + response = await aiObjects._callWithModel(model, prompt, contentPart.data, options) + logger.info(f"✅ Content part processed successfully with model: {model.name}") + return response + else: + # Part too large - chunk it + chunks = await self.chunkContentPartForAi(contentPart, model, options, prompt) + if not chunks: + raise ValueError(f"Failed to chunk content part for model {model.name}") + + logger.info(f"Starting to process {len(chunks)} chunks with model {model.name}") + + if progressCallback: + progressCallback(0.0, f"Starting to process {len(chunks)} chunks") + + chunkResults = [] + for idx, chunk in enumerate(chunks): + chunkNum = idx + 1 + chunkData = chunk.get('data', '') + logger.info(f"Processing chunk {chunkNum}/{len(chunks)} with model {model.name}") + + if progressCallback: + progressCallback(chunkNum / len(chunks), f"Processing chunk {chunkNum}/{len(chunks)}") + + try: + chunkResponse = await aiObjects._callWithModel(model, prompt, chunkData, options) + chunkResults.append(chunkResponse) + logger.info(f"✅ Chunk {chunkNum}/{len(chunks)} processed successfully") + + if progressCallback: + progressCallback(chunkNum / len(chunks), f"Chunk {chunkNum}/{len(chunks)} processed") + except Exception as e: + logger.error(f"❌ Error processing chunk {chunkNum}/{len(chunks)}: {str(e)}") + raise + + # Merge chunk results + mergedContent = self.mergeChunkResults(chunkResults) + + logger.info(f"✅ Content part chunked and processed with model: {model.name} ({len(chunks)} chunks)") + return AiCallResponse( + content=mergedContent, + modelName=model.name, + priceUsd=sum(r.priceUsd for r in chunkResults), + processingTime=sum(r.processingTime for r in chunkResults), + bytesSent=sum(r.bytesSent for r in chunkResults), + bytesReceived=sum(r.bytesReceived for r in chunkResults), + errorCount=sum(r.errorCount for r in chunkResults) + ) + + except Exception as e: + lastError = e + error_msg = str(e) if str(e) else f"{type(e).__name__}" + logger.warning(f"❌ Model {model.name} failed for content part: {error_msg}", exc_info=True) + + if attempt < len(failoverModelList) - 1: + logger.info(f"🔄 Trying next failover model...") + continue + else: + logger.error(f"💥 All {len(failoverModelList)} models failed for content part") + break + + # All models failed + return self._createErrorResponse(f"All models failed: {str(lastError)}", 0, 0) + + def _createErrorResponse(self, errorMsg: str, inputBytes: int, outputBytes: int) -> AiCallResponse: + """Create an error response.""" + return AiCallResponse( + content=errorMsg, + modelName="error", + priceUsd=0.0, + processingTime=0.0, + bytesSent=inputBytes, + bytesReceived=outputBytes, + errorCount=1 + ) + + async def processContentPartsWithAi( + self, + request: AiCallRequest, + aiObjects, # Pass interface for AI calls + progressCallback=None + ) -> AiCallResponse: + """Process content parts with model-aware chunking and AI calls. + + Moved from interfaceAiObjects.callWithContentParts() - entry point for content parts processing. + """ + prompt = request.prompt + options = request.options + contentParts = request.contentParts + + # Get failover models + availableModels = modelRegistry.getAvailableModels() + failoverModelList = modelSelector.getFailoverModelList(prompt, "", options, availableModels) + + if not failoverModelList: + return self._createErrorResponse("No suitable models found", 0, 0) + + # Process each content part + allResults = [] + for contentPart in contentParts: + partResult = await self.processContentPartWithFallback( + contentPart, prompt, options, failoverModelList, aiObjects, progressCallback + ) + allResults.append(partResult) + + # Merge all results using unified mergePartResults + mergedContent = self.mergePartResults(allResults) + + return AiCallResponse( + content=mergedContent, + modelName="multiple", + priceUsd=sum(r.priceUsd for r in allResults), + processingTime=sum(r.processingTime for r in allResults), + bytesSent=sum(r.bytesSent for r in allResults), + bytesReceived=sum(r.bytesReceived for r in allResults), + errorCount=sum(r.errorCount for r in allResults) + ) + + +# Module-level function for use by subPipeline and ExtractionService +def applyMerging(parts: List[ContentPart], strategy: MergeStrategy) -> List[ContentPart]: + """Apply merging strategy to parts with intelligent token-aware merging. + + Moved from interfaceAiObjects.py to resolve dependency violations. + Can be used as module-level function or called from ExtractionService methods. + """ + logger.debug(f"applyMerging called with {len(parts)} parts") + + # Import merging dependencies (now local imports ✅) + from .merging.mergerText import TextMerger + from .merging.mergerTable import TableMerger + from .merging.mergerDefault import DefaultMerger + from .subMerger import IntelligentTokenAwareMerger + + # Check if intelligent merging is enabled + if strategy.useIntelligentMerging: + modelCapabilities = strategy.capabilities or {} + subMerger = IntelligentTokenAwareMerger(modelCapabilities) + + # Use intelligent merging for all parts + merged = subMerger.mergeChunksIntelligently(parts, strategy.prompt or "") + + # Calculate and log optimization stats + stats = subMerger.calculateOptimizationStats(parts, merged) + logger.info(f"🧠 Intelligent merging stats: {stats}") + logger.debug(f"Intelligent merging: {stats['original_ai_calls']} → {stats['optimized_ai_calls']} calls ({stats['reduction_percent']}% reduction)") + + return merged + + # Fallback to traditional merging + textMerger = TextMerger() + tableMerger = TableMerger() + defaultMerger = DefaultMerger() + + # Group by typeGroup + textParts = [p for p in parts if p.typeGroup == "text"] + tableParts = [p for p in parts if p.typeGroup == "table"] + structureParts = [p for p in parts if p.typeGroup == "structure"] + otherParts = [p for p in parts if p.typeGroup not in ("text", "table", "structure")] + + logger.debug(f"Grouped - text: {len(textParts)}, table: {len(tableParts)}, structure: {len(structureParts)}, other: {len(otherParts)}") + + merged: List[ContentPart] = [] + + if textParts: + textMerged = textMerger.merge(textParts, strategy) + logger.debug(f"TextMerger merged {len(textParts)} parts into {len(textMerged)} parts") + merged.extend(textMerged) + if tableParts: + tableMerged = tableMerger.merge(tableParts, strategy) + logger.debug(f"TableMerger merged {len(tableParts)} parts into {len(tableMerged)} parts") + merged.extend(tableMerged) + if structureParts: + # For now, treat structure like text + structureMerged = textMerger.merge(structureParts, strategy) + logger.debug(f"StructureMerger merged {len(structureParts)} parts into {len(structureMerged)} parts") + merged.extend(structureMerged) + if otherParts: + otherMerged = defaultMerger.merge(otherParts, strategy) + logger.debug(f"DefaultMerger merged {len(otherParts)} parts into {len(otherMerged)} parts") + merged.extend(otherMerged) + + logger.debug(f"applyMerging returning {len(merged)} parts") + return merged + diff --git a/modules/services/serviceExtraction/subPipeline.py b/modules/services/serviceExtraction/subPipeline.py index 510bcca8..a1ad6b04 100644 --- a/modules/services/serviceExtraction/subPipeline.py +++ b/modules/services/serviceExtraction/subPipeline.py @@ -34,7 +34,8 @@ def runExtraction(extractorRegistry: ExtractorRegistry, chunkerRegistry: Chunker # Apply merging strategy if provided (preserve existing logic) if options.mergeStrategy: - from modules.interfaces.interfaceAiObjects import applyMerging + # Use module-level applyMerging function + from .mainServiceExtraction import applyMerging parts = applyMerging(parts, options.mergeStrategy) return ContentExtracted(id=makeId(), parts=parts) diff --git a/modules/services/serviceSecurity/mainServiceSecurity.py b/modules/services/serviceSecurity/mainServiceSecurity.py new file mode 100644 index 00000000..76bae54e --- /dev/null +++ b/modules/services/serviceSecurity/mainServiceSecurity.py @@ -0,0 +1,128 @@ +""" +Security service for token management operations. +Provides centralized access to token refresh and management functionality. +""" + +import logging +from typing import Optional, Callable + +from modules.datamodels.datamodelSecurity import Token +from modules.security.tokenManager import TokenManager + +logger = logging.getLogger(__name__) + + +class SecurityService: + """Security service providing token management operations.""" + + def __init__(self, services): + """Initialize security service with service center access. + + Args: + services: Service center instance providing access to interfaces + """ + self.services = services + self._tokenManager = TokenManager() + + def getFreshToken(self, connectionId: str, secondsBeforeExpiry: int = 30 * 60) -> Optional[Token]: + """Get a fresh token for a connection, refreshing when expiring soon. + + Reads the latest stored token via interface layer, then + uses ensureFreshToken to refresh if needed and persists the refreshed + token via interface layer. + + Args: + connectionId: ID of the connection to get token for + secondsBeforeExpiry: Threshold window to proactively refresh (default: 30 minutes) + + Returns: + Token object or None if not found/expired + """ + try: + # Use interface from services instead of getRootInterface() + interfaceDbApp = self.services.interfaceDbApp + + token = interfaceDbApp.getConnectionToken(connectionId) + if not token: + return None + + return self._tokenManager.ensureFreshToken( + token, + secondsBeforeExpiry=secondsBeforeExpiry, + saveCallback=lambda t: interfaceDbApp.saveConnectionToken(t) + ) + except Exception as e: + logger.error(f"getFreshToken: Error fetching or refreshing token for connection {connectionId}: {e}") + return None + + def refreshToken(self, oldToken: Token) -> Optional[Token]: + """Refresh an expired token using the appropriate OAuth service. + + Args: + oldToken: Token object to refresh + + Returns: + Refreshed Token object or None if refresh failed + """ + try: + return self._tokenManager.refreshToken(oldToken) + except Exception as e: + logger.error(f"refreshToken: Error refreshing token: {e}") + return None + + def ensureFreshToken(self, token: Token, *, secondsBeforeExpiry: int = 30 * 60, + saveCallback: Optional[Callable[[Token], None]] = None) -> Optional[Token]: + """Ensure a token is fresh; refresh if expiring within threshold. + + Args: + token: Existing token to validate/refresh + secondsBeforeExpiry: Threshold window to proactively refresh (default: 30 minutes) + saveCallback: Optional function to persist a refreshed token + + Returns: + A fresh token (refreshed or original) or None if refresh failed + """ + try: + return self._tokenManager.ensureFreshToken( + token, + secondsBeforeExpiry=secondsBeforeExpiry, + saveCallback=saveCallback + ) + except Exception as e: + logger.error(f"ensureFreshToken: Error ensuring fresh token: {e}") + return None + + def refreshMicrosoftToken(self, refreshToken: str, userId: str, oldToken: Token) -> Optional[Token]: + """Refresh Microsoft OAuth token using refresh token. + + Args: + refreshToken: Microsoft refresh token + userId: User ID owning the token + oldToken: Previous token object to preserve connection ID + + Returns: + New Token object or None if refresh failed + """ + try: + return self._tokenManager.refreshMicrosoftToken(refreshToken, userId, oldToken) + except Exception as e: + logger.error(f"refreshMicrosoftToken: Error refreshing Microsoft token: {e}") + return None + + def refreshGoogleToken(self, refreshToken: str, userId: str, oldToken: Token) -> Optional[Token]: + """Refresh Google OAuth token using refresh token. + + Args: + refreshToken: Google refresh token + userId: User ID owning the token + oldToken: Previous token object to preserve connection ID + + Returns: + New Token object or None if refresh failed + """ + try: + return self._tokenManager.refreshGoogleToken(refreshToken, userId, oldToken) + except Exception as e: + logger.error(f"refreshGoogleToken: Error refreshing Google token: {e}") + return None + diff --git a/modules/services/serviceSharepoint/mainServiceSharepoint.py b/modules/services/serviceSharepoint/mainServiceSharepoint.py index 70fc52ff..e7f24648 100644 --- a/modules/services/serviceSharepoint/mainServiceSharepoint.py +++ b/modules/services/serviceSharepoint/mainServiceSharepoint.py @@ -47,9 +47,12 @@ class SharepointService: logger.error("UserConnection must have an 'id' field") return False - # Get a fresh token for this specific connection - from modules.security.tokenManager import TokenManager - token = TokenManager().getFreshToken(connectionId) + # Get a fresh token for this specific connection via security service + if not self.services: + logger.error("Service center not available for token access") + return False + + token = self.services.security.getFreshToken(connectionId) if not token: logger.error(f"No token found for connection {connectionId}") return False diff --git a/modules/services/serviceUtils/mainServiceUtils.py b/modules/services/serviceUtils/mainServiceUtils.py index bbee6540..8e106233 100644 --- a/modules/services/serviceUtils/mainServiceUtils.py +++ b/modules/services/serviceUtils/mainServiceUtils.py @@ -155,11 +155,11 @@ class UtilsService: def storeDebugMessageAndDocuments(self, message, currentUser): """ - Wrapper to store debug messages and documents via shared debugLogger. - Mirrors storeDebugMessageAndDocuments() in modules.shared.debugLogger. + Wrapper to store debug messages and documents via interfaceDbChatObjects. + Mirrors storeDebugMessageAndDocuments() in modules.interfaces.interfaceDbChatObjects. """ try: - from modules.shared.debugLogger import storeDebugMessageAndDocuments as _storeDebugMessageAndDocuments + from modules.interfaces.interfaceDbChatObjects import storeDebugMessageAndDocuments as _storeDebugMessageAndDocuments _storeDebugMessageAndDocuments(message, currentUser) except Exception: # Silent fail to never break main flow diff --git a/modules/shared/callbackRegistry.py b/modules/shared/callbackRegistry.py new file mode 100644 index 00000000..0529ff1d --- /dev/null +++ b/modules/shared/callbackRegistry.py @@ -0,0 +1,70 @@ +""" +Callback registry for decoupled event notifications. + +Allows interfaces to notify about changes without knowing about features. +Features can register callbacks to be notified when automations change. +""" + +import logging +from typing import Callable, List, Dict, Any +import asyncio + +logger = logging.getLogger(__name__) + + +class CallbackRegistry: + """Registry for callbacks that can be triggered by interfaces without knowing about features.""" + + def __init__(self): + self._callbacks: Dict[str, List[Callable]] = {} + + def register(self, event_type: str, callback: Callable): + """Register a callback for a specific event type. + + Args: + event_type: Type of event (e.g., 'automation.changed') + callback: Async or sync callback function + """ + if event_type not in self._callbacks: + self._callbacks[event_type] = [] + self._callbacks[event_type].append(callback) + logger.debug(f"Registered callback for event type: {event_type}") + + def unregister(self, event_type: str, callback: Callable): + """Unregister a callback for a specific event type.""" + if event_type in self._callbacks: + try: + self._callbacks[event_type].remove(callback) + logger.debug(f"Unregistered callback for event type: {event_type}") + except ValueError: + logger.warning(f"Callback not found for event type: {event_type}") + + async def trigger(self, event_type: str, *args, **kwargs): + """Trigger all callbacks registered for an event type. + + Args: + event_type: Type of event to trigger + *args, **kwargs: Arguments to pass to callbacks + """ + if event_type not in self._callbacks: + return + + callbacks = self._callbacks[event_type].copy() # Copy to avoid modification during iteration + + for callback in callbacks: + try: + if asyncio.iscoroutinefunction(callback): + await callback(*args, **kwargs) + else: + callback(*args, **kwargs) + except Exception as e: + logger.error(f"Error executing callback for {event_type}: {str(e)}", exc_info=True) + + def has_callbacks(self, event_type: str) -> bool: + """Check if there are any callbacks registered for an event type.""" + return event_type in self._callbacks and len(self._callbacks[event_type]) > 0 + + +# Global singleton instance +callbackRegistry = CallbackRegistry() + diff --git a/modules/shared/debugLogger.py b/modules/shared/debugLogger.py index 6ee78bc7..2af3f329 100644 --- a/modules/shared/debugLogger.py +++ b/modules/shared/debugLogger.py @@ -145,131 +145,3 @@ def debugLogToFile(message: str, context: str = "DEBUG") -> None: # Don't log debug errors to avoid recursion pass -def storeDebugMessageAndDocuments(message, currentUser) -> None: - """ - Store message and documents (metadata and file bytes) for debugging purposes. - Structure: {log_dir}/debug/messages/m_round_task_action_timestamp/documentlist_label/ - - message.json, message_text.txt - - document_###_metadata.json - - document_###_ (actual file bytes) - - Args: - message: ChatMessage object to store - currentUser: Current user for component interface access - """ - try: - import json - - # Create base debug directory (use base debug dir, not prompts subdirectory) - baseDebugDir = _getBaseDebugDir() - debug_root = os.path.join(baseDebugDir, 'messages') - _ensureDir(debug_root) - - # Generate timestamp - timestamp = datetime.now(UTC).strftime('%Y%m%d-%H%M%S-%f')[:-3] - - # Create message folder name: m_round_task_action_timestamp - # Use actual values from message, not defaults - round_str = str(message.roundNumber) if message.roundNumber is not None else "0" - task_str = str(message.taskNumber) if message.taskNumber is not None else "0" - action_str = str(message.actionNumber) if message.actionNumber is not None else "0" - message_folder = f"{timestamp}_m_{round_str}_{task_str}_{action_str}" - - message_path = os.path.join(debug_root, message_folder) - os.makedirs(message_path, exist_ok=True) - - # Store message data - use dict() instead of model_dump() for compatibility - message_file = os.path.join(message_path, "message.json") - with open(message_file, "w", encoding="utf-8") as f: - # Convert message to dict manually to avoid model_dump() issues - message_dict = { - "id": message.id, - "workflowId": message.workflowId, - "parentMessageId": message.parentMessageId, - "message": message.message, - "role": message.role, - "status": message.status, - "sequenceNr": message.sequenceNr, - "publishedAt": message.publishedAt, - "roundNumber": message.roundNumber, - "taskNumber": message.taskNumber, - "actionNumber": message.actionNumber, - "documentsLabel": message.documentsLabel, - "actionId": message.actionId, - "actionMethod": message.actionMethod, - "actionName": message.actionName, - "success": message.success, - "documents": [] - } - json.dump(message_dict, f, indent=2, ensure_ascii=False, default=str) - - # Store message content as text - if message.message: - message_text_file = os.path.join(message_path, "message_text.txt") - with open(message_text_file, "w", encoding="utf-8") as f: - f.write(str(message.message)) - - # Store documents if provided - if message.documents and len(message.documents) > 0: - # Group documents by documentsLabel - documents_by_label = {} - for doc in message.documents: - label = message.documentsLabel or 'default' - if label not in documents_by_label: - documents_by_label[label] = [] - documents_by_label[label].append(doc) - - # Create subfolder for each document label - for label, docs in documents_by_label.items(): - # Sanitize label for filesystem - safe_label = "".join(c for c in str(label) if c.isalnum() or c in (' ', '-', '_')).rstrip() - safe_label = safe_label.replace(' ', '_') - if not safe_label: - safe_label = "default" - - label_folder = os.path.join(message_path, safe_label) - _ensureDir(label_folder) - - # Store each document - for i, doc in enumerate(docs): - # Create document metadata file - doc_meta = { - "id": doc.id, - "messageId": doc.messageId, - "fileId": doc.fileId, - "fileName": doc.fileName, - "fileSize": doc.fileSize, - "mimeType": doc.mimeType, - "roundNumber": doc.roundNumber, - "taskNumber": doc.taskNumber, - "actionNumber": doc.actionNumber, - "actionId": doc.actionId - } - - doc_meta_file = os.path.join(label_folder, f"document_{i+1:03d}_metadata.json") - with open(doc_meta_file, "w", encoding="utf-8") as f: - json.dump(doc_meta, f, indent=2, ensure_ascii=False, default=str) - - # Also store the actual file bytes next to metadata for debugging - try: - # Lazy import to avoid circular deps at module load - from modules.interfaces import interfaceDbComponentObjects as comp - componentInterface = comp.getInterface(currentUser) - file_bytes = componentInterface.getFileData(doc.fileId) - if file_bytes: - # Build a safe filename preserving original name - safe_name = doc.fileName or f"document_{i+1:03d}" - # Avoid path traversal - safe_name = os.path.basename(safe_name) - doc_file_path = os.path.join(label_folder, f"document_{i+1:03d}_" + safe_name) - with open(doc_file_path, "wb") as df: - df.write(file_bytes) - else: - pass - except Exception as e: - pass - - except Exception as e: - # Silent fail - don't break main flow - pass - diff --git a/modules/workflows/methods/methodAi.py b/modules/workflows/methods/methodAi.py index 1e837f62..c60469c8 100644 --- a/modules/workflows/methods/methodAi.py +++ b/modules/workflows/methods/methodAi.py @@ -214,138 +214,6 @@ class MethodAi(MethodBase): ) - @action - async def extractContent(self, parameters: Dict[str, Any]) -> ActionResult: - """ - Extract content from documents (separate from AI calls). - - This action performs pure content extraction without AI processing. - The extracted ContentParts can then be used by subsequent AI processing actions. - - Parameters: - - documentList (list, required): Document reference(s) to extract content from. - - extractionOptions (dict, optional): Extraction options (if not provided, defaults are used). - - Returns: - - ActionResult with ActionDocument containing ContentExtracted objects - - ContentExtracted.parts contains List[ContentPart] (already chunked if needed) - """ - try: - # Init progress logger - workflowId = self.services.workflow.id if self.services.workflow else f"no-workflow-{int(time.time())}" - operationId = f"ai_extract_{workflowId}_{int(time.time())}" - - # Extract documentList from parameters dict - from modules.datamodels.datamodelDocref import DocumentReferenceList - documentListParam = parameters.get("documentList") - if not documentListParam: - return ActionResult.isFailure(error="documentList is required") - - # Convert to DocumentReferenceList if needed - if isinstance(documentListParam, DocumentReferenceList): - documentList = documentListParam - elif isinstance(documentListParam, str): - documentList = DocumentReferenceList.from_string_list([documentListParam]) - elif isinstance(documentListParam, list): - documentList = DocumentReferenceList.from_string_list(documentListParam) - else: - return ActionResult.isFailure(error=f"Invalid documentList type: {type(documentListParam)}") - - # Start progress tracking - self.services.chat.progressLogStart( - operationId, - "Extracting content from documents", - "Content Extraction", - f"Documents: {len(documentList.references)}" - ) - - # Get ChatDocuments from documentList - self.services.chat.progressLogUpdate(operationId, 0.2, "Loading documents") - chatDocuments = self.services.chat.getChatDocumentsFromDocumentList(documentList) - - if not chatDocuments: - self.services.chat.progressLogFinish(operationId, False) - return ActionResult.isFailure(error="No documents found in documentList") - - logger.info(f"Extracting content from {len(chatDocuments)} documents") - - # Prepare extraction options - self.services.chat.progressLogUpdate(operationId, 0.3, "Preparing extraction options") - extractionOptionsParam = parameters.get("extractionOptions") - - # Convert dict to ExtractionOptions object if needed, or create defaults - if extractionOptionsParam: - if isinstance(extractionOptionsParam, dict): - # Convert dict to ExtractionOptions object - extractionOptions = ExtractionOptions(**extractionOptionsParam) - elif isinstance(extractionOptionsParam, ExtractionOptions): - extractionOptions = extractionOptionsParam - else: - # Invalid type, use defaults - extractionOptions = None - else: - extractionOptions = None - - # If extractionOptions not provided, create defaults - if not extractionOptions: - # Default extraction options for pure content extraction (no AI processing) - extractionOptions = ExtractionOptions( - prompt="Extract all content from the document", - mergeStrategy=MergeStrategy( - mergeType="concatenate", - groupBy="typeGroup", - orderBy="id" - ), - processDocumentsIndividually=True - ) - - # Get parent log ID for document-level operations - parentLogId = self.services.chat.getOperationLogId(operationId) - - # Call extraction service - self.services.chat.progressLogUpdate(operationId, 0.4, "Initiating") - self.services.chat.progressLogUpdate(operationId, 0.5, f"Extracting content from {len(chatDocuments)} documents") - extractedResults = self.services.extraction.extractContent(chatDocuments, extractionOptions) - - # Build ActionDocuments from ContentExtracted results - self.services.chat.progressLogUpdate(operationId, 0.8, "Building result documents") - actionDocuments = [] - # Map extracted results back to original documents by index (results are in same order) - for i, extracted in enumerate(extractedResults): - # Get original document name if available - originalDoc = chatDocuments[i] if i < len(chatDocuments) else None - if originalDoc and hasattr(originalDoc, 'fileName') and originalDoc.fileName: - # Use original filename with "extracted_" prefix - baseName = originalDoc.fileName.rsplit('.', 1)[0] if '.' in originalDoc.fileName else originalDoc.fileName - documentName = f"{baseName}_extracted_{extracted.id}.json" - else: - # Fallback to generic name with index - documentName = f"document_{i+1:03d}_extracted_{extracted.id}.json" - - # Store ContentExtracted object in ActionDocument.documentData - actionDoc = ActionDocument( - documentName=documentName, - documentData=extracted, # ContentExtracted object - mimeType="application/json" - ) - actionDocuments.append(actionDoc) - - self.services.chat.progressLogFinish(operationId, True) - - return ActionResult.isSuccess(documents=actionDocuments) - - except Exception as e: - logger.error(f"Error in content extraction: {str(e)}") - - # Complete progress tracking with failure - try: - self.services.chat.progressLogFinish(operationId, False) - except: - pass # Don't fail on progress logging errors - - return ActionResult.isFailure(error=str(e)) - - @action async def webResearch(self, parameters: Dict[str, Any]) -> ActionResult: """ @@ -707,171 +575,6 @@ class MethodAi(MethodBase): return output.getvalue() - @action - async def reformat(self, parameters: Dict[str, Any]) -> ActionResult: - """ - GENERAL: - - Purpose: Reformat/transform documents with specific transformation rules (e.g., extract arrays, reshape data, apply custom formatting). - - Input requirements: documentList (required); inputFormat and outputFormat (required); transformationRules (optional). - - Output format: Document in target format with applied transformation rules. - - CRITICAL: If input is already in standardized JSON format, uses automatic rendering system with transformation rules. - - Parameters: - - documentList (list, required): Document reference(s) to reformat. - - inputFormat (str, required): Source format (json, csv, xlsx, txt, etc.). - - outputFormat (str, required): Target format (csv, json, xlsx, txt, etc.). - - transformationRules (str, optional): Specific transformation instructions (e.g., "Extract prime numbers array and format as CSV with 10 columns per row"). - - columnsPerRow (int, optional): For CSV output, number of columns per row. Default: auto-detect. - - totalRows (int, optional): For CSV output, total number of rows to create. Default: auto-detect. - - delimiter (str, optional): For CSV output, delimiter character. Default: comma (,). - - includeHeader (bool, optional): For CSV output, whether to include header row. Default: True. - - language (str, optional): Language for output (e.g., 'de', 'en', 'fr'). Default: 'en'. - """ - documentList = parameters.get("documentList", []) - if not documentList: - return ActionResult.isFailure(error="documentList is required") - - inputFormat = parameters.get("inputFormat") - outputFormat = parameters.get("outputFormat") - if not inputFormat or not outputFormat: - return ActionResult.isFailure(error="inputFormat and outputFormat are required") - - transformationRules = parameters.get("transformationRules") - columnsPerRow = parameters.get("columnsPerRow") - totalRows = parameters.get("totalRows") - delimiter = parameters.get("delimiter", ",") - includeHeader = parameters.get("includeHeader", True) - language = parameters.get("language", "en") - - # Normalize formats (remove leading dot if present) - normalizedInputFormat = inputFormat.strip().lstrip('.').lower() - normalizedOutputFormat = outputFormat.strip().lstrip('.').lower() - - # Get documents - from modules.datamodels.datamodelDocref import DocumentReferenceList - if isinstance(documentList, DocumentReferenceList): - docRefList = documentList - elif isinstance(documentList, list): - docRefList = DocumentReferenceList.from_string_list(documentList) - else: - docRefList = DocumentReferenceList.from_string_list([documentList]) - - chatDocuments = self.services.chat.getChatDocumentsFromDocumentList(docRefList) - if not chatDocuments: - return ActionResult.isFailure(error="No documents found in documentList") - - # Check if input is standardized JSON format - if so, use direct rendering with transformation - if normalizedInputFormat == "json" and len(chatDocuments) == 1: - try: - import json - doc = chatDocuments[0] - # ChatDocument doesn't have documentData - need to load file content using fileId - docBytes = self.services.chat.getFileData(doc.fileId) - if not docBytes: - raise ValueError(f"No file data found for fileId={doc.fileId}") - - # Decode bytes to string - docData = docBytes.decode('utf-8') - - # Try to parse as JSON - if isinstance(docData, str): - jsonData = json.loads(docData) - elif isinstance(docData, dict): - jsonData = docData - else: - jsonData = None - - # Check if it's standardized JSON format (has "documents" or "sections") - if jsonData and (isinstance(jsonData, dict) and ("documents" in jsonData or "sections" in jsonData)): - # Apply transformation rules if provided - if transformationRules: - # Use AI to apply transformation rules to JSON - aiPrompt = f"Apply the following transformation rules to the JSON document: {transformationRules}" - if normalizedOutputFormat == "csv": - aiPrompt += f" Output format: CSV with delimiter '{delimiter}'" - if columnsPerRow: - aiPrompt += f", {columnsPerRow} columns per row" - if totalRows: - aiPrompt += f", {totalRows} total rows" - if not includeHeader: - aiPrompt += ", no header row" - - # Use process to apply transformation - return await self.process({ - "aiPrompt": aiPrompt, - "documentList": documentList, - "resultType": normalizedOutputFormat - }) - else: - # No transformation rules - use direct rendering - from modules.services.serviceGeneration.mainServiceGeneration import GenerationService - generationService = GenerationService(self.services) - - # Ensure format is "documents" array - if "documents" not in jsonData: - jsonData = {"documents": [{"sections": jsonData.get("sections", []), "metadata": jsonData.get("metadata", {})}]} - - # Get title - title = jsonData.get("metadata", {}).get("title", doc.documentName or "Reformatted Document") - - # Render with options - renderOptions = {} - if normalizedOutputFormat == "csv": - renderOptions["delimiter"] = delimiter - renderOptions["columnsPerRow"] = columnsPerRow - renderOptions["includeHeader"] = includeHeader - - rendered_content, mime_type = await generationService.renderReport( - jsonData, normalizedOutputFormat, title, None, None - ) - - # Apply CSV options if needed - if normalizedOutputFormat == "csv" and renderOptions: - rendered_content = self._applyCsvOptions(rendered_content, renderOptions) - - from modules.datamodels.datamodelChat import ActionDocument - actionDoc = ActionDocument( - documentName=f"{doc.documentName.rsplit('.', 1)[0] if '.' in doc.documentName else doc.documentName}.{normalizedOutputFormat}", - documentData=rendered_content, - mimeType=mime_type, - sourceJson=jsonData # Preserve source JSON for structure validation - ) - - return ActionResult.isSuccess(documents=[actionDoc]) - - except Exception as e: - logger.warning(f"Direct rendering failed, falling back to AI reformatting: {str(e)}") - # Fall through to AI-based reformatting - - # Fallback: Use AI for reformatting with transformation rules - aiPrompt = f"Reformat the provided document(s) from {normalizedInputFormat.upper()} format to {normalizedOutputFormat.upper()} format." - - if transformationRules: - aiPrompt += f" Apply the following transformation rules: {transformationRules}" - - if normalizedOutputFormat == "csv": - aiPrompt += f" Use '{delimiter}' as the delimiter character." - if columnsPerRow: - aiPrompt += f" Format the output with {columnsPerRow} columns per row." - if totalRows: - aiPrompt += f" Create exactly {totalRows} rows total." - if not includeHeader: - aiPrompt += " Do not include a header row." - else: - aiPrompt += " Include a header row with column names." - - if language and language != "en": - aiPrompt += f" Use language: {language}." - - aiPrompt += " Preserve all data and ensure accurate transformation. Maintain data integrity." - - return await self.process({ - "aiPrompt": aiPrompt, - "documentList": documentList, - "resultType": normalizedOutputFormat - }) - - @action async def convertDocument(self, parameters: Dict[str, Any]) -> ActionResult: """ @@ -955,160 +658,10 @@ class MethodAi(MethodBase): }) - @action - async def extractTables(self, parameters: Dict[str, Any]) -> ActionResult: - """ - GENERAL: - - Purpose: Extract tables from documents, preserving structure and data. - - Input requirements: documentList (required); optional tableFormat. - - Output format: JSON by default (structured table data), or CSV/XLSX if specified. - - Parameters: - - documentList (list, required): Document reference(s) to extract tables from. - - tableFormat (str, optional): Output format for tables - json, csv, or xlsx. Default: json. - - includeHeaders (bool, optional): Include table headers. Default: True. - """ - documentList = parameters.get("documentList", []) - if not documentList: - return ActionResult.isFailure(error="documentList is required") - - tableFormat = parameters.get("tableFormat", "json") - includeHeaders = parameters.get("includeHeaders", True) - - # Map tableFormat to resultType - formatMap = { - "json": "json", - "csv": "csv", - "xlsx": "xlsx", - "xls": "xlsx" - } - resultType = formatMap.get(tableFormat.lower(), "json") - - aiPrompt = "Extract all tables from the provided document(s)." - if includeHeaders: - aiPrompt += " Include table headers and preserve the table structure." - else: - aiPrompt += " Extract table data without headers." - aiPrompt += " Maintain accurate data types (numbers as numbers, dates as dates, etc.) and preserve all table relationships." - - if resultType == "json": - aiPrompt += " Structure each table as a JSON object with headers and rows as arrays." - elif resultType == "csv": - aiPrompt += " Output each table as CSV format with proper comma separation." - elif resultType == "xlsx": - aiPrompt += " Structure the output as an Excel spreadsheet with tables properly formatted." - - return await self.process({ - "aiPrompt": aiPrompt, - "documentList": documentList, - "resultType": resultType - }) - - # ============================================================================ - # Content Generation Wrappers + # Content Generation Wrapper # ============================================================================ - @action - async def generateReport(self, parameters: Dict[str, Any]) -> ActionResult: - """ - GENERAL: - - Purpose: Generate comprehensive reports from input documents/data with analysis and insights. - - Input requirements: documentList (optional, can generate from scratch); optional reportType, sections. - - Output format: Document in specified format (default: docx). - - Parameters: - - documentList (list, optional): Input documents/data to base the report on. - - reportType (str, optional): Type of report - summary, analysis, executive, detailed. Default: analysis. - - sections (list, optional): Specific sections to include (e.g., ["introduction", "findings", "recommendations"]). - - title (str, optional): Report title. - - resultType (str, optional): Output format (docx, pdf, md, etc.). Default: docx. - """ - documentList = parameters.get("documentList", []) - reportType = parameters.get("reportType", "analysis") - sections = parameters.get("sections", []) - title = parameters.get("title") - resultType = parameters.get("resultType", "docx") - - reportTypeInstructions = { - "summary": "Create a summary report with key highlights and main points.", - "analysis": "Create an analytical report with insights, findings, and detailed examination.", - "executive": "Create an executive summary report suitable for senior management with key insights and recommendations.", - "detailed": "Create a comprehensive detailed report covering all aspects with in-depth analysis." - } - - aiPrompt = f"Generate a {reportType} report." - if title: - aiPrompt += f" Title: {title}." - aiPrompt += f" {reportTypeInstructions.get(reportType.lower(), reportTypeInstructions['analysis'])}" - - if sections: - sectionsStr = ", ".join(sections) - aiPrompt += f" Include the following sections: {sectionsStr}." - else: - aiPrompt += " Include standard report sections such as introduction, main content, analysis, findings, and conclusions." - - if documentList: - aiPrompt += " Base the report on the provided input documents, analyzing and synthesizing the information." - else: - aiPrompt += " Create a professional, well-structured report." - - processParams = { - "aiPrompt": aiPrompt, - "resultType": resultType - } - if documentList: - processParams["documentList"] = documentList - - return await self.process(processParams) - - - @action - async def generateChart(self, parameters: Dict[str, Any]) -> ActionResult: - """ - GENERAL: - - Purpose: Generate charts/graphs from data in documents or structured data. - - Input requirements: documentList (required); optional chartType, title, labels. - - Output format: Image (png or jpg). - - Parameters: - - documentList (list, required): Documents containing data to visualize (CSV, Excel, JSON, etc.). - - chartType (str, optional): Type of chart - bar, line, pie, scatter, area, etc. Default: bar. - - title (str, optional): Chart title. - - xAxisLabel (str, optional): X-axis label. - - yAxisLabel (str, optional): Y-axis label. - - resultType (str, optional): Image format (png or jpg). Default: png. - """ - documentList = parameters.get("documentList", []) - if not documentList: - return ActionResult.isFailure(error="documentList is required") - - chartType = parameters.get("chartType", "bar") - title = parameters.get("title") - xAxisLabel = parameters.get("xAxisLabel") - yAxisLabel = parameters.get("yAxisLabel") - resultType = parameters.get("resultType", "png") - - # Ensure resultType is an image format - if resultType.lower() not in ["png", "jpg", "jpeg"]: - resultType = "png" - - aiPrompt = f"Generate a {chartType} chart from the provided data." - if title: - aiPrompt += f" Chart title: {title}." - if xAxisLabel: - aiPrompt += f" X-axis label: {xAxisLabel}." - if yAxisLabel: - aiPrompt += f" Y-axis label: {yAxisLabel}." - aiPrompt += " Create a clear, professional chart with appropriate labels, legends, and formatting. Ensure the chart is visually appealing and easy to read." - - return await self.process({ - "aiPrompt": aiPrompt, - "documentList": documentList, - "resultType": resultType - }) - - @action async def generateDocument(self, parameters: Dict[str, Any]) -> ActionResult: """ @@ -1146,137 +699,3 @@ class MethodAi(MethodBase): processParams["documentList"] = documentList return await self.process(processParams) - - - # ============================================================================ - # Analysis & Comparison Wrappers - # ============================================================================ - - @action - async def analyzeDocuments(self, parameters: Dict[str, Any]) -> ActionResult: - """ - GENERAL: - - Purpose: Analyze documents and find insights, patterns, trends, and key information. - - Input requirements: documentList (required); optional analysisType, focus. - - Output format: Analysis report in specified format (default: txt). - - Parameters: - - documentList (list, required): Document(s) to analyze. - - analysisType (str, optional): Type of analysis - general, financial, technical, sentiment, etc. Default: general. - - focus (str, optional): Specific aspect to focus on (e.g., "trends", "risks", "opportunities"). - - resultType (str, optional): Output format (txt, md, docx, json, etc.). Default: txt. - """ - documentList = parameters.get("documentList", []) - if not documentList: - return ActionResult.isFailure(error="documentList is required") - - analysisType = parameters.get("analysisType", "general") - focus = parameters.get("focus") - resultType = parameters.get("resultType", "txt") - - aiPrompt = f"Analyze the provided document(s) and find insights, patterns, and key information." - aiPrompt += f" Perform a {analysisType} analysis." - if focus: - aiPrompt += f" Focus specifically on: {focus}." - aiPrompt += " Identify trends, important findings, relationships, and provide actionable insights. Present the analysis in a clear, structured format." - - return await self.process({ - "aiPrompt": aiPrompt, - "documentList": documentList, - "resultType": resultType - }) - - - @action - async def compareDocuments(self, parameters: Dict[str, Any]) -> ActionResult: - """ - GENERAL: - - Purpose: Compare multiple documents and identify differences, similarities, and changes. - - Input requirements: documentList (required, should contain 2+ documents); optional comparisonType, focus. - - Output format: Comparison report in specified format (default: txt). - - Parameters: - - documentList (list, required): Two or more documents to compare. - - comparisonType (str, optional): Type of comparison - differences, similarities, changes, full. Default: full. - - focus (str, optional): Specific aspect to focus on (e.g., "content", "structure", "data", "formatting"). - - resultType (str, optional): Output format (txt, md, docx, json, etc.). Default: txt. - """ - documentList = parameters.get("documentList", []) - if not documentList: - return ActionResult.isFailure(error="documentList is required") - - if isinstance(documentList, str): - documentList = [documentList] - - if len(documentList) < 2: - return ActionResult.isFailure(error="At least 2 documents are required for comparison") - - comparisonType = parameters.get("comparisonType", "full") - focus = parameters.get("focus") - resultType = parameters.get("resultType", "txt") - - comparisonInstructions = { - "differences": "Focus on identifying and highlighting all differences between the documents.", - "similarities": "Focus on identifying commonalities, shared content, and similarities.", - "changes": "Identify what has changed between versions, what was added, removed, or modified.", - "full": "Provide a comprehensive comparison including both differences and similarities." - } - - aiPrompt = f"Compare the provided documents." - aiPrompt += f" {comparisonInstructions.get(comparisonType.lower(), comparisonInstructions['full'])}" - if focus: - aiPrompt += f" Focus specifically on: {focus}." - aiPrompt += " Present the comparison in a clear, structured format that makes differences and similarities easy to understand." - - return await self.process({ - "aiPrompt": aiPrompt, - "documentList": documentList, - "resultType": resultType - }) - - - @action - async def validateData(self, parameters: Dict[str, Any]) -> ActionResult: - """ - GENERAL: - - Purpose: Validate data quality, structure, completeness, and correctness in documents/data files. - - Input requirements: documentList (required); optional validationRules, schema. - - Output format: Validation report in JSON or text format (default: json). - - Parameters: - - documentList (list, required): Documents/data files to validate. - - validationRules (list, optional): Specific validation rules to check (e.g., ["required_fields", "data_types", "ranges"]). - - schema (dict, optional): Expected data schema/structure to validate against. - - resultType (str, optional): Output format (json, txt, md, etc.). Default: json. - """ - documentList = parameters.get("documentList", []) - if not documentList: - return ActionResult.isFailure(error="documentList is required") - - validationRules = parameters.get("validationRules", []) - schema = parameters.get("schema") - resultType = parameters.get("resultType", "json") - - aiPrompt = "Validate the data quality, structure, completeness, and correctness in the provided documents." - - if validationRules: - rulesStr = ", ".join(validationRules) - aiPrompt += f" Apply the following validation rules: {rulesStr}." - else: - aiPrompt += " Check for data completeness, correct data types, required fields, data consistency, and any anomalies or errors." - - if schema: - import json - schemaStr = json.dumps(schema, indent=2) - aiPrompt += f" Validate against the following expected schema: {schemaStr}." - - if resultType == "json": - aiPrompt += " Provide the validation results as structured JSON with validation status, errors, warnings, and details for each check." - else: - aiPrompt += " Provide a detailed validation report listing all findings, errors, warnings, and pass/fail status for each validation check." - - return await self.process({ - "aiPrompt": aiPrompt, - "documentList": documentList, - "resultType": resultType - }) diff --git a/modules/workflows/methods/methodContext.py b/modules/workflows/methods/methodContext.py new file mode 100644 index 00000000..e974606c --- /dev/null +++ b/modules/workflows/methods/methodContext.py @@ -0,0 +1,337 @@ +""" +Context and workflow information method module. +Handles workflow context queries and document indexing. +""" + +import time +import json +import logging +from typing import Dict, Any, List +from datetime import datetime, UTC + +from modules.workflows.methods.methodBase import MethodBase, action +from modules.datamodels.datamodelChat import ActionResult, ActionDocument +from modules.datamodels.datamodelExtraction import ExtractionOptions, MergeStrategy + +logger = logging.getLogger(__name__) + +class MethodContext(MethodBase): + """Context and workflow information methods.""" + + def __init__(self, services): + super().__init__(services) + self.name = "context" + self.description = "Context and workflow information methods" + + @action + async def getDocumentIndex(self, parameters: Dict[str, Any]) -> ActionResult: + """ + GENERAL: + - Purpose: Generate a comprehensive index of all documents available in the current workflow, including documents from all rounds and tasks. + - Input requirements: No input documents required. Optional resultType parameter. + - Output format: Structured document index in JSON format (default) or text format, listing all documents with their references, metadata, and organization by rounds/tasks. + + Parameters: + - resultType (str, optional): Output format (json, txt, md). Default: json. + """ + try: + workflow = self.services.workflow + if not workflow: + return ActionResult.isFailure( + error="No workflow available" + ) + + resultType = parameters.get("resultType", "json").lower().strip().lstrip('.') + + # Get available documents index from chat service + documentsIndex = self.services.chat.getAvailableDocuments(workflow) + + if not documentsIndex or documentsIndex == "No documents available" or documentsIndex == "NO DOCUMENTS AVAILABLE - This workflow has no documents to process.": + # Return empty index structure + if resultType == "json": + indexData = { + "workflowId": getattr(workflow, 'id', 'unknown'), + "totalDocuments": 0, + "rounds": [], + "documentReferences": [] + } + indexContent = json.dumps(indexData, indent=2, ensure_ascii=False) + else: + indexContent = "Document Index\n==============\n\nNo documents available in this workflow.\n" + else: + # Parse the document index string to extract structured information + indexData = self._parseDocumentIndex(documentsIndex, workflow) + + if resultType == "json": + indexContent = json.dumps(indexData, indent=2, ensure_ascii=False) + elif resultType == "md": + indexContent = self._formatAsMarkdown(indexData) + else: # txt + indexContent = self._formatAsText(indexData, documentsIndex) + + # Generate meaningful filename + workflowContext = self.services.chat.getWorkflowContext() + filename = self._generateMeaningfulFileName( + "document_index", + resultType if resultType in ["json", "txt", "md"] else "json", + workflowContext, + "getDocumentIndex" + ) + + # Create ActionDocument + document = ActionDocument( + documentName=filename, + documentData=indexContent, + mimeType="application/json" if resultType == "json" else "text/plain" + ) + + return ActionResult.isSuccess(documents=[document]) + + except Exception as e: + logger.error(f"Error generating document index: {str(e)}") + return ActionResult.isFailure( + error=f"Failed to generate document index: {str(e)}" + ) + + def _parseDocumentIndex(self, documentsIndex: str, workflow: Any) -> Dict[str, Any]: + """Parse the document index string into structured data.""" + try: + indexData = { + "workflowId": getattr(workflow, 'id', 'unknown'), + "generatedAt": datetime.now(UTC).isoformat(), + "totalDocuments": 0, + "rounds": [], + "documentReferences": [] + } + + # Extract document references from the index string + lines = documentsIndex.split('\n') + currentRound = None + currentDocList = None + + for line in lines: + line = line.strip() + if not line: + continue + + # Check for round headers + if "Current round documents:" in line: + currentRound = "current" + continue + elif "Past rounds documents:" in line: + currentRound = "past" + continue + + # Check for document list references (docList:...) + if line.startswith("- docList:"): + docListRef = line.replace("- docList:", "").strip() + currentDocList = { + "reference": docListRef, + "round": currentRound, + "documents": [] + } + indexData["rounds"].append(currentDocList) + continue + + # Check for individual document references (docItem:...) + if line.startswith(" - docItem:") or line.startswith("- docItem:"): + docItemRef = line.replace(" - docItem:", "").replace("- docItem:", "").strip() + indexData["documentReferences"].append({ + "reference": docItemRef, + "round": currentRound, + "docList": currentDocList["reference"] if currentDocList else None + }) + indexData["totalDocuments"] += 1 + if currentDocList: + currentDocList["documents"].append(docItemRef) + + return indexData + + except Exception as e: + logger.error(f"Error parsing document index: {str(e)}") + return { + "workflowId": getattr(workflow, 'id', 'unknown'), + "error": f"Failed to parse document index: {str(e)}", + "rawIndex": documentsIndex + } + + def _formatAsMarkdown(self, indexData: Dict[str, Any]) -> str: + """Format document index as Markdown.""" + try: + md = f"# Document Index\n\n" + md += f"**Workflow ID:** {indexData.get('workflowId', 'unknown')}\n\n" + md += f"**Generated At:** {indexData.get('generatedAt', 'unknown')}\n\n" + md += f"**Total Documents:** {indexData.get('totalDocuments', 0)}\n\n" + + if indexData.get('rounds'): + md += "## Documents by Round\n\n" + for roundInfo in indexData['rounds']: + roundLabel = roundInfo.get('round', 'unknown').title() + md += f"### {roundLabel} Round\n\n" + md += f"**Document List:** `{roundInfo.get('reference', 'unknown')}`\n\n" + if roundInfo.get('documents'): + md += "**Documents:**\n\n" + for docRef in roundInfo['documents']: + md += f"- `{docRef}`\n" + md += "\n" + + if indexData.get('documentReferences'): + md += "## All Document References\n\n" + for docRef in indexData['documentReferences']: + md += f"- `{docRef.get('reference', 'unknown')}`\n" + + return md + + except Exception as e: + logger.error(f"Error formatting as Markdown: {str(e)}") + return f"# Document Index\n\nError formatting index: {str(e)}\n" + + def _formatAsText(self, indexData: Dict[str, Any], rawIndex: str) -> str: + """Format document index as plain text.""" + try: + text = "Document Index\n" + text += "=" * 50 + "\n\n" + text += f"Workflow ID: {indexData.get('workflowId', 'unknown')}\n" + text += f"Generated At: {indexData.get('generatedAt', 'unknown')}\n" + text += f"Total Documents: {indexData.get('totalDocuments', 0)}\n\n" + + # Include the raw formatted index for readability + text += rawIndex + + return text + + except Exception as e: + logger.error(f"Error formatting as text: {str(e)}") + return f"Document Index\n\nError formatting index: {str(e)}\n\nRaw index:\n{rawIndex}\n" + + @action + async def extractContent(self, parameters: Dict[str, Any]) -> ActionResult: + """ + Extract content from documents (separate from AI calls). + + This action performs pure content extraction without AI processing. + The extracted ContentParts can then be used by subsequent AI processing actions. + + Parameters: + - documentList (list, required): Document reference(s) to extract content from. + - extractionOptions (dict, optional): Extraction options (if not provided, defaults are used). + + Returns: + - ActionResult with ActionDocument containing ContentExtracted objects + - ContentExtracted.parts contains List[ContentPart] (already chunked if needed) + """ + try: + # Init progress logger + workflowId = self.services.workflow.id if self.services.workflow else f"no-workflow-{int(time.time())}" + operationId = f"context_extract_{workflowId}_{int(time.time())}" + + # Extract documentList from parameters dict + from modules.datamodels.datamodelDocref import DocumentReferenceList + documentListParam = parameters.get("documentList") + if not documentListParam: + return ActionResult.isFailure(error="documentList is required") + + # Convert to DocumentReferenceList if needed + if isinstance(documentListParam, DocumentReferenceList): + documentList = documentListParam + elif isinstance(documentListParam, str): + documentList = DocumentReferenceList.from_string_list([documentListParam]) + elif isinstance(documentListParam, list): + documentList = DocumentReferenceList.from_string_list(documentListParam) + else: + return ActionResult.isFailure(error=f"Invalid documentList type: {type(documentListParam)}") + + # Start progress tracking + self.services.chat.progressLogStart( + operationId, + "Extracting content from documents", + "Content Extraction", + f"Documents: {len(documentList.references)}" + ) + + # Get ChatDocuments from documentList + self.services.chat.progressLogUpdate(operationId, 0.2, "Loading documents") + chatDocuments = self.services.chat.getChatDocumentsFromDocumentList(documentList) + + if not chatDocuments: + self.services.chat.progressLogFinish(operationId, False) + return ActionResult.isFailure(error="No documents found in documentList") + + logger.info(f"Extracting content from {len(chatDocuments)} documents") + + # Prepare extraction options + self.services.chat.progressLogUpdate(operationId, 0.3, "Preparing extraction options") + extractionOptionsParam = parameters.get("extractionOptions") + + # Convert dict to ExtractionOptions object if needed, or create defaults + if extractionOptionsParam: + if isinstance(extractionOptionsParam, dict): + # Convert dict to ExtractionOptions object + extractionOptions = ExtractionOptions(**extractionOptionsParam) + elif isinstance(extractionOptionsParam, ExtractionOptions): + extractionOptions = extractionOptionsParam + else: + # Invalid type, use defaults + extractionOptions = None + else: + extractionOptions = None + + # If extractionOptions not provided, create defaults + if not extractionOptions: + # Default extraction options for pure content extraction (no AI processing) + extractionOptions = ExtractionOptions( + prompt="Extract all content from the document", + mergeStrategy=MergeStrategy( + mergeType="concatenate", + groupBy="typeGroup", + orderBy="id" + ), + processDocumentsIndividually=True + ) + + # Get parent log ID for document-level operations + parentLogId = self.services.chat.getOperationLogId(operationId) + + # Call extraction service + self.services.chat.progressLogUpdate(operationId, 0.4, "Initiating") + self.services.chat.progressLogUpdate(operationId, 0.5, f"Extracting content from {len(chatDocuments)} documents") + extractedResults = self.services.extraction.extractContent(chatDocuments, extractionOptions) + + # Build ActionDocuments from ContentExtracted results + self.services.chat.progressLogUpdate(operationId, 0.8, "Building result documents") + actionDocuments = [] + # Map extracted results back to original documents by index (results are in same order) + for i, extracted in enumerate(extractedResults): + # Get original document name if available + originalDoc = chatDocuments[i] if i < len(chatDocuments) else None + if originalDoc and hasattr(originalDoc, 'fileName') and originalDoc.fileName: + # Use original filename with "extracted_" prefix + baseName = originalDoc.fileName.rsplit('.', 1)[0] if '.' in originalDoc.fileName else originalDoc.fileName + documentName = f"{baseName}_extracted_{extracted.id}.json" + else: + # Fallback to generic name with index + documentName = f"document_{i+1:03d}_extracted_{extracted.id}.json" + + # Store ContentExtracted object in ActionDocument.documentData + actionDoc = ActionDocument( + documentName=documentName, + documentData=extracted, # ContentExtracted object + mimeType="application/json" + ) + actionDocuments.append(actionDoc) + + self.services.chat.progressLogFinish(operationId, True) + + return ActionResult.isSuccess(documents=actionDocuments) + + except Exception as e: + logger.error(f"Error in content extraction: {str(e)}") + + # Complete progress tracking with failure + try: + self.services.chat.progressLogFinish(operationId, False) + except: + pass # Don't fail on progress logging errors + + return ActionResult.isFailure(error=str(e)) + diff --git a/modules/workflows/processing/core/messageCreator.py b/modules/workflows/processing/core/messageCreator.py index 55222ece..79ec66a6 100644 --- a/modules/workflows/processing/core/messageCreator.py +++ b/modules/workflows/processing/core/messageCreator.py @@ -210,8 +210,14 @@ class MessageCreator: taskProgress = str(taskIndex) # Enhanced completion message with criteria details - if reviewResult and hasattr(reviewResult, 'reason'): - completionMessage = f"🎯 **Task {taskProgress}**\n\n✅ {reviewResult.reason or 'Task completed successfully'}" + # Prefer userMessage (user-friendly in user's language), fallback to reason + if reviewResult: + if hasattr(reviewResult, 'userMessage') and reviewResult.userMessage: + completionMessage = f"🎯 **Task {taskProgress}**\n\n✅ {reviewResult.userMessage}" + elif hasattr(reviewResult, 'reason') and reviewResult.reason: + completionMessage = f"🎯 **Task {taskProgress}**\n\n✅ {reviewResult.reason}" + else: + completionMessage = f"🎯 **Task {taskProgress}**\n\n✅ Task completed successfully" else: completionMessage = f"🎯 **Task {taskProgress}**\n\n✅ Task completed successfully" diff --git a/modules/workflows/processing/shared/promptGenerationActionsDynamic.py b/modules/workflows/processing/shared/promptGenerationActionsDynamic.py index a58467fb..266c2e4f 100644 --- a/modules/workflows/processing/shared/promptGenerationActionsDynamic.py +++ b/modules/workflows/processing/shared/promptGenerationActionsDynamic.py @@ -350,34 +350,36 @@ Return ONLY JSON (no markdown, no explanations). The decision MUST: - Match parameter names exactly as defined in AVAILABLE_METHODS {{ - "status": "continue", - "reason": "Brief reason explaining why continuing", - "nextAction": "Selected_action_from_ACTIONS", + "status": "continue" | "success", + "reason": "Brief reason explaining why continuing or why task is complete", + "userMessage": "User-friendly message in language '{{KEY:USER_LANGUAGE}}' explaining the task status (1 sentence, first person, friendly tone)", + "nextAction": "Selected_action_from_ACTIONS" | null, "nextActionParameters": {{ "documentList": ["docItem::", "docList: