From 16338a6f940a54946272a731cf8f7f12a5fb6d0d Mon Sep 17 00:00:00 2001 From: ValueOn AG Date: Sat, 17 May 2025 11:38:38 +0200 Subject: [PATCH] erd cleanup --- app.py | 30 ++-- modules/{ => agents}/agentAnalyst.py | 2 +- modules/{ => agents}/agentCoach.py | 2 +- modules/{ => agents}/agentCoder.py | 15 +- modules/{ => agents}/agentDocumentation.py | 10 +- modules/{ => agents}/agentEmail.py | 15 +- modules/{ => agents}/agentWebcrawler.py | 8 +- .../connectors}/connectorAiAnthropic.py | 2 +- .../connectors}/connectorAiOpenai.py | 2 +- .../connectors}/connectorDbJson.py | 0 modules/{ => interfaces}/gatewayAccess.py | 0 modules/{ => interfaces}/gatewayInterface.py | 14 +- modules/{ => interfaces}/gatewayModel.py | 0 modules/{ => interfaces}/lucydomAccess.py | 0 modules/{ => interfaces}/lucydomInterface.py | 18 +-- modules/{ => interfaces}/lucydomModel.py | 25 +--- {routes => modules/routes}/routeAttributes.py | 8 +- {routes => modules/routes}/routeFiles.py | 8 +- {routes => modules/routes}/routeGeneral.py | 17 ++- {routes => modules/routes}/routeMandates.py | 6 +- {routes => modules/routes}/routeMsft.py | 8 +- {routes => modules/routes}/routePrompts.py | 6 +- {routes => modules/routes}/routeUsers.py | 6 +- {routes => modules/routes}/routeWorkflows.py | 8 +- modules/{ => security}/auth.py | 4 +- modules/{ => shared}/configuration.py | 0 modules/{ => shared}/defAttributes.py | 0 modules/{ => shared}/mimeUtils.py | 0 modules/workflow/agentBase.py | 128 +++++++++++++++++ .../agentRegistry.py} | 129 +----------------- modules/{ => workflow}/documentProcessor.py | 0 modules/workflow/workflowAgentsRegistry.py | 10 ++ modules/{ => workflow}/workflowManager.py | 10 +- 33 files changed, 237 insertions(+), 254 deletions(-) rename modules/{ => agents}/agentAnalyst.py (99%) rename modules/{ => agents}/agentCoach.py (99%) rename modules/{ => agents}/agentCoder.py (99%) rename modules/{ => agents}/agentDocumentation.py (99%) rename modules/{ => agents}/agentEmail.py (98%) rename modules/{ => agents}/agentWebcrawler.py (99%) rename {connectors => modules/connectors}/connectorAiAnthropic.py (99%) rename {connectors => modules/connectors}/connectorAiOpenai.py (98%) rename {connectors => modules/connectors}/connectorDbJson.py (100%) rename modules/{ => interfaces}/gatewayAccess.py (100%) rename modules/{ => interfaces}/gatewayInterface.py (97%) rename modules/{ => interfaces}/gatewayModel.py (100%) rename modules/{ => interfaces}/lucydomAccess.py (100%) rename modules/{ => interfaces}/lucydomInterface.py (98%) rename modules/{ => interfaces}/lucydomModel.py (92%) rename {routes => modules/routes}/routeAttributes.py (89%) rename {routes => modules/routes}/routeFiles.py (96%) rename {routes => modules/routes}/routeGeneral.py (82%) rename {routes => modules/routes}/routeMandates.py (97%) rename {routes => modules/routes}/routeMsft.py (98%) rename {routes => modules/routes}/routePrompts.py (96%) rename {routes => modules/routes}/routeUsers.py (98%) rename {routes => modules/routes}/routeWorkflows.py (98%) rename modules/{ => security}/auth.py (97%) rename modules/{ => shared}/configuration.py (100%) rename modules/{ => shared}/defAttributes.py (100%) rename modules/{ => shared}/mimeUtils.py (100%) create mode 100644 modules/workflow/agentBase.py rename modules/{workflowAgentsRegistry.py => workflow/agentRegistry.py} (53%) rename modules/{ => workflow}/documentProcessor.py (100%) create mode 100644 modules/workflow/workflowAgentsRegistry.py rename modules/{ => workflow}/workflowManager.py (99%) diff --git a/app.py b/app.py index 068fcaed..dba65587 100644 --- a/app.py +++ b/app.py @@ -3,7 +3,6 @@ os.environ["NUMEXPR_MAX_THREADS"] = "12" from fastapi import FastAPI, HTTPException, Depends, Body, status, Response from fastapi.middleware.cors import CORSMiddleware -from fastapi.staticfiles import StaticFiles from contextlib import asynccontextmanager import logging @@ -11,7 +10,7 @@ from logging.handlers import RotatingFileHandler from datetime import timedelta import pathlib -from modules.configuration import APP_CONFIG +from modules.shared.configuration import APP_CONFIG def initLogging(): # Get log level from config (default to INFO if not found) @@ -59,8 +58,7 @@ logger = logging.getLogger(__name__) instanceLabel = APP_CONFIG.get("APP_ENV_LABEL") # Import models - import generically for INITIALIZATION -import modules.gatewayModel as gatewayModel -from modules.gatewayInterface import getGatewayInterface +from modules.interfaces.gatewayInterface import getGatewayInterface gateway = getGatewayInterface() @@ -99,35 +97,27 @@ app.add_middleware( max_age=86400 # Increased caching for preflight requests ) -# Static folder for frontend - work with absolute path -baseDir = pathlib.Path(__file__).parent -staticFolder = baseDir / "static" -os.makedirs(staticFolder, exist_ok=True) - -# Mount static files with proper configuration -app.mount("/static", StaticFiles(directory=str(staticFolder), html=True), name="static") - # Include all routers -from routes.routeGeneral import router as generalRouter +from modules.routes.routeGeneral import router as generalRouter app.include_router(generalRouter) -from routes.routeAttributes import router as attributesRouter +from modules.routes.routeAttributes import router as attributesRouter app.include_router(attributesRouter) -from routes.routeMandates import router as mandateRouter +from modules.routes.routeMandates import router as mandateRouter app.include_router(mandateRouter) -from routes.routeUsers import router as userRouter +from modules.routes.routeUsers import router as userRouter app.include_router(userRouter) -from routes.routeFiles import router as fileRouter +from modules.routes.routeFiles import router as fileRouter app.include_router(fileRouter) -from routes.routePrompts import router as promptRouter +from modules.routes.routePrompts import router as promptRouter app.include_router(promptRouter) -from routes.routeWorkflows import router as workflowRouter +from modules.routes.routeWorkflows import router as workflowRouter app.include_router(workflowRouter) -from routes.routeMsft import router as msftRouter +from modules.routes.routeMsft import router as msftRouter app.include_router(msftRouter) diff --git a/modules/agentAnalyst.py b/modules/agents/agentAnalyst.py similarity index 99% rename from modules/agentAnalyst.py rename to modules/agents/agentAnalyst.py index 93107371..1e14332b 100644 --- a/modules/agentAnalyst.py +++ b/modules/agents/agentAnalyst.py @@ -12,7 +12,7 @@ import pandas as pd import matplotlib.pyplot as plt import seaborn as sns -from modules.workflowAgentsRegistry import AgentBase +from modules.workflow.agentBase import AgentBase logger = logging.getLogger(__name__) diff --git a/modules/agentCoach.py b/modules/agents/agentCoach.py similarity index 99% rename from modules/agentCoach.py rename to modules/agents/agentCoach.py index 8f97cf78..13fd5dbb 100644 --- a/modules/agentCoach.py +++ b/modules/agents/agentCoach.py @@ -8,7 +8,7 @@ from typing import Dict, Any, List import json from datetime import datetime -from modules.workflowAgentsRegistry import AgentBase +from modules.workflow.agentBase import AgentBase logger = logging.getLogger(__name__) diff --git a/modules/agentCoder.py b/modules/agents/agentCoder.py similarity index 99% rename from modules/agentCoder.py rename to modules/agents/agentCoder.py index 8950e6c7..798a744e 100644 --- a/modules/agentCoder.py +++ b/modules/agents/agentCoder.py @@ -1,19 +1,22 @@ """ -Simple Coder Agent for execution of Python code. -Modified to pass expected output document names to the generated code. +Coder agent for generating and executing code. +Provides code generation, execution, and improvement capabilities. """ import logging +from typing import Dict, Any, List, Tuple import json import os +import sys import subprocess import tempfile import shutil -import sys -from typing import Dict, Any, List, Tuple +import venv +import importlib.util +from datetime import datetime -from modules.workflowAgentsRegistry import AgentBase -from modules.configuration import APP_CONFIG +from modules.workflow.agentBase import AgentBase +from modules.shared.configuration import APP_CONFIG logger = logging.getLogger(__name__) diff --git a/modules/agentDocumentation.py b/modules/agents/agentDocumentation.py similarity index 99% rename from modules/agentDocumentation.py rename to modules/agents/agentDocumentation.py index 259795fe..11ca8e1f 100644 --- a/modules/agentDocumentation.py +++ b/modules/agents/agentDocumentation.py @@ -1,13 +1,15 @@ """ -Documentation agent for creating documentation, reports, and structured content. -Reimagined with an output-first, AI-driven approach with multi-step document generation. +Documentation agent for generating structured documentation. +Provides comprehensive documentation generation capabilities. """ import logging -import json from typing import Dict, Any, List +import json +import re +from datetime import datetime -from modules.workflowAgentsRegistry import AgentBase +from modules.workflow.agentBase import AgentBase logger = logging.getLogger(__name__) diff --git a/modules/agentEmail.py b/modules/agents/agentEmail.py similarity index 98% rename from modules/agentEmail.py rename to modules/agents/agentEmail.py index 116e58fc..19b84948 100644 --- a/modules/agentEmail.py +++ b/modules/agents/agentEmail.py @@ -1,18 +1,21 @@ """ -Email agent for creating draft emails with Microsoft Graph API. -Creates HTML-formatted email templates with attachments based on input documents. +Email agent for generating and sending emails. +Provides email template generation and sending capabilities. """ import logging +from typing import Dict, Any, List, Tuple import json -import base64 import os import requests +import base64 +from datetime import datetime +import re +from bs4 import BeautifulSoup import msal -from typing import Dict, Any, List, Optional -from modules.configuration import APP_CONFIG +from modules.shared.configuration import APP_CONFIG -from modules.workflowAgentsRegistry import AgentBase +from modules.workflow.agentBase import AgentBase logger = logging.getLogger(__name__) diff --git a/modules/agentWebcrawler.py b/modules/agents/agentWebcrawler.py similarity index 99% rename from modules/agentWebcrawler.py rename to modules/agents/agentWebcrawler.py index 56cac5b1..7ea7c9c5 100644 --- a/modules/agentWebcrawler.py +++ b/modules/agents/agentWebcrawler.py @@ -1,6 +1,6 @@ """ -Webcrawler agent for research and retrieval of information from the web. -Reimagined with an output-first, AI-driven approach. +Web crawler agent for gathering and analyzing web content. +Provides web research and content extraction capabilities. """ import logging @@ -14,8 +14,8 @@ from bs4 import BeautifulSoup import requests import markdown -from modules.workflowAgentsRegistry import AgentBase -from modules.configuration import APP_CONFIG +from modules.workflow.agentBase import AgentBase +from modules.shared.configuration import APP_CONFIG logger = logging.getLogger(__name__) diff --git a/connectors/connectorAiAnthropic.py b/modules/connectors/connectorAiAnthropic.py similarity index 99% rename from connectors/connectorAiAnthropic.py rename to modules/connectors/connectorAiAnthropic.py index 4b0b2412..b1e435ee 100644 --- a/connectors/connectorAiAnthropic.py +++ b/modules/connectors/connectorAiAnthropic.py @@ -2,7 +2,7 @@ import logging import httpx from typing import Dict, Any, List, Union from fastapi import HTTPException -from modules.configuration import APP_CONFIG +from modules.shared.configuration import APP_CONFIG # Configure logger logger = logging.getLogger(__name__) diff --git a/connectors/connectorAiOpenai.py b/modules/connectors/connectorAiOpenai.py similarity index 98% rename from connectors/connectorAiOpenai.py rename to modules/connectors/connectorAiOpenai.py index fd03af86..ad79f445 100644 --- a/connectors/connectorAiOpenai.py +++ b/modules/connectors/connectorAiOpenai.py @@ -3,7 +3,7 @@ import base64 import httpx from typing import Dict, Any, List, Union from fastapi import HTTPException -from modules.configuration import APP_CONFIG +from modules.shared.configuration import APP_CONFIG # Configure logger logger = logging.getLogger(__name__) diff --git a/connectors/connectorDbJson.py b/modules/connectors/connectorDbJson.py similarity index 100% rename from connectors/connectorDbJson.py rename to modules/connectors/connectorDbJson.py diff --git a/modules/gatewayAccess.py b/modules/interfaces/gatewayAccess.py similarity index 100% rename from modules/gatewayAccess.py rename to modules/interfaces/gatewayAccess.py diff --git a/modules/gatewayInterface.py b/modules/interfaces/gatewayInterface.py similarity index 97% rename from modules/gatewayInterface.py rename to modules/interfaces/gatewayInterface.py index 7a236768..6ab523d4 100644 --- a/modules/gatewayInterface.py +++ b/modules/interfaces/gatewayInterface.py @@ -9,9 +9,9 @@ from typing import Dict, Any, List, Optional, Union import importlib from passlib.context import CryptContext -from connectors.connectorDbJson import DatabaseConnector -from modules.configuration import APP_CONFIG -from modules.gatewayAccess import _uam, _canModify +from modules.connectors.connectorDbJson import DatabaseConnector +from modules.shared.configuration import APP_CONFIG +from modules.interfaces.gatewayAccess import _uam, _canModify logger = logging.getLogger(__name__) @@ -31,14 +31,6 @@ class GatewayInterface: self.mandateId = mandateId self.userId = userId - # Import data model module - try: - self.modelModule = importlib.import_module("modules.gatewayModel") - logger.info("gatewayModel successfully imported") - except ImportError as e: - logger.error(f"Error importing gatewayModel: {e}") - raise - # Initialize database self._initializeDatabase() diff --git a/modules/gatewayModel.py b/modules/interfaces/gatewayModel.py similarity index 100% rename from modules/gatewayModel.py rename to modules/interfaces/gatewayModel.py diff --git a/modules/lucydomAccess.py b/modules/interfaces/lucydomAccess.py similarity index 100% rename from modules/lucydomAccess.py rename to modules/interfaces/lucydomAccess.py diff --git a/modules/lucydomInterface.py b/modules/interfaces/lucydomInterface.py similarity index 98% rename from modules/lucydomInterface.py rename to modules/interfaces/lucydomInterface.py index e63ef736..21214117 100644 --- a/modules/lucydomInterface.py +++ b/modules/interfaces/lucydomInterface.py @@ -13,15 +13,15 @@ import importlib import hashlib import json -from modules.mimeUtils import isTextMimeType, determineContentEncoding -from modules.lucydomAccess import LucyDOMAccess +from modules.shared.mimeUtils import isTextMimeType, determineContentEncoding +from modules.interfaces.lucydomAccess import LucyDOMAccess # DYNAMIC PART: Connectors to the Interface -from connectors.connectorDbJson import DatabaseConnector -from connectors.connectorAiOpenai import ChatService +from modules.connectors.connectorDbJson import DatabaseConnector +from modules.connectors.connectorAiOpenai import ChatService # Basic Configurations -from modules.configuration import APP_CONFIG +from modules.shared.configuration import APP_CONFIG logger = logging.getLogger(__name__) # Custom exceptions for file handling @@ -61,14 +61,6 @@ class LucyDOMInterface: self.userLanguage = "en" # Default user language self.aiService = None # Will be set externally - # Import data model module - try: - self.modelModule = importlib.import_module("modules.lucydomModel") - logger.info("lucydomModel successfully imported") - except ImportError as e: - logger.error(f"Error importing lucydomModel: {e}") - raise - # Initialize database connector self._initializeDatabase() diff --git a/modules/lucydomModel.py b/modules/interfaces/lucydomModel.py similarity index 92% rename from modules/lucydomModel.py rename to modules/interfaces/lucydomModel.py index 3c71414c..c7bb6d37 100644 --- a/modules/lucydomModel.py +++ b/modules/interfaces/lucydomModel.py @@ -183,20 +183,6 @@ class Workflow(BaseModel): # Agent and Workflow Task Models -class AgentTask(BaseModel): - """Standardized task structure for agent processing""" - taskId: str = Field(description="Unique ID for this task") - workflowId: str = Field(description="ID of the parent workflow") - prompt: str = Field(description="The main instruction for the agent") - inputDocuments: List[Document] = Field(default=[], description="List of document objects to process") - outputSpecifications: List[Dict[str, str]] = Field(default=[], description="List of required output documents") - context: Dict[str, Any] = Field(description="Additional contextual information") - - label: Label = Field( - default=Label(default="Agent Task", translations={"en": "Agent Task", "fr": "Tâche d'agent"}), - description="Label for the class" - ) - class AgentResult(BaseModel): """Result structure returned by agent processing""" feedback: str = Field(description="Text response explaining what the agent did") @@ -218,6 +204,8 @@ class AgentInfo(BaseModel): description="Label for the class" ) + + class InputDocument(BaseModel): """Input document specification for a task""" label: str = Field(description="Document label in the format 'filename.ext'") @@ -268,15 +256,6 @@ class WorkflowStatus(BaseModel): description="Label for the class" ) -class WorkflowLabels(BaseModel): - """Global workflow labels and messages""" - systemName: str = Field(default="AI Assistant") - workflowStatusMessages: WorkflowStatus = Field(default_factory=WorkflowStatus) - - label: Label = Field( - default=Label(default="Workflow Labels", translations={"en": "Workflow Labels", "fr": "Étiquettes du workflow"}), - description="Label for the class" - ) # Request models for the API diff --git a/routes/routeAttributes.py b/modules/routes/routeAttributes.py similarity index 89% rename from routes/routeAttributes.py rename to modules/routes/routeAttributes.py index 7a085da3..b9cd752f 100644 --- a/routes/routeAttributes.py +++ b/modules/routes/routeAttributes.py @@ -2,14 +2,14 @@ from fastapi import APIRouter, HTTPException, Depends, Path, Response from typing import List, Dict, Any from fastapi import status -from modules.auth import getCurrentActiveUser, getUserContext +from modules.security.auth import getCurrentActiveUser, getUserContext # Import the attribute definition and helper functions -from modules.defAttributes import AttributeDefinition, getModelAttributes +from modules.shared.defAttributes import AttributeDefinition, getModelAttributes # Import the model modules (without specific classes) -import modules.gatewayModel as gatewayModel -import modules.lucydomModel as lucydomModel +import modules.interfaces.gatewayModel as gatewayModel +import modules.interfaces.lucydomModel as lucydomModel modelClasses = { # Gateway model classes diff --git a/routes/routeFiles.py b/modules/routes/routeFiles.py similarity index 96% rename from routes/routeFiles.py rename to modules/routes/routeFiles.py index f1026d03..524016ba 100644 --- a/routes/routeFiles.py +++ b/modules/routes/routeFiles.py @@ -6,12 +6,12 @@ from datetime import datetime from dataclasses import dataclass import io -from modules.auth import getCurrentActiveUser, getUserContext -from modules.configuration import APP_CONFIG +from modules.security.auth import getCurrentActiveUser, getUserContext +from modules.shared.configuration import APP_CONFIG # Import interfaces -from modules.lucydomInterface import getLucydomInterface, FileError, FileNotFoundError, FileStorageError, FilePermissionError, FileDeletionError -from modules.lucydomModel import FileItem +from modules.interfaces.lucydomInterface import getLucydomInterface, FileError, FileNotFoundError, FileStorageError, FilePermissionError, FileDeletionError +from modules.interfaces.lucydomModel import FileItem # Configure logger logger = logging.getLogger(__name__) diff --git a/routes/routeGeneral.py b/modules/routes/routeGeneral.py similarity index 82% rename from routes/routeGeneral.py rename to modules/routes/routeGeneral.py index 0b5d3d33..0aa60055 100644 --- a/routes/routeGeneral.py +++ b/modules/routes/routeGeneral.py @@ -1,26 +1,31 @@ from fastapi import APIRouter, HTTPException, Depends, Body, status, Response from fastapi.responses import FileResponse from fastapi.security import OAuth2PasswordRequestForm +from fastapi.staticfiles import StaticFiles from typing import Dict, Any from datetime import timedelta import pathlib import os -from modules.configuration import APP_CONFIG -from modules.auth import ( +from modules.shared.configuration import APP_CONFIG +from modules.security.auth import ( createAccessToken, getCurrentActiveUser, getUserContext, ACCESS_TOKEN_EXPIRE_MINUTES ) -import modules.gatewayModel as gatewayModel -from modules.gatewayInterface import getGatewayInterface +import modules.interfaces.gatewayModel as gatewayModel +from modules.interfaces.gatewayInterface import getGatewayInterface router = APIRouter() -# Static folder for favicon -baseDir = pathlib.Path(__file__).parent.parent +# Static folder setup - using absolute path from app root +baseDir = pathlib.Path(__file__).parent.parent.parent # Go up to gateway root staticFolder = baseDir / "static" +os.makedirs(staticFolder, exist_ok=True) + +# Mount static files +router.mount("/static", StaticFiles(directory=str(staticFolder), html=True), name="static") @router.get("/favicon.ico") async def favicon(): diff --git a/routes/routeMandates.py b/modules/routes/routeMandates.py similarity index 97% rename from routes/routeMandates.py rename to modules/routes/routeMandates.py index ab71cc7e..981508f9 100644 --- a/routes/routeMandates.py +++ b/modules/routes/routeMandates.py @@ -5,9 +5,9 @@ from datetime import datetime from dataclasses import dataclass # Import interfaces -from modules.auth import getCurrentActiveUser, getUserContext -from modules.gatewayInterface import getGatewayInterface -from modules.gatewayModel import Mandate +from modules.security.auth import getCurrentActiveUser, getUserContext +from modules.interfaces.gatewayInterface import getGatewayInterface +from modules.interfaces.gatewayModel import Mandate # Determine all attributes of the model def getModelAttributes(modelClass): diff --git a/routes/routeMsft.py b/modules/routes/routeMsft.py similarity index 98% rename from routes/routeMsft.py rename to modules/routes/routeMsft.py index 917ab74e..77fb9f9a 100644 --- a/routes/routeMsft.py +++ b/modules/routes/routeMsft.py @@ -7,10 +7,10 @@ from typing import Dict, Any, Optional, List from datetime import datetime, timedelta import secrets -from modules.auth import getCurrentActiveUser, getUserContext, createAccessToken, ACCESS_TOKEN_EXPIRE_MINUTES -from modules.configuration import APP_CONFIG -from modules.lucydomInterface import getLucydomInterface -from modules.gatewayInterface import getGatewayInterface +from modules.security.auth import getCurrentActiveUser, getUserContext, createAccessToken, ACCESS_TOKEN_EXPIRE_MINUTES +from modules.shared.configuration import APP_CONFIG +from modules.interfaces.lucydomInterface import getLucydomInterface +from modules.interfaces.gatewayInterface import getGatewayInterface # Configure logger logger = logging.getLogger(__name__) diff --git a/routes/routePrompts.py b/modules/routes/routePrompts.py similarity index 96% rename from routes/routePrompts.py rename to modules/routes/routePrompts.py index 0e9216e7..a42ca865 100644 --- a/routes/routePrompts.py +++ b/modules/routes/routePrompts.py @@ -5,11 +5,11 @@ from datetime import datetime from dataclasses import dataclass # Import auth module -from modules.auth import getCurrentActiveUser, getUserContext +from modules.security.auth import getCurrentActiveUser, getUserContext # Import interfaces -from modules.lucydomInterface import getLucydomInterface -from modules.lucydomModel import Prompt +from modules.interfaces.lucydomInterface import getLucydomInterface +from modules.interfaces.lucydomModel import Prompt # Get all attributes of the model def getModelAttributes(modelClass): diff --git a/routes/routeUsers.py b/modules/routes/routeUsers.py similarity index 98% rename from routes/routeUsers.py rename to modules/routes/routeUsers.py index f7fa02c3..af53b75c 100644 --- a/routes/routeUsers.py +++ b/modules/routes/routeUsers.py @@ -8,11 +8,11 @@ import time import traceback # Import auth module -from modules.auth import getCurrentActiveUser, getUserContext +from modules.security.auth import getCurrentActiveUser, getUserContext # Import interfaces -from modules.gatewayInterface import getGatewayInterface -from modules.gatewayModel import User +from modules.interfaces.gatewayInterface import getGatewayInterface +from modules.interfaces.gatewayModel import User # Set up logger logger = logging.getLogger(__name__) diff --git a/routes/routeWorkflows.py b/modules/routes/routeWorkflows.py similarity index 98% rename from routes/routeWorkflows.py rename to modules/routes/routeWorkflows.py index 622198b4..3165b156 100644 --- a/routes/routeWorkflows.py +++ b/modules/routes/routeWorkflows.py @@ -12,12 +12,12 @@ from dataclasses import dataclass from datetime import datetime # Import interfaces -from modules.lucydomInterface import getLucydomInterface -from modules.auth import getCurrentActiveUser, getUserContext -from modules.workflowManager import getWorkflowManager +from modules.interfaces.lucydomInterface import getLucydomInterface +from modules.security.auth import getCurrentActiveUser, getUserContext +from modules.workflow.workflowManager import getWorkflowManager # Import models -from modules.lucydomModel import UserInputRequest +from modules.interfaces.lucydomModel import UserInputRequest # Configure logger logger = logging.getLogger(__name__) diff --git a/modules/auth.py b/modules/security/auth.py similarity index 97% rename from modules/auth.py rename to modules/security/auth.py index 6fdf7e2f..7506ea12 100644 --- a/modules/auth.py +++ b/modules/security/auth.py @@ -10,8 +10,8 @@ from fastapi.security import OAuth2PasswordBearer from jose import JWTError, jwt import logging -from modules.gatewayInterface import getGatewayInterface -from modules.configuration import APP_CONFIG +from modules.interfaces.gatewayInterface import getGatewayInterface +from modules.shared.configuration import APP_CONFIG # Get Config Data SECRET_KEY = APP_CONFIG.get("APP_JWT_SECRET_SECRET") diff --git a/modules/configuration.py b/modules/shared/configuration.py similarity index 100% rename from modules/configuration.py rename to modules/shared/configuration.py diff --git a/modules/defAttributes.py b/modules/shared/defAttributes.py similarity index 100% rename from modules/defAttributes.py rename to modules/shared/defAttributes.py diff --git a/modules/mimeUtils.py b/modules/shared/mimeUtils.py similarity index 100% rename from modules/mimeUtils.py rename to modules/shared/mimeUtils.py diff --git a/modules/workflow/agentBase.py b/modules/workflow/agentBase.py new file mode 100644 index 00000000..05223388 --- /dev/null +++ b/modules/workflow/agentBase.py @@ -0,0 +1,128 @@ +""" +Agent Base Module. +Provides the base class for all chat agents. +Defines the standardized interface for task processing. +""" + +import os +import logging +import uuid +from datetime import datetime +from typing import Dict, Any, List, Optional +from modules.shared.mimeUtils import isTextMimeType, determineContentEncoding + +logger = logging.getLogger(__name__) + +class AgentBase: + """ + Base class for all chat agents. + Defines the standardized interface for task processing. + """ + + def __init__(self): + """Initialize the base agent.""" + self.name = "base" + self.label = "Base Agent" + self.description = "Base agent functionality" + self.capabilities = [] + self.mydom = None + self.workflowManager = None # Will be set by workflow manager + + def setDependencies(self, mydom=None): + """Set external dependencies for the agent.""" + self.mydom = mydom + + def getAgentInfo(self) -> Dict[str, Any]: + """ + Return standardized information about the agent's capabilities. + + Returns: + Dictionary with name, description, and capabilities + """ + return { + "name": self.name, + "description": self.description, + "capabilities": self.capabilities + } + + async def processTask(self, task: Dict[str, Any]) -> Dict[str, Any]: + """ + Process a standardized task structure and return results. + This method must be implemented by all concrete agent classes. + + Args: + task: A dictionary containing: + - taskId: Unique ID for this task + - workflowId: ID of the parent workflow + - prompt: The main instruction for the agent + - inputDocuments: List of document objects to process + - outputSpecifications: List of required output documents + - context: Additional contextual information including: + - workflow: The complete workflow object + - workflowRound: Current workflow round + - agentType: Type of agent + - timestamp: Task timestamp + - language: User language + + Returns: + A dictionary containing: + - feedback: Text response explaining what the agent did + - documents: List of document objects created by the agent, + each containing a "base64Encoded" flag in addition to "label" and "content" + """ + # Base implementation - should be overridden by specialized agents + logger.warning(f"Agent {self.name} is using the default implementation of processTask") + return { + "feedback": f"The processTask method was not implemented by agent '{self.name}'.", + "documents": [] + } + + def determineBase64EncodingFlag(self, filename: str, content: Any, mimeType: str = None) -> bool: + """Wrapper for the utility function""" + return determineContentEncoding(filename, content, mimeType) + + def isTextMimeType(self, mimeType: str) -> bool: + """Wrapper for the utility function""" + return isTextMimeType(mimeType) + + def formatAgentDocumentOutput(self, label: str, content: Any, mimeType: str = None) -> Dict[str, Any]: + """ + Format agent output as a document. + + Args: + label: Label for the document + content: Content of the document + mimeType: Optional MIME type for the document + """ + # Create document structure + doc = { + "id": str(uuid.uuid4()), + "name": label, + "ext": "txt", # Default extension + "data": content, + "base64Encoded": False, + "metadata": { + "isText": True + } + } + + # Set MIME type if provided + if mimeType: + doc["mimeType"] = mimeType + # Update extension based on MIME type + if mimeType == "text/markdown": + doc["ext"] = "md" + elif mimeType == "text/html": + doc["ext"] = "html" + elif mimeType == "text/csv": + doc["ext"] = "csv" + elif mimeType == "application/json": + doc["ext"] = "json" + elif mimeType.startswith("image/"): + doc["ext"] = mimeType.split("/")[1] + doc["metadata"]["isText"] = False + elif mimeType == "application/pdf": + doc["ext"] = "pdf" + doc["metadata"]["isText"] = False + + return doc \ No newline at end of file diff --git a/modules/workflowAgentsRegistry.py b/modules/workflow/agentRegistry.py similarity index 53% rename from modules/workflowAgentsRegistry.py rename to modules/workflow/agentRegistry.py index 0d3e03b9..2665df25 100644 --- a/modules/workflowAgentsRegistry.py +++ b/modules/workflow/agentRegistry.py @@ -7,132 +7,11 @@ Optimized for the standardized task processing pattern. import os import logging import importlib -import uuid -from datetime import datetime from typing import Dict, Any, List, Optional -from modules.mimeUtils import isTextMimeType, determineContentEncoding - +from .agentBase import AgentBase logger = logging.getLogger(__name__) -""" -Updates to the AgentBase class in workflowAgentsRegistry.py to include base64Encoded flag handling. -""" - -class AgentBase: - """ - Base class for all chat agents. - Defines the standardized interface for task processing. - """ - - def __init__(self): - """Initialize the base agent.""" - self.name = "base-agent" - self.label = "(Template)" - self.description = "Basic agent functionality" - self.capabilities = [] - self.mydom = None - self.workflowManager = None # Will be set by workflow manager - - def setDependencies(self, mydom=None): - """Set external dependencies for the agent.""" - self.mydom = mydom - - def getAgentInfo(self) -> Dict[str, Any]: - """ - Return standardized information about the agent's capabilities. - - Returns: - Dictionary with name, description, and capabilities - """ - return { - "name": self.name, - "description": self.description, - "capabilities": self.capabilities - } - - async def processTask(self, task: Dict[str, Any]) -> Dict[str, Any]: - """ - Process a standardized task structure and return results. - This method must be implemented by all concrete agent classes. - - Args: - task: A dictionary containing: - - taskId: Unique ID for this task - - workflowId: ID of the parent workflow - - prompt: The main instruction for the agent - - inputDocuments: List of document objects to process - - outputSpecifications: List of required output documents - - context: Additional contextual information including: - - workflow: The complete workflow object - - workflowRound: Current workflow round - - agentType: Type of agent - - timestamp: Task timestamp - - language: User language - - Returns: - A dictionary containing: - - feedback: Text response explaining what the agent did - - documents: List of document objects created by the agent, - each containing a "base64Encoded" flag in addition to "label" and "content" - """ - # Base implementation - should be overridden by specialized agents - logger.warning(f"Agent {self.name} is using the default implementation of processTask") - return { - "feedback": f"The processTask method was not implemented by agent '{self.name}'.", - "documents": [] - } - - def determineBase64EncodingFlag(self, filename: str, content: Any, mimeType: str = None) -> bool: - """Wrapper for the utility function""" - return determineContentEncoding(filename, content, mimeType) - - def isTextMimeType(self, mimeType: str) -> bool: - """Wrapper for the utility function""" - return isTextMimeType(mimeType) - - def formatAgentDocumentOutput(self, label: str, content: Any, mimeType: str = None) -> Dict[str, Any]: - """ - Format agent output as a document. - - Args: - label: Label for the document - content: Content of the document - mimeType: Optional MIME type for the document - """ - # Create document structure - doc = { - "id": str(uuid.uuid4()), - "name": label, - "ext": "txt", # Default extension - "data": content, - "base64Encoded": False, - "metadata": { - "isText": True - } - } - - # Set MIME type if provided - if mimeType: - doc["mimeType"] = mimeType - # Update extension based on MIME type - if mimeType == "text/markdown": - doc["ext"] = "md" - elif mimeType == "text/html": - doc["ext"] = "html" - elif mimeType == "text/csv": - doc["ext"] = "csv" - elif mimeType == "application/json": - doc["ext"] = "json" - elif mimeType.startswith("image/"): - doc["ext"] = mimeType.split("/")[1] - doc["metadata"]["isText"] = False - elif mimeType == "application/pdf": - doc["ext"] = "pdf" - doc["metadata"]["isText"] = False - - return doc - class AgentRegistry: """Central registry for all available agents in the system.""" @@ -160,7 +39,7 @@ class AgentRegistry: # List of agent modules to load agentModules = [] - agentDir = os.path.dirname(__file__) + agentDir = os.path.join(os.path.dirname(os.path.dirname(__file__)), "agents") # Search the directory for agent modules for filename in os.listdir(agentDir): @@ -177,7 +56,7 @@ class AgentRegistry: for moduleName in agentModules: try: # Import the module - module = importlib.import_module(f"modules.{moduleName}") + module = importlib.import_module(f"modules.agents.{moduleName}") # Look for agent class or get_*_agent function agentName = moduleName.split("agent")[-1] @@ -273,4 +152,4 @@ class AgentRegistry: # Singleton factory for the agent registry def getAgentRegistry(): - return AgentRegistry.getInstance() \ No newline at end of file + return AgentRegistry.getInstance() \ No newline at end of file diff --git a/modules/documentProcessor.py b/modules/workflow/documentProcessor.py similarity index 100% rename from modules/documentProcessor.py rename to modules/workflow/documentProcessor.py diff --git a/modules/workflow/workflowAgentsRegistry.py b/modules/workflow/workflowAgentsRegistry.py new file mode 100644 index 00000000..f742a1ac --- /dev/null +++ b/modules/workflow/workflowAgentsRegistry.py @@ -0,0 +1,10 @@ +""" +Agent Registry Module. +Provides a central registry system for all available agents. +Optimized for the standardized task processing pattern. +""" + +from .agentBase import AgentBase +from .agentRegistry import AgentRegistry, getAgentRegistry + +__all__ = ['AgentBase', 'AgentRegistry', 'getAgentRegistry'] \ No newline at end of file diff --git a/modules/workflowManager.py b/modules/workflow/workflowManager.py similarity index 99% rename from modules/workflowManager.py rename to modules/workflow/workflowManager.py index 7187d7a6..5082b563 100644 --- a/modules/workflowManager.py +++ b/modules/workflow/workflowManager.py @@ -14,12 +14,12 @@ from datetime import datetime, timedelta from typing import Dict, Any, List, Optional, Union, Tuple import time -from modules.mimeUtils import isTextMimeType, determineContentEncoding +from modules.shared.mimeUtils import isTextMimeType, determineContentEncoding # Required imports -from modules.workflowAgentsRegistry import getAgentRegistry -from modules.lucydomInterface import getLucydomInterface as domInterface -from modules.documentProcessor import getDocumentContents +from modules.workflow.agentRegistry import getAgentRegistry +from modules.interfaces.lucydomInterface import getLucydomInterface as domInterface +from modules.workflow.documentProcessor import getDocumentContents # Configure logger logger = logging.getLogger(__name__) @@ -461,7 +461,7 @@ JSON_OUTPUT = {{ "objFinalDocuments": ["label",...], # document label in the format 'filename.ext' "objWorkplan": [ {{ - "agent": "agent_name", # Name of an available agent + "agent": "agentName", # Name of an available agent "prompt": "Specific instructions to the agent, that he knows what to do with which documents and which output to provide." "outputDocuments": [ {{