erd cleanup

app.py

@@ -3,7 +3,6 @@ os.environ["NUMEXPR_MAX_THREADS"] = "12"
 
 from fastapi import FastAPI, HTTPException, Depends, Body, status, Response
 from fastapi.middleware.cors import CORSMiddleware
-from fastapi.staticfiles import StaticFiles
 from contextlib import asynccontextmanager
 
 import logging
@@ -11,7 +10,7 @@ from logging.handlers import RotatingFileHandler
 from datetime import timedelta
 import pathlib
 
-from modules.configuration import APP_CONFIG
+from modules.shared.configuration import APP_CONFIG
 
 def initLogging():
     # Get log level from config (default to INFO if not found)
@@ -59,8 +58,7 @@ logger = logging.getLogger(__name__)
 instanceLabel = APP_CONFIG.get("APP_ENV_LABEL")
 
 # Import models - import generically for INITIALIZATION
-import modules.gatewayModel as gatewayModel
-from modules.gatewayInterface import getGatewayInterface
+from modules.interfaces.gatewayInterface import getGatewayInterface
 gateway = getGatewayInterface()
 
 
@@ -99,35 +97,27 @@ app.add_middleware(
     max_age=86400  # Increased caching for preflight requests
 )
 
-# Static folder for frontend - work with absolute path
-baseDir = pathlib.Path(__file__).parent
-staticFolder = baseDir / "static"
-os.makedirs(staticFolder, exist_ok=True)
-
-# Mount static files with proper configuration
-app.mount("/static", StaticFiles(directory=str(staticFolder), html=True), name="static")
-
 # Include all routers
-from routes.routeGeneral import router as generalRouter
+from modules.routes.routeGeneral import router as generalRouter
 app.include_router(generalRouter)
 
-from routes.routeAttributes import router as attributesRouter
+from modules.routes.routeAttributes import router as attributesRouter
 app.include_router(attributesRouter)
 
-from routes.routeMandates import router as mandateRouter
+from modules.routes.routeMandates import router as mandateRouter
 app.include_router(mandateRouter)
 
-from routes.routeUsers import router as userRouter
+from modules.routes.routeUsers import router as userRouter
 app.include_router(userRouter)
 
-from routes.routeFiles import router as fileRouter
+from modules.routes.routeFiles import router as fileRouter
 app.include_router(fileRouter)
 
-from routes.routePrompts import router as promptRouter
+from modules.routes.routePrompts import router as promptRouter
 app.include_router(promptRouter)
 
-from routes.routeWorkflows import router as workflowRouter
+from modules.routes.routeWorkflows import router as workflowRouter
 app.include_router(workflowRouter)
 
-from routes.routeMsft import router as msftRouter
+from modules.routes.routeMsft import router as msftRouter
 app.include_router(msftRouter)

modules/agents/agentAnalyst.py

@@ -12,7 +12,7 @@ import pandas as pd
 import matplotlib.pyplot as plt
 import seaborn as sns
 
-from modules.workflowAgentsRegistry import AgentBase
+from modules.workflow.agentBase import AgentBase
 
 logger = logging.getLogger(__name__)
 

modules/agents/agentCoach.py

@@ -8,7 +8,7 @@ from typing import Dict, Any, List
 import json
 from datetime import datetime
 
-from modules.workflowAgentsRegistry import AgentBase
+from modules.workflow.agentBase import AgentBase
 
 logger = logging.getLogger(__name__)
 

modules/agents/agentCoder.py

@@ -1,19 +1,22 @@
 """
-Simple Coder Agent for execution of Python code.
-Modified to pass expected output document names to the generated code.
+Coder agent for generating and executing code.
+Provides code generation, execution, and improvement capabilities.
 """
 
 import logging
+from typing import Dict, Any, List, Tuple
 import json
 import os
+import sys
 import subprocess
 import tempfile
 import shutil
-import sys
-from typing import Dict, Any, List, Tuple
+import venv
+import importlib.util
+from datetime import datetime
 
-from modules.workflowAgentsRegistry import AgentBase
-from modules.configuration import APP_CONFIG
+from modules.workflow.agentBase import AgentBase
+from modules.shared.configuration import APP_CONFIG
 
 logger = logging.getLogger(__name__)
 

modules/agents/agentDocumentation.py

@@ -1,13 +1,15 @@
 """
-Documentation agent for creating documentation, reports, and structured content.
-Reimagined with an output-first, AI-driven approach with multi-step document generation.
+Documentation agent for generating structured documentation.
+Provides comprehensive documentation generation capabilities.
 """
 
 import logging
-import json
 from typing import Dict, Any, List
+import json
+import re
+from datetime import datetime
 
-from modules.workflowAgentsRegistry import AgentBase
+from modules.workflow.agentBase import AgentBase
 
 logger = logging.getLogger(__name__)
 

modules/agents/agentEmail.py

@@ -1,18 +1,21 @@
 """
-Email agent for creating draft emails with Microsoft Graph API.
-Creates HTML-formatted email templates with attachments based on input documents.
+Email agent for generating and sending emails.
+Provides email template generation and sending capabilities.
 """
 
 import logging
+from typing import Dict, Any, List, Tuple
 import json
-import base64
 import os
 import requests
+import base64
+from datetime import datetime
+import re
+from bs4 import BeautifulSoup
 import msal
-from typing import Dict, Any, List, Optional
-from modules.configuration import APP_CONFIG
+from modules.shared.configuration import APP_CONFIG
 
-from modules.workflowAgentsRegistry import AgentBase
+from modules.workflow.agentBase import AgentBase
 
 logger = logging.getLogger(__name__)
 

modules/agents/agentWebcrawler.py

@@ -1,6 +1,6 @@
 """
-Webcrawler agent for research and retrieval of information from the web.
-Reimagined with an output-first, AI-driven approach.
+Web crawler agent for gathering and analyzing web content.
+Provides web research and content extraction capabilities.
 """
 
 import logging
@@ -14,8 +14,8 @@ from bs4 import BeautifulSoup
 import requests
 import markdown
 
-from modules.workflowAgentsRegistry import AgentBase
-from modules.configuration import APP_CONFIG
+from modules.workflow.agentBase import AgentBase
+from modules.shared.configuration import APP_CONFIG
 
 logger = logging.getLogger(__name__)
 

modules/connectors/connectorAiAnthropic.py

@@ -2,7 +2,7 @@ import logging
 import httpx
 from typing import Dict, Any, List, Union
 from fastapi import HTTPException
-from modules.configuration import APP_CONFIG
+from modules.shared.configuration import APP_CONFIG
 
 # Configure logger
 logger = logging.getLogger(__name__)

modules/connectors/connectorAiOpenai.py

@@ -3,7 +3,7 @@ import base64
 import httpx
 from typing import Dict, Any, List, Union
 from fastapi import HTTPException
-from modules.configuration import APP_CONFIG
+from modules.shared.configuration import APP_CONFIG
 
 # Configure logger
 logger = logging.getLogger(__name__)

modules/interfaces/gatewayInterface.py

@@ -9,9 +9,9 @@ from typing import Dict, Any, List, Optional, Union
 import importlib
 from passlib.context import CryptContext
 
-from connectors.connectorDbJson import DatabaseConnector
-from modules.configuration import APP_CONFIG
-from modules.gatewayAccess import _uam, _canModify
+from modules.connectors.connectorDbJson import DatabaseConnector
+from modules.shared.configuration import APP_CONFIG
+from modules.interfaces.gatewayAccess import _uam, _canModify
 
 logger = logging.getLogger(__name__)
 
@@ -31,14 +31,6 @@ class GatewayInterface:
         self.mandateId = mandateId
         self.userId = userId
         
-        # Import data model module
-        try:
-            self.modelModule = importlib.import_module("modules.gatewayModel")
-            logger.info("gatewayModel successfully imported")
-        except ImportError as e:
-            logger.error(f"Error importing gatewayModel: {e}")
-            raise
-        
         # Initialize database
         self._initializeDatabase()
         

modules/interfaces/lucydomInterface.py

@@ -13,15 +13,15 @@ import importlib
 import hashlib
 import json
 
-from modules.mimeUtils import isTextMimeType, determineContentEncoding
-from modules.lucydomAccess import LucyDOMAccess
+from modules.shared.mimeUtils import isTextMimeType, determineContentEncoding
+from modules.interfaces.lucydomAccess import LucyDOMAccess
 
 # DYNAMIC PART: Connectors to the Interface
-from connectors.connectorDbJson import DatabaseConnector
-from connectors.connectorAiOpenai import ChatService
+from modules.connectors.connectorDbJson import DatabaseConnector
+from modules.connectors.connectorAiOpenai import ChatService
 
 # Basic Configurations
-from modules.configuration import APP_CONFIG
+from modules.shared.configuration import APP_CONFIG
 logger = logging.getLogger(__name__)
 
 # Custom exceptions for file handling
@@ -61,14 +61,6 @@ class LucyDOMInterface:
         self.userLanguage = "en"  # Default user language
         self.aiService = None  # Will be set externally
         
-        # Import data model module
-        try:
-            self.modelModule = importlib.import_module("modules.lucydomModel")
-            logger.info("lucydomModel successfully imported")
-        except ImportError as e:
-            logger.error(f"Error importing lucydomModel: {e}")
-            raise
-        
         # Initialize database connector
         self._initializeDatabase()
         

modules/interfaces/lucydomModel.py

@@ -183,20 +183,6 @@ class Workflow(BaseModel):
 
 # Agent and Workflow Task Models
 
-class AgentTask(BaseModel):
-    """Standardized task structure for agent processing"""
-    taskId: str = Field(description="Unique ID for this task")
-    workflowId: str = Field(description="ID of the parent workflow")
-    prompt: str = Field(description="The main instruction for the agent")
-    inputDocuments: List[Document] = Field(default=[], description="List of document objects to process")
-    outputSpecifications: List[Dict[str, str]] = Field(default=[], description="List of required output documents")
-    context: Dict[str, Any] = Field(description="Additional contextual information")
-
-    label: Label = Field(
-        default=Label(default="Agent Task", translations={"en": "Agent Task", "fr": "Tâche d'agent"}),
-        description="Label for the class"
-    )
-
 class AgentResult(BaseModel):
     """Result structure returned by agent processing"""
     feedback: str = Field(description="Text response explaining what the agent did")
@@ -218,6 +204,8 @@ class AgentInfo(BaseModel):
         description="Label for the class"
     )
 
+
+
 class InputDocument(BaseModel):
     """Input document specification for a task"""
     label: str = Field(description="Document label in the format 'filename.ext'")
@@ -268,15 +256,6 @@ class WorkflowStatus(BaseModel):
         description="Label for the class"
     )
 
-class WorkflowLabels(BaseModel):
-    """Global workflow labels and messages"""
-    systemName: str = Field(default="AI Assistant")
-    workflowStatusMessages: WorkflowStatus = Field(default_factory=WorkflowStatus)
-
-    label: Label = Field(
-        default=Label(default="Workflow Labels", translations={"en": "Workflow Labels", "fr": "Étiquettes du workflow"}),
-        description="Label for the class"
-    )
 
 # Request models for the API
 

modules/routes/routeAttributes.py

@@ -2,14 +2,14 @@ from fastapi import APIRouter, HTTPException, Depends, Path, Response
 from typing import List, Dict, Any
 from fastapi import status
 
-from modules.auth import getCurrentActiveUser, getUserContext
+from modules.security.auth import getCurrentActiveUser, getUserContext
 
 # Import the attribute definition and helper functions
-from modules.defAttributes import AttributeDefinition, getModelAttributes
+from modules.shared.defAttributes import AttributeDefinition, getModelAttributes
 
 # Import the model modules (without specific classes)
-import modules.gatewayModel as gatewayModel
-import modules.lucydomModel as lucydomModel
+import modules.interfaces.gatewayModel as gatewayModel
+import modules.interfaces.lucydomModel as lucydomModel
 
 modelClasses = {
     # Gateway model classes

modules/routes/routeFiles.py

@@ -6,12 +6,12 @@ from datetime import datetime
 from dataclasses import dataclass
 import io
 
-from modules.auth import getCurrentActiveUser, getUserContext
-from modules.configuration import APP_CONFIG
+from modules.security.auth import getCurrentActiveUser, getUserContext
+from modules.shared.configuration import APP_CONFIG
 
 # Import interfaces
-from modules.lucydomInterface import getLucydomInterface, FileError, FileNotFoundError, FileStorageError, FilePermissionError, FileDeletionError
-from modules.lucydomModel import FileItem
+from modules.interfaces.lucydomInterface import getLucydomInterface, FileError, FileNotFoundError, FileStorageError, FilePermissionError, FileDeletionError
+from modules.interfaces.lucydomModel import FileItem
 
 # Configure logger
 logger = logging.getLogger(__name__)

modules/routes/routeGeneral.py

@@ -1,26 +1,31 @@
 from fastapi import APIRouter, HTTPException, Depends, Body, status, Response
 from fastapi.responses import FileResponse
 from fastapi.security import OAuth2PasswordRequestForm
+from fastapi.staticfiles import StaticFiles
 from typing import Dict, Any
 from datetime import timedelta
 import pathlib
 import os
 
-from modules.configuration import APP_CONFIG
-from modules.auth import (
+from modules.shared.configuration import APP_CONFIG
+from modules.security.auth import (
     createAccessToken,
     getCurrentActiveUser, 
     getUserContext,
     ACCESS_TOKEN_EXPIRE_MINUTES
 )
-import modules.gatewayModel as gatewayModel
-from modules.gatewayInterface import getGatewayInterface
+import modules.interfaces.gatewayModel as gatewayModel
+from modules.interfaces.gatewayInterface import getGatewayInterface
 
 router = APIRouter()
 
-# Static folder for favicon
-baseDir = pathlib.Path(__file__).parent.parent
+# Static folder setup - using absolute path from app root
+baseDir = pathlib.Path(__file__).parent.parent.parent  # Go up to gateway root
 staticFolder = baseDir / "static"
+os.makedirs(staticFolder, exist_ok=True)
+
+# Mount static files
+router.mount("/static", StaticFiles(directory=str(staticFolder), html=True), name="static")
 
 @router.get("/favicon.ico")
 async def favicon():

modules/routes/routeMandates.py

@@ -5,9 +5,9 @@ from datetime import datetime
 from dataclasses import dataclass
 
 # Import interfaces
-from modules.auth import getCurrentActiveUser, getUserContext
-from modules.gatewayInterface import getGatewayInterface
-from modules.gatewayModel import Mandate
+from modules.security.auth import getCurrentActiveUser, getUserContext
+from modules.interfaces.gatewayInterface import getGatewayInterface
+from modules.interfaces.gatewayModel import Mandate
 
 # Determine all attributes of the model
 def getModelAttributes(modelClass):

modules/routes/routeMsft.py

@@ -7,10 +7,10 @@ from typing import Dict, Any, Optional, List
 from datetime import datetime, timedelta
 import secrets
 
-from modules.auth import getCurrentActiveUser, getUserContext, createAccessToken, ACCESS_TOKEN_EXPIRE_MINUTES
-from modules.configuration import APP_CONFIG
-from modules.lucydomInterface import getLucydomInterface
-from modules.gatewayInterface import getGatewayInterface
+from modules.security.auth import getCurrentActiveUser, getUserContext, createAccessToken, ACCESS_TOKEN_EXPIRE_MINUTES
+from modules.shared.configuration import APP_CONFIG
+from modules.interfaces.lucydomInterface import getLucydomInterface
+from modules.interfaces.gatewayInterface import getGatewayInterface
 
 # Configure logger
 logger = logging.getLogger(__name__)

modules/routes/routePrompts.py

@@ -5,11 +5,11 @@ from datetime import datetime
 from dataclasses import dataclass
 
 # Import auth module
-from modules.auth import getCurrentActiveUser, getUserContext
+from modules.security.auth import getCurrentActiveUser, getUserContext
 
 # Import interfaces
-from modules.lucydomInterface import getLucydomInterface
-from modules.lucydomModel import Prompt
+from modules.interfaces.lucydomInterface import getLucydomInterface
+from modules.interfaces.lucydomModel import Prompt
 
 # Get all attributes of the model
 def getModelAttributes(modelClass):

modules/routes/routeUsers.py

@@ -8,11 +8,11 @@ import time
 import traceback
 
 # Import auth module
-from modules.auth import getCurrentActiveUser, getUserContext
+from modules.security.auth import getCurrentActiveUser, getUserContext
 
 # Import interfaces
-from modules.gatewayInterface import getGatewayInterface
-from modules.gatewayModel import User
+from modules.interfaces.gatewayInterface import getGatewayInterface
+from modules.interfaces.gatewayModel import User
 
 # Set up logger
 logger = logging.getLogger(__name__)

modules/routes/routeWorkflows.py

@@ -12,12 +12,12 @@ from dataclasses import dataclass
 from datetime import datetime
 
 # Import interfaces
-from modules.lucydomInterface import getLucydomInterface
-from modules.auth import getCurrentActiveUser, getUserContext
-from modules.workflowManager import getWorkflowManager
+from modules.interfaces.lucydomInterface import getLucydomInterface
+from modules.security.auth import getCurrentActiveUser, getUserContext
+from modules.workflow.workflowManager import getWorkflowManager
 
 # Import models
-from modules.lucydomModel import UserInputRequest
+from modules.interfaces.lucydomModel import UserInputRequest
 
 # Configure logger
 logger = logging.getLogger(__name__)

modules/security/auth.py

@@ -10,8 +10,8 @@ from fastapi.security import OAuth2PasswordBearer
 from jose import JWTError, jwt
 import logging
 
-from modules.gatewayInterface import getGatewayInterface
-from modules.configuration import APP_CONFIG
+from modules.interfaces.gatewayInterface import getGatewayInterface
+from modules.shared.configuration import APP_CONFIG
 
 # Get Config Data
 SECRET_KEY = APP_CONFIG.get("APP_JWT_SECRET_SECRET")

modules/workflow/agentBase.py

@@ -0,0 +1,128 @@
+"""
+Agent Base Module.
+Provides the base class for all chat agents.
+Defines the standardized interface for task processing.
+"""
+
+import os
+import logging
+import uuid
+from datetime import datetime
+from typing import Dict, Any, List, Optional
+from modules.shared.mimeUtils import isTextMimeType, determineContentEncoding
+
+logger = logging.getLogger(__name__)
+
+class AgentBase:
+    """
+    Base class for all chat agents.
+    Defines the standardized interface for task processing.
+    """
+    
+    def __init__(self):
+        """Initialize the base agent."""
+        self.name = "base"
+        self.label = "Base Agent"
+        self.description = "Base agent functionality"
+        self.capabilities = []
+        self.mydom = None
+        self.workflowManager = None  # Will be set by workflow manager
+    
+    def setDependencies(self, mydom=None):
+        """Set external dependencies for the agent."""
+        self.mydom = mydom
+    
+    def getAgentInfo(self) -> Dict[str, Any]:
+        """
+        Return standardized information about the agent's capabilities.
+        
+        Returns:
+            Dictionary with name, description, and capabilities
+        """
+        return {
+            "name": self.name,
+            "description": self.description,
+            "capabilities": self.capabilities
+        }
+    
+    async def processTask(self, task: Dict[str, Any]) -> Dict[str, Any]:
+        """
+        Process a standardized task structure and return results.
+        This method must be implemented by all concrete agent classes.
+        
+        Args:
+            task: A dictionary containing:
+                - taskId: Unique ID for this task
+                - workflowId: ID of the parent workflow
+                - prompt: The main instruction for the agent
+                - inputDocuments: List of document objects to process
+                - outputSpecifications: List of required output documents
+                - context: Additional contextual information including:
+                    - workflow: The complete workflow object
+                    - workflowRound: Current workflow round
+                    - agentType: Type of agent
+                    - timestamp: Task timestamp
+                    - language: User language
+        
+        Returns:
+            A dictionary containing:
+                - feedback: Text response explaining what the agent did
+                - documents: List of document objects created by the agent, 
+                  each containing a "base64Encoded" flag in addition to "label" and "content"
+        """
+        # Base implementation - should be overridden by specialized agents
+        logger.warning(f"Agent {self.name} is using the default implementation of processTask")
+        return {
+            "feedback": f"The processTask method was not implemented by agent '{self.name}'.",
+            "documents": []
+        }
+    
+    def determineBase64EncodingFlag(self, filename: str, content: Any, mimeType: str = None) -> bool:
+        """Wrapper for the utility function"""
+        return determineContentEncoding(filename, content, mimeType)
+
+    def isTextMimeType(self, mimeType: str) -> bool:
+        """Wrapper for the utility function"""
+        return isTextMimeType(mimeType)
+
+    def formatAgentDocumentOutput(self, label: str, content: Any, mimeType: str = None) -> Dict[str, Any]:
+        """
+        Format agent output as a document.
+        
+        Args:
+            label: Label for the document
+            content: Content of the document
+            mimeType: Optional MIME type for the document
+        """
+        # Create document structure
+        doc = {
+            "id": str(uuid.uuid4()),
+            "name": label,
+            "ext": "txt",  # Default extension
+            "data": content,
+            "base64Encoded": False,
+            "metadata": {
+                "isText": True
+            }
+        }
+        
+        # Set MIME type if provided
+        if mimeType:
+            doc["mimeType"] = mimeType
+            # Update extension based on MIME type
+            if mimeType == "text/markdown":
+                doc["ext"] = "md"
+            elif mimeType == "text/html":
+                doc["ext"] = "html"
+            elif mimeType == "text/csv":
+                doc["ext"] = "csv"
+            elif mimeType == "application/json":
+                doc["ext"] = "json"
+            elif mimeType.startswith("image/"):
+                doc["ext"] = mimeType.split("/")[1]
+                doc["metadata"]["isText"] = False
+            elif mimeType == "application/pdf":
+                doc["ext"] = "pdf"
+                doc["metadata"]["isText"] = False
+        
+        return doc 

modules/workflow/agentRegistry.py

@@ -7,132 +7,11 @@ Optimized for the standardized task processing pattern.
 import os
 import logging
 import importlib
-import uuid
-from datetime import datetime
 from typing import Dict, Any, List, Optional
-from modules.mimeUtils import isTextMimeType, determineContentEncoding
-
+from .agentBase import AgentBase
 
 logger = logging.getLogger(__name__)
 
-"""
-Updates to the AgentBase class in workflowAgentsRegistry.py to include base64Encoded flag handling.
-"""
-
-class AgentBase:
-    """
-    Base class for all chat agents.
-    Defines the standardized interface for task processing.
-    """
-    
-    def __init__(self):
-        """Initialize the base agent."""
-        self.name = "base-agent"
-        self.label = "(Template)"
-        self.description = "Basic agent functionality"
-        self.capabilities = []
-        self.mydom = None
-        self.workflowManager = None  # Will be set by workflow manager
-    
-    def setDependencies(self, mydom=None):
-        """Set external dependencies for the agent."""
-        self.mydom = mydom
-    
-    def getAgentInfo(self) -> Dict[str, Any]:
-        """
-        Return standardized information about the agent's capabilities.
-        
-        Returns:
-            Dictionary with name, description, and capabilities
-        """
-        return {
-            "name": self.name,
-            "description": self.description,
-            "capabilities": self.capabilities
-        }
-    
-    async def processTask(self, task: Dict[str, Any]) -> Dict[str, Any]:
-        """
-        Process a standardized task structure and return results.
-        This method must be implemented by all concrete agent classes.
-        
-        Args:
-            task: A dictionary containing:
-                - taskId: Unique ID for this task
-                - workflowId: ID of the parent workflow
-                - prompt: The main instruction for the agent
-                - inputDocuments: List of document objects to process
-                - outputSpecifications: List of required output documents
-                - context: Additional contextual information including:
-                    - workflow: The complete workflow object
-                    - workflowRound: Current workflow round
-                    - agentType: Type of agent
-                    - timestamp: Task timestamp
-                    - language: User language
-        
-        Returns:
-            A dictionary containing:
-                - feedback: Text response explaining what the agent did
-                - documents: List of document objects created by the agent, 
-                  each containing a "base64Encoded" flag in addition to "label" and "content"
-        """
-        # Base implementation - should be overridden by specialized agents
-        logger.warning(f"Agent {self.name} is using the default implementation of processTask")
-        return {
-            "feedback": f"The processTask method was not implemented by agent '{self.name}'.",
-            "documents": []
-        }
-    
-    def determineBase64EncodingFlag(self, filename: str, content: Any, mimeType: str = None) -> bool:
-        """Wrapper for the utility function"""
-        return determineContentEncoding(filename, content, mimeType)
-
-    def isTextMimeType(self, mimeType: str) -> bool:
-        """Wrapper for the utility function"""
-        return isTextMimeType(mimeType)
-
-    def formatAgentDocumentOutput(self, label: str, content: Any, mimeType: str = None) -> Dict[str, Any]:
-        """
-        Format agent output as a document.
-        
-        Args:
-            label: Label for the document
-            content: Content of the document
-            mimeType: Optional MIME type for the document
-        """
-        # Create document structure
-        doc = {
-            "id": str(uuid.uuid4()),
-            "name": label,
-            "ext": "txt",  # Default extension
-            "data": content,
-            "base64Encoded": False,
-            "metadata": {
-                "isText": True
-            }
-        }
-        
-        # Set MIME type if provided
-        if mimeType:
-            doc["mimeType"] = mimeType
-            # Update extension based on MIME type
-            if mimeType == "text/markdown":
-                doc["ext"] = "md"
-            elif mimeType == "text/html":
-                doc["ext"] = "html"
-            elif mimeType == "text/csv":
-                doc["ext"] = "csv"
-            elif mimeType == "application/json":
-                doc["ext"] = "json"
-            elif mimeType.startswith("image/"):
-                doc["ext"] = mimeType.split("/")[1]
-                doc["metadata"]["isText"] = False
-            elif mimeType == "application/pdf":
-                doc["ext"] = "pdf"
-                doc["metadata"]["isText"] = False
-        
-        return doc
-
 class AgentRegistry:
     """Central registry for all available agents in the system."""
     
@@ -160,7 +39,7 @@ class AgentRegistry:
         
         # List of agent modules to load
         agentModules = []
-        agentDir = os.path.dirname(__file__)
+        agentDir = os.path.join(os.path.dirname(os.path.dirname(__file__)), "agents")
         
         # Search the directory for agent modules
         for filename in os.listdir(agentDir):
@@ -177,7 +56,7 @@ class AgentRegistry:
         for moduleName in agentModules:
             try:
                 # Import the module
-                module = importlib.import_module(f"modules.{moduleName}")
+                module = importlib.import_module(f"modules.agents.{moduleName}")
                 
                 # Look for agent class or get_*_agent function
                 agentName = moduleName.split("agent")[-1]
@@ -273,4 +152,4 @@ class AgentRegistry:
 
 # Singleton factory for the agent registry
 def getAgentRegistry():
-    return AgentRegistry.getInstance()
+    return AgentRegistry.getInstance() 

modules/workflow/workflowAgentsRegistry.py

@@ -0,0 +1,10 @@
+"""
+Agent Registry Module.
+Provides a central registry system for all available agents.
+Optimized for the standardized task processing pattern.
+"""
+
+from .agentBase import AgentBase
+from .agentRegistry import AgentRegistry, getAgentRegistry
+
+__all__ = ['AgentBase', 'AgentRegistry', 'getAgentRegistry']

modules/workflow/workflowManager.py

@@ -14,12 +14,12 @@ from datetime import datetime, timedelta
 from typing import Dict, Any, List, Optional, Union, Tuple
 import time
 
-from modules.mimeUtils import isTextMimeType, determineContentEncoding
+from modules.shared.mimeUtils import isTextMimeType, determineContentEncoding
 
 # Required imports
-from modules.workflowAgentsRegistry import getAgentRegistry
-from modules.lucydomInterface import getLucydomInterface as domInterface
-from modules.documentProcessor import getDocumentContents
+from modules.workflow.agentRegistry import getAgentRegistry
+from modules.interfaces.lucydomInterface import getLucydomInterface as domInterface
+from modules.workflow.documentProcessor import getDocumentContents
 
 # Configure logger
 logger = logging.getLogger(__name__)
@@ -461,7 +461,7 @@ JSON_OUTPUT = {{
     "objFinalDocuments": ["label",...], # document label in the format 'filename.ext'
     "objWorkplan": [
         {{
-            "agent": "agent_name",  # Name of an available agent
+            "agent": "agentName",  # Name of an available agent
             "prompt": "Specific instructions to the agent, that he knows what to do with which documents and which output to provide."
             "outputDocuments": [
                 {{