From daddf417be7a0393445aae0d6aa845a9094f8018 Mon Sep 17 00:00:00 2001 From: ValueOn AG Date: Fri, 24 Oct 2025 23:57:17 +0200 Subject: [PATCH] Cleaned all import lists --- app.py | 1 - modules/aicore/aicorePluginAnthropic.py | 2 +- modules/aicore/aicorePluginInternal.py | 2 +- modules/aicore/aicorePluginOpenai.py | 3 +- modules/aicore/aicorePluginPerplexity.py | 3 +- modules/aicore/aicorePluginTavily.py | 1 - modules/connectors/connectorDbJson.py | 8 +- modules/connectors/connectorDbPostgre.py | 33 +-- modules/connectors/connectorVoiceGoogle.py | 3 - modules/datamodels/datamodelAi.py | 2 +- modules/datamodels/datamodelChat.py | 56 ++-- modules/datamodels/datamodelFiles.py | 15 +- modules/datamodels/datamodelNeutralizer.py | 6 +- modules/datamodels/datamodelSecurity.py | 8 +- modules/datamodels/datamodelUam.py | 8 +- modules/datamodels/datamodelUtils.py | 4 +- modules/datamodels/datamodelVoice.py | 7 +- .../chatPlayground/mainChatPlayground.py | 1 - modules/features/syncDelta/mainSyncDelta.py | 2 - modules/interfaces/interfaceAiObjects.py | 14 +- modules/interfaces/interfaceDbAppAccess.py | 2 - modules/interfaces/interfaceDbAppObjects.py | 51 ++-- modules/interfaces/interfaceDbChatAccess.py | 2 +- modules/interfaces/interfaceDbChatObjects.py | 33 +-- .../interfaces/interfaceDbComponentAccess.py | 7 +- .../interfaces/interfaceDbComponentObjects.py | 15 +- modules/interfaces/interfaceVoiceObjects.py | 1 - modules/routes/routeAdmin.py | 1 - modules/routes/routeAttributes.py | 10 +- modules/routes/routeChatPlayground.py | 2 - modules/routes/routeDataConnections.py | 9 +- modules/routes/routeDataFiles.py | 15 +- modules/routes/routeDataMandates.py | 11 +- modules/routes/routeDataPrompts.py | 10 +- modules/routes/routeDataUsers.py | 7 - modules/routes/routeSecurityGoogle.py | 11 +- modules/routes/routeSecurityLocal.py | 6 +- modules/routes/routeSecurityMsft.py | 10 +- modules/routes/routeVoiceGoogle.py | 5 +- modules/routes/routeWorkflows.py | 5 - modules/security/auth.py | 3 - modules/security/tokenManager.py | 1 - modules/security/tokenRefreshMiddleware.py | 1 - modules/security/tokenRefreshService.py | 7 +- modules/services/serviceAi/mainServiceAi.py | 4 +- modules/services/serviceAi/subCoreAi.py | 1 - .../serviceAi/subDocumentGeneration.py | 4 +- .../serviceAi/subDocumentProcessing.py | 2 +- .../services/serviceAi/subSharedAiUtils.py | 1 - modules/services/serviceAi/subWebResearch.py | 3 +- .../services/serviceExtraction/subMerger.py | 2 +- .../serviceGeneration/renderers/registry.py | 1 - .../renderers/rendererCsv.py | 2 - .../renderers/rendererImage.py | 1 - .../renderers/rendererPdf.py | 1 - .../mainServiceNormalization.py | 1 - .../mainServiceSharepoint.py | 2 - .../services/serviceUtils/mainServiceUtils.py | 1 - .../serviceWorkflow/mainServiceWorkflow.py | 1 - modules/shared/attributeUtils.py | 274 ++++-------------- modules/shared/timezoneUtils.py | 18 +- modules/workflows/methods/methodBase.py | 2 - modules/workflows/methods/methodSharepoint.py | 2 - .../adaptive/adaptiveLearningEngine.py | 1 - .../processing/adaptive/contentValidator.py | 1 - .../processing/adaptive/learningEngine.py | 1 - .../workflows/processing/modes/modeReact.py | 1 - .../processing/shared/executionState.py | 1 - .../processing/shared/methodDiscovery.py | 2 - .../processing/shared/placeholderFactory.py | 1 - .../promptGenerationActionsActionplan.py | 1 - .../shared/promptGenerationTaskplan.py | 1 - .../workflows/processing/workflowProcessor.py | 6 +- modules/workflows/workflowManager.py | 2 +- tool_security_encrypt_all_env_files.py | 1 - 75 files changed, 216 insertions(+), 530 deletions(-) diff --git a/app.py b/app.py index 2f7dbaf3..e91a7892 100644 --- a/app.py +++ b/app.py @@ -1,6 +1,5 @@ import os import sys -import asyncio from urllib.parse import quote_plus os.environ["NUMEXPR_MAX_THREADS"] = "12" diff --git a/modules/aicore/aicorePluginAnthropic.py b/modules/aicore/aicorePluginAnthropic.py index 83482d6f..1ac63516 100644 --- a/modules/aicore/aicorePluginAnthropic.py +++ b/modules/aicore/aicorePluginAnthropic.py @@ -1,7 +1,7 @@ import logging import httpx import os -from typing import Dict, Any, List, Union +from typing import Dict, Any, List from fastapi import HTTPException from modules.shared.configuration import APP_CONFIG from modules.aicore.aicoreBase import BaseConnectorAi diff --git a/modules/aicore/aicorePluginInternal.py b/modules/aicore/aicorePluginInternal.py index bb28de50..77f31511 100644 --- a/modules/aicore/aicorePluginInternal.py +++ b/modules/aicore/aicorePluginInternal.py @@ -1,5 +1,5 @@ import logging -from typing import Dict, Any, List, Union +from typing import List from modules.aicore.aicoreBase import BaseConnectorAi from modules.datamodels.datamodelAi import AiModel, PriorityEnum, ProcessingModeEnum, OperationTypeEnum, AiModelCall, AiModelResponse, createOperationTypeRatings diff --git a/modules/aicore/aicorePluginOpenai.py b/modules/aicore/aicorePluginOpenai.py index 49667903..71aa0a95 100644 --- a/modules/aicore/aicorePluginOpenai.py +++ b/modules/aicore/aicorePluginOpenai.py @@ -1,7 +1,6 @@ import logging -import base64 import httpx -from typing import Dict, Any, List, Union +from typing import List from fastapi import HTTPException from modules.shared.configuration import APP_CONFIG from modules.aicore.aicoreBase import BaseConnectorAi diff --git a/modules/aicore/aicorePluginPerplexity.py b/modules/aicore/aicorePluginPerplexity.py index 1ff9abc9..ef1a87bc 100644 --- a/modules/aicore/aicorePluginPerplexity.py +++ b/modules/aicore/aicorePluginPerplexity.py @@ -1,7 +1,6 @@ import logging import httpx -import asyncio -from typing import Dict, Any, List, Union, Optional +from typing import List from fastapi import HTTPException from modules.shared.configuration import APP_CONFIG from modules.aicore.aicoreBase import BaseConnectorAi diff --git a/modules/aicore/aicorePluginTavily.py b/modules/aicore/aicorePluginTavily.py index ee4b40ed..7fa914e5 100644 --- a/modules/aicore/aicorePluginTavily.py +++ b/modules/aicore/aicorePluginTavily.py @@ -8,7 +8,6 @@ from dataclasses import dataclass from typing import Optional, List from tavily import AsyncTavilyClient from modules.shared.configuration import APP_CONFIG -from modules.shared.timezoneUtils import get_utc_timestamp from modules.aicore.aicoreBase import BaseConnectorAi from modules.datamodels.datamodelAi import AiModel, PriorityEnum, ProcessingModeEnum, OperationTypeEnum, AiModelResponse, createOperationTypeRatings diff --git a/modules/connectors/connectorDbJson.py b/modules/connectors/connectorDbJson.py index b0b6b586..999814db 100644 --- a/modules/connectors/connectorDbJson.py +++ b/modules/connectors/connectorDbJson.py @@ -1,14 +1,12 @@ import json import os -from typing import List, Dict, Any, Optional, Union, TypedDict +from typing import List, Dict, Any, Optional, TypedDict import logging -from datetime import datetime import uuid from pydantic import BaseModel import threading import time -from modules.shared.attributeUtils import to_dict from modules.shared.timezoneUtils import get_utc_timestamp logger = logging.getLogger(__name__) @@ -567,7 +565,7 @@ class DatabaseConnector: # If record is a Pydantic model, convert to dict if isinstance(record, BaseModel): - record = to_dict(record) + record = record.model_dump() # Save record self._saveRecord(table, record["id"], record) @@ -582,7 +580,7 @@ class DatabaseConnector: # If record is a Pydantic model, convert to dict if isinstance(record, BaseModel): - record = to_dict(record) + record = record.model_dump() # CRITICAL: Ensure we never modify the ID if "id" in record and str(record["id"]) != recordId: diff --git a/modules/connectors/connectorDbPostgre.py b/modules/connectors/connectorDbPostgre.py index 77441399..ef937a7c 100644 --- a/modules/connectors/connectorDbPostgre.py +++ b/modules/connectors/connectorDbPostgre.py @@ -1,16 +1,11 @@ import psycopg2 import psycopg2.extras -import json -import os import logging from typing import List, Dict, Any, Optional, Union, get_origin, get_args -from datetime import datetime import uuid from pydantic import BaseModel, Field import threading -import time -from modules.shared.attributeUtils import to_dict, ModelMixin from modules.shared.timezoneUtils import get_utc_timestamp from modules.shared.configuration import APP_CONFIG @@ -19,7 +14,7 @@ logger = logging.getLogger(__name__) # No mapping needed - table name = Pydantic model name exactly -class SystemTable(BaseModel, ModelMixin): +class SystemTable(BaseModel): """Data model for system table entries""" table_name: str = Field( @@ -39,22 +34,13 @@ class SystemTable(BaseModel, ModelMixin): def _get_model_fields(model_class) -> Dict[str, str]: """Get all fields from Pydantic model and map to SQL types.""" - # Pydantic v2 uses model_fields instead of __fields__ - if hasattr(model_class, "model_fields"): - model_fields = model_class.model_fields - elif hasattr(model_class, "__fields__"): - model_fields = model_class.__fields__ - else: - return {} + # Pydantic v2 + model_fields = model_class.model_fields fields = {} for field_name, field_info in model_fields.items(): - # Pydantic v2 uses annotation instead of type_ - field_type = ( - field_info.annotation - if hasattr(field_info, "annotation") - else field_info.type_ - ) + # Pydantic v2 + field_type = field_info.annotation # Check for JSONB fields (Dict, List, or complex types) if ( @@ -538,11 +524,8 @@ class DatabaseConnector: # If not valid JSON, convert to JSON string value = json.dumps(value) elif hasattr(value, 'model_dump'): - # Handle Pydantic v2 models + # Handle Pydantic models value = json.dumps(value.model_dump()) - elif hasattr(value, 'dict'): - # Handle Pydantic v1 models - value = json.dumps(value.dict()) else: # Convert other types to JSON value = json.dumps(value) @@ -912,7 +895,7 @@ class DatabaseConnector: """Creates a new record in a table based on Pydantic model class.""" # If record is a Pydantic model, convert to dict if isinstance(record, BaseModel): - record = to_dict(record) + record = record.model_dump() elif isinstance(record, dict): record = record.copy() else: @@ -947,7 +930,7 @@ class DatabaseConnector: # If record is a Pydantic model, convert to dict if isinstance(record, BaseModel): - record = to_dict(record) + record = record.model_dump() elif isinstance(record, dict): record = record.copy() else: diff --git a/modules/connectors/connectorVoiceGoogle.py b/modules/connectors/connectorVoiceGoogle.py index f51b9f2f..2bf4aff3 100644 --- a/modules/connectors/connectorVoiceGoogle.py +++ b/modules/connectors/connectorVoiceGoogle.py @@ -3,12 +3,9 @@ Google Cloud Speech-to-Text and Translation Connector Replaces Azure Speech Services with Google Cloud APIs """ -import os -import io import json import html import logging -import asyncio from typing import Dict, Optional, Any from google.cloud import speech from google.cloud import translate_v2 as translate diff --git a/modules/datamodels/datamodelAi.py b/modules/datamodels/datamodelAi.py index 7643dc79..baec853b 100644 --- a/modules/datamodels/datamodelAi.py +++ b/modules/datamodels/datamodelAi.py @@ -1,4 +1,4 @@ -from typing import Optional, List, Dict, Any, Literal, Callable, TYPE_CHECKING, Tuple +from typing import Optional, List, Dict, Any, Callable, TYPE_CHECKING, Tuple from pydantic import BaseModel, Field from enum import Enum diff --git a/modules/datamodels/datamodelChat.py b/modules/datamodels/datamodelChat.py index c75ea70b..93f8bf62 100644 --- a/modules/datamodels/datamodelChat.py +++ b/modules/datamodels/datamodelChat.py @@ -3,12 +3,12 @@ from typing import List, Dict, Any, Optional from enum import Enum from pydantic import BaseModel, Field -from modules.shared.attributeUtils import register_model_labels, ModelMixin +from modules.shared.attributeUtils import register_model_labels from modules.shared.timezoneUtils import get_utc_timestamp import uuid -class ChatStat(BaseModel, ModelMixin): +class ChatStat(BaseModel): id: str = Field( default_factory=lambda: str(uuid.uuid4()), description="Primary key" ) @@ -43,7 +43,7 @@ register_model_labels( ) -class ChatLog(BaseModel, ModelMixin): +class ChatLog(BaseModel): id: str = Field( default_factory=lambda: str(uuid.uuid4()), description="Primary key" ) @@ -79,7 +79,7 @@ register_model_labels( ) -class ChatDocument(BaseModel, ModelMixin): +class ChatDocument(BaseModel): id: str = Field( default_factory=lambda: str(uuid.uuid4()), description="Primary key" ) @@ -114,7 +114,7 @@ register_model_labels( ) -class ContentMetadata(BaseModel, ModelMixin): +class ContentMetadata(BaseModel): size: int = Field(description="Content size in bytes") pages: Optional[int] = Field( None, description="Number of pages for multi-page content" @@ -151,7 +151,7 @@ register_model_labels( ) -class ContentItem(BaseModel, ModelMixin): +class ContentItem(BaseModel): label: str = Field(description="Content label") data: str = Field(description="Extracted text content") metadata: ContentMetadata = Field(description="Content metadata") @@ -168,7 +168,7 @@ register_model_labels( ) -class ChatContentExtracted(BaseModel, ModelMixin): +class ChatContentExtracted(BaseModel): id: str = Field(description="Reference to source ChatDocument") contents: List[ContentItem] = Field( default_factory=list, description="List of content items" @@ -185,7 +185,7 @@ register_model_labels( ) -class ChatMessage(BaseModel, ModelMixin): +class ChatMessage(BaseModel): id: str = Field( default_factory=lambda: str(uuid.uuid4()), description="Primary key" ) @@ -263,7 +263,7 @@ register_model_labels( ) -class ChatWorkflow(BaseModel, ModelMixin): +class ChatWorkflow(BaseModel): id: str = Field( default_factory=lambda: str(uuid.uuid4()), description="Primary key", @@ -420,7 +420,7 @@ register_model_labels( ) -class UserInputRequest(BaseModel, ModelMixin): +class UserInputRequest(BaseModel): prompt: str = Field(description="Prompt for the user") listFileId: List[str] = Field(default_factory=list, description="List of file IDs") userLanguage: str = Field(default="en", description="User's preferred language") @@ -437,7 +437,7 @@ register_model_labels( ) -class ActionDocument(BaseModel, ModelMixin): +class ActionDocument(BaseModel): """Clear document structure for action results""" documentName: str = Field(description="Name of the document") @@ -456,7 +456,7 @@ register_model_labels( ) -class ActionResult(BaseModel, ModelMixin): +class ActionResult(BaseModel): """Clean action result with documents as primary output IMPORTANT: Action methods should NOT set resultLabel in their return value. @@ -497,7 +497,7 @@ register_model_labels( ) -class ActionSelection(BaseModel, ModelMixin): +class ActionSelection(BaseModel): method: str = Field(description="Method to execute (e.g., web, document, ai)") name: str = Field( description="Action name within the method (e.g., search, extract)" @@ -514,7 +514,7 @@ register_model_labels( ) -class ActionParameters(BaseModel, ModelMixin): +class ActionParameters(BaseModel): parameters: Dict[str, Any] = Field( default_factory=dict, description="Parameters to execute the selected action" ) @@ -529,7 +529,7 @@ register_model_labels( ) -class ObservationPreview(BaseModel, ModelMixin): +class ObservationPreview(BaseModel): name: str = Field(description="Document name or URL label") mime: str = Field(description="MIME type or kind") snippet: str = Field(description="Short snippet or summary") @@ -546,7 +546,7 @@ register_model_labels( ) -class Observation(BaseModel, ModelMixin): +class Observation(BaseModel): success: bool = Field(description="Action execution success flag") resultLabel: str = Field(description="Deterministic label for produced documents") documentsCount: int = Field(description="Number of produced documents") @@ -592,7 +592,7 @@ register_model_labels( ) -class DocumentExchange(BaseModel, ModelMixin): +class DocumentExchange(BaseModel): documentsLabel: str = Field(description="Label for the set of documents") documents: List[str] = Field( default_factory=list, description="List of document references" @@ -609,7 +609,7 @@ register_model_labels( ) -class ActionItem(BaseModel, ModelMixin): +class ActionItem(BaseModel): id: str = Field(..., description="Action ID") execMethod: str = Field(..., description="Method to execute") execAction: str = Field(..., description="Action to perform") @@ -675,7 +675,7 @@ register_model_labels( ) -class TaskResult(BaseModel, ModelMixin): +class TaskResult(BaseModel): taskId: str = Field(..., description="Task ID") status: TaskStatus = Field(default=TaskStatus.PENDING, description="Task status") success: bool = Field(..., description="Whether the task was successful") @@ -696,7 +696,7 @@ register_model_labels( ) -class TaskItem(BaseModel, ModelMixin): +class TaskItem(BaseModel): id: str = Field(..., description="Task ID") workflowId: str = Field(..., description="Workflow ID") userInput: str = Field(..., description="User input that triggered the task") @@ -747,7 +747,7 @@ register_model_labels( ) -class TaskStep(BaseModel, ModelMixin): +class TaskStep(BaseModel): id: str objective: str dependencies: Optional[list[str]] = Field(default_factory=list) @@ -775,7 +775,7 @@ register_model_labels( ) -class TaskHandover(BaseModel, ModelMixin): +class TaskHandover(BaseModel): taskId: str = Field(description="Target task ID") sourceTask: Optional[str] = Field(None, description="Source task ID") inputDocuments: List[DocumentExchange] = Field( @@ -824,7 +824,7 @@ register_model_labels( ) -class TaskContext(BaseModel, ModelMixin): +class TaskContext(BaseModel): task_step: TaskStep workflow: Optional["ChatWorkflow"] = None workflow_id: Optional[str] = None @@ -856,7 +856,7 @@ class TaskContext(BaseModel, ModelMixin): self.improvements.append(improvement) -class ReviewContext(BaseModel, ModelMixin): +class ReviewContext(BaseModel): task_step: TaskStep task_actions: Optional[list] = Field(default_factory=list) action_results: Optional[list] = Field(default_factory=list) @@ -865,7 +865,7 @@ class ReviewContext(BaseModel, ModelMixin): previous_results: Optional[list[str]] = Field(default_factory=list) -class ReviewResult(BaseModel, ModelMixin): +class ReviewResult(BaseModel): status: str reason: Optional[str] = None improvements: Optional[list[str]] = Field(default_factory=list) @@ -896,7 +896,7 @@ register_model_labels( ) -class TaskPlan(BaseModel, ModelMixin): +class TaskPlan(BaseModel): overview: str tasks: list[TaskStep] userMessage: Optional[str] = Field( @@ -918,7 +918,7 @@ register_model_labels( TaskContext.update_forward_refs() -class PromptPlaceholder(BaseModel, ModelMixin): +class PromptPlaceholder(BaseModel): label: str content: str summaryAllowed: bool = Field( @@ -938,7 +938,7 @@ register_model_labels( ) -class PromptBundle(BaseModel, ModelMixin): +class PromptBundle(BaseModel): prompt: str placeholders: List[PromptPlaceholder] = Field(default_factory=list) diff --git a/modules/datamodels/datamodelFiles.py b/modules/datamodels/datamodelFiles.py index 41cb0cb6..6dcb3a0e 100644 --- a/modules/datamodels/datamodelFiles.py +++ b/modules/datamodels/datamodelFiles.py @@ -2,13 +2,13 @@ from typing import Dict, Any, Optional, Union from pydantic import BaseModel, Field -from modules.shared.attributeUtils import register_model_labels, ModelMixin +from modules.shared.attributeUtils import register_model_labels from modules.shared.timezoneUtils import get_utc_timestamp import uuid import base64 -class FileItem(BaseModel, ModelMixin): +class FileItem(BaseModel): id: str = Field(default_factory=lambda: str(uuid.uuid4()), description="Primary key", frontend_type="text", frontend_readonly=True, frontend_required=False) mandateId: str = Field(description="ID of the mandate this file belongs to", frontend_type="text", frontend_readonly=True, frontend_required=False) fileName: str = Field(description="Name of the file", frontend_type="text", frontend_readonly=False, frontend_required=True) @@ -17,8 +17,6 @@ class FileItem(BaseModel, ModelMixin): fileSize: int = Field(description="Size of the file in bytes", frontend_type="integer", frontend_readonly=True, frontend_required=False) creationDate: float = Field(default_factory=get_utc_timestamp, description="Date when the file was created (UTC timestamp in seconds)", frontend_type="timestamp", frontend_readonly=True, frontend_required=False) - def to_dict(self) -> Dict[str, Any]: - return super().to_dict() register_model_labels( "FileItem", {"en": "File Item", "fr": "Élément de fichier"}, @@ -33,7 +31,7 @@ register_model_labels( }, ) -class FilePreview(BaseModel, ModelMixin): +class FilePreview(BaseModel): content: Union[str, bytes] = Field(description="File content (text or binary)") mimeType: str = Field(description="MIME type of the file") fileName: str = Field(description="Original fileName") @@ -41,8 +39,9 @@ class FilePreview(BaseModel, ModelMixin): encoding: Optional[str] = Field(None, description="Text encoding if content is text") size: int = Field(description="Size of the content in bytes") - def to_dict(self) -> Dict[str, Any]: - data = super().to_dict() + def toDictWithBase64Encoding(self) -> Dict[str, Any]: + """Convert to dictionary with base64 encoding for binary content.""" + data = self.model_dump() if isinstance(data.get("content"), bytes): data["content"] = base64.b64encode(data["content"]).decode("utf-8") return data @@ -59,7 +58,7 @@ register_model_labels( }, ) -class FileData(BaseModel, ModelMixin): +class FileData(BaseModel): id: str = Field(default_factory=lambda: str(uuid.uuid4()), description="Primary key") data: str = Field(description="File data content") base64Encoded: bool = Field(description="Whether the data is base64 encoded") diff --git a/modules/datamodels/datamodelNeutralizer.py b/modules/datamodels/datamodelNeutralizer.py index 998cb17f..93f751e5 100644 --- a/modules/datamodels/datamodelNeutralizer.py +++ b/modules/datamodels/datamodelNeutralizer.py @@ -3,10 +3,10 @@ import uuid from typing import Optional from pydantic import BaseModel, Field -from modules.shared.attributeUtils import register_model_labels, ModelMixin +from modules.shared.attributeUtils import register_model_labels -class DataNeutraliserConfig(BaseModel, ModelMixin): +class DataNeutraliserConfig(BaseModel): id: str = Field(default_factory=lambda: str(uuid.uuid4()), description="Unique ID of the configuration", frontend_type="text", frontend_readonly=True, frontend_required=False) mandateId: str = Field(description="ID of the mandate this configuration belongs to", frontend_type="text", frontend_readonly=True, frontend_required=True) userId: str = Field(description="ID of the user who created this configuration", frontend_type="text", frontend_readonly=True, frontend_required=True) @@ -28,7 +28,7 @@ register_model_labels( }, ) -class DataNeutralizerAttributes(BaseModel, ModelMixin): +class DataNeutralizerAttributes(BaseModel): id: str = Field(default_factory=lambda: str(uuid.uuid4()), description="Unique ID of the attribute mapping (used as UID in neutralized files)", frontend_type="text", frontend_readonly=True, frontend_required=False) mandateId: str = Field(description="ID of the mandate this attribute belongs to", frontend_type="text", frontend_readonly=True, frontend_required=True) userId: str = Field(description="ID of the user who created this attribute", frontend_type="text", frontend_readonly=True, frontend_required=True) diff --git a/modules/datamodels/datamodelSecurity.py b/modules/datamodels/datamodelSecurity.py index 6184d236..cb629a01 100644 --- a/modules/datamodels/datamodelSecurity.py +++ b/modules/datamodels/datamodelSecurity.py @@ -1,8 +1,8 @@ """Security models: Token and AuthEvent.""" from typing import Optional -from pydantic import BaseModel, Field, ConfigDict -from modules.shared.attributeUtils import register_model_labels, ModelMixin +from pydantic import BaseModel, Field +from modules.shared.attributeUtils import register_model_labels from modules.shared.timezoneUtils import get_utc_timestamp from .datamodelUam import AuthAuthority from enum import Enum @@ -14,7 +14,7 @@ class TokenStatus(str, Enum): REVOKED = "revoked" -class Token(BaseModel, ModelMixin): +class Token(BaseModel): id: Optional[str] = None userId: str authority: AuthAuthority @@ -74,7 +74,7 @@ register_model_labels( ) -class AuthEvent(BaseModel, ModelMixin): +class AuthEvent(BaseModel): id: str = Field( default_factory=lambda: str(uuid.uuid4()), description="Unique ID of the auth event", diff --git a/modules/datamodels/datamodelUam.py b/modules/datamodels/datamodelUam.py index 8bd24d8c..747bfc27 100644 --- a/modules/datamodels/datamodelUam.py +++ b/modules/datamodels/datamodelUam.py @@ -4,7 +4,7 @@ import uuid from typing import Optional from enum import Enum from pydantic import BaseModel, Field, EmailStr -from modules.shared.attributeUtils import register_model_labels, ModelMixin +from modules.shared.attributeUtils import register_model_labels from modules.shared.timezoneUtils import get_utc_timestamp @@ -24,7 +24,7 @@ class ConnectionStatus(str, Enum): REVOKED = "revoked" PENDING = "pending" -class Mandate(BaseModel, ModelMixin): +class Mandate(BaseModel): id: str = Field(default_factory=lambda: str(uuid.uuid4()), description="Unique ID of the mandate", frontend_type="text", frontend_readonly=True, frontend_required=False) name: str = Field(description="Name of the mandate", frontend_type="text", frontend_readonly=False, frontend_required=True) language: str = Field(default="en", description="Default language of the mandate", frontend_type="select", frontend_readonly=False, frontend_required=True, frontend_options=[ @@ -45,7 +45,7 @@ register_model_labels( }, ) -class UserConnection(BaseModel, ModelMixin): +class UserConnection(BaseModel): id: str = Field(default_factory=lambda: str(uuid.uuid4()), description="Unique ID of the connection", frontend_type="text", frontend_readonly=True, frontend_required=False) userId: str = Field(description="ID of the user this connection belongs to", frontend_type="text", frontend_readonly=True, frontend_required=False) authority: AuthAuthority = Field(description="Authentication authority", frontend_type="select", frontend_readonly=True, frontend_required=False, frontend_options=[ @@ -90,7 +90,7 @@ register_model_labels( }, ) -class User(BaseModel, ModelMixin): +class User(BaseModel): id: str = Field(default_factory=lambda: str(uuid.uuid4()), description="Unique ID of the user", frontend_type="text", frontend_readonly=True, frontend_required=False) username: str = Field(description="Username for login", frontend_type="text", frontend_readonly=False, frontend_required=True) email: Optional[EmailStr] = Field(None, description="Email address of the user", frontend_type="email", frontend_readonly=False, frontend_required=True) diff --git a/modules/datamodels/datamodelUtils.py b/modules/datamodels/datamodelUtils.py index ccd01c04..c928cd47 100644 --- a/modules/datamodels/datamodelUtils.py +++ b/modules/datamodels/datamodelUtils.py @@ -1,11 +1,11 @@ """Utility datamodels: Prompt.""" from pydantic import BaseModel, Field -from modules.shared.attributeUtils import register_model_labels, ModelMixin +from modules.shared.attributeUtils import register_model_labels import uuid -class Prompt(BaseModel, ModelMixin): +class Prompt(BaseModel): id: str = Field(default_factory=lambda: str(uuid.uuid4()), description="Primary key", frontend_type="text", frontend_readonly=True, frontend_required=False) mandateId: str = Field(description="ID of the mandate this prompt belongs to", frontend_type="text", frontend_readonly=True, frontend_required=False) content: str = Field(description="Content of the prompt", frontend_type="textarea", frontend_readonly=False, frontend_required=True) diff --git a/modules/datamodels/datamodelVoice.py b/modules/datamodels/datamodelVoice.py index 274a507b..8be43b58 100644 --- a/modules/datamodels/datamodelVoice.py +++ b/modules/datamodels/datamodelVoice.py @@ -1,13 +1,12 @@ """Voice settings datamodel.""" -from typing import Dict, Any, Optional from pydantic import BaseModel, Field -from modules.shared.attributeUtils import register_model_labels, ModelMixin +from modules.shared.attributeUtils import register_model_labels from modules.shared.timezoneUtils import get_utc_timestamp import uuid -class VoiceSettings(BaseModel, ModelMixin): +class VoiceSettings(BaseModel): id: str = Field(default_factory=lambda: str(uuid.uuid4()), description="Primary key", frontend_type="text", frontend_readonly=True, frontend_required=False) userId: str = Field(description="ID of the user these settings belong to", frontend_type="text", frontend_readonly=True, frontend_required=True) mandateId: str = Field(description="ID of the mandate these settings belong to", frontend_type="text", frontend_readonly=True, frontend_required=False) @@ -19,8 +18,6 @@ class VoiceSettings(BaseModel, ModelMixin): creationDate: float = Field(default_factory=get_utc_timestamp, description="Date when the settings were created (UTC timestamp in seconds)", frontend_type="timestamp", frontend_readonly=True, frontend_required=False) lastModified: float = Field(default_factory=get_utc_timestamp, description="Date when the settings were last modified (UTC timestamp in seconds)", frontend_type="timestamp", frontend_readonly=True, frontend_required=False) - def to_dict(self) -> Dict[str, Any]: - return super().to_dict() register_model_labels( "VoiceSettings", diff --git a/modules/features/chatPlayground/mainChatPlayground.py b/modules/features/chatPlayground/mainChatPlayground.py index 3a4e015f..54d50cb4 100644 --- a/modules/features/chatPlayground/mainChatPlayground.py +++ b/modules/features/chatPlayground/mainChatPlayground.py @@ -1,5 +1,4 @@ import logging -import asyncio from typing import Optional from modules.datamodels.datamodelUam import User diff --git a/modules/features/syncDelta/mainSyncDelta.py b/modules/features/syncDelta/mainSyncDelta.py index e3d363a3..fa8c1f93 100644 --- a/modules/features/syncDelta/mainSyncDelta.py +++ b/modules/features/syncDelta/mainSyncDelta.py @@ -5,7 +5,6 @@ This module handles the synchronization of tickets to SharePoint using the new Graph API-based connector architecture. """ -import asyncio import logging import os import io @@ -13,7 +12,6 @@ import pandas as pd import csv as csv_module from io import StringIO, BytesIO from datetime import datetime, UTC -from typing import Dict, Any, List, Optional from modules.services import getInterface as getServices logger = logging.getLogger(__name__) diff --git a/modules/interfaces/interfaceAiObjects.py b/modules/interfaces/interfaceAiObjects.py index 51fa90af..57c4d2b4 100644 --- a/modules/interfaces/interfaceAiObjects.py +++ b/modules/interfaces/interfaceAiObjects.py @@ -1,5 +1,6 @@ import logging import asyncio +import uuid from typing import Dict, Any, List, Union, Tuple, Optional from dataclasses import dataclass import time @@ -17,6 +18,7 @@ from modules.datamodels.datamodelAi import ( AiModelCall, AiModelResponse, ) +from modules.datamodels.datamodelExtraction import ContentPart # Dynamic model registry - models are now loaded from connectors via aicore system @@ -299,7 +301,7 @@ class AiObjects: for i, result in enumerate(partResults): if result.content: content_part = ContentPart( - id=makeId(), + id=str(uuid.uuid4()), parentId=None, label=f"ai_result_{i}", typeGroup="text", # Default to text for AI results @@ -339,14 +341,12 @@ class AiObjects: return "" # Convert AiCallResponse results to ContentParts for merging - from modules.datamodels.datamodelExtraction import ContentPart - from modules.services.serviceExtraction.subUtils import makeId content_parts = [] for i, result in enumerate(chunkResults): if result.content: content_part = ContentPart( - id=makeId(), + id=str(uuid.uuid4()), parentId=None, label=f"chunk_result_{i}", typeGroup="text", # Default to text for AI results @@ -1075,15 +1075,15 @@ Format your response in a clear, professional manner that would be helpful for s """List available models, optionally filtered by connector type.""" models = modelRegistry.getAvailableModels() if connectorType: - return [model.dict() for model in models if model.connectorType == connectorType] - return [model.dict() for model in models] + return [model.model_dump() for model in models if model.connectorType == connectorType] + return [model.model_dump() for model in models] async def getModelInfo(self, modelName: str) -> Dict[str, Any]: """Get information about a specific model.""" model = modelRegistry.getModel(modelName) if not model: raise ValueError(f"Model {modelName} not found") - return model.dict() + return model.model_dump() async def getModelsByTag(self, tag: str) -> List[str]: diff --git a/modules/interfaces/interfaceDbAppAccess.py b/modules/interfaces/interfaceDbAppAccess.py index 5520152a..1bb9126c 100644 --- a/modules/interfaces/interfaceDbAppAccess.py +++ b/modules/interfaces/interfaceDbAppAccess.py @@ -4,10 +4,8 @@ Access control for the Application. import logging from typing import Dict, Any, List, Optional -from datetime import datetime from modules.datamodels.datamodelUam import UserPrivilege, User, UserInDB, Mandate from modules.datamodels.datamodelSecurity import AuthEvent -from modules.shared.timezoneUtils import get_utc_now # Configure logger logger = logging.getLogger(__name__) diff --git a/modules/interfaces/interfaceDbAppObjects.py b/modules/interfaces/interfaceDbAppObjects.py index 2151fae9..e37c98f8 100644 --- a/modules/interfaces/interfaceDbAppObjects.py +++ b/modules/interfaces/interfaceDbAppObjects.py @@ -3,19 +3,14 @@ Interface to the Gateway system. Manages users and mandates for authentication. """ -from datetime import datetime, timedelta, UTC -import os import logging -from typing import Dict, Any, List, Optional, Union -import importlib -import json +from typing import Dict, Any, List, Optional from passlib.context import CryptContext import uuid -import re from modules.connectors.connectorDbPostgre import DatabaseConnector from modules.shared.configuration import APP_CONFIG -from modules.shared.timezoneUtils import get_utc_now, get_utc_timestamp +from modules.shared.timezoneUtils import get_utc_timestamp from modules.interfaces.interfaceDbAppAccess import AppAccess from modules.datamodels.datamodelUam import ( User, @@ -259,7 +254,7 @@ class AppObjects: filteredUsers = self._uam(UserInDB, users) # Convert to User models - return [User.from_dict(user) for user in filteredUsers] + return [User(**user) for user in filteredUsers] def getUserByUsername(self, username: str) -> Optional[User]: """Returns a user by username.""" @@ -272,7 +267,7 @@ class AppObjects: # Find user by username for user_dict in users: if user_dict.get("username") == username: - return User.from_dict(user_dict) + return User(**user_dict) logger.info(f"No user found with username {username}") return None @@ -295,7 +290,7 @@ class AppObjects: # Apply access control filteredUsers = self._uam(UserInDB, [user_dict]) if filteredUsers: - return User.from_dict(filteredUsers[0]) + return User(**filteredUsers[0]) return None return None @@ -398,7 +393,7 @@ class AppObjects: # Clear cache to ensure fresh data (already done above) - return User.from_dict(createdUser[0]) + return User(**createdUser[0]) except ValueError as e: logger.error(f"Error creating user: {str(e)}") @@ -416,9 +411,9 @@ class AppObjects: raise ValueError(f"User {userId} not found") # Update user data using model - updatedData = user.to_dict() + updatedData = user.model_dump() updatedData.update(updateData) - updatedUser = User.from_dict(updatedData) + updatedUser = User(**updatedData) # Update user record self.db.recordModify(UserInDB, userId, updatedUser) @@ -647,7 +642,7 @@ class AppObjects: """Returns all mandates based on user access level.""" allMandates = self.db.getRecordset(Mandate) filteredMandates = self._uam(Mandate, allMandates) - return [Mandate.from_dict(mandate) for mandate in filteredMandates] + return [Mandate(**mandate) for mandate in filteredMandates] def getMandate(self, mandateId: str) -> Optional[Mandate]: """Returns a mandate by ID if user has access.""" @@ -659,7 +654,7 @@ class AppObjects: if not filteredMandates: return None - return Mandate.from_dict(filteredMandates[0]) + return Mandate(**filteredMandates[0]) def createMandate(self, name: str, language: str = "en") -> Mandate: """Creates a new mandate if user has permission.""" @@ -674,7 +669,7 @@ class AppObjects: if not createdRecord or not createdRecord.get("id"): raise ValueError("Failed to create mandate record") - return Mandate.from_dict(createdRecord) + return Mandate(**createdRecord) def updateMandate(self, mandateId: str, updateData: Dict[str, Any]) -> Mandate: """Updates a mandate if user has access.""" @@ -689,9 +684,9 @@ class AppObjects: raise ValueError(f"Mandate {mandateId} not found") # Update mandate data using model - updatedData = mandate.to_dict() + updatedData = mandate.model_dump() updatedData.update(updateData) - updatedMandate = Mandate.from_dict(updatedData) + updatedMandate = Mandate(**updatedData) # Update mandate record self.db.recordModify(Mandate, mandateId, updatedMandate) @@ -793,10 +788,10 @@ class AppObjects: # Continue with saving the new token even if deletion fails # Convert to dict and ensure all fields are properly set - token_dict = token.to_dict() + token_dict = token.model_dump() # Ensure userId is set to current user # Convert to dict and ensure all fields are properly set - token_dict = token.to_dict() + token_dict = token.model_dump() # Ensure userId is set to current user token_dict["userId"] = self.currentUser.id @@ -830,7 +825,7 @@ class AppObjects: token.createdAt = get_utc_timestamp() # Convert to dict and ensure all fields are properly set - token_dict = token.to_dict() + token_dict = token.model_dump() # Ensure userId is set to current user token_dict["userId"] = self.currentUser.id @@ -1087,7 +1082,7 @@ class AppObjects: if not filtered_configs: return None - return DataNeutraliserConfig.from_dict(filtered_configs[0]) + return DataNeutraliserConfig(**filtered_configs[0]) except Exception as e: logger.error(f"Error getting neutralization config: {str(e)}") @@ -1103,11 +1098,11 @@ class AppObjects: if existing_config: # Update existing config - update_data = existing_config.to_dict() + update_data = existing_config.model_dump() update_data.update(config_data) update_data["updatedAt"] = get_utc_timestamp() - updated_config = DataNeutraliserConfig.from_dict(update_data) + updated_config = DataNeutraliserConfig(**update_data) self.db.recordModify( DataNeutraliserConfig, existing_config.id, updated_config ) @@ -1118,10 +1113,10 @@ class AppObjects: config_data["mandateId"] = self.mandateId config_data["userId"] = self.userId - new_config = DataNeutraliserConfig.from_dict(config_data) + new_config = DataNeutraliserConfig(**config_data) created_record = self.db.recordCreate(DataNeutraliserConfig, new_config) - return DataNeutraliserConfig.from_dict(created_record) + return DataNeutraliserConfig(**created_record) except Exception as e: logger.error(f"Error creating/updating neutralization config: {str(e)}") @@ -1142,7 +1137,7 @@ class AppObjects: filtered_attributes = self._uam(DataNeutralizerAttributes, attributes) return [ - DataNeutralizerAttributes.from_dict(attr) + DataNeutralizerAttributes(**attr) for attr in filtered_attributes ] @@ -1217,7 +1212,7 @@ def getRootInterface() -> AppObjects: # Convert to User model (use helper compatible with our models) user_data = users[0] - rootUser = User.from_dict(user_data) + rootUser = User(**user_data) # Create root interface with the root user _rootAppObjects = AppObjects(rootUser) diff --git a/modules/interfaces/interfaceDbChatAccess.py b/modules/interfaces/interfaceDbChatAccess.py index f37c8e96..3facab4d 100644 --- a/modules/interfaces/interfaceDbChatAccess.py +++ b/modules/interfaces/interfaceDbChatAccess.py @@ -5,7 +5,7 @@ Handles user access management and permission checks. from typing import Dict, Any, List, Optional from modules.datamodels.datamodelUam import User, UserPrivilege -from modules.datamodels.datamodelChat import ChatWorkflow, ChatMessage, ChatLog, ChatStat, ChatDocument +from modules.datamodels.datamodelChat import ChatWorkflow class ChatAccess: """ diff --git a/modules/interfaces/interfaceDbChatObjects.py b/modules/interfaces/interfaceDbChatObjects.py index 31db88d8..92dcccf6 100644 --- a/modules/interfaces/interfaceDbChatObjects.py +++ b/modules/interfaces/interfaceDbChatObjects.py @@ -81,23 +81,14 @@ class ChatObjects: object_fields = {} # Get field information from the Pydantic model - model_fields = {} - if hasattr(model_class, '__fields__'): - model_fields = model_class.__fields__ - elif hasattr(model_class, 'model_fields'): - model_fields = model_class.model_fields + model_fields = model_class.model_fields for field_name, value in data.items(): # Check if this field should be stored as JSONB in the database if field_name in model_fields: field_info = model_fields[field_name] - # Handle both Pydantic v1 and v2 - if hasattr(field_info, 'type_'): - field_type = field_info.type_ # Pydantic v1 - elif hasattr(field_info, 'annotation'): - field_type = field_info.annotation # Pydantic v2 - else: - field_type = type(value) # Fallback + # Pydantic v2 only + field_type = field_info.annotation # Always route relational/object fields to object_fields for separate handling if field_name in ['documents', 'stats']: @@ -511,13 +502,8 @@ class ChatObjects: # Create documents in normalized documents table created_documents = [] for doc_data in documents_to_create: - # Convert to dict if it's a Pydantic object - if hasattr(doc_data, 'model_dump'): - doc_dict = doc_data.model_dump() # Pydantic v2 - elif hasattr(doc_data, 'to_dict'): - doc_dict = doc_data.to_dict() - else: - doc_dict = doc_data + # Use the document data directly + doc_dict = doc_data doc_dict["messageId"] = createdMessage["id"] created_doc = self.createDocument(doc_dict) @@ -626,14 +612,7 @@ class ChatObjects: documents_data = object_fields['documents'] try: for doc_data in documents_data: - if hasattr(doc_data, 'model_dump'): - doc_dict = doc_data.model_dump() # Pydantic v2 - elif hasattr(doc_data, 'dict'): - doc_dict = doc_data.dict() # Pydantic v1 - elif hasattr(doc_data, 'to_dict'): - doc_dict = doc_data.to_dict() - else: - doc_dict = doc_data + doc_dict = doc_data doc_dict["messageId"] = messageId self.createDocument(doc_dict) except Exception as e: diff --git a/modules/interfaces/interfaceDbComponentAccess.py b/modules/interfaces/interfaceDbComponentAccess.py index 3d259789..36c3cfff 100644 --- a/modules/interfaces/interfaceDbComponentAccess.py +++ b/modules/interfaces/interfaceDbComponentAccess.py @@ -5,11 +5,10 @@ Handles user access management and permission checks. import logging from typing import Dict, Any, List, Optional -from modules.datamodels.datamodelUam import User, UserInDB +from modules.datamodels.datamodelUam import User from modules.datamodels.datamodelUtils import Prompt -from modules.datamodels.datamodelFiles import FileItem, FileData -from modules.datamodels.datamodelVoice import VoiceSettings -from modules.datamodels.datamodelChat import ChatWorkflow, ChatMessage, ChatLog +from modules.datamodels.datamodelFiles import FileItem +from modules.datamodels.datamodelChat import ChatWorkflow # Configure logger logger = logging.getLogger(__name__) diff --git a/modules/interfaces/interfaceDbComponentObjects.py b/modules/interfaces/interfaceDbComponentObjects.py index 9df9364b..d2a74e69 100644 --- a/modules/interfaces/interfaceDbComponentObjects.py +++ b/modules/interfaces/interfaceDbComponentObjects.py @@ -7,8 +7,7 @@ import os import logging import base64 import hashlib -from datetime import datetime, UTC -from typing import Dict, Any, List, Optional, Union +from typing import Dict, Any, List, Optional from modules.connectors.connectorDbPostgre import DatabaseConnector from modules.interfaces.interfaceDbComponentAccess import ComponentAccess @@ -263,7 +262,7 @@ class ComponentObjects: filteredPrompts = self._uam(Prompt, allPrompts) # Convert to Prompt objects - return [Prompt.from_dict(prompt) for prompt in filteredPrompts] + return [Prompt(**prompt) for prompt in filteredPrompts] except Exception as e: logger.error(f"Error getting prompts: {str(e)}") @@ -276,7 +275,7 @@ class ComponentObjects: return None filteredPrompts = self._uam(Prompt, prompts) - return Prompt.from_dict(filteredPrompts[0]) if filteredPrompts else None + return Prompt(**filteredPrompts[0]) if filteredPrompts else None def createPrompt(self, promptData: Dict[str, Any]) -> Dict[str, Any]: """Creates a new prompt if user has permission.""" @@ -308,7 +307,7 @@ class ComponentObjects: if not updatedPrompt: raise ValueError("Failed to retrieve updated prompt") - return updatedPrompt.to_dict() + return updatedPrompt.model_dump() except Exception as e: logger.error(f"Error updating prompt: {str(e)}") @@ -886,7 +885,7 @@ class ComponentObjects: if not settings_data.get("lastModified"): settings_data["lastModified"] = get_utc_timestamp() - return VoiceSettings.from_dict(settings_data) + return VoiceSettings(**settings_data) except Exception as e: logger.error(f"Error getting voice settings: {str(e)}") @@ -945,7 +944,7 @@ class ComponentObjects: raise ValueError("Failed to retrieve updated voice settings") logger.info(f"Updated voice settings for user {userId}") - return updatedSettings.to_dict() + return updatedSettings.model_dump() except Exception as e: logger.error(f"Error updating voice settings: {str(e)}") @@ -997,7 +996,7 @@ class ComponentObjects: } createdRecord = self.createVoiceSettings(defaultSettings) - return VoiceSettings.from_dict(createdRecord) + return VoiceSettings(**createdRecord) except Exception as e: logger.error(f"Error getting or creating voice settings: {str(e)}") diff --git a/modules/interfaces/interfaceVoiceObjects.py b/modules/interfaces/interfaceVoiceObjects.py index d7047a5a..a1d0b96c 100644 --- a/modules/interfaces/interfaceVoiceObjects.py +++ b/modules/interfaces/interfaceVoiceObjects.py @@ -6,7 +6,6 @@ Handles voice operations including speech-to-text, text-to-speech, and translati import logging from typing import Dict, Any, Optional, List -from datetime import datetime, UTC from modules.connectors.connectorVoiceGoogle import ConnectorGoogleSpeech from modules.datamodels.datamodelVoice import VoiceSettings diff --git a/modules/routes/routeAdmin.py b/modules/routes/routeAdmin.py index 992ad596..5eec28d6 100644 --- a/modules/routes/routeAdmin.py +++ b/modules/routes/routeAdmin.py @@ -6,7 +6,6 @@ import logging from pathlib import Path as FilePath from typing import Dict, Any, List from fastapi import HTTPException, status -from datetime import datetime from modules.shared.configuration import APP_CONFIG from modules.security.auth import limiter, getCurrentUser diff --git a/modules/routes/routeAttributes.py b/modules/routes/routeAttributes.py index cf6095c9..5ada9a4e 100644 --- a/modules/routes/routeAttributes.py +++ b/modules/routes/routeAttributes.py @@ -1,17 +1,11 @@ -from fastapi import APIRouter, HTTPException, Depends, Path, Response, Request -from typing import List, Dict, Any +from fastapi import APIRouter, HTTPException, Path, Response, Request from fastapi import status -import inspect -import importlib -import os -from pydantic import BaseModel import logging # Import auth module -from modules.security.auth import limiter, getCurrentUser +from modules.security.auth import limiter # Import the attribute definition and helper functions -from modules.datamodels.datamodelUam import User from modules.shared.attributeUtils import getModelClasses, getModelAttributeDefinitions, AttributeResponse, AttributeDefinition # Configure logger diff --git a/modules/routes/routeChatPlayground.py b/modules/routes/routeChatPlayground.py index 8957f632..300b415b 100644 --- a/modules/routes/routeChatPlayground.py +++ b/modules/routes/routeChatPlayground.py @@ -6,14 +6,12 @@ Implements the endpoints for chat playground workflow management. import logging from typing import Optional, Dict, Any from fastapi import APIRouter, HTTPException, Depends, Body, Path, Query, Request -from datetime import datetime # Import auth modules from modules.security.auth import limiter, getCurrentUser # Import interfaces import modules.interfaces.interfaceDbChatObjects as interfaceDbChatObjects -from modules.interfaces.interfaceDbChatObjects import getInterface # Import models from modules.datamodels.datamodelChat import ChatWorkflow, UserInputRequest diff --git a/modules/routes/routeDataConnections.py b/modules/routes/routeDataConnections.py index 0ca6b492..1c49fa13 100644 --- a/modules/routes/routeDataConnections.py +++ b/modules/routes/routeDataConnections.py @@ -11,14 +11,13 @@ SECURITY NOTE: from fastapi import APIRouter, HTTPException, Depends, Body, Path, Request, Response from typing import List, Dict, Any, Optional from fastapi import status -from datetime import datetime import logging import json from modules.datamodels.datamodelUam import User, UserConnection, AuthAuthority, ConnectionStatus from modules.datamodels.datamodelSecurity import Token from modules.security.auth import getCurrentUser, limiter -from modules.interfaces.interfaceDbAppObjects import getInterface, getRootInterface +from modules.interfaces.interfaceDbAppObjects import getInterface from modules.shared.timezoneUtils import get_utc_timestamp # Configure logger @@ -197,7 +196,7 @@ async def create_connection( ) # Save connection record - models now handle timestamp serialization automatically - interface.db.recordModify(UserConnection, connection.id, connection.to_dict()) + interface.db.recordModify(UserConnection, connection.id, connection.model_dump()) return connection @@ -251,7 +250,7 @@ async def update_connection( connection.lastChecked = get_utc_timestamp() # Update connection - models now handle timestamp serialization automatically - interface.db.recordModify(UserConnection, connectionId, connection.to_dict()) + interface.db.recordModify(UserConnection, connectionId, connection.model_dump()) # Get token status for the updated connection @@ -386,7 +385,7 @@ async def disconnect_service( connection.lastChecked = get_utc_timestamp() # Update connection record - models now handle timestamp serialization automatically - interface.db.recordModify(UserConnection, connectionId, connection.to_dict()) + interface.db.recordModify(UserConnection, connectionId, connection.model_dump()) return {"message": "Service disconnected successfully"} diff --git a/modules/routes/routeDataFiles.py b/modules/routes/routeDataFiles.py index 7ec556a7..0facb8f6 100644 --- a/modules/routes/routeDataFiles.py +++ b/modules/routes/routeDataFiles.py @@ -1,14 +1,7 @@ from fastapi import APIRouter, HTTPException, Depends, File, UploadFile, Form, Path, Request, status, Query, Response, Body -from fastapi.responses import JSONResponse, FileResponse -from typing import List, Dict, Any, Optional, Union +from fastapi.responses import JSONResponse +from typing import List, Dict, Any, Optional import logging -from datetime import datetime, timezone -from dataclasses import dataclass -import io -import inspect -import importlib -import os -from pydantic import BaseModel # Import auth module from modules.security.auth import limiter, getCurrentUser @@ -16,7 +9,7 @@ from modules.security.auth import limiter, getCurrentUser # Import interfaces import modules.interfaces.interfaceDbComponentObjects as interfaceDbComponentObjects from modules.datamodels.datamodelFiles import FileItem, FilePreview -from modules.shared.attributeUtils import getModelAttributeDefinitions, AttributeResponse, AttributeDefinition +from modules.shared.attributeUtils import getModelAttributeDefinitions from modules.datamodels.datamodelUam import User # Configure logger @@ -103,7 +96,7 @@ async def upload_file( fileItem.workflowId = workflowId # Convert FileItem to dictionary for JSON response - fileMeta = fileItem.to_dict() + fileMeta = fileItem.model_dump() # Response with duplicate information return JSONResponse({ diff --git a/modules/routes/routeDataMandates.py b/modules/routes/routeDataMandates.py index 00375e4e..97c4477e 100644 --- a/modules/routes/routeDataMandates.py +++ b/modules/routes/routeDataMandates.py @@ -4,21 +4,16 @@ Implements the endpoints for mandate management. """ from fastapi import APIRouter, HTTPException, Depends, Body, Path, Request, Response -from typing import List, Dict, Any, Optional +from typing import List, Dict, Any from fastapi import status -from datetime import datetime import logging -import inspect -import importlib -import os -from pydantic import BaseModel # Import auth module from modules.security.auth import limiter, getCurrentUser # Import interfaces import modules.interfaces.interfaceDbAppObjects as interfaceDbAppObjects -from modules.shared.attributeUtils import getModelAttributeDefinitions, AttributeResponse, AttributeDefinition +from modules.shared.attributeUtils import getModelAttributeDefinitions # Import the model classes from modules.datamodels.datamodelUam import Mandate, User @@ -136,7 +131,7 @@ async def update_mandate( ) # Update mandate - updatedMandate = appInterface.updateMandate(mandateId, mandateData.to_dict()) + updatedMandate = appInterface.updateMandate(mandateId, mandateData.model_dump()) if not updatedMandate: raise HTTPException( diff --git a/modules/routes/routeDataPrompts.py b/modules/routes/routeDataPrompts.py index 51968808..19b28798 100644 --- a/modules/routes/routeDataPrompts.py +++ b/modules/routes/routeDataPrompts.py @@ -1,12 +1,7 @@ -from fastapi import APIRouter, HTTPException, Depends, Body, Query, Path, Request, Response -from typing import List, Dict, Any, Optional +from fastapi import APIRouter, HTTPException, Depends, Body, Path, Request +from typing import List, Dict, Any from fastapi import status -from datetime import datetime import logging -import inspect -import importlib -import os -from pydantic import BaseModel # Import auth module from modules.security.auth import limiter, getCurrentUser @@ -14,7 +9,6 @@ from modules.security.auth import limiter, getCurrentUser # Import interfaces import modules.interfaces.interfaceDbComponentObjects as interfaceDbComponentObjects from modules.datamodels.datamodelUtils import Prompt -from modules.shared.attributeUtils import getModelAttributeDefinitions, AttributeResponse, AttributeDefinition from modules.datamodels.datamodelUam import User # Configure logger diff --git a/modules/routes/routeDataUsers.py b/modules/routes/routeDataUsers.py index afd286d6..7a0ea735 100644 --- a/modules/routes/routeDataUsers.py +++ b/modules/routes/routeDataUsers.py @@ -6,12 +6,7 @@ Implements the endpoints for user management. from fastapi import APIRouter, HTTPException, Depends, Body, Path, Request, Response from typing import List, Dict, Any, Optional from fastapi import status -from datetime import datetime import logging -import inspect -import importlib -import os -from pydantic import BaseModel # Import interfaces and models import modules.interfaces.interfaceDbAppObjects as interfaceDbAppObjects @@ -19,8 +14,6 @@ from modules.security.auth import getCurrentUser, limiter, getCurrentUser # Import the attribute definition and helper functions from modules.datamodels.datamodelUam import User, UserPrivilege -from modules.shared.attributeUtils import AttributeDefinition -from modules.shared.attributeUtils import getModelAttributeDefinitions, AttributeResponse # Configure logger logger = logging.getLogger(__name__) diff --git a/modules/routes/routeSecurityGoogle.py b/modules/routes/routeSecurityGoogle.py index 6ebd5e0a..8177da8d 100644 --- a/modules/routes/routeSecurityGoogle.py +++ b/modules/routes/routeSecurityGoogle.py @@ -7,17 +7,14 @@ from fastapi.responses import HTMLResponse, RedirectResponse, JSONResponse import logging import json from typing import Dict, Any, Optional -from datetime import datetime, timedelta from requests_oauthlib import OAuth2Session import httpx from modules.shared.configuration import APP_CONFIG from modules.interfaces.interfaceDbAppObjects import getInterface, getRootInterface from modules.datamodels.datamodelUam import AuthAuthority, User, ConnectionStatus, UserConnection -from modules.datamodels.datamodelSecurity import Token from modules.security.auth import getCurrentUser, limiter -from modules.shared.attributeUtils import ModelMixin -from modules.shared.timezoneUtils import get_utc_now, create_expiration_timestamp, get_utc_timestamp +from modules.shared.timezoneUtils import create_expiration_timestamp, get_utc_timestamp # Configure logger logger = logging.getLogger(__name__) @@ -377,7 +374,7 @@ async def auth_callback(code: str, state: str, request: Request) -> HTMLResponse window.opener.postMessage({{ type: 'google_auth_success', access_token: {json.dumps(token_response["access_token"])}, - token_data: {json.dumps(token.to_dict())} + token_data: {json.dumps(token.model_dump())} }}, '*'); }} setTimeout(() => window.close(), 1000); @@ -471,7 +468,7 @@ async def auth_callback(code: str, state: str, request: Request) -> HTMLResponse # Update connection record directly from modules.datamodels.datamodelUam import UserConnection - rootInterface.db.recordModify(UserConnection, connection_id, connection.to_dict()) + rootInterface.db.recordModify(UserConnection, connection_id, connection.model_dump()) # Save token @@ -731,7 +728,7 @@ async def refresh_token( google_connection.expiresAt = float(current_token.expiresAt) if current_token.expiresAt else google_connection.expiresAt google_connection.lastChecked = get_utc_timestamp() google_connection.status = ConnectionStatus.ACTIVE - appInterface.db.recordModify(UserConnection, google_connection.id, google_connection.to_dict()) + appInterface.db.recordModify(UserConnection, google_connection.id, google_connection.model_dump()) # Calculate time until expiration current_time = get_utc_timestamp() diff --git a/modules/routes/routeSecurityLocal.py b/modules/routes/routeSecurityLocal.py index f24aee78..d705d5c6 100644 --- a/modules/routes/routeSecurityLocal.py +++ b/modules/routes/routeSecurityLocal.py @@ -5,12 +5,11 @@ Routes for local security and authentication. from fastapi import APIRouter, HTTPException, status, Depends, Request, Response, Body from fastapi.security import OAuth2PasswordRequestForm import logging -from typing import Dict, Any, Optional -from datetime import datetime, timedelta +from typing import Dict, Any +from datetime import datetime from fastapi.responses import JSONResponse, HTMLResponse, RedirectResponse import uuid from jose import jwt -from pydantic import BaseModel # Import auth modules from modules.security.auth import getCurrentUser, limiter, SECRET_KEY, ALGORITHM @@ -18,7 +17,6 @@ from modules.security.jwtService import createAccessToken, createRefreshToken, s from modules.interfaces.interfaceDbAppObjects import getInterface, getRootInterface from modules.datamodels.datamodelUam import User, UserInDB, AuthAuthority, UserPrivilege from modules.datamodels.datamodelSecurity import Token -from modules.shared.attributeUtils import ModelMixin # Configure logger logger = logging.getLogger(__name__) diff --git a/modules/routes/routeSecurityMsft.py b/modules/routes/routeSecurityMsft.py index be682549..6c16c37e 100644 --- a/modules/routes/routeSecurityMsft.py +++ b/modules/routes/routeSecurityMsft.py @@ -7,7 +7,6 @@ from fastapi.responses import HTMLResponse, RedirectResponse, JSONResponse import logging import json from typing import Dict, Any, Optional -from datetime import datetime, timedelta import msal import httpx @@ -17,8 +16,7 @@ from modules.datamodels.datamodelUam import AuthAuthority, User, ConnectionStatu from modules.datamodels.datamodelSecurity import Token from modules.security.auth import getCurrentUser, limiter from modules.security.jwtService import createAccessToken -from modules.shared.attributeUtils import ModelMixin -from modules.shared.timezoneUtils import get_utc_now, create_expiration_timestamp, get_utc_timestamp +from modules.shared.timezoneUtils import create_expiration_timestamp, get_utc_timestamp # Configure logger logger = logging.getLogger(__name__) @@ -234,7 +232,7 @@ async def auth_callback(code: str, state: str, request: Request) -> HTMLResponse appInterface.saveAccessToken(jwt_token_obj) # Convert token to dict and ensure proper timestamp handling - token_dict = jwt_token_obj.to_dict() + token_dict = jwt_token_obj.model_dump() # Remove datetime conversion logic - models now handle this automatically # The token model already returns float timestamps @@ -341,7 +339,7 @@ async def auth_callback(code: str, state: str, request: Request) -> HTMLResponse connection.externalEmail = user_info.get("mail") # Update connection record directly - rootInterface.db.recordModify(UserConnection, connection_id, connection.to_dict()) + rootInterface.db.recordModify(UserConnection, connection_id, connection.model_dump()) # Save token @@ -588,7 +586,7 @@ async def refresh_token( msft_connection.status = ConnectionStatus.ACTIVE # Save updated connection - appInterface.db.recordModify(UserConnection, msft_connection.id, msft_connection.to_dict()) + appInterface.db.recordModify(UserConnection, msft_connection.id, msft_connection.model_dump()) # Calculate time until expiration current_time = get_utc_timestamp() diff --git a/modules/routes/routeVoiceGoogle.py b/modules/routes/routeVoiceGoogle.py index aa7bc409..17c6e73d 100644 --- a/modules/routes/routeVoiceGoogle.py +++ b/modules/routes/routeVoiceGoogle.py @@ -4,17 +4,14 @@ Replaces Azure voice services with Google Cloud Speech-to-Text and Translation Includes WebSocket support for real-time voice streaming """ -import os import logging import json import base64 -import asyncio from fastapi import APIRouter, File, Form, UploadFile, Depends, HTTPException, Body, WebSocket, WebSocketDisconnect from fastapi.responses import Response from typing import Optional, Dict, Any, List from modules.security.auth import getCurrentUser from modules.datamodels.datamodelUam import User -from modules.interfaces.interfaceDbComponentObjects import getInterface from modules.interfaces.interfaceVoiceObjects import getVoiceInterface, VoiceObjects logger = logging.getLogger(__name__) @@ -391,7 +388,7 @@ async def get_voice_settings(current_user: User = Depends(getCurrentUser)): return { "success": True, "data": { - "user_settings": voice_settings.to_dict(), + "user_settings": voice_settings.model_dump(), "default_settings": { "sttLanguage": "de-DE", "ttsLanguage": "de-DE", diff --git a/modules/routes/routeWorkflows.py b/modules/routes/routeWorkflows.py index 5ca0f43b..b3ff1a87 100644 --- a/modules/routes/routeWorkflows.py +++ b/modules/routes/routeWorkflows.py @@ -3,13 +3,9 @@ Workflow routes for the backend API. Implements the endpoints for workflow management according to the state machine. """ -import os -import json import logging from typing import List, Dict, Any, Optional from fastapi import APIRouter, HTTPException, Depends, Body, Path, Query, Response, status, Request -from fastapi.responses import HTMLResponse, RedirectResponse, JSONResponse -from datetime import datetime, timedelta # Import auth modules from modules.security.auth import limiter, getCurrentUser @@ -28,7 +24,6 @@ from modules.datamodels.datamodelChat import ( ) from modules.shared.attributeUtils import getModelAttributeDefinitions, AttributeResponse from modules.datamodels.datamodelUam import User -from modules.shared.timezoneUtils import get_utc_timestamp # Configure logger diff --git a/modules/security/auth.py b/modules/security/auth.py index b11c67d3..ad349f19 100644 --- a/modules/security/auth.py +++ b/modules/security/auth.py @@ -3,8 +3,6 @@ Authentication module for backend API. Handles JWT-based authentication, token generation, and user context. """ -from datetime import datetime, timedelta, timezone -import uuid from typing import Optional, Dict, Any, Tuple from fastapi import Depends, HTTPException, status, Request, Response from fastapi.security import HTTPBearer, HTTPAuthorizationCredentials @@ -14,7 +12,6 @@ from slowapi import Limiter from slowapi.util import get_remote_address from modules.shared.configuration import APP_CONFIG -from modules.shared.timezoneUtils import get_utc_now, get_utc_timestamp from modules.interfaces.interfaceDbAppObjects import getRootInterface from modules.datamodels.datamodelUam import User, AuthAuthority from modules.datamodels.datamodelSecurity import Token diff --git a/modules/security/tokenManager.py b/modules/security/tokenManager.py index e8a8d9be..07c5c2a9 100644 --- a/modules/security/tokenManager.py +++ b/modules/security/tokenManager.py @@ -5,7 +5,6 @@ Handles all token operations including automatic refresh for backend services. import logging import httpx -from datetime import datetime from typing import Optional, Dict, Any, Callable from modules.datamodels.datamodelSecurity import Token diff --git a/modules/security/tokenRefreshMiddleware.py b/modules/security/tokenRefreshMiddleware.py index 5f1f2c21..2ced531c 100644 --- a/modules/security/tokenRefreshMiddleware.py +++ b/modules/security/tokenRefreshMiddleware.py @@ -8,7 +8,6 @@ when API endpoints are accessed, providing seamless user experience. import logging from fastapi import Request, Response from starlette.middleware.base import BaseHTTPMiddleware -from starlette.responses import Response as StarletteResponse from typing import Callable import asyncio from modules.security.tokenRefreshService import token_refresh_service diff --git a/modules/security/tokenRefreshService.py b/modules/security/tokenRefreshService.py index 3962aa78..ff7ac720 100644 --- a/modules/security/tokenRefreshService.py +++ b/modules/security/tokenRefreshService.py @@ -7,11 +7,8 @@ to ensure users don't experience token expiration issues. """ import logging -from typing import Optional, Dict, Any, List -from datetime import datetime, timedelta -from modules.interfaces.interfaceDbAppObjects import getInterface -from modules.datamodels.datamodelUam import User, UserConnection, AuthAuthority -from modules.datamodels.datamodelSecurity import Token +from typing import Dict, Any +from modules.datamodels.datamodelUam import UserConnection, AuthAuthority from modules.shared.timezoneUtils import get_utc_timestamp from modules.shared.auditLogger import audit_logger diff --git a/modules/services/serviceAi/mainServiceAi.py b/modules/services/serviceAi/mainServiceAi.py index e4782588..a79bad3a 100644 --- a/modules/services/serviceAi/mainServiceAi.py +++ b/modules/services/serviceAi/mainServiceAi.py @@ -1,16 +1,14 @@ import logging -from typing import Dict, Any, List, Optional, Tuple, Union +from typing import Dict, Any, List, Optional, Union from modules.datamodels.datamodelChat import PromptPlaceholder, ChatDocument from modules.services.serviceExtraction.mainServiceExtraction import ExtractionService from modules.datamodels.datamodelAi import AiCallRequest, AiCallOptions, OperationTypeEnum, PriorityEnum -from modules.datamodels.datamodelExtraction import ChunkResult, ContentExtracted from modules.aicore.aicorePluginTavily import WebResearchRequest, WebResearchResult from modules.interfaces.interfaceAiObjects import AiObjects from modules.services.serviceAi.subCoreAi import SubCoreAi from modules.services.serviceAi.subDocumentProcessing import SubDocumentProcessing from modules.services.serviceAi.subWebResearch import SubWebResearch from modules.services.serviceAi.subDocumentGeneration import SubDocumentGeneration -from modules.services.serviceAi.subSharedAiUtils import sanitizePromptContent logger = logging.getLogger(__name__) diff --git a/modules/services/serviceAi/subCoreAi.py b/modules/services/serviceAi/subCoreAi.py index 80992b4b..c0f6d6a4 100644 --- a/modules/services/serviceAi/subCoreAi.py +++ b/modules/services/serviceAi/subCoreAi.py @@ -3,7 +3,6 @@ import logging from typing import Dict, Any, List, Optional, Tuple, Union from modules.datamodels.datamodelChat import PromptPlaceholder, ChatDocument from modules.datamodels.datamodelAi import AiCallRequest, AiCallOptions, OperationTypeEnum, PriorityEnum, ProcessingModeEnum -from modules.datamodels.datamodelExtraction import ContentPart from modules.services.serviceAi.subSharedAiUtils import ( buildPromptWithPlaceholders, extractTextFromContentParts, diff --git a/modules/services/serviceAi/subDocumentGeneration.py b/modules/services/serviceAi/subDocumentGeneration.py index 86f13e2d..82d1df67 100644 --- a/modules/services/serviceAi/subDocumentGeneration.py +++ b/modules/services/serviceAi/subDocumentGeneration.py @@ -3,9 +3,9 @@ import json import logging import time from datetime import datetime, UTC -from typing import Dict, Any, List, Optional, Tuple, Union +from typing import Dict, Any, List, Optional from modules.datamodels.datamodelChat import ChatDocument -from modules.datamodels.datamodelAi import AiCallOptions, OperationTypeEnum +from modules.datamodels.datamodelAi import AiCallOptions logger = logging.getLogger(__name__) diff --git a/modules/services/serviceAi/subDocumentProcessing.py b/modules/services/serviceAi/subDocumentProcessing.py index 8cdca91c..5664f218 100644 --- a/modules/services/serviceAi/subDocumentProcessing.py +++ b/modules/services/serviceAi/subDocumentProcessing.py @@ -2,7 +2,7 @@ import json import logging import re import time -from typing import Dict, Any, List, Optional, Tuple, Union +from typing import Dict, Any, List, Optional from modules.datamodels.datamodelChat import ChatDocument from modules.datamodels.datamodelAi import AiCallRequest, AiCallOptions, OperationTypeEnum, PriorityEnum from modules.datamodels.datamodelExtraction import ChunkResult, ContentExtracted, PartResult, ExtractionOptions, MergeStrategy diff --git a/modules/services/serviceAi/subSharedAiUtils.py b/modules/services/serviceAi/subSharedAiUtils.py index 198f8aee..a4b825e9 100644 --- a/modules/services/serviceAi/subSharedAiUtils.py +++ b/modules/services/serviceAi/subSharedAiUtils.py @@ -8,7 +8,6 @@ to maintain DRY principles and ensure consistency. import re import logging from typing import Dict, Any, List, Optional, Union -from modules.datamodels.datamodelChat import PromptPlaceholder logger = logging.getLogger(__name__) diff --git a/modules/services/serviceAi/subWebResearch.py b/modules/services/serviceAi/subWebResearch.py index deb9f9b2..fd6e88ac 100644 --- a/modules/services/serviceAi/subWebResearch.py +++ b/modules/services/serviceAi/subWebResearch.py @@ -1,7 +1,6 @@ import logging -from typing import Dict, Any, List, Optional, Tuple, Union +from typing import Optional from modules.aicore.aicorePluginTavily import WebResearchRequest, WebResearchResult -from modules.interfaces.interfaceAiObjects import AiObjects from modules.shared.configuration import APP_CONFIG logger = logging.getLogger(__name__) diff --git a/modules/services/serviceExtraction/subMerger.py b/modules/services/serviceExtraction/subMerger.py index da0bbfcd..aa9bf1f5 100644 --- a/modules/services/serviceExtraction/subMerger.py +++ b/modules/services/serviceExtraction/subMerger.py @@ -1,7 +1,7 @@ """ Intelligent Token-Aware Merger for optimizing AI calls based on LLM token limits. """ -from typing import List, Dict, Any, Tuple +from typing import List, Dict, Any import logging from modules.datamodels.datamodelExtraction import ContentPart from .subUtils import makeId diff --git a/modules/services/serviceGeneration/renderers/registry.py b/modules/services/serviceGeneration/renderers/registry.py index 909cfb2c..4c9032a4 100644 --- a/modules/services/serviceGeneration/renderers/registry.py +++ b/modules/services/serviceGeneration/renderers/registry.py @@ -4,7 +4,6 @@ Renderer registry for automatic discovery and registration of renderers. import logging import importlib -import pkgutil from typing import Dict, Type, List, Optional from .rendererBaseTemplate import BaseRenderer diff --git a/modules/services/serviceGeneration/renderers/rendererCsv.py b/modules/services/serviceGeneration/renderers/rendererCsv.py index 782e7d4a..5ad2d4bc 100644 --- a/modules/services/serviceGeneration/renderers/rendererCsv.py +++ b/modules/services/serviceGeneration/renderers/rendererCsv.py @@ -4,8 +4,6 @@ CSV renderer for report generation. from .rendererBaseTemplate import BaseRenderer from typing import Dict, Any, Tuple, List -import csv -import io class RendererCsv(BaseRenderer): """Renders content to CSV format with format-specific extraction.""" diff --git a/modules/services/serviceGeneration/renderers/rendererImage.py b/modules/services/serviceGeneration/renderers/rendererImage.py index a0797bfa..4db495dc 100644 --- a/modules/services/serviceGeneration/renderers/rendererImage.py +++ b/modules/services/serviceGeneration/renderers/rendererImage.py @@ -4,7 +4,6 @@ Image renderer for report generation using AI image generation. from .rendererBaseTemplate import BaseRenderer from typing import Dict, Any, Tuple, List -import base64 import logging logger = logging.getLogger(__name__) diff --git a/modules/services/serviceGeneration/renderers/rendererPdf.py b/modules/services/serviceGeneration/renderers/rendererPdf.py index f6681a79..b80e6197 100644 --- a/modules/services/serviceGeneration/renderers/rendererPdf.py +++ b/modules/services/serviceGeneration/renderers/rendererPdf.py @@ -6,7 +6,6 @@ from .rendererBaseTemplate import BaseRenderer from typing import Dict, Any, Tuple, List import io import base64 -from datetime import datetime, UTC try: from reportlab.lib.pagesizes import letter, A4 diff --git a/modules/services/serviceNormalization/mainServiceNormalization.py b/modules/services/serviceNormalization/mainServiceNormalization.py index d8696fa5..7ba3d3b2 100644 --- a/modules/services/serviceNormalization/mainServiceNormalization.py +++ b/modules/services/serviceNormalization/mainServiceNormalization.py @@ -1,7 +1,6 @@ import json import os from typing import Any, Dict, List, Set -from datetime import datetime, UTC class NormalizationService: diff --git a/modules/services/serviceSharepoint/mainServiceSharepoint.py b/modules/services/serviceSharepoint/mainServiceSharepoint.py index 66a6a7cf..ec05f881 100644 --- a/modules/services/serviceSharepoint/mainServiceSharepoint.py +++ b/modules/services/serviceSharepoint/mainServiceSharepoint.py @@ -1,11 +1,9 @@ """Connector for SharePoint operations using Microsoft Graph API.""" import logging -import json import aiohttp import asyncio from typing import Dict, Any, List, Optional -from datetime import datetime, UTC logger = logging.getLogger(__name__) diff --git a/modules/services/serviceUtils/mainServiceUtils.py b/modules/services/serviceUtils/mainServiceUtils.py index 85944d28..1824eab7 100644 --- a/modules/services/serviceUtils/mainServiceUtils.py +++ b/modules/services/serviceUtils/mainServiceUtils.py @@ -4,7 +4,6 @@ Provides centralized access to configuration, events, and other utilities. """ import logging -import os from typing import Any, Optional, Dict, Callable, List from modules.shared.configuration import APP_CONFIG from modules.shared.eventManagement import eventManager diff --git a/modules/services/serviceWorkflow/mainServiceWorkflow.py b/modules/services/serviceWorkflow/mainServiceWorkflow.py index 8c6acd84..50d14611 100644 --- a/modules/services/serviceWorkflow/mainServiceWorkflow.py +++ b/modules/services/serviceWorkflow/mainServiceWorkflow.py @@ -1,5 +1,4 @@ import logging -import uuid from typing import Dict, Any, List, Optional from modules.datamodels.datamodelUam import User, UserConnection from modules.datamodels.datamodelChat import ChatDocument, ChatMessage, ChatStat, ChatLog diff --git a/modules/shared/attributeUtils.py b/modules/shared/attributeUtils.py index cf5c1216..6c56e855 100644 --- a/modules/shared/attributeUtils.py +++ b/modules/shared/attributeUtils.py @@ -3,52 +3,14 @@ Shared utilities for model attributes and labels. """ from pydantic import BaseModel, Field, ConfigDict -from typing import Dict, Any, List, Type, Optional, Union +from typing import Dict, Any, List, Type, Optional import inspect import importlib import os -from datetime import datetime - - -class ModelMixin: - """Mixin class that provides serialization methods for Pydantic models.""" - - def to_dict(self) -> Dict[str, Any]: - """ - Convert a Pydantic model to a dictionary. - Handles both Pydantic v1 and v2. - All timestamp fields remain as float values. - - Returns: - Dict[str, Any]: Dictionary representation of the model - """ - # Get the raw dictionary - if hasattr(self, "model_dump"): - data: Dict[str, Any] = self.model_dump() # Pydantic v2 - else: - data: Dict[str, Any] = self.dict() # Pydantic v1 - - # All fields (including timestamps) remain in their original format - # No conversions needed - timestamps are already float - - return data - - @classmethod - def from_dict(cls, data: Dict[str, Any]) -> "ModelMixin": - """ - Create a Pydantic model instance from a dictionary. - - Args: - data: Dictionary containing the model data - - Returns: - ModelMixin: New instance of the model class - """ - return cls(**data) # Define the AttributeDefinition class here instead of importing it -class AttributeDefinition(BaseModel, ModelMixin): +class AttributeDefinition(BaseModel): """Definition of a model attribute with its metadata.""" name: str @@ -72,39 +34,7 @@ class AttributeDefinition(BaseModel, ModelMixin): MODEL_LABELS: Dict[str, Dict[str, Dict[str, str]]] = {} -def to_dict(model: BaseModel) -> Dict[str, Any]: - """ - Convert a Pydantic model to a dictionary. - Handles both Pydantic v1 and v2. - - Args: - model: The Pydantic model instance to convert - - Returns: - Dict[str, Any]: Dictionary representation of the model - """ - if hasattr(model, "model_dump"): - return model.model_dump() # Pydantic v2 - return model.dict() # Pydantic v1 - - -def from_dict(model_class: Type[BaseModel], data: Dict[str, Any]) -> BaseModel: - """ - Create a Pydantic model instance from a dictionary. - - Args: - model_class: The Pydantic model class to instantiate - data: Dictionary containing the model data - - Returns: - BaseModel: New instance of the model class - """ - return model_class(**data) - - -def register_model_labels( - model_name: str, model_label: Dict[str, str], labels: Dict[str, Dict[str, str]] -): +def register_model_labels(model_name: str, model_label: Dict[str, str], labels: Dict[str, Dict[str, str]]): """ Register labels for a model's attributes and the model itself. @@ -154,9 +84,7 @@ def get_model_label(model_name: str, language: str = "en") -> str: return model_label.get(language, model_label.get("en", model_name)) -def getModelAttributeDefinitions( - modelClass: Type[BaseModel] = None, userLanguage: str = "en" -) -> Dict[str, Any]: +def getModelAttributeDefinitions(modelClass: Type[BaseModel] = None, userLanguage: str = "en") -> Dict[str, Any]: """ Get attribute definitions for a model class. @@ -175,145 +103,71 @@ def getModelAttributeDefinitions( labels = get_model_labels(model_name, userLanguage) model_label = get_model_label(model_name, userLanguage) - # Handle both Pydantic v1 and v2 - if hasattr(modelClass, "model_fields"): # Pydantic v2 - fields = modelClass.model_fields - for name, field in fields.items(): - # Extract frontend metadata from field info - field_info = field.field_info if hasattr(field, "field_info") else None - # Check both direct attributes and extra field for frontend metadata - frontend_type = None - frontend_readonly = False - frontend_required = field.is_required() - frontend_options = None + # Pydantic v2 only + fields = modelClass.model_fields + for name, field in fields.items(): + # Extract frontend metadata from field info + field_info = field.field_info if hasattr(field, "field_info") else None + # Check both direct attributes and extra field for frontend metadata + frontend_type = None + frontend_readonly = False + frontend_required = field.is_required() + frontend_options = None - if field_info: - # Try direct attributes first - frontend_type = getattr(field_info, "frontend_type", None) - frontend_readonly = getattr(field_info, "frontend_readonly", False) - frontend_required = getattr( - field_info, "frontend_required", frontend_required - ) - frontend_options = getattr(field_info, "frontend_options", None) - - # If not found, check extra field - if hasattr(field_info, "extra") and field_info.extra: - if frontend_type is None: - frontend_type = field_info.extra.get("frontend_type") - if not frontend_readonly: - frontend_readonly = field_info.extra.get( - "frontend_readonly", False - ) - if ( - frontend_required == field.is_required() - ): # Only override if we didn't get it from direct attribute - frontend_required = field_info.extra.get( - "frontend_required", frontend_required - ) - if frontend_options is None: - frontend_options = field_info.extra.get("frontend_options") - - # Use frontend type if available, otherwise fall back to Python type - field_type = ( - frontend_type - if frontend_type - else ( - field.annotation.__name__ - if hasattr(field.annotation, "__name__") - else str(field.annotation) - ) + if field_info: + # Try direct attributes first + frontend_type = getattr(field_info, "frontend_type", None) + frontend_readonly = getattr(field_info, "frontend_readonly", False) + frontend_required = getattr( + field_info, "frontend_required", frontend_required ) + frontend_options = getattr(field_info, "frontend_options", None) - attributes.append( - { - "name": name, - "type": field_type, - "required": frontend_required, - "description": field.description - if hasattr(field, "description") - else "", - "label": labels.get(name, name), - "placeholder": f"Please enter {labels.get(name, name)}", - "editable": not frontend_readonly, - "visible": True, - "order": len(attributes), - "readonly": frontend_readonly, - "options": frontend_options, - } + # If not found, check extra field + if hasattr(field_info, "extra") and field_info.extra: + if frontend_type is None: + frontend_type = field_info.extra.get("frontend_type") + if not frontend_readonly: + frontend_readonly = field_info.extra.get( + "frontend_readonly", False + ) + if ( + frontend_required == field.is_required() + ): # Only override if we didn't get it from direct attribute + frontend_required = field_info.extra.get( + "frontend_required", frontend_required + ) + if frontend_options is None: + frontend_options = field_info.extra.get("frontend_options") + + # Use frontend type if available, otherwise fall back to Python type + field_type = ( + frontend_type + if frontend_type + else ( + field.annotation.__name__ + if hasattr(field.annotation, "__name__") + else str(field.annotation) ) - else: # Pydantic v1 - fields = modelClass.__fields__ - for name, field in fields.items(): - # Extract frontend metadata from field info - field_info = field.field_info if hasattr(field, "field_info") else None - # Check both direct attributes and extra field for frontend metadata - frontend_type = None - frontend_readonly = False - frontend_required = field.required - frontend_options = None + ) - if field_info: - # Try direct attributes first - frontend_type = getattr(field_info, "frontend_type", None) - frontend_readonly = getattr(field_info, "frontend_readonly", False) - frontend_required = getattr( - field_info, "frontend_required", frontend_required - ) - frontend_options = getattr(field_info, "frontend_options", None) - - # If not found, check extra field - if hasattr(field_info, "extra") and field_info.extra: - if frontend_type is None: - frontend_type = field_info.extra.get("frontend_type") - if not frontend_readonly: - frontend_readonly = field_info.extra.get( - "frontend_readonly", False - ) - if ( - frontend_required == field.required - ): # Only override if we didn't get it from direct attribute - frontend_required = field_info.extra.get( - "frontend_required", frontend_required - ) - if frontend_options is None: - frontend_options = field_info.extra.get("frontend_options") - - # Use frontend type if available, otherwise fall back to Python type - # Handle both Pydantic v1 and v2 - if hasattr(field, 'type_'): - field_annotation = field.type_ # Pydantic v1 - elif hasattr(field, 'annotation'): - field_annotation = field.annotation # Pydantic v2 - else: - field_annotation = type(None) # Fallback - - field_type = ( - frontend_type - if frontend_type - else ( - field_annotation.__name__ - if hasattr(field_annotation, "__name__") - else str(field_annotation) - ) - ) - - attributes.append( - { - "name": name, - "type": field_type, - "required": frontend_required, - "description": field.field_info.description - if hasattr(field.field_info, "description") - else "", - "label": labels.get(name, name), - "placeholder": f"Please enter {labels.get(name, name)}", - "editable": not frontend_readonly, - "visible": True, - "order": len(attributes), - "readonly": frontend_readonly, - "options": frontend_options, - } - ) + attributes.append( + { + "name": name, + "type": field_type, + "required": frontend_required, + "description": field.description + if hasattr(field, "description") + else "", + "label": labels.get(name, name), + "placeholder": f"Please enter {labels.get(name, name)}", + "editable": not frontend_readonly, + "visible": True, + "order": len(attributes), + "readonly": frontend_readonly, + "options": frontend_options, + } + ) return {"model": model_label, "attributes": attributes} diff --git a/modules/shared/timezoneUtils.py b/modules/shared/timezoneUtils.py index 93011060..1656f0f6 100644 --- a/modules/shared/timezoneUtils.py +++ b/modules/shared/timezoneUtils.py @@ -3,8 +3,7 @@ Timezone utilities for consistent timestamp handling across the gateway. Ensures all timestamps are properly handled as UTC. """ -from datetime import datetime, timezone, timedelta -from typing import Union, Optional +from datetime import datetime, timezone import time def get_utc_now() -> datetime: @@ -25,21 +24,6 @@ def get_utc_timestamp() -> float: """ return time.time() -def to_utc_timestamp(dt: datetime) -> float: - """ - Convert datetime object to UTC timestamp. - - Args: - dt (datetime): Datetime object to convert - - Returns: - float: UTC timestamp in seconds - """ - if dt.tzinfo is None: - # If naive datetime, assume it's UTC - dt = dt.replace(tzinfo=timezone.utc) - return dt.timestamp() - def create_expiration_timestamp(expires_in_seconds: int) -> float: """ Create a new expiration timestamp from seconds until expiration. diff --git a/modules/workflows/methods/methodBase.py b/modules/workflows/methods/methodBase.py index 0fa61ee9..2850f370 100644 --- a/modules/workflows/methods/methodBase.py +++ b/modules/workflows/methods/methodBase.py @@ -1,7 +1,5 @@ -from enum import Enum from typing import Dict, List, Optional, Any, Literal from datetime import datetime, UTC -from pydantic import BaseModel, Field import logging from functools import wraps diff --git a/modules/workflows/methods/methodSharepoint.py b/modules/workflows/methods/methodSharepoint.py index f0d020c2..2eff2107 100644 --- a/modules/workflows/methods/methodSharepoint.py +++ b/modules/workflows/methods/methodSharepoint.py @@ -4,11 +4,9 @@ Handles SharePoint document operations using the SharePoint service. """ import logging -import json import re from typing import Dict, Any, List, Optional from datetime import datetime, UTC -import base64 from urllib.parse import urlparse import aiohttp import asyncio diff --git a/modules/workflows/processing/adaptive/adaptiveLearningEngine.py b/modules/workflows/processing/adaptive/adaptiveLearningEngine.py index cc21d42a..107821e7 100644 --- a/modules/workflows/processing/adaptive/adaptiveLearningEngine.py +++ b/modules/workflows/processing/adaptive/adaptiveLearningEngine.py @@ -1,7 +1,6 @@ # adaptiveLearningEngine.py # Enhanced learning engine that tracks validation patterns and adapts prompts -import json import logging from typing import Dict, Any, List, Optional from datetime import datetime, timezone diff --git a/modules/workflows/processing/adaptive/contentValidator.py b/modules/workflows/processing/adaptive/contentValidator.py index 66c44b1e..3ef531d5 100644 --- a/modules/workflows/processing/adaptive/contentValidator.py +++ b/modules/workflows/processing/adaptive/contentValidator.py @@ -3,7 +3,6 @@ import logging import json -import re from typing import List, Dict, Any logger = logging.getLogger(__name__) diff --git a/modules/workflows/processing/adaptive/learningEngine.py b/modules/workflows/processing/adaptive/learningEngine.py index f18d1d17..29cfd53b 100644 --- a/modules/workflows/processing/adaptive/learningEngine.py +++ b/modules/workflows/processing/adaptive/learningEngine.py @@ -1,7 +1,6 @@ # learningEngine.py # Learning engine for adaptive React mode -import json import logging from typing import Dict, Any, List from datetime import datetime, timezone diff --git a/modules/workflows/processing/modes/modeReact.py b/modules/workflows/processing/modes/modeReact.py index 33686561..0dc580bf 100644 --- a/modules/workflows/processing/modes/modeReact.py +++ b/modules/workflows/processing/modes/modeReact.py @@ -12,7 +12,6 @@ from modules.datamodels.datamodelChat import ( ActionResult ) from modules.datamodels.datamodelChat import ChatWorkflow -from modules.datamodels.datamodelAi import AiCallOptions, OperationTypeEnum, ProcessingModeEnum, PriorityEnum from modules.workflows.processing.modes.modeBase import BaseMode from modules.workflows.processing.shared.executionState import TaskExecutionState, shouldContinue from modules.workflows.processing.shared.promptGenerationActionsReact import ( diff --git a/modules/workflows/processing/shared/executionState.py b/modules/workflows/processing/shared/executionState.py index 8504033a..2e1cd04f 100644 --- a/modules/workflows/processing/shared/executionState.py +++ b/modules/workflows/processing/shared/executionState.py @@ -3,7 +3,6 @@ import logging from typing import List -from datetime import datetime, UTC from modules.datamodels.datamodelChat import TaskStep from modules.datamodels.datamodelChat import ActionResult diff --git a/modules/workflows/processing/shared/methodDiscovery.py b/modules/workflows/processing/shared/methodDiscovery.py index 64389dd5..6e515d85 100644 --- a/modules/workflows/processing/shared/methodDiscovery.py +++ b/modules/workflows/processing/shared/methodDiscovery.py @@ -1,13 +1,11 @@ # methodDiscovery.py # Method discovery and management for workflow execution -import json import logging import importlib import pkgutil import inspect from typing import Any, Dict, List -from modules.datamodels.datamodelChat import TaskContext, ReviewContext, DocumentExchange from modules.workflows.methods.methodBase import MethodBase # Set up logger diff --git a/modules/workflows/processing/shared/placeholderFactory.py b/modules/workflows/processing/shared/placeholderFactory.py index 0e6d73f1..a4c148ae 100644 --- a/modules/workflows/processing/shared/placeholderFactory.py +++ b/modules/workflows/processing/shared/placeholderFactory.py @@ -32,7 +32,6 @@ Following placeholders are populated directly by prompt builders with according import json import logging from typing import Dict, Any, List -from modules.datamodels.datamodelChat import ChatDocument logger = logging.getLogger(__name__) from modules.workflows.processing.shared.methodDiscovery import (methods, discoverMethods) diff --git a/modules/workflows/processing/shared/promptGenerationActionsActionplan.py b/modules/workflows/processing/shared/promptGenerationActionsActionplan.py index ad8b0f8d..76413d24 100644 --- a/modules/workflows/processing/shared/promptGenerationActionsActionplan.py +++ b/modules/workflows/processing/shared/promptGenerationActionsActionplan.py @@ -3,7 +3,6 @@ Actionplan Mode Prompt Generation Handles prompt templates and extraction functions for actionplan mode action handling. """ -import json import logging from typing import Dict, Any, List from modules.datamodels.datamodelChat import PromptBundle, PromptPlaceholder diff --git a/modules/workflows/processing/shared/promptGenerationTaskplan.py b/modules/workflows/processing/shared/promptGenerationTaskplan.py index 54e80b50..702a7868 100644 --- a/modules/workflows/processing/shared/promptGenerationTaskplan.py +++ b/modules/workflows/processing/shared/promptGenerationTaskplan.py @@ -3,7 +3,6 @@ Task Planning Prompt Generation Handles prompt templates and extraction functions for task planning phase. """ -import json import logging from typing import Dict, Any, List from modules.datamodels.datamodelChat import PromptBundle, PromptPlaceholder diff --git a/modules/workflows/processing/workflowProcessor.py b/modules/workflows/processing/workflowProcessor.py index a80dba87..4d7edcca 100644 --- a/modules/workflows/processing/workflowProcessor.py +++ b/modules/workflows/processing/workflowProcessor.py @@ -3,7 +3,7 @@ import logging from typing import Dict, Any, Optional, List -from modules.datamodels.datamodelChat import TaskStep, TaskContext, TaskPlan, TaskResult, ReviewResult +from modules.datamodels.datamodelChat import TaskStep, TaskContext, TaskPlan, TaskResult from modules.datamodels.datamodelChat import ChatWorkflow from modules.workflows.processing.modes.modeBase import BaseMode from modules.workflows.processing.modes.modeActionplan import ActionplanMode @@ -289,7 +289,7 @@ class WorkflowProcessor: if hasattr(taskResult, 'met_criteria'): # This is a ReviewResult object met = taskResult.met_criteria if taskResult.met_criteria else [] - reviewResult = taskResult.to_dict() + reviewResult = taskResult.model_dump() else: # This is a TaskResult object met = [] @@ -302,7 +302,7 @@ class WorkflowProcessor: handoverData = { 'task_id': taskStep.id, 'task_description': taskStep.objective, - 'actions': [action.to_dict() for action in taskActions] if taskActions else [], + 'actions': [action.model_dump() for action in taskActions] if taskActions else [], 'review_result': reviewResult, 'workflow_id': workflow.id, 'handover_time': self.services.utils.timestampGetUtc() diff --git a/modules/workflows/workflowManager.py b/modules/workflows/workflowManager.py index d52365ef..9bfc6070 100644 --- a/modules/workflows/workflowManager.py +++ b/modules/workflows/workflowManager.py @@ -10,7 +10,7 @@ from modules.datamodels.datamodelChat import ( ChatWorkflow, ChatDocument ) -from modules.datamodels.datamodelChat import TaskItem, TaskStatus, TaskContext +from modules.datamodels.datamodelChat import TaskContext from modules.workflows.processing.workflowProcessor import WorkflowProcessor, WorkflowStoppedException diff --git a/tool_security_encrypt_all_env_files.py b/tool_security_encrypt_all_env_files.py index e3c40bea..df9cb71a 100644 --- a/tool_security_encrypt_all_env_files.py +++ b/tool_security_encrypt_all_env_files.py @@ -21,7 +21,6 @@ Usage: """ import sys -import os import argparse import shutil from pathlib import Path