fixed import chain, removed invalid imports by moving logic between modules
This commit is contained in:
ValueOn AG
parent
61f42259e5
commit
9f46ca3b03
31 changed files with 3058 additions and 1863 deletions
11
app.py
11
app.py
|
|
@ -16,6 +16,7 @@ from datetime import datetime
|
|||
from modules.shared.configuration import APP_CONFIG
|
||||
from modules.shared.eventManagement import eventManager
|
||||
from modules.features import featuresLifecycle as featuresLifecycle
|
||||
from modules.interfaces.interfaceDbAppObjects import getRootInterface
|
||||
|
||||
class DailyRotatingFileHandler(RotatingFileHandler):
|
||||
"""
|
||||
|
|
@ -275,15 +276,21 @@ instanceLabel = APP_CONFIG.get("APP_ENV_LABEL")
|
|||
async def lifespan(app: FastAPI):
|
||||
logger.info("Application is starting up")
|
||||
|
||||
# Get event user for feature lifecycle (system-level user for background operations)
|
||||
rootInterface = getRootInterface()
|
||||
eventUser = rootInterface.getUserByUsername("event")
|
||||
if not eventUser:
|
||||
logger.error("Could not get event user - some features may not start properly")
|
||||
|
||||
# --- Init Managers ---
|
||||
await featuresLifecycle.start()
|
||||
await featuresLifecycle.start(eventUser)
|
||||
eventManager.start()
|
||||
|
||||
yield
|
||||
|
||||
# --- Stop Managers ---
|
||||
eventManager.stop()
|
||||
await featuresLifecycle.stop()
|
||||
await featuresLifecycle.stop(eventUser)
|
||||
logger.info("Application has been shut down")
|
||||
|
||||
|
||||
|
|
|
|||
326
modules/.$DEPENDENCY_DIAGRAM.drawio.bkp
Normal file
326
modules/.$DEPENDENCY_DIAGRAM.drawio.bkp
Normal file
|
|
@ -0,0 +1,326 @@
|
|||
<mxfile host="app.diagrams.net">
|
||||
<diagram name="Module Dependencies" id="dependency-diagram">
|
||||
<mxGraphModel dx="1422" dy="794" grid="1" gridSize="10" guides="1" tooltips="1" connect="1" arrows="1" fold="1" page="1" pageScale="1" pageWidth="1400" pageHeight="1200" math="0" shadow="0">
|
||||
<root>
|
||||
<mxCell id="0" />
|
||||
<mxCell id="1" parent="0" />
|
||||
|
||||
<!-- Foundation Layer -->
|
||||
<mxCell id="shared" value="shared/
(Foundation)" style="rounded=1;whiteSpace=wrap;html=1;fillColor=#e1f5ff;strokeColor=#01579b;strokeWidth=2;fontStyle=1" vertex="1" parent="1">
|
||||
<mxGeometry x="600" y="100" width="120" height="60" as="geometry" />
|
||||
</mxCell>
|
||||
|
||||
<mxCell id="datamodels" value="datamodels/
(Foundation)" style="rounded=1;whiteSpace=wrap;html=1;fillColor=#e1f5ff;strokeColor=#01579b;strokeWidth=2;fontStyle=1" vertex="1" parent="1">
|
||||
<mxGeometry x="400" y="100" width="120" height="60" as="geometry" />
|
||||
</mxCell>
|
||||
|
||||
<mxCell id="aicore" value="aicore/
(Infrastructure)" style="rounded=1;whiteSpace=wrap;html=1;fillColor=#e1f5ff;strokeColor=#01579b;strokeWidth=2;fontStyle=1" vertex="1" parent="1">
|
||||
<mxGeometry x="200" y="100" width="120" height="60" as="geometry" />
|
||||
</mxCell>
|
||||
|
||||
<mxCell id="connectors" value="connectors/
(Infrastructure)" style="rounded=1;whiteSpace=wrap;html=1;fillColor=#e1f5ff;strokeColor=#01579b;strokeWidth=2;fontStyle=1" vertex="1" parent="1">
|
||||
<mxGeometry x="800" y="100" width="120" height="60" as="geometry" />
|
||||
</mxCell>
|
||||
|
||||
<!-- Data Layer -->
|
||||
<mxCell id="interfaces" value="interfaces/
(Data Access)
✅ No violations" style="rounded=1;whiteSpace=wrap;html=1;fillColor=#f3e5f5;strokeColor=#4a148c;strokeWidth=3;fontStyle=1" vertex="1" parent="1">
|
||||
<mxGeometry x="500" y="250" width="200" height="80" as="geometry" />
|
||||
</mxCell>
|
||||
|
||||
<!-- Business Logic Layer -->
|
||||
<mxCell id="services" value="services/
(Business Logic)
✅ Unidirectional" style="rounded=1;whiteSpace=wrap;html=1;fillColor=#e8f5e9;strokeColor=#1b5e20;strokeWidth=2;fontStyle=1" vertex="1" parent="1">
|
||||
<mxGeometry x="300" y="400" width="200" height="80" as="geometry" />
|
||||
</mxCell>
|
||||
|
||||
<mxCell id="workflows" value="workflows/
(Business Logic)" style="rounded=1;whiteSpace=wrap;html=1;fillColor=#e8f5e9;strokeColor=#1b5e20;strokeWidth=2;fontStyle=1" vertex="1" parent="1">
|
||||
<mxGeometry x="700" y="400" width="200" height="80" as="geometry" />
|
||||
</mxCell>
|
||||
|
||||
<!-- Feature Layer -->
|
||||
<mxCell id="features" value="features/
(Features)
✅ Unidirectional" style="rounded=1;whiteSpace=wrap;html=1;fillColor=#fff3e0;strokeColor=#e65100;strokeWidth=2;fontStyle=1" vertex="1" parent="1">
|
||||
<mxGeometry x="500" y="550" width="200" height="80" as="geometry" />
|
||||
</mxCell>
|
||||
|
||||
<!-- API Layer -->
|
||||
<mxCell id="routes" value="routes/
(API Layer)" style="rounded=1;whiteSpace=wrap;html=1;fillColor=#fce4ec;strokeColor=#880e4f;strokeWidth=2;fontStyle=1" vertex="1" parent="1">
|
||||
<mxGeometry x="300" y="700" width="200" height="80" as="geometry" />
|
||||
</mxCell>
|
||||
|
||||
<mxCell id="security" value="security/
(Security)" style="rounded=1;whiteSpace=wrap;html=1;fillColor=#fce4ec;strokeColor=#880e4f;strokeWidth=2;fontStyle=1" vertex="1" parent="1">
|
||||
<mxGeometry x="700" y="700" width="200" height="80" as="geometry" />
|
||||
</mxCell>
|
||||
|
||||
<!-- Foundation dependencies -->
|
||||
<mxCell id="edge1" value="" style="endArrow=classic;html=1;rounded=0;strokeWidth=2;strokeColor=#01579b" edge="1" parent="1" source="datamodels" target="shared">
|
||||
<mxGeometry width="50" height="50" relative="1" as="geometry">
|
||||
<mxPoint x="400" y="300" as="sourcePoint" />
|
||||
<mxPoint x="450" y="250" as="targetPoint" />
|
||||
</mxGeometry>
|
||||
</mxCell>
|
||||
|
||||
<mxCell id="edge2" value="" style="endArrow=classic;html=1;rounded=0;strokeWidth=2;strokeColor=#01579b" edge="1" parent="1" source="aicore" target="datamodels">
|
||||
<mxGeometry width="50" height="50" relative="1" as="geometry">
|
||||
<mxPoint x="300" y="300" as="sourcePoint" />
|
||||
<mxPoint x="350" y="250" as="targetPoint" />
|
||||
</mxGeometry>
|
||||
</mxCell>
|
||||
|
||||
<mxCell id="edge3" value="" style="endArrow=classic;html=1;rounded=0;strokeWidth=2;strokeColor=#01579b" edge="1" parent="1" source="aicore" target="shared">
|
||||
<mxGeometry width="50" height="50" relative="1" as="geometry">
|
||||
<mxPoint x="300" y="300" as="sourcePoint" />
|
||||
<mxPoint x="350" y="250" as="targetPoint" />
|
||||
</mxGeometry>
|
||||
</mxCell>
|
||||
|
||||
<mxCell id="edge4" value="" style="endArrow=classic;html=1;rounded=0;strokeWidth=2;strokeColor=#01579b" edge="1" parent="1" source="connectors" target="datamodels">
|
||||
<mxGeometry width="50" height="50" relative="1" as="geometry">
|
||||
<mxPoint x="800" y="300" as="sourcePoint" />
|
||||
<mxPoint x="450" y="250" as="targetPoint" />
|
||||
</mxGeometry>
|
||||
</mxCell>
|
||||
|
||||
<mxCell id="edge5" value="" style="endArrow=classic;html=1;rounded=0;strokeWidth=2;strokeColor=#01579b" edge="1" parent="1" source="connectors" target="shared">
|
||||
<mxGeometry width="50" height="50" relative="1" as="geometry">
|
||||
<mxPoint x="800" y="300" as="sourcePoint" />
|
||||
<mxPoint x="650" y="250" as="targetPoint" />
|
||||
</mxGeometry>
|
||||
</mxCell>
|
||||
|
||||
<!-- Interface layer dependencies -->
|
||||
<mxCell id="edge6" value="" style="endArrow=classic;html=1;rounded=0;strokeWidth=2;strokeColor=#4a148c" edge="1" parent="1" source="interfaces" target="aicore">
|
||||
<mxGeometry width="50" height="50" relative="1" as="geometry">
|
||||
<mxPoint x="500" y="400" as="sourcePoint" />
|
||||
<mxPoint x="350" y="300" as="targetPoint" />
|
||||
</mxGeometry>
|
||||
</mxCell>
|
||||
|
||||
<mxCell id="edge7" value="" style="endArrow=classic;html=1;rounded=0;strokeWidth=2;strokeColor=#4a148c" edge="1" parent="1" source="interfaces" target="connectors">
|
||||
<mxGeometry width="50" height="50" relative="1" as="geometry">
|
||||
<mxPoint x="600" y="400" as="sourcePoint" />
|
||||
<mxPoint x="750" y="300" as="targetPoint" />
|
||||
</mxGeometry>
|
||||
</mxCell>
|
||||
|
||||
<mxCell id="edge8" value="" style="endArrow=classic;html=1;rounded=0;strokeWidth=2;strokeColor=#4a148c" edge="1" parent="1" source="interfaces" target="datamodels">
|
||||
<mxGeometry width="50" height="50" relative="1" as="geometry">
|
||||
<mxPoint x="550" y="400" as="sourcePoint" />
|
||||
<mxPoint x="450" y="300" as="targetPoint" />
|
||||
</mxGeometry>
|
||||
</mxCell>
|
||||
|
||||
<mxCell id="edge9" value="callbackRegistry" style="endArrow=classic;html=1;rounded=0;strokeWidth=2;strokeColor=#4a148c;dashed=1" edge="1" parent="1" source="interfaces" target="shared">
|
||||
<mxGeometry width="50" height="50" relative="1" as="geometry">
|
||||
<mxPoint x="600" y="400" as="sourcePoint" />
|
||||
<mxPoint x="650" y="300" as="targetPoint" />
|
||||
</mxGeometry>
|
||||
</mxCell>
|
||||
|
||||
<!-- Service layer dependencies -->
|
||||
<mxCell id="edge10" value="✅" style="endArrow=classic;html=1;rounded=0;strokeWidth=3;strokeColor=#1b5e20" edge="1" parent="1" source="services" target="interfaces">
|
||||
<mxGeometry width="50" height="50" relative="1" as="geometry">
|
||||
<mxPoint x="400" y="500" as="sourcePoint" />
|
||||
<mxPoint x="550" y="450" as="targetPoint" />
|
||||
</mxGeometry>
|
||||
</mxCell>
|
||||
|
||||
<mxCell id="edge11" value="" style="endArrow=classic;html=1;rounded=0;strokeWidth=2;strokeColor=#1b5e20" edge="1" parent="1" source="services" target="aicore">
|
||||
<mxGeometry width="50" height="50" relative="1" as="geometry">
|
||||
<mxPoint x="350" y="500" as="sourcePoint" />
|
||||
<mxPoint x="250" y="300" as="targetPoint" />
|
||||
</mxGeometry>
|
||||
</mxCell>
|
||||
|
||||
<mxCell id="edge12" value="" style="endArrow=classic;html=1;rounded=0;strokeWidth=2;strokeColor=#1b5e20" edge="1" parent="1" source="services" target="datamodels">
|
||||
<mxGeometry width="50" height="50" relative="1" as="geometry">
|
||||
<mxPoint x="400" y="500" as="sourcePoint" />
|
||||
<mxPoint x="450" y="300" as="targetPoint" />
|
||||
</mxGeometry>
|
||||
</mxCell>
|
||||
|
||||
<mxCell id="edge13" value="" style="endArrow=classic;html=1;rounded=0;strokeWidth=2;strokeColor=#1b5e20" edge="1" parent="1" source="services" target="shared">
|
||||
<mxGeometry width="50" height="50" relative="1" as="geometry">
|
||||
<mxPoint x="450" y="500" as="sourcePoint" />
|
||||
<mxPoint x="650" y="300" as="targetPoint" />
|
||||
</mxGeometry>
|
||||
</mxCell>
|
||||
|
||||
<mxCell id="edge14" value="" style="endArrow=classic;html=1;rounded=0;strokeWidth=2;strokeColor=#1b5e20" edge="1" parent="1" source="services" target="security">
|
||||
<mxGeometry width="50" height="50" relative="1" as="geometry">
|
||||
<mxPoint x="400" y="500" as="sourcePoint" />
|
||||
<mxPoint x="800" y="650" as="targetPoint" />
|
||||
</mxGeometry>
|
||||
</mxCell>
|
||||
|
||||
<!-- Workflow layer dependencies -->
|
||||
<mxCell id="edge15" value="" style="endArrow=classic;html=1;rounded=0;strokeWidth=2;strokeColor=#1b5e20" edge="1" parent="1" source="workflows" target="aicore">
|
||||
<mxGeometry width="50" height="50" relative="1" as="geometry">
|
||||
<mxPoint x="750" y="500" as="sourcePoint" />
|
||||
<mxPoint x="250" y="300" as="targetPoint" />
|
||||
</mxGeometry>
|
||||
</mxCell>
|
||||
|
||||
<mxCell id="edge16" value="" style="endArrow=classic;html=1;rounded=0;strokeWidth=2;strokeColor=#1b5e20" edge="1" parent="1" source="workflows" target="datamodels">
|
||||
<mxGeometry width="50" height="50" relative="1" as="geometry">
|
||||
<mxPoint x="800" y="500" as="sourcePoint" />
|
||||
<mxPoint x="450" y="300" as="targetPoint" />
|
||||
</mxGeometry>
|
||||
</mxCell>
|
||||
|
||||
<mxCell id="edge17" value="" style="endArrow=classic;html=1;rounded=0;strokeWidth=2;strokeColor=#1b5e20" edge="1" parent="1" source="workflows" target="services">
|
||||
<mxGeometry width="50" height="50" relative="1" as="geometry">
|
||||
<mxPoint x="750" y="500" as="sourcePoint" />
|
||||
<mxPoint x="450" y="450" as="targetPoint" />
|
||||
</mxGeometry>
|
||||
</mxCell>
|
||||
|
||||
<mxCell id="edge18" value="" style="endArrow=classic;html=1;rounded=0;strokeWidth=2;strokeColor=#1b5e20" edge="1" parent="1" source="workflows" target="shared">
|
||||
<mxGeometry width="50" height="50" relative="1" as="geometry">
|
||||
<mxPoint x="800" y="500" as="sourcePoint" />
|
||||
<mxPoint x="650" y="300" as="targetPoint" />
|
||||
</mxGeometry>
|
||||
</mxCell>
|
||||
|
||||
<!-- Feature layer dependencies -->
|
||||
<mxCell id="edge19" value="✅" style="endArrow=classic;html=1;rounded=0;strokeWidth=3;strokeColor=#e65100" edge="1" parent="1" source="features" target="interfaces">
|
||||
<mxGeometry width="50" height="50" relative="1" as="geometry">
|
||||
<mxPoint x="550" y="600" as="sourcePoint" />
|
||||
<mxPoint x="600" y="450" as="targetPoint" />
|
||||
</mxGeometry>
|
||||
</mxCell>
|
||||
|
||||
<mxCell id="edge20" value="✅" style="endArrow=classic;html=1;rounded=0;strokeWidth=3;strokeColor=#e65100" edge="1" parent="1" source="features" target="services">
|
||||
<mxGeometry width="50" height="50" relative="1" as="geometry">
|
||||
<mxPoint x="550" y="600" as="sourcePoint" />
|
||||
<mxPoint x="450" y="500" as="targetPoint" />
|
||||
</mxGeometry>
|
||||
</mxCell>
|
||||
|
||||
<mxCell id="edge21" value="" style="endArrow=classic;html=1;rounded=0;strokeWidth=2;strokeColor=#e65100" edge="1" parent="1" source="features" target="datamodels">
|
||||
<mxGeometry width="50" height="50" relative="1" as="geometry">
|
||||
<mxPoint x="550" y="600" as="sourcePoint" />
|
||||
<mxPoint x="450" y="300" as="targetPoint" />
|
||||
</mxGeometry>
|
||||
</mxCell>
|
||||
|
||||
<mxCell id="edge22" value="" style="endArrow=classic;html=1;rounded=0;strokeWidth=2;strokeColor=#e65100" edge="1" parent="1" source="features" target="workflows">
|
||||
<mxGeometry width="50" height="50" relative="1" as="geometry">
|
||||
<mxPoint x="600" y="600" as="sourcePoint" />
|
||||
<mxPoint x="750" y="500" as="targetPoint" />
|
||||
</mxGeometry>
|
||||
</mxCell>
|
||||
|
||||
<mxCell id="edge23" value="" style="endArrow=classic;html=1;rounded=0;strokeWidth=2;strokeColor=#e65100" edge="1" parent="1" source="features" target="shared">
|
||||
<mxGeometry width="50" height="50" relative="1" as="geometry">
|
||||
<mxPoint x="600" y="600" as="sourcePoint" />
|
||||
<mxPoint x="650" y="300" as="targetPoint" />
|
||||
</mxGeometry>
|
||||
</mxCell>
|
||||
|
||||
<!-- Routes dependencies -->
|
||||
<mxCell id="edge24" value="" style="endArrow=classic;html=1;rounded=0;strokeWidth=2;strokeColor=#880e4f" edge="1" parent="1" source="routes" target="interfaces">
|
||||
<mxGeometry width="50" height="50" relative="1" as="geometry">
|
||||
<mxPoint x="400" y="800" as="sourcePoint" />
|
||||
<mxPoint x="550" y="450" as="targetPoint" />
|
||||
</mxGeometry>
|
||||
</mxCell>
|
||||
|
||||
<mxCell id="edge25" value="" style="endArrow=classic;html=1;rounded=0;strokeWidth=2;strokeColor=#880e4f" edge="1" parent="1" source="routes" target="features">
|
||||
<mxGeometry width="50" height="50" relative="1" as="geometry">
|
||||
<mxPoint x="400" y="800" as="sourcePoint" />
|
||||
<mxPoint x="550" y="600" as="targetPoint" />
|
||||
</mxGeometry>
|
||||
</mxCell>
|
||||
|
||||
<mxCell id="edge26" value="" style="endArrow=classic;html=1;rounded=0;strokeWidth=2;strokeColor=#880e4f" edge="1" parent="1" source="routes" target="services">
|
||||
<mxGeometry width="50" height="50" relative="1" as="geometry">
|
||||
<mxPoint x="400" y="800" as="sourcePoint" />
|
||||
<mxPoint x="450" y="500" as="targetPoint" />
|
||||
</mxGeometry>
|
||||
</mxCell>
|
||||
|
||||
<mxCell id="edge27" value="" style="endArrow=classic;html=1;rounded=0;strokeWidth=2;strokeColor=#880e4f" edge="1" parent="1" source="routes" target="security">
|
||||
<mxGeometry width="50" height="50" relative="1" as="geometry">
|
||||
<mxPoint x="500" y="800" as="sourcePoint" />
|
||||
<mxPoint x="800" y="700" as="targetPoint" />
|
||||
</mxGeometry>
|
||||
</mxCell>
|
||||
|
||||
<mxCell id="edge28" value="" style="endArrow=classic;html=1;rounded=0;strokeWidth=2;strokeColor=#880e4f" edge="1" parent="1" source="routes" target="datamodels">
|
||||
<mxGeometry width="50" height="50" relative="1" as="geometry">
|
||||
<mxPoint x="400" y="800" as="sourcePoint" />
|
||||
<mxPoint x="450" y="300" as="targetPoint" />
|
||||
</mxGeometry>
|
||||
</mxCell>
|
||||
|
||||
<mxCell id="edge29" value="" style="endArrow=classic;html=1;rounded=0;strokeWidth=2;strokeColor=#880e4f" edge="1" parent="1" source="routes" target="shared">
|
||||
<mxGeometry width="50" height="50" relative="1" as="geometry">
|
||||
<mxPoint x="450" y="800" as="sourcePoint" />
|
||||
<mxPoint x="650" y="300" as="targetPoint" />
|
||||
</mxGeometry>
|
||||
</mxCell>
|
||||
|
||||
<!-- Security dependencies -->
|
||||
<mxCell id="edge30" value="" style="endArrow=classic;html=1;rounded=0;strokeWidth=2;strokeColor=#880e4f" edge="1" parent="1" source="security" target="interfaces">
|
||||
<mxGeometry width="50" height="50" relative="1" as="geometry">
|
||||
<mxPoint x="800" y="800" as="sourcePoint" />
|
||||
<mxPoint x="600" y="450" as="targetPoint" />
|
||||
</mxGeometry>
|
||||
</mxCell>
|
||||
|
||||
<mxCell id="edge31" value="" style="endArrow=classic;html=1;rounded=0;strokeWidth=2;strokeColor=#880e4f" edge="1" parent="1" source="security" target="datamodels">
|
||||
<mxGeometry width="50" height="50" relative="1" as="geometry">
|
||||
<mxPoint x="800" y="800" as="sourcePoint" />
|
||||
<mxPoint x="450" y="300" as="targetPoint" />
|
||||
</mxGeometry>
|
||||
</mxCell>
|
||||
|
||||
<mxCell id="edge32" value="" style="endArrow=classic;html=1;rounded=0;strokeWidth=2;strokeColor=#880e4f" edge="1" parent="1" source="security" target="shared">
|
||||
<mxGeometry width="50" height="50" relative="1" as="geometry">
|
||||
<mxPoint x="800" y="800" as="sourcePoint" />
|
||||
<mxPoint x="650" y="300" as="targetPoint" />
|
||||
</mxGeometry>
|
||||
</mxCell>
|
||||
|
||||
<!-- Legend -->
|
||||
<mxCell id="legend" value="Legend" style="text;html=1;strokeColor=none;fillColor=none;align=center;verticalAlign=middle;whiteSpace=wrap;rounded=0;fontStyle=1;fontSize=14" vertex="1" parent="1">
|
||||
<mxGeometry x="1000" y="100" width="100" height="30" as="geometry" />
|
||||
</mxCell>
|
||||
|
||||
<mxCell id="legend1" value="Foundation Layer" style="rounded=1;whiteSpace=wrap;html=1;fillColor=#e1f5ff;strokeColor=#01579b;strokeWidth=2" vertex="1" parent="1">
|
||||
<mxGeometry x="1000" y="140" width="150" height="30" as="geometry" />
|
||||
</mxCell>
|
||||
|
||||
<mxCell id="legend2" value="Data Access Layer" style="rounded=1;whiteSpace=wrap;html=1;fillColor=#f3e5f5;strokeColor=#4a148c;strokeWidth=2" vertex="1" parent="1">
|
||||
<mxGeometry x="1000" y="180" width="150" height="30" as="geometry" />
|
||||
</mxCell>
|
||||
|
||||
<mxCell id="legend3" value="Business Logic Layer" style="rounded=1;whiteSpace=wrap;html=1;fillColor=#e8f5e9;strokeColor=#1b5e20;strokeWidth=2" vertex="1" parent="1">
|
||||
<mxGeometry x="1000" y="220" width="150" height="30" as="geometry" />
|
||||
</mxCell>
|
||||
|
||||
<mxCell id="legend4" value="Feature Layer" style="rounded=1;whiteSpace=wrap;html=1;fillColor=#fff3e0;strokeColor=#e65100;strokeWidth=2" vertex="1" parent="1">
|
||||
<mxGeometry x="1000" y="260" width="150" height="30" as="geometry" />
|
||||
</mxCell>
|
||||
|
||||
<mxCell id="legend5" value="API Layer" style="rounded=1;whiteSpace=wrap;html=1;fillColor=#fce4ec;strokeColor=#880e4f;strokeWidth=2" vertex="1" parent="1">
|
||||
<mxGeometry x="1000" y="300" width="150" height="30" as="geometry" />
|
||||
</mxCell>
|
||||
|
||||
<mxCell id="legend6" value="✅ Correct Direction" style="text;html=1;strokeColor=none;fillColor=none;align=left;verticalAlign=middle;whiteSpace=wrap;rounded=0" vertex="1" parent="1">
|
||||
<mxGeometry x="1000" y="350" width="150" height="20" as="geometry" />
|
||||
</mxCell>
|
||||
|
||||
<mxCell id="legend7" value="--- Callback Registry" style="text;html=1;strokeColor=none;fillColor=none;align=left;verticalAlign=middle;whiteSpace=wrap;rounded=0;dashed=1" vertex="1" parent="1">
|
||||
<mxGeometry x="1000" y="380" width="150" height="20" as="geometry" />
|
||||
</mxCell>
|
||||
|
||||
<!-- Status Box -->
|
||||
<mxCell id="status" value="✅ ZERO VIOLATIONS
Perfect Architectural Compliance" style="rounded=1;whiteSpace=wrap;html=1;fillColor=#c8e6c9;strokeColor=#2e7d32;strokeWidth=3;fontStyle=1;fontSize=12" vertex="1" parent="1">
|
||||
<mxGeometry x="1000" y="450" width="200" height="60" as="geometry" />
|
||||
</mxCell>
|
||||
|
||||
</root>
|
||||
</mxGraphModel>
|
||||
</diagram>
|
||||
</mxfile>
|
||||
|
||||
407
modules/AUTOMATION_FEATURE_ANALYSIS.md
Normal file
407
modules/AUTOMATION_FEATURE_ANALYSIS.md
Normal file
|
|
@ -0,0 +1,407 @@
|
|||
# Automation Feature Analysis: Moving Automation Handler to Features Layer
|
||||
|
||||
## Executive Summary
|
||||
|
||||
**Status: ✅ HIGHLY RECOMMENDED - Architectural Improvement**
|
||||
|
||||
Moving automation workflow handler functionality from `interfaces/interfaceDbChatObjects.py` to a new feature in `features/` is **architecturally correct** and aligns with separation of concerns.
|
||||
|
||||
---
|
||||
|
||||
## Current Architecture Analysis
|
||||
|
||||
### Current Location: `interfaces/interfaceDbChatObjects.py`
|
||||
|
||||
**Automation-related methods:**
|
||||
1. `executeAutomation(automationId: str)` - Executes automation workflow immediately (test mode)
|
||||
2. `syncAutomationEvents()` - Syncs scheduler with all active automations
|
||||
3. `_createAutomationEventHandler(automationId: str)` - Creates event handler for scheduled execution
|
||||
4. `_parseScheduleToCron(schedule: str)` - Parses schedule string to cron kwargs
|
||||
5. `_planToPrompt(plan: Dict)` - Converts plan structure to prompt string
|
||||
6. `_replacePlaceholders(template: str, placeholders: Dict)` - Replaces placeholders in template
|
||||
|
||||
**Dependencies:**
|
||||
- Uses `getAutomationDefinition()` - Database access (should stay in interface)
|
||||
- Uses `chatStart()` from `features.chatPlayground` - Already imports from features
|
||||
- Uses `eventManager` from `shared.eventManagement` - Foundation layer
|
||||
- Creates workflows using `WorkflowModeEnum.WORKFLOW_AUTOMATION`
|
||||
|
||||
---
|
||||
|
||||
## Why This Should Be a Feature
|
||||
|
||||
### 1. **Business Logic vs. Data Access**
|
||||
|
||||
**Current Problem:**
|
||||
- Automation execution logic is **business logic** (orchestration, workflow creation)
|
||||
- It's mixed with **data access** (interface layer)
|
||||
- Interface layer should only provide data access, not business orchestration
|
||||
|
||||
**After Move:**
|
||||
- Interface layer: `getAutomationDefinition()`, `saveAutomationDefinition()` (data access)
|
||||
- Feature layer: `executeAutomation()`, `syncAutomationEvents()` (business logic)
|
||||
|
||||
### 2. **Feature Pattern Consistency**
|
||||
|
||||
**Existing Features Pattern:**
|
||||
- `features/chatPlayground/` - Chat workflow execution
|
||||
- `features/chatAlthaus/` - Scheduled data updates
|
||||
- `features/syncDelta/` - Sync management
|
||||
- `features/neutralizePlayground/` - Neutralization workflows
|
||||
|
||||
**Automation Handler Pattern:**
|
||||
- Scheduled execution (like `chatAlthaus`)
|
||||
- Workflow orchestration (like `chatPlayground`)
|
||||
- Event-driven (like `syncDelta`)
|
||||
|
||||
**Conclusion:** Automation handler fits the feature pattern perfectly.
|
||||
|
||||
### 3. **Dependency Direction**
|
||||
|
||||
**Current Violation:**
|
||||
- `interfaces/` imports from `features/` (line 1927: `from modules.features.chatPlayground.mainChatPlayground import chatStart`)
|
||||
- This creates bidirectional dependency: `interfaces/ ↔ features/`
|
||||
|
||||
**After Move:**
|
||||
- `features/automation/` imports from `interfaces/` (correct direction)
|
||||
- `features/automation/` imports from `features/chatPlayground/` (feature-to-feature)
|
||||
- Eliminates `interfaces/ → features/` import
|
||||
|
||||
### 4. **Separation of Concerns**
|
||||
|
||||
**Interface Layer Should:**
|
||||
- ✅ Provide data access (`getAutomationDefinition`, `saveAutomationDefinition`)
|
||||
- ✅ Handle CRUD operations
|
||||
- ❌ NOT orchestrate workflows
|
||||
- ❌ NOT manage scheduling
|
||||
- ❌ NOT execute business logic
|
||||
|
||||
**Feature Layer Should:**
|
||||
- ✅ Orchestrate workflows
|
||||
- ✅ Manage scheduling
|
||||
- ✅ Execute business logic
|
||||
- ✅ Coordinate between services and interfaces
|
||||
|
||||
---
|
||||
|
||||
## Proposed Architecture
|
||||
|
||||
### New Structure: `features/automation/`
|
||||
|
||||
```
|
||||
features/automation/
|
||||
├── mainAutomation.py # Main automation service
|
||||
│ ├── executeAutomation() # Execute automation workflow
|
||||
│ ├── syncAutomationEvents() # Sync scheduler with automations
|
||||
│ └── _createAutomationEventHandler() # Create event handler
|
||||
└── subAutomationUtils.py # Utility functions
|
||||
├── _parseScheduleToCron() # Parse schedule to cron
|
||||
├── _planToPrompt() # Convert plan to prompt
|
||||
└── _replacePlaceholders() # Replace template placeholders
|
||||
```
|
||||
|
||||
### Interface Layer (Keep)
|
||||
|
||||
```
|
||||
interfaces/interfaceDbChatObjects.py
|
||||
├── getAutomationDefinition() # Data access - KEEP
|
||||
├── saveAutomationDefinition() # Data access - KEEP
|
||||
└── (other CRUD methods) # Data access - KEEP
|
||||
```
|
||||
|
||||
### Feature Lifecycle Integration
|
||||
|
||||
```
|
||||
features/featuresLifecycle.py
|
||||
├── start()
|
||||
│ └── from features.automation import mainAutomation
|
||||
│ mainAutomation.startScheduler(eventUser)
|
||||
└── stop()
|
||||
└── mainAutomation.stopScheduler()
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Detailed Functionality Analysis
|
||||
|
||||
### Functions to Move
|
||||
|
||||
#### 1. `executeAutomation(automationId: str)` → `features/automation/mainAutomation.py`
|
||||
|
||||
**Current Implementation:**
|
||||
- Loads automation definition (calls interface method)
|
||||
- Replaces placeholders in template
|
||||
- Creates UserInputRequest
|
||||
- Calls `chatStart()` from `features.chatPlayground`
|
||||
- Returns ChatWorkflow
|
||||
|
||||
**Dependencies:**
|
||||
- `getAutomationDefinition()` - Interface method (import from interface)
|
||||
- `_replacePlaceholders()` - Utility (move to feature)
|
||||
- `_planToPrompt()` - Utility (move to feature)
|
||||
- `chatStart()` - Feature method (import from feature)
|
||||
- `getInterface()` - Interface factory (import from interface)
|
||||
|
||||
**After Move:**
|
||||
```python
|
||||
# features/automation/mainAutomation.py
|
||||
from modules.interfaces.interfaceDbChatObjects import getInterface as getChatInterface
|
||||
from modules.interfaces.interfaceDbAppObjects import getInterface as getAppInterface
|
||||
from modules.features.chatPlayground.mainChatPlayground import chatStart
|
||||
from .subAutomationUtils import replacePlaceholders, planToPrompt
|
||||
|
||||
async def executeAutomation(automationId: str, chatInterface) -> ChatWorkflow:
|
||||
"""Execute automation workflow immediately."""
|
||||
# Load automation (uses interface)
|
||||
automation = chatInterface.getAutomationDefinition(automationId)
|
||||
# ... rest of logic
|
||||
```
|
||||
|
||||
#### 2. `syncAutomationEvents()` → `features/automation/mainAutomation.py`
|
||||
|
||||
**Current Implementation:**
|
||||
- Gets all automation definitions (calls interface method)
|
||||
- Parses schedules
|
||||
- Registers cron jobs with eventManager
|
||||
- Creates event handlers
|
||||
|
||||
**Dependencies:**
|
||||
- `getRecordset()` - Interface method (via chatInterface)
|
||||
- `_parseScheduleToCron()` - Utility (move to feature)
|
||||
- `_createAutomationEventHandler()` - Handler creation (move to feature)
|
||||
- `eventManager` - Foundation layer (import from shared)
|
||||
|
||||
**After Move:**
|
||||
```python
|
||||
# features/automation/mainAutomation.py
|
||||
from modules.shared.eventManagement import eventManager
|
||||
from modules.interfaces.interfaceDbChatObjects import getInterface as getChatInterface
|
||||
from .subAutomationUtils import parseScheduleToCron
|
||||
|
||||
async def syncAutomationEvents(chatInterface) -> Dict[str, Any]:
|
||||
"""Sync scheduler with all active automations."""
|
||||
# Get automations (uses interface)
|
||||
allAutomations = chatInterface.db.getRecordset(AutomationDefinition)
|
||||
# ... rest of logic
|
||||
```
|
||||
|
||||
#### 3. `_createAutomationEventHandler(automationId: str)` → `features/automation/mainAutomation.py`
|
||||
|
||||
**Current Implementation:**
|
||||
- Creates async handler function
|
||||
- Gets event user
|
||||
- Loads automation
|
||||
- Executes automation with creator user context
|
||||
|
||||
**Dependencies:**
|
||||
- `getRootInterface()` - Interface factory (import from interface)
|
||||
- `getInterface()` - Interface factories (import from interfaces)
|
||||
- `executeAutomation()` - Will be in same module
|
||||
|
||||
**After Move:**
|
||||
```python
|
||||
# features/automation/mainAutomation.py
|
||||
def createAutomationEventHandler(automationId: str):
|
||||
"""Create event handler function for scheduled automation."""
|
||||
async def handler():
|
||||
# Uses interfaces and executeAutomation from same module
|
||||
await executeAutomation(automationId, eventInterface)
|
||||
return handler
|
||||
```
|
||||
|
||||
#### 4. Utility Functions → `features/automation/subAutomationUtils.py`
|
||||
|
||||
**Functions:**
|
||||
- `_parseScheduleToCron(schedule: str)` - Parse schedule to cron kwargs
|
||||
- `_planToPrompt(plan: Dict)` - Convert plan to prompt string
|
||||
- `_replacePlaceholders(template: str, placeholders: Dict)` - Replace placeholders
|
||||
|
||||
**Dependencies:**
|
||||
- No external dependencies (pure utility functions)
|
||||
|
||||
---
|
||||
|
||||
## Migration Plan
|
||||
|
||||
### Phase 1: Create Feature Structure
|
||||
|
||||
1. Create `features/automation/` directory
|
||||
2. Create `features/automation/__init__.py`
|
||||
3. Create `features/automation/mainAutomation.py`
|
||||
4. Create `features/automation/subAutomationUtils.py`
|
||||
|
||||
### Phase 2: Move Functions
|
||||
|
||||
1. Move utility functions to `subAutomationUtils.py`
|
||||
2. Move `executeAutomation()` to `mainAutomation.py`
|
||||
3. Move `syncAutomationEvents()` to `mainAutomation.py`
|
||||
4. Move `_createAutomationEventHandler()` to `mainAutomation.py`
|
||||
|
||||
### Phase 3: Update Dependencies
|
||||
|
||||
1. Update `features/featuresLifecycle.py` to use new feature
|
||||
2. Update `routes/routeAdminAutomationEvents.py` to use new feature
|
||||
3. Update any other call sites
|
||||
|
||||
### Phase 4: Cleanup Interface
|
||||
|
||||
1. Remove moved functions from `interfaceDbChatObjects.py`
|
||||
2. Keep only data access methods (`getAutomationDefinition`, etc.)
|
||||
3. Remove import from `features.chatPlayground` from interface
|
||||
|
||||
### Phase 5: Update Documentation
|
||||
|
||||
1. Update `BIDIRECTIONAL_IMPORTS.md` to reflect resolved dependency
|
||||
2. Document new feature structure
|
||||
|
||||
---
|
||||
|
||||
## Benefits
|
||||
|
||||
### ✅ Architectural Benefits
|
||||
|
||||
1. **Correct Separation of Concerns**
|
||||
- Interface layer: Data access only
|
||||
- Feature layer: Business logic and orchestration
|
||||
|
||||
2. **Resolves Bidirectional Dependency**
|
||||
- Eliminates `interfaces/ → features/` import
|
||||
- Only `features/ → interfaces/` remains (correct direction)
|
||||
|
||||
3. **Consistency with Existing Patterns**
|
||||
- Matches other feature implementations
|
||||
- Follows established architecture
|
||||
|
||||
4. **Better Testability**
|
||||
- Feature logic can be tested independently
|
||||
- Interface layer remains focused
|
||||
|
||||
### ✅ Maintainability Benefits
|
||||
|
||||
1. **Clearer Code Organization**
|
||||
- Automation logic in one place
|
||||
- Easier to find and modify
|
||||
|
||||
2. **Reduced Coupling**
|
||||
- Interface layer doesn't depend on features
|
||||
- Features depend on interfaces (correct direction)
|
||||
|
||||
3. **Easier to Extend**
|
||||
- New automation features can be added to feature module
|
||||
- Interface layer remains stable
|
||||
|
||||
---
|
||||
|
||||
## Risks and Considerations
|
||||
|
||||
### 🟢 Low Risk
|
||||
|
||||
- **Functionality preservation** - Logic doesn't change, only location
|
||||
- **Interface methods remain** - Data access methods stay in interface
|
||||
- **Lazy imports** - Already using lazy imports for event handlers
|
||||
|
||||
### 🟡 Medium Risk
|
||||
|
||||
- **Call site updates** - Need to update routes and lifecycle
|
||||
- **Interface method access** - Feature needs to call interface methods
|
||||
- **Event user context** - Need to ensure proper user context handling
|
||||
|
||||
### 🔴 Potential Issues
|
||||
|
||||
1. **Interface method access** - Feature needs `chatInterface` instance
|
||||
- **Solution:** Pass interface instance as parameter or create in feature
|
||||
|
||||
2. **Event handler context** - Event handlers need interface access
|
||||
- **Solution:** Create interface instances in handler (already doing this)
|
||||
|
||||
3. **Backward compatibility** - Existing code calling `chatInterface.executeAutomation()`
|
||||
- **Solution:** Update all call sites, or create wrapper method in interface (deprecated)
|
||||
|
||||
---
|
||||
|
||||
## Call Sites Analysis
|
||||
|
||||
### Current Call Sites
|
||||
|
||||
1. **`features/featuresLifecycle.py` (line 20)**
|
||||
```python
|
||||
await chatInterface.syncAutomationEvents()
|
||||
```
|
||||
**Update:** Import and call feature directly
|
||||
|
||||
2. **`routes/routeAdminAutomationEvents.py` (line 97)**
|
||||
```python
|
||||
result = await chatInterface.syncAutomationEvents()
|
||||
```
|
||||
**Update:** Import and call feature directly
|
||||
|
||||
3. **`interfaces/interfaceDbChatObjects.py` (lines 1683, 1714, 1744)**
|
||||
```python
|
||||
asyncio.create_task(self.syncAutomationEvents())
|
||||
```
|
||||
**Update:** Remove (will be handled by feature lifecycle)
|
||||
|
||||
4. **`_createAutomationEventHandler()` (line 2105)**
|
||||
```python
|
||||
await creatorInterface.executeAutomation(automationId)
|
||||
```
|
||||
**Update:** Call feature method instead
|
||||
|
||||
### Proposed Call Sites
|
||||
|
||||
1. **`features/featuresLifecycle.py`**
|
||||
```python
|
||||
from modules.features.automation import mainAutomation
|
||||
await mainAutomation.syncAutomationEvents(chatInterface)
|
||||
```
|
||||
|
||||
2. **`routes/routeAdminAutomationEvents.py`**
|
||||
```python
|
||||
from modules.features.automation import mainAutomation
|
||||
result = await mainAutomation.syncAutomationEvents(chatInterface)
|
||||
```
|
||||
|
||||
3. **`features/automation/mainAutomation.py` (event handler)**
|
||||
```python
|
||||
from .mainAutomation import executeAutomation
|
||||
await executeAutomation(automationId, eventInterface)
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Implementation Checklist
|
||||
|
||||
- [ ] Create `features/automation/` directory structure
|
||||
- [ ] Create `features/automation/__init__.py`
|
||||
- [ ] Create `features/automation/mainAutomation.py`
|
||||
- [ ] Create `features/automation/subAutomationUtils.py`
|
||||
- [ ] Move `_parseScheduleToCron()` to `subAutomationUtils.py`
|
||||
- [ ] Move `_planToPrompt()` to `subAutomationUtils.py`
|
||||
- [ ] Move `_replacePlaceholders()` to `subAutomationUtils.py`
|
||||
- [ ] Move `executeAutomation()` to `mainAutomation.py`
|
||||
- [ ] Move `syncAutomationEvents()` to `mainAutomation.py`
|
||||
- [ ] Move `_createAutomationEventHandler()` to `mainAutomation.py`
|
||||
- [ ] Update `features/featuresLifecycle.py` to use feature
|
||||
- [ ] Update `routes/routeAdminAutomationEvents.py` to use feature
|
||||
- [ ] Remove moved functions from `interfaceDbChatObjects.py`
|
||||
- [ ] Remove `features.chatPlayground` import from `interfaceDbChatObjects.py`
|
||||
- [ ] Update `BIDIRECTIONAL_IMPORTS.md`
|
||||
- [ ] Test automation execution (manual)
|
||||
- [ ] Test automation scheduling (scheduled)
|
||||
- [ ] Verify no circular dependencies
|
||||
|
||||
---
|
||||
|
||||
## Conclusion
|
||||
|
||||
**✅ STRONGLY RECOMMENDED: Move to Features Layer**
|
||||
|
||||
This refactoring is:
|
||||
- **Architecturally correct** - Business logic belongs in features, not interfaces
|
||||
- **Resolves dependency violation** - Eliminates `interfaces/ → features/` import
|
||||
- **Consistent with patterns** - Matches existing feature implementations
|
||||
- **Low risk** - Logic doesn't change, only location
|
||||
- **Improves maintainability** - Clearer separation of concerns
|
||||
|
||||
**Recommendation: PROCEED** with moving automation handler functionality to `features/automation/` following the plan above.
|
||||
|
||||
406
modules/BIDIRECTIONAL_IMPORTS.md
Normal file
406
modules/BIDIRECTIONAL_IMPORTS.md
Normal file
|
|
@ -0,0 +1,406 @@
|
|||
# Bidirectional Import Analysis
|
||||
|
||||
## Summary
|
||||
|
||||
After refactoring extraction functions and automation handler, **ALL bidirectional dependencies have been RESOLVED**:
|
||||
|
||||
**Current Status:**
|
||||
- ✅ **interfaces/ → services/**: **RESOLVED** (no imports)
|
||||
- ✅ **interfaces/ → features/**: **RESOLVED** (uses callback registry, no direct imports)
|
||||
- ✅ **services/ → interfaces/**: **UNIDIRECTIONAL** (correct dependency direction)
|
||||
- ✅ **services/ → features/**: **NONE** (no imports)
|
||||
- ✅ **features/ → interfaces/**: **UNIDIRECTIONAL** (correct dependency direction)
|
||||
- ✅ **features/ → services/**: **1 lazy import** (correct direction)
|
||||
|
||||
**Result:** ✅ **ZERO VIOLATIONS** - Perfect architectural compliance achieved.
|
||||
|
||||
---
|
||||
|
||||
## Dependency Diagram
|
||||
|
||||
### Mermaid Diagram
|
||||
|
||||
```mermaid
|
||||
graph TB
|
||||
%% Foundation Layer (no dependencies)
|
||||
shared[shared/<br/>Foundation]
|
||||
datamodels[datamodels/<br/>Foundation]
|
||||
aicore[aicore/<br/>Infrastructure]
|
||||
connectors[connectors/<br/>Infrastructure]
|
||||
|
||||
%% Data Layer
|
||||
interfaces[interfaces/<br/>Data Access<br/>✅ No violations]
|
||||
|
||||
%% Business Logic Layer
|
||||
services[services/<br/>Business Logic<br/>✅ Unidirectional]
|
||||
workflows[workflows/<br/>Business Logic]
|
||||
|
||||
%% Feature Layer
|
||||
features[features/<br/>Features<br/>✅ Unidirectional]
|
||||
|
||||
%% API Layer
|
||||
routes[routes/<br/>API Layer]
|
||||
security[security/<br/>Security]
|
||||
|
||||
%% Foundation dependencies
|
||||
datamodels -->|imports| shared
|
||||
aicore -->|imports| datamodels
|
||||
aicore -->|imports| shared
|
||||
connectors -->|imports| datamodels
|
||||
connectors -->|imports| shared
|
||||
|
||||
%% Interface layer (foundation only)
|
||||
interfaces -->|imports| aicore
|
||||
interfaces -->|imports| connectors
|
||||
interfaces -->|imports| datamodels
|
||||
interfaces -.->|callbackRegistry| shared
|
||||
|
||||
%% Service layer (interfaces only)
|
||||
services -->|✅ imports| interfaces
|
||||
services -->|imports| aicore
|
||||
services -->|imports| datamodels
|
||||
services -->|imports| security
|
||||
services -->|imports| shared
|
||||
|
||||
%% Workflow layer
|
||||
workflows -->|imports| aicore
|
||||
workflows -->|imports| datamodels
|
||||
workflows -->|imports| services
|
||||
workflows -->|imports| shared
|
||||
|
||||
%% Feature layer (interfaces + services)
|
||||
features -->|✅ imports| interfaces
|
||||
features -->|✅ imports| services
|
||||
features -->|imports| datamodels
|
||||
features -->|imports| workflows
|
||||
features -->|imports| shared
|
||||
|
||||
%% API layer
|
||||
routes -->|imports| interfaces
|
||||
routes -->|imports| features
|
||||
routes -->|imports| services
|
||||
routes -->|imports| security
|
||||
routes -->|imports| datamodels
|
||||
routes -->|imports| shared
|
||||
|
||||
%% Security layer
|
||||
security -->|imports| interfaces
|
||||
security -->|imports| datamodels
|
||||
security -->|imports| shared
|
||||
|
||||
%% Styling
|
||||
classDef foundation fill:#e1f5ff,stroke:#01579b,stroke-width:2px
|
||||
classDef data fill:#f3e5f5,stroke:#4a148c,stroke-width:3px
|
||||
classDef business fill:#e8f5e9,stroke:#1b5e20,stroke-width:2px
|
||||
classDef feature fill:#fff3e0,stroke:#e65100,stroke-width:2px
|
||||
classDef api fill:#fce4ec,stroke:#880e4f,stroke-width:2px
|
||||
|
||||
class shared,datamodels,aicore,connectors foundation
|
||||
class interfaces data
|
||||
class services,workflows business
|
||||
class features feature
|
||||
class routes,security api
|
||||
```
|
||||
|
||||
### Draw.io Diagram
|
||||
|
||||
A detailed draw.io diagram is available in `DEPENDENCY_DIAGRAM.drawio` with:
|
||||
- Color-coded layers (Foundation, Data, Business Logic, Features, API)
|
||||
- Arrow directions showing import relationships
|
||||
- ✅ markers on correct dependency directions
|
||||
- Dashed line for callback registry pattern
|
||||
- Status box showing zero violations
|
||||
|
||||
**Key Visual Elements:**
|
||||
- **Thick green arrows (✅)**: Correct dependency directions (services→interfaces, features→interfaces, features→services)
|
||||
- **Dashed purple line**: Callback registry pattern (interfaces→shared, decoupled from features)
|
||||
- **Color coding**: Each layer has distinct colors for easy identification
|
||||
- **Status indicators**: ✅ markers show compliance, "No violations" labels confirm architectural correctness
|
||||
|
||||
---
|
||||
|
||||
## Detailed Analysis
|
||||
|
||||
### 1. interfaces/ → services/ ✅ RESOLVED
|
||||
|
||||
**Current State:**
|
||||
- ✅ **No imports from services/** in `interfaces/`
|
||||
- All extraction-related functions moved to `services/serviceExtraction/`
|
||||
- Dependency violations resolved
|
||||
|
||||
**Impact:** Major architectural improvement - interfaces no longer depend on services.
|
||||
|
||||
---
|
||||
|
||||
### 2. interfaces/ → features/ ✅ RESOLVED
|
||||
|
||||
**Previous State:**
|
||||
- `interfaceDbChatObjects.py` (line 1754): Lazy import in `_triggerAutomationSync()` helper method
|
||||
```python
|
||||
from modules.features.automation import syncAutomationEvents
|
||||
```
|
||||
|
||||
**Current State:**
|
||||
- ✅ **No imports from features/** in `interfaces/`
|
||||
- Uses callback registry pattern (`shared.callbackRegistry`) for decoupled notifications
|
||||
- Interface triggers callbacks without knowing which features are listening
|
||||
- Feature registers callback in `featuresLifecycle.py` startup
|
||||
|
||||
**Refactoring:**
|
||||
- Created `shared/callbackRegistry.py` - decoupled event notification system
|
||||
- Interface calls `callbackRegistry.trigger('automation.changed', self)` instead of importing feature
|
||||
- Feature registers callback on startup: `callbackRegistry.register('automation.changed', onAutomationChanged)`
|
||||
|
||||
**Impact:** Perfect separation - interface doesn't know about features, uses shared callback registry.
|
||||
|
||||
---
|
||||
|
||||
### 3. services/ → interfaces/ ✅ CORRECT DIRECTION
|
||||
|
||||
**Current State:**
|
||||
- `serviceAi/mainServiceAi.py`: Imports `AiObjects` from `interfaceAiObjects`
|
||||
- `serviceExtraction/mainServiceExtraction.py`: Lazy import from `interfaceDbComponentObjects`
|
||||
- `serviceUtils/mainServiceUtils.py`: Lazy import from `interfaceDbChatObjects`
|
||||
- `serviceTicket/mainServiceTicket.py`: Imports from `interfaceTicketObjects`
|
||||
- `services/__init__.py`: Lazy imports from multiple interfaces
|
||||
|
||||
**Impact:** This is **correct** - services should use interfaces for data access. This follows the dependency rule: `services/` → `interfaces/` ✅
|
||||
|
||||
**Note:** This is **unidirectional** (services → interfaces), not bidirectional.
|
||||
|
||||
---
|
||||
|
||||
### 4. services/ → features/ ✅ NONE
|
||||
|
||||
**Current State:**
|
||||
- ✅ **No imports from features/** in `services/`
|
||||
|
||||
**Impact:** Services correctly do not depend on features.
|
||||
|
||||
---
|
||||
|
||||
### 5. features/ → interfaces/ ✅ CORRECT DIRECTION
|
||||
|
||||
**Current State:**
|
||||
- `features/automation/mainAutomation.py`: Imports from `interfaceDbChatObjects`, `interfaceDbAppObjects`
|
||||
- `features/featuresLifecycle.py`: Imports from `interfaceDbAppObjects`, `interfaceDbChatObjects` (lazy)
|
||||
|
||||
**Impact:** This is **correct** - features should use interfaces for data access. This follows the dependency rule: `features/` → `interfaces/` ✅
|
||||
|
||||
**Note:** This is **unidirectional** (features → interfaces), not bidirectional.
|
||||
|
||||
---
|
||||
|
||||
### 6. features/ → services/ ✅ CORRECT DIRECTION
|
||||
|
||||
**Current State:**
|
||||
- `features/neutralizePlayground/mainNeutralizePlayground.py` (line 123): Lazy import from `serviceSharepoint`
|
||||
|
||||
**Impact:** This is **correct** - features can use services. This follows the dependency rule: `features/` → `services/` ✅
|
||||
|
||||
---
|
||||
|
||||
## Complete Import Matrix (Fact-Based)
|
||||
|
||||
### aicore/
|
||||
- **Imports from:** `datamodels/`, `shared/`
|
||||
- **Imported by:** `interfaces/`, `services/`, `workflows/`
|
||||
- **Bidirectional:** None ✅
|
||||
|
||||
### connectors/
|
||||
- **Imports from:** `datamodels/`, `shared/`
|
||||
- **Imported by:** `interfaces/`
|
||||
- **Bidirectional:** None ✅
|
||||
|
||||
### datamodels/
|
||||
- **Imports from:** `shared/`
|
||||
- **Imported by:** `aicore/`, `connectors/`, `features/`, `interfaces/`, `routes/`, `security/`, `services/`, `workflows/`
|
||||
- **Bidirectional:** None ✅
|
||||
|
||||
### features/
|
||||
- **Imports from:** `datamodels/`, `interfaces/`, `services/`, `shared/`, `workflows/`
|
||||
- **Imported by:** `routes/`
|
||||
- **✅ UNIDIRECTIONAL:** No longer imported by `interfaces/`
|
||||
|
||||
**Detailed imports:**
|
||||
- From `interfaces/`: `interfaceDbChatObjects`, `interfaceDbAppObjects`
|
||||
- From `services/`: `serviceSharepoint` (lazy, in `neutralizePlayground`)
|
||||
- From `features/`: `chatPlayground` (in `automation`), `syncDelta`, `chatAlthaus` (in `featuresLifecycle`)
|
||||
|
||||
### interfaces/
|
||||
- **Imports from:** `aicore/`, `connectors/`, `datamodels/`, `shared/`
|
||||
- **Imported by:** `features/`, `routes/`, `security/`, `services/`
|
||||
- **✅ RESOLVED:** No longer imports from `services/` or `features/`
|
||||
|
||||
**Detailed imports:**
|
||||
- From `shared/`: `callbackRegistry` (for decoupled event notifications), `eventManagement` (for event removal in delete), `timeUtils`, `configuration`, `debugLogger`
|
||||
- From `interfaces/`: Internal imports (`interfaceDbChatAccess`, `interfaceDbAppAccess`, `interfaceDbComponentAccess`)
|
||||
|
||||
### routes/
|
||||
- **Imports from:** `datamodels/`, `features/`, `interfaces/`, `security/`, `services/`, `shared/`
|
||||
- **Imported by:** None (top-level API layer)
|
||||
- **Bidirectional:** None ✅
|
||||
|
||||
**Detailed imports:**
|
||||
- From `interfaces/`: `interfaceDbChatObjects`, `interfaceDbAppObjects`, `interfaceDbComponentObjects`, `interfaceVoiceObjects`
|
||||
- From `features/**: `features.automation`, `features.chatPlayground`, `features.neutralizePlayground`
|
||||
|
||||
### security/
|
||||
- **Imports from:** `datamodels/`, `interfaces/`, `shared/`
|
||||
- **Imported by:** `routes/`, `services/`
|
||||
- **Bidirectional:** None ✅
|
||||
|
||||
**Detailed imports:**
|
||||
- From `interfaces/**: `interfaceDbAppObjects` (lazy imports)
|
||||
|
||||
### services/
|
||||
- **Imports from:** `aicore/`, `datamodels/`, `interfaces/`, `security/`, `shared/`
|
||||
- **Imported by:** `features/`, `routes/`, `workflows/`
|
||||
- **✅ UNIDIRECTIONAL:** Only imports from `interfaces/` (correct direction)
|
||||
- **✅ RESOLVED:** No longer imported by `interfaces/`
|
||||
|
||||
**Detailed imports:**
|
||||
- From `interfaces/**: `interfaceAiObjects`, `interfaceDbComponentObjects` (lazy), `interfaceDbChatObjects` (lazy), `interfaceTicketObjects`
|
||||
- From `services/**: Internal imports (service-to-service)
|
||||
|
||||
### shared/
|
||||
- **Imports from:** None (foundation layer)
|
||||
- **Imported by:** `aicore/`, `connectors/`, `datamodels/`, `features/`, `interfaces/`, `routes/`, `security/`, `services/`, `workflows/`
|
||||
- **Bidirectional:** None ✅
|
||||
|
||||
### workflows/
|
||||
- **Imports from:** `aicore/`, `datamodels/`, `services/`, `shared/`
|
||||
- **Imported by:** `features/`
|
||||
- **Bidirectional:** None ✅
|
||||
|
||||
**Detailed imports:**
|
||||
- From `services/**: `serviceGeneration` (lazy, in `methodAi.py`)
|
||||
|
||||
---
|
||||
|
||||
## Refactoring Impact Summary
|
||||
|
||||
### Before Refactoring:
|
||||
- ❌ **interfaces/ ↔ services/**: Bidirectional (violations)
|
||||
- `interfaces/` imported from `services/serviceExtraction/` (6 violations)
|
||||
- `services/` imported from `interfaces/` (correct)
|
||||
- ❌ **interfaces/ ↔ features/**: Bidirectional (violation)
|
||||
- `interfaces/` imported from `features.chatPlayground` (1 violation)
|
||||
- `features/` imported from `interfaces/` (correct)
|
||||
|
||||
### After Refactoring:
|
||||
- ✅ **interfaces/ → services/**: RESOLVED (no imports)
|
||||
- ✅ **services/ → interfaces/**: UNIDIRECTIONAL (correct direction)
|
||||
- ✅ **interfaces/ → features/**: RESOLVED (uses callback registry pattern)
|
||||
- ✅ **features/ → interfaces/**: UNIDIRECTIONAL (correct direction)
|
||||
- ✅ **features/ → services/**: CORRECT DIRECTION (1 lazy import)
|
||||
|
||||
---
|
||||
|
||||
## Specific Import Details
|
||||
|
||||
### interfaces/ → features/ ✅ RESOLVED
|
||||
|
||||
**Previous:** `interfaceDbChatObjects.py` (line 1754) had lazy import from `features.automation`
|
||||
```python
|
||||
from modules.features.automation import syncAutomationEvents
|
||||
```
|
||||
|
||||
**Current:** Uses `shared.callbackRegistry` pattern (line 1754):
|
||||
- Interface calls: `callbackRegistry.trigger('automation.changed', self)`
|
||||
- Feature registers callback in `featuresLifecycle.py`: `callbackRegistry.register('automation.changed', onAutomationChanged)`
|
||||
- **Zero direct imports** from features in interfaces
|
||||
- **Verification:** `grep "from modules.features" interfaces/` returns no matches ✅
|
||||
|
||||
### features/ → services/ (1 import, correct direction)
|
||||
|
||||
**File:** `features/neutralizePlayground/mainNeutralizePlayground.py`
|
||||
- **Line:** 123
|
||||
- **Import:** `from modules.services.serviceSharepoint.mainServiceSharepoint import SharepointService`
|
||||
- **Type:** Lazy import (inside method)
|
||||
- **Context:** Used for SharePoint file processing
|
||||
- **Status:** ✅ CORRECT - Features can import from services
|
||||
|
||||
---
|
||||
|
||||
## Recommendations
|
||||
|
||||
### ✅ Completed Improvements
|
||||
1. **Resolved interfaces/ → services/ violations** - Moved extraction functions to `serviceExtraction/`
|
||||
2. **Resolved interfaces/ → features/ violations** - Moved automation handler to `features/automation/`
|
||||
3. **Eliminated ALL bidirectional dependencies** - `interfaces/` no longer imports from `services/` or `features/`
|
||||
4. **Implemented callback registry pattern** - Decoupled event notifications using `shared.callbackRegistry`
|
||||
5. **Follows dependency rules perfectly** - All dependencies now follow correct direction:
|
||||
- `services/` → `interfaces/` ✅
|
||||
- `features/` → `interfaces/` ✅
|
||||
- `features/` → `services/` ✅
|
||||
- `interfaces/` → `shared/` only ✅
|
||||
|
||||
### Best Practices
|
||||
- ✅ Use lazy imports (inside functions) for dependencies when appropriate
|
||||
- ✅ Services correctly depend on interfaces (unidirectional)
|
||||
- ✅ Features correctly depend on interfaces and services (unidirectional)
|
||||
- ✅ Interfaces completely independent (only foundation layers)
|
||||
- ✅ Use callback registry for decoupled event notifications
|
||||
- ✅ Document dependency relationships clearly
|
||||
- ✅ Monitor for circular import errors
|
||||
|
||||
---
|
||||
|
||||
## Dependency Rule Compliance
|
||||
|
||||
### Current Rules:
|
||||
- ✅ **features/** → **services/** ✅ (correct)
|
||||
- ✅ **services/** → **interfaces/** ✅ (correct)
|
||||
- ✅ **features/** → **interfaces/** ✅ (correct)
|
||||
|
||||
### Status:
|
||||
- ✅ **interfaces/** → **services/**: **RESOLVED** (was violation, now compliant)
|
||||
- ✅ **services/** → **interfaces/**: **COMPLIANT** (correct direction)
|
||||
- ✅ **features/** → **interfaces/**: **COMPLIANT** (correct direction)
|
||||
- ✅ **features/** → **services/**: **COMPLIANT** (correct direction)
|
||||
- ✅ **interfaces/** → **features/**: **RESOLVED** (was violation, now uses callback registry)
|
||||
|
||||
---
|
||||
|
||||
## Conclusion
|
||||
|
||||
The refactoring successfully resolved **ALL bidirectional dependencies**:
|
||||
- ✅ **interfaces/ ↔ services/**: RESOLVED
|
||||
- ✅ **interfaces/ ↔ features/**: RESOLVED (using callback registry pattern)
|
||||
|
||||
The architecture now follows the intended dependency rules **perfectly**:
|
||||
- `interfaces/` only imports from foundation layers (`aicore/`, `connectors/`, `datamodels/`, `shared/`)
|
||||
- `services/` imports from `interfaces/` (correct direction)
|
||||
- `features/` imports from `interfaces/` and `services/` (correct direction)
|
||||
- **Zero violations** - perfect architectural compliance achieved through callback registry pattern
|
||||
|
||||
---
|
||||
|
||||
## Architecture Layers
|
||||
|
||||
The codebase follows a clean layered architecture:
|
||||
|
||||
1. **Foundation Layer** (`shared/`, `datamodels/`)
|
||||
- No dependencies on other modules
|
||||
- Used by all layers
|
||||
|
||||
2. **Infrastructure Layer** (`aicore/`, `connectors/`)
|
||||
- Depends only on foundation
|
||||
- Provides core capabilities
|
||||
|
||||
3. **Data Access Layer** (`interfaces/`)
|
||||
- Depends only on foundation and infrastructure
|
||||
- Provides data access abstraction
|
||||
|
||||
4. **Business Logic Layer** (`services/`, `workflows/`)
|
||||
- Depends on interfaces and foundation
|
||||
- Implements business logic
|
||||
|
||||
5. **Feature Layer** (`features/`)
|
||||
- Depends on interfaces, services, and foundation
|
||||
- Implements user-facing features
|
||||
|
||||
6. **API Layer** (`routes/`, `security/`)
|
||||
- Depends on all layers
|
||||
- Provides HTTP API endpoints
|
||||
1
modules/DEPENDENCY_DIAGRAM.drawio
Normal file
1
modules/DEPENDENCY_DIAGRAM.drawio
Normal file
|
|
@ -0,0 +1 @@
|
|||
<mxfile host="Electron" modified="2025-12-02T16:28:53.356Z" agent="5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) draw.io/20.3.0 Chrome/104.0.5112.114 Electron/20.1.3 Safari/537.36" etag="L77fz7tGmuPPsr8RbEBA" version="20.3.0" type="device"><diagram name="Module Dependencies" id="dependency-diagram">5Z1bb6M4FMc/DdLOQyUuIZfHNm1nK3Wno8lepH1zwSSoBFeGNO1++jWNSXzLNKW2MR1ppCbHBJLz//n4cIAzXjRfP3/F4HH1B0ph4YV++uxFl14YBmEYes0/P31pLcFkZ1niPKW2g2GR/wep0afWTZ7CituwRqio80femKCyhEnN2QDGaMtvlqGCP+ojWELJsEhAIVv/ydN6Ra0j3z8M/A7z5ao9dNiOrEG7NTVUK5CiLWOKrrxojhGqd6/Wz3NYNO5rHbP73PWR0f03w7CsT/kAOT6G6e5TT6DYQN587YVjL4yI46OL367RpkxBnaPyC/3y9UvrEtyMwWanAdl0u8pruHgESTO6JRQQ26peF3Q4y4tijgqEXz8bwSCLs4zYqxqjB8iM+EE8md3vR1pnh80+UFkv6PEVv5qaniCu4TNjol74CtEa1viFbEJHwxlVhEIZtQptGYVb24oRd0xtgEK13O/64HbygnperQLxKlg386SSlGCHfh014phXY2xRDJAnCENJiNbMiXBTZhgQd2ySeoPhJxRiH7naWO1bVILGb4TlacEO/VqKTPtUJC9riDPiLlkRdohT5JJEMLLpeUKGqi/MiHcVerO5N43J6DfU7C1HxWtEq7TKlkUwzmKVbCMQjKaJJFtkYiJNeNlmI1m2/WRjZZtqkK2C+ClXiXYY4CS72FR5SdQiG9+iZZ4cU+2vMk9zTOYh0QwUeufaNIvhTCVacB/D0Lcy18a8ZiNFTmBMsy3CD1nR5KqiaMzIW6p9NkEmQvAbKYKfMUUyCJo1RRbkMMBnaNTcw+zJMhL0fJVYcBw3K4YNsWbC9JnZFIt4r1ZI1Zo5oc6/3zRzBrxArHfOZAkcwUQlw3Tqw1FmRYZImDMTmzJUMNngvH5RrDztACfFgpo/oQ5i7LKqA0yXMJBEkJwMy/S8KZCQd0kBqipPeL8eRFCHkGOJ8THvVWiDE8WZ7mtJB+AlpJuzZYrmp/zU34w/Y4U7WxuGTbL5xNd0VD6mR/iOSIp7kHMkyLmfZu0udr+NfootwYg7Es5191+w3dHOEdKOXiXf/+yTKQjdpoA9+2YJEOlwhAIxuHamILJLQTRMChyMA0MlYOQ2AWLtZwCxQKzMDGVFiIdLgoPxQBsFY7sUjHuggNbfTqFArDyyFLCrhSMUxMcKFR9dFSSc9FIwGS4FYpxwhISxLhImdkmYDpcER/MD8aJmZxLE/MAwCTOJhAQUxT1IHn7AZV61lQgbZEQXKahWtAr0UUwcTB60BQsxeTCMSHtnDhstDpVujXBEP7kAcQoP/JUwlgaRFEeIEMtMcVcipOhjNp0M+qg2agHBwWxSTAI7QxBaDgvhUCFwNIXQFg0spxBBH/VGLSA4mCSI2nWGwHaS0EfJUQ8E3FVMVzDQFQvEitXYcGbQR73xHRgIN944nhqIVYHBpAZ91Bv1UOBobiBO48HkBn3UHPWQwK8YjnCgLSKIHJg+Y+yj4qiJA/eSRG3RwHaSKFcbLVaS6N2Rp+DA3w06gEqS/ExNVyLEIqXhG5Z6rS1qIcLJdUIbD2+ej2rmoY/KohYMHE0cjYFgeKlodR8eCEIq4QgHYlzvzMGbGahmDvooLupZF9zLG7VBYDlvDPsoLtKnB06BgH0eZQDJolhcnHbFwPJlx7CP4qIGDPgY8dkhkIKKZgj6qC1qgMDJ0wRtENg+TeijrKgFAgevNYm3s3aGQCxLTQxD0EdNUQMEjp4pGosFppNDRVFxCBg4eH4gKtcZAcvnB5GiiugSAnzYH8AZghjJu4Ngt5wc9VE+1AKCo4uCNhAsLwpROFQQHFwW9EUDexAUcEn0lSC4ZcwMCjV8rnnxeXFLVJItuc4T1ASKfFk2BMEmkhND0zMiT0BxTgfWeZo2h1F2teAR45tS0Pe7Bpnt7XUfalIR+Ke0F1N1qYj848yc2qVip4gcoA+9Dj3ap0WSx3JfNxOeVnQEC1RTVp+n5QjINWsz4ux3d2Mz4+ypdWfLVynEhl1m4H5/3y4D/t43vLXnb/mCAG3IZQbr93fcMuFmVaNUs26WC+5MOy29Ln5/FycDLo5UC6JZFyvK2UwbuTnCTRM58uqy7SZnIHkpYFZ/JHUxoUR8ohKhNiXkmvLZ2VmjAX0glrw8+kxs/xqc9PTsRwQ5dVHVIUhVk2AuN/hjZ8a/Vz/uyJ+/b+5uz/+8ufu2YDrNfYc4282ac5w0fmv6BTddGMl8Wj8WOSgTqDV6JVM4TpTrcAgnaRRK0UvuQsun+0ai20gxp5RN6Tp0EyZvD43td+drh/8gILr6Hw==</diagram></mxfile>
|
||||
199
modules/FEATURES_TO_INTERFACES_IMPORTS.md
Normal file
199
modules/FEATURES_TO_INTERFACES_IMPORTS.md
Normal file
|
|
@ -0,0 +1,199 @@
|
|||
# Features → Interfaces Import Analysis
|
||||
|
||||
## Summary
|
||||
|
||||
This document details all imports from `features/` modules to `interfaces/` modules.
|
||||
|
||||
**Total Feature Modules:** 6
|
||||
**Modules Importing from Interfaces:** 2 (only `getRootInterface` - system-level function)
|
||||
**Total Interface Imports:** 2 (both for `getRootInterface` only)
|
||||
|
||||
**✅ REFACTORED:** All feature modules now use `services.getInterface()` to access interfaces, following the same pattern as `chatPlayground`. Only `getRootInterface()` remains as a direct import (system-level function, not user-specific).
|
||||
|
||||
---
|
||||
|
||||
## Detailed Import List
|
||||
|
||||
### 1. `features/featuresLifecycle.py`
|
||||
|
||||
**Imports:**
|
||||
- **Line 2:** `from modules.interfaces.interfaceDbAppObjects import getRootInterface`
|
||||
- **Type:** Direct import
|
||||
- **Usage:** Gets root interface to retrieve event user for feature initialization
|
||||
- **Context:** Used in `start()` function to get event user for automation, syncDelta, and chatAlthaus features
|
||||
- **Note:** `getRootInterface()` is a system-level function (no user required), so it remains as direct import
|
||||
|
||||
**Refactored:**
|
||||
- ✅ **Removed:** `from modules.interfaces.interfaceDbChatObjects import getInterface as getChatInterface`
|
||||
- ✅ **Added:** `from modules.services import getInterface as getServices`
|
||||
- ✅ **Changed:** Now uses `services.interfaceDbChat` instead of direct interface import
|
||||
- **Pattern:** `services = getServices(eventUser, None)` then `services.interfaceDbChat`
|
||||
|
||||
**Purpose:** Feature lifecycle management - initializes and manages all features on startup/shutdown.
|
||||
|
||||
---
|
||||
|
||||
### 2. `features/automation/mainAutomation.py`
|
||||
|
||||
**Imports:**
|
||||
- **Line 14:** `from modules.interfaces.interfaceDbAppObjects import getRootInterface`
|
||||
- **Type:** Direct import
|
||||
- **Usage:** Gets root interface to retrieve event user (system-level function)
|
||||
- **Context:** Used in `createAutomationEventHandler()` to get event user
|
||||
- **Note:** `getRootInterface()` is a system-level function (no user required), so it remains as direct import
|
||||
|
||||
**Refactored:**
|
||||
- ✅ **Removed:** `from modules.interfaces.interfaceDbChatObjects import getInterface as getChatInterface`
|
||||
- ✅ **Removed:** `from modules.interfaces.interfaceDbAppObjects import getInterface as getAppInterface`
|
||||
- ✅ **Added:** `from modules.services import getInterface as getServices`
|
||||
- ✅ **Changed:** All interface access now goes through services:
|
||||
- `executeAutomation()`: Uses `services.interfaceDbApp` and `services.interfaceDbChat`
|
||||
- `createAutomationEventHandler()`: Uses `eventServices.interfaceDbChat` and `eventServices.interfaceDbApp`
|
||||
- **Pattern:** `services = getServices(user, None)` then `services.interfaceDbChat` or `services.interfaceDbApp`
|
||||
|
||||
**Purpose:** Automation workflow execution and scheduling - handles automated workflow triggers and event scheduling.
|
||||
|
||||
---
|
||||
|
||||
### 3. `features/chatPlayground/mainChatPlayground.py`
|
||||
|
||||
**Imports:**
|
||||
- ❌ **No imports from interfaces/**
|
||||
- Uses `modules.services.getInterface` instead (indirect access through services layer)
|
||||
|
||||
**Purpose:** Chat playground feature - interactive chat interface.
|
||||
|
||||
---
|
||||
|
||||
### 4. `features/chatAlthaus/mainChatAlthaus.py`
|
||||
|
||||
**Imports:**
|
||||
- ❌ **No imports from interfaces/**
|
||||
- Uses `modules.services.getInterface` instead (indirect access through services layer)
|
||||
|
||||
**Purpose:** Chat Althaus data scheduler - scheduled data updates for Althaus preprocessing.
|
||||
|
||||
---
|
||||
|
||||
### 5. `features/syncDelta/mainSyncDelta.py`
|
||||
|
||||
**Imports:**
|
||||
- ❌ **No imports from interfaces/**
|
||||
- Uses `modules.services.getInterface` instead (indirect access through services layer)
|
||||
|
||||
**Purpose:** Delta Group sync manager - synchronizes tickets to SharePoint.
|
||||
|
||||
---
|
||||
|
||||
### 6. `features/neutralizePlayground/mainNeutralizePlayground.py`
|
||||
|
||||
**Imports:**
|
||||
- ❌ **No imports from interfaces/**
|
||||
- Uses `modules.services.getInterface` instead (indirect access through services layer)
|
||||
|
||||
**Purpose:** Neutralization playground - UI wrapper for data neutralization service.
|
||||
|
||||
---
|
||||
|
||||
## Import Statistics
|
||||
|
||||
### By Interface Module
|
||||
|
||||
**`interfaceDbAppObjects`:**
|
||||
- `getRootInterface`: 2 imports (system-level function, remains as direct import)
|
||||
- `features/featuresLifecycle.py` (line 2)
|
||||
- `features/automation/mainAutomation.py` (line 14)
|
||||
|
||||
**`interfaceDbChatObjects`:**
|
||||
- ✅ **Removed:** All direct imports refactored to use services layer
|
||||
- Now accessed via `services.interfaceDbChat` after calling `getServices(user, None)`
|
||||
|
||||
**`interfaceDbAppObjects.getInterface`:**
|
||||
- ✅ **Removed:** All direct imports refactored to use services layer
|
||||
- Now accessed via `services.interfaceDbApp` after calling `getServices(user, None)`
|
||||
|
||||
### By Import Type
|
||||
|
||||
- **Direct imports:** 2 (only `getRootInterface` - system-level function)
|
||||
- `features/featuresLifecycle.py`: 1
|
||||
- `features/automation/mainAutomation.py`: 1
|
||||
|
||||
- **Services-based access:** All user-specific interface access now goes through services layer
|
||||
- Pattern: `services = getServices(user, None)` then `services.interfaceDbChat` or `services.interfaceDbApp`
|
||||
|
||||
---
|
||||
|
||||
## Architectural Notes
|
||||
|
||||
### ✅ Correct Dependency Direction
|
||||
|
||||
All imports follow the correct architectural direction:
|
||||
- **features/** → **interfaces/** ✅
|
||||
|
||||
This is compliant with the dependency rules:
|
||||
- Features can import from interfaces (correct)
|
||||
- Features can import from services (correct)
|
||||
- Features do NOT import from other features (except internal feature-to-feature imports)
|
||||
|
||||
### Import Patterns
|
||||
|
||||
1. **Direct Interface Access:**
|
||||
- `features/automation/mainAutomation.py` - Directly imports interfaces for automation management
|
||||
- `features/featuresLifecycle.py` - Directly imports `getRootInterface` for feature initialization
|
||||
|
||||
2. **Indirect Access via Services:**
|
||||
- `features/chatPlayground/mainChatPlayground.py`
|
||||
- `features/chatAlthaus/mainChatAlthaus.py`
|
||||
- `features/syncDelta/mainSyncDelta.py`
|
||||
- `features/neutralizePlayground/mainNeutralizePlayground.py`
|
||||
|
||||
These features use `modules.services.getInterface` which provides a service layer abstraction.
|
||||
|
||||
### Usage Context
|
||||
|
||||
**`interfaceDbAppObjects`:**
|
||||
- Used for user management (`getRootInterface`, `getAppInterface`)
|
||||
- Primarily for getting event user and creator user in automation context
|
||||
|
||||
**`interfaceDbChatObjects`:**
|
||||
- Used for chat/automation data access (`getChatInterface`)
|
||||
- Used for automation execution, event syncing, and workflow management
|
||||
|
||||
---
|
||||
|
||||
## Summary Table
|
||||
|
||||
| Feature Module | Interface Imports | Import Type | Purpose |
|
||||
|---------------|-------------------|-------------|---------|
|
||||
| `featuresLifecycle.py` | `interfaceDbAppObjects.getRootInterface` | Direct | Feature initialization (system-level) |
|
||||
| `featuresLifecycle.py` | ✅ Via `services.interfaceDbChat` | Services | Automation event sync |
|
||||
| `automation/mainAutomation.py` | `interfaceDbAppObjects.getRootInterface` | Direct | Event user access (system-level) |
|
||||
| `automation/mainAutomation.py` | ✅ Via `services.interfaceDbChat` | Services | Automation execution |
|
||||
| `automation/mainAutomation.py` | ✅ Via `services.interfaceDbApp` | Services | User management |
|
||||
| `chatPlayground/mainChatPlayground.py` | None | - | Uses services layer |
|
||||
| `chatAlthaus/mainChatAlthaus.py` | None | - | Uses services layer |
|
||||
| `syncDelta/mainSyncDelta.py` | None | - | Uses services layer |
|
||||
| `neutralizePlayground/mainNeutralizePlayground.py` | None | - | Uses services layer |
|
||||
|
||||
---
|
||||
|
||||
## Conclusion
|
||||
|
||||
**Total Interface Imports:** 2 (only `getRootInterface` - system-level function)
|
||||
- 2 direct imports (both for `getRootInterface`)
|
||||
|
||||
**Modules Using Direct Interface Imports:** 2 out of 6 (only for `getRootInterface`)
|
||||
- `features/featuresLifecycle.py`
|
||||
- `features/automation/mainAutomation.py`
|
||||
|
||||
**✅ REFACTORED:** All user-specific interface access now goes through services layer
|
||||
- Pattern: `services = getServices(user, None)` then `services.interfaceDbChat` or `services.interfaceDbApp`
|
||||
- Consistent with other feature modules (`chatPlayground`, `chatAlthaus`, `syncDelta`, `neutralizePlayground`)
|
||||
|
||||
**Architectural Compliance:** ✅ **PERFECT**
|
||||
- All imports follow correct direction (features → interfaces)
|
||||
- Only system-level function (`getRootInterface`) remains as direct import
|
||||
- All user-specific interface access goes through services layer
|
||||
- Clean separation maintained
|
||||
- Consistent pattern across all feature modules
|
||||
|
||||
12
modules/features/automation/__init__.py
Normal file
12
modules/features/automation/__init__.py
Normal file
|
|
@ -0,0 +1,12 @@
|
|||
"""
|
||||
Automation feature - handles automated workflow execution and scheduling.
|
||||
|
||||
Moved from interfaces/interfaceDbChatObjects.py to follow proper architectural separation:
|
||||
- Interface layer: Data access only (getAutomationDefinition, etc.)
|
||||
- Feature layer: Business logic and orchestration (executeAutomation, syncAutomationEvents)
|
||||
"""
|
||||
|
||||
from .mainAutomation import executeAutomation, syncAutomationEvents, createAutomationEventHandler
|
||||
|
||||
__all__ = ['executeAutomation', 'syncAutomationEvents', 'createAutomationEventHandler']
|
||||
|
||||
287
modules/features/automation/mainAutomation.py
Normal file
287
modules/features/automation/mainAutomation.py
Normal file
|
|
@ -0,0 +1,287 @@
|
|||
"""
|
||||
Main automation service - handles automation workflow execution and scheduling.
|
||||
|
||||
Moved from interfaces/interfaceDbChatObjects.py to follow proper architectural separation.
|
||||
"""
|
||||
|
||||
import logging
|
||||
import json
|
||||
from typing import Dict, Any
|
||||
|
||||
from modules.datamodels.datamodelChat import ChatWorkflow, UserInputRequest, WorkflowModeEnum, AutomationDefinition
|
||||
from modules.shared.timeUtils import getUtcTimestamp
|
||||
from modules.shared.eventManagement import eventManager
|
||||
from modules.services import getInterface as getServices
|
||||
from modules.features.chatPlayground.mainChatPlayground import chatStart
|
||||
from .subAutomationUtils import parseScheduleToCron, planToPrompt, replacePlaceholders
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
async def executeAutomation(automationId: str, chatInterface) -> ChatWorkflow:
|
||||
"""Execute automation workflow immediately (test mode) with placeholder replacement.
|
||||
|
||||
Args:
|
||||
automationId: ID of automation to execute
|
||||
chatInterface: ChatObjects interface instance for data access
|
||||
|
||||
Returns:
|
||||
ChatWorkflow instance created by automation execution
|
||||
"""
|
||||
executionStartTime = getUtcTimestamp()
|
||||
executionLog = {
|
||||
"timestamp": executionStartTime,
|
||||
"workflowId": None,
|
||||
"status": "running",
|
||||
"messages": []
|
||||
}
|
||||
|
||||
try:
|
||||
# 1. Load automation definition
|
||||
automation = chatInterface.getAutomationDefinition(automationId)
|
||||
if not automation:
|
||||
raise ValueError(f"Automation {automationId} not found")
|
||||
|
||||
executionLog["messages"].append(f"Started execution at {executionStartTime}")
|
||||
|
||||
# 2. Replace placeholders in template to generate plan
|
||||
template = automation.get("template", "")
|
||||
placeholders = automation.get("placeholders", {})
|
||||
planJson = replacePlaceholders(template, placeholders)
|
||||
try:
|
||||
plan = json.loads(planJson)
|
||||
except json.JSONDecodeError as e:
|
||||
logger.error(f"Failed to parse plan JSON after placeholder replacement: {str(e)}")
|
||||
logger.error(f"Template: {template[:500]}...")
|
||||
logger.error(f"Placeholders: {placeholders}")
|
||||
logger.error(f"Generated planJson (first 1000 chars): {planJson[:1000]}")
|
||||
logger.error(f"Error position: line {e.lineno}, column {e.colno}, char {e.pos}")
|
||||
if e.pos:
|
||||
start = max(0, e.pos - 100)
|
||||
end = min(len(planJson), e.pos + 100)
|
||||
logger.error(f"Context around error: ...{planJson[start:end]}...")
|
||||
raise ValueError(f"Invalid JSON after placeholder replacement: {str(e)}")
|
||||
executionLog["messages"].append("Template placeholders replaced successfully")
|
||||
|
||||
# 3. Get user who created automation
|
||||
creatorUserId = automation.get("_createdBy")
|
||||
|
||||
# CRITICAL: Automation MUST run as creator user only, or fail
|
||||
if not creatorUserId:
|
||||
errorMsg = f"Automation {automationId} has no creator user (_createdBy field missing). Cannot execute automation."
|
||||
logger.error(errorMsg)
|
||||
executionLog["messages"].append(errorMsg)
|
||||
raise ValueError(errorMsg)
|
||||
|
||||
# Get user from database using services
|
||||
services = getServices(chatInterface.currentUser, None)
|
||||
creatorUser = services.interfaceDbApp.getUser(creatorUserId)
|
||||
if not creatorUser:
|
||||
raise ValueError(f"Creator user {creatorUserId} not found")
|
||||
|
||||
executionLog["messages"].append(f"Using creator user: {creatorUserId}")
|
||||
|
||||
# 4. Create UserInputRequest from plan
|
||||
# Embed plan JSON in prompt for TemplateMode to extract
|
||||
promptText = planToPrompt(plan)
|
||||
planJsonStr = json.dumps(plan)
|
||||
# Embed plan as JSON comment so TemplateMode can extract it
|
||||
promptWithPlan = f"{promptText}\n\n<!--TEMPLATE_PLAN_START-->\n{planJsonStr}\n<!--TEMPLATE_PLAN_END-->"
|
||||
|
||||
userInput = UserInputRequest(
|
||||
prompt=promptWithPlan,
|
||||
listFileId=[],
|
||||
userLanguage=creatorUser.language or "en"
|
||||
)
|
||||
|
||||
executionLog["messages"].append("Starting workflow execution")
|
||||
|
||||
# 5. Start workflow using chatStart
|
||||
workflow = await chatStart(
|
||||
currentUser=creatorUser,
|
||||
userInput=userInput,
|
||||
workflowMode=WorkflowModeEnum.WORKFLOW_AUTOMATION,
|
||||
workflowId=None
|
||||
)
|
||||
|
||||
executionLog["workflowId"] = workflow.id
|
||||
executionLog["status"] = "completed"
|
||||
executionLog["messages"].append(f"Workflow {workflow.id} started successfully")
|
||||
logger.info(f"Started workflow {workflow.id} with plan containing {len(plan.get('tasks', []))} tasks (plan embedded in userInput)")
|
||||
|
||||
# Set workflow name with "automated" prefix
|
||||
automationLabel = automation.get("label", "Unknown Automation")
|
||||
workflowName = f"automated: {automationLabel}"
|
||||
workflow = chatInterface.updateWorkflow(workflow.id, {"name": workflowName})
|
||||
logger.info(f"Set workflow {workflow.id} name to: {workflowName}")
|
||||
|
||||
# Update automation with execution log
|
||||
executionLogs = automation.get("executionLogs", [])
|
||||
executionLogs.append(executionLog)
|
||||
# Keep only last 50 executions
|
||||
if len(executionLogs) > 50:
|
||||
executionLogs = executionLogs[-50:]
|
||||
|
||||
chatInterface.db.recordModify(
|
||||
AutomationDefinition,
|
||||
automationId,
|
||||
{"executionLogs": executionLogs}
|
||||
)
|
||||
|
||||
return workflow
|
||||
except Exception as e:
|
||||
# Log error to execution log
|
||||
executionLog["status"] = "error"
|
||||
executionLog["messages"].append(f"Error: {str(e)}")
|
||||
|
||||
# Update automation with execution log even on error
|
||||
try:
|
||||
automation = chatInterface.getAutomationDefinition(automationId)
|
||||
if automation:
|
||||
executionLogs = automation.get("executionLogs", [])
|
||||
executionLogs.append(executionLog)
|
||||
if len(executionLogs) > 50:
|
||||
executionLogs = executionLogs[-50:]
|
||||
chatInterface.db.recordModify(
|
||||
AutomationDefinition,
|
||||
automationId,
|
||||
{"executionLogs": executionLogs}
|
||||
)
|
||||
except Exception as logError:
|
||||
logger.error(f"Error saving execution log: {str(logError)}")
|
||||
|
||||
raise
|
||||
|
||||
|
||||
async def syncAutomationEvents(chatInterface, eventUser) -> Dict[str, Any]:
|
||||
"""Automation event handler - syncs scheduler with all active automations.
|
||||
|
||||
Args:
|
||||
chatInterface: ChatObjects interface instance for data access
|
||||
eventUser: System-level event user for accessing automations
|
||||
|
||||
Returns:
|
||||
Dictionary with sync results (synced count and event IDs)
|
||||
"""
|
||||
# Get all automation definitions (for current mandate)
|
||||
allAutomations = chatInterface.db.getRecordset(AutomationDefinition)
|
||||
filtered = chatInterface._uam(AutomationDefinition, allAutomations)
|
||||
|
||||
registeredEvents = {}
|
||||
|
||||
for automation in filtered:
|
||||
automationId = automation.get("id")
|
||||
isActive = automation.get("active", False)
|
||||
currentEventId = automation.get("eventId")
|
||||
schedule = automation.get("schedule")
|
||||
|
||||
if not schedule:
|
||||
logger.warning(f"Automation {automationId} has no schedule, skipping")
|
||||
continue
|
||||
|
||||
try:
|
||||
# Parse schedule to cron kwargs
|
||||
cronKwargs = parseScheduleToCron(schedule)
|
||||
|
||||
if isActive:
|
||||
# Remove existing event if present (handles schedule changes)
|
||||
if currentEventId:
|
||||
try:
|
||||
eventManager.remove(currentEventId)
|
||||
except Exception as e:
|
||||
logger.warning(f"Error removing old event {currentEventId}: {str(e)}")
|
||||
|
||||
# Register new event
|
||||
newEventId = f"automation.{automationId}"
|
||||
|
||||
# Create event handler function
|
||||
handler = createAutomationEventHandler(automationId, eventUser)
|
||||
|
||||
# Register cron job
|
||||
eventManager.registerCron(
|
||||
jobId=newEventId,
|
||||
func=handler,
|
||||
cronKwargs=cronKwargs,
|
||||
replaceExisting=True
|
||||
)
|
||||
|
||||
# Update automation with new eventId
|
||||
if currentEventId != newEventId:
|
||||
chatInterface.db.recordModify(
|
||||
AutomationDefinition,
|
||||
automationId,
|
||||
{"eventId": newEventId}
|
||||
)
|
||||
|
||||
registeredEvents[automationId] = newEventId
|
||||
else:
|
||||
# Remove event if exists
|
||||
if currentEventId:
|
||||
try:
|
||||
eventManager.remove(currentEventId)
|
||||
chatInterface.db.recordModify(
|
||||
AutomationDefinition,
|
||||
automationId,
|
||||
{"eventId": None}
|
||||
)
|
||||
except Exception as e:
|
||||
logger.warning(f"Error removing event {currentEventId}: {str(e)}")
|
||||
except Exception as e:
|
||||
logger.error(f"Error syncing automation {automationId}: {str(e)}")
|
||||
|
||||
return {
|
||||
"synced": len(registeredEvents),
|
||||
"events": registeredEvents
|
||||
}
|
||||
|
||||
|
||||
def createAutomationEventHandler(automationId: str, eventUser):
|
||||
"""Create event handler function for a specific automation.
|
||||
|
||||
Args:
|
||||
automationId: ID of automation to create handler for
|
||||
eventUser: System-level event user for accessing automations (captured in closure)
|
||||
|
||||
Returns:
|
||||
Async handler function for scheduled automation execution
|
||||
"""
|
||||
async def handler():
|
||||
try:
|
||||
if not eventUser:
|
||||
logger.error("Event user not available for automation execution")
|
||||
return
|
||||
|
||||
# Get services for event user (provides access to interfaces)
|
||||
eventServices = getServices(eventUser, None)
|
||||
|
||||
# Load automation using event user context
|
||||
automation = eventServices.interfaceDbChat.getAutomationDefinition(automationId)
|
||||
if not automation or not automation.get("active"):
|
||||
logger.warning(f"Automation {automationId} not found or not active, skipping execution")
|
||||
return
|
||||
|
||||
# Get creator user
|
||||
creatorUserId = automation.get("_createdBy")
|
||||
if not creatorUserId:
|
||||
logger.error(f"Automation {automationId} has no creator user")
|
||||
return
|
||||
|
||||
# Get creator user from database using services
|
||||
eventServices = getServices(eventUser, None)
|
||||
creatorUser = eventServices.interfaceDbApp.getUser(creatorUserId)
|
||||
if not creatorUser:
|
||||
logger.error(f"Creator user {creatorUserId} not found for automation {automationId}")
|
||||
return
|
||||
|
||||
# Get services for creator user (provides access to interfaces)
|
||||
creatorServices = getServices(creatorUser, None)
|
||||
|
||||
# Execute automation with creator user's context
|
||||
# executeAutomation is in same module, so we can call it directly
|
||||
await executeAutomation(automationId, creatorServices.interfaceDbChat)
|
||||
logger.info(f"Successfully executed automation {automationId} as user {creatorUserId}")
|
||||
except Exception as e:
|
||||
logger.error(f"Error executing automation {automationId}: {str(e)}")
|
||||
|
||||
return handler
|
||||
|
||||
108
modules/features/automation/subAutomationUtils.py
Normal file
108
modules/features/automation/subAutomationUtils.py
Normal file
|
|
@ -0,0 +1,108 @@
|
|||
"""
|
||||
Utility functions for automation feature.
|
||||
|
||||
Moved from interfaces/interfaceDbChatObjects.py.
|
||||
"""
|
||||
|
||||
import json
|
||||
from typing import Dict, Any
|
||||
|
||||
|
||||
def parseScheduleToCron(schedule: str) -> Dict[str, Any]:
|
||||
"""Parse schedule string to cron kwargs for APScheduler"""
|
||||
parts = schedule.split()
|
||||
if len(parts) != 5:
|
||||
raise ValueError(f"Invalid schedule format: {schedule}")
|
||||
|
||||
return {
|
||||
"minute": parts[0],
|
||||
"hour": parts[1],
|
||||
"day": parts[2],
|
||||
"month": parts[3],
|
||||
"day_of_week": parts[4]
|
||||
}
|
||||
|
||||
|
||||
def planToPrompt(plan: Dict) -> str:
|
||||
"""Convert plan structure to prompt string for workflow execution"""
|
||||
return plan.get("userMessage", plan.get("overview", "Execute automation workflow"))
|
||||
|
||||
|
||||
def replacePlaceholders(template: str, placeholders: Dict[str, str]) -> str:
|
||||
"""Replace placeholders in template with actual values. Placeholder format: {{KEY:PLACEHOLDER_NAME}}"""
|
||||
result = template
|
||||
for placeholderName, value in placeholders.items():
|
||||
pattern = f"{{{{KEY:{placeholderName}}}}}"
|
||||
|
||||
# Check if placeholder is in an array context like ["{{KEY:...}}"]
|
||||
# If value is a JSON array/dict, we should replace the entire ["{{KEY:...}}"] with the array
|
||||
arrayPattern = f'["{pattern}"]'
|
||||
if arrayPattern in result:
|
||||
# Check if value is a JSON array/dict
|
||||
isArrayValue = False
|
||||
arrayValue = None
|
||||
|
||||
if isinstance(value, (list, dict)):
|
||||
isArrayValue = True
|
||||
arrayValue = json.dumps(value)
|
||||
elif isinstance(value, str):
|
||||
try:
|
||||
parsed = json.loads(value)
|
||||
if isinstance(parsed, (list, dict)):
|
||||
isArrayValue = True
|
||||
arrayValue = value # Already valid JSON string
|
||||
except (json.JSONDecodeError, ValueError):
|
||||
pass
|
||||
|
||||
if isArrayValue:
|
||||
# Replace ["{{KEY:...}}"] with the array value
|
||||
result = result.replace(arrayPattern, arrayValue)
|
||||
continue # Skip the regular replacement below
|
||||
|
||||
# Regular replacement - check if in quoted context
|
||||
patternStart = result.find(pattern)
|
||||
isQuoted = False
|
||||
if patternStart > 0:
|
||||
charBefore = result[patternStart - 1] if patternStart > 0 else None
|
||||
patternEnd = patternStart + len(pattern)
|
||||
charAfter = result[patternEnd] if patternEnd < len(result) else None
|
||||
if charBefore == '"' and charAfter == '"':
|
||||
isQuoted = True
|
||||
|
||||
# Handle different value types
|
||||
if isinstance(value, (list, dict)):
|
||||
# Python list/dict - convert to JSON
|
||||
replacement = json.dumps(value)
|
||||
elif isinstance(value, str):
|
||||
# String value - check if it's a JSON string representing list/dict
|
||||
try:
|
||||
parsed = json.loads(value)
|
||||
if isinstance(parsed, (list, dict)):
|
||||
# It's a JSON string of a list/dict
|
||||
if isQuoted:
|
||||
# In quoted context, escape the JSON string
|
||||
escaped = json.dumps(value)
|
||||
replacement = escaped[1:-1] # Remove outer quotes
|
||||
else:
|
||||
# In unquoted context, use JSON directly
|
||||
replacement = value
|
||||
else:
|
||||
# It's a JSON string of a primitive
|
||||
if isQuoted:
|
||||
escaped = json.dumps(value)
|
||||
replacement = escaped[1:-1]
|
||||
else:
|
||||
replacement = value
|
||||
except (json.JSONDecodeError, ValueError):
|
||||
# Not valid JSON - treat as plain string
|
||||
if isQuoted:
|
||||
escaped = json.dumps(value)
|
||||
replacement = escaped[1:-1]
|
||||
else:
|
||||
replacement = value
|
||||
else:
|
||||
# Numbers, booleans, None - convert to string
|
||||
replacement = str(value)
|
||||
result = result.replace(pattern, replacement)
|
||||
return result
|
||||
|
||||
|
|
@ -1,211 +0,0 @@
|
|||
# Komponentendiagramm: Kunden-Chatbot Althaus
|
||||
|
||||
## Übersicht
|
||||
|
||||
Dieses Diagramm zeigt die High-Level-Architektur der Althaus Chatbot-Anwendung mit allen beteiligten Komponenten, Datenflüssen und Kommunikationswegen.
|
||||
|
||||
## Komponentendiagramm
|
||||
|
||||
```mermaid
|
||||
graph TB
|
||||
subgraph "PowerOn Chat UI"
|
||||
ChatUI[Chat Interface]
|
||||
end
|
||||
|
||||
subgraph "PowerOn Platform"
|
||||
Gateway[Gateway Backend<br/>Event Scheduler & Data Query API]
|
||||
GatewayDB[(PostgreSQL)]
|
||||
AIServices[Dynamic AI, Tavily]
|
||||
end
|
||||
|
||||
subgraph "Tenant althaus-ag.ch"
|
||||
subgraph "PowerOn PreProcessing"
|
||||
PreProcessing[Pre-Processing Service]
|
||||
PreProcessingDB[(PostgreSQL<br/>Memory DB)]
|
||||
end
|
||||
|
||||
subgraph "MSFT Services"
|
||||
PowerBI[Power BI]
|
||||
TenantServices[Azure DC, DNA Center]
|
||||
end
|
||||
end
|
||||
|
||||
%% Hauptkommunikation
|
||||
ChatUI -->|"Data Queries<br/>User/Password Auth"| Gateway
|
||||
Gateway -->|"SQL Queries<br/>X-PP-API-Key"| PreProcessing
|
||||
Gateway -->|"Config Update<br/>Daily 01:00 UTC"| PreProcessing
|
||||
|
||||
%% Datenfluss
|
||||
PowerBI -->|"Rohdaten"| PreProcessing
|
||||
PreProcessing --> PreProcessingDB
|
||||
PreProcessingDB -->|"Query Results"| Gateway
|
||||
Gateway --> ChatUI
|
||||
Gateway --> GatewayDB
|
||||
|
||||
%% Styling
|
||||
classDef platform fill:#e1f5ff,stroke:#01579b,stroke-width:2px
|
||||
classDef frontend fill:#f3e5f5,stroke:#4a148c,stroke-width:2px
|
||||
classDef preprocessing fill:#fff3e0,stroke:#e65100,stroke-width:2px
|
||||
classDef customer fill:#e8f5e9,stroke:#1b5e20,stroke-width:2px
|
||||
classDef database fill:#fce4ec,stroke:#880e4f,stroke-width:2px
|
||||
|
||||
class Gateway,AIServices platform
|
||||
class ChatUI frontend
|
||||
class PreProcessing preprocessing
|
||||
class PowerBI,TenantServices customer
|
||||
class GatewayDB,PreProcessingDB database
|
||||
```
|
||||
|
||||
## Komponentenbeschreibungen
|
||||
|
||||
### 1. Gateway Backend (gateway.poweron-center.net)
|
||||
|
||||
**Hauptkomponenten:**
|
||||
- **FastAPI Application**: Zentrale Backend-Anwendung der PowerOn Platform
|
||||
- **Event Scheduler (chatAlthaus)**:
|
||||
- Täglicher Scheduler um 01:00 UTC
|
||||
- Sendet Konfigurations-Updates an Pre-Processing Service
|
||||
- Verwendet `X-PP-API-Key` Header für Authentifizierung
|
||||
- **Configuration Management**:
|
||||
- Verwaltung von Secrets und Environment-Variablen
|
||||
- Verschlüsselung/Entschlüsselung von Secrets
|
||||
- Unterstützt verschiedene Umgebungen (dev, int, prod)
|
||||
- **Data Query API**:
|
||||
- `POST /api/v1/dataquery/query` - SQL Query ausführen
|
||||
- `GET /api/v1/dataquery/schema` - Datenbankschema abrufen
|
||||
- `GET /api/v1/dataquery/schema/{table_name}` - Tabellenschema abrufen
|
||||
- **PostgreSQL Database**: Zentrale Datenbank für Gateway-Daten
|
||||
|
||||
**Technologie:**
|
||||
- Python/FastAPI
|
||||
- PostgreSQL
|
||||
- APScheduler für Event-Management
|
||||
|
||||
**Externe AI-Services:**
|
||||
- **Dynamic AI**: LLM Service für AI-Anfragen
|
||||
- **Tavily**: Web-Such-Service für Web-Recherchen
|
||||
|
||||
### 2. PowerOn Chat UI (althaus-chat.poweron-center.net)
|
||||
|
||||
**Hauptkomponenten:**
|
||||
- **React Application**: Frontend-Interface für den Chatbot
|
||||
- **Authentication**: User/Password-basierte Authentifizierung mit JWT-Token
|
||||
|
||||
**Kommunikation:**
|
||||
- Nutzt 3 Data Query Endpunkte vom Gateway
|
||||
- Authentifiziert sich mit User/Password beim Gateway
|
||||
- Erhält Antworten über Gateway API
|
||||
|
||||
**Technologie:**
|
||||
- React
|
||||
- REST API Calls
|
||||
|
||||
### 3. Tenant althaus-ag.ch
|
||||
|
||||
#### 3.1 PowerOn PreProcessing
|
||||
|
||||
**Hauptkomponenten:**
|
||||
- **FastAPI Application**: Pre-Processing Service im Azure-Tenant des Kunden
|
||||
- **Pre-Processing API**:
|
||||
- `POST /api/v1/dataprocessor/update-db-with-config` - Datenbank mit Konfiguration aktualisieren
|
||||
- Authentifizierung: `X-PP-API-Key` Header
|
||||
- **PostgreSQL Memory Database**:
|
||||
- Speichert verarbeitete Daten
|
||||
- Wird vom Chat für Queries genutzt
|
||||
|
||||
**Datenfluss:**
|
||||
- Empfängt Rohdaten aus Power BI Semantikmodell
|
||||
- Verarbeitet Daten nach konfigurierten Schritten (keep, fillna, to_numeric, dropna, etc.)
|
||||
- Speichert verarbeitete Daten in Memory Database
|
||||
- Beantwortet SQL-Queries vom Gateway
|
||||
|
||||
**Technologie:**
|
||||
- Python/FastAPI
|
||||
- PostgreSQL
|
||||
- Azure App Service (im Kunden-Tenant althaus-ag.ch)
|
||||
|
||||
#### 3.2 MSFT Services
|
||||
|
||||
**Power BI Semantikmodell:**
|
||||
- Datenquelle für Rohdaten
|
||||
- Wird vom Pre-Processing Service gelesen
|
||||
|
||||
**Azure Domänen-Controller:**
|
||||
- Authentifizierungs-Service
|
||||
- Wird vom Gateway für Authentifizierung genutzt
|
||||
|
||||
**DNA Center:**
|
||||
- Netzwerk-Management-Service
|
||||
- Wird vom Gateway genutzt
|
||||
|
||||
## Datenfluss
|
||||
|
||||
### 1. Datenaktualisierung (Scheduled)
|
||||
```
|
||||
Power BI Semantikmodell (Tenant althaus-ag.ch)
|
||||
→ PowerOn PreProcessing (verarbeitet Daten)
|
||||
→ PostgreSQL Memory DB (speichert verarbeitete Daten)
|
||||
|
||||
Gateway Event Scheduler (01:00 UTC täglich)
|
||||
→ POST /api/v1/dataprocessor/update-db-with-config
|
||||
→ PowerOn PreProcessing (aktualisiert Konfiguration)
|
||||
```
|
||||
|
||||
### 2. Chat-Interaktion (User Request)
|
||||
```
|
||||
PowerOn Chat UI
|
||||
→ POST /api/v1/dataquery/query (mit User/Password Auth)
|
||||
→ Gateway Data Query API
|
||||
→ POST /api/v1/dataquery/query (mit X-PP-API-Key)
|
||||
→ PowerOn PreProcessing
|
||||
→ PostgreSQL Memory DB (führt Query aus)
|
||||
→ PowerOn PreProcessing (gibt Ergebnisse zurück)
|
||||
→ Gateway Data Query API
|
||||
→ PowerOn Chat UI (zeigt Antwort)
|
||||
```
|
||||
|
||||
### 3. AI-Integration
|
||||
```
|
||||
PowerOn Chat UI
|
||||
→ Gateway (vermittelt AI-Anfragen)
|
||||
→ Dynamic AI & Tavily (in PowerOn Platform)
|
||||
→ Gateway (kombiniert Ergebnisse)
|
||||
→ PowerOn Chat UI (zeigt Antwort)
|
||||
```
|
||||
|
||||
## Authentifizierung
|
||||
|
||||
### Gateway → PowerOn PreProcessing
|
||||
- **Header**: `X-PP-API-Key`
|
||||
- **Wert**: Aus Gateway Config (`PREPROCESS_ALTHAUS_CHAT_SECRET`)
|
||||
- **Verwendung**: Event Scheduler und Data Query API
|
||||
|
||||
### PowerOn Chat UI → Gateway
|
||||
- **Methode**: User/Password
|
||||
- **Token**: JWT Token (nach erfolgreicher Authentifizierung)
|
||||
- **Verwendung**: Alle API-Calls vom Chat Frontend
|
||||
|
||||
### Weitere Authentifizierung
|
||||
- Gateway nutzt Azure Domänen-Controller für zusätzliche Authentifizierung
|
||||
- Verschiedene API-Endpunkte können unterschiedliche Authentifizierungsmechanismen haben
|
||||
|
||||
## Deployment
|
||||
|
||||
- **PowerOn Platform**: gateway.poweron-center.net
|
||||
- **PowerOn Chat UI**: althaus-chat.poweron-center.net
|
||||
- **PowerOn PreProcessing**: Azure App Service im Kunden-Tenant (althaus-ag.ch)
|
||||
- URL: `poweron-althaus-preprocess-prod-e3fegaatc7faency.switzerlandnorth-01.azurewebsites.net`
|
||||
- **Tenant althaus-ag.ch**: Enthält PowerOn PreProcessing und MSFT Services (Power BI, Azure DC, DNA Center) im Azure-Tenant von Althaus AG
|
||||
|
||||
## Konfiguration
|
||||
|
||||
### Gateway Config Keys
|
||||
- `PREPROCESS_ALTHAUS_CHAT_SECRET`: API-Key für Pre-Processing Service
|
||||
- `APP_ENV_TYPE`: Umgebung (dev, int, prod)
|
||||
- Weitere Gateway-spezifische Konfigurationen
|
||||
|
||||
### Pre-Processing Config
|
||||
- Konfiguration wird als JSON im Gateway Code definiert
|
||||
- Wird täglich um 01:00 UTC an Pre-Processing Service gesendet
|
||||
- Definiert Tabellen, Spalten, Verarbeitungsschritte
|
||||
|
||||
|
|
@ -1,24 +1,37 @@
|
|||
import logging
|
||||
from modules.interfaces.interfaceDbAppObjects import getRootInterface
|
||||
from modules.services import getInterface as getServices
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
async def start() -> None:
|
||||
""" Start feature triggers and background managers """
|
||||
|
||||
# Provide Event User
|
||||
rootInterface = getRootInterface()
|
||||
eventUser = rootInterface.getUserByUsername("event")
|
||||
async def start(eventUser) -> None:
|
||||
""" Start feature triggers and background managers
|
||||
|
||||
Args:
|
||||
eventUser: System-level event user for background operations (provided by app.py)
|
||||
"""
|
||||
|
||||
# Feature Automation Events
|
||||
if eventUser:
|
||||
try:
|
||||
from modules.interfaces.interfaceDbChatObjects import getInterface as getChatInterface
|
||||
chatInterface = getChatInterface(eventUser)
|
||||
await chatInterface.syncAutomationEvents()
|
||||
from modules.features.automation import syncAutomationEvents
|
||||
from modules.shared.callbackRegistry import callbackRegistry
|
||||
|
||||
# Get services for event user (provides access to interfaces)
|
||||
services = getServices(eventUser, None)
|
||||
|
||||
# Register callback for automation changes
|
||||
async def onAutomationChanged(chatInterface):
|
||||
"""Callback triggered when automations are created/updated/deleted."""
|
||||
await syncAutomationEvents(chatInterface, eventUser)
|
||||
|
||||
callbackRegistry.register('automation.changed', onAutomationChanged)
|
||||
logger.info("Registered automation change callback")
|
||||
|
||||
# Initial sync on startup - use interface from services
|
||||
await syncAutomationEvents(services.interfaceDbChat, eventUser)
|
||||
logger.info("Automation events synced on startup")
|
||||
except Exception as e:
|
||||
logger.error(f"Error syncing automation events on startup: {str(e)}")
|
||||
logger.error(f"Error setting up automation events on startup: {str(e)}")
|
||||
# Don't fail startup if automation sync fails
|
||||
|
||||
# Feature SyncDelta
|
||||
|
|
@ -36,8 +49,21 @@ async def start() -> None:
|
|||
|
||||
|
||||
|
||||
async def stop() -> None:
|
||||
""" Stop feature triggers and background managers """
|
||||
async def stop(eventUser) -> None:
|
||||
""" Stop feature triggers and background managers
|
||||
|
||||
Args:
|
||||
eventUser: System-level event user (provided by app.py)
|
||||
"""
|
||||
|
||||
# Unregister automation callback
|
||||
try:
|
||||
from modules.shared.callbackRegistry import callbackRegistry
|
||||
# Note: We'd need to store the callback reference to unregister it properly
|
||||
# For now, callbacks will remain registered (acceptable for shutdown)
|
||||
logger.info("Automation callbacks remain registered (will be cleaned up on process exit)")
|
||||
except Exception as e:
|
||||
logger.warning(f"Error during automation callback cleanup: {str(e)}")
|
||||
|
||||
# Feature ...
|
||||
|
||||
|
|
|
|||
|
|
@ -75,15 +75,7 @@ class AiObjects:
|
|||
|
||||
|
||||
# AI for Extraction, Processing, Generation
|
||||
async def call(self, request: AiCallRequest, progressCallback=None) -> AiCallResponse:
|
||||
"""Call AI model for text generation with model-aware chunking."""
|
||||
# Handle content parts (unified path)
|
||||
if hasattr(request, 'contentParts') and request.contentParts:
|
||||
return await self._callWithContentParts(request, progressCallback)
|
||||
# Handle traditional text/context calls
|
||||
return await self._callWithTextContext(request)
|
||||
|
||||
async def _callWithTextContext(self, request: AiCallRequest) -> AiCallResponse:
|
||||
async def callWithTextContext(self, request: AiCallRequest) -> AiCallResponse:
|
||||
"""Call AI model for traditional text/context calls with fallback mechanism."""
|
||||
prompt = request.prompt
|
||||
context = request.context or ""
|
||||
|
|
@ -148,412 +140,6 @@ class AiObjects:
|
|||
errorCount=1
|
||||
)
|
||||
|
||||
async def _callWithContentParts(self, request: AiCallRequest, progressCallback=None) -> AiCallResponse:
|
||||
"""Process content parts with model-aware chunking (unified for single and multiple parts)."""
|
||||
prompt = request.prompt
|
||||
options = request.options
|
||||
contentParts = request.contentParts
|
||||
|
||||
# Get failover models
|
||||
availableModels = modelRegistry.getAvailableModels()
|
||||
failoverModelList = modelSelector.getFailoverModelList(prompt, "", options, availableModels)
|
||||
|
||||
if not failoverModelList:
|
||||
return self._createErrorResponse("No suitable models found", 0, 0)
|
||||
|
||||
# Process each content part
|
||||
allResults = []
|
||||
for contentPart in contentParts:
|
||||
partResult = await self._processContentPartWithFallback(contentPart, prompt, options, failoverModelList, progressCallback)
|
||||
allResults.append(partResult)
|
||||
|
||||
# Merge all results
|
||||
mergedContent = self._mergePartResults(allResults)
|
||||
|
||||
return AiCallResponse(
|
||||
content=mergedContent,
|
||||
modelName="multiple",
|
||||
priceUsd=sum(r.priceUsd for r in allResults),
|
||||
processingTime=sum(r.processingTime for r in allResults),
|
||||
bytesSent=sum(r.bytesSent for r in allResults),
|
||||
bytesReceived=sum(r.bytesReceived for r in allResults),
|
||||
errorCount=sum(r.errorCount for r in allResults)
|
||||
)
|
||||
|
||||
async def _processContentPartWithFallback(self, contentPart, prompt: str, options, failoverModelList, progressCallback=None) -> AiCallResponse:
|
||||
"""Process a single content part with model-aware chunking and fallback."""
|
||||
lastError = None
|
||||
|
||||
# Check if this is an image - Vision models need special handling
|
||||
isImage = (contentPart.typeGroup == "image") or (contentPart.mimeType and contentPart.mimeType.startswith("image/"))
|
||||
|
||||
# Determine the correct operation type based on content type
|
||||
# Images should use IMAGE_ANALYSE, not the generic operation type
|
||||
actualOperationType = options.operationType
|
||||
if isImage:
|
||||
actualOperationType = OperationTypeEnum.IMAGE_ANALYSE
|
||||
# Get vision-capable models for images
|
||||
availableModels = modelRegistry.getAvailableModels()
|
||||
visionFailoverList = modelSelector.getFailoverModelList(prompt, "", AiCallOptions(operationType=actualOperationType), availableModels)
|
||||
if visionFailoverList:
|
||||
logger.debug(f"Using {len(visionFailoverList)} vision-capable models for image processing")
|
||||
failoverModelList = visionFailoverList
|
||||
|
||||
for attempt, model in enumerate(failoverModelList):
|
||||
try:
|
||||
logger.info(f"Processing content part with model: {model.name} (attempt {attempt + 1}/{len(failoverModelList)})")
|
||||
|
||||
# Special handling for images with Vision models
|
||||
if isImage and hasattr(model, 'functionCall'):
|
||||
# Call model's functionCall directly (for Vision models this is callAiImage)
|
||||
from modules.datamodels.datamodelAi import AiModelCall, AiCallOptions as AiCallOpts
|
||||
|
||||
try:
|
||||
# Validate and prepare image data
|
||||
if not contentPart.data:
|
||||
raise ValueError("Image content part has no data")
|
||||
|
||||
# Ensure mimeType is valid
|
||||
mimeType = contentPart.mimeType or "image/jpeg"
|
||||
if not mimeType.startswith("image/"):
|
||||
raise ValueError(f"Invalid mimeType for image: {mimeType}")
|
||||
|
||||
# Prepare base64 data
|
||||
if isinstance(contentPart.data, str):
|
||||
# Already base64 encoded - validate it
|
||||
try:
|
||||
base64.b64decode(contentPart.data, validate=True)
|
||||
base64Data = contentPart.data
|
||||
except Exception as e:
|
||||
raise ValueError(f"Invalid base64 data in contentPart: {str(e)}")
|
||||
elif isinstance(contentPart.data, bytes):
|
||||
# Binary data - encode to base64
|
||||
base64Data = base64.b64encode(contentPart.data).decode('utf-8')
|
||||
else:
|
||||
raise ValueError(f"Unsupported data type for image: {type(contentPart.data)}")
|
||||
|
||||
# Create data URL
|
||||
imageDataUrl = f"data:{mimeType};base64,{base64Data}"
|
||||
|
||||
modelCall = AiModelCall(
|
||||
messages=[
|
||||
{
|
||||
"role": "user",
|
||||
"content": [
|
||||
{"type": "text", "text": prompt or ""},
|
||||
{
|
||||
"type": "image_url",
|
||||
"image_url": {
|
||||
"url": imageDataUrl
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
],
|
||||
model=model,
|
||||
options=AiCallOpts(operationType=actualOperationType)
|
||||
)
|
||||
|
||||
modelResponse = await model.functionCall(modelCall)
|
||||
|
||||
if not modelResponse.success:
|
||||
raise ValueError(f"Model call failed: {modelResponse.error}")
|
||||
|
||||
logger.info(f"✅ Image content part processed successfully with model: {model.name}")
|
||||
|
||||
# Convert to AiCallResponse format
|
||||
# Note: AiModelResponse doesn't have priceUsd, and processingTime can be None
|
||||
# Calculate processing time if not provided (fallback to 0.0)
|
||||
processingTime = getattr(modelResponse, 'processingTime', None)
|
||||
if processingTime is None:
|
||||
processingTime = 0.0
|
||||
|
||||
return AiCallResponse(
|
||||
content=modelResponse.content,
|
||||
modelName=model.name,
|
||||
priceUsd=0.0, # Price will be calculated elsewhere if needed
|
||||
processingTime=processingTime,
|
||||
bytesSent=0, # Will be calculated elsewhere
|
||||
bytesReceived=0, # Will be calculated elsewhere
|
||||
errorCount=0
|
||||
)
|
||||
except Exception as e:
|
||||
# Image processing failed with this model
|
||||
lastError = e
|
||||
logger.warning(f"❌ Image processing failed with model {model.name}: {str(e)}")
|
||||
|
||||
# If this is not the last model, try the next one
|
||||
if attempt < len(failoverModelList) - 1:
|
||||
logger.info(f"🔄 Trying next fallback model for image processing...")
|
||||
continue
|
||||
else:
|
||||
# All models failed
|
||||
logger.error(f"💥 All {len(failoverModelList)} models failed for image processing")
|
||||
raise
|
||||
|
||||
# For non-image parts, check if part fits in model context
|
||||
# Calculate available space accounting for prompt, system message, and output reservation
|
||||
partSize = len(contentPart.data.encode('utf-8')) if contentPart.data else 0
|
||||
|
||||
# Use same calculation as _chunkContentPart to determine actual available space
|
||||
modelContextTokens = model.contextLength
|
||||
modelMaxOutputTokens = model.maxTokens
|
||||
|
||||
# Reserve tokens for prompt, system message, output, and message overhead
|
||||
promptTokens = len(prompt.encode('utf-8')) / 4 if prompt else 0
|
||||
systemMessageTokens = 10 # ~40 bytes = 10 tokens
|
||||
outputTokens = modelMaxOutputTokens
|
||||
messageOverheadTokens = 100
|
||||
totalReservedTokens = promptTokens + systemMessageTokens + messageOverheadTokens + outputTokens
|
||||
|
||||
# Available tokens for content (with 80% safety margin)
|
||||
availableContentTokens = int((modelContextTokens - totalReservedTokens) * 0.8)
|
||||
if availableContentTokens < 100:
|
||||
availableContentTokens = max(100, int(modelContextTokens * 0.1))
|
||||
|
||||
# Convert to bytes (1 token ≈ 4 bytes)
|
||||
availableContentBytes = availableContentTokens * 4
|
||||
|
||||
logger.debug(f"Size check for {model.name}: partSize={partSize} bytes, availableContentBytes={availableContentBytes} bytes (contextLength={modelContextTokens} tokens, reserved={totalReservedTokens:.0f} tokens)")
|
||||
|
||||
if partSize <= availableContentBytes:
|
||||
# Part fits - call AI directly
|
||||
response = await self._callWithModel(model, prompt, contentPart.data, options)
|
||||
logger.info(f"✅ Content part processed successfully with model: {model.name}")
|
||||
return response
|
||||
else:
|
||||
# Part too large - chunk it (pass prompt to account for it in chunk size calculation)
|
||||
chunks = await self._chunkContentPart(contentPart, model, options, prompt)
|
||||
if not chunks:
|
||||
raise ValueError(f"Failed to chunk content part for model {model.name}")
|
||||
|
||||
logger.info(f"Starting to process {len(chunks)} chunks with model {model.name}")
|
||||
|
||||
# Log progress if callback provided
|
||||
if progressCallback:
|
||||
progressCallback(0.0, f"Starting to process {len(chunks)} chunks")
|
||||
|
||||
# Process each chunk
|
||||
chunkResults = []
|
||||
for idx, chunk in enumerate(chunks):
|
||||
chunkNum = idx + 1
|
||||
chunkData = chunk.get('data', '')
|
||||
chunkSize = len(chunkData.encode('utf-8')) if chunkData else 0
|
||||
logger.info(f"Processing chunk {chunkNum}/{len(chunks)} with model {model.name}, chunk size: {chunkSize} bytes")
|
||||
|
||||
# Calculate and log progress
|
||||
if progressCallback:
|
||||
progress = chunkNum / len(chunks)
|
||||
progressCallback(progress, f"Processing chunk {chunkNum}/{len(chunks)}")
|
||||
|
||||
try:
|
||||
chunkResponse = await self._callWithModel(model, prompt, chunkData, options)
|
||||
chunkResults.append(chunkResponse)
|
||||
logger.info(f"✅ Chunk {chunkNum}/{len(chunks)} processed successfully")
|
||||
|
||||
# Log completion progress
|
||||
if progressCallback:
|
||||
progressCallback(chunkNum / len(chunks), f"Chunk {chunkNum}/{len(chunks)} processed")
|
||||
except Exception as e:
|
||||
logger.error(f"❌ Error processing chunk {chunkNum}/{len(chunks)}: {str(e)}")
|
||||
raise
|
||||
|
||||
# Merge chunk results
|
||||
mergedContent = self._mergeChunkResults(chunkResults)
|
||||
totalPrice = sum(r.priceUsd for r in chunkResults)
|
||||
totalTime = sum(r.processingTime for r in chunkResults)
|
||||
totalBytesSent = sum(r.bytesSent for r in chunkResults)
|
||||
totalBytesReceived = sum(r.bytesReceived for r in chunkResults)
|
||||
totalErrors = sum(r.errorCount for r in chunkResults)
|
||||
|
||||
logger.info(f"✅ Content part chunked and processed with model: {model.name} ({len(chunks)} chunks)")
|
||||
return AiCallResponse(
|
||||
content=mergedContent,
|
||||
modelName=model.name,
|
||||
priceUsd=totalPrice,
|
||||
processingTime=totalTime,
|
||||
bytesSent=totalBytesSent,
|
||||
bytesReceived=totalBytesReceived,
|
||||
errorCount=totalErrors
|
||||
)
|
||||
|
||||
except Exception as e:
|
||||
lastError = e
|
||||
error_msg = str(e) if str(e) else f"{type(e).__name__}"
|
||||
error_detail = f"❌ Model {model.name} failed for content part: {error_msg}"
|
||||
if hasattr(e, 'detail') and e.detail:
|
||||
error_detail += f" | Detail: {e.detail}"
|
||||
if hasattr(e, 'status_code'):
|
||||
error_detail += f" | Status: {e.status_code}"
|
||||
logger.warning(error_detail, exc_info=True)
|
||||
|
||||
if attempt < len(failoverModelList) - 1:
|
||||
logger.info(f"🔄 Trying next failover model...")
|
||||
continue
|
||||
else:
|
||||
logger.error(f"💥 All {len(failoverModelList)} models failed for content part")
|
||||
break
|
||||
|
||||
# All models failed
|
||||
return self._createErrorResponse(f"All models failed: {str(lastError)}", 0, 0)
|
||||
|
||||
async def _chunkContentPart(self, contentPart, model, options, prompt: str = "") -> List[Dict[str, Any]]:
|
||||
"""Chunk a content part based on model capabilities, accounting for prompt, system message overhead, and maxTokens output."""
|
||||
# Calculate model-specific chunk sizes
|
||||
modelContextTokens = model.contextLength # Total context in tokens
|
||||
modelMaxOutputTokens = model.maxTokens # Maximum output tokens
|
||||
|
||||
# Reserve tokens for:
|
||||
# 1. Prompt (user message)
|
||||
promptTokens = len(prompt.encode('utf-8')) / 4 if prompt else 0
|
||||
|
||||
# 2. System message wrapper ("Context from documents:\n")
|
||||
systemMessageTokens = 10 # ~40 bytes = 10 tokens
|
||||
|
||||
# 3. Max output tokens (model will reserve space for completion)
|
||||
outputTokens = modelMaxOutputTokens
|
||||
|
||||
# 4. JSON structure and message overhead (~100 tokens)
|
||||
messageOverheadTokens = 100
|
||||
|
||||
# Total reserved tokens = input overhead + output reservation
|
||||
totalReservedTokens = promptTokens + systemMessageTokens + messageOverheadTokens + outputTokens
|
||||
|
||||
# Available tokens for content = context length - reserved tokens
|
||||
# Use 80% of available for safety margin
|
||||
availableContentTokens = int((modelContextTokens - totalReservedTokens) * 0.8)
|
||||
|
||||
# Ensure we have at least some space
|
||||
if availableContentTokens < 100:
|
||||
logger.warning(f"Very limited space for content: {availableContentTokens} tokens available. Model: {model.name}, contextLength: {modelContextTokens}, maxTokens: {modelMaxOutputTokens}, prompt: {promptTokens:.0f} tokens")
|
||||
availableContentTokens = max(100, int(modelContextTokens * 0.1)) # Fallback to 10% of context
|
||||
|
||||
# Convert tokens to bytes (1 token ≈ 4 bytes)
|
||||
availableContentBytes = availableContentTokens * 4
|
||||
|
||||
logger.debug(f"Chunking calculation for {model.name}: contextLength={modelContextTokens} tokens, maxTokens={modelMaxOutputTokens} tokens, prompt={promptTokens:.0f} tokens, reserved={totalReservedTokens:.0f} tokens, available={availableContentTokens} tokens ({availableContentBytes} bytes)")
|
||||
|
||||
# Use 70% of available content bytes for text chunks (conservative)
|
||||
textChunkSize = int(availableContentBytes * 0.7)
|
||||
imageChunkSize = int(availableContentBytes * 0.8) # 80% for image chunks
|
||||
|
||||
# Build chunking options
|
||||
chunkingOptions = {
|
||||
"textChunkSize": textChunkSize,
|
||||
"imageChunkSize": imageChunkSize,
|
||||
"maxSize": availableContentBytes,
|
||||
"chunkAllowed": True
|
||||
}
|
||||
|
||||
# Get appropriate chunker
|
||||
from modules.services.serviceExtraction.subRegistry import ChunkerRegistry
|
||||
chunkerRegistry = ChunkerRegistry()
|
||||
chunker = chunkerRegistry.resolve(contentPart.typeGroup)
|
||||
|
||||
if not chunker:
|
||||
logger.warning(f"No chunker found for typeGroup: {contentPart.typeGroup}")
|
||||
return []
|
||||
|
||||
# Chunk the content part
|
||||
try:
|
||||
chunks = chunker.chunk(contentPart, chunkingOptions)
|
||||
logger.debug(f"Created {len(chunks)} chunks for {contentPart.typeGroup} part")
|
||||
return chunks
|
||||
except Exception as e:
|
||||
logger.error(f"Chunking failed for {contentPart.typeGroup}: {str(e)}")
|
||||
return []
|
||||
|
||||
def _mergePartResults(self, partResults: List[AiCallResponse]) -> str:
|
||||
"""Merge part results using the existing sophisticated merging system."""
|
||||
if not partResults:
|
||||
return ""
|
||||
|
||||
# Convert AiCallResponse results to ContentParts for merging
|
||||
from modules.datamodels.datamodelExtraction import ContentPart
|
||||
from modules.services.serviceExtraction.subUtils import makeId
|
||||
|
||||
content_parts = []
|
||||
for i, result in enumerate(partResults):
|
||||
if result.content:
|
||||
content_part = ContentPart(
|
||||
id=str(uuid.uuid4()),
|
||||
parentId=None,
|
||||
label=f"ai_result_{i}",
|
||||
typeGroup="text", # Default to text for AI results
|
||||
mimeType="text/plain",
|
||||
data=result.content,
|
||||
metadata={
|
||||
"aiResult": True,
|
||||
"modelName": result.modelName,
|
||||
"priceUsd": result.priceUsd,
|
||||
"processingTime": result.processingTime,
|
||||
"bytesSent": result.bytesSent,
|
||||
"bytesReceived": result.bytesReceived
|
||||
}
|
||||
)
|
||||
content_parts.append(content_part)
|
||||
|
||||
# Use existing merging system
|
||||
merge_strategy = MergeStrategy(
|
||||
useIntelligentMerging=True,
|
||||
groupBy="typeGroup",
|
||||
orderBy="id",
|
||||
mergeType="concatenate"
|
||||
)
|
||||
|
||||
merged_parts = applyMerging(content_parts, merge_strategy)
|
||||
|
||||
# Convert merged parts back to final string
|
||||
final_content = "\n\n".join([part.data for part in merged_parts])
|
||||
|
||||
logger.info(f"Merged {len(partResults)} AI results using existing merging system")
|
||||
return final_content.strip()
|
||||
|
||||
def _mergeChunkResults(self, chunkResults: List[AiCallResponse]) -> str:
|
||||
"""Merge chunk results using the existing sophisticated merging system."""
|
||||
if not chunkResults:
|
||||
return ""
|
||||
|
||||
# Convert AiCallResponse results to ContentParts for merging
|
||||
|
||||
content_parts = []
|
||||
for i, result in enumerate(chunkResults):
|
||||
if result.content:
|
||||
content_part = ContentPart(
|
||||
id=str(uuid.uuid4()),
|
||||
parentId=None,
|
||||
label=f"chunk_result_{i}",
|
||||
typeGroup="text", # Default to text for AI results
|
||||
mimeType="text/plain",
|
||||
data=result.content,
|
||||
metadata={
|
||||
"aiResult": True,
|
||||
"chunk": True,
|
||||
"modelName": result.modelName,
|
||||
"priceUsd": result.priceUsd,
|
||||
"processingTime": result.processingTime,
|
||||
"bytesSent": result.bytesSent,
|
||||
"bytesReceived": result.bytesReceived
|
||||
}
|
||||
)
|
||||
content_parts.append(content_part)
|
||||
|
||||
# Use existing merging system
|
||||
merge_strategy = MergeStrategy(
|
||||
useIntelligentMerging=True,
|
||||
groupBy="typeGroup",
|
||||
orderBy="id",
|
||||
mergeType="concatenate"
|
||||
)
|
||||
|
||||
merged_parts = applyMerging(content_parts, merge_strategy)
|
||||
|
||||
# Convert merged parts back to final string
|
||||
final_content = "\n\n".join([part.data for part in merged_parts])
|
||||
|
||||
logger.info(f"Merged {len(chunkResults)} chunk results using existing merging system")
|
||||
return final_content.strip()
|
||||
|
||||
def _createErrorResponse(self, errorMsg: str, inputBytes: int, outputBytes: int) -> AiCallResponse:
|
||||
"""Create an error response."""
|
||||
return AiCallResponse(
|
||||
|
|
@ -659,64 +245,4 @@ class AiObjects:
|
|||
return [model.displayName for model in models]
|
||||
|
||||
|
||||
def applyMerging(parts: List[ContentPart], strategy: MergeStrategy) -> List[ContentPart]:
|
||||
"""Apply merging strategy to parts with intelligent token-aware merging."""
|
||||
logger.debug(f"applyMerging called with {len(parts)} parts")
|
||||
|
||||
# Import merging dependencies
|
||||
from modules.services.serviceExtraction.merging.mergerText import TextMerger
|
||||
from modules.services.serviceExtraction.merging.mergerTable import TableMerger
|
||||
from modules.services.serviceExtraction.merging.mergerDefault import DefaultMerger
|
||||
from modules.services.serviceExtraction.subMerger import IntelligentTokenAwareMerger
|
||||
|
||||
# Check if intelligent merging is enabled
|
||||
if strategy.useIntelligentMerging:
|
||||
modelCapabilities = strategy.capabilities or {}
|
||||
subMerger = IntelligentTokenAwareMerger(modelCapabilities)
|
||||
|
||||
# Use intelligent merging for all parts
|
||||
merged = subMerger.mergeChunksIntelligently(parts, strategy.prompt or "")
|
||||
|
||||
# Calculate and log optimization stats
|
||||
stats = subMerger.calculateOptimizationStats(parts, merged)
|
||||
logger.info(f"🧠 Intelligent merging stats: {stats}")
|
||||
logger.debug(f"Intelligent merging: {stats['original_ai_calls']} → {stats['optimized_ai_calls']} calls ({stats['reduction_percent']}% reduction)")
|
||||
|
||||
return merged
|
||||
|
||||
# Fallback to traditional merging
|
||||
textMerger = TextMerger()
|
||||
tableMerger = TableMerger()
|
||||
defaultMerger = DefaultMerger()
|
||||
|
||||
# Group by typeGroup
|
||||
textParts = [p for p in parts if p.typeGroup == "text"]
|
||||
tableParts = [p for p in parts if p.typeGroup == "table"]
|
||||
structureParts = [p for p in parts if p.typeGroup == "structure"]
|
||||
otherParts = [p for p in parts if p.typeGroup not in ("text", "table", "structure")]
|
||||
|
||||
logger.debug(f"Grouped - text: {len(textParts)}, table: {len(tableParts)}, structure: {len(structureParts)}, other: {len(otherParts)}")
|
||||
|
||||
merged: List[ContentPart] = []
|
||||
|
||||
if textParts:
|
||||
textMerged = textMerger.merge(textParts, strategy)
|
||||
logger.debug(f"TextMerger merged {len(textParts)} parts into {len(textMerged)} parts")
|
||||
merged.extend(textMerged)
|
||||
if tableParts:
|
||||
tableMerged = tableMerger.merge(tableParts, strategy)
|
||||
logger.debug(f"TableMerger merged {len(tableParts)} parts into {len(tableMerged)} parts")
|
||||
merged.extend(tableMerged)
|
||||
if structureParts:
|
||||
# For now, treat structure like text
|
||||
structureMerged = textMerger.merge(structureParts, strategy)
|
||||
logger.debug(f"StructureMerger merged {len(structureParts)} parts into {len(structureMerged)} parts")
|
||||
merged.extend(structureMerged)
|
||||
if otherParts:
|
||||
otherMerged = defaultMerger.merge(otherParts, strategy)
|
||||
logger.debug(f"DefaultMerger merged {len(otherParts)} parts into {len(otherMerged)} parts")
|
||||
merged.extend(otherMerged)
|
||||
|
||||
logger.debug(f"applyMerging returning {len(merged)} parts")
|
||||
return merged
|
||||
|
||||
|
|
|
|||
|
|
@ -37,6 +37,136 @@ logger = logging.getLogger(__name__)
|
|||
# Singleton factory for Chat instances
|
||||
_chatInterfaces = {}
|
||||
|
||||
|
||||
def storeDebugMessageAndDocuments(message, currentUser) -> None:
|
||||
"""
|
||||
Store message and documents (metadata and file bytes) for debugging purposes.
|
||||
Structure: {log_dir}/debug/messages/m_round_task_action_timestamp/documentlist_label/
|
||||
- message.json, message_text.txt
|
||||
- document_###_metadata.json
|
||||
- document_###_<original_filename> (actual file bytes)
|
||||
|
||||
Args:
|
||||
message: ChatMessage object to store
|
||||
currentUser: Current user for component interface access
|
||||
"""
|
||||
try:
|
||||
import os
|
||||
from datetime import datetime, UTC
|
||||
from modules.shared.debugLogger import _getBaseDebugDir, _ensureDir
|
||||
from modules.interfaces.interfaceDbComponentObjects import getInterface
|
||||
|
||||
# Create base debug directory (use base debug dir, not prompts subdirectory)
|
||||
baseDebugDir = _getBaseDebugDir()
|
||||
debug_root = os.path.join(baseDebugDir, 'messages')
|
||||
_ensureDir(debug_root)
|
||||
|
||||
# Generate timestamp
|
||||
timestamp = datetime.now(UTC).strftime('%Y%m%d-%H%M%S-%f')[:-3]
|
||||
|
||||
# Create message folder name: m_round_task_action_timestamp
|
||||
# Use actual values from message, not defaults
|
||||
round_str = str(message.roundNumber) if message.roundNumber is not None else "0"
|
||||
task_str = str(message.taskNumber) if message.taskNumber is not None else "0"
|
||||
action_str = str(message.actionNumber) if message.actionNumber is not None else "0"
|
||||
message_folder = f"{timestamp}_m_{round_str}_{task_str}_{action_str}"
|
||||
|
||||
message_path = os.path.join(debug_root, message_folder)
|
||||
os.makedirs(message_path, exist_ok=True)
|
||||
|
||||
# Store message data - use dict() instead of model_dump() for compatibility
|
||||
message_file = os.path.join(message_path, "message.json")
|
||||
with open(message_file, "w", encoding="utf-8") as f:
|
||||
# Convert message to dict manually to avoid model_dump() issues
|
||||
message_dict = {
|
||||
"id": message.id,
|
||||
"workflowId": message.workflowId,
|
||||
"parentMessageId": message.parentMessageId,
|
||||
"message": message.message,
|
||||
"role": message.role,
|
||||
"status": message.status,
|
||||
"sequenceNr": message.sequenceNr,
|
||||
"publishedAt": message.publishedAt,
|
||||
"roundNumber": message.roundNumber,
|
||||
"taskNumber": message.taskNumber,
|
||||
"actionNumber": message.actionNumber,
|
||||
"documentsLabel": message.documentsLabel,
|
||||
"actionId": message.actionId,
|
||||
"actionMethod": message.actionMethod,
|
||||
"actionName": message.actionName,
|
||||
"success": message.success,
|
||||
"documents": []
|
||||
}
|
||||
json.dump(message_dict, f, indent=2, ensure_ascii=False, default=str)
|
||||
|
||||
# Store message content as text
|
||||
if message.message:
|
||||
message_text_file = os.path.join(message_path, "message_text.txt")
|
||||
with open(message_text_file, "w", encoding="utf-8") as f:
|
||||
f.write(str(message.message))
|
||||
|
||||
# Store documents if provided
|
||||
if message.documents and len(message.documents) > 0:
|
||||
# Group documents by documentsLabel
|
||||
documents_by_label = {}
|
||||
for doc in message.documents:
|
||||
label = message.documentsLabel or 'default'
|
||||
if label not in documents_by_label:
|
||||
documents_by_label[label] = []
|
||||
documents_by_label[label].append(doc)
|
||||
|
||||
# Create subfolder for each document label
|
||||
for label, docs in documents_by_label.items():
|
||||
# Sanitize label for filesystem
|
||||
safe_label = "".join(c for c in str(label) if c.isalnum() or c in (' ', '-', '_')).rstrip()
|
||||
safe_label = safe_label.replace(' ', '_')
|
||||
if not safe_label:
|
||||
safe_label = "default"
|
||||
|
||||
label_folder = os.path.join(message_path, safe_label)
|
||||
_ensureDir(label_folder)
|
||||
|
||||
# Store each document
|
||||
for i, doc in enumerate(docs):
|
||||
# Create document metadata file
|
||||
doc_meta = {
|
||||
"id": doc.id,
|
||||
"messageId": doc.messageId,
|
||||
"fileId": doc.fileId,
|
||||
"fileName": doc.fileName,
|
||||
"fileSize": doc.fileSize,
|
||||
"mimeType": doc.mimeType,
|
||||
"roundNumber": doc.roundNumber,
|
||||
"taskNumber": doc.taskNumber,
|
||||
"actionNumber": doc.actionNumber,
|
||||
"actionId": doc.actionId
|
||||
}
|
||||
|
||||
doc_meta_file = os.path.join(label_folder, f"document_{i+1:03d}_metadata.json")
|
||||
with open(doc_meta_file, "w", encoding="utf-8") as f:
|
||||
json.dump(doc_meta, f, indent=2, ensure_ascii=False, default=str)
|
||||
|
||||
# Also store the actual file bytes next to metadata for debugging
|
||||
try:
|
||||
componentInterface = getInterface(currentUser)
|
||||
file_bytes = componentInterface.getFileData(doc.fileId)
|
||||
if file_bytes:
|
||||
# Build a safe filename preserving original name
|
||||
safe_name = doc.fileName or f"document_{i+1:03d}"
|
||||
# Avoid path traversal
|
||||
safe_name = os.path.basename(safe_name)
|
||||
doc_file_path = os.path.join(label_folder, f"document_{i+1:03d}_" + safe_name)
|
||||
with open(doc_file_path, "wb") as df:
|
||||
df.write(file_bytes)
|
||||
else:
|
||||
pass
|
||||
except Exception as e:
|
||||
pass
|
||||
|
||||
except Exception as e:
|
||||
# Silent fail - don't break main flow
|
||||
pass
|
||||
|
||||
class ChatObjects:
|
||||
"""
|
||||
Interface to Chat database and AI Connectors.
|
||||
|
|
@ -893,7 +1023,6 @@ class ChatObjects:
|
|||
)
|
||||
|
||||
# Debug: Store message and documents for debugging - only if debug enabled
|
||||
from modules.shared.debugLogger import storeDebugMessageAndDocuments
|
||||
storeDebugMessageAndDocuments(chat_message, self.currentUser)
|
||||
|
||||
return chat_message
|
||||
|
|
@ -1550,8 +1679,8 @@ class ChatObjects:
|
|||
if createdAutomation.get("executionLogs") is None:
|
||||
createdAutomation["executionLogs"] = []
|
||||
|
||||
# Trigger sync (async, don't wait)
|
||||
asyncio.create_task(self.syncAutomationEvents())
|
||||
# Trigger automation change callback (async, don't wait)
|
||||
asyncio.create_task(self._notifyAutomationChanged())
|
||||
|
||||
return createdAutomation
|
||||
except Exception as e:
|
||||
|
|
@ -1581,8 +1710,8 @@ class ChatObjects:
|
|||
if updatedAutomation.get("executionLogs") is None:
|
||||
updatedAutomation["executionLogs"] = []
|
||||
|
||||
# Trigger sync (async, don't wait)
|
||||
asyncio.create_task(self.syncAutomationEvents())
|
||||
# Trigger automation change callback (async, don't wait)
|
||||
asyncio.create_task(self._notifyAutomationChanged())
|
||||
|
||||
return updatedAutomation
|
||||
except Exception as e:
|
||||
|
|
@ -1611,374 +1740,22 @@ class ChatObjects:
|
|||
# Delete automation from database
|
||||
self.db.recordDelete(AutomationDefinition, automationId)
|
||||
|
||||
# Trigger sync (async, don't wait)
|
||||
asyncio.create_task(self.syncAutomationEvents())
|
||||
# Trigger automation change callback (async, don't wait)
|
||||
asyncio.create_task(self._notifyAutomationChanged())
|
||||
|
||||
return True
|
||||
except Exception as e:
|
||||
logger.error(f"Error deleting automation definition: {str(e)}")
|
||||
raise
|
||||
|
||||
def _replacePlaceholders(self, template: str, placeholders: Dict[str, str]) -> str:
|
||||
"""Replace placeholders in template with actual values. Placeholder format: {{KEY:PLACEHOLDER_NAME}}"""
|
||||
result = template
|
||||
for placeholderName, value in placeholders.items():
|
||||
pattern = f"{{{{KEY:{placeholderName}}}}}"
|
||||
|
||||
# Check if placeholder is in an array context like ["{{KEY:...}}"]
|
||||
# If value is a JSON array/dict, we should replace the entire ["{{KEY:...}}"] with the array
|
||||
arrayPattern = f'["{pattern}"]'
|
||||
if arrayPattern in result:
|
||||
# Check if value is a JSON array/dict
|
||||
isArrayValue = False
|
||||
arrayValue = None
|
||||
|
||||
if isinstance(value, (list, dict)):
|
||||
isArrayValue = True
|
||||
arrayValue = json.dumps(value)
|
||||
elif isinstance(value, str):
|
||||
try:
|
||||
parsed = json.loads(value)
|
||||
if isinstance(parsed, (list, dict)):
|
||||
isArrayValue = True
|
||||
arrayValue = value # Already valid JSON string
|
||||
except (json.JSONDecodeError, ValueError):
|
||||
pass
|
||||
|
||||
if isArrayValue:
|
||||
# Replace ["{{KEY:...}}"] with the array value
|
||||
result = result.replace(arrayPattern, arrayValue)
|
||||
continue # Skip the regular replacement below
|
||||
|
||||
# Regular replacement - check if in quoted context
|
||||
patternStart = result.find(pattern)
|
||||
isQuoted = False
|
||||
if patternStart > 0:
|
||||
charBefore = result[patternStart - 1] if patternStart > 0 else None
|
||||
patternEnd = patternStart + len(pattern)
|
||||
charAfter = result[patternEnd] if patternEnd < len(result) else None
|
||||
if charBefore == '"' and charAfter == '"':
|
||||
isQuoted = True
|
||||
|
||||
# Handle different value types
|
||||
if isinstance(value, (list, dict)):
|
||||
# Python list/dict - convert to JSON
|
||||
replacement = json.dumps(value)
|
||||
elif isinstance(value, str):
|
||||
# String value - check if it's a JSON string representing list/dict
|
||||
try:
|
||||
parsed = json.loads(value)
|
||||
if isinstance(parsed, (list, dict)):
|
||||
# It's a JSON string of a list/dict
|
||||
if isQuoted:
|
||||
# In quoted context, escape the JSON string
|
||||
escaped = json.dumps(value)
|
||||
replacement = escaped[1:-1] # Remove outer quotes
|
||||
else:
|
||||
# In unquoted context, use JSON directly
|
||||
replacement = value
|
||||
else:
|
||||
# It's a JSON string of a primitive
|
||||
if isQuoted:
|
||||
escaped = json.dumps(value)
|
||||
replacement = escaped[1:-1]
|
||||
else:
|
||||
replacement = value
|
||||
except (json.JSONDecodeError, ValueError):
|
||||
# Not valid JSON - treat as plain string
|
||||
if isQuoted:
|
||||
escaped = json.dumps(value)
|
||||
replacement = escaped[1:-1]
|
||||
else:
|
||||
replacement = value
|
||||
else:
|
||||
# Numbers, booleans, None - convert to string
|
||||
replacement = str(value)
|
||||
result = result.replace(pattern, replacement)
|
||||
return result
|
||||
|
||||
def _parseScheduleToCron(self, schedule: str) -> Dict[str, Any]:
|
||||
"""Parse schedule string to cron kwargs for APScheduler"""
|
||||
parts = schedule.split()
|
||||
if len(parts) != 5:
|
||||
raise ValueError(f"Invalid schedule format: {schedule}")
|
||||
|
||||
return {
|
||||
"minute": parts[0],
|
||||
"hour": parts[1],
|
||||
"day": parts[2],
|
||||
"month": parts[3],
|
||||
"day_of_week": parts[4]
|
||||
}
|
||||
|
||||
async def executeAutomation(self, automationId: str) -> ChatWorkflow:
|
||||
"""Execute automation workflow immediately (test mode) with placeholder replacement"""
|
||||
executionStartTime = getUtcTimestamp()
|
||||
executionLog = {
|
||||
"timestamp": executionStartTime,
|
||||
"workflowId": None,
|
||||
"status": "running",
|
||||
"messages": []
|
||||
}
|
||||
|
||||
async def _notifyAutomationChanged(self):
|
||||
"""Notify registered callbacks about automation changes (decoupled from features)."""
|
||||
try:
|
||||
# 1. Load automation definition
|
||||
automation = self.getAutomationDefinition(automationId)
|
||||
if not automation:
|
||||
raise ValueError(f"Automation {automationId} not found")
|
||||
|
||||
executionLog["messages"].append(f"Started execution at {executionStartTime}")
|
||||
|
||||
# 2. Replace placeholders in template to generate plan
|
||||
template = automation.get("template", "")
|
||||
placeholders = automation.get("placeholders", {})
|
||||
planJson = self._replacePlaceholders(template, placeholders)
|
||||
try:
|
||||
plan = json.loads(planJson)
|
||||
except json.JSONDecodeError as e:
|
||||
logger.error(f"Failed to parse plan JSON after placeholder replacement: {str(e)}")
|
||||
logger.error(f"Template: {template[:500]}...")
|
||||
logger.error(f"Placeholders: {placeholders}")
|
||||
logger.error(f"Generated planJson (first 1000 chars): {planJson[:1000]}")
|
||||
logger.error(f"Error position: line {e.lineno}, column {e.colno}, char {e.pos}")
|
||||
if e.pos:
|
||||
start = max(0, e.pos - 100)
|
||||
end = min(len(planJson), e.pos + 100)
|
||||
logger.error(f"Context around error: ...{planJson[start:end]}...")
|
||||
raise ValueError(f"Invalid JSON after placeholder replacement: {str(e)}")
|
||||
executionLog["messages"].append("Template placeholders replaced successfully")
|
||||
|
||||
# 3. Get user who created automation
|
||||
creator_user_id = automation.get("_createdBy")
|
||||
|
||||
# If _createdBy is missing, try to fix it by setting it to current user
|
||||
# This handles automations created before _createdBy was required
|
||||
if not creator_user_id:
|
||||
logger.warning(f"Automation {automationId} has no creator user, setting to current user {self.userId}")
|
||||
try:
|
||||
# Update the automation to set _createdBy
|
||||
self.db.recordModify(
|
||||
AutomationDefinition,
|
||||
automationId,
|
||||
{"_createdBy": self.userId}
|
||||
)
|
||||
creator_user_id = self.userId
|
||||
automation["_createdBy"] = self.userId
|
||||
logger.info(f"Fixed automation {automationId} by setting _createdBy to {self.userId}")
|
||||
executionLog["messages"].append(f"Fixed missing _createdBy field, set to user {self.userId}")
|
||||
except Exception as e:
|
||||
logger.error(f"Error fixing automation {automationId}: {str(e)}")
|
||||
raise ValueError(f"Automation {automationId} has no creator user and could not be fixed")
|
||||
|
||||
# Get user from database
|
||||
from modules.interfaces.interfaceDbAppObjects import getInterface as getAppInterface
|
||||
appInterface = getAppInterface(self.currentUser)
|
||||
creator_user = appInterface.getUser(creator_user_id)
|
||||
if not creator_user:
|
||||
raise ValueError(f"Creator user {creator_user_id} not found")
|
||||
|
||||
executionLog["messages"].append(f"Using creator user: {creator_user_id}")
|
||||
|
||||
# 4. Create UserInputRequest from plan
|
||||
# Embed plan JSON in prompt for TemplateMode to extract
|
||||
promptText = self._planToPrompt(plan)
|
||||
planJson = json.dumps(plan)
|
||||
# Embed plan as JSON comment so TemplateMode can extract it
|
||||
promptWithPlan = f"{promptText}\n\n<!--TEMPLATE_PLAN_START-->\n{planJson}\n<!--TEMPLATE_PLAN_END-->"
|
||||
|
||||
userInput = UserInputRequest(
|
||||
prompt=promptWithPlan,
|
||||
listFileId=[],
|
||||
userLanguage=creator_user.language or "en"
|
||||
)
|
||||
|
||||
executionLog["messages"].append("Starting workflow execution")
|
||||
|
||||
# 5. Start workflow using chatStart
|
||||
from modules.features.chatPlayground.mainChatPlayground import chatStart
|
||||
|
||||
workflow = await chatStart(
|
||||
currentUser=creator_user,
|
||||
userInput=userInput,
|
||||
workflowMode=WorkflowModeEnum.WORKFLOW_AUTOMATION,
|
||||
workflowId=None
|
||||
)
|
||||
|
||||
executionLog["workflowId"] = workflow.id
|
||||
executionLog["status"] = "completed"
|
||||
executionLog["messages"].append(f"Workflow {workflow.id} started successfully")
|
||||
logger.info(f"Started workflow {workflow.id} with plan containing {len(plan.get('tasks', []))} tasks (plan embedded in userInput)")
|
||||
|
||||
# Set workflow name with "automated" prefix
|
||||
automationLabel = automation.get("label", "Unknown Automation")
|
||||
workflowName = f"automated: {automationLabel}"
|
||||
workflow = self.updateWorkflow(workflow.id, {"name": workflowName})
|
||||
logger.info(f"Set workflow {workflow.id} name to: {workflowName}")
|
||||
|
||||
# Update automation with execution log
|
||||
executionLogs = automation.get("executionLogs", [])
|
||||
executionLogs.append(executionLog)
|
||||
# Keep only last 50 executions
|
||||
if len(executionLogs) > 50:
|
||||
executionLogs = executionLogs[-50:]
|
||||
|
||||
self.db.recordModify(
|
||||
AutomationDefinition,
|
||||
automationId,
|
||||
{"executionLogs": executionLogs}
|
||||
)
|
||||
|
||||
return workflow
|
||||
from modules.shared.callbackRegistry import callbackRegistry
|
||||
# Trigger callbacks without knowing which features are listening
|
||||
await callbackRegistry.trigger('automation.changed', self)
|
||||
except Exception as e:
|
||||
# Log error to execution log
|
||||
executionLog["status"] = "error"
|
||||
executionLog["messages"].append(f"Error: {str(e)}")
|
||||
|
||||
# Update automation with execution log even on error
|
||||
try:
|
||||
automation = self.getAutomationDefinition(automationId)
|
||||
if automation:
|
||||
executionLogs = automation.get("executionLogs", [])
|
||||
executionLogs.append(executionLog)
|
||||
if len(executionLogs) > 50:
|
||||
executionLogs = executionLogs[-50:]
|
||||
self.db.recordModify(
|
||||
AutomationDefinition,
|
||||
automationId,
|
||||
{"executionLogs": executionLogs}
|
||||
)
|
||||
except Exception as logError:
|
||||
logger.error(f"Error saving execution log: {str(logError)}")
|
||||
|
||||
raise
|
||||
|
||||
def _planToPrompt(self, plan: Dict) -> str:
|
||||
"""Convert plan structure to prompt string for workflow execution"""
|
||||
return plan.get("userMessage", plan.get("overview", "Execute automation workflow"))
|
||||
|
||||
async def syncAutomationEvents(self) -> Dict[str, Any]:
|
||||
"""Automation event handler - syncs scheduler with all active automations."""
|
||||
from modules.shared.eventManagement import eventManager
|
||||
|
||||
# Get all automation definitions (for current mandate)
|
||||
allAutomations = self.db.getRecordset(AutomationDefinition)
|
||||
filtered = self._uam(AutomationDefinition, allAutomations)
|
||||
|
||||
registered_events = {}
|
||||
|
||||
for automation in filtered:
|
||||
automation_id = automation.get("id")
|
||||
is_active = automation.get("active", False)
|
||||
current_event_id = automation.get("eventId")
|
||||
schedule = automation.get("schedule")
|
||||
|
||||
if not schedule:
|
||||
logger.warning(f"Automation {automation_id} has no schedule, skipping")
|
||||
continue
|
||||
|
||||
try:
|
||||
# Parse schedule to cron kwargs
|
||||
cron_kwargs = self._parseScheduleToCron(schedule)
|
||||
|
||||
if is_active:
|
||||
# Remove existing event if present (handles schedule changes)
|
||||
if current_event_id:
|
||||
try:
|
||||
eventManager.remove(current_event_id)
|
||||
except Exception as e:
|
||||
logger.warning(f"Error removing old event {current_event_id}: {str(e)}")
|
||||
|
||||
# Register new event
|
||||
new_event_id = f"automation.{automation_id}"
|
||||
|
||||
# Create event handler function
|
||||
handler = self._createAutomationEventHandler(automation_id)
|
||||
|
||||
# Register cron job
|
||||
eventManager.registerCron(
|
||||
jobId=new_event_id,
|
||||
func=handler,
|
||||
cronKwargs=cron_kwargs,
|
||||
replaceExisting=True
|
||||
)
|
||||
|
||||
# Update automation with new eventId
|
||||
if current_event_id != new_event_id:
|
||||
self.db.recordModify(
|
||||
AutomationDefinition,
|
||||
automation_id,
|
||||
{"eventId": new_event_id}
|
||||
)
|
||||
|
||||
registered_events[automation_id] = new_event_id
|
||||
else:
|
||||
# Remove event if exists
|
||||
if current_event_id:
|
||||
try:
|
||||
eventManager.remove(current_event_id)
|
||||
self.db.recordModify(
|
||||
AutomationDefinition,
|
||||
automation_id,
|
||||
{"eventId": None}
|
||||
)
|
||||
except Exception as e:
|
||||
logger.warning(f"Error removing event {current_event_id}: {str(e)}")
|
||||
except Exception as e:
|
||||
logger.error(f"Error syncing automation {automation_id}: {str(e)}")
|
||||
|
||||
return {
|
||||
"synced": len(registered_events),
|
||||
"events": registered_events
|
||||
}
|
||||
|
||||
def _createAutomationEventHandler(self, automationId: str):
|
||||
"""Create event handler function for a specific automation"""
|
||||
async def handler():
|
||||
try:
|
||||
# Get event user to access automation (event user can access all automations)
|
||||
from modules.interfaces.interfaceDbAppObjects import getRootInterface
|
||||
from modules.interfaces.interfaceDbAppObjects import getInterface as getAppInterface
|
||||
from modules.interfaces.interfaceDbChatObjects import getInterface as getChatInterface
|
||||
|
||||
rootInterface = getRootInterface()
|
||||
eventUser = rootInterface.getUserByUsername("event")
|
||||
|
||||
if not eventUser:
|
||||
logger.error("Could not get event user for automation execution")
|
||||
return
|
||||
|
||||
# Create ChatObjects interface for event user (to access automation)
|
||||
eventInterface = getChatInterface(eventUser)
|
||||
|
||||
# Load automation using event user context
|
||||
automation = eventInterface.getAutomationDefinition(automationId)
|
||||
if not automation or not automation.get("active"):
|
||||
logger.warning(f"Automation {automationId} not found or not active, skipping execution")
|
||||
return
|
||||
|
||||
# Get creator user
|
||||
creator_user_id = automation.get("_createdBy")
|
||||
if not creator_user_id:
|
||||
logger.error(f"Automation {automationId} has no creator user")
|
||||
return
|
||||
|
||||
# Get creator user from database
|
||||
appInterface = getAppInterface(eventUser)
|
||||
creator_user = appInterface.getUser(creator_user_id)
|
||||
if not creator_user:
|
||||
logger.error(f"Creator user {creator_user_id} not found for automation {automationId}")
|
||||
return
|
||||
|
||||
# Create ChatObjects interface for creator user
|
||||
creatorInterface = getChatInterface(creator_user)
|
||||
|
||||
# Execute automation with creator user's context
|
||||
await creatorInterface.executeAutomation(automationId)
|
||||
logger.info(f"Successfully executed automation {automationId} as user {creator_user_id}")
|
||||
except Exception as e:
|
||||
logger.error(f"Error executing automation {automationId}: {str(e)}")
|
||||
|
||||
return handler
|
||||
logger.error(f"Error notifying automation change: {str(e)}")
|
||||
|
||||
|
||||
def getInterface(currentUser: Optional[User] = None) -> 'ChatObjects':
|
||||
|
|
|
|||
|
|
@ -86,15 +86,21 @@ async def sync_all_automation_events(
|
|||
requireSysadmin(currentUser)
|
||||
|
||||
try:
|
||||
chatInterface = interfaceDbChatObjects.getInterface(currentUser)
|
||||
from modules.interfaces.interfaceDbChatObjects import getInterface as getChatInterface
|
||||
from modules.interfaces.interfaceDbAppObjects import getRootInterface
|
||||
from modules.features.automation import syncAutomationEvents
|
||||
|
||||
if not hasattr(chatInterface, 'syncAutomationEvents'):
|
||||
chatInterface = getChatInterface(currentUser)
|
||||
# Get event user for sync operation (routes can import from interfaces)
|
||||
rootInterface = getRootInterface()
|
||||
eventUser = rootInterface.getUserByUsername("event")
|
||||
if not eventUser:
|
||||
raise HTTPException(
|
||||
status_code=501,
|
||||
detail="Automation methods not available"
|
||||
status_code=500,
|
||||
detail="Event user not available"
|
||||
)
|
||||
|
||||
result = await chatInterface.syncAutomationEvents()
|
||||
result = await syncAutomationEvents(chatInterface, eventUser)
|
||||
return {
|
||||
"success": True,
|
||||
"synced": result.get("synced", 0),
|
||||
|
|
|
|||
|
|
@ -84,6 +84,9 @@ class Services:
|
|||
from .serviceWeb.mainServiceWeb import WebService
|
||||
self.web = PublicService(WebService(self))
|
||||
|
||||
from .serviceSecurity.mainServiceSecurity import SecurityService
|
||||
self.security = PublicService(SecurityService(self))
|
||||
|
||||
|
||||
def getInterface(user: User, workflow: ChatWorkflow) -> Services:
|
||||
return Services(user, workflow)
|
||||
|
|
|
|||
|
|
@ -48,6 +48,18 @@ class AiService:
|
|||
logger.info("Initializing ExtractionService...")
|
||||
self.extractionService = ExtractionService(self.services)
|
||||
|
||||
async def callAi(self, request: AiCallRequest, progressCallback=None):
|
||||
"""Router: handles content parts via extractionService, text context via interface.
|
||||
|
||||
Replaces direct calls to self.aiObjects.call() to route content parts processing
|
||||
through serviceExtraction layer.
|
||||
"""
|
||||
if hasattr(request, 'contentParts') and request.contentParts:
|
||||
return await self.extractionService.processContentPartsWithAi(
|
||||
request, self.aiObjects, progressCallback
|
||||
)
|
||||
return await self.aiObjects.callWithTextContext(request)
|
||||
|
||||
async def ensureAiObjectsInitialized(self):
|
||||
"""Ensure aiObjects is initialized and submodules are ready."""
|
||||
if self.aiObjects is None:
|
||||
|
|
@ -141,7 +153,7 @@ Respond with ONLY a JSON object in this exact format:
|
|||
)
|
||||
)
|
||||
|
||||
response = await self.aiObjects.call(request)
|
||||
response = await self.callAi(request)
|
||||
|
||||
# Parse AI response using structured parsing with AiCallOptions model
|
||||
try:
|
||||
|
|
@ -251,7 +263,7 @@ Respond with ONLY a JSON object in this exact format:
|
|||
else:
|
||||
self.services.utils.writeDebugFile(iterationPrompt, f"{debugPrefix}_prompt_iteration_{iteration}")
|
||||
|
||||
response = await self.aiObjects.call(request)
|
||||
response = await self.callAi(request)
|
||||
result = response.content
|
||||
|
||||
# Update progress after AI call
|
||||
|
|
@ -582,7 +594,7 @@ If no trackable items can be identified, return: {{"kpis": []}}
|
|||
# Write KPI definition prompt to debug file
|
||||
self.services.utils.writeDebugFile(kpiDefinitionPrompt, f"{debugPrefix}_kpi_definition_prompt")
|
||||
|
||||
response = await self.aiObjects.call(request)
|
||||
response = await self.callAi(request)
|
||||
|
||||
# Write KPI definition response to debug file
|
||||
self.services.utils.writeDebugFile(response.content, f"{debugPrefix}_kpi_definition_response")
|
||||
|
|
@ -966,7 +978,7 @@ If no trackable items can be identified, return: {{"kpis": []}}
|
|||
options=options
|
||||
)
|
||||
|
||||
response = await self.aiObjects.call(request)
|
||||
response = await self.callAi(request)
|
||||
|
||||
if response.content:
|
||||
# Build document data for image
|
||||
|
|
@ -1011,7 +1023,7 @@ If no trackable items can be identified, return: {{"kpis": []}}
|
|||
options=options
|
||||
)
|
||||
|
||||
response = await self.aiObjects.call(request)
|
||||
response = await self.callAi(request)
|
||||
|
||||
if response.content:
|
||||
metadata = AiResponseMetadata(
|
||||
|
|
@ -1046,7 +1058,7 @@ If no trackable items can be identified, return: {{"kpis": []}}
|
|||
options.compressContext = False
|
||||
|
||||
# Process contentParts for generation prompt (if provided)
|
||||
# Use generic _callWithContentParts() which handles all content types (images, text, etc.)
|
||||
# Use generic callWithContentParts() which handles all content types (images, text, etc.)
|
||||
# This automatically processes images with vision models and merges all results
|
||||
if contentParts:
|
||||
# Filter out binary/other parts that shouldn't be processed
|
||||
|
|
|
|||
|
|
@ -3,7 +3,6 @@ from typing import Dict, Any, List, Optional
|
|||
from modules.datamodels.datamodelUam import User, UserConnection
|
||||
from modules.datamodels.datamodelChat import ChatDocument, ChatMessage, ChatStat, ChatLog
|
||||
from modules.datamodels.datamodelAi import AiCallOptions, OperationTypeEnum, PriorityEnum, ProcessingModeEnum
|
||||
from modules.security.tokenManager import TokenManager
|
||||
from modules.shared.progressLogger import ProgressLogger
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
|
@ -306,9 +305,9 @@ class ChatService:
|
|||
token = None
|
||||
token_status = "unknown"
|
||||
try:
|
||||
# Get a fresh token via TokenManager convenience method
|
||||
# Get a fresh token via security service
|
||||
logger.debug(f"Getting fresh token for connection {connection.id}")
|
||||
token = TokenManager().getFreshToken(connection.id)
|
||||
token = self.services.security.getFreshToken(connection.id)
|
||||
if token:
|
||||
if hasattr(token, 'expiresAt') and token.expiresAt:
|
||||
current_time = self.services.utils.timestampGetUtc()
|
||||
|
|
@ -389,7 +388,7 @@ class ChatService:
|
|||
Token object or None if not found/expired
|
||||
"""
|
||||
try:
|
||||
return TokenManager().getFreshToken(connectionId)
|
||||
return self.services.security.getFreshToken(connectionId)
|
||||
except Exception as e:
|
||||
logger.error(f"Error getting fresh token for connection {connectionId}: {str(e)}")
|
||||
return None
|
||||
|
|
|
|||
|
|
@ -3,13 +3,15 @@ import uuid
|
|||
import logging
|
||||
import time
|
||||
import asyncio
|
||||
import base64
|
||||
|
||||
from .subRegistry import ExtractorRegistry, ChunkerRegistry
|
||||
from .subPipeline import runExtraction
|
||||
from modules.datamodels.datamodelExtraction import ContentExtracted, ContentPart, MergeStrategy, ExtractionOptions, PartResult
|
||||
from modules.datamodels.datamodelChat import ChatDocument
|
||||
from modules.datamodels.datamodelAi import AiCallResponse, AiCallRequest, AiCallOptions
|
||||
from modules.datamodels.datamodelAi import AiCallResponse, AiCallRequest, AiCallOptions, OperationTypeEnum, AiModelCall
|
||||
from modules.aicore.aicoreModelRegistry import modelRegistry
|
||||
from modules.aicore.aicoreModelSelector import modelSelector
|
||||
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
|
@ -498,7 +500,7 @@ class ExtractionService:
|
|||
# Merge results using existing merging system
|
||||
if operationId:
|
||||
self.services.chat.progressLogUpdate(operationId, 0.9, f"Merging {len(partResults)} part results")
|
||||
mergedContent = self._mergePartResults(partResults, options)
|
||||
mergedContent = self.mergePartResults(partResults, options)
|
||||
|
||||
# Save merged extraction content to debug
|
||||
self.services.utils.writeDebugFile(mergedContent or '', "extraction_merged_text")
|
||||
|
|
@ -660,54 +662,473 @@ class ExtractionService:
|
|||
logger.info(f"Completed processing {len(processedResults)} parts")
|
||||
return processedResults
|
||||
|
||||
def _mergePartResults(
|
||||
def _convertToContentParts(
|
||||
self, partResults: Union[List[PartResult], List[AiCallResponse]]
|
||||
) -> List[ContentPart]:
|
||||
"""Convert part results to ContentParts (internal helper for consolidation).
|
||||
|
||||
Handles both PartResult (from extraction workflow) and AiCallResponse (from content parts processing).
|
||||
"""
|
||||
content_parts = []
|
||||
|
||||
if not partResults:
|
||||
return content_parts
|
||||
|
||||
# Detect input type and convert accordingly
|
||||
if isinstance(partResults[0], PartResult):
|
||||
# Existing logic for PartResult (from processDocumentsPerChunk)
|
||||
for part_result in partResults:
|
||||
content_part = ContentPart(
|
||||
id=part_result.originalPart.id,
|
||||
parentId=part_result.originalPart.parentId,
|
||||
label=part_result.originalPart.label,
|
||||
typeGroup=part_result.originalPart.typeGroup, # Use original typeGroup
|
||||
mimeType=part_result.originalPart.mimeType,
|
||||
data=part_result.aiResult, # Use AI result as data
|
||||
metadata={
|
||||
**part_result.originalPart.metadata,
|
||||
"aiResult": True,
|
||||
"partIndex": part_result.partIndex,
|
||||
"documentId": part_result.documentId,
|
||||
"processingTime": part_result.processingTime,
|
||||
"success": part_result.metadata.get("success", False)
|
||||
}
|
||||
)
|
||||
content_parts.append(content_part)
|
||||
elif isinstance(partResults[0], AiCallResponse):
|
||||
# Logic from interfaceAiObjects (from content parts processing)
|
||||
for i, result in enumerate(partResults):
|
||||
if result.content:
|
||||
content_part = ContentPart(
|
||||
id=str(uuid.uuid4()),
|
||||
parentId=None,
|
||||
label=f"ai_result_{i}",
|
||||
typeGroup="text", # Default to text for AI results
|
||||
mimeType="text/plain",
|
||||
data=result.content,
|
||||
metadata={
|
||||
"aiResult": True,
|
||||
"modelName": result.modelName,
|
||||
"priceUsd": result.priceUsd,
|
||||
"processingTime": result.processingTime,
|
||||
"bytesSent": result.bytesSent,
|
||||
"bytesReceived": result.bytesReceived
|
||||
}
|
||||
)
|
||||
content_parts.append(content_part)
|
||||
|
||||
return content_parts
|
||||
|
||||
def mergePartResults(
|
||||
self,
|
||||
partResults: List[PartResult],
|
||||
partResults: Union[List[PartResult], List[AiCallResponse]],
|
||||
options: Optional[AiCallOptions] = None
|
||||
) -> str:
|
||||
"""Merge part results using existing sophisticated merging system."""
|
||||
) -> str:
|
||||
"""Unified merge for both PartResult and AiCallResponse.
|
||||
|
||||
Consolidated from both interfaceAiObjects.py and existing serviceExtraction method.
|
||||
"""
|
||||
if not partResults:
|
||||
return ""
|
||||
|
||||
# Convert PartResults back to ContentParts for existing merger system
|
||||
content_parts = []
|
||||
for part_result in partResults:
|
||||
# Create ContentPart from PartResult with proper typeGroup
|
||||
content_part = ContentPart(
|
||||
id=part_result.originalPart.id,
|
||||
parentId=part_result.originalPart.parentId,
|
||||
label=part_result.originalPart.label,
|
||||
typeGroup=part_result.originalPart.typeGroup, # Use original typeGroup
|
||||
mimeType=part_result.originalPart.mimeType,
|
||||
data=part_result.aiResult, # Use AI result as data
|
||||
metadata={
|
||||
**part_result.originalPart.metadata,
|
||||
"aiResult": True,
|
||||
"partIndex": part_result.partIndex,
|
||||
"documentId": part_result.documentId,
|
||||
"processingTime": part_result.processingTime,
|
||||
"success": part_result.metadata.get("success", False)
|
||||
}
|
||||
# Convert to ContentParts using unified helper
|
||||
content_parts = self._convertToContentParts(partResults)
|
||||
|
||||
# Determine merge strategy based on input type
|
||||
if isinstance(partResults[0], PartResult):
|
||||
# Use strategy for extraction workflow (group by document, order by part index)
|
||||
merge_strategy = MergeStrategy(
|
||||
useIntelligentMerging=True,
|
||||
groupBy="documentId", # Group by document
|
||||
orderBy="partIndex", # Order by part index
|
||||
mergeType="concatenate"
|
||||
)
|
||||
else:
|
||||
# Default strategy for content parts workflow
|
||||
merge_strategy = MergeStrategy(
|
||||
useIntelligentMerging=True,
|
||||
groupBy="typeGroup",
|
||||
orderBy="id",
|
||||
mergeType="concatenate"
|
||||
)
|
||||
content_parts.append(content_part)
|
||||
|
||||
# Use existing merging strategy from options
|
||||
merge_strategy = MergeStrategy(
|
||||
useIntelligentMerging=True,
|
||||
groupBy="documentId", # Group by document
|
||||
orderBy="partIndex", # Order by part index
|
||||
mergeType="concatenate"
|
||||
)
|
||||
|
||||
|
||||
# Apply existing merging logic using the sophisticated merging system
|
||||
from modules.interfaces.interfaceAiObjects import applyMerging
|
||||
# Apply merging
|
||||
merged_parts = applyMerging(content_parts, merge_strategy)
|
||||
|
||||
# Convert merged parts back to final string
|
||||
# Convert back to string
|
||||
final_content = "\n\n".join([part.data for part in merged_parts])
|
||||
|
||||
logger.info(f"Merged {len(partResults)} parts using existing sophisticated merging system")
|
||||
logger.info(f"Merged {len(partResults)} parts using unified merging system")
|
||||
return final_content.strip()
|
||||
|
||||
async def chunkContentPartForAi(self, contentPart, model, options, prompt: str = "") -> List[Dict[str, Any]]:
|
||||
"""Chunk a content part based on model capabilities, accounting for prompt, system message overhead, and maxTokens output.
|
||||
|
||||
Moved from interfaceAiObjects.py - model-aware chunking for AI processing.
|
||||
Complementary to existing size-based chunking in extraction pipeline.
|
||||
"""
|
||||
# Calculate model-specific chunk sizes
|
||||
modelContextTokens = model.contextLength # Total context in tokens
|
||||
modelMaxOutputTokens = model.maxTokens # Maximum output tokens
|
||||
|
||||
# Reserve tokens for:
|
||||
# 1. Prompt (user message)
|
||||
promptTokens = len(prompt.encode('utf-8')) / 4 if prompt else 0
|
||||
|
||||
# 2. System message wrapper ("Context from documents:\n")
|
||||
systemMessageTokens = 10 # ~40 bytes = 10 tokens
|
||||
|
||||
# 3. Max output tokens (model will reserve space for completion)
|
||||
outputTokens = modelMaxOutputTokens
|
||||
|
||||
# 4. JSON structure and message overhead (~100 tokens)
|
||||
messageOverheadTokens = 100
|
||||
|
||||
# Total reserved tokens = input overhead + output reservation
|
||||
totalReservedTokens = promptTokens + systemMessageTokens + messageOverheadTokens + outputTokens
|
||||
|
||||
# Available tokens for content = context length - reserved tokens
|
||||
# Use 80% of available for safety margin
|
||||
availableContentTokens = int((modelContextTokens - totalReservedTokens) * 0.8)
|
||||
|
||||
# Ensure we have at least some space
|
||||
if availableContentTokens < 100:
|
||||
logger.warning(f"Very limited space for content: {availableContentTokens} tokens available. Model: {model.name}, contextLength: {modelContextTokens}, maxTokens: {modelMaxOutputTokens}, prompt: {promptTokens:.0f} tokens")
|
||||
availableContentTokens = max(100, int(modelContextTokens * 0.1)) # Fallback to 10% of context
|
||||
|
||||
# Convert tokens to bytes (1 token ≈ 4 bytes)
|
||||
availableContentBytes = availableContentTokens * 4
|
||||
|
||||
logger.debug(f"Chunking calculation for {model.name}: contextLength={modelContextTokens} tokens, maxTokens={modelMaxOutputTokens} tokens, prompt={promptTokens:.0f} tokens, reserved={totalReservedTokens:.0f} tokens, available={availableContentTokens} tokens ({availableContentBytes} bytes)")
|
||||
|
||||
# Use 70% of available content bytes for text chunks (conservative)
|
||||
textChunkSize = int(availableContentBytes * 0.7)
|
||||
imageChunkSize = int(availableContentBytes * 0.8) # 80% for image chunks
|
||||
|
||||
# Build chunking options
|
||||
chunkingOptions = {
|
||||
"textChunkSize": textChunkSize,
|
||||
"imageChunkSize": imageChunkSize,
|
||||
"maxSize": availableContentBytes,
|
||||
"chunkAllowed": True
|
||||
}
|
||||
|
||||
# Get appropriate chunker (uses existing ChunkerRegistry ✅)
|
||||
chunker = self._chunkerRegistry.resolve(contentPart.typeGroup)
|
||||
|
||||
if not chunker:
|
||||
logger.warning(f"No chunker found for typeGroup: {contentPart.typeGroup}")
|
||||
return []
|
||||
|
||||
# Chunk the content part
|
||||
try:
|
||||
chunks = chunker.chunk(contentPart, chunkingOptions)
|
||||
logger.debug(f"Created {len(chunks)} chunks for {contentPart.typeGroup} part")
|
||||
return chunks
|
||||
except Exception as e:
|
||||
logger.error(f"Chunking failed for {contentPart.typeGroup}: {str(e)}")
|
||||
return []
|
||||
|
||||
async def processContentPartWithFallback(self, contentPart, prompt: str, options, failoverModelList, aiObjects, progressCallback=None) -> AiCallResponse:
|
||||
"""Process a single content part with model-aware chunking and fallback.
|
||||
|
||||
Moved from interfaceAiObjects.py - orchestrates chunking and merging.
|
||||
Calls aiObjects._callWithModel() for actual AI calls.
|
||||
"""
|
||||
lastError = None
|
||||
|
||||
# Check if this is an image - Vision models need special handling
|
||||
isImage = (contentPart.typeGroup == "image") or (contentPart.mimeType and contentPart.mimeType.startswith("image/"))
|
||||
|
||||
# Determine the correct operation type based on content type
|
||||
actualOperationType = options.operationType
|
||||
if isImage:
|
||||
actualOperationType = OperationTypeEnum.IMAGE_ANALYSE
|
||||
# Get vision-capable models for images
|
||||
availableModels = modelRegistry.getAvailableModels()
|
||||
visionFailoverList = modelSelector.getFailoverModelList(prompt, "", AiCallOptions(operationType=actualOperationType), availableModels)
|
||||
if visionFailoverList:
|
||||
logger.debug(f"Using {len(visionFailoverList)} vision-capable models for image processing")
|
||||
failoverModelList = visionFailoverList
|
||||
|
||||
for attempt, model in enumerate(failoverModelList):
|
||||
try:
|
||||
logger.info(f"Processing content part with model: {model.name} (attempt {attempt + 1}/{len(failoverModelList)})")
|
||||
|
||||
# Special handling for images with Vision models
|
||||
if isImage and hasattr(model, 'functionCall'):
|
||||
try:
|
||||
if not contentPart.data:
|
||||
raise ValueError("Image content part has no data")
|
||||
|
||||
mimeType = contentPart.mimeType or "image/jpeg"
|
||||
if not mimeType.startswith("image/"):
|
||||
raise ValueError(f"Invalid mimeType for image: {mimeType}")
|
||||
|
||||
# Prepare base64 data
|
||||
if isinstance(contentPart.data, str):
|
||||
try:
|
||||
base64.b64decode(contentPart.data, validate=True)
|
||||
base64Data = contentPart.data
|
||||
except Exception as e:
|
||||
raise ValueError(f"Invalid base64 data in contentPart: {str(e)}")
|
||||
elif isinstance(contentPart.data, bytes):
|
||||
base64Data = base64.b64encode(contentPart.data).decode('utf-8')
|
||||
else:
|
||||
raise ValueError(f"Unsupported data type for image: {type(contentPart.data)}")
|
||||
|
||||
imageDataUrl = f"data:{mimeType};base64,{base64Data}"
|
||||
|
||||
modelCall = AiModelCall(
|
||||
messages=[
|
||||
{
|
||||
"role": "user",
|
||||
"content": [
|
||||
{"type": "text", "text": prompt or ""},
|
||||
{
|
||||
"type": "image_url",
|
||||
"image_url": {"url": imageDataUrl}
|
||||
}
|
||||
]
|
||||
}
|
||||
],
|
||||
model=model,
|
||||
options=AiCallOptions(operationType=actualOperationType)
|
||||
)
|
||||
|
||||
modelResponse = await model.functionCall(modelCall)
|
||||
|
||||
if not modelResponse.success:
|
||||
raise ValueError(f"Model call failed: {modelResponse.error}")
|
||||
|
||||
logger.info(f"✅ Image content part processed successfully with model: {model.name}")
|
||||
|
||||
processingTime = getattr(modelResponse, 'processingTime', None) or 0.0
|
||||
|
||||
return AiCallResponse(
|
||||
content=modelResponse.content,
|
||||
modelName=model.name,
|
||||
priceUsd=0.0,
|
||||
processingTime=processingTime,
|
||||
bytesSent=0,
|
||||
bytesReceived=0,
|
||||
errorCount=0
|
||||
)
|
||||
except Exception as e:
|
||||
lastError = e
|
||||
logger.warning(f"❌ Image processing failed with model {model.name}: {str(e)}")
|
||||
|
||||
if attempt < len(failoverModelList) - 1:
|
||||
logger.info(f"🔄 Trying next fallback model for image processing...")
|
||||
continue
|
||||
else:
|
||||
logger.error(f"💥 All {len(failoverModelList)} models failed for image processing")
|
||||
raise
|
||||
|
||||
# For non-image parts, check if part fits in model context
|
||||
partSize = len(contentPart.data.encode('utf-8')) if contentPart.data else 0
|
||||
|
||||
modelContextTokens = model.contextLength
|
||||
modelMaxOutputTokens = model.maxTokens
|
||||
|
||||
promptTokens = len(prompt.encode('utf-8')) / 4 if prompt else 0
|
||||
systemMessageTokens = 10
|
||||
outputTokens = modelMaxOutputTokens
|
||||
messageOverheadTokens = 100
|
||||
totalReservedTokens = promptTokens + systemMessageTokens + messageOverheadTokens + outputTokens
|
||||
|
||||
availableContentTokens = int((modelContextTokens - totalReservedTokens) * 0.8)
|
||||
if availableContentTokens < 100:
|
||||
availableContentTokens = max(100, int(modelContextTokens * 0.1))
|
||||
|
||||
availableContentBytes = availableContentTokens * 4
|
||||
|
||||
logger.debug(f"Size check for {model.name}: partSize={partSize} bytes, availableContentBytes={availableContentBytes} bytes")
|
||||
|
||||
if partSize <= availableContentBytes:
|
||||
# Part fits - call AI directly via aiObjects interface
|
||||
response = await aiObjects._callWithModel(model, prompt, contentPart.data, options)
|
||||
logger.info(f"✅ Content part processed successfully with model: {model.name}")
|
||||
return response
|
||||
else:
|
||||
# Part too large - chunk it
|
||||
chunks = await self.chunkContentPartForAi(contentPart, model, options, prompt)
|
||||
if not chunks:
|
||||
raise ValueError(f"Failed to chunk content part for model {model.name}")
|
||||
|
||||
logger.info(f"Starting to process {len(chunks)} chunks with model {model.name}")
|
||||
|
||||
if progressCallback:
|
||||
progressCallback(0.0, f"Starting to process {len(chunks)} chunks")
|
||||
|
||||
chunkResults = []
|
||||
for idx, chunk in enumerate(chunks):
|
||||
chunkNum = idx + 1
|
||||
chunkData = chunk.get('data', '')
|
||||
logger.info(f"Processing chunk {chunkNum}/{len(chunks)} with model {model.name}")
|
||||
|
||||
if progressCallback:
|
||||
progressCallback(chunkNum / len(chunks), f"Processing chunk {chunkNum}/{len(chunks)}")
|
||||
|
||||
try:
|
||||
chunkResponse = await aiObjects._callWithModel(model, prompt, chunkData, options)
|
||||
chunkResults.append(chunkResponse)
|
||||
logger.info(f"✅ Chunk {chunkNum}/{len(chunks)} processed successfully")
|
||||
|
||||
if progressCallback:
|
||||
progressCallback(chunkNum / len(chunks), f"Chunk {chunkNum}/{len(chunks)} processed")
|
||||
except Exception as e:
|
||||
logger.error(f"❌ Error processing chunk {chunkNum}/{len(chunks)}: {str(e)}")
|
||||
raise
|
||||
|
||||
# Merge chunk results
|
||||
mergedContent = self.mergeChunkResults(chunkResults)
|
||||
|
||||
logger.info(f"✅ Content part chunked and processed with model: {model.name} ({len(chunks)} chunks)")
|
||||
return AiCallResponse(
|
||||
content=mergedContent,
|
||||
modelName=model.name,
|
||||
priceUsd=sum(r.priceUsd for r in chunkResults),
|
||||
processingTime=sum(r.processingTime for r in chunkResults),
|
||||
bytesSent=sum(r.bytesSent for r in chunkResults),
|
||||
bytesReceived=sum(r.bytesReceived for r in chunkResults),
|
||||
errorCount=sum(r.errorCount for r in chunkResults)
|
||||
)
|
||||
|
||||
except Exception as e:
|
||||
lastError = e
|
||||
error_msg = str(e) if str(e) else f"{type(e).__name__}"
|
||||
logger.warning(f"❌ Model {model.name} failed for content part: {error_msg}", exc_info=True)
|
||||
|
||||
if attempt < len(failoverModelList) - 1:
|
||||
logger.info(f"🔄 Trying next failover model...")
|
||||
continue
|
||||
else:
|
||||
logger.error(f"💥 All {len(failoverModelList)} models failed for content part")
|
||||
break
|
||||
|
||||
# All models failed
|
||||
return self._createErrorResponse(f"All models failed: {str(lastError)}", 0, 0)
|
||||
|
||||
def _createErrorResponse(self, errorMsg: str, inputBytes: int, outputBytes: int) -> AiCallResponse:
|
||||
"""Create an error response."""
|
||||
return AiCallResponse(
|
||||
content=errorMsg,
|
||||
modelName="error",
|
||||
priceUsd=0.0,
|
||||
processingTime=0.0,
|
||||
bytesSent=inputBytes,
|
||||
bytesReceived=outputBytes,
|
||||
errorCount=1
|
||||
)
|
||||
|
||||
async def processContentPartsWithAi(
|
||||
self,
|
||||
request: AiCallRequest,
|
||||
aiObjects, # Pass interface for AI calls
|
||||
progressCallback=None
|
||||
) -> AiCallResponse:
|
||||
"""Process content parts with model-aware chunking and AI calls.
|
||||
|
||||
Moved from interfaceAiObjects.callWithContentParts() - entry point for content parts processing.
|
||||
"""
|
||||
prompt = request.prompt
|
||||
options = request.options
|
||||
contentParts = request.contentParts
|
||||
|
||||
# Get failover models
|
||||
availableModels = modelRegistry.getAvailableModels()
|
||||
failoverModelList = modelSelector.getFailoverModelList(prompt, "", options, availableModels)
|
||||
|
||||
if not failoverModelList:
|
||||
return self._createErrorResponse("No suitable models found", 0, 0)
|
||||
|
||||
# Process each content part
|
||||
allResults = []
|
||||
for contentPart in contentParts:
|
||||
partResult = await self.processContentPartWithFallback(
|
||||
contentPart, prompt, options, failoverModelList, aiObjects, progressCallback
|
||||
)
|
||||
allResults.append(partResult)
|
||||
|
||||
# Merge all results using unified mergePartResults
|
||||
mergedContent = self.mergePartResults(allResults)
|
||||
|
||||
return AiCallResponse(
|
||||
content=mergedContent,
|
||||
modelName="multiple",
|
||||
priceUsd=sum(r.priceUsd for r in allResults),
|
||||
processingTime=sum(r.processingTime for r in allResults),
|
||||
bytesSent=sum(r.bytesSent for r in allResults),
|
||||
bytesReceived=sum(r.bytesReceived for r in allResults),
|
||||
errorCount=sum(r.errorCount for r in allResults)
|
||||
)
|
||||
|
||||
|
||||
# Module-level function for use by subPipeline and ExtractionService
|
||||
def applyMerging(parts: List[ContentPart], strategy: MergeStrategy) -> List[ContentPart]:
|
||||
"""Apply merging strategy to parts with intelligent token-aware merging.
|
||||
|
||||
Moved from interfaceAiObjects.py to resolve dependency violations.
|
||||
Can be used as module-level function or called from ExtractionService methods.
|
||||
"""
|
||||
logger.debug(f"applyMerging called with {len(parts)} parts")
|
||||
|
||||
# Import merging dependencies (now local imports ✅)
|
||||
from .merging.mergerText import TextMerger
|
||||
from .merging.mergerTable import TableMerger
|
||||
from .merging.mergerDefault import DefaultMerger
|
||||
from .subMerger import IntelligentTokenAwareMerger
|
||||
|
||||
# Check if intelligent merging is enabled
|
||||
if strategy.useIntelligentMerging:
|
||||
modelCapabilities = strategy.capabilities or {}
|
||||
subMerger = IntelligentTokenAwareMerger(modelCapabilities)
|
||||
|
||||
# Use intelligent merging for all parts
|
||||
merged = subMerger.mergeChunksIntelligently(parts, strategy.prompt or "")
|
||||
|
||||
# Calculate and log optimization stats
|
||||
stats = subMerger.calculateOptimizationStats(parts, merged)
|
||||
logger.info(f"🧠 Intelligent merging stats: {stats}")
|
||||
logger.debug(f"Intelligent merging: {stats['original_ai_calls']} → {stats['optimized_ai_calls']} calls ({stats['reduction_percent']}% reduction)")
|
||||
|
||||
return merged
|
||||
|
||||
# Fallback to traditional merging
|
||||
textMerger = TextMerger()
|
||||
tableMerger = TableMerger()
|
||||
defaultMerger = DefaultMerger()
|
||||
|
||||
# Group by typeGroup
|
||||
textParts = [p for p in parts if p.typeGroup == "text"]
|
||||
tableParts = [p for p in parts if p.typeGroup == "table"]
|
||||
structureParts = [p for p in parts if p.typeGroup == "structure"]
|
||||
otherParts = [p for p in parts if p.typeGroup not in ("text", "table", "structure")]
|
||||
|
||||
logger.debug(f"Grouped - text: {len(textParts)}, table: {len(tableParts)}, structure: {len(structureParts)}, other: {len(otherParts)}")
|
||||
|
||||
merged: List[ContentPart] = []
|
||||
|
||||
if textParts:
|
||||
textMerged = textMerger.merge(textParts, strategy)
|
||||
logger.debug(f"TextMerger merged {len(textParts)} parts into {len(textMerged)} parts")
|
||||
merged.extend(textMerged)
|
||||
if tableParts:
|
||||
tableMerged = tableMerger.merge(tableParts, strategy)
|
||||
logger.debug(f"TableMerger merged {len(tableParts)} parts into {len(tableMerged)} parts")
|
||||
merged.extend(tableMerged)
|
||||
if structureParts:
|
||||
# For now, treat structure like text
|
||||
structureMerged = textMerger.merge(structureParts, strategy)
|
||||
logger.debug(f"StructureMerger merged {len(structureParts)} parts into {len(structureMerged)} parts")
|
||||
merged.extend(structureMerged)
|
||||
if otherParts:
|
||||
otherMerged = defaultMerger.merge(otherParts, strategy)
|
||||
logger.debug(f"DefaultMerger merged {len(otherParts)} parts into {len(otherMerged)} parts")
|
||||
merged.extend(otherMerged)
|
||||
|
||||
logger.debug(f"applyMerging returning {len(merged)} parts")
|
||||
return merged
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -34,7 +34,8 @@ def runExtraction(extractorRegistry: ExtractorRegistry, chunkerRegistry: Chunker
|
|||
|
||||
# Apply merging strategy if provided (preserve existing logic)
|
||||
if options.mergeStrategy:
|
||||
from modules.interfaces.interfaceAiObjects import applyMerging
|
||||
# Use module-level applyMerging function
|
||||
from .mainServiceExtraction import applyMerging
|
||||
parts = applyMerging(parts, options.mergeStrategy)
|
||||
|
||||
return ContentExtracted(id=makeId(), parts=parts)
|
||||
|
|
|
|||
128
modules/services/serviceSecurity/mainServiceSecurity.py
Normal file
128
modules/services/serviceSecurity/mainServiceSecurity.py
Normal file
|
|
@ -0,0 +1,128 @@
|
|||
"""
|
||||
Security service for token management operations.
|
||||
Provides centralized access to token refresh and management functionality.
|
||||
"""
|
||||
|
||||
import logging
|
||||
from typing import Optional, Callable
|
||||
|
||||
from modules.datamodels.datamodelSecurity import Token
|
||||
from modules.security.tokenManager import TokenManager
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class SecurityService:
|
||||
"""Security service providing token management operations."""
|
||||
|
||||
def __init__(self, services):
|
||||
"""Initialize security service with service center access.
|
||||
|
||||
Args:
|
||||
services: Service center instance providing access to interfaces
|
||||
"""
|
||||
self.services = services
|
||||
self._tokenManager = TokenManager()
|
||||
|
||||
def getFreshToken(self, connectionId: str, secondsBeforeExpiry: int = 30 * 60) -> Optional[Token]:
|
||||
"""Get a fresh token for a connection, refreshing when expiring soon.
|
||||
|
||||
Reads the latest stored token via interface layer, then
|
||||
uses ensureFreshToken to refresh if needed and persists the refreshed
|
||||
token via interface layer.
|
||||
|
||||
Args:
|
||||
connectionId: ID of the connection to get token for
|
||||
secondsBeforeExpiry: Threshold window to proactively refresh (default: 30 minutes)
|
||||
|
||||
Returns:
|
||||
Token object or None if not found/expired
|
||||
"""
|
||||
try:
|
||||
# Use interface from services instead of getRootInterface()
|
||||
interfaceDbApp = self.services.interfaceDbApp
|
||||
|
||||
token = interfaceDbApp.getConnectionToken(connectionId)
|
||||
if not token:
|
||||
return None
|
||||
|
||||
return self._tokenManager.ensureFreshToken(
|
||||
token,
|
||||
secondsBeforeExpiry=secondsBeforeExpiry,
|
||||
saveCallback=lambda t: interfaceDbApp.saveConnectionToken(t)
|
||||
)
|
||||
except Exception as e:
|
||||
logger.error(f"getFreshToken: Error fetching or refreshing token for connection {connectionId}: {e}")
|
||||
return None
|
||||
|
||||
def refreshToken(self, oldToken: Token) -> Optional[Token]:
|
||||
"""Refresh an expired token using the appropriate OAuth service.
|
||||
|
||||
Args:
|
||||
oldToken: Token object to refresh
|
||||
|
||||
Returns:
|
||||
Refreshed Token object or None if refresh failed
|
||||
"""
|
||||
try:
|
||||
return self._tokenManager.refreshToken(oldToken)
|
||||
except Exception as e:
|
||||
logger.error(f"refreshToken: Error refreshing token: {e}")
|
||||
return None
|
||||
|
||||
def ensureFreshToken(self, token: Token, *, secondsBeforeExpiry: int = 30 * 60,
|
||||
saveCallback: Optional[Callable[[Token], None]] = None) -> Optional[Token]:
|
||||
"""Ensure a token is fresh; refresh if expiring within threshold.
|
||||
|
||||
Args:
|
||||
token: Existing token to validate/refresh
|
||||
secondsBeforeExpiry: Threshold window to proactively refresh (default: 30 minutes)
|
||||
saveCallback: Optional function to persist a refreshed token
|
||||
|
||||
Returns:
|
||||
A fresh token (refreshed or original) or None if refresh failed
|
||||
"""
|
||||
try:
|
||||
return self._tokenManager.ensureFreshToken(
|
||||
token,
|
||||
secondsBeforeExpiry=secondsBeforeExpiry,
|
||||
saveCallback=saveCallback
|
||||
)
|
||||
except Exception as e:
|
||||
logger.error(f"ensureFreshToken: Error ensuring fresh token: {e}")
|
||||
return None
|
||||
|
||||
def refreshMicrosoftToken(self, refreshToken: str, userId: str, oldToken: Token) -> Optional[Token]:
|
||||
"""Refresh Microsoft OAuth token using refresh token.
|
||||
|
||||
Args:
|
||||
refreshToken: Microsoft refresh token
|
||||
userId: User ID owning the token
|
||||
oldToken: Previous token object to preserve connection ID
|
||||
|
||||
Returns:
|
||||
New Token object or None if refresh failed
|
||||
"""
|
||||
try:
|
||||
return self._tokenManager.refreshMicrosoftToken(refreshToken, userId, oldToken)
|
||||
except Exception as e:
|
||||
logger.error(f"refreshMicrosoftToken: Error refreshing Microsoft token: {e}")
|
||||
return None
|
||||
|
||||
def refreshGoogleToken(self, refreshToken: str, userId: str, oldToken: Token) -> Optional[Token]:
|
||||
"""Refresh Google OAuth token using refresh token.
|
||||
|
||||
Args:
|
||||
refreshToken: Google refresh token
|
||||
userId: User ID owning the token
|
||||
oldToken: Previous token object to preserve connection ID
|
||||
|
||||
Returns:
|
||||
New Token object or None if refresh failed
|
||||
"""
|
||||
try:
|
||||
return self._tokenManager.refreshGoogleToken(refreshToken, userId, oldToken)
|
||||
except Exception as e:
|
||||
logger.error(f"refreshGoogleToken: Error refreshing Google token: {e}")
|
||||
return None
|
||||
|
||||
|
|
@ -47,9 +47,12 @@ class SharepointService:
|
|||
logger.error("UserConnection must have an 'id' field")
|
||||
return False
|
||||
|
||||
# Get a fresh token for this specific connection
|
||||
from modules.security.tokenManager import TokenManager
|
||||
token = TokenManager().getFreshToken(connectionId)
|
||||
# Get a fresh token for this specific connection via security service
|
||||
if not self.services:
|
||||
logger.error("Service center not available for token access")
|
||||
return False
|
||||
|
||||
token = self.services.security.getFreshToken(connectionId)
|
||||
if not token:
|
||||
logger.error(f"No token found for connection {connectionId}")
|
||||
return False
|
||||
|
|
|
|||
|
|
@ -155,11 +155,11 @@ class UtilsService:
|
|||
|
||||
def storeDebugMessageAndDocuments(self, message, currentUser):
|
||||
"""
|
||||
Wrapper to store debug messages and documents via shared debugLogger.
|
||||
Mirrors storeDebugMessageAndDocuments() in modules.shared.debugLogger.
|
||||
Wrapper to store debug messages and documents via interfaceDbChatObjects.
|
||||
Mirrors storeDebugMessageAndDocuments() in modules.interfaces.interfaceDbChatObjects.
|
||||
"""
|
||||
try:
|
||||
from modules.shared.debugLogger import storeDebugMessageAndDocuments as _storeDebugMessageAndDocuments
|
||||
from modules.interfaces.interfaceDbChatObjects import storeDebugMessageAndDocuments as _storeDebugMessageAndDocuments
|
||||
_storeDebugMessageAndDocuments(message, currentUser)
|
||||
except Exception:
|
||||
# Silent fail to never break main flow
|
||||
|
|
|
|||
70
modules/shared/callbackRegistry.py
Normal file
70
modules/shared/callbackRegistry.py
Normal file
|
|
@ -0,0 +1,70 @@
|
|||
"""
|
||||
Callback registry for decoupled event notifications.
|
||||
|
||||
Allows interfaces to notify about changes without knowing about features.
|
||||
Features can register callbacks to be notified when automations change.
|
||||
"""
|
||||
|
||||
import logging
|
||||
from typing import Callable, List, Dict, Any
|
||||
import asyncio
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class CallbackRegistry:
|
||||
"""Registry for callbacks that can be triggered by interfaces without knowing about features."""
|
||||
|
||||
def __init__(self):
|
||||
self._callbacks: Dict[str, List[Callable]] = {}
|
||||
|
||||
def register(self, event_type: str, callback: Callable):
|
||||
"""Register a callback for a specific event type.
|
||||
|
||||
Args:
|
||||
event_type: Type of event (e.g., 'automation.changed')
|
||||
callback: Async or sync callback function
|
||||
"""
|
||||
if event_type not in self._callbacks:
|
||||
self._callbacks[event_type] = []
|
||||
self._callbacks[event_type].append(callback)
|
||||
logger.debug(f"Registered callback for event type: {event_type}")
|
||||
|
||||
def unregister(self, event_type: str, callback: Callable):
|
||||
"""Unregister a callback for a specific event type."""
|
||||
if event_type in self._callbacks:
|
||||
try:
|
||||
self._callbacks[event_type].remove(callback)
|
||||
logger.debug(f"Unregistered callback for event type: {event_type}")
|
||||
except ValueError:
|
||||
logger.warning(f"Callback not found for event type: {event_type}")
|
||||
|
||||
async def trigger(self, event_type: str, *args, **kwargs):
|
||||
"""Trigger all callbacks registered for an event type.
|
||||
|
||||
Args:
|
||||
event_type: Type of event to trigger
|
||||
*args, **kwargs: Arguments to pass to callbacks
|
||||
"""
|
||||
if event_type not in self._callbacks:
|
||||
return
|
||||
|
||||
callbacks = self._callbacks[event_type].copy() # Copy to avoid modification during iteration
|
||||
|
||||
for callback in callbacks:
|
||||
try:
|
||||
if asyncio.iscoroutinefunction(callback):
|
||||
await callback(*args, **kwargs)
|
||||
else:
|
||||
callback(*args, **kwargs)
|
||||
except Exception as e:
|
||||
logger.error(f"Error executing callback for {event_type}: {str(e)}", exc_info=True)
|
||||
|
||||
def has_callbacks(self, event_type: str) -> bool:
|
||||
"""Check if there are any callbacks registered for an event type."""
|
||||
return event_type in self._callbacks and len(self._callbacks[event_type]) > 0
|
||||
|
||||
|
||||
# Global singleton instance
|
||||
callbackRegistry = CallbackRegistry()
|
||||
|
||||
|
|
@ -145,131 +145,3 @@ def debugLogToFile(message: str, context: str = "DEBUG") -> None:
|
|||
# Don't log debug errors to avoid recursion
|
||||
pass
|
||||
|
||||
def storeDebugMessageAndDocuments(message, currentUser) -> None:
|
||||
"""
|
||||
Store message and documents (metadata and file bytes) for debugging purposes.
|
||||
Structure: {log_dir}/debug/messages/m_round_task_action_timestamp/documentlist_label/
|
||||
- message.json, message_text.txt
|
||||
- document_###_metadata.json
|
||||
- document_###_<original_filename> (actual file bytes)
|
||||
|
||||
Args:
|
||||
message: ChatMessage object to store
|
||||
currentUser: Current user for component interface access
|
||||
"""
|
||||
try:
|
||||
import json
|
||||
|
||||
# Create base debug directory (use base debug dir, not prompts subdirectory)
|
||||
baseDebugDir = _getBaseDebugDir()
|
||||
debug_root = os.path.join(baseDebugDir, 'messages')
|
||||
_ensureDir(debug_root)
|
||||
|
||||
# Generate timestamp
|
||||
timestamp = datetime.now(UTC).strftime('%Y%m%d-%H%M%S-%f')[:-3]
|
||||
|
||||
# Create message folder name: m_round_task_action_timestamp
|
||||
# Use actual values from message, not defaults
|
||||
round_str = str(message.roundNumber) if message.roundNumber is not None else "0"
|
||||
task_str = str(message.taskNumber) if message.taskNumber is not None else "0"
|
||||
action_str = str(message.actionNumber) if message.actionNumber is not None else "0"
|
||||
message_folder = f"{timestamp}_m_{round_str}_{task_str}_{action_str}"
|
||||
|
||||
message_path = os.path.join(debug_root, message_folder)
|
||||
os.makedirs(message_path, exist_ok=True)
|
||||
|
||||
# Store message data - use dict() instead of model_dump() for compatibility
|
||||
message_file = os.path.join(message_path, "message.json")
|
||||
with open(message_file, "w", encoding="utf-8") as f:
|
||||
# Convert message to dict manually to avoid model_dump() issues
|
||||
message_dict = {
|
||||
"id": message.id,
|
||||
"workflowId": message.workflowId,
|
||||
"parentMessageId": message.parentMessageId,
|
||||
"message": message.message,
|
||||
"role": message.role,
|
||||
"status": message.status,
|
||||
"sequenceNr": message.sequenceNr,
|
||||
"publishedAt": message.publishedAt,
|
||||
"roundNumber": message.roundNumber,
|
||||
"taskNumber": message.taskNumber,
|
||||
"actionNumber": message.actionNumber,
|
||||
"documentsLabel": message.documentsLabel,
|
||||
"actionId": message.actionId,
|
||||
"actionMethod": message.actionMethod,
|
||||
"actionName": message.actionName,
|
||||
"success": message.success,
|
||||
"documents": []
|
||||
}
|
||||
json.dump(message_dict, f, indent=2, ensure_ascii=False, default=str)
|
||||
|
||||
# Store message content as text
|
||||
if message.message:
|
||||
message_text_file = os.path.join(message_path, "message_text.txt")
|
||||
with open(message_text_file, "w", encoding="utf-8") as f:
|
||||
f.write(str(message.message))
|
||||
|
||||
# Store documents if provided
|
||||
if message.documents and len(message.documents) > 0:
|
||||
# Group documents by documentsLabel
|
||||
documents_by_label = {}
|
||||
for doc in message.documents:
|
||||
label = message.documentsLabel or 'default'
|
||||
if label not in documents_by_label:
|
||||
documents_by_label[label] = []
|
||||
documents_by_label[label].append(doc)
|
||||
|
||||
# Create subfolder for each document label
|
||||
for label, docs in documents_by_label.items():
|
||||
# Sanitize label for filesystem
|
||||
safe_label = "".join(c for c in str(label) if c.isalnum() or c in (' ', '-', '_')).rstrip()
|
||||
safe_label = safe_label.replace(' ', '_')
|
||||
if not safe_label:
|
||||
safe_label = "default"
|
||||
|
||||
label_folder = os.path.join(message_path, safe_label)
|
||||
_ensureDir(label_folder)
|
||||
|
||||
# Store each document
|
||||
for i, doc in enumerate(docs):
|
||||
# Create document metadata file
|
||||
doc_meta = {
|
||||
"id": doc.id,
|
||||
"messageId": doc.messageId,
|
||||
"fileId": doc.fileId,
|
||||
"fileName": doc.fileName,
|
||||
"fileSize": doc.fileSize,
|
||||
"mimeType": doc.mimeType,
|
||||
"roundNumber": doc.roundNumber,
|
||||
"taskNumber": doc.taskNumber,
|
||||
"actionNumber": doc.actionNumber,
|
||||
"actionId": doc.actionId
|
||||
}
|
||||
|
||||
doc_meta_file = os.path.join(label_folder, f"document_{i+1:03d}_metadata.json")
|
||||
with open(doc_meta_file, "w", encoding="utf-8") as f:
|
||||
json.dump(doc_meta, f, indent=2, ensure_ascii=False, default=str)
|
||||
|
||||
# Also store the actual file bytes next to metadata for debugging
|
||||
try:
|
||||
# Lazy import to avoid circular deps at module load
|
||||
from modules.interfaces import interfaceDbComponentObjects as comp
|
||||
componentInterface = comp.getInterface(currentUser)
|
||||
file_bytes = componentInterface.getFileData(doc.fileId)
|
||||
if file_bytes:
|
||||
# Build a safe filename preserving original name
|
||||
safe_name = doc.fileName or f"document_{i+1:03d}"
|
||||
# Avoid path traversal
|
||||
safe_name = os.path.basename(safe_name)
|
||||
doc_file_path = os.path.join(label_folder, f"document_{i+1:03d}_" + safe_name)
|
||||
with open(doc_file_path, "wb") as df:
|
||||
df.write(file_bytes)
|
||||
else:
|
||||
pass
|
||||
except Exception as e:
|
||||
pass
|
||||
|
||||
except Exception as e:
|
||||
# Silent fail - don't break main flow
|
||||
pass
|
||||
|
||||
|
|
|
|||
|
|
@ -214,138 +214,6 @@ class MethodAi(MethodBase):
|
|||
)
|
||||
|
||||
|
||||
@action
|
||||
async def extractContent(self, parameters: Dict[str, Any]) -> ActionResult:
|
||||
"""
|
||||
Extract content from documents (separate from AI calls).
|
||||
|
||||
This action performs pure content extraction without AI processing.
|
||||
The extracted ContentParts can then be used by subsequent AI processing actions.
|
||||
|
||||
Parameters:
|
||||
- documentList (list, required): Document reference(s) to extract content from.
|
||||
- extractionOptions (dict, optional): Extraction options (if not provided, defaults are used).
|
||||
|
||||
Returns:
|
||||
- ActionResult with ActionDocument containing ContentExtracted objects
|
||||
- ContentExtracted.parts contains List[ContentPart] (already chunked if needed)
|
||||
"""
|
||||
try:
|
||||
# Init progress logger
|
||||
workflowId = self.services.workflow.id if self.services.workflow else f"no-workflow-{int(time.time())}"
|
||||
operationId = f"ai_extract_{workflowId}_{int(time.time())}"
|
||||
|
||||
# Extract documentList from parameters dict
|
||||
from modules.datamodels.datamodelDocref import DocumentReferenceList
|
||||
documentListParam = parameters.get("documentList")
|
||||
if not documentListParam:
|
||||
return ActionResult.isFailure(error="documentList is required")
|
||||
|
||||
# Convert to DocumentReferenceList if needed
|
||||
if isinstance(documentListParam, DocumentReferenceList):
|
||||
documentList = documentListParam
|
||||
elif isinstance(documentListParam, str):
|
||||
documentList = DocumentReferenceList.from_string_list([documentListParam])
|
||||
elif isinstance(documentListParam, list):
|
||||
documentList = DocumentReferenceList.from_string_list(documentListParam)
|
||||
else:
|
||||
return ActionResult.isFailure(error=f"Invalid documentList type: {type(documentListParam)}")
|
||||
|
||||
# Start progress tracking
|
||||
self.services.chat.progressLogStart(
|
||||
operationId,
|
||||
"Extracting content from documents",
|
||||
"Content Extraction",
|
||||
f"Documents: {len(documentList.references)}"
|
||||
)
|
||||
|
||||
# Get ChatDocuments from documentList
|
||||
self.services.chat.progressLogUpdate(operationId, 0.2, "Loading documents")
|
||||
chatDocuments = self.services.chat.getChatDocumentsFromDocumentList(documentList)
|
||||
|
||||
if not chatDocuments:
|
||||
self.services.chat.progressLogFinish(operationId, False)
|
||||
return ActionResult.isFailure(error="No documents found in documentList")
|
||||
|
||||
logger.info(f"Extracting content from {len(chatDocuments)} documents")
|
||||
|
||||
# Prepare extraction options
|
||||
self.services.chat.progressLogUpdate(operationId, 0.3, "Preparing extraction options")
|
||||
extractionOptionsParam = parameters.get("extractionOptions")
|
||||
|
||||
# Convert dict to ExtractionOptions object if needed, or create defaults
|
||||
if extractionOptionsParam:
|
||||
if isinstance(extractionOptionsParam, dict):
|
||||
# Convert dict to ExtractionOptions object
|
||||
extractionOptions = ExtractionOptions(**extractionOptionsParam)
|
||||
elif isinstance(extractionOptionsParam, ExtractionOptions):
|
||||
extractionOptions = extractionOptionsParam
|
||||
else:
|
||||
# Invalid type, use defaults
|
||||
extractionOptions = None
|
||||
else:
|
||||
extractionOptions = None
|
||||
|
||||
# If extractionOptions not provided, create defaults
|
||||
if not extractionOptions:
|
||||
# Default extraction options for pure content extraction (no AI processing)
|
||||
extractionOptions = ExtractionOptions(
|
||||
prompt="Extract all content from the document",
|
||||
mergeStrategy=MergeStrategy(
|
||||
mergeType="concatenate",
|
||||
groupBy="typeGroup",
|
||||
orderBy="id"
|
||||
),
|
||||
processDocumentsIndividually=True
|
||||
)
|
||||
|
||||
# Get parent log ID for document-level operations
|
||||
parentLogId = self.services.chat.getOperationLogId(operationId)
|
||||
|
||||
# Call extraction service
|
||||
self.services.chat.progressLogUpdate(operationId, 0.4, "Initiating")
|
||||
self.services.chat.progressLogUpdate(operationId, 0.5, f"Extracting content from {len(chatDocuments)} documents")
|
||||
extractedResults = self.services.extraction.extractContent(chatDocuments, extractionOptions)
|
||||
|
||||
# Build ActionDocuments from ContentExtracted results
|
||||
self.services.chat.progressLogUpdate(operationId, 0.8, "Building result documents")
|
||||
actionDocuments = []
|
||||
# Map extracted results back to original documents by index (results are in same order)
|
||||
for i, extracted in enumerate(extractedResults):
|
||||
# Get original document name if available
|
||||
originalDoc = chatDocuments[i] if i < len(chatDocuments) else None
|
||||
if originalDoc and hasattr(originalDoc, 'fileName') and originalDoc.fileName:
|
||||
# Use original filename with "extracted_" prefix
|
||||
baseName = originalDoc.fileName.rsplit('.', 1)[0] if '.' in originalDoc.fileName else originalDoc.fileName
|
||||
documentName = f"{baseName}_extracted_{extracted.id}.json"
|
||||
else:
|
||||
# Fallback to generic name with index
|
||||
documentName = f"document_{i+1:03d}_extracted_{extracted.id}.json"
|
||||
|
||||
# Store ContentExtracted object in ActionDocument.documentData
|
||||
actionDoc = ActionDocument(
|
||||
documentName=documentName,
|
||||
documentData=extracted, # ContentExtracted object
|
||||
mimeType="application/json"
|
||||
)
|
||||
actionDocuments.append(actionDoc)
|
||||
|
||||
self.services.chat.progressLogFinish(operationId, True)
|
||||
|
||||
return ActionResult.isSuccess(documents=actionDocuments)
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Error in content extraction: {str(e)}")
|
||||
|
||||
# Complete progress tracking with failure
|
||||
try:
|
||||
self.services.chat.progressLogFinish(operationId, False)
|
||||
except:
|
||||
pass # Don't fail on progress logging errors
|
||||
|
||||
return ActionResult.isFailure(error=str(e))
|
||||
|
||||
|
||||
@action
|
||||
async def webResearch(self, parameters: Dict[str, Any]) -> ActionResult:
|
||||
"""
|
||||
|
|
@ -707,171 +575,6 @@ class MethodAi(MethodBase):
|
|||
return output.getvalue()
|
||||
|
||||
|
||||
@action
|
||||
async def reformat(self, parameters: Dict[str, Any]) -> ActionResult:
|
||||
"""
|
||||
GENERAL:
|
||||
- Purpose: Reformat/transform documents with specific transformation rules (e.g., extract arrays, reshape data, apply custom formatting).
|
||||
- Input requirements: documentList (required); inputFormat and outputFormat (required); transformationRules (optional).
|
||||
- Output format: Document in target format with applied transformation rules.
|
||||
- CRITICAL: If input is already in standardized JSON format, uses automatic rendering system with transformation rules.
|
||||
|
||||
Parameters:
|
||||
- documentList (list, required): Document reference(s) to reformat.
|
||||
- inputFormat (str, required): Source format (json, csv, xlsx, txt, etc.).
|
||||
- outputFormat (str, required): Target format (csv, json, xlsx, txt, etc.).
|
||||
- transformationRules (str, optional): Specific transformation instructions (e.g., "Extract prime numbers array and format as CSV with 10 columns per row").
|
||||
- columnsPerRow (int, optional): For CSV output, number of columns per row. Default: auto-detect.
|
||||
- totalRows (int, optional): For CSV output, total number of rows to create. Default: auto-detect.
|
||||
- delimiter (str, optional): For CSV output, delimiter character. Default: comma (,).
|
||||
- includeHeader (bool, optional): For CSV output, whether to include header row. Default: True.
|
||||
- language (str, optional): Language for output (e.g., 'de', 'en', 'fr'). Default: 'en'.
|
||||
"""
|
||||
documentList = parameters.get("documentList", [])
|
||||
if not documentList:
|
||||
return ActionResult.isFailure(error="documentList is required")
|
||||
|
||||
inputFormat = parameters.get("inputFormat")
|
||||
outputFormat = parameters.get("outputFormat")
|
||||
if not inputFormat or not outputFormat:
|
||||
return ActionResult.isFailure(error="inputFormat and outputFormat are required")
|
||||
|
||||
transformationRules = parameters.get("transformationRules")
|
||||
columnsPerRow = parameters.get("columnsPerRow")
|
||||
totalRows = parameters.get("totalRows")
|
||||
delimiter = parameters.get("delimiter", ",")
|
||||
includeHeader = parameters.get("includeHeader", True)
|
||||
language = parameters.get("language", "en")
|
||||
|
||||
# Normalize formats (remove leading dot if present)
|
||||
normalizedInputFormat = inputFormat.strip().lstrip('.').lower()
|
||||
normalizedOutputFormat = outputFormat.strip().lstrip('.').lower()
|
||||
|
||||
# Get documents
|
||||
from modules.datamodels.datamodelDocref import DocumentReferenceList
|
||||
if isinstance(documentList, DocumentReferenceList):
|
||||
docRefList = documentList
|
||||
elif isinstance(documentList, list):
|
||||
docRefList = DocumentReferenceList.from_string_list(documentList)
|
||||
else:
|
||||
docRefList = DocumentReferenceList.from_string_list([documentList])
|
||||
|
||||
chatDocuments = self.services.chat.getChatDocumentsFromDocumentList(docRefList)
|
||||
if not chatDocuments:
|
||||
return ActionResult.isFailure(error="No documents found in documentList")
|
||||
|
||||
# Check if input is standardized JSON format - if so, use direct rendering with transformation
|
||||
if normalizedInputFormat == "json" and len(chatDocuments) == 1:
|
||||
try:
|
||||
import json
|
||||
doc = chatDocuments[0]
|
||||
# ChatDocument doesn't have documentData - need to load file content using fileId
|
||||
docBytes = self.services.chat.getFileData(doc.fileId)
|
||||
if not docBytes:
|
||||
raise ValueError(f"No file data found for fileId={doc.fileId}")
|
||||
|
||||
# Decode bytes to string
|
||||
docData = docBytes.decode('utf-8')
|
||||
|
||||
# Try to parse as JSON
|
||||
if isinstance(docData, str):
|
||||
jsonData = json.loads(docData)
|
||||
elif isinstance(docData, dict):
|
||||
jsonData = docData
|
||||
else:
|
||||
jsonData = None
|
||||
|
||||
# Check if it's standardized JSON format (has "documents" or "sections")
|
||||
if jsonData and (isinstance(jsonData, dict) and ("documents" in jsonData or "sections" in jsonData)):
|
||||
# Apply transformation rules if provided
|
||||
if transformationRules:
|
||||
# Use AI to apply transformation rules to JSON
|
||||
aiPrompt = f"Apply the following transformation rules to the JSON document: {transformationRules}"
|
||||
if normalizedOutputFormat == "csv":
|
||||
aiPrompt += f" Output format: CSV with delimiter '{delimiter}'"
|
||||
if columnsPerRow:
|
||||
aiPrompt += f", {columnsPerRow} columns per row"
|
||||
if totalRows:
|
||||
aiPrompt += f", {totalRows} total rows"
|
||||
if not includeHeader:
|
||||
aiPrompt += ", no header row"
|
||||
|
||||
# Use process to apply transformation
|
||||
return await self.process({
|
||||
"aiPrompt": aiPrompt,
|
||||
"documentList": documentList,
|
||||
"resultType": normalizedOutputFormat
|
||||
})
|
||||
else:
|
||||
# No transformation rules - use direct rendering
|
||||
from modules.services.serviceGeneration.mainServiceGeneration import GenerationService
|
||||
generationService = GenerationService(self.services)
|
||||
|
||||
# Ensure format is "documents" array
|
||||
if "documents" not in jsonData:
|
||||
jsonData = {"documents": [{"sections": jsonData.get("sections", []), "metadata": jsonData.get("metadata", {})}]}
|
||||
|
||||
# Get title
|
||||
title = jsonData.get("metadata", {}).get("title", doc.documentName or "Reformatted Document")
|
||||
|
||||
# Render with options
|
||||
renderOptions = {}
|
||||
if normalizedOutputFormat == "csv":
|
||||
renderOptions["delimiter"] = delimiter
|
||||
renderOptions["columnsPerRow"] = columnsPerRow
|
||||
renderOptions["includeHeader"] = includeHeader
|
||||
|
||||
rendered_content, mime_type = await generationService.renderReport(
|
||||
jsonData, normalizedOutputFormat, title, None, None
|
||||
)
|
||||
|
||||
# Apply CSV options if needed
|
||||
if normalizedOutputFormat == "csv" and renderOptions:
|
||||
rendered_content = self._applyCsvOptions(rendered_content, renderOptions)
|
||||
|
||||
from modules.datamodels.datamodelChat import ActionDocument
|
||||
actionDoc = ActionDocument(
|
||||
documentName=f"{doc.documentName.rsplit('.', 1)[0] if '.' in doc.documentName else doc.documentName}.{normalizedOutputFormat}",
|
||||
documentData=rendered_content,
|
||||
mimeType=mime_type,
|
||||
sourceJson=jsonData # Preserve source JSON for structure validation
|
||||
)
|
||||
|
||||
return ActionResult.isSuccess(documents=[actionDoc])
|
||||
|
||||
except Exception as e:
|
||||
logger.warning(f"Direct rendering failed, falling back to AI reformatting: {str(e)}")
|
||||
# Fall through to AI-based reformatting
|
||||
|
||||
# Fallback: Use AI for reformatting with transformation rules
|
||||
aiPrompt = f"Reformat the provided document(s) from {normalizedInputFormat.upper()} format to {normalizedOutputFormat.upper()} format."
|
||||
|
||||
if transformationRules:
|
||||
aiPrompt += f" Apply the following transformation rules: {transformationRules}"
|
||||
|
||||
if normalizedOutputFormat == "csv":
|
||||
aiPrompt += f" Use '{delimiter}' as the delimiter character."
|
||||
if columnsPerRow:
|
||||
aiPrompt += f" Format the output with {columnsPerRow} columns per row."
|
||||
if totalRows:
|
||||
aiPrompt += f" Create exactly {totalRows} rows total."
|
||||
if not includeHeader:
|
||||
aiPrompt += " Do not include a header row."
|
||||
else:
|
||||
aiPrompt += " Include a header row with column names."
|
||||
|
||||
if language and language != "en":
|
||||
aiPrompt += f" Use language: {language}."
|
||||
|
||||
aiPrompt += " Preserve all data and ensure accurate transformation. Maintain data integrity."
|
||||
|
||||
return await self.process({
|
||||
"aiPrompt": aiPrompt,
|
||||
"documentList": documentList,
|
||||
"resultType": normalizedOutputFormat
|
||||
})
|
||||
|
||||
|
||||
@action
|
||||
async def convertDocument(self, parameters: Dict[str, Any]) -> ActionResult:
|
||||
"""
|
||||
|
|
@ -955,160 +658,10 @@ class MethodAi(MethodBase):
|
|||
})
|
||||
|
||||
|
||||
@action
|
||||
async def extractTables(self, parameters: Dict[str, Any]) -> ActionResult:
|
||||
"""
|
||||
GENERAL:
|
||||
- Purpose: Extract tables from documents, preserving structure and data.
|
||||
- Input requirements: documentList (required); optional tableFormat.
|
||||
- Output format: JSON by default (structured table data), or CSV/XLSX if specified.
|
||||
|
||||
Parameters:
|
||||
- documentList (list, required): Document reference(s) to extract tables from.
|
||||
- tableFormat (str, optional): Output format for tables - json, csv, or xlsx. Default: json.
|
||||
- includeHeaders (bool, optional): Include table headers. Default: True.
|
||||
"""
|
||||
documentList = parameters.get("documentList", [])
|
||||
if not documentList:
|
||||
return ActionResult.isFailure(error="documentList is required")
|
||||
|
||||
tableFormat = parameters.get("tableFormat", "json")
|
||||
includeHeaders = parameters.get("includeHeaders", True)
|
||||
|
||||
# Map tableFormat to resultType
|
||||
formatMap = {
|
||||
"json": "json",
|
||||
"csv": "csv",
|
||||
"xlsx": "xlsx",
|
||||
"xls": "xlsx"
|
||||
}
|
||||
resultType = formatMap.get(tableFormat.lower(), "json")
|
||||
|
||||
aiPrompt = "Extract all tables from the provided document(s)."
|
||||
if includeHeaders:
|
||||
aiPrompt += " Include table headers and preserve the table structure."
|
||||
else:
|
||||
aiPrompt += " Extract table data without headers."
|
||||
aiPrompt += " Maintain accurate data types (numbers as numbers, dates as dates, etc.) and preserve all table relationships."
|
||||
|
||||
if resultType == "json":
|
||||
aiPrompt += " Structure each table as a JSON object with headers and rows as arrays."
|
||||
elif resultType == "csv":
|
||||
aiPrompt += " Output each table as CSV format with proper comma separation."
|
||||
elif resultType == "xlsx":
|
||||
aiPrompt += " Structure the output as an Excel spreadsheet with tables properly formatted."
|
||||
|
||||
return await self.process({
|
||||
"aiPrompt": aiPrompt,
|
||||
"documentList": documentList,
|
||||
"resultType": resultType
|
||||
})
|
||||
|
||||
|
||||
# ============================================================================
|
||||
# Content Generation Wrappers
|
||||
# Content Generation Wrapper
|
||||
# ============================================================================
|
||||
|
||||
@action
|
||||
async def generateReport(self, parameters: Dict[str, Any]) -> ActionResult:
|
||||
"""
|
||||
GENERAL:
|
||||
- Purpose: Generate comprehensive reports from input documents/data with analysis and insights.
|
||||
- Input requirements: documentList (optional, can generate from scratch); optional reportType, sections.
|
||||
- Output format: Document in specified format (default: docx).
|
||||
|
||||
Parameters:
|
||||
- documentList (list, optional): Input documents/data to base the report on.
|
||||
- reportType (str, optional): Type of report - summary, analysis, executive, detailed. Default: analysis.
|
||||
- sections (list, optional): Specific sections to include (e.g., ["introduction", "findings", "recommendations"]).
|
||||
- title (str, optional): Report title.
|
||||
- resultType (str, optional): Output format (docx, pdf, md, etc.). Default: docx.
|
||||
"""
|
||||
documentList = parameters.get("documentList", [])
|
||||
reportType = parameters.get("reportType", "analysis")
|
||||
sections = parameters.get("sections", [])
|
||||
title = parameters.get("title")
|
||||
resultType = parameters.get("resultType", "docx")
|
||||
|
||||
reportTypeInstructions = {
|
||||
"summary": "Create a summary report with key highlights and main points.",
|
||||
"analysis": "Create an analytical report with insights, findings, and detailed examination.",
|
||||
"executive": "Create an executive summary report suitable for senior management with key insights and recommendations.",
|
||||
"detailed": "Create a comprehensive detailed report covering all aspects with in-depth analysis."
|
||||
}
|
||||
|
||||
aiPrompt = f"Generate a {reportType} report."
|
||||
if title:
|
||||
aiPrompt += f" Title: {title}."
|
||||
aiPrompt += f" {reportTypeInstructions.get(reportType.lower(), reportTypeInstructions['analysis'])}"
|
||||
|
||||
if sections:
|
||||
sectionsStr = ", ".join(sections)
|
||||
aiPrompt += f" Include the following sections: {sectionsStr}."
|
||||
else:
|
||||
aiPrompt += " Include standard report sections such as introduction, main content, analysis, findings, and conclusions."
|
||||
|
||||
if documentList:
|
||||
aiPrompt += " Base the report on the provided input documents, analyzing and synthesizing the information."
|
||||
else:
|
||||
aiPrompt += " Create a professional, well-structured report."
|
||||
|
||||
processParams = {
|
||||
"aiPrompt": aiPrompt,
|
||||
"resultType": resultType
|
||||
}
|
||||
if documentList:
|
||||
processParams["documentList"] = documentList
|
||||
|
||||
return await self.process(processParams)
|
||||
|
||||
|
||||
@action
|
||||
async def generateChart(self, parameters: Dict[str, Any]) -> ActionResult:
|
||||
"""
|
||||
GENERAL:
|
||||
- Purpose: Generate charts/graphs from data in documents or structured data.
|
||||
- Input requirements: documentList (required); optional chartType, title, labels.
|
||||
- Output format: Image (png or jpg).
|
||||
|
||||
Parameters:
|
||||
- documentList (list, required): Documents containing data to visualize (CSV, Excel, JSON, etc.).
|
||||
- chartType (str, optional): Type of chart - bar, line, pie, scatter, area, etc. Default: bar.
|
||||
- title (str, optional): Chart title.
|
||||
- xAxisLabel (str, optional): X-axis label.
|
||||
- yAxisLabel (str, optional): Y-axis label.
|
||||
- resultType (str, optional): Image format (png or jpg). Default: png.
|
||||
"""
|
||||
documentList = parameters.get("documentList", [])
|
||||
if not documentList:
|
||||
return ActionResult.isFailure(error="documentList is required")
|
||||
|
||||
chartType = parameters.get("chartType", "bar")
|
||||
title = parameters.get("title")
|
||||
xAxisLabel = parameters.get("xAxisLabel")
|
||||
yAxisLabel = parameters.get("yAxisLabel")
|
||||
resultType = parameters.get("resultType", "png")
|
||||
|
||||
# Ensure resultType is an image format
|
||||
if resultType.lower() not in ["png", "jpg", "jpeg"]:
|
||||
resultType = "png"
|
||||
|
||||
aiPrompt = f"Generate a {chartType} chart from the provided data."
|
||||
if title:
|
||||
aiPrompt += f" Chart title: {title}."
|
||||
if xAxisLabel:
|
||||
aiPrompt += f" X-axis label: {xAxisLabel}."
|
||||
if yAxisLabel:
|
||||
aiPrompt += f" Y-axis label: {yAxisLabel}."
|
||||
aiPrompt += " Create a clear, professional chart with appropriate labels, legends, and formatting. Ensure the chart is visually appealing and easy to read."
|
||||
|
||||
return await self.process({
|
||||
"aiPrompt": aiPrompt,
|
||||
"documentList": documentList,
|
||||
"resultType": resultType
|
||||
})
|
||||
|
||||
|
||||
@action
|
||||
async def generateDocument(self, parameters: Dict[str, Any]) -> ActionResult:
|
||||
"""
|
||||
|
|
@ -1146,137 +699,3 @@ class MethodAi(MethodBase):
|
|||
processParams["documentList"] = documentList
|
||||
|
||||
return await self.process(processParams)
|
||||
|
||||
|
||||
# ============================================================================
|
||||
# Analysis & Comparison Wrappers
|
||||
# ============================================================================
|
||||
|
||||
@action
|
||||
async def analyzeDocuments(self, parameters: Dict[str, Any]) -> ActionResult:
|
||||
"""
|
||||
GENERAL:
|
||||
- Purpose: Analyze documents and find insights, patterns, trends, and key information.
|
||||
- Input requirements: documentList (required); optional analysisType, focus.
|
||||
- Output format: Analysis report in specified format (default: txt).
|
||||
|
||||
Parameters:
|
||||
- documentList (list, required): Document(s) to analyze.
|
||||
- analysisType (str, optional): Type of analysis - general, financial, technical, sentiment, etc. Default: general.
|
||||
- focus (str, optional): Specific aspect to focus on (e.g., "trends", "risks", "opportunities").
|
||||
- resultType (str, optional): Output format (txt, md, docx, json, etc.). Default: txt.
|
||||
"""
|
||||
documentList = parameters.get("documentList", [])
|
||||
if not documentList:
|
||||
return ActionResult.isFailure(error="documentList is required")
|
||||
|
||||
analysisType = parameters.get("analysisType", "general")
|
||||
focus = parameters.get("focus")
|
||||
resultType = parameters.get("resultType", "txt")
|
||||
|
||||
aiPrompt = f"Analyze the provided document(s) and find insights, patterns, and key information."
|
||||
aiPrompt += f" Perform a {analysisType} analysis."
|
||||
if focus:
|
||||
aiPrompt += f" Focus specifically on: {focus}."
|
||||
aiPrompt += " Identify trends, important findings, relationships, and provide actionable insights. Present the analysis in a clear, structured format."
|
||||
|
||||
return await self.process({
|
||||
"aiPrompt": aiPrompt,
|
||||
"documentList": documentList,
|
||||
"resultType": resultType
|
||||
})
|
||||
|
||||
|
||||
@action
|
||||
async def compareDocuments(self, parameters: Dict[str, Any]) -> ActionResult:
|
||||
"""
|
||||
GENERAL:
|
||||
- Purpose: Compare multiple documents and identify differences, similarities, and changes.
|
||||
- Input requirements: documentList (required, should contain 2+ documents); optional comparisonType, focus.
|
||||
- Output format: Comparison report in specified format (default: txt).
|
||||
|
||||
Parameters:
|
||||
- documentList (list, required): Two or more documents to compare.
|
||||
- comparisonType (str, optional): Type of comparison - differences, similarities, changes, full. Default: full.
|
||||
- focus (str, optional): Specific aspect to focus on (e.g., "content", "structure", "data", "formatting").
|
||||
- resultType (str, optional): Output format (txt, md, docx, json, etc.). Default: txt.
|
||||
"""
|
||||
documentList = parameters.get("documentList", [])
|
||||
if not documentList:
|
||||
return ActionResult.isFailure(error="documentList is required")
|
||||
|
||||
if isinstance(documentList, str):
|
||||
documentList = [documentList]
|
||||
|
||||
if len(documentList) < 2:
|
||||
return ActionResult.isFailure(error="At least 2 documents are required for comparison")
|
||||
|
||||
comparisonType = parameters.get("comparisonType", "full")
|
||||
focus = parameters.get("focus")
|
||||
resultType = parameters.get("resultType", "txt")
|
||||
|
||||
comparisonInstructions = {
|
||||
"differences": "Focus on identifying and highlighting all differences between the documents.",
|
||||
"similarities": "Focus on identifying commonalities, shared content, and similarities.",
|
||||
"changes": "Identify what has changed between versions, what was added, removed, or modified.",
|
||||
"full": "Provide a comprehensive comparison including both differences and similarities."
|
||||
}
|
||||
|
||||
aiPrompt = f"Compare the provided documents."
|
||||
aiPrompt += f" {comparisonInstructions.get(comparisonType.lower(), comparisonInstructions['full'])}"
|
||||
if focus:
|
||||
aiPrompt += f" Focus specifically on: {focus}."
|
||||
aiPrompt += " Present the comparison in a clear, structured format that makes differences and similarities easy to understand."
|
||||
|
||||
return await self.process({
|
||||
"aiPrompt": aiPrompt,
|
||||
"documentList": documentList,
|
||||
"resultType": resultType
|
||||
})
|
||||
|
||||
|
||||
@action
|
||||
async def validateData(self, parameters: Dict[str, Any]) -> ActionResult:
|
||||
"""
|
||||
GENERAL:
|
||||
- Purpose: Validate data quality, structure, completeness, and correctness in documents/data files.
|
||||
- Input requirements: documentList (required); optional validationRules, schema.
|
||||
- Output format: Validation report in JSON or text format (default: json).
|
||||
|
||||
Parameters:
|
||||
- documentList (list, required): Documents/data files to validate.
|
||||
- validationRules (list, optional): Specific validation rules to check (e.g., ["required_fields", "data_types", "ranges"]).
|
||||
- schema (dict, optional): Expected data schema/structure to validate against.
|
||||
- resultType (str, optional): Output format (json, txt, md, etc.). Default: json.
|
||||
"""
|
||||
documentList = parameters.get("documentList", [])
|
||||
if not documentList:
|
||||
return ActionResult.isFailure(error="documentList is required")
|
||||
|
||||
validationRules = parameters.get("validationRules", [])
|
||||
schema = parameters.get("schema")
|
||||
resultType = parameters.get("resultType", "json")
|
||||
|
||||
aiPrompt = "Validate the data quality, structure, completeness, and correctness in the provided documents."
|
||||
|
||||
if validationRules:
|
||||
rulesStr = ", ".join(validationRules)
|
||||
aiPrompt += f" Apply the following validation rules: {rulesStr}."
|
||||
else:
|
||||
aiPrompt += " Check for data completeness, correct data types, required fields, data consistency, and any anomalies or errors."
|
||||
|
||||
if schema:
|
||||
import json
|
||||
schemaStr = json.dumps(schema, indent=2)
|
||||
aiPrompt += f" Validate against the following expected schema: {schemaStr}."
|
||||
|
||||
if resultType == "json":
|
||||
aiPrompt += " Provide the validation results as structured JSON with validation status, errors, warnings, and details for each check."
|
||||
else:
|
||||
aiPrompt += " Provide a detailed validation report listing all findings, errors, warnings, and pass/fail status for each validation check."
|
||||
|
||||
return await self.process({
|
||||
"aiPrompt": aiPrompt,
|
||||
"documentList": documentList,
|
||||
"resultType": resultType
|
||||
})
|
||||
|
|
|
|||
337
modules/workflows/methods/methodContext.py
Normal file
337
modules/workflows/methods/methodContext.py
Normal file
|
|
@ -0,0 +1,337 @@
|
|||
"""
|
||||
Context and workflow information method module.
|
||||
Handles workflow context queries and document indexing.
|
||||
"""
|
||||
|
||||
import time
|
||||
import json
|
||||
import logging
|
||||
from typing import Dict, Any, List
|
||||
from datetime import datetime, UTC
|
||||
|
||||
from modules.workflows.methods.methodBase import MethodBase, action
|
||||
from modules.datamodels.datamodelChat import ActionResult, ActionDocument
|
||||
from modules.datamodels.datamodelExtraction import ExtractionOptions, MergeStrategy
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
class MethodContext(MethodBase):
|
||||
"""Context and workflow information methods."""
|
||||
|
||||
def __init__(self, services):
|
||||
super().__init__(services)
|
||||
self.name = "context"
|
||||
self.description = "Context and workflow information methods"
|
||||
|
||||
@action
|
||||
async def getDocumentIndex(self, parameters: Dict[str, Any]) -> ActionResult:
|
||||
"""
|
||||
GENERAL:
|
||||
- Purpose: Generate a comprehensive index of all documents available in the current workflow, including documents from all rounds and tasks.
|
||||
- Input requirements: No input documents required. Optional resultType parameter.
|
||||
- Output format: Structured document index in JSON format (default) or text format, listing all documents with their references, metadata, and organization by rounds/tasks.
|
||||
|
||||
Parameters:
|
||||
- resultType (str, optional): Output format (json, txt, md). Default: json.
|
||||
"""
|
||||
try:
|
||||
workflow = self.services.workflow
|
||||
if not workflow:
|
||||
return ActionResult.isFailure(
|
||||
error="No workflow available"
|
||||
)
|
||||
|
||||
resultType = parameters.get("resultType", "json").lower().strip().lstrip('.')
|
||||
|
||||
# Get available documents index from chat service
|
||||
documentsIndex = self.services.chat.getAvailableDocuments(workflow)
|
||||
|
||||
if not documentsIndex or documentsIndex == "No documents available" or documentsIndex == "NO DOCUMENTS AVAILABLE - This workflow has no documents to process.":
|
||||
# Return empty index structure
|
||||
if resultType == "json":
|
||||
indexData = {
|
||||
"workflowId": getattr(workflow, 'id', 'unknown'),
|
||||
"totalDocuments": 0,
|
||||
"rounds": [],
|
||||
"documentReferences": []
|
||||
}
|
||||
indexContent = json.dumps(indexData, indent=2, ensure_ascii=False)
|
||||
else:
|
||||
indexContent = "Document Index\n==============\n\nNo documents available in this workflow.\n"
|
||||
else:
|
||||
# Parse the document index string to extract structured information
|
||||
indexData = self._parseDocumentIndex(documentsIndex, workflow)
|
||||
|
||||
if resultType == "json":
|
||||
indexContent = json.dumps(indexData, indent=2, ensure_ascii=False)
|
||||
elif resultType == "md":
|
||||
indexContent = self._formatAsMarkdown(indexData)
|
||||
else: # txt
|
||||
indexContent = self._formatAsText(indexData, documentsIndex)
|
||||
|
||||
# Generate meaningful filename
|
||||
workflowContext = self.services.chat.getWorkflowContext()
|
||||
filename = self._generateMeaningfulFileName(
|
||||
"document_index",
|
||||
resultType if resultType in ["json", "txt", "md"] else "json",
|
||||
workflowContext,
|
||||
"getDocumentIndex"
|
||||
)
|
||||
|
||||
# Create ActionDocument
|
||||
document = ActionDocument(
|
||||
documentName=filename,
|
||||
documentData=indexContent,
|
||||
mimeType="application/json" if resultType == "json" else "text/plain"
|
||||
)
|
||||
|
||||
return ActionResult.isSuccess(documents=[document])
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Error generating document index: {str(e)}")
|
||||
return ActionResult.isFailure(
|
||||
error=f"Failed to generate document index: {str(e)}"
|
||||
)
|
||||
|
||||
def _parseDocumentIndex(self, documentsIndex: str, workflow: Any) -> Dict[str, Any]:
|
||||
"""Parse the document index string into structured data."""
|
||||
try:
|
||||
indexData = {
|
||||
"workflowId": getattr(workflow, 'id', 'unknown'),
|
||||
"generatedAt": datetime.now(UTC).isoformat(),
|
||||
"totalDocuments": 0,
|
||||
"rounds": [],
|
||||
"documentReferences": []
|
||||
}
|
||||
|
||||
# Extract document references from the index string
|
||||
lines = documentsIndex.split('\n')
|
||||
currentRound = None
|
||||
currentDocList = None
|
||||
|
||||
for line in lines:
|
||||
line = line.strip()
|
||||
if not line:
|
||||
continue
|
||||
|
||||
# Check for round headers
|
||||
if "Current round documents:" in line:
|
||||
currentRound = "current"
|
||||
continue
|
||||
elif "Past rounds documents:" in line:
|
||||
currentRound = "past"
|
||||
continue
|
||||
|
||||
# Check for document list references (docList:...)
|
||||
if line.startswith("- docList:"):
|
||||
docListRef = line.replace("- docList:", "").strip()
|
||||
currentDocList = {
|
||||
"reference": docListRef,
|
||||
"round": currentRound,
|
||||
"documents": []
|
||||
}
|
||||
indexData["rounds"].append(currentDocList)
|
||||
continue
|
||||
|
||||
# Check for individual document references (docItem:...)
|
||||
if line.startswith(" - docItem:") or line.startswith("- docItem:"):
|
||||
docItemRef = line.replace(" - docItem:", "").replace("- docItem:", "").strip()
|
||||
indexData["documentReferences"].append({
|
||||
"reference": docItemRef,
|
||||
"round": currentRound,
|
||||
"docList": currentDocList["reference"] if currentDocList else None
|
||||
})
|
||||
indexData["totalDocuments"] += 1
|
||||
if currentDocList:
|
||||
currentDocList["documents"].append(docItemRef)
|
||||
|
||||
return indexData
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Error parsing document index: {str(e)}")
|
||||
return {
|
||||
"workflowId": getattr(workflow, 'id', 'unknown'),
|
||||
"error": f"Failed to parse document index: {str(e)}",
|
||||
"rawIndex": documentsIndex
|
||||
}
|
||||
|
||||
def _formatAsMarkdown(self, indexData: Dict[str, Any]) -> str:
|
||||
"""Format document index as Markdown."""
|
||||
try:
|
||||
md = f"# Document Index\n\n"
|
||||
md += f"**Workflow ID:** {indexData.get('workflowId', 'unknown')}\n\n"
|
||||
md += f"**Generated At:** {indexData.get('generatedAt', 'unknown')}\n\n"
|
||||
md += f"**Total Documents:** {indexData.get('totalDocuments', 0)}\n\n"
|
||||
|
||||
if indexData.get('rounds'):
|
||||
md += "## Documents by Round\n\n"
|
||||
for roundInfo in indexData['rounds']:
|
||||
roundLabel = roundInfo.get('round', 'unknown').title()
|
||||
md += f"### {roundLabel} Round\n\n"
|
||||
md += f"**Document List:** `{roundInfo.get('reference', 'unknown')}`\n\n"
|
||||
if roundInfo.get('documents'):
|
||||
md += "**Documents:**\n\n"
|
||||
for docRef in roundInfo['documents']:
|
||||
md += f"- `{docRef}`\n"
|
||||
md += "\n"
|
||||
|
||||
if indexData.get('documentReferences'):
|
||||
md += "## All Document References\n\n"
|
||||
for docRef in indexData['documentReferences']:
|
||||
md += f"- `{docRef.get('reference', 'unknown')}`\n"
|
||||
|
||||
return md
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Error formatting as Markdown: {str(e)}")
|
||||
return f"# Document Index\n\nError formatting index: {str(e)}\n"
|
||||
|
||||
def _formatAsText(self, indexData: Dict[str, Any], rawIndex: str) -> str:
|
||||
"""Format document index as plain text."""
|
||||
try:
|
||||
text = "Document Index\n"
|
||||
text += "=" * 50 + "\n\n"
|
||||
text += f"Workflow ID: {indexData.get('workflowId', 'unknown')}\n"
|
||||
text += f"Generated At: {indexData.get('generatedAt', 'unknown')}\n"
|
||||
text += f"Total Documents: {indexData.get('totalDocuments', 0)}\n\n"
|
||||
|
||||
# Include the raw formatted index for readability
|
||||
text += rawIndex
|
||||
|
||||
return text
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Error formatting as text: {str(e)}")
|
||||
return f"Document Index\n\nError formatting index: {str(e)}\n\nRaw index:\n{rawIndex}\n"
|
||||
|
||||
@action
|
||||
async def extractContent(self, parameters: Dict[str, Any]) -> ActionResult:
|
||||
"""
|
||||
Extract content from documents (separate from AI calls).
|
||||
|
||||
This action performs pure content extraction without AI processing.
|
||||
The extracted ContentParts can then be used by subsequent AI processing actions.
|
||||
|
||||
Parameters:
|
||||
- documentList (list, required): Document reference(s) to extract content from.
|
||||
- extractionOptions (dict, optional): Extraction options (if not provided, defaults are used).
|
||||
|
||||
Returns:
|
||||
- ActionResult with ActionDocument containing ContentExtracted objects
|
||||
- ContentExtracted.parts contains List[ContentPart] (already chunked if needed)
|
||||
"""
|
||||
try:
|
||||
# Init progress logger
|
||||
workflowId = self.services.workflow.id if self.services.workflow else f"no-workflow-{int(time.time())}"
|
||||
operationId = f"context_extract_{workflowId}_{int(time.time())}"
|
||||
|
||||
# Extract documentList from parameters dict
|
||||
from modules.datamodels.datamodelDocref import DocumentReferenceList
|
||||
documentListParam = parameters.get("documentList")
|
||||
if not documentListParam:
|
||||
return ActionResult.isFailure(error="documentList is required")
|
||||
|
||||
# Convert to DocumentReferenceList if needed
|
||||
if isinstance(documentListParam, DocumentReferenceList):
|
||||
documentList = documentListParam
|
||||
elif isinstance(documentListParam, str):
|
||||
documentList = DocumentReferenceList.from_string_list([documentListParam])
|
||||
elif isinstance(documentListParam, list):
|
||||
documentList = DocumentReferenceList.from_string_list(documentListParam)
|
||||
else:
|
||||
return ActionResult.isFailure(error=f"Invalid documentList type: {type(documentListParam)}")
|
||||
|
||||
# Start progress tracking
|
||||
self.services.chat.progressLogStart(
|
||||
operationId,
|
||||
"Extracting content from documents",
|
||||
"Content Extraction",
|
||||
f"Documents: {len(documentList.references)}"
|
||||
)
|
||||
|
||||
# Get ChatDocuments from documentList
|
||||
self.services.chat.progressLogUpdate(operationId, 0.2, "Loading documents")
|
||||
chatDocuments = self.services.chat.getChatDocumentsFromDocumentList(documentList)
|
||||
|
||||
if not chatDocuments:
|
||||
self.services.chat.progressLogFinish(operationId, False)
|
||||
return ActionResult.isFailure(error="No documents found in documentList")
|
||||
|
||||
logger.info(f"Extracting content from {len(chatDocuments)} documents")
|
||||
|
||||
# Prepare extraction options
|
||||
self.services.chat.progressLogUpdate(operationId, 0.3, "Preparing extraction options")
|
||||
extractionOptionsParam = parameters.get("extractionOptions")
|
||||
|
||||
# Convert dict to ExtractionOptions object if needed, or create defaults
|
||||
if extractionOptionsParam:
|
||||
if isinstance(extractionOptionsParam, dict):
|
||||
# Convert dict to ExtractionOptions object
|
||||
extractionOptions = ExtractionOptions(**extractionOptionsParam)
|
||||
elif isinstance(extractionOptionsParam, ExtractionOptions):
|
||||
extractionOptions = extractionOptionsParam
|
||||
else:
|
||||
# Invalid type, use defaults
|
||||
extractionOptions = None
|
||||
else:
|
||||
extractionOptions = None
|
||||
|
||||
# If extractionOptions not provided, create defaults
|
||||
if not extractionOptions:
|
||||
# Default extraction options for pure content extraction (no AI processing)
|
||||
extractionOptions = ExtractionOptions(
|
||||
prompt="Extract all content from the document",
|
||||
mergeStrategy=MergeStrategy(
|
||||
mergeType="concatenate",
|
||||
groupBy="typeGroup",
|
||||
orderBy="id"
|
||||
),
|
||||
processDocumentsIndividually=True
|
||||
)
|
||||
|
||||
# Get parent log ID for document-level operations
|
||||
parentLogId = self.services.chat.getOperationLogId(operationId)
|
||||
|
||||
# Call extraction service
|
||||
self.services.chat.progressLogUpdate(operationId, 0.4, "Initiating")
|
||||
self.services.chat.progressLogUpdate(operationId, 0.5, f"Extracting content from {len(chatDocuments)} documents")
|
||||
extractedResults = self.services.extraction.extractContent(chatDocuments, extractionOptions)
|
||||
|
||||
# Build ActionDocuments from ContentExtracted results
|
||||
self.services.chat.progressLogUpdate(operationId, 0.8, "Building result documents")
|
||||
actionDocuments = []
|
||||
# Map extracted results back to original documents by index (results are in same order)
|
||||
for i, extracted in enumerate(extractedResults):
|
||||
# Get original document name if available
|
||||
originalDoc = chatDocuments[i] if i < len(chatDocuments) else None
|
||||
if originalDoc and hasattr(originalDoc, 'fileName') and originalDoc.fileName:
|
||||
# Use original filename with "extracted_" prefix
|
||||
baseName = originalDoc.fileName.rsplit('.', 1)[0] if '.' in originalDoc.fileName else originalDoc.fileName
|
||||
documentName = f"{baseName}_extracted_{extracted.id}.json"
|
||||
else:
|
||||
# Fallback to generic name with index
|
||||
documentName = f"document_{i+1:03d}_extracted_{extracted.id}.json"
|
||||
|
||||
# Store ContentExtracted object in ActionDocument.documentData
|
||||
actionDoc = ActionDocument(
|
||||
documentName=documentName,
|
||||
documentData=extracted, # ContentExtracted object
|
||||
mimeType="application/json"
|
||||
)
|
||||
actionDocuments.append(actionDoc)
|
||||
|
||||
self.services.chat.progressLogFinish(operationId, True)
|
||||
|
||||
return ActionResult.isSuccess(documents=actionDocuments)
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Error in content extraction: {str(e)}")
|
||||
|
||||
# Complete progress tracking with failure
|
||||
try:
|
||||
self.services.chat.progressLogFinish(operationId, False)
|
||||
except:
|
||||
pass # Don't fail on progress logging errors
|
||||
|
||||
return ActionResult.isFailure(error=str(e))
|
||||
|
||||
|
|
@ -210,8 +210,14 @@ class MessageCreator:
|
|||
taskProgress = str(taskIndex)
|
||||
|
||||
# Enhanced completion message with criteria details
|
||||
if reviewResult and hasattr(reviewResult, 'reason'):
|
||||
completionMessage = f"🎯 **Task {taskProgress}**\n\n✅ {reviewResult.reason or 'Task completed successfully'}"
|
||||
# Prefer userMessage (user-friendly in user's language), fallback to reason
|
||||
if reviewResult:
|
||||
if hasattr(reviewResult, 'userMessage') and reviewResult.userMessage:
|
||||
completionMessage = f"🎯 **Task {taskProgress}**\n\n✅ {reviewResult.userMessage}"
|
||||
elif hasattr(reviewResult, 'reason') and reviewResult.reason:
|
||||
completionMessage = f"🎯 **Task {taskProgress}**\n\n✅ {reviewResult.reason}"
|
||||
else:
|
||||
completionMessage = f"🎯 **Task {taskProgress}**\n\n✅ Task completed successfully"
|
||||
else:
|
||||
completionMessage = f"🎯 **Task {taskProgress}**\n\n✅ Task completed successfully"
|
||||
|
||||
|
|
|
|||
|
|
@ -350,34 +350,36 @@ Return ONLY JSON (no markdown, no explanations). The decision MUST:
|
|||
- Match parameter names exactly as defined in AVAILABLE_METHODS
|
||||
|
||||
{{
|
||||
"status": "continue",
|
||||
"reason": "Brief reason explaining why continuing",
|
||||
"nextAction": "Selected_action_from_ACTIONS",
|
||||
"status": "continue" | "success",
|
||||
"reason": "Brief reason explaining why continuing or why task is complete",
|
||||
"userMessage": "User-friendly message in language '{{KEY:USER_LANGUAGE}}' explaining the task status (1 sentence, first person, friendly tone)",
|
||||
"nextAction": "Selected_action_from_ACTIONS" | null,
|
||||
"nextActionParameters": {{
|
||||
"documentList": ["docItem:<documentId>:<filename>", "docList:<label>"],
|
||||
"connectionReference": "connection:reference_from_AVAILABLE_CONNECTIONS_INDEX",
|
||||
"parameter1": "value1",
|
||||
"parameter2": "value2"
|
||||
}},
|
||||
"nextActionObjective": "Clear description of what this action will achieve based on improvement suggestions"
|
||||
}} | null,
|
||||
"nextActionObjective": "Clear description of what this action will achieve based on improvement suggestions" | null
|
||||
}}
|
||||
|
||||
=== RULES ===
|
||||
1. Return ONLY JSON - no markdown, no explanations
|
||||
2. If "continue": MUST provide nextAction and nextActionParameters
|
||||
3. nextAction: SPECIFIC action from AVAILABLE_METHODS (do not invent)
|
||||
4. nextActionParameters: concrete parameters (check AVAILABLE_METHODS for valid names)
|
||||
5. documentList: ONLY exact references from AVAILABLE_DOCUMENTS_INDEX (do not invent/modify)
|
||||
2. userMessage: REQUIRED - Provide a user-friendly message in language '{{KEY:USER_LANGUAGE}}' explaining the task status (for "continue": explain what's being done next; for "success": explain what was accomplished)
|
||||
3. If "continue": MUST provide nextAction and nextActionParameters
|
||||
4. nextAction: SPECIFIC action from AVAILABLE_METHODS (do not invent)
|
||||
5. nextActionParameters: concrete parameters (check AVAILABLE_METHODS for valid names)
|
||||
6. documentList: ONLY exact references from AVAILABLE_DOCUMENTS_INDEX (do not invent/modify)
|
||||
- For individual documents: ALWAYS use docItem:<documentId>:<filename> format (include filename)
|
||||
- For document lists: use docList:<label> format
|
||||
- Copy references EXACTLY as shown in AVAILABLE_DOCUMENTS_INDEX (including filename)
|
||||
6. connectionReference: ONLY exact label from AVAILABLE_CONNECTIONS_INDEX (required if action needs connection)
|
||||
7. nextActionObjective: describe what this action will achieve based on the FIRST improvement suggestion from CONTENT VALIDATION
|
||||
8. CRITICAL: Use structureComparison.gap to specify the missing part in nextActionParameters
|
||||
9. Do NOT repeat failed actions - suggest DIFFERENT approach
|
||||
10. If ACTION HISTORY shows repeated actions, suggest a fundamentally different approach
|
||||
11. nextActionObjective must directly address the highest priority improvement suggestion from CONTENT VALIDATION
|
||||
12. If validation shows partial data delivered, next action should CONTINUE from where it stopped, not restart
|
||||
7. connectionReference: ONLY exact label from AVAILABLE_CONNECTIONS_INDEX (required if action needs connection)
|
||||
8. nextActionObjective: describe what this action will achieve based on the FIRST improvement suggestion from CONTENT VALIDATION
|
||||
9. CRITICAL: Use structureComparison.gap to specify the missing part in nextActionParameters
|
||||
10. Do NOT repeat failed actions - suggest DIFFERENT approach
|
||||
11. If ACTION HISTORY shows repeated actions, suggest a fundamentally different approach
|
||||
12. nextActionObjective must directly address the highest priority improvement suggestion from CONTENT VALIDATION
|
||||
13. If validation shows partial data delivered, next action should CONTINUE from where it stopped, not restart
|
||||
|
||||
"""
|
||||
|
||||
|
|
|
|||
|
|
@ -18,7 +18,14 @@ logger = logging.getLogger(__name__)
|
|||
def generateTaskPlanningPrompt(services, context: Any) -> PromptBundle:
|
||||
"""Define placeholders first, then the template; return PromptBundle."""
|
||||
# Extract user language from services
|
||||
userLanguage = getattr(services, 'currentUserLanguage', None) or 'en'
|
||||
# Prefer currentUserLanguage (set from user intention analysis), fallback to user.language, then 'en'
|
||||
userLanguage = getattr(services, 'currentUserLanguage', None)
|
||||
if not userLanguage:
|
||||
userLanguage = getattr(services.user, 'language', None) if hasattr(services, 'user') and services.user else None
|
||||
if not userLanguage:
|
||||
userLanguage = 'en'
|
||||
|
||||
logger.debug(f"Task planning prompt using user language: {userLanguage}")
|
||||
|
||||
# Extract workflowIntent from workflow object if available
|
||||
workflowIntent = {}
|
||||
|
|
|
|||
|
|
@ -55,10 +55,14 @@ class WorkflowProcessor:
|
|||
f"Mode: {workflow.workflowMode.value if hasattr(workflow.workflowMode, 'value') else workflow.workflowMode}"
|
||||
)
|
||||
|
||||
# Initialize currentUserLanguage to empty at workflow start
|
||||
self.services.currentUserLanguage = ""
|
||||
# currentUserLanguage should already be set from user intention analysis in _sendFirstMessage
|
||||
# Do NOT reset it here, as it contains the detected language from the user's input
|
||||
# Only initialize if not already set (should not happen in normal flow)
|
||||
if not hasattr(self.services, 'currentUserLanguage') or not self.services.currentUserLanguage:
|
||||
self.services.currentUserLanguage = getattr(self.services.user, 'language', None) or 'en'
|
||||
|
||||
logger.info(f"=== STARTING TASK PLAN GENERATION ===")
|
||||
logger.info(f"Using user language: {self.services.currentUserLanguage}")
|
||||
logger.info(f"Workflow ID: {workflow.id}")
|
||||
logger.info(f"User Input: {userInput}")
|
||||
modeValue = workflow.workflowMode.value if hasattr(workflow.workflowMode, 'value') else workflow.workflowMode
|
||||
|
|
|
|||
|
|
@ -186,11 +186,18 @@ class WorkflowManager:
|
|||
# Now send the first message (which will also process the documents again, but that's fine)
|
||||
await self._sendFirstMessage(userInput)
|
||||
|
||||
# Route to fast path for simple requests (skip for automation mode)
|
||||
if not skipComplexityDetection and complexity == "simple":
|
||||
# Check if workflow history is needed before deciding on fast path
|
||||
# Use AI intention analysis result only (no keyword matching)
|
||||
needsHistory = getattr(self.services, '_needsWorkflowHistory', False)
|
||||
hasHistory = self._checkIfHistoryAvailable()
|
||||
|
||||
# Route to fast path for simple requests (skip for automation mode and if history is needed)
|
||||
if not skipComplexityDetection and complexity == "simple" and not (needsHistory and hasHistory):
|
||||
logger.info("Routing to fast path for simple request")
|
||||
await self._executeFastPath(userInput, documents)
|
||||
return # Fast path completes the workflow
|
||||
elif needsHistory and hasHistory:
|
||||
logger.info(f"Skipping fast path - workflow history is needed (from AI intention analysis) and available")
|
||||
|
||||
# Route to full workflow for moderate/complex requests or automation mode
|
||||
logger.info(f"Routing to full workflow for {complexity} request" + (" (automation mode)" if skipComplexityDetection else ""))
|
||||
|
|
@ -369,7 +376,8 @@ class WorkflowManager:
|
|||
"6) dataType: What type of data/content they want (numbers|text|documents|analysis|code|unknown).\n"
|
||||
"7) expectedFormats: What file format(s) they expect - provide matching file format extensions list (e.g., [\"xlsx\", \"pdf\"]). If format is unclear or not specified, use empty list [].\n"
|
||||
"8) qualityRequirements: Quality requirements they have (accuracy, completeness) as {accuracyThreshold: 0.0-1.0, completenessThreshold: 0.0-1.0}.\n"
|
||||
"9) successCriteria: Specific success criteria that define completion (array of strings).\n\n"
|
||||
"9) successCriteria: Specific success criteria that define completion (array of strings).\n"
|
||||
"10) needsWorkflowHistory: Boolean indicating if this request needs previous workflow rounds/history to be understood or completed (e.g., 'continue', 'retry', 'fix', 'improve', 'update', 'modify', 'based on previous', 'build on', references to earlier work). Return true if the request is a continuation, retry, modification, or builds upon previous work.\n\n"
|
||||
"Rules:\n"
|
||||
"- If total content (intent + data) is < 10% of model max tokens, do not extract; return empty contextItems and keep intent compact and self-contained.\n"
|
||||
"- If content exceeds that threshold, move bulky parts into contextItems; keep intent short and clear.\n"
|
||||
|
|
@ -394,7 +402,8 @@ class WorkflowManager:
|
|||
" \"accuracyThreshold\": 0.0-1.0,\n"
|
||||
" \"completenessThreshold\": 0.0-1.0\n"
|
||||
" },\n"
|
||||
" \"successCriteria\": [\"specific criterion 1\", \"specific criterion 2\"]\n"
|
||||
" \"successCriteria\": [\"specific criterion 1\", \"specific criterion 2\"],\n"
|
||||
" \"needsWorkflowHistory\": true|false\n"
|
||||
"}\n\n"
|
||||
f"User message:\n{self.services.utils.sanitizePromptContent(userInput.prompt, 'userinput')}"
|
||||
)
|
||||
|
|
@ -431,9 +440,15 @@ class WorkflowManager:
|
|||
'expectedFormats': parsed.get('expectedFormats', []),
|
||||
'qualityRequirements': parsed.get('qualityRequirements', {}),
|
||||
'successCriteria': parsed.get('successCriteria', []),
|
||||
'languageUserDetected': detectedLanguage
|
||||
'languageUserDetected': detectedLanguage,
|
||||
'needsWorkflowHistory': parsed.get('needsWorkflowHistory', False)
|
||||
}
|
||||
|
||||
# Store needsWorkflowHistory in services for fast path decision
|
||||
needsHistoryFromIntention = parsed.get('needsWorkflowHistory', False)
|
||||
if isinstance(needsHistoryFromIntention, bool):
|
||||
setattr(self.services, '_needsWorkflowHistory', needsHistoryFromIntention)
|
||||
|
||||
# Store workflowIntent in workflow object for reuse
|
||||
if hasattr(self.services, 'workflow') and self.services.workflow:
|
||||
self.services.workflow._workflowIntent = workflowIntent
|
||||
|
|
@ -1089,3 +1104,22 @@ class WorkflowManager:
|
|||
logger.error(f"Error during content neutralization: {str(e)}")
|
||||
# Return original content on error
|
||||
return contentBytes
|
||||
|
||||
def _checkIfHistoryAvailable(self) -> bool:
|
||||
"""Check if workflow history is available (previous rounds exist).
|
||||
|
||||
Returns True if there are previous workflow rounds with messages.
|
||||
"""
|
||||
try:
|
||||
from modules.workflows.processing.shared.placeholderFactory import getPreviousRoundContext
|
||||
|
||||
history = getPreviousRoundContext(self.services)
|
||||
|
||||
# Check if history contains actual content (not just "No previous round context available")
|
||||
if history and history != "No previous round context available":
|
||||
return True
|
||||
|
||||
return False
|
||||
except Exception as e:
|
||||
logger.error(f"Error checking if history is available: {str(e)}")
|
||||
return False
|
||||
|
|
|
|||
Loading…
Reference in a new issue