From a20c44aadfb6c0f8a880ab25d42ab9d37fcf8759 Mon Sep 17 00:00:00 2001 From: Ida Dittrich Date: Wed, 19 Nov 2025 16:02:12 +0100 Subject: [PATCH 01/25] integrated PEK data modell and according routes to get data --- app.py | 3 + docs/PEK_datamodel_desc.md | 418 ++++++++ .../01-overview.md | 169 +++ .../02-datamodels.md | 976 ++++++++++++++++++ .../03-interfaces.md | 845 +++++++++++++++ .../04-feature-logic.md | 780 ++++++++++++++ .../05-routes.md | 332 ++++++ .../06-router-registration.md | 45 + .../07-environment.md | 50 + .../08-lifecycle.md | 43 + .../09-database-schema.md | 247 +++++ .../10-security.md | 90 ++ .../11-testing.md | 51 + .../12-troubleshooting.md | 33 + .../13-summary.md | 136 +++ .../README.md | 42 + env_dev.env | 7 + ...51119-100038-001-intentanalysis_prompt.txt | 48 + ...119-100041-002-intentanalysis_response.txt | 10 + ...51119-103736-003-intentanalysis_prompt.txt | 48 + ...119-103742-004-intentanalysis_response.txt | 16 + ...51119-103802-005-intentanalysis_prompt.txt | 48 + ...119-103808-006-intentanalysis_response.txt | 16 + ...51119-104317-007-intentanalysis_prompt.txt | 88 ++ ...119-104324-008-intentanalysis_response.txt | 18 + ...51119-104747-009-intentanalysis_prompt.txt | 88 ++ ...119-104754-010-intentanalysis_response.txt | 17 + ...51119-104856-011-intentanalysis_prompt.txt | 88 ++ ...119-104902-012-intentanalysis_response.txt | 17 + ...51119-105033-013-intentanalysis_prompt.txt | 88 ++ ...119-105039-014-intentanalysis_response.txt | 17 + ...51119-110433-015-intentanalysis_prompt.txt | 89 ++ ...119-110440-016-intentanalysis_response.txt | 17 + ...51119-125234-017-intentanalysis_prompt.txt | 89 ++ ...119-125239-018-intentanalysis_response.txt | 16 + ...51119-125657-019-intentanalysis_prompt.txt | 89 ++ ...119-125702-020-intentanalysis_response.txt | 16 + ...51119-125750-021-intentanalysis_prompt.txt | 89 ++ ...119-125756-022-intentanalysis_response.txt | 20 + ...51119-141840-023-intentanalysis_prompt.txt | 89 ++ ...119-141848-024-intentanalysis_response.txt | 16 + ...51119-144901-025-intentanalysis_prompt.txt | 89 ++ ...119-144909-026-intentanalysis_response.txt | 17 + ...51119-145121-027-intentanalysis_prompt.txt | 89 ++ ...119-145129-028-intentanalysis_response.txt | 17 + ...51119-145151-029-intentanalysis_prompt.txt | 89 ++ ...119-145202-030-intentanalysis_response.txt | 18 + ...51119-145222-031-intentanalysis_prompt.txt | 89 ++ ...119-145229-032-intentanalysis_response.txt | 14 + ...51119-145449-033-intentanalysis_prompt.txt | 89 ++ ...119-145452-034-intentanalysis_response.txt | 12 + modules/datamodels/datamodelRealEstate.py | 667 ++++++++++++ modules/features/realEstate/__init__.py | 4 + modules/features/realEstate/mainRealEstate.py | 769 ++++++++++++++ .../interfaces/interfaceDbRealEstateAccess.py | 87 ++ .../interfaceDbRealEstateObjects.py | 713 +++++++++++++ modules/routes/routeRealEstate.py | 637 ++++++++++++ 57 files changed, 8804 insertions(+) create mode 100644 docs/PEK_datamodel_desc.md create mode 100644 docs/real-estate-feature-integration-guide/01-overview.md create mode 100644 docs/real-estate-feature-integration-guide/02-datamodels.md create mode 100644 docs/real-estate-feature-integration-guide/03-interfaces.md create mode 100644 docs/real-estate-feature-integration-guide/04-feature-logic.md create mode 100644 docs/real-estate-feature-integration-guide/05-routes.md create mode 100644 docs/real-estate-feature-integration-guide/06-router-registration.md create mode 100644 docs/real-estate-feature-integration-guide/07-environment.md create mode 100644 docs/real-estate-feature-integration-guide/08-lifecycle.md create mode 100644 docs/real-estate-feature-integration-guide/09-database-schema.md create mode 100644 docs/real-estate-feature-integration-guide/10-security.md create mode 100644 docs/real-estate-feature-integration-guide/11-testing.md create mode 100644 docs/real-estate-feature-integration-guide/12-troubleshooting.md create mode 100644 docs/real-estate-feature-integration-guide/13-summary.md create mode 100644 docs/real-estate-feature-integration-guide/README.md create mode 100644 logs/debug/prompts/20251119-100038-001-intentanalysis_prompt.txt create mode 100644 logs/debug/prompts/20251119-100041-002-intentanalysis_response.txt create mode 100644 logs/debug/prompts/20251119-103736-003-intentanalysis_prompt.txt create mode 100644 logs/debug/prompts/20251119-103742-004-intentanalysis_response.txt create mode 100644 logs/debug/prompts/20251119-103802-005-intentanalysis_prompt.txt create mode 100644 logs/debug/prompts/20251119-103808-006-intentanalysis_response.txt create mode 100644 logs/debug/prompts/20251119-104317-007-intentanalysis_prompt.txt create mode 100644 logs/debug/prompts/20251119-104324-008-intentanalysis_response.txt create mode 100644 logs/debug/prompts/20251119-104747-009-intentanalysis_prompt.txt create mode 100644 logs/debug/prompts/20251119-104754-010-intentanalysis_response.txt create mode 100644 logs/debug/prompts/20251119-104856-011-intentanalysis_prompt.txt create mode 100644 logs/debug/prompts/20251119-104902-012-intentanalysis_response.txt create mode 100644 logs/debug/prompts/20251119-105033-013-intentanalysis_prompt.txt create mode 100644 logs/debug/prompts/20251119-105039-014-intentanalysis_response.txt create mode 100644 logs/debug/prompts/20251119-110433-015-intentanalysis_prompt.txt create mode 100644 logs/debug/prompts/20251119-110440-016-intentanalysis_response.txt create mode 100644 logs/debug/prompts/20251119-125234-017-intentanalysis_prompt.txt create mode 100644 logs/debug/prompts/20251119-125239-018-intentanalysis_response.txt create mode 100644 logs/debug/prompts/20251119-125657-019-intentanalysis_prompt.txt create mode 100644 logs/debug/prompts/20251119-125702-020-intentanalysis_response.txt create mode 100644 logs/debug/prompts/20251119-125750-021-intentanalysis_prompt.txt create mode 100644 logs/debug/prompts/20251119-125756-022-intentanalysis_response.txt create mode 100644 logs/debug/prompts/20251119-141840-023-intentanalysis_prompt.txt create mode 100644 logs/debug/prompts/20251119-141848-024-intentanalysis_response.txt create mode 100644 logs/debug/prompts/20251119-144901-025-intentanalysis_prompt.txt create mode 100644 logs/debug/prompts/20251119-144909-026-intentanalysis_response.txt create mode 100644 logs/debug/prompts/20251119-145121-027-intentanalysis_prompt.txt create mode 100644 logs/debug/prompts/20251119-145129-028-intentanalysis_response.txt create mode 100644 logs/debug/prompts/20251119-145151-029-intentanalysis_prompt.txt create mode 100644 logs/debug/prompts/20251119-145202-030-intentanalysis_response.txt create mode 100644 logs/debug/prompts/20251119-145222-031-intentanalysis_prompt.txt create mode 100644 logs/debug/prompts/20251119-145229-032-intentanalysis_response.txt create mode 100644 logs/debug/prompts/20251119-145449-033-intentanalysis_prompt.txt create mode 100644 logs/debug/prompts/20251119-145452-034-intentanalysis_response.txt create mode 100644 modules/datamodels/datamodelRealEstate.py create mode 100644 modules/features/realEstate/__init__.py create mode 100644 modules/features/realEstate/mainRealEstate.py create mode 100644 modules/interfaces/interfaceDbRealEstateAccess.py create mode 100644 modules/interfaces/interfaceDbRealEstateObjects.py create mode 100644 modules/routes/routeRealEstate.py diff --git a/app.py b/app.py index a167503c..0e8eef2d 100644 --- a/app.py +++ b/app.py @@ -406,6 +406,9 @@ app.include_router(workflowRouter) from modules.routes.routeChatPlayground import router as chatPlaygroundRouter app.include_router(chatPlaygroundRouter) +from modules.routes.routeRealEstate import router as realEstateRouter +app.include_router(realEstateRouter) + from modules.routes.routeSecurityLocal import router as localRouter app.include_router(localRouter) diff --git a/docs/PEK_datamodel_desc.md b/docs/PEK_datamodel_desc.md new file mode 100644 index 00000000..d2b94e5e --- /dev/null +++ b/docs/PEK_datamodel_desc.md @@ -0,0 +1,418 @@ +# Datenmodell Architektur-Planungs-App + +## Übersicht + +Dieses Datenmodell bildet die Grundlage für eine Schweizer Architektur-Planungs-App zur Verwaltung von Bauprojekten, Parzellen, Dokumenten und regulatorischen Kontextinformationen. + +## Wichtige Hinweise zum Datenmodell + +**Objektmodell vs. Datenbank-Repräsentation:** + +Dieses Dokument beschreibt ein **Objektmodell** für die Arbeit im Code. Es handelt sich **NICHT** um ein relationales Datenbankmodell mit Junction Tables. + +- **Im Code-Modell**: Alle Beziehungen werden als Objektreferenzen oder Listen von Objekten dargestellt (z.B. `dokumente: list[Dokument]`, `parzellen: list[Parzelle]`). +- **Für die Datenbank-Serialisierung**: Bei der Persistierung können Junction Tables verwendet werden, um n:m-Beziehungen in der Datenbank abzubilden. Dies ist jedoch ein Implementierungsdetail der Datenbank-Schicht und gehört nicht zum Hauptmodell. + +**Systemattribute:** + +Alle Datenobjekte haben automatisch die folgenden Systemattribute: +- `_createdAt`: Float (Timestamp UTC) +- `_createdBy`: String (User-ID) +- `_modifiedAt`: Float (Timestamp UTC) +- `_modifiedBy`: String (User-ID) + +**Timestamps:** +- Alle Timestamps sind im Float-Format UTC im Datenmodell gespeichert. +- Die Darstellung im UI erfolgt mit der lokalen Zeitzone des Benutzers. + +## Datenfluss-Diagramm + +```mermaid +--- +title: Hauptflüsse - Architektur-Planungs-App +--- +flowchart LR + subgraph Admin[Administrative Ebene] + Land[Land
Schweiz] + Kanton[Kanton
z.B. Zürich] + Gemeinde[Gemeinde
z.B. Zürich Stadt] + Land --> Kanton + Kanton --> Gemeinde + end + + subgraph Geo[Geografische Daten] + GeoPolylinie[GeoPolylinie
Linie/Polygon] + GeoPunkt[GeoPunkt
Koordinaten] + GeoPolylinie --> GeoPunkt + end + + subgraph Core[Kern-Business-Logik] + Projekt[Projekt
Bauprojekt] + Parzelle[Parzelle
Grundstück mit
Bauparametern] + Gemeinde --> Parzelle + Projekt --> Parzelle + Projekt --> GeoPolylinie + Parzelle --> GeoPolylinie + end + + subgraph Support[Unterstützende Daten] + Dokument[Dokument
Dateien & URLs] + Kontext[Kontext
Zusatzinfos] + end + + style Land fill:#50C878,stroke:#2D7A4A,stroke-width:2px,color:#fff + style Kanton fill:#50C878,stroke:#2D7A4A,stroke-width:2px,color:#fff + style Gemeinde fill:#50C878,stroke:#2D7A4A,stroke-width:2px,color:#fff + style Parzelle fill:#FF6B6B,stroke:#C92A2A,stroke-width:4px,color:#fff + style Projekt fill:#FF6B6B,stroke:#C92A2A,stroke-width:4px,color:#fff + style Dokument fill:#F5A623,stroke:#C17D11,stroke-width:2px,color:#fff + style GeoPolylinie fill:#F5A623,stroke:#C17D11,stroke-width:2px,color:#fff + style GeoPunkt fill:#F5A623,stroke:#C17D11,stroke-width:2px,color:#fff + style Kontext fill:#F5A623,stroke:#C17D11,stroke-width:2px,color:#fff +``` + +--- + +## Alle Datenobjekte als Tabellen + +### Übersichtstabelle + +| Objekt | Typ | Beschreibung | Hauptfelder | +|--------|-----|--------------|-------------| +| **Projekt** | Kernentität | Bauprojekt mit Status und Perimeter | id, label, statusProzess, perimeter, baulinie, parzellen | +| **Parzelle** | Hauptentität | Grundstück mit Bauparametern | id, label, plz, bauzone, AZ, BZ, perimeter, baulinie, laermschutzzone, hochwasserschutzzone, grundwasserschutzzone | +| **Dokument** | Unterstützend | Dateien und URLs mit Versionierung | id, label, dokumentTyp, quelle, mimeType, kategorienTags | +| **Kontext** | Unterstützend | Flexible Zusatzinformationen | id, thema, inhalt | +| **GeoPolylinie** | Hilfsobjekt | Geometrische Linie/Polygon | id, closed, punkte | +| **Land** | Admin | Nationale Ebene | id, label, abk | +| **Kanton** | Admin | Kantonale Ebene mit Baurecht | id, label, abk, Baureglement | +| **Gemeinde** | Admin | Gemeinde-Ebene mit BZO | id, label, plz, BZO | +| **GeoPunkt** | Hilfsobjekt | 3D-Koordinate | koordinatensystem, x, y, z, referenz | +| **GeoTag** | Enum | Geopunkt-Kategorien | - | +| **JaNein** | Enum | Drei-wertiger Status | "", "Ja", "Nein" | +| **StatusProzess** | Enum | Projektstatus | 7 Werte | +| **DokumentTyp** | Enum | Dokumenttyp | 6 Werte | + +--- + +## Zentrale Entitäten + +### 1. Projekt +**Das Kernobjekt, das ein Bauprojekt repräsentiert.** + +| Feld | Datentyp | Pflicht | Beschreibung | +|------|----------|---------|--------------| +| `id` | UUID | ✓ | Eindeutiger Identifier | +| `label` | String | ✓ | Projektbezeichnung | +| `statusProzess` | Enum[StatusProzess] | - | Projektstatus: Eingang, Analyse, Studie, Planung, Baurechtsverfahren, Umsetzung, Archiv | +| `perimeter` | GeoPolylinie | - | Umhüllende aller Parzellen des Projektes | +| `baulinie` | GeoPolylinie | - | Baulinie des Projektes | +| `parzellen` | list[Parzelle] | - | Alle Parzellen des Projektes | +| `dokumente` | list[Dokument] | - | Projektspezifische Dokumente | +| `kontextInformationen` | list[Kontext] | - | Projektspezifische Kontextinfos | + +--- + +### 2. Parzelle +**Repräsentiert ein Grundstück mit allen baurechtlichen Eigenschaften als ein einheitliches Objekt.** + +#### Grunddaten + +| Feld | Datentyp | Pflicht | Beschreibung | +|------|----------|---------|--------------| +| `id` | UUID | ✓ | Eindeutiger Identifier | +| `label` | String | ✓ | Parzellenbezeichnung | +| `parzellenAliasTags` | list[String] | - | Weitere Parzellennamen oder Flurnamen | +| `eigentuemerschaft` | String | - | Eigentümer der Parzelle | +| `strasseNr` | String | - | Straße und Hausnummer | +| `plz` | String | - | Postleitzahl der Parzelle (insbesondere bei Gemeinden mit mehreren PLZ) | + +#### Geografischer Kontext + +| Feld | Datentyp | Pflicht | Beschreibung | +|------|----------|---------|--------------| +| `perimeter` | GeoPolylinie | - | Parzellengrenze als geschlossene GeoPolylinie | +| `baulinie` | GeoPolylinie | - | Baulinie der Parzelle | +| `kontextGemeinde` | Gemeinde | - | Gemeinde der Parzelle | + +#### Bebauungsparameter + +| Feld | Datentyp | Pflicht | Beschreibung | +|------|----------|---------|--------------| +| `bauzone` | String | - | Bauzonenbezeichnung (z.B. W3, WG2, etc.) | +| `az` | Float | - | Ausnützungsziffer | +| `bz` | Float | - | Bebauungsziffer | +| `vollgeschossZahl` | Integer | - | Anzahl zulässiger Vollgeschosse | +| `anrechenbarDachgeschoss` | Float | - | Anrechenbarer Anteil Dachgeschoss (0.0 - 1.0) | +| `anrechenbarUntergeschoss` | Float | - | Anrechenbarer Anteil Untergeschoss (0.0 - 1.0) | +| `gebaeudehoeheMax` | Float | - | Maximale Gebäudehöhe in Metern | + +#### Abstandsregelungen + +| Feld | Datentyp | Pflicht | Beschreibung | +|------|----------|---------|--------------| +| `regelnGrenzabstand` | list[String] | - | Regelungen zum Grenzabstand | +| `regelnMehrlaengenzuschlag` | list[String] | - | Regelungen zum Mehrlängenzuschlag | +| `regelnMehrhoehenzuschlag` | list[String] | - | Regelungen zum Mehrhöhenzuschlag | + +#### Eigenschaften (Ja/Nein) + +| Feld | Datentyp | Pflicht | Beschreibung | +|------|----------|---------|--------------| +| `parzelleBebaut` | JaNein | - | Ist die Parzelle bebaut? ("", "Ja", "Nein") | +| `parzelleErschlossen` | JaNein | - | Ist die Parzelle erschlossen? ("", "Ja", "Nein") | +| `parzelleHanglage` | JaNein | - | Liegt die Parzelle in Hanglage? ("", "Ja", "Nein") | + +#### Schutzzonen + +| Feld | Datentyp | Pflicht | Beschreibung | +|------|----------|---------|--------------| +| `laermschutzzone` | String | - | Lärmschutzzone (z.B. "II") | +| `hochwasserschutzzone` | String | - | Hochwasserschutzzone (z.B. "tief") | +| `grundwasserschutzzone` | String | - | Grundwasserschutzzone | + +#### Beziehungen + +| Feld | Datentyp | Pflicht | Beschreibung | +|------|----------|---------|--------------| +| `parzellenNachbarschaft` | list[Parzelle] | - | Nachbarparzellen | +| `dokumente` | list[Dokument] | - | Parzellenspezifische Dokumente | +| `kontextInformationen` | list[Kontext] | - | Parzellenspezifische Kontextinfos | + +--- + +### 3. Dokument +**Unterstützendes Datenobjekt zur Verwaltung von Dateien und URLs mit Versionierung.** + +| Feld | Datentyp | Pflicht | Beschreibung | +|------|----------|---------|--------------| +| `id` | UUID | ✓ | Eindeutiger Identifier | +| `label` | String | ✓ | Dokumentbezeichnung | +| `versionsbezeichnung` | String | - | Versionsnummer oder -bezeichnung (z.B. "v1.0", "Rev. A") | +| `dokumentTyp` | Enum[DokumentTyp] | - | Typ des Dokuments (siehe DokumentTyp-Enum) | +| `dokumentReferenz` | String | ✓ | Dateipfad oder URL | +| `quelle` | String | - | Quelle des Dokuments | +| `mimeType` | String | - | MIME-Type des Dokuments (z.B. "application/pdf", "image/png") | +| `kategorienTags` | list[String] | - | Kategorisierung des Dokuments | + +**Hinweis:** +Aktuelle Dokumente (z.B. aktuelle Baureglemente, BZO) können anhand des `dokumentTyp`-Attributs identifiziert werden. Die entsprechenden Dokumente finden sich in der `dokumente`-Liste der jeweiligen Entität (Kanton, Gemeinde). + +#### Beispiel-Kategorien (nicht abschliessend) + +| Kategorie | Beschreibung | +|-----------|--------------| +| `Kataster Objekte` | Amtliche Vermessung | +| `Kataster Werkeleitungen` | Leitungskataster | +| `Kataster Belastete Standorte` | Altlasten | +| `Kataster Bäume` | Baumkataster | +| `Zonenplan` | Zonenpläne | +| `Planungs- und Baugesetz (PGB)` | Kantonale Baugesetze | +| `Bau- und Zonenordnung (BZO)` | Gemeinde BZO | +| `Parkplatzverordnung` | Parkplatzregelungen | +| `Eigentümerauskunft` | Grundbuch-Auszüge Eigentümer | +| `Grundbuchauszug` | Vollständige Grundbuch-Auszüge | +| `Bauherrschaft` | Dokumente von der Bauherrschaft | +| `Planung` | Planungsdokumente | + +--- + +### 4. Geografische Entitäten + +#### GeoPolylinie +**Repräsentiert eine Linie oder ein Polygon aus mehreren GeoPunkten.** + +| Feld | Datentyp | Pflicht | Beschreibung | +|------|----------|---------|--------------| +| `id` | UUID | ✓ | Eindeutiger Identifier | +| `closed` | Boolean | ✓ | Ist die GeoPolylinie geschlossen (Polygon)? | +| `punkte` | list[GeoPunkt] | ✓ | Liste der GeoPunkte, die die GeoPolylinie bilden | + +**Verwendung:** +- Parzellenperimeter (geschlossene GeoPolylinie) +- Baulinie (offene oder geschlossene GeoPolylinie) +- Projektperimeter (geschlossene GeoPolylinie) + +#### GeoPunkt +**Repräsentiert einen 3D-Punkt mit Referenzangabe.** + +| Feld | Datentyp | Pflicht | Beschreibung | +|------|----------|---------|--------------| +| `koordinatensystem` | String | ✓ | Koordinatensystem (z.B. "LV95", "EPSG:2056") | +| `x` | Float | ✓ | Ostwert (E) [m], typisch 2'480'000 - 2'840'000 | +| `y` | Float | ✓ | Nordwert (N) [m], typisch 1'070'000 - 1'300'000 | +| `z` | Float | - | Höhe über Meer [m] | +| `referenz` | Enum[GeoTag] | - | Kategorisierung des Punktes | + +**Verwendung:** +- Als Teil einer GeoPolylinie (Parzellenperimeter, Baulinie) +- Einzelne Referenzpunkte + +#### GeoTag (Enum) + +| Kategorie | Beschreibung | +|-----------|--------------| +| `K1` | Fixpunkt höchster Genauigkeit | +| `K2` | Fixpunkt mittlerer Genauigkeit | +| `K3` | Fixpunkt niedriger Genauigkeit | +| `Geometer` | Vom Geometer vermessener Punkt | + +**Beispiel (Zürich Hauptbahnhof):** +- `koordinatensystem`: "LV95" oder "EPSG:2056" +- `x`: 2'683'140 [m] +- `y`: 1'247'850 [m] +- `z`: 408 [m] +- `referenz`: "K1" (oder ein anderer GeoTag-Wert) + +--- + +### 5. Administrative Hierarchie + +#### Land + +| Feld | Datentyp | Pflicht | Beschreibung | +|------|----------|---------|--------------| +| `id` | UUID | ✓ | Eindeutiger Identifier | +| `label` | String | ✓ | Landesname (z.B. "Schweiz") | +| `abk` | String | - | Abkürzung (z.B. "CH") | +| `dokumente` | list[Dokument] | - | Nationale Gesetze | +| `kontextInformationen` | list[Kontext] | - | Nationale Kontextinformationen | + +--- + +#### Kanton + +| Feld | Datentyp | Pflicht | Beschreibung | +|------|----------|---------|--------------| +| `id` | UUID | ✓ | Eindeutiger Identifier | +| `label` | String | ✓ | Kantonsname (z.B. "Zürich") | +| `id_land` | [land] | eindeutiger Link zum land, also in welchem land kanton liegt | +| `abk` | String | - | Abkürzung (z.B. "ZH") | +| `dokumente` | list[Dokument] | - | Kantonale Dokumente | +| `kontextInformationen` | list[Kontext] | - | Kantonsspezifische Kontextinfos | + +--- + +#### Gemeinde + +| Feld | Datentyp | Pflicht | Beschreibung | +|------|----------|---------|--------------| +| `id` | UUID | ✓ | Eindeutiger Identifier | +| `label` | String | ✓ | Gemeindename (z.B. "Zürich") | +| `id_kanton` | [kanton] | eindeutiger Link zur gemeinde, also im welchem kanton gemeinde liegt | +| `plz` | String | - | Postleitzahl (bei Gemeinden mit mehreren PLZ kann dies eine Haupt-PLZ sein) | +| `dokumente` | list[Dokument] | - | Gemeindedokumente | +| `kontextInformationen` | list[Kontext] | - | Gemeindespezifische Kontextinfos | + +**Hinweis:** +Bei Gemeinden mit mehreren Postleitzahlen (z.B. Zürich, Bern) wird die konkrete PLZ der Parzelle im Attribut `plz` der Parzelle erfasst. + +--- + +### 6. Kontext +**Unterstützendes Datenobjekt für flexible Zusatzinformationen und Hinweise.** + +| Feld | Datentyp | Pflicht | Beschreibung | +|------|----------|---------|--------------| +| `id` | UUID | ✓ | Eindeutiger Identifier | +| `thema` | String | ✓ | Bezeichnung des Themas | +| `inhalt` | String | ✓ | Detaillierte Information (Text) | + +**Verwendung:** +Kontext-Objekte werden als Listen in den jeweiligen Entitäten gespeichert: +- `projekt.kontextInformationen: list[Kontext]` +- `parzelle.kontextInformationen: list[Kontext]` +- `land.kontextInformationen: list[Kontext]` +- `kanton.kontextInformationen: list[Kontext]` +- `gemeinde.kontextInformationen: list[Kontext]` + +#### Beispielthemen (nicht abschliessend) + +| Themenbereich | Beispiele | +|---------------|-----------| +| **Nutzung** | Vorgaben zur Erdgeschossnutzung (Wohnen erlaubt oder Pflicht für Gewerbe) | +| **Rechte** | Dienstbarkeiten (Wegrechte, Nähebaurechte, etc.) | +| **Parkierung** | Anforderung Parkplätze (Berechnung / Reduktionsfaktoren) | +| **Ausnützung** | Ausnützungsübertragungen | +| **Umwelt** | Schadstoffbelastungen auf Parzellen | +| **Planung** | Aktive Gestaltungspläne | +| **Lärm** | Lärmempfindlichkeitsstufen | +| **Energie** | Mögliche Wärmenutzung (Wärmeverbundnetze; Fernwärme, Anergie) | +| **Natur** | Baumbestand auf privaten Grundstücken | +| **Schutz** | Isos (Ortsbild, Schutzstatus, Denkmalschutz, Weilergebiet, etc.) | +| **Gefahren** | Naturgefahren (z.B. Objektschutzmassnahmen (Hochwasser)) | +| **Revision** | Verweis auf aktuell in oder zukünftig in Revision befindlichen Normen/Gesetze (z.B. Revision PBG mit aktuell negativer Vorwirkung) | + +**Design-Rationale:** +Das Kontext-Objekt ermöglicht flexibles Hinzufügen von projektspezifischen, parzellen-spezifischen oder regionalen Informationen ohne Schemaänderungen. + +--- + +### 7. Hilfsentitäten & Enumerationen + +#### JaNein (Enum) +**Drei-wertiger Zustand für optionale Ja/Nein-Fragen.** + +| Wert | Bedeutung | +|------|-----------| +| `""` (leer) | Unbekannt / Nicht erfasst | +| `"Ja"` | Ja / Zutreffend | +| `"Nein"` | Nein / Nicht zutreffend | + +**Verwendung:** +- `parzelleBebaut`: Ist die Parzelle bebaut? +- `parzelleErschlossen`: Ist die Parzelle erschlossen? +- `parzelleHanglage`: Liegt die Parzelle in Hanglage? + +--- + +#### StatusProzess (Enum) +**Projektstatus zur Nachverfolgung des Projektfortschritts.** + +| Wert | Beschreibung | +|------|--------------| +| `Eingang` | Projekt wurde eingereicht | +| `Analyse` | Projekt wird analysiert | +| `Studie` | Machbarkeitsstudie läuft | +| `Planung` | Planungsphase | +| `Baurechtsverfahren` | Baubewilligung läuft | +| `Umsetzung` | Bauprojekt in Umsetzung | +| `Archiv` | Projekt abgeschlossen | + +--- + +#### DokumentTyp (Enum) +**Dokumenttyp zur Kategorisierung von Dokumenten, insbesondere für kantonale und kommunale Dokumente.** + +| Wert | Beschreibung | +|------|--------------| +| `kantonBaureglementAktuell` | Aktuelles Baureglement (Kanton) | +| `kantonBaureglementRevision` | Baureglement in Revision (Kanton) | +| `kantonBauverordnungAktuell` | Aktuelle Bauverordnung (Kanton) | +| `kantonBauverordnungRevision` | Bauverordnung in Revision (Kanton) | +| `gemeindeBzoAktuell` | Aktuelle Bau- und Zonenordnung (BZO) (Gemeinde) | +| `gemeindeBzoRevision` | BZO in Revision (Gemeinde) | + +**Verwendung:** +- Dokumente in der `dokumente`-Liste von Kanton oder Gemeinde können über `dokumentTyp` identifiziert werden +- Ermöglicht die Suche nach aktuellen oder in Revision befindlichen Dokumenten + +--- + +## Q & A + +1. **Versionierung**: Sollen Änderungen an Parzellen historisiert werden? --> Vorerst nicht +2. **Mehrsprachigkeit**: Labels in DE/FR/IT? --> Wir im Pydantic Model später umgesetzt +3. **Benutzer & Rollen**: Wer darf was bearbeiten? --> In der App über Roles und Permissions gesteuert +4. **Workflow-Engine**: Für Statusübergänge und Genehmigungen? --> In der App über Workflow-Engine gesteuert +5. **Integration**: Anbindung an amtliche Geodaten (z.B. Swisstopo API)? --> In der App über Integrationen gesteuert +6. **Berechnungen**: Sollen Ausnützungsberechnungen automatisiert werden? --> In der App über Berechnungen gesteuert + +--- + +## Nächste Schritte + +1. **Validierung**: Review mit PEK +2. **Prototyp**: Implementierung der Datenmodell-Klassen +3. **GIS-Integration**: PostGIS aufsetzen, Test-Geodaten importieren +4. **API-Design**: RESTful API (FastAPI) mit OpenAPI-Dokumentation diff --git a/docs/real-estate-feature-integration-guide/01-overview.md b/docs/real-estate-feature-integration-guide/01-overview.md new file mode 100644 index 00000000..f0691689 --- /dev/null +++ b/docs/real-estate-feature-integration-guide/01-overview.md @@ -0,0 +1,169 @@ +# Überblick und Projektstruktur + +[← Zurück zum Inhaltsverzeichnis](README.md) | [Weiter: Datenmodell erstellen →](02-datamodels.md) + +## Überblick + +Das Feature "realEstate" bietet eine **stateless API** für Real Estate-Datenbankoperationen mit AI-basierter natürlicher Sprachverarbeitung: + +### Hauptfunktionalität + +- **Natürliche Sprache → CRUD-Operationen**: User-Input wird mit AI analysiert und in entsprechende Datenbankoperationen übersetzt +- **Direkte Datenbankabfragen**: SQL-Queries können direkt ausgeführt werden +- **CRUD-Operationen**: Erstellen, Lesen, Aktualisieren und Löschen von Real Estate-Entitäten +- **PostgreSQL-Datenbankzugriff**: Über den bestehenden DatabaseConnector +- **RESTful API-Endpunkte**: Einfache, stateless Endpunkte ohne Session-Management +- **Benutzerauthentifizierung und Zugriffskontrolle**: Integriert in bestehende UAM-Struktur + +### Architektur-Ansatz + +**Stateless Design:** +- Keine Chat-Sessions notwendig (optional für zukünftige Erweiterungen) +- Jeder Request ist unabhängig +- Direkter Flow: User-Input → AI-Analyse → CRUD-Operation → Ergebnis +- Keine Query-History (kann optional später hinzugefügt werden) + +**AI-Integration:** +- Nutzt `serviceAi` für Intent-Erkennung +- Übersetzt natürliche Sprache in CRUD-Operationen +- Unterstützt CREATE, READ, UPDATE, DELETE, QUERY-Intents + +**Real Estate-Datenmodell-Entitäten:** +- **Kernentitäten**: Projekt, Parzelle +- **Unterstützend**: Dokument, Kontext +- **Geografisch**: GeoPolylinie, GeoPunkt +- **Administrativ**: Land, Kanton, Gemeinde + +### Architektur-Komponenten + +Die Architektur folgt dem Muster bestehender Features: +- **Routes** (`modules/routes/`) - API-Endpunkte (stateless) +- **Features** (`modules/features/`) - Geschäftslogik mit AI-Integration +- **Interfaces** (`modules/interfaces/`) - Datenbankzugriff (CRUD-Operationen) +- **DataModels** (`modules/datamodels/`) - Pydantic-Modelle für Real Estate-Entitäten +- **Services** (`modules/services/`) - AI-Service für Intent-Analyse + +--- + +## Projektstruktur + +``` +gateway/ +├── modules/ +│ ├── routes/ +│ │ └── routeRealEstate.py # NEU: Stateless API-Endpunkte +│ │ ├── POST /api/realestate/command # Natürliche Sprache → CRUD +│ │ └── POST /api/realestate/query # Direkte SQL-Query +│ │ +│ ├── features/ +│ │ └── realEstate/ +│ │ └── mainRealEstate.py # NEU: Feature-Logik mit AI-Integration +│ │ ├── processNaturalLanguageCommand() # Hauptfunktion +│ │ ├── analyzeUserIntent() # AI-basierte Intent-Analyse +│ │ └── executeIntentBasedOperation() # CRUD-Ausführung +│ │ +│ ├── interfaces/ +│ │ ├── interfaceDbRealEstateAccess.py # NEU: Zugriffskontrolle +│ │ ├── interfaceDbRealEstateObjects.py # NEU: CRUD-Interface +│ │ └── interfaceDbRealEstateChatObjects.py # OPTIONAL: Für Session-Support +│ │ +│ ├── datamodels/ +│ │ ├── datamodelRealEstate.py # NEU: Real Estate Datenmodelle +│ │ │ ├── Projekt, Parzelle, Dokument, etc. +│ │ └── datamodelRealEstateChat.py # OPTIONAL: Für Session-Support +│ │ +│ ├── services/ +│ │ └── serviceAi/ # BEREITS VORHANDEN +│ │ └── mainServiceAi.py # Wird für Intent-Analyse genutzt +│ │ +│ └── connectors/ +│ └── connectorDbPostgre.py # BEREITS VORHANDEN +│ +├── app.py # Router-Registrierung +├── env_dev.env # Environment-Konfiguration +└── modules/features/featuresLifecycle.py # Feature-Lifecycle +``` + +### Wichtige Dateien + +**Erforderlich:** +- `routeRealEstate.py` - API-Endpunkte (stateless) +- `mainRealEstate.py` - Feature-Logik mit AI-Integration +- `interfaceDbRealEstateAccess.py` - Zugriffskontrolle +- `interfaceDbRealEstateObjects.py` - CRUD-Interface +- `datamodelRealEstate.py` - Datenmodelle + +**Optional (für zukünftige Session-Unterstützung):** +- `interfaceDbRealEstateChatObjects.py` - Session-Management +- `datamodelRealEstateChat.py` - Session-Modelle + +--- + +## Datenfluss: User-Input → Ergebnis + +### Flow: Natürliche Sprache ohne Session + +``` +1. User sendet Request + POST /api/realestate/command + Body: { "userInput": "Erstelle ein neues Projekt namens 'Hauptstrasse 42'" } + +2. Route empfängt Request + routeRealEstate.process_command() + → Auth: getCurrentUser() + +3. Feature-Logik verarbeitet + mainRealEstate.processNaturalLanguageCommand() + → Services initialisieren: getServices(currentUser, workflow=None) + +4. AI analysiert Intent + analyzeUserIntent(services.ai, userInput) + → services.ai.callAiPlanning(intentPrompt) + → AI gibt zurück: { "intent": "CREATE", "entity": "Projekt", "parameters": {...} } + +5. CRUD-Operation ausführen + executeIntentBasedOperation(intent, entity, parameters) + → getRealEstateInterface(currentUser) + → RealEstateObjects.createProjekt(projekt) + → DatabaseConnector.recordCreate(Projekt) + +6. Ergebnis zurückgeben + HTTP 200 OK + { "success": true, "intent": "CREATE", "result": {...} } +``` + +### Flow: Direkte SQL-Query + +``` +1. User sendet Request + POST /api/realestate/query + Body: { "queryText": "SELECT * FROM Projekt WHERE plz = '8000'" } + +2. Route empfängt Request + routeRealEstate.execute_direct_query() + → Auth: getCurrentUser() + +3. Query ausführen + getChatInterface(currentUser) + → RealEstateChatObjects.executeQuery(queryText) + → DatabaseConnector.executeQuery(sql) + +4. Ergebnis zurückgeben + HTTP 200 OK + { "rows": [...], "columns": [...], "rowCount": 15 } +``` + +--- + +## Vorteile des stateless Ansatzes + +- **Einfachheit**: Kein Session-Management notwendig +- **Performance**: Weniger Datenbank-Operationen pro Request +- **Skalierbarkeit**: Stateless Requests sind einfacher zu skalieren +- **Flexibilität**: Jeder Request ist unabhängig +- **Erweiterbarkeit**: Session-Support kann später optional hinzugefügt werden + +--- + +**Nächster Schritt:** [02-datamodels.md](02-datamodels.md) + diff --git a/docs/real-estate-feature-integration-guide/02-datamodels.md b/docs/real-estate-feature-integration-guide/02-datamodels.md new file mode 100644 index 00000000..376f5121 --- /dev/null +++ b/docs/real-estate-feature-integration-guide/02-datamodels.md @@ -0,0 +1,976 @@ +# Schritt 1: Datenmodell erstellen + +[← Zurück zur Übersicht](README.md) | [Weiter: Interface erstellen →](03-interfaces.md) + +## Real Estate-Datenmodelle + +**Datei:** `modules/datamodels/datamodelRealEstate.py` + +Das Feature arbeitet **stateless** ohne Session-Management. Die Datenmodelle definieren die Struktur der Real Estate-Entitäten, die über die API verwaltet werden können. + +**Hinweis:** Die Real Estate-Datenmodell-Entitäten (Projekt, Parzelle, Dokument, etc.) werden in `datamodelRealEstate.py` definiert. Diese werden direkt über CRUD-Operationen verwaltet, ohne zusätzliche Chat-Interface-Modelle. + +### Warum keine Chat-Interface-Modelle? + +Das Feature arbeitet **stateless** ohne Session-Management. Alle Operationen arbeiten direkt auf den Real Estate-Datenmodellen: + +#### Stateless Design + +- **Keine Session-Modelle**: Keine `RealEstateChatSession` notwendig +- **Keine Query-History**: Queries werden nicht gespeichert (kann optional später hinzugefügt werden) +- **Direkte CRUD-Operationen**: User-Input → AI-Analyse → CRUD → Ergebnis +- **Einfache Architektur**: Weniger Komplexität, bessere Performance + +#### Real Estate-Modelle + +Die Real Estate-Modelle (`Projekt`, `Parzelle`, `Dokument`, etc.): +- Repräsentieren die **tatsächlichen Geschäftsdaten** der Immobilien-Projekte +- Werden über lange Zeiträume gepflegt und verändert +- Haben komplexe Beziehungen zueinander (Projekt → Parzellen → Dokumente) +- Werden direkt über CRUD-Operationen verwaltet + +#### Datenfluss (stateless) + +``` +User Input (natürliche Sprache) + ↓ +AI-Analyse (Intent-Erkennung) + ↓ +CRUD-Operation identifizieren + ↓ +Real Estate-Modelle + ↓ +Datenbank-Operation + ↓ +Ergebnis zurückgeben + (keine Session, keine History) +``` + +--- + +### Stateless vs. Session-basiert + +**Real Estate Feature (stateless):** +- Direkte CRUD-Operationen auf Real Estate-Modellen +- Keine Session-Modelle notwendig +- Keine Query-History +- Einfacher und schneller + +**Chat-System (session-basiert):** +- Verwendet `ChatWorkflow` für komplexe AI-Workflows +- Verwendet `ChatDocument` für Datei-Verknüpfungen +- Session-Management für Multi-Step-Operationen +- Für komplexe Workflows mit Planung und Review + +**Unterschied:** +- Real Estate Feature ist für **einfache CRUD-Operationen** optimiert +- Chat-System ist für **komplexe AI-Workflows** optimiert +- Beide können parallel existieren und für verschiedene Use Cases genutzt werden + +--- + +### Warum nicht das bestehende `ChatWorkflow` verwenden? + +Sie fragen sich vielleicht: **Kann ich nicht einfach das bestehende `ChatWorkflow` aus `datamodelChat.py` verwenden?** + +Die kurze Antwort: **Für stateless CRUD-Operationen ist `ChatWorkflow` zu komplex**. Das Real Estate Feature arbeitet ohne Session-Management und nutzt direkt die Real Estate-Modelle. + +#### Unterschiedliche Anwendungsfälle + +| **Aspekt** | **ChatWorkflow (bestehend)** | **Real Estate Feature (stateless)** | +|------------|------------------------------|-------------------------------------| +| **Zweck** | Komplexe AI-gesteuerte Workflows mit mehreren Tasks/Actions | Einfache CRUD-Operationen | +| **Komplexität** | Hoch: Tasks, Actions, Rounds, Workflow-Modi, Retries | Niedrig: Direkte CRUD-Operationen | +| **Session** | Session-Management für Multi-Step-Workflows | Keine Session, stateless | +| **Verarbeitung** | Multi-Step AI-Workflows mit Planung, Review, Iteration | Direkte CRUD: User-Input → AI-Analyse → CRUD → Ergebnis | +| **Ergebnisse** | `ChatMessage` mit `documents`, `ActionResult` | Direkte CRUD-Ergebnisse (Projekt, Parzelle, etc.) | + +#### Warum `ChatWorkflow` nicht passt: + +1. **Zu komplex**: `ChatWorkflow` hat viele Felder, die für einfache CRUD-Operationen nicht relevant sind +2. **Session-basiert**: `ChatWorkflow` benötigt Session-Management, das wir nicht brauchen +3. **Falsches Abstraktionsniveau**: `ChatWorkflow` ist für komplexe AI-Workflows, Real Estate braucht einfache CRUD-Operationen + +#### Die richtige Lösung: Direkte CRUD-Operationen + +Stattdessen arbeiten wir **direkt** mit den Real Estate-Modellen: + +```python +# Stateless CRUD-Operationen +User Input → AI-Analyse → CRUD-Operation → Ergebnis +# Keine Session, keine History, einfach und schnell +``` + +#### Wann könnte man `ChatWorkflow` verwenden? + +Sie könnten `ChatWorkflow` verwenden, wenn Sie: +- ✅ **Komplexe AI-Workflows** für Real Estate implementieren wollen (z.B. "Analysiere alle Projekte und erstelle einen Bericht") +- ✅ **Multi-Step-Verarbeitung** benötigen (z.B. "Lade Daten → Transformiere → Erstelle Visualisierung") +- ✅ **Planung und Review** brauchen (z.B. "Prüfe alle Parzellen auf Konformität") + +Aber für **einfache CRUD-Operationen** ist der stateless Ansatz die bessere Wahl. + +--- + +## Real Estate-Datenmodell-Implementierung: + +Die Real Estate-Datenmodell-Entitäten müssen separat in `modules/datamodels/datamodelRealEstate.py` implementiert werden. +Siehe `../PEK_datamodel_desc.md` für die vollständige Spezifikation aller Felder und Beziehungen (PEK ist ein Beispiel für eine Real Estate-Firma, das Modell ist aber allgemein verwendbar). + +### Wichtige Hinweise zum Datenmodell + +**Objektmodell vs. Datenbank-Repräsentation:** + +Dieses Dokument beschreibt ein **Objektmodell** für die Arbeit im Code. Es handelt sich **NICHT** um ein relationales Datenbankmodell mit Junction Tables. + +- **Im Code-Modell**: Alle Beziehungen werden als Objektreferenzen oder Listen von Objekten dargestellt (z.B. `dokumente: list[Dokument]`, `parzellen: list[Parzelle]`). +- **Für die Datenbank-Serialisierung**: Bei der Persistierung können Junction Tables verwendet werden, um n:m-Beziehungen in der Datenbank abzubilden. Dies ist jedoch ein Implementierungsdetail der Datenbank-Schicht und gehört nicht zum Hauptmodell. + +**Systemattribute:** + +Alle Datenobjekte haben automatisch die folgenden Systemattribute: +- `_createdAt`: Float (Timestamp UTC) +- `_createdBy`: String (User-ID) +- `_modifiedAt`: Float (Timestamp UTC) +- `_modifiedBy`: String (User-ID) + +**Timestamps:** +- Alle Timestamps sind im Float-Format UTC im Datenmodell gespeichert. +- Die Darstellung im UI erfolgt mit der lokalen Zeitzone des Benutzers. + +**Wichtige Punkte für die Implementierung:** +- Objektbeziehungen wie `parzellen: list[Parzelle]` werden als JSONB in PostgreSQL gespeichert +- Einzelne Objektreferenzen wie `kontextKanton: Optional[str]` werden als String-ID (Foreign Key) gespeichert +- Administrative Hierarchie: `Kanton` benötigt `id_land` (Foreign Key zu Land), `Gemeinde` benötigt `id_kanton` (Foreign Key zu Kanton) +- Alle Entitäten benötigen `mandateId` für Mandaten-Isolation +- Systemattribute (`_createdAt`, `_createdBy`, etc.) werden automatisch vom DatabaseConnector hinzugefügt + +### Datenfluss-Diagramm + +```mermaid +--- +title: Hauptflüsse - Architektur-Planungs-App +--- +flowchart LR + subgraph Admin[Administrative Ebene] + Land[Land
Schweiz] + Kanton[Kanton
z.B. Zürich] + Gemeinde[Gemeinde
z.B. Zürich Stadt] + Land --> Kanton + Kanton --> Gemeinde + end + + subgraph Geo[Geografische Daten] + GeoPolylinie[GeoPolylinie
Linie/Polygon] + GeoPunkt[GeoPunkt
Koordinaten] + GeoPolylinie --> GeoPunkt + end + + subgraph Core[Kern-Business-Logik] + Projekt[Projekt
Bauprojekt] + Parzelle[Parzelle
Grundstück mit
Bauparametern] + Gemeinde --> Parzelle + Projekt --> Parzelle + Projekt --> GeoPolylinie + Parzelle --> GeoPolylinie + end + + subgraph Support[Unterstützende Daten] + Dokument[Dokument
Dateien & URLs] + Kontext[Kontext
Zusatzinfos] + end +``` + +### Übersichtstabelle aller Entitäten + +| Objekt | Typ | Beschreibung | Hauptfelder | +|--------|-----|--------------|-------------| +| **Projekt** | Kernentität | Bauprojekt mit Status und Perimeter | id, label, statusProzess, perimeter, baulinie, parzellen | +| **Parzelle** | Hauptentität | Grundstück mit Bauparametern | id, label, plz, bauzone, AZ, BZ, perimeter, baulinie, laermschutzzone, hochwasserschutzzone, grundwasserschutzzone | +| **Dokument** | Unterstützend | Dateien und URLs mit Versionierung | id, label, dokumentTyp, quelle, mimeType, kategorienTags | +| **Kontext** | Unterstützend | Flexible Zusatzinformationen | id, thema, inhalt | +| **GeoPolylinie** | Hilfsobjekt | Geometrische Linie/Polygon | id, closed, punkte | +| **Land** | Admin | Nationale Ebene | id, label, abk | +| **Kanton** | Admin | Kantonale Ebene mit Baurecht | id, label, id_land, abk | +| **Gemeinde** | Admin | Gemeinde-Ebene mit BZO | id, label, id_kanton, plz | +| **GeoPunkt** | Hilfsobjekt | 3D-Koordinate | koordinatensystem, x, y, z, referenz | +| **GeoTag** | Enum | Geopunkt-Kategorien | K1, K2, K3, Geometer | +| **JaNein** | Enum | Drei-wertiger Status | "", "Ja", "Nein" | +| **StatusProzess** | Enum | Projektstatus | 7 Werte (Eingang, Analyse, Studie, Planung, Baurechtsverfahren, Umsetzung, Archiv) | +| **DokumentTyp** | Enum | Dokumenttyp | 6 Werte (kantonBaureglementAktuell, kantonBaureglementRevision, etc.) | + +### Beispiel: Vollständige Pydantic-Modelle für Real Estate-Entitäten + +Hier ist ein Beispiel, wie die Pydantic-Modelle für die Real Estate-Entitäten aussehen sollten: + +```python +""" +Real Estate data models for Architektur-Planungs-App. +Implements a general Swiss architecture planning data model. +(PEK is one example implementation, but the model is general-purpose) +""" + +from typing import List, Dict, Any, Optional, ForwardRef +from enum import Enum +from pydantic import BaseModel, Field +from modules.shared.attributeUtils import registerModelLabels +from modules.shared.timeUtils import getUtcTimestamp +import uuid + +# ===== Enums ===== + +class StatusProzess(str, Enum): + """Project process status""" + EINGANG = "Eingang" + ANALYSE = "Analyse" + STUDIE = "Studie" + PLANUNG = "Planung" + BAURECHTSVERFAHREN = "Baurechtsverfahren" + UMSETZUNG = "Umsetzung" + ARCHIV = "Archiv" + + +class DokumentTyp(str, Enum): + """Document type for categorization""" + KANTON_BAUREGLEMENT_AKTUELL = "kantonBaureglementAktuell" + KANTON_BAUREGLEMENT_REVISION = "kantonBaureglementRevision" + KANTON_BAUVERORDNUNG_AKTUELL = "kantonBauverordnungAktuell" + KANTON_BAUVERORDNUNG_REVISION = "kantonBauverordnungRevision" + GEMEINDE_BZO_AKTUELL = "gemeindeBzoAktuell" + GEMEINDE_BZO_REVISION = "gemeindeBzoRevision" + + +class JaNein(str, Enum): + """Three-valued state for optional yes/no questions""" + UNBEKANNT = "" # Empty string for unknown/not captured + JA = "Ja" + NEIN = "Nein" + + +class GeoTag(str, Enum): + """Geopoint categories""" + K1 = "K1" # Fixpunkt höchster Genauigkeit + K2 = "K2" # Fixpunkt mittlerer Genauigkeit + K3 = "K3" # Fixpunkt niedriger Genauigkeit + GEOMETER = "Geometer" # Vom Geometer vermessener Punkt + + +# ===== Helper Models (must be defined before main models) ===== + +class GeoPunkt(BaseModel): + """Represents a 3D point with reference.""" + koordinatensystem: str = Field( + description="Coordinate system (e.g. 'LV95', 'EPSG:2056')", + frontend_type="text", + frontend_readonly=False, + frontend_required=True, + ) + x: float = Field( + description="East value (E) [m], typically 2'480'000 - 2'840'000", + frontend_type="number", + frontend_readonly=False, + frontend_required=True, + ) + y: float = Field( + description="North value (N) [m], typically 1'070'000 - 1'300'000", + frontend_type="number", + frontend_readonly=False, + frontend_required=True, + ) + z: Optional[float] = Field( + None, + description="Height above sea level [m]", + frontend_type="number", + frontend_readonly=False, + frontend_required=False, + ) + referenz: Optional[GeoTag] = Field( + None, + description="Point categorization", + frontend_type="select", + frontend_readonly=False, + frontend_required=False, + ) + + +class GeoPolylinie(BaseModel): + """Represents a line or polygon from multiple GeoPunkte.""" + id: str = Field( + default_factory=lambda: str(uuid.uuid4()), + description="Primary key", + ) + closed: bool = Field( + description="Is the GeoPolylinie closed (polygon)?", + frontend_type="boolean", + frontend_readonly=False, + frontend_required=True, + ) + punkte: List[GeoPunkt] = Field( + default_factory=list, + description="List of GeoPunkte forming the GeoPolylinie", + frontend_type="json", + frontend_readonly=False, + frontend_required=True, + ) + + +class Dokument(BaseModel): + """Supporting data object for file and URL management with versioning.""" + id: str = Field( + default_factory=lambda: str(uuid.uuid4()), + description="Primary key", + frontend_type="text", + frontend_readonly=True, + frontend_required=False, + ) + mandateId: str = Field( + description="ID of the mandate this document belongs to", + frontend_type="text", + frontend_readonly=True, + frontend_required=False, + ) + label: str = Field( + description="Document label", + frontend_type="text", + frontend_readonly=False, + frontend_required=True, + ) + versionsbezeichnung: Optional[str] = Field( + None, + description="Version number or designation (e.g. 'v1.0', 'Rev. A')", + frontend_type="text", + frontend_readonly=False, + frontend_required=False, + ) + dokumentTyp: Optional[DokumentTyp] = Field( + None, + description="Document type", + frontend_type="select", + frontend_readonly=False, + frontend_required=False, + ) + dokumentReferenz: str = Field( + description="File path or URL", + frontend_type="text", + frontend_readonly=False, + frontend_required=True, + ) + quelle: Optional[str] = Field( + None, + description="Source of the document", + frontend_type="text", + frontend_readonly=False, + frontend_required=False, + ) + mimeType: Optional[str] = Field( + None, + description="MIME type of the document (e.g. 'application/pdf', 'image/png')", + frontend_type="text", + frontend_readonly=False, + frontend_required=False, + ) + kategorienTags: List[str] = Field( + default_factory=list, + description="Document categorization tags", + frontend_type="json", + frontend_readonly=False, + frontend_required=False, + ) + + +# Beispiel-Kategorien für Dokumente (nicht abschließend): +# - "Kataster Objekte" - Amtliche Vermessung +# - "Kataster Werkeleitungen" - Leitungskataster +# - "Kataster Belastete Standorte" - Altlasten +# - "Kataster Bäume" - Baumkataster +# - "Zonenplan" - Zonenpläne +# - "Planungs- und Baugesetz (PGB)" - Kantonale Baugesetze +# - "Bau- und Zonenordnung (BZO)" - Gemeinde BZO +# - "Parkplatzverordnung" - Parkplatzregelungen +# - "Eigentümerauskunft" - Grundbuch-Auszüge Eigentümer +# - "Grundbuchauszug" - Vollständige Grundbuch-Auszüge +# - "Bauherrschaft" - Dokumente von der Bauherrschaft +# - "Planung" - Planungsdokumente +# +# Hinweis: Aktuelle Dokumente (z.B. aktuelle Baureglemente, BZO) können anhand des +# `dokumentTyp`-Attributs identifiziert werden. Die entsprechenden Dokumente finden sich +# in der `dokumente`-Liste der jeweiligen Entität (Kanton, Gemeinde). + + +class Kontext(BaseModel): + """Supporting data object for flexible additional information.""" + id: str = Field( + default_factory=lambda: str(uuid.uuid4()), + description="Primary key", + ) + thema: str = Field( + description="Theme designation", + frontend_type="text", + frontend_readonly=False, + frontend_required=True, + ) + inhalt: str = Field( + description="Detailed information (text)", + frontend_type="textarea", + frontend_readonly=False, + frontend_required=True, + ) + + +# Beispielthemen für Kontext (nicht abschließend): +# - "Nutzung" - Vorgaben zur Erdgeschossnutzung (Wohnen erlaubt oder Pflicht für Gewerbe) +# - "Rechte" - Dienstbarkeiten (Wegrechte, Nähebaurechte, etc.) +# - "Parkierung" - Anforderung Parkplätze (Berechnung / Reduktionsfaktoren) +# - "Ausnützung" - Ausnützungsübertragungen +# - "Umwelt" - Schadstoffbelastungen auf Parzellen +# - "Planung" - Aktive Gestaltungspläne +# - "Lärm" - Lärmempfindlichkeitsstufen +# - "Energie" - Mögliche Wärmenutzung (Wärmeverbundnetze; Fernwärme, Anergie) +# - "Natur" - Baumbestand auf privaten Grundstücken +# - "Schutz" - Isos (Ortsbild, Schutzstatus, Denkmalschutz, Weilergebiet, etc.) +# - "Gefahren" - Naturgefahren (z.B. Objektschutzmassnahmen (Hochwasser)) +# - "Revision" - Verweis auf aktuell in oder zukünftig in Revision befindlichen Normen/Gesetze +# +# Verwendung: Kontext-Objekte werden als Listen in den jeweiligen Entitäten gespeichert: +# - projekt.kontextInformationen: list[Kontext] +# - parzelle.kontextInformationen: list[Kontext] +# - land.kontextInformationen: list[Kontext] +# - kanton.kontextInformationen: list[Kontext] +# - gemeinde.kontextInformationen: list[Kontext] +# +# Design-Rationale: Das Kontext-Objekt ermöglicht flexibles Hinzufügen von projektspezifischen, +# parzellen-spezifischen oder regionalen Informationen ohne Schemaänderungen. + + +class Land(BaseModel): + """National level administrative entity.""" + id: str = Field( + default_factory=lambda: str(uuid.uuid4()), + description="Primary key", + frontend_type="text", + frontend_readonly=True, + frontend_required=False, + ) + mandateId: str = Field( + description="ID of the mandate", + frontend_type="text", + frontend_readonly=True, + frontend_required=False, + ) + label: str = Field( + description="Country name (e.g. 'Schweiz')", + frontend_type="text", + frontend_readonly=False, + frontend_required=True, + ) + abk: Optional[str] = Field( + None, + description="Abbreviation (e.g. 'CH')", + frontend_type="text", + frontend_readonly=False, + frontend_required=False, + ) + dokumente: List[Dokument] = Field( + default_factory=list, + description="National laws/documents", + frontend_type="json", + frontend_readonly=False, + frontend_required=False, + ) + kontextInformationen: List[Kontext] = Field( + default_factory=list, + description="National context information", + frontend_type="json", + frontend_readonly=False, + frontend_required=False, + ) + + +class Kanton(BaseModel): + """Cantonal level administrative entity.""" + id: str = Field( + default_factory=lambda: str(uuid.uuid4()), + description="Primary key", + frontend_type="text", + frontend_readonly=True, + frontend_required=False, + ) + mandateId: str = Field( + description="ID of the mandate", + frontend_type="text", + frontend_readonly=True, + frontend_required=False, + ) + label: str = Field( + description="Canton name (e.g. 'Zürich')", + frontend_type="text", + frontend_readonly=False, + frontend_required=True, + ) + id_land: Optional[str] = Field( + None, + description="Land ID (Foreign Key) - eindeutiger Link zum Land, in welchem Land der Kanton liegt", + frontend_type="text", + frontend_readonly=False, + frontend_required=False, + ) + abk: Optional[str] = Field( + None, + description="Abbreviation (e.g. 'ZH')", + frontend_type="text", + frontend_readonly=False, + frontend_required=False, + ) + dokumente: List[Dokument] = Field( + default_factory=list, + description="Cantonal documents", + frontend_type="json", + frontend_readonly=False, + frontend_required=False, + ) + kontextInformationen: List[Kontext] = Field( + default_factory=list, + description="Canton-specific context information", + frontend_type="json", + frontend_readonly=False, + frontend_required=False, + ) + + +class Gemeinde(BaseModel): + """Municipal level administrative entity.""" + id: str = Field( + default_factory=lambda: str(uuid.uuid4()), + description="Primary key", + frontend_type="text", + frontend_readonly=True, + frontend_required=False, + ) + mandateId: str = Field( + description="ID of the mandate", + frontend_type="text", + frontend_readonly=True, + frontend_required=False, + ) + label: str = Field( + description="Municipality name (e.g. 'Zürich')", + frontend_type="text", + frontend_readonly=False, + frontend_required=True, + ) + id_kanton: Optional[str] = Field( + None, + description="Kanton ID (Foreign Key) - eindeutiger Link zum Kanton, in welchem Kanton die Gemeinde liegt", + frontend_type="text", + frontend_readonly=False, + frontend_required=False, + ) + plz: Optional[str] = Field( + None, + description="Postal code (for municipalities with multiple PLZ, this can be a main PLZ). Bei Gemeinden mit mehreren Postleitzahlen wird die konkrete PLZ der Parzelle im Attribut `plz` der Parzelle erfasst.", + frontend_type="text", + frontend_readonly=False, + frontend_required=False, + ) + dokumente: List[Dokument] = Field( + default_factory=list, + description="Municipal documents", + frontend_type="json", + frontend_readonly=False, + frontend_required=False, + ) + kontextInformationen: List[Kontext] = Field( + default_factory=list, + description="Municipality-specific context information", + frontend_type="json", + frontend_readonly=False, + frontend_required=False, + ) + + +# ===== Main Models (use ForwardRef for circular references) ===== + +# Forward references for circular dependencies +ParzelleRef = ForwardRef('Parzelle') + + +class Parzelle(BaseModel): + """Represents a plot with all building law properties.""" + id: str = Field( + default_factory=lambda: str(uuid.uuid4()), + description="Primary key", + frontend_type="text", + frontend_readonly=True, + frontend_required=False, + ) + mandateId: str = Field( + description="ID of the mandate", + frontend_type="text", + frontend_readonly=True, + frontend_required=False, + ) + + # Grunddaten + label: str = Field( + description="Plot designation", + frontend_type="text", + frontend_readonly=False, + frontend_required=True, + ) + parzellenAliasTags: List[str] = Field( + default_factory=list, + description="Additional plot names or field names", + frontend_type="json", + frontend_readonly=False, + frontend_required=False, + ) + eigentuemerschaft: Optional[str] = Field( + None, + description="Owner of the plot", + frontend_type="text", + frontend_readonly=False, + frontend_required=False, + ) + strasseNr: Optional[str] = Field( + None, + description="Street and house number", + frontend_type="text", + frontend_readonly=False, + frontend_required=False, + ) + plz: Optional[str] = Field( + None, + description="Postal code of the plot", + frontend_type="text", + frontend_readonly=False, + frontend_required=False, + ) + + # Geografischer Kontext + perimeter: Optional[GeoPolylinie] = Field( + None, + description="Plot boundary as closed GeoPolylinie", + frontend_type="json", + frontend_readonly=False, + frontend_required=False, + ) + baulinie: Optional[GeoPolylinie] = Field( + None, + description="Building line of the plot", + frontend_type="json", + frontend_readonly=False, + frontend_required=False, + ) + kontextLand: Optional[str] = Field( + None, + description="Land ID (Foreign Key)", + frontend_type="text", + frontend_readonly=False, + frontend_required=False, + ) + kontextKanton: Optional[str] = Field( + None, + description="Canton ID (Foreign Key)", + frontend_type="text", + frontend_readonly=False, + frontend_required=False, + ) + kontextGemeinde: Optional[str] = Field( + None, + description="Municipality ID (Foreign Key)", + frontend_type="text", + frontend_readonly=False, + frontend_required=False, + ) + + # Bebauungsparameter + bauzone: Optional[str] = Field( + None, + description="Building zone designation (e.g. W3, WG2, etc.)", + frontend_type="text", + frontend_readonly=False, + frontend_required=False, + ) + az: Optional[float] = Field( + None, + description="Ausnützungsziffer", + frontend_type="number", + frontend_readonly=False, + frontend_required=False, + ) + bz: Optional[float] = Field( + None, + description="Bebauungsziffer", + frontend_type="number", + frontend_readonly=False, + frontend_required=False, + ) + vollgeschossZahl: Optional[int] = Field( + None, + description="Number of allowed full floors", + frontend_type="number", + frontend_readonly=False, + frontend_required=False, + ) + anrechenbarDachgeschoss: Optional[float] = Field( + None, + description="Accountable portion of attic (0.0 - 1.0)", + frontend_type="number", + frontend_readonly=False, + frontend_required=False, + ) + anrechenbarUntergeschoss: Optional[float] = Field( + None, + description="Accountable portion of basement (0.0 - 1.0)", + frontend_type="number", + frontend_readonly=False, + frontend_required=False, + ) + gebaeudehoeheMax: Optional[float] = Field( + None, + description="Maximum building height in meters", + frontend_type="number", + frontend_readonly=False, + frontend_required=False, + ) + + # Abstandsregelungen + regelnGrenzabstand: List[str] = Field( + default_factory=list, + description="Regulations for boundary distance", + frontend_type="json", + frontend_readonly=False, + frontend_required=False, + ) + regelnMehrlaengenzuschlag: List[str] = Field( + default_factory=list, + description="Regulations for additional length surcharge", + frontend_type="json", + frontend_readonly=False, + frontend_required=False, + ) + regelnMehrhoehenzuschlag: List[str] = Field( + default_factory=list, + description="Regulations for additional height surcharge", + frontend_type="json", + frontend_readonly=False, + frontend_required=False, + ) + + # Eigenschaften (Ja/Nein) + parzelleBebaut: Optional[JaNein] = Field( + None, + description="Is the plot built?", + frontend_type="select", + frontend_readonly=False, + frontend_required=False, + ) + parzelleErschlossen: Optional[JaNein] = Field( + None, + description="Is the plot developed?", + frontend_type="select", + frontend_readonly=False, + frontend_required=False, + ) + parzelleHanglage: Optional[JaNein] = Field( + None, + description="Is the plot on a slope?", + frontend_type="select", + frontend_readonly=False, + frontend_required=False, + ) + + # Schutzzonen + laermschutzzone: Optional[str] = Field( + None, + description="Noise protection zone (e.g. 'II')", + frontend_type="text", + frontend_readonly=False, + frontend_required=False, + ) + hochwasserschutzzone: Optional[str] = Field( + None, + description="Flood protection zone (e.g. 'tief')", + frontend_type="text", + frontend_readonly=False, + frontend_required=False, + ) + grundwasserschutzzone: Optional[str] = Field( + None, + description="Groundwater protection zone", + frontend_type="text", + frontend_readonly=False, + frontend_required=False, + ) + + # Beziehungen (stored as JSONB in database) + parzellenNachbarschaft: List[Dict[str, Any]] = Field( + default_factory=list, + description="Neighboring plots (stored as list of Parzelle IDs or full objects)", + frontend_type="json", + frontend_readonly=False, + frontend_required=False, + ) + dokumente: List[Dokument] = Field( + default_factory=list, + description="Plot-specific documents", + frontend_type="json", + frontend_readonly=False, + frontend_required=False, + ) + kontextInformationen: List[Kontext] = Field( + default_factory=list, + description="Plot-specific context information", + frontend_type="json", + frontend_readonly=False, + frontend_required=False, + ) + + +class Projekt(BaseModel): + """Core object representing a construction project.""" + id: str = Field( + default_factory=lambda: str(uuid.uuid4()), + description="Primary key", + frontend_type="text", + frontend_readonly=True, + frontend_required=False, + ) + mandateId: str = Field( + description="ID of the mandate", + frontend_type="text", + frontend_readonly=True, + frontend_required=False, + ) + label: str = Field( + description="Project designation", + frontend_type="text", + frontend_readonly=False, + frontend_required=True, + ) + statusProzess: Optional[StatusProzess] = Field( + None, + description="Project status", + frontend_type="select", + frontend_readonly=False, + frontend_required=False, + ) + perimeter: Optional[GeoPolylinie] = Field( + None, + description="Envelope of all plots in the project", + frontend_type="json", + frontend_readonly=False, + frontend_required=False, + ) + baulinie: Optional[GeoPolylinie] = Field( + None, + description="Building line of the project", + frontend_type="json", + frontend_readonly=False, + frontend_required=False, + ) + parzellen: List[Parzelle] = Field( + default_factory=list, + description="All plots of the project", + frontend_type="json", + frontend_readonly=False, + frontend_required=False, + ) + dokumente: List[Dokument] = Field( + default_factory=list, + description="Project-specific documents", + frontend_type="json", + frontend_readonly=False, + frontend_required=False, + ) + kontextInformationen: List[Kontext] = Field( + default_factory=list, + description="Project-specific context information", + frontend_type="json", + frontend_readonly=False, + frontend_required=False, + ) + + +# Resolve forward references +Parzelle.model_rebuild() +Projekt.model_rebuild() + + +# Register labels for frontend +registerModelLabels( + "Projekt", + {"en": "Project", "fr": "Projet", "de": "Projekt"}, + { + "id": {"en": "ID", "fr": "ID", "de": "ID"}, + "label": {"en": "Label", "fr": "Libellé", "de": "Bezeichnung"}, + "statusProzess": {"en": "Process Status", "fr": "Statut du processus", "de": "Prozessstatus"}, + # ... more labels + }, +) + +# Similar registerModelLabels calls for all other models... +``` + +**Wichtige Hinweise zur Implementierung:** + +1. **Forward References**: Für zirkuläre Referenzen (z.B. `Parzelle` → `parzellenNachbarschaft: list[Parzelle]`) verwenden Sie `ForwardRef` oder speichern Sie nur IDs als Strings. + +2. **JSONB-Speicherung**: Listen von Objekten (`list[Parzelle]`, `list[Dokument]`) werden automatisch als JSONB gespeichert. Der DatabaseConnector erkennt `List`-Typen automatisch. + +3. **Foreign Keys**: Einzelne Objektreferenzen wie `kontextKanton: Optional[str]` werden als String-ID gespeichert. Sie können später im Interface die vollständigen Objekte laden. + +4. **MandateId**: Alle Entitäten benötigen `mandateId` für Mandaten-Isolation. + +5. **Systemattribute**: `_createdAt`, `_createdBy`, `_modifiedAt`, `_modifiedBy` werden automatisch vom DatabaseConnector hinzugefügt - Sie müssen sie nicht im Modell definieren. + +--- + +**WICHTIG:** Die obigen Real Estate-Modelle (`Projekt`, `Parzelle`, etc.) sind die **tatsächlichen Datenmodelle**, die Sie implementieren müssen. Diese werden in `modules/datamodels/datamodelRealEstate.py` erstellt. + +**Keine Chat-Interface-Modelle notwendig:** +- Das Feature arbeitet **stateless** ohne Session-Management +- Alle Operationen arbeiten direkt auf den Real Estate-Modellen +- Keine `RealEstateChatSession`, `RealEstateQuery` oder `RealEstateQueryResult` notwendig +- CRUD-Operationen werden direkt ausgeführt und Ergebnisse direkt zurückgegeben + +### Wichtige Punkte: + +1. **UUID als ID**: Alle Modelle verwenden `uuid.uuid4()` für eindeutige IDs +2. **MandateId**: Jedes Modell benötigt `mandateId` für Mandaten-Isolation +3. **Frontend-Metadaten**: `frontend_type`, `frontend_readonly`, `frontend_required` für UI-Generierung +4. **registerModelLabels**: Registriert Labels für Mehrsprachigkeit +5. **JSONB-Felder**: `Dict[str, Any]` und `List[...]` werden automatisch als JSONB in PostgreSQL gespeichert +6. **Foreign Keys**: Administrative Hierarchie wird über Foreign Keys abgebildet: + - `Kanton.id_land` → `Land.id` + - `Gemeinde.id_kanton` → `Kanton.id` + - `Parzelle.kontextLand` → `Land.id` (Optional) + - `Parzelle.kontextKanton` → `Kanton.id` (Optional) + - `Parzelle.kontextGemeinde` → `Gemeinde.id` (Optional) + +--- + +## Q & A - Häufige Fragen + +1. **Versionierung**: Sollen Änderungen an Parzellen historisiert werden? + → Vorerst nicht + +2. **Mehrsprachigkeit**: Labels in DE/FR/IT? + → Wird im Pydantic Model über `registerModelLabels` umgesetzt + +3. **Benutzer & Rollen**: Wer darf was bearbeiten? + → In der App über Roles und Permissions gesteuert (UAM-System) + +4. **Workflow-Engine**: Für Statusübergänge und Genehmigungen? + → In der App über Workflow-Engine gesteuert (optional, kann später integriert werden) + +5. **Integration**: Anbindung an amtliche Geodaten (z.B. Swisstopo API)? + → In der App über Integrationen gesteuert (optional) + +6. **Berechnungen**: Sollen Ausnützungsberechnungen automatisiert werden? + → In der App über Berechnungen gesteuert (optional) + +--- + +[← Zurück zur Übersicht](README.md) | [Weiter: Interface erstellen →](03-interfaces.md) + diff --git a/docs/real-estate-feature-integration-guide/03-interfaces.md b/docs/real-estate-feature-integration-guide/03-interfaces.md new file mode 100644 index 00000000..a90c31b9 --- /dev/null +++ b/docs/real-estate-feature-integration-guide/03-interfaces.md @@ -0,0 +1,845 @@ +# Schritt 2: Interface erstellen + +[← Zurück: Datenmodell erstellen](02-datamodels.md) | [Weiter: Feature-Logik implementieren →](04-feature-logic.md) + +## Übersicht: Was sind Interfaces? + +**Interfaces** sind aktive Klassen, die den **Datenbankzugriff** implementieren. Sie unterscheiden sich von **Datamodels** (die nur die Datenstruktur definieren): + +| **Aspekt** | **Datamodels** | **Interfaces** | +|------------|----------------|----------------| +| **Zweck** | Definiert **WAS** (Datenstruktur) | Implementiert **WIE** (Datenzugriff) | +| **Inhalt** | Pydantic-Modelle mit Feldern und Validierung | Klassen mit CRUD-Methoden (`create`, `get`, `update`, `delete`) | +| **Beispiel** | `class Projekt(BaseModel): ...` | `def createProjekt(...) -> Projekt: ...` | +| **Aktivität** | Passiv (nur Struktur) | Aktiv (führt Operationen aus) | + +**Analogie:** +- **Datamodel** = Bauplan (beschreibt das Haus) +- **Interface** = Bauunternehmer (baut das Haus) + +--- + +## Struktur: Real Estate CRUD-Interface + +Da das Feature **stateless** arbeitet, benötigen wir nur **ein Interface** für CRUD-Operationen auf Real Estate-Entitäten: + +### Real Estate-Datenmodelle → Real Estate CRUD-Interface + +**Datamodel:** `datamodelRealEstate.py` +- `Projekt` +- `Parzelle` +- `Dokument` +- `Kanton`, `Gemeinde`, `Land` +- `GeoPolylinie`, `GeoPunkt` +- `Kontext` +- etc. + +**Interface:** `interfaceDbRealEstateObjects.py` +- `RealEstateObjects` (Haupt-Interface) +- `RealEstateAccess` (Zugriffskontrolle) +- Methoden: `createProjekt()`, `getParzelle()`, `updateDokument()`, etc. + +**Optional:** `interfaceDbRealEstateChatObjects.py` (nur für direkte SQL-Queries) +- `RealEstateChatObjects` - Für direkte Query-Ausführung ohne Session +- Methoden: `executeQuery()` - Führt SQL direkt aus + +--- + +## Warum nur ein Haupt-Interface? + +1. **Stateless Design**: + - Keine Session-Verwaltung notwendig + - Direkte CRUD-Operationen auf Real Estate-Modellen + +2. **Einfache Architektur**: + - Ein Interface für alle CRUD-Operationen + - Weniger Komplexität, bessere Wartbarkeit + +3. **Optionales Query-Interface**: + - Nur für direkte SQL-Queries (stateless) + - Keine Session-Management-Funktionen + +--- + +## Zu erstellende Dateien + +### Schritt 2a: Real Estate CRUD-Interface (ERFORDERLICH) + +**Zwei separate Dateien** (wie bei anderen Features): + +#### Datei 1: `modules/interfaces/interfaceDbRealEstateAccess.py` + +**Enthält:** +- `RealEstateAccess` - Zugriffskontrolle für Real Estate-Entitäten +- Methoden: `uam()`, `canModify()` + +**Zweck:** Prüft Zugriffsrechte und filtert Daten basierend auf Benutzerprivilegien + +#### Datei 2: `modules/interfaces/interfaceDbRealEstateObjects.py` + +**Enthält:** +- `RealEstateObjects` - Haupt-Interface für CRUD-Operationen +- `getInterface()` - Factory-Funktion +- Nutzt `RealEstateAccess` aus der Access-Datei + +**Zweck:** Verwaltet Real Estate-Entitäten (Projekt, Parzelle, Dokument, etc.) + +**Nutzt:** +- `datamodelRealEstate.py` (Projekt, Parzelle, Dokument, etc.) +- `interfaceDbRealEstateAccess.py` (für Zugriffskontrolle) + +**Wann benötigt:** Für alle CRUD-Operationen auf Real Estate-Entitäten (z.B. Projekte erstellen/bearbeiten, Parzellen verwalten). Dies ist das Haupt-Interface für das Feature. + +--- + +### Schritt 2b: Query-Interface (OPTIONAL - nur für direkte SQL-Queries) + +**Eine Datei** für stateless Query-Ausführung: + +#### Datei: `modules/interfaces/interfaceDbRealEstateChatObjects.py` + +**Enthält:** +- `RealEstateChatObjects` - Interface für direkte SQL-Query-Ausführung +- `getInterface()` - Factory-Funktion +- Methoden: `executeQuery()` - Führt SQL direkt aus (stateless) + +**Zweck:** Direkte SQL-Query-Ausführung ohne Session-Management + +**Nutzt:** +- `connectorDbPostgre.DatabaseConnector` für direkte SQL-Ausführung +- Keine Chat-Modelle (stateless) + +**Wann benötigt:** Nur wenn Sie direkte SQL-Queries ausführen möchten (z.B. für komplexe SELECT-Queries). Für CRUD-Operationen verwenden Sie das Real Estate CRUD-Interface. + +--- + +## Übersicht: Dateien und ihre Beziehungen + +``` +┌─────────────────────────────────────────────────────────────┐ +│ DATAMODELS (Struktur) │ +├─────────────────────────────────────────────────────────────┤ +│ datamodelRealEstate.py │ +│ ├── Projekt │ +│ ├── Parzelle │ +│ ├── Dokument │ +│ ├── Kanton, Gemeinde, Land │ +│ ├── GeoPolylinie, GeoPunkt │ +│ ├── Kontext │ +│ └── ... │ +└─────────────────────────────────────────────────────────────┘ + │ + │ nutzt + ▼ +┌─────────────────────────────────────────────────────────────┐ +│ INTERFACES (Zugriff) │ +├─────────────────────────────────────────────────────────────┤ +│ REAL ESTATE CRUD-INTERFACE (ERFORDERLICH) │ +│ │ +│ interfaceDbRealEstateAccess.py │ +│ └── RealEstateAccess │ +│ ├── uam() │ +│ └── canModify() │ +│ │ +│ interfaceDbRealEstateObjects.py │ +│ ├── RealEstateObjects │ +│ │ ├── createProjekt() │ +│ │ ├── getProjekt() │ +│ │ ├── updateProjekt() │ +│ │ ├── deleteProjekt() │ +│ │ ├── createParzelle() │ +│ │ ├── getParzelle() │ +│ │ └── ... (CRUD für alle Entitäten) │ +│ └── getInterface() │ +│ └── nutzt RealEstateAccess │ +│ │ +│ QUERY-INTERFACE (OPTIONAL - nur für direkte SQL) │ +│ │ +│ interfaceDbRealEstateChatObjects.py │ +│ ├── RealEstateChatObjects │ +│ │ └── executeQuery() # Direkte SQL-Ausführung │ +│ └── getInterface() │ +│ └── Keine Access-Klasse (stateless) │ +└─────────────────────────────────────────────────────────────┘ +``` + +--- + +## Interface-Struktur: Access vs. Objects + +Jedes Interface besteht aus **zwei Klassen**: + +### 1. `*Access` Klasse (Zugriffskontrolle) + +**Zweck:** Prüft, wer was sehen/dürfen darf + +**Methoden:** +- `uam()` - Filtert Daten basierend auf Benutzerprivilegien +- `canModify()` - Prüft, ob Benutzer ändern darf + +**Beispiel:** `RealEstateChatAccess` + +### 2. `*Objects` Klasse (Haupt-Interface) + +**Zweck:** Führt CRUD-Operationen aus + +**Methoden:** +- `create*()` - Erstellt neue Einträge +- `get*()` - Lädt Einträge +- `update*()` - Aktualisiert Einträge +- `delete*()` - Löscht Einträge + +**Nutzt:** `*Access` für Zugriffskontrolle + +**Beispiel:** `RealEstateChatObjects` + +**Warum getrennt?** +- Separation of Concerns: Zugriffskontrolle ist separate Verantwortlichkeit +- Wiederverwendbarkeit: Access-Klasse kann von mehreren Interfaces genutzt werden +- Testbarkeit: Zugriffskontrolle kann unabhängig getestet werden + +--- + +## Implementierung: Real Estate CRUD-Interface + +Das Real Estate CRUD-Interface besteht aus **zwei separaten Dateien**, genau wie bei anderen Features (`interfaceDbAppObjects.py` + `interfaceDbAppAccess.py`). + +### Datei 1: Access-Implementierung + +**Datei:** `modules/interfaces/interfaceDbRealEstateAccess.py` + +```python +""" +Access control for Real Estate interface. +Handles user access management and permission checks. +""" + +import logging +from typing import Dict, Any, List, Optional +from modules.datamodels.datamodelUam import User, UserPrivilege + +logger = logging.getLogger(__name__) + + +class RealEstateAccess: + """ + Access control class for Real Estate interface. + Handles user access management and permission checks. + """ + + def __init__(self, currentUser: User, db): + """Initialize with user context.""" + self.currentUser = currentUser + self.mandateId = currentUser.mandateId + self.userId = currentUser.id + + if not self.mandateId or not self.userId: + raise ValueError("Invalid user context: mandateId and userId are required") + + self.db = db + + def uam(self, model_class: type, recordset: List[Dict[str, Any]]) -> List[Dict[str, Any]]: + """ + Unified user access management function that filters data based on user privileges. + + Args: + model_class: Pydantic model class for the table + recordset: Recordset to filter based on access rules + + Returns: + Filtered recordset with access control attributes + """ + from modules.datamodels.datamodelUam import UserPrivilege + + userPrivilege = self.currentUser.privilege + filtered_records = [] + + # System admins see all records + if userPrivilege == UserPrivilege.SYSADMIN: + filtered_records = recordset + # Admins see records in their mandate + elif userPrivilege == UserPrivilege.ADMIN: + filtered_records = [r for r in recordset if r.get("mandateId", "-") == self.mandateId] + # Regular users see only their records + else: + filtered_records = [ + r for r in recordset + if r.get("mandateId", "-") == self.mandateId and r.get("_createdBy") == self.userId + ] + + # Add access control attributes + for record in filtered_records: + record["_hideView"] = False + record["_hideEdit"] = not self.canModify(model_class, record.get("id")) + record["_hideDelete"] = not self.canModify(model_class, record.get("id")) + + return filtered_records + + def canModify(self, model_class: type, recordId: Optional[str] = None) -> bool: + """Checks if the current user can modify records.""" + from modules.datamodels.datamodelUam import UserPrivilege + + userPrivilege = self.currentUser.privilege + + if userPrivilege == UserPrivilege.SYSADMIN: + return True + + if recordId is not None: + records = self.db.getRecordset(model_class, recordFilter={"id": recordId}) + if not records: + return False + + record = records[0] + + if userPrivilege == UserPrivilege.ADMIN and record.get("mandateId", "-") == self.mandateId: + return True + + if (record.get("mandateId", "-") == self.mandateId and + record.get("_createdBy") == self.userId): + return True + + return False + else: + return True # Regular users can create records +``` + +--- + +### Datei 2: Objects-Implementierung + +**Datei:** `modules/interfaces/interfaceDbRealEstateObjects.py` + +```python +""" +Interface to Real Estate database objects. +Uses PostgreSQL connector for data access with user/mandate filtering. +Handles CRUD operations on Real Estate entities (Projekt, Parzelle, etc.). +""" + +import logging +from typing import Dict, Any, List, Optional +from modules.datamodels.datamodelRealEstate import ( + Projekt, + Parzelle, + Dokument, + Kanton, + Gemeinde, + Land, + GeoPolylinie, + GeoPunkt, + Kontext, + StatusProzess, +) +from modules.datamodels.datamodelUam import User +from modules.connectors.connectorDbPostgre import DatabaseConnector +from modules.shared.configuration import APP_CONFIG +# Import Access-Klasse aus separater Datei +from modules.interfaces.interfaceDbRealEstateAccess import RealEstateAccess + +logger = logging.getLogger(__name__) + +# Singleton factory for Real Estate interfaces +_realEstateInterfaces = {} + + +class RealEstateObjects: + """ + Interface to Real Estate database objects. + Uses PostgreSQL connector for data access with user/mandate filtering. + Handles CRUD operations on Real Estate entities. + """ + + def __init__(self, currentUser: Optional[User] = None): + """Initializes the Real Estate Interface.""" + self.currentUser = currentUser + self.userId = currentUser.id if currentUser else None + self.mandateId = currentUser.mandateId if currentUser else None + self.access = None + + # Initialize database + self._initializeDatabase() + + # Set user context if provided + if currentUser: + self.setUserContext(currentUser) + + def _initializeDatabase(self): + """Initialize PostgreSQL database connection.""" + try: + # Get database configuration from environment + dbHost = APP_CONFIG.get("DB_APP_HOST", "localhost") + dbDatabase = APP_CONFIG.get("DB_APP_DATABASE", "poweron_app") + dbUser = APP_CONFIG.get("DB_APP_USER") + dbPassword = APP_CONFIG.get("DB_APP_PASSWORD_SECRET") + dbPort = int(APP_CONFIG.get("DB_APP_PORT", 5432)) + + # Initialize database connector + self.db = DatabaseConnector( + dbHost=dbHost, + dbDatabase=dbDatabase, + dbUser=dbUser, + dbPassword=dbPassword, + dbPort=dbPort, + userId=self.userId if self.userId else None, + ) + + logger.info(f"Real Estate database connector initialized for database: {dbDatabase}") + except Exception as e: + logger.error(f"Error initializing Real Estate database: {e}") + raise + + def setUserContext(self, currentUser: User): + """Sets the user context for the interface.""" + self.currentUser = currentUser + self.userId = currentUser.id + self.mandateId = currentUser.mandateId + + if not self.userId or not self.mandateId: + raise ValueError("Invalid user context: id and mandateId are required") + + # Initialize access control + self.access = RealEstateAccess(self.currentUser, self.db) + + # Update database context + self.db.updateContext(self.userId) + + # ===== Projekt Methods ===== + + def createProjekt(self, projekt: Projekt) -> Projekt: + """Create a new project.""" + # Ensure mandateId is set + if not projekt.mandateId: + projekt.mandateId = self.mandateId + + # Apply access control + self.access.uam(Projekt, []) + + # Save to database + self.db.recordCreate(Projekt, projekt.model_dump()) + + return projekt + + def getProjekt(self, projektId: str) -> Optional[Projekt]: + """Get a project by ID.""" + records = self.db.getRecordset( + Projekt, + recordFilter={"id": projektId} + ) + + if not records: + return None + + # Apply access control + filtered = self.access.uam(Projekt, records) + + if not filtered: + return None + + return Projekt(**filtered[0]) + + def updateProjekt(self, projektId: str, updateData: Dict[str, Any]) -> Optional[Projekt]: + """Update a project.""" + projekt = self.getProjekt(projektId) + if not projekt: + return None + + # Check if user can modify + if not self.access.canModify(Projekt, projektId): + raise PermissionError(f"User {self.userId} cannot modify project {projektId}") + + # Update fields + for key, value in updateData.items(): + if hasattr(projekt, key): + setattr(projekt, key, value) + + # Save to database + self.db.recordModify(Projekt, projektId, projekt.model_dump()) + + return projekt + + def deleteProjekt(self, projektId: str) -> bool: + """Delete a project.""" + projekt = self.getProjekt(projektId) + if not projekt: + return False + + # Check if user can modify + if not self.access.canModify(Projekt, projektId): + raise PermissionError(f"User {self.userId} cannot delete project {projektId}") + + return self.db.recordDelete(Projekt, projektId) + + # ===== Parzelle Methods ===== + + def createParzelle(self, parzelle: Parzelle) -> Parzelle: + """Create a new plot.""" + if not parzelle.mandateId: + parzelle.mandateId = self.mandateId + + self.access.uam(Parzelle, []) + self.db.recordCreate(Parzelle, parzelle.model_dump()) + + return parzelle + + def getParzelle(self, parzelleId: str) -> Optional[Parzelle]: + """Get a plot by ID.""" + records = self.db.getRecordset( + Parzelle, + recordFilter={"id": parzelleId} + ) + + if not records: + return None + + filtered = self.access.uam(Parzelle, records) + + if not filtered: + return None + + return Parzelle(**filtered[0]) + + def updateParzelle(self, parzelleId: str, updateData: Dict[str, Any]) -> Optional[Parzelle]: + """Update a plot.""" + parzelle = self.getParzelle(parzelleId) + if not parzelle: + return None + + if not self.access.canModify(Parzelle, parzelleId): + raise PermissionError(f"User {self.userId} cannot modify plot {parzelleId}") + + for key, value in updateData.items(): + if hasattr(parzelle, key): + setattr(parzelle, key, value) + + self.db.recordModify(Parzelle, parzelleId, parzelle.model_dump()) + + return parzelle + + def deleteParzelle(self, parzelleId: str) -> bool: + """Delete a plot.""" + parzelle = self.getParzelle(parzelleId) + if not parzelle: + return False + + if not self.access.canModify(Parzelle, parzelleId): + raise PermissionError(f"User {self.userId} cannot delete plot {parzelleId}") + + return self.db.recordDelete(Parzelle, parzelleId) + + # ===== Dokument Methods ===== + + def createDokument(self, dokument: Dokument) -> Dokument: + """Create a new document.""" + if not dokument.mandateId: + dokument.mandateId = self.mandateId + + self.access.uam(Dokument, []) + self.db.recordCreate(Dokument, dokument.model_dump()) + + return dokument + + def getDokument(self, dokumentId: str) -> Optional[Dokument]: + """Get a document by ID.""" + records = self.db.getRecordset( + Dokument, + recordFilter={"id": dokumentId} + ) + + if not records: + return None + + filtered = self.access.uam(Dokument, records) + + if not filtered: + return None + + return Dokument(**filtered[0]) + + # ... weitere CRUD-Methoden für andere Entitäten (Kanton, Gemeinde, Land, etc.) + + +def getInterface(currentUser: User) -> RealEstateObjects: + """ + Factory function to get or create a Real Estate interface instance for a user. + Uses singleton pattern per user. + """ + userKey = f"{currentUser.id}_{currentUser.mandateId}" + + if userKey not in _realEstateInterfaces: + _realEstateInterfaces[userKey] = RealEstateObjects(currentUser) + + return _realEstateInterfaces[userKey] +``` + +## Wichtige Punkte: + +1. **DatabaseConnector**: Nutzt `connectorDbPostgre.DatabaseConnector` für Datenbankzugriff +2. **Access Control**: `RealEstateAccess` implementiert Benutzer- und Mandaten-Filterung +3. **Singleton Pattern**: `getInterface()` erstellt pro User eine Instanz +4. **CRUD-Operationen**: `recordCreate`, `recordModify`, `recordDelete`, `getRecordset` vom Connector +5. **MandateId**: Wird automatisch gesetzt, wenn nicht vorhanden + +--- + +## Schritt 2b: Query-Interface (OPTIONAL - nur für direkte SQL-Queries) + +### Wann benötigt? + +**Kurze Antwort:** Nur wenn Sie **direkte SQL-Queries** ausführen möchten (z.B. für komplexe SELECT-Queries). Für CRUD-Operationen verwenden Sie das Real Estate CRUD-Interface. + +#### Szenario 1: Alles über CRUD-Interface (EMPFOHLEN) + +**Strukturiert und sicher:** + +```python +# User schreibt: "Erstelle Projekt 'Test'" +# Feature-Logik nutzt CRUD-Interface: +realEstateInterface = getRealEstateInterface(currentUser) +projekt = realEstateInterface.createProjekt(Projekt( + mandateId=currentUser.mandateId, # Automatisch gesetzt + label="Test" # Validierung durch Pydantic +)) +``` + +**Vorteile:** +- ✅ **Validierung**: Pydantic-Modelle prüfen alle Felder automatisch +- ✅ **Zugriffskontrolle**: `RealEstateAccess` prüft Berechtigungen +- ✅ **Sicherheit**: Kein SQL-Injection-Risiko +- ✅ **Geschäftslogik**: Automatisches Setzen von Systemfeldern (`_createdBy`, `_createdAt`) +- ✅ **Typsicherheit**: Fehler werden zur Entwicklungszeit erkannt +- ✅ **Wartbarkeit**: Zentrale CRUD-Methoden, einfach zu testen + +#### Szenario 2: Direkte SQL-Queries (OPTIONAL) + +**Nur für komplexe SELECT-Queries:** + +```python +# Für komplexe Queries, die nicht über CRUD-Methoden abgedeckt sind +chatInterface = getChatInterface(currentUser) +results = chatInterface.executeQuery( + "SELECT p.*, COUNT(parz.id) as parzellen_count FROM Projekt p LEFT JOIN Parzelle parz ON parz.projektId = p.id GROUP BY p.id" +) +``` + +**Warnung:** +- ⚠️ **Nur für SELECT-Queries**: Keine INSERT/UPDATE/DELETE über direkte SQL-Queries +- ⚠️ **Validierung erforderlich**: Queries sollten validiert werden +- ⚠️ **SQL-Injection-Risiko**: Immer Parameterisierung verwenden + +#### Empfehlung + +**Sie benötigen das Query-Interface, wenn Sie:** + +- ✅ **Komplexe SELECT-Queries** benötigen (z.B. JOINs, Aggregationen) +- ✅ **Flexible Query-Ausführung** benötigen (nicht über CRUD-Methoden abgedeckt) + +**Sie benötigen es NICHT, wenn Sie:** + +- ✅ **Nur CRUD-Operationen** benötigen (verwenden Sie das CRUD-Interface) +- ✅ **Einfache Queries** haben (können über CRUD-Methoden abgedeckt werden) + +**Für Production-Systeme:** +- **CRUD-Operationen**: Immer über CRUD-Interface +- **Komplexe Queries**: Optional über Query-Interface (nur SELECT) + +### Struktur: Query-Interface (stateless) + +**Eine Datei** für direkte SQL-Query-Ausführung: + +#### Datei: `modules/interfaces/interfaceDbRealEstateChatObjects.py` + +```python +""" +Interface for direct SQL query execution (stateless). +Uses PostgreSQL connector for direct query execution without session management. +""" + +import logging +from typing import Dict, Any, Optional +from modules.datamodels.datamodelUam import User +from modules.connectors.connectorDbPostgre import DatabaseConnector +from modules.shared.configuration import APP_CONFIG + +logger = logging.getLogger(__name__) + +# Singleton factory +_realEstateChatInterfaces = {} + + +class RealEstateChatObjects: + """Interface for direct SQL query execution (stateless).""" + + def __init__(self, currentUser: Optional[User] = None): + """Initialize the Query Interface.""" + self.currentUser = currentUser + self.userId = currentUser.id if currentUser else None + self.mandateId = currentUser.mandateId if currentUser else None + + # Initialize database + self._initializeDatabase() + + # Set user context if provided + if currentUser: + self.setUserContext(currentUser) + + def _initializeDatabase(self): + """Initialize PostgreSQL database connection.""" + try: + dbHost = APP_CONFIG.get("DB_APP_HOST", "localhost") + dbDatabase = APP_CONFIG.get("DB_APP_DATABASE", "poweron_app") + dbUser = APP_CONFIG.get("DB_APP_USER") + dbPassword = APP_CONFIG.get("DB_APP_PASSWORD_SECRET") + dbPort = int(APP_CONFIG.get("DB_APP_PORT", 5432)) + + self.db = DatabaseConnector( + dbHost=dbHost, + dbDatabase=dbDatabase, + dbUser=dbUser, + dbPassword=dbPassword, + dbPort=dbPort, + userId=self.userId if self.userId else None, + ) + + logger.info(f"Real Estate Query database connector initialized for database: {dbDatabase}") + except Exception as e: + logger.error(f"Error initializing Real Estate Query database: {e}") + raise + + def setUserContext(self, currentUser: User): + """Sets the user context for the interface.""" + self.currentUser = currentUser + self.userId = currentUser.id + self.mandateId = currentUser.mandateId + + if not self.userId or not self.mandateId: + raise ValueError("Invalid user context: id and mandateId are required") + + # Update database context + self.db.updateContext(self.userId) + + # ===== Database Query Execution ===== + + def executeQuery(self, queryText: str, parameters: Optional[Dict[str, Any]] = None) -> Dict[str, Any]: + """ + Execute a SQL query directly on the database (stateless). + + WARNING: This method executes raw SQL. Ensure proper validation and sanitization + before calling this method. Consider implementing query whitelisting or + only allowing SELECT statements for production use. + + Args: + queryText: SQL query string (preferably SELECT only) + parameters: Optional parameters for parameterized queries + + Returns: + Dictionary with 'rows' (list of dicts), 'columns' (list of column names), + 'rowCount' (int), and 'executionTime' (float) + """ + import time + + try: + start_time = time.time() + + # Ensure connection is alive + self.db._ensure_connection() + + with self.db.connection.cursor() as cursor: + # Execute query + if parameters: + # Use parameterized query for safety + cursor.execute(queryText, parameters) + else: + cursor.execute(queryText) + + # Fetch results + rows = cursor.fetchall() + + # Convert to list of dictionaries + result_rows = [dict(row) for row in rows] + + # Get column names + columns = [desc[0] for desc in cursor.description] if cursor.description else [] + + execution_time = time.time() - start_time + + return { + "rows": result_rows, + "columns": columns, + "rowCount": len(result_rows), + "executionTime": execution_time, + } + except Exception as e: + logger.error(f"Error executing query: {e}") + raise + + +def getInterface(currentUser: User) -> RealEstateChatObjects: + """ + Factory function to get or create a Query interface instance for a user. + Uses singleton pattern per user. + """ + userKey = f"{currentUser.id}_{currentUser.mandateId}" + + if userKey not in _realEstateChatInterfaces: + _realEstateChatInterfaces[userKey] = RealEstateChatObjects(currentUser) + + return _realEstateChatInterfaces[userKey] +``` + +### Hinweise zur Implementierung + +1. **Stateless**: Keine Session-Management-Funktionen +2. **Nur für Queries**: Primär für SELECT-Queries gedacht +3. **Sicherheit**: Immer Parameterisierung verwenden +4. **Validierung**: Queries sollten validiert werden (z.B. nur SELECT erlauben) + +### Beispiel: Beide Interfaces zusammen nutzen + +```python +from modules.interfaces.interfaceDbRealEstateChatObjects import getInterface as getChatInterface +from modules.interfaces.interfaceDbRealEstateObjects import getInterface as getRealEstateInterface + +# CRUD-Interface für strukturierte Operationen +realEstateInterface = getRealEstateInterface(currentUser) +projekt = realEstateInterface.createProjekt(Projekt( + mandateId=currentUser.mandateId, + label="Neues Projekt" +)) + +# Query-Interface für komplexe SELECT-Queries (optional) +chatInterface = getChatInterface(currentUser) +results = chatInterface.executeQuery( + "SELECT p.*, COUNT(parz.id) as parzellen_count FROM Projekt p LEFT JOIN Parzelle parz ON parz.projektId = p.id GROUP BY p.id" +) +``` + +--- + +## Zusammenfassung: Benötigte Dateien + +### Erforderlich (für CRUD-Operationen): + +1. ✅ `modules/datamodels/datamodelRealEstate.py` + - Real Estate-Datenmodelle (Projekt, Parzelle, Dokument, etc.) + +2. ✅ `modules/interfaces/interfaceDbRealEstateAccess.py` + - Zugriffskontrolle für Real Estate-Entitäten (RealEstateAccess) + +3. ✅ `modules/interfaces/interfaceDbRealEstateObjects.py` + - Real Estate CRUD-Interface (RealEstateObjects) + - Nutzt `interfaceDbRealEstateAccess.py` + - Haupt-Interface für alle CRUD-Operationen + +### Optional (für direkte SQL-Queries): + +4. ⚠️ `modules/interfaces/interfaceDbRealEstateChatObjects.py` + - Query-Interface für direkte SQL-Ausführung (RealEstateChatObjects) + - Stateless, keine Session-Management + - Nur wenn Sie komplexe SELECT-Queries benötigen + +--- + +[← Zurück: Datenmodell erstellen](02-datamodels.md) | [Weiter: Feature-Logik implementieren →](04-feature-logic.md) + + + diff --git a/docs/real-estate-feature-integration-guide/04-feature-logic.md b/docs/real-estate-feature-integration-guide/04-feature-logic.md new file mode 100644 index 00000000..cd0b1758 --- /dev/null +++ b/docs/real-estate-feature-integration-guide/04-feature-logic.md @@ -0,0 +1,780 @@ +# Schritt 3: Feature-Logik implementieren + +[← Zurück: Interface erstellen](03-interfaces.md) | [Weiter: Routen erstellen →](05-routes.md) + +**Datei:** `modules/features/realEstate/mainRealEstate.py` + +Die Feature-Logik enthält die Geschäftslogik für das Feature. Sie wird von den Routen aufgerufen und arbeitet **stateless** ohne Session-Management. + +## Übersicht: Stateless Feature-Logik mit AI-Integration + +Die Feature-Logik verwendet **AI**, um natürliche Sprache direkt in CRUD-Operationen zu übersetzen - ohne Session-Management: + +``` +User Input (natürliche Sprache) + ↓ +AI-Analyse (Intent-Erkennung) + ↓ +CRUD-Operation identifizieren + ↓ +Parameter extrahieren + ↓ +Interface CRUD-Methode aufrufen + ↓ +Datenbank-Operation ausführen + ↓ +Ergebnis zurückgeben (keine Session-Speicherung) +``` + +**Beispiel:** +- User: "Erstelle ein neues Projekt namens 'Hauptstrasse 42'" +- AI analysiert → Intent: CREATE, Entity: Projekt, Parameter: {label: "Hauptstrasse 42"} +- Feature-Logik ruft auf → `interface.createProjekt(Projekt(label="Hauptstrasse 42"))` +- Ergebnis wird direkt zurückgegeben (keine Session, keine History) + +## AI-Integration: Services initialisieren + +Um AI zu verwenden, müssen Sie die **Services** initialisieren. Services sind eine zentrale Schnittstelle zu verschiedenen Systemkomponenten (AI, Chat, Database, etc.). + +### Services-Initialisierung + +```python +from modules.services import getInterface as getServices + +# Services für einen User erhalten +services = getServices(currentUser, workflow=None) + +# AI-Service verfügbar über: +aiService = services.ai # Für AI-Aufrufe +``` + +**Wichtig:** Services werden normalerweise im Feature-Logik-Modul initialisiert und an Funktionen weitergegeben. + +--- + +## AI-basierte Intent-Erkennung und CRUD-Operationen + +### Schritt 1: Intent-Analyse mit AI + +Die AI analysiert User-Input und identifiziert: +- **Intent**: CREATE, READ, UPDATE, DELETE, QUERY +- **Entity**: Projekt, Parzelle, Dokument, etc. +- **Parameter**: Extrahierte Werte aus dem User-Input + +### Schritt 2: CRUD-Operation ausführen + +Basierend auf der AI-Analyse wird die entsprechende Interface-Methode aufgerufen. + +--- + +## Beispiel-Implementierung: + +```python +""" +Real Estate feature main logic. +Handles chat interface for database queries with AI-powered natural language processing. +""" + +import logging +import json +from typing import Optional, Dict, Any, List +from modules.datamodels.datamodelUam import User +from modules.datamodels.datamodelRealEstate import ( + Projekt, + Parzelle, + StatusProzess, +) +from modules.interfaces.interfaceDbRealEstateChatObjects import getInterface as getChatInterface +from modules.interfaces.interfaceDbRealEstateObjects import getInterface as getRealEstateInterface +from modules.services import getInterface as getServices + +logger = logging.getLogger(__name__) + + +# ===== Direkte Query-Ausführung (stateless) ===== + +async def executeDirectQuery( + currentUser: User, + queryText: str, + parameters: Optional[Dict[str, Any]] = None, +) -> Dict[str, Any]: + """ + Execute a database query directly without session management. + + Args: + currentUser: Current authenticated user + queryText: SQL query text + parameters: Optional parameters for parameterized queries + + Returns: + Dictionary containing query result (rows, columns, rowCount) + + Note: + - No session or query history is saved + - Query is executed directly and result is returned + - For production, validate and sanitize queries before execution + """ + try: + chatInterface = getChatInterface(currentUser) + + # Execute query directly (no session tracking) + result = chatInterface.executeQuery(queryText, parameters) + + logger.info( + f"Query executed successfully: {result['rowCount']} rows in {result.get('executionTime', 0):.3f}s" + ) + + return { + "status": "success", + "rows": result["rows"], + "columns": result["columns"], + "rowCount": result["rowCount"], + "executionTime": result.get("executionTime", 0), + } + + except Exception as e: + logger.error(f"Error executing query: {str(e)}") + raise + + +# ===== AI-basierte Intent-Erkennung und CRUD-Operationen ===== + +async def processNaturalLanguageCommand( + currentUser: User, + userInput: str, +) -> Dict[str, Any]: + """ + Process natural language user input and execute corresponding CRUD operations. + + Uses AI to analyze user intent and extract parameters, then executes the appropriate + CRUD operation through the interface. Works stateless without session management. + + Args: + currentUser: Current authenticated user + userInput: Natural language command from user + + Returns: + Dictionary containing operation result and metadata + + Example user inputs: + - "Erstelle ein neues Projekt namens 'Hauptstrasse 42'" + - "Zeige mir alle Projekte in Zürich" + - "Aktualisiere Projekt XYZ mit Status 'Planung'" + - "Lösche Parzelle ABC" + - "SELECT * FROM Projekt WHERE plz = '8000'" + """ + try: + # Initialize services for AI access + services = getServices(currentUser, workflow=None) + aiService = services.ai + + # Step 1: Analyze user intent with AI + intentAnalysis = await analyzeUserIntent(aiService, userInput) + + logger.info(f"Intent analysis result: {intentAnalysis}") + + # Step 2: Execute CRUD operation based on intent + result = await executeIntentBasedOperation( + currentUser=currentUser, + intent=intentAnalysis["intent"], + entity=intentAnalysis["entity"], + parameters=intentAnalysis["parameters"], + ) + + return { + "success": True, + "intent": intentAnalysis["intent"], + "entity": intentAnalysis["entity"], + "result": result, + } + + except Exception as e: + logger.error(f"Error processing natural language command: {str(e)}") + raise + + +async def analyzeUserIntent( + aiService, + userInput: str +) -> Dict[str, Any]: + """ + Use AI to analyze user input and extract intent, entity, and parameters. + + Args: + aiService: AI service instance + userInput: Natural language user input + + Returns: + Dictionary with 'intent', 'entity', and 'parameters' + """ + # Create a structured prompt for intent analysis + intentPrompt = f""" +Analyze the following user command and extract the intent, entity, and parameters. + +User Command: "{userInput}" + +Available intents: +- CREATE: User wants to create a new entity +- READ: User wants to read/query entities +- UPDATE: User wants to update an existing entity +- DELETE: User wants to delete an entity +- QUERY: User wants to execute a database query + +Available entities: +- Projekt: Real estate project +- Parzelle: Plot/parcel +- Dokument: Document +- Kanton: Canton +- Gemeinde: Municipality + +Return a JSON object with the following structure: +{{ + "intent": "CREATE|READ|UPDATE|DELETE|QUERY", + "entity": "Projekt|Parzelle|Dokument|Kanton|Gemeinde|null", + "parameters": {{ + // Extracted parameters from user input + // For CREATE/UPDATE: include all relevant fields + // For READ: include filter criteria + // For DELETE: include entity ID if mentioned + // For QUERY: include query text or natural language query + }}, + "confidence": 0.0-1.0 // Confidence score for the analysis +}} + +Examples: +- Input: "Erstelle ein neues Projekt namens 'Hauptstrasse 42'" + Output: {{"intent": "CREATE", "entity": "Projekt", "parameters": {{"label": "Hauptstrasse 42"}}, "confidence": 0.95}} + +- Input: "Zeige mir alle Projekte" + Output: {{"intent": "READ", "entity": "Projekt", "parameters": {{}}, "confidence": 0.9}} + +- Input: "SELECT * FROM Projekt WHERE plz = '8000'" + Output: {{"intent": "QUERY", "entity": null, "parameters": {{"queryText": "SELECT * FROM Projekt WHERE plz = '8000'", "queryType": "sql"}}, "confidence": 1.0}} +""" + + try: + # Use AI planning call for structured JSON response + response = await aiService.callAiPlanning( + prompt=intentPrompt, + debugType="intentanalysis" + ) + + # Parse JSON response + intentData = json.loads(response) + + # Validate response structure + if "intent" not in intentData or "entity" not in intentData: + raise ValueError("Invalid intent analysis response structure") + + return intentData + + except json.JSONDecodeError as e: + logger.error(f"Failed to parse AI intent analysis response: {e}") + logger.error(f"Raw response: {response}") + raise ValueError(f"AI returned invalid JSON: {str(e)}") + except Exception as e: + logger.error(f"Error analyzing user intent: {str(e)}") + raise + + +async def executeIntentBasedOperation( + currentUser: User, + intent: str, + entity: Optional[str], + parameters: Dict[str, Any], +) -> Dict[str, Any]: + """ + Execute CRUD operation based on analyzed intent. + + Args: + currentUser: Current authenticated user + intent: Intent from AI analysis (CREATE, READ, UPDATE, DELETE, QUERY) + entity: Entity type from AI analysis + parameters: Extracted parameters from AI analysis + + Returns: + Operation result + """ + try: + if intent == "QUERY": + # Execute database query directly (stateless) + queryText = parameters.get("queryText", "") + + result = await executeDirectQuery( + currentUser=currentUser, + queryText=queryText, + parameters=parameters.get("queryParameters"), + ) + return result + + elif intent == "CREATE": + # Create new entity + realEstateInterface = getRealEstateInterface(currentUser) + + if entity == "Projekt": + projekt = Projekt( + mandateId=currentUser.mandateId, + label=parameters.get("label", ""), + statusProzess=StatusProzess(parameters.get("statusProzess", "EINGANG")) if parameters.get("statusProzess") else None, + ) + created = realEstateInterface.createProjekt(projekt) + return {"operation": "CREATE", "entity": "Projekt", "result": created.model_dump()} + + elif entity == "Parzelle": + parzelle = Parzelle( + mandateId=currentUser.mandateId, + label=parameters.get("label", ""), + # ... weitere Parameter + ) + created = realEstateInterface.createParzelle(parzelle) + return {"operation": "CREATE", "entity": "Parzelle", "result": created.model_dump()} + + else: + raise ValueError(f"CREATE operation not supported for entity: {entity}") + + elif intent == "READ": + # Read entities + realEstateInterface = getRealEstateInterface(currentUser) + + if entity == "Projekt": + # Apply filters from parameters + projektId = parameters.get("id") + if projektId: + projekt = realEstateInterface.getProjekt(projektId) + return {"operation": "READ", "entity": "Projekt", "result": projekt.model_dump() if projekt else None} + else: + # List all projects (with optional filters) + # Note: You may need to implement getProjekte() method + raise NotImplementedError("List operation needs to be implemented") + + else: + raise ValueError(f"READ operation not supported for entity: {entity}") + + elif intent == "UPDATE": + # Update existing entity + realEstateInterface = getRealEstateInterface(currentUser) + + if entity == "Projekt": + projektId = parameters.get("id") + if not projektId: + raise ValueError("UPDATE operation requires entity ID") + + # Get existing projekt + projekt = realEstateInterface.getProjekt(projektId) + if not projekt: + raise ValueError(f"Projekt {projektId} not found") + + # Update fields + updateData = {k: v for k, v in parameters.items() if k != "id"} + updated = realEstateInterface.updateProjekt(projektId, updateData) + return {"operation": "UPDATE", "entity": "Projekt", "result": updated.model_dump()} + + else: + raise ValueError(f"UPDATE operation not supported for entity: {entity}") + + elif intent == "DELETE": + # Delete entity + realEstateInterface = getRealEstateInterface(currentUser) + + if entity == "Projekt": + projektId = parameters.get("id") + if not projektId: + raise ValueError("DELETE operation requires entity ID") + + success = realEstateInterface.deleteProjekt(projektId) + return {"operation": "DELETE", "entity": "Projekt", "success": success} + + else: + raise ValueError(f"DELETE operation not supported for entity: {entity}") + + else: + raise ValueError(f"Unknown intent: {intent}") + + except Exception as e: + logger.error(f"Error executing intent-based operation: {str(e)}") + raise + + +# ===== Erweiterte Query-Funktion mit AI-Unterstützung ===== + +async def executeNaturalLanguageQuery( + currentUser: User, + naturalLanguageQuery: str, +) -> Dict[str, Any]: + """ + Execute a natural language query by translating it to SQL using AI. + + Args: + currentUser: Current authenticated user + naturalLanguageQuery: Natural language query (e.g., "Zeige mir alle Projekte in Zürich") + + Returns: + Query result with metadata (stateless, no session) + """ + try: + services = getServices(currentUser, workflow=None) + aiService = services.ai + + # Step 1: Translate natural language to SQL using AI + sqlQuery = await translateNaturalLanguageToSQL(aiService, naturalLanguageQuery, currentUser.mandateId) + + logger.info(f"Translated '{naturalLanguageQuery}' to SQL: {sqlQuery}") + + # Step 2: Execute the SQL query directly (stateless) + result = await executeDirectQuery( + currentUser=currentUser, + queryText=sqlQuery, + ) + + return result + + except Exception as e: + logger.error(f"Error executing natural language query: {str(e)}") + raise + + +async def translateNaturalLanguageToSQL( + aiService, + naturalLanguageQuery: str, + mandateId: str +) -> str: + """ + Use AI to translate natural language query to SQL. + + Args: + aiService: AI service instance + naturalLanguageQuery: Natural language query + mandateId: User's mandate ID for filtering + + Returns: + SQL query string with mandateId filter applied + """ + translationPrompt = f""" +Translate the following natural language query into a valid PostgreSQL SQL SELECT statement. + +Natural Language Query: "{naturalLanguageQuery}" + +Available tables and their fields: +- Projekt: id, mandateId, label, statusProzess, perimeter, baulinie, parzellen (JSONB), dokumente (JSONB) +- Parzelle: id, mandateId, label, strasseNr, plz, bauzone, az, bz, kontextKanton, kontextGemeinde +- Dokument: id, mandateId, label, dokumentTyp, dokumentReferenz, mimeType +- Kanton: id, mandateId, label, abk +- Gemeinde: id, mandateId, label, plz + +Rules: +1. Always include 'mandateId' filter based on user context (use placeholder {{mandateId}}) +2. Only use SELECT statements (no INSERT, UPDATE, DELETE) +3. Return ONLY the SQL query, no explanations +4. Use proper PostgreSQL syntax +5. For text searches, use ILIKE for case-insensitive matching + +Examples: +- Input: "Zeige mir alle Projekte" + Output: SELECT * FROM Projekt WHERE mandateId = '{{mandateId}}' + +- Input: "Zeige mir alle Parzellen in Zürich" + Output: SELECT p.* FROM Parzelle p JOIN Gemeinde g ON p.kontextGemeinde = g.id WHERE g.label ILIKE '%Zürich%' AND p.mandateId = '{{mandateId}}' + +- Input: "Wie viele Projekte haben Status 'Planung'?" + Output: SELECT COUNT(*) as count FROM Projekt WHERE statusProzess = 'Planung' AND mandateId = '{{mandateId}}' + +Now translate this query: +""" + + try: + # Use AI planning call for SQL generation + response = await aiService.callAiPlanning( + prompt=translationPrompt, + debugType="sqltranslation" + ) + + # Clean response (remove markdown code blocks if present) + sqlQuery = response.strip() + if sqlQuery.startswith("```sql"): + sqlQuery = sqlQuery[6:] + if sqlQuery.startswith("```"): + sqlQuery = sqlQuery[3:] + if sqlQuery.endswith("```"): + sqlQuery = sqlQuery[:-3] + sqlQuery = sqlQuery.strip() + + # Replace placeholder with actual mandateId + sqlQuery = sqlQuery.replace("{{mandateId}}", mandateId) + + return sqlQuery + + except Exception as e: + logger.error(f"Error translating natural language to SQL: {str(e)}") + raise ValueError(f"Failed to translate query: {str(e)}") +``` + +## Wichtige Punkte: + +### 1. Services-Initialisierung + +- **`getServices(currentUser, workflow=None)`** - Initialisiert Services für AI-Zugriff +- **`services.ai`** - Zugriff auf AI-Service für AI-Aufrufe + +### 2. AI-Aufrufe + +- **`callAiPlanning()`** - Für strukturierte JSON-Antworten (Intent-Analyse, SQL-Übersetzung) +- **`callAiText()`** - Für einfache Text-Generierung +- **`callAiDocuments()`** - Für Dokumenten-Verarbeitung + +### 3. Intent-Analyse + +Die AI analysiert User-Input und gibt zurück: +- **Intent**: CREATE, READ, UPDATE, DELETE, QUERY +- **Entity**: Projekt, Parzelle, Dokument, etc. +- **Parameters**: Extrahierte Werte aus dem Input + +### 4. CRUD-Operationen + +Basierend auf der Intent-Analyse: +- **CREATE** → `interface.createProjekt()`, `interface.createParzelle()`, etc. +- **READ** → `interface.getProjekt()`, `interface.getParzelle()`, etc. +- **UPDATE** → `interface.updateProjekt()`, etc. +- **DELETE** → `interface.deleteProjekt()`, etc. +- **QUERY** → `interface.executeQuery()` oder `executeDatabaseQuery()` + +### 5. Natural Language to SQL + +- AI übersetzt natürliche Sprache in SQL-Queries +- Automatische Validierung und Sanitization empfohlen +- MandateId-Filter wird automatisch hinzugefügt + +### 6. Error Handling + +- Umfassendes Error Handling für AI-Aufrufe +- JSON-Parsing mit Fallback +- Logging für Debugging + +--- + +## Beispiel-Verwendung: + +```python +# In einer Route (stateless): +@router.post("/command") +async def process_command( + userInput: str = Body(...), + currentUser: User = Depends(getCurrentUser) +): + result = await processNaturalLanguageCommand( + currentUser=currentUser, + userInput=userInput + ) + return result + +# Direkte Query (stateless): +@router.post("/query") +async def execute_query( + queryText: str = Body(...), + currentUser: User = Depends(getCurrentUser) +): + result = await executeDirectQuery( + currentUser=currentUser, + queryText=queryText + ) + return result +``` + +**User-Input-Beispiele:** +- `"Erstelle ein neues Projekt namens 'Hauptstrasse 42'"` +- `"Zeige mir alle Projekte in Zürich"` +- `"Aktualisiere Projekt XYZ mit Status 'Planung'"` +- `"Wie viele Parzellen haben Bauzone W3?"` + +--- + +## Vollständiger Flow: User-Input → CRUD-Operation (stateless) + +### Beispiel: "Erstelle ein neues Projekt namens 'Hauptstrasse 42'" + +``` +1. User sendet HTTP POST Request + POST /api/realestate/command + Body: {"userInput": "Erstelle ein neues Projekt namens 'Hauptstrasse 42'"} + +2. Route ruft Feature-Logik auf + → processNaturalLanguageCommand(currentUser, userInput) + # Keine Session-ID notwendig! + +3. Feature-Logik initialisiert Services + → services = getServices(currentUser, workflow=None) + → aiService = services.ai + +4. AI analysiert User-Input + → analyzeUserIntent(aiService, userInput) + → AI gibt zurück: + { + "intent": "CREATE", + "entity": "Projekt", + "parameters": {"label": "Hauptstrasse 42"}, + "confidence": 0.95 + } + +5. Feature-Logik führt CRUD-Operation aus + → executeIntentBasedOperation(intent="CREATE", entity="Projekt", ...) + → realEstateInterface = getRealEstateInterface(currentUser) + → projekt = Projekt(mandateId=..., label="Hauptstrasse 42") + → created = realEstateInterface.createProjekt(projekt) + +6. Interface speichert in Datenbank + → DatabaseConnector.recordCreate(Projekt, projekt.model_dump()) + → PostgreSQL INSERT INTO Projekt ... + +7. Ergebnis wird direkt zurückgegeben + → Route gibt HTTP Response zurück + → Keine Session-Speicherung, keine History + → Frontend zeigt Erfolg +``` + +--- + +## AI-Service Methoden im Detail + +### `callAiPlanning()` - Für strukturierte Antworten + +**Verwendung:** Intent-Analyse, SQL-Übersetzung, strukturierte Daten-Extraktion + +```python +response = await aiService.callAiPlanning( + prompt=intentPrompt, + debugType="intentanalysis" # Optional: für Debug-Dateien +) +# Response ist JSON-String, muss geparst werden +intentData = json.loads(response) +``` + +**Vorteile:** +- Optimiert für strukturierte JSON-Antworten +- Verwendet beste Modelle für Planungs-Aufgaben +- Automatisches Debug-File-Writing + +### `callAiText()` - Für einfache Text-Generierung + +**Verwendung:** Text-Generierung, Zusammenfassungen, Erklärungen + +```python +response = await aiService.callAiText( + prompt="Erkläre mir...", + documents=None, # Optional: Dokumente für Kontext + options=AiCallOptions(...) +) +# Response ist direkt Text-String +``` + +### `callAiDocuments()` - Für Dokumenten-Verarbeitung + +**Verwendung:** Dokumenten-Analyse, Extraktion, Generierung mit Dokumenten-Kontext + +```python +response = await aiService.callAiDocuments( + prompt="Analysiere diese Dokumente...", + documents=[ChatDocument(...), ...], + options=AiCallOptions(...), + outputFormat="json" # Optional: Format für Output +) +``` + +--- + +## Best Practices für AI-Integration + +### 1. Prompt-Engineering + +- **Klare Struktur**: Definieren Sie genau, welche Antwort Sie erwarten +- **Beispiele**: Geben Sie Beispiele für bessere Ergebnisse +- **Format**: Spezifizieren Sie das erwartete Format (JSON, SQL, etc.) + +### 2. Error Handling + +- **JSON-Parsing**: Immer try/except für JSON-Parsing +- **Fallback**: Planen Sie Fallback-Strategien bei AI-Fehlern +- **Validierung**: Validieren Sie AI-Antworten vor Verwendung + +### 3. Sicherheit + +- **Query-Validierung**: Validieren Sie SQL-Queries vor Ausführung +- **Parameter-Sanitization**: Sanitizen Sie alle Parameter +- **MandateId-Filter**: Stellen Sie sicher, dass MandateId immer gefiltert wird + +### 4. Performance + +- **Caching**: Cache häufige AI-Antworten wenn möglich +- **Model-Auswahl**: Lassen Sie das System automatisch das beste Modell wählen +- **Async**: Nutzen Sie async/await für nicht-blockierende Operationen + +### 5. Debugging + +- **Debug-Files**: Nutzen Sie `debugType` Parameter für Debug-Dateien +- **Logging**: Loggen Sie alle AI-Aufrufe und Antworten +- **Confidence-Scores**: Nutzen Sie Confidence-Scores für Fehlerbehandlung + +--- + +## Erweiterte Features + +### Schema-Aware Prompting + +Sie können das Datenbank-Schema in Prompts einbinden: + +```python +# Lade Schema-Informationen +schemaInfo = getDatabaseSchema() # Ihre Funktion + +prompt = f""" +Available database schema: +{schemaInfo} + +User query: "{userInput}" +... +""" +``` + +### Context-Aware Operations (Optional) + +Falls Sie später Kontext zwischen Queries benötigen, können Sie optional eine Session verwenden: + +```python +# Optional: Session für Kontext (nur wenn nötig) +# Für stateless Operationen nicht notwendig + +# Falls Session gewünscht: +sessionId = parameters.get("sessionId") # Optional +if sessionId: + previousQueries = interface.getQueries(sessionId=sessionId) + context = "\n".join([q.queryText for q in previousQueries[-5:]]) +else: + context = "" # Kein Kontext bei stateless Operationen + +prompt = f""" +{context if context else ""} +User query: "{userInput}" +... +""" +``` + +### Multi-Step Operations + +Für komplexe Operationen können Sie mehrere AI-Calls machen: + +```python +# Schritt 1: Intent-Analyse +intent = await analyzeUserIntent(aiService, userInput) + +# Schritt 2: Parameter-Validierung +if intent["intent"] == "CREATE": + validatedParams = await validateParameters(aiService, intent["parameters"]) + +# Schritt 3: CRUD-Operation +result = await executeIntentBasedOperation(...) +``` + +--- + +[← Zurück: Interface erstellen](03-interfaces.md) | [Weiter: Routen erstellen →](05-routes.md) + + + diff --git a/docs/real-estate-feature-integration-guide/05-routes.md b/docs/real-estate-feature-integration-guide/05-routes.md new file mode 100644 index 00000000..04908fd7 --- /dev/null +++ b/docs/real-estate-feature-integration-guide/05-routes.md @@ -0,0 +1,332 @@ +# Schritt 4: Routen erstellen + +[← Zurück: Feature-Logik implementieren](04-feature-logic.md) | [Weiter: Router registrieren →](06-router-registration.md) + +**Datei:** `modules/routes/routeRealEstate.py` + +Die Routen definieren die REST-API-Endpunkte für das Feature. Das Feature arbeitet **stateless** ohne Session-Management. + +## Route-Struktur + +``` +/api/realestate/ + ├── POST /command → Natürliche Sprache → CRUD-Operation + └── POST /query → Direkte SQL-Query +``` + +## Beispiel-Implementierung: + +```python +""" +Real Estate routes for the backend API. +Implements stateless endpoints for real estate database operations with AI-powered natural language processing. +""" + +import logging +from typing import Optional, Dict, Any +from fastapi import APIRouter, HTTPException, Depends, Body, Request +from modules.security.auth import limiter, getCurrentUser +from modules.datamodels.datamodelUam import User +from modules.features.realEstate.mainRealEstate import ( + processNaturalLanguageCommand, + executeDirectQuery, +) + +# Configure logger +logger = logging.getLogger(__name__) + +# Create router for real estate endpoints +router = APIRouter( + prefix="/api/realestate", + tags=["Real Estate"], + responses={404: {"description": "Not found"}} +) + + +# ===== Stateless Command Endpoint ===== + +@router.post("/command", response_model=Dict[str, Any]) +@limiter.limit("120/minute") +async def process_command( + request: Request, + userInput: str = Body(..., embed=True, description="Natural language command"), + currentUser: User = Depends(getCurrentUser) +) -> Dict[str, Any]: + """ + Process natural language command and execute corresponding CRUD operation. + + Uses AI to analyze user intent and extract parameters, then executes the appropriate + CRUD operation. Works stateless without session management. + + Example user inputs: + - "Erstelle ein neues Projekt namens 'Hauptstrasse 42'" + - "Zeige mir alle Projekte in Zürich" + - "Aktualisiere Projekt XYZ mit Status 'Planung'" + - "Lösche Parzelle ABC" + - "SELECT * FROM Projekt WHERE plz = '8000'" + + Returns: + { + "success": true, + "intent": "CREATE|READ|UPDATE|DELETE|QUERY", + "entity": "Projekt|Parzelle|...|null", + "result": {...} + } + """ + try: + result = await processNaturalLanguageCommand( + currentUser=currentUser, + userInput=userInput + ) + return result + except ValueError as e: + logger.error(f"Validation error: {str(e)}") + raise HTTPException( + status_code=400, + detail=str(e) + ) + except Exception as e: + logger.error(f"Error processing command: {str(e)}") + raise HTTPException( + status_code=500, + detail=str(e) + ) + + +# ===== Stateless Query Endpoint ===== + +@router.post("/query", response_model=Dict[str, Any]) +@limiter.limit("120/minute") +async def execute_query( + request: Request, + queryText: str = Body(..., embed=True, description="SQL query text"), + parameters: Optional[Dict[str, Any]] = Body(None, embed=True, description="Optional query parameters for parameterized queries"), + currentUser: User = Depends(getCurrentUser) +) -> Dict[str, Any]: + """ + Execute a direct SQL query without session management. + + Executes the query directly and returns the result. No query history is saved. + + WARNING: This endpoint executes raw SQL queries. Ensure proper validation + and sanitization on the frontend. Consider implementing query whitelisting + or only allowing SELECT statements for production use. + + Returns: + { + "status": "success", + "rows": [...], + "columns": [...], + "rowCount": 15, + "executionTime": 0.123 + } + """ + try: + result = await executeDirectQuery( + currentUser=currentUser, + queryText=queryText, + parameters=parameters, + ) + return result + except ValueError as e: + logger.error(f"Validation error: {str(e)}") + raise HTTPException( + status_code=400, + detail=str(e) + ) + except Exception as e: + logger.error(f"Error executing query: {str(e)}") + raise HTTPException( + status_code=500, + detail=str(e) + ) +``` + +## Wichtige Punkte: + +### 1. Stateless Design + +- **Keine Session-Management**: Alle Endpunkte arbeiten stateless +- **Direkte Verarbeitung**: User-Input wird direkt verarbeitet und Ergebnis zurückgegeben +- **Keine History**: Queries werden nicht gespeichert (kann optional später hinzugefügt werden) + +### 2. API-Endpunkte + +**`POST /api/realestate/command`** +- Verarbeitet natürliche Sprache +- Nutzt AI für Intent-Analyse +- Führt CRUD-Operationen aus +- Gibt Ergebnis direkt zurück + +**`POST /api/realestate/query`** +- Führt direkte SQL-Queries aus +- Keine Session notwendig +- Gibt Query-Ergebnis direkt zurück + +### 3. Sicherheit + +- **Rate Limiting**: `@limiter.limit("120/minute")` für API-Schutz +- **Authentication**: `Depends(getCurrentUser)` für alle Endpunkte +- **Query-Validierung**: WICHTIG - Validieren Sie SQL-Queries vor Ausführung +- **MandateId-Filter**: Wird automatisch durch Interfaces angewendet + +### 4. Error Handling + +- Umfassendes Error Handling mit HTTPException +- Unterschiedliche Status-Codes: 400 (Validation), 404 (Not Found), 500 (Server Error) +- Detaillierte Fehlermeldungen für Debugging + +### 5. Response-Struktur + +**Command-Endpunkt:** +```json +{ + "success": true, + "intent": "CREATE", + "entity": "Projekt", + "result": { + "operation": "CREATE", + "entity": "Projekt", + "result": { + "id": "projekt_123", + "label": "Hauptstrasse 42", + ... + } + } +} +``` + +**Query-Endpunkt:** +```json +{ + "status": "success", + "rows": [ + {"id": "...", "label": "...", ...} + ], + "columns": ["id", "label", ...], + "rowCount": 15, + "executionTime": 0.123 +} +``` + +--- + +## Beispiel-Requests + +### Command-Endpunkt + +```bash +# CREATE Operation +POST /api/realestate/command +Content-Type: application/json +Authorization: Bearer + +{ + "userInput": "Erstelle ein neues Projekt namens 'Hauptstrasse 42'" +} + +# READ Operation +POST /api/realestate/command +{ + "userInput": "Zeige mir alle Projekte in Zürich" +} + +# UPDATE Operation +POST /api/realestate/command +{ + "userInput": "Aktualisiere Projekt XYZ mit Status 'Planung'" +} + +# DELETE Operation +POST /api/realestate/command +{ + "userInput": "Lösche Parzelle ABC" +} + +# QUERY Operation (SQL wird erkannt) +POST /api/realestate/command +{ + "userInput": "SELECT * FROM Projekt WHERE plz = '8000'" +} +``` + +### Query-Endpunkt + +```bash +# Direkte SQL-Query +POST /api/realestate/query +Content-Type: application/json +Authorization: Bearer + +{ + "queryText": "SELECT * FROM Projekt WHERE plz = '8000'" +} + +# Parameterized Query +POST /api/realestate/query +{ + "queryText": "SELECT * FROM Projekt WHERE plz = $1", + "parameters": {"$1": "8000"} +} +``` + +--- + +## Flow: Route → Feature-Logik + +### Command-Endpunkt Flow + +``` +POST /api/realestate/command + ↓ +routeRealEstate.process_command() + ↓ +getCurrentUser() # Auth + ↓ +processNaturalLanguageCommand(currentUser, userInput) + ↓ +mainRealEstate.processNaturalLanguageCommand() + ↓ +analyzeUserIntent() → executeIntentBasedOperation() + ↓ +return Dict mit Ergebnis +``` + +### Query-Endpunkt Flow + +``` +POST /api/realestate/query + ↓ +routeRealEstate.execute_query() + ↓ +getCurrentUser() # Auth + ↓ +executeDirectQuery(currentUser, queryText, parameters) + ↓ +mainRealEstate.executeDirectQuery() + ↓ +getChatInterface(currentUser) + ↓ +RealEstateChatObjects.executeQuery(queryText) + ↓ +DatabaseConnector.executeQuery(sql) + ↓ +return Dict mit rows, columns, rowCount +``` + +--- + +## Vorteile des stateless Ansatzes + +- **Einfachheit**: Kein Session-Management notwendig +- **Performance**: Weniger Datenbank-Operationen pro Request +- **Skalierbarkeit**: Stateless Requests sind einfacher zu skalieren +- **Flexibilität**: Jeder Request ist unabhängig +- **Schnell**: Direkte Verarbeitung ohne Overhead + +--- + +[← Zurück: Feature-Logik implementieren](04-feature-logic.md) | [Weiter: Router registrieren →](06-router-registration.md) + + + diff --git a/docs/real-estate-feature-integration-guide/06-router-registration.md b/docs/real-estate-feature-integration-guide/06-router-registration.md new file mode 100644 index 00000000..430c88b2 --- /dev/null +++ b/docs/real-estate-feature-integration-guide/06-router-registration.md @@ -0,0 +1,45 @@ +# Schritt 5: Router registrieren + +[← Zurück: Routen erstellen](05-routes.md) | [Weiter: Environment-Konfiguration →](07-environment.md) + +**Datei:** `app.py` + +Der Router muss in der Hauptanwendung registriert werden. + +## Änderung in app.py: + +```python +# ... existing imports ... + +# Include all routers + +from modules.routes.routeAdmin import router as generalRouter +app.include_router(generalRouter) + +# ... existing routers ... + +from modules.routes.routeChatPlayground import router as chatPlaygroundRouter +app.include_router(chatPlaygroundRouter) + +# NEU: Real Estate Router hinzufügen (Chat-Interface) +from modules.routes.routeRealEstate import router as realEstateRouter +app.include_router(realEstateRouter) + +# NEU: Real Estate Data Router hinzufügen (falls CRUD-API gewünscht) +# from modules.routes.routeRealEstateData import router as realEstateDataRouter +# app.include_router(realEstateDataRouter) + +from modules.routes.routeSecurityLocal import router as localRouter +app.include_router(localRouter) + +# ... rest of routers ... +``` + +**Wichtig**: Die Reihenfolge der Router-Registrierung kann wichtig sein, wenn es Überschneidungen in den Pfaden gibt. Allgemeinere Routen sollten nach spezifischeren Routen kommen. + +--- + +[← Zurück: Routen erstellen](05-routes.md) | [Weiter: Environment-Konfiguration →](07-environment.md) + + + diff --git a/docs/real-estate-feature-integration-guide/07-environment.md b/docs/real-estate-feature-integration-guide/07-environment.md new file mode 100644 index 00000000..73765e61 --- /dev/null +++ b/docs/real-estate-feature-integration-guide/07-environment.md @@ -0,0 +1,50 @@ +# Schritt 6: Environment-Konfiguration + +[← Zurück: Router registrieren](06-router-registration.md) | [Weiter: Feature Lifecycle →](08-lifecycle.md) + +**Datei:** `env_dev.env` + +Für das realEstate-Feature benötigen wir keine zusätzlichen Environment-Variablen, da es die bereits vorhandenen PostgreSQL-Konfigurationen nutzt: + +```env +# PostgreSQL Storage (bereits vorhanden) +DB_APP_HOST=localhost +DB_APP_DATABASE=poweron_app +DB_APP_USER=poweron_dev +DB_APP_PASSWORD_SECRET = DEV_ENC:... +DB_APP_PORT=5432 +``` + +**Optional**: Falls Sie eine separate Datenbank für Real Estate verwenden möchten, können Sie zusätzliche Variablen hinzufügen: + +```env +# Optional: Separate Real Estate Database +DB_REALESTATE_HOST=localhost +DB_REALESTATE_DATABASE=poweron_realestate +DB_REALESTATE_USER=poweron_dev +DB_REALESTATE_PASSWORD_SECRET = DEV_ENC:... +DB_REALESTATE_PORT=5432 +``` + +In diesem Fall müssten Sie die `_initializeDatabase()` Methode im Interface anpassen: + +```python +def _initializeDatabase(self): + """Initialize PostgreSQL database connection.""" + try: + # Use Real Estate specific config if available, otherwise fall back to APP config + dbHost = APP_CONFIG.get("DB_REALESTATE_HOST") or APP_CONFIG.get("DB_APP_HOST", "localhost") + dbDatabase = APP_CONFIG.get("DB_REALESTATE_DATABASE") or APP_CONFIG.get("DB_APP_DATABASE", "poweron_app") + dbUser = APP_CONFIG.get("DB_REALESTATE_USER") or APP_CONFIG.get("DB_APP_USER") + dbPassword = APP_CONFIG.get("DB_REALESTATE_PASSWORD_SECRET") or APP_CONFIG.get("DB_APP_PASSWORD_SECRET") + dbPort = int(APP_CONFIG.get("DB_REALESTATE_PORT") or APP_CONFIG.get("DB_APP_PORT", 5432)) + + # ... rest of initialization ... +``` + +--- + +[← Zurück: Router registrieren](06-router-registration.md) | [Weiter: Feature Lifecycle →](08-lifecycle.md) + + + diff --git a/docs/real-estate-feature-integration-guide/08-lifecycle.md b/docs/real-estate-feature-integration-guide/08-lifecycle.md new file mode 100644 index 00000000..0946e331 --- /dev/null +++ b/docs/real-estate-feature-integration-guide/08-lifecycle.md @@ -0,0 +1,43 @@ +# Schritt 7: Feature Lifecycle (optional) + +[← Zurück: Environment-Konfiguration](07-environment.md) | [Weiter: Datenbank-Schema →](09-database-schema.md) + +**Datei:** `modules/features/featuresLifecycle.py` + +Falls Ihr Feature Hintergrundprozesse oder Initialisierung beim Start benötigt, können Sie diese hier hinzufügen: + +```python +async def start() -> None: + """ Start feature triggers and background managers """ + + # Provide Event User + rootInterface = getRootInterface() + eventUser = rootInterface.getUserByUsername("event") + + # ... existing features ... + + # Feature RealEstate (optional) + # from modules.features.realEstate import mainRealEstate + # mainRealEstate.initializeFeature(eventUser) + # logger.info("Real Estate feature initialized") + + return True + + +async def stop() -> None: + """ Stop feature triggers and background managers """ + + # Feature RealEstate cleanup (optional) + # from modules.features.realEstate import mainRealEstate + # mainRealEstate.cleanupFeature() + # logger.info("Real Estate feature cleaned up") + + return True +``` + +--- + +[← Zurück: Environment-Konfiguration](07-environment.md) | [Weiter: Datenbank-Schema →](09-database-schema.md) + + + diff --git a/docs/real-estate-feature-integration-guide/09-database-schema.md b/docs/real-estate-feature-integration-guide/09-database-schema.md new file mode 100644 index 00000000..bd094003 --- /dev/null +++ b/docs/real-estate-feature-integration-guide/09-database-schema.md @@ -0,0 +1,247 @@ +# Datenbank-Schema + +[← Zurück: Feature Lifecycle](08-lifecycle.md) | [Weiter: Sicherheitshinweise →](10-security.md) + +Die Datenbank-Tabellen werden automatisch vom `DatabaseConnector` erstellt, basierend auf den Pydantic-Modellen: + +## Chat-Interface Tabellen: + +- **RealEstateQuery**: Speichert Abfragen +- **RealEstateQueryResult**: Speichert Abfrageergebnisse (mit JSONB für `rowData`) +- **RealEstateChatSession**: Speichert Chat-Sessions + +## Real Estate-Datenmodell Tabellen: + +Die folgenden Tabellen werden basierend auf den Real Estate-Datenmodell-Entitäten erstellt: + +- **Projekt**: Bauprojekte (mit `parzellen`, `dokumente`, `kontextInformationen` als JSONB) +- **Parzelle**: Grundstücke mit Bauparametern (mit `parzellenNachbarschaft`, `dokumente`, `kontextInformationen` als JSONB) +- **Dokument**: Dateien und URLs +- **Kontext**: Zusatzinformationen +- **GeoPolylinie**: Geometrische Linien/Polygone (mit `punkte` als JSONB) +- **GeoPunkt**: 3D-Koordinaten +- **Land**: Nationale Ebene (mit `dokumente`, `kontextInformationen` als JSONB) +- **Kanton**: Kantonale Ebene (mit `dokumente`, `kontextInformationen` als JSONB) +- **Gemeinde**: Gemeinde-Ebene (mit `dokumente`, `kontextInformationen` als JSONB) + +--- + +## Automatische Tabellenerstellung + +### Wie funktioniert die automatische Tabellenerstellung? + +Der `DatabaseConnector` erstellt Tabellen **automatisch beim ersten Zugriff** auf ein Pydantic-Modell. Sie müssen keine SQL-CREATE-TABLE-Statements manuell schreiben. + +#### 1. Ablauf der Tabellenerstellung: + +``` +1. Code ruft z.B. `db.recordCreate(Projekt, projekt_data)` auf + ↓ +2. DatabaseConnector ruft `_ensureTableExists(Projekt)` auf + ↓ +3. Prüft ob Tabelle "Projekt" existiert (über information_schema) + ↓ +4. Wenn NICHT vorhanden: + → Ruft `_create_table_from_model()` auf + → Extrahiert Felder aus Pydantic-Modell mit `_get_model_fields()` + → Mappt Python-Typen zu SQL-Typen + → Erstellt CREATE TABLE Statement + → Führt SQL aus + → Erstellt Indexes für Foreign Keys +``` + +#### 2. Typ-Mapping (Python → PostgreSQL): + +Der `DatabaseConnector` mappt automatisch Pydantic-Feldtypen zu PostgreSQL-Datentypen: + +| Python/Pydantic Typ | PostgreSQL Typ | Beispiel | +|---------------------|----------------|----------| +| `str` oder `Optional[str]` | `TEXT` | `label: str` → `"label" TEXT` | +| `int` | `INTEGER` | `vollgeschossZahl: int` → `"vollgeschossZahl" INTEGER` | +| `float` | `DOUBLE PRECISION` | `az: float` → `"az" DOUBLE PRECISION` | +| `bool` | `BOOLEAN` | `closed: bool` → `"closed" BOOLEAN` | +| `Dict[str, Any]` oder `dict` | `JSONB` | `parameters: Dict[str, Any]` → `"parameters" JSONB` | +| `List[...]` oder `list` | `JSONB` | `parzellen: List[Parzelle]` → `"parzellen" JSONB` | +| `Optional[Enum]` | `TEXT` | `statusProzess: StatusProzess` → `"statusProzess" TEXT` | + +**Spezielle Felder:** +- Felder mit Namen `*Id` (z.B. `kontextKantonId`) erhalten automatisch einen Index +- Systemfelder werden automatisch hinzugefügt: `_createdAt`, `_createdBy`, `_modifiedAt`, `_modifiedBy` + +#### 3. Beispiel: CREATE TABLE Statement + +Für das `Projekt`-Modell würde automatisch folgendes SQL erstellt: + +```sql +CREATE TABLE IF NOT EXISTS "Projekt" ( + "id" VARCHAR(255) PRIMARY KEY, + "mandateId" TEXT, + "label" TEXT, + "statusProzess" TEXT, + "perimeter" JSONB, + "baulinie" JSONB, + "parzellen" JSONB, + "dokumente" JSONB, + "kontextInformationen" JSONB, + "_createdAt" DOUBLE PRECISION, + "_modifiedAt" DOUBLE PRECISION, + "_createdBy" VARCHAR(255), + "_modifiedBy" VARCHAR(255) +); + +-- Automatisch erstellte Indexes für Foreign Keys: +CREATE INDEX IF NOT EXISTS "idx_Projekt_mandateId" ON "Projekt" ("mandateId"); +``` + +#### 4. Automatische Schema-Migrationen + +**Wichtig:** Der Connector unterstützt **additive Migrationen**: + +- Wenn eine Tabelle bereits existiert, werden **fehlende Spalten automatisch hinzugefügt** +- **Bestehende Spalten werden NICHT gelöscht oder geändert** +- Wenn Sie ein neues Feld zum Pydantic-Modell hinzufügen, wird es beim nächsten Zugriff automatisch als Spalte hinzugefügt + +**Beispiel:** +```python +# Ursprüngliches Modell +class Projekt(BaseModel): + id: str + label: str + statusProzess: Optional[StatusProzess] + +# Später: Neues Feld hinzugefügt +class Projekt(BaseModel): + id: str + label: str + statusProzess: Optional[StatusProzess] + beschreibung: Optional[str] # NEU + +# Beim nächsten recordCreate() wird automatisch ausgeführt: +# ALTER TABLE "Projekt" ADD COLUMN "beschreibung" TEXT +``` + +#### 5. Wann werden Tabellen erstellt? + +Tabellen werden erstellt, wenn Sie **zum ersten Mal** eine der folgenden Operationen ausführen: + +- `db.recordCreate(model_class, data)` - Erstellt Record +- `db.recordUpdate(model_class, recordId, data)` - Aktualisiert Record +- `db.getRecordset(model_class)` - Lädt Records +- `db.getRecord(model_class, recordId)` - Lädt einen Record + +**Beispiel:** +```python +# Beim ersten Aufruf wird die Tabelle "Projekt" automatisch erstellt +interface = getInterface(currentUser) +projekt = interface.createProjekt(label="Mein Projekt") +# → Tabelle "Projekt" wird jetzt in PostgreSQL erstellt +``` + +#### 6. Manuelle Tabellenerstellung (optional) + +Falls Sie Tabellen manuell erstellen möchten (z.B. für Initialisierung), können Sie: + +```python +from modules.connectors.connectorDbPostgre import DatabaseConnector +from modules.datamodels.datamodelRealEstate import Projekt, Parzelle + +# Connector initialisieren +db = DatabaseConnector( + dbHost="localhost", + dbDatabase="poweron_app", + dbUser="poweron_dev", + dbPassword="...", + dbPort=5432 +) + +# Tabellen explizit erstellen +db._ensureTableExists(Projekt) +db._ensureTableExists(Parzelle) +# ... weitere Modelle +``` + +#### 7. Wichtige Hinweise: + +✅ **Automatisch:** +- Tabellenerstellung beim ersten Zugriff +- Spalten-Erstellung basierend auf Pydantic-Feldern +- Index-Erstellung für Foreign Keys (`*Id` Felder) +- Systemfelder (`_createdAt`, etc.) werden automatisch hinzugefügt + +❌ **NICHT automatisch:** +- Foreign Key Constraints (werden nicht erstellt - Sie müssen sie manuell hinzufügen falls gewünscht) +- Unique Constraints (außer PRIMARY KEY auf `id`) +- Check Constraints +- Trigger oder Stored Procedures + +⚠️ **Einschränkungen:** +- **Keine Schema-Änderungen**: Wenn Sie einen Feldtyp ändern (z.B. `str` → `int`), wird die Spalte NICHT automatisch geändert +- **Keine Spalten-Löschung**: Gelöschte Felder im Modell werden nicht aus der Datenbank entfernt +- **Case-Sensitive**: Tabellennamen werden exakt wie der Klassenname verwendet (z.B. `Projekt`, nicht `projekt`) + +#### 8. Beispiel: Vollständiger Ablauf + +```python +# 1. Pydantic-Modell definieren +class Projekt(BaseModel): + id: str = Field(default_factory=lambda: str(uuid.uuid4())) + mandateId: str + label: str + statusProzess: Optional[StatusProzess] + parzellen: List[Parzelle] = Field(default_factory=list) + +# 2. Interface initialisieren (erstellt noch keine Tabellen) +interface = getInterface(currentUser) + +# 3. Ersten Record erstellen (erstellt jetzt die Tabelle!) +projekt = interface.createProjekt( + label="Mein erstes Projekt", + statusProzess=StatusProzess.PLANUNG +) +# → Intern wird ausgeführt: +# 1. _ensureTableExists(Projekt) aufgerufen +# 2. Tabelle "Projekt" existiert nicht → wird erstellt +# 3. CREATE TABLE "Projekt" (...) wird ausgeführt +# 4. Record wird eingefügt + +# 4. Weitere Records können jetzt ohne Tabellenerstellung erstellt werden +projekt2 = interface.createProjekt(label="Zweites Projekt") +# → Tabelle existiert bereits, nur INSERT wird ausgeführt +``` + +--- + +## Zusammenfassung: + +- ✅ **Tabellenname** = Klassenname des Pydantic-Modells (z.B. `Projekt`) +- ✅ **Spalten** = Alle Felder aus dem Pydantic-Modell +- ✅ **Typen** = Automatisch gemappt (str→TEXT, List→JSONB, etc.) +- ✅ **Systemfelder** = Automatisch hinzugefügt (`_createdAt`, `_createdBy`, etc.) +- ✅ **Indexes** = Automatisch für Felder mit `*Id` Suffix +- ✅ **Migrationen** = Additive Migrationen (neue Spalten werden hinzugefügt) +- ⚠️ **Keine Constraints** = Foreign Keys, Unique, Check müssen manuell erstellt werden + +## Beispiel-Abfragen auf Real Estate-Datenmodell: + +```sql +-- Alle Parzellen in einer bestimmten Gemeinde +SELECT * FROM Parzelle WHERE plz = '8000' ORDER BY label; + +-- Projekte mit Status "Planung" +SELECT * FROM Projekt WHERE "statusProzess" = 'Planung'; + +-- Parzellen mit bestimmter Bauzone +SELECT label, az, bz, gebaeudehoeheMax FROM Parzelle WHERE bauzone = 'W3'; + +-- Dokumente eines Projekts +SELECT * FROM Dokument WHERE id IN ( + SELECT unnest(dokumente::jsonb->>'id') FROM Projekt WHERE id = '...' +); +``` + +--- + +[← Zurück: Feature Lifecycle](08-lifecycle.md) | [Weiter: Sicherheitshinweise →](10-security.md) + + + diff --git a/docs/real-estate-feature-integration-guide/10-security.md b/docs/real-estate-feature-integration-guide/10-security.md new file mode 100644 index 00000000..f5ab6b2e --- /dev/null +++ b/docs/real-estate-feature-integration-guide/10-security.md @@ -0,0 +1,90 @@ +# Sicherheitshinweise + +[← Zurück: Datenbank-Schema](09-database-schema.md) | [Weiter: Testing →](11-testing.md) + +## ⚠️ WICHTIG: Query-Validierung + +Die aktuelle Implementierung erlaubt die Ausführung von **rohen SQL-Queries**. Für Produktion sollten Sie: + +1. **Query-Whitelisting**: Nur erlaubte Queries zulassen +2. **Nur SELECT**: Nur SELECT-Statements erlauben (keine INSERT/UPDATE/DELETE) +3. **Parameterized Queries**: Immer Parameterized Queries verwenden +4. **Query-Parsing**: SQL-Parser verwenden zur Validierung +5. **Rate Limiting**: Strikte Rate Limits setzen (bereits implementiert) + +## Beispiel für Query-Validierung: + +```python +def validateQuery(queryText: str) -> bool: + """ + Validate that query is safe to execute. + Only allows SELECT statements on Real Estate data model tables. + """ + query_lower = queryText.strip().lower() + + # Only allow SELECT statements + if not query_lower.startswith('select'): + return False + + # Block dangerous keywords + dangerous_keywords = [ + 'drop', 'delete', 'insert', 'update', 'alter', 'create', + 'truncate', 'grant', 'revoke', 'exec', 'execute', 'call' + ] + for keyword in dangerous_keywords: + if keyword in query_lower: + return False + + # Only allow queries on Real Estate data model tables + allowed_tables = [ + 'projekt', 'parzelle', 'dokument', 'kontext', + 'geopolylinie', 'geopunkt', 'land', 'kanton', 'gemeinde' + ] + + # Check if query references allowed tables + # Simple check - in production, use SQL parser + query_contains_allowed_table = any( + f'from {table}' in query_lower or f'join {table}' in query_lower + for table in allowed_tables + ) + + if not query_contains_allowed_table: + # Allow queries that don't specify table explicitly (might be subqueries) + # But log for review + logger.warning(f"Query does not reference known Real Estate tables: {queryText[:100]}") + + return True +``` + +## Erweiterte Validierung mit SQL-Parser: + +Für Produktion sollten Sie einen SQL-Parser verwenden: + +```python +from sqlparse import parse, tokens + +def validateQueryAdvanced(queryText: str) -> bool: + """Advanced query validation using SQL parser.""" + try: + parsed = parse(queryText)[0] + + # Check statement type + if parsed.get_type() != 'SELECT': + return False + + # Extract table names and validate + # Implementation depends on SQL parser library + # ... + + return True + except Exception as e: + logger.error(f"Query parsing failed: {e}") + return False +``` + +--- + +[← Zurück: Datenbank-Schema](09-database-schema.md) | [Weiter: Testing →](11-testing.md) + + + diff --git a/docs/real-estate-feature-integration-guide/11-testing.md b/docs/real-estate-feature-integration-guide/11-testing.md new file mode 100644 index 00000000..2ec33c3a --- /dev/null +++ b/docs/real-estate-feature-integration-guide/11-testing.md @@ -0,0 +1,51 @@ +# Testing + +[← Zurück: Sicherheitshinweise](10-security.md) | [Weiter: Troubleshooting →](12-troubleshooting.md) + +## Manuelle API-Tests mit curl: + +```bash +# 1. Login (erhalten Sie Token) +curl -X POST "http://localhost:8000/api/local/auth/login" \ + -H "Content-Type: application/x-www-form-urlencoded" \ + -d "username=youruser&password=yourpass" + +# 2. Session erstellen +curl -X POST "http://localhost:8000/api/realestate/sessions" \ + -H "Authorization: Bearer YOUR_TOKEN" \ + -H "Content-Type: application/json" \ + -d '{"title": "Parzellen-Analyse Zürich"}' + +# 3. Query ausführen - Beispiel: Alle Parzellen in Zürich +curl -X POST "http://localhost:8000/api/realestate/sessions/SESSION_ID/queries" \ + -H "Authorization: Bearer YOUR_TOKEN" \ + -H "Content-Type: application/json" \ + -d '{ + "queryText": "SELECT label, plz, bauzone, az, bz, gebaeudehoeheMax FROM Parzelle WHERE plz = ''8000'' ORDER BY label LIMIT 20", + "queryType": "sql" + }' + +# 4. Query ausführen - Beispiel: Projekte mit Status "Planung" +curl -X POST "http://localhost:8000/api/realestate/sessions/SESSION_ID/queries" \ + -H "Authorization: Bearer YOUR_TOKEN" \ + -H "Content-Type: application/json" \ + -d '{ + "queryText": "SELECT id, label, \"statusProzess\" FROM Projekt WHERE \"statusProzess\" = ''Planung''", + "queryType": "sql" + }' + +# 5. Queries abrufen +curl -X GET "http://localhost:8000/api/realestate/sessions/SESSION_ID/queries" \ + -H "Authorization: Bearer YOUR_TOKEN" +``` + +## Swagger UI: + +Nach dem Start der Anwendung können Sie die API unter `http://localhost:8000/docs` testen. + +--- + +[← Zurück: Sicherheitshinweise](10-security.md) | [Weiter: Troubleshooting →](12-troubleshooting.md) + + + diff --git a/docs/real-estate-feature-integration-guide/12-troubleshooting.md b/docs/real-estate-feature-integration-guide/12-troubleshooting.md new file mode 100644 index 00000000..9c7b2475 --- /dev/null +++ b/docs/real-estate-feature-integration-guide/12-troubleshooting.md @@ -0,0 +1,33 @@ +# Troubleshooting + +[← Zurück: Testing](11-testing.md) | [Weiter: Zusammenfassung →](13-summary.md) + +## Problem: Datenbankverbindung schlägt fehl + +**Lösung**: Überprüfen Sie die Environment-Variablen in `env_dev.env`: +- `DB_APP_HOST` +- `DB_APP_DATABASE` +- `DB_APP_USER` +- `DB_APP_PASSWORD_SECRET` +- `DB_APP_PORT` + +## Problem: Tabellen werden nicht erstellt + +**Lösung**: Der Connector erstellt Tabellen beim ersten Zugriff. Stellen Sie sicher, dass: +- Die Datenbank existiert +- Der Benutzer CREATE-Rechte hat +- Die Verbindung erfolgreich ist + +## Problem: Access Denied Fehler + +**Lösung**: Überprüfen Sie: +- User hat gültiges `mandateId` +- User hat entsprechende Privilegien +- Access Control Logik im Interface + +--- + +[← Zurück: Testing](11-testing.md) | [Weiter: Zusammenfassung →](13-summary.md) + + + diff --git a/docs/real-estate-feature-integration-guide/13-summary.md b/docs/real-estate-feature-integration-guide/13-summary.md new file mode 100644 index 00000000..240af342 --- /dev/null +++ b/docs/real-estate-feature-integration-guide/13-summary.md @@ -0,0 +1,136 @@ +# Zusammenfassung + +[← Zurück: Troubleshooting](12-troubleshooting.md) | [← Zurück zur Übersicht](README.md) + +## Dateinamen-Konvention: + +**Wichtig:** Die Dateien sind nach Funktionalität benannt: + +| Datei | Zweck | Enthält | +|-------|-------|---------| +| `datamodelRealEstateChat.py` | Chat-Interface Modelle | `RealEstateQuery`, `RealEstateQueryResult`, `RealEstateChatSession` | +| `datamodelRealEstate.py` | Real Estate-Datenmodelle | `Projekt`, `Parzelle`, `Dokument`, etc. (allgemein verwendbar) | +| `interfaceDbRealEstateChatObjects.py` | Chat-Interface Interface | Methoden für Sessions und Queries | +| `interfaceDbRealEstateObjects.py` | Real Estate CRUD Interface | Methoden für Projekt, Parzelle, etc. (optional) | + +**Hinweis:** Das Modell ist allgemein für alle Real Estate-Firmen verwendbar. PEK ist nur ein Beispiel. + +--- + +## Zu erstellende Dateien: + +1. **`modules/datamodels/datamodelRealEstateChat.py`** (Chat-Interface Modelle) + - Pydantic-Modelle: `RealEstateQuery`, `RealEstateQueryResult`, `RealEstateChatSession` + - Enums: `QueryStatusEnum` + +2. **`modules/datamodels/datamodelRealEstate.py`** (Real Estate-Datenmodell) + - Pydantic-Modelle: `Projekt`, `Parzelle`, `Dokument`, `Kontext`, `GeoPolylinie`, `GeoPunkt`, `Land`, `Kanton`, `Gemeinde` + - Enums: `StatusProzess`, `DokumentTyp`, `JaNein`, `GeoTag` + - Siehe `../PEK_datamodel_desc.md` für vollständige Spezifikation (PEK ist ein Beispiel, das Modell ist allgemein verwendbar) + +3. **`modules/interfaces/interfaceDbRealEstateChatObjects.py`** (Chat-Interface) + - `RealEstateChatObjects` Klasse für Datenbankzugriff (Chat-Sessions, Queries) + - `RealEstateChatAccess` Klasse für Zugriffskontrolle + - `getInterface()` Factory-Funktion + +4. **`modules/interfaces/interfaceDbRealEstateObjects.py`** (NEU - für Real Estate-Datenmodell CRUD) + - `RealEstateObjects` Klasse für CRUD-Operationen auf Real Estate-Entitäten (Projekt, Parzelle, etc.) + - `RealEstateAccess` Klasse für Zugriffskontrolle + - Methoden für Projekt, Parzelle, Dokument, etc. + - **Hinweis:** Diese Datei ist für CRUD-Operationen auf die Real Estate-Entitäten. Das Chat-Interface nutzt `interfaceDbRealEstateChatObjects.py` (siehe Punkt 3). + - **Optional:** Falls Sie eine separate CRUD-API benötigen (das Chat-Interface kann auch direkt SQL-Queries verwenden) + +5. **`modules/features/realEstate/mainRealEstate.py`** + - Feature-Logik-Funktionen: `createSession`, `executeDatabaseQuery`, etc. + +6. **`modules/routes/routeRealEstate.py`** + - FastAPI Router mit allen Endpunkten für Chat-Interface + +7. **`modules/routes/routeRealEstateData.py`** (NEU - für Real Estate-Datenmodell) + - FastAPI Router für CRUD-Operationen auf Real Estate-Entitäten + - Endpunkte für Projekt, Parzelle, Dokument, etc. + - **Optional:** Falls Sie eine separate CRUD-API benötigen (das Chat-Interface kann auch direkt SQL-Queries verwenden) + +## Zu modifizierende Dateien: + +1. **`app.py`** + - Router-Registrierung für `routeRealEstate` hinzufügen (Chat-Interface) + - Router-Registrierung für `routeRealEstateData` hinzufügen (falls CRUD-API gewünscht) + +2. **`env_dev.env`** (optional) + - Separate Datenbank-Konfiguration falls gewünscht + - PostGIS-Konfiguration falls geografische Abfragen benötigt werden + +3. **`modules/features/featuresLifecycle.py`** (optional) + - Feature-Initialisierung falls benötigt + - Initialisierung von Standard-Daten (z.B. Land "Schweiz", Kantone, Gemeinden) + +## Datenmodell-Implementierung: + +**Wichtig:** Bevor Sie das Chat-Interface nutzen können, müssen Sie die Real Estate-Datenmodell-Entitäten implementieren: + +1. **Erstellen Sie `modules/datamodels/datamodelRealEstate.py`** mit allen Entitäten aus `../PEK_datamodel_desc.md` + - **Hinweis:** PEK ist ein Beispiel für eine Real Estate-Firma, aber das Modell ist allgemein verwendbar für alle Real Estate-Firmen +2. **Beachten Sie die Objektbeziehungen**: + - `parzellen: list[Parzelle]` wird als JSONB gespeichert + - `kontextKanton: Kanton` wird als String-ID gespeichert (Foreign Key) +3. **Implementieren Sie die Enums** entsprechend der Spezifikation +4. **Testen Sie die Tabellenerstellung** durch den DatabaseConnector + +--- + +## Nächste Schritte + +1. **Real Estate-Datenmodell-Implementierung**: + - Erstellen Sie die Pydantic-Modelle für alle Real Estate-Entitäten (`Projekt`, `Parzelle`, `Dokument`, `Kontext`, `GeoPolylinie`, `GeoPunkt`, `Land`, `Kanton`, `Gemeinde`) + - Implementieren Sie die Enums (`StatusProzess`, `DokumentTyp`, `JaNein`, `GeoTag`) + - Siehe `../PEK_datamodel_desc.md` für vollständige Spezifikation (PEK ist ein Beispiel, das Modell ist allgemein verwendbar) + +2. **Query-Validierung implementieren**: Siehe [Sicherheitshinweise](10-security.md) + - Besonders wichtig für Real Estate-Datenmodell: Nur SELECT-Statements erlauben + - Whitelist für erlaubte Tabellen (Projekt, Parzelle, etc.) + +3. **Natural Language Processing**: + - Implementieren Sie NLP für `queryType="natural"` + - Beispiele: "Zeige mir alle Parzellen in Zürich" → SQL-Query + - Nutzen Sie AI-Modelle zur SQL-Generierung aus natürlicher Sprache + +4. **Geografische Abfragen**: + - PostGIS-Integration für räumliche Abfragen + - Beispiel: "Zeige alle Parzellen innerhalb eines bestimmten Perimeters" + - Nutzung von GeoPolylinie und GeoPunkt für GIS-Funktionen + +5. **Query-History**: Erweiterte Historie-Funktionen + - Speichern häufig verwendeter Queries + - Query-Templates für häufige Abfragen (z.B. "Parzellen nach Bauzone") + +6. **Export-Funktionen**: CSV/Excel-Export von Ergebnissen + - Export von Parzellen-Listen + - Export von Projekt-Übersichten + +7. **Caching**: Query-Ergebnisse cachen für wiederholte Abfragen + - Besonders für administrative Daten (Land, Kanton, Gemeinde) + +8. **Permissions**: Erweiterte Berechtigungen für bestimmte Tabellen + - Mandaten-basierte Filterung für Projekte und Parzellen + - Rollen-basierte Zugriffe (z.B. nur Leserechte für bestimmte Benutzer) + +--- + +## Architektur-Zusammenfassung + +Dieses Feature folgt dem etablierten Muster des Projekts: +- **Separation of Concerns**: Routes → Features → Interfaces → Connectors +- **Dependency Injection**: Interfaces werden über Factory-Funktionen erstellt +- **Access Control**: Mandaten- und Benutzer-basierte Filterung +- **Type Safety**: Pydantic-Modelle für Validierung +- **Async Support**: Asynchrone Verarbeitung für Skalierbarkeit + +Die Implementierung ist modular und erweiterbar. Sie können weitere Funktionen hinzufügen, ohne die bestehende Struktur zu ändern. + +--- + +[← Zurück: Troubleshooting](12-troubleshooting.md) | [← Zurück zur Übersicht](README.md) + + + diff --git a/docs/real-estate-feature-integration-guide/README.md b/docs/real-estate-feature-integration-guide/README.md new file mode 100644 index 00000000..95b0346d --- /dev/null +++ b/docs/real-estate-feature-integration-guide/README.md @@ -0,0 +1,42 @@ +# Feature Integration Guide: realEstate + +Diese Dokumentation erklärt Schritt für Schritt, wie Sie ein neues Feature "realEstate" in das Gateway-Projekt integrieren. Das Feature ermöglicht es, über ein Chat-Interface Datenbankabfragen auf Real Estate-Daten (Architektur-Planungs-App) durchzuführen. + +**Referenz:** Das zugrundeliegende Datenmodell ist in `../PEK_datamodel_desc.md` beschrieben (PEK ist ein Beispiel für eine Real Estate-Firma, das Modell ist aber allgemein verwendbar). + +## Inhaltsverzeichnis + +1. [Überblick und Projektstruktur](01-overview.md) +2. [Schritt 1: Datenmodell erstellen](02-datamodels.md) +3. [Schritt 2: Interface erstellen](03-interfaces.md) +4. [Schritt 3: Feature-Logik implementieren](04-feature-logic.md) +5. [Schritt 4: Routen erstellen](05-routes.md) +6. [Schritt 5: Router registrieren](06-router-registration.md) +7. [Schritt 6: Environment-Konfiguration](07-environment.md) +8. [Schritt 7: Feature Lifecycle (optional)](08-lifecycle.md) +9. [Datenbank-Schema und Tabellenerstellung](09-database-schema.md) +10. [Sicherheitshinweise](10-security.md) +11. [Testing](11-testing.md) +12. [Troubleshooting](12-troubleshooting.md) +13. [Zusammenfassung](13-summary.md) + +--- + +## Schnellstart + +Für eine schnelle Übersicht über alle zu erstellenden Dateien, siehe [Zusammenfassung](13-summary.md). + +## Architektur-Überblick + +Die Architektur folgt dem Muster bestehender Features wie `chatPlayground`: +- **Routes** (`modules/routes/`) - API-Endpunkte +- **Features** (`modules/features/`) - Geschäftslogik +- **Interfaces** (`modules/interfaces/`) - Datenbankzugriff +- **DataModels** (`modules/datamodels/`) - Pydantic-Modelle + +--- + +**Nächster Schritt:** [01-overview.md](01-overview.md) + + + diff --git a/env_dev.env b/env_dev.env index da72e528..df0de269 100644 --- a/env_dev.env +++ b/env_dev.env @@ -29,6 +29,13 @@ DB_MANAGEMENT_USER=poweron_dev DB_MANAGEMENT_PASSWORD_SECRET = DEV_ENC:Z0FBQUFBQm8xSUpEUldqSTVpUnFqdGhITDYzT3RScGlMYVdTMmZhOXdudDRCc3dhdllOd3l6MS1vWHY2MjVsTUF1Sk9saEJOSk9ONUlBZjQwb2c2T1gtWWJhcXFzVVVXd01xc0U0b0lJX0JyVDRxaDhNS01JcWs9 DB_MANAGEMENT_PORT=5432 +# PostgreSQL Storage (new) +DB_REALESTATE_HOST=localhost +DB_REALESTATE_DATABASE=poweron_realestate +DB_REALESTATE_USER=poweron_dev +DB_REALESTATE_PASSWORD_SECRET = DEV_ENC:Z0FBQUFBQm8xSUpEcUIxNEFfQ2xnS0RrSC1KNnUxTlVvTGZoMHgzaEI4Z3NlVzVROTVLak5Ubi1vaEZubFZaMTFKMGd6MXAxekN2d2NvMy1hRjg2UVhybktlcFA5anZ1WjFlQmZhcXdwaGhWdzRDc3ExeUhzWTg9 +DB_REALESTATE_PORT=5432 + # Security Configuration APP_JWT_KEY_SECRET = DEV_ENC:Z0FBQUFBQm8xSUpERjlrSktmZHVuQnJ1VVJDdndLaUcxZGJsT2ZlUFRlcFdOZ001RnlzM2FhLWhRV2tjWWFhaWQwQ3hkcUFvbThMcndxSjFpYTdfRV9OZGhTcksxbXFTZWg5MDZvOHpCVXBHcDJYaHlJM0tyNWRZckZsVHpQcmxTZHJoZUs1M3lfU2ljRnJaTmNSQ0w0X085OXI0QW80M2xfQnJqZmZ6VEh3TUltX0xzeE42SGtZPQ== APP_TOKEN_EXPIRY=300 diff --git a/logs/debug/prompts/20251119-100038-001-intentanalysis_prompt.txt b/logs/debug/prompts/20251119-100038-001-intentanalysis_prompt.txt new file mode 100644 index 00000000..7f329236 --- /dev/null +++ b/logs/debug/prompts/20251119-100038-001-intentanalysis_prompt.txt @@ -0,0 +1,48 @@ + +Analyze the following user command and extract the intent, entity, and parameters. + +User Command: "Erstelle ein neues Projekt namens 'Hauptstrasse 42'" + +Available intents: +- CREATE: User wants to create a new entity +- READ: User wants to read/query entities +- UPDATE: User wants to update an existing entity +- DELETE: User wants to delete an entity +- QUERY: User wants to execute a database query (SQL statements) + +Available entities: +- Projekt: Real estate project +- Parzelle: Plot/parcel +- Dokument: Document +- Kanton: Canton +- Gemeinde: Municipality + +Return a JSON object with the following structure: +{ + "intent": "CREATE|READ|UPDATE|DELETE|QUERY", + "entity": "Projekt|Parzelle|Dokument|Kanton|Gemeinde|null", + "parameters": { + // Extracted parameters from user input + // For CREATE/UPDATE: include all relevant fields (label, statusProzess, etc.) + // For READ: include filter criteria (id, label, plz, etc.) + // For DELETE: include entity ID if mentioned + // For QUERY: include queryText if SQL is detected + }, + "confidence": 0.0-1.0 // Confidence score for the analysis +} + +Examples: +- Input: "Erstelle ein neues Projekt namens 'Hauptstrasse 42'" + Output: {"intent": "CREATE", "entity": "Projekt", "parameters": {"label": "Hauptstrasse 42"}, "confidence": 0.95} + +- Input: "Zeige mir alle Projekte" + Output: {"intent": "READ", "entity": "Projekt", "parameters": {}, "confidence": 0.9} + +- Input: "Aktualisiere Projekt XYZ mit Status 'Planung'" + Output: {"intent": "UPDATE", "entity": "Projekt", "parameters": {"id": "XYZ", "statusProzess": "Planung"}, "confidence": 0.85} + +- Input: "SELECT * FROM Projekt WHERE plz = '8000'" + Output: {"intent": "QUERY", "entity": null, "parameters": {"queryText": "SELECT * FROM Projekt WHERE plz = '8000'", "queryType": "sql"}, "confidence": 1.0} + +- Input: "Lösche Parzelle ABC" + Output: {"intent": "DELETE", "entity": "Parzelle", "parameters": {"id": "ABC"}, "confidence": 0.9} diff --git a/logs/debug/prompts/20251119-100041-002-intentanalysis_response.txt b/logs/debug/prompts/20251119-100041-002-intentanalysis_response.txt new file mode 100644 index 00000000..202b02aa --- /dev/null +++ b/logs/debug/prompts/20251119-100041-002-intentanalysis_response.txt @@ -0,0 +1,10 @@ +```json +{ + "intent": "CREATE", + "entity": "Projekt", + "parameters": { + "label": "Hauptstrasse 42" + }, + "confidence": 0.95 +} +``` \ No newline at end of file diff --git a/logs/debug/prompts/20251119-103736-003-intentanalysis_prompt.txt b/logs/debug/prompts/20251119-103736-003-intentanalysis_prompt.txt new file mode 100644 index 00000000..69cfe85c --- /dev/null +++ b/logs/debug/prompts/20251119-103736-003-intentanalysis_prompt.txt @@ -0,0 +1,48 @@ + +Analyze the following user command and extract the intent, entity, and parameters. + +User Command: "Zeige mir alle Projekte in Zürich." + +Available intents: +- CREATE: User wants to create a new entity +- READ: User wants to read/query entities +- UPDATE: User wants to update an existing entity +- DELETE: User wants to delete an entity +- QUERY: User wants to execute a database query (SQL statements) + +Available entities: +- Projekt: Real estate project +- Parzelle: Plot/parcel +- Dokument: Document +- Kanton: Canton +- Gemeinde: Municipality + +Return a JSON object with the following structure: +{ + "intent": "CREATE|READ|UPDATE|DELETE|QUERY", + "entity": "Projekt|Parzelle|Dokument|Kanton|Gemeinde|null", + "parameters": { + // Extracted parameters from user input + // For CREATE/UPDATE: include all relevant fields (label, statusProzess, etc.) + // For READ: include filter criteria (id, label, plz, etc.) + // For DELETE: include entity ID if mentioned + // For QUERY: include queryText if SQL is detected + }, + "confidence": 0.0-1.0 // Confidence score for the analysis +} + +Examples: +- Input: "Erstelle ein neues Projekt namens 'Hauptstrasse 42'" + Output: {"intent": "CREATE", "entity": "Projekt", "parameters": {"label": "Hauptstrasse 42"}, "confidence": 0.95} + +- Input: "Zeige mir alle Projekte" + Output: {"intent": "READ", "entity": "Projekt", "parameters": {}, "confidence": 0.9} + +- Input: "Aktualisiere Projekt XYZ mit Status 'Planung'" + Output: {"intent": "UPDATE", "entity": "Projekt", "parameters": {"id": "XYZ", "statusProzess": "Planung"}, "confidence": 0.85} + +- Input: "SELECT * FROM Projekt WHERE plz = '8000'" + Output: {"intent": "QUERY", "entity": null, "parameters": {"queryText": "SELECT * FROM Projekt WHERE plz = '8000'", "queryType": "sql"}, "confidence": 1.0} + +- Input: "Lösche Parzelle ABC" + Output: {"intent": "DELETE", "entity": "Parzelle", "parameters": {"id": "ABC"}, "confidence": 0.9} diff --git a/logs/debug/prompts/20251119-103742-004-intentanalysis_response.txt b/logs/debug/prompts/20251119-103742-004-intentanalysis_response.txt new file mode 100644 index 00000000..12bd99bc --- /dev/null +++ b/logs/debug/prompts/20251119-103742-004-intentanalysis_response.txt @@ -0,0 +1,16 @@ +```json +{ + "intent": "READ", + "entity": "Projekt", + "parameters": { + "location": "Zürich" + }, + "confidence": 0.95 +} +``` + +**Analysis:** +- **Intent**: READ - The command "Zeige mir" (Show me) clearly indicates a query/read operation +- **Entity**: Projekt - The user explicitly asks for "Projekte" (projects) +- **Parameters**: The location filter "in Zürich" is extracted as a search criterion. This could refer to a municipality (Gemeinde) named Zürich or projects located in the Zürich area +- **Confidence**: 0.95 - High confidence as the intent and entity are clearly stated, with an explicit location filter \ No newline at end of file diff --git a/logs/debug/prompts/20251119-103802-005-intentanalysis_prompt.txt b/logs/debug/prompts/20251119-103802-005-intentanalysis_prompt.txt new file mode 100644 index 00000000..69cfe85c --- /dev/null +++ b/logs/debug/prompts/20251119-103802-005-intentanalysis_prompt.txt @@ -0,0 +1,48 @@ + +Analyze the following user command and extract the intent, entity, and parameters. + +User Command: "Zeige mir alle Projekte in Zürich." + +Available intents: +- CREATE: User wants to create a new entity +- READ: User wants to read/query entities +- UPDATE: User wants to update an existing entity +- DELETE: User wants to delete an entity +- QUERY: User wants to execute a database query (SQL statements) + +Available entities: +- Projekt: Real estate project +- Parzelle: Plot/parcel +- Dokument: Document +- Kanton: Canton +- Gemeinde: Municipality + +Return a JSON object with the following structure: +{ + "intent": "CREATE|READ|UPDATE|DELETE|QUERY", + "entity": "Projekt|Parzelle|Dokument|Kanton|Gemeinde|null", + "parameters": { + // Extracted parameters from user input + // For CREATE/UPDATE: include all relevant fields (label, statusProzess, etc.) + // For READ: include filter criteria (id, label, plz, etc.) + // For DELETE: include entity ID if mentioned + // For QUERY: include queryText if SQL is detected + }, + "confidence": 0.0-1.0 // Confidence score for the analysis +} + +Examples: +- Input: "Erstelle ein neues Projekt namens 'Hauptstrasse 42'" + Output: {"intent": "CREATE", "entity": "Projekt", "parameters": {"label": "Hauptstrasse 42"}, "confidence": 0.95} + +- Input: "Zeige mir alle Projekte" + Output: {"intent": "READ", "entity": "Projekt", "parameters": {}, "confidence": 0.9} + +- Input: "Aktualisiere Projekt XYZ mit Status 'Planung'" + Output: {"intent": "UPDATE", "entity": "Projekt", "parameters": {"id": "XYZ", "statusProzess": "Planung"}, "confidence": 0.85} + +- Input: "SELECT * FROM Projekt WHERE plz = '8000'" + Output: {"intent": "QUERY", "entity": null, "parameters": {"queryText": "SELECT * FROM Projekt WHERE plz = '8000'", "queryType": "sql"}, "confidence": 1.0} + +- Input: "Lösche Parzelle ABC" + Output: {"intent": "DELETE", "entity": "Parzelle", "parameters": {"id": "ABC"}, "confidence": 0.9} diff --git a/logs/debug/prompts/20251119-103808-006-intentanalysis_response.txt b/logs/debug/prompts/20251119-103808-006-intentanalysis_response.txt new file mode 100644 index 00000000..62d89d2a --- /dev/null +++ b/logs/debug/prompts/20251119-103808-006-intentanalysis_response.txt @@ -0,0 +1,16 @@ +```json +{ + "intent": "READ", + "entity": "Projekt", + "parameters": { + "location": "Zürich" + }, + "confidence": 0.95 +} +``` + +**Analysis:** +- **Intent**: READ - The command "Zeige mir" (Show me) clearly indicates a query/read operation +- **Entity**: Projekt - The user explicitly asks for "Projekte" (projects) +- **Parameters**: The location filter "in Zürich" is extracted as a search criterion. This could refer to a municipality (Gemeinde) named Zürich or projects located in the Zürich area +- **Confidence**: 0.95 - High confidence as the intent and entity are clearly stated in the command \ No newline at end of file diff --git a/logs/debug/prompts/20251119-104317-007-intentanalysis_prompt.txt b/logs/debug/prompts/20251119-104317-007-intentanalysis_prompt.txt new file mode 100644 index 00000000..bb7b48bc --- /dev/null +++ b/logs/debug/prompts/20251119-104317-007-intentanalysis_prompt.txt @@ -0,0 +1,88 @@ + +Analyze the following user command and extract the intent, entity, and parameters. + +User Command: "Zeige mir alle Projekte in Zürich." + +Available intents: +- CREATE: User wants to create a new entity +- READ: User wants to read/query entities +- UPDATE: User wants to update an existing entity +- DELETE: User wants to delete an entity +- QUERY: User wants to execute a database query (SQL statements) + +Available entities and their fields: + +**Projekt** (Real estate project): +- id: string (primary key) +- mandateId: string (mandate ID) +- label: string (project designation/name) +- statusProzess: string enum (Eingang, Analyse, Studie, Planung, Baurechtsverfahren, Umsetzung, Archiv) +- perimeter: GeoPolylinie (geographic boundary, JSONB) +- baulinie: GeoPolylinie (building line, JSONB) +- parzellen: List[Parzelle] (plots belonging to project, JSONB) +- dokumente: List[Dokument] (documents, JSONB) +- kontextInformationen: List[Kontext] (context info, JSONB) + +**Parzelle** (Plot/parcel): +- id: string (primary key) +- mandateId: string (mandate ID) +- label: string (plot designation) +- strasseNr: string (street and house number) +- plz: string (postal code) +- kontextGemeinde: string (municipality ID, Foreign Key) +- kontextKanton: string (canton ID, Foreign Key) +- kontextLand: string (country ID, Foreign Key) +- bauzone: string (building zone, e.g. W3, WG2) +- az: float (Ausnützungsziffer) +- bz: float (Bebauungsziffer) +- vollgeschossZahl: int (number of allowed full floors) +- gebaeudehoeheMax: float (maximum building height in meters) +- laermschutzzone: string (noise protection zone) +- hochwasserschutzzone: string (flood protection zone) +- grundwasserschutzzone: string (groundwater protection zone) +- parzelleBebaut: JaNein enum (is plot built) +- parzelleErschlossen: JaNein enum (is plot developed) +- parzelleHanglage: JaNein enum (is plot on slope) + +**Important relationships:** +- Projekte contain Parzellen (projects have plots) +- Location queries (city, postal code) should use Parzelle fields (plz, kontextGemeinde), NOT Projekt fields +- Projekt does NOT have location fields directly - location is stored in associated Parzellen + +Return a JSON object with the following structure: +{ + "intent": "CREATE|READ|UPDATE|DELETE|QUERY", + "entity": "Projekt|Parzelle|Dokument|Kanton|Gemeinde|null", + "parameters": { + // Extracted parameters from user input + // For CREATE/UPDATE: include all relevant fields using EXACT field names from above + // For READ: include filter criteria using EXACT field names (id, label, plz, kontextGemeinde, etc.) + // For DELETE: include entity ID if mentioned + // For QUERY: include queryText if SQL is detected + // IMPORTANT: Use only field names that exist in the entity definition above + }, + "confidence": 0.0-1.0 // Confidence score for the analysis +} + +Examples: +- Input: "Erstelle ein neues Projekt namens 'Hauptstrasse 42'" + Output: {"intent": "CREATE", "entity": "Projekt", "parameters": {"label": "Hauptstrasse 42"}, "confidence": 0.95} + +- Input: "Zeige mir alle Projekte" + Output: {"intent": "READ", "entity": "Projekt", "parameters": {}, "confidence": 0.9} + +- Input: "Zeige mir Projekte in Zürich" + Output: {"intent": "READ", "entity": "Parzelle", "parameters": {"kontextGemeinde": "Zürich"}, "confidence": 0.9} + Note: Location queries should query Parzelle, not Projekt directly + +- Input: "Zeige mir Parzellen mit PLZ 8000" + Output: {"intent": "READ", "entity": "Parzelle", "parameters": {"plz": "8000"}, "confidence": 0.95} + +- Input: "Aktualisiere Projekt XYZ mit Status 'Planung'" + Output: {"intent": "UPDATE", "entity": "Projekt", "parameters": {"id": "XYZ", "statusProzess": "Planung"}, "confidence": 0.85} + +- Input: "SELECT * FROM Projekt WHERE label = 'Test'" + Output: {"intent": "QUERY", "entity": null, "parameters": {"queryText": "SELECT * FROM Projekt WHERE label = 'Test'", "queryType": "sql"}, "confidence": 1.0} + +- Input: "Lösche Parzelle ABC" + Output: {"intent": "DELETE", "entity": "Parzelle", "parameters": {"id": "ABC"}, "confidence": 0.9} diff --git a/logs/debug/prompts/20251119-104324-008-intentanalysis_response.txt b/logs/debug/prompts/20251119-104324-008-intentanalysis_response.txt new file mode 100644 index 00000000..037f72d0 --- /dev/null +++ b/logs/debug/prompts/20251119-104324-008-intentanalysis_response.txt @@ -0,0 +1,18 @@ +```json +{ + "intent": "READ", + "entity": "Parzelle", + "parameters": { + "kontextGemeinde": "Zürich" + }, + "confidence": 0.9 +} +``` + +**Reasoning:** +- The user wants to see/display ("Zeige mir") projects, which indicates a READ intent +- The location filter "in Zürich" refers to a city/municipality +- According to the schema, **Projekt does NOT have location fields directly** - location information is stored in the associated **Parzelle** entity through the `kontextGemeinde` field +- Therefore, to find projects in Zürich, we need to query **Parzelle** with the filter `kontextGemeinde: "Zürich"` +- The system can then retrieve the associated projects through the relationship (Projekte contain Parzellen) +- Confidence is 0.9 because the intent is clear, though the user said "Projekte" while we're technically querying "Parzelle" first to filter by location \ No newline at end of file diff --git a/logs/debug/prompts/20251119-104747-009-intentanalysis_prompt.txt b/logs/debug/prompts/20251119-104747-009-intentanalysis_prompt.txt new file mode 100644 index 00000000..bb7b48bc --- /dev/null +++ b/logs/debug/prompts/20251119-104747-009-intentanalysis_prompt.txt @@ -0,0 +1,88 @@ + +Analyze the following user command and extract the intent, entity, and parameters. + +User Command: "Zeige mir alle Projekte in Zürich." + +Available intents: +- CREATE: User wants to create a new entity +- READ: User wants to read/query entities +- UPDATE: User wants to update an existing entity +- DELETE: User wants to delete an entity +- QUERY: User wants to execute a database query (SQL statements) + +Available entities and their fields: + +**Projekt** (Real estate project): +- id: string (primary key) +- mandateId: string (mandate ID) +- label: string (project designation/name) +- statusProzess: string enum (Eingang, Analyse, Studie, Planung, Baurechtsverfahren, Umsetzung, Archiv) +- perimeter: GeoPolylinie (geographic boundary, JSONB) +- baulinie: GeoPolylinie (building line, JSONB) +- parzellen: List[Parzelle] (plots belonging to project, JSONB) +- dokumente: List[Dokument] (documents, JSONB) +- kontextInformationen: List[Kontext] (context info, JSONB) + +**Parzelle** (Plot/parcel): +- id: string (primary key) +- mandateId: string (mandate ID) +- label: string (plot designation) +- strasseNr: string (street and house number) +- plz: string (postal code) +- kontextGemeinde: string (municipality ID, Foreign Key) +- kontextKanton: string (canton ID, Foreign Key) +- kontextLand: string (country ID, Foreign Key) +- bauzone: string (building zone, e.g. W3, WG2) +- az: float (Ausnützungsziffer) +- bz: float (Bebauungsziffer) +- vollgeschossZahl: int (number of allowed full floors) +- gebaeudehoeheMax: float (maximum building height in meters) +- laermschutzzone: string (noise protection zone) +- hochwasserschutzzone: string (flood protection zone) +- grundwasserschutzzone: string (groundwater protection zone) +- parzelleBebaut: JaNein enum (is plot built) +- parzelleErschlossen: JaNein enum (is plot developed) +- parzelleHanglage: JaNein enum (is plot on slope) + +**Important relationships:** +- Projekte contain Parzellen (projects have plots) +- Location queries (city, postal code) should use Parzelle fields (plz, kontextGemeinde), NOT Projekt fields +- Projekt does NOT have location fields directly - location is stored in associated Parzellen + +Return a JSON object with the following structure: +{ + "intent": "CREATE|READ|UPDATE|DELETE|QUERY", + "entity": "Projekt|Parzelle|Dokument|Kanton|Gemeinde|null", + "parameters": { + // Extracted parameters from user input + // For CREATE/UPDATE: include all relevant fields using EXACT field names from above + // For READ: include filter criteria using EXACT field names (id, label, plz, kontextGemeinde, etc.) + // For DELETE: include entity ID if mentioned + // For QUERY: include queryText if SQL is detected + // IMPORTANT: Use only field names that exist in the entity definition above + }, + "confidence": 0.0-1.0 // Confidence score for the analysis +} + +Examples: +- Input: "Erstelle ein neues Projekt namens 'Hauptstrasse 42'" + Output: {"intent": "CREATE", "entity": "Projekt", "parameters": {"label": "Hauptstrasse 42"}, "confidence": 0.95} + +- Input: "Zeige mir alle Projekte" + Output: {"intent": "READ", "entity": "Projekt", "parameters": {}, "confidence": 0.9} + +- Input: "Zeige mir Projekte in Zürich" + Output: {"intent": "READ", "entity": "Parzelle", "parameters": {"kontextGemeinde": "Zürich"}, "confidence": 0.9} + Note: Location queries should query Parzelle, not Projekt directly + +- Input: "Zeige mir Parzellen mit PLZ 8000" + Output: {"intent": "READ", "entity": "Parzelle", "parameters": {"plz": "8000"}, "confidence": 0.95} + +- Input: "Aktualisiere Projekt XYZ mit Status 'Planung'" + Output: {"intent": "UPDATE", "entity": "Projekt", "parameters": {"id": "XYZ", "statusProzess": "Planung"}, "confidence": 0.85} + +- Input: "SELECT * FROM Projekt WHERE label = 'Test'" + Output: {"intent": "QUERY", "entity": null, "parameters": {"queryText": "SELECT * FROM Projekt WHERE label = 'Test'", "queryType": "sql"}, "confidence": 1.0} + +- Input: "Lösche Parzelle ABC" + Output: {"intent": "DELETE", "entity": "Parzelle", "parameters": {"id": "ABC"}, "confidence": 0.9} diff --git a/logs/debug/prompts/20251119-104754-010-intentanalysis_response.txt b/logs/debug/prompts/20251119-104754-010-intentanalysis_response.txt new file mode 100644 index 00000000..9df10c17 --- /dev/null +++ b/logs/debug/prompts/20251119-104754-010-intentanalysis_response.txt @@ -0,0 +1,17 @@ +```json +{ + "intent": "READ", + "entity": "Parzelle", + "parameters": { + "kontextGemeinde": "Zürich" + }, + "confidence": 0.9 +} +``` + +**Reasoning:** +- The user wants to see/display ("Zeige mir") projects, which indicates a READ intent +- The location filter "in Zürich" refers to a city/municipality +- According to the schema, **Projekt does NOT have location fields directly** - location information is stored in the associated **Parzelle** entity through the `kontextGemeinde` field +- Therefore, to find projects in Zürich, we need to query **Parzelle** entities with `kontextGemeinde: "Zürich"`, which will give us the plots (and their associated projects) in that municipality +- The confidence is 0.9 because the intent and location are clear, though the user said "Projekte" but we need to query via "Parzelle" due to the data model structure \ No newline at end of file diff --git a/logs/debug/prompts/20251119-104856-011-intentanalysis_prompt.txt b/logs/debug/prompts/20251119-104856-011-intentanalysis_prompt.txt new file mode 100644 index 00000000..bbfb0711 --- /dev/null +++ b/logs/debug/prompts/20251119-104856-011-intentanalysis_prompt.txt @@ -0,0 +1,88 @@ + +Analyze the following user command and extract the intent, entity, and parameters. + +User Command: "Erstell eine neue Parzelle in Zürich." + +Available intents: +- CREATE: User wants to create a new entity +- READ: User wants to read/query entities +- UPDATE: User wants to update an existing entity +- DELETE: User wants to delete an entity +- QUERY: User wants to execute a database query (SQL statements) + +Available entities and their fields: + +**Projekt** (Real estate project): +- id: string (primary key) +- mandateId: string (mandate ID) +- label: string (project designation/name) +- statusProzess: string enum (Eingang, Analyse, Studie, Planung, Baurechtsverfahren, Umsetzung, Archiv) +- perimeter: GeoPolylinie (geographic boundary, JSONB) +- baulinie: GeoPolylinie (building line, JSONB) +- parzellen: List[Parzelle] (plots belonging to project, JSONB) +- dokumente: List[Dokument] (documents, JSONB) +- kontextInformationen: List[Kontext] (context info, JSONB) + +**Parzelle** (Plot/parcel): +- id: string (primary key) +- mandateId: string (mandate ID) +- label: string (plot designation) +- strasseNr: string (street and house number) +- plz: string (postal code) +- kontextGemeinde: string (municipality ID, Foreign Key) +- kontextKanton: string (canton ID, Foreign Key) +- kontextLand: string (country ID, Foreign Key) +- bauzone: string (building zone, e.g. W3, WG2) +- az: float (Ausnützungsziffer) +- bz: float (Bebauungsziffer) +- vollgeschossZahl: int (number of allowed full floors) +- gebaeudehoeheMax: float (maximum building height in meters) +- laermschutzzone: string (noise protection zone) +- hochwasserschutzzone: string (flood protection zone) +- grundwasserschutzzone: string (groundwater protection zone) +- parzelleBebaut: JaNein enum (is plot built) +- parzelleErschlossen: JaNein enum (is plot developed) +- parzelleHanglage: JaNein enum (is plot on slope) + +**Important relationships:** +- Projekte contain Parzellen (projects have plots) +- Location queries (city, postal code) should use Parzelle fields (plz, kontextGemeinde), NOT Projekt fields +- Projekt does NOT have location fields directly - location is stored in associated Parzellen + +Return a JSON object with the following structure: +{ + "intent": "CREATE|READ|UPDATE|DELETE|QUERY", + "entity": "Projekt|Parzelle|Dokument|Kanton|Gemeinde|null", + "parameters": { + // Extracted parameters from user input + // For CREATE/UPDATE: include all relevant fields using EXACT field names from above + // For READ: include filter criteria using EXACT field names (id, label, plz, kontextGemeinde, etc.) + // For DELETE: include entity ID if mentioned + // For QUERY: include queryText if SQL is detected + // IMPORTANT: Use only field names that exist in the entity definition above + }, + "confidence": 0.0-1.0 // Confidence score for the analysis +} + +Examples: +- Input: "Erstelle ein neues Projekt namens 'Hauptstrasse 42'" + Output: {"intent": "CREATE", "entity": "Projekt", "parameters": {"label": "Hauptstrasse 42"}, "confidence": 0.95} + +- Input: "Zeige mir alle Projekte" + Output: {"intent": "READ", "entity": "Projekt", "parameters": {}, "confidence": 0.9} + +- Input: "Zeige mir Projekte in Zürich" + Output: {"intent": "READ", "entity": "Parzelle", "parameters": {"kontextGemeinde": "Zürich"}, "confidence": 0.9} + Note: Location queries should query Parzelle, not Projekt directly + +- Input: "Zeige mir Parzellen mit PLZ 8000" + Output: {"intent": "READ", "entity": "Parzelle", "parameters": {"plz": "8000"}, "confidence": 0.95} + +- Input: "Aktualisiere Projekt XYZ mit Status 'Planung'" + Output: {"intent": "UPDATE", "entity": "Projekt", "parameters": {"id": "XYZ", "statusProzess": "Planung"}, "confidence": 0.85} + +- Input: "SELECT * FROM Projekt WHERE label = 'Test'" + Output: {"intent": "QUERY", "entity": null, "parameters": {"queryText": "SELECT * FROM Projekt WHERE label = 'Test'", "queryType": "sql"}, "confidence": 1.0} + +- Input: "Lösche Parzelle ABC" + Output: {"intent": "DELETE", "entity": "Parzelle", "parameters": {"id": "ABC"}, "confidence": 0.9} diff --git a/logs/debug/prompts/20251119-104902-012-intentanalysis_response.txt b/logs/debug/prompts/20251119-104902-012-intentanalysis_response.txt new file mode 100644 index 00000000..f18102ca --- /dev/null +++ b/logs/debug/prompts/20251119-104902-012-intentanalysis_response.txt @@ -0,0 +1,17 @@ +```json +{ + "intent": "CREATE", + "entity": "Parzelle", + "parameters": { + "kontextGemeinde": "Zürich" + }, + "confidence": 0.9 +} +``` + +**Analysis:** +- **Intent**: CREATE - The user explicitly wants to create ("Erstell") a new entity +- **Entity**: Parzelle - The user specifies "eine neue Parzelle" (a new plot/parcel) +- **Parameters**: + - `kontextGemeinde`: "Zürich" - The location (municipality) where the plot should be created +- **Confidence**: 0.9 - High confidence as the intent and entity are clearly stated, though only minimal information (location) is provided for the creation \ No newline at end of file diff --git a/logs/debug/prompts/20251119-105033-013-intentanalysis_prompt.txt b/logs/debug/prompts/20251119-105033-013-intentanalysis_prompt.txt new file mode 100644 index 00000000..f5c8b8af --- /dev/null +++ b/logs/debug/prompts/20251119-105033-013-intentanalysis_prompt.txt @@ -0,0 +1,88 @@ + +Analyze the following user command and extract the intent, entity, and parameters. + +User Command: "Zeige mir alle Parzellen in Zürich." + +Available intents: +- CREATE: User wants to create a new entity +- READ: User wants to read/query entities +- UPDATE: User wants to update an existing entity +- DELETE: User wants to delete an entity +- QUERY: User wants to execute a database query (SQL statements) + +Available entities and their fields: + +**Projekt** (Real estate project): +- id: string (primary key) +- mandateId: string (mandate ID) +- label: string (project designation/name) +- statusProzess: string enum (Eingang, Analyse, Studie, Planung, Baurechtsverfahren, Umsetzung, Archiv) +- perimeter: GeoPolylinie (geographic boundary, JSONB) +- baulinie: GeoPolylinie (building line, JSONB) +- parzellen: List[Parzelle] (plots belonging to project, JSONB) +- dokumente: List[Dokument] (documents, JSONB) +- kontextInformationen: List[Kontext] (context info, JSONB) + +**Parzelle** (Plot/parcel): +- id: string (primary key) +- mandateId: string (mandate ID) +- label: string (plot designation) +- strasseNr: string (street and house number) +- plz: string (postal code) +- kontextGemeinde: string (municipality ID, Foreign Key) +- kontextKanton: string (canton ID, Foreign Key) +- kontextLand: string (country ID, Foreign Key) +- bauzone: string (building zone, e.g. W3, WG2) +- az: float (Ausnützungsziffer) +- bz: float (Bebauungsziffer) +- vollgeschossZahl: int (number of allowed full floors) +- gebaeudehoeheMax: float (maximum building height in meters) +- laermschutzzone: string (noise protection zone) +- hochwasserschutzzone: string (flood protection zone) +- grundwasserschutzzone: string (groundwater protection zone) +- parzelleBebaut: JaNein enum (is plot built) +- parzelleErschlossen: JaNein enum (is plot developed) +- parzelleHanglage: JaNein enum (is plot on slope) + +**Important relationships:** +- Projekte contain Parzellen (projects have plots) +- Location queries (city, postal code) should use Parzelle fields (plz, kontextGemeinde), NOT Projekt fields +- Projekt does NOT have location fields directly - location is stored in associated Parzellen + +Return a JSON object with the following structure: +{ + "intent": "CREATE|READ|UPDATE|DELETE|QUERY", + "entity": "Projekt|Parzelle|Dokument|Kanton|Gemeinde|null", + "parameters": { + // Extracted parameters from user input + // For CREATE/UPDATE: include all relevant fields using EXACT field names from above + // For READ: include filter criteria using EXACT field names (id, label, plz, kontextGemeinde, etc.) + // For DELETE: include entity ID if mentioned + // For QUERY: include queryText if SQL is detected + // IMPORTANT: Use only field names that exist in the entity definition above + }, + "confidence": 0.0-1.0 // Confidence score for the analysis +} + +Examples: +- Input: "Erstelle ein neues Projekt namens 'Hauptstrasse 42'" + Output: {"intent": "CREATE", "entity": "Projekt", "parameters": {"label": "Hauptstrasse 42"}, "confidence": 0.95} + +- Input: "Zeige mir alle Projekte" + Output: {"intent": "READ", "entity": "Projekt", "parameters": {}, "confidence": 0.9} + +- Input: "Zeige mir Projekte in Zürich" + Output: {"intent": "READ", "entity": "Parzelle", "parameters": {"kontextGemeinde": "Zürich"}, "confidence": 0.9} + Note: Location queries should query Parzelle, not Projekt directly + +- Input: "Zeige mir Parzellen mit PLZ 8000" + Output: {"intent": "READ", "entity": "Parzelle", "parameters": {"plz": "8000"}, "confidence": 0.95} + +- Input: "Aktualisiere Projekt XYZ mit Status 'Planung'" + Output: {"intent": "UPDATE", "entity": "Projekt", "parameters": {"id": "XYZ", "statusProzess": "Planung"}, "confidence": 0.85} + +- Input: "SELECT * FROM Projekt WHERE label = 'Test'" + Output: {"intent": "QUERY", "entity": null, "parameters": {"queryText": "SELECT * FROM Projekt WHERE label = 'Test'", "queryType": "sql"}, "confidence": 1.0} + +- Input: "Lösche Parzelle ABC" + Output: {"intent": "DELETE", "entity": "Parzelle", "parameters": {"id": "ABC"}, "confidence": 0.9} diff --git a/logs/debug/prompts/20251119-105039-014-intentanalysis_response.txt b/logs/debug/prompts/20251119-105039-014-intentanalysis_response.txt new file mode 100644 index 00000000..0e156b67 --- /dev/null +++ b/logs/debug/prompts/20251119-105039-014-intentanalysis_response.txt @@ -0,0 +1,17 @@ +```json +{ + "intent": "READ", + "entity": "Parzelle", + "parameters": { + "kontextGemeinde": "Zürich" + }, + "confidence": 0.95 +} +``` + +**Reasoning:** +- The user command "Zeige mir alle Parzellen in Zürich" translates to "Show me all plots in Zurich" +- **Intent**: READ - The user wants to query/retrieve existing entities +- **Entity**: Parzelle - The user explicitly asks for "Parzellen" (plots/parcels) +- **Parameters**: The location filter "in Zürich" maps to the `kontextGemeinde` field of the Parzelle entity, which stores the municipality information +- **Confidence**: 0.95 - High confidence because the intent is clear (show/display), the entity is explicitly mentioned (Parzellen), and the location parameter (Zürich) clearly maps to the kontextGemeinde field \ No newline at end of file diff --git a/logs/debug/prompts/20251119-110433-015-intentanalysis_prompt.txt b/logs/debug/prompts/20251119-110433-015-intentanalysis_prompt.txt new file mode 100644 index 00000000..6c53a768 --- /dev/null +++ b/logs/debug/prompts/20251119-110433-015-intentanalysis_prompt.txt @@ -0,0 +1,89 @@ + +Analyze the following user command and extract the intent, entity, and parameters. + +User Command: "Zeige mir alle Parzellen in Zürich." + +Available intents: +- CREATE: User wants to create a new entity +- READ: User wants to read/query entities +- UPDATE: User wants to update an existing entity +- DELETE: User wants to delete an entity +- QUERY: User wants to execute a database query (SQL statements) + +Available entities and their fields: + +**Projekt** (Real estate project): +- id: string (primary key) +- mandateId: string (mandate ID) +- label: string (project designation/name) +- statusProzess: string enum (Eingang, Analyse, Studie, Planung, Baurechtsverfahren, Umsetzung, Archiv) +- perimeter: GeoPolylinie (geographic boundary, JSONB) +- baulinie: GeoPolylinie (building line, JSONB) +- parzellen: List[Parzelle] (plots belonging to project, JSONB) +- dokumente: List[Dokument] (documents, JSONB) +- kontextInformationen: List[Kontext] (context info, JSONB) + +**Parzelle** (Plot/parcel): +- id: string (primary key) +- mandateId: string (mandate ID) +- label: string (plot designation) +- strasseNr: string (street and house number) +- plz: string (postal code) +- kontextGemeinde: string (municipality ID, Foreign Key to Gemeinde table) +- bauzone: string (building zone, e.g. W3, WG2) +- az: float (Ausnützungsziffer) +- bz: float (Bebauungsziffer) +- vollgeschossZahl: int (number of allowed full floors) +- gebaeudehoeheMax: float (maximum building height in meters) +- laermschutzzone: string (noise protection zone) +- hochwasserschutzzone: string (flood protection zone) +- grundwasserschutzzone: string (groundwater protection zone) +- parzelleBebaut: JaNein enum (is plot built) +- parzelleErschlossen: JaNein enum (is plot developed) +- parzelleHanglage: JaNein enum (is plot on slope) + +**Important relationships:** +- Projekte contain Parzellen (projects have plots) +- Parzelle links to Gemeinde (via kontextGemeinde) +- Gemeinde links to Kanton (via id_kanton) +- Kanton links to Land (via id_land) +- Location queries (city, postal code) should use Parzelle.kontextGemeinde (municipality name will be resolved to ID) +- Projekt does NOT have location fields directly - location is stored in associated Parzellen + +Return a JSON object with the following structure: +{ + "intent": "CREATE|READ|UPDATE|DELETE|QUERY", + "entity": "Projekt|Parzelle|Dokument|Kanton|Gemeinde|null", + "parameters": { + // Extracted parameters from user input + // For CREATE/UPDATE: include all relevant fields using EXACT field names from above + // For READ: include filter criteria using EXACT field names (id, label, plz, kontextGemeinde, etc.) + // For DELETE: include entity ID if mentioned + // For QUERY: include queryText if SQL is detected + // IMPORTANT: Use only field names that exist in the entity definition above + }, + "confidence": 0.0-1.0 // Confidence score for the analysis +} + +Examples: +- Input: "Erstelle ein neues Projekt namens 'Hauptstrasse 42'" + Output: {"intent": "CREATE", "entity": "Projekt", "parameters": {"label": "Hauptstrasse 42"}, "confidence": 0.95} + +- Input: "Zeige mir alle Projekte" + Output: {"intent": "READ", "entity": "Projekt", "parameters": {}, "confidence": 0.9} + +- Input: "Zeige mir Projekte in Zürich" + Output: {"intent": "READ", "entity": "Parzelle", "parameters": {"kontextGemeinde": "Zürich"}, "confidence": 0.9} + Note: Location queries should query Parzelle, not Projekt directly + +- Input: "Zeige mir Parzellen mit PLZ 8000" + Output: {"intent": "READ", "entity": "Parzelle", "parameters": {"plz": "8000"}, "confidence": 0.95} + +- Input: "Aktualisiere Projekt XYZ mit Status 'Planung'" + Output: {"intent": "UPDATE", "entity": "Projekt", "parameters": {"id": "XYZ", "statusProzess": "Planung"}, "confidence": 0.85} + +- Input: "SELECT * FROM Projekt WHERE label = 'Test'" + Output: {"intent": "QUERY", "entity": null, "parameters": {"queryText": "SELECT * FROM Projekt WHERE label = 'Test'", "queryType": "sql"}, "confidence": 1.0} + +- Input: "Lösche Parzelle ABC" + Output: {"intent": "DELETE", "entity": "Parzelle", "parameters": {"id": "ABC"}, "confidence": 0.9} diff --git a/logs/debug/prompts/20251119-110440-016-intentanalysis_response.txt b/logs/debug/prompts/20251119-110440-016-intentanalysis_response.txt new file mode 100644 index 00000000..331d501b --- /dev/null +++ b/logs/debug/prompts/20251119-110440-016-intentanalysis_response.txt @@ -0,0 +1,17 @@ +```json +{ + "intent": "READ", + "entity": "Parzelle", + "parameters": { + "kontextGemeinde": "Zürich" + }, + "confidence": 0.95 +} +``` + +**Explanation:** +- **Intent**: READ - The user wants to query/view existing data ("Zeige mir" = "Show me") +- **Entity**: Parzelle - The user explicitly asks for "Parzellen" (plots/parcels) +- **Parameters**: + - `kontextGemeinde`: "Zürich" - The location filter is specified as Zürich, which is a municipality (Gemeinde). According to the schema, location information for Parzellen is stored in the `kontextGemeinde` field +- **Confidence**: 0.95 - High confidence as the command is clear and unambiguous, directly requesting plots in a specific location \ No newline at end of file diff --git a/logs/debug/prompts/20251119-125234-017-intentanalysis_prompt.txt b/logs/debug/prompts/20251119-125234-017-intentanalysis_prompt.txt new file mode 100644 index 00000000..41ca3352 --- /dev/null +++ b/logs/debug/prompts/20251119-125234-017-intentanalysis_prompt.txt @@ -0,0 +1,89 @@ + +Analyze the following user command and extract the intent, entity, and parameters. + +User Command: "Füge die Gemeinde Zürich in das System hinzu." + +Available intents: +- CREATE: User wants to create a new entity +- READ: User wants to read/query entities +- UPDATE: User wants to update an existing entity +- DELETE: User wants to delete an entity +- QUERY: User wants to execute a database query (SQL statements) + +Available entities and their fields: + +**Projekt** (Real estate project): +- id: string (primary key) +- mandateId: string (mandate ID) +- label: string (project designation/name) +- statusProzess: string enum (Eingang, Analyse, Studie, Planung, Baurechtsverfahren, Umsetzung, Archiv) +- perimeter: GeoPolylinie (geographic boundary, JSONB) +- baulinie: GeoPolylinie (building line, JSONB) +- parzellen: List[Parzelle] (plots belonging to project, JSONB) +- dokumente: List[Dokument] (documents, JSONB) +- kontextInformationen: List[Kontext] (context info, JSONB) + +**Parzelle** (Plot/parcel): +- id: string (primary key) +- mandateId: string (mandate ID) +- label: string (plot designation) +- strasseNr: string (street and house number) +- plz: string (postal code) +- kontextGemeinde: string (municipality ID, Foreign Key to Gemeinde table) +- bauzone: string (building zone, e.g. W3, WG2) +- az: float (Ausnützungsziffer) +- bz: float (Bebauungsziffer) +- vollgeschossZahl: int (number of allowed full floors) +- gebaeudehoeheMax: float (maximum building height in meters) +- laermschutzzone: string (noise protection zone) +- hochwasserschutzzone: string (flood protection zone) +- grundwasserschutzzone: string (groundwater protection zone) +- parzelleBebaut: JaNein enum (is plot built) +- parzelleErschlossen: JaNein enum (is plot developed) +- parzelleHanglage: JaNein enum (is plot on slope) + +**Important relationships:** +- Projekte contain Parzellen (projects have plots) +- Parzelle links to Gemeinde (via kontextGemeinde) +- Gemeinde links to Kanton (via id_kanton) +- Kanton links to Land (via id_land) +- Location queries (city, postal code) should use Parzelle.kontextGemeinde (municipality name will be resolved to ID) +- Projekt does NOT have location fields directly - location is stored in associated Parzellen + +Return a JSON object with the following structure: +{ + "intent": "CREATE|READ|UPDATE|DELETE|QUERY", + "entity": "Projekt|Parzelle|Dokument|Kanton|Gemeinde|null", + "parameters": { + // Extracted parameters from user input + // For CREATE/UPDATE: include all relevant fields using EXACT field names from above + // For READ: include filter criteria using EXACT field names (id, label, plz, kontextGemeinde, etc.) + // For DELETE: include entity ID if mentioned + // For QUERY: include queryText if SQL is detected + // IMPORTANT: Use only field names that exist in the entity definition above + }, + "confidence": 0.0-1.0 // Confidence score for the analysis +} + +Examples: +- Input: "Erstelle ein neues Projekt namens 'Hauptstrasse 42'" + Output: {"intent": "CREATE", "entity": "Projekt", "parameters": {"label": "Hauptstrasse 42"}, "confidence": 0.95} + +- Input: "Zeige mir alle Projekte" + Output: {"intent": "READ", "entity": "Projekt", "parameters": {}, "confidence": 0.9} + +- Input: "Zeige mir Projekte in Zürich" + Output: {"intent": "READ", "entity": "Parzelle", "parameters": {"kontextGemeinde": "Zürich"}, "confidence": 0.9} + Note: Location queries should query Parzelle, not Projekt directly + +- Input: "Zeige mir Parzellen mit PLZ 8000" + Output: {"intent": "READ", "entity": "Parzelle", "parameters": {"plz": "8000"}, "confidence": 0.95} + +- Input: "Aktualisiere Projekt XYZ mit Status 'Planung'" + Output: {"intent": "UPDATE", "entity": "Projekt", "parameters": {"id": "XYZ", "statusProzess": "Planung"}, "confidence": 0.85} + +- Input: "SELECT * FROM Projekt WHERE label = 'Test'" + Output: {"intent": "QUERY", "entity": null, "parameters": {"queryText": "SELECT * FROM Projekt WHERE label = 'Test'", "queryType": "sql"}, "confidence": 1.0} + +- Input: "Lösche Parzelle ABC" + Output: {"intent": "DELETE", "entity": "Parzelle", "parameters": {"id": "ABC"}, "confidence": 0.9} diff --git a/logs/debug/prompts/20251119-125239-018-intentanalysis_response.txt b/logs/debug/prompts/20251119-125239-018-intentanalysis_response.txt new file mode 100644 index 00000000..e278914c --- /dev/null +++ b/logs/debug/prompts/20251119-125239-018-intentanalysis_response.txt @@ -0,0 +1,16 @@ +```json +{ + "intent": "CREATE", + "entity": "Gemeinde", + "parameters": { + "label": "Zürich" + }, + "confidence": 0.95 +} +``` + +**Analysis:** +- **Intent**: CREATE - The verb "Füge...hinzu" (add) clearly indicates the user wants to create a new entity +- **Entity**: Gemeinde - The user explicitly mentions "die Gemeinde Zürich" (the municipality Zürich) +- **Parameters**: The municipality name "Zürich" is extracted as the label/identifier +- **Confidence**: 0.95 - High confidence due to clear intent and explicit entity mention \ No newline at end of file diff --git a/logs/debug/prompts/20251119-125657-019-intentanalysis_prompt.txt b/logs/debug/prompts/20251119-125657-019-intentanalysis_prompt.txt new file mode 100644 index 00000000..41ca3352 --- /dev/null +++ b/logs/debug/prompts/20251119-125657-019-intentanalysis_prompt.txt @@ -0,0 +1,89 @@ + +Analyze the following user command and extract the intent, entity, and parameters. + +User Command: "Füge die Gemeinde Zürich in das System hinzu." + +Available intents: +- CREATE: User wants to create a new entity +- READ: User wants to read/query entities +- UPDATE: User wants to update an existing entity +- DELETE: User wants to delete an entity +- QUERY: User wants to execute a database query (SQL statements) + +Available entities and their fields: + +**Projekt** (Real estate project): +- id: string (primary key) +- mandateId: string (mandate ID) +- label: string (project designation/name) +- statusProzess: string enum (Eingang, Analyse, Studie, Planung, Baurechtsverfahren, Umsetzung, Archiv) +- perimeter: GeoPolylinie (geographic boundary, JSONB) +- baulinie: GeoPolylinie (building line, JSONB) +- parzellen: List[Parzelle] (plots belonging to project, JSONB) +- dokumente: List[Dokument] (documents, JSONB) +- kontextInformationen: List[Kontext] (context info, JSONB) + +**Parzelle** (Plot/parcel): +- id: string (primary key) +- mandateId: string (mandate ID) +- label: string (plot designation) +- strasseNr: string (street and house number) +- plz: string (postal code) +- kontextGemeinde: string (municipality ID, Foreign Key to Gemeinde table) +- bauzone: string (building zone, e.g. W3, WG2) +- az: float (Ausnützungsziffer) +- bz: float (Bebauungsziffer) +- vollgeschossZahl: int (number of allowed full floors) +- gebaeudehoeheMax: float (maximum building height in meters) +- laermschutzzone: string (noise protection zone) +- hochwasserschutzzone: string (flood protection zone) +- grundwasserschutzzone: string (groundwater protection zone) +- parzelleBebaut: JaNein enum (is plot built) +- parzelleErschlossen: JaNein enum (is plot developed) +- parzelleHanglage: JaNein enum (is plot on slope) + +**Important relationships:** +- Projekte contain Parzellen (projects have plots) +- Parzelle links to Gemeinde (via kontextGemeinde) +- Gemeinde links to Kanton (via id_kanton) +- Kanton links to Land (via id_land) +- Location queries (city, postal code) should use Parzelle.kontextGemeinde (municipality name will be resolved to ID) +- Projekt does NOT have location fields directly - location is stored in associated Parzellen + +Return a JSON object with the following structure: +{ + "intent": "CREATE|READ|UPDATE|DELETE|QUERY", + "entity": "Projekt|Parzelle|Dokument|Kanton|Gemeinde|null", + "parameters": { + // Extracted parameters from user input + // For CREATE/UPDATE: include all relevant fields using EXACT field names from above + // For READ: include filter criteria using EXACT field names (id, label, plz, kontextGemeinde, etc.) + // For DELETE: include entity ID if mentioned + // For QUERY: include queryText if SQL is detected + // IMPORTANT: Use only field names that exist in the entity definition above + }, + "confidence": 0.0-1.0 // Confidence score for the analysis +} + +Examples: +- Input: "Erstelle ein neues Projekt namens 'Hauptstrasse 42'" + Output: {"intent": "CREATE", "entity": "Projekt", "parameters": {"label": "Hauptstrasse 42"}, "confidence": 0.95} + +- Input: "Zeige mir alle Projekte" + Output: {"intent": "READ", "entity": "Projekt", "parameters": {}, "confidence": 0.9} + +- Input: "Zeige mir Projekte in Zürich" + Output: {"intent": "READ", "entity": "Parzelle", "parameters": {"kontextGemeinde": "Zürich"}, "confidence": 0.9} + Note: Location queries should query Parzelle, not Projekt directly + +- Input: "Zeige mir Parzellen mit PLZ 8000" + Output: {"intent": "READ", "entity": "Parzelle", "parameters": {"plz": "8000"}, "confidence": 0.95} + +- Input: "Aktualisiere Projekt XYZ mit Status 'Planung'" + Output: {"intent": "UPDATE", "entity": "Projekt", "parameters": {"id": "XYZ", "statusProzess": "Planung"}, "confidence": 0.85} + +- Input: "SELECT * FROM Projekt WHERE label = 'Test'" + Output: {"intent": "QUERY", "entity": null, "parameters": {"queryText": "SELECT * FROM Projekt WHERE label = 'Test'", "queryType": "sql"}, "confidence": 1.0} + +- Input: "Lösche Parzelle ABC" + Output: {"intent": "DELETE", "entity": "Parzelle", "parameters": {"id": "ABC"}, "confidence": 0.9} diff --git a/logs/debug/prompts/20251119-125702-020-intentanalysis_response.txt b/logs/debug/prompts/20251119-125702-020-intentanalysis_response.txt new file mode 100644 index 00000000..069e404e --- /dev/null +++ b/logs/debug/prompts/20251119-125702-020-intentanalysis_response.txt @@ -0,0 +1,16 @@ +```json +{ + "intent": "CREATE", + "entity": "Gemeinde", + "parameters": { + "label": "Zürich" + }, + "confidence": 0.95 +} +``` + +**Analysis:** +- **Intent**: CREATE - The user explicitly wants to add ("Füge...hinzu") a new entity to the system +- **Entity**: Gemeinde - The user specifically mentions "die Gemeinde Zürich" (the municipality Zürich) +- **Parameters**: The municipality name "Zürich" is extracted as the label/identifier +- **Confidence**: 0.95 - High confidence as the command is clear and unambiguous with explicit CREATE intent and entity type \ No newline at end of file diff --git a/logs/debug/prompts/20251119-125750-021-intentanalysis_prompt.txt b/logs/debug/prompts/20251119-125750-021-intentanalysis_prompt.txt new file mode 100644 index 00000000..853df98b --- /dev/null +++ b/logs/debug/prompts/20251119-125750-021-intentanalysis_prompt.txt @@ -0,0 +1,89 @@ + +Analyze the following user command and extract the intent, entity, and parameters. + +User Command: "Füge ein neues Projekt mit Parzelle in Zürich hinzu." + +Available intents: +- CREATE: User wants to create a new entity +- READ: User wants to read/query entities +- UPDATE: User wants to update an existing entity +- DELETE: User wants to delete an entity +- QUERY: User wants to execute a database query (SQL statements) + +Available entities and their fields: + +**Projekt** (Real estate project): +- id: string (primary key) +- mandateId: string (mandate ID) +- label: string (project designation/name) +- statusProzess: string enum (Eingang, Analyse, Studie, Planung, Baurechtsverfahren, Umsetzung, Archiv) +- perimeter: GeoPolylinie (geographic boundary, JSONB) +- baulinie: GeoPolylinie (building line, JSONB) +- parzellen: List[Parzelle] (plots belonging to project, JSONB) +- dokumente: List[Dokument] (documents, JSONB) +- kontextInformationen: List[Kontext] (context info, JSONB) + +**Parzelle** (Plot/parcel): +- id: string (primary key) +- mandateId: string (mandate ID) +- label: string (plot designation) +- strasseNr: string (street and house number) +- plz: string (postal code) +- kontextGemeinde: string (municipality ID, Foreign Key to Gemeinde table) +- bauzone: string (building zone, e.g. W3, WG2) +- az: float (Ausnützungsziffer) +- bz: float (Bebauungsziffer) +- vollgeschossZahl: int (number of allowed full floors) +- gebaeudehoeheMax: float (maximum building height in meters) +- laermschutzzone: string (noise protection zone) +- hochwasserschutzzone: string (flood protection zone) +- grundwasserschutzzone: string (groundwater protection zone) +- parzelleBebaut: JaNein enum (is plot built) +- parzelleErschlossen: JaNein enum (is plot developed) +- parzelleHanglage: JaNein enum (is plot on slope) + +**Important relationships:** +- Projekte contain Parzellen (projects have plots) +- Parzelle links to Gemeinde (via kontextGemeinde) +- Gemeinde links to Kanton (via id_kanton) +- Kanton links to Land (via id_land) +- Location queries (city, postal code) should use Parzelle.kontextGemeinde (municipality name will be resolved to ID) +- Projekt does NOT have location fields directly - location is stored in associated Parzellen + +Return a JSON object with the following structure: +{ + "intent": "CREATE|READ|UPDATE|DELETE|QUERY", + "entity": "Projekt|Parzelle|Dokument|Kanton|Gemeinde|null", + "parameters": { + // Extracted parameters from user input + // For CREATE/UPDATE: include all relevant fields using EXACT field names from above + // For READ: include filter criteria using EXACT field names (id, label, plz, kontextGemeinde, etc.) + // For DELETE: include entity ID if mentioned + // For QUERY: include queryText if SQL is detected + // IMPORTANT: Use only field names that exist in the entity definition above + }, + "confidence": 0.0-1.0 // Confidence score for the analysis +} + +Examples: +- Input: "Erstelle ein neues Projekt namens 'Hauptstrasse 42'" + Output: {"intent": "CREATE", "entity": "Projekt", "parameters": {"label": "Hauptstrasse 42"}, "confidence": 0.95} + +- Input: "Zeige mir alle Projekte" + Output: {"intent": "READ", "entity": "Projekt", "parameters": {}, "confidence": 0.9} + +- Input: "Zeige mir Projekte in Zürich" + Output: {"intent": "READ", "entity": "Parzelle", "parameters": {"kontextGemeinde": "Zürich"}, "confidence": 0.9} + Note: Location queries should query Parzelle, not Projekt directly + +- Input: "Zeige mir Parzellen mit PLZ 8000" + Output: {"intent": "READ", "entity": "Parzelle", "parameters": {"plz": "8000"}, "confidence": 0.95} + +- Input: "Aktualisiere Projekt XYZ mit Status 'Planung'" + Output: {"intent": "UPDATE", "entity": "Projekt", "parameters": {"id": "XYZ", "statusProzess": "Planung"}, "confidence": 0.85} + +- Input: "SELECT * FROM Projekt WHERE label = 'Test'" + Output: {"intent": "QUERY", "entity": null, "parameters": {"queryText": "SELECT * FROM Projekt WHERE label = 'Test'", "queryType": "sql"}, "confidence": 1.0} + +- Input: "Lösche Parzelle ABC" + Output: {"intent": "DELETE", "entity": "Parzelle", "parameters": {"id": "ABC"}, "confidence": 0.9} diff --git a/logs/debug/prompts/20251119-125756-022-intentanalysis_response.txt b/logs/debug/prompts/20251119-125756-022-intentanalysis_response.txt new file mode 100644 index 00000000..85c1db10 --- /dev/null +++ b/logs/debug/prompts/20251119-125756-022-intentanalysis_response.txt @@ -0,0 +1,20 @@ +```json +{ + "intent": "CREATE", + "entity": "Projekt", + "parameters": { + "parzellen": [ + { + "kontextGemeinde": "Zürich" + } + ] + }, + "confidence": 0.85 +} +``` + +**Reasoning:** +- The user wants to create ("Füge hinzu") a new project ("neues Projekt") +- The project should include a plot ("mit Parzelle") in Zürich +- Since Projekt contains Parzellen as a nested list, and the location information (Zürich) belongs to Parzelle (via kontextGemeinde field), I've structured the parameters to reflect this relationship +- The confidence is 0.85 because while the intent is clear, the user hasn't provided other typical project details like a label/name for the project itself \ No newline at end of file diff --git a/logs/debug/prompts/20251119-141840-023-intentanalysis_prompt.txt b/logs/debug/prompts/20251119-141840-023-intentanalysis_prompt.txt new file mode 100644 index 00000000..69983369 --- /dev/null +++ b/logs/debug/prompts/20251119-141840-023-intentanalysis_prompt.txt @@ -0,0 +1,89 @@ + +Analyze the following user command and extract the intent, entity, and parameters. + +User Command: "Füge die Gemeinde Winterthur hinzu" + +Available intents: +- CREATE: User wants to create a new entity +- READ: User wants to read/query entities +- UPDATE: User wants to update an existing entity +- DELETE: User wants to delete an entity +- QUERY: User wants to execute a database query (SQL statements) + +Available entities and their fields: + +**Projekt** (Real estate project): +- id: string (primary key) +- mandateId: string (mandate ID) +- label: string (project designation/name) +- statusProzess: string enum (Eingang, Analyse, Studie, Planung, Baurechtsverfahren, Umsetzung, Archiv) +- perimeter: GeoPolylinie (geographic boundary, JSONB) +- baulinie: GeoPolylinie (building line, JSONB) +- parzellen: List[Parzelle] (plots belonging to project, JSONB) +- dokumente: List[Dokument] (documents, JSONB) +- kontextInformationen: List[Kontext] (context info, JSONB) + +**Parzelle** (Plot/parcel): +- id: string (primary key) +- mandateId: string (mandate ID) +- label: string (plot designation) +- strasseNr: string (street and house number) +- plz: string (postal code) +- kontextGemeinde: string (municipality ID, Foreign Key to Gemeinde table) +- bauzone: string (building zone, e.g. W3, WG2) +- az: float (Ausnützungsziffer) +- bz: float (Bebauungsziffer) +- vollgeschossZahl: int (number of allowed full floors) +- gebaeudehoeheMax: float (maximum building height in meters) +- laermschutzzone: string (noise protection zone) +- hochwasserschutzzone: string (flood protection zone) +- grundwasserschutzzone: string (groundwater protection zone) +- parzelleBebaut: JaNein enum (is plot built) +- parzelleErschlossen: JaNein enum (is plot developed) +- parzelleHanglage: JaNein enum (is plot on slope) + +**Important relationships:** +- Projekte contain Parzellen (projects have plots) +- Parzelle links to Gemeinde (via kontextGemeinde) +- Gemeinde links to Kanton (via id_kanton) +- Kanton links to Land (via id_land) +- Location queries (city, postal code) should use Parzelle.kontextGemeinde (municipality name will be resolved to ID) +- Projekt does NOT have location fields directly - location is stored in associated Parzellen + +Return a JSON object with the following structure: +{ + "intent": "CREATE|READ|UPDATE|DELETE|QUERY", + "entity": "Projekt|Parzelle|Dokument|Kanton|Gemeinde|null", + "parameters": { + // Extracted parameters from user input + // For CREATE/UPDATE: include all relevant fields using EXACT field names from above + // For READ: include filter criteria using EXACT field names (id, label, plz, kontextGemeinde, etc.) + // For DELETE: include entity ID if mentioned + // For QUERY: include queryText if SQL is detected + // IMPORTANT: Use only field names that exist in the entity definition above + }, + "confidence": 0.0-1.0 // Confidence score for the analysis +} + +Examples: +- Input: "Erstelle ein neues Projekt namens 'Hauptstrasse 42'" + Output: {"intent": "CREATE", "entity": "Projekt", "parameters": {"label": "Hauptstrasse 42"}, "confidence": 0.95} + +- Input: "Zeige mir alle Projekte" + Output: {"intent": "READ", "entity": "Projekt", "parameters": {}, "confidence": 0.9} + +- Input: "Zeige mir Projekte in Zürich" + Output: {"intent": "READ", "entity": "Parzelle", "parameters": {"kontextGemeinde": "Zürich"}, "confidence": 0.9} + Note: Location queries should query Parzelle, not Projekt directly + +- Input: "Zeige mir Parzellen mit PLZ 8000" + Output: {"intent": "READ", "entity": "Parzelle", "parameters": {"plz": "8000"}, "confidence": 0.95} + +- Input: "Aktualisiere Projekt XYZ mit Status 'Planung'" + Output: {"intent": "UPDATE", "entity": "Projekt", "parameters": {"id": "XYZ", "statusProzess": "Planung"}, "confidence": 0.85} + +- Input: "SELECT * FROM Projekt WHERE label = 'Test'" + Output: {"intent": "QUERY", "entity": null, "parameters": {"queryText": "SELECT * FROM Projekt WHERE label = 'Test'", "queryType": "sql"}, "confidence": 1.0} + +- Input: "Lösche Parzelle ABC" + Output: {"intent": "DELETE", "entity": "Parzelle", "parameters": {"id": "ABC"}, "confidence": 0.9} diff --git a/logs/debug/prompts/20251119-141848-024-intentanalysis_response.txt b/logs/debug/prompts/20251119-141848-024-intentanalysis_response.txt new file mode 100644 index 00000000..5b3aa3d5 --- /dev/null +++ b/logs/debug/prompts/20251119-141848-024-intentanalysis_response.txt @@ -0,0 +1,16 @@ +```json +{ + "intent": "CREATE", + "entity": "Gemeinde", + "parameters": { + "label": "Winterthur" + }, + "confidence": 0.95 +} +``` + +**Analysis:** +- **Intent**: CREATE - The verb "Füge...hinzu" (add) clearly indicates the user wants to create a new entity +- **Entity**: Gemeinde - The user explicitly mentions "die Gemeinde Winterthur" (the municipality Winterthur) +- **Parameters**: The name/label of the municipality is "Winterthur" +- **Confidence**: 0.95 - High confidence as the command is clear and unambiguous \ No newline at end of file diff --git a/logs/debug/prompts/20251119-144901-025-intentanalysis_prompt.txt b/logs/debug/prompts/20251119-144901-025-intentanalysis_prompt.txt new file mode 100644 index 00000000..b13d0efe --- /dev/null +++ b/logs/debug/prompts/20251119-144901-025-intentanalysis_prompt.txt @@ -0,0 +1,89 @@ + +Analyze the following user command and extract the intent, entity, and parameters. + +User Command: "Wie viele Projekte in Zürich gibt es?" + +Available intents: +- CREATE: User wants to create a new entity +- READ: User wants to read/query entities +- UPDATE: User wants to update an existing entity +- DELETE: User wants to delete an entity +- QUERY: User wants to execute a database query (SQL statements) + +Available entities and their fields: + +**Projekt** (Real estate project): +- id: string (primary key) +- mandateId: string (mandate ID) +- label: string (project designation/name) +- statusProzess: string enum (Eingang, Analyse, Studie, Planung, Baurechtsverfahren, Umsetzung, Archiv) +- perimeter: GeoPolylinie (geographic boundary, JSONB) +- baulinie: GeoPolylinie (building line, JSONB) +- parzellen: List[Parzelle] (plots belonging to project, JSONB) +- dokumente: List[Dokument] (documents, JSONB) +- kontextInformationen: List[Kontext] (context info, JSONB) + +**Parzelle** (Plot/parcel): +- id: string (primary key) +- mandateId: string (mandate ID) +- label: string (plot designation) +- strasseNr: string (street and house number) +- plz: string (postal code) +- kontextGemeinde: string (municipality ID, Foreign Key to Gemeinde table) +- bauzone: string (building zone, e.g. W3, WG2) +- az: float (Ausnützungsziffer) +- bz: float (Bebauungsziffer) +- vollgeschossZahl: int (number of allowed full floors) +- gebaeudehoeheMax: float (maximum building height in meters) +- laermschutzzone: string (noise protection zone) +- hochwasserschutzzone: string (flood protection zone) +- grundwasserschutzzone: string (groundwater protection zone) +- parzelleBebaut: JaNein enum (is plot built) +- parzelleErschlossen: JaNein enum (is plot developed) +- parzelleHanglage: JaNein enum (is plot on slope) + +**Important relationships:** +- Projekte contain Parzellen (projects have plots) +- Parzelle links to Gemeinde (via kontextGemeinde) +- Gemeinde links to Kanton (via id_kanton) +- Kanton links to Land (via id_land) +- Location queries (city, postal code) should use Parzelle.kontextGemeinde (municipality name will be resolved to ID) +- Projekt does NOT have location fields directly - location is stored in associated Parzellen + +Return a JSON object with the following structure: +{ + "intent": "CREATE|READ|UPDATE|DELETE|QUERY", + "entity": "Projekt|Parzelle|Dokument|Kanton|Gemeinde|null", + "parameters": { + // Extracted parameters from user input + // For CREATE/UPDATE: include all relevant fields using EXACT field names from above + // For READ: include filter criteria using EXACT field names (id, label, plz, kontextGemeinde, etc.) + // For DELETE: include entity ID if mentioned + // For QUERY: include queryText if SQL is detected + // IMPORTANT: Use only field names that exist in the entity definition above + }, + "confidence": 0.0-1.0 // Confidence score for the analysis +} + +Examples: +- Input: "Erstelle ein neues Projekt namens 'Hauptstrasse 42'" + Output: {"intent": "CREATE", "entity": "Projekt", "parameters": {"label": "Hauptstrasse 42"}, "confidence": 0.95} + +- Input: "Zeige mir alle Projekte" + Output: {"intent": "READ", "entity": "Projekt", "parameters": {}, "confidence": 0.9} + +- Input: "Zeige mir Projekte in Zürich" + Output: {"intent": "READ", "entity": "Parzelle", "parameters": {"kontextGemeinde": "Zürich"}, "confidence": 0.9} + Note: Location queries should query Parzelle, not Projekt directly + +- Input: "Zeige mir Parzellen mit PLZ 8000" + Output: {"intent": "READ", "entity": "Parzelle", "parameters": {"plz": "8000"}, "confidence": 0.95} + +- Input: "Aktualisiere Projekt XYZ mit Status 'Planung'" + Output: {"intent": "UPDATE", "entity": "Projekt", "parameters": {"id": "XYZ", "statusProzess": "Planung"}, "confidence": 0.85} + +- Input: "SELECT * FROM Projekt WHERE label = 'Test'" + Output: {"intent": "QUERY", "entity": null, "parameters": {"queryText": "SELECT * FROM Projekt WHERE label = 'Test'", "queryType": "sql"}, "confidence": 1.0} + +- Input: "Lösche Parzelle ABC" + Output: {"intent": "DELETE", "entity": "Parzelle", "parameters": {"id": "ABC"}, "confidence": 0.9} diff --git a/logs/debug/prompts/20251119-144909-026-intentanalysis_response.txt b/logs/debug/prompts/20251119-144909-026-intentanalysis_response.txt new file mode 100644 index 00000000..59197959 --- /dev/null +++ b/logs/debug/prompts/20251119-144909-026-intentanalysis_response.txt @@ -0,0 +1,17 @@ +```json +{ + "intent": "READ", + "entity": "Parzelle", + "parameters": { + "kontextGemeinde": "Zürich" + }, + "confidence": 0.9 +} +``` + +**Reasoning:** +- The user asks "How many projects in Zürich are there?" which is a READ/query operation +- Since location information (city) is stored in Parzelle via `kontextGemeinde`, not directly in Projekt, we need to query Parzelle entities +- The parameter `kontextGemeinde` is set to "Zürich" to filter plots by municipality +- To get projects in Zürich, the system will need to find all Parzellen with kontextGemeinde="Zürich" and then identify the associated projects +- Confidence is 0.9 because the intent is clear, though the user asks about "projects" while we're querying "Parzelle" (which is the correct approach given the data model) \ No newline at end of file diff --git a/logs/debug/prompts/20251119-145121-027-intentanalysis_prompt.txt b/logs/debug/prompts/20251119-145121-027-intentanalysis_prompt.txt new file mode 100644 index 00000000..6f20254d --- /dev/null +++ b/logs/debug/prompts/20251119-145121-027-intentanalysis_prompt.txt @@ -0,0 +1,89 @@ + +Analyze the following user command and extract the intent, entity, and parameters. + +User Command: "welche projekte in Zürich gibt es?" + +Available intents: +- CREATE: User wants to create a new entity +- READ: User wants to read/query entities +- UPDATE: User wants to update an existing entity +- DELETE: User wants to delete an entity +- QUERY: User wants to execute a database query (SQL statements) + +Available entities and their fields: + +**Projekt** (Real estate project): +- id: string (primary key) +- mandateId: string (mandate ID) +- label: string (project designation/name) +- statusProzess: string enum (Eingang, Analyse, Studie, Planung, Baurechtsverfahren, Umsetzung, Archiv) +- perimeter: GeoPolylinie (geographic boundary, JSONB) +- baulinie: GeoPolylinie (building line, JSONB) +- parzellen: List[Parzelle] (plots belonging to project, JSONB) +- dokumente: List[Dokument] (documents, JSONB) +- kontextInformationen: List[Kontext] (context info, JSONB) + +**Parzelle** (Plot/parcel): +- id: string (primary key) +- mandateId: string (mandate ID) +- label: string (plot designation) +- strasseNr: string (street and house number) +- plz: string (postal code) +- kontextGemeinde: string (municipality ID, Foreign Key to Gemeinde table) +- bauzone: string (building zone, e.g. W3, WG2) +- az: float (Ausnützungsziffer) +- bz: float (Bebauungsziffer) +- vollgeschossZahl: int (number of allowed full floors) +- gebaeudehoeheMax: float (maximum building height in meters) +- laermschutzzone: string (noise protection zone) +- hochwasserschutzzone: string (flood protection zone) +- grundwasserschutzzone: string (groundwater protection zone) +- parzelleBebaut: JaNein enum (is plot built) +- parzelleErschlossen: JaNein enum (is plot developed) +- parzelleHanglage: JaNein enum (is plot on slope) + +**Important relationships:** +- Projekte contain Parzellen (projects have plots) +- Parzelle links to Gemeinde (via kontextGemeinde) +- Gemeinde links to Kanton (via id_kanton) +- Kanton links to Land (via id_land) +- Location queries (city, postal code) should use Parzelle.kontextGemeinde (municipality name will be resolved to ID) +- Projekt does NOT have location fields directly - location is stored in associated Parzellen + +Return a JSON object with the following structure: +{ + "intent": "CREATE|READ|UPDATE|DELETE|QUERY", + "entity": "Projekt|Parzelle|Dokument|Kanton|Gemeinde|null", + "parameters": { + // Extracted parameters from user input + // For CREATE/UPDATE: include all relevant fields using EXACT field names from above + // For READ: include filter criteria using EXACT field names (id, label, plz, kontextGemeinde, etc.) + // For DELETE: include entity ID if mentioned + // For QUERY: include queryText if SQL is detected + // IMPORTANT: Use only field names that exist in the entity definition above + }, + "confidence": 0.0-1.0 // Confidence score for the analysis +} + +Examples: +- Input: "Erstelle ein neues Projekt namens 'Hauptstrasse 42'" + Output: {"intent": "CREATE", "entity": "Projekt", "parameters": {"label": "Hauptstrasse 42"}, "confidence": 0.95} + +- Input: "Zeige mir alle Projekte" + Output: {"intent": "READ", "entity": "Projekt", "parameters": {}, "confidence": 0.9} + +- Input: "Zeige mir Projekte in Zürich" + Output: {"intent": "READ", "entity": "Parzelle", "parameters": {"kontextGemeinde": "Zürich"}, "confidence": 0.9} + Note: Location queries should query Parzelle, not Projekt directly + +- Input: "Zeige mir Parzellen mit PLZ 8000" + Output: {"intent": "READ", "entity": "Parzelle", "parameters": {"plz": "8000"}, "confidence": 0.95} + +- Input: "Aktualisiere Projekt XYZ mit Status 'Planung'" + Output: {"intent": "UPDATE", "entity": "Projekt", "parameters": {"id": "XYZ", "statusProzess": "Planung"}, "confidence": 0.85} + +- Input: "SELECT * FROM Projekt WHERE label = 'Test'" + Output: {"intent": "QUERY", "entity": null, "parameters": {"queryText": "SELECT * FROM Projekt WHERE label = 'Test'", "queryType": "sql"}, "confidence": 1.0} + +- Input: "Lösche Parzelle ABC" + Output: {"intent": "DELETE", "entity": "Parzelle", "parameters": {"id": "ABC"}, "confidence": 0.9} diff --git a/logs/debug/prompts/20251119-145129-028-intentanalysis_response.txt b/logs/debug/prompts/20251119-145129-028-intentanalysis_response.txt new file mode 100644 index 00000000..b6c6eaae --- /dev/null +++ b/logs/debug/prompts/20251119-145129-028-intentanalysis_response.txt @@ -0,0 +1,17 @@ +```json +{ + "intent": "READ", + "entity": "Parzelle", + "parameters": { + "kontextGemeinde": "Zürich" + }, + "confidence": 0.9 +} +``` + +**Reasoning:** +- The user is asking "which projects in Zürich exist?" which is a READ/query operation +- Since location information (city/municipality) is stored in Parzelle (via `kontextGemeinde`), not directly in Projekt, we need to query Parzelle entities +- "Zürich" is the municipality name that maps to the `kontextGemeinde` field +- The system will need to find all Parzellen in Zürich and then retrieve their associated Projekte +- High confidence (0.9) as the intent and location are clearly stated \ No newline at end of file diff --git a/logs/debug/prompts/20251119-145151-029-intentanalysis_prompt.txt b/logs/debug/prompts/20251119-145151-029-intentanalysis_prompt.txt new file mode 100644 index 00000000..c2b228d9 --- /dev/null +++ b/logs/debug/prompts/20251119-145151-029-intentanalysis_prompt.txt @@ -0,0 +1,89 @@ + +Analyze the following user command and extract the intent, entity, and parameters. + +User Command: "Welche Projekte in Zürich gibt es?" + +Available intents: +- CREATE: User wants to create a new entity +- READ: User wants to read/query entities +- UPDATE: User wants to update an existing entity +- DELETE: User wants to delete an entity +- QUERY: User wants to execute a database query (SQL statements) + +Available entities and their fields: + +**Projekt** (Real estate project): +- id: string (primary key) +- mandateId: string (mandate ID) +- label: string (project designation/name) +- statusProzess: string enum (Eingang, Analyse, Studie, Planung, Baurechtsverfahren, Umsetzung, Archiv) +- perimeter: GeoPolylinie (geographic boundary, JSONB) +- baulinie: GeoPolylinie (building line, JSONB) +- parzellen: List[Parzelle] (plots belonging to project, JSONB) +- dokumente: List[Dokument] (documents, JSONB) +- kontextInformationen: List[Kontext] (context info, JSONB) + +**Parzelle** (Plot/parcel): +- id: string (primary key) +- mandateId: string (mandate ID) +- label: string (plot designation) +- strasseNr: string (street and house number) +- plz: string (postal code) +- kontextGemeinde: string (municipality ID, Foreign Key to Gemeinde table) +- bauzone: string (building zone, e.g. W3, WG2) +- az: float (Ausnützungsziffer) +- bz: float (Bebauungsziffer) +- vollgeschossZahl: int (number of allowed full floors) +- gebaeudehoeheMax: float (maximum building height in meters) +- laermschutzzone: string (noise protection zone) +- hochwasserschutzzone: string (flood protection zone) +- grundwasserschutzzone: string (groundwater protection zone) +- parzelleBebaut: JaNein enum (is plot built) +- parzelleErschlossen: JaNein enum (is plot developed) +- parzelleHanglage: JaNein enum (is plot on slope) + +**Important relationships:** +- Projekte contain Parzellen (projects have plots) +- Parzelle links to Gemeinde (via kontextGemeinde) +- Gemeinde links to Kanton (via id_kanton) +- Kanton links to Land (via id_land) +- Location queries (city, postal code) should use Parzelle.kontextGemeinde (municipality name will be resolved to ID) +- Projekt does NOT have location fields directly - location is stored in associated Parzellen + +Return a JSON object with the following structure: +{ + "intent": "CREATE|READ|UPDATE|DELETE|QUERY", + "entity": "Projekt|Parzelle|Dokument|Kanton|Gemeinde|null", + "parameters": { + // Extracted parameters from user input + // For CREATE/UPDATE: include all relevant fields using EXACT field names from above + // For READ: include filter criteria using EXACT field names (id, label, plz, kontextGemeinde, etc.) + // For DELETE: include entity ID if mentioned + // For QUERY: include queryText if SQL is detected + // IMPORTANT: Use only field names that exist in the entity definition above + }, + "confidence": 0.0-1.0 // Confidence score for the analysis +} + +Examples: +- Input: "Erstelle ein neues Projekt namens 'Hauptstrasse 42'" + Output: {"intent": "CREATE", "entity": "Projekt", "parameters": {"label": "Hauptstrasse 42"}, "confidence": 0.95} + +- Input: "Zeige mir alle Projekte" + Output: {"intent": "READ", "entity": "Projekt", "parameters": {}, "confidence": 0.9} + +- Input: "Zeige mir Projekte in Zürich" + Output: {"intent": "READ", "entity": "Parzelle", "parameters": {"kontextGemeinde": "Zürich"}, "confidence": 0.9} + Note: Location queries should query Parzelle, not Projekt directly + +- Input: "Zeige mir Parzellen mit PLZ 8000" + Output: {"intent": "READ", "entity": "Parzelle", "parameters": {"plz": "8000"}, "confidence": 0.95} + +- Input: "Aktualisiere Projekt XYZ mit Status 'Planung'" + Output: {"intent": "UPDATE", "entity": "Projekt", "parameters": {"id": "XYZ", "statusProzess": "Planung"}, "confidence": 0.85} + +- Input: "SELECT * FROM Projekt WHERE label = 'Test'" + Output: {"intent": "QUERY", "entity": null, "parameters": {"queryText": "SELECT * FROM Projekt WHERE label = 'Test'", "queryType": "sql"}, "confidence": 1.0} + +- Input: "Lösche Parzelle ABC" + Output: {"intent": "DELETE", "entity": "Parzelle", "parameters": {"id": "ABC"}, "confidence": 0.9} diff --git a/logs/debug/prompts/20251119-145202-030-intentanalysis_response.txt b/logs/debug/prompts/20251119-145202-030-intentanalysis_response.txt new file mode 100644 index 00000000..4cc5d8b0 --- /dev/null +++ b/logs/debug/prompts/20251119-145202-030-intentanalysis_response.txt @@ -0,0 +1,18 @@ +```json +{ + "intent": "READ", + "entity": "Parzelle", + "parameters": { + "kontextGemeinde": "Zürich" + }, + "confidence": 0.9 +} +``` + +**Reasoning:** +- The user is asking "Which projects in Zürich exist?" (Welche Projekte in Zürich gibt es?) +- This is a READ operation (querying for existing data) +- Since location information (city/municipality) is stored in **Parzelle** via the `kontextGemeinde` field, not directly in Projekt, we need to query Parzelle entities +- The parameter `kontextGemeinde` is set to "Zürich" to filter plots by municipality +- To get projects in Zürich, the system will need to find Parzellen with kontextGemeinde="Zürich" and then retrieve their associated Projekte +- Confidence is 0.9 because the intent is clear, though the user asked about "Projekte" while we're technically querying via "Parzelle" due to the data model structure \ No newline at end of file diff --git a/logs/debug/prompts/20251119-145222-031-intentanalysis_prompt.txt b/logs/debug/prompts/20251119-145222-031-intentanalysis_prompt.txt new file mode 100644 index 00000000..d67478d1 --- /dev/null +++ b/logs/debug/prompts/20251119-145222-031-intentanalysis_prompt.txt @@ -0,0 +1,89 @@ + +Analyze the following user command and extract the intent, entity, and parameters. + +User Command: "Welche Gemeinden haben wir bereits hinzugefügt?" + +Available intents: +- CREATE: User wants to create a new entity +- READ: User wants to read/query entities +- UPDATE: User wants to update an existing entity +- DELETE: User wants to delete an entity +- QUERY: User wants to execute a database query (SQL statements) + +Available entities and their fields: + +**Projekt** (Real estate project): +- id: string (primary key) +- mandateId: string (mandate ID) +- label: string (project designation/name) +- statusProzess: string enum (Eingang, Analyse, Studie, Planung, Baurechtsverfahren, Umsetzung, Archiv) +- perimeter: GeoPolylinie (geographic boundary, JSONB) +- baulinie: GeoPolylinie (building line, JSONB) +- parzellen: List[Parzelle] (plots belonging to project, JSONB) +- dokumente: List[Dokument] (documents, JSONB) +- kontextInformationen: List[Kontext] (context info, JSONB) + +**Parzelle** (Plot/parcel): +- id: string (primary key) +- mandateId: string (mandate ID) +- label: string (plot designation) +- strasseNr: string (street and house number) +- plz: string (postal code) +- kontextGemeinde: string (municipality ID, Foreign Key to Gemeinde table) +- bauzone: string (building zone, e.g. W3, WG2) +- az: float (Ausnützungsziffer) +- bz: float (Bebauungsziffer) +- vollgeschossZahl: int (number of allowed full floors) +- gebaeudehoeheMax: float (maximum building height in meters) +- laermschutzzone: string (noise protection zone) +- hochwasserschutzzone: string (flood protection zone) +- grundwasserschutzzone: string (groundwater protection zone) +- parzelleBebaut: JaNein enum (is plot built) +- parzelleErschlossen: JaNein enum (is plot developed) +- parzelleHanglage: JaNein enum (is plot on slope) + +**Important relationships:** +- Projekte contain Parzellen (projects have plots) +- Parzelle links to Gemeinde (via kontextGemeinde) +- Gemeinde links to Kanton (via id_kanton) +- Kanton links to Land (via id_land) +- Location queries (city, postal code) should use Parzelle.kontextGemeinde (municipality name will be resolved to ID) +- Projekt does NOT have location fields directly - location is stored in associated Parzellen + +Return a JSON object with the following structure: +{ + "intent": "CREATE|READ|UPDATE|DELETE|QUERY", + "entity": "Projekt|Parzelle|Dokument|Kanton|Gemeinde|null", + "parameters": { + // Extracted parameters from user input + // For CREATE/UPDATE: include all relevant fields using EXACT field names from above + // For READ: include filter criteria using EXACT field names (id, label, plz, kontextGemeinde, etc.) + // For DELETE: include entity ID if mentioned + // For QUERY: include queryText if SQL is detected + // IMPORTANT: Use only field names that exist in the entity definition above + }, + "confidence": 0.0-1.0 // Confidence score for the analysis +} + +Examples: +- Input: "Erstelle ein neues Projekt namens 'Hauptstrasse 42'" + Output: {"intent": "CREATE", "entity": "Projekt", "parameters": {"label": "Hauptstrasse 42"}, "confidence": 0.95} + +- Input: "Zeige mir alle Projekte" + Output: {"intent": "READ", "entity": "Projekt", "parameters": {}, "confidence": 0.9} + +- Input: "Zeige mir Projekte in Zürich" + Output: {"intent": "READ", "entity": "Parzelle", "parameters": {"kontextGemeinde": "Zürich"}, "confidence": 0.9} + Note: Location queries should query Parzelle, not Projekt directly + +- Input: "Zeige mir Parzellen mit PLZ 8000" + Output: {"intent": "READ", "entity": "Parzelle", "parameters": {"plz": "8000"}, "confidence": 0.95} + +- Input: "Aktualisiere Projekt XYZ mit Status 'Planung'" + Output: {"intent": "UPDATE", "entity": "Projekt", "parameters": {"id": "XYZ", "statusProzess": "Planung"}, "confidence": 0.85} + +- Input: "SELECT * FROM Projekt WHERE label = 'Test'" + Output: {"intent": "QUERY", "entity": null, "parameters": {"queryText": "SELECT * FROM Projekt WHERE label = 'Test'", "queryType": "sql"}, "confidence": 1.0} + +- Input: "Lösche Parzelle ABC" + Output: {"intent": "DELETE", "entity": "Parzelle", "parameters": {"id": "ABC"}, "confidence": 0.9} diff --git a/logs/debug/prompts/20251119-145229-032-intentanalysis_response.txt b/logs/debug/prompts/20251119-145229-032-intentanalysis_response.txt new file mode 100644 index 00000000..4e0657b7 --- /dev/null +++ b/logs/debug/prompts/20251119-145229-032-intentanalysis_response.txt @@ -0,0 +1,14 @@ +```json +{ + "intent": "READ", + "entity": "Gemeinde", + "parameters": {}, + "confidence": 0.95 +} +``` + +**Reasoning:** +- The user is asking "Which municipalities have we already added?" (Welche Gemeinden haben wir bereits hinzugefügt?) +- This is clearly a READ operation to query existing Gemeinde entities +- No specific filters are mentioned, so the user wants to see all Gemeinde records +- The confidence is high (0.95) because the intent and entity are explicitly stated in the question \ No newline at end of file diff --git a/logs/debug/prompts/20251119-145449-033-intentanalysis_prompt.txt b/logs/debug/prompts/20251119-145449-033-intentanalysis_prompt.txt new file mode 100644 index 00000000..26aa68b0 --- /dev/null +++ b/logs/debug/prompts/20251119-145449-033-intentanalysis_prompt.txt @@ -0,0 +1,89 @@ + +Analyze the following user command and extract the intent, entity, and parameters. + +User Command: "Füge eine neue Parzelle hinzu die in Zürich liegt und die Adresse Ueberlandstrasse 11 8050 hat." + +Available intents: +- CREATE: User wants to create a new entity +- READ: User wants to read/query entities +- UPDATE: User wants to update an existing entity +- DELETE: User wants to delete an entity +- QUERY: User wants to execute a database query (SQL statements) + +Available entities and their fields: + +**Projekt** (Real estate project): +- id: string (primary key) +- mandateId: string (mandate ID) +- label: string (project designation/name) +- statusProzess: string enum (Eingang, Analyse, Studie, Planung, Baurechtsverfahren, Umsetzung, Archiv) +- perimeter: GeoPolylinie (geographic boundary, JSONB) +- baulinie: GeoPolylinie (building line, JSONB) +- parzellen: List[Parzelle] (plots belonging to project, JSONB) +- dokumente: List[Dokument] (documents, JSONB) +- kontextInformationen: List[Kontext] (context info, JSONB) + +**Parzelle** (Plot/parcel): +- id: string (primary key) +- mandateId: string (mandate ID) +- label: string (plot designation) +- strasseNr: string (street and house number) +- plz: string (postal code) +- kontextGemeinde: string (municipality ID, Foreign Key to Gemeinde table) +- bauzone: string (building zone, e.g. W3, WG2) +- az: float (Ausnützungsziffer) +- bz: float (Bebauungsziffer) +- vollgeschossZahl: int (number of allowed full floors) +- gebaeudehoeheMax: float (maximum building height in meters) +- laermschutzzone: string (noise protection zone) +- hochwasserschutzzone: string (flood protection zone) +- grundwasserschutzzone: string (groundwater protection zone) +- parzelleBebaut: JaNein enum (is plot built) +- parzelleErschlossen: JaNein enum (is plot developed) +- parzelleHanglage: JaNein enum (is plot on slope) + +**Important relationships:** +- Projekte contain Parzellen (projects have plots) +- Parzelle links to Gemeinde (via kontextGemeinde) +- Gemeinde links to Kanton (via id_kanton) +- Kanton links to Land (via id_land) +- Location queries (city, postal code) should use Parzelle.kontextGemeinde (municipality name will be resolved to ID) +- Projekt does NOT have location fields directly - location is stored in associated Parzellen + +Return a JSON object with the following structure: +{ + "intent": "CREATE|READ|UPDATE|DELETE|QUERY", + "entity": "Projekt|Parzelle|Dokument|Kanton|Gemeinde|null", + "parameters": { + // Extracted parameters from user input + // For CREATE/UPDATE: include all relevant fields using EXACT field names from above + // For READ: include filter criteria using EXACT field names (id, label, plz, kontextGemeinde, etc.) + // For DELETE: include entity ID if mentioned + // For QUERY: include queryText if SQL is detected + // IMPORTANT: Use only field names that exist in the entity definition above + }, + "confidence": 0.0-1.0 // Confidence score for the analysis +} + +Examples: +- Input: "Erstelle ein neues Projekt namens 'Hauptstrasse 42'" + Output: {"intent": "CREATE", "entity": "Projekt", "parameters": {"label": "Hauptstrasse 42"}, "confidence": 0.95} + +- Input: "Zeige mir alle Projekte" + Output: {"intent": "READ", "entity": "Projekt", "parameters": {}, "confidence": 0.9} + +- Input: "Zeige mir Projekte in Zürich" + Output: {"intent": "READ", "entity": "Parzelle", "parameters": {"kontextGemeinde": "Zürich"}, "confidence": 0.9} + Note: Location queries should query Parzelle, not Projekt directly + +- Input: "Zeige mir Parzellen mit PLZ 8000" + Output: {"intent": "READ", "entity": "Parzelle", "parameters": {"plz": "8000"}, "confidence": 0.95} + +- Input: "Aktualisiere Projekt XYZ mit Status 'Planung'" + Output: {"intent": "UPDATE", "entity": "Projekt", "parameters": {"id": "XYZ", "statusProzess": "Planung"}, "confidence": 0.85} + +- Input: "SELECT * FROM Projekt WHERE label = 'Test'" + Output: {"intent": "QUERY", "entity": null, "parameters": {"queryText": "SELECT * FROM Projekt WHERE label = 'Test'", "queryType": "sql"}, "confidence": 1.0} + +- Input: "Lösche Parzelle ABC" + Output: {"intent": "DELETE", "entity": "Parzelle", "parameters": {"id": "ABC"}, "confidence": 0.9} diff --git a/logs/debug/prompts/20251119-145452-034-intentanalysis_response.txt b/logs/debug/prompts/20251119-145452-034-intentanalysis_response.txt new file mode 100644 index 00000000..379e6909 --- /dev/null +++ b/logs/debug/prompts/20251119-145452-034-intentanalysis_response.txt @@ -0,0 +1,12 @@ +```json +{ + "intent": "CREATE", + "entity": "Parzelle", + "parameters": { + "kontextGemeinde": "Zürich", + "strasseNr": "Ueberlandstrasse 11", + "plz": "8050" + }, + "confidence": 0.95 +} +``` \ No newline at end of file diff --git a/modules/datamodels/datamodelRealEstate.py b/modules/datamodels/datamodelRealEstate.py new file mode 100644 index 00000000..fa9e717e --- /dev/null +++ b/modules/datamodels/datamodelRealEstate.py @@ -0,0 +1,667 @@ +""" +Real Estate data models for Architektur-Planungs-App. +Implements a general Swiss architecture planning data model. +(PEK is one example implementation, but the model is general-purpose) +""" + +from typing import List, Dict, Any, Optional, ForwardRef +from enum import Enum +from pydantic import BaseModel, Field +from modules.shared.attributeUtils import registerModelLabels +from modules.shared.timeUtils import getUtcTimestamp +import uuid + +# ===== Enums ===== + +class StatusProzess(str, Enum): + """Project process status""" + EINGANG = "Eingang" + ANALYSE = "Analyse" + STUDIE = "Studie" + PLANUNG = "Planung" + BAURECHTSVERFAHREN = "Baurechtsverfahren" + UMSETZUNG = "Umsetzung" + ARCHIV = "Archiv" + + +class DokumentTyp(str, Enum): + """Document type for categorization""" + KANTON_BAUREGLEMENT_AKTUELL = "kantonBaureglementAktuell" + KANTON_BAUREGLEMENT_REVISION = "kantonBaureglementRevision" + KANTON_BAUVERORDNUNG_AKTUELL = "kantonBauverordnungAktuell" + KANTON_BAUVERORDNUNG_REVISION = "kantonBauverordnungRevision" + GEMEINDE_BZO_AKTUELL = "gemeindeBzoAktuell" + GEMEINDE_BZO_REVISION = "gemeindeBzoRevision" + + +class JaNein(str, Enum): + """Three-valued state for optional yes/no questions""" + UNBEKANNT = "" # Empty string for unknown/not captured + JA = "Ja" + NEIN = "Nein" + + +class GeoTag(str, Enum): + """Geopoint categories""" + K1 = "K1" # Fixpunkt höchster Genauigkeit + K2 = "K2" # Fixpunkt mittlerer Genauigkeit + K3 = "K3" # Fixpunkt niedriger Genauigkeit + GEOMETER = "Geometer" # Vom Geometer vermessener Punkt + + +# ===== Helper Models (must be defined before main models) ===== + +class GeoPunkt(BaseModel): + """Represents a 3D point with reference.""" + koordinatensystem: str = Field( + description="Coordinate system (e.g. 'LV95', 'EPSG:2056')", + frontend_type="text", + frontend_readonly=False, + frontend_required=True, + ) + x: float = Field( + description="East value (E) [m], typically 2'480'000 - 2'840'000", + frontend_type="number", + frontend_readonly=False, + frontend_required=True, + ) + y: float = Field( + description="North value (N) [m], typically 1'070'000 - 1'300'000", + frontend_type="number", + frontend_readonly=False, + frontend_required=True, + ) + z: Optional[float] = Field( + None, + description="Height above sea level [m]", + frontend_type="number", + frontend_readonly=False, + frontend_required=False, + ) + referenz: Optional[GeoTag] = Field( + None, + description="Point categorization", + frontend_type="select", + frontend_readonly=False, + frontend_required=False, + ) + + +class GeoPolylinie(BaseModel): + """Represents a line or polygon from multiple GeoPunkte.""" + id: str = Field( + default_factory=lambda: str(uuid.uuid4()), + description="Primary key", + ) + closed: bool = Field( + description="Is the GeoPolylinie closed (polygon)?", + frontend_type="boolean", + frontend_readonly=False, + frontend_required=True, + ) + punkte: List[GeoPunkt] = Field( + default_factory=list, + description="List of GeoPunkte forming the GeoPolylinie", + frontend_type="json", + frontend_readonly=False, + frontend_required=True, + ) + + +class Dokument(BaseModel): + """Supporting data object for file and URL management with versioning.""" + id: str = Field( + default_factory=lambda: str(uuid.uuid4()), + description="Primary key", + frontend_type="text", + frontend_readonly=True, + frontend_required=False, + ) + mandateId: str = Field( + description="ID of the mandate this document belongs to", + frontend_type="text", + frontend_readonly=True, + frontend_required=False, + ) + label: str = Field( + description="Document label", + frontend_type="text", + frontend_readonly=False, + frontend_required=True, + ) + versionsbezeichnung: Optional[str] = Field( + None, + description="Version number or designation (e.g. 'v1.0', 'Rev. A')", + frontend_type="text", + frontend_readonly=False, + frontend_required=False, + ) + dokumentTyp: Optional[DokumentTyp] = Field( + None, + description="Document type", + frontend_type="select", + frontend_readonly=False, + frontend_required=False, + ) + dokumentReferenz: str = Field( + description="File path or URL", + frontend_type="text", + frontend_readonly=False, + frontend_required=True, + ) + quelle: Optional[str] = Field( + None, + description="Source of the document", + frontend_type="text", + frontend_readonly=False, + frontend_required=False, + ) + mimeType: Optional[str] = Field( + None, + description="MIME type of the document (e.g. 'application/pdf', 'image/png')", + frontend_type="text", + frontend_readonly=False, + frontend_required=False, + ) + kategorienTags: List[str] = Field( + default_factory=list, + description="Document categorization tags", + frontend_type="json", + frontend_readonly=False, + frontend_required=False, + ) + + +class Kontext(BaseModel): + """Supporting data object for flexible additional information.""" + id: str = Field( + default_factory=lambda: str(uuid.uuid4()), + description="Primary key", + ) + thema: str = Field( + description="Theme designation", + frontend_type="text", + frontend_readonly=False, + frontend_required=True, + ) + inhalt: str = Field( + description="Detailed information (text)", + frontend_type="textarea", + frontend_readonly=False, + frontend_required=True, + ) + + +class Land(BaseModel): + """National level administrative entity.""" + id: str = Field( + default_factory=lambda: str(uuid.uuid4()), + description="Primary key", + frontend_type="text", + frontend_readonly=True, + frontend_required=False, + ) + mandateId: str = Field( + description="ID of the mandate", + frontend_type="text", + frontend_readonly=True, + frontend_required=False, + ) + label: str = Field( + description="Country name (e.g. 'Schweiz')", + frontend_type="text", + frontend_readonly=False, + frontend_required=True, + ) + abk: Optional[str] = Field( + None, + description="Abbreviation (e.g. 'CH')", + frontend_type="text", + frontend_readonly=False, + frontend_required=False, + ) + dokumente: List[Dokument] = Field( + default_factory=list, + description="National laws/documents", + frontend_type="json", + frontend_readonly=False, + frontend_required=False, + ) + kontextInformationen: List[Kontext] = Field( + default_factory=list, + description="National context information", + frontend_type="json", + frontend_readonly=False, + frontend_required=False, + ) + + +class Kanton(BaseModel): + """Cantonal level administrative entity.""" + id: str = Field( + default_factory=lambda: str(uuid.uuid4()), + description="Primary key", + frontend_type="text", + frontend_readonly=True, + frontend_required=False, + ) + mandateId: str = Field( + description="ID of the mandate", + frontend_type="text", + frontend_readonly=True, + frontend_required=False, + ) + label: str = Field( + description="Canton name (e.g. 'Zürich')", + frontend_type="text", + frontend_readonly=False, + frontend_required=True, + ) + id_land: Optional[str] = Field( + None, + description="Land ID (Foreign Key) - eindeutiger Link zum Land, in welchem Land der Kanton liegt", + frontend_type="text", + frontend_readonly=False, + frontend_required=False, + ) + abk: Optional[str] = Field( + None, + description="Abbreviation (e.g. 'ZH')", + frontend_type="text", + frontend_readonly=False, + frontend_required=False, + ) + dokumente: List[Dokument] = Field( + default_factory=list, + description="Cantonal documents", + frontend_type="json", + frontend_readonly=False, + frontend_required=False, + ) + kontextInformationen: List[Kontext] = Field( + default_factory=list, + description="Canton-specific context information", + frontend_type="json", + frontend_readonly=False, + frontend_required=False, + ) + + +class Gemeinde(BaseModel): + """Municipal level administrative entity.""" + id: str = Field( + default_factory=lambda: str(uuid.uuid4()), + description="Primary key", + frontend_type="text", + frontend_readonly=True, + frontend_required=False, + ) + mandateId: str = Field( + description="ID of the mandate", + frontend_type="text", + frontend_readonly=True, + frontend_required=False, + ) + label: str = Field( + description="Municipality name (e.g. 'Zürich')", + frontend_type="text", + frontend_readonly=False, + frontend_required=True, + ) + id_kanton: Optional[str] = Field( + None, + description="Kanton ID (Foreign Key) - eindeutiger Link zum Kanton, in welchem Kanton die Gemeinde liegt", + frontend_type="text", + frontend_readonly=False, + frontend_required=False, + ) + plz: Optional[str] = Field( + None, + description="Postal code (for municipalities with multiple PLZ, this can be a main PLZ). Bei Gemeinden mit mehreren Postleitzahlen wird die konkrete PLZ der Parzelle im Attribut `plz` der Parzelle erfasst.", + frontend_type="text", + frontend_readonly=False, + frontend_required=False, + ) + dokumente: List[Dokument] = Field( + default_factory=list, + description="Municipal documents", + frontend_type="json", + frontend_readonly=False, + frontend_required=False, + ) + kontextInformationen: List[Kontext] = Field( + default_factory=list, + description="Municipality-specific context information", + frontend_type="json", + frontend_readonly=False, + frontend_required=False, + ) + + +# ===== Main Models (use ForwardRef for circular references) ===== + +# Forward references for circular dependencies +ParzelleRef = ForwardRef('Parzelle') + + +class Parzelle(BaseModel): + """Represents a plot with all building law properties.""" + id: str = Field( + default_factory=lambda: str(uuid.uuid4()), + description="Primary key", + frontend_type="text", + frontend_readonly=True, + frontend_required=False, + ) + mandateId: str = Field( + description="ID of the mandate", + frontend_type="text", + frontend_readonly=True, + frontend_required=False, + ) + + # Grunddaten + label: str = Field( + description="Plot designation", + frontend_type="text", + frontend_readonly=False, + frontend_required=True, + ) + parzellenAliasTags: List[str] = Field( + default_factory=list, + description="Additional plot names or field names", + frontend_type="json", + frontend_readonly=False, + frontend_required=False, + ) + eigentuemerschaft: Optional[str] = Field( + None, + description="Owner of the plot", + frontend_type="text", + frontend_readonly=False, + frontend_required=False, + ) + strasseNr: Optional[str] = Field( + None, + description="Street and house number", + frontend_type="text", + frontend_readonly=False, + frontend_required=False, + ) + plz: Optional[str] = Field( + None, + description="Postal code of the plot", + frontend_type="text", + frontend_readonly=False, + frontend_required=False, + ) + + # Geografischer Kontext + perimeter: Optional[GeoPolylinie] = Field( + None, + description="Plot boundary as closed GeoPolylinie", + frontend_type="json", + frontend_readonly=False, + frontend_required=False, + ) + baulinie: Optional[GeoPolylinie] = Field( + None, + description="Building line of the plot", + frontend_type="json", + frontend_readonly=False, + frontend_required=False, + ) + + kontextGemeinde: Optional[str] = Field( + None, + description="Municipality ID (Foreign Key)", + frontend_type="text", + frontend_readonly=False, + frontend_required=False, + ) + + # Bebauungsparameter + bauzone: Optional[str] = Field( + None, + description="Building zone designation (e.g. W3, WG2, etc.)", + frontend_type="text", + frontend_readonly=False, + frontend_required=False, + ) + az: Optional[float] = Field( + None, + description="Ausnützungsziffer", + frontend_type="number", + frontend_readonly=False, + frontend_required=False, + ) + bz: Optional[float] = Field( + None, + description="Bebauungsziffer", + frontend_type="number", + frontend_readonly=False, + frontend_required=False, + ) + vollgeschossZahl: Optional[int] = Field( + None, + description="Number of allowed full floors", + frontend_type="number", + frontend_readonly=False, + frontend_required=False, + ) + anrechenbarDachgeschoss: Optional[float] = Field( + None, + description="Accountable portion of attic (0.0 - 1.0)", + frontend_type="number", + frontend_readonly=False, + frontend_required=False, + ) + anrechenbarUntergeschoss: Optional[float] = Field( + None, + description="Accountable portion of basement (0.0 - 1.0)", + frontend_type="number", + frontend_readonly=False, + frontend_required=False, + ) + gebaeudehoeheMax: Optional[float] = Field( + None, + description="Maximum building height in meters", + frontend_type="number", + frontend_readonly=False, + frontend_required=False, + ) + + # Abstandsregelungen + regelnGrenzabstand: List[str] = Field( + default_factory=list, + description="Regulations for boundary distance", + frontend_type="json", + frontend_readonly=False, + frontend_required=False, + ) + regelnMehrlaengenzuschlag: List[str] = Field( + default_factory=list, + description="Regulations for additional length surcharge", + frontend_type="json", + frontend_readonly=False, + frontend_required=False, + ) + regelnMehrhoehenzuschlag: List[str] = Field( + default_factory=list, + description="Regulations for additional height surcharge", + frontend_type="json", + frontend_readonly=False, + frontend_required=False, + ) + + # Eigenschaften (Ja/Nein) + parzelleBebaut: Optional[JaNein] = Field( + None, + description="Is the plot built?", + frontend_type="select", + frontend_readonly=False, + frontend_required=False, + ) + parzelleErschlossen: Optional[JaNein] = Field( + None, + description="Is the plot developed?", + frontend_type="select", + frontend_readonly=False, + frontend_required=False, + ) + parzelleHanglage: Optional[JaNein] = Field( + None, + description="Is the plot on a slope?", + frontend_type="select", + frontend_readonly=False, + frontend_required=False, + ) + + # Schutzzonen + laermschutzzone: Optional[str] = Field( + None, + description="Noise protection zone (e.g. 'II')", + frontend_type="text", + frontend_readonly=False, + frontend_required=False, + ) + hochwasserschutzzone: Optional[str] = Field( + None, + description="Flood protection zone (e.g. 'tief')", + frontend_type="text", + frontend_readonly=False, + frontend_required=False, + ) + grundwasserschutzzone: Optional[str] = Field( + None, + description="Groundwater protection zone", + frontend_type="text", + frontend_readonly=False, + frontend_required=False, + ) + + # Beziehungen (stored as JSONB in database) + parzellenNachbarschaft: List[Dict[str, Any]] = Field( + default_factory=list, + description="Neighboring plots (stored as list of Parzelle IDs or full objects)", + frontend_type="json", + frontend_readonly=False, + frontend_required=False, + ) + dokumente: List[Dokument] = Field( + default_factory=list, + description="Plot-specific documents", + frontend_type="json", + frontend_readonly=False, + frontend_required=False, + ) + kontextInformationen: List[Kontext] = Field( + default_factory=list, + description="Plot-specific context information", + frontend_type="json", + frontend_readonly=False, + frontend_required=False, + ) + + +class Projekt(BaseModel): + """Core object representing a construction project.""" + id: str = Field( + default_factory=lambda: str(uuid.uuid4()), + description="Primary key", + frontend_type="text", + frontend_readonly=True, + frontend_required=False, + ) + mandateId: str = Field( + description="ID of the mandate", + frontend_type="text", + frontend_readonly=True, + frontend_required=False, + ) + label: str = Field( + description="Project designation", + frontend_type="text", + frontend_readonly=False, + frontend_required=True, + ) + statusProzess: Optional[StatusProzess] = Field( + None, + description="Project status", + frontend_type="select", + frontend_readonly=False, + frontend_required=False, + ) + perimeter: Optional[GeoPolylinie] = Field( + None, + description="Envelope of all plots in the project", + frontend_type="json", + frontend_readonly=False, + frontend_required=False, + ) + baulinie: Optional[GeoPolylinie] = Field( + None, + description="Building line of the project", + frontend_type="json", + frontend_readonly=False, + frontend_required=False, + ) + parzellen: List[Parzelle] = Field( + default_factory=list, + description="All plots of the project", + frontend_type="json", + frontend_readonly=False, + frontend_required=False, + ) + dokumente: List[Dokument] = Field( + default_factory=list, + description="Project-specific documents", + frontend_type="json", + frontend_readonly=False, + frontend_required=False, + ) + kontextInformationen: List[Kontext] = Field( + default_factory=list, + description="Project-specific context information", + frontend_type="json", + frontend_readonly=False, + frontend_required=False, + ) + + +# Resolve forward references +Parzelle.model_rebuild() +Projekt.model_rebuild() + + +# Register labels for frontend +registerModelLabels( + "Projekt", + {"en": "Project", "fr": "Projet", "de": "Projekt"}, + { + "id": {"en": "ID", "fr": "ID", "de": "ID"}, + "label": {"en": "Label", "fr": "Libellé", "de": "Bezeichnung"}, + "statusProzess": {"en": "Process Status", "fr": "Statut du processus", "de": "Prozessstatus"}, + "mandateId": {"en": "Mandate ID", "fr": "ID du mandat", "de": "Mandats-ID"}, + }, +) + +registerModelLabels( + "Parzelle", + {"en": "Plot", "fr": "Parcelle", "de": "Parzelle"}, + { + "id": {"en": "ID", "fr": "ID", "de": "ID"}, + "label": {"en": "Label", "fr": "Libellé", "de": "Bezeichnung"}, + "mandateId": {"en": "Mandate ID", "fr": "ID du mandat", "de": "Mandats-ID"}, + }, +) + +registerModelLabels( + "Dokument", + {"en": "Document", "fr": "Document", "de": "Dokument"}, + { + "id": {"en": "ID", "fr": "ID", "de": "ID"}, + "label": {"en": "Label", "fr": "Libellé", "de": "Bezeichnung"}, + }, +) + diff --git a/modules/features/realEstate/__init__.py b/modules/features/realEstate/__init__.py new file mode 100644 index 00000000..48368b52 --- /dev/null +++ b/modules/features/realEstate/__init__.py @@ -0,0 +1,4 @@ +""" +Real Estate feature module. +""" + diff --git a/modules/features/realEstate/mainRealEstate.py b/modules/features/realEstate/mainRealEstate.py new file mode 100644 index 00000000..5396a20a --- /dev/null +++ b/modules/features/realEstate/mainRealEstate.py @@ -0,0 +1,769 @@ +""" +Real Estate feature main logic. +Handles database operations with AI-powered natural language processing. +Stateless implementation without session management. +""" + +import logging +import json +from typing import Optional, Dict, Any +from modules.datamodels.datamodelUam import User +from modules.datamodels.datamodelRealEstate import ( + Projekt, + Parzelle, + StatusProzess, +) +from modules.services import getInterface as getServices +from modules.interfaces.interfaceDbRealEstateObjects import getInterface as getRealEstateInterface + +logger = logging.getLogger(__name__) + + +# ===== Direkte Query-Ausführung (stateless) ===== + +async def executeDirectQuery( + currentUser: User, + queryText: str, + parameters: Optional[Dict[str, Any]] = None, +) -> Dict[str, Any]: + """ + Execute a database query directly without session management. + + Args: + currentUser: Current authenticated user + queryText: SQL query text + parameters: Optional parameters for parameterized queries + + Returns: + Dictionary containing query result (rows, columns, rowCount) + + Note: + - No session or query history is saved + - Query is executed directly and result is returned + - For production, validate and sanitize queries before execution + - TODO: Implement actual database query execution via interface + """ + try: + logger.info(f"Executing direct query for user {currentUser.id} (mandate: {currentUser.mandateId})") + logger.debug(f"Query text: {queryText}") + if parameters: + logger.debug(f"Query parameters: {parameters}") + + # Execute query via Real Estate interface (stateless) + realEstateInterface = getRealEstateInterface(currentUser) + result = realEstateInterface.executeQuery(queryText, parameters) + + logger.info( + f"Query executed successfully: {result['rowCount']} rows in {result.get('executionTime', 0):.3f}s" + ) + + return { + "status": "success", + "rows": result["rows"], + "columns": result["columns"], + "rowCount": result["rowCount"], + "executionTime": result.get("executionTime", 0), + } + + except Exception as e: + logger.error(f"Error executing query: {str(e)}", exc_info=True) + raise + + +# ===== AI-basierte Intent-Erkennung und CRUD-Operationen ===== + +async def processNaturalLanguageCommand( + currentUser: User, + userInput: str, +) -> Dict[str, Any]: + """ + Process natural language user input and execute corresponding CRUD operations. + + Uses AI to analyze user intent and extract parameters, then executes the appropriate + CRUD operation through the interface. Works stateless without session management. + + Args: + currentUser: Current authenticated user + userInput: Natural language command from user + + Returns: + Dictionary containing operation result and metadata + + Example user inputs: + - "Erstelle ein neues Projekt namens 'Hauptstrasse 42'" + - "Zeige mir alle Projekte in Zürich" + - "Aktualisiere Projekt XYZ mit Status 'Planung'" + - "Lösche Parzelle ABC" + - "SELECT * FROM Projekt WHERE plz = '8000'" + """ + try: + logger.info(f"Processing natural language command for user {currentUser.id} (mandate: {currentUser.mandateId})") + logger.debug(f"User input: {userInput}") + + # Initialize services for AI access + services = getServices(currentUser, workflow=None) + aiService = services.ai + + # Step 1: Analyze user intent with AI + intentAnalysis = await analyzeUserIntent(aiService, userInput) + + logger.info(f"Intent analysis result: intent={intentAnalysis.get('intent')}, entity={intentAnalysis.get('entity')}") + + # Step 2: Execute CRUD operation based on intent + result = await executeIntentBasedOperation( + currentUser=currentUser, + intent=intentAnalysis["intent"], + entity=intentAnalysis.get("entity"), + parameters=intentAnalysis.get("parameters", {}), + ) + + return { + "success": True, + "intent": intentAnalysis["intent"], + "entity": intentAnalysis.get("entity"), + "result": result, + } + + except Exception as e: + logger.error(f"Error processing natural language command: {str(e)}", exc_info=True) + raise + + +async def analyzeUserIntent( + aiService, + userInput: str +) -> Dict[str, Any]: + """ + Use AI to analyze user input and extract intent, entity, and parameters. + + Args: + aiService: AI service instance + userInput: Natural language user input + + Returns: + Dictionary with 'intent', 'entity', and 'parameters' + """ + # Create a structured prompt for intent analysis with accurate field information + intentPrompt = f""" +Analyze the following user command and extract the intent, entity, and parameters. + +User Command: "{userInput}" + +Available intents: +- CREATE: User wants to create a new entity +- READ: User wants to read/query entities +- UPDATE: User wants to update an existing entity +- DELETE: User wants to delete an entity +- QUERY: User wants to execute a database query (SQL statements) + +Available entities and their fields: + +**Projekt** (Real estate project): +- id: string (primary key) +- mandateId: string (mandate ID) +- label: string (project designation/name) +- statusProzess: string enum (Eingang, Analyse, Studie, Planung, Baurechtsverfahren, Umsetzung, Archiv) +- perimeter: GeoPolylinie (geographic boundary, JSONB) +- baulinie: GeoPolylinie (building line, JSONB) +- parzellen: List[Parzelle] (plots belonging to project, JSONB) +- dokumente: List[Dokument] (documents, JSONB) +- kontextInformationen: List[Kontext] (context info, JSONB) + +**Parzelle** (Plot/parcel): +- id: string (primary key) +- mandateId: string (mandate ID) +- label: string (plot designation) +- strasseNr: string (street and house number) +- plz: string (postal code) +- kontextGemeinde: string (municipality ID, Foreign Key to Gemeinde table) +- bauzone: string (building zone, e.g. W3, WG2) +- az: float (Ausnützungsziffer) +- bz: float (Bebauungsziffer) +- vollgeschossZahl: int (number of allowed full floors) +- gebaeudehoeheMax: float (maximum building height in meters) +- laermschutzzone: string (noise protection zone) +- hochwasserschutzzone: string (flood protection zone) +- grundwasserschutzzone: string (groundwater protection zone) +- parzelleBebaut: JaNein enum (is plot built) +- parzelleErschlossen: JaNein enum (is plot developed) +- parzelleHanglage: JaNein enum (is plot on slope) + +**Important relationships:** +- Projekte contain Parzellen (projects have plots) +- Parzelle links to Gemeinde (via kontextGemeinde) +- Gemeinde links to Kanton (via id_kanton) +- Kanton links to Land (via id_land) +- Location queries (city, postal code) should use Parzelle.kontextGemeinde (municipality name will be resolved to ID) +- Projekt does NOT have location fields directly - location is stored in associated Parzellen + +Return a JSON object with the following structure: +{{ + "intent": "CREATE|READ|UPDATE|DELETE|QUERY", + "entity": "Projekt|Parzelle|Dokument|Kanton|Gemeinde|null", + "parameters": {{ + // Extracted parameters from user input + // For CREATE/UPDATE: include all relevant fields using EXACT field names from above + // For READ: include filter criteria using EXACT field names (id, label, plz, kontextGemeinde, etc.) + // For DELETE: include entity ID if mentioned + // For QUERY: include queryText if SQL is detected + // IMPORTANT: Use only field names that exist in the entity definition above + }}, + "confidence": 0.0-1.0 // Confidence score for the analysis +}} + +Examples: +- Input: "Erstelle ein neues Projekt namens 'Hauptstrasse 42'" + Output: {{"intent": "CREATE", "entity": "Projekt", "parameters": {{"label": "Hauptstrasse 42"}}, "confidence": 0.95}} + +- Input: "Zeige mir alle Projekte" + Output: {{"intent": "READ", "entity": "Projekt", "parameters": {{}}, "confidence": 0.9}} + +- Input: "Zeige mir Projekte in Zürich" + Output: {{"intent": "READ", "entity": "Parzelle", "parameters": {{"kontextGemeinde": "Zürich"}}, "confidence": 0.9}} + Note: Location queries should query Parzelle, not Projekt directly + +- Input: "Zeige mir Parzellen mit PLZ 8000" + Output: {{"intent": "READ", "entity": "Parzelle", "parameters": {{"plz": "8000"}}, "confidence": 0.95}} + +- Input: "Aktualisiere Projekt XYZ mit Status 'Planung'" + Output: {{"intent": "UPDATE", "entity": "Projekt", "parameters": {{"id": "XYZ", "statusProzess": "Planung"}}, "confidence": 0.85}} + +- Input: "SELECT * FROM Projekt WHERE label = 'Test'" + Output: {{"intent": "QUERY", "entity": null, "parameters": {{"queryText": "SELECT * FROM Projekt WHERE label = 'Test'", "queryType": "sql"}}, "confidence": 1.0}} + +- Input: "Lösche Parzelle ABC" + Output: {{"intent": "DELETE", "entity": "Parzelle", "parameters": {{"id": "ABC"}}, "confidence": 0.9}} +""" + + try: + # Use AI planning call for structured JSON response + response = await aiService.callAiPlanning( + prompt=intentPrompt, + debugType="intentanalysis" + ) + + # Extract JSON from response (handles markdown code blocks) + jsonStart = response.find('{') + jsonEnd = response.rfind('}') + 1 + + if jsonStart == -1 or jsonEnd == 0: + raise ValueError("No JSON found in AI response") + + jsonStr = response[jsonStart:jsonEnd] + + # Parse JSON response + intentData = json.loads(jsonStr) + + # Validate response structure + if "intent" not in intentData: + raise ValueError("Invalid intent analysis response: missing 'intent' field") + + # Ensure parameters exists + if "parameters" not in intentData: + intentData["parameters"] = {} + + logger.debug(f"Parsed intent analysis: {intentData}") + + return intentData + + except json.JSONDecodeError as e: + logger.error(f"Failed to parse AI intent analysis response: {e}") + logger.error(f"Raw response: {response}") + raise ValueError(f"AI returned invalid JSON: {str(e)}") + except Exception as e: + logger.error(f"Error analyzing user intent: {str(e)}", exc_info=True) + raise + + +async def executeIntentBasedOperation( + currentUser: User, + intent: str, + entity: Optional[str], + parameters: Dict[str, Any], +) -> Dict[str, Any]: + """ + Execute CRUD operation based on analyzed intent. + + Args: + currentUser: Current authenticated user + intent: Intent from AI analysis (CREATE, READ, UPDATE, DELETE, QUERY) + entity: Entity type from AI analysis + parameters: Extracted parameters from AI analysis + + Returns: + Operation result + + Note: + - TODO: Implement actual interface calls once datamodels are ready + - Currently returns test responses showing what would be executed + """ + try: + logger.info(f"Executing intent-based operation: intent={intent}, entity={entity}") + logger.debug(f"Parameters: {parameters}") + + if intent == "QUERY": + # Execute database query directly (stateless) + queryText = parameters.get("queryText", "") + + if not queryText: + raise ValueError("QUERY intent requires queryText in parameters") + + result = await executeDirectQuery( + currentUser=currentUser, + queryText=queryText, + parameters=parameters.get("queryParameters"), + ) + return result + + elif intent == "CREATE": + # Create new entity + realEstateInterface = getRealEstateInterface(currentUser) + + if entity == "Projekt": + # Create Projekt from parameters + projekt = Projekt( + mandateId=currentUser.mandateId, + label=parameters.get("label", ""), + statusProzess=StatusProzess(parameters.get("statusProzess", "EINGANG")) if parameters.get("statusProzess") else None, + ) + created = realEstateInterface.createProjekt(projekt) + return { + "operation": "CREATE", + "entity": "Projekt", + "result": created.model_dump() + } + + elif entity == "Parzelle": + # Create Parzelle from parameters + parzelle = Parzelle( + mandateId=currentUser.mandateId, + label=parameters.get("label", ""), + strasseNr=parameters.get("strasseNr"), + plz=parameters.get("plz"), + bauzone=parameters.get("bauzone"), + kontextGemeinde=parameters.get("kontextGemeinde"), + ) + created = realEstateInterface.createParzelle(parzelle) + return { + "operation": "CREATE", + "entity": "Parzelle", + "result": created.model_dump() + } + elif entity == "Gemeinde": + # Create Gemeinde from parameters + from modules.datamodels.datamodelRealEstate import Gemeinde + gemeinde = Gemeinde( + mandateId=currentUser.mandateId, + label=parameters.get("label", ""), + id_kanton=parameters.get("id_kanton"), + plz=parameters.get("plz"), + ) + created = realEstateInterface.createGemeinde(gemeinde) + return { + "operation": "CREATE", + "entity": "Gemeinde", + "result": created.model_dump() + } + elif entity == "Kanton": + # Create Kanton from parameters + from modules.datamodels.datamodelRealEstate import Kanton + kanton = Kanton( + mandateId=currentUser.mandateId, + label=parameters.get("label", ""), + id_land=parameters.get("id_land"), + abk=parameters.get("abk"), + ) + created = realEstateInterface.createKanton(kanton) + return { + "operation": "CREATE", + "entity": "Kanton", + "result": created.model_dump() + } + elif entity == "Land": + # Create Land from parameters + from modules.datamodels.datamodelRealEstate import Land + land = Land( + mandateId=currentUser.mandateId, + label=parameters.get("label", ""), + abk=parameters.get("abk"), + ) + created = realEstateInterface.createLand(land) + return { + "operation": "CREATE", + "entity": "Land", + "result": created.model_dump() + } + elif entity == "Dokument": + # Create Dokument from parameters + from modules.datamodels.datamodelRealEstate import Dokument + dokument = Dokument( + mandateId=currentUser.mandateId, + label=parameters.get("label", ""), + dokumentReferenz=parameters.get("dokumentReferenz", ""), + versionsbezeichnung=parameters.get("versionsbezeichnung"), + dokumentTyp=parameters.get("dokumentTyp"), + quelle=parameters.get("quelle"), + mimeType=parameters.get("mimeType"), + ) + created = realEstateInterface.createDokument(dokument) + return { + "operation": "CREATE", + "entity": "Dokument", + "result": created.model_dump() + } + else: + raise ValueError(f"CREATE operation not supported for entity: {entity}") + + elif intent == "READ": + # Read entities + realEstateInterface = getRealEstateInterface(currentUser) + + if entity == "Projekt": + projektId = parameters.get("id") + if projektId: + # Get single Projekt by ID + projekt = realEstateInterface.getProjekt(projektId) + if not projekt: + raise ValueError(f"Projekt {projektId} not found") + return { + "operation": "READ", + "entity": "Projekt", + "result": projekt.model_dump() + } + else: + # List all Projekte (with optional filters) + # Validate filter fields against Projekt model + validProjektFields = {"id", "mandateId", "label", "statusProzess"} + recordFilter = { + k: v for k, v in parameters.items() + if k != "id" and k in validProjektFields + } + # Warn about invalid fields + invalidFields = {k: v for k, v in parameters.items() if k not in validProjektFields and k != "id"} + if invalidFields: + logger.warning(f"Invalid filter fields for Projekt ignored: {list(invalidFields.keys())}") + logger.info("Note: Location queries should use Parzelle entity, not Projekt") + + projekte = realEstateInterface.getProjekte(recordFilter=recordFilter if recordFilter else None) + return { + "operation": "READ", + "entity": "Projekt", + "result": [p.model_dump() for p in projekte] + } + elif entity == "Parzelle": + parzelleId = parameters.get("id") + if parzelleId: + # Get single Parzelle by ID + parzelle = realEstateInterface.getParzelle(parzelleId) + if not parzelle: + raise ValueError(f"Parzelle {parzelleId} not found") + return { + "operation": "READ", + "entity": "Parzelle", + "result": parzelle.model_dump() + } + else: + # List all Parzellen (with optional filters) + # Validate filter fields against Parzelle model + # Note: kontextKanton and kontextLand are NOT direct fields on Parzelle + # Parzelle links to Gemeinde, Gemeinde links to Kanton, Kanton links to Land + validParzelleFields = { + "id", "mandateId", "label", "strasseNr", "plz", + "kontextGemeinde", # Only direct link - Gemeinde → Kanton → Land + "bauzone", "az", "bz", "vollgeschossZahl", "gebaeudehoeheMax", + "laermschutzzone", "hochwasserschutzzone", "grundwasserschutzzone", + "parzelleBebaut", "parzelleErschlossen", "parzelleHanglage" + } + recordFilter = { + k: v for k, v in parameters.items() + if k != "id" and k in validParzelleFields + } + # Warn about invalid fields + invalidFields = {k: v for k, v in parameters.items() if k not in validParzelleFields and k != "id"} + if invalidFields: + logger.warning(f"Invalid filter fields for Parzelle ignored: {list(invalidFields.keys())}") + + parzellen = realEstateInterface.getParzellen(recordFilter=recordFilter if recordFilter else None) + return { + "operation": "READ", + "entity": "Parzelle", + "result": [p.model_dump() for p in parzellen] + } + elif entity == "Gemeinde": + from modules.datamodels.datamodelRealEstate import Gemeinde + gemeindeId = parameters.get("id") + if gemeindeId: + gemeinde = realEstateInterface.getGemeinde(gemeindeId) + if not gemeinde: + raise ValueError(f"Gemeinde {gemeindeId} not found") + return { + "operation": "READ", + "entity": "Gemeinde", + "result": gemeinde.model_dump() + } + else: + recordFilter = {k: v for k, v in parameters.items() if k != "id"} + gemeinden = realEstateInterface.getGemeinden(recordFilter=recordFilter if recordFilter else None) + return { + "operation": "READ", + "entity": "Gemeinde", + "result": [g.model_dump() for g in gemeinden] + } + elif entity == "Kanton": + from modules.datamodels.datamodelRealEstate import Kanton + kantonId = parameters.get("id") + if kantonId: + kanton = realEstateInterface.getKanton(kantonId) + if not kanton: + raise ValueError(f"Kanton {kantonId} not found") + return { + "operation": "READ", + "entity": "Kanton", + "result": kanton.model_dump() + } + else: + recordFilter = {k: v for k, v in parameters.items() if k != "id"} + kantone = realEstateInterface.getKantone(recordFilter=recordFilter if recordFilter else None) + return { + "operation": "READ", + "entity": "Kanton", + "result": [k.model_dump() for k in kantone] + } + elif entity == "Land": + from modules.datamodels.datamodelRealEstate import Land + landId = parameters.get("id") + if landId: + land = realEstateInterface.getLand(landId) + if not land: + raise ValueError(f"Land {landId} not found") + return { + "operation": "READ", + "entity": "Land", + "result": land.model_dump() + } + else: + recordFilter = {k: v for k, v in parameters.items() if k != "id"} + laender = realEstateInterface.getLaender(recordFilter=recordFilter if recordFilter else None) + return { + "operation": "READ", + "entity": "Land", + "result": [l.model_dump() for l in laender] + } + elif entity == "Dokument": + from modules.datamodels.datamodelRealEstate import Dokument + dokumentId = parameters.get("id") + if dokumentId: + dokument = realEstateInterface.getDokument(dokumentId) + if not dokument: + raise ValueError(f"Dokument {dokumentId} not found") + return { + "operation": "READ", + "entity": "Dokument", + "result": dokument.model_dump() + } + else: + recordFilter = {k: v for k, v in parameters.items() if k != "id"} + dokumente = realEstateInterface.getDokumente(recordFilter=recordFilter if recordFilter else None) + return { + "operation": "READ", + "entity": "Dokument", + "result": [d.model_dump() for d in dokumente] + } + else: + raise ValueError(f"READ operation not supported for entity: {entity}") + + elif intent == "UPDATE": + # Update existing entity + realEstateInterface = getRealEstateInterface(currentUser) + + if entity == "Projekt": + projektId = parameters.get("id") + if not projektId: + raise ValueError("UPDATE operation requires entity ID") + + # Get existing projekt + projekt = realEstateInterface.getProjekt(projektId) + if not projekt: + raise ValueError(f"Projekt {projektId} not found") + + # Update fields + updateData = {k: v for k, v in parameters.items() if k != "id"} + updated = realEstateInterface.updateProjekt(projektId, updateData) + return { + "operation": "UPDATE", + "entity": "Projekt", + "result": updated.model_dump() + } + elif entity == "Parzelle": + parzelleId = parameters.get("id") + if not parzelleId: + raise ValueError("UPDATE operation requires entity ID") + + # Get existing parzelle + parzelle = realEstateInterface.getParzelle(parzelleId) + if not parzelle: + raise ValueError(f"Parzelle {parzelleId} not found") + + # Update fields + updateData = {k: v for k, v in parameters.items() if k != "id"} + updated = realEstateInterface.updateParzelle(parzelleId, updateData) + return { + "operation": "UPDATE", + "entity": "Parzelle", + "result": updated.model_dump() + } + elif entity == "Gemeinde": + from modules.datamodels.datamodelRealEstate import Gemeinde + gemeindeId = parameters.get("id") + if not gemeindeId: + raise ValueError("UPDATE operation requires entity ID") + + gemeinde = realEstateInterface.getGemeinde(gemeindeId) + if not gemeinde: + raise ValueError(f"Gemeinde {gemeindeId} not found") + + updateData = {k: v for k, v in parameters.items() if k != "id"} + updated = realEstateInterface.updateGemeinde(gemeindeId, updateData) + return { + "operation": "UPDATE", + "entity": "Gemeinde", + "result": updated.model_dump() + } + elif entity == "Kanton": + from modules.datamodels.datamodelRealEstate import Kanton + kantonId = parameters.get("id") + if not kantonId: + raise ValueError("UPDATE operation requires entity ID") + + kanton = realEstateInterface.getKanton(kantonId) + if not kanton: + raise ValueError(f"Kanton {kantonId} not found") + + updateData = {k: v for k, v in parameters.items() if k != "id"} + updated = realEstateInterface.updateKanton(kantonId, updateData) + return { + "operation": "UPDATE", + "entity": "Kanton", + "result": updated.model_dump() + } + elif entity == "Land": + from modules.datamodels.datamodelRealEstate import Land + landId = parameters.get("id") + if not landId: + raise ValueError("UPDATE operation requires entity ID") + + land = realEstateInterface.getLand(landId) + if not land: + raise ValueError(f"Land {landId} not found") + + updateData = {k: v for k, v in parameters.items() if k != "id"} + updated = realEstateInterface.updateLand(landId, updateData) + return { + "operation": "UPDATE", + "entity": "Land", + "result": updated.model_dump() + } + elif entity == "Dokument": + from modules.datamodels.datamodelRealEstate import Dokument + dokumentId = parameters.get("id") + if not dokumentId: + raise ValueError("UPDATE operation requires entity ID") + + dokument = realEstateInterface.getDokument(dokumentId) + if not dokument: + raise ValueError(f"Dokument {dokumentId} not found") + + updateData = {k: v for k, v in parameters.items() if k != "id"} + updated = realEstateInterface.updateDokument(dokumentId, updateData) + return { + "operation": "UPDATE", + "entity": "Dokument", + "result": updated.model_dump() + } + else: + raise ValueError(f"UPDATE operation not supported for entity: {entity}") + + elif intent == "DELETE": + # Delete entity + realEstateInterface = getRealEstateInterface(currentUser) + + if entity == "Projekt": + projektId = parameters.get("id") + if not projektId: + raise ValueError("DELETE operation requires entity ID") + + success = realEstateInterface.deleteProjekt(projektId) + return { + "operation": "DELETE", + "entity": "Projekt", + "success": success + } + elif entity == "Parzelle": + parzelleId = parameters.get("id") + if not parzelleId: + raise ValueError("DELETE operation requires entity ID") + + success = realEstateInterface.deleteParzelle(parzelleId) + return { + "operation": "DELETE", + "entity": "Parzelle", + "success": success + } + elif entity == "Gemeinde": + from modules.datamodels.datamodelRealEstate import Gemeinde + gemeindeId = parameters.get("id") + if not gemeindeId: + raise ValueError("DELETE operation requires entity ID") + + success = realEstateInterface.deleteGemeinde(gemeindeId) + return { + "operation": "DELETE", + "entity": "Gemeinde", + "success": success + } + elif entity == "Kanton": + from modules.datamodels.datamodelRealEstate import Kanton + kantonId = parameters.get("id") + if not kantonId: + raise ValueError("DELETE operation requires entity ID") + + success = realEstateInterface.deleteKanton(kantonId) + return { + "operation": "DELETE", + "entity": "Kanton", + "success": success + } + elif entity == "Land": + from modules.datamodels.datamodelRealEstate import Land + landId = parameters.get("id") + if not landId: + raise ValueError("DELETE operation requires entity ID") + + success = realEstateInterface.deleteLand(landId) + return { + "operation": "DELETE", + "entity": "Land", + "success": success + } + elif entity == "Dokument": + from modules.datamodels.datamodelRealEstate import Dokument + dokumentId = parameters.get("id") + if not dokumentId: + raise ValueError("DELETE operation requires entity ID") + + success = realEstateInterface.deleteDokument(dokumentId) + return { + "operation": "DELETE", + "entity": "Dokument", + "success": success + } + else: + raise ValueError(f"DELETE operation not supported for entity: {entity}") + + else: + raise ValueError(f"Unknown intent: {intent}") + + except Exception as e: + logger.error(f"Error executing intent-based operation: {str(e)}", exc_info=True) + raise + diff --git a/modules/interfaces/interfaceDbRealEstateAccess.py b/modules/interfaces/interfaceDbRealEstateAccess.py new file mode 100644 index 00000000..9a25293f --- /dev/null +++ b/modules/interfaces/interfaceDbRealEstateAccess.py @@ -0,0 +1,87 @@ +""" +Access control for Real Estate interface. +Handles user access management and permission checks. +""" + +import logging +from typing import Dict, Any, List, Optional +from modules.datamodels.datamodelUam import User, UserPrivilege + +logger = logging.getLogger(__name__) + + +class RealEstateAccess: + """ + Access control class for Real Estate interface. + Handles user access management and permission checks. + """ + + def __init__(self, currentUser: User, db): + """Initialize with user context.""" + self.currentUser = currentUser + self.mandateId = currentUser.mandateId + self.userId = currentUser.id + self.privilege = currentUser.privilege + + if not self.mandateId or not self.userId: + raise ValueError("Invalid user context: mandateId and userId are required") + + self.db = db + + def uam(self, model_class: type, recordset: List[Dict[str, Any]]) -> List[Dict[str, Any]]: + """ + Unified user access management function that filters data based on user privileges. + + Args: + model_class: Pydantic model class for the table + recordset: Recordset to filter based on access rules + + Returns: + Filtered recordset with access control attributes + """ + filtered_records = [] + + # System admins see all records + if self.privilege == UserPrivilege.SYSADMIN: + filtered_records = recordset + # Admins see records in their mandate + elif self.privilege == UserPrivilege.ADMIN: + filtered_records = [r for r in recordset if r.get("mandateId", "-") == self.mandateId] + # Regular users see only their records + else: + filtered_records = [ + r for r in recordset + if r.get("mandateId", "-") == self.mandateId and r.get("_createdBy") == self.userId + ] + + # Add access control attributes + for record in filtered_records: + record["_hideView"] = False + record["_hideEdit"] = not self.canModify(model_class, record.get("id")) + record["_hideDelete"] = not self.canModify(model_class, record.get("id")) + + return filtered_records + + def canModify(self, model_class: type, recordId: Optional[str] = None) -> bool: + """Checks if the current user can modify records.""" + if self.privilege == UserPrivilege.SYSADMIN: + return True + + if recordId is not None: + records = self.db.getRecordset(model_class, recordFilter={"id": recordId}) + if not records: + return False + + record = records[0] + + if self.privilege == UserPrivilege.ADMIN and record.get("mandateId", "-") == self.mandateId: + return True + + if (record.get("mandateId", "-") == self.mandateId and + record.get("_createdBy") == self.userId): + return True + + return False + else: + return True # Regular users can create records + diff --git a/modules/interfaces/interfaceDbRealEstateObjects.py b/modules/interfaces/interfaceDbRealEstateObjects.py new file mode 100644 index 00000000..2c6d1554 --- /dev/null +++ b/modules/interfaces/interfaceDbRealEstateObjects.py @@ -0,0 +1,713 @@ +""" +Interface to Real Estate database objects. +Uses PostgreSQL connector for data access with user/mandate filtering. +Handles CRUD operations on Real Estate entities (Projekt, Parzelle, etc.). +""" + +import logging +from typing import Dict, Any, List, Optional +from modules.datamodels.datamodelRealEstate import ( + Projekt, + Parzelle, + Dokument, + Kanton, + Gemeinde, + Land, + GeoPolylinie, + GeoPunkt, + Kontext, + StatusProzess, +) +from modules.datamodels.datamodelUam import User +from modules.connectors.connectorDbPostgre import DatabaseConnector +from modules.shared.configuration import APP_CONFIG +# Import Access-Klasse aus separater Datei +from modules.interfaces.interfaceDbRealEstateAccess import RealEstateAccess + +logger = logging.getLogger(__name__) + +# Singleton factory for Real Estate interfaces +_realEstateInterfaces = {} + + +class RealEstateObjects: + """ + Interface to Real Estate database objects. + Uses PostgreSQL connector for data access with user/mandate filtering. + Handles CRUD operations on Real Estate entities. + """ + + def __init__(self, currentUser: Optional[User] = None): + """Initializes the Real Estate Interface.""" + self.currentUser = currentUser + self.userId = currentUser.id if currentUser else None + self.mandateId = currentUser.mandateId if currentUser else None + self.access = None + + # Initialize database + self._initializeDatabase() + + # Set user context if provided + if currentUser: + self.setUserContext(currentUser) + + def _initializeDatabase(self): + """Initialize PostgreSQL database connection.""" + try: + # Get database configuration from environment + dbHost = APP_CONFIG.get("DB_REALESTATE_HOST", "localhost") + dbDatabase = APP_CONFIG.get("DB_REALESTATE_DATABASE", "poweron_realestate") + dbUser = APP_CONFIG.get("DB_REALESTATE_USER") + dbPassword = APP_CONFIG.get("DB_REALESTATE_PASSWORD_SECRET") + dbPort = int(APP_CONFIG.get("DB_REALESTATE_PORT", 5432)) + + # Initialize database connector + self.db = DatabaseConnector( + dbHost=dbHost, + dbDatabase=dbDatabase, + dbUser=dbUser, + dbPassword=dbPassword, + dbPort=dbPort, + userId=self.userId if self.userId else None, + ) + + # Initialize database system (creates database and system table if needed) + # Note: This is also called in DatabaseConnector.__init__, but we call it explicitly + # for consistency with other interfaces and to ensure proper initialization + self.db.initDbSystem() + + # Ensure all supporting tables are created (Land, Kanton, Gemeinde, Dokument) + # These tables are needed for foreign key relationships + self._ensureSupportingTablesExist() + + logger.info(f"Real Estate database connector initialized for database: {dbDatabase}") + except Exception as e: + logger.error(f"Error initializing Real Estate database: {e}") + raise + + def _ensureSupportingTablesExist(self): + """Ensure all supporting tables (Land, Kanton, Gemeinde, Dokument) are created.""" + try: + # These tables are created on-demand when first accessed, but we ensure they exist here + # to avoid errors when resolving location names to IDs + self.db._ensureTableExists(Land) + self.db._ensureTableExists(Kanton) + self.db._ensureTableExists(Gemeinde) + self.db._ensureTableExists(Dokument) + logger.debug("Supporting tables (Land, Kanton, Gemeinde, Dokument) verified/created") + except Exception as e: + logger.warning(f"Error ensuring supporting tables exist: {e}") + # Don't raise - tables will be created on-demand anyway + + def setUserContext(self, currentUser: User): + """Sets the user context for the interface.""" + self.currentUser = currentUser + self.userId = currentUser.id + self.mandateId = currentUser.mandateId + + if not self.userId or not self.mandateId: + raise ValueError("Invalid user context: id and mandateId are required") + + # Initialize access control + self.access = RealEstateAccess(self.currentUser, self.db) + + # Update database context + self.db.updateContext(self.userId) + + # ===== Projekt Methods ===== + + def createProjekt(self, projekt: Projekt) -> Projekt: + """Create a new project.""" + # Ensure mandateId is set + if not projekt.mandateId: + projekt.mandateId = self.mandateId + + # Apply access control + self.access.uam(Projekt, []) + + # Save to database + self.db.recordCreate(Projekt, projekt.model_dump()) + + return projekt + + def getProjekt(self, projektId: str) -> Optional[Projekt]: + """Get a project by ID.""" + records = self.db.getRecordset( + Projekt, + recordFilter={"id": projektId} + ) + + if not records: + return None + + # Apply access control + filtered = self.access.uam(Projekt, records) + + if not filtered: + return None + + return Projekt(**filtered[0]) + + def getProjekte(self, recordFilter: Optional[Dict[str, Any]] = None) -> List[Projekt]: + """Get all projects matching the filter.""" + records = self.db.getRecordset(Projekt, recordFilter=recordFilter or {}) + + # Apply access control + filtered = self.access.uam(Projekt, records) + + return [Projekt(**r) for r in filtered] + + def updateProjekt(self, projektId: str, updateData: Dict[str, Any]) -> Optional[Projekt]: + """Update a project.""" + projekt = self.getProjekt(projektId) + if not projekt: + return None + + # Check if user can modify + if not self.access.canModify(Projekt, projektId): + raise PermissionError(f"User {self.userId} cannot modify project {projektId}") + + # Update fields + for key, value in updateData.items(): + if hasattr(projekt, key): + setattr(projekt, key, value) + + # Save to database + self.db.recordModify(Projekt, projektId, projekt.model_dump()) + + return projekt + + def deleteProjekt(self, projektId: str) -> bool: + """Delete a project.""" + projekt = self.getProjekt(projektId) + if not projekt: + return False + + # Check if user can modify + if not self.access.canModify(Projekt, projektId): + raise PermissionError(f"User {self.userId} cannot delete project {projektId}") + + return self.db.recordDelete(Projekt, projektId) + + # ===== Parzelle Methods ===== + + def createParzelle(self, parzelle: Parzelle) -> Parzelle: + """Create a new plot.""" + if not parzelle.mandateId: + parzelle.mandateId = self.mandateId + + self.access.uam(Parzelle, []) + self.db.recordCreate(Parzelle, parzelle.model_dump()) + + return parzelle + + def getParzelle(self, parzelleId: str) -> Optional[Parzelle]: + """Get a plot by ID.""" + records = self.db.getRecordset( + Parzelle, + recordFilter={"id": parzelleId} + ) + + if not records: + return None + + filtered = self.access.uam(Parzelle, records) + + if not filtered: + return None + + return Parzelle(**filtered[0]) + + def getParzellen(self, recordFilter: Optional[Dict[str, Any]] = None) -> List[Parzelle]: + """Get all plots matching the filter.""" + # Resolve location names to IDs if needed + if recordFilter: + recordFilter = self._resolveLocationFilters(recordFilter) + + records = self.db.getRecordset(Parzelle, recordFilter=recordFilter or {}) + + # Apply access control + filtered = self.access.uam(Parzelle, records) + + return [Parzelle(**r) for r in filtered] + + def _resolveLocationFilters(self, recordFilter: Dict[str, Any]) -> Dict[str, Any]: + """ + Resolve location names to IDs for foreign key fields. + Only handles kontextGemeinde (Parzelle → Gemeinde). + Note: Parzelle does NOT have direct links to Kanton or Land. + The relationship is: Parzelle → Gemeinde → Kanton → Land + """ + resolvedFilter = recordFilter.copy() + + # Resolve Gemeinde name to ID + # This is the only direct location link on Parzelle + if "kontextGemeinde" in resolvedFilter: + gemeindeValue = resolvedFilter["kontextGemeinde"] + # Check if it's a name (not a UUID-like string) + if not self._isUUID(gemeindeValue): + gemeindeId = self._resolveGemeindeByName(gemeindeValue) + if gemeindeId: + resolvedFilter["kontextGemeinde"] = gemeindeId + logger.debug(f"Resolved Gemeinde name '{gemeindeValue}' to ID '{gemeindeId}'") + else: + logger.warning(f"Gemeinde '{gemeindeValue}' not found, filter may return no results") + # Keep the original value - query will return empty if not found + + # Note: kontextKanton and kontextLand are NOT fields on Parzelle + # If they appear in the filter, they will be filtered out by the validation in mainRealEstate.py + + return resolvedFilter + + def _isUUID(self, value: str) -> bool: + """Check if a string looks like a UUID.""" + import re + uuid_pattern = re.compile(r'^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$', re.IGNORECASE) + return bool(uuid_pattern.match(value)) + + def _resolveGemeindeByName(self, name: str) -> Optional[str]: + """Resolve Gemeinde name to ID by looking up in Gemeinde table.""" + try: + # First try exact match + gemeinden = self.db.getRecordset( + Gemeinde, + recordFilter={"label": name} + ) + if gemeinden: + gemeindeId = gemeinden[0].get("id") + logger.debug(f"Found Gemeinde '{name}' with ID '{gemeindeId}'") + return gemeindeId + + # If no exact match, try case-insensitive search via SQL query + # This handles cases where the name might have different casing + self.db._ensure_connection() + with self.db.connection.cursor() as cursor: + cursor.execute( + 'SELECT "id" FROM "Gemeinde" WHERE LOWER("label") = LOWER(%s) LIMIT 1', + (name,) + ) + result = cursor.fetchone() + if result: + # psycopg2 returns tuples, so result[0] is the id + gemeindeId = result[0] + logger.debug(f"Found Gemeinde '{name}' (case-insensitive) with ID '{gemeindeId}'") + return gemeindeId + + logger.warning(f"Gemeinde '{name}' not found in database") + return None + except Exception as e: + logger.error(f"Error resolving Gemeinde by name '{name}': {e}", exc_info=True) + return None + + def _resolveKantonByName(self, name: str) -> Optional[str]: + """Resolve Kanton name to ID by looking up in Kanton table.""" + try: + # First try exact match + kantone = self.db.getRecordset( + Kanton, + recordFilter={"label": name} + ) + if kantone: + kantonId = kantone[0].get("id") + logger.debug(f"Found Kanton '{name}' with ID '{kantonId}'") + return kantonId + + # Try case-insensitive search + self.db._ensure_connection() + with self.db.connection.cursor() as cursor: + cursor.execute( + 'SELECT "id" FROM "Kanton" WHERE LOWER("label") = LOWER(%s) LIMIT 1', + (name,) + ) + result = cursor.fetchone() + if result: + # psycopg2 returns tuples, so result[0] is the id + kantonId = result[0] + logger.debug(f"Found Kanton '{name}' (case-insensitive) with ID '{kantonId}'") + return kantonId + + logger.warning(f"Kanton '{name}' not found in database") + return None + except Exception as e: + logger.error(f"Error resolving Kanton by name '{name}': {e}", exc_info=True) + return None + + def _resolveLandByName(self, name: str) -> Optional[str]: + """Resolve Land name to ID by looking up in Land table.""" + try: + # First try exact match + laender = self.db.getRecordset( + Land, + recordFilter={"label": name} + ) + if laender: + landId = laender[0].get("id") + logger.debug(f"Found Land '{name}' with ID '{landId}'") + return landId + + # Try case-insensitive search + self.db._ensure_connection() + with self.db.connection.cursor() as cursor: + cursor.execute( + 'SELECT "id" FROM "Land" WHERE LOWER("label") = LOWER(%s) LIMIT 1', + (name,) + ) + result = cursor.fetchone() + if result: + # psycopg2 returns tuples, so result[0] is the id + landId = result[0] + logger.debug(f"Found Land '{name}' (case-insensitive) with ID '{landId}'") + return landId + + logger.warning(f"Land '{name}' not found in database") + return None + except Exception as e: + logger.error(f"Error resolving Land by name '{name}': {e}", exc_info=True) + return None + + def updateParzelle(self, parzelleId: str, updateData: Dict[str, Any]) -> Optional[Parzelle]: + """Update a plot.""" + parzelle = self.getParzelle(parzelleId) + if not parzelle: + return None + + if not self.access.canModify(Parzelle, parzelleId): + raise PermissionError(f"User {self.userId} cannot modify plot {parzelleId}") + + for key, value in updateData.items(): + if hasattr(parzelle, key): + setattr(parzelle, key, value) + + self.db.recordModify(Parzelle, parzelleId, parzelle.model_dump()) + + return parzelle + + def deleteParzelle(self, parzelleId: str) -> bool: + """Delete a plot.""" + parzelle = self.getParzelle(parzelleId) + if not parzelle: + return False + + if not self.access.canModify(Parzelle, parzelleId): + raise PermissionError(f"User {self.userId} cannot delete plot {parzelleId}") + + return self.db.recordDelete(Parzelle, parzelleId) + + # ===== Dokument Methods ===== + + def createDokument(self, dokument: Dokument) -> Dokument: + """Create a new document.""" + if not dokument.mandateId: + dokument.mandateId = self.mandateId + + self.access.uam(Dokument, []) + self.db.recordCreate(Dokument, dokument.model_dump()) + + return dokument + + def getDokument(self, dokumentId: str) -> Optional[Dokument]: + """Get a document by ID.""" + records = self.db.getRecordset( + Dokument, + recordFilter={"id": dokumentId} + ) + + if not records: + return None + + filtered = self.access.uam(Dokument, records) + + if not filtered: + return None + + return Dokument(**filtered[0]) + + def getDokumente(self, recordFilter: Optional[Dict[str, Any]] = None) -> List[Dokument]: + """Get all documents matching the filter.""" + records = self.db.getRecordset(Dokument, recordFilter=recordFilter or {}) + filtered = self.access.uam(Dokument, records) + return [Dokument(**r) for r in filtered] + + def updateDokument(self, dokumentId: str, updateData: Dict[str, Any]) -> Optional[Dokument]: + """Update a document.""" + dokument = self.getDokument(dokumentId) + if not dokument: + return None + + if not self.access.canModify(Dokument, dokumentId): + raise PermissionError(f"User {self.userId} cannot modify document {dokumentId}") + + for key, value in updateData.items(): + if hasattr(dokument, key): + setattr(dokument, key, value) + + self.db.recordModify(Dokument, dokumentId, dokument.model_dump()) + return dokument + + def deleteDokument(self, dokumentId: str) -> bool: + """Delete a document.""" + dokument = self.getDokument(dokumentId) + if not dokument: + return False + + if not self.access.canModify(Dokument, dokumentId): + raise PermissionError(f"User {self.userId} cannot delete document {dokumentId}") + + return self.db.recordDelete(Dokument, dokumentId) + + # ===== Gemeinde Methods ===== + + def createGemeinde(self, gemeinde: Gemeinde) -> Gemeinde: + """Create a new municipality.""" + if not gemeinde.mandateId: + gemeinde.mandateId = self.mandateId + + self.access.uam(Gemeinde, []) + self.db.recordCreate(Gemeinde, gemeinde.model_dump()) + + return gemeinde + + def getGemeinde(self, gemeindeId: str) -> Optional[Gemeinde]: + """Get a municipality by ID.""" + records = self.db.getRecordset( + Gemeinde, + recordFilter={"id": gemeindeId} + ) + + if not records: + return None + + filtered = self.access.uam(Gemeinde, records) + + if not filtered: + return None + + return Gemeinde(**filtered[0]) + + def getGemeinden(self, recordFilter: Optional[Dict[str, Any]] = None) -> List[Gemeinde]: + """Get all municipalities matching the filter.""" + records = self.db.getRecordset(Gemeinde, recordFilter=recordFilter or {}) + filtered = self.access.uam(Gemeinde, records) + return [Gemeinde(**r) for r in filtered] + + def updateGemeinde(self, gemeindeId: str, updateData: Dict[str, Any]) -> Optional[Gemeinde]: + """Update a municipality.""" + gemeinde = self.getGemeinde(gemeindeId) + if not gemeinde: + return None + + if not self.access.canModify(Gemeinde, gemeindeId): + raise PermissionError(f"User {self.userId} cannot modify municipality {gemeindeId}") + + for key, value in updateData.items(): + if hasattr(gemeinde, key): + setattr(gemeinde, key, value) + + self.db.recordModify(Gemeinde, gemeindeId, gemeinde.model_dump()) + return gemeinde + + def deleteGemeinde(self, gemeindeId: str) -> bool: + """Delete a municipality.""" + gemeinde = self.getGemeinde(gemeindeId) + if not gemeinde: + return False + + if not self.access.canModify(Gemeinde, gemeindeId): + raise PermissionError(f"User {self.userId} cannot delete municipality {gemeindeId}") + + return self.db.recordDelete(Gemeinde, gemeindeId) + + # ===== Kanton Methods ===== + + def createKanton(self, kanton: Kanton) -> Kanton: + """Create a new canton.""" + if not kanton.mandateId: + kanton.mandateId = self.mandateId + + self.access.uam(Kanton, []) + self.db.recordCreate(Kanton, kanton.model_dump()) + + return kanton + + def getKanton(self, kantonId: str) -> Optional[Kanton]: + """Get a canton by ID.""" + records = self.db.getRecordset( + Kanton, + recordFilter={"id": kantonId} + ) + + if not records: + return None + + filtered = self.access.uam(Kanton, records) + + if not filtered: + return None + + return Kanton(**filtered[0]) + + def getKantone(self, recordFilter: Optional[Dict[str, Any]] = None) -> List[Kanton]: + """Get all cantons matching the filter.""" + records = self.db.getRecordset(Kanton, recordFilter=recordFilter or {}) + filtered = self.access.uam(Kanton, records) + return [Kanton(**r) for r in filtered] + + def updateKanton(self, kantonId: str, updateData: Dict[str, Any]) -> Optional[Kanton]: + """Update a canton.""" + kanton = self.getKanton(kantonId) + if not kanton: + return None + + if not self.access.canModify(Kanton, kantonId): + raise PermissionError(f"User {self.userId} cannot modify canton {kantonId}") + + for key, value in updateData.items(): + if hasattr(kanton, key): + setattr(kanton, key, value) + + self.db.recordModify(Kanton, kantonId, kanton.model_dump()) + return kanton + + def deleteKanton(self, kantonId: str) -> bool: + """Delete a canton.""" + kanton = self.getKanton(kantonId) + if not kanton: + return False + + if not self.access.canModify(Kanton, kantonId): + raise PermissionError(f"User {self.userId} cannot delete canton {kantonId}") + + return self.db.recordDelete(Kanton, kantonId) + + # ===== Land Methods ===== + + def createLand(self, land: Land) -> Land: + """Create a new country.""" + if not land.mandateId: + land.mandateId = self.mandateId + + self.access.uam(Land, []) + self.db.recordCreate(Land, land.model_dump()) + + return land + + def getLand(self, landId: str) -> Optional[Land]: + """Get a country by ID.""" + records = self.db.getRecordset( + Land, + recordFilter={"id": landId} + ) + + if not records: + return None + + filtered = self.access.uam(Land, records) + + if not filtered: + return None + + return Land(**filtered[0]) + + def getLaender(self, recordFilter: Optional[Dict[str, Any]] = None) -> List[Land]: + """Get all countries matching the filter.""" + records = self.db.getRecordset(Land, recordFilter=recordFilter or {}) + filtered = self.access.uam(Land, records) + return [Land(**r) for r in filtered] + + def updateLand(self, landId: str, updateData: Dict[str, Any]) -> Optional[Land]: + """Update a country.""" + land = self.getLand(landId) + if not land: + return None + + if not self.access.canModify(Land, landId): + raise PermissionError(f"User {self.userId} cannot modify country {landId}") + + for key, value in updateData.items(): + if hasattr(land, key): + setattr(land, key, value) + + self.db.recordModify(Land, landId, land.model_dump()) + return land + + def deleteLand(self, landId: str) -> bool: + """Delete a country.""" + land = self.getLand(landId) + if not land: + return False + + if not self.access.canModify(Land, landId): + raise PermissionError(f"User {self.userId} cannot delete country {landId}") + + return self.db.recordDelete(Land, landId) + + # ===== Direct Query Execution (stateless) ===== + + def executeQuery(self, queryText: str, parameters: Optional[Dict[str, Any]] = None) -> Dict[str, Any]: + """ + Execute a SQL query directly on the database (stateless). + + WARNING: This method executes raw SQL. Ensure proper validation and sanitization + before calling this method. Consider implementing query whitelisting or + only allowing SELECT statements for production use. + + Args: + queryText: SQL query string (preferably SELECT only) + parameters: Optional parameters for parameterized queries + + Returns: + Dictionary with 'rows' (list of dicts), 'columns' (list of column names), + 'rowCount' (int), and 'executionTime' (float) + """ + import time + + try: + start_time = time.time() + + # Ensure connection is alive + self.db._ensure_connection() + + with self.db.connection.cursor() as cursor: + # Execute query + if parameters: + # Use parameterized query for safety + cursor.execute(queryText, parameters) + else: + cursor.execute(queryText) + + # Fetch results + rows = cursor.fetchall() + + # Convert to list of dictionaries + result_rows = [] + if rows: + columns = [desc[0] for desc in cursor.description] if cursor.description else [] + result_rows = [dict(zip(columns, row)) for row in rows] + else: + columns = [] + + execution_time = time.time() - start_time + + return { + "rows": result_rows, + "columns": columns, + "rowCount": len(result_rows), + "executionTime": execution_time, + } + except Exception as e: + logger.error(f"Error executing query: {e}", exc_info=True) + raise + + +def getInterface(currentUser: User) -> RealEstateObjects: + """ + Factory function to get or create a Real Estate interface instance for a user. + Uses singleton pattern per user. + """ + userKey = f"{currentUser.id}_{currentUser.mandateId}" + + if userKey not in _realEstateInterfaces: + _realEstateInterfaces[userKey] = RealEstateObjects(currentUser) + + return _realEstateInterfaces[userKey] + diff --git a/modules/routes/routeRealEstate.py b/modules/routes/routeRealEstate.py new file mode 100644 index 00000000..b5e34ef3 --- /dev/null +++ b/modules/routes/routeRealEstate.py @@ -0,0 +1,637 @@ +""" +Real Estate routes for the backend API. +Implements stateless endpoints for real estate database operations with AI-powered natural language processing. +""" + +import logging +import json +from typing import Optional, Dict, Any, List, Union +from fastapi import APIRouter, HTTPException, Depends, Body, Request, Query, Path, status + +# Import auth modules +from modules.security.auth import limiter, getCurrentUser + +# Import models +from modules.datamodels.datamodelUam import User +from modules.datamodels.datamodelPagination import PaginationParams, PaginatedResponse, PaginationMetadata +from modules.datamodels.datamodelRealEstate import ( + Projekt, + Parzelle, + Dokument, + Gemeinde, + Kanton, + Land, +) + +# Import interfaces +from modules.interfaces.interfaceDbRealEstateObjects import getInterface as getRealEstateInterface + +# Import feature logic +from modules.features.realEstate.mainRealEstate import ( + processNaturalLanguageCommand, + executeDirectQuery, +) + +# Import attribute utilities for model schema +from modules.shared.attributeUtils import getModelAttributeDefinitions + +# Configure logger +logger = logging.getLogger(__name__) + +# Create router for real estate endpoints +router = APIRouter( + prefix="/api/realestate", + tags=["Real Estate"], + responses={ + 404: {"description": "Not found"}, + 400: {"description": "Bad request"}, + 401: {"description": "Unauthorized"}, + 403: {"description": "Forbidden"}, + 500: {"description": "Internal server error"} + } +) + + +@router.post("/command", response_model=Dict[str, Any]) +@limiter.limit("120/minute") +async def process_command( + request: Request, + userInput: str = Body(..., embed=True, description="Natural language command"), + currentUser: User = Depends(getCurrentUser) +) -> Dict[str, Any]: + """ + Process natural language command and execute corresponding CRUD operation. + + Uses AI to analyze user intent and extract parameters, then executes the appropriate + CRUD operation. Works stateless without session management. + + Example user inputs: + - "Erstelle ein neues Projekt namens 'Hauptstrasse 42'" + - "Zeige mir alle Projekte in Zürich" + - "Aktualisiere Projekt XYZ mit Status 'Planung'" + - "Lösche Parzelle ABC" + - "SELECT * FROM Projekt WHERE plz = '8000'" + + Headers: + - X-CSRF-Token: CSRF token (required for security) + + Returns: + { + "success": true, + "intent": "CREATE|READ|UPDATE|DELETE|QUERY", + "entity": "Projekt|Parzelle|...|null", + "result": {...} + } + """ + try: + # Validate CSRF token (middleware also checks, but explicit validation for better error messages) + csrf_token = request.headers.get("X-CSRF-Token") or request.headers.get("x-csrf-token") + if not csrf_token: + logger.warning(f"CSRF token missing for POST /api/realestate/command from user {currentUser.id}") + raise HTTPException( + status_code=status.HTTP_403_FORBIDDEN, + detail="CSRF token missing. Please include X-CSRF-Token header." + ) + + # Basic CSRF token format validation + if not isinstance(csrf_token, str) or len(csrf_token) < 16 or len(csrf_token) > 64: + logger.warning(f"Invalid CSRF token format for POST /api/realestate/command from user {currentUser.id}") + raise HTTPException( + status_code=status.HTTP_403_FORBIDDEN, + detail="Invalid CSRF token format" + ) + + # Validate token is hex string + try: + int(csrf_token, 16) + except ValueError: + logger.warning(f"CSRF token is not a valid hex string for POST /api/realestate/command from user {currentUser.id}") + raise HTTPException( + status_code=status.HTTP_403_FORBIDDEN, + detail="Invalid CSRF token format" + ) + + logger.info(f"Processing command request from user {currentUser.id} (mandate: {currentUser.mandateId})") + logger.debug(f"User input: {userInput}") + + # Process natural language command with AI + result = await processNaturalLanguageCommand( + currentUser=currentUser, + userInput=userInput + ) + + return result + + except ValueError as e: + logger.error(f"Validation error in process_command: {str(e)}", exc_info=True) + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail=f"Validation error: {str(e)}" + ) + except HTTPException: + raise + except Exception as e: + logger.error(f"Error processing command: {str(e)}", exc_info=True) + raise HTTPException( + status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, + detail=f"Error processing command: {str(e)}" + ) + +@router.post("/query", response_model=Dict[str, Any]) +@limiter.limit("120/minute") +async def execute_query( + request: Request, + body: Dict[str, Any] = Body(...), + currentUser: User = Depends(getCurrentUser) +) -> Dict[str, Any]: + """ + Execute a direct SQL query without session management. + + Executes the query directly and returns the result. No query history is saved. + + Request body: + { + "queryText": "SELECT * FROM Projekt WHERE plz = '8000'", + "parameters": { // Optional + "$1": "8000" + } + } + + Headers: + - X-CSRF-Token: CSRF token (required for security) + + WARNING: This endpoint executes raw SQL queries. Ensure proper validation + and sanitization on the frontend. Consider implementing query whitelisting + or only allowing SELECT statements for production use. + + Returns: + { + "status": "success", + "rows": [...], + "columns": [...], + "rowCount": 15, + "executionTime": 0.123 + } + """ + try: + # Validate CSRF token (middleware also checks, but explicit validation for better error messages) + csrf_token = request.headers.get("X-CSRF-Token") or request.headers.get("x-csrf-token") + if not csrf_token: + logger.warning(f"CSRF token missing for POST /api/realestate/query from user {currentUser.id}") + raise HTTPException( + status_code=status.HTTP_403_FORBIDDEN, + detail="CSRF token missing. Please include X-CSRF-Token header." + ) + + # Basic CSRF token format validation + if not isinstance(csrf_token, str) or len(csrf_token) < 16 or len(csrf_token) > 64: + logger.warning(f"Invalid CSRF token format for POST /api/realestate/query from user {currentUser.id}") + raise HTTPException( + status_code=status.HTTP_403_FORBIDDEN, + detail="Invalid CSRF token format" + ) + + # Validate token is hex string + try: + int(csrf_token, 16) + except ValueError: + logger.warning(f"CSRF token is not a valid hex string for POST /api/realestate/query from user {currentUser.id}") + raise HTTPException( + status_code=status.HTTP_403_FORBIDDEN, + detail="Invalid CSRF token format" + ) + + # Extract fields from body + queryText = body.get("queryText") + if not queryText: + raise ValueError("queryText is required") + + parameters = body.get("parameters") + + logger.info(f"Processing query request from user {currentUser.id} (mandate: {currentUser.mandateId})") + logger.debug(f"Query text: {queryText}") + if parameters: + logger.debug(f"Query parameters: {parameters}") + + # Execute direct query + result = await executeDirectQuery( + currentUser=currentUser, + queryText=queryText, + parameters=parameters, + ) + + return result + + except ValueError as e: + logger.error(f"Validation error in execute_query: {str(e)}", exc_info=True) + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail=f"Validation error: {str(e)}" + ) + except HTTPException: + raise + except Exception as e: + logger.error(f"Error executing query: {str(e)}", exc_info=True) + raise HTTPException( + status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, + detail=f"Error executing query: {str(e)}" + ) + + +@router.get("/tables", response_model=Dict[str, Any]) +@limiter.limit("120/minute") +async def get_available_tables( + request: Request, + currentUser: User = Depends(getCurrentUser) +) -> Dict[str, Any]: + """ + Get all available real estate tables. + + Returns a list of available table names with their descriptions. + + Headers: + - X-CSRF-Token: CSRF token (required for security) + + Example: + - GET /api/realestate/tables + """ + try: + # Validate CSRF token if provided + csrf_token = request.headers.get("X-CSRF-Token") or request.headers.get("x-csrf-token") + if not csrf_token: + logger.warning(f"CSRF token missing for GET /api/realestate/tables from user {currentUser.id}") + raise HTTPException( + status_code=status.HTTP_403_FORBIDDEN, + detail="CSRF token missing. Please include X-CSRF-Token header." + ) + + # Basic CSRF token format validation + if not isinstance(csrf_token, str) or len(csrf_token) < 16 or len(csrf_token) > 64: + logger.warning(f"Invalid CSRF token format for GET /api/realestate/tables from user {currentUser.id}") + raise HTTPException( + status_code=status.HTTP_403_FORBIDDEN, + detail="Invalid CSRF token format" + ) + + # Validate token is hex string + try: + int(csrf_token, 16) + except ValueError: + logger.warning(f"CSRF token is not a valid hex string for GET /api/realestate/tables from user {currentUser.id}") + raise HTTPException( + status_code=status.HTTP_403_FORBIDDEN, + detail="Invalid CSRF token format" + ) + + logger.info(f"Getting available tables for user {currentUser.id} (mandate: {currentUser.mandateId})") + + # Define available tables with descriptions + tables = [ + { + "name": "Projekt", + "description": "Real estate projects", + "model": "Projekt" + }, + { + "name": "Parzelle", + "description": "Plots/parcels", + "model": "Parzelle" + }, + { + "name": "Dokument", + "description": "Documents", + "model": "Dokument" + }, + { + "name": "Gemeinde", + "description": "Municipalities", + "model": "Gemeinde" + }, + { + "name": "Kanton", + "description": "Cantons", + "model": "Kanton" + }, + { + "name": "Land", + "description": "Countries", + "model": "Land" + }, + ] + + return { + "tables": tables, + "count": len(tables) + } + + except HTTPException: + raise + except Exception as e: + logger.error(f"Error getting available tables: {str(e)}", exc_info=True) + raise HTTPException( + status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, + detail=f"Error getting available tables: {str(e)}" + ) + + +@router.get("/table/{table}", response_model=PaginatedResponse[Any]) +@limiter.limit("120/minute") +async def get_table_data( + request: Request, + table: str = Path(..., description="Table name (Projekt, Parzelle, Dokument, Gemeinde, Kanton, Land)"), + pagination: Optional[str] = Query(None, description="JSON-encoded PaginationParams object"), + currentUser: User = Depends(getCurrentUser) +) -> PaginatedResponse[Dict[str, Any]]: + """ + Get all data from a specific real estate table with optional pagination. + + Available tables: + - Projekt: Real estate projects + - Parzelle: Plots/parcels + - Dokument: Documents + - Gemeinde: Municipalities + - Kanton: Cantons + - Land: Countries + + Query Parameters: + - pagination: JSON-encoded PaginationParams object, or None for no pagination + + Headers: + - X-CSRF-Token: CSRF token (required for security) + + Examples: + - GET /api/realestate/table/Projekt (no pagination - returns all items) + - GET /api/realestate/table/Parzelle?pagination={"page":1,"pageSize":10,"sort":[]} + - GET /api/realestate/table/Gemeinde?pagination={"page":2,"pageSize":20,"sort":[{"field":"label","direction":"asc"}]} + """ + try: + # Validate CSRF token if provided + csrf_token = request.headers.get("X-CSRF-Token") or request.headers.get("x-csrf-token") + if not csrf_token: + logger.warning(f"CSRF token missing for GET /api/realestate/table/{table} from user {currentUser.id}") + raise HTTPException( + status_code=status.HTTP_403_FORBIDDEN, + detail="CSRF token missing. Please include X-CSRF-Token header." + ) + + # Basic CSRF token format validation + if not isinstance(csrf_token, str) or len(csrf_token) < 16 or len(csrf_token) > 64: + logger.warning(f"Invalid CSRF token format for GET /api/realestate/table/{table} from user {currentUser.id}") + raise HTTPException( + status_code=status.HTTP_403_FORBIDDEN, + detail="Invalid CSRF token format" + ) + + # Validate token is hex string + try: + int(csrf_token, 16) + except ValueError: + logger.warning(f"CSRF token is not a valid hex string for GET /api/realestate/table/{table} from user {currentUser.id}") + raise HTTPException( + status_code=status.HTTP_403_FORBIDDEN, + detail="Invalid CSRF token format" + ) + + logger.info(f"Getting table data for '{table}' from user {currentUser.id} (mandate: {currentUser.mandateId})") + + # Map table names to model classes and getter methods + table_mapping = { + "Projekt": (Projekt, "getProjekte"), + "Parzelle": (Parzelle, "getParzellen"), + "Dokument": (Dokument, "getDokumente"), + "Gemeinde": (Gemeinde, "getGemeinden"), + "Kanton": (Kanton, "getKantone"), + "Land": (Land, "getLaender"), + } + + # Validate table name + if table not in table_mapping: + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail=f"Invalid table name '{table}'. Available tables: {', '.join(table_mapping.keys())}" + ) + + # Get interface and fetch data + realEstateInterface = getRealEstateInterface(currentUser) + model_class, method_name = table_mapping[table] + getter_method = getattr(realEstateInterface, method_name) + + # Fetch all records (no filter for now) + records = getter_method(recordFilter=None) + + # Keep records as model instances (like routeDataFiles does with FileItem) + # FastAPI will automatically serialize Pydantic models to JSON + items = records + + # If table is empty, create an empty instance with all fields set to None/empty + # This allows the frontend to extract column structure from the response + # All fields will be None/empty - no IDs or other values generated + if not items: + try: + # Get all model fields + model_fields = model_class.model_fields + empty_values = {} + + # Set all fields to None - explicitly set every field to None + # This ensures no default_factory is called and no IDs are generated + for field_name in model_fields.keys(): + empty_values[field_name] = None + + # Create instance with all None values + # Use model_validate with allow_none=True or construct directly + empty_instance = model_class.model_construct(**empty_values) + items = [empty_instance] + logger.debug(f"Created empty instance for {table} with all fields set to None") + except Exception as e: + logger.warning(f"Could not create empty instance for {table}: {str(e)}. Returning empty list.") + items = [] + + # Parse pagination parameter + paginationParams = None + if pagination: + try: + paginationDict = json.loads(pagination) + paginationParams = PaginationParams(**paginationDict) if paginationDict else None + except (json.JSONDecodeError, ValueError) as e: + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail=f"Invalid pagination parameter: {str(e)}" + ) + + # Apply pagination if requested + if paginationParams: + # Apply sorting if specified + if paginationParams.sort: + for sort_field in reversed(paginationParams.sort): # Reverse to apply in priority order + field_name = sort_field.field + direction = sort_field.direction.lower() + + def sort_key(item): + # Access attribute from model instance + value = getattr(item, field_name, None) + # Handle None values - put them at the end for asc, at the start for desc + if value is None: + return (1, None) # Use tuple to ensure None values sort consistently + return (0, value) + + items.sort(key=sort_key, reverse=(direction == "desc")) + + # Apply pagination + total_items = len(items) + total_pages = (total_items + paginationParams.pageSize - 1) // paginationParams.pageSize # Ceiling division + start_idx = (paginationParams.page - 1) * paginationParams.pageSize + end_idx = start_idx + paginationParams.pageSize + paginated_items = items[start_idx:end_idx] + + return PaginatedResponse( + items=paginated_items, + pagination=PaginationMetadata( + currentPage=paginationParams.page, + pageSize=paginationParams.pageSize, + totalItems=total_items, + totalPages=total_pages, + sort=paginationParams.sort, + filters=paginationParams.filters + ) + ) + else: + # No pagination - return all items (as model instances, like routeDataFiles) + return PaginatedResponse( + items=items, + pagination=None + ) + + except HTTPException: + raise + except Exception as e: + logger.error(f"Error getting table data for '{table}': {str(e)}", exc_info=True) + raise HTTPException( + status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, + detail=f"Error getting table data: {str(e)}" + ) + + +@router.post("/table/{table}", response_model=Dict[str, Any]) +@limiter.limit("120/minute") +async def create_table_record( + request: Request, + table: str = Path(..., description="Table name (Projekt, Parzelle, Dokument, Gemeinde, Kanton, Land)"), + data: Dict[str, Any] = Body(..., description="Record data to create"), + currentUser: User = Depends(getCurrentUser) +) -> Dict[str, Any]: + """ + Create a new record in a specific real estate table. + + Available tables: + - Projekt: Real estate projects + - Parzelle: Plots/parcels + - Dokument: Documents + - Gemeinde: Municipalities + - Kanton: Cantons + - Land: Countries + + Request Body: + - JSON object with fields matching the table's data model + + Headers: + - X-CSRF-Token: CSRF token (required for security) + + Examples: + - POST /api/realestate/table/Projekt + Body: {"label": "Hauptstrasse 42", "statusProzess": "Eingang"} + - POST /api/realestate/table/Parzelle + Body: {"label": "Parzelle 1", "strasseNr": "Hauptstrasse 42", "plz": "8000", "bauzone": "W3"} + """ + try: + # Validate CSRF token + csrf_token = request.headers.get("X-CSRF-Token") or request.headers.get("x-csrf-token") + if not csrf_token: + logger.warning(f"CSRF token missing for POST /api/realestate/table/{table} from user {currentUser.id}") + raise HTTPException( + status_code=status.HTTP_403_FORBIDDEN, + detail="CSRF token missing. Please include X-CSRF-Token header." + ) + + # Basic CSRF token format validation + if not isinstance(csrf_token, str) or len(csrf_token) < 16 or len(csrf_token) > 64: + logger.warning(f"Invalid CSRF token format for POST /api/realestate/table/{table} from user {currentUser.id}") + raise HTTPException( + status_code=status.HTTP_403_FORBIDDEN, + detail="Invalid CSRF token format" + ) + + # Validate token is hex string + try: + int(csrf_token, 16) + except ValueError: + logger.warning(f"CSRF token is not a valid hex string for POST /api/realestate/table/{table} from user {currentUser.id}") + raise HTTPException( + status_code=status.HTTP_403_FORBIDDEN, + detail="Invalid CSRF token format" + ) + + logger.info(f"Creating record in table '{table}' for user {currentUser.id} (mandate: {currentUser.mandateId})") + logger.debug(f"Record data: {data}") + + # Map table names to model classes and create methods + table_mapping = { + "Projekt": (Projekt, "createProjekt"), + "Parzelle": (Parzelle, "createParzelle"), + "Dokument": (Dokument, "createDokument"), + "Gemeinde": (Gemeinde, "createGemeinde"), + "Kanton": (Kanton, "createKanton"), + "Land": (Land, "createLand"), + } + + # Validate table name + if table not in table_mapping: + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail=f"Invalid table name '{table}'. Available tables: {', '.join(table_mapping.keys())}" + ) + + # Get interface + realEstateInterface = getRealEstateInterface(currentUser) + model_class, method_name = table_mapping[table] + create_method = getattr(realEstateInterface, method_name) + + # Ensure mandateId is set (will be set by interface if missing) + if "mandateId" not in data: + data["mandateId"] = currentUser.mandateId + + # Create model instance from data + try: + model_instance = model_class(**data) + except Exception as e: + logger.error(f"Error creating {table} model instance: {str(e)}", exc_info=True) + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail=f"Invalid data for {table}: {str(e)}" + ) + + # Create record + try: + created_record = create_method(model_instance) + + # Convert to dictionary for response + if hasattr(created_record, 'model_dump'): + return created_record.model_dump() + else: + return created_record + + except Exception as e: + logger.error(f"Error creating {table} record: {str(e)}", exc_info=True) + raise HTTPException( + status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, + detail=f"Error creating {table} record: {str(e)}" + ) + + except HTTPException: + raise + except Exception as e: + logger.error(f"Error creating record in table '{table}': {str(e)}", exc_info=True) + raise HTTPException( + status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, + detail=f"Error creating record: {str(e)}" + ) + From 397aa68b05bb7abc126a2bf5d042f4f27454ee04 Mon Sep 17 00:00:00 2001 From: Ida Dittrich Date: Tue, 25 Nov 2025 10:06:33 +0100 Subject: [PATCH 02/25] feat: real estate integration --- .../02-datamodels.md | 976 ------------------ .../03-interfaces.md | 665 +----------- .../04-feature-logic.md | 640 +----------- .../05-routes.md | 337 +++--- .../06-router-registration.md | 45 - .../07-environment.md | 2 + .../08-lifecycle.md | 43 - .../09-database-schema.md | 247 ----- .../10-security.md | 90 -- .../11-testing.md | 51 - .../12-troubleshooting.md | 33 - .../13-summary.md | 136 --- .../README.md | 3 + ...51120-154518-035-intentanalysis_prompt.txt | 89 ++ ...120-154524-036-intentanalysis_response.txt | 18 + ...51120-154559-037-intentanalysis_prompt.txt | 89 ++ ...120-154607-038-intentanalysis_response.txt | 17 + ...51120-154715-039-intentanalysis_prompt.txt | 89 ++ ...120-154718-040-intentanalysis_response.txt | 10 + ...51120-154742-041-intentanalysis_prompt.txt | 89 ++ ...120-154746-042-intentanalysis_response.txt | 10 + 21 files changed, 633 insertions(+), 3046 deletions(-) delete mode 100644 docs/real-estate-feature-integration-guide/02-datamodels.md delete mode 100644 docs/real-estate-feature-integration-guide/06-router-registration.md delete mode 100644 docs/real-estate-feature-integration-guide/08-lifecycle.md delete mode 100644 docs/real-estate-feature-integration-guide/09-database-schema.md delete mode 100644 docs/real-estate-feature-integration-guide/10-security.md delete mode 100644 docs/real-estate-feature-integration-guide/11-testing.md delete mode 100644 docs/real-estate-feature-integration-guide/12-troubleshooting.md delete mode 100644 docs/real-estate-feature-integration-guide/13-summary.md create mode 100644 logs/debug/prompts/20251120-154518-035-intentanalysis_prompt.txt create mode 100644 logs/debug/prompts/20251120-154524-036-intentanalysis_response.txt create mode 100644 logs/debug/prompts/20251120-154559-037-intentanalysis_prompt.txt create mode 100644 logs/debug/prompts/20251120-154607-038-intentanalysis_response.txt create mode 100644 logs/debug/prompts/20251120-154715-039-intentanalysis_prompt.txt create mode 100644 logs/debug/prompts/20251120-154718-040-intentanalysis_response.txt create mode 100644 logs/debug/prompts/20251120-154742-041-intentanalysis_prompt.txt create mode 100644 logs/debug/prompts/20251120-154746-042-intentanalysis_response.txt diff --git a/docs/real-estate-feature-integration-guide/02-datamodels.md b/docs/real-estate-feature-integration-guide/02-datamodels.md deleted file mode 100644 index 376f5121..00000000 --- a/docs/real-estate-feature-integration-guide/02-datamodels.md +++ /dev/null @@ -1,976 +0,0 @@ -# Schritt 1: Datenmodell erstellen - -[← Zurück zur Übersicht](README.md) | [Weiter: Interface erstellen →](03-interfaces.md) - -## Real Estate-Datenmodelle - -**Datei:** `modules/datamodels/datamodelRealEstate.py` - -Das Feature arbeitet **stateless** ohne Session-Management. Die Datenmodelle definieren die Struktur der Real Estate-Entitäten, die über die API verwaltet werden können. - -**Hinweis:** Die Real Estate-Datenmodell-Entitäten (Projekt, Parzelle, Dokument, etc.) werden in `datamodelRealEstate.py` definiert. Diese werden direkt über CRUD-Operationen verwaltet, ohne zusätzliche Chat-Interface-Modelle. - -### Warum keine Chat-Interface-Modelle? - -Das Feature arbeitet **stateless** ohne Session-Management. Alle Operationen arbeiten direkt auf den Real Estate-Datenmodellen: - -#### Stateless Design - -- **Keine Session-Modelle**: Keine `RealEstateChatSession` notwendig -- **Keine Query-History**: Queries werden nicht gespeichert (kann optional später hinzugefügt werden) -- **Direkte CRUD-Operationen**: User-Input → AI-Analyse → CRUD → Ergebnis -- **Einfache Architektur**: Weniger Komplexität, bessere Performance - -#### Real Estate-Modelle - -Die Real Estate-Modelle (`Projekt`, `Parzelle`, `Dokument`, etc.): -- Repräsentieren die **tatsächlichen Geschäftsdaten** der Immobilien-Projekte -- Werden über lange Zeiträume gepflegt und verändert -- Haben komplexe Beziehungen zueinander (Projekt → Parzellen → Dokumente) -- Werden direkt über CRUD-Operationen verwaltet - -#### Datenfluss (stateless) - -``` -User Input (natürliche Sprache) - ↓ -AI-Analyse (Intent-Erkennung) - ↓ -CRUD-Operation identifizieren - ↓ -Real Estate-Modelle - ↓ -Datenbank-Operation - ↓ -Ergebnis zurückgeben - (keine Session, keine History) -``` - ---- - -### Stateless vs. Session-basiert - -**Real Estate Feature (stateless):** -- Direkte CRUD-Operationen auf Real Estate-Modellen -- Keine Session-Modelle notwendig -- Keine Query-History -- Einfacher und schneller - -**Chat-System (session-basiert):** -- Verwendet `ChatWorkflow` für komplexe AI-Workflows -- Verwendet `ChatDocument` für Datei-Verknüpfungen -- Session-Management für Multi-Step-Operationen -- Für komplexe Workflows mit Planung und Review - -**Unterschied:** -- Real Estate Feature ist für **einfache CRUD-Operationen** optimiert -- Chat-System ist für **komplexe AI-Workflows** optimiert -- Beide können parallel existieren und für verschiedene Use Cases genutzt werden - ---- - -### Warum nicht das bestehende `ChatWorkflow` verwenden? - -Sie fragen sich vielleicht: **Kann ich nicht einfach das bestehende `ChatWorkflow` aus `datamodelChat.py` verwenden?** - -Die kurze Antwort: **Für stateless CRUD-Operationen ist `ChatWorkflow` zu komplex**. Das Real Estate Feature arbeitet ohne Session-Management und nutzt direkt die Real Estate-Modelle. - -#### Unterschiedliche Anwendungsfälle - -| **Aspekt** | **ChatWorkflow (bestehend)** | **Real Estate Feature (stateless)** | -|------------|------------------------------|-------------------------------------| -| **Zweck** | Komplexe AI-gesteuerte Workflows mit mehreren Tasks/Actions | Einfache CRUD-Operationen | -| **Komplexität** | Hoch: Tasks, Actions, Rounds, Workflow-Modi, Retries | Niedrig: Direkte CRUD-Operationen | -| **Session** | Session-Management für Multi-Step-Workflows | Keine Session, stateless | -| **Verarbeitung** | Multi-Step AI-Workflows mit Planung, Review, Iteration | Direkte CRUD: User-Input → AI-Analyse → CRUD → Ergebnis | -| **Ergebnisse** | `ChatMessage` mit `documents`, `ActionResult` | Direkte CRUD-Ergebnisse (Projekt, Parzelle, etc.) | - -#### Warum `ChatWorkflow` nicht passt: - -1. **Zu komplex**: `ChatWorkflow` hat viele Felder, die für einfache CRUD-Operationen nicht relevant sind -2. **Session-basiert**: `ChatWorkflow` benötigt Session-Management, das wir nicht brauchen -3. **Falsches Abstraktionsniveau**: `ChatWorkflow` ist für komplexe AI-Workflows, Real Estate braucht einfache CRUD-Operationen - -#### Die richtige Lösung: Direkte CRUD-Operationen - -Stattdessen arbeiten wir **direkt** mit den Real Estate-Modellen: - -```python -# Stateless CRUD-Operationen -User Input → AI-Analyse → CRUD-Operation → Ergebnis -# Keine Session, keine History, einfach und schnell -``` - -#### Wann könnte man `ChatWorkflow` verwenden? - -Sie könnten `ChatWorkflow` verwenden, wenn Sie: -- ✅ **Komplexe AI-Workflows** für Real Estate implementieren wollen (z.B. "Analysiere alle Projekte und erstelle einen Bericht") -- ✅ **Multi-Step-Verarbeitung** benötigen (z.B. "Lade Daten → Transformiere → Erstelle Visualisierung") -- ✅ **Planung und Review** brauchen (z.B. "Prüfe alle Parzellen auf Konformität") - -Aber für **einfache CRUD-Operationen** ist der stateless Ansatz die bessere Wahl. - ---- - -## Real Estate-Datenmodell-Implementierung: - -Die Real Estate-Datenmodell-Entitäten müssen separat in `modules/datamodels/datamodelRealEstate.py` implementiert werden. -Siehe `../PEK_datamodel_desc.md` für die vollständige Spezifikation aller Felder und Beziehungen (PEK ist ein Beispiel für eine Real Estate-Firma, das Modell ist aber allgemein verwendbar). - -### Wichtige Hinweise zum Datenmodell - -**Objektmodell vs. Datenbank-Repräsentation:** - -Dieses Dokument beschreibt ein **Objektmodell** für die Arbeit im Code. Es handelt sich **NICHT** um ein relationales Datenbankmodell mit Junction Tables. - -- **Im Code-Modell**: Alle Beziehungen werden als Objektreferenzen oder Listen von Objekten dargestellt (z.B. `dokumente: list[Dokument]`, `parzellen: list[Parzelle]`). -- **Für die Datenbank-Serialisierung**: Bei der Persistierung können Junction Tables verwendet werden, um n:m-Beziehungen in der Datenbank abzubilden. Dies ist jedoch ein Implementierungsdetail der Datenbank-Schicht und gehört nicht zum Hauptmodell. - -**Systemattribute:** - -Alle Datenobjekte haben automatisch die folgenden Systemattribute: -- `_createdAt`: Float (Timestamp UTC) -- `_createdBy`: String (User-ID) -- `_modifiedAt`: Float (Timestamp UTC) -- `_modifiedBy`: String (User-ID) - -**Timestamps:** -- Alle Timestamps sind im Float-Format UTC im Datenmodell gespeichert. -- Die Darstellung im UI erfolgt mit der lokalen Zeitzone des Benutzers. - -**Wichtige Punkte für die Implementierung:** -- Objektbeziehungen wie `parzellen: list[Parzelle]` werden als JSONB in PostgreSQL gespeichert -- Einzelne Objektreferenzen wie `kontextKanton: Optional[str]` werden als String-ID (Foreign Key) gespeichert -- Administrative Hierarchie: `Kanton` benötigt `id_land` (Foreign Key zu Land), `Gemeinde` benötigt `id_kanton` (Foreign Key zu Kanton) -- Alle Entitäten benötigen `mandateId` für Mandaten-Isolation -- Systemattribute (`_createdAt`, `_createdBy`, etc.) werden automatisch vom DatabaseConnector hinzugefügt - -### Datenfluss-Diagramm - -```mermaid ---- -title: Hauptflüsse - Architektur-Planungs-App ---- -flowchart LR - subgraph Admin[Administrative Ebene] - Land[Land
Schweiz] - Kanton[Kanton
z.B. Zürich] - Gemeinde[Gemeinde
z.B. Zürich Stadt] - Land --> Kanton - Kanton --> Gemeinde - end - - subgraph Geo[Geografische Daten] - GeoPolylinie[GeoPolylinie
Linie/Polygon] - GeoPunkt[GeoPunkt
Koordinaten] - GeoPolylinie --> GeoPunkt - end - - subgraph Core[Kern-Business-Logik] - Projekt[Projekt
Bauprojekt] - Parzelle[Parzelle
Grundstück mit
Bauparametern] - Gemeinde --> Parzelle - Projekt --> Parzelle - Projekt --> GeoPolylinie - Parzelle --> GeoPolylinie - end - - subgraph Support[Unterstützende Daten] - Dokument[Dokument
Dateien & URLs] - Kontext[Kontext
Zusatzinfos] - end -``` - -### Übersichtstabelle aller Entitäten - -| Objekt | Typ | Beschreibung | Hauptfelder | -|--------|-----|--------------|-------------| -| **Projekt** | Kernentität | Bauprojekt mit Status und Perimeter | id, label, statusProzess, perimeter, baulinie, parzellen | -| **Parzelle** | Hauptentität | Grundstück mit Bauparametern | id, label, plz, bauzone, AZ, BZ, perimeter, baulinie, laermschutzzone, hochwasserschutzzone, grundwasserschutzzone | -| **Dokument** | Unterstützend | Dateien und URLs mit Versionierung | id, label, dokumentTyp, quelle, mimeType, kategorienTags | -| **Kontext** | Unterstützend | Flexible Zusatzinformationen | id, thema, inhalt | -| **GeoPolylinie** | Hilfsobjekt | Geometrische Linie/Polygon | id, closed, punkte | -| **Land** | Admin | Nationale Ebene | id, label, abk | -| **Kanton** | Admin | Kantonale Ebene mit Baurecht | id, label, id_land, abk | -| **Gemeinde** | Admin | Gemeinde-Ebene mit BZO | id, label, id_kanton, plz | -| **GeoPunkt** | Hilfsobjekt | 3D-Koordinate | koordinatensystem, x, y, z, referenz | -| **GeoTag** | Enum | Geopunkt-Kategorien | K1, K2, K3, Geometer | -| **JaNein** | Enum | Drei-wertiger Status | "", "Ja", "Nein" | -| **StatusProzess** | Enum | Projektstatus | 7 Werte (Eingang, Analyse, Studie, Planung, Baurechtsverfahren, Umsetzung, Archiv) | -| **DokumentTyp** | Enum | Dokumenttyp | 6 Werte (kantonBaureglementAktuell, kantonBaureglementRevision, etc.) | - -### Beispiel: Vollständige Pydantic-Modelle für Real Estate-Entitäten - -Hier ist ein Beispiel, wie die Pydantic-Modelle für die Real Estate-Entitäten aussehen sollten: - -```python -""" -Real Estate data models for Architektur-Planungs-App. -Implements a general Swiss architecture planning data model. -(PEK is one example implementation, but the model is general-purpose) -""" - -from typing import List, Dict, Any, Optional, ForwardRef -from enum import Enum -from pydantic import BaseModel, Field -from modules.shared.attributeUtils import registerModelLabels -from modules.shared.timeUtils import getUtcTimestamp -import uuid - -# ===== Enums ===== - -class StatusProzess(str, Enum): - """Project process status""" - EINGANG = "Eingang" - ANALYSE = "Analyse" - STUDIE = "Studie" - PLANUNG = "Planung" - BAURECHTSVERFAHREN = "Baurechtsverfahren" - UMSETZUNG = "Umsetzung" - ARCHIV = "Archiv" - - -class DokumentTyp(str, Enum): - """Document type for categorization""" - KANTON_BAUREGLEMENT_AKTUELL = "kantonBaureglementAktuell" - KANTON_BAUREGLEMENT_REVISION = "kantonBaureglementRevision" - KANTON_BAUVERORDNUNG_AKTUELL = "kantonBauverordnungAktuell" - KANTON_BAUVERORDNUNG_REVISION = "kantonBauverordnungRevision" - GEMEINDE_BZO_AKTUELL = "gemeindeBzoAktuell" - GEMEINDE_BZO_REVISION = "gemeindeBzoRevision" - - -class JaNein(str, Enum): - """Three-valued state for optional yes/no questions""" - UNBEKANNT = "" # Empty string for unknown/not captured - JA = "Ja" - NEIN = "Nein" - - -class GeoTag(str, Enum): - """Geopoint categories""" - K1 = "K1" # Fixpunkt höchster Genauigkeit - K2 = "K2" # Fixpunkt mittlerer Genauigkeit - K3 = "K3" # Fixpunkt niedriger Genauigkeit - GEOMETER = "Geometer" # Vom Geometer vermessener Punkt - - -# ===== Helper Models (must be defined before main models) ===== - -class GeoPunkt(BaseModel): - """Represents a 3D point with reference.""" - koordinatensystem: str = Field( - description="Coordinate system (e.g. 'LV95', 'EPSG:2056')", - frontend_type="text", - frontend_readonly=False, - frontend_required=True, - ) - x: float = Field( - description="East value (E) [m], typically 2'480'000 - 2'840'000", - frontend_type="number", - frontend_readonly=False, - frontend_required=True, - ) - y: float = Field( - description="North value (N) [m], typically 1'070'000 - 1'300'000", - frontend_type="number", - frontend_readonly=False, - frontend_required=True, - ) - z: Optional[float] = Field( - None, - description="Height above sea level [m]", - frontend_type="number", - frontend_readonly=False, - frontend_required=False, - ) - referenz: Optional[GeoTag] = Field( - None, - description="Point categorization", - frontend_type="select", - frontend_readonly=False, - frontend_required=False, - ) - - -class GeoPolylinie(BaseModel): - """Represents a line or polygon from multiple GeoPunkte.""" - id: str = Field( - default_factory=lambda: str(uuid.uuid4()), - description="Primary key", - ) - closed: bool = Field( - description="Is the GeoPolylinie closed (polygon)?", - frontend_type="boolean", - frontend_readonly=False, - frontend_required=True, - ) - punkte: List[GeoPunkt] = Field( - default_factory=list, - description="List of GeoPunkte forming the GeoPolylinie", - frontend_type="json", - frontend_readonly=False, - frontend_required=True, - ) - - -class Dokument(BaseModel): - """Supporting data object for file and URL management with versioning.""" - id: str = Field( - default_factory=lambda: str(uuid.uuid4()), - description="Primary key", - frontend_type="text", - frontend_readonly=True, - frontend_required=False, - ) - mandateId: str = Field( - description="ID of the mandate this document belongs to", - frontend_type="text", - frontend_readonly=True, - frontend_required=False, - ) - label: str = Field( - description="Document label", - frontend_type="text", - frontend_readonly=False, - frontend_required=True, - ) - versionsbezeichnung: Optional[str] = Field( - None, - description="Version number or designation (e.g. 'v1.0', 'Rev. A')", - frontend_type="text", - frontend_readonly=False, - frontend_required=False, - ) - dokumentTyp: Optional[DokumentTyp] = Field( - None, - description="Document type", - frontend_type="select", - frontend_readonly=False, - frontend_required=False, - ) - dokumentReferenz: str = Field( - description="File path or URL", - frontend_type="text", - frontend_readonly=False, - frontend_required=True, - ) - quelle: Optional[str] = Field( - None, - description="Source of the document", - frontend_type="text", - frontend_readonly=False, - frontend_required=False, - ) - mimeType: Optional[str] = Field( - None, - description="MIME type of the document (e.g. 'application/pdf', 'image/png')", - frontend_type="text", - frontend_readonly=False, - frontend_required=False, - ) - kategorienTags: List[str] = Field( - default_factory=list, - description="Document categorization tags", - frontend_type="json", - frontend_readonly=False, - frontend_required=False, - ) - - -# Beispiel-Kategorien für Dokumente (nicht abschließend): -# - "Kataster Objekte" - Amtliche Vermessung -# - "Kataster Werkeleitungen" - Leitungskataster -# - "Kataster Belastete Standorte" - Altlasten -# - "Kataster Bäume" - Baumkataster -# - "Zonenplan" - Zonenpläne -# - "Planungs- und Baugesetz (PGB)" - Kantonale Baugesetze -# - "Bau- und Zonenordnung (BZO)" - Gemeinde BZO -# - "Parkplatzverordnung" - Parkplatzregelungen -# - "Eigentümerauskunft" - Grundbuch-Auszüge Eigentümer -# - "Grundbuchauszug" - Vollständige Grundbuch-Auszüge -# - "Bauherrschaft" - Dokumente von der Bauherrschaft -# - "Planung" - Planungsdokumente -# -# Hinweis: Aktuelle Dokumente (z.B. aktuelle Baureglemente, BZO) können anhand des -# `dokumentTyp`-Attributs identifiziert werden. Die entsprechenden Dokumente finden sich -# in der `dokumente`-Liste der jeweiligen Entität (Kanton, Gemeinde). - - -class Kontext(BaseModel): - """Supporting data object for flexible additional information.""" - id: str = Field( - default_factory=lambda: str(uuid.uuid4()), - description="Primary key", - ) - thema: str = Field( - description="Theme designation", - frontend_type="text", - frontend_readonly=False, - frontend_required=True, - ) - inhalt: str = Field( - description="Detailed information (text)", - frontend_type="textarea", - frontend_readonly=False, - frontend_required=True, - ) - - -# Beispielthemen für Kontext (nicht abschließend): -# - "Nutzung" - Vorgaben zur Erdgeschossnutzung (Wohnen erlaubt oder Pflicht für Gewerbe) -# - "Rechte" - Dienstbarkeiten (Wegrechte, Nähebaurechte, etc.) -# - "Parkierung" - Anforderung Parkplätze (Berechnung / Reduktionsfaktoren) -# - "Ausnützung" - Ausnützungsübertragungen -# - "Umwelt" - Schadstoffbelastungen auf Parzellen -# - "Planung" - Aktive Gestaltungspläne -# - "Lärm" - Lärmempfindlichkeitsstufen -# - "Energie" - Mögliche Wärmenutzung (Wärmeverbundnetze; Fernwärme, Anergie) -# - "Natur" - Baumbestand auf privaten Grundstücken -# - "Schutz" - Isos (Ortsbild, Schutzstatus, Denkmalschutz, Weilergebiet, etc.) -# - "Gefahren" - Naturgefahren (z.B. Objektschutzmassnahmen (Hochwasser)) -# - "Revision" - Verweis auf aktuell in oder zukünftig in Revision befindlichen Normen/Gesetze -# -# Verwendung: Kontext-Objekte werden als Listen in den jeweiligen Entitäten gespeichert: -# - projekt.kontextInformationen: list[Kontext] -# - parzelle.kontextInformationen: list[Kontext] -# - land.kontextInformationen: list[Kontext] -# - kanton.kontextInformationen: list[Kontext] -# - gemeinde.kontextInformationen: list[Kontext] -# -# Design-Rationale: Das Kontext-Objekt ermöglicht flexibles Hinzufügen von projektspezifischen, -# parzellen-spezifischen oder regionalen Informationen ohne Schemaänderungen. - - -class Land(BaseModel): - """National level administrative entity.""" - id: str = Field( - default_factory=lambda: str(uuid.uuid4()), - description="Primary key", - frontend_type="text", - frontend_readonly=True, - frontend_required=False, - ) - mandateId: str = Field( - description="ID of the mandate", - frontend_type="text", - frontend_readonly=True, - frontend_required=False, - ) - label: str = Field( - description="Country name (e.g. 'Schweiz')", - frontend_type="text", - frontend_readonly=False, - frontend_required=True, - ) - abk: Optional[str] = Field( - None, - description="Abbreviation (e.g. 'CH')", - frontend_type="text", - frontend_readonly=False, - frontend_required=False, - ) - dokumente: List[Dokument] = Field( - default_factory=list, - description="National laws/documents", - frontend_type="json", - frontend_readonly=False, - frontend_required=False, - ) - kontextInformationen: List[Kontext] = Field( - default_factory=list, - description="National context information", - frontend_type="json", - frontend_readonly=False, - frontend_required=False, - ) - - -class Kanton(BaseModel): - """Cantonal level administrative entity.""" - id: str = Field( - default_factory=lambda: str(uuid.uuid4()), - description="Primary key", - frontend_type="text", - frontend_readonly=True, - frontend_required=False, - ) - mandateId: str = Field( - description="ID of the mandate", - frontend_type="text", - frontend_readonly=True, - frontend_required=False, - ) - label: str = Field( - description="Canton name (e.g. 'Zürich')", - frontend_type="text", - frontend_readonly=False, - frontend_required=True, - ) - id_land: Optional[str] = Field( - None, - description="Land ID (Foreign Key) - eindeutiger Link zum Land, in welchem Land der Kanton liegt", - frontend_type="text", - frontend_readonly=False, - frontend_required=False, - ) - abk: Optional[str] = Field( - None, - description="Abbreviation (e.g. 'ZH')", - frontend_type="text", - frontend_readonly=False, - frontend_required=False, - ) - dokumente: List[Dokument] = Field( - default_factory=list, - description="Cantonal documents", - frontend_type="json", - frontend_readonly=False, - frontend_required=False, - ) - kontextInformationen: List[Kontext] = Field( - default_factory=list, - description="Canton-specific context information", - frontend_type="json", - frontend_readonly=False, - frontend_required=False, - ) - - -class Gemeinde(BaseModel): - """Municipal level administrative entity.""" - id: str = Field( - default_factory=lambda: str(uuid.uuid4()), - description="Primary key", - frontend_type="text", - frontend_readonly=True, - frontend_required=False, - ) - mandateId: str = Field( - description="ID of the mandate", - frontend_type="text", - frontend_readonly=True, - frontend_required=False, - ) - label: str = Field( - description="Municipality name (e.g. 'Zürich')", - frontend_type="text", - frontend_readonly=False, - frontend_required=True, - ) - id_kanton: Optional[str] = Field( - None, - description="Kanton ID (Foreign Key) - eindeutiger Link zum Kanton, in welchem Kanton die Gemeinde liegt", - frontend_type="text", - frontend_readonly=False, - frontend_required=False, - ) - plz: Optional[str] = Field( - None, - description="Postal code (for municipalities with multiple PLZ, this can be a main PLZ). Bei Gemeinden mit mehreren Postleitzahlen wird die konkrete PLZ der Parzelle im Attribut `plz` der Parzelle erfasst.", - frontend_type="text", - frontend_readonly=False, - frontend_required=False, - ) - dokumente: List[Dokument] = Field( - default_factory=list, - description="Municipal documents", - frontend_type="json", - frontend_readonly=False, - frontend_required=False, - ) - kontextInformationen: List[Kontext] = Field( - default_factory=list, - description="Municipality-specific context information", - frontend_type="json", - frontend_readonly=False, - frontend_required=False, - ) - - -# ===== Main Models (use ForwardRef for circular references) ===== - -# Forward references for circular dependencies -ParzelleRef = ForwardRef('Parzelle') - - -class Parzelle(BaseModel): - """Represents a plot with all building law properties.""" - id: str = Field( - default_factory=lambda: str(uuid.uuid4()), - description="Primary key", - frontend_type="text", - frontend_readonly=True, - frontend_required=False, - ) - mandateId: str = Field( - description="ID of the mandate", - frontend_type="text", - frontend_readonly=True, - frontend_required=False, - ) - - # Grunddaten - label: str = Field( - description="Plot designation", - frontend_type="text", - frontend_readonly=False, - frontend_required=True, - ) - parzellenAliasTags: List[str] = Field( - default_factory=list, - description="Additional plot names or field names", - frontend_type="json", - frontend_readonly=False, - frontend_required=False, - ) - eigentuemerschaft: Optional[str] = Field( - None, - description="Owner of the plot", - frontend_type="text", - frontend_readonly=False, - frontend_required=False, - ) - strasseNr: Optional[str] = Field( - None, - description="Street and house number", - frontend_type="text", - frontend_readonly=False, - frontend_required=False, - ) - plz: Optional[str] = Field( - None, - description="Postal code of the plot", - frontend_type="text", - frontend_readonly=False, - frontend_required=False, - ) - - # Geografischer Kontext - perimeter: Optional[GeoPolylinie] = Field( - None, - description="Plot boundary as closed GeoPolylinie", - frontend_type="json", - frontend_readonly=False, - frontend_required=False, - ) - baulinie: Optional[GeoPolylinie] = Field( - None, - description="Building line of the plot", - frontend_type="json", - frontend_readonly=False, - frontend_required=False, - ) - kontextLand: Optional[str] = Field( - None, - description="Land ID (Foreign Key)", - frontend_type="text", - frontend_readonly=False, - frontend_required=False, - ) - kontextKanton: Optional[str] = Field( - None, - description="Canton ID (Foreign Key)", - frontend_type="text", - frontend_readonly=False, - frontend_required=False, - ) - kontextGemeinde: Optional[str] = Field( - None, - description="Municipality ID (Foreign Key)", - frontend_type="text", - frontend_readonly=False, - frontend_required=False, - ) - - # Bebauungsparameter - bauzone: Optional[str] = Field( - None, - description="Building zone designation (e.g. W3, WG2, etc.)", - frontend_type="text", - frontend_readonly=False, - frontend_required=False, - ) - az: Optional[float] = Field( - None, - description="Ausnützungsziffer", - frontend_type="number", - frontend_readonly=False, - frontend_required=False, - ) - bz: Optional[float] = Field( - None, - description="Bebauungsziffer", - frontend_type="number", - frontend_readonly=False, - frontend_required=False, - ) - vollgeschossZahl: Optional[int] = Field( - None, - description="Number of allowed full floors", - frontend_type="number", - frontend_readonly=False, - frontend_required=False, - ) - anrechenbarDachgeschoss: Optional[float] = Field( - None, - description="Accountable portion of attic (0.0 - 1.0)", - frontend_type="number", - frontend_readonly=False, - frontend_required=False, - ) - anrechenbarUntergeschoss: Optional[float] = Field( - None, - description="Accountable portion of basement (0.0 - 1.0)", - frontend_type="number", - frontend_readonly=False, - frontend_required=False, - ) - gebaeudehoeheMax: Optional[float] = Field( - None, - description="Maximum building height in meters", - frontend_type="number", - frontend_readonly=False, - frontend_required=False, - ) - - # Abstandsregelungen - regelnGrenzabstand: List[str] = Field( - default_factory=list, - description="Regulations for boundary distance", - frontend_type="json", - frontend_readonly=False, - frontend_required=False, - ) - regelnMehrlaengenzuschlag: List[str] = Field( - default_factory=list, - description="Regulations for additional length surcharge", - frontend_type="json", - frontend_readonly=False, - frontend_required=False, - ) - regelnMehrhoehenzuschlag: List[str] = Field( - default_factory=list, - description="Regulations for additional height surcharge", - frontend_type="json", - frontend_readonly=False, - frontend_required=False, - ) - - # Eigenschaften (Ja/Nein) - parzelleBebaut: Optional[JaNein] = Field( - None, - description="Is the plot built?", - frontend_type="select", - frontend_readonly=False, - frontend_required=False, - ) - parzelleErschlossen: Optional[JaNein] = Field( - None, - description="Is the plot developed?", - frontend_type="select", - frontend_readonly=False, - frontend_required=False, - ) - parzelleHanglage: Optional[JaNein] = Field( - None, - description="Is the plot on a slope?", - frontend_type="select", - frontend_readonly=False, - frontend_required=False, - ) - - # Schutzzonen - laermschutzzone: Optional[str] = Field( - None, - description="Noise protection zone (e.g. 'II')", - frontend_type="text", - frontend_readonly=False, - frontend_required=False, - ) - hochwasserschutzzone: Optional[str] = Field( - None, - description="Flood protection zone (e.g. 'tief')", - frontend_type="text", - frontend_readonly=False, - frontend_required=False, - ) - grundwasserschutzzone: Optional[str] = Field( - None, - description="Groundwater protection zone", - frontend_type="text", - frontend_readonly=False, - frontend_required=False, - ) - - # Beziehungen (stored as JSONB in database) - parzellenNachbarschaft: List[Dict[str, Any]] = Field( - default_factory=list, - description="Neighboring plots (stored as list of Parzelle IDs or full objects)", - frontend_type="json", - frontend_readonly=False, - frontend_required=False, - ) - dokumente: List[Dokument] = Field( - default_factory=list, - description="Plot-specific documents", - frontend_type="json", - frontend_readonly=False, - frontend_required=False, - ) - kontextInformationen: List[Kontext] = Field( - default_factory=list, - description="Plot-specific context information", - frontend_type="json", - frontend_readonly=False, - frontend_required=False, - ) - - -class Projekt(BaseModel): - """Core object representing a construction project.""" - id: str = Field( - default_factory=lambda: str(uuid.uuid4()), - description="Primary key", - frontend_type="text", - frontend_readonly=True, - frontend_required=False, - ) - mandateId: str = Field( - description="ID of the mandate", - frontend_type="text", - frontend_readonly=True, - frontend_required=False, - ) - label: str = Field( - description="Project designation", - frontend_type="text", - frontend_readonly=False, - frontend_required=True, - ) - statusProzess: Optional[StatusProzess] = Field( - None, - description="Project status", - frontend_type="select", - frontend_readonly=False, - frontend_required=False, - ) - perimeter: Optional[GeoPolylinie] = Field( - None, - description="Envelope of all plots in the project", - frontend_type="json", - frontend_readonly=False, - frontend_required=False, - ) - baulinie: Optional[GeoPolylinie] = Field( - None, - description="Building line of the project", - frontend_type="json", - frontend_readonly=False, - frontend_required=False, - ) - parzellen: List[Parzelle] = Field( - default_factory=list, - description="All plots of the project", - frontend_type="json", - frontend_readonly=False, - frontend_required=False, - ) - dokumente: List[Dokument] = Field( - default_factory=list, - description="Project-specific documents", - frontend_type="json", - frontend_readonly=False, - frontend_required=False, - ) - kontextInformationen: List[Kontext] = Field( - default_factory=list, - description="Project-specific context information", - frontend_type="json", - frontend_readonly=False, - frontend_required=False, - ) - - -# Resolve forward references -Parzelle.model_rebuild() -Projekt.model_rebuild() - - -# Register labels for frontend -registerModelLabels( - "Projekt", - {"en": "Project", "fr": "Projet", "de": "Projekt"}, - { - "id": {"en": "ID", "fr": "ID", "de": "ID"}, - "label": {"en": "Label", "fr": "Libellé", "de": "Bezeichnung"}, - "statusProzess": {"en": "Process Status", "fr": "Statut du processus", "de": "Prozessstatus"}, - # ... more labels - }, -) - -# Similar registerModelLabels calls for all other models... -``` - -**Wichtige Hinweise zur Implementierung:** - -1. **Forward References**: Für zirkuläre Referenzen (z.B. `Parzelle` → `parzellenNachbarschaft: list[Parzelle]`) verwenden Sie `ForwardRef` oder speichern Sie nur IDs als Strings. - -2. **JSONB-Speicherung**: Listen von Objekten (`list[Parzelle]`, `list[Dokument]`) werden automatisch als JSONB gespeichert. Der DatabaseConnector erkennt `List`-Typen automatisch. - -3. **Foreign Keys**: Einzelne Objektreferenzen wie `kontextKanton: Optional[str]` werden als String-ID gespeichert. Sie können später im Interface die vollständigen Objekte laden. - -4. **MandateId**: Alle Entitäten benötigen `mandateId` für Mandaten-Isolation. - -5. **Systemattribute**: `_createdAt`, `_createdBy`, `_modifiedAt`, `_modifiedBy` werden automatisch vom DatabaseConnector hinzugefügt - Sie müssen sie nicht im Modell definieren. - ---- - -**WICHTIG:** Die obigen Real Estate-Modelle (`Projekt`, `Parzelle`, etc.) sind die **tatsächlichen Datenmodelle**, die Sie implementieren müssen. Diese werden in `modules/datamodels/datamodelRealEstate.py` erstellt. - -**Keine Chat-Interface-Modelle notwendig:** -- Das Feature arbeitet **stateless** ohne Session-Management -- Alle Operationen arbeiten direkt auf den Real Estate-Modellen -- Keine `RealEstateChatSession`, `RealEstateQuery` oder `RealEstateQueryResult` notwendig -- CRUD-Operationen werden direkt ausgeführt und Ergebnisse direkt zurückgegeben - -### Wichtige Punkte: - -1. **UUID als ID**: Alle Modelle verwenden `uuid.uuid4()` für eindeutige IDs -2. **MandateId**: Jedes Modell benötigt `mandateId` für Mandaten-Isolation -3. **Frontend-Metadaten**: `frontend_type`, `frontend_readonly`, `frontend_required` für UI-Generierung -4. **registerModelLabels**: Registriert Labels für Mehrsprachigkeit -5. **JSONB-Felder**: `Dict[str, Any]` und `List[...]` werden automatisch als JSONB in PostgreSQL gespeichert -6. **Foreign Keys**: Administrative Hierarchie wird über Foreign Keys abgebildet: - - `Kanton.id_land` → `Land.id` - - `Gemeinde.id_kanton` → `Kanton.id` - - `Parzelle.kontextLand` → `Land.id` (Optional) - - `Parzelle.kontextKanton` → `Kanton.id` (Optional) - - `Parzelle.kontextGemeinde` → `Gemeinde.id` (Optional) - ---- - -## Q & A - Häufige Fragen - -1. **Versionierung**: Sollen Änderungen an Parzellen historisiert werden? - → Vorerst nicht - -2. **Mehrsprachigkeit**: Labels in DE/FR/IT? - → Wird im Pydantic Model über `registerModelLabels` umgesetzt - -3. **Benutzer & Rollen**: Wer darf was bearbeiten? - → In der App über Roles und Permissions gesteuert (UAM-System) - -4. **Workflow-Engine**: Für Statusübergänge und Genehmigungen? - → In der App über Workflow-Engine gesteuert (optional, kann später integriert werden) - -5. **Integration**: Anbindung an amtliche Geodaten (z.B. Swisstopo API)? - → In der App über Integrationen gesteuert (optional) - -6. **Berechnungen**: Sollen Ausnützungsberechnungen automatisiert werden? - → In der App über Berechnungen gesteuert (optional) - ---- - -[← Zurück zur Übersicht](README.md) | [Weiter: Interface erstellen →](03-interfaces.md) - diff --git a/docs/real-estate-feature-integration-guide/03-interfaces.md b/docs/real-estate-feature-integration-guide/03-interfaces.md index a90c31b9..628b6e5a 100644 --- a/docs/real-estate-feature-integration-guide/03-interfaces.md +++ b/docs/real-estate-feature-integration-guide/03-interfaces.md @@ -39,9 +39,7 @@ Da das Feature **stateless** arbeitet, benötigen wir nur **ein Interface** für - `RealEstateAccess` (Zugriffskontrolle) - Methoden: `createProjekt()`, `getParzelle()`, `updateDokument()`, etc. -**Optional:** `interfaceDbRealEstateChatObjects.py` (nur für direkte SQL-Queries) -- `RealEstateChatObjects` - Für direkte Query-Ausführung ohne Session -- Methoden: `executeQuery()` - Führt SQL direkt aus +**Hinweis:** Das Haupt-Interface enthält auch `executeQuery()` für direkte SQL-Queries (stateless) --- @@ -55,8 +53,9 @@ Da das Feature **stateless** arbeitet, benötigen wir nur **ein Interface** für - Ein Interface für alle CRUD-Operationen - Weniger Komplexität, bessere Wartbarkeit -3. **Optionales Query-Interface**: - - Nur für direkte SQL-Queries (stateless) +3. **Query-Funktionalität**: + - `executeQuery()` ist direkt im Haupt-Interface verfügbar + - Für direkte SQL-Queries (stateless) - Keine Session-Management-Funktionen --- @@ -92,27 +91,6 @@ Da das Feature **stateless** arbeitet, benötigen wir nur **ein Interface** für --- -### Schritt 2b: Query-Interface (OPTIONAL - nur für direkte SQL-Queries) - -**Eine Datei** für stateless Query-Ausführung: - -#### Datei: `modules/interfaces/interfaceDbRealEstateChatObjects.py` - -**Enthält:** -- `RealEstateChatObjects` - Interface für direkte SQL-Query-Ausführung -- `getInterface()` - Factory-Funktion -- Methoden: `executeQuery()` - Führt SQL direkt aus (stateless) - -**Zweck:** Direkte SQL-Query-Ausführung ohne Session-Management - -**Nutzt:** -- `connectorDbPostgre.DatabaseConnector` für direkte SQL-Ausführung -- Keine Chat-Modelle (stateless) - -**Wann benötigt:** Nur wenn Sie direkte SQL-Queries ausführen möchten (z.B. für komplexe SELECT-Queries). Für CRUD-Operationen verwenden Sie das Real Estate CRUD-Interface. - ---- - ## Übersicht: Dateien und ihre Beziehungen ``` @@ -153,13 +131,9 @@ Da das Feature **stateless** arbeitet, benötigen wir nur **ein Interface** für │ └── getInterface() │ │ └── nutzt RealEstateAccess │ │ │ -│ QUERY-INTERFACE (OPTIONAL - nur für direkte SQL) │ │ │ -│ interfaceDbRealEstateChatObjects.py │ -│ ├── RealEstateChatObjects │ -│ │ └── executeQuery() # Direkte SQL-Ausführung │ -│ └── getInterface() │ -│ └── Keine Access-Klasse (stateless) │ +│ HINWEIS: executeQuery() ist im Haupt-Interface verfügbar │ +│ (kein separates Chat-Interface notwendig) │ └─────────────────────────────────────────────────────────────┘ ``` @@ -177,7 +151,7 @@ Jedes Interface besteht aus **zwei Klassen**: - `uam()` - Filtert Daten basierend auf Benutzerprivilegien - `canModify()` - Prüft, ob Benutzer ändern darf -**Beispiel:** `RealEstateChatAccess` +**Beispiel:** `RealEstateAccess` ### 2. `*Objects` Klasse (Haupt-Interface) @@ -185,13 +159,15 @@ Jedes Interface besteht aus **zwei Klassen**: **Methoden:** - `create*()` - Erstellt neue Einträge -- `get*()` - Lädt Einträge +- `get*()` - Lädt einzelne Einträge nach ID +- `get*()` (Plural) - Lädt Listen von Einträgen mit optionalen Filtern - `update*()` - Aktualisiert Einträge - `delete*()` - Löscht Einträge +- `executeQuery()` - Führt direkte SQL-Queries aus (stateless) **Nutzt:** `*Access` für Zugriffskontrolle -**Beispiel:** `RealEstateChatObjects` +**Beispiel:** `RealEstateObjects` **Warum getrennt?** - Separation of Concerns: Zugriffskontrolle ist separate Verantwortlichkeit @@ -208,100 +184,14 @@ Das Real Estate CRUD-Interface besteht aus **zwei separaten Dateien**, genau wie **Datei:** `modules/interfaces/interfaceDbRealEstateAccess.py` -```python -""" -Access control for Real Estate interface. -Handles user access management and permission checks. -""" +**Enthält:** +- `RealEstateAccess` Klasse +- Methoden: `uam()`, `canModify()` -import logging -from typing import Dict, Any, List, Optional -from modules.datamodels.datamodelUam import User, UserPrivilege - -logger = logging.getLogger(__name__) - - -class RealEstateAccess: - """ - Access control class for Real Estate interface. - Handles user access management and permission checks. - """ - - def __init__(self, currentUser: User, db): - """Initialize with user context.""" - self.currentUser = currentUser - self.mandateId = currentUser.mandateId - self.userId = currentUser.id - - if not self.mandateId or not self.userId: - raise ValueError("Invalid user context: mandateId and userId are required") - - self.db = db - - def uam(self, model_class: type, recordset: List[Dict[str, Any]]) -> List[Dict[str, Any]]: - """ - Unified user access management function that filters data based on user privileges. - - Args: - model_class: Pydantic model class for the table - recordset: Recordset to filter based on access rules - - Returns: - Filtered recordset with access control attributes - """ - from modules.datamodels.datamodelUam import UserPrivilege - - userPrivilege = self.currentUser.privilege - filtered_records = [] - - # System admins see all records - if userPrivilege == UserPrivilege.SYSADMIN: - filtered_records = recordset - # Admins see records in their mandate - elif userPrivilege == UserPrivilege.ADMIN: - filtered_records = [r for r in recordset if r.get("mandateId", "-") == self.mandateId] - # Regular users see only their records - else: - filtered_records = [ - r for r in recordset - if r.get("mandateId", "-") == self.mandateId and r.get("_createdBy") == self.userId - ] - - # Add access control attributes - for record in filtered_records: - record["_hideView"] = False - record["_hideEdit"] = not self.canModify(model_class, record.get("id")) - record["_hideDelete"] = not self.canModify(model_class, record.get("id")) - - return filtered_records - - def canModify(self, model_class: type, recordId: Optional[str] = None) -> bool: - """Checks if the current user can modify records.""" - from modules.datamodels.datamodelUam import UserPrivilege - - userPrivilege = self.currentUser.privilege - - if userPrivilege == UserPrivilege.SYSADMIN: - return True - - if recordId is not None: - records = self.db.getRecordset(model_class, recordFilter={"id": recordId}) - if not records: - return False - - record = records[0] - - if userPrivilege == UserPrivilege.ADMIN and record.get("mandateId", "-") == self.mandateId: - return True - - if (record.get("mandateId", "-") == self.mandateId and - record.get("_createdBy") == self.userId): - return True - - return False - else: - return True # Regular users can create records -``` +**Funktionalität:** +- **`uam()`**: Filtert Datensätze basierend auf Benutzerprivilegien (SYSADMIN sieht alles, ADMIN sieht Mandat, User sieht nur eigene) +- **`canModify()`**: Prüft, ob Benutzer Datensätze ändern/löschen darf +- Fügt Zugriffskontroll-Attribute hinzu: `_hideView`, `_hideEdit`, `_hideDelete` --- @@ -309,267 +199,27 @@ class RealEstateAccess: **Datei:** `modules/interfaces/interfaceDbRealEstateObjects.py` -```python -""" -Interface to Real Estate database objects. -Uses PostgreSQL connector for data access with user/mandate filtering. -Handles CRUD operations on Real Estate entities (Projekt, Parzelle, etc.). -""" +**Enthält:** +- `RealEstateObjects` Klasse (Haupt-Interface) +- `getInterface()` Factory-Funktion (Singleton-Pattern) -import logging -from typing import Dict, Any, List, Optional -from modules.datamodels.datamodelRealEstate import ( - Projekt, - Parzelle, - Dokument, - Kanton, - Gemeinde, - Land, - GeoPolylinie, - GeoPunkt, - Kontext, - StatusProzess, -) -from modules.datamodels.datamodelUam import User -from modules.connectors.connectorDbPostgre import DatabaseConnector -from modules.shared.configuration import APP_CONFIG -# Import Access-Klasse aus separater Datei -from modules.interfaces.interfaceDbRealEstateAccess import RealEstateAccess +**Datenbank-Konfiguration:** +- Verwendet `DB_REALESTATE_*` Umgebungsvariablen (nicht `DB_APP_*`) +- Variablen: `DB_REALESTATE_HOST`, `DB_REALESTATE_DATABASE`, `DB_REALESTATE_USER`, `DB_REALESTATE_PASSWORD_SECRET`, `DB_REALESTATE_PORT` -logger = logging.getLogger(__name__) +**CRUD-Methoden für alle Entitäten:** +- **Projekt**: `createProjekt()`, `getProjekt()`, `getProjekte()`, `updateProjekt()`, `deleteProjekt()` +- **Parzelle**: `createParzelle()`, `getParzelle()`, `getParzellen()`, `updateParzelle()`, `deleteParzelle()` +- **Dokument**: `createDokument()`, `getDokument()`, `getDokumente()`, `updateDokument()`, `deleteDokument()` +- **Gemeinde**: `createGemeinde()`, `getGemeinde()`, `getGemeinden()`, `updateGemeinde()`, `deleteGemeinde()` +- **Kanton**: `createKanton()`, `getKanton()`, `getKantone()`, `updateKanton()`, `deleteKanton()` +- **Land**: `createLand()`, `getLand()`, `getLaender()`, `updateLand()`, `deleteLand()` -# Singleton factory for Real Estate interfaces -_realEstateInterfaces = {} - - -class RealEstateObjects: - """ - Interface to Real Estate database objects. - Uses PostgreSQL connector for data access with user/mandate filtering. - Handles CRUD operations on Real Estate entities. - """ - - def __init__(self, currentUser: Optional[User] = None): - """Initializes the Real Estate Interface.""" - self.currentUser = currentUser - self.userId = currentUser.id if currentUser else None - self.mandateId = currentUser.mandateId if currentUser else None - self.access = None - - # Initialize database - self._initializeDatabase() - - # Set user context if provided - if currentUser: - self.setUserContext(currentUser) - - def _initializeDatabase(self): - """Initialize PostgreSQL database connection.""" - try: - # Get database configuration from environment - dbHost = APP_CONFIG.get("DB_APP_HOST", "localhost") - dbDatabase = APP_CONFIG.get("DB_APP_DATABASE", "poweron_app") - dbUser = APP_CONFIG.get("DB_APP_USER") - dbPassword = APP_CONFIG.get("DB_APP_PASSWORD_SECRET") - dbPort = int(APP_CONFIG.get("DB_APP_PORT", 5432)) - - # Initialize database connector - self.db = DatabaseConnector( - dbHost=dbHost, - dbDatabase=dbDatabase, - dbUser=dbUser, - dbPassword=dbPassword, - dbPort=dbPort, - userId=self.userId if self.userId else None, - ) - - logger.info(f"Real Estate database connector initialized for database: {dbDatabase}") - except Exception as e: - logger.error(f"Error initializing Real Estate database: {e}") - raise - - def setUserContext(self, currentUser: User): - """Sets the user context for the interface.""" - self.currentUser = currentUser - self.userId = currentUser.id - self.mandateId = currentUser.mandateId - - if not self.userId or not self.mandateId: - raise ValueError("Invalid user context: id and mandateId are required") - - # Initialize access control - self.access = RealEstateAccess(self.currentUser, self.db) - - # Update database context - self.db.updateContext(self.userId) - - # ===== Projekt Methods ===== - - def createProjekt(self, projekt: Projekt) -> Projekt: - """Create a new project.""" - # Ensure mandateId is set - if not projekt.mandateId: - projekt.mandateId = self.mandateId - - # Apply access control - self.access.uam(Projekt, []) - - # Save to database - self.db.recordCreate(Projekt, projekt.model_dump()) - - return projekt - - def getProjekt(self, projektId: str) -> Optional[Projekt]: - """Get a project by ID.""" - records = self.db.getRecordset( - Projekt, - recordFilter={"id": projektId} - ) - - if not records: - return None - - # Apply access control - filtered = self.access.uam(Projekt, records) - - if not filtered: - return None - - return Projekt(**filtered[0]) - - def updateProjekt(self, projektId: str, updateData: Dict[str, Any]) -> Optional[Projekt]: - """Update a project.""" - projekt = self.getProjekt(projektId) - if not projekt: - return None - - # Check if user can modify - if not self.access.canModify(Projekt, projektId): - raise PermissionError(f"User {self.userId} cannot modify project {projektId}") - - # Update fields - for key, value in updateData.items(): - if hasattr(projekt, key): - setattr(projekt, key, value) - - # Save to database - self.db.recordModify(Projekt, projektId, projekt.model_dump()) - - return projekt - - def deleteProjekt(self, projektId: str) -> bool: - """Delete a project.""" - projekt = self.getProjekt(projektId) - if not projekt: - return False - - # Check if user can modify - if not self.access.canModify(Projekt, projektId): - raise PermissionError(f"User {self.userId} cannot delete project {projektId}") - - return self.db.recordDelete(Projekt, projektId) - - # ===== Parzelle Methods ===== - - def createParzelle(self, parzelle: Parzelle) -> Parzelle: - """Create a new plot.""" - if not parzelle.mandateId: - parzelle.mandateId = self.mandateId - - self.access.uam(Parzelle, []) - self.db.recordCreate(Parzelle, parzelle.model_dump()) - - return parzelle - - def getParzelle(self, parzelleId: str) -> Optional[Parzelle]: - """Get a plot by ID.""" - records = self.db.getRecordset( - Parzelle, - recordFilter={"id": parzelleId} - ) - - if not records: - return None - - filtered = self.access.uam(Parzelle, records) - - if not filtered: - return None - - return Parzelle(**filtered[0]) - - def updateParzelle(self, parzelleId: str, updateData: Dict[str, Any]) -> Optional[Parzelle]: - """Update a plot.""" - parzelle = self.getParzelle(parzelleId) - if not parzelle: - return None - - if not self.access.canModify(Parzelle, parzelleId): - raise PermissionError(f"User {self.userId} cannot modify plot {parzelleId}") - - for key, value in updateData.items(): - if hasattr(parzelle, key): - setattr(parzelle, key, value) - - self.db.recordModify(Parzelle, parzelleId, parzelle.model_dump()) - - return parzelle - - def deleteParzelle(self, parzelleId: str) -> bool: - """Delete a plot.""" - parzelle = self.getParzelle(parzelleId) - if not parzelle: - return False - - if not self.access.canModify(Parzelle, parzelleId): - raise PermissionError(f"User {self.userId} cannot delete plot {parzelleId}") - - return self.db.recordDelete(Parzelle, parzelleId) - - # ===== Dokument Methods ===== - - def createDokument(self, dokument: Dokument) -> Dokument: - """Create a new document.""" - if not dokument.mandateId: - dokument.mandateId = self.mandateId - - self.access.uam(Dokument, []) - self.db.recordCreate(Dokument, dokument.model_dump()) - - return dokument - - def getDokument(self, dokumentId: str) -> Optional[Dokument]: - """Get a document by ID.""" - records = self.db.getRecordset( - Dokument, - recordFilter={"id": dokumentId} - ) - - if not records: - return None - - filtered = self.access.uam(Dokument, records) - - if not filtered: - return None - - return Dokument(**filtered[0]) - - # ... weitere CRUD-Methoden für andere Entitäten (Kanton, Gemeinde, Land, etc.) - - -def getInterface(currentUser: User) -> RealEstateObjects: - """ - Factory function to get or create a Real Estate interface instance for a user. - Uses singleton pattern per user. - """ - userKey = f"{currentUser.id}_{currentUser.mandateId}" - - if userKey not in _realEstateInterfaces: - _realEstateInterfaces[userKey] = RealEstateObjects(currentUser) - - return _realEstateInterfaces[userKey] -``` +**Zusätzliche Funktionalität:** +- **List-Methoden**: Alle Entitäten haben `get*()` (Plural) Methoden für Listen mit optionalen Filtern +- **Location-Resolution**: `getParzellen()` löst automatisch Gemeinde-Namen zu IDs auf +- **Query-Ausführung**: `executeQuery()` für direkte SQL-Queries (stateless) +- **Supporting Tables**: Automatische Erstellung von Land, Kanton, Gemeinde, Dokument Tabellen bei Initialisierung ## Wichtige Punkte: @@ -578,241 +228,34 @@ def getInterface(currentUser: User) -> RealEstateObjects: 3. **Singleton Pattern**: `getInterface()` erstellt pro User eine Instanz 4. **CRUD-Operationen**: `recordCreate`, `recordModify`, `recordDelete`, `getRecordset` vom Connector 5. **MandateId**: Wird automatisch gesetzt, wenn nicht vorhanden +6. **List-Methoden**: Alle Entitäten haben `get*()` (Plural) Methoden für Listen mit optionalen Filtern +7. **Location-Resolution**: Parzelle-Filter können Gemeinde-Namen enthalten, die automatisch zu IDs aufgelöst werden +8. **Query-Ausführung**: `executeQuery()` ist direkt im Haupt-Interface verfügbar (kein separates Chat-Interface notwendig) +9. **Datenbank-Initialisierung**: Unterstützende Tabellen (Land, Kanton, Gemeinde, Dokument) werden automatisch erstellt --- -## Schritt 2b: Query-Interface (OPTIONAL - nur für direkte SQL-Queries) +## Query-Ausführung: executeQuery() -### Wann benötigt? +Das Haupt-Interface `RealEstateObjects` enthält die Methode `executeQuery()` für direkte SQL-Queries. **Kein separates Chat-Interface ist notwendig.** -**Kurze Antwort:** Nur wenn Sie **direkte SQL-Queries** ausführen möchten (z.B. für komplexe SELECT-Queries). Für CRUD-Operationen verwenden Sie das Real Estate CRUD-Interface. +### Verwendung -#### Szenario 1: Alles über CRUD-Interface (EMPFOHLEN) +**Für CRUD-Operationen (EMPFOHLEN):** +- Verwenden Sie die strukturierten CRUD-Methoden (`createProjekt()`, `getProjekte()`, etc.) +- Vorteile: Validierung, Zugriffskontrolle, Typsicherheit, keine SQL-Injection-Risiken -**Strukturiert und sicher:** +**Für komplexe SELECT-Queries (OPTIONAL):** +- Verwenden Sie `executeQuery()` direkt im Haupt-Interface +- **Warnung**: Nur für SELECT-Queries, immer Parameterisierung verwenden, Queries validieren -```python -# User schreibt: "Erstelle Projekt 'Test'" -# Feature-Logik nutzt CRUD-Interface: -realEstateInterface = getRealEstateInterface(currentUser) -projekt = realEstateInterface.createProjekt(Projekt( - mandateId=currentUser.mandateId, # Automatisch gesetzt - label="Test" # Validierung durch Pydantic -)) -``` - -**Vorteile:** -- ✅ **Validierung**: Pydantic-Modelle prüfen alle Felder automatisch -- ✅ **Zugriffskontrolle**: `RealEstateAccess` prüft Berechtigungen -- ✅ **Sicherheit**: Kein SQL-Injection-Risiko -- ✅ **Geschäftslogik**: Automatisches Setzen von Systemfeldern (`_createdBy`, `_createdAt`) -- ✅ **Typsicherheit**: Fehler werden zur Entwicklungszeit erkannt -- ✅ **Wartbarkeit**: Zentrale CRUD-Methoden, einfach zu testen - -#### Szenario 2: Direkte SQL-Queries (OPTIONAL) - -**Nur für komplexe SELECT-Queries:** - -```python -# Für komplexe Queries, die nicht über CRUD-Methoden abgedeckt sind -chatInterface = getChatInterface(currentUser) -results = chatInterface.executeQuery( - "SELECT p.*, COUNT(parz.id) as parzellen_count FROM Projekt p LEFT JOIN Parzelle parz ON parz.projektId = p.id GROUP BY p.id" -) -``` - -**Warnung:** -- ⚠️ **Nur für SELECT-Queries**: Keine INSERT/UPDATE/DELETE über direkte SQL-Queries -- ⚠️ **Validierung erforderlich**: Queries sollten validiert werden -- ⚠️ **SQL-Injection-Risiko**: Immer Parameterisierung verwenden - -#### Empfehlung - -**Sie benötigen das Query-Interface, wenn Sie:** - -- ✅ **Komplexe SELECT-Queries** benötigen (z.B. JOINs, Aggregationen) -- ✅ **Flexible Query-Ausführung** benötigen (nicht über CRUD-Methoden abgedeckt) - -**Sie benötigen es NICHT, wenn Sie:** - -- ✅ **Nur CRUD-Operationen** benötigen (verwenden Sie das CRUD-Interface) -- ✅ **Einfache Queries** haben (können über CRUD-Methoden abgedeckt werden) - -**Für Production-Systeme:** -- **CRUD-Operationen**: Immer über CRUD-Interface -- **Komplexe Queries**: Optional über Query-Interface (nur SELECT) - -### Struktur: Query-Interface (stateless) - -**Eine Datei** für direkte SQL-Query-Ausführung: - -#### Datei: `modules/interfaces/interfaceDbRealEstateChatObjects.py` - -```python -""" -Interface for direct SQL query execution (stateless). -Uses PostgreSQL connector for direct query execution without session management. -""" - -import logging -from typing import Dict, Any, Optional -from modules.datamodels.datamodelUam import User -from modules.connectors.connectorDbPostgre import DatabaseConnector -from modules.shared.configuration import APP_CONFIG - -logger = logging.getLogger(__name__) - -# Singleton factory -_realEstateChatInterfaces = {} - - -class RealEstateChatObjects: - """Interface for direct SQL query execution (stateless).""" - - def __init__(self, currentUser: Optional[User] = None): - """Initialize the Query Interface.""" - self.currentUser = currentUser - self.userId = currentUser.id if currentUser else None - self.mandateId = currentUser.mandateId if currentUser else None - - # Initialize database - self._initializeDatabase() - - # Set user context if provided - if currentUser: - self.setUserContext(currentUser) - - def _initializeDatabase(self): - """Initialize PostgreSQL database connection.""" - try: - dbHost = APP_CONFIG.get("DB_APP_HOST", "localhost") - dbDatabase = APP_CONFIG.get("DB_APP_DATABASE", "poweron_app") - dbUser = APP_CONFIG.get("DB_APP_USER") - dbPassword = APP_CONFIG.get("DB_APP_PASSWORD_SECRET") - dbPort = int(APP_CONFIG.get("DB_APP_PORT", 5432)) - - self.db = DatabaseConnector( - dbHost=dbHost, - dbDatabase=dbDatabase, - dbUser=dbUser, - dbPassword=dbPassword, - dbPort=dbPort, - userId=self.userId if self.userId else None, - ) - - logger.info(f"Real Estate Query database connector initialized for database: {dbDatabase}") - except Exception as e: - logger.error(f"Error initializing Real Estate Query database: {e}") - raise - - def setUserContext(self, currentUser: User): - """Sets the user context for the interface.""" - self.currentUser = currentUser - self.userId = currentUser.id - self.mandateId = currentUser.mandateId - - if not self.userId or not self.mandateId: - raise ValueError("Invalid user context: id and mandateId are required") - - # Update database context - self.db.updateContext(self.userId) - - # ===== Database Query Execution ===== - - def executeQuery(self, queryText: str, parameters: Optional[Dict[str, Any]] = None) -> Dict[str, Any]: - """ - Execute a SQL query directly on the database (stateless). - - WARNING: This method executes raw SQL. Ensure proper validation and sanitization - before calling this method. Consider implementing query whitelisting or - only allowing SELECT statements for production use. - - Args: - queryText: SQL query string (preferably SELECT only) - parameters: Optional parameters for parameterized queries - - Returns: - Dictionary with 'rows' (list of dicts), 'columns' (list of column names), - 'rowCount' (int), and 'executionTime' (float) - """ - import time - - try: - start_time = time.time() - - # Ensure connection is alive - self.db._ensure_connection() - - with self.db.connection.cursor() as cursor: - # Execute query - if parameters: - # Use parameterized query for safety - cursor.execute(queryText, parameters) - else: - cursor.execute(queryText) - - # Fetch results - rows = cursor.fetchall() - - # Convert to list of dictionaries - result_rows = [dict(row) for row in rows] - - # Get column names - columns = [desc[0] for desc in cursor.description] if cursor.description else [] - - execution_time = time.time() - start_time - - return { - "rows": result_rows, - "columns": columns, - "rowCount": len(result_rows), - "executionTime": execution_time, - } - except Exception as e: - logger.error(f"Error executing query: {e}") - raise - - -def getInterface(currentUser: User) -> RealEstateChatObjects: - """ - Factory function to get or create a Query interface instance for a user. - Uses singleton pattern per user. - """ - userKey = f"{currentUser.id}_{currentUser.mandateId}" - - if userKey not in _realEstateChatInterfaces: - _realEstateChatInterfaces[userKey] = RealEstateChatObjects(currentUser) - - return _realEstateChatInterfaces[userKey] -``` - -### Hinweise zur Implementierung +### Hinweise zur Query-Ausführung 1. **Stateless**: Keine Session-Management-Funktionen 2. **Nur für Queries**: Primär für SELECT-Queries gedacht 3. **Sicherheit**: Immer Parameterisierung verwenden 4. **Validierung**: Queries sollten validiert werden (z.B. nur SELECT erlauben) -### Beispiel: Beide Interfaces zusammen nutzen - -```python -from modules.interfaces.interfaceDbRealEstateChatObjects import getInterface as getChatInterface -from modules.interfaces.interfaceDbRealEstateObjects import getInterface as getRealEstateInterface - -# CRUD-Interface für strukturierte Operationen -realEstateInterface = getRealEstateInterface(currentUser) -projekt = realEstateInterface.createProjekt(Projekt( - mandateId=currentUser.mandateId, - label="Neues Projekt" -)) - -# Query-Interface für komplexe SELECT-Queries (optional) -chatInterface = getChatInterface(currentUser) -results = chatInterface.executeQuery( - "SELECT p.*, COUNT(parz.id) as parzellen_count FROM Projekt p LEFT JOIN Parzelle parz ON parz.projektId = p.id GROUP BY p.id" -) -``` - --- ## Zusammenfassung: Benötigte Dateien @@ -830,12 +273,12 @@ results = chatInterface.executeQuery( - Nutzt `interfaceDbRealEstateAccess.py` - Haupt-Interface für alle CRUD-Operationen -### Optional (für direkte SQL-Queries): +### Hinweis zu Query-Ausführung: -4. ⚠️ `modules/interfaces/interfaceDbRealEstateChatObjects.py` - - Query-Interface für direkte SQL-Ausführung (RealEstateChatObjects) +4. ✅ `executeQuery()` ist bereits im Haupt-Interface verfügbar + - Kein separates Chat-Interface notwendig + - Direkt in `RealEstateObjects` verfügbar - Stateless, keine Session-Management - - Nur wenn Sie komplexe SELECT-Queries benötigen --- diff --git a/docs/real-estate-feature-integration-guide/04-feature-logic.md b/docs/real-estate-feature-integration-guide/04-feature-logic.md index cd0b1758..9f3daba9 100644 --- a/docs/real-estate-feature-integration-guide/04-feature-logic.md +++ b/docs/real-estate-feature-integration-guide/04-feature-logic.md @@ -29,7 +29,7 @@ Ergebnis zurückgeben (keine Session-Speicherung) **Beispiel:** - User: "Erstelle ein neues Projekt namens 'Hauptstrasse 42'" - AI analysiert → Intent: CREATE, Entity: Projekt, Parameter: {label: "Hauptstrasse 42"} -- Feature-Logik ruft auf → `interface.createProjekt(Projekt(label="Hauptstrasse 42"))` +- Feature-Logik ruft auf → Interface-Methode `createProjekt()` mit extrahierten Parametern - Ergebnis wird direkt zurückgegeben (keine Session, keine History) ## AI-Integration: Services initialisieren @@ -38,17 +38,9 @@ Um AI zu verwenden, müssen Sie die **Services** initialisieren. Services sind e ### Services-Initialisierung -```python -from modules.services import getInterface as getServices - -# Services für einen User erhalten -services = getServices(currentUser, workflow=None) - -# AI-Service verfügbar über: -aiService = services.ai # Für AI-Aufrufe -``` - -**Wichtig:** Services werden normalerweise im Feature-Logik-Modul initialisiert und an Funktionen weitergegeben. +**Wichtig:** +- Services werden normalerweise im Feature-Logik-Modul initialisiert und an Funktionen weitergegeben +- Für Query-Ausführung wird `getRealEstateInterface()` verwendet, nicht `getChatInterface()` --- @@ -65,449 +57,6 @@ Die AI analysiert User-Input und identifiziert: Basierend auf der AI-Analyse wird die entsprechende Interface-Methode aufgerufen. ---- - -## Beispiel-Implementierung: - -```python -""" -Real Estate feature main logic. -Handles chat interface for database queries with AI-powered natural language processing. -""" - -import logging -import json -from typing import Optional, Dict, Any, List -from modules.datamodels.datamodelUam import User -from modules.datamodels.datamodelRealEstate import ( - Projekt, - Parzelle, - StatusProzess, -) -from modules.interfaces.interfaceDbRealEstateChatObjects import getInterface as getChatInterface -from modules.interfaces.interfaceDbRealEstateObjects import getInterface as getRealEstateInterface -from modules.services import getInterface as getServices - -logger = logging.getLogger(__name__) - - -# ===== Direkte Query-Ausführung (stateless) ===== - -async def executeDirectQuery( - currentUser: User, - queryText: str, - parameters: Optional[Dict[str, Any]] = None, -) -> Dict[str, Any]: - """ - Execute a database query directly without session management. - - Args: - currentUser: Current authenticated user - queryText: SQL query text - parameters: Optional parameters for parameterized queries - - Returns: - Dictionary containing query result (rows, columns, rowCount) - - Note: - - No session or query history is saved - - Query is executed directly and result is returned - - For production, validate and sanitize queries before execution - """ - try: - chatInterface = getChatInterface(currentUser) - - # Execute query directly (no session tracking) - result = chatInterface.executeQuery(queryText, parameters) - - logger.info( - f"Query executed successfully: {result['rowCount']} rows in {result.get('executionTime', 0):.3f}s" - ) - - return { - "status": "success", - "rows": result["rows"], - "columns": result["columns"], - "rowCount": result["rowCount"], - "executionTime": result.get("executionTime", 0), - } - - except Exception as e: - logger.error(f"Error executing query: {str(e)}") - raise - - -# ===== AI-basierte Intent-Erkennung und CRUD-Operationen ===== - -async def processNaturalLanguageCommand( - currentUser: User, - userInput: str, -) -> Dict[str, Any]: - """ - Process natural language user input and execute corresponding CRUD operations. - - Uses AI to analyze user intent and extract parameters, then executes the appropriate - CRUD operation through the interface. Works stateless without session management. - - Args: - currentUser: Current authenticated user - userInput: Natural language command from user - - Returns: - Dictionary containing operation result and metadata - - Example user inputs: - - "Erstelle ein neues Projekt namens 'Hauptstrasse 42'" - - "Zeige mir alle Projekte in Zürich" - - "Aktualisiere Projekt XYZ mit Status 'Planung'" - - "Lösche Parzelle ABC" - - "SELECT * FROM Projekt WHERE plz = '8000'" - """ - try: - # Initialize services for AI access - services = getServices(currentUser, workflow=None) - aiService = services.ai - - # Step 1: Analyze user intent with AI - intentAnalysis = await analyzeUserIntent(aiService, userInput) - - logger.info(f"Intent analysis result: {intentAnalysis}") - - # Step 2: Execute CRUD operation based on intent - result = await executeIntentBasedOperation( - currentUser=currentUser, - intent=intentAnalysis["intent"], - entity=intentAnalysis["entity"], - parameters=intentAnalysis["parameters"], - ) - - return { - "success": True, - "intent": intentAnalysis["intent"], - "entity": intentAnalysis["entity"], - "result": result, - } - - except Exception as e: - logger.error(f"Error processing natural language command: {str(e)}") - raise - - -async def analyzeUserIntent( - aiService, - userInput: str -) -> Dict[str, Any]: - """ - Use AI to analyze user input and extract intent, entity, and parameters. - - Args: - aiService: AI service instance - userInput: Natural language user input - - Returns: - Dictionary with 'intent', 'entity', and 'parameters' - """ - # Create a structured prompt for intent analysis - intentPrompt = f""" -Analyze the following user command and extract the intent, entity, and parameters. - -User Command: "{userInput}" - -Available intents: -- CREATE: User wants to create a new entity -- READ: User wants to read/query entities -- UPDATE: User wants to update an existing entity -- DELETE: User wants to delete an entity -- QUERY: User wants to execute a database query - -Available entities: -- Projekt: Real estate project -- Parzelle: Plot/parcel -- Dokument: Document -- Kanton: Canton -- Gemeinde: Municipality - -Return a JSON object with the following structure: -{{ - "intent": "CREATE|READ|UPDATE|DELETE|QUERY", - "entity": "Projekt|Parzelle|Dokument|Kanton|Gemeinde|null", - "parameters": {{ - // Extracted parameters from user input - // For CREATE/UPDATE: include all relevant fields - // For READ: include filter criteria - // For DELETE: include entity ID if mentioned - // For QUERY: include query text or natural language query - }}, - "confidence": 0.0-1.0 // Confidence score for the analysis -}} - -Examples: -- Input: "Erstelle ein neues Projekt namens 'Hauptstrasse 42'" - Output: {{"intent": "CREATE", "entity": "Projekt", "parameters": {{"label": "Hauptstrasse 42"}}, "confidence": 0.95}} - -- Input: "Zeige mir alle Projekte" - Output: {{"intent": "READ", "entity": "Projekt", "parameters": {{}}, "confidence": 0.9}} - -- Input: "SELECT * FROM Projekt WHERE plz = '8000'" - Output: {{"intent": "QUERY", "entity": null, "parameters": {{"queryText": "SELECT * FROM Projekt WHERE plz = '8000'", "queryType": "sql"}}, "confidence": 1.0}} -""" - - try: - # Use AI planning call for structured JSON response - response = await aiService.callAiPlanning( - prompt=intentPrompt, - debugType="intentanalysis" - ) - - # Parse JSON response - intentData = json.loads(response) - - # Validate response structure - if "intent" not in intentData or "entity" not in intentData: - raise ValueError("Invalid intent analysis response structure") - - return intentData - - except json.JSONDecodeError as e: - logger.error(f"Failed to parse AI intent analysis response: {e}") - logger.error(f"Raw response: {response}") - raise ValueError(f"AI returned invalid JSON: {str(e)}") - except Exception as e: - logger.error(f"Error analyzing user intent: {str(e)}") - raise - - -async def executeIntentBasedOperation( - currentUser: User, - intent: str, - entity: Optional[str], - parameters: Dict[str, Any], -) -> Dict[str, Any]: - """ - Execute CRUD operation based on analyzed intent. - - Args: - currentUser: Current authenticated user - intent: Intent from AI analysis (CREATE, READ, UPDATE, DELETE, QUERY) - entity: Entity type from AI analysis - parameters: Extracted parameters from AI analysis - - Returns: - Operation result - """ - try: - if intent == "QUERY": - # Execute database query directly (stateless) - queryText = parameters.get("queryText", "") - - result = await executeDirectQuery( - currentUser=currentUser, - queryText=queryText, - parameters=parameters.get("queryParameters"), - ) - return result - - elif intent == "CREATE": - # Create new entity - realEstateInterface = getRealEstateInterface(currentUser) - - if entity == "Projekt": - projekt = Projekt( - mandateId=currentUser.mandateId, - label=parameters.get("label", ""), - statusProzess=StatusProzess(parameters.get("statusProzess", "EINGANG")) if parameters.get("statusProzess") else None, - ) - created = realEstateInterface.createProjekt(projekt) - return {"operation": "CREATE", "entity": "Projekt", "result": created.model_dump()} - - elif entity == "Parzelle": - parzelle = Parzelle( - mandateId=currentUser.mandateId, - label=parameters.get("label", ""), - # ... weitere Parameter - ) - created = realEstateInterface.createParzelle(parzelle) - return {"operation": "CREATE", "entity": "Parzelle", "result": created.model_dump()} - - else: - raise ValueError(f"CREATE operation not supported for entity: {entity}") - - elif intent == "READ": - # Read entities - realEstateInterface = getRealEstateInterface(currentUser) - - if entity == "Projekt": - # Apply filters from parameters - projektId = parameters.get("id") - if projektId: - projekt = realEstateInterface.getProjekt(projektId) - return {"operation": "READ", "entity": "Projekt", "result": projekt.model_dump() if projekt else None} - else: - # List all projects (with optional filters) - # Note: You may need to implement getProjekte() method - raise NotImplementedError("List operation needs to be implemented") - - else: - raise ValueError(f"READ operation not supported for entity: {entity}") - - elif intent == "UPDATE": - # Update existing entity - realEstateInterface = getRealEstateInterface(currentUser) - - if entity == "Projekt": - projektId = parameters.get("id") - if not projektId: - raise ValueError("UPDATE operation requires entity ID") - - # Get existing projekt - projekt = realEstateInterface.getProjekt(projektId) - if not projekt: - raise ValueError(f"Projekt {projektId} not found") - - # Update fields - updateData = {k: v for k, v in parameters.items() if k != "id"} - updated = realEstateInterface.updateProjekt(projektId, updateData) - return {"operation": "UPDATE", "entity": "Projekt", "result": updated.model_dump()} - - else: - raise ValueError(f"UPDATE operation not supported for entity: {entity}") - - elif intent == "DELETE": - # Delete entity - realEstateInterface = getRealEstateInterface(currentUser) - - if entity == "Projekt": - projektId = parameters.get("id") - if not projektId: - raise ValueError("DELETE operation requires entity ID") - - success = realEstateInterface.deleteProjekt(projektId) - return {"operation": "DELETE", "entity": "Projekt", "success": success} - - else: - raise ValueError(f"DELETE operation not supported for entity: {entity}") - - else: - raise ValueError(f"Unknown intent: {intent}") - - except Exception as e: - logger.error(f"Error executing intent-based operation: {str(e)}") - raise - - -# ===== Erweiterte Query-Funktion mit AI-Unterstützung ===== - -async def executeNaturalLanguageQuery( - currentUser: User, - naturalLanguageQuery: str, -) -> Dict[str, Any]: - """ - Execute a natural language query by translating it to SQL using AI. - - Args: - currentUser: Current authenticated user - naturalLanguageQuery: Natural language query (e.g., "Zeige mir alle Projekte in Zürich") - - Returns: - Query result with metadata (stateless, no session) - """ - try: - services = getServices(currentUser, workflow=None) - aiService = services.ai - - # Step 1: Translate natural language to SQL using AI - sqlQuery = await translateNaturalLanguageToSQL(aiService, naturalLanguageQuery, currentUser.mandateId) - - logger.info(f"Translated '{naturalLanguageQuery}' to SQL: {sqlQuery}") - - # Step 2: Execute the SQL query directly (stateless) - result = await executeDirectQuery( - currentUser=currentUser, - queryText=sqlQuery, - ) - - return result - - except Exception as e: - logger.error(f"Error executing natural language query: {str(e)}") - raise - - -async def translateNaturalLanguageToSQL( - aiService, - naturalLanguageQuery: str, - mandateId: str -) -> str: - """ - Use AI to translate natural language query to SQL. - - Args: - aiService: AI service instance - naturalLanguageQuery: Natural language query - mandateId: User's mandate ID for filtering - - Returns: - SQL query string with mandateId filter applied - """ - translationPrompt = f""" -Translate the following natural language query into a valid PostgreSQL SQL SELECT statement. - -Natural Language Query: "{naturalLanguageQuery}" - -Available tables and their fields: -- Projekt: id, mandateId, label, statusProzess, perimeter, baulinie, parzellen (JSONB), dokumente (JSONB) -- Parzelle: id, mandateId, label, strasseNr, plz, bauzone, az, bz, kontextKanton, kontextGemeinde -- Dokument: id, mandateId, label, dokumentTyp, dokumentReferenz, mimeType -- Kanton: id, mandateId, label, abk -- Gemeinde: id, mandateId, label, plz - -Rules: -1. Always include 'mandateId' filter based on user context (use placeholder {{mandateId}}) -2. Only use SELECT statements (no INSERT, UPDATE, DELETE) -3. Return ONLY the SQL query, no explanations -4. Use proper PostgreSQL syntax -5. For text searches, use ILIKE for case-insensitive matching - -Examples: -- Input: "Zeige mir alle Projekte" - Output: SELECT * FROM Projekt WHERE mandateId = '{{mandateId}}' - -- Input: "Zeige mir alle Parzellen in Zürich" - Output: SELECT p.* FROM Parzelle p JOIN Gemeinde g ON p.kontextGemeinde = g.id WHERE g.label ILIKE '%Zürich%' AND p.mandateId = '{{mandateId}}' - -- Input: "Wie viele Projekte haben Status 'Planung'?" - Output: SELECT COUNT(*) as count FROM Projekt WHERE statusProzess = 'Planung' AND mandateId = '{{mandateId}}' - -Now translate this query: -""" - - try: - # Use AI planning call for SQL generation - response = await aiService.callAiPlanning( - prompt=translationPrompt, - debugType="sqltranslation" - ) - - # Clean response (remove markdown code blocks if present) - sqlQuery = response.strip() - if sqlQuery.startswith("```sql"): - sqlQuery = sqlQuery[6:] - if sqlQuery.startswith("```"): - sqlQuery = sqlQuery[3:] - if sqlQuery.endswith("```"): - sqlQuery = sqlQuery[:-3] - sqlQuery = sqlQuery.strip() - - # Replace placeholder with actual mandateId - sqlQuery = sqlQuery.replace("{{mandateId}}", mandateId) - - return sqlQuery - - except Exception as e: - logger.error(f"Error translating natural language to SQL: {str(e)}") - raise ValueError(f"Failed to translate query: {str(e)}") -``` - ## Wichtige Punkte: ### 1. Services-Initialisierung @@ -530,60 +79,37 @@ Die AI analysiert User-Input und gibt zurück: ### 4. CRUD-Operationen -Basierend auf der Intent-Analyse: -- **CREATE** → `interface.createProjekt()`, `interface.createParzelle()`, etc. -- **READ** → `interface.getProjekt()`, `interface.getParzelle()`, etc. -- **UPDATE** → `interface.updateProjekt()`, etc. -- **DELETE** → `interface.deleteProjekt()`, etc. -- **QUERY** → `interface.executeQuery()` oder `executeDatabaseQuery()` +Basierend auf der Intent-Analyse werden folgende Operationen unterstützt: -### 5. Natural Language to SQL +**Unterstützte Entities:** +- Projekt, Parzelle, Gemeinde, Kanton, Land, Dokument -- AI übersetzt natürliche Sprache in SQL-Queries -- Automatische Validierung und Sanitization empfohlen -- MandateId-Filter wird automatisch hinzugefügt +**CREATE** → `interface.createProjekt()`, `interface.createParzelle()`, `interface.createGemeinde()`, `interface.createKanton()`, `interface.createLand()`, `interface.createDokument()` + +**READ** → +- Einzelne Entität: `interface.getProjekt(id)`, `interface.getParzelle(id)`, etc. +- Liste mit Filtern: `interface.getProjekte(recordFilter)`, `interface.getParzellen(recordFilter)`, etc. +- **Wichtig:** READ-Operationen validieren Filter-Felder gegen das Datenmodell + +**UPDATE** → `interface.updateProjekt(id, updateData)`, `interface.updateParzelle(id, updateData)`, etc. + +**DELETE** → `interface.deleteProjekt(id)`, `interface.deleteParzelle(id)`, etc. + +**QUERY** → `interface.executeQuery(queryText, parameters)` für direkte SQL-Ausführung + +### 5. Field Validation + +- **READ-Operationen** validieren Filter-Felder gegen das Datenmodell +- Ungültige Felder werden ignoriert und geloggt +- **Wichtig:** Location-Queries sollten Parzelle-Entity verwenden, nicht Projekt direkt ### 6. Error Handling - Umfassendes Error Handling für AI-Aufrufe -- JSON-Parsing mit Fallback -- Logging für Debugging - ---- - -## Beispiel-Verwendung: - -```python -# In einer Route (stateless): -@router.post("/command") -async def process_command( - userInput: str = Body(...), - currentUser: User = Depends(getCurrentUser) -): - result = await processNaturalLanguageCommand( - currentUser=currentUser, - userInput=userInput - ) - return result - -# Direkte Query (stateless): -@router.post("/query") -async def execute_query( - queryText: str = Body(...), - currentUser: User = Depends(getCurrentUser) -): - result = await executeDirectQuery( - currentUser=currentUser, - queryText=queryText - ) - return result -``` - -**User-Input-Beispiele:** -- `"Erstelle ein neues Projekt namens 'Hauptstrasse 42'"` -- `"Zeige mir alle Projekte in Zürich"` -- `"Aktualisiere Projekt XYZ mit Status 'Planung'"` -- `"Wie viele Parzellen haben Bauzone W3?"` +- JSON-Parsing mit Fallback (extrahiert JSON aus Markdown-Code-Blöcken) +- Validierung der AI-Response-Struktur +- Logging für Debugging mit `exc_info=True` für vollständige Stack-Traces +- Entity-Validierung: Prüft ob Entity existiert vor Update/Delete --- @@ -597,32 +123,26 @@ async def execute_query( Body: {"userInput": "Erstelle ein neues Projekt namens 'Hauptstrasse 42'"} 2. Route ruft Feature-Logik auf - → processNaturalLanguageCommand(currentUser, userInput) - # Keine Session-ID notwendig! + → processNaturalLanguageCommand() wird aufgerufen + → Keine Session-ID notwendig! 3. Feature-Logik initialisiert Services - → services = getServices(currentUser, workflow=None) - → aiService = services.ai + → Services werden für den aktuellen User initialisiert + → AI-Service wird aus Services abgerufen 4. AI analysiert User-Input - → analyzeUserIntent(aiService, userInput) - → AI gibt zurück: - { - "intent": "CREATE", - "entity": "Projekt", - "parameters": {"label": "Hauptstrasse 42"}, - "confidence": 0.95 - } + → analyzeUserIntent() wird aufgerufen + → AI gibt zurück: Intent "CREATE", Entity "Projekt", Parameter {"label": "Hauptstrasse 42"} 5. Feature-Logik führt CRUD-Operation aus - → executeIntentBasedOperation(intent="CREATE", entity="Projekt", ...) - → realEstateInterface = getRealEstateInterface(currentUser) - → projekt = Projekt(mandateId=..., label="Hauptstrasse 42") - → created = realEstateInterface.createProjekt(projekt) + → executeIntentBasedOperation() wird mit Intent und Parametern aufgerufen + → Real Estate Interface wird initialisiert + → Projekt-Objekt wird aus Parametern erstellt + → createProjekt() wird aufgerufen 6. Interface speichert in Datenbank - → DatabaseConnector.recordCreate(Projekt, projekt.model_dump()) - → PostgreSQL INSERT INTO Projekt ... + → Datenbank-Operation wird über Interface ausgeführt + → PostgreSQL INSERT wird durchgeführt 7. Ergebnis wird direkt zurückgegeben → Route gibt HTTP Response zurück @@ -638,15 +158,6 @@ async def execute_query( **Verwendung:** Intent-Analyse, SQL-Übersetzung, strukturierte Daten-Extraktion -```python -response = await aiService.callAiPlanning( - prompt=intentPrompt, - debugType="intentanalysis" # Optional: für Debug-Dateien -) -# Response ist JSON-String, muss geparst werden -intentData = json.loads(response) -``` - **Vorteile:** - Optimiert für strukturierte JSON-Antworten - Verwendet beste Modelle für Planungs-Aufgaben @@ -656,27 +167,13 @@ intentData = json.loads(response) **Verwendung:** Text-Generierung, Zusammenfassungen, Erklärungen -```python -response = await aiService.callAiText( - prompt="Erkläre mir...", - documents=None, # Optional: Dokumente für Kontext - options=AiCallOptions(...) -) -# Response ist direkt Text-String -``` +**Hinweis:** Response ist direkt ein Text-String (kein JSON-Parsing nötig) ### `callAiDocuments()` - Für Dokumenten-Verarbeitung **Verwendung:** Dokumenten-Analyse, Extraktion, Generierung mit Dokumenten-Kontext -```python -response = await aiService.callAiDocuments( - prompt="Analysiere diese Dokumente...", - documents=[ChatDocument(...), ...], - options=AiCallOptions(...), - outputFormat="json" # Optional: Format für Output -) -``` +**Hinweis:** Unterstützt optionales `outputFormat` Parameter für strukturierte Ausgaben (z.B. "json") --- @@ -718,59 +215,18 @@ response = await aiService.callAiDocuments( ### Schema-Aware Prompting -Sie können das Datenbank-Schema in Prompts einbinden: - -```python -# Lade Schema-Informationen -schemaInfo = getDatabaseSchema() # Ihre Funktion - -prompt = f""" -Available database schema: -{schemaInfo} - -User query: "{userInput}" -... -""" -``` +Sie können das Datenbank-Schema in Prompts einbinden, um der AI besseren Kontext zu geben. Laden Sie Schema-Informationen dynamisch und fügen Sie diese in den Prompt ein. ### Context-Aware Operations (Optional) -Falls Sie später Kontext zwischen Queries benötigen, können Sie optional eine Session verwenden: - -```python -# Optional: Session für Kontext (nur wenn nötig) -# Für stateless Operationen nicht notwendig - -# Falls Session gewünscht: -sessionId = parameters.get("sessionId") # Optional -if sessionId: - previousQueries = interface.getQueries(sessionId=sessionId) - context = "\n".join([q.queryText for q in previousQueries[-5:]]) -else: - context = "" # Kein Kontext bei stateless Operationen - -prompt = f""" -{context if context else ""} -User query: "{userInput}" -... -""" -``` +Falls Sie später Kontext zwischen Queries benötigen, können Sie optional eine Session verwenden. Für stateless Operationen ist dies normalerweise nicht notwendig. Falls gewünscht, können Sie vorherige Queries aus einer Session laden und als Kontext in den Prompt einbinden. ### Multi-Step Operations Für komplexe Operationen können Sie mehrere AI-Calls machen: - -```python -# Schritt 1: Intent-Analyse -intent = await analyzeUserIntent(aiService, userInput) - -# Schritt 2: Parameter-Validierung -if intent["intent"] == "CREATE": - validatedParams = await validateParameters(aiService, intent["parameters"]) - -# Schritt 3: CRUD-Operation -result = await executeIntentBasedOperation(...) -``` +1. **Intent-Analyse**: Zuerst den User-Intent analysieren +2. **Parameter-Validierung**: Optional Parameter mit AI validieren +3. **CRUD-Operation**: Die eigentliche Operation ausführen --- diff --git a/docs/real-estate-feature-integration-guide/05-routes.md b/docs/real-estate-feature-integration-guide/05-routes.md index 04908fd7..6163248a 100644 --- a/docs/real-estate-feature-integration-guide/05-routes.md +++ b/docs/real-estate-feature-integration-guide/05-routes.md @@ -10,136 +10,11 @@ Die Routen definieren die REST-API-Endpunkte für das Feature. Das Feature arbei ``` /api/realestate/ - ├── POST /command → Natürliche Sprache → CRUD-Operation - └── POST /query → Direkte SQL-Query -``` - -## Beispiel-Implementierung: - -```python -""" -Real Estate routes for the backend API. -Implements stateless endpoints for real estate database operations with AI-powered natural language processing. -""" - -import logging -from typing import Optional, Dict, Any -from fastapi import APIRouter, HTTPException, Depends, Body, Request -from modules.security.auth import limiter, getCurrentUser -from modules.datamodels.datamodelUam import User -from modules.features.realEstate.mainRealEstate import ( - processNaturalLanguageCommand, - executeDirectQuery, -) - -# Configure logger -logger = logging.getLogger(__name__) - -# Create router for real estate endpoints -router = APIRouter( - prefix="/api/realestate", - tags=["Real Estate"], - responses={404: {"description": "Not found"}} -) - - -# ===== Stateless Command Endpoint ===== - -@router.post("/command", response_model=Dict[str, Any]) -@limiter.limit("120/minute") -async def process_command( - request: Request, - userInput: str = Body(..., embed=True, description="Natural language command"), - currentUser: User = Depends(getCurrentUser) -) -> Dict[str, Any]: - """ - Process natural language command and execute corresponding CRUD operation. - - Uses AI to analyze user intent and extract parameters, then executes the appropriate - CRUD operation. Works stateless without session management. - - Example user inputs: - - "Erstelle ein neues Projekt namens 'Hauptstrasse 42'" - - "Zeige mir alle Projekte in Zürich" - - "Aktualisiere Projekt XYZ mit Status 'Planung'" - - "Lösche Parzelle ABC" - - "SELECT * FROM Projekt WHERE plz = '8000'" - - Returns: - { - "success": true, - "intent": "CREATE|READ|UPDATE|DELETE|QUERY", - "entity": "Projekt|Parzelle|...|null", - "result": {...} - } - """ - try: - result = await processNaturalLanguageCommand( - currentUser=currentUser, - userInput=userInput - ) - return result - except ValueError as e: - logger.error(f"Validation error: {str(e)}") - raise HTTPException( - status_code=400, - detail=str(e) - ) - except Exception as e: - logger.error(f"Error processing command: {str(e)}") - raise HTTPException( - status_code=500, - detail=str(e) - ) - - -# ===== Stateless Query Endpoint ===== - -@router.post("/query", response_model=Dict[str, Any]) -@limiter.limit("120/minute") -async def execute_query( - request: Request, - queryText: str = Body(..., embed=True, description="SQL query text"), - parameters: Optional[Dict[str, Any]] = Body(None, embed=True, description="Optional query parameters for parameterized queries"), - currentUser: User = Depends(getCurrentUser) -) -> Dict[str, Any]: - """ - Execute a direct SQL query without session management. - - Executes the query directly and returns the result. No query history is saved. - - WARNING: This endpoint executes raw SQL queries. Ensure proper validation - and sanitization on the frontend. Consider implementing query whitelisting - or only allowing SELECT statements for production use. - - Returns: - { - "status": "success", - "rows": [...], - "columns": [...], - "rowCount": 15, - "executionTime": 0.123 - } - """ - try: - result = await executeDirectQuery( - currentUser=currentUser, - queryText=queryText, - parameters=parameters, - ) - return result - except ValueError as e: - logger.error(f"Validation error: {str(e)}") - raise HTTPException( - status_code=400, - detail=str(e) - ) - except Exception as e: - logger.error(f"Error executing query: {str(e)}") - raise HTTPException( - status_code=500, - detail=str(e) - ) + ├── POST /command → Natürliche Sprache → CRUD-Operation + ├── POST /query → Direkte SQL-Query + ├── GET /tables → Liste aller verfügbaren Tabellen + ├── GET /table/{table} → Daten aus einer Tabelle (mit optionaler Pagination) + └── POST /table/{table} → Neuen Datensatz in einer Tabelle erstellen ``` ## Wichtige Punkte: @@ -157,118 +32,74 @@ async def execute_query( - Nutzt AI für Intent-Analyse - Führt CRUD-Operationen aus - Gibt Ergebnis direkt zurück +- **CSRF-Token erforderlich** (X-CSRF-Token Header) **`POST /api/realestate/query`** - Führt direkte SQL-Queries aus +- Request Body: `{"queryText": "...", "parameters": {...}}` - Keine Session notwendig - Gibt Query-Ergebnis direkt zurück +- **CSRF-Token erforderlich** (X-CSRF-Token Header) + +**`GET /api/realestate/tables`** +- Gibt Liste aller verfügbaren Tabellen zurück +- Enthält Tabellennamen, Beschreibungen und Model-Namen +- **CSRF-Token erforderlich** (X-CSRF-Token Header) + +**`GET /api/realestate/table/{table}`** +- Gibt alle Daten aus einer spezifischen Tabelle zurück +- Unterstützt optionale Pagination über Query-Parameter +- Sortierung und Filterung möglich +- Leere Tabellen geben ein leeres Modell-Instanz zurück (für Schema-Extraktion) +- **CSRF-Token erforderlich** (X-CSRF-Token Header) + +**`POST /api/realestate/table/{table}`** +- Erstellt einen neuen Datensatz in einer spezifischen Tabelle +- Request Body enthält die Datensatz-Daten +- mandateId wird automatisch gesetzt falls nicht vorhanden +- **CSRF-Token erforderlich** (X-CSRF-Token Header) ### 3. Sicherheit -- **Rate Limiting**: `@limiter.limit("120/minute")` für API-Schutz -- **Authentication**: `Depends(getCurrentUser)` für alle Endpunkte +- **Rate Limiting**: 120 Requests pro Minute für alle Endpunkte +- **Authentication**: Alle Endpunkte erfordern authentifizierte User +- **CSRF-Token-Validierung**: Alle Endpunkte validieren CSRF-Token im X-CSRF-Token Header + - Token muss hexadezimaler String sein + - Token-Länge: 16-64 Zeichen + - Fehlende oder ungültige Token führen zu 403 Forbidden - **Query-Validierung**: WICHTIG - Validieren Sie SQL-Queries vor Ausführung - **MandateId-Filter**: Wird automatisch durch Interfaces angewendet ### 4. Error Handling - Umfassendes Error Handling mit HTTPException -- Unterschiedliche Status-Codes: 400 (Validation), 404 (Not Found), 500 (Server Error) +- Unterschiedliche Status-Codes: + - **400 Bad Request**: Validierungsfehler (z.B. fehlende Parameter, ungültige Daten) + - **403 Forbidden**: CSRF-Token fehlt oder ist ungültig + - **404 Not Found**: Ressource nicht gefunden + - **500 Internal Server Error**: Server-Fehler - Detaillierte Fehlermeldungen für Debugging +- Logging mit vollständigen Stack-Traces (`exc_info=True`) ### 5. Response-Struktur **Command-Endpunkt:** -```json -{ - "success": true, - "intent": "CREATE", - "entity": "Projekt", - "result": { - "operation": "CREATE", - "entity": "Projekt", - "result": { - "id": "projekt_123", - "label": "Hauptstrasse 42", - ... - } - } -} -``` +- Erfolgreiche Response enthält: `success`, `intent`, `entity`, `result` +- Result enthält die Operation-Details und das erstellte/aktualisierte/gelesene Objekt **Query-Endpunkt:** -```json -{ - "status": "success", - "rows": [ - {"id": "...", "label": "...", ...} - ], - "columns": ["id", "label", ...], - "rowCount": 15, - "executionTime": 0.123 -} -``` +- Response enthält: `status`, `rows`, `columns`, `rowCount`, `executionTime` ---- +**Tables-Endpunkt:** +- Response enthält: `tables` (Array mit Tabellen-Informationen), `count` -## Beispiel-Requests +**Table GET-Endpunkt:** +- Response ist eine `PaginatedResponse` mit `items` und `pagination` Metadata +- Ohne Pagination: Alle Items werden zurückgegeben +- Mit Pagination: Enthält `currentPage`, `pageSize`, `totalItems`, `totalPages`, `sort`, `filters` -### Command-Endpunkt - -```bash -# CREATE Operation -POST /api/realestate/command -Content-Type: application/json -Authorization: Bearer - -{ - "userInput": "Erstelle ein neues Projekt namens 'Hauptstrasse 42'" -} - -# READ Operation -POST /api/realestate/command -{ - "userInput": "Zeige mir alle Projekte in Zürich" -} - -# UPDATE Operation -POST /api/realestate/command -{ - "userInput": "Aktualisiere Projekt XYZ mit Status 'Planung'" -} - -# DELETE Operation -POST /api/realestate/command -{ - "userInput": "Lösche Parzelle ABC" -} - -# QUERY Operation (SQL wird erkannt) -POST /api/realestate/command -{ - "userInput": "SELECT * FROM Projekt WHERE plz = '8000'" -} -``` - -### Query-Endpunkt - -```bash -# Direkte SQL-Query -POST /api/realestate/query -Content-Type: application/json -Authorization: Bearer - -{ - "queryText": "SELECT * FROM Projekt WHERE plz = '8000'" -} - -# Parameterized Query -POST /api/realestate/query -{ - "queryText": "SELECT * FROM Projekt WHERE plz = $1", - "parameters": {"$1": "8000"} -} -``` +**Table POST-Endpunkt:** +- Response ist das erstellte Objekt als Dictionary (model_dump()) --- @@ -279,6 +110,8 @@ POST /api/realestate/query ``` POST /api/realestate/command ↓ +CSRF-Token-Validierung + ↓ routeRealEstate.process_command() ↓ getCurrentUser() # Auth @@ -297,25 +130,85 @@ return Dict mit Ergebnis ``` POST /api/realestate/query ↓ +CSRF-Token-Validierung + ↓ routeRealEstate.execute_query() ↓ getCurrentUser() # Auth ↓ +Body-Parsing (queryText, parameters) + ↓ executeDirectQuery(currentUser, queryText, parameters) ↓ mainRealEstate.executeDirectQuery() ↓ -getChatInterface(currentUser) +getRealEstateInterface(currentUser) ↓ -RealEstateChatObjects.executeQuery(queryText) - ↓ -DatabaseConnector.executeQuery(sql) +Interface.executeQuery(queryText, parameters) ↓ return Dict mit rows, columns, rowCount ``` +### Table-Endpunkte Flow + +**GET /api/realestate/table/{table}:** +``` +GET /api/realestate/table/{table} + ↓ +CSRF-Token-Validierung + ↓ +getCurrentUser() # Auth + ↓ +Tabellen-Name validieren + ↓ +getRealEstateInterface(currentUser) + ↓ +Interface.getProjekte() / getParzellen() / etc. + ↓ +Pagination anwenden (falls angegeben) + ↓ +return PaginatedResponse +``` + +**POST /api/realestate/table/{table}:** +``` +POST /api/realestate/table/{table} + ↓ +CSRF-Token-Validierung + ↓ +getCurrentUser() # Auth + ↓ +Tabellen-Name validieren + ↓ +Model-Instanz aus Body-Daten erstellen + ↓ +getRealEstateInterface(currentUser) + ↓ +Interface.createProjekt() / createParzelle() / etc. + ↓ +return erstelltes Objekt +``` + --- +## Verfügbare Tabellen + +Die folgenden Tabellen sind verfügbar: +- **Projekt**: Real estate projects +- **Parzelle**: Plots/parcels +- **Dokument**: Documents +- **Gemeinde**: Municipalities +- **Kanton**: Cantons +- **Land**: Countries + +## Pagination + +Der `GET /api/realestate/table/{table}` Endpunkt unterstützt Pagination über einen Query-Parameter: + +- **Parameter**: `pagination` (JSON-encoded string) +- **Format**: `{"page": 1, "pageSize": 10, "sort": [{"field": "label", "direction": "asc"}], "filters": []}` +- **Ohne Pagination**: Alle Datensätze werden zurückgegeben + ## Vorteile des stateless Ansatzes - **Einfachheit**: Kein Session-Management notwendig diff --git a/docs/real-estate-feature-integration-guide/06-router-registration.md b/docs/real-estate-feature-integration-guide/06-router-registration.md deleted file mode 100644 index 430c88b2..00000000 --- a/docs/real-estate-feature-integration-guide/06-router-registration.md +++ /dev/null @@ -1,45 +0,0 @@ -# Schritt 5: Router registrieren - -[← Zurück: Routen erstellen](05-routes.md) | [Weiter: Environment-Konfiguration →](07-environment.md) - -**Datei:** `app.py` - -Der Router muss in der Hauptanwendung registriert werden. - -## Änderung in app.py: - -```python -# ... existing imports ... - -# Include all routers - -from modules.routes.routeAdmin import router as generalRouter -app.include_router(generalRouter) - -# ... existing routers ... - -from modules.routes.routeChatPlayground import router as chatPlaygroundRouter -app.include_router(chatPlaygroundRouter) - -# NEU: Real Estate Router hinzufügen (Chat-Interface) -from modules.routes.routeRealEstate import router as realEstateRouter -app.include_router(realEstateRouter) - -# NEU: Real Estate Data Router hinzufügen (falls CRUD-API gewünscht) -# from modules.routes.routeRealEstateData import router as realEstateDataRouter -# app.include_router(realEstateDataRouter) - -from modules.routes.routeSecurityLocal import router as localRouter -app.include_router(localRouter) - -# ... rest of routers ... -``` - -**Wichtig**: Die Reihenfolge der Router-Registrierung kann wichtig sein, wenn es Überschneidungen in den Pfaden gibt. Allgemeinere Routen sollten nach spezifischeren Routen kommen. - ---- - -[← Zurück: Routen erstellen](05-routes.md) | [Weiter: Environment-Konfiguration →](07-environment.md) - - - diff --git a/docs/real-estate-feature-integration-guide/07-environment.md b/docs/real-estate-feature-integration-guide/07-environment.md index 73765e61..e4b6646c 100644 --- a/docs/real-estate-feature-integration-guide/07-environment.md +++ b/docs/real-estate-feature-integration-guide/07-environment.md @@ -48,3 +48,5 @@ def _initializeDatabase(self): + + diff --git a/docs/real-estate-feature-integration-guide/08-lifecycle.md b/docs/real-estate-feature-integration-guide/08-lifecycle.md deleted file mode 100644 index 0946e331..00000000 --- a/docs/real-estate-feature-integration-guide/08-lifecycle.md +++ /dev/null @@ -1,43 +0,0 @@ -# Schritt 7: Feature Lifecycle (optional) - -[← Zurück: Environment-Konfiguration](07-environment.md) | [Weiter: Datenbank-Schema →](09-database-schema.md) - -**Datei:** `modules/features/featuresLifecycle.py` - -Falls Ihr Feature Hintergrundprozesse oder Initialisierung beim Start benötigt, können Sie diese hier hinzufügen: - -```python -async def start() -> None: - """ Start feature triggers and background managers """ - - # Provide Event User - rootInterface = getRootInterface() - eventUser = rootInterface.getUserByUsername("event") - - # ... existing features ... - - # Feature RealEstate (optional) - # from modules.features.realEstate import mainRealEstate - # mainRealEstate.initializeFeature(eventUser) - # logger.info("Real Estate feature initialized") - - return True - - -async def stop() -> None: - """ Stop feature triggers and background managers """ - - # Feature RealEstate cleanup (optional) - # from modules.features.realEstate import mainRealEstate - # mainRealEstate.cleanupFeature() - # logger.info("Real Estate feature cleaned up") - - return True -``` - ---- - -[← Zurück: Environment-Konfiguration](07-environment.md) | [Weiter: Datenbank-Schema →](09-database-schema.md) - - - diff --git a/docs/real-estate-feature-integration-guide/09-database-schema.md b/docs/real-estate-feature-integration-guide/09-database-schema.md deleted file mode 100644 index bd094003..00000000 --- a/docs/real-estate-feature-integration-guide/09-database-schema.md +++ /dev/null @@ -1,247 +0,0 @@ -# Datenbank-Schema - -[← Zurück: Feature Lifecycle](08-lifecycle.md) | [Weiter: Sicherheitshinweise →](10-security.md) - -Die Datenbank-Tabellen werden automatisch vom `DatabaseConnector` erstellt, basierend auf den Pydantic-Modellen: - -## Chat-Interface Tabellen: - -- **RealEstateQuery**: Speichert Abfragen -- **RealEstateQueryResult**: Speichert Abfrageergebnisse (mit JSONB für `rowData`) -- **RealEstateChatSession**: Speichert Chat-Sessions - -## Real Estate-Datenmodell Tabellen: - -Die folgenden Tabellen werden basierend auf den Real Estate-Datenmodell-Entitäten erstellt: - -- **Projekt**: Bauprojekte (mit `parzellen`, `dokumente`, `kontextInformationen` als JSONB) -- **Parzelle**: Grundstücke mit Bauparametern (mit `parzellenNachbarschaft`, `dokumente`, `kontextInformationen` als JSONB) -- **Dokument**: Dateien und URLs -- **Kontext**: Zusatzinformationen -- **GeoPolylinie**: Geometrische Linien/Polygone (mit `punkte` als JSONB) -- **GeoPunkt**: 3D-Koordinaten -- **Land**: Nationale Ebene (mit `dokumente`, `kontextInformationen` als JSONB) -- **Kanton**: Kantonale Ebene (mit `dokumente`, `kontextInformationen` als JSONB) -- **Gemeinde**: Gemeinde-Ebene (mit `dokumente`, `kontextInformationen` als JSONB) - ---- - -## Automatische Tabellenerstellung - -### Wie funktioniert die automatische Tabellenerstellung? - -Der `DatabaseConnector` erstellt Tabellen **automatisch beim ersten Zugriff** auf ein Pydantic-Modell. Sie müssen keine SQL-CREATE-TABLE-Statements manuell schreiben. - -#### 1. Ablauf der Tabellenerstellung: - -``` -1. Code ruft z.B. `db.recordCreate(Projekt, projekt_data)` auf - ↓ -2. DatabaseConnector ruft `_ensureTableExists(Projekt)` auf - ↓ -3. Prüft ob Tabelle "Projekt" existiert (über information_schema) - ↓ -4. Wenn NICHT vorhanden: - → Ruft `_create_table_from_model()` auf - → Extrahiert Felder aus Pydantic-Modell mit `_get_model_fields()` - → Mappt Python-Typen zu SQL-Typen - → Erstellt CREATE TABLE Statement - → Führt SQL aus - → Erstellt Indexes für Foreign Keys -``` - -#### 2. Typ-Mapping (Python → PostgreSQL): - -Der `DatabaseConnector` mappt automatisch Pydantic-Feldtypen zu PostgreSQL-Datentypen: - -| Python/Pydantic Typ | PostgreSQL Typ | Beispiel | -|---------------------|----------------|----------| -| `str` oder `Optional[str]` | `TEXT` | `label: str` → `"label" TEXT` | -| `int` | `INTEGER` | `vollgeschossZahl: int` → `"vollgeschossZahl" INTEGER` | -| `float` | `DOUBLE PRECISION` | `az: float` → `"az" DOUBLE PRECISION` | -| `bool` | `BOOLEAN` | `closed: bool` → `"closed" BOOLEAN` | -| `Dict[str, Any]` oder `dict` | `JSONB` | `parameters: Dict[str, Any]` → `"parameters" JSONB` | -| `List[...]` oder `list` | `JSONB` | `parzellen: List[Parzelle]` → `"parzellen" JSONB` | -| `Optional[Enum]` | `TEXT` | `statusProzess: StatusProzess` → `"statusProzess" TEXT` | - -**Spezielle Felder:** -- Felder mit Namen `*Id` (z.B. `kontextKantonId`) erhalten automatisch einen Index -- Systemfelder werden automatisch hinzugefügt: `_createdAt`, `_createdBy`, `_modifiedAt`, `_modifiedBy` - -#### 3. Beispiel: CREATE TABLE Statement - -Für das `Projekt`-Modell würde automatisch folgendes SQL erstellt: - -```sql -CREATE TABLE IF NOT EXISTS "Projekt" ( - "id" VARCHAR(255) PRIMARY KEY, - "mandateId" TEXT, - "label" TEXT, - "statusProzess" TEXT, - "perimeter" JSONB, - "baulinie" JSONB, - "parzellen" JSONB, - "dokumente" JSONB, - "kontextInformationen" JSONB, - "_createdAt" DOUBLE PRECISION, - "_modifiedAt" DOUBLE PRECISION, - "_createdBy" VARCHAR(255), - "_modifiedBy" VARCHAR(255) -); - --- Automatisch erstellte Indexes für Foreign Keys: -CREATE INDEX IF NOT EXISTS "idx_Projekt_mandateId" ON "Projekt" ("mandateId"); -``` - -#### 4. Automatische Schema-Migrationen - -**Wichtig:** Der Connector unterstützt **additive Migrationen**: - -- Wenn eine Tabelle bereits existiert, werden **fehlende Spalten automatisch hinzugefügt** -- **Bestehende Spalten werden NICHT gelöscht oder geändert** -- Wenn Sie ein neues Feld zum Pydantic-Modell hinzufügen, wird es beim nächsten Zugriff automatisch als Spalte hinzugefügt - -**Beispiel:** -```python -# Ursprüngliches Modell -class Projekt(BaseModel): - id: str - label: str - statusProzess: Optional[StatusProzess] - -# Später: Neues Feld hinzugefügt -class Projekt(BaseModel): - id: str - label: str - statusProzess: Optional[StatusProzess] - beschreibung: Optional[str] # NEU - -# Beim nächsten recordCreate() wird automatisch ausgeführt: -# ALTER TABLE "Projekt" ADD COLUMN "beschreibung" TEXT -``` - -#### 5. Wann werden Tabellen erstellt? - -Tabellen werden erstellt, wenn Sie **zum ersten Mal** eine der folgenden Operationen ausführen: - -- `db.recordCreate(model_class, data)` - Erstellt Record -- `db.recordUpdate(model_class, recordId, data)` - Aktualisiert Record -- `db.getRecordset(model_class)` - Lädt Records -- `db.getRecord(model_class, recordId)` - Lädt einen Record - -**Beispiel:** -```python -# Beim ersten Aufruf wird die Tabelle "Projekt" automatisch erstellt -interface = getInterface(currentUser) -projekt = interface.createProjekt(label="Mein Projekt") -# → Tabelle "Projekt" wird jetzt in PostgreSQL erstellt -``` - -#### 6. Manuelle Tabellenerstellung (optional) - -Falls Sie Tabellen manuell erstellen möchten (z.B. für Initialisierung), können Sie: - -```python -from modules.connectors.connectorDbPostgre import DatabaseConnector -from modules.datamodels.datamodelRealEstate import Projekt, Parzelle - -# Connector initialisieren -db = DatabaseConnector( - dbHost="localhost", - dbDatabase="poweron_app", - dbUser="poweron_dev", - dbPassword="...", - dbPort=5432 -) - -# Tabellen explizit erstellen -db._ensureTableExists(Projekt) -db._ensureTableExists(Parzelle) -# ... weitere Modelle -``` - -#### 7. Wichtige Hinweise: - -✅ **Automatisch:** -- Tabellenerstellung beim ersten Zugriff -- Spalten-Erstellung basierend auf Pydantic-Feldern -- Index-Erstellung für Foreign Keys (`*Id` Felder) -- Systemfelder (`_createdAt`, etc.) werden automatisch hinzugefügt - -❌ **NICHT automatisch:** -- Foreign Key Constraints (werden nicht erstellt - Sie müssen sie manuell hinzufügen falls gewünscht) -- Unique Constraints (außer PRIMARY KEY auf `id`) -- Check Constraints -- Trigger oder Stored Procedures - -⚠️ **Einschränkungen:** -- **Keine Schema-Änderungen**: Wenn Sie einen Feldtyp ändern (z.B. `str` → `int`), wird die Spalte NICHT automatisch geändert -- **Keine Spalten-Löschung**: Gelöschte Felder im Modell werden nicht aus der Datenbank entfernt -- **Case-Sensitive**: Tabellennamen werden exakt wie der Klassenname verwendet (z.B. `Projekt`, nicht `projekt`) - -#### 8. Beispiel: Vollständiger Ablauf - -```python -# 1. Pydantic-Modell definieren -class Projekt(BaseModel): - id: str = Field(default_factory=lambda: str(uuid.uuid4())) - mandateId: str - label: str - statusProzess: Optional[StatusProzess] - parzellen: List[Parzelle] = Field(default_factory=list) - -# 2. Interface initialisieren (erstellt noch keine Tabellen) -interface = getInterface(currentUser) - -# 3. Ersten Record erstellen (erstellt jetzt die Tabelle!) -projekt = interface.createProjekt( - label="Mein erstes Projekt", - statusProzess=StatusProzess.PLANUNG -) -# → Intern wird ausgeführt: -# 1. _ensureTableExists(Projekt) aufgerufen -# 2. Tabelle "Projekt" existiert nicht → wird erstellt -# 3. CREATE TABLE "Projekt" (...) wird ausgeführt -# 4. Record wird eingefügt - -# 4. Weitere Records können jetzt ohne Tabellenerstellung erstellt werden -projekt2 = interface.createProjekt(label="Zweites Projekt") -# → Tabelle existiert bereits, nur INSERT wird ausgeführt -``` - ---- - -## Zusammenfassung: - -- ✅ **Tabellenname** = Klassenname des Pydantic-Modells (z.B. `Projekt`) -- ✅ **Spalten** = Alle Felder aus dem Pydantic-Modell -- ✅ **Typen** = Automatisch gemappt (str→TEXT, List→JSONB, etc.) -- ✅ **Systemfelder** = Automatisch hinzugefügt (`_createdAt`, `_createdBy`, etc.) -- ✅ **Indexes** = Automatisch für Felder mit `*Id` Suffix -- ✅ **Migrationen** = Additive Migrationen (neue Spalten werden hinzugefügt) -- ⚠️ **Keine Constraints** = Foreign Keys, Unique, Check müssen manuell erstellt werden - -## Beispiel-Abfragen auf Real Estate-Datenmodell: - -```sql --- Alle Parzellen in einer bestimmten Gemeinde -SELECT * FROM Parzelle WHERE plz = '8000' ORDER BY label; - --- Projekte mit Status "Planung" -SELECT * FROM Projekt WHERE "statusProzess" = 'Planung'; - --- Parzellen mit bestimmter Bauzone -SELECT label, az, bz, gebaeudehoeheMax FROM Parzelle WHERE bauzone = 'W3'; - --- Dokumente eines Projekts -SELECT * FROM Dokument WHERE id IN ( - SELECT unnest(dokumente::jsonb->>'id') FROM Projekt WHERE id = '...' -); -``` - ---- - -[← Zurück: Feature Lifecycle](08-lifecycle.md) | [Weiter: Sicherheitshinweise →](10-security.md) - - - diff --git a/docs/real-estate-feature-integration-guide/10-security.md b/docs/real-estate-feature-integration-guide/10-security.md deleted file mode 100644 index f5ab6b2e..00000000 --- a/docs/real-estate-feature-integration-guide/10-security.md +++ /dev/null @@ -1,90 +0,0 @@ -# Sicherheitshinweise - -[← Zurück: Datenbank-Schema](09-database-schema.md) | [Weiter: Testing →](11-testing.md) - -## ⚠️ WICHTIG: Query-Validierung - -Die aktuelle Implementierung erlaubt die Ausführung von **rohen SQL-Queries**. Für Produktion sollten Sie: - -1. **Query-Whitelisting**: Nur erlaubte Queries zulassen -2. **Nur SELECT**: Nur SELECT-Statements erlauben (keine INSERT/UPDATE/DELETE) -3. **Parameterized Queries**: Immer Parameterized Queries verwenden -4. **Query-Parsing**: SQL-Parser verwenden zur Validierung -5. **Rate Limiting**: Strikte Rate Limits setzen (bereits implementiert) - -## Beispiel für Query-Validierung: - -```python -def validateQuery(queryText: str) -> bool: - """ - Validate that query is safe to execute. - Only allows SELECT statements on Real Estate data model tables. - """ - query_lower = queryText.strip().lower() - - # Only allow SELECT statements - if not query_lower.startswith('select'): - return False - - # Block dangerous keywords - dangerous_keywords = [ - 'drop', 'delete', 'insert', 'update', 'alter', 'create', - 'truncate', 'grant', 'revoke', 'exec', 'execute', 'call' - ] - for keyword in dangerous_keywords: - if keyword in query_lower: - return False - - # Only allow queries on Real Estate data model tables - allowed_tables = [ - 'projekt', 'parzelle', 'dokument', 'kontext', - 'geopolylinie', 'geopunkt', 'land', 'kanton', 'gemeinde' - ] - - # Check if query references allowed tables - # Simple check - in production, use SQL parser - query_contains_allowed_table = any( - f'from {table}' in query_lower or f'join {table}' in query_lower - for table in allowed_tables - ) - - if not query_contains_allowed_table: - # Allow queries that don't specify table explicitly (might be subqueries) - # But log for review - logger.warning(f"Query does not reference known Real Estate tables: {queryText[:100]}") - - return True -``` - -## Erweiterte Validierung mit SQL-Parser: - -Für Produktion sollten Sie einen SQL-Parser verwenden: - -```python -from sqlparse import parse, tokens - -def validateQueryAdvanced(queryText: str) -> bool: - """Advanced query validation using SQL parser.""" - try: - parsed = parse(queryText)[0] - - # Check statement type - if parsed.get_type() != 'SELECT': - return False - - # Extract table names and validate - # Implementation depends on SQL parser library - # ... - - return True - except Exception as e: - logger.error(f"Query parsing failed: {e}") - return False -``` - ---- - -[← Zurück: Datenbank-Schema](09-database-schema.md) | [Weiter: Testing →](11-testing.md) - - - diff --git a/docs/real-estate-feature-integration-guide/11-testing.md b/docs/real-estate-feature-integration-guide/11-testing.md deleted file mode 100644 index 2ec33c3a..00000000 --- a/docs/real-estate-feature-integration-guide/11-testing.md +++ /dev/null @@ -1,51 +0,0 @@ -# Testing - -[← Zurück: Sicherheitshinweise](10-security.md) | [Weiter: Troubleshooting →](12-troubleshooting.md) - -## Manuelle API-Tests mit curl: - -```bash -# 1. Login (erhalten Sie Token) -curl -X POST "http://localhost:8000/api/local/auth/login" \ - -H "Content-Type: application/x-www-form-urlencoded" \ - -d "username=youruser&password=yourpass" - -# 2. Session erstellen -curl -X POST "http://localhost:8000/api/realestate/sessions" \ - -H "Authorization: Bearer YOUR_TOKEN" \ - -H "Content-Type: application/json" \ - -d '{"title": "Parzellen-Analyse Zürich"}' - -# 3. Query ausführen - Beispiel: Alle Parzellen in Zürich -curl -X POST "http://localhost:8000/api/realestate/sessions/SESSION_ID/queries" \ - -H "Authorization: Bearer YOUR_TOKEN" \ - -H "Content-Type: application/json" \ - -d '{ - "queryText": "SELECT label, plz, bauzone, az, bz, gebaeudehoeheMax FROM Parzelle WHERE plz = ''8000'' ORDER BY label LIMIT 20", - "queryType": "sql" - }' - -# 4. Query ausführen - Beispiel: Projekte mit Status "Planung" -curl -X POST "http://localhost:8000/api/realestate/sessions/SESSION_ID/queries" \ - -H "Authorization: Bearer YOUR_TOKEN" \ - -H "Content-Type: application/json" \ - -d '{ - "queryText": "SELECT id, label, \"statusProzess\" FROM Projekt WHERE \"statusProzess\" = ''Planung''", - "queryType": "sql" - }' - -# 5. Queries abrufen -curl -X GET "http://localhost:8000/api/realestate/sessions/SESSION_ID/queries" \ - -H "Authorization: Bearer YOUR_TOKEN" -``` - -## Swagger UI: - -Nach dem Start der Anwendung können Sie die API unter `http://localhost:8000/docs` testen. - ---- - -[← Zurück: Sicherheitshinweise](10-security.md) | [Weiter: Troubleshooting →](12-troubleshooting.md) - - - diff --git a/docs/real-estate-feature-integration-guide/12-troubleshooting.md b/docs/real-estate-feature-integration-guide/12-troubleshooting.md deleted file mode 100644 index 9c7b2475..00000000 --- a/docs/real-estate-feature-integration-guide/12-troubleshooting.md +++ /dev/null @@ -1,33 +0,0 @@ -# Troubleshooting - -[← Zurück: Testing](11-testing.md) | [Weiter: Zusammenfassung →](13-summary.md) - -## Problem: Datenbankverbindung schlägt fehl - -**Lösung**: Überprüfen Sie die Environment-Variablen in `env_dev.env`: -- `DB_APP_HOST` -- `DB_APP_DATABASE` -- `DB_APP_USER` -- `DB_APP_PASSWORD_SECRET` -- `DB_APP_PORT` - -## Problem: Tabellen werden nicht erstellt - -**Lösung**: Der Connector erstellt Tabellen beim ersten Zugriff. Stellen Sie sicher, dass: -- Die Datenbank existiert -- Der Benutzer CREATE-Rechte hat -- Die Verbindung erfolgreich ist - -## Problem: Access Denied Fehler - -**Lösung**: Überprüfen Sie: -- User hat gültiges `mandateId` -- User hat entsprechende Privilegien -- Access Control Logik im Interface - ---- - -[← Zurück: Testing](11-testing.md) | [Weiter: Zusammenfassung →](13-summary.md) - - - diff --git a/docs/real-estate-feature-integration-guide/13-summary.md b/docs/real-estate-feature-integration-guide/13-summary.md deleted file mode 100644 index 240af342..00000000 --- a/docs/real-estate-feature-integration-guide/13-summary.md +++ /dev/null @@ -1,136 +0,0 @@ -# Zusammenfassung - -[← Zurück: Troubleshooting](12-troubleshooting.md) | [← Zurück zur Übersicht](README.md) - -## Dateinamen-Konvention: - -**Wichtig:** Die Dateien sind nach Funktionalität benannt: - -| Datei | Zweck | Enthält | -|-------|-------|---------| -| `datamodelRealEstateChat.py` | Chat-Interface Modelle | `RealEstateQuery`, `RealEstateQueryResult`, `RealEstateChatSession` | -| `datamodelRealEstate.py` | Real Estate-Datenmodelle | `Projekt`, `Parzelle`, `Dokument`, etc. (allgemein verwendbar) | -| `interfaceDbRealEstateChatObjects.py` | Chat-Interface Interface | Methoden für Sessions und Queries | -| `interfaceDbRealEstateObjects.py` | Real Estate CRUD Interface | Methoden für Projekt, Parzelle, etc. (optional) | - -**Hinweis:** Das Modell ist allgemein für alle Real Estate-Firmen verwendbar. PEK ist nur ein Beispiel. - ---- - -## Zu erstellende Dateien: - -1. **`modules/datamodels/datamodelRealEstateChat.py`** (Chat-Interface Modelle) - - Pydantic-Modelle: `RealEstateQuery`, `RealEstateQueryResult`, `RealEstateChatSession` - - Enums: `QueryStatusEnum` - -2. **`modules/datamodels/datamodelRealEstate.py`** (Real Estate-Datenmodell) - - Pydantic-Modelle: `Projekt`, `Parzelle`, `Dokument`, `Kontext`, `GeoPolylinie`, `GeoPunkt`, `Land`, `Kanton`, `Gemeinde` - - Enums: `StatusProzess`, `DokumentTyp`, `JaNein`, `GeoTag` - - Siehe `../PEK_datamodel_desc.md` für vollständige Spezifikation (PEK ist ein Beispiel, das Modell ist allgemein verwendbar) - -3. **`modules/interfaces/interfaceDbRealEstateChatObjects.py`** (Chat-Interface) - - `RealEstateChatObjects` Klasse für Datenbankzugriff (Chat-Sessions, Queries) - - `RealEstateChatAccess` Klasse für Zugriffskontrolle - - `getInterface()` Factory-Funktion - -4. **`modules/interfaces/interfaceDbRealEstateObjects.py`** (NEU - für Real Estate-Datenmodell CRUD) - - `RealEstateObjects` Klasse für CRUD-Operationen auf Real Estate-Entitäten (Projekt, Parzelle, etc.) - - `RealEstateAccess` Klasse für Zugriffskontrolle - - Methoden für Projekt, Parzelle, Dokument, etc. - - **Hinweis:** Diese Datei ist für CRUD-Operationen auf die Real Estate-Entitäten. Das Chat-Interface nutzt `interfaceDbRealEstateChatObjects.py` (siehe Punkt 3). - - **Optional:** Falls Sie eine separate CRUD-API benötigen (das Chat-Interface kann auch direkt SQL-Queries verwenden) - -5. **`modules/features/realEstate/mainRealEstate.py`** - - Feature-Logik-Funktionen: `createSession`, `executeDatabaseQuery`, etc. - -6. **`modules/routes/routeRealEstate.py`** - - FastAPI Router mit allen Endpunkten für Chat-Interface - -7. **`modules/routes/routeRealEstateData.py`** (NEU - für Real Estate-Datenmodell) - - FastAPI Router für CRUD-Operationen auf Real Estate-Entitäten - - Endpunkte für Projekt, Parzelle, Dokument, etc. - - **Optional:** Falls Sie eine separate CRUD-API benötigen (das Chat-Interface kann auch direkt SQL-Queries verwenden) - -## Zu modifizierende Dateien: - -1. **`app.py`** - - Router-Registrierung für `routeRealEstate` hinzufügen (Chat-Interface) - - Router-Registrierung für `routeRealEstateData` hinzufügen (falls CRUD-API gewünscht) - -2. **`env_dev.env`** (optional) - - Separate Datenbank-Konfiguration falls gewünscht - - PostGIS-Konfiguration falls geografische Abfragen benötigt werden - -3. **`modules/features/featuresLifecycle.py`** (optional) - - Feature-Initialisierung falls benötigt - - Initialisierung von Standard-Daten (z.B. Land "Schweiz", Kantone, Gemeinden) - -## Datenmodell-Implementierung: - -**Wichtig:** Bevor Sie das Chat-Interface nutzen können, müssen Sie die Real Estate-Datenmodell-Entitäten implementieren: - -1. **Erstellen Sie `modules/datamodels/datamodelRealEstate.py`** mit allen Entitäten aus `../PEK_datamodel_desc.md` - - **Hinweis:** PEK ist ein Beispiel für eine Real Estate-Firma, aber das Modell ist allgemein verwendbar für alle Real Estate-Firmen -2. **Beachten Sie die Objektbeziehungen**: - - `parzellen: list[Parzelle]` wird als JSONB gespeichert - - `kontextKanton: Kanton` wird als String-ID gespeichert (Foreign Key) -3. **Implementieren Sie die Enums** entsprechend der Spezifikation -4. **Testen Sie die Tabellenerstellung** durch den DatabaseConnector - ---- - -## Nächste Schritte - -1. **Real Estate-Datenmodell-Implementierung**: - - Erstellen Sie die Pydantic-Modelle für alle Real Estate-Entitäten (`Projekt`, `Parzelle`, `Dokument`, `Kontext`, `GeoPolylinie`, `GeoPunkt`, `Land`, `Kanton`, `Gemeinde`) - - Implementieren Sie die Enums (`StatusProzess`, `DokumentTyp`, `JaNein`, `GeoTag`) - - Siehe `../PEK_datamodel_desc.md` für vollständige Spezifikation (PEK ist ein Beispiel, das Modell ist allgemein verwendbar) - -2. **Query-Validierung implementieren**: Siehe [Sicherheitshinweise](10-security.md) - - Besonders wichtig für Real Estate-Datenmodell: Nur SELECT-Statements erlauben - - Whitelist für erlaubte Tabellen (Projekt, Parzelle, etc.) - -3. **Natural Language Processing**: - - Implementieren Sie NLP für `queryType="natural"` - - Beispiele: "Zeige mir alle Parzellen in Zürich" → SQL-Query - - Nutzen Sie AI-Modelle zur SQL-Generierung aus natürlicher Sprache - -4. **Geografische Abfragen**: - - PostGIS-Integration für räumliche Abfragen - - Beispiel: "Zeige alle Parzellen innerhalb eines bestimmten Perimeters" - - Nutzung von GeoPolylinie und GeoPunkt für GIS-Funktionen - -5. **Query-History**: Erweiterte Historie-Funktionen - - Speichern häufig verwendeter Queries - - Query-Templates für häufige Abfragen (z.B. "Parzellen nach Bauzone") - -6. **Export-Funktionen**: CSV/Excel-Export von Ergebnissen - - Export von Parzellen-Listen - - Export von Projekt-Übersichten - -7. **Caching**: Query-Ergebnisse cachen für wiederholte Abfragen - - Besonders für administrative Daten (Land, Kanton, Gemeinde) - -8. **Permissions**: Erweiterte Berechtigungen für bestimmte Tabellen - - Mandaten-basierte Filterung für Projekte und Parzellen - - Rollen-basierte Zugriffe (z.B. nur Leserechte für bestimmte Benutzer) - ---- - -## Architektur-Zusammenfassung - -Dieses Feature folgt dem etablierten Muster des Projekts: -- **Separation of Concerns**: Routes → Features → Interfaces → Connectors -- **Dependency Injection**: Interfaces werden über Factory-Funktionen erstellt -- **Access Control**: Mandaten- und Benutzer-basierte Filterung -- **Type Safety**: Pydantic-Modelle für Validierung -- **Async Support**: Asynchrone Verarbeitung für Skalierbarkeit - -Die Implementierung ist modular und erweiterbar. Sie können weitere Funktionen hinzufügen, ohne die bestehende Struktur zu ändern. - ---- - -[← Zurück: Troubleshooting](12-troubleshooting.md) | [← Zurück zur Übersicht](README.md) - - - diff --git a/docs/real-estate-feature-integration-guide/README.md b/docs/real-estate-feature-integration-guide/README.md index 95b0346d..a7d50f76 100644 --- a/docs/real-estate-feature-integration-guide/README.md +++ b/docs/real-estate-feature-integration-guide/README.md @@ -40,3 +40,6 @@ Die Architektur folgt dem Muster bestehender Features wie `chatPlayground`: + + + diff --git a/logs/debug/prompts/20251120-154518-035-intentanalysis_prompt.txt b/logs/debug/prompts/20251120-154518-035-intentanalysis_prompt.txt new file mode 100644 index 00000000..cd1151aa --- /dev/null +++ b/logs/debug/prompts/20251120-154518-035-intentanalysis_prompt.txt @@ -0,0 +1,89 @@ + +Analyze the following user command and extract the intent, entity, and parameters. + +User Command: "Wie viele Parzellen in Zürich gibt es?" + +Available intents: +- CREATE: User wants to create a new entity +- READ: User wants to read/query entities +- UPDATE: User wants to update an existing entity +- DELETE: User wants to delete an entity +- QUERY: User wants to execute a database query (SQL statements) + +Available entities and their fields: + +**Projekt** (Real estate project): +- id: string (primary key) +- mandateId: string (mandate ID) +- label: string (project designation/name) +- statusProzess: string enum (Eingang, Analyse, Studie, Planung, Baurechtsverfahren, Umsetzung, Archiv) +- perimeter: GeoPolylinie (geographic boundary, JSONB) +- baulinie: GeoPolylinie (building line, JSONB) +- parzellen: List[Parzelle] (plots belonging to project, JSONB) +- dokumente: List[Dokument] (documents, JSONB) +- kontextInformationen: List[Kontext] (context info, JSONB) + +**Parzelle** (Plot/parcel): +- id: string (primary key) +- mandateId: string (mandate ID) +- label: string (plot designation) +- strasseNr: string (street and house number) +- plz: string (postal code) +- kontextGemeinde: string (municipality ID, Foreign Key to Gemeinde table) +- bauzone: string (building zone, e.g. W3, WG2) +- az: float (Ausnützungsziffer) +- bz: float (Bebauungsziffer) +- vollgeschossZahl: int (number of allowed full floors) +- gebaeudehoeheMax: float (maximum building height in meters) +- laermschutzzone: string (noise protection zone) +- hochwasserschutzzone: string (flood protection zone) +- grundwasserschutzzone: string (groundwater protection zone) +- parzelleBebaut: JaNein enum (is plot built) +- parzelleErschlossen: JaNein enum (is plot developed) +- parzelleHanglage: JaNein enum (is plot on slope) + +**Important relationships:** +- Projekte contain Parzellen (projects have plots) +- Parzelle links to Gemeinde (via kontextGemeinde) +- Gemeinde links to Kanton (via id_kanton) +- Kanton links to Land (via id_land) +- Location queries (city, postal code) should use Parzelle.kontextGemeinde (municipality name will be resolved to ID) +- Projekt does NOT have location fields directly - location is stored in associated Parzellen + +Return a JSON object with the following structure: +{ + "intent": "CREATE|READ|UPDATE|DELETE|QUERY", + "entity": "Projekt|Parzelle|Dokument|Kanton|Gemeinde|null", + "parameters": { + // Extracted parameters from user input + // For CREATE/UPDATE: include all relevant fields using EXACT field names from above + // For READ: include filter criteria using EXACT field names (id, label, plz, kontextGemeinde, etc.) + // For DELETE: include entity ID if mentioned + // For QUERY: include queryText if SQL is detected + // IMPORTANT: Use only field names that exist in the entity definition above + }, + "confidence": 0.0-1.0 // Confidence score for the analysis +} + +Examples: +- Input: "Erstelle ein neues Projekt namens 'Hauptstrasse 42'" + Output: {"intent": "CREATE", "entity": "Projekt", "parameters": {"label": "Hauptstrasse 42"}, "confidence": 0.95} + +- Input: "Zeige mir alle Projekte" + Output: {"intent": "READ", "entity": "Projekt", "parameters": {}, "confidence": 0.9} + +- Input: "Zeige mir Projekte in Zürich" + Output: {"intent": "READ", "entity": "Parzelle", "parameters": {"kontextGemeinde": "Zürich"}, "confidence": 0.9} + Note: Location queries should query Parzelle, not Projekt directly + +- Input: "Zeige mir Parzellen mit PLZ 8000" + Output: {"intent": "READ", "entity": "Parzelle", "parameters": {"plz": "8000"}, "confidence": 0.95} + +- Input: "Aktualisiere Projekt XYZ mit Status 'Planung'" + Output: {"intent": "UPDATE", "entity": "Projekt", "parameters": {"id": "XYZ", "statusProzess": "Planung"}, "confidence": 0.85} + +- Input: "SELECT * FROM Projekt WHERE label = 'Test'" + Output: {"intent": "QUERY", "entity": null, "parameters": {"queryText": "SELECT * FROM Projekt WHERE label = 'Test'", "queryType": "sql"}, "confidence": 1.0} + +- Input: "Lösche Parzelle ABC" + Output: {"intent": "DELETE", "entity": "Parzelle", "parameters": {"id": "ABC"}, "confidence": 0.9} diff --git a/logs/debug/prompts/20251120-154524-036-intentanalysis_response.txt b/logs/debug/prompts/20251120-154524-036-intentanalysis_response.txt new file mode 100644 index 00000000..25bb5a4e --- /dev/null +++ b/logs/debug/prompts/20251120-154524-036-intentanalysis_response.txt @@ -0,0 +1,18 @@ +```json +{ + "intent": "READ", + "entity": "Parzelle", + "parameters": { + "kontextGemeinde": "Zürich", + "aggregation": "count" + }, + "confidence": 0.95 +} +``` + +**Explanation:** +- The user is asking "How many plots are there in Zürich?" which is a READ/query operation +- The entity is "Parzelle" (plot/parcel) since the user explicitly asks about "Parzellen" +- The location "Zürich" maps to the `kontextGemeinde` field (municipality) +- The phrase "Wie viele" (how many) indicates a count aggregation is needed +- High confidence (0.95) because the intent and entity are clearly stated in German \ No newline at end of file diff --git a/logs/debug/prompts/20251120-154559-037-intentanalysis_prompt.txt b/logs/debug/prompts/20251120-154559-037-intentanalysis_prompt.txt new file mode 100644 index 00000000..b13d0efe --- /dev/null +++ b/logs/debug/prompts/20251120-154559-037-intentanalysis_prompt.txt @@ -0,0 +1,89 @@ + +Analyze the following user command and extract the intent, entity, and parameters. + +User Command: "Wie viele Projekte in Zürich gibt es?" + +Available intents: +- CREATE: User wants to create a new entity +- READ: User wants to read/query entities +- UPDATE: User wants to update an existing entity +- DELETE: User wants to delete an entity +- QUERY: User wants to execute a database query (SQL statements) + +Available entities and their fields: + +**Projekt** (Real estate project): +- id: string (primary key) +- mandateId: string (mandate ID) +- label: string (project designation/name) +- statusProzess: string enum (Eingang, Analyse, Studie, Planung, Baurechtsverfahren, Umsetzung, Archiv) +- perimeter: GeoPolylinie (geographic boundary, JSONB) +- baulinie: GeoPolylinie (building line, JSONB) +- parzellen: List[Parzelle] (plots belonging to project, JSONB) +- dokumente: List[Dokument] (documents, JSONB) +- kontextInformationen: List[Kontext] (context info, JSONB) + +**Parzelle** (Plot/parcel): +- id: string (primary key) +- mandateId: string (mandate ID) +- label: string (plot designation) +- strasseNr: string (street and house number) +- plz: string (postal code) +- kontextGemeinde: string (municipality ID, Foreign Key to Gemeinde table) +- bauzone: string (building zone, e.g. W3, WG2) +- az: float (Ausnützungsziffer) +- bz: float (Bebauungsziffer) +- vollgeschossZahl: int (number of allowed full floors) +- gebaeudehoeheMax: float (maximum building height in meters) +- laermschutzzone: string (noise protection zone) +- hochwasserschutzzone: string (flood protection zone) +- grundwasserschutzzone: string (groundwater protection zone) +- parzelleBebaut: JaNein enum (is plot built) +- parzelleErschlossen: JaNein enum (is plot developed) +- parzelleHanglage: JaNein enum (is plot on slope) + +**Important relationships:** +- Projekte contain Parzellen (projects have plots) +- Parzelle links to Gemeinde (via kontextGemeinde) +- Gemeinde links to Kanton (via id_kanton) +- Kanton links to Land (via id_land) +- Location queries (city, postal code) should use Parzelle.kontextGemeinde (municipality name will be resolved to ID) +- Projekt does NOT have location fields directly - location is stored in associated Parzellen + +Return a JSON object with the following structure: +{ + "intent": "CREATE|READ|UPDATE|DELETE|QUERY", + "entity": "Projekt|Parzelle|Dokument|Kanton|Gemeinde|null", + "parameters": { + // Extracted parameters from user input + // For CREATE/UPDATE: include all relevant fields using EXACT field names from above + // For READ: include filter criteria using EXACT field names (id, label, plz, kontextGemeinde, etc.) + // For DELETE: include entity ID if mentioned + // For QUERY: include queryText if SQL is detected + // IMPORTANT: Use only field names that exist in the entity definition above + }, + "confidence": 0.0-1.0 // Confidence score for the analysis +} + +Examples: +- Input: "Erstelle ein neues Projekt namens 'Hauptstrasse 42'" + Output: {"intent": "CREATE", "entity": "Projekt", "parameters": {"label": "Hauptstrasse 42"}, "confidence": 0.95} + +- Input: "Zeige mir alle Projekte" + Output: {"intent": "READ", "entity": "Projekt", "parameters": {}, "confidence": 0.9} + +- Input: "Zeige mir Projekte in Zürich" + Output: {"intent": "READ", "entity": "Parzelle", "parameters": {"kontextGemeinde": "Zürich"}, "confidence": 0.9} + Note: Location queries should query Parzelle, not Projekt directly + +- Input: "Zeige mir Parzellen mit PLZ 8000" + Output: {"intent": "READ", "entity": "Parzelle", "parameters": {"plz": "8000"}, "confidence": 0.95} + +- Input: "Aktualisiere Projekt XYZ mit Status 'Planung'" + Output: {"intent": "UPDATE", "entity": "Projekt", "parameters": {"id": "XYZ", "statusProzess": "Planung"}, "confidence": 0.85} + +- Input: "SELECT * FROM Projekt WHERE label = 'Test'" + Output: {"intent": "QUERY", "entity": null, "parameters": {"queryText": "SELECT * FROM Projekt WHERE label = 'Test'", "queryType": "sql"}, "confidence": 1.0} + +- Input: "Lösche Parzelle ABC" + Output: {"intent": "DELETE", "entity": "Parzelle", "parameters": {"id": "ABC"}, "confidence": 0.9} diff --git a/logs/debug/prompts/20251120-154607-038-intentanalysis_response.txt b/logs/debug/prompts/20251120-154607-038-intentanalysis_response.txt new file mode 100644 index 00000000..4974d042 --- /dev/null +++ b/logs/debug/prompts/20251120-154607-038-intentanalysis_response.txt @@ -0,0 +1,17 @@ +```json +{ + "intent": "READ", + "entity": "Parzelle", + "parameters": { + "kontextGemeinde": "Zürich" + }, + "confidence": 0.9 +} +``` + +**Reasoning:** +- The user asks "How many projects in Zürich are there?" which is a READ/query operation +- Since location information (city/municipality) is stored in **Parzelle** via the `kontextGemeinde` field, not directly in Projekt, we need to query Parzelle entities +- "Zürich" maps to the `kontextGemeinde` parameter +- The system will need to find all Parzellen in Zürich and then count the associated unique projects +- Confidence is 0.9 because the intent is clear, though the user asks about "Projekte" while we're querying via "Parzelle" (which is the correct approach based on the data model) \ No newline at end of file diff --git a/logs/debug/prompts/20251120-154715-039-intentanalysis_prompt.txt b/logs/debug/prompts/20251120-154715-039-intentanalysis_prompt.txt new file mode 100644 index 00000000..fd553fd5 --- /dev/null +++ b/logs/debug/prompts/20251120-154715-039-intentanalysis_prompt.txt @@ -0,0 +1,89 @@ + +Analyze the following user command and extract the intent, entity, and parameters. + +User Command: "Welche Gemeinden gibt es in der datenbank?" + +Available intents: +- CREATE: User wants to create a new entity +- READ: User wants to read/query entities +- UPDATE: User wants to update an existing entity +- DELETE: User wants to delete an entity +- QUERY: User wants to execute a database query (SQL statements) + +Available entities and their fields: + +**Projekt** (Real estate project): +- id: string (primary key) +- mandateId: string (mandate ID) +- label: string (project designation/name) +- statusProzess: string enum (Eingang, Analyse, Studie, Planung, Baurechtsverfahren, Umsetzung, Archiv) +- perimeter: GeoPolylinie (geographic boundary, JSONB) +- baulinie: GeoPolylinie (building line, JSONB) +- parzellen: List[Parzelle] (plots belonging to project, JSONB) +- dokumente: List[Dokument] (documents, JSONB) +- kontextInformationen: List[Kontext] (context info, JSONB) + +**Parzelle** (Plot/parcel): +- id: string (primary key) +- mandateId: string (mandate ID) +- label: string (plot designation) +- strasseNr: string (street and house number) +- plz: string (postal code) +- kontextGemeinde: string (municipality ID, Foreign Key to Gemeinde table) +- bauzone: string (building zone, e.g. W3, WG2) +- az: float (Ausnützungsziffer) +- bz: float (Bebauungsziffer) +- vollgeschossZahl: int (number of allowed full floors) +- gebaeudehoeheMax: float (maximum building height in meters) +- laermschutzzone: string (noise protection zone) +- hochwasserschutzzone: string (flood protection zone) +- grundwasserschutzzone: string (groundwater protection zone) +- parzelleBebaut: JaNein enum (is plot built) +- parzelleErschlossen: JaNein enum (is plot developed) +- parzelleHanglage: JaNein enum (is plot on slope) + +**Important relationships:** +- Projekte contain Parzellen (projects have plots) +- Parzelle links to Gemeinde (via kontextGemeinde) +- Gemeinde links to Kanton (via id_kanton) +- Kanton links to Land (via id_land) +- Location queries (city, postal code) should use Parzelle.kontextGemeinde (municipality name will be resolved to ID) +- Projekt does NOT have location fields directly - location is stored in associated Parzellen + +Return a JSON object with the following structure: +{ + "intent": "CREATE|READ|UPDATE|DELETE|QUERY", + "entity": "Projekt|Parzelle|Dokument|Kanton|Gemeinde|null", + "parameters": { + // Extracted parameters from user input + // For CREATE/UPDATE: include all relevant fields using EXACT field names from above + // For READ: include filter criteria using EXACT field names (id, label, plz, kontextGemeinde, etc.) + // For DELETE: include entity ID if mentioned + // For QUERY: include queryText if SQL is detected + // IMPORTANT: Use only field names that exist in the entity definition above + }, + "confidence": 0.0-1.0 // Confidence score for the analysis +} + +Examples: +- Input: "Erstelle ein neues Projekt namens 'Hauptstrasse 42'" + Output: {"intent": "CREATE", "entity": "Projekt", "parameters": {"label": "Hauptstrasse 42"}, "confidence": 0.95} + +- Input: "Zeige mir alle Projekte" + Output: {"intent": "READ", "entity": "Projekt", "parameters": {}, "confidence": 0.9} + +- Input: "Zeige mir Projekte in Zürich" + Output: {"intent": "READ", "entity": "Parzelle", "parameters": {"kontextGemeinde": "Zürich"}, "confidence": 0.9} + Note: Location queries should query Parzelle, not Projekt directly + +- Input: "Zeige mir Parzellen mit PLZ 8000" + Output: {"intent": "READ", "entity": "Parzelle", "parameters": {"plz": "8000"}, "confidence": 0.95} + +- Input: "Aktualisiere Projekt XYZ mit Status 'Planung'" + Output: {"intent": "UPDATE", "entity": "Projekt", "parameters": {"id": "XYZ", "statusProzess": "Planung"}, "confidence": 0.85} + +- Input: "SELECT * FROM Projekt WHERE label = 'Test'" + Output: {"intent": "QUERY", "entity": null, "parameters": {"queryText": "SELECT * FROM Projekt WHERE label = 'Test'", "queryType": "sql"}, "confidence": 1.0} + +- Input: "Lösche Parzelle ABC" + Output: {"intent": "DELETE", "entity": "Parzelle", "parameters": {"id": "ABC"}, "confidence": 0.9} diff --git a/logs/debug/prompts/20251120-154718-040-intentanalysis_response.txt b/logs/debug/prompts/20251120-154718-040-intentanalysis_response.txt new file mode 100644 index 00000000..e97121ac --- /dev/null +++ b/logs/debug/prompts/20251120-154718-040-intentanalysis_response.txt @@ -0,0 +1,10 @@ +```json +{ + "intent": "READ", + "entity": "Gemeinde", + "parameters": {}, + "confidence": 0.95 +} +``` + +The user is asking "Which municipalities are there in the database?" in German. This is a straightforward READ query for all Gemeinde (municipality) entities without any filter criteria. \ No newline at end of file diff --git a/logs/debug/prompts/20251120-154742-041-intentanalysis_prompt.txt b/logs/debug/prompts/20251120-154742-041-intentanalysis_prompt.txt new file mode 100644 index 00000000..f1df4e02 --- /dev/null +++ b/logs/debug/prompts/20251120-154742-041-intentanalysis_prompt.txt @@ -0,0 +1,89 @@ + +Analyze the following user command and extract the intent, entity, and parameters. + +User Command: "Erstell ein neues Projekt" + +Available intents: +- CREATE: User wants to create a new entity +- READ: User wants to read/query entities +- UPDATE: User wants to update an existing entity +- DELETE: User wants to delete an entity +- QUERY: User wants to execute a database query (SQL statements) + +Available entities and their fields: + +**Projekt** (Real estate project): +- id: string (primary key) +- mandateId: string (mandate ID) +- label: string (project designation/name) +- statusProzess: string enum (Eingang, Analyse, Studie, Planung, Baurechtsverfahren, Umsetzung, Archiv) +- perimeter: GeoPolylinie (geographic boundary, JSONB) +- baulinie: GeoPolylinie (building line, JSONB) +- parzellen: List[Parzelle] (plots belonging to project, JSONB) +- dokumente: List[Dokument] (documents, JSONB) +- kontextInformationen: List[Kontext] (context info, JSONB) + +**Parzelle** (Plot/parcel): +- id: string (primary key) +- mandateId: string (mandate ID) +- label: string (plot designation) +- strasseNr: string (street and house number) +- plz: string (postal code) +- kontextGemeinde: string (municipality ID, Foreign Key to Gemeinde table) +- bauzone: string (building zone, e.g. W3, WG2) +- az: float (Ausnützungsziffer) +- bz: float (Bebauungsziffer) +- vollgeschossZahl: int (number of allowed full floors) +- gebaeudehoeheMax: float (maximum building height in meters) +- laermschutzzone: string (noise protection zone) +- hochwasserschutzzone: string (flood protection zone) +- grundwasserschutzzone: string (groundwater protection zone) +- parzelleBebaut: JaNein enum (is plot built) +- parzelleErschlossen: JaNein enum (is plot developed) +- parzelleHanglage: JaNein enum (is plot on slope) + +**Important relationships:** +- Projekte contain Parzellen (projects have plots) +- Parzelle links to Gemeinde (via kontextGemeinde) +- Gemeinde links to Kanton (via id_kanton) +- Kanton links to Land (via id_land) +- Location queries (city, postal code) should use Parzelle.kontextGemeinde (municipality name will be resolved to ID) +- Projekt does NOT have location fields directly - location is stored in associated Parzellen + +Return a JSON object with the following structure: +{ + "intent": "CREATE|READ|UPDATE|DELETE|QUERY", + "entity": "Projekt|Parzelle|Dokument|Kanton|Gemeinde|null", + "parameters": { + // Extracted parameters from user input + // For CREATE/UPDATE: include all relevant fields using EXACT field names from above + // For READ: include filter criteria using EXACT field names (id, label, plz, kontextGemeinde, etc.) + // For DELETE: include entity ID if mentioned + // For QUERY: include queryText if SQL is detected + // IMPORTANT: Use only field names that exist in the entity definition above + }, + "confidence": 0.0-1.0 // Confidence score for the analysis +} + +Examples: +- Input: "Erstelle ein neues Projekt namens 'Hauptstrasse 42'" + Output: {"intent": "CREATE", "entity": "Projekt", "parameters": {"label": "Hauptstrasse 42"}, "confidence": 0.95} + +- Input: "Zeige mir alle Projekte" + Output: {"intent": "READ", "entity": "Projekt", "parameters": {}, "confidence": 0.9} + +- Input: "Zeige mir Projekte in Zürich" + Output: {"intent": "READ", "entity": "Parzelle", "parameters": {"kontextGemeinde": "Zürich"}, "confidence": 0.9} + Note: Location queries should query Parzelle, not Projekt directly + +- Input: "Zeige mir Parzellen mit PLZ 8000" + Output: {"intent": "READ", "entity": "Parzelle", "parameters": {"plz": "8000"}, "confidence": 0.95} + +- Input: "Aktualisiere Projekt XYZ mit Status 'Planung'" + Output: {"intent": "UPDATE", "entity": "Projekt", "parameters": {"id": "XYZ", "statusProzess": "Planung"}, "confidence": 0.85} + +- Input: "SELECT * FROM Projekt WHERE label = 'Test'" + Output: {"intent": "QUERY", "entity": null, "parameters": {"queryText": "SELECT * FROM Projekt WHERE label = 'Test'", "queryType": "sql"}, "confidence": 1.0} + +- Input: "Lösche Parzelle ABC" + Output: {"intent": "DELETE", "entity": "Parzelle", "parameters": {"id": "ABC"}, "confidence": 0.9} diff --git a/logs/debug/prompts/20251120-154746-042-intentanalysis_response.txt b/logs/debug/prompts/20251120-154746-042-intentanalysis_response.txt new file mode 100644 index 00000000..e873906e --- /dev/null +++ b/logs/debug/prompts/20251120-154746-042-intentanalysis_response.txt @@ -0,0 +1,10 @@ +```json +{ + "intent": "CREATE", + "entity": "Projekt", + "parameters": {}, + "confidence": 0.95 +} +``` + +The user command "Erstell ein neues Projekt" (Create a new project) clearly indicates a CREATE intent for a Projekt entity. No specific parameters like project name (label) or other fields are mentioned in this command, so the parameters object is empty. The confidence is high (0.95) because the intent and entity are unambiguous. \ No newline at end of file From f12ac49d323fac729e6819fdc1ee521068582716 Mon Sep 17 00:00:00 2001 From: Ida Dittrich Date: Thu, 27 Nov 2025 13:18:07 +0100 Subject: [PATCH 03/25] finished pagination feature with filter and search options --- docs/code-documentation/aicore-component.md | 1046 +++++++ .../architecture-overview.md | 209 ++ .../connectors-component.md | 1241 +++++++++ .../datamodels-interfaces-component.md | 1832 +++++++++++++ docs/code-documentation/features-component.md | 981 +++++++ .../gateway-development-framework.md | 2281 ++++++++++++++++ .../interactive-workflow-planning-ui.md | 1133 ++++++++ docs/code-documentation/security-api.md | 508 ++++ docs/code-documentation/security-component.md | 1399 ++++++++++ .../services-api-reference.md | 2399 +++++++++++++++++ docs/code-documentation/services-component.md | 1530 +++++++++++ .../code-documentation/workflows-component.md | 1245 +++++++++ .../workflow-routes-frontend.md | 1677 ++++++++++++ .../20251127-113948-147_m_1_0_0/message.json | 19 + .../message_text.txt | 1 + .../20251127-113957-841_m_1_1_0/message.json | 19 + .../message_text.txt | 6 + .../20251127-113958-980_m_1_1_0/message.json | 19 + .../message_text.txt | 3 + .../20251127-114024-001_m_1_1_1/message.json | 19 + .../message_text.txt | 4 + .../document_001_metadata.json | 12 + .../document_001_test_document_memo.docx | Bin 0 -> 36938 bytes .../document_002_metadata.json | 12 + .../document_002_structured_content.json | 106 + .../20251127-114110-763_m_1_1_2/message.json | 19 + .../message_text.txt | 4 + .../document_001_memo_analysis_report.txt | 24 + .../document_001_metadata.json | 12 + .../document_002_metadata.json | 12 + .../document_002_structured_content_1.json | 134 + .../20251127-114147-275_m_1_1_3/message.json | 19 + .../message_text.txt | 4 + .../document_001_memo_analysis_summary.txt | 10 + .../document_001_metadata.json | 12 + .../document_002_metadata.json | 12 + .../document_002_structured_content_2.json | 52 + .../20251127-114229-639_m_1_1_4/message.json | 19 + .../message_text.txt | 4 + .../document_001_memo_analysis_report_1.txt | 22 + .../document_001_metadata.json | 12 + .../document_002_metadata.json | 12 + .../document_002_structured_content_3.json | 136 + .../20251127-114301-221_m_1_1_5/message.json | 19 + .../message_text.txt | 4 + .../document_001_memo_summary.txt | 12 + .../document_001_metadata.json | 12 + .../document_002_metadata.json | 12 + .../document_002_structured_content_4.json | 62 + .../20251127-114309-479_m_1_1_0/message.json | 19 + .../message_text.txt | 4 + .../20251127-114310-696_m_1_0_0/message.json | 19 + .../message_text.txt | 4 + ...251127-113945-043-userintention_prompt.txt | 28 + ...1127-113947-044-userintention_response.txt | 8 + ...51127-113949-045-intentanalysis_prompt.txt | 30 + ...127-113951-046-intentanalysis_response.txt | 14 + .../20251127-113951-047-taskplan_prompt.txt | 100 + .../20251127-113957-048-taskplan_response.txt | 20 + .../20251127-113959-049-actionplan_prompt.txt | 87 + ...0251127-114003-050-actionplan_response.txt | 10 + .../20251127-114003-051-paramplan_prompt.txt | 62 + ...20251127-114005-052-paramplan_response.txt | 10 + ...-114007-053-document_generation_prompt.txt | 84 + ...14016-054-document_generation_response.txt | 89 + ...7-055-document_generation_final_result.txt | 105 + ...127-114022-056-renderer_styling_prompt.txt | 66 + ...7-114022-057-renderer_styling_response.txt | 56 + ...14022-058-document_generation_response.txt | 4 + ...27-114024-059-contentvalidation_prompt.txt | 57 + ...-114031-060-contentvalidation_response.txt | 24 + .../20251127-114031-061-refinement_prompt.txt | 79 + ...0251127-114032-062-refinement_response.txt | 6 + .../20251127-114033-063-actionplan_prompt.txt | 92 + ...0251127-114037-064-actionplan_response.txt | 10 + .../20251127-114038-065-paramplan_prompt.txt | 62 + ...20251127-114040-066-paramplan_response.txt | 10 + ...1127-114102-067-extraction_merged_text.txt | 103 + ...-114103-068-document_generation_prompt.txt | 189 ++ ...14109-069-document_generation_response.txt | 117 + ...9-070-document_generation_final_result.txt | 133 + ...14109-071-document_generation_response.txt | 4 + ...27-114110-072-contentvalidation_prompt.txt | 57 + ...-114115-073-contentvalidation_response.txt | 24 + .../20251127-114115-074-refinement_prompt.txt | 77 + ...0251127-114116-075-refinement_response.txt | 6 + .../20251127-114118-076-actionplan_prompt.txt | 96 + ...0251127-114127-077-actionplan_response.txt | 10 + .../20251127-114127-078-paramplan_prompt.txt | 62 + ...20251127-114129-079-paramplan_response.txt | 10 + ...1127-114140-080-extraction_merged_text.txt | 11 + ...-114141-081-document_generation_prompt.txt | 97 + ...14146-082-document_generation_response.txt | 49 + ...6-083-document_generation_final_result.txt | 51 + ...14146-084-document_generation_response.txt | 4 + ...27-114147-085-contentvalidation_prompt.txt | 57 + ...-114155-086-contentvalidation_response.txt | 24 + .../20251127-114155-087-refinement_prompt.txt | 75 + ...0251127-114157-088-refinement_response.txt | 6 + .../20251127-114158-089-actionplan_prompt.txt | 100 + ...0251127-114159-090-actionplan_response.txt | 10 + .../20251127-114200-091-paramplan_prompt.txt | 62 + ...20251127-114201-092-paramplan_response.txt | 10 + ...1127-114214-093-extraction_merged_text.txt | 65 + ...-114214-094-document_generation_prompt.txt | 151 ++ ...14228-095-document_generation_response.txt | 121 + ...8-096-document_generation_final_result.txt | 135 + ...14228-097-document_generation_response.txt | 4 + ...27-114229-098-contentvalidation_prompt.txt | 57 + ...-114235-099-contentvalidation_response.txt | 24 + .../20251127-114235-100-refinement_prompt.txt | 80 + ...0251127-114237-101-refinement_response.txt | 6 + .../20251127-114238-102-actionplan_prompt.txt | 104 + ...0251127-114240-103-actionplan_response.txt | 10 + .../20251127-114241-104-paramplan_prompt.txt | 62 + ...20251127-114242-105-paramplan_response.txt | 10 + ...1127-114250-106-extraction_merged_text.txt | 14 + ...-114251-107-document_generation_prompt.txt | 100 + ...14259-108-document_generation_response.txt | 57 + ...9-109-document_generation_final_result.txt | 61 + ...14259-110-document_generation_response.txt | 4 + ...27-114301-111-contentvalidation_prompt.txt | 57 + ...-114306-112-contentvalidation_response.txt | 24 + .../20251127-114306-113-refinement_prompt.txt | 75 + ...0251127-114308-114-refinement_response.txt | 6 + modules/datamodels/datamodelPagination.py | 13 +- modules/interfaces/interfaceDbAppObjects.py | 160 +- modules/interfaces/interfaceDbChatObjects.py | 160 +- .../interfaces/interfaceDbComponentObjects.py | 160 +- 129 files changed, 22653 insertions(+), 11 deletions(-) create mode 100644 docs/code-documentation/aicore-component.md create mode 100644 docs/code-documentation/architecture-overview.md create mode 100644 docs/code-documentation/connectors-component.md create mode 100644 docs/code-documentation/datamodels-interfaces-component.md create mode 100644 docs/code-documentation/features-component.md create mode 100644 docs/code-documentation/gateway-development-framework.md create mode 100644 docs/code-documentation/interactive-workflow-planning-ui.md create mode 100644 docs/code-documentation/security-api.md create mode 100644 docs/code-documentation/security-component.md create mode 100644 docs/code-documentation/services-api-reference.md create mode 100644 docs/code-documentation/services-component.md create mode 100644 docs/code-documentation/workflows-component.md create mode 100644 docs/frontend-documentation/workflow-routes-frontend.md create mode 100644 logs/debug/messages/20251127-113948-147_m_1_0_0/message.json create mode 100644 logs/debug/messages/20251127-113948-147_m_1_0_0/message_text.txt create mode 100644 logs/debug/messages/20251127-113957-841_m_1_1_0/message.json create mode 100644 logs/debug/messages/20251127-113957-841_m_1_1_0/message_text.txt create mode 100644 logs/debug/messages/20251127-113958-980_m_1_1_0/message.json create mode 100644 logs/debug/messages/20251127-113958-980_m_1_1_0/message_text.txt create mode 100644 logs/debug/messages/20251127-114024-001_m_1_1_1/message.json create mode 100644 logs/debug/messages/20251127-114024-001_m_1_1_1/message_text.txt create mode 100644 logs/debug/messages/20251127-114024-001_m_1_1_1/round1_task1_action1_results/document_001_metadata.json create mode 100644 logs/debug/messages/20251127-114024-001_m_1_1_1/round1_task1_action1_results/document_001_test_document_memo.docx create mode 100644 logs/debug/messages/20251127-114024-001_m_1_1_1/round1_task1_action1_results/document_002_metadata.json create mode 100644 logs/debug/messages/20251127-114024-001_m_1_1_1/round1_task1_action1_results/document_002_structured_content.json create mode 100644 logs/debug/messages/20251127-114110-763_m_1_1_2/message.json create mode 100644 logs/debug/messages/20251127-114110-763_m_1_1_2/message_text.txt create mode 100644 logs/debug/messages/20251127-114110-763_m_1_1_2/round1_task1_action2_results/document_001_memo_analysis_report.txt create mode 100644 logs/debug/messages/20251127-114110-763_m_1_1_2/round1_task1_action2_results/document_001_metadata.json create mode 100644 logs/debug/messages/20251127-114110-763_m_1_1_2/round1_task1_action2_results/document_002_metadata.json create mode 100644 logs/debug/messages/20251127-114110-763_m_1_1_2/round1_task1_action2_results/document_002_structured_content_1.json create mode 100644 logs/debug/messages/20251127-114147-275_m_1_1_3/message.json create mode 100644 logs/debug/messages/20251127-114147-275_m_1_1_3/message_text.txt create mode 100644 logs/debug/messages/20251127-114147-275_m_1_1_3/round1_task1_action3_results/document_001_memo_analysis_summary.txt create mode 100644 logs/debug/messages/20251127-114147-275_m_1_1_3/round1_task1_action3_results/document_001_metadata.json create mode 100644 logs/debug/messages/20251127-114147-275_m_1_1_3/round1_task1_action3_results/document_002_metadata.json create mode 100644 logs/debug/messages/20251127-114147-275_m_1_1_3/round1_task1_action3_results/document_002_structured_content_2.json create mode 100644 logs/debug/messages/20251127-114229-639_m_1_1_4/message.json create mode 100644 logs/debug/messages/20251127-114229-639_m_1_1_4/message_text.txt create mode 100644 logs/debug/messages/20251127-114229-639_m_1_1_4/round1_task1_action4_results/document_001_memo_analysis_report_1.txt create mode 100644 logs/debug/messages/20251127-114229-639_m_1_1_4/round1_task1_action4_results/document_001_metadata.json create mode 100644 logs/debug/messages/20251127-114229-639_m_1_1_4/round1_task1_action4_results/document_002_metadata.json create mode 100644 logs/debug/messages/20251127-114229-639_m_1_1_4/round1_task1_action4_results/document_002_structured_content_3.json create mode 100644 logs/debug/messages/20251127-114301-221_m_1_1_5/message.json create mode 100644 logs/debug/messages/20251127-114301-221_m_1_1_5/message_text.txt create mode 100644 logs/debug/messages/20251127-114301-221_m_1_1_5/round1_task1_action5_results/document_001_memo_summary.txt create mode 100644 logs/debug/messages/20251127-114301-221_m_1_1_5/round1_task1_action5_results/document_001_metadata.json create mode 100644 logs/debug/messages/20251127-114301-221_m_1_1_5/round1_task1_action5_results/document_002_metadata.json create mode 100644 logs/debug/messages/20251127-114301-221_m_1_1_5/round1_task1_action5_results/document_002_structured_content_4.json create mode 100644 logs/debug/messages/20251127-114309-479_m_1_1_0/message.json create mode 100644 logs/debug/messages/20251127-114309-479_m_1_1_0/message_text.txt create mode 100644 logs/debug/messages/20251127-114310-696_m_1_0_0/message.json create mode 100644 logs/debug/messages/20251127-114310-696_m_1_0_0/message_text.txt create mode 100644 logs/debug/prompts/20251127-113945-043-userintention_prompt.txt create mode 100644 logs/debug/prompts/20251127-113947-044-userintention_response.txt create mode 100644 logs/debug/prompts/20251127-113949-045-intentanalysis_prompt.txt create mode 100644 logs/debug/prompts/20251127-113951-046-intentanalysis_response.txt create mode 100644 logs/debug/prompts/20251127-113951-047-taskplan_prompt.txt create mode 100644 logs/debug/prompts/20251127-113957-048-taskplan_response.txt create mode 100644 logs/debug/prompts/20251127-113959-049-actionplan_prompt.txt create mode 100644 logs/debug/prompts/20251127-114003-050-actionplan_response.txt create mode 100644 logs/debug/prompts/20251127-114003-051-paramplan_prompt.txt create mode 100644 logs/debug/prompts/20251127-114005-052-paramplan_response.txt create mode 100644 logs/debug/prompts/20251127-114007-053-document_generation_prompt.txt create mode 100644 logs/debug/prompts/20251127-114016-054-document_generation_response.txt create mode 100644 logs/debug/prompts/20251127-114017-055-document_generation_final_result.txt create mode 100644 logs/debug/prompts/20251127-114022-056-renderer_styling_prompt.txt create mode 100644 logs/debug/prompts/20251127-114022-057-renderer_styling_response.txt create mode 100644 logs/debug/prompts/20251127-114022-058-document_generation_response.txt create mode 100644 logs/debug/prompts/20251127-114024-059-contentvalidation_prompt.txt create mode 100644 logs/debug/prompts/20251127-114031-060-contentvalidation_response.txt create mode 100644 logs/debug/prompts/20251127-114031-061-refinement_prompt.txt create mode 100644 logs/debug/prompts/20251127-114032-062-refinement_response.txt create mode 100644 logs/debug/prompts/20251127-114033-063-actionplan_prompt.txt create mode 100644 logs/debug/prompts/20251127-114037-064-actionplan_response.txt create mode 100644 logs/debug/prompts/20251127-114038-065-paramplan_prompt.txt create mode 100644 logs/debug/prompts/20251127-114040-066-paramplan_response.txt create mode 100644 logs/debug/prompts/20251127-114102-067-extraction_merged_text.txt create mode 100644 logs/debug/prompts/20251127-114103-068-document_generation_prompt.txt create mode 100644 logs/debug/prompts/20251127-114109-069-document_generation_response.txt create mode 100644 logs/debug/prompts/20251127-114109-070-document_generation_final_result.txt create mode 100644 logs/debug/prompts/20251127-114109-071-document_generation_response.txt create mode 100644 logs/debug/prompts/20251127-114110-072-contentvalidation_prompt.txt create mode 100644 logs/debug/prompts/20251127-114115-073-contentvalidation_response.txt create mode 100644 logs/debug/prompts/20251127-114115-074-refinement_prompt.txt create mode 100644 logs/debug/prompts/20251127-114116-075-refinement_response.txt create mode 100644 logs/debug/prompts/20251127-114118-076-actionplan_prompt.txt create mode 100644 logs/debug/prompts/20251127-114127-077-actionplan_response.txt create mode 100644 logs/debug/prompts/20251127-114127-078-paramplan_prompt.txt create mode 100644 logs/debug/prompts/20251127-114129-079-paramplan_response.txt create mode 100644 logs/debug/prompts/20251127-114140-080-extraction_merged_text.txt create mode 100644 logs/debug/prompts/20251127-114141-081-document_generation_prompt.txt create mode 100644 logs/debug/prompts/20251127-114146-082-document_generation_response.txt create mode 100644 logs/debug/prompts/20251127-114146-083-document_generation_final_result.txt create mode 100644 logs/debug/prompts/20251127-114146-084-document_generation_response.txt create mode 100644 logs/debug/prompts/20251127-114147-085-contentvalidation_prompt.txt create mode 100644 logs/debug/prompts/20251127-114155-086-contentvalidation_response.txt create mode 100644 logs/debug/prompts/20251127-114155-087-refinement_prompt.txt create mode 100644 logs/debug/prompts/20251127-114157-088-refinement_response.txt create mode 100644 logs/debug/prompts/20251127-114158-089-actionplan_prompt.txt create mode 100644 logs/debug/prompts/20251127-114159-090-actionplan_response.txt create mode 100644 logs/debug/prompts/20251127-114200-091-paramplan_prompt.txt create mode 100644 logs/debug/prompts/20251127-114201-092-paramplan_response.txt create mode 100644 logs/debug/prompts/20251127-114214-093-extraction_merged_text.txt create mode 100644 logs/debug/prompts/20251127-114214-094-document_generation_prompt.txt create mode 100644 logs/debug/prompts/20251127-114228-095-document_generation_response.txt create mode 100644 logs/debug/prompts/20251127-114228-096-document_generation_final_result.txt create mode 100644 logs/debug/prompts/20251127-114228-097-document_generation_response.txt create mode 100644 logs/debug/prompts/20251127-114229-098-contentvalidation_prompt.txt create mode 100644 logs/debug/prompts/20251127-114235-099-contentvalidation_response.txt create mode 100644 logs/debug/prompts/20251127-114235-100-refinement_prompt.txt create mode 100644 logs/debug/prompts/20251127-114237-101-refinement_response.txt create mode 100644 logs/debug/prompts/20251127-114238-102-actionplan_prompt.txt create mode 100644 logs/debug/prompts/20251127-114240-103-actionplan_response.txt create mode 100644 logs/debug/prompts/20251127-114241-104-paramplan_prompt.txt create mode 100644 logs/debug/prompts/20251127-114242-105-paramplan_response.txt create mode 100644 logs/debug/prompts/20251127-114250-106-extraction_merged_text.txt create mode 100644 logs/debug/prompts/20251127-114251-107-document_generation_prompt.txt create mode 100644 logs/debug/prompts/20251127-114259-108-document_generation_response.txt create mode 100644 logs/debug/prompts/20251127-114259-109-document_generation_final_result.txt create mode 100644 logs/debug/prompts/20251127-114259-110-document_generation_response.txt create mode 100644 logs/debug/prompts/20251127-114301-111-contentvalidation_prompt.txt create mode 100644 logs/debug/prompts/20251127-114306-112-contentvalidation_response.txt create mode 100644 logs/debug/prompts/20251127-114306-113-refinement_prompt.txt create mode 100644 logs/debug/prompts/20251127-114308-114-refinement_response.txt diff --git a/docs/code-documentation/aicore-component.md b/docs/code-documentation/aicore-component.md new file mode 100644 index 00000000..336eb71d --- /dev/null +++ b/docs/code-documentation/aicore-component.md @@ -0,0 +1,1046 @@ +# AI Core Component Documentation + +## Overview + +The `aicore` module is the **centralized AI infrastructure layer** that provides a **plugin-based architecture** for integrating multiple AI providers (OpenAI, Anthropic, Perplexity, Tavily) into the application. It acts as an abstraction layer between high-level AI services and specific AI provider APIs, enabling dynamic model discovery, intelligent model selection, and automatic failover. + +**Key Responsibilities:** +- Dynamic discovery and registration of AI connectors (plugins) +- Model registry with unified model metadata +- Intelligent model selection based on operation type, context size, and optimization criteria +- Automatic failover between models +- Standardized interface for AI operations across all providers + +## Architecture + +### System Architecture Overview + +```mermaid +graph TB + subgraph "Application Layer" + Routes[FastAPI Routes
routeWorkflows.py
routeChatPlayground.py] + end + + subgraph "Service Layer" + AiService[AiService
mainServiceAi.py] + Methods[callAiPlanning
callAiDocuments
callAiText] + AiService --> Methods + end + + subgraph "Interface Layer" + AiObjects[AiObjects
interfaceAiObjects.py] + CallHandler[call request
Handles failover & model calls] + AiObjects --> CallHandler + end + + subgraph "AI Core Layer" + Registry[ModelRegistry
discoverConnectors
registerConnector
getAvailableModels] + Selector[ModelSelector
selectModel
getFailoverModelList
scoring logic] + Base[BaseConnectorAi
getModels
getConnectorType
getCachedModels] + + Registry -.-> Selector + Selector -.-> Base + end + + subgraph "Plugin Connectors" + OpenAI[aicorePluginOpenai] + Anthropic[aicorePluginAnthropic] + Perplexity[aicorePluginPerplexity] + Tavily[aicorePluginTavily] + end + + subgraph "AI Provider APIs" + OpenAI_API[OpenAI API
api.openai.com] + Anthropic_API[Anthropic API
api.anthropic.com] + Perplexity_API[Perplexity API
api.perplexity.ai] + Tavily_API[Tavily API
api.tavily.com] + end + + Routes --> AiService + AiService --> AiObjects + AiObjects --> Registry + AiObjects --> Selector + + Base --> OpenAI + Base --> Anthropic + Base --> Perplexity + Base --> Tavily + + OpenAI --> OpenAI_API + Anthropic --> Anthropic_API + Perplexity --> Perplexity_API + Tavily --> Tavily_API + + style Routes fill:#e1f5ff + style AiService fill:#fff3e0 + style AiObjects fill:#f3e5f5 + style Registry fill:#e8f5e9 + style Selector fill:#e8f5e9 + style Base fill:#e8f5e9 + style OpenAI fill:#fff9c4 + style Anthropic fill:#fff9c4 + style Perplexity fill:#fff9c4 + style Tavily fill:#fff9c4 +``` + +### Component Structure + +The aicore module is organized into several key files: + +- **aicoreBase.py**: Defines the abstract base class that all AI connectors must inherit from, establishing the contract for plugin implementations +- **aicoreModelRegistry.py**: Manages the centralized registry of all available AI models across all connectors +- **aicoreModelSelector.py**: Implements the intelligent model selection algorithm based on multiple criteria +- **aicorePlugin*.py files**: Individual connector implementations for each AI provider (OpenAI, Anthropic, Perplexity, Tavily, and potentially internal systems) + +Each plugin file follows the naming convention `aicorePlugin.py`, which enables the automatic discovery mechanism to find and register them at startup without requiring manual configuration. + +### Core Components + +#### 1. **BaseConnectorAi** (`aicoreBase.py`) +The abstract base class that establishes the contract for all AI connector implementations. This class ensures that every AI provider plugin implements a consistent interface, making the system extensible and maintainable. + +**Core Responsibilities:** + +The base connector defines several essential methods that every plugin must implement: + +- **Model Discovery**: Each connector provides its list of available models through `getModels()`, which returns comprehensive metadata about each model including capabilities, costs, and performance characteristics +- **Connector Identification**: The `getConnectorType()` method returns a unique identifier string for the connector (such as "openai" or "anthropic"), used throughout the system for routing and logging +- **Cached Model Access**: The `getCachedModels()` method provides performance optimization by returning cached model metadata with automatic TTL (Time-To-Live) validation +- **Model Lookup**: Utility methods like `getModelByDisplayName()` enable quick retrieval of specific models by their unique identifiers +- **Cache Management**: The `clearCache()` method allows manual cache invalidation when model configurations need immediate refresh + +**Critical Design Principle - Unique Display Names:** + +The system enforces a strict uniqueness constraint on model display names across all connectors. While the `name` field (used for actual API calls) can be duplicated across different model instances (for example, "gpt-4o" might have multiple instances for different use cases), the `displayName` must be globally unique. This serves as the primary key in the model registry and prevents configuration conflicts. Examples of unique display names include "OpenAI GPT-4o", "OpenAI GPT-4o Instance Vision", and "Anthropic Claude 3 Opus". + +**Performance Optimization Through Caching:** + +To minimize unnecessary operations, the base connector implements a sophisticated caching mechanism with a 5-minute TTL. When `getCachedModels()` is called, the system checks if cached data exists and if the last update timestamp is within the 300-second window. If the cache is still valid, it returns the cached models immediately, avoiding the overhead of regenerating model metadata. If the cache has expired, it automatically refreshes by calling `getModels()` and updates both the cache and timestamp. This approach significantly reduces computational overhead during high-frequency operations while ensuring data freshness. + +#### 2. **ModelRegistry** (`aicoreModelRegistry.py`) +The centralized registry serves as the single source of truth for all available AI models in the system. It acts as a dynamic inventory management system, automatically discovering, validating, and organizing models from all registered connectors. + +**Automatic Plugin Discovery:** + +The registry implements a sophisticated auto-discovery mechanism that scans the aicore directory for any files matching the pattern `aicorePlugin*.py`. This pattern-based discovery enables zero-configuration extensibility - developers can add new AI providers simply by creating a properly named file, and the system automatically detects and integrates it during startup. The discovery process imports each plugin module, inspects its classes to find those inheriting from BaseConnectorAi, and instantiates them for registration. + +**Dynamic Registration and Validation:** + +When a connector is registered through `registerConnector()`, the registry performs critical validation steps. It calls the connector's `getCachedModels()` method to retrieve all available models, then validates that each model's `displayName` is unique across the entire registry. If a duplicate is detected, the registration fails with a detailed error message identifying both the existing and conflicting model configurations. This strict validation prevents configuration errors that could lead to unpredictable model selection behavior. + +**Intelligent Refresh Mechanism:** + +The registry maintains model freshness through a dual-refresh strategy. First, it implements automatic periodic refresh with a 5-minute interval - when any query method is called, the system checks if the last refresh timestamp exceeds this threshold and triggers an automatic update if needed. Second, it provides a `refreshModels()` method with a force parameter, allowing manual refresh operations that bypass the TTL check. This is particularly useful during development or when connector configurations change dynamically. + +**Comprehensive Query Interface:** + +The registry exposes a rich query interface for model retrieval: + +- **Direct Lookup**: `getModel(displayName)` provides O(1) access to specific models using their unique identifier +- **Complete Inventory**: `getModels()` returns the full catalog of registered models +- **Connector Filtering**: `getModelsByConnector(connectorType)` enables retrieval of all models from a specific provider +- **Availability Filtering**: `getAvailableModels()` returns only models currently marked as available, filtering out any disabled or problematic models +- **Reverse Lookup**: `getConnectorForModel(displayName)` retrieves the connector instance responsible for a specific model, enabling direct connector interaction +- **Statistical Analysis**: `getModelStats()` provides aggregate metrics including model counts by connector, capability, and priority + +**Singleton Pattern:** + +The registry is implemented as a global singleton instance (modelRegistry) that can be imported and used throughout the application, ensuring consistent model access and preventing duplicate registries. + +#### 3. **ModelSelector** (`aicoreModelSelector.py`) +The intelligent model selection engine implements a sophisticated scoring algorithm that evaluates available models against multiple criteria to determine the optimal choice for each AI operation. Rather than using hard-coded rules or simple priority lists, the selector employs a weighted scoring system that considers operation compatibility, resource constraints, and performance preferences to create a ranked failover list. + +**Selection Algorithm:** + +```mermaid +flowchart TD + Start[AI Call Request] --> GetModels[Get Available Models
from Registry] + GetModels --> OpFilter[Filter by Operation Type
MUST support requested operation] + OpFilter --> SizeFilter[Filter by Prompt Size
Prompt must fit within 80% of context] + SizeFilter --> Scoring[Calculate Score for Each Model] + + Scoring --> Score1[Operation Type Rating × 1000
PRIMARY sorting criteria] + Scoring --> Score2[Size Rating
How well prompt+context fits] + Scoring --> Score3[Processing Mode Rating
Compatibility score] + Scoring --> Score4[Priority Rating
Speed/Quality/Cost preference] + + Score1 --> Combine[Combine All Scores] + Score2 --> Combine + Score3 --> Combine + Score4 --> Combine + + Combine --> Sort[Sort by Total Score
Descending] + Sort --> Failover[Create Failover List] + Failover --> Return[Return Best Model
+ Fallback Models] + + style Start fill:#e1f5ff + style OpFilter fill:#fff3e0 + style SizeFilter fill:#fff3e0 + style Scoring fill:#f3e5f5 + style Sort fill:#e8f5e9 + style Return fill:#c8e6c9 +``` + +**Detailed Algorithm Process:** + +**Phase 1: Operation Type Filtering (Mandatory Constraint)** + +The first filtering phase is absolute - a model must explicitly support the requested operation type to be considered. Each model in the registry declares its supported operations through an `operationTypes` list, where each operation (such as PLAN, DATA_ANALYSE, DATA_GENERATE, IMAGE_ANALYSE) is associated with a performance rating from 1-10. Models lacking the required operation type are immediately excluded from consideration, regardless of their other characteristics. This ensures that specialized operations like image analysis are only routed to vision-capable models, and web search operations are directed to appropriate connectors. + +**Phase 2: Context Size Validation (Resource Constraint)** + +After operation filtering, the selector validates that each remaining model can physically accommodate the input. The system calculates the approximate token count for both the prompt and context (using a 4-byte-per-token approximation), then compares this against 80% of each model's declared context length. This 80% threshold provides a safety margin for message formatting overhead, system prompts, and output token reservation. Models with insufficient context capacity are filtered out, preventing runtime failures due to context length violations. For models with zero context length (indicating unlimited capacity), this check is bypassed. + +**Phase 3: Multi-Factor Scoring (Quality Assessment)** + +Each model that passes both mandatory filters receives a composite score calculated from four weighted components: + +- **Operation Type Rating (Primary Factor)**: Multiplied by 1000 to establish it as the dominant sorting criterion. A model rated 9/10 for DATA_ANALYSE will score 9000 points from this factor alone, while a model rated 7/10 scores only 7000. This massive weighting ensures that operation-specific optimization takes precedence over other factors. + +- **Size Efficiency Rating**: Measures how efficiently the model utilizes its context window. If the prompt+context fits comfortably (total size ≤ 80% of capacity), the rating equals (actual size / maximum allowed size), rewarding larger models for handling substantial content. If the content exceeds the limit (shouldn't happen after filtering, but serves as safety), the rating inverts to (maximum / actual), penalizing undersized models. + +- **Processing Mode Compatibility**: Evaluates alignment between the model's processing mode (BASIC, ADVANCED, DETAILED) and the requested mode. Perfect matches score 1.0, while compatible mismatches receive fractional scores (e.g., 0.5 for ADVANCED model handling BASIC request). This allows flexible matching while preferring mode-appropriate models. + +- **Priority Optimization**: Applies user preference for speed, quality, or cost efficiency. For SPEED priority, models with high `speedRating` values score better. For QUALITY, `qualityRating` dominates. For COST, the system inverts cost metrics to favor inexpensive models while adding weighted bonuses for speed and quality. BALANCED priority treats all factors equally. + +**Phase 4: Ranking and Failover List Generation** + +After scoring, models are sorted in descending order by their composite scores. The resulting list represents an optimal failover chain - the first model is the best match for the specific request, while subsequent models serve as progressively less optimal but still viable alternatives. This ranked list is returned for use by the call handler, which attempts models in order until one succeeds. + +**Primary Methods:** + +The selector exposes two main methods: `selectModel()` returns only the top-ranked model (index 0 of the failover list), while `getFailoverModelList()` returns the complete ranked list for failover handling. Both methods accept the same parameters: the prompt text, context data, AI call options, and the list of available models. + +**Global Singleton:** + +Like the registry, the selector is implemented as a global singleton (modelSelector) for consistent access throughout the application. + +#### 4. **Plugin Connectors** (`aicorePlugin*.py`) +Each plugin connector represents a concrete implementation of the BaseConnectorAi interface, tailored to a specific AI provider's API specifications and capabilities. These plugins serve as translation layers between the system's standardized interface and the provider-specific API requirements. + +**Architectural Pattern:** + +Each connector follows a consistent architectural pattern with four main components: + +**Initialization and Configuration:** +The constructor loads provider-specific configuration from the application's environment settings, including API keys, endpoint URLs, and any provider-specific parameters. It also initializes an HTTP client (typically using httpx) with appropriate timeouts, retry logic, and authentication headers. This separation of configuration from code enables easy deployment across different environments without code changes. + +**Connector Identification:** +The `getConnectorType()` method returns a simple string identifier for the connector, such as "openai", "anthropic", "perplexity", or "tavily". This identifier is used throughout the system for logging, routing, and model attribution. It must be unique across all connectors and is stored in every model's metadata. + +**Model Catalog Definition:** +The `getModels()` method returns a comprehensive list of AiModel instances, each representing a distinct AI model or model configuration. Each model entry includes: + +- **Identity**: Unique `displayName` (e.g., "OpenAI GPT-4o") and API `name` (e.g., "gpt-4o") +- **Technical Specifications**: Context window size in tokens, maximum output tokens, default temperature +- **Economic Metrics**: Cost per 1000 input tokens and output tokens, enabling accurate cost tracking +- **Performance Characteristics**: Speed rating (1-10) indicating response time, quality rating (1-10) for output quality +- **Operational Capabilities**: List of supported operation types with performance ratings for each +- **Execution Reference**: A callable reference (`functionCall`) pointing to the method that handles API communication +- **Strategic Attributes**: Priority classification (SPEED, QUALITY, COST, BALANCED) and processing mode (BASIC, ADVANCED, DETAILED) + +**API Communication Implementation:** + +Connectors implement one or more call methods (such as `callAiBasic()`, `callAiImage()`, or specialized methods) that handle the actual communication with the AI provider's API. These methods: + +- Accept standardized `AiModelCall` objects containing messages, model reference, and options +- Transform the standardized request format into the provider's specific API format (different providers use varying JSON schemas for requests) +- Execute HTTP requests with appropriate error handling, timeouts, and retry logic +- Parse provider-specific response formats back into standardized `AiModelResponse` objects +- Calculate actual costs based on token usage reported by the provider +- Handle provider-specific error codes and translate them into meaningful exceptions + +**Provider-Specific Adaptations:** + +Each connector adapts to its provider's unique characteristics: + +- **OpenAI Connectors**: Support both text completion and vision capabilities, handle rate limiting, manage model versioning +- **Anthropic Connectors**: Implement Claude-specific message formatting, handle thinking tokens, manage conversation context +- **Perplexity Connectors**: Integrate web search capabilities, handle citation extraction, manage search-enhanced responses +- **Tavily Connectors**: Implement web crawling protocols, handle structured data extraction, manage crawl depth and scope + +## Connection to serviceAi + +The `aicore` module is the **foundation layer** that `serviceAi` (AI Service) builds upon. Here's how they connect: + +### Integration Flow + +```mermaid +sequenceDiagram + participant App as Application
(app.py) + participant Service as Service Layer
(mainServiceAi.py) + participant Interface as Interface Layer
(interfaceAiObjects.py) + participant Core as AI Core
(aicore/) + participant Provider as AI Provider APIs + + App->>Service: HTTP Request + Service->>Interface: callAiDocuments/Planning + Interface->>Core: AiCallRequest + Core->>Core: Model Selection + Core->>Provider: API Call + Provider-->>Core: API Response + Core-->>Interface: AiCallResponse + Interface-->>Service: Processed Result + Service-->>App: HTTP Response +``` + +### Initialization Sequence + +```mermaid +sequenceDiagram + participant App as app.py + participant Lifecycle as featuresLifecycle + participant Service as AiService + participant AiObjects as AiObjects + participant Registry as ModelRegistry + participant Plugins as Plugin Connectors + + App->>Lifecycle: lifespan startup + Lifecycle->>Lifecycle: start() + Lifecycle->>Service: create AiService + Service->>AiObjects: AiObjects.create() + + AiObjects->>AiObjects: _discoverAndRegisterConnectors() + AiObjects->>Registry: discoverConnectors() + + Registry->>Registry: Scan aicore folder
for aicorePlugin*.py + Registry->>Plugins: Import & instantiate connectors + + loop For each discovered connector + AiObjects->>Registry: registerConnector(connector) + Registry->>Plugins: connector.getModels() + Plugins-->>Registry: List[AiModel] + Registry->>Registry: Validate displayName uniqueness + Registry->>Registry: Store models with displayName as key + end + + Registry-->>AiObjects: Registration complete + AiObjects-->>Service: Initialized with all models + Service-->>Lifecycle: AiService ready + Lifecycle-->>App: Startup complete + + Note over Registry: Models cached for 5 minutes
with auto-refresh +``` + +### Service-to-Core Communication + +The communication between the service layer and aicore follows a well-defined request-response pattern with multiple abstraction layers, each serving a specific purpose in the overall architecture. + +**High-Level Service Operations:** + +The AiService class (in `mainServiceAi.py`) provides domain-specific methods that application features and workflows can invoke. These methods abstract away the complexity of AI operations, presenting simple interfaces like `callAiPlanning()` for task planning and `callAiDocuments()` for document processing. + +When `callAiPlanning()` is invoked, it handles prompt construction by integrating placeholders and building a complete prompt string. It then creates an AiCallRequest object configured specifically for planning operations - with operation type set to PLAN, priority set to QUALITY (since planning requires accurate reasoning), and processing mode set to DETAILED (to ensure comprehensive analysis). This request is passed to `aiObjects.call()`, initiating the core AI processing chain. + +The `callAiDocuments()` method follows a similar pattern but with more flexibility - it accepts custom options, handles document attachments, and can process various output formats. It manages document extraction, prompt building with continuation contexts, and result formatting, while delegating the actual AI communication to the aicore layer. + +**Interface Layer Orchestration:** + +The AiObjects class (in `interfaceAiObjects.py`) serves as the orchestration layer, coordinating between the service layer's high-level requests and the aicore's model selection and execution capabilities. When its `call()` method receives an AiCallRequest, it follows a three-phase process: + +**Phase 1 - Model Selection:** +The interface queries the modelRegistry to retrieve all currently available models. It then invokes the modelSelector's `getFailoverModelList()` method, passing the request's prompt, context, and options. The selector returns a prioritized list of suitable models, ranked from most to least optimal for the specific request characteristics. + +**Phase 2 - Failover Execution:** +The interface iterates through the failover list, attempting each model in sequence. For each attempt, it calls the internal `_callWithModel()` method, which constructs a standardized AiModelCall object and invokes the model's `functionCall` reference. This reference points to the connector's API communication method, which executes the actual HTTP request to the AI provider. + +If the model call succeeds, the interface immediately returns the AiCallResponse to the service layer, completing the request. If an exception occurs (due to API errors, rate limits, or other issues), the interface logs the error with detailed context and proceeds to the next model in the failover list. + +**Phase 3 - Completion or Failure:** +If any model succeeds, the operation completes successfully. If all models in the failover list fail (a rare but possible scenario during API outages or configuration errors), the interface returns an AiCallResponse with an error message and error count, allowing the service layer to handle the failure gracefully. + +**Cross-Cutting Concerns:** + +Throughout this communication flow, several cross-cutting concerns are handled automatically: + +- **Metrics Collection**: Every AI call records timing, token usage, costs, and error counts for monitoring and optimization +- **Progress Tracking**: Long-running operations emit progress updates through callbacks for user feedback +- **Content Chunking**: Large content that exceeds model context limits is automatically chunked and processed in segments +- **Token Management**: The system calculates token usage estimates and reserves appropriate context space for prompts, system messages, and expected outputs + +### Key Integration Points + +1. **Model Selection**: `serviceAi` delegates to `modelSelector` for choosing the right model +2. **Failover Handling**: `AiObjects.call()` automatically tries multiple models if one fails +3. **Operation Types**: `serviceAi` defines operation types (PLAN, DATA_ANALYSE, etc.) that `aicore` uses for selection +4. **Standardized Interface**: All AI calls go through `AiCallRequest`/`AiCallResponse` regardless of provider + +## Connection to the Application + +### Application Flow + +```mermaid +sequenceDiagram + participant User + participant Route as FastAPI Route + participant Workflow as Workflow/Feature + participant Service as AiService + participant Objects as AiObjects + participant Registry as ModelRegistry + participant Selector as ModelSelector + participant Plugin as Plugin Connector + participant API as AI Provider API + + User->>Route: HTTP Request + Route->>Workflow: Call workflow + Workflow->>Service: callAiDocuments() + Service->>Objects: aiObjects.call(request) + Objects->>Registry: getAvailableModels() + Registry-->>Objects: List of models + Objects->>Selector: getFailoverModelList() + Selector-->>Objects: Sorted model list + + loop Try each model until success + Objects->>Plugin: model.functionCall() + Plugin->>API: HTTP Request + + alt Success + API-->>Plugin: Response + Plugin-->>Objects: AiModelResponse + Objects-->>Service: AiCallResponse + else Error + API-->>Plugin: Error + Plugin-->>Objects: Exception + Objects->>Objects: Try next model + end + end + + Service-->>Workflow: Result + Workflow-->>Route: Response + Route-->>User: HTTP Response + + Note over Objects,Plugin: Automatic failover
tries next best model +``` + +### Example: Chat Workflow + +**User Request**: "Analyze this document and extract key information" + +```mermaid +sequenceDiagram + participant User + participant Route as Route Handler
routeChatPlayground.py + participant Workflow as Workflow Layer + participant AiService as AiService
mainServiceAi.py + participant AiObjects as AiObjects
interfaceAiObjects.py + participant Registry as ModelRegistry + participant Selector as ModelSelector + participant Connector as aicorePluginOpenai.py + participant OpenAI as OpenAI API + + User->>Route: POST /chat/message
"Analyze document" + Route->>Workflow: featureWorkflow.run(request) + + Workflow->>AiService: callAiDocuments()
operationType=DATA_EXTRACT + Note over AiService: Build prompt with placeholders + + AiService->>AiObjects: aiObjects.call(request) + AiObjects->>Registry: getAvailableModels() + Registry-->>AiObjects: List of models + + AiObjects->>Selector: getFailoverModelList() + Note over Selector: Filter by DATA_EXTRACT
Score and sort models + Selector-->>AiObjects: [GPT-3.5, GPT-4, ...] + + AiObjects->>Connector: model.functionCall(AiModelCall) + Note over Connector: Format for OpenAI API + + Connector->>OpenAI: HTTP POST with messages + OpenAI-->>Connector: JSON response + + Connector-->>AiObjects: AiModelResponse + AiObjects-->>AiService: AiCallResponse + Note over AiService: Handle looping if needed + + AiService-->>Workflow: Extracted content + Workflow-->>Route: Result with documents + Route-->>User: HTTP 200 + JSON response + + Note over User,OpenAI: Full request/response cycle
with automatic failover +``` + +**Detailed Flow Breakdown:** + +**Step 1: HTTP Request Reception** +When a user sends a chat message through the frontend, it arrives as an HTTP POST request to the `/chat/message` endpoint defined in `routeChatPlayground.py`. The route handler receives a ChatMessageRequest containing the user's message, any attached documents, and conversation context. The handler immediately delegates to the workflow system by calling `featureWorkflow.run(request)`, which orchestrates the entire chat processing pipeline. + +**Step 2: Workflow Orchestration** +The workflow layer (living between routes and services) analyzes the user's request to determine the appropriate processing strategy. For a document analysis request, it identifies that document extraction is needed and invokes `serviceCenter.ai.callAiDocuments()`. This call includes the constructed prompt ("Extract key information from documents"), the attached chat documents, and explicitly configured options specifying DATA_EXTRACT as the operation type - signaling that this is an information extraction task rather than generation or analysis. + +**Step 3: Service Layer Processing** +The AiService receives the document processing request and performs several preparatory operations. It builds the complete prompt by replacing any placeholder markers with actual content (such as document titles, user context, or system instructions). It validates the documents and converts them into the appropriate format for AI processing. For lengthy responses that might span multiple AI generations, it sets up a looping mechanism that can handle continuation contexts. Finally, it creates an AiCallRequest object and passes it to `aiObjects.call()`, transitioning into the core AI layer. + +**Step 4: Intelligent Model Selection** +The AiObjects interface queries the modelRegistry to retrieve all currently available and healthy models. It then invokes the modelSelector with the full request context - passing the prompt text, any additional context, and the configured options. The selector executes its multi-phase filtering and scoring algorithm, ultimately returning a prioritized failover list. For a DATA_EXTRACT operation, this list typically starts with fast, cost-efficient models (like GPT-3.5 Turbo or Claude Haiku) since extraction doesn't require the highest reasoning capabilities. + +**Step 5: Model Execution with Failover** +AiObjects begins iterating through the failover list, attempting each model in sequence. For the first model (assume GPT-3.5 Turbo from OpenAI), it constructs an AiModelCall object containing the formatted messages and invokes the model's registered `functionCall`, which points to the OpenAI connector's API method. The connector transforms the standardized request into OpenAI's specific JSON format, adds authentication headers, and sends an HTTP POST request to `api.openai.com/v1/chat/completions`. + +If the OpenAI API responds successfully, the connector parses the JSON response, extracts the generated text, calculates costs based on reported token usage, and wraps everything in an AiModelResponse object. This response flows back through AiObjects, which converts it to an AiCallResponse and returns it to the service layer. + +If the API call fails (network timeout, rate limit, API error), the connector throws an exception. AiObjects catches this exception, logs detailed error information including the model name and error type, and immediately proceeds to the next model in the failover list. This process continues until either a model succeeds or the entire list is exhausted. + +**Step 6: Response Assembly and Delivery** +Once the AiService receives a successful AiCallResponse, it processes the content according to the request specifications. For document extraction, this might involve parsing structured JSON from the AI's response, validating the extracted data against expected schemas, and formatting it for frontend consumption. The processed result flows back up through the workflow layer, which adds any workflow-specific metadata (execution time, step logs), and finally reaches the route handler. The handler constructs an HTTP response with appropriate status codes and headers, delivering the extracted information back to the waiting frontend client. + +**Error Handling Throughout:** +At every step, comprehensive error handling ensures graceful degradation. If document processing fails, the workflow might retry with different parameters or return a helpful error message. If all AI models fail, the system returns a structured error response rather than crashing. Each failure point is logged with sufficient context for debugging and monitoring. + +### Configuration + +**Environment-Based Secrets Management:** + +The aicore system loads all sensitive configuration through the application's central `APP_CONFIG` system, which reads from environment files (env_dev.env, env_int.env, env_prod.env) based on the deployment environment. Each AI provider connector requires its API key stored under a standardized naming convention: `Connector_Ai_API_SECRET`. For example, the OpenAI connector looks for `Connector_AiOpenai_API_SECRET`, while Anthropic uses `Connector_AiAnthropic_API_SECRET`. This convention enables consistent configuration management across all providers and environments. + +Additional provider-specific settings follow similar naming patterns with descriptive suffixes. The SECRET suffix indicates that these values contain sensitive information and should never be committed to version control or exposed in logs. Configuration loading happens during connector initialization, allowing different API keys per environment without code changes. + +**Plugin-Level Model Configuration:** + +Each plugin file contains hard-coded model definitions specifying technical and economic characteristics. These configurations include: + +- **Capacity Parameters**: Context window sizes (in tokens) define maximum input lengths, while max token settings limit output generation length +- **Economic Metrics**: Input and output costs per 1000 tokens enable accurate cost tracking and budget management +- **Performance Characteristics**: Speed ratings (1-10 scale) indicate typical response time, while quality ratings reflect output sophistication and accuracy +- **Operational Capabilities**: Operation type ratings specify which tasks each model handles well, with ratings from 1-10 for supported operations +- **Strategic Classifications**: Priority tags (SPEED, QUALITY, COST, BALANCED) and processing mode designations (BASIC, ADVANCED, DETAILED) guide selection algorithms + +These plugin-level configurations represent the static characteristics of models and change only when model capabilities are updated or new models are added. They're versioned with the code rather than stored in environment variables, since they're not environment-specific or sensitive. + +## Key Features + +### 1. **Dynamic Plugin Architecture** + +```mermaid +graph LR + subgraph "Auto-Discovery Process" + Scan[Scan aicore folder
for aicorePlugin*.py] + Import[Import module dynamically] + Find[Find BaseConnectorAi
subclasses] + Instantiate[Instantiate connector] + Register[Register in ModelRegistry] + end + + subgraph "Plugin Files" + P1[aicorePluginOpenai.py] + P2[aicorePluginAnthropic.py] + P3[aicorePluginPerplexity.py] + P4[aicorePluginTavily.py] + P5[aicorePlugin*.py
Add new plugins here] + end + + Scan --> P1 + Scan --> P2 + Scan --> P3 + Scan --> P4 + Scan --> P5 + + P1 --> Import + P2 --> Import + P3 --> Import + P4 --> Import + P5 --> Import + + Import --> Find + Find --> Instantiate + Instantiate --> Register + + Register --> Models[All Models Available
in ModelRegistry] + + style Scan fill:#e1f5ff + style Register fill:#c8e6c9 + style P5 fill:#fff9c4 + style Models fill:#e8f5e9 +``` + +**Key Benefits:** +- New AI providers can be added by creating `aicorePlugin*.py` files +- No code changes needed in core logic +- Automatic discovery and registration + +### 2. **Intelligent Model Selection** + +The model selection engine goes far beyond simple rule-based routing by implementing a sophisticated multi-criteria decision system: + +**Holistic Evaluation:** +Rather than selecting models based on a single factor, the selector considers operation type compatibility (can this model handle planning vs. extraction?), resource constraints (will the prompt fit?), performance preferences (does the user prioritize speed or quality?), and cost implications. Each factor contributes to a weighted score that reflects the model's overall suitability. + +**Context-Aware Decisions:** +The selector analyzes not just what operation is requested, but also the size and complexity of the input. A simple data extraction from a small document might route to a fast, economical model like GPT-3.5 Turbo, while complex multi-document analysis with a large prompt routes to more capable models like GPT-4 or Claude Opus. This context-awareness optimizes the trade-off between cost and capability. + +**Ranked Failover Lists:** +Instead of returning a single "best" model, the selector produces a complete ranked list representing a spectrum from optimal to acceptable. This ranked list serves as a failover chain - if the top model fails due to rate limits or transient errors, the system immediately tries the second-ranked model without user intervention or workflow delays. This approach significantly improves system reliability and reduces user-facing errors. + +### 3. **Automatic Failover** + +```mermaid +flowchart TD + Start[AI Call Request] --> GetList[Get Failover Model List
Sorted by Score] + GetList --> Loop{Models
Available?} + + Loop -->|Yes| Try[Try Model #N] + Try --> Call[Call model.functionCall] + + Call --> Success{Success?} + Success -->|Yes| Return[Return Response] + Success -->|No| Log[Log Error with Details] + + Log --> More{More Models
in List?} + More -->|Yes| Next[Try Next Model] + Next --> Loop + More -->|No| Fail[All Models Failed] + + Loop -->|No| Error[Return Error Response] + Fail --> Error + Return --> End[Response to Caller] + Error --> End + + style Start fill:#e1f5ff + style Try fill:#fff3e0 + style Success fill:#f3e5f5 + style Return fill:#c8e6c9 + style Error fill:#ffcdd2 + style Next fill:#fff9c4 +``` + +**Key Benefits:** +- If primary model fails, automatically tries next best +- Logs each attempt with detailed error information +- Ensures high availability of AI operations +- No manual intervention required + +### 4. **Model Caching** + +```mermaid +stateDiagram-v2 + [*] --> Empty: System Start + Empty --> Loading: First Request + Loading --> Cached: getModels() called + Cached --> Valid: Check TTL + Valid --> Cached: TTL < 5 min + Valid --> Expired: TTL >= 5 min + Expired --> Loading: Refresh + Loading --> Cached: Cache Updated + Cached --> [*]: Return Models + + note right of Cached + Models cached for 5 minutes + Reduces API calls + Improves performance + end note + + note right of Loading + Calls connector.getModels() + Updates _last_cache_update + Stores in _models_cache + end note +``` + +**Key Benefits:** +- 5-minute TTL cache for model metadata +- Reduces repeated API calls +- Improves performance +- Manual cache clearing available via `clearCache()` + +### 5. **Unified Interface** + +One of the aicore system's most powerful design principles is its provider-agnostic abstraction layer: + +**Universal Request Format:** +Regardless of whether the eventual API call goes to OpenAI, Anthropic, Perplexity, or any other provider, the requesting code always uses the same AiCallRequest structure. This insulates application code from the complexity and variability of different provider APIs. Developers can write workflow logic once, and the system handles all provider-specific transformations transparently. + +**Standardized Response Structure:** +Every AI operation returns an AiCallResponse object with the same structure and semantics, whether it came from GPT-4, Claude, or a specialized search model. This consistency simplifies response handling code - no need for provider-specific parsing logic or conditional handling based on which model was used. + +**Consistent Error Semantics:** +Different AI providers report errors in vastly different formats - OpenAI uses different status codes and error structures than Anthropic, which differs from Perplexity. The aicore connectors translate all these provider-specific error formats into consistent error responses with standardized error counts and messages. This enables unified error handling logic throughout the application. + +**Normalized Metrics:** +Cost calculations, timing measurements, and token usage reporting follow the same format regardless of provider. This enables apples-to-apples comparisons of different models' performance and economics, facilitating data-driven decisions about model selection strategies. + +### 6. **Operation Type System** + +The operation type taxonomy provides semantic categorization of AI tasks, enabling intelligent routing and specialized model selection: + +**Task-Based Classification:** +Rather than selecting models based on generic "intelligence" levels, the system classifies each request by what it's trying to accomplish. This task-based approach recognizes that different models excel at different types of operations - a model optimized for rapid extraction might not be ideal for deep analytical reasoning, even if both are "capable" in an abstract sense. + +**Operation Type Catalog:** + +- **PLAN**: Strategic reasoning operations including task decomposition, action sequencing, and decision planning. These operations require strong logical reasoning and the ability to consider multiple factors simultaneously. Typically routed to high-capability models like GPT-4 or Claude Opus. + +- **DATA_ANALYSE**: Analytical operations that examine data to identify patterns, draw insights, or make assessments. Requires good comprehension and reasoning but not necessarily creative generation. Often uses balanced models that provide good analysis without premium costs. + +- **DATA_GENERATE**: Creative content generation including report writing, document creation, and structured output generation. Emphasizes coherent, well-structured output over analytical depth. Can often use mid-tier models effectively. + +- **DATA_EXTRACT**: Information extraction and parsing operations that pull structured data from unstructured sources. Speed and accuracy matter more than sophisticated reasoning. Frequently routed to fast, economical models like GPT-3.5 Turbo or Claude Haiku. + +- **IMAGE_ANALYSE**: Vision operations including image understanding, OCR, visual question answering, and scene description. Requires specialized vision-capable models with multimodal understanding. Automatically routes to GPT-4 Vision, Claude Vision, or similar models. + +- **IMAGE_GENERATE**: Image creation and generation operations. Routes to specialized generative models like DALL-E or Stable Diffusion connectors. + +- **WEB_SEARCH**: Real-time web search operations that query current information. Routes to search-specialized connectors like Perplexity that integrate web search APIs. + +- **WEB_CRAWL**: Web content extraction and crawling operations. Routes to specialized web crawling connectors like Tavily that handle website traversal and content extraction. + +**Performance Rating System:** +Each model declares not just which operations it supports, but how well it performs each operation on a 1-10 scale. A model might rate 9/10 for DATA_ANALYSE but only 6/10 for DATA_GENERATE, reflecting its strengths in analytical over creative tasks. These ratings form the primary sorting criterion in model selection, ensuring task-appropriate routing. + +### 7. **Content-Aware Chunking** + +When content exceeds a model's context capacity, the system employs sophisticated chunking strategies rather than simply failing: + +**Model-Specific Chunk Sizing:** +Chunking decisions are based on each model's specific capabilities rather than using universal chunk sizes. A model with a 128K token context window receives much larger chunks than one with a 16K limit. The system calculates optimal chunk sizes by considering the model's total context length, subtracting reserved space for prompts and system messages, and applying a safety margin (typically 70-80% utilization). + +**Comprehensive Token Accounting:** +Naive chunking might only consider content size, but the aicore system accounts for all token consumers: the user prompt (which repeats with each chunk), system message overhead (message formatting and instructions), output token reservation (space the model needs for its response), and protocol overhead (JSON structure and metadata). This comprehensive accounting prevents context overflow errors during generation. + +**Intelligent Result Merging:** +After processing multiple chunks, their results must be intelligently combined. Simple concatenation can produce disjointed or redundant output. The system employs content-type-aware merging strategies - text chunks are merged with appropriate spacing and deduplication, structured data is merged while preserving relationships, and vision results are aggregated with context preservation. The merging system maintains coherence across chunk boundaries, producing results that read as unified responses rather than fragmented pieces. + +**Progressive Processing:** +For very large documents, chunking enables progressive processing where each chunk can be processed as soon as it's prepared, rather than waiting for the entire document. This streaming approach reduces perceived latency and enables progress reporting to users, showing incremental completion rather than a black box wait. + +## Data Models + +### Core Data Models (`datamodelAi.py`) + +```mermaid +classDiagram + class AiModel { + +string name + +string displayName + +string connectorType + +string apiUrl + +float temperature + +int maxTokens + +int contextLength + +float costPer1kTokensInput + +float costPer1kTokensOutput + +int speedRating + +int qualityRating + +callable functionCall + +PriorityEnum priority + +ProcessingModeEnum processingMode + +List~OperationTypeRating~ operationTypes + +string version + +callable calculatePriceUsd + } + + class AiCallRequest { + +string prompt + +string context + +AiCallOptions options + +List~ContentPart~ contentParts + } + + class AiCallOptions { + +OperationTypeEnum operationType + +PriorityEnum priority + +ProcessingModeEnum processingMode + +bool compressPrompt + +bool compressContext + } + + class AiCallResponse { + +string content + +string modelName + +float priceUsd + +float processingTime + +int bytesSent + +int bytesReceived + +int errorCount + } + + class OperationTypeEnum { + <> + PLAN + DATA_ANALYSE + DATA_GENERATE + DATA_EXTRACT + IMAGE_ANALYSE + IMAGE_GENERATE + WEB_SEARCH + WEB_CRAWL + } + + class PriorityEnum { + <> + BALANCED + SPEED + QUALITY + COST + } + + class ProcessingModeEnum { + <> + BASIC + ADVANCED + DETAILED + } + + AiCallRequest --> AiCallOptions + AiCallOptions --> OperationTypeEnum + AiCallOptions --> PriorityEnum + AiCallOptions --> ProcessingModeEnum + AiModel --> PriorityEnum + AiModel --> ProcessingModeEnum + + note for AiModel "Unique displayName required\nacross all connectors" + note for AiCallRequest "Input to AI system" + note for AiCallResponse "Output from AI system" +``` + +**Core Data Model Descriptions:** + +**AiModel:** Represents a complete model configuration with all metadata required for selection, execution, and cost tracking. The `name` field contains the API-level identifier used in actual provider calls, while `displayName` serves as the globally unique identifier within the registry. Technical specifications like `contextLength` (maximum input tokens) and `maxTokens` (maximum output tokens) inform chunking and validation logic. Economic fields (`costPer1kTokensInput`, `costPer1kTokensOutput`) enable precise cost tracking across all operations. Performance metrics (`speedRating`, `qualityRating`) influence selection algorithms. The `functionCall` field holds a callable reference to the connector method that executes API communication. The `operationTypes` list defines which operation types this model supports and how well it performs each, using ratings from 1-10. + +**AiCallRequest:** Encapsulates all information needed to execute an AI operation. The `prompt` contains the primary instruction or question, while optional `context` provides supporting information. The `options` object configures operation behavior including type, priority, and processing mode. For multi-modal requests (like vision operations), the `contentParts` list can contain multiple pieces of content with different MIME types. + +**AiCallOptions:** Configures how an AI operation should be executed. The `operationType` determines what kind of operation this is (planning, analysis, generation, etc.), which drives model selection. The `priority` indicates whether to optimize for speed, quality, cost, or balance. The `processingMode` suggests the depth of processing required (basic for simple tasks, detailed for complex reasoning). Boolean flags like `compressPrompt` and `compressContext` control whether the system should attempt content compression to fit context limits. + +**AiCallResponse:** Contains the complete result of an AI operation including the generated `content`, the `modelName` that produced it, and comprehensive metrics. Cost tracking is provided via `priceUsd`, calculated based on actual token usage reported by the provider. Performance metrics include `processingTime` (wall-clock time for the operation), `bytesSent` and `bytesReceived` (for network monitoring), and `errorCount` (zero for success, greater than zero indicating partial or complete failure). + +## Best Practices + +### Adding a New AI Provider + +The plugin architecture makes adding new AI providers straightforward through a four-step process: + +**Step 1: Create the Plugin File** + +Create a new file in the `modules/aicore` directory following the naming convention `aicorePlugin.py`, where `` is a descriptive name for the AI service (e.g., `aicorePluginCohere` for Cohere AI). The filename itself triggers automatic discovery - the system scans for any file matching the `aicorePlugin*.py` pattern during initialization. + +**Step 2: Implement the Connector Class** + +Within your plugin file, create a class that inherits from BaseConnectorAi. This class must implement several required methods: + +**Connector Identification:** +The `getConnectorType()` method returns a simple string identifier (lowercase, no spaces) that uniquely identifies this connector throughout the system. This identifier appears in logs, model metadata, and routing decisions. + +**Model Catalog Definition:** +The `getModels()` method returns a list of AiModel instances, one for each model configuration you want to expose. Each AiModel requires comprehensive metadata including: +- A unique displayName that differs from all other models in the system (e.g., "Cohere Command-R Plus") +- The API model name used in actual API calls +- Technical specifications (context length, max output tokens, temperature) +- Economic data (input and output costs per 1000 tokens) +- Performance ratings (speed and quality on 1-10 scales) +- Operational capabilities defined via `createOperationTypeRatings()`, specifying which operation types the model supports and how well (rating 1-10 for each) +- A reference to the callable method that handles API communication (typically a method on your connector class) + +**API Communication Method:** +Implement one or more async methods (like `callAi()`) that accept an AiModelCall object and return an AiModelResponse. This method handles the actual HTTP communication with your provider's API. It must: +- Extract messages from the AiModelCall +- Transform them into the provider's expected JSON format +- Execute the HTTP request with proper authentication and error handling +- Parse the provider's response format +- Extract the generated text and any usage statistics +- Calculate costs based on token usage +- Return everything wrapped in an AiModelResponse object + +**Step 3: Configure Environment Variables** + +Add the necessary configuration to your environment files (env_dev.env, env_int.env, env_prod.env). At minimum, this includes the API key for authentication, but might also include endpoint URLs, organization IDs, or other provider-specific settings. Use descriptive configuration key names following the convention `Connector_Ai__SECRET` for sensitive values. + +**Step 4: Automatic Integration** + +No manual registration or configuration code changes are required. When the application next starts, the modelRegistry's discovery mechanism automatically: +- Scans the aicore directory +- Finds your new plugin file +- Imports the module +- Instantiates your connector class +- Calls getModels() to retrieve available models +- Validates displayName uniqueness +- Registers all models in the global registry + +Your new AI provider is now fully integrated and will participate in model selection for appropriate operation types. The system logs will show discovery and registration messages confirming successful integration. + +### Model Selection Guidelines + +- **PLAN operations**: Use high-quality models (GPT-4, Claude 3 Opus) +- **DATA_GENERATE**: Balanced models for quality/cost trade-off +- **DATA_EXTRACT**: Speed-optimized models for bulk processing +- **IMAGE_ANALYSE**: Vision-capable models only +- **WEB_SEARCH**: Specialized search connectors (Perplexity, Tavily) + +### Error Handling Philosophy + +The aicore system implements a comprehensive error handling strategy designed for resilience and observability: + +**Automatic Failover:** +When you invoke `aiObjects.call()` with a request, the system automatically attempts multiple models from the failover list until one succeeds. Each failure is logged with detailed context (model name, error type, error message) but doesn't interrupt the execution flow. Only if all models in the failover list fail does the method return an error response. + +**Graceful Degradation:** +Rather than throwing exceptions that crash workflows, the system returns AiCallResponse objects even in failure scenarios. These error responses have `errorCount` greater than zero and contain descriptive error messages in the `content` field. This allows calling code to inspect the errorCount property and decide how to handle partial failures - whether to retry with different parameters, fall back to alternative processing paths, or present user-friendly error messages. + +**Comprehensive Logging:** +Every error is logged with sufficient context for debugging: the attempted model's displayName, the operation type, the error type (network timeout, API error, rate limit, etc.), and the full error message. This creates an audit trail for troubleshooting production issues without requiring verbose debug logging during normal operations. + +**Error Classification:** +The system distinguishes between transient errors (network timeouts, temporary API issues) that warrant trying another model, and permanent errors (authentication failures, malformed requests) that indicate configuration problems requiring immediate attention. Transient errors trigger failover silently, while permanent errors are logged at higher severity levels. + +## Performance Considerations + +### Caching +- Model registry caches for 5 minutes +- Connector models cached individually +- Reduces discovery overhead + +### Failover Strategy +- Models sorted by score (best first) +- Failed models logged with detailed errors +- Next best model tried automatically + +### Chunking +- Large content automatically chunked based on model limits +- Conservative 70-80% utilization for safety +- Intelligent merging of chunk results + +### Cost Optimization +- Model selector considers cost ratings +- Price calculated per call for tracking +- Can prioritize by cost with `PriorityEnum.COST` + +## Troubleshooting + +### Common Issues + +1. **"No models available"** + - Check API keys in environment configuration + - Verify connector plugins exist in `aicore/` folder + - Check logs for connector initialization errors + +2. **"No suitable model found"** + - Check if operation type is supported by any model + - Verify prompt size isn't too large for all models + - Review model filtering criteria in logs + +3. **"All models failed"** + - Check API connectivity and keys + - Review model-specific error messages in logs + - Verify request format is correct + +4. **"Duplicate displayName"** + - Each model must have unique `displayName` + - Check all plugin files for name conflicts + - Naming convention: ` ` + +## Future Enhancements + +- **Streaming Support**: Real-time response streaming for chat interfaces +- **Model Health Monitoring**: Track success rates and performance metrics +- **Cost Budgets**: Automatic model selection based on budget constraints +- **Custom Scoring**: User-defined scoring functions for model selection +- **A/B Testing**: Compare different models for the same operation +- **Rate Limiting**: Built-in rate limit handling per provider + +## Quick Reference + +### Common Usage Patterns + +**1. Making AI Calls:** + +There are two primary approaches for invoking AI operations in the system: + +**Via AiService (Recommended Approach):** +The recommended pattern uses the high-level service methods like `callAiPlanning()`, `callAiDocuments()`, or `callAiText()`. These methods are accessed through the serviceCenter and handle all complexity internally. For planning operations, you call `serviceCenter.ai.callAiPlanning()` with a prompt string and optional placeholder list. Placeholders allow dynamic content injection - the system replaces markers like `{TASK}` with actual content before sending to the AI. This approach provides automatic prompt building, placeholder resolution, and response formatting. + +**Direct via AiObjects (Advanced Use):** +For specialized scenarios requiring fine-grained control, you can construct an AiCallRequest manually and invoke `aiObjects.call()` directly. This requires creating an AiCallOptions object with explicit operation type and priority settings, then awaiting the call. The response object contains the generated content plus metrics like token usage, processing time, and costs. This approach is typically used within service implementations or for custom AI workflows. + +**2. Querying Available Models:** + +The modelRegistry provides comprehensive model inventory access: + +**Complete Inventory Access:** +Calling `modelRegistry.getAvailableModels()` returns all currently available and healthy models across all registered connectors. This list automatically excludes any models marked as unavailable due to configuration issues or connector errors. + +**Connector-Specific Filtering:** +Use `modelRegistry.getModelsByConnector("openai")` to retrieve only models from a specific provider. This is useful when implementing provider-specific features or debugging connector issues. Pass the connector type string (openai, anthropic, perplexity, tavily) as the parameter. + +**Direct Model Lookup:** +For retrieving a specific model's full metadata, use `modelRegistry.getModel("OpenAI GPT-4o")` with the exact displayName. This returns the complete AiModel object including capabilities, costs, ratings, and the functionCall reference. + +**Statistical Overview:** +The `modelRegistry.getModelStats()` method provides aggregate statistics including total model count, availability counts, breakdowns by connector type, capability distribution, and priority classifications. This is valuable for monitoring system health and model distribution. + +**3. Understanding Model Selection:** + +To understand how the system selects models for specific requests: + +**Generating Failover Lists:** +Invoke `modelSelector.getFailoverModelList()` with your prompt, context, options, and the list of available models. The selector executes its full filtering and scoring algorithm, returning a ranked list ordered from most to least suitable. The first element represents the optimal choice, while subsequent elements serve as fallback options. + +**Analyzing Selection Results:** +Each model in the failover list has been validated for operation type compatibility and context size constraints. Their ordering reflects the composite score from operation ratings, size efficiency, processing mode alignment, and priority preferences. Examining this list helps understand why specific models were chosen or excluded for particular operations. + +### Operation Types Reference + +| Operation Type | Description | Best Models | Use Case | +|---------------|-------------|-------------|----------| +| `PLAN` | Task planning, action selection | GPT-4, Claude Opus | Workflow planning, decision making | +| `DATA_ANALYSE` | Data analysis and insights | GPT-4, Claude Sonnet | Document analysis, pattern detection | +| `DATA_GENERATE` | Content generation | GPT-4, Claude Sonnet | Report creation, document generation | +| `DATA_EXTRACT` | Information extraction | GPT-3.5, Claude Haiku | Text extraction, data parsing | +| `IMAGE_ANALYSE` | Image/vision analysis | GPT-4 Vision, Claude Vision | Image understanding, OCR | +| `IMAGE_GENERATE` | Image generation | DALL-E, Stable Diffusion | Image creation | +| `WEB_SEARCH` | Web search operations | Perplexity | Real-time web search | +| `WEB_CRAWL` | Web crawling | Tavily | Website content extraction | + +### Priority Reference + +| Priority | Description | Selection Behavior | +|----------|-------------|-------------------| +| `BALANCED` | Balance speed, quality, cost | Default selection | +| `SPEED` | Prioritize fast response | Favor high speedRating models | +| `QUALITY` | Prioritize high-quality output | Favor high qualityRating models | +| `COST` | Prioritize low cost | Favor low-cost models | + +### Processing Mode Reference + +| Mode | Description | When to Use | +|------|-------------|-------------| +| `BASIC` | Simple, straightforward processing | Quick tasks, simple questions | +| `ADVANCED` | Complex reasoning required | Multi-step tasks, analysis | +| `DETAILED` | Comprehensive, thorough output | Planning, detailed generation | + +### Module Import Structure + +The aicore system is organized across several module paths for clean separation of concerns: + +**Core Infrastructure Components:** +- The base connector interface lives at `modules.aicore.aicoreBase` and exports BaseConnectorAi +- The global model registry singleton is imported from `modules.aicore.aicoreModelRegistry` as modelRegistry +- The global model selector singleton is imported from `modules.aicore.aicoreModelSelector` as modelSelector + +**Data Model Definitions:** +All AI-related data models are centralized in `modules.datamodels.datamodelAi`, including: +- AiModel: Complete model metadata and configuration +- AiCallRequest and AiCallResponse: Request/response wrapper objects +- AiCallOptions: Configuration options for AI operations +- OperationTypeEnum, PriorityEnum, ProcessingModeEnum: Enumeration types for operation classification + +**Interface and Service Layers:** +- The AiObjects interface class is available at `modules.interfaces.interfaceAiObjects` +- The high-level AiService class is located at `modules.services.serviceAi.mainServiceAi` + +Most application code interacts with the service layer rather than importing core components directly, maintaining proper architectural separation. + +## Summary + +The `aicore` module is the **backbone of AI operations** in the application, providing: +- **Abstraction**: Single interface for multiple AI providers +- **Intelligence**: Smart model selection and automatic failover +- **Flexibility**: Plugin architecture for easy provider addition +- **Reliability**: Caching, failover, and error handling +- **Performance**: Context-aware chunking and optimization + +It connects to `serviceAi` as the **foundation layer**, enabling high-level AI services to operate without knowledge of specific AI provider implementations. The entire system integrates seamlessly into the application through the service layer architecture. + +--- + +**Related Documentation:** +- [Services API Reference](./services-api-reference.md) +- [Architecture Overview](./architecture-overview.md) +- [Security Component](./security-component.md) + diff --git a/docs/code-documentation/architecture-overview.md b/docs/code-documentation/architecture-overview.md new file mode 100644 index 00000000..5d3495c8 --- /dev/null +++ b/docs/code-documentation/architecture-overview.md @@ -0,0 +1,209 @@ +# Architecture Overview + +High-level architecture diagram of the Gateway project. + +```mermaid +graph TB + %% Entry Point + App[app.py
FastAPI Application] + + %% Middleware Layer + App --> Security[Security
Auth, CSRF, JWT, Token Refresh] + App --> CORS[CORS Middleware] + + %% API Layer + App --> Routes[Routes
API Endpoints] + + %% Business Logic Layer + Routes --> Features[Features
Business Logic Modules] + Routes --> Services[Services
Service Layer] + + %% Features can use Services + Features --> Services + + %% Data Access Layer + Services --> Interfaces[Interfaces
Data Access Layer] + Features --> Interfaces + + %% External Connections + Interfaces --> Connectors[Connectors
External System Connections] + Interfaces --> Database[(Database
PostgreSQL)] + + %% Connectors connect to external systems + Connectors --> Database + Connectors --> External[External Systems
Jira, ClickUp, Google, etc.] + + %% Shared Resources + App -.-> Shared[Shared Modules
Configuration, Logging, Utils] + Routes -.-> Shared + Features -.-> Shared + Services -.-> Shared + Interfaces -.-> Shared + + %% Data Models used throughout + Routes -.-> DataModels[Data Models
Request/Response Schemas] + Features -.-> DataModels + Services -.-> DataModels + Interfaces -.-> DataModels + + %% Feature Lifecycle Management + App --> FeaturesLifecycle[Features Lifecycle
Startup/Shutdown Management] + FeaturesLifecycle --> Features + + %% Styling + classDef entryPoint fill:#e1f5ff,stroke:#01579b,stroke-width:3px + classDef apiLayer fill:#f3e5f5,stroke:#4a148c,stroke-width:2px + classDef businessLogic fill:#e8f5e9,stroke:#1b5e20,stroke-width:2px + classDef dataAccess fill:#fff3e0,stroke:#e65100,stroke-width:2px + classDef external fill:#fce4ec,stroke:#880e4f,stroke-width:2px + classDef shared fill:#f5f5f5,stroke:#424242,stroke-width:1px,stroke-dasharray: 5 5 + + class App entryPoint + class Routes,Security,CORS apiLayer + class Features,Services businessLogic + class Interfaces,Connectors dataAccess + class Database,External external + class Shared,DataModels,FeaturesLifecycle shared +``` + +## Data Flow Diagram + +The following sequence diagram shows how data flows through the same architectural layers from the architecture diagram above. + +```mermaid +sequenceDiagram + participant Client + participant App as app.py
FastAPI Application + participant Security as Security
Auth, CSRF, JWT + participant Routes as Routes
API Endpoints + participant Features as Features
Business Logic + participant Services as Services
Service Layer + participant Interfaces as Interfaces
Data Access Layer + participant Connectors as Connectors
External Connections + participant Database as Database
PostgreSQL + participant External as External Systems
Jira, ClickUp, etc. + + %% Request Flow + Client->>App: HTTP Request + App->>Security: Validate Auth & CSRF + Security-->>App: Authenticated + App->>Routes: Validated Request + Routes->>Routes: Validate Data Models + + alt Route delegates to Features + Routes->>Features: Delegate Request + Features->>Services: Use Service (optional) + Services-->>Features: Service Result + Features->>Interfaces: Request Data Access + else Route delegates to Services + Routes->>Services: Delegate Request + Services->>Interfaces: Request Data Access + end + + Interfaces->>Connectors: Query Request + + alt Database Query + Connectors->>Database: Execute Query + Database-->>Connectors: Raw Data + else External API Call + Connectors->>External: API Call + External-->>Connectors: API Response + end + + %% Response Flow + Connectors-->>Interfaces: Raw Data + Interfaces->>Interfaces: Transform to Domain Objects + Interfaces-->>Features: Domain Objects + Interfaces-->>Services: Domain Objects + + Features->>Features: Process Business Logic + Services->>Services: Process Service Logic + + Features-->>Routes: Processed Data + Services-->>Routes: Processed Data + + Routes->>Routes: Serialize to Response Models + Routes-->>App: HTTP Response + App-->>Client: HTTP Response +``` + +### Request Flow (Top to Bottom) + +1. **Client** sends HTTP request to **app.py** +2. **app.py** forwards to **Security** middleware for authentication and CSRF validation +3. **Security** validates and returns authenticated request to **app.py** +4. **app.py** forwards validated request to **Routes** +5. **Routes** validate request data using Data Models, then delegate to either: + - **Features** (which may use **Services**), or + - **Services** directly +6. **Features/Services** call **Interfaces** for data access +7. **Interfaces** use **Connectors** to execute queries +8. **Connectors** query **Database** or call **External Systems** + +### Response Flow (Bottom to Top) + +1. **Database/External Systems** return raw data to **Connectors** +2. **Connectors** pass raw data to **Interfaces** +3. **Interfaces** transform raw data into domain objects +4. **Interfaces** return domain objects to **Features/Services** +5. **Features/Services** process business logic and return processed data to **Routes** +6. **Routes** serialize data to response models and return HTTP response to **app.py** +7. **app.py** returns HTTP response to **Client** + +### Data Transformations + +- **Routes**: HTTP Request ↔ Validated Data Models (Pydantic) +- **Features/Services**: Data Models ↔ Domain Objects (Business Logic Processing) +- **Interfaces**: Domain Objects ↔ Raw Data (SQL/API Format) +- **Connectors**: Raw Data ↔ Database Queries/API Calls + +## Layer Descriptions + +### Entry Point Layer +**app.py** - The FastAPI application entry point that orchestrates the entire system. It initializes logging, configures CORS and security middleware, registers all route routers, and manages the application lifecycle (startup/shutdown). This is where the application server starts and all components are wired together. + +### API Layer +**Routes** - HTTP endpoints that define the REST API surface. Routes receive client requests, validate input using data models, delegate to features or services for business logic, and return structured responses. Each route module handles a specific domain (e.g., Real Estate, Chat, Workflows, Security). + +**Security** - Middleware and services that handle authentication, authorization, CSRF protection, JWT token management, and token refresh. Ensures all requests are properly authenticated and authorized before reaching business logic. See [Security Component Documentation](./security-component.md) for detailed documentation. + +**CORS** - Cross-Origin Resource Sharing middleware that controls which external domains can access the API, enabling secure cross-origin requests from web applications. + +### Business Logic Layer +**Features** - Domain-specific business logic modules that implement core functionality for specific use cases (e.g., Real Estate management, Chat workflows, Data neutralization). Features are stateless and orchestrate services to fulfill business requirements. They can be called directly from routes or managed by the Features Lifecycle for background processing. + +**Services** - Reusable, composable service components that provide cross-cutting functionality (AI processing, document extraction, content generation, chat operations, ticket management, etc.). Services encapsulate complex operations and can be used by multiple features. They typically use interfaces to access data and may call other services. + +### Data Access Layer +**Interfaces** - Abstraction layer that provides a clean, domain-oriented API for accessing data. Interfaces hide the complexity of database connections and external system integrations, offering high-level methods for CRUD operations. They handle user context, access control, and data transformation between the application and persistence layers. + +**Connectors** - Concrete implementations that handle low-level communication with external systems. Database connectors manage PostgreSQL connections, query execution, and transaction handling. External connectors integrate with third-party services (Jira, ClickUp, Google Voice, SharePoint) using their specific APIs and protocols. + +### External Systems Layer +**Database** - PostgreSQL databases that persist application data. Multiple databases may exist for different domains (e.g., chat data, real estate data, management data). Connectors handle all database interactions. + +**External Systems** - Third-party services and APIs that the application integrates with. These include ticketing systems (Jira, ClickUp), cloud services (Google Voice, SharePoint), and other external platforms. Connectors abstract away the specifics of each integration. + +### Shared Resources Layer +**Shared Modules** - Common utilities and infrastructure used throughout the application. Includes configuration management, logging utilities, time/date helpers, JSON processing, attribute utilities, and audit logging. These modules provide cross-cutting concerns that don't belong to any specific domain. + +**Data Models** - Pydantic models that define data structures for requests, responses, and database entities. They provide validation, serialization, and type safety across all layers. Models are organized by domain (e.g., Real Estate, Chat, Security, AI). + +**Features Lifecycle** - Manages the startup and shutdown of features that require background processing, scheduled tasks, or event-driven operations. Coordinates initialization and cleanup of features that need persistent processes or event listeners. + +## Request/Response Flow Summary + +For a detailed visual representation, see the [Data Flow Diagram](#data-flow-diagram) above. + +**Simplified Request Flow**: `Client Request` → `CORS` → `Security (Auth/CSRF)` → `Routes` → `Features/Services` → `Interfaces` → `Connectors` → `Database/External Systems` + +**Simplified Response Flow**: `Database/External Systems` → `Connectors` → `Interfaces` → `Features/Services` → `Routes` → `Transform & Log` → `Client Response` + +## Key Architectural Patterns + +- **Layered Architecture**: Clear separation between API, business logic, and data access layers +- **Dependency Injection**: Services and interfaces are injected where needed +- **Interface Abstraction**: Interfaces abstract away database and connector details +- **Stateless Design**: Features operate statelessly without session management +- **Shared Utilities**: Common functionality centralized in shared modules + diff --git a/docs/code-documentation/connectors-component.md b/docs/code-documentation/connectors-component.md new file mode 100644 index 00000000..252fabfd --- /dev/null +++ b/docs/code-documentation/connectors-component.md @@ -0,0 +1,1241 @@ +# Connectors Component Documentation + +## Table of Contents + +1. [Overview](#overview) +2. [Architecture](#architecture) +3. [Database Connectors](#database-connectors) +4. [Voice Connector](#voice-connector) +5. [Ticket Connectors](#ticket-connectors) +6. [Integration Patterns](#integration-patterns) +7. [Configuration](#configuration) +8. [Design Principles](#design-principles) + +--- + +## Overview + +The Connectors component provides abstraction layers for external systems and data storage mechanisms. It acts as the bridge between the application's business logic and external services, databases, and third-party APIs. This component implements the **Adapter Pattern** to provide consistent interfaces regardless of the underlying technology. + +### Purpose and Scope + +The connectors component serves three primary functions: + +1. **Data Persistence** - Abstracts database operations for both JSON file-based and PostgreSQL storage +2. **Voice Processing** - Integrates Google Cloud Speech services for voice recognition, translation, and synthesis +3. **Ticket Management** - Connects to external ticketing systems (JIRA, ClickUp) for synchronization + +### Component Structure + +``` +modules/connectors/ +├── connectorDbJson.py # JSON file-based database +├── connectorDbPostgre.py # PostgreSQL database +├── connectorVoiceGoogle.py # Google Cloud Speech services +├── connectorTicketsJira.py # JIRA integration +└── connectorTicketsClickup.py # ClickUp integration +``` + +--- + +## Architecture + +### Connector Hierarchy + +```mermaid +graph TD + A[Application Layer
routes/, workflows/, features/] --> B[Interface Layer
modules/interfaces/] + B --> C[Connector Layer
modules/connectors/] + C --> D[External Systems] + + B --> B1[interfaceDbAppObjects.py
AppObjects] + B --> B2[interfaceDbChatObjects.py
ChatObjects] + B --> B3[interfaceVoiceObjects.py
VoiceObjects] + B --> B4[interfaceTicketObjects.py
TicketInterface] + + C --> C1[connectorDbJson.py
DatabaseConnector] + C --> C2[connectorDbPostgre.py
DatabaseConnector] + C --> C3[connectorVoiceGoogle.py
ConnectorGoogleSpeech] + C --> C4[connectorTicketsJira.py
ConnectorTicketJira] + C --> C5[connectorTicketsClickup.py
ConnectorTicketClickup] + + B1 --> C1 + B1 --> C2 + B2 --> C1 + B2 --> C2 + B3 --> C3 + B4 --> C4 + B4 --> C5 + + C1 --> D1[JSON Files] + C2 --> D2[PostgreSQL] + C3 --> D3[Google Cloud APIs] + C4 --> D4[JIRA API] + C5 --> D5[ClickUp API] + + style A fill:#e1f5ff + style B fill:#fff9e6 + style C fill:#e8f5e9 + style D fill:#fce4ec +``` + +### Layered Architecture Pattern + +The connectors follow a three-tier architecture: + +1. **Application Layer**: Business logic, workflows, services +2. **Interface Layer**: Domain-specific abstractions (AppObjects, ChatObjects, etc.) +3. **Connector Layer**: Technology-specific implementations +4. **External Systems**: Databases, APIs, cloud services + +This separation ensures: +- **Loose Coupling**: Application code doesn't depend on specific technologies +- **Testability**: Connectors can be mocked or swapped +- **Flexibility**: Easy migration between storage backends or service providers +- **Maintainability**: Changes to external systems are isolated to connector layer + +--- + +## Database Connectors + +### Overview + +The application supports two database connector implementations that provide identical public APIs but different storage mechanisms. This allows deployment flexibility without code changes. + +### DatabaseConnector Interface + +Both database connectors implement a common interface using duck typing (no formal interface class). They provide: + +- **CRUD Operations**: Create, read, update, delete records +- **Schema Management**: Dynamic table creation from Pydantic models +- **Context Management**: User-aware operations for audit trails +- **Concurrency Control**: Thread-safe operations with locking mechanisms +- **Initial Record Tracking**: System table for bootstrap data + +### JSON Database Connector + +#### Purpose and Use Cases + +The JSON connector is ideal for: +- **Development Environments**: Fast setup without database infrastructure +- **Small Deployments**: Low-volume applications +- **Portable Data**: Easy backup and version control +- **Testing**: Simplified test data management + +#### Storage Structure + +```mermaid +graph LR + A[Database Host Directory] --> B[Database Name Directory] + B --> C[Table1 Directory] + B --> D[Table2 Directory] + B --> E[_system.json] + + C --> C1[record1.json] + C --> C2[record2.json] + C --> C3[_metadata.json] + + D --> D1[record3.json] + D --> D2[record4.json] + + style E fill:#ffeb3b + style C3 fill:#ffeb3b +``` + +**File System Layout:** +- Each database is a directory +- Each table is a subdirectory +- Each record is a separate JSON file +- Metadata files track record IDs and indexes +- System table stores initial record references + +#### Key Features + +**Atomic Operations:** +- Temporary file creation with validation +- Atomic move operations to prevent corruption +- Lock management for concurrent access + +**Caching Strategy:** +- In-memory table cache for performance +- Metadata cache for quick record lookups +- Intelligent cache invalidation + +**Concurrency Control:** +- File-level locks with timeout protection +- Table-level locks for metadata operations +- Deadlock prevention through lock ordering + +### PostgreSQL Database Connector + +#### Purpose and Use Cases + +The PostgreSQL connector is designed for: +- **Production Environments**: High-performance, reliable storage +- **Multi-User Systems**: Concurrent access with ACID guarantees +- **Large Datasets**: Efficient querying and indexing +- **Scalability**: Horizontal and vertical scaling capabilities + +#### Schema Architecture + +```mermaid +erDiagram + _system ||--o{ Tables : tracks_initial_records + Tables ||--o{ Records : contains + + _system { + varchar table_name PK + varchar initial_id + double _createdAt + double _modifiedAt + } + + Tables { + varchar id PK + text field1 + jsonb field2 + double _createdAt + double _modifiedAt + varchar _createdBy + varchar _modifiedBy + } +``` + +#### Dynamic Schema Generation + +The connector automatically: +- Creates tables from Pydantic models +- Maps Python types to SQL types +- Adds metadata columns automatically +- Creates indexes for foreign key fields +- Performs additive migrations (adds missing columns) + +**Type Mapping:** +- `str` → `TEXT` +- `int` → `INTEGER` +- `float` → `DOUBLE PRECISION` +- `bool` → `BOOLEAN` +- `dict/list` → `JSONB` (enables flexible document storage) + +#### JSONB Support + +The connector uses PostgreSQL's JSONB type for complex fields: +- Efficient binary JSON storage +- Indexable JSON content +- Native JSON operators +- Flexible schema evolution + +### Database Connector Selection + +The system selects connectors through import statements in interface files: + +```mermaid +graph TD + A[Interface Initialization] --> B{Check DB_HOST Config} + B -->|File Path| C[Import connectorDbJson] + B -->|Host:Port| D[Import connectorDbPostgre] + C --> E[Create DatabaseConnector] + D --> E + E --> F[Initialize System] + + style B fill:#fff3e0 +``` + +**Selection Criteria:** +- Configuration-driven through `config.ini` +- Import statement determines implementation +- Transparent to application layer +- No runtime switching (decided at startup) + +### Common Operations Flow + +```mermaid +sequenceDiagram + participant App as Application + participant Iface as Interface Layer + participant DB as DatabaseConnector + participant Storage as Storage Backend + + App->>Iface: getRecordset(Model, filters) + Iface->>DB: getRecordset(model_class, recordFilter) + + alt PostgreSQL + DB->>Storage: SELECT * FROM table WHERE... + Storage-->>DB: Rows + else JSON + DB->>Storage: Read files from directory + Storage-->>DB: JSON objects + end + + DB->>DB: Apply filters + DB->>DB: Handle JSONB parsing + DB-->>Iface: List of records + Iface->>Iface: Apply UAM filters + Iface-->>App: Filtered records + + Note over DB,Storage: Both connectors provide
identical interface +``` + +### Transaction Handling + +**PostgreSQL:** +- Uses database transactions +- ACID compliance +- Automatic rollback on errors +- Connection pooling and retry logic + +**JSON:** +- File-level atomicity +- Lock-based isolation +- Manual rollback through file operations +- Lock timeout protection + +### Performance Considerations + +| Aspect | JSON Connector | PostgreSQL Connector | +|--------|---------------|---------------------| +| **Read Speed** | Fast for small datasets, degrades with size | Consistent, optimized with indexes | +| **Write Speed** | Fast for single records | Fast with connection pooling | +| **Concurrent Access** | Limited by file locking | Excellent with MVCC | +| **Query Capability** | In-memory filtering only | Full SQL with JSONB operators | +| **Scalability** | Limited to single server | Horizontal and vertical scaling | +| **Memory Usage** | High (full table caching) | Low (database managed) | + +--- + +## Voice Connector + +### Overview + +The `ConnectorGoogleSpeech` provides integration with Google Cloud AI services for voice processing, offering a complete pipeline for speech recognition, translation, and text-to-speech synthesis. + +### Architecture + +```mermaid +graph TB + A[Voice Interface] --> B[ConnectorGoogleSpeech] + + B --> C[Speech-to-Text Client] + B --> D[Translation Client] + B --> E[Text-to-Speech Client] + + C --> F[Google Cloud Speech-to-Text API] + D --> G[Google Cloud Translation API] + E --> H[Google Cloud Text-to-Speech API] + + F --> I[Audio Processing] + G --> J[Language Translation] + H --> K[Voice Synthesis] + + style B fill:#e8f5e9 + style F fill:#e3f2fd + style G fill:#e3f2fd + style H fill:#e3f2fd +``` + +### Core Capabilities + +#### 1. Speech-to-Text Processing + +**Audio Format Support:** +- WEBM OPUS (primary web recording format) +- WAV (Linear PCM) +- MP3 +- FLAC +- OGG + +**Processing Pipeline:** + +```mermaid +sequenceDiagram + participant Client + participant Connector + participant Validator + participant API as Google Speech API + + Client->>Connector: speechToText(audioContent) + Connector->>Validator: validateAudioFormat() + + Validator->>Validator: Detect format + Validator->>Validator: Extract sample rate + Validator->>Validator: Determine channels + Validator-->>Connector: Format metadata + + Connector->>API: recognize(config, audio) + + alt Success + API-->>Connector: Transcription + confidence + Connector-->>Client: Success response + else API Error + API-->>Connector: Error + Connector->>Connector: Try fallback configs + Connector->>API: recognize(fallback_config) + API-->>Connector: Result + Connector-->>Client: Response + end +``` + +**Audio Format Detection:** +- Magic byte pattern recognition +- Header parsing for metadata extraction +- Automatic format-specific configuration +- Deep scanning for ambiguous formats + +**Fallback Strategy:** +Multiple configurations tried automatically: +1. Detected format with detected parameters +2. Alternative encodings (LINEAR16, WEBM_OPUS) +3. Standard sample rates (8kHz, 16kHz, 44.1kHz, 48kHz) +4. Different recognition models (latest_long, phone_call, latest_short) + +#### 2. Translation Services + +**Features:** +- Automatic language detection +- HTML entity decoding +- Bidirectional translation +- Preserves text formatting + +**Translation Flow:** + +```mermaid +graph LR + A[Input Text] --> B[Google Translation API] + B --> C[Detect Source Language] + C --> D[Translate to Target] + D --> E[Decode HTML Entities] + E --> F[Return Result] + + style B fill:#e3f2fd +``` + +#### 3. Text-to-Speech Synthesis + +**Voice Selection:** +- Language-specific voices +- Gender-based voice selection +- Neural voice quality +- Multiple voice variants per language + +**Synthesis Process:** + +```mermaid +sequenceDiagram + participant Client + participant Connector + participant API as Google TTS API + + Client->>Connector: textToSpeech(text, language, voice) + + alt Voice Specified + Connector->>API: synthesize_speech(voice) + else No Voice + Connector->>Client: Error: no default voice + end + + API->>API: Generate audio + API-->>Connector: MP3 audio data + Connector-->>Client: Audio content + metadata +``` + +### Complete Pipeline: Speech-to-Translated-Text + +The connector provides an integrated pipeline: + +```mermaid +graph TD + A[Audio Input] --> B[Speech-to-Text] + B --> C[Original Text] + C --> D[Translation] + D --> E[Translated Text] + + B -.->|Confidence Score| F[Metadata] + D -.->|Source Language| F + F --> G[Complete Response] + + style A fill:#ffebee + style C fill:#fff3e0 + style E fill:#e8f5e9 + style G fill:#e1f5fe +``` + +**Use Cases:** +- Real-time voice translation +- Multilingual voice assistants +- International call centers +- Language learning applications + +### Authentication and Configuration + +**Credential Management:** +- Service account JSON key stored in configuration +- Parsed and loaded at initialization +- No file system dependency +- Credentials object creation from JSON + +**Configuration Parameters:** +- `Connector_GoogleSpeech_API_KEY_SECRET`: Service account JSON (encrypted) + +### Error Handling and Resilience + +**Retry Mechanisms:** +- Multiple encoding attempts +- Sample rate fallbacks +- Model fallbacks +- Graceful degradation + +**Validation:** +- Audio length verification +- Format compatibility checks +- Content quality analysis +- Silence detection + +### Integration Points + +```mermaid +graph TB + A[Routes Layer] --> B[VoiceObjects Interface] + B --> C[ConnectorGoogleSpeech] + + A1[/voice-google/speech-to-text] --> B + A2[/voice-google/translate] --> B + A3[/voice-google/text-to-speech] --> B + A4[/voice-google/languages] --> B + A5[/voice-google/voices] --> B + A6[WebSocket /ws/realtime-interpreter] --> B + + style A1 fill:#e8f5e9 + style A2 fill:#e8f5e9 + style A3 fill:#e8f5e9 + style A4 fill:#fff3e0 + style A5 fill:#fff3e0 + style A6 fill:#ffebee +``` + +--- + +## Ticket Connectors + +### Overview + +Ticket connectors provide unified access to external project management and ticketing systems. They enable bidirectional synchronization of tasks and tickets with external platforms. + +### Common Interface Pattern + +Both ticket connectors implement a common base pattern: + +**Core Operations:** +- `readAttributes()`: Fetch field metadata from the system +- `readTasks()`: Read tickets/tasks with pagination +- `writeTasks()`: Update tickets/tasks in bulk + +```mermaid +classDiagram + class TicketBase { + <> + +readAttributes() list~TicketFieldAttribute~ + +readTasks(limit) list~dict~ + +writeTasks(tasklist) None + } + + class ConnectorTicketJira { + -apiUsername: str + -apiToken: str + -apiUrl: str + -projectCode: str + -ticketType: str + +readAttributes() + +readTasks() + +writeTasks() + } + + class ConnectorTicketClickup { + -apiToken: str + -teamId: str + -listId: str + -apiUrl: str + +readAttributes() + +readTasks() + +writeTasks() + } + + TicketBase <|-- ConnectorTicketJira + TicketBase <|-- ConnectorTicketClickup +``` + +### JIRA Connector + +#### Authentication and Configuration + +**Required Parameters:** +- `apiUsername`: JIRA account username +- `apiToken`: API authentication token +- `apiUrl`: JIRA instance URL +- `projectCode`: Project identifier +- `ticketType`: Issue type filter + +#### Field Discovery + +```mermaid +sequenceDiagram + participant App + participant Connector + participant JIRA as JIRA API + + App->>Connector: readAttributes() + Connector->>JIRA: POST /search/jql + JIRA-->>Connector: Issue with all fields + + alt Fields Available + Connector->>Connector: Extract field mappings + Connector-->>App: List of attributes + else No Fields + Connector->>JIRA: GET /field + JIRA-->>Connector: All field definitions + Connector-->>App: Field list + end +``` + +**Field Mapping:** +- Maps JIRA field IDs to human-readable names +- Supports custom fields +- Handles complex field types (ADF, arrays, objects) + +#### Pagination Strategy + +The connector uses JIRA's cursor-based pagination: + +```mermaid +graph TD + A[Start] --> B[Initial Request] + B --> C{Issues Returned?} + C -->|No| D[End] + C -->|Yes| E[Process Issues] + E --> F{Has Next Page Token?} + F -->|No| D + F -->|Yes| G[Request Next Page] + G --> H{Safety Cap Reached?} + H -->|Yes| D + H -->|No| C + + style D fill:#e8f5e9 + style H fill:#ffebee +``` + +**Pagination Features:** +- Cursor-based continuation +- Duplicate detection +- Safety cap (1000 pages max) +- Configurable page size +- Loop prevention + +#### Task Updates + +**Update Flow:** + +```mermaid +sequenceDiagram + participant App + participant Connector + participant JIRA + + App->>Connector: writeTasks([task1, task2]) + + loop For each task + Connector->>Connector: Extract task ID + Connector->>Connector: Map fields + Connector->>Connector: Convert to ADF format + Connector->>JIRA: PUT /issue/{id} + + alt Success + JIRA-->>Connector: 204 No Content + else Error + JIRA-->>Connector: Error response + Connector->>Connector: Log error + end + end + + Connector-->>App: Complete +``` + +**Field Processing:** +- Automatic ADF (Atlassian Document Format) conversion for rich text +- Custom field handling +- Empty field validation +- Selective field updates + +### ClickUp Connector + +#### Authentication and Configuration + +**Required Parameters:** +- `apiToken`: ClickUp API token +- `teamId`: Workspace/team identifier +- `listId`: Optional list filter +- `apiUrl`: API endpoint (default: https://api.clickup.com/api/v2) + +#### Hierarchical Data Access + +```mermaid +graph TD + A[Team Level] --> B[Space Level] + B --> C[Folder Level] + C --> D[List Level] + D --> E[Task Level] + E --> F[Subtask Level] + + G[Connector] -.->|listId specified| D + G -.->|listId not specified| A + + style G fill:#fff3e0 +``` + +**Access Patterns:** +- List-specific access when `listId` provided +- Team-wide search when no `listId` +- Automatic subtask inclusion + +#### Field Discovery + +ClickUp provides both: +1. **Custom Fields**: From list-specific field API +2. **Core Fields**: Standard task properties + +```mermaid +graph LR + A[readAttributes] --> B{listId Present?} + B -->|Yes| C[GET /list/id/field] + B -->|No| D[Return Core Fields Only] + + C --> E[Merge Custom + Core Fields] + D --> F[Return Fields] + E --> F + + style C fill:#e3f2fd +``` + +**Core Fields:** +- ID +- Name +- Status +- Assignees +- Date Created +- Due Date + +#### Task Retrieval + +**Pagination:** +- Page-based pagination +- Configurable page size (100 default) +- Automatic page iteration + +```mermaid +sequenceDiagram + participant Connector + participant API as ClickUp API + + loop Until no more tasks + Connector->>API: GET /list/id/task?page={n} + API-->>Connector: Tasks array + + alt Tasks returned < page size + Connector->>Connector: Stop pagination + else More tasks possible + Connector->>Connector: Increment page + end + end +``` + +#### Task Updates + +**Update Strategy:** + +```mermaid +graph TD + A[Task Update Request] --> B[Extract Task ID] + B --> C[Extract Fields] + C --> D{Field Type?} + + D -->|name/summary| E[Update name] + D -->|status| F[Update status] + D -->|custom field| G[Add to custom_fields array] + D -->|other| H[Add to description] + + E --> I[Build Payload] + F --> I + G --> I + H --> I + + I --> J[PUT /task/id] + + style J fill:#e3f2fd +``` + +**Field Mapping:** +- Heuristic field name matching +- Custom field special handling +- Best-effort unknown field mapping + +### Ticket Interface Integration + +The ticket connectors are wrapped by the `TicketInterface` for field mapping: + +```mermaid +graph TB + A[Workflow/Feature] --> B[TicketService] + B --> C[TicketInterface] + C --> D[Connector Factory] + + D -->|connectorType='Jira'| E[ConnectorTicketJira] + D -->|connectorType='ClickUp'| F[ConnectorTicketClickup] + + C --> G[Task Sync Definition] + G --> H[Field Mapping] + + E --> I[External System] + F --> I + + style C fill:#fff3e0 + style G fill:#e8f5e9 +``` + +**Task Sync Definition:** +- Maps internal field names to external field paths +- Specifies read/write directions +- Handles nested field access +- Enables field transformations + +**Example Flow:** + +```mermaid +sequenceDiagram + participant Workflow + participant Interface as TicketInterface + participant Connector + participant External as External System + + Workflow->>Interface: exportTicketsAsList() + Interface->>Connector: readTasks() + Connector->>External: API Request + External-->>Connector: Raw tickets + Connector-->>Interface: Tickets list + Interface->>Interface: _transformTicketRecords() + Interface-->>Workflow: Transformed data + + Note over Interface: Applies field mapping
from sync definition +``` + +--- + +## Integration Patterns + +### Connector Initialization Patterns + +#### 1. Singleton Pattern (Database Connectors) + +Database connectors use singleton-like patterns through interface factories: + +```mermaid +graph TD + A[Request 1] --> B[getAppInterface] + C[Request 2] --> B + D[Request 3] --> B + + B --> E{Instance Exists?} + E -->|No| F[Create AppObjects] + E -->|Yes| G[Return Cached Instance] + + F --> H[Initialize DatabaseConnector] + H --> I[Store in _gatewayInterfaces] + + G --> J[Return Interface] + I --> J + + style B fill:#fff3e0 + style I fill:#e8f5e9 +``` + +**Benefits:** +- Reuses database connections +- Maintains context consistency +- Reduces initialization overhead +- Per-user instances for security + +#### 2. Factory Pattern (Ticket Connectors) + +Ticket connectors use factory pattern for runtime selection: + +```mermaid +graph TD + A[Service Request] --> B[createTicketInterfaceByType] + B --> C{Connector Type?} + + C -->|'jira'| D[Import JIRA Connector] + C -->|'clickup'| E[Import ClickUp Connector] + C -->|unknown| F[Raise ValueError] + + D --> G[Create Connector Instance] + E --> G + + G --> H[Wrap in TicketInterface] + H --> I[Return Interface] + + style B fill:#fff3e0 +``` + +**Advantages:** +- Runtime connector selection +- Easy addition of new connectors +- Consistent interface wrapping +- Configuration-driven behavior + +#### 3. Dependency Injection (Voice Connector) + +Voice connector uses lazy initialization with dependency injection: + +```mermaid +sequenceDiagram + participant Route + participant Interface as VoiceObjects + participant Connector as ConnectorGoogleSpeech + + Route->>Interface: getVoiceInterface(user) + Interface->>Interface: _getGoogleSpeechConnector() + + alt First Call + Interface->>Connector: __init__() + Connector->>Connector: Load credentials + Connector->>Connector: Initialize clients + Connector-->>Interface: Connector instance + Interface->>Interface: Cache connector + else Subsequent Calls + Interface-->>Route: Cached connector + end + + Interface-->>Route: Voice interface +``` + +### Context Management Pattern + +All connectors support context updates for audit trails: + +```mermaid +graph LR + A[User Login] --> B[Create Interface] + B --> C[Initialize Connector] + C --> D[Set userId Context] + + E[User Switch] --> F[updateContext] + F --> G[Update userId] + F --> H[Clear Caches] + + I[All Operations] --> J[Include userId in metadata] + J --> K[_createdBy] + J --> L[_modifiedBy] + + style D fill:#e8f5e9 + style G fill:#fff3e0 +``` + +**Context Metadata:** +- `_createdBy`: User who created record +- `_modifiedBy`: User who last modified record +- `_createdAt`: Creation timestamp +- `_modifiedAt`: Modification timestamp + +### Error Handling Pattern + +Connectors implement consistent error handling: + +```mermaid +graph TD + A[Operation Start] --> B{Try Operation} + B -->|Success| C[Return Result] + B -->|Error| D[Log Error] + + D --> E{Retry Possible?} + E -->|Yes| F[Execute Fallback] + E -->|No| G[Return Error Response] + + F --> H{Success?} + H -->|Yes| C + H -->|No| G + + G --> I[Structured Error Response] + + style C fill:#e8f5e9 + style G fill:#ffebee +``` + +**Error Response Structure:** +- Consistent dictionary format +- `success`: Boolean indicator +- `error`: Descriptive error message +- Additional context fields +- No exceptions propagated to application layer + +--- + +## Configuration + +### Database Configuration + +Each database interface reads specific configuration keys: + +**App Database (User/Mandate Management):** +- `DB_APP_HOST`: Database host or file path +- `DB_APP_DATABASE`: Database name +- `DB_APP_USER`: Database username +- `DB_APP_PASSWORD_SECRET`: Encrypted password +- `DB_APP_PORT`: Database port (default: 5432) + +**Chat Database:** +- `DB_CHAT_HOST` +- `DB_CHAT_DATABASE` +- `DB_CHAT_USER` +- `DB_CHAT_PASSWORD_SECRET` +- `DB_CHAT_PORT` + +**Management Database:** +- `DB_MANAGEMENT_HOST` +- `DB_MANAGEMENT_DATABASE` +- `DB_MANAGEMENT_USER` +- `DB_MANAGEMENT_PASSWORD_SECRET` +- `DB_MANAGEMENT_PORT` + +**Real Estate Database:** +- `DB_REAL_ESTATE_HOST` +- `DB_REAL_ESTATE_DATABASE` +- `DB_REAL_ESTATE_USER` +- `DB_REAL_ESTATE_PASSWORD_SECRET` +- `DB_REAL_ESTATE_PORT` + +### Voice Connector Configuration + +**Google Cloud Credentials:** +- `Connector_GoogleSpeech_API_KEY_SECRET`: Complete service account JSON key (encrypted) + +### Ticket Connector Configuration + +Ticket connectors receive configuration at runtime through `connectorParams`: + +**JIRA Configuration:** +- `apiUsername`: JIRA username +- `apiToken`: API token +- `apiUrl`: JIRA instance URL +- `projectCode`: Project key +- `ticketType`: Issue type filter + +**ClickUp Configuration:** +- `apiToken`: ClickUp API token +- `teamId`: Workspace ID +- `listId`: Optional list ID +- `apiUrl`: API base URL + +### Configuration Flow + +```mermaid +sequenceDiagram + participant Config as config.ini + participant Security as Security Module + participant Interface + participant Connector + + Config->>Security: Read encrypted values + Security->>Security: Decrypt secrets + Security-->>Interface: Configuration values + + Interface->>Connector: Initialize with config + Connector->>Connector: Validate configuration + + alt Valid Config + Connector->>Connector: Establish connections + Connector-->>Interface: Ready + else Invalid Config + Connector-->>Interface: Raise Exception + end +``` + +--- + +## Design Principles + +### 1. Abstraction and Encapsulation + +**Principle:** Hide implementation details behind consistent interfaces. + +```mermaid +graph LR + A[Application Code] --> B[Interface Layer] + B -.->|Never directly accesses| C[Connector Details] + B --> D[Public API] + D --> C + + style B fill:#e8f5e9 + style C fill:#ffebee +``` + +**Benefits:** +- Technology independence +- Easy testing with mocks +- Simplified application code +- Future-proof architecture + +### 2. Duck Typing over Formal Interfaces + +**Rationale:** Python's duck typing provides flexibility without interface boilerplate. + +Both database connectors provide identical methods without inheriting from a common base class. This allows: +- Natural Python idioms +- Easy addition of connector-specific features +- No multiple inheritance complexity +- Freedom in implementation + +### 3. Configuration over Code + +**Principle:** Behavior should be configurable without code changes. + +```mermaid +graph TD + A[Deployment Requirements] --> B[Configuration Files] + B --> C[Runtime Behavior] + + B1[Development] --> B + B2[Staging] --> B + B3[Production] --> B + + C --> C1[Connector Selection] + C --> C2[Connection Parameters] + C --> C3[Feature Flags] + + style B fill:#fff3e0 +``` + +**Implementation:** +- External configuration files +- Environment-specific settings +- Encrypted secrets support +- No hardcoded credentials + +### 4. Fail-Safe Defaults + +**Principle:** System should work out-of-the-box with sensible defaults. + +**Examples:** +- JSON connector for development (no DB setup) +- Default sample rates for audio processing +- Automatic format detection +- Graceful degradation + +### 5. Explicit Error Handling + +**Principle:** Errors should be caught, logged, and returned as data structures. + +```mermaid +graph TD + A[Operation] --> B{Success?} + B -->|Yes| C[Return Success Response] + B -->|No| D[Catch Exception] + + D --> E[Log Error with Context] + E --> F[Create Error Response] + F --> G[Return Error Structure] + + style C fill:#e8f5e9 + style G fill:#ffebee +``` + +**Benefits:** +- No unexpected exceptions +- Consistent error format +- Rich error context for debugging +- Application can handle errors gracefully + +### 6. Single Responsibility + +**Principle:** Each connector has one clear purpose. + +- **Database Connectors**: Only handle data persistence +- **Voice Connector**: Only handle voice processing +- **Ticket Connectors**: Only handle external ticket systems + +Business logic, validation, and transformations belong in higher layers. + +### 7. Dependency Inversion + +**Principle:** High-level modules don't depend on low-level modules. + +```mermaid +graph TD + A[Workflow Layer] --> B[Service Layer] + B --> C[Interface Layer] + C --> D[Connector Layer] + + A -.->|Does not depend on| D + B -.->|Does not depend on| D + + style A fill:#e3f2fd + style D fill:#e8f5e9 +``` + +The application depends on interfaces (duck-typed contracts), not concrete implementations. + +### 8. Idempotency Where Possible + +**Principle:** Operations should be safe to retry. + +**Implementation:** +- Record updates are idempotent (same result if repeated) +- Duplicate detection in pagination +- Transaction rollback on errors +- Atomic file operations + +### 9. Progressive Enhancement + +**Principle:** Core functionality works simply; advanced features add complexity only when needed. + +**Examples:** +- Basic audio format → Automatic fallbacks +- Simple field mapping → Complex transformations +- Single database → Multiple database support +- Direct API calls → Retry logic + +### 10. Audit Trail by Design + +**Principle:** All data modifications tracked automatically. + +```mermaid +graph LR + A[Create/Modify Operation] --> B[Add Metadata] + B --> C[_createdAt] + B --> D[_createdBy] + B --> E[_modifiedAt] + B --> F[_modifiedBy] + + G[User Context] --> B + H[Current Timestamp] --> B + + style B fill:#fff3e0 +``` + +**Benefits:** +- Automatic compliance +- Debugging support +- Security auditing +- User accountability + +--- + +## Summary + +The Connectors component provides a robust, flexible abstraction layer for external system integration. Key strengths include: + +- **Technology Independence**: Application code unaware of specific storage or service implementations +- **Flexibility**: Easy swapping between implementations without code changes +- **Reliability**: Comprehensive error handling and retry mechanisms +- **Performance**: Optimized for each technology (caching for JSON, connection pooling for PostgreSQL) +- **Maintainability**: Clear separation of concerns and consistent patterns +- **Extensibility**: New connectors can be added with minimal impact + +The component enables the application to work seamlessly across different deployment scenarios while maintaining clean architecture and separation of concerns. + diff --git a/docs/code-documentation/datamodels-interfaces-component.md b/docs/code-documentation/datamodels-interfaces-component.md new file mode 100644 index 00000000..8fd749c2 --- /dev/null +++ b/docs/code-documentation/datamodels-interfaces-component.md @@ -0,0 +1,1832 @@ +# Datamodels and Interfaces Component + +## Overview + +The Datamodels and Interfaces components form the core data layer of the Gateway application. They provide a clean separation between data structures (datamodels) and data access logic (interfaces), enabling type-safe, maintainable, and scalable data operations throughout the application. + +## Component Architecture + +```mermaid +graph TB + subgraph "Application Layer" + App[Application
Routes, Services, Features] + end + + subgraph "Interfaces Layer" + IF_RealEstate[Real Estate Interface] + IF_Chat[Chat Interface] + IF_App[App Interface] + IF_Component[Component Interface] + IF_AI[AI Interface] + IF_Ticket[Ticket Interface] + IF_Voice[Voice Interface] + end + + subgraph "Access Control Layer" + AC_RealEstate[Real Estate Access] + AC_Chat[Chat Access] + AC_App[App Access] + AC_Component[Component Access] + end + + subgraph "Database Connectors" + Connector_Postgre[PostgreSQL Connector] + end + + subgraph "Databases" + DB_RealEstate[(Real Estate
Database)] + DB_Chat[(Chat
Database)] + DB_App[(App
Database)] + DB_Component[(Component
Database)] + end + + subgraph "External Systems" + External_AI[AI APIs
OpenAI, Anthropic, etc.] + External_Tickets[Ticket Systems
Jira, ClickUp] + External_Voice[Voice Services
Google Cloud] + end + + App -.->|uses| IF_RealEstate + App -.->|uses| IF_Chat + App -.->|uses| IF_App + App -.->|uses| IF_Component + App -.->|uses| IF_AI + App -.->|uses| IF_Ticket + App -.->|uses| IF_Voice + + IF_RealEstate --> AC_RealEstate + IF_Chat --> AC_Chat + IF_App --> AC_App + IF_Component --> AC_Component + + AC_RealEstate --> Connector_Postgre + AC_Chat --> Connector_Postgre + AC_App --> Connector_Postgre + AC_Component --> Connector_Postgre + + Connector_Postgre --> DB_RealEstate + Connector_Postgre --> DB_Chat + Connector_Postgre --> DB_App + Connector_Postgre --> DB_Component + + IF_AI --> External_AI + IF_Ticket --> External_Tickets + IF_Voice --> External_Voice + + IF_RealEstate -.->|uses| DM_RealEstate[Real Estate
Datamodels] + IF_Chat -.->|uses| DM_Chat[Chat
Datamodels] + IF_App -.->|uses| DM_UAM[User & Mandate
Datamodels] + IF_Component -.->|uses| DM_Files[File
Datamodels] + IF_AI -.->|uses| DM_AI[AI
Datamodels] +``` + +## Data Flow + +```mermaid +sequenceDiagram + participant Route as API Route + participant Service as Service Layer + participant Interface as Interface + participant Access as Access Control + participant Connector as Database Connector + participant DB as Database + + Route->>Service: Request with User Context + Service->>Interface: Initialize with User + Interface->>Access: Check Permissions + Access-->>Interface: Permission Granted + Service->>Interface: CRUD Operation + Interface->>Access: Validate Access + Access-->>Interface: Access Validated + Interface->>Connector: Execute Query + Connector->>DB: SQL Query + DB-->>Connector: Result Set + Connector-->>Interface: Data Objects + Interface->>Access: Apply Filtering + Access-->>Interface: Filtered Data + Interface-->>Service: Datamodel Instances + Service-->>Route: Response Data +``` + +## Component Structure + +### Datamodels Structure + +``` +modules/datamodels/ +├── datamodelRealEstate.py # Real estate domain models +├── datamodelChat.py # Chat workflow models +├── datamodelAi.py # AI operation models +├── datamodelUam.py # User and mandate models +├── datamodelSecurity.py # Security and authentication models +├── datamodelFiles.py # File management models +├── datamodelDocument.py # Document structure models +├── datamodelExtraction.py # Content extraction models +├── datamodelPagination.py # Pagination models +├── datamodelVoice.py # Voice settings models +├── datamodelTickets.py # Ticket system models +├── datamodelNeutralizer.py # Data neutralization models +├── datamodelTools.py # Tool definitions +├── datamodelUtils.py # Utility models +└── __init__.py # Package exports +``` + +### Interfaces Structure + +``` +modules/interfaces/ +├── interfaceDbRealEstateObjects.py # Real estate data access +├── interfaceDbRealEstateAccess.py # Real estate access control +├── interfaceDbChatObjects.py # Chat data access +├── interfaceDbChatAccess.py # Chat access control +├── interfaceDbAppObjects.py # App/user management access +├── interfaceDbAppAccess.py # App access control +├── interfaceDbComponentObjects.py # Component management access +├── interfaceDbComponentAccess.py # Component access control +├── interfaceAiObjects.py # AI operations interface +├── interfaceTicketObjects.py # Ticket system interface +└── interfaceVoiceObjects.py # Voice operations interface +``` + +--- + +## Datamodels Component + +### datamodelRealEstate.py + +```mermaid +erDiagram + Projekt { + string id PK + string mandateId + string label + string statusProzess + json perimeter + json baulinie + json parzellen + json dokumente + json kontextInformationen + } + + Parzelle { + string id PK + string mandateId + string label + string kontextGemeinde FK + json perimeter + json baulinie + string bauzone + float az + float bz + json dokumente + json kontextInformationen + } + + Land { + string id PK + string mandateId + string label + string abk + json dokumente + json kontextInformationen + } + + Kanton { + string id PK + string mandateId + string label + string id_land FK + string abk + json dokumente + json kontextInformationen + } + + Gemeinde { + string id PK + string mandateId + string label + string id_kanton FK + string plz + json dokumente + json kontextInformationen + } + + Dokument { + string id PK + string mandateId + string label + string versionsbezeichnung + string dokumentTyp + string dokumentReferenz + string quelle + string mimeType + json kategorienTags + } + + Kontext { + string id PK + string thema + string inhalt + } + + GeoPunkt { + string koordinatensystem + float x + float y + float z + string referenz + } + + GeoPolylinie { + string id PK + bool closed + json punkte + } + + Projekt ||--o{ Parzelle : contains + Projekt ||--o{ Dokument : references + Projekt ||--o{ Kontext : has + + Parzelle ||--o{ GeoPolylinie : contains + Parzelle ||--o{ GeoPunkt : contains + Parzelle }o--|| Gemeinde : located_in + + Land ||--o{ Kanton : contains + Kanton ||--o{ Gemeinde : contains + + Land ||--o{ Dokument : has + Land ||--o{ Kontext : has + Kanton ||--o{ Dokument : has + Kanton ||--o{ Kontext : has + Gemeinde ||--o{ Dokument : has + Gemeinde ||--o{ Kontext : has + + GeoPolylinie ||--o{ GeoPunkt : contains +``` + +### datamodelChat.py + +```mermaid +erDiagram + ChatWorkflow { + string id PK + string mandateId + string status + string name + int currentRound + int currentTask + int currentAction + int totalTasks + int totalActions + float lastActivity + float startedAt + string workflowMode + int maxSteps + json logs + json messages + json stats + json tasks + } + + ChatMessage { + string id PK + string workflowId FK + string parentMessageId FK + string message + string summary + string role + string status + int sequenceNr + float publishedAt + bool success + string actionId + json documents + } + + ChatLog { + string id PK + string workflowId FK + string message + string type + float timestamp + string status + float progress + json performance + } + + ChatStat { + string id PK + string workflowId FK + float processingTime + int bytesSent + int bytesReceived + int errorCount + string process + string engine + float priceUsd + } + + ChatDocument { + string id PK + string messageId FK + string fileId FK + string fileName + int fileSize + string mimeType + int roundNumber + int taskNumber + int actionNumber + string actionId + } + + TaskPlan { + string overview + json tasks + string userMessage + } + + TaskItem { + string id PK + string workflowId FK + string userInput + string status + string error + float startedAt + float finishedAt + json actionList + int retryCount + int retryMax + bool rollbackOnFailure + json dependencies + string feedback + float processingTime + json resultLabels + } + + TaskStep { + string id PK + string objective + json dependencies + json successCriteria + string estimatedComplexity + string userMessage + string dataType + json expectedFormats + json qualityRequirements + } + + ActionItem { + string id PK + string execMethod + string execAction + json execParameters + string execResultLabel + json expectedDocumentFormats + string userMessage + string status + string error + int retryCount + int retryMax + float processingTime + float timestamp + string result + } + + ActionResult { + bool success + string error + json documents + string resultLabel + } + + AutomationDefinition { + string id PK + string mandateId + string label + string schedule + string template + json placeholders + bool active + string eventId + string status + json executionLogs + } + + ChatWorkflow ||--o{ ChatMessage : contains + ChatWorkflow ||--o{ ChatLog : contains + ChatWorkflow ||--o{ ChatStat : contains + ChatWorkflow ||--o{ TaskPlan : has + ChatWorkflow ||--o{ AutomationDefinition : defines + + ChatMessage ||--o{ ChatDocument : references + TaskPlan ||--o{ TaskStep : contains + TaskItem ||--o{ ActionItem : contains + ActionItem ||--o{ ActionResult : produces +``` + +### datamodelAi.py + +```mermaid +erDiagram + AiModel { + string name PK + string displayName + string connectorType + string apiUrl + float temperature + int maxTokens + int contextLength + float costPer1kTokensInput + float costPer1kTokensOutput + int speedRating + int qualityRating + string priority + string processingMode + json operationTypes + int minContextLength + bool isAvailable + string version + string lastUpdated + } + + OperationTypeRating { + string operationType + int rating + } + + AiCallOptions { + string operationType + string priority + bool compressPrompt + bool compressContext + bool processDocumentsIndividually + float maxCost + int maxProcessingTime + string processingMode + string resultFormat + float safetyMargin + float temperature + int maxParts + } + + AiCallRequest { + string prompt + string context + json options + json contentParts + } + + AiCallResponse { + string content + string modelName + float priceUsd + float processingTime + int bytesSent + int bytesReceived + int errorCount + } + + AiModelCall { + json messages + json model + json options + } + + AiModelResponse { + string content + bool success + string error + string modelId + float processingTime + json tokensUsed + json metadata + } + + AiModel ||--o{ OperationTypeRating : has + AiCallRequest ||--|| AiCallOptions : uses + AiCallRequest ||--|| AiCallResponse : produces + AiCallRequest }o--|| AiModel : uses + AiModelCall }o--|| AiModel : uses + AiModelCall ||--|| AiCallOptions : uses + AiModelCall ||--|| AiModelResponse : produces +``` + +### datamodelUam.py + +```mermaid +erDiagram + Mandate { + string id PK + string name + string language + bool enabled + } + + User { + string id PK + string username + string email + string fullName + string language + bool enabled + string privilege + string authenticationAuthority + string mandateId FK + } + + UserConnection { + string id PK + string userId FK + string authority + string externalId + string externalUsername + string externalEmail + string status + float connectedAt + float lastChecked + float expiresAt + string tokenStatus + float tokenExpiresAt + } + + UserInDB { + string id PK + string username + string email + string fullName + string language + bool enabled + string privilege + string authenticationAuthority + string mandateId FK + string hashedPassword + } + + Mandate ||--o{ User : contains + User ||--o{ UserConnection : has + User ||--|| UserInDB : extends +``` + +### datamodelSecurity.py + +```mermaid +erDiagram + Token { + string id PK + string userId FK + string authority + string connectionId FK + string tokenAccess + string tokenType + float expiresAt + string tokenRefresh + float createdAt + string status + float revokedAt + string revokedBy + string reason + string sessionId + string mandateId FK + } + + AuthEvent { + string id PK + string userId FK + string eventType + float timestamp + string ipAddress + string userAgent + bool success + string details + } + + Token ||--o{ AuthEvent : generates +``` + +### datamodelFiles.py + +```mermaid +erDiagram + FileItem { + string id PK + string mandateId FK + string fileName + string mimeType + string fileHash + int fileSize + float creationDate + } + + FilePreview { + string content + string mimeType + string fileName + bool isText + string encoding + int size + } + + FileData { + string id PK + string data + bool base64Encoded + } + + FileItem ||--|| FilePreview : generates + FileItem ||--|| FileData : contains +``` + +### datamodelDocument.py + +```mermaid +erDiagram + StructuredDocument { + json metadata + json sections + string summary + json tags + } + + DocumentMetadata { + string title + string author + datetime createdAt + json sourceDocuments + string extractionMethod + string version + } + + DocumentSection { + string id PK + string title + string contentType + json elements + int order + json metadata + } + + Paragraph { + string text + json formatting + json metadata + } + + Heading { + string text + int level + json metadata + } + + CodeBlock { + string code + string language + json metadata + } + + Image { + string data + string altText + string caption + json metadata + } + + BulletList { + json items + string listType + json metadata + } + + ListItem { + string text + json subitems + json metadata + } + + TableData { + json headers + json rows + string caption + json metadata + } + + StructuredDocument ||--|| DocumentMetadata : has + StructuredDocument ||--o{ DocumentSection : contains + DocumentSection ||--o{ Paragraph : can_contain + DocumentSection ||--o{ Heading : can_contain + DocumentSection ||--o{ CodeBlock : can_contain + DocumentSection ||--o{ Image : can_contain + DocumentSection ||--o{ BulletList : can_contain + DocumentSection ||--o{ TableData : can_contain + BulletList ||--o{ ListItem : contains + ListItem ||--o{ ListItem : contains +``` + +### datamodelExtraction.py + +```mermaid +erDiagram + ContentExtracted { + string id PK + json parts + json summary + } + + ContentPart { + string id PK + string parentId FK + string label + string typeGroup + string mimeType + string data + json metadata + } + + ExtractionOptions { + string prompt + string operationType + bool processDocumentsIndividually + int imageMaxPixels + int imageQuality + json mergeStrategy + bool chunkAllowed + int maxSize + int textChunkSize + int imageChunkSize + bool enableParallelProcessing + int maxConcurrentChunks + } + + MergeStrategy { + string groupBy + string orderBy + string mergeType + int maxSize + json textMerge + json tableMerge + json structureMerge + json aiResultMerge + bool preserveChunks + string chunkSeparator + bool preserveMetadata + json metadataFields + string onError + bool validateContent + bool useIntelligentMerging + string prompt + json capabilities + } + + PartResult { + json originalPart + string aiResult + int partIndex + string documentId + float processingTime + json metadata + } + + ChunkResult { + json originalChunk + string aiResult + int chunkIndex + string documentId + float processingTime + json metadata + } + + ContentExtracted ||--o{ ContentPart : contains + ContentPart ||--o{ ContentPart : parent_of + ContentExtracted ||--|| ExtractionOptions : uses + ExtractionOptions ||--|| MergeStrategy : uses + ContentPart ||--|| PartResult : produces + ContentPart ||--|| ChunkResult : produces +``` + +### datamodelPagination.py + +```mermaid +erDiagram + PaginationParams { + int page + int pageSize + string sortBy + string sortOrder + } + + PaginationRequest { + json params + json sortFields + } + + SortField { + string field + string order + } + + PaginatedResult { + json items + json metadata + json params + } + + PaginationMetadata { + int page + int pageSize + int totalItems + int totalPages + bool hasNext + bool hasPrevious + } + + PaginationRequest ||--|| PaginationParams : uses + PaginationRequest ||--o{ SortField : contains + PaginatedResult ||--|| PaginationMetadata : has + PaginatedResult ||--o{ PaginationParams : uses +``` + +### datamodelVoice.py + +```mermaid +erDiagram + VoiceSettings { + string id PK + string userId FK + string language + string voice + json settings + } +``` + +### datamodelTickets.py + +```mermaid +erDiagram + TicketFieldAttribute { + string fieldName PK + string fieldType + json fieldConfig + } +``` + +### datamodelNeutralizer.py + +```mermaid +erDiagram + DataNeutraliserConfig { + string id PK + string mandateId FK + string name + bool enabled + json attributes + } + + DataNeutralizerAttributes { + string fieldName PK + string neutralizationType + json options + } + + DataNeutraliserConfig ||--o{ DataNeutralizerAttributes : contains +``` + +### datamodelUtils.py + +```mermaid +erDiagram + Prompt { + string id PK + string name + string content + json metadata + } +``` + +### datamodelTools.py + +```mermaid +erDiagram + CountryCodes { + string ISO2Code PK + string tavilyName + string perplexityName + } +``` + +**Note**: `CountryCodes` is a utility class (not a Pydantic BaseModel) that provides static methods for country code mapping. It contains a mapping dictionary but is not persisted to the database. + +### datamodelJson.py + +No database models - contains JSON template constants and supported section types. + +--- + +## Interfaces Component + +### Overview + +The Interfaces component provides a clean abstraction layer for data access operations. Interfaces handle CRUD operations, user context management, access control, and integration with database connectors and external systems. + +### Objects vs Access Files + +Interfaces are split into two file types: + +#### Objects Files (`interface*Objects.py`) + +**Purpose**: Business logic and CRUD operations for data entities. + +**Responsibilities**: +- CRUD operations (Create, Read, Update, Delete) +- Data validation and transformation +- Business rule enforcement +- Database/external system communication +- User context management +- Pagination and filtering + +**Pattern**: Each Objects file contains methods for manipulating domain entities (e.g., `createProjekt()`, `getWorkflow()`, `updateUser()`). + +#### Access Files (`interface*Access.py`) + +**Purpose**: Permission checking and data filtering based on user privileges. + +**Responsibilities**: +- User privilege validation +- Mandate-based filtering +- Record ownership checking +- Access control attribute generation (`_hideView`, `_hideEdit`, `_hideDelete`) +- Permission decision logic + +**Pattern**: Access files contain two main methods: +- `uam()`: Unified Access Management - filters recordsets and adds access control flags +- `canModify()`: Checks if user can create/update/delete records + +**Relationship**: + +```mermaid +graph TB + Objects[Objects File] --> Access[Access File] + Access --> UAM[uam Method] + Access --> CanModify[canModify Method] + + Objects --> CRUD[CRUD Operations] + CRUD --> AccessCheck{Check Access} + AccessCheck -->|Read| UAM + AccessCheck -->|Write| CanModify + + UAM --> Filter[Filter Records] + UAM --> Flags[Add Access Flags] + + CanModify --> Permission{Permission?} + Permission -->|Yes| Allow[Allow Operation] + Permission -->|No| Deny[Deny Operation] +``` + +### Interface Types + +Interfaces are categorized into two types based on their data source: + +#### Database Interfaces + +**Why Database Connectors?**: These interfaces manage persistent data stored in PostgreSQL databases. They use database connectors to: +- Store structured data with relationships +- Ensure data consistency and integrity +- Provide ACID transactions +- Support complex queries and filtering +- Enable mandate-based data isolation + +**Characteristics**: +- Use `DatabaseConnector` for PostgreSQL access +- Implement Access classes for permission control +- Support pagination and sorting +- Apply mandate-based filtering automatically +- Track record ownership (`_createdBy`) + +#### External System Interfaces + +**Why External Connectors?**: These interfaces integrate with external APIs and services. They use connectors to: +- Communicate with third-party systems +- Transform data between formats +- Handle API authentication and rate limiting +- Provide abstraction over external service complexity + +**Characteristics**: +- Use specialized connectors (e.g., `ConnectorGoogleSpeech`, `ConnectorTicketJira`) +- May not require user context (system-level operations) +- Focus on data transformation and synchronization +- Handle external API errors and retries + +### Real Estate Interface (`interfaceDbRealEstateObjects.py`) + +**Type**: Database Interface +**Database**: PostgreSQL (Real Estate database) +**Access Control**: `interfaceDbRealEstateAccess.py` → `RealEstateAccess` + +**Purpose**: Manages real estate domain data including projects, parcels, administrative entities, and geographic information. + +**Why Database Connector**: Real estate data requires persistent storage with complex relationships (projects → parcels → administrative units), geographic data (polygons, points), and mandate-based isolation for multi-tenant scenarios. + +**CRUD Operations**: + +```mermaid +graph TB + subgraph "Projekt Operations" + PCreate[createProjekt] + PGet[getProjekt] + PGetAll[getProjekte] + PUpdate[updateProjekt] + PDelete[deleteProjekt] + end + + subgraph "Parzelle Operations" + ParCreate[createParzelle] + ParGet[getParzelle] + ParGetAll[getParzellen] + ParUpdate[updateParzelle] + ParDelete[deleteParzelle] + end + + subgraph "Dokument Operations" + DocCreate[createDokument] + DocGet[getDokument] + DocGetAll[getDokumente] + DocUpdate[updateDokument] + DocDelete[deleteDokument] + end + + subgraph "Administrative Hierarchy" + GemCreate[createGemeinde] + GemGet[getGemeinde] + GemGetAll[getGemeinden] + GemUpdate[updateGemeinde] + GemDelete[deleteGemeinde] + + KanCreate[createKanton] + KanGet[getKanton] + KanGetAll[getKantone] + KanUpdate[updateKanton] + KanDelete[deleteKanton] + + LanCreate[createLand] + LanGet[getLand] + LanGetAll[getLaender] + LanUpdate[updateLand] + LanDelete[deleteLand] + end + + subgraph "Kontext Operations" + KonCreate[createKontext] + KonGet[getKontext] + KonGetAll[getKontexte] + KonUpdate[updateKontext] + KonDelete[deleteKontext] + end +``` + +**Complete CRUD List**: + +**Projekt**: +- `createProjekt(projekt: Projekt) → Projekt` +- `getProjekt(projektId: str) → Optional[Projekt]` +- `getProjekte(recordFilter: Optional[Dict]) → List[Projekt]` +- `updateProjekt(projektId: str, updateData: Dict) → Optional[Projekt]` +- `deleteProjekt(projektId: str) → bool` + +**Parzelle**: +- `createParzelle(parzelle: Parzelle) → Parzelle` +- `getParzelle(parzelleId: str) → Optional[Parzelle]` +- `getParzellen(recordFilter: Optional[Dict]) → List[Parzelle]` +- `updateParzelle(parzelleId: str, updateData: Dict) → Optional[Parzelle]` +- `deleteParzelle(parzelleId: str) → bool` + +**Dokument**: +- `createDokument(dokument: Dokument) → Dokument` +- `getDokument(dokumentId: str) → Optional[Dokument]` +- `getDokumente(recordFilter: Optional[Dict]) → List[Dokument]` +- `updateDokument(dokumentId: str, updateData: Dict) → Optional[Dokument]` +- `deleteDokument(dokumentId: str) → bool` + +**Gemeinde**: +- `createGemeinde(gemeinde: Gemeinde) → Gemeinde` +- `getGemeinde(gemeindeId: str) → Optional[Gemeinde]` +- `getGemeinden(recordFilter: Optional[Dict]) → List[Gemeinde]` +- `updateGemeinde(gemeindeId: str, updateData: Dict) → Optional[Gemeinde]` +- `deleteGemeinde(gemeindeId: str) → bool` + +**Kanton**: +- `createKanton(kanton: Kanton) → Kanton` +- `getKanton(kantonId: str) → Optional[Kanton]` +- `getKantone(recordFilter: Optional[Dict]) → List[Kanton]` +- `updateKanton(kantonId: str, updateData: Dict) → Optional[Kanton]` +- `deleteKanton(kantonId: str) → bool` + +**Land**: +- `createLand(land: Land) → Land` +- `getLand(landId: str) → Optional[Land]` +- `getLaender(recordFilter: Optional[Dict]) → List[Land]` +- `updateLand(landId: str, updateData: Dict) → Optional[Land]` +- `deleteLand(landId: str) → bool` + +**Kontext**: +- `createKontext(kontext: Kontext) → Kontext` +- `getKontext(kontextId: str) → Optional[Kontext]` +- `getKontexte(recordFilter: Optional[Dict]) → List[Kontext]` +- `updateKontext(kontextId: str, updateData: Dict) → Optional[Kontext]` +- `deleteKontext(kontextId: str) → bool` + +**Access Control Flow**: + +```mermaid +graph TB + Request[CRUD Request] --> Objects[RealEstateObjects] + Objects --> Access[RealEstateAccess] + + Access --> CheckPriv[Check Privilege] + CheckPriv -->|SYSADMIN| AllData[All Records] + CheckPriv -->|ADMIN| MandateData[Mandate Records] + CheckPriv -->|USER| OwnData[Own Records] + + AllData --> UAM[uam Method] + MandateData --> UAM + OwnData --> UAM + + UAM --> Filter[Filter by mandateId] + UAM --> CheckOwn[Check _createdBy] + UAM --> AddFlags[Add _hideView/_hideEdit/_hideDelete] + + Filter --> Return[Return Filtered Data] + CheckOwn --> Return + AddFlags --> Return +``` + +**Key Features**: +- Geographic data support (GeoPolylinie, GeoPunkt) +- Multi-level administrative hierarchy (Land → Kanton → Gemeinde) +- Location name resolution (converts names to IDs for filtering) +- Document versioning and management +- Context information for projects and administrative units + +### Chat Interface (`interfaceDbChatObjects.py`) + +**Type**: Database Interface +**Database**: PostgreSQL (Chat database) +**Access Control**: `interfaceDbChatAccess.py` → `ChatAccess` + +**Purpose**: Manages chat workflows, messages, logs, statistics, and automation definitions for AI-powered conversation workflows. + +**Why Database Connector**: Chat workflows require persistent storage for conversation history, workflow state, performance metrics, and automation configurations. Data must be queryable, filterable, and mandate-isolated. + +**CRUD Operations**: + +```mermaid +graph TB + subgraph "ChatWorkflow Operations" + WfGet[getWorkflows - with pagination] + WfGetOne[getWorkflow - by ID] + WfCreate[createWorkflow] + WfUpdate[updateWorkflow] + WfDelete[deleteWorkflow - cascade] + end + + subgraph "ChatMessage Operations" + MsgGet[getMessages - by workflowId, pagination] + MsgCreate[createMessage] + MsgUpdate[updateMessage] + MsgDelete[deleteMessage] + MsgDeleteFile[deleteFileFromMessage] + end + + subgraph "ChatDocument Operations" + DocGet[getDocuments - by messageId] + DocCreate[createDocument] + end + + subgraph "ChatLog Operations" + LogGet[getLogs - by workflowId, pagination] + LogCreate[createLog] + end + + subgraph "ChatStat Operations" + StatGet[getStats - by workflowId] + StatCreate[createStat] + end + + subgraph "AutomationDefinition Operations" + AutoGet[getAllAutomationDefinitions - pagination] + AutoGetOne[getAutomationDefinition - by ID] + AutoCreate[createAutomationDefinition] + AutoUpdate[updateAutomationDefinition] + AutoDelete[deleteAutomationDefinition] + end + + subgraph "Utility Operations" + Unified[getUnifiedChatData - workflow snapshot] + end +``` + +**Complete CRUD List**: + +**ChatWorkflow**: +- `getWorkflows(pagination: Optional[PaginationParams]) → Union[List[Dict], PaginatedResult]` +- `getWorkflow(workflowId: str) → Optional[ChatWorkflow]` +- `createWorkflow(workflowData: Dict) → ChatWorkflow` +- `updateWorkflow(workflowId: str, workflowData: Dict) → ChatWorkflow` +- `deleteWorkflow(workflowId: str) → bool` (cascades to messages, logs, stats) + +**ChatMessage**: +- `getMessages(workflowId: str, pagination: Optional[PaginationParams]) → Union[List[ChatMessage], PaginatedResult]` +- `createMessage(messageData: Dict) → ChatMessage` +- `updateMessage(messageId: str, messageData: Dict) → Dict` +- `deleteMessage(workflowId: str, messageId: str) → bool` +- `deleteFileFromMessage(workflowId: str, messageId: str, fileId: str) → bool` + +**ChatDocument**: +- `getDocuments(messageId: str) → List[ChatDocument]` +- `createDocument(documentData: Dict) → ChatDocument` + +**ChatLog**: +- `getLogs(workflowId: str, pagination: Optional[PaginationParams]) → Union[List[ChatLog], PaginatedResult]` +- `createLog(logData: Dict) → ChatLog` + +**ChatStat**: +- `getStats(workflowId: str) → List[ChatStat]` +- `createStat(statData: Dict) → ChatStat` + +**AutomationDefinition**: +- `getAllAutomationDefinitions(pagination: Optional[PaginationParams]) → Union[List[Dict], PaginatedResult]` +- `getAutomationDefinition(automationId: str) → Optional[Dict]` +- `createAutomationDefinition(automationData: Dict) → Dict` +- `updateAutomationDefinition(automationId: str, automationData: Dict) → Dict` +- `deleteAutomationDefinition(automationId: str) → bool` + +**Utility Methods**: +- `getUnifiedChatData(workflowId: str, afterTimestamp: Optional[float]) → Dict` (returns workflow snapshot with messages, logs, stats) + +**Access Control Flow**: + +```mermaid +graph TB + Request[CRUD Request] --> Objects[ChatObjects] + Objects --> Access[ChatAccess] + + Access --> CheckPriv[Check Privilege] + CheckPriv -->|SYSADMIN| AllWorkflows[All Workflows] + CheckPriv -->|ADMIN| MandateWorkflows[Mandate Workflows] + CheckPriv -->|USER| OwnWorkflows[Own Workflows] + + AllWorkflows --> UAM[uam Method] + MandateWorkflows --> UAM + OwnWorkflows --> UAM + + UAM --> FilterWorkflow[Filter by workflowId mandate] + UAM --> CheckOwn[Check _createdBy] + UAM --> AddFlags[Add access flags] + + FilterWorkflow --> CheckChild[Check Child Access] + CheckChild -->|Message| CheckWorkflow[Check workflow ownership] + CheckChild -->|Log| CheckWorkflow + CheckChild -->|Stat| CheckWorkflow + + CheckWorkflow --> Return[Return Filtered Data] + CheckOwn --> Return + AddFlags --> Return +``` + +**Key Features**: +- Multi-round workflow support with state tracking +- Normalized data model (workflows, messages, logs, stats in separate tables) +- Cascade delete (deleting workflow removes all related data) +- Pagination support for large datasets +- Unified data retrieval for workflow snapshots +- Automation workflow definitions +- Document attachment management + +### App Interface (`interfaceDbAppObjects.py`) + +**Type**: Database Interface +**Database**: PostgreSQL (App database) +**Access Control**: `interfaceDbAppAccess.py` → `AppAccess` + +**Purpose**: Manages users, mandates, authentication tokens, and application-level configuration. + +**Why Database Connector**: User accounts, mandates, and authentication data require secure, persistent storage with strict access control. This is the foundation for all other interfaces' user context. + +**CRUD Operations**: + +```mermaid +graph TB + subgraph "User Operations" + UserGet[getUsersByMandate - pagination] + UserGetByUsername[getUserByUsername] + UserGetOne[getUser - by ID] + UserCreate[createUser - with password hash] + UserUpdate[updateUser] + UserDelete[deleteUser] + end + + subgraph "UserConnection Operations" + ConnGet[getUserConnections - by userId] + ConnGetToken[getConnectionToken - by connectionId] + end + + subgraph "Mandate Operations" + ManGet[getAllMandates - pagination] + ManGetOne[getMandate - by ID] + ManCreate[createMandate] + ManUpdate[updateMandate] + ManDelete[deleteMandate] + end + + subgraph "Neutralization Config" + NeuGet[getNeutralizationConfig] + NeuCreate[createOrUpdateNeutralizationConfig] + NeuGetAttrs[getNeutralizationAttributes] + NeuDeleteAttrs[deleteNeutralizationAttributes] + end + + subgraph "Initialization" + InitRoot[getRootInterface - system init] + InitRecords[_initRootMandate, _initAdminUser, _initEventUser] + end +``` + +**Complete CRUD List**: + +**User**: +- `getUsersByMandate(mandateId: str, pagination: Optional[PaginationParams]) → Union[List[User], PaginatedResult]` +- `getUserByUsername(username: str) → Optional[User]` +- `getUser(userId: str) → Optional[User]` +- `createUser(userData: Dict, password: Optional[str]) → User` (hashes password with Argon2) +- `updateUser(userId: str, updateData: Union[Dict, User]) → User` +- `deleteUser(userId: str) → bool` + +**UserConnection**: +- `getUserConnections(userId: str) → List[UserConnection]` +- `getConnectionToken(connectionId: str) → Optional[Token]` + +**Mandate**: +- `getAllMandates(pagination: Optional[PaginationParams]) → Union[List[Mandate], PaginatedResult]` +- `getMandate(mandateId: str) → Optional[Mandate]` +- `createMandate(name: str, language: str) → Mandate` +- `updateMandate(mandateId: str, updateData: Dict) → Mandate` +- `deleteMandate(mandateId: str) → bool` + +**DataNeutraliserConfig**: +- `getNeutralizationConfig() → Optional[DataNeutraliserConfig]` +- `createOrUpdateNeutralizationConfig(configData: Dict) → DataNeutraliserConfig` +- `getNeutralizationAttributes(file_id: str) → List[Dict]` +- `deleteNeutralizationAttributes(file_id: str) → bool` + +**Special Methods**: +- `getRootInterface() → AppObjects` (creates interface with system admin privileges for initialization) +- `getInitialId(model_class: type) → Optional[str]` (gets first record ID for a model) + +**Access Control Flow**: + +```mermaid +graph TB + Request[CRUD Request] --> Objects[AppObjects] + Objects --> Access[AppAccess] + + Access --> CheckPriv[Check Privilege] + CheckPriv -->|SYSADMIN| FullAccess[Full System Access] + CheckPriv -->|ADMIN| MandateAccess[Mandate Access Only] + CheckPriv -->|USER| SelfAccess[Own User Record Only] + + FullAccess --> UserOps[User Operations] + MandateAccess --> UserOps + SelfAccess --> UserOps + + UserOps --> CheckOwn[Check Ownership] + CheckOwn -->|User CRUD| SYSADMINOnly{SYSADMIN?} + CheckOwn -->|Mandate CRUD| SYSADMINOnly + CheckOwn -->|Own Profile| Allow[Allow] + + SYSADMINOnly -->|Yes| Allow + SYSADMINOnly -->|No| Deny[Deny] +``` + +**Key Features**: +- Password hashing with Argon2 +- Multi-provider authentication support (local, external) +- System initialization (Root mandate, Admin user, Event user) +- Mandate-based user isolation +- Token management for external connections +- Data neutralization configuration + +### Component Interface (`interfaceDbComponentObjects.py`) + +**Type**: Database Interface +**Database**: PostgreSQL (Component/Management database) +**Access Control**: `interfaceDbComponentAccess.py` → `ComponentAccess` + +**Purpose**: Manages component-level data including files, prompts, and voice settings used across the application. + +**Why Database Connector**: Files, prompts, and voice settings require persistent storage with metadata, ownership tracking, and mandate isolation. Files need binary storage with preview generation. + +**CRUD Operations**: + +```mermaid +graph TB + subgraph "File Operations" + FileGetAll[getAllFiles - pagination] + FileGet[getFile - by ID] + FileCreate[createFile - name, mimeType, content] + FileUpdate[updateFile] + FileDelete[deleteFile] + FileGetData[getFileData - binary content] + FileGetContent[getFileContent - preview] + FileCreateData[createFileData - store binary] + end + + subgraph "Prompt Operations" + PromptGetAll[getAllPrompts - pagination] + PromptGet[getPrompt - by ID] + PromptCreate[createPrompt] + PromptUpdate[updatePrompt] + PromptDelete[deletePrompt] + end + + subgraph "VoiceSettings Operations" + VoiceGet[getVoiceSettings - by userId] + VoiceCreate[createVoiceSettings] + VoiceUpdate[updateVoiceSettings] + VoiceDelete[deleteVoiceSettings] + VoiceGetOrCreate[getOrCreateVoiceSettings] + end + + subgraph "Utilities" + MimeType[getMimeType - from fileName] + end +``` + +**Complete CRUD List**: + +**FileItem**: +- `getAllFiles(pagination: Optional[PaginationParams]) → Union[List[FileItem], PaginatedResult]` +- `getFile(fileId: str) → Optional[FileItem]` +- `createFile(name: str, mimeType: str, content: bytes) → FileItem` (creates FileItem and FileData) +- `updateFile(fileId: str, updateData: Dict) → Dict` +- `deleteFile(fileId: str) → bool` (deletes FileItem and FileData) +- `getFileData(fileId: str) → Optional[bytes]` (raw binary content) +- `getFileContent(fileId: str) → Optional[FilePreview]` (generates preview) +- `createFileData(fileId: str, data: bytes) → bool` (stores binary data) + +**Prompt**: +- `getAllPrompts(pagination: Optional[PaginationParams]) → Union[List[Prompt], PaginatedResult]` +- `getPrompt(promptId: str) → Optional[Prompt]` +- `createPrompt(promptData: Dict) → Dict` +- `updatePrompt(promptId: str, updateData: Dict) → Dict` +- `deletePrompt(promptId: str) → bool` + +**VoiceSettings**: +- `getVoiceSettings(userId: Optional[str]) → Optional[VoiceSettings]` +- `createVoiceSettings(settingsData: Dict) → Dict` +- `updateVoiceSettings(userId: str, updateData: Dict) → Dict` +- `deleteVoiceSettings(userId: str) → bool` +- `getOrCreateVoiceSettings(userId: Optional[str]) → VoiceSettings` + +**Utility Methods**: +- `getMimeType(fileName: str) → str` (detects MIME type from extension) + +**Access Control Flow**: + +```mermaid +graph TB + Request[CRUD Request] --> Objects[ComponentObjects] + Objects --> Access[ComponentAccess] + + Access --> CheckPriv[Check Privilege] + CheckPriv -->|SYSADMIN| AllFiles[All Files] + CheckPriv -->|ADMIN| MandateFiles[Mandate Files] + CheckPriv -->|USER| OwnFiles[Own Files] + + AllFiles --> UAM[uam Method] + MandateFiles --> UAM + OwnFiles --> UAM + + UAM --> FilterMandate[Filter by mandateId] + UAM --> CheckOwn[Check _createdBy] + UAM --> AddFlags[Add access flags] + + FilterMandate --> Return[Return Filtered Data] + CheckOwn --> Return + AddFlags --> Return +``` + +**Key Features**: +- Binary file storage with metadata +- Automatic preview generation (text, images, etc.) +- MIME type detection +- Prompt template management with initialization +- Voice settings per user +- File hash calculation for deduplication + +### AI Interface (`interfaceAiObjects.py`) + +**Type**: External System Interface +**Connectors**: Dynamic discovery via `modelRegistry` +**Access Control**: None (system-level operations) + +**Purpose**: Provides centralized AI operations with dynamic model discovery, automatic model selection, and failover handling. + +**Why External Connector**: AI operations require communication with external APIs (OpenAI, Anthropic, Perplexity, Tavily) and internal AI services. The interface abstracts model selection, handles API calls, manages failover, and tracks costs. + +**Operations**: + +```mermaid +graph TB + subgraph "AI Call Operations" + Call[call - main entry point] + CallText[call with text/context] + CallParts[call with content parts] + end + + subgraph "Model Information" + ModelInfo[getModelInfo - by displayName] + ModelsByTag[getModelsByTag - filter by tag] + end + + subgraph "Internal Processing" + SelectModel[_selectModel - dynamic selection] + ProcessPart[_processContentPartWithFallback] + MergeResults[_mergePartResults] + CallWithModel[_callWithModel - execute API call] + end +``` + +**Complete Operations List**: + +**AI Calls**: +- `call(request: AiCallRequest, progressCallback=None) → AiCallResponse` (main entry point, handles text/context or content parts) +- `getModelInfo(displayName: str) → Dict[str, Any]` (get model metadata) +- `getModelsByTag(tag: str) → List[str]` (filter models by tag) + +**Internal Methods** (used by call): +- `_selectModel(prompt: str, context: str, options: AiCallOptions) → str` (selects best model) +- `_callWithTextContext(request: AiCallRequest) → AiCallResponse` (handles traditional text/context calls) +- `_callWithContentParts(request: AiCallRequest, progressCallback) → AiCallResponse` (handles content parts with chunking) +- `_processContentPartWithFallback(...) → AiCallResponse` (processes single part with failover) +- `_callWithModel(model, prompt, context, options) → AiCallResponse` (executes actual API call) + +**Model Selection and Failover Flow**: + +```mermaid +graph TB + Request[AI Call Request] --> CheckType{Request Type?} + + CheckType -->|Text/Context| TextPath[Text/Context Path] + CheckType -->|Content Parts| PartsPath[Content Parts Path] + + TextPath --> GetFailover[Get Failover Model List] + PartsPath --> GetFailover + + GetFailover --> SelectModel[Select Best Model] + SelectModel --> FilterOp[Filter by Operation Type] + FilterOp --> RateCap[Rate by Capabilities] + RateCap --> ApplyPriority[Apply Priority Rules] + ApplyPriority --> TryModel[Try Model Call] + + TryModel --> Success{Success?} + Success -->|Yes| Return[Return Response] + Success -->|No| NextModel{More Models?} + + NextModel -->|Yes| TryModel + NextModel -->|No| Error[Return Error Response] + + PartsPath --> ProcessEach[Process Each Part] + ProcessEach --> Chunk[Model-Aware Chunking] + Chunk --> TryModel + ProcessEach --> Merge[Merge Results] + Merge --> Return +``` + +**Supported External Systems**: +- **OpenAI**: GPT models via `aicorePluginOpenai` +- **Anthropic**: Claude models via `aicorePluginAnthropic` +- **Perplexity**: Search-enabled models via `aicorePluginPerplexity` +- **Tavily**: Web search API via `aicorePluginTavily` +- **Internal**: Custom models via `aicorePluginInternal` + +**Key Features**: +- Dynamic model discovery (auto-registers available connectors) +- Operation type-based model selection (e.g., IMAGE_ANALYSE, TEXT_GENERATION) +- Automatic failover (tries multiple models on failure) +- Model-aware chunking (respects model context limits) +- Content part processing (handles images, text, tables) +- Cost tracking (calculates USD cost per call) +- Progress callbacks for long-running operations + +### Ticket Interface (`interfaceTicketObjects.py`) + +**Type**: External System Interface +**Connectors**: `ConnectorTicketJira`, `ConnectorTicketClickup` +**Access Control**: Connector-level (API credentials) + +**Purpose**: Synchronizes data with external ticket systems (Jira, ClickUp) by transforming tickets to/from list format for Excel-like operations. + +**Why External Connector**: Ticket systems are external services with their own APIs. The interface provides bidirectional synchronization, field mapping, and data transformation between the application's list format and ticket system formats. + +**Operations**: + +```mermaid +graph TB + subgraph "Factory Method" + Factory[createTicketInterfaceByType - connectorType, params] + end + + subgraph "Export Operations" + Export[exportTicketsAsList - read from external] + end + + subgraph "Import Operations" + Import[importListToTickets - write to external] + end + + subgraph "Internal Transformation" + Transform[_transformTicketRecords - field mapping] + Extract[_extractFieldValue - path-based extraction] + FormatDate[_formatDateForExcel - date formatting] + FilterEmpty[_filterEmptyRecords - validation] + end +``` + +**Complete Operations List**: + +**Factory**: +- `createTicketInterfaceByType(taskSyncDefinition: dict, connectorType: str, connectorParams: dict) → TicketInterface` (creates interface with appropriate connector) + +**Export**: +- `exportTicketsAsList() → list[dict]` (reads tickets from external system, transforms to list format) + +**Import**: +- `importListToTickets(records: list[dict]) → None` (transforms list format, writes to external system) + +**Internal Methods**: +- `_transformTicketRecords(tasks: list[dict], includePut: bool) → list[dict]` (transforms according to task_sync_definition) +- `_extractFieldValue(issue_data: dict, field_path: list[str], field_name: str) → Any` (extracts value using path) +- `_formatDateForExcel(date_value: Any) → Optional[str]` (formats dates for Excel compatibility) +- `_isDateField(field_name: str) → bool` (detects date fields) +- `_filterEmptyRecords(records: list[dict]) → list[dict]` (removes invalid records) + +**Synchronization Flow**: + +```mermaid +graph TB + subgraph "Export Flow" + ExportStart[exportTicketsAsList] --> ReadConn[Read from Connector] + ReadConn --> GetTasks[Get Tasks from API] + GetTasks --> Transform[Transform Fields] + Transform --> MapFields[Map via task_sync_definition] + MapFields --> FormatDates[Format Dates] + FormatDates --> Filter[Filter Empty Records] + Filter --> ReturnList[Return List Format] + end + + subgraph "Import Flow" + ImportStart[importListToTickets] --> ReceiveList[Receive List Format] + ReceiveList --> ExtractFields[Extract Fields] + ExtractFields --> MapToTicket[Map to Ticket Format] + MapToTicket --> BuildUpdate[Build Update Payload] + BuildUpdate --> WriteConn[Write via Connector] + WriteConn --> UpdateAPI[Update External API] + end + + subgraph "Field Mapping" + SyncDef[task_sync_definition] --> Direction[Direction: get/put] + SyncDef --> Path[Field Path: nested access] + Direction --> Transform + Path --> MapFields + Path --> MapToTicket + end +``` + +**Supported External Systems**: +- **Jira**: Via `ConnectorTicketJira` (uses Jira REST API) +- **ClickUp**: Via `ConnectorTicketClickup` (uses ClickUp API) + +**Key Features**: +- Dynamic connector selection (Jira or ClickUp) +- Field mapping configuration (task_sync_definition maps fields bidirectionally) +- Path-based field extraction (supports nested JSON structures) +- Date format handling (converts various formats to Excel-compatible) +- Bidirectional sync (export to list, import from list) +- Empty record filtering (validates records have IDs) + +### Voice Interface (`interfaceVoiceObjects.py`) + +**Type**: External System Interface +**Connector**: `ConnectorGoogleSpeech` +**Access Control**: User context for settings (stored in Component database) + +**Purpose**: Provides speech-to-text, text-to-speech, and translation services using Google Cloud APIs. + +**Why External Connector**: Voice operations require Google Cloud Speech-to-Text, Text-to-Speech, and Translation APIs. The interface abstracts API complexity, handles audio format conversion, and manages user voice settings (stored in database via Component interface). + +**Operations**: + +```mermaid +graph TB + subgraph "Speech-to-Text Operations" + STT[speechToText - audio to text] + STTTrans[speechToTranslatedText - audio to translated text] + end + + subgraph "Text-to-Speech Operations" + TTS[textToSpeech - text to audio] + TTSTrans[textToTranslatedSpeech - text to translated audio] + end + + subgraph "Translation Operations" + Trans[translateText - text translation] + end + + subgraph "Voice Settings" + GetSettings[getVoiceSettings - from Component DB] + CreateSettings[createVoiceSettings - to Component DB] + UpdateSettings[updateVoiceSettings - in Component DB] + GetOrCreate[getOrCreateVoiceSettings] + end + + subgraph "Metadata Operations" + GetLangs[getAvailableLanguages - from Google API] + GetVoices[getAvailableVoices - from Google API] + end +``` + +**Complete Operations List**: + +**Speech-to-Text**: +- `speechToText(audioContent: bytes, language: str, sampleRate: Optional[int], channels: Optional[int]) → Dict[str, Any]` (converts audio to text) +- `speechToTranslatedText(audioContent: bytes, fromLanguage: str, toLanguage: str, sampleRate: Optional[int], channels: Optional[int]) → Dict[str, Any]` (converts audio to translated text) + +**Text-to-Speech**: +- `textToSpeech(text: str, language: Optional[str], voice: Optional[str]) → Dict[str, Any]` (converts text to audio) +- `textToTranslatedSpeech(text: str, fromLanguage: str, toLanguage: str, voice: Optional[str]) → Dict[str, Any]` (converts text to translated audio) + +**Translation**: +- `translateText(text: str, sourceLanguage: str, targetLanguage: str) → Dict[str, Any]` (translates text) + +**Voice Settings** (delegates to Component interface): +- `getVoiceSettings(userId: str) → Optional[VoiceSettings]` +- `createVoiceSettings(settingsData: Dict) → Optional[VoiceSettings]` +- `updateVoiceSettings(userId: str, settingsData: Dict) → Optional[VoiceSettings]` +- `getOrCreateVoiceSettings(userId: str) → Optional[VoiceSettings]` + +**Metadata**: +- `getAvailableLanguages() → Dict[str, Any]` (lists supported languages from Google API) +- `getAvailableVoices(languageCode: Optional[str]) → Dict[str, Any]` (lists available voices, optionally filtered by language) + +**Operation Flow**: + +```mermaid +graph TB + Request[Voice Operation Request] --> Voice[VoiceObjects] + Voice --> CheckSettings{Need Settings?} + + CheckSettings -->|Yes| GetSettings[Get from Component DB] + CheckSettings -->|No| Connector[Get Connector] + + GetSettings --> Connector + Connector --> GoogleAPI[Google Cloud API] + + GoogleAPI --> STT{Operation Type?} + STT -->|Speech-to-Text| STTAPI[Speech-to-Text API] + STT -->|Text-to-Speech| TTSAPI[Text-to-Speech API] + STT -->|Translation| TransAPI[Translation API] + + STTAPI --> Process[Process Response] + TTSAPI --> Process + TransAPI --> Process + + Process --> Format[Format Response] + Format --> Return[Return Result] +``` + +**Supported External Systems**: +- **Google Cloud Speech-to-Text**: Audio transcription +- **Google Cloud Text-to-Speech**: Audio synthesis +- **Google Cloud Translation**: Text translation + +**Key Features**: +- Multi-language support (detects and supports many languages) +- Audio format handling (auto-detects sample rate, channels) +- Combined operations (speech-to-translated-text, text-to-translated-speech) +- User voice preferences (stored in Component database) +- Language and voice discovery (queries Google API for available options) +- Error handling with detailed error messages + +### Access Control Summary + +**Privilege Levels**: +1. **SYSADMIN**: Full system access, all mandates +2. **ADMIN**: Full access within mandate +3. **USER**: Access to own records only + +**Access Methods**: +- `uam()`: Filters recordsets by privilege, adds `_hideView`, `_hideEdit`, `_hideDelete` flags +- `canModify()`: Checks if user can create/update/delete records based on ownership and privilege + +**Singleton Pattern**: Interfaces use factory functions that cache instances per user context for efficient memory usage. + +**User Context**: All database interfaces require user context (User object with mandateId, userId, privilege) for access control and data filtering. + +--- \ No newline at end of file diff --git a/docs/code-documentation/features-component.md b/docs/code-documentation/features-component.md new file mode 100644 index 00000000..a84b138d --- /dev/null +++ b/docs/code-documentation/features-component.md @@ -0,0 +1,981 @@ +# Features Component Documentation + +Comprehensive documentation of the Features layer in the Gateway application, explaining the architecture, patterns, and implementation details of all feature modules and their relationship to connectors, services, and workflows. + +## Table of Contents + +1. [Overview](#overview) +2. [What is a Feature?](#what-is-a-feature) +3. [Features vs Services vs Workflows](#features-vs-services-vs-workflows) +4. [Feature Architecture](#feature-architecture) +5. [Feature Lifecycle Management](#feature-lifecycle-management) +6. [Connectors in the Architecture](#connectors-in-the-architecture) +7. [Individual Features](#individual-features) +8. [Feature Patterns and Best Practices](#feature-patterns-and-best-practices) + +--- + +## Overview + +The **Features Layer** is a domain-specific business logic layer that implements core functionality for specific use cases. Features serve as **temporary solutions** that bridge the gap between initial requirements and full service implementation or workflow integration. They provide rapid prototyping capabilities while maintaining clean architectural boundaries. + +```mermaid +graph TB + subgraph "Application Layers" + Routes[Routes Layer
API Endpoints] + Features[Features Layer
Domain-Specific Logic] + Services[Services Layer
Reusable Components] + Workflows[Workflows Layer
Orchestration Engine] + Interfaces[Interfaces Layer
Data Access] + Connectors[Connectors Layer
External Systems] + end + + Routes --> Features + Routes --> Services + Routes --> Workflows + Features --> Services + Features --> Interfaces + Workflows --> Services + Services --> Interfaces + Interfaces --> Connectors + + style Features fill:#e8f5e9,stroke:#1b5e20,stroke-width:3px + style Connectors fill:#fff3e0,stroke:#e65100,stroke-width:2px +``` + +### Key Characteristics + +- **Domain-Specific**: Each feature addresses a specific business domain or use case +- **Temporary by Design**: Features are intended to be migrated to services or workflows over time +- **Stateless**: Features operate without maintaining session state +- **Service-Dependent**: Features leverage services for cross-cutting functionality +- **Interface-Dependent**: Features use interfaces to access data through connectors +- **Lifecycle-Managed**: Background features are managed through the Features Lifecycle system + +--- + +## What is a Feature? + +A **Feature** is a domain-specific business logic module that implements functionality for a particular use case. Features are designed to: + +1. **Rapid Prototyping**: Enable quick implementation of new functionality without full service architecture +2. **Domain Encapsulation**: Group related business logic for a specific domain (e.g., Real Estate, Chat, Data Synchronization) +3. **Temporary Solutions**: Serve as interim implementations before migration to services or workflows +4. **Orchestration**: Coordinate between services, interfaces, and external systems to fulfill business requirements +5. **Background Processing**: Support scheduled tasks, event-driven operations, and background managers + +### Feature Lifecycle Philosophy + +Features follow a natural evolution path: + +```mermaid +graph LR + A[Initial Requirement] --> B[Feature Implementation] + B --> C{Stability & Usage} + C -->|Mature| D[Service Migration] + C -->|Complex Workflow| E[Workflow Integration] + C -->|Still Experimental| B + + D --> F[Production Service] + E --> G[Workflow Component] + + style B fill:#fff3e0,stroke:#e65100 + style D fill:#e8f5e9,stroke:#1b5e20 + style E fill:#e1f5ff,stroke:#01579b +``` + +**When to Use Features:** +- New functionality that needs rapid development +- Domain-specific logic that may not be reusable +- Experimental or proof-of-concept implementations +- Background tasks requiring scheduled execution +- Integrations that are still being refined + +**When to Migrate to Services:** +- Functionality becomes reusable across multiple domains +- The feature is stable and well-tested +- Multiple features or routes need the same functionality +- The logic should be part of the core service layer + +**When to Migrate to Workflows:** +- The feature involves complex multi-step user interactions +- Task planning and adaptive learning are required +- The feature needs workflow orchestration capabilities +- User interactions require state management and progress tracking + +--- + +## Features vs Services vs Workflows + +Understanding the distinction between Features, Services, and Workflows is crucial for architectural decisions. + +```mermaid +graph TB + subgraph "Comparison Matrix" + A[Route Request] --> B{What Type?} + B -->|Domain-Specific
Single Use Case| C[Feature] + B -->|Reusable
Cross-Cutting| D[Service] + B -->|Complex Multi-Step
User Interaction| E[Workflow] + + C --> F[Uses Services] + C --> G[Uses Interfaces] + D --> H[Uses Other Services] + D --> G + E --> D + E --> I[Uses Methods] + + G --> J[Uses Connectors] + end + + style C fill:#fff3e0,stroke:#e65100 + style D fill:#e8f5e9,stroke:#1b5e20 + style E fill:#e1f5ff,stroke:#01579b + style J fill:#fce4ec,stroke:#880e4f +``` + +| Aspect | Feature | Service | Workflow | +|--------|---------|---------|----------| +| **Purpose** | Domain-specific business logic | Cross-cutting, reusable functionality | Complex multi-step orchestration | +| **Scope** | Single use case or domain | Multiple use cases | User interaction flows | +| **Reusability** | Low (domain-specific) | High (cross-domain) | Medium (workflow patterns) | +| **State Management** | Stateless | Stateless | Stateful (workflow state) | +| **Dependencies** | Uses services and interfaces | Uses other services and interfaces | Uses services and methods | +| **Lifecycle** | Temporary, may migrate | Permanent core component | Permanent orchestration engine | +| **Examples** | Real Estate queries, Chat Althaus scheduler | AI processing, Document extraction | Chat workflows, Task planning | + +### Decision Flow + +```mermaid +flowchart TD + Start[New Functionality Required] --> Q1{Is it reusable
across domains?} + Q1 -->|Yes| Service[Implement as Service] + Q1 -->|No| Q2{Does it require
complex multi-step
user interaction?} + Q2 -->|Yes| Workflow[Implement as Workflow] + Q2 -->|No| Q3{Is it domain-specific
or experimental?} + Q3 -->|Yes| Feature[Implement as Feature] + Q3 -->|No| Service + + Feature --> Q4{Feature Matures} + Q4 -->|Stable & Reusable| Service + Q4 -->|Complex Interactions| Workflow + Q4 -->|Still Experimental| Feature + + style Feature fill:#fff3e0,stroke:#e65100 + style Service fill:#e8f5e9,stroke:#1b5e20 + style Workflow fill:#e1f5ff,stroke:#01579b +``` + +--- + +## Feature Architecture + +### High-Level Architecture + +```mermaid +graph TB + subgraph "Entry Point" + App[app.py
FastAPI Application] + Lifecycle[Features Lifecycle
featuresLifecycle.py] + end + + subgraph "API Layer" + Routes[Routes
routeRealEstate.py
routeChatPlayground.py
routeDataNeutralization.py] + end + + subgraph "Feature Layer" + RE[Real Estate Feature
mainRealEstate.py] + CA[Chat Althaus Feature
mainChatAlthaus.py] + SD[Sync Delta Feature
mainSyncDelta.py] + CP[Chat Playground Feature
mainChatPlayground.py] + NP[Neutralize Playground Feature
mainNeutralizePlayground.py] + end + + subgraph "Service Layer" + Services[Services Container
AI, Chat, SharePoint, etc.] + end + + subgraph "Interface Layer" + Interfaces[Interfaces
Database, Ticket, etc.] + end + + subgraph "Connector Layer" + DBConn[Database Connector
connectorDbPostgre.py] + TicketConn[Ticket Connectors
connectorTicketsJira.py
connectorTicketsClickup.py] + VoiceConn[Voice Connector
connectorVoiceGoogle.py] + JsonConn[JSON Connector
connectorDbJson.py] + end + + subgraph "External Systems" + DB[(PostgreSQL Database)] + Jira[Jira API] + ClickUp[ClickUp API] + SharePoint[SharePoint API] + GoogleVoice[Google Voice API] + end + + App --> Lifecycle + App --> Routes + Lifecycle --> CA + Lifecycle --> SD + Routes --> RE + Routes --> CP + Routes --> NP + + RE --> Services + CA --> Services + SD --> Services + CP --> Services + NP --> Services + + Services --> Interfaces + Interfaces --> DBConn + Interfaces --> TicketConn + Interfaces --> VoiceConn + Interfaces --> JsonConn + + DBConn --> DB + TicketConn --> Jira + TicketConn --> ClickUp + VoiceConn --> GoogleVoice + Services --> SharePoint + + style Features fill:#fff3e0,stroke:#e65100 + style Connectors fill:#fce4ec,stroke:#880e4f +``` + +### Feature Request Flow + +```mermaid +sequenceDiagram + participant Client + participant Route as Route
routeRealEstate.py + participant Feature as Feature
mainRealEstate.py + participant Service as Service
Services Container + participant Interface as Interface
interfaceDbRealEstateObjects.py + participant Connector as Connector
connectorDbPostgre.py + participant DB as Database
PostgreSQL + + Client->>Route: HTTP Request + Route->>Route: Validate Request Data + Route->>Feature: Call Feature Function + Feature->>Service: Use Service (e.g., AI Service) + Service-->>Feature: Service Result + Feature->>Interface: Request Data Access + Interface->>Connector: Execute Query + Connector->>DB: SQL Query + DB-->>Connector: Raw Data + Connector-->>Interface: Raw Data + Interface->>Interface: Transform to Domain Objects + Interface-->>Feature: Domain Objects + Feature->>Feature: Process Business Logic + Feature-->>Route: Processed Result + Route->>Route: Serialize Response + Route-->>Client: HTTP Response +``` + +--- + +## Feature Lifecycle Management + +Features that require background processing, scheduled tasks, or event-driven operations are managed through the **Features Lifecycle** system. + +### Lifecycle Architecture + +```mermaid +graph TB + subgraph "Application Startup" + App[app.py
FastAPI Application] + Lifespan[Lifespan Context Manager] + end + + subgraph "Features Lifecycle" + Lifecycle[featuresLifecycle.py] + Start[start function] + Stop[stop function] + end + + subgraph "Background Features" + SyncDelta[SyncDelta Manager
startSyncManager] + ChatAlthaus[ChatAlthaus Manager
startDataScheduler] + AutomationEvents[Automation Events
syncAutomationEvents] + end + + App --> Lifespan + Lifespan -->|On Startup| Lifecycle + Lifecycle --> Start + Start --> SyncDelta + Start --> ChatAlthaus + Start --> AutomationEvents + + Lifespan -->|On Shutdown| Lifecycle + Lifecycle --> Stop + Stop --> SyncDelta + Stop --> ChatAlthaus + + style Lifecycle fill:#e8f5e9,stroke:#1b5e20 + style SyncDelta fill:#fff3e0,stroke:#e65100 + style ChatAlthaus fill:#fff3e0,stroke:#e65100 +``` + +### Lifecycle Sequence + +```mermaid +sequenceDiagram + participant App as app.py + participant Lifespan as Lifespan Manager + participant Lifecycle as featuresLifecycle + participant EventUser as Event User + participant SyncDelta as SyncDelta Manager + participant ChatAlthaus as ChatAlthaus Manager + + App->>Lifespan: Application Startup + Lifespan->>Lifecycle: start() + Lifecycle->>EventUser: getRootInterface().getUserByUsername("event") + EventUser-->>Lifecycle: Event User Object + + Lifecycle->>ChatAlthaus: syncAutomationEvents() + ChatAlthaus-->>Lifecycle: Events Synced + + Lifecycle->>SyncDelta: startSyncManager(eventUser) + SyncDelta->>SyncDelta: Initialize Background Thread + SyncDelta-->>Lifecycle: Manager Started + + Lifecycle->>ChatAlthaus: startDataScheduler(eventUser) + ChatAlthaus->>ChatAlthaus: Initialize Scheduler + ChatAlthaus-->>Lifecycle: Scheduler Started + + Lifecycle->>ChatAlthaus: performDataUpdate(eventUser) + ChatAlthaus-->>Lifecycle: Initial Update Complete + + Lifecycle-->>Lifespan: Startup Complete + Lifespan-->>App: Application Ready + + Note over App: Application Running... + + App->>Lifespan: Application Shutdown + Lifespan->>Lifecycle: stop() + Lifecycle->>SyncDelta: Stop Manager + Lifecycle->>ChatAlthaus: Stop Scheduler + Lifecycle-->>Lifespan: Shutdown Complete +``` + +### Lifecycle-Managed Features + +Features managed through the lifecycle system include: + +1. **SyncDelta**: Background synchronization manager for ticket synchronization +2. **ChatAlthaus**: Scheduled data updates for Althaus preprocessing service +3. **Automation Events**: Event synchronization for chat automation + +These features run continuously in the background and require proper initialization and cleanup during application startup and shutdown. + +--- + +## Connectors in the Architecture + +Connectors are the lowest-level abstraction for communicating with external systems. They provide concrete implementations for database connections, API integrations, and external service communication. + +### Connector Architecture + +```mermaid +graph TB + subgraph "Connector Types" + DBConn[Database Connectors
connectorDbPostgre.py
connectorDbJson.py] + TicketConn[Ticket Connectors
connectorTicketsJira.py
connectorTicketsClickup.py] + VoiceConn[Voice Connectors
connectorVoiceGoogle.py] + end + + subgraph "Connector Responsibilities" + Connection[Connection Management
Establish & Maintain Connections] + Query[Query Execution
Execute Queries & API Calls] + Transform[Data Transformation
Raw Data ↔ Application Format] + Error[Error Handling
Connection Errors & Retries] + end + + subgraph "External Systems" + PostgreSQL[(PostgreSQL Database)] + JSONFile[JSON Files] + JiraAPI[Jira API] + ClickUpAPI[ClickUp API] + GoogleVoiceAPI[Google Voice API] + end + + DBConn --> Connection + TicketConn --> Connection + VoiceConn --> Connection + + Connection --> Query + Query --> Transform + Transform --> Error + + DBConn --> PostgreSQL + DBConn --> JSONFile + TicketConn --> JiraAPI + TicketConn --> ClickUpAPI + VoiceConn --> GoogleVoiceAPI + + style DBConn fill:#fce4ec,stroke:#880e4f + style TicketConn fill:#fce4ec,stroke:#880e4f + style VoiceConn fill:#fce4ec,stroke:#880e4f +``` + +### Connector Usage Flow + +```mermaid +sequenceDiagram + participant Feature as Feature + participant Service as Service + participant Interface as Interface + participant Connector as Connector + participant External as External System + + Feature->>Service: Use Service + Service->>Interface: Request Data Access + Interface->>Connector: Initialize Connection + Connector->>External: Establish Connection + External-->>Connector: Connection Established + + Interface->>Connector: Execute Query/API Call + Connector->>Connector: Format Request + Connector->>External: Send Request + External-->>Connector: Raw Response + Connector->>Connector: Parse Response + Connector-->>Interface: Formatted Data + Interface->>Interface: Transform to Domain Objects + Interface-->>Service: Domain Objects + Service-->>Feature: Processed Result +``` + +### Available Connectors + +#### Database Connectors + +**connectorDbPostgre.py** - PostgreSQL Database Connector +- Manages PostgreSQL database connections +- Executes SQL queries with parameterization +- Handles JSONB column types +- Provides transaction support +- Used by: Real Estate interfaces, Chat interfaces, Application interfaces + +**connectorDbJson.py** - JSON File Database Connector +- Provides file-based data storage using JSON +- Useful for development and testing +- Lightweight alternative to PostgreSQL +- Used by: Development environments, Testing scenarios + +#### Ticket Connectors + +**connectorTicketsJira.py** - Jira Ticket Connector +- Integrates with Jira REST API +- Manages Jira tickets, issues, and projects +- Handles field mapping and synchronization +- Used by: SyncDelta feature, Ticket interfaces + +**connectorTicketsClickup.py** - ClickUp Ticket Connector +- Integrates with ClickUp API +- Manages ClickUp tasks and lists +- Handles task synchronization +- Used by: Ticket interfaces + +#### Voice Connectors + +**connectorVoiceGoogle.py** - Google Voice Connector +- Integrates with Google Voice API +- Handles voice transcription and processing +- Manages voice data and audio files +- Used by: Voice-related services and features + +### Connector Integration Pattern + +Connectors are never directly accessed by features. Instead, they follow this integration pattern: + +```mermaid +graph LR + A[Feature] --> B[Service] + B --> C[Interface] + C --> D[Connector] + D --> E[External System] + + style A fill:#fff3e0,stroke:#e65100 + style B fill:#e8f5e9,stroke:#1b5e20 + style C fill:#e1f5ff,stroke:#01579b + style D fill:#fce4ec,stroke:#880e4f + style E fill:#f5f5f5,stroke:#424242 +``` + +**Why This Pattern?** +- **Abstraction**: Interfaces hide connector implementation details +- **Flexibility**: Connectors can be swapped without affecting features +- **Testability**: Interfaces can be mocked for testing +- **Consistency**: All data access follows the same pattern +- **User Context**: Interfaces handle user context and access control + +--- + +## Individual Features + +### Real Estate Feature + +**Location**: `modules/features/realEstate/mainRealEstate.py` + +**Purpose**: Provides AI-powered natural language processing for Real Estate database operations. Enables users to interact with Real Estate data using natural language commands that are translated into CRUD operations. + +**Architecture**: + +```mermaid +graph TB + subgraph "Real Estate Feature" + Route[routeRealEstate.py] + Feature[mainRealEstate.py] + Intent[Intent Analysis
analyzeUserIntent] + CRUD[CRUD Operations
executeIntentBasedOperation] + Query[Direct Queries
executeDirectQuery] + end + + subgraph "Dependencies" + AIService[AI Service
Intent Recognition] + REInterface[Real Estate Interface
interfaceDbRealEstateObjects.py] + DBConnector[Database Connector
connectorDbPostgre.py] + end + + subgraph "Data Models" + REModels[Real Estate Models
Projekt, Parzelle, etc.] + end + + Route --> Feature + Feature --> Intent + Feature --> CRUD + Feature --> Query + + Intent --> AIService + CRUD --> REInterface + Query --> REInterface + + REInterface --> DBConnector + REInterface --> REModels + + style Feature fill:#fff3e0,stroke:#e65100 +``` + +**Key Functions**: +- `processNaturalLanguageCommand()`: Main entry point for natural language processing +- `analyzeUserIntent()`: Uses AI to analyze user input and extract intent, entity, and parameters +- `executeIntentBasedOperation()`: Executes CRUD operations based on analyzed intent +- `executeDirectQuery()`: Executes direct SQL queries without AI processing + +**Connector Usage**: +- Uses **Database Connector** (`connectorDbPostgre.py`) through the Real Estate Interface +- Accesses PostgreSQL database for Real Estate data +- Handles CRUD operations on entities like Projekt, Parzelle, Dokument + +**Service Integration**: +- Uses **AI Service** for intent recognition and natural language understanding +- Leverages AI planning capabilities to analyze user commands + +**Migration Path**: +- May evolve into a **Service** if Real Estate operations become reusable across domains +- Could integrate with **Workflows** for complex multi-step Real Estate processes + +--- + +### Chat Althaus Feature + +**Location**: `modules/features/chatAlthaus/mainChatAlthaus.py` + +**Purpose**: Manages scheduled data updates for the Althaus preprocessing service. Triggers daily updates to synchronize database configuration with external preprocessing service. + +**Architecture**: + +```mermaid +graph TB + subgraph "Chat Althaus Feature" + Lifecycle[featuresLifecycle.py] + Manager[ManagerChatAlthaus] + Scheduler[Data Scheduler] + Updater[Data Updater
updateDatabaseWithConfig] + end + + subgraph "Dependencies" + Services[Services Container] + HTTPClient[HTTP Client
aiohttp] + Config[Configuration
APP_CONFIG] + end + + subgraph "External System" + AlthausAPI[Althaus Preprocessing API
Azure Function] + end + + Lifecycle --> Manager + Manager --> Scheduler + Manager --> Updater + + Updater --> Services + Updater --> HTTPClient + Updater --> Config + + Updater --> AlthausAPI + + style Manager fill:#fff3e0,stroke:#e65100 +``` + +**Key Functions**: +- `startDataScheduler()`: Initializes and starts the scheduled data update manager +- `performDataUpdate()`: Executes immediate data update +- `updateDatabaseWithConfig()`: Sends configuration to Althaus preprocessing service + +**Scheduling**: +- Runs daily at 01:00 UTC +- Uses background scheduler for automated execution +- Managed through Features Lifecycle system + +**Connector Usage**: +- Uses **HTTP Client** (aiohttp) for API communication +- No database connector (uses external API) + +**Service Integration**: +- Uses **Services Container** for configuration access +- Leverages shared configuration utilities + +**Migration Path**: +- Could become a **Service** if data synchronization becomes a core capability +- May integrate with **Workflows** for complex data processing pipelines + +--- + +### Sync Delta Feature + +**Location**: `modules/features/syncDelta/mainSyncDelta.py` + +**Purpose**: Synchronizes tickets between Jira and SharePoint. Manages bidirectional synchronization of ticket data, supporting both CSV and Excel file formats. + +**Architecture**: + +```mermaid +graph TB + subgraph "Sync Delta Feature" + Lifecycle[featuresLifecycle.py] + Manager[ManagerSyncDelta] + Sync[syncTicketsOverSharepoint] + Merge[Data Merging Logic] + Audit[Audit Logging] + end + + subgraph "Dependencies" + TicketService[Ticket Service] + SharePointService[SharePoint Service] + JiraConnector[Jira Connector
connectorTicketsJira.py] + end + + subgraph "External Systems" + Jira[Jira API] + SharePoint[SharePoint API] + end + + Lifecycle --> Manager + Manager --> Sync + Sync --> Merge + Sync --> Audit + + Sync --> TicketService + Sync --> SharePointService + + TicketService --> JiraConnector + JiraConnector --> Jira + SharePointService --> SharePoint + + style Manager fill:#fff3e0,stroke:#e65100 +``` + +**Key Functions**: +- `startSyncManager()`: Initializes background synchronization manager +- `syncTicketsOverSharepoint()`: Performs synchronization between Jira and SharePoint +- `initializeInterface()`: Sets up connectors and validates connections +- `_logAuditEvent()`: Logs synchronization events for auditing + +**Synchronization Modes**: +- **CSV Mode**: Uses CSV files for data exchange +- **Excel Mode**: Uses Excel (.xlsx) files for data exchange + +**Connector Usage**: +- Uses **Jira Connector** (`connectorTicketsJira.py`) through Ticket Service +- Uses **SharePoint Service** for file operations +- Manages field mapping between Jira and SharePoint formats + +**Service Integration**: +- Uses **Ticket Service** for ticket interface creation +- Uses **SharePoint Service** for file upload/download +- Leverages **Services Container** for configuration and utilities + +**Migration Path**: +- Likely candidate for **Service** migration as ticket synchronization becomes core functionality +- Could integrate with **Workflows** for complex synchronization scenarios + +--- + +### Chat Playground Feature + +**Location**: `modules/features/chatPlayground/mainChatPlayground.py` + +**Purpose**: Provides entry point for chat workflow functionality. Acts as a thin wrapper around the WorkflowManager for chat-based interactions. + +**Architecture**: + +```mermaid +graph TB + subgraph "Chat Playground Feature" + Route[routeChatPlayground.py] + Feature[mainChatPlayground.py] + Start[chatStart] + Stop[chatStop] + end + + subgraph "Workflow System" + WorkflowManager[WorkflowManager] + WorkflowProcessor[WorkflowProcessor] + Methods[Workflow Methods] + end + + subgraph "Dependencies" + Services[Services Container] + end + + Route --> Feature + Feature --> Start + Feature --> Stop + + Start --> WorkflowManager + Stop --> WorkflowManager + + WorkflowManager --> WorkflowProcessor + WorkflowProcessor --> Methods + + WorkflowManager --> Services + + style Feature fill:#fff3e0,stroke:#e65100 + style WorkflowManager fill:#e1f5ff,stroke:#01579b +``` + +**Key Functions**: +- `chatStart()`: Starts a new chat workflow or continues an existing one +- `chatStop()`: Stops a running chat workflow + +**Relationship to Workflows**: +- This feature is a **bridge** between routes and the workflow system +- Delegates all processing to WorkflowManager +- Demonstrates how features can integrate with workflows + +**Connector Usage**: +- No direct connector usage (delegates to workflows) +- Workflows use connectors through services and methods + +**Service Integration**: +- Uses **Services Container** for workflow management +- Leverages workflow services for chat operations + +**Migration Path**: +- Already integrated with **Workflows** system +- May be simplified or removed as workflows become the primary interface + +--- + +### Neutralize Playground Feature + +**Location**: `modules/features/neutralizePlayground/mainNeutralizePlayground.py` + +**Purpose**: Provides a playground interface for data neutralization functionality. Wraps the Neutralization Service for testing and experimentation. + +**Architecture**: + +```mermaid +graph TB + subgraph "Neutralize Playground Feature" + Route[routeDataNeutralization.py] + Feature[NeutralizationPlayground] + ProcessText[processText] + ProcessFiles[processFiles] + CleanAttributes[cleanAttributes] + Stats[getStats] + Config[getConfig/saveConfig] + end + + subgraph "Dependencies" + NeutralizationService[Neutralization Service] + end + + Route --> Feature + Feature --> ProcessText + Feature --> ProcessFiles + Feature --> CleanAttributes + Feature --> Stats + Feature --> Config + + ProcessText --> NeutralizationService + ProcessFiles --> NeutralizationService + CleanAttributes --> NeutralizationService + Stats --> NeutralizationService + Config --> NeutralizationService + + style Feature fill:#fff3e0,stroke:#e65100 + style NeutralizationService fill:#e8f5e9,stroke:#1b5e20 +``` + +**Key Functions**: +- `processText()`: Processes text for data neutralization +- `processFiles()`: Processes files for data neutralization +- `cleanAttributes()`: Cleans neutralization attributes +- `getStats()`: Retrieves neutralization statistics +- `getConfig()` / `saveConfig()`: Manages neutralization configuration + +**Purpose as Playground**: +- Provides testing interface for neutralization functionality +- Allows experimentation with neutralization patterns +- Demonstrates service usage patterns + +**Connector Usage**: +- No direct connector usage (uses Neutralization Service) +- Service handles all data access internally + +**Service Integration**: +- Wraps **Neutralization Service** for easy access +- Provides playground-specific functionality + +**Migration Path**: +- May be removed once neutralization is fully integrated +- Functionality may move directly to routes using the service + +--- + +## Feature Patterns and Best Practices + +### Pattern 1: Stateless Feature Design + +Features should be stateless and operate without session management. Each request should be independent and self-contained. + +```mermaid +graph LR + A[Request] --> B[Feature Function] + B --> C[Process Request] + C --> D[Return Result] + + style B fill:#fff3e0,stroke:#e65100 +``` + +**Benefits**: +- Simpler implementation +- Better scalability +- Easier testing +- No state management overhead + +### Pattern 2: Service Delegation + +Features should delegate cross-cutting functionality to services rather than implementing it directly. + +```mermaid +graph TB + A[Feature] --> B{Needs Functionality} + B -->|AI Processing| C[AI Service] + B -->|Data Access| D[Interface] + B -->|File Operations| E[File Service] + B -->|Other| F[Other Services] + + style A fill:#fff3e0,stroke:#e65100 + style C fill:#e8f5e9,stroke:#1b5e20 + style D fill:#e1f5ff,stroke:#01579b +``` + +**Benefits**: +- Code reuse +- Consistent behavior +- Easier maintenance +- Better separation of concerns + +### Pattern 3: Interface Abstraction + +Features should never directly access connectors. All data access should go through interfaces. + +```mermaid +graph LR + A[Feature] --> B[Interface] + B --> C[Connector] + C --> D[External System] + + style A fill:#fff3e0,stroke:#e65100 + style B fill:#e1f5ff,stroke:#01579b + style C fill:#fce4ec,stroke:#880e4f +``` + +**Benefits**: +- Abstraction of implementation details +- Flexibility to change connectors +- Consistent data access patterns +- User context handling + +### Pattern 4: Background Processing + +Features requiring background processing should use the Features Lifecycle system. + +```mermaid +sequenceDiagram + participant App as Application + participant Lifecycle as Features Lifecycle + participant Feature as Background Feature + participant Scheduler as Scheduler + + App->>Lifecycle: Startup + Lifecycle->>Feature: Initialize Manager + Feature->>Scheduler: Start Background Task + Scheduler-->>Feature: Task Running + + Note over Feature,Scheduler: Background Processing... + + App->>Lifecycle: Shutdown + Lifecycle->>Feature: Stop Manager + Feature->>Scheduler: Stop Background Task + Scheduler-->>Feature: Task Stopped +``` + +**Benefits**: +- Proper lifecycle management +- Clean startup/shutdown +- Resource management +- Error handling + +### Pattern 5: Migration Planning + +Features should be designed with migration in mind. Consider future migration to services or workflows during design. + +```mermaid +graph LR + A[Feature Design] --> B{Consider Migration} + B -->|Reusable Logic| C[Design as Service] + B -->|Complex Flow| D[Design for Workflow] + B -->|Temporary| E[Keep as Feature] + + style A fill:#fff3e0,stroke:#e65100 + style C fill:#e8f5e9,stroke:#1b5e20 + style D fill:#e1f5ff,stroke:#01579b +``` + +**Best Practices**: +- Document migration path +- Keep dependencies minimal +- Use standard patterns +- Plan for refactoring + +--- + +## Summary + +The Features component provides a flexible, domain-specific business logic layer that enables rapid development while maintaining architectural boundaries. Features serve as temporary solutions that bridge the gap between initial requirements and full service or workflow implementation. + +**Key Takeaways**: + +1. **Features are Temporary**: Designed to be migrated to services or workflows as they mature +2. **Domain-Specific**: Each feature addresses a specific business domain or use case +3. **Service-Dependent**: Features leverage services for cross-cutting functionality +4. **Interface-Abstracted**: Features access data through interfaces, never directly through connectors +5. **Lifecycle-Managed**: Background features are managed through the Features Lifecycle system +6. **Connector Integration**: Connectors are accessed through interfaces, providing abstraction and flexibility + +The architecture supports a natural evolution path from features to services or workflows, ensuring that the codebase remains maintainable and scalable as functionality matures. + diff --git a/docs/code-documentation/gateway-development-framework.md b/docs/code-documentation/gateway-development-framework.md new file mode 100644 index 00000000..9ce651a0 --- /dev/null +++ b/docs/code-documentation/gateway-development-framework.md @@ -0,0 +1,2281 @@ +# Gateway Development Framework: Connectors → Interfaces → Services → Workflows + +This document explains the gateway's code logic and development framework to build market customer journey features. It focuses on how connectors, interfaces, services, and workflows compose a standardized services landscape that can be consumed by routes, features, and agent models to perform tasks and actions. + +--- + +## Purpose + +- **Unify external tools**: Combine many third‑party APIs and utilities behind a consistent interface. + +- **Standardize service design**: Model capabilities as reusable services with clear contracts. + +- **Enable workflow automation**: Let agent models orchestrate multi‑step tasks using the centralized services. + +- **Abstract complexity**: Hide implementation details behind clean, well-defined APIs. + +- **Enforce security and governance**: Apply consistent access control, audit trails, and data isolation across all layers. + +--- + +## High‑Level Architecture + +The Gateway follows a layered architecture pattern with clear separation of concerns: + +1. **Connectors**: Vendor-specific adapters for external systems (databases, APIs, cloud services) handling auth, transport, retries, and basic mapping. + +2. **Interfaces**: Normalization layer exposing common contracts independent of any single vendor. Provides CRUD operations, access control, and data transformation. + +3. **Services**: Business‑level capabilities built on interfaces, composed into feature‑ready functions. Orchestrate multiple interfaces and apply business rules. + +4. **Service Center**: Central registry/factory (`Services` class) that instantiates and exposes services with consistent configuration, user context, and lifecycle management. + +5. **Workflows & Methods**: Orchestration engine that calls services to perform tasks/actions. Methods provide extensible, plugin-like actions that workflows can invoke. + +**Data/control flow**: Client or Workflow → Service Center → Service → Interface → Connector → External Tool/Database + +--- + +## Directory Overview (gateway) + +``` +gateway/ +├── modules/ +│ ├── connectors/ # Vendor-specific adapters +│ │ ├── connectorDbPostgre.py # PostgreSQL database +│ │ ├── connectorDbJson.py # JSON file-based database +│ │ ├── connectorVoiceGoogle.py # Google Cloud Speech services +│ │ ├── connectorTicketsJira.py # JIRA integration +│ │ └── connectorTicketsClickup.py # ClickUp integration +│ │ +│ ├── datamodels/ # Pydantic models defining data structures +│ │ ├── datamodelRealEstate.py +│ │ ├── datamodelChat.py +│ │ ├── datamodelAi.py +│ │ ├── datamodelUam.py # User & Mandate models +│ │ └── ... +│ │ +│ ├── interfaces/ # Data access layer +│ │ ├── interfaceDbRealEstateObjects.py # CRUD operations +│ │ ├── interfaceDbRealEstateAccess.py # Access control +│ │ ├── interfaceDbChatObjects.py +│ │ ├── interfaceDbChatAccess.py +│ │ ├── interfaceDbAppObjects.py +│ │ ├── interfaceDbComponentObjects.py +│ │ ├── interfaceAiObjects.py # AI operations +│ │ ├── interfaceTicketObjects.py # Ticket systems +│ │ └── interfaceVoiceObjects.py # Voice operations +│ │ +│ ├── services/ # Business-level capabilities +│ │ ├── __init__.py # Services container (Service Center) +│ │ ├── serviceAi/ # AI operations +│ │ ├── serviceChat/ # Workflow & document management +│ │ ├── serviceExtraction/ # Content extraction +│ │ ├── serviceGeneration/ # Document generation +│ │ ├── serviceNeutralization/ # Data anonymization +│ │ ├── serviceSharepoint/ # SharePoint integration +│ │ ├── serviceTicket/ # Ticket system integration +│ │ └── serviceUtils/ # Common utilities +│ │ +│ ├── workflows/ # Orchestration engine +│ │ ├── workflowManager.py # Main orchestration controller +│ │ ├── processing/ # Processing logic +│ │ │ ├── workflowProcessor.py +│ │ │ ├── core/ # Core components +│ │ │ ├── modes/ # Execution modes +│ │ │ └── shared/ # Shared utilities +│ │ └── methods/ # Extensible action methods +│ │ ├── methodBase.py +│ │ ├── methodAi.py +│ │ └── ... +│ │ +│ ├── routes/ # HTTP endpoints exposing capabilities +│ │ ├── routeChatPlayground.py +│ │ ├── routeWorkflows.py +│ │ └── ... +│ │ +│ ├── features/ # Domain-specific business logic +│ │ └── mainChatPlayground.py +│ │ +│ ├── security/ # Authentication, authorization, token management +│ │ ├── auth.py +│ │ ├── jwtService.py +│ │ ├── tokenManager.py +│ │ └── ... +│ │ +│ └── shared/ # Cross-cutting utilities +│ ├── config.py +│ ├── logging.py +│ └── ... +``` + +--- + +## 1) Connectors: Many External Tools, One Adapter Shape + +**Role**: Provide the lowest-level integration with external systems (databases, APIs, SDKs, auth, retries). + +**Responsibility**: + +- **Authentication and credential handling**: Manage API keys, OAuth tokens, database credentials +- **Transport**: HTTP/WebSocket clients, connection pooling, retry logic, circuit breaking +- **Response normalization**: Map vendor-specific responses to minimal internal shapes +- **Error handling**: Transform external errors into consistent internal error structures + +**Output**: Vendor‑flavored data mapped to connector models, not directly used by workflows or services. + +**Key Guidelines**: + +- Keep connectors vendor‑specific and replaceable (e.g., `connectorDbPostgre.py` vs `connectorDbJson.py`) +- No business logic; only integration concerns and basic mapping +- Use duck typing (no formal interfaces) for flexibility +- Handle retries, timeouts, and connection management internally +- Return structured error responses, never raise exceptions to application layer + +**Example Connector Types**: + +- **Database Connectors**: PostgreSQL (`connectorDbPostgre.py`), JSON file-based (`connectorDbJson.py`) +- **Voice Connectors**: Google Cloud Speech (`connectorVoiceGoogle.py`) +- **Ticket Connectors**: JIRA (`connectorTicketsJira.py`), ClickUp (`connectorTicketsClickup.py`) + +--- + +## 2) Interfaces: Stable Contracts Over Connectors + +**Role**: Define capability‑oriented contracts (e.g., `ChatObjects`, `AppObjects`, `AiObjects`) and map connector outputs into interface DTOs. + +**Responsibility**: + +- **Normalize differing vendors**: Convert vendor-specific data into consistent domain objects +- **Hide vendor peculiarities**: Abstract away implementation details behind clean, typed DTOs +- **Provide CRUD operations**: Create, Read, Update, Delete methods for domain entities +- **Enforce access control**: Apply user privilege checks and mandate-based filtering +- **Offer capability toggles**: Sensible defaults and configuration options + +**Output**: Clean, stable methods used by services (e.g., `getWorkflow()`, `createMessage()`, `call()`). + +**Interface Structure**: + +Interfaces are split into two file types: + +- **Objects Files** (`interface*Objects.py`): CRUD operations and business logic +- **Access Files** (`interface*Access.py`): Permission checking and data filtering + +**Key Guidelines**: + +- Prefer capability names over vendor names (e.g., `ChatObjects` not `PostgreChatObjects`) +- Keep interfaces small, cohesive, and testable with mocks +- Always require user context for database interfaces (enables access control) +- Use Pydantic models (datamodels) for type safety +- Apply Unified Access Management (UAM) for all database queries + +**Example Interface Types**: + +- **Database Interfaces**: `interfaceDbChatObjects`, `interfaceDbAppObjects`, `interfaceDbRealEstateObjects` +- **External System Interfaces**: `interfaceAiObjects`, `interfaceTicketObjects`, `interfaceVoiceObjects` + +--- + +## 3) Services: Business‑Level Capabilities + +**Role**: Compose one or more interfaces to implement feature‑ready operations (e.g., "answer question with web grounding", "extract and analyze documents"). + +**Responsibility**: + +- **Apply business rules**: Validation, guardrails, transformations, data enrichment +- **Orchestrate multiple interfaces**: Coordinate between interfaces and other services +- **Emit domain events/metrics**: Track operations, costs, performance +- **Enforce security policies**: Apply additional security checks beyond interface layer +- **Handle complex workflows**: Multi-step operations with error recovery + +**Output**: High‑level operations that workflows and routes can call atomically. + +**Service Container Pattern**: + +All services are initialized through the `Services` container. Initialize with user context using `Services(user=current_user, workflow=current_workflow)`, then access services via `services.ai.callAiDocuments()`, `services.chat.storeMessageWithDocuments()`, etc. + +**Key Guidelines**: + +- Services depend on interfaces, not connectors directly +- Keep input/output DTOs explicit and versioned when necessary +- Services can call other services via `self.services` +- Use `PublicService` wrapper to expose only public methods +- Keep services stateless (no session state, use database for persistence) + +**Core Services**: + +- **AI Service**: AI model operations, planning, document processing +- **Chat Service**: Workflow management, message handling, document resolution +- **Extraction Service**: Multi-format document extraction and processing +- **Generation Service**: Document rendering in various formats +- **Neutralization Service**: Data anonymization for GDPR compliance +- **SharePoint Service**: SharePoint integration +- **Ticket Service**: Ticket system integration (Jira, ClickUp) +- **Utils Service**: Common utilities (config, events, time, debug) + +--- + +## 4) Centralized Service Center + +**Role**: A registry/factory (`Services` class) that instantiates and exposes services with consistent configuration and lifecycle. + +**Responsibility**: + +- **Discoverability**: List/get services by capability key (e.g., `services.ai`, `services.chat`) +- **Configuration**: Environment, credentials, routing to specific vendors +- **Cross‑cutting**: User context, workflow context, interface access +- **Lifecycle management**: Initialize services with proper dependencies +- **Access control**: Provide user context to all services and interfaces + +**Usage Pattern**: + +1. Route receives request with authenticated user (via `getCurrentUser` dependency) +2. Create Services container with user context using `Services(user=currentUser)` +3. Call service method with typed input (e.g., `services.ai.callAiDocuments()`) +4. Receive typed output + +**Service Center Structure**: + +The `Services` class initializes with user and optional workflow context. It initializes interfaces via `getChatInterface()`, `getAppInterface()`, `getComponentInterface()`, and wraps all services in `PublicService` wrappers (e.g., `PublicService(AiService(self))`). + +**Key Features**: + +- **User Context**: Every service has access to `self.services.user` for access control +- **Workflow Context**: Services can access `self.services.workflow` for workflow-aware operations +- **Interface Access**: Services access interfaces via `self.services.interfaceDbChat`, etc. +- **Service Composition**: Services call other services via `self.services.otherService.method()` + +--- + +## 5) Workflows & Agent Models + +**Role**: Coordinate tasks and actions by invoking services in sequence/branches/loops. + +**Responsibility**: + +- **Maintain execution state**: Track workflow progress, round/task/action counters +- **Choose actions**: Use agent models (AI) or predefined plans to determine next steps +- **Handle retries/compensation**: Retry failed tasks with improvements, rollback on failure +- **Record audit logs**: Track all workflow steps, decisions, and outcomes +- **Manage document flow**: Resolve document references, track document lineage + +**Typical Pattern**: + +1. **Ingest user intent/context**: Analyze user input, extract documents, detect language +2. **Plan next action**: Use AI to generate task plan or follow predefined JSON plan +3. **Call services via Service Center**: Invoke services to perform operations +4. **Persist outputs**: Store results, update state, decide next step +5. **Generate feedback**: Create completion messages, summarize results + +**Workflow Modes**: + +- **Actionplan Mode**: Batch planning with quality review and intelligent retry +- **Dynamic Mode**: Iterative, just-in-time action generation +- **Automation Mode**: Predefined JSON-based deterministic execution + +**Method System**: + +Workflows invoke actions through an extensible method system: + +- **Methods**: Plugin-like classes that expose actions via `@action` decorator +- **Actions**: Async methods that perform specific operations (e.g., `methodAi.process()`, `methodSharepoint.search()`) +- **Automatic Discovery**: Methods are discovered at runtime via introspection +- **Signature Generation**: Action signatures are generated for AI prompt generation + +--- + +## Standardized Interface Example (Actual Implementation) + +Interfaces like `ChatObjects` provide methods such as `getWorkflow()` and `createMessage()`. The `AiObjects` interface provides `call()` for AI model operations. Vendors like OpenAI/Anthropic implement `AiObjects` through connectors; database connectors implement `ChatObjects`. Services compose these interfaces. + +--- + +## Example Service Composition (Actual Implementation) + +The `AiService.callAiDocuments()` method demonstrates service composition: + +**Steps**: +1. `ExtractionService.extractContent()` → extracts content from documents +2. `AiObjects.call()` → processes with AI model +3. `ChatService.storeWorkflowStat()` → records statistics + +**Outputs**: AI-generated content, processing statistics, cost tracking + +--- + +## Adding a New Capability + +### Step 1: Create Connector (if needed) + +Add vendor adapter in `modules/connectors/` (e.g., `connectorNewVendor.py`). The connector class should initialize with configuration, handle API calls, and return structured responses with `{"success": True/False, "data": ...}` format. + +### Step 2: Create Interface + +Implement capability contract in `modules/interfaces/` (e.g., `interfaceNewCapabilityObjects.py`). The interface class should initialize with user context, use the connector, and provide normalized methods like `performOperation()` that return domain objects. + +### Step 3: Create Service + +Compose the interface in `modules/services/serviceNewCapability/mainServiceNewCapability.py`. The service class should initialize with the services container, access the interface, and provide business-level methods like `performBusinessOperation()` that apply validation, call the interface, and enrich results. + +### Step 4: Register in Service Center + +Wire into `Services` class in `modules/services/__init__.py`. Import the service class and wrap it in `PublicService()` (e.g., `self.newCapability = PublicService(NewCapabilityService(self))`). + +### Step 5: Expose via Route (if needed) + +Add HTTP endpoint in `modules/routes/routeNewCapability.py`. Create a route handler that uses `getCurrentUser` dependency, creates a `Services` instance, calls the service method, and returns the result. + +### Step 6: Use in Workflows (if needed) + +Create method action in `modules/workflows/methods/methodNewCapability.py`. Inherit from `MethodBase`, use the `@action` decorator on async methods, and return `ActionResult` objects with success status and documents. + +--- + +## Adding a New Database Domain + +Adding a completely new database domain (like Real Estate, Projects, Inventory) requires creating datamodels, database interfaces, and access control. This section covers creating a new domain from scratch. + +### Overview + +A new database domain consists of: +1. **Datamodels**: Pydantic models defining data structures +2. **Database Interface Objects**: CRUD operations for domain entities +3. **Database Interface Access**: Access control and permission checking +4. **Database Configuration**: Connection settings for the new database +5. **Service Integration**: Optional service layer for business logic + +### Step 1: Create Datamodels + +Create a new datamodel file in `modules/datamodels/datamodel[Domain].py` (e.g., `datamodelRealEstate.py`, `datamodelProject.py`). + +**Structure**: +- Define Pydantic models inheriting from `BaseModel` +- Include enums for status fields and categories +- Add helper models for complex nested structures +- Use Field() with frontend metadata for UI generation +- Include standard fields: `id`, `mandateId`, `_createdBy`, `_createdAt`, `_modifiedBy`, `_modifiedAt` + +**Example Structure**: +``` +datamodel[Domain].py +├── Enums (StatusEnum, CategoryEnum, etc.) +├── Helper Models (GeoPoint, Address, etc.) +├── Main Entity Models +│ ├── Entity1 (id, mandateId, fields, timestamps) +│ ├── Entity2 (id, mandateId, fields, timestamps) +│ └── Entity3 (id, mandateId, fields, timestamps) +└── Relationship Models (if needed) +``` + +**Key Requirements**: +- All main entities must have `id: str` (UUID) +- All main entities must have `mandateId: str` for multi-tenant isolation +- Include audit fields: `_createdBy`, `_createdAt`, `_modifiedBy`, `_modifiedAt` +- Use `Field()` with `frontend_type`, `frontend_readonly`, `frontend_required` for UI metadata +- Define relationships using ForwardRef if models reference each other + +**Example**: +``` +from pydantic import BaseModel, Field +from enum import Enum +import uuid + +class StatusEnum(str, Enum): + ACTIVE = "active" + INACTIVE = "inactive" + ARCHIVED = "archived" + +class Project(BaseModel): + id: str = Field(default_factory=lambda: str(uuid.uuid4())) + mandateId: str + name: str = Field(..., frontend_type="text", frontend_required=True) + status: StatusEnum = Field(..., frontend_type="select") + description: Optional[str] = Field(None, frontend_type="textarea") + _createdBy: Optional[str] = None + _createdAt: Optional[int] = None + _modifiedBy: Optional[str] = None + _modifiedAt: Optional[int] = None +``` + +### Step 2: Create Database Interface Objects + +Create `modules/interfaces/interfaceDb[Domain]Objects.py` (e.g., `interfaceDbRealEstateObjects.py`). + +**Structure**: +- `[Domain]Objects` class that initializes database connector +- CRUD methods for each entity: `create[Entity]()`, `get[Entity]()`, `get[Entities]()`, `update[Entity]()`, `delete[Entity]()` +- Query execution method: `executeQuery()` for custom SQL queries +- User context management: `setUserContext()` + +**Key Components**: + +**1. Database Initialization**: +``` +def _initializeDatabase(self): + dbHost = APP_CONFIG.get("DB_[DOMAIN]_HOST", "localhost") + dbDatabase = APP_CONFIG.get("DB_[DOMAIN]_DATABASE", "poweron_[domain]") + dbUser = APP_CONFIG.get("DB_[DOMAIN]_USER") + dbPassword = APP_CONFIG.get("DB_[DOMAIN]_PASSWORD_SECRET") + dbPort = int(APP_CONFIG.get("DB_[DOMAIN]_PORT", 5432)) + + self.db = DatabaseConnector( + dbHost=dbHost, + dbDatabase=dbDatabase, + dbUser=dbUser, + dbPassword=dbPassword, + dbPort=dbPort, + userId=self.userId if self.userId else None, + ) + + self.db.initDbSystem() +``` + +**2. CRUD Pattern**: +``` +def create[Entity](self, entity: [Entity]) -> [Entity]: + # Ensure mandateId is set + if not entity.mandateId: + entity.mandateId = self.mandateId + + # Apply access control + self.access.uam([Entity], []) + + # Save to database + self.db.recordCreate([Entity], entity.model_dump()) + + return entity + +def get[Entity](self, entityId: str) -> Optional[[Entity]]: + records = self.db.getRecordset( + [Entity], + recordFilter={"id": entityId} + ) + + if not records: + return None + + # Apply access control + filtered = self.access.uam([Entity], records) + + if not filtered: + return None + + return [Entity](**filtered[0]) + +def get[Entities](self, filters: Optional[Dict] = None) -> List[[Entity]]: + records = self.db.getRecordset([Entity], recordFilter=filters or {}) + filtered = self.access.uam([Entity], records) + return [Entity](**r) for r in filtered] + +def update[Entity](self, entityId: str, updates: Dict) -> Optional[[Entity]]: + # Check access control + self.access.canModify([Entity], entityId) + + # Update in database + self.db.recordUpdate([Entity], entityId, updates) + + # Return updated entity + return self.get[Entity](entityId) + +def delete[Entity](self, entityId: str) -> bool: + # Check access control + self.access.canModify([Entity], entityId) + + # Delete from database + self.db.recordDelete([Entity], entityId) + + return True +``` + +**3. Singleton Factory Pattern**: +``` +_[domain]Interfaces = {} + +def get[Domain]Interface(currentUser: Optional[User] = None) -> [Domain]Objects: + """Factory function to get or create interface instance.""" + userId = currentUser.id if currentUser else None + if userId not in _[domain]Interfaces: + _[domain]Interfaces[userId] = [Domain]Objects(currentUser) + return _[domain]Interfaces[userId] +``` + +### Step 3: Create Database Interface Access + +Create `modules/interfaces/interfaceDb[Domain]Access.py` (e.g., `interfaceDbRealEstateAccess.py`). + +**Structure**: +- `[Domain]Access` class that handles permission checking +- `uam()` method for Unified Access Management (filtering and flagging) +- `canModify()` method for write permission checking + +**Key Components**: + +**1. Access Control Class**: +``` +class [Domain]Access: + def __init__(self, currentUser: User, db: DatabaseConnector): + self.currentUser = currentUser + self.userId = currentUser.id + self.mandateId = currentUser.mandateId + self.userRole = currentUser.role + self.db = db + + def uam(self, model: Type[BaseModel], records: List[Dict]) -> List[Dict]: + """Unified Access Management: Filter records and add access flags.""" + if self.userRole == "SYSADMIN": + # SYSADMIN sees all records + filtered = records + elif self.userRole == "ADMIN": + # ADMIN sees all records in their mandate + filtered = [r for r in records if r.get("mandateId") == self.mandateId] + else: + # USER sees only their own records + filtered = [r for r in records if r.get("_createdBy") == self.userId] + + # Add access flags + for record in filtered: + record["_hideView"] = False + record["_hideEdit"] = not self.canModify(model, record.get("id")) + record["_hideDelete"] = not self.canModify(model, record.get("id")) + + return filtered + + def canModify(self, model: Type[BaseModel], recordId: str) -> bool: + """Check if user can modify a record.""" + if self.userRole == "SYSADMIN": + return True + + # Get record to check ownership + records = self.db.getRecordset(model, recordFilter={"id": recordId}) + if not records: + return False + + record = records[0] + + if self.userRole == "ADMIN": + # ADMIN can modify records in their mandate + return record.get("mandateId") == self.mandateId + else: + # USER can only modify their own records + return record.get("_createdBy") == self.userId +``` + +**2. Import in Objects File**: +``` +from modules.interfaces.interfaceDb[Domain]Access import [Domain]Access +``` + +### Step 4: Configure Database Connection + +Add database configuration to `config.ini`: + +``` +[Database] +DB_[DOMAIN]_HOST=localhost +DB_[DOMAIN]_DATABASE=poweron_[domain] +DB_[DOMAIN]_USER=postgres +DB_[DOMAIN]_PASSWORD_SECRET=your_password +DB_[DOMAIN]_PORT=5432 +``` + +**Database Creation**: +- The `DatabaseConnector.initDbSystem()` method automatically creates the database if it doesn't exist +- Tables are created on-demand when first accessed via `_ensureTableExists()` +- No manual database schema creation needed + +### Step 5: Register Interface in Services (Optional) + +If you need business logic, create a service that uses the interface: + +**Create Service** (`modules/services/service[Domain]/mainService[Domain].py`): +``` +class [Domain]Service: + def __init__(self, services: 'Services'): + self.services = services + + def get[Domain]Interface(self) -> [Domain]Objects: + """Get interface instance with current user context.""" + return get[Domain]Interface(self.services.workflow.currentUser) + + def performBusinessOperation(self, ...): + """Business-level method that uses interface.""" + interface = self.get[Domain]Interface() + # Apply business logic + # Call interface methods + # Return enriched results +``` + +**Register in Service Center** (`modules/services/__init__.py`): +``` +from modules.services.service[Domain].mainService[Domain] import [Domain]Service + +class Services: + def __init__(self, ...): + ... + self.[domain] = PublicService([Domain]Service(self)) +``` + +### Step 6: Create Routes (Optional) + +If you need HTTP endpoints, create `modules/routes/route[Domain].py`: + +``` +from fastapi import APIRouter, Depends +from modules.features.shared.dependencies import getCurrentUser +from modules.datamodels.datamodelUam import User +from modules.services import Services +from modules.interfaces.interfaceDb[Domain]Objects import get[Domain]Interface + +router = APIRouter() + +@router.get("/[domain]/entities") +async def getEntities( + currentUser: User = Depends(getCurrentUser) +): + """Get all entities.""" + interface = get[Domain]Interface(currentUser) + entities = interface.get[Entities]() + return {"success": True, "data": [e.model_dump() for e in entities]} +``` + +### Step 7: Use in Workflows (Optional) + +If you need workflow actions, create `modules/workflows/methods/method[Domain].py`: + +``` +from modules.workflows.methods.methodBase import MethodBase +from modules.workflows.methods.methodBase import action +from modules.workflows.methods.methodBase import ActionResult + +class Method[Domain](MethodBase): + name = "[domain]" + description = "[Domain] operations" + + def __init__(self, services): + super().__init__(services) + + @action + async def performOperation(self, parameters: Dict[str, Any]) -> ActionResult: + """Perform domain operation.""" + interface = get[Domain]Interface(self.services.workflow.currentUser) + # Perform operation + # Return ActionResult +``` + +### Complete Example: Real Estate Domain + +**Datamodels** (`datamodelRealEstate.py`): +- `Projekt`, `Parzelle`, `Dokument`, `Kanton`, `Gemeinde`, `Land` +- `GeoPunkt`, `GeoPolylinie` (geographic data) +- `Kontext` (context/notes) +- Enums: `StatusProzess`, `DokumentTyp`, `GeoTag` + +**Interface Objects** (`interfaceDbRealEstateObjects.py`): +- `RealEstateObjects` class +- CRUD methods for all entities +- `executeQuery()` for custom SQL +- Database initialization with `DB_REALESTATE_*` config + +**Interface Access** (`interfaceDbRealEstateAccess.py`): +- `RealEstateAccess` class +- `uam()` method for filtering +- `canModify()` method for permissions + +**Configuration**: +``` +DB_REALESTATE_HOST=localhost +DB_REALESTATE_DATABASE=poweron_realestate +DB_REALESTATE_USER=postgres +DB_REALESTATE_PASSWORD_SECRET=... +DB_REALESTATE_PORT=5432 +``` + +### Best Practices + +**1. Naming Conventions**: +- Datamodel file: `datamodel[Domain].py` (PascalCase domain name) +- Interface Objects: `interfaceDb[Domain]Objects.py` +- Interface Access: `interfaceDb[Domain]Access.py` +- Database config: `DB_[DOMAIN]_*` (uppercase with underscores) + +**2. Mandate Isolation**: +- Always set `mandateId` on create operations +- Filter by `mandateId` in access control +- Never expose data across mandates + +**3. Access Control**: +- Always call `self.access.uam()` before returning records +- Always call `self.access.canModify()` before write operations +- Respect role hierarchy: SYSADMIN > ADMIN > USER + +**4. Error Handling**: +- Validate user context before operations +- Handle missing records gracefully (return None, not raise) +- Log errors with context (user ID, mandate ID, operation) + +**5. Database Management**: +- Let `DatabaseConnector` handle table creation automatically +- Use `_ensureTableExists()` for supporting tables with foreign keys +- Don't manually create database schemas + +**6. Testing**: +- Test CRUD operations with different user roles +- Test mandate isolation (users can't see other mandates' data) +- Test access control (users can't modify others' records) + +### Common Patterns + +**Pattern 1: Simple Domain (Single Entity)** +- One main entity model +- Basic CRUD operations +- Standard access control + +**Pattern 2: Hierarchical Domain (Parent-Child)** +- Multiple related entities +- Foreign key relationships +- Cascade operations (delete children when parent deleted) + +**Pattern 3: Complex Domain (Multiple Entities + Relationships)** +- Multiple entities with relationships +- Supporting tables (lookup tables, reference data) +- Custom query methods for complex operations + +--- + +## Security & Governance + +### Access Control + +- **RBAC**: Role-based access control enforced at Interface layer (`interface*Access.py`) + - **SYSADMIN**: Full system access, all mandates + - **ADMIN**: Full access within mandate + - **USER**: Access to own records only +- **UAM**: Unified Access Management filters recordsets by privilege and adds access flags (`_hideView`, `_hideEdit`, `_hideDelete`) + +### Secrets Management + +- **Centralized Configuration**: Credentials stored in `config.ini` with encryption +- **Interface-Level Access**: Connectors receive credentials through interfaces, not directly +- **No Leakage**: Credentials never exposed to workflows or services + +### Audit + +- **Automatic Tracking**: All database operations include `_createdBy`, `_modifiedBy`, `_createdAt`, `_modifiedAt` +- **Workflow Logging**: Workflow steps logged via `ChatService.storeLog()` +- **Security Events**: Authentication events logged via `auditLogger.logSecurityEvent()` + +### Quotas + +- **Rate Limiting**: Applied at route level using `slowapi.Limiter` +- **Token Refresh Limits**: OAuth token refresh limited to 3 attempts per hour per connection +- **Cost Tracking**: AI operations track costs via `ChatService.storeWorkflowStat()` + +--- + +## Observability + +### Structured Logging + +- **Layer-Specific Loggers**: Each layer uses module-specific loggers (e.g., `logging.getLogger("modules.services.serviceAi")`) +- **Context Information**: Logs include user ID, workflow ID, operation context +- **Error Details**: Exceptions logged with full stack traces and context + +### Tracing + +- **Operation IDs**: Long-running operations use unique operation IDs for tracking +- **Progress Logging**: `ChatService.progressLogStart()`, `progressLogUpdate()`, `progressLogFinish()` +- **Workflow State**: Workflow state persisted to database for debugging + +### Metrics + +- **Per-Capability Tracking**: Services track operation counts, costs, processing time +- **Workflow Statistics**: `ChatStat` records track bytes sent/received, error counts, prices +- **Performance Monitoring**: Processing time tracked for all AI calls and service operations + +--- + +## Minimal Request Lifecycle + +```mermaid +sequenceDiagram + participant Client + participant Route + participant Services as Service Center + participant Service + participant Interface + participant Connector + participant External as External System/DB + + Client->>Route: HTTP Request + Route->>Route: Authenticate (getCurrentUser) + Route->>Services: Create(user, workflow) + Services->>Services: Initialize interfaces + Services->>Services: Initialize services + Services-->>Route: services instance + + Route->>Service: services.capability.operation() + Service->>Interface: interface.method(params) + Interface->>Interface: Apply access control (UAM) + Interface->>Connector: connector.operation(params) + Connector->>External: API call / DB query + External-->>Connector: Response + Connector-->>Interface: Normalized data + Interface-->>Service: Domain object + Service-->>Services: Business result + Services-->>Route: Result + Route-->>Client: HTTP Response +``` + +**Steps**: +1. Route receives request or workflow triggers an action +2. Service Center resolves service instance and validates user context +3. Service executes using interfaces; interfaces call connectors +4. Results propagate back; logs/metrics recorded; workflow advances state + +--- + +## Benefits + +- **Replace vendors without breaking services**: Interfaces shield changes (e.g., swap PostgreSQL for JSON connector) +- **Accelerate feature delivery**: Services are reusable building blocks +- **Improve reliability and security**: Centralized policies and observability +- **Empower workflows/agents**: Perform complex tasks with simple, typed calls +- **Type safety**: Pydantic models ensure data consistency +- **Testability**: Clear boundaries enable mocking and unit testing +- **Maintainability**: Separation of concerns makes code easier to understand and modify + +--- + +## Quick Map to Code (for orientation) + +- `gateway/modules/connectors/` → Vendor adapters (e.g., `connectorDbPostgre.py`, `connectorVoiceGoogle.py`) +- `gateway/modules/interfaces/` → Capability contracts (e.g., `interfaceDbChatObjects.py`, `interfaceAiObjects.py`) +- `gateway/modules/services/` → Composed capabilities (e.g., `serviceAi/mainServiceAi.py`, `serviceChat/mainServiceChat.py`) +- `gateway/modules/workflows/` → Orchestrations/agents (e.g., `workflowManager.py`, `methods/methodAi.py`) +- `gateway/modules/routes/` → HTTP endpoints (e.g., `routeChatPlayground.py`, `routeWorkflows.py`) + +This framework is the backbone for market customer journey features: build once as services, reuse everywhere in workflows. + +--- + +## Visuals + +### Layered Architecture + +```mermaid +flowchart TB + subgraph ClientOrWorkflow[Client / Workflow Engine] + C[Feature or Agent Task] + end + + subgraph ServiceCenter[Service Center] + SC[Services Container\nUser Context, Interfaces, Services] + end + + subgraph Services[Services] + S1[AI Service] + S2[Chat Service] + S3[Extraction Service] + S4[Generation Service] + end + + subgraph Interfaces[Interfaces] + I1[ChatObjects] + I2[AppObjects] + I3[AiObjects] + I4[ComponentObjects] + end + + subgraph Connectors[Connectors] + K1[PostgreSQL Connector] + K2[JSON Connector] + K3[Google Speech Connector] + K4[AI Provider Connectors] + end + + subgraph External[External Systems] + E1[(PostgreSQL Database)] + E2[Google Cloud APIs] + E3[AI APIs\nOpenAI, Anthropic] + end + + C --> SC --> S1 & S2 & S3 & S4 + S1 --> I3 + S2 --> I1 + S3 --> I4 + S4 --> I1 & I4 + + I1 --> K1 + I2 --> K1 + I3 --> K4 + I4 --> K1 + + K1 --> E1 + K2 --> E1 + K3 --> E2 + K4 --> E3 +``` + +### Request / Action Sequence + +```mermaid +sequenceDiagram + participant Client as Client / Workflow + participant SC as Service Center + participant S as Service + participant I as Interface + participant AC as Access Control + participant K as Connector + participant EXT as External Tool/DB + + Client->>SC: Request capability (e.g., services.ai.callAiDocuments) + SC->>SC: Initialize with user context + SC->>S: Get service instance + S->>I: Call normalized method (e.g., aiObjects.call) + I->>AC: Check permissions (UAM) + AC-->>I: Permission granted + I->>K: Prepare vendor-specific request + K->>EXT: API/DB call (auth, retries) + EXT-->>K: Response + K-->>I: Map to normalized DTO + I-->>S: Return normalized result + S->>S: Apply business logic + S-->>SC: Business output (validated, enriched) + SC-->>Client: Typed response, telemetry recorded +``` + +### Service Center Components + +```mermaid +graph LR + subgraph SC[Service Center - Services Class] + REG[Service Registry] + CTX[User Context] + WF[Workflow Context] + INT[Interface Access] + FAC[Service Factory] + end + + REG --> FAC + CTX --> FAC + WF --> FAC + INT --> FAC + + FAC -->|builds| Svc[(Service Instances)] + + subgraph Layers[Below Services] + IF[Interfaces] + CON[Connectors] + end + + Svc --> IF --> CON + + subgraph Services[Services] + AI[AI Service] + Chat[Chat Service] + Extract[Extraction Service] + Gen[Generation Service] + end + + Svc --> AI & Chat & Extract & Gen +``` + +### Workflow State Machine (Conceptual) + +```mermaid +stateDiagram-v2 + [*] --> Plan + + Plan: Decide next action (AI or rules) + Plan --> CallService: needs external capability + Plan --> Done: no more steps + + CallService: Invoke via Service Center + CallService --> HandleResult + + HandleResult: Persist, evaluate, log + HandleResult --> Plan: more work + HandleResult --> Done: goal achieved + + Done --> [*] +``` + +### Interface Access Control Flow + +```mermaid +sequenceDiagram + participant Service + participant Interface as Interface Objects + participant Access as Access Control + participant Connector + participant DB as Database + + Service->>Interface: CRUD Operation + Interface->>Access: Check permissions (uam) + Access->>Access: Check user privilege + Access->>Access: Filter by mandateId + Access->>Access: Check ownership (_createdBy) + Access->>Access: Add access flags + Access-->>Interface: Filtered data + flags + Interface->>Connector: Execute query + Connector->>DB: SQL Query + DB-->>Connector: Results + Connector-->>Interface: Raw data + Interface->>Interface: Transform to datamodel + Interface-->>Service: Domain objects with access flags +``` + +--- + +## Development Best Practices + +### 1. Always Use Service Center + +✅ **GOOD**: Use Service Center via `Services(user=current_user)` and call `services.ai.callAiDocuments()`, `services.chat.storeMessageWithDocuments()`, etc. + +❌ **BAD**: Direct interface access bypasses the service layer (e.g., calling `getChatInterface(user).getWorkflow()` directly). + +### 2. Keep Services Stateless + +✅ **GOOD**: Stateless services use the database for persistence (e.g., `self.services.interfaceDbApp.getCache()`). + +❌ **BAD**: Stateful services store data in instance variables (e.g., `self.cache = {}`). + +### 3. Use Datamodels for Type Safety + +✅ **GOOD**: Use Pydantic models like `ChatWorkflow`, `ChatMessage` from `modules.datamodels.datamodelChat`. Create instances with `ChatWorkflow(**data)` and return typed results. + +❌ **BAD**: Use raw dictionaries without type safety. + +### 4. Apply Access Control + +✅ **GOOD**: Interfaces apply UAM automatically (e.g., `self.interfaceDbChat.getWorkflows()` filters by user privilege). + +❌ **BAD**: Bypass access control by calling connectors directly (e.g., `self.connector.getRecordset()` has no filtering). + +### 5. Handle Errors Gracefully + +✅ **GOOD**: Return structured errors with `{"success": True/False, "data": ..., "error": ...}` format. Log exceptions with context. + +❌ **BAD**: Let exceptions propagate to callers without handling. + +--- + +## Workflow Engineering + +Workflow engineering is the process of designing, building, and maintaining workflows that orchestrate multi-step tasks using the gateway's service layer. Workflows transform user requests into structured execution plans, coordinate action execution, and manage state throughout the process. + +### Understanding Workflow Architecture + +Workflows operate at the highest level of the gateway architecture, orchestrating services to accomplish complex goals. They provide: + +- **Intelligent Planning**: AI-powered task breakdown and action generation +- **State Management**: Track progress, maintain context, and handle errors +- **Document Flow**: Manage document references and lineage throughout execution +- **Adaptive Execution**: Retry failed tasks, learn from results, improve over time +- **Multi-Mode Support**: Different execution strategies for different use cases + +### Workflow Components + +**WorkflowManager**: Main orchestration controller that manages workflow lifecycle (`workflowStart()`, `workflowStop()`, `_workflowProcess()`) + +**WorkflowProcessor**: Delegates to mode-specific implementations (Actionplan, Dynamic, Automation) + +**TaskPlanner**: Generates structured task plans from user input using AI + +**ActionExecutor**: Executes individual actions by invoking methods from the global methods catalog + +**MessageCreator**: Creates and persists workflow messages with document associations + +**Method System**: Extensible plugin framework for defining reusable actions + +### Workflow Execution Pipeline + +Every workflow follows a four-stage pipeline: + +1. **Send First Message**: Analyze user intent, extract documents, detect language, normalize request +2. **Plan Tasks**: Generate structured task plan with objectives, success criteria, and dependencies +3. **Execute Tasks**: Execute each task sequentially, maintaining context between tasks +4. **Process Results**: Generate feedback, create completion message, update workflow status + +--- + +## Workflow Modes + +The gateway supports three distinct workflow modes, each optimized for different use cases: + +### Actionplan Mode + +**Strategy**: Batch planning with quality review and intelligent retry + +**Characteristics**: +- Plans all actions upfront before execution begins +- Reviews results against success criteria after execution +- Retries failed tasks up to 3 times with cumulative improvements +- Best for complex multi-step workflows with specific requirements + +**Use Cases**: Data processing pipelines, document analysis with requirements, complex transformations + +**Execution Flow**: +1. Generate complete action plan for entire task +2. Execute all actions sequentially +3. Review results against success criteria +4. Retry with improvements if criteria not met +5. Return final result + +### Dynamic Mode + +**Strategy**: Iterative, just-in-time action generation + +**Characteristics**: +- Generates one action at a time based on current state +- Each action's result influences the next action +- Workflow path emerges organically based on findings +- Limited by `maxSteps` (default: 5) to prevent infinite loops + +**Use Cases**: Research workflows, exploratory data analysis, iterative problem solving, uncertain paths + +**Execution Flow**: +1. Generate single next action based on current context +2. Execute action immediately +3. Evaluate if task objective is met +4. Continue if objective not met and under max steps +5. Return result when objective met or max steps reached + +### Automation Mode + +**Strategy**: Predefined JSON-based deterministic execution + +**Characteristics**: +- No AI planning or action generation +- User provides complete task and action plan in JSON format +- Deterministic execution (same input always produces same sequence) +- Fastest execution time (no planning overhead) + +**Use Cases**: Repeated workflows, automated jobs, batch processing, template execution, routine operations + +**Execution Flow**: +1. Parse predefined JSON plan from user input +2. Execute actions in order specified in JSON +3. Collect results without review +4. Return execution summary + +--- + +## Building New Workflows + +New workflows are typically built using Actionplan or Dynamic modes, where AI generates the execution plan based on user input. This section covers how to create workflows that adapt to user requests. + +### Starting a New Workflow + +**Entry Point**: `WorkflowManager.workflowStart()` + +**Required Parameters**: +- `userInput`: UserInputRequest containing prompt, file IDs, and language +- `workflowMode`: WorkflowModeEnum (WORKFLOW_ACTIONPLAN, WORKFLOW_DYNAMIC, or WORKFLOW_AUTOMATION) +- `workflowId`: Optional ID to continue existing workflow + +**Process**: +1. Create or load `ChatWorkflow` record in database +2. Initialize workflow state (status="running", currentRound=1, counters=0) +3. Discover and update method instances with current services +4. Launch asynchronous processing pipeline +5. Return workflow object immediately (non-blocking) + +**Example Flow**: +``` +Route → chatStart() → WorkflowManager.workflowStart() → _workflowProcess() +``` + +### Workflow Input Processing + +The first stage (`_sendFirstMessage()`) processes user input: + +**Intent Analysis**: AI analyzes user input to extract: +- Detected language (ISO 639-1 code) +- Normalized request (full, explicit restatement) +- Core intent (primary goals and requirements) +- Bulky context items (large data blocks extracted as separate documents) + +**Document Management**: +- Processes user-uploaded files (converts file IDs to ChatDocument objects) +- Extracts large content blocks from prompt (code snippets, tables, lists) +- Creates document records in component database +- Applies neutralization if enabled in user settings +- Associates documents with labels (e.g., "round1_usercontext") + +**Message Creation**: Creates first message with role="user", status="first", and all associated documents + +### Task Planning + +The second stage (`_planTasks()`) generates structured task plans: + +**Planning Process**: +1. Uses cleaned user intent from previous stage +2. Calls `WorkflowProcessor.generateTaskPlan()` which delegates to mode-specific implementation +3. For Actionplan/Dynamic modes: Uses `TaskPlanner.generateTaskPlan()` with AI +4. For Automation mode: Parses predefined JSON plan from user input + +**TaskPlan Structure**: +- `overview`: High-level description of the plan +- `tasks`: Array of TaskStep objects +- `userMessage`: Original user request + +**TaskStep Structure**: +- `id`: Unique task identifier +- `objective`: What the task should accomplish +- `dependencies`: Array of task IDs this task depends on +- `successCriteria`: Array of measurable criteria for task completion +- `estimatedComplexity`: Complexity estimate (simple, medium, complex) +- `userMessage`: User-facing description of the task + +**AI Planning**: Uses `services.ai.callAiPlanning()` with quality settings to generate detailed task breakdown. The AI receives: +- User prompt and normalized intent +- Available methods and actions (from method discovery) +- Available documents and connections +- Workflow context and history + +### Task Execution + +The third stage (`_executeTasks()`) executes each task sequentially: + +**For Each Task**: +1. Build `TaskContext` containing: + - Task details (objective, success criteria, dependencies) + - Workflow state (current round, task, action numbers) + - Available documents (from current and previous rounds) + - Available connections (user's OAuth connections) + - Previous task results (for context and dependencies) + +2. Call `WorkflowProcessor.executeTask()` which delegates to mode-specific execution + +3. Receive `TaskResult` with: + - `success`: Boolean indicating task completion status + - `feedback`: Human-readable summary of what was accomplished + - `documents`: List of ChatDocument objects created during task execution + - `reviewResult`: Optional ReviewResult if quality review was performed + +4. Prepare task handover data for subsequent tasks + +5. Accumulate results for use by dependent tasks + +**Mode-Specific Execution**: + +**Actionplan Mode**: +- Generates complete action plan for entire task upfront +- Executes all actions sequentially +- Reviews results against success criteria +- Retries with improvements if criteria not met (max 3 attempts) + +**Dynamic Mode**: +- Generates single next action based on current state +- Executes action immediately +- Evaluates if task objective is met +- Continues generating actions until objective met or max steps reached + +**Automation Mode**: +- Uses predefined action list from JSON plan +- Executes actions in order specified +- No retry logic or quality review + +### Action Execution + +Actions are executed by `ActionExecutor.executeSingleAction()`: + +**Process**: +1. Resolve parameters (document references, connections, etc.) +2. Look up method in global methods catalog +3. Validate action exists within method +4. Invoke action method with parameters +5. Extract result text from ActionDocument objects +6. Convert ActionDocuments to ChatDocuments for persistence +7. Create action completion message +8. Return ActionResult with success status and documents + +**Action Invocation**: Actions are invoked using compound names (e.g., "ai.process", "sharepoint.search") or separate method/action names. + +**Document References**: Actions receive document references in three formats: +- `docItem::`: Single document by ID +- `docList: