From 07a2d279df2c304269f9c2af4315548299313cd2 Mon Sep 17 00:00:00 2001 From: ValueOn AG Date: Thu, 25 Sep 2025 16:59:44 +0200 Subject: [PATCH] cleaned model space centralized --- modules/connectors/connectorTicketsClickup.py | 26 +- modules/connectors/connectorTicketsJira.py | 27 +- modules/connectors/connectorWebTavily.py | 316 +-- modules/datamodels/__init__.py | 17 + .../datamodelAi.py} | 2 + modules/datamodels/datamodelChat.py | 262 +++ modules/datamodels/datamodelFiles.py | 84 + modules/datamodels/datamodelNeutralizer.py | 56 + modules/datamodels/datamodelSecurity.py | 87 + .../datamodelTickets.py} | 15 +- modules/datamodels/datamodelUam.py | 154 ++ modules/datamodels/datamodelUtils.py | 26 + modules/datamodels/datamodelVoice.py | 43 + .../datamodelWeb.py} | 57 +- modules/datamodels/datamodelWorkflow.py | 446 ++++ .../chatPlayground/mainChatPlayground.py | 5 +- .../mainNeutralizePlayground.py | 5 +- modules/features/syncDelta/mainSyncDelta.py | 12 +- modules/interfaces/interfaceAiObjects.py | 2 +- modules/interfaces/interfaceAppAccess.py | 3 +- modules/interfaces/interfaceAppModel.py | 566 ----- modules/interfaces/interfaceAppObjects.py | 10 +- modules/interfaces/interfaceChatAccess.py | 4 +- modules/interfaces/interfaceChatObjects.py | 8 +- .../interfaces/interfaceComponentAccess.py | 8 +- modules/interfaces/interfaceComponentModel.py | 264 --- .../interfaces/interfaceComponentObjects.py | 8 +- modules/interfaces/interfaceTicketObjects.py | 45 +- modules/interfaces/interfaceWebObjects.py | 118 +- modules/routes/routeAdmin.py | 2 +- modules/routes/routeAttributes.py | 2 +- modules/routes/routeChatPlayground.py | 8 +- modules/routes/routeDataConnections.py | 3 +- modules/routes/routeDataFiles.py | 4 +- modules/routes/routeDataMandates.py | 2 +- modules/routes/routeDataNeutralization.py | 3 +- modules/routes/routeDataPrompts.py | 4 +- modules/routes/routeDataUsers.py | 7 +- modules/routes/routeSecurityAdmin.py | 3 +- modules/routes/routeSecurityGoogle.py | 9 +- modules/routes/routeSecurityLocal.py | 9 +- modules/routes/routeSecurityMsft.py | 3 +- modules/routes/routeVoiceGoogle.py | 2 +- modules/routes/routeWorkflows.py | 6 +- modules/security/auth.py | 3 +- modules/security/tokenManager.py | 3 +- modules/security/tokenRefreshService.py | 3 +- modules/services/__init__.py | 7 +- modules/services/serviceAi/mainServiceAi.py | 17 +- .../mainServiceDocumentExtraction.py | 9 +- .../mainServiceNeutralization.py | 2 +- modules/services/serviceWeb/mainServiceWeb.py | 49 + .../serviceWorkflow/mainServiceWorkflow.py | 5 +- modules/workflows/methods/methodAi.py | 2 +- modules/workflows/methods/methodDocument.py | 5 +- modules/workflows/methods/methodOutlook.py | 113 +- modules/workflows/methods/methodSharepoint.py | 2 +- modules/workflows/methods/methodWeb.py | 79 +- .../workflows/processing/executionState.py | 3 +- modules/workflows/processing/handlingTasks.py | 8 +- modules/workflows/processing/promptFactory.py | 3 +- modules/workflows/workflowManager.py | 8 +- static/favicon.ico | Bin 3870 -> 0 bytes tests/__init__.py | 1 - tests/fixtures/__init__.py | 0 tests/fixtures/tavily_responses.py | 71 - tests/methods/__init__.py | 0 tests/methods/test_method_web.py | 248 --- tests/test_graph_search.py | 311 --- tests/test_neutralizer/apprun.py | 263 --- tests/test_neutralizer/logs/log_mapping.csv | 17 - .../logs/log_replacements.csv | 1947 ----------------- tests/test_neutralizer/neutralizer.py | 334 --- .../output/neutralized_Case.md | 608 ----- .../output/neutralized_customers.csv | 6 - .../output/neutralized_cv_lara_meier.txt | 69 - .../output/neutralized_employees.csv | 20 - .../output/neutralized_english.txt | 9 - .../output/neutralized_example.json | 54 - .../output/neutralized_example.xml | 78 - .../output/neutralized_french.txt | 10 - .../output/neutralized_german.txt | 24 - .../neutralized_geschaeftsstrategie.txt | 54 - .../output/neutralized_geschäfte.csv | 6 - .../output/neutralized_italian.txt | 9 - .../output/neutralized_kunden.csv | 6 - .../output/neutralized_mitarbeiter.csv | 6 - .../output/neutralized_swiss.txt | 10 - .../output/neutralized_transactions.csv | 6 - tests/test_neutralizer/patterns.py | 396 ---- tests/test_neutralizer/testdata/Case.md | 608 ----- tests/test_neutralizer/testdata/customers.csv | 6 - .../testdata/cv_lara_meier.txt | 73 - tests/test_neutralizer/testdata/employees.csv | 6 - tests/test_neutralizer/testdata/english.txt | 12 - tests/test_neutralizer/testdata/example.json | 54 - tests/test_neutralizer/testdata/example.xml | 79 - tests/test_neutralizer/testdata/french.txt | 12 - tests/test_neutralizer/testdata/german.txt | 12 - .../testdata/geschaeftsstrategie.txt | 56 - tests/test_neutralizer/testdata/geschäfte.csv | 6 - tests/test_neutralizer/testdata/italian.txt | 13 - tests/test_neutralizer/testdata/kunden.csv | 6 - .../test_neutralizer/testdata/mitarbeiter.csv | 6 - tests/test_neutralizer/testdata/swiss.txt | 13 - .../testdata/transactions.csv | 6 - tests/test_neutralizer/zdocu.html | 157 -- 107 files changed, 1651 insertions(+), 7101 deletions(-) create mode 100644 modules/datamodels/__init__.py rename modules/{interfaces/interfaceAiModel.py => datamodels/datamodelAi.py} (99%) create mode 100644 modules/datamodels/datamodelChat.py create mode 100644 modules/datamodels/datamodelFiles.py create mode 100644 modules/datamodels/datamodelNeutralizer.py create mode 100644 modules/datamodels/datamodelSecurity.py rename modules/{interfaces/interfaceTicketModel.py => datamodels/datamodelTickets.py} (51%) create mode 100644 modules/datamodels/datamodelUam.py create mode 100644 modules/datamodels/datamodelUtils.py create mode 100644 modules/datamodels/datamodelVoice.py rename modules/{interfaces/interfaceWebModel.py => datamodels/datamodelWeb.py} (69%) create mode 100644 modules/datamodels/datamodelWorkflow.py delete mode 100644 modules/interfaces/interfaceAppModel.py delete mode 100644 modules/interfaces/interfaceComponentModel.py create mode 100644 modules/services/serviceWeb/mainServiceWeb.py delete mode 100644 static/favicon.ico delete mode 100644 tests/__init__.py delete mode 100644 tests/fixtures/__init__.py delete mode 100644 tests/fixtures/tavily_responses.py delete mode 100644 tests/methods/__init__.py delete mode 100644 tests/methods/test_method_web.py delete mode 100644 tests/test_graph_search.py delete mode 100644 tests/test_neutralizer/apprun.py delete mode 100644 tests/test_neutralizer/logs/log_mapping.csv delete mode 100644 tests/test_neutralizer/logs/log_replacements.csv delete mode 100644 tests/test_neutralizer/neutralizer.py delete mode 100644 tests/test_neutralizer/output/neutralized_Case.md delete mode 100644 tests/test_neutralizer/output/neutralized_customers.csv delete mode 100644 tests/test_neutralizer/output/neutralized_cv_lara_meier.txt delete mode 100644 tests/test_neutralizer/output/neutralized_employees.csv delete mode 100644 tests/test_neutralizer/output/neutralized_english.txt delete mode 100644 tests/test_neutralizer/output/neutralized_example.json delete mode 100644 tests/test_neutralizer/output/neutralized_example.xml delete mode 100644 tests/test_neutralizer/output/neutralized_french.txt delete mode 100644 tests/test_neutralizer/output/neutralized_german.txt delete mode 100644 tests/test_neutralizer/output/neutralized_geschaeftsstrategie.txt delete mode 100644 tests/test_neutralizer/output/neutralized_geschäfte.csv delete mode 100644 tests/test_neutralizer/output/neutralized_italian.txt delete mode 100644 tests/test_neutralizer/output/neutralized_kunden.csv delete mode 100644 tests/test_neutralizer/output/neutralized_mitarbeiter.csv delete mode 100644 tests/test_neutralizer/output/neutralized_swiss.txt delete mode 100644 tests/test_neutralizer/output/neutralized_transactions.csv delete mode 100644 tests/test_neutralizer/patterns.py delete mode 100644 tests/test_neutralizer/testdata/Case.md delete mode 100644 tests/test_neutralizer/testdata/customers.csv delete mode 100644 tests/test_neutralizer/testdata/cv_lara_meier.txt delete mode 100644 tests/test_neutralizer/testdata/employees.csv delete mode 100644 tests/test_neutralizer/testdata/english.txt delete mode 100644 tests/test_neutralizer/testdata/example.json delete mode 100644 tests/test_neutralizer/testdata/example.xml delete mode 100644 tests/test_neutralizer/testdata/french.txt delete mode 100644 tests/test_neutralizer/testdata/german.txt delete mode 100644 tests/test_neutralizer/testdata/geschaeftsstrategie.txt delete mode 100644 tests/test_neutralizer/testdata/geschäfte.csv delete mode 100644 tests/test_neutralizer/testdata/italian.txt delete mode 100644 tests/test_neutralizer/testdata/kunden.csv delete mode 100644 tests/test_neutralizer/testdata/mitarbeiter.csv delete mode 100644 tests/test_neutralizer/testdata/swiss.txt delete mode 100644 tests/test_neutralizer/testdata/transactions.csv delete mode 100644 tests/test_neutralizer/zdocu.html diff --git a/modules/connectors/connectorTicketsClickup.py b/modules/connectors/connectorTicketsClickup.py index 7e07a830..7d92f54a 100644 --- a/modules/connectors/connectorTicketsClickup.py +++ b/modules/connectors/connectorTicketsClickup.py @@ -1,15 +1,12 @@ -"""ClickUp connector for CRUD operations (compatible with TicketInterface).""" +"""ClickUp connector for CRUD operations (compatible with TicketInterface). + +This module defines its own minimal abstractions to avoid coupling. +""" -from dataclasses import dataclass from typing import Optional import logging import aiohttp - -from modules.interfaces.interfaceTicketModel import ( - TicketBase, - TicketFieldAttribute, - Task, -) +from modules.datamodels.datamodelTickets import TicketBase, TicketFieldAttribute logger = logging.getLogger(__name__) @@ -68,11 +65,11 @@ class ConnectorTicketClickup(TicketBase): logger.error(f"ClickUp read_attributes error: {e}") return attributes - async def read_tasks(self, *, limit: int = 0) -> list[Task]: + async def read_tasks(self, *, limit: int = 0) -> list[dict]: """Read tasks from ClickUp, always returning full task records. If list_id is set, read from that list; otherwise read from team. """ - tasks: list[Task] = [] + tasks: list[dict] = [] try: async with aiohttp.ClientSession() as session: page = 0 @@ -94,7 +91,7 @@ class ConnectorTicketClickup(TicketBase): data = await response.json() items = data.get("tasks", []) for item in items: - tasks.append(Task(data=item)) + tasks.append(item) if limit and len(tasks) >= limit: return tasks @@ -105,12 +102,11 @@ class ConnectorTicketClickup(TicketBase): logger.error(f"ClickUp read_tasks error: {e}") return tasks - async def write_tasks(self, tasklist: list[Task]) -> None: - """Update tasks in ClickUp. Expects Task.data to contain {'ID' or 'id' or 'task_id', 'fields': {...}}""" + async def write_tasks(self, tasklist: list[dict]) -> None: + """Update tasks in ClickUp. Expects each item to contain {'ID' or 'id' or 'task_id', 'fields': {...}}""" try: async with aiohttp.ClientSession() as session: - for task in tasklist: - data = task.data + for data in tasklist: taskId = data.get("ID") or data.get("id") or data.get("task_id") fields = data.get("fields", {}) if not taskId or not isinstance(fields, dict) or not fields: diff --git a/modules/connectors/connectorTicketsJira.py b/modules/connectors/connectorTicketsJira.py index 553120c9..eb665036 100644 --- a/modules/connectors/connectorTicketsJira.py +++ b/modules/connectors/connectorTicketsJira.py @@ -1,12 +1,13 @@ -"""Jira connector for CRUD operations (neutralized to generic ticket interface).""" +"""Jira connector for CRUD operations (neutralized to generic ticket interface). + +This module defines its own minimal abstractions to avoid coupling. +""" -from dataclasses import dataclass import logging import aiohttp import asyncio import json - -from modules.interfaces.interfaceTicketModel import (TicketBase, TicketFieldAttribute, Task, ) +from modules.datamodels.datamodelTickets import TicketBase, TicketFieldAttribute logger = logging.getLogger(__name__) @@ -129,7 +130,7 @@ class ConnectorTicketJira(TicketBase): logger.error(f"Error while calling fields API: {str(e)}") return [] - async def read_tasks(self, *, limit: int = 0) -> list[Task]: + async def read_tasks(self, *, limit: int = 0) -> list[dict]: """ Read tasks from Jira with pagination support. @@ -137,7 +138,7 @@ class ConnectorTicketJira(TicketBase): limit: Maximum number of tasks to retrieve. 0 means no limit. Returns: - list[Task]: List of tasks with their data + list[dict]: List of tasks with their data """ # Build JQL dynamically; allow empty or '*' issue_type to mean "all types" if self.ticketType and self.ticketType != "*": @@ -148,7 +149,7 @@ class ConnectorTicketJira(TicketBase): # Initialize variables for pagination (cursor-based /search/jql) max_results = 100 next_page_token: str | None = None - tasks = [] + tasks: list[dict] = [] page_counter = 0 max_pages_safety_cap = 1000 seen_issue_ids: set[str] = set() @@ -202,8 +203,7 @@ class ConnectorTicketJira(TicketBase): continue if issue_id: seen_issue_ids.add(issue_id) - task = Task(data=issue) - tasks.append(task) + tasks.append(issue) new_items_added += 1 # Check limit @@ -253,20 +253,19 @@ class ConnectorTicketJira(TicketBase): logger.error(f"Unexpected error while fetching Jira tasks: {str(e)}") raise - async def write_tasks(self, tasklist: list[Task]) -> None: + async def write_tasks(self, tasklist: list[dict]) -> None: """ Write/update tasks to Jira. Args: - tasklist: List of Task objects containing task data to update + tasklist: List of dicts containing task data to update """ headers = {"Accept": "application/json", "Content-Type": "application/json"} auth = aiohttp.BasicAuth(self.apiUsername, self.apiToken) try: async with aiohttp.ClientSession() as session: - for task in tasklist: - task_data = task.data + for task_data in tasklist: task_id = ( task_data.get("ID") or task_data.get("id") @@ -274,7 +273,7 @@ class ConnectorTicketJira(TicketBase): ) if not task_id: - logger.warning("Task missing ID or key, skipping update") + logger.warning("Ticket update missing ID or key, skipping") continue # Extract fields to update from task data diff --git a/modules/connectors/connectorWebTavily.py b/modules/connectors/connectorWebTavily.py index 6270d10f..10ac105d 100644 --- a/modules/connectors/connectorWebTavily.py +++ b/modules/connectors/connectorWebTavily.py @@ -1,62 +1,50 @@ -"""Tavily web search class.""" +"""Tavily web search class. +""" import logging -import os +import asyncio from dataclasses import dataclass -from modules.interfaces.interfaceWebModel import ( - WebCrawlBase, - WebCrawlDocumentData, - WebCrawlRequest, - WebCrawlResultItem, - WebScrapeActionDocument, - WebScrapeActionResult, - WebScrapeBase, - WebScrapeDocumentData, - WebScrapeRequest, - WebScrapeResultItem, - WebSearchBase, - WebSearchRequest, +from tavily import AsyncTavilyClient +from modules.shared.configuration import APP_CONFIG +from modules.shared.timezoneUtils import get_utc_timestamp +from modules.datamodels.datamodelWeb import ( WebSearchActionResult, WebSearchActionDocument, WebSearchDocumentData, WebSearchResultItem, - WebCrawlActionDocument, WebCrawlActionResult, - get_web_search_min_results, - get_web_search_max_results, + WebCrawlActionDocument, + WebCrawlDocumentData, + WebCrawlResultItem, + WebScrapeActionResult, + WebScrapeActionDocument, + WebSearchDocumentData as WebScrapeDocumentData, + WebScrapeResultItem, ) -# from modules.interfaces.interfaceChatModel import ActionResult, ActionDocument -from tavily import AsyncTavilyClient -from modules.shared.timezoneUtils import get_utc_timestamp -from modules.shared.configuration import APP_CONFIG - logger = logging.getLogger(__name__) - -# Cached configuration values are loaded into the connector instance on creation - - @dataclass -class TavilySearchResult: +class WebSearchResult: title: str url: str - @dataclass -class TavilyCrawlResult: +class WebCrawlResult: url: str content: str - @dataclass -class ConnectorTavily(WebSearchBase, WebCrawlBase, WebScrapeBase): +class ConnectorWeb: client: AsyncTavilyClient = None # Cached settings loaded at initialization time crawl_timeout: int = 30 crawl_max_retries: int = 3 crawl_retry_delay: int = 2 + # Cached web search constraints (camelCase per project style) + webSearchMinResults: int = 1 + webSearchMaxResults: int = 20 @classmethod async def create(cls): @@ -72,16 +60,14 @@ class ConnectorTavily(WebSearchBase, WebCrawlBase, WebScrapeBase): crawl_timeout=crawl_timeout, crawl_max_retries=crawl_max_retries, crawl_retry_delay=crawl_retry_delay, + webSearchMinResults=int(APP_CONFIG.get("Web_Search_MIN_RESULTS", "1")), + webSearchMaxResults=int(APP_CONFIG.get("Web_Search_MAX_RESULTS", "20")), ) - async def search_urls(self, request: WebSearchRequest) -> WebSearchActionResult: - """Handles the web search request. - - Takes a query and returns a list of URLs. - """ - # Step 1: Search + # Standardized methods returning ActionResults for the interface to consume + async def search(self, request) -> "WebSearchActionResult": try: - search_results = await self._search( + raw_results = await self._search( query=request.query, max_results=request.max_results, search_depth=request.search_depth, @@ -96,33 +82,59 @@ class ConnectorTavily(WebSearchBase, WebCrawlBase, WebScrapeBase): except Exception as e: return WebSearchActionResult(success=False, error=str(e)) - # Step 2: Build ActionResult - try: - result = self._build_search_action_result(search_results, request.query) - except Exception as e: - return WebSearchActionResult(success=False, error=str(e)) + result_items = [ + WebSearchResultItem(title=result.title, url=result.url) + for result in raw_results + ] - return result + document_data = WebSearchDocumentData( + query=request.query, + results=result_items, + total_count=len(result_items), + ) - async def crawl_urls(self, request: WebCrawlRequest) -> WebCrawlActionResult: - """Crawls the given URLs and returns the extracted text content.""" - # Step 1: Crawl + document = WebSearchActionDocument( + documentName=f"web_search_results_{get_utc_timestamp()}.json", + documentData=document_data, + mimeType="application/json", + ) + + return WebSearchActionResult( + success=True, documents=[document], resultLabel="web_search_results" + ) + + async def crawl(self, request) -> "WebCrawlActionResult": try: - crawl_results = await self._crawl(request.urls) + raw_results = await self._crawl( + [str(u) for u in request.urls], + extract_depth=request.extract_depth, + format=request.format, + ) except Exception as e: return WebCrawlActionResult(success=False, error=str(e)) - # Step 2: Build ActionResult - try: - result = self._build_crawl_action_result(crawl_results, request.urls) - except Exception as e: - return WebCrawlActionResult(success=False, error=str(e)) + result_items = [ + WebCrawlResultItem(url=result.url, content=result.content) + for result in raw_results + ] - return result + document_data = WebCrawlDocumentData( + urls=[str(u) for u in request.urls], + results=result_items, + total_count=len(result_items), + ) - async def scrape(self, request: WebScrapeRequest) -> WebScrapeActionResult: - """Turns a query in a list of urls with extracted content.""" - # Step 1: Search + document = WebCrawlActionDocument( + documentName=f"web_crawl_results_{get_utc_timestamp()}.json", + documentData=document_data, + mimeType="application/json", + ) + + return WebCrawlActionResult( + success=True, documents=[document], resultLabel="web_crawl_results" + ) + + async def scrape(self, request) -> "WebScrapeActionResult": try: search_results = await self._search( query=request.query, @@ -139,7 +151,6 @@ class ConnectorTavily(WebSearchBase, WebCrawlBase, WebScrapeBase): except Exception as e: return WebScrapeActionResult(success=False, error=str(e)) - # Step 2: Crawl try: urls = [result.url for result in search_results] crawl_results = await self._crawl( @@ -150,13 +161,90 @@ class ConnectorTavily(WebSearchBase, WebCrawlBase, WebScrapeBase): except Exception as e: return WebScrapeActionResult(success=False, error=str(e)) - # Step 3: Build ActionResult - try: - result = self._build_scrape_action_result(crawl_results, request.query) - except Exception as e: - return WebScrapeActionResult(success=False, error=str(e)) + result_items = [ + WebScrapeResultItem(url=result.url, content=result.content) + for result in crawl_results + ] - return result + document_data = WebScrapeDocumentData( + query=request.query, + results=result_items, + total_count=len(result_items), + ) + + document = WebScrapeActionDocument( + documentName=f"web_scrape_results_{get_utc_timestamp()}.json", + documentData=document_data, + mimeType="application/json", + ) + + return WebScrapeActionResult( + success=True, documents=[document], resultLabel="web_scrape_results" + ) + + async def _search_urls_raw(self, + *, + query: str, + max_results: int, + search_depth: str | None = None, + time_range: str | None = None, + topic: str | None = None, + include_domains: list[str] | None = None, + exclude_domains: list[str] | None = None, + language: str | None = None, + include_answer: bool | None = None, + include_raw_content: bool | None = None, + ) -> list["WebSearchResult"]: + return await self._search( + query=query, + max_results=max_results, + search_depth=search_depth, + time_range=time_range, + topic=topic, + include_domains=include_domains, + exclude_domains=exclude_domains, + language=language, + include_answer=include_answer, + include_raw_content=include_raw_content, + ) + + async def _crawl_urls_raw(self, + *, + urls: list[str], + extract_depth: str | None = None, + format: str | None = None, + ) -> list["WebCrawlResult"]: + return await self._crawl(urls, extract_depth=extract_depth, format=format) + + async def _scrape_raw(self, + *, + query: str, + max_results: int, + search_depth: str | None = None, + time_range: str | None = None, + topic: str | None = None, + include_domains: list[str] | None = None, + exclude_domains: list[str] | None = None, + language: str | None = None, + include_answer: bool | None = None, + include_raw_content: bool | None = None, + extract_depth: str | None = None, + format: str | None = None, + ) -> list["WebCrawlResult"]: + search_results = await self._search( + query=query, + max_results=max_results, + search_depth=search_depth, + time_range=time_range, + topic=topic, + include_domains=include_domains, + exclude_domains=exclude_domains, + language=language, + include_answer=include_answer, + include_raw_content=include_raw_content, + ) + urls = [result.url for result in search_results] + return await self._crawl(urls, extract_depth=extract_depth, format=format) async def _search( self, @@ -170,11 +258,11 @@ class ConnectorTavily(WebSearchBase, WebCrawlBase, WebScrapeBase): language: str | None = None, include_answer: bool | None = None, include_raw_content: bool | None = None, - ) -> list[TavilySearchResult]: + ) -> list[WebSearchResult]: """Calls the Tavily API to perform a web search.""" - # Make sure max_results is within the allowed range - min_results = get_web_search_min_results() - max_allowed_results = get_web_search_max_results() + # Make sure max_results is within the allowed range (use cached values) + min_results = self.webSearchMinResults + max_allowed_results = self.webSearchMaxResults if max_results < min_results or max_results > max_allowed_results: raise ValueError(f"max_results must be between {min_results} and {max_allowed_results}") @@ -201,45 +289,17 @@ class ConnectorTavily(WebSearchBase, WebCrawlBase, WebScrapeBase): response = await self.client.search(**kwargs) return [ - TavilySearchResult(title=result["title"], url=result["url"]) + WebSearchResult(title=result["title"], url=result["url"]) for result in response["results"] ] - def _build_search_action_result( - self, search_results: list[TavilySearchResult], query: str = "" - ) -> WebSearchActionResult: - """Builds the ActionResult from the search results.""" - # Convert to result items - result_items = [ - WebSearchResultItem(title=result.title, url=result.url) - for result in search_results - ] - - # Create document data with all results - document_data = WebSearchDocumentData( - query=query, results=result_items, total_count=len(result_items) - ) - - # Create single document - document = WebSearchActionDocument( - documentName=f"web_search_results_{get_utc_timestamp()}.json", - documentData=document_data, - mimeType="application/json", - ) - - return WebSearchActionResult( - success=True, documents=[document], resultLabel="web_search_results" - ) - async def _crawl( self, urls: list, extract_depth: str | None = None, format: str | None = None, - ) -> list[TavilyCrawlResult]: + ) -> list[WebCrawlResult]: """Calls the Tavily API to extract text content from URLs with retry logic.""" - import asyncio - max_retries = self.crawl_max_retries retry_delay = self.crawl_retry_delay timeout = self.crawl_timeout @@ -258,7 +318,7 @@ class ConnectorTavily(WebSearchBase, WebCrawlBase, WebScrapeBase): ) return [ - TavilyCrawlResult(url=result["url"], content=result["raw_content"]) + WebCrawlResult(url=result["url"], content=result["raw_content"]) for result in response["results"] ] @@ -277,59 +337,3 @@ class ConnectorTavily(WebSearchBase, WebCrawlBase, WebScrapeBase): await asyncio.sleep(retry_delay) else: raise Exception(f"Crawl failed after {max_retries + 1} attempts: {str(e)}") - - def _build_crawl_action_result( - self, crawl_results: list[TavilyCrawlResult], urls: list[str] = None - ) -> WebCrawlActionResult: - """Builds the ActionResult from the crawl results.""" - # Convert to result items - result_items = [ - WebCrawlResultItem(url=result.url, content=result.content) - for result in crawl_results - ] - - # Create document data with all results - document_data = WebCrawlDocumentData( - urls=urls or [result.url for result in crawl_results], - results=result_items, - total_count=len(result_items), - ) - - # Create single document - document = WebCrawlActionDocument( - documentName=f"web_crawl_results_{get_utc_timestamp()}.json", - documentData=document_data, - mimeType="application/json", - ) - - return WebCrawlActionResult( - success=True, documents=[document], resultLabel="web_crawl_results" - ) - - def _build_scrape_action_result( - self, crawl_results: list[TavilyCrawlResult], query: str = "" - ) -> WebScrapeActionResult: - """Builds the ActionResult from the scrape results.""" - # Convert to result items - result_items = [ - WebScrapeResultItem(url=result.url, content=result.content) - for result in crawl_results - ] - - # Create document data with all results - document_data = WebScrapeDocumentData( - query=query, - results=result_items, - total_count=len(result_items), - ) - - # Create single document - document = WebScrapeActionDocument( - documentName=f"web_scrape_results_{get_utc_timestamp()}.json", - documentData=document_data, - mimeType="application/json", - ) - - return WebScrapeActionResult( - success=True, documents=[document], resultLabel="web_scrape_results" - ) diff --git a/modules/datamodels/__init__.py b/modules/datamodels/__init__.py new file mode 100644 index 00000000..bc18cabd --- /dev/null +++ b/modules/datamodels/__init__.py @@ -0,0 +1,17 @@ +""" +Unified modules.datamodels package. + +Usage examples: + from modules.datamodels import ai + from modules.datamodels import web +""" +from . import datamodelAi as ai +from . import datamodelWeb as web +from . import datamodelUam as uam +from . import datamodelSecurity as security +from . import datamodelNeutralizer as neutralizer +from . import datamodelWorkflow as workflow +from . import datamodelChat as chat +from . import datamodelFiles as files +from . import datamodelVoice as voice +from . import datamodelUtils as utils diff --git a/modules/interfaces/interfaceAiModel.py b/modules/datamodels/datamodelAi.py similarity index 99% rename from modules/interfaces/interfaceAiModel.py rename to modules/datamodels/datamodelAi.py index 6bd541b2..0d3a8831 100644 --- a/modules/interfaces/interfaceAiModel.py +++ b/modules/datamodels/datamodelAi.py @@ -28,3 +28,5 @@ class AiCallResponse(BaseModel): modelName: str = Field(description="Selected model name") usedTokens: Optional[int] = Field(default=None, description="Estimated used tokens") costEstimate: Optional[float] = Field(default=None, description="Estimated cost of the call") + + diff --git a/modules/datamodels/datamodelChat.py b/modules/datamodels/datamodelChat.py new file mode 100644 index 00000000..afb92454 --- /dev/null +++ b/modules/datamodels/datamodelChat.py @@ -0,0 +1,262 @@ +"""Chat models: ChatWorkflow, ChatMessage, ChatLog, ChatStat, ChatDocument.""" + +from typing import List, Dict, Any, Optional +from pydantic import BaseModel, Field +from modules.shared.attributeUtils import register_model_labels, ModelMixin +from modules.shared.timezoneUtils import get_utc_timestamp +import uuid + + +class ChatStat(BaseModel, ModelMixin): + id: str = Field(default_factory=lambda: str(uuid.uuid4()), description="Primary key") + workflowId: Optional[str] = Field(None, description="Foreign key to workflow (for workflow stats)") + messageId: Optional[str] = Field(None, description="Foreign key to message (for message stats)") + processingTime: Optional[float] = Field(None, description="Processing time in seconds") + tokenCount: Optional[int] = Field(None, description="Number of tokens processed") + bytesSent: Optional[int] = Field(None, description="Number of bytes sent") + bytesReceived: Optional[int] = Field(None, description="Number of bytes received") + successRate: Optional[float] = Field(None, description="Success rate of operations") + errorCount: Optional[int] = Field(None, description="Number of errors encountered") + + +register_model_labels( + "ChatStat", + {"en": "Chat Statistics", "fr": "Statistiques de chat"}, + { + "id": {"en": "ID", "fr": "ID"}, + "workflowId": {"en": "Workflow ID", "fr": "ID du workflow"}, + "messageId": {"en": "Message ID", "fr": "ID du message"}, + "processingTime": {"en": "Processing Time", "fr": "Temps de traitement"}, + "tokenCount": {"en": "Token Count", "fr": "Nombre de tokens"}, + "bytesSent": {"en": "Bytes Sent", "fr": "Octets envoyés"}, + "bytesReceived": {"en": "Bytes Received", "fr": "Octets reçus"}, + "successRate": {"en": "Success Rate", "fr": "Taux de succès"}, + "errorCount": {"en": "Error Count", "fr": "Nombre d'erreurs"}, + }, +) + + +class ChatLog(BaseModel, ModelMixin): + id: str = Field(default_factory=lambda: str(uuid.uuid4()), description="Primary key") + workflowId: str = Field(description="Foreign key to workflow") + message: str = Field(description="Log message") + type: str = Field(description="Log type (info, warning, error, etc.)") + timestamp: float = Field(default_factory=get_utc_timestamp, description="When the log entry was created (UTC timestamp in seconds)") + status: Optional[str] = Field(None, description="Status of the log entry") + progress: Optional[float] = Field(None, description="Progress indicator (0.0 to 1.0)") + performance: Optional[Dict[str, Any]] = Field(None, description="Performance metrics") + + +register_model_labels( + "ChatLog", + {"en": "Chat Log", "fr": "Journal de chat"}, + { + "id": {"en": "ID", "fr": "ID"}, + "workflowId": {"en": "Workflow ID", "fr": "ID du flux de travail"}, + "message": {"en": "Message", "fr": "Message"}, + "type": {"en": "Type", "fr": "Type"}, + "timestamp": {"en": "Timestamp", "fr": "Horodatage"}, + "status": {"en": "Status", "fr": "Statut"}, + "progress": {"en": "Progress", "fr": "Progression"}, + "performance": {"en": "Performance", "fr": "Performance"}, + }, +) + + +class ChatDocument(BaseModel, ModelMixin): + id: str = Field(default_factory=lambda: str(uuid.uuid4()), description="Primary key") + messageId: str = Field(description="Foreign key to message") + fileId: str = Field(description="Foreign key to file") + fileName: str = Field(description="Name of the file") + fileSize: int = Field(description="Size of the file") + mimeType: str = Field(description="MIME type of the file") + roundNumber: Optional[int] = Field(None, description="Round number in workflow") + taskNumber: Optional[int] = Field(None, description="Task number within round") + actionNumber: Optional[int] = Field(None, description="Action number within task") + actionId: Optional[str] = Field(None, description="ID of the action that created this document") + + +register_model_labels( + "ChatDocument", + {"en": "Chat Document", "fr": "Document de chat"}, + { + "id": {"en": "ID", "fr": "ID"}, + "messageId": {"en": "Message ID", "fr": "ID du message"}, + "fileId": {"en": "File ID", "fr": "ID du fichier"}, + "fileName": {"en": "File Name", "fr": "Nom du fichier"}, + "fileSize": {"en": "File Size", "fr": "Taille du fichier"}, + "mimeType": {"en": "MIME Type", "fr": "Type MIME"}, + "roundNumber": {"en": "Round Number", "fr": "Numéro de tour"}, + "taskNumber": {"en": "Task Number", "fr": "Numéro de tâche"}, + "actionNumber": {"en": "Action Number", "fr": "Numéro d'action"}, + "actionId": {"en": "Action ID", "fr": "ID de l'action"}, + }, +) + + +class ContentMetadata(BaseModel, ModelMixin): + size: int = Field(description="Content size in bytes") + pages: Optional[int] = Field(None, description="Number of pages for multi-page content") + error: Optional[str] = Field(None, description="Processing error if any") + width: Optional[int] = Field(None, description="Width in pixels for images/videos") + height: Optional[int] = Field(None, description="Height in pixels for images/videos") + colorMode: Optional[str] = Field(None, description="Color mode") + fps: Optional[float] = Field(None, description="Frames per second for videos") + durationSec: Optional[float] = Field(None, description="Duration in seconds for media") + mimeType: str = Field(description="MIME type of the content") + base64Encoded: bool = Field(description="Whether the data is base64 encoded") + + +register_model_labels( + "ContentMetadata", + {"en": "Content Metadata", "fr": "Métadonnées du contenu"}, + { + "size": {"en": "Size", "fr": "Taille"}, + "pages": {"en": "Pages", "fr": "Pages"}, + "error": {"en": "Error", "fr": "Erreur"}, + "width": {"en": "Width", "fr": "Largeur"}, + "height": {"en": "Height", "fr": "Hauteur"}, + "colorMode": {"en": "Color Mode", "fr": "Mode de couleur"}, + "fps": {"en": "FPS", "fr": "IPS"}, + "durationSec": {"en": "Duration", "fr": "Durée"}, + "mimeType": {"en": "MIME Type", "fr": "Type MIME"}, + "base64Encoded": {"en": "Base64 Encoded", "fr": "Encodé en Base64"}, + }, +) + + +class ContentItem(BaseModel, ModelMixin): + label: str = Field(description="Content label") + data: str = Field(description="Extracted text content") + metadata: ContentMetadata = Field(description="Content metadata") + + +register_model_labels( + "ContentItem", + {"en": "Content Item", "fr": "Élément de contenu"}, + { + "label": {"en": "Label", "fr": "Étiquette"}, + "data": {"en": "Data", "fr": "Données"}, + "metadata": {"en": "Metadata", "fr": "Métadonnées"}, + }, +) + + +class ExtractedContent(BaseModel, ModelMixin): + id: str = Field(description="Reference to source ChatDocument") + contents: List[ContentItem] = Field(default_factory=list, description="List of content items") + + +register_model_labels( + "ExtractedContent", + {"en": "Extracted Content", "fr": "Contenu extrait"}, + { + "id": {"en": "Object ID", "fr": "ID de l'objet"}, + "contents": {"en": "Contents", "fr": "Contenus"}, + }, +) + +class ChatMessage(BaseModel, ModelMixin): + id: str = Field(default_factory=lambda: str(uuid.uuid4()), description="Primary key") + workflowId: str = Field(description="Foreign key to workflow") + parentMessageId: Optional[str] = Field(None, description="Parent message ID for threading") + documents: List[ChatDocument] = Field(default_factory=list, description="Associated documents") + documentsLabel: Optional[str] = Field(None, description="Label for the set of documents") + message: Optional[str] = Field(None, description="Message content") + role: str = Field(description="Role of the message sender") + status: str = Field(description="Status of the message (first, step, last)") + sequenceNr: int = Field(description="Sequence number of the message (set automatically)") + publishedAt: float = Field(default_factory=get_utc_timestamp, description="When the message was published (UTC timestamp in seconds)") + stats: Optional[ChatStat] = Field(None, description="Statistics for this message") + success: Optional[bool] = Field(None, description="Whether the message processing was successful") + actionId: Optional[str] = Field(None, description="ID of the action that produced this message") + actionMethod: Optional[str] = Field(None, description="Method of the action that produced this message") + actionName: Optional[str] = Field(None, description="Name of the action that produced this message") + roundNumber: Optional[int] = Field(None, description="Round number in workflow") + taskNumber: Optional[int] = Field(None, description="Task number within round") + actionNumber: Optional[int] = Field(None, description="Action number within task") + taskProgress: Optional[str] = Field(None, description="Task progress status: pending, running, success, fail, retry") + actionProgress: Optional[str] = Field(None, description="Action progress status: pending, running, success, fail") + + +register_model_labels( + "ChatMessage", + {"en": "Chat Message", "fr": "Message de chat"}, + { + "id": {"en": "ID", "fr": "ID"}, + "workflowId": {"en": "Workflow ID", "fr": "ID du flux de travail"}, + "parentMessageId": {"en": "Parent Message ID", "fr": "ID du message parent"}, + "documents": {"en": "Documents", "fr": "Documents"}, + "documentsLabel": {"en": "Documents Label", "fr": "Label des documents"}, + "message": {"en": "Message", "fr": "Message"}, + "role": {"en": "Role", "fr": "Rôle"}, + "status": {"en": "Status", "fr": "Statut"}, + "sequenceNr": {"en": "Sequence Number", "fr": "Numéro de séquence"}, + "publishedAt": {"en": "Published At", "fr": "Publié le"}, + "stats": {"en": "Statistics", "fr": "Statistiques"}, + "success": {"en": "Success", "fr": "Succès"}, + "actionId": {"en": "Action ID", "fr": "ID de l'action"}, + "actionMethod": {"en": "Action Method", "fr": "Méthode de l'action"}, + "actionName": {"en": "Action Name", "fr": "Nom de l'action"}, + "roundNumber": {"en": "Round Number", "fr": "Numéro de tour"}, + "taskNumber": {"en": "Task Number", "fr": "Numéro de tâche"}, + "actionNumber": {"en": "Action Number", "fr": "Numéro d'action"}, + "taskProgress": {"en": "Task Progress", "fr": "Progression de la tâche"}, + "actionProgress": {"en": "Action Progress", "fr": "Progression de l'action"}, + }, +) + + +class ChatWorkflow(BaseModel, ModelMixin): + id: str = Field(default_factory=lambda: str(uuid.uuid4()), description="Primary key", frontend_type="text", frontend_readonly=True, frontend_required=False) + mandateId: str = Field(description="ID of the mandate this workflow belongs to", frontend_type="text", frontend_readonly=True, frontend_required=False) + status: str = Field(description="Current status of the workflow", frontend_type="select", frontend_readonly=False, frontend_required=False, frontend_options=[ + {"value": "running", "label": {"en": "Running", "fr": "En cours"}}, + {"value": "completed", "label": {"en": "Completed", "fr": "Terminé"}}, + {"value": "stopped", "label": {"en": "Stopped", "fr": "Arrêté"}}, + {"value": "error", "label": {"en": "Error", "fr": "Erreur"}}, + ]) + name: Optional[str] = Field(None, description="Name of the workflow", frontend_type="text", frontend_readonly=False, frontend_required=True) + currentRound: int = Field(description="Current round number", frontend_type="integer", frontend_readonly=True, frontend_required=False) + currentTask: int = Field(default=0, description="Current task number", frontend_type="integer", frontend_readonly=True, frontend_required=False) + currentAction: int = Field(default=0, description="Current action number", frontend_type="integer", frontend_readonly=True, frontend_required=False) + totalTasks: int = Field(default=0, description="Total number of tasks in the workflow", frontend_type="integer", frontend_readonly=True, frontend_required=False) + totalActions: int = Field(default=0, description="Total number of actions in the workflow", frontend_type="integer", frontend_readonly=True, frontend_required=False) + lastActivity: float = Field(default_factory=get_utc_timestamp, description="Timestamp of last activity (UTC timestamp in seconds)", frontend_type="timestamp", frontend_readonly=True, frontend_required=False) + startedAt: float = Field(default_factory=get_utc_timestamp, description="When the workflow started (UTC timestamp in seconds)", frontend_type="timestamp", frontend_readonly=True, frontend_required=False) + logs: List[ChatLog] = Field(default_factory=list, description="Workflow logs", frontend_type="text", frontend_readonly=True, frontend_required=False) + messages: List[ChatMessage] = Field(default_factory=list, description="Messages in the workflow", frontend_type="text", frontend_readonly=True, frontend_required=False) + stats: Optional[ChatStat] = Field(None, description="Workflow statistics", frontend_type="text", frontend_readonly=True, frontend_required=False) + tasks: list = Field(default_factory=list, description="List of tasks in the workflow", frontend_type="text", frontend_readonly=True, frontend_required=False) + workflowMode: str = Field(default="Actionplan", description="Workflow mode selector", frontend_type="select", frontend_readonly=False, frontend_required=False, frontend_options=[ + {"value": "Actionplan", "label": {"en": "Action Plan", "fr": "Plan d'actions"}}, + {"value": "React", "label": {"en": "React", "fr": "Réactif"}}, + ]) + maxSteps: int = Field(default=5, description="Maximum number of iterations in react mode", frontend_type="integer", frontend_readonly=False, frontend_required=False) + + +register_model_labels( + "ChatWorkflow", + {"en": "Chat Workflow", "fr": "Flux de travail de chat"}, + { + "id": {"en": "ID", "fr": "ID"}, + "mandateId": {"en": "Mandate ID", "fr": "ID du mandat"}, + "status": {"en": "Status", "fr": "Statut"}, + "name": {"en": "Name", "fr": "Nom"}, + "currentRound": {"en": "Current Round", "fr": "Tour actuel"}, + "currentTask": {"en": "Current Task", "fr": "Tâche actuelle"}, + "currentAction": {"en": "Current Action", "fr": "Action actuelle"}, + "totalTasks": {"en": "Total Tasks", "fr": "Total des tâches"}, + "totalActions": {"en": "Total Actions", "fr": "Total des actions"}, + "lastActivity": {"en": "Last Activity", "fr": "Dernière activité"}, + "startedAt": {"en": "Started At", "fr": "Démarré le"}, + "logs": {"en": "Logs", "fr": "Journaux"}, + "messages": {"en": "Messages", "fr": "Messages"}, + "stats": {"en": "Statistics", "fr": "Statistiques"}, + "tasks": {"en": "Tasks", "fr": "Tâches"}, + "workflowMode": {"en": "Workflow Mode", "fr": "Mode de workflow"}, + "maxSteps": {"en": "Max Steps", "fr": "Étapes max"}, + }, +) + + diff --git a/modules/datamodels/datamodelFiles.py b/modules/datamodels/datamodelFiles.py new file mode 100644 index 00000000..d561c79e --- /dev/null +++ b/modules/datamodels/datamodelFiles.py @@ -0,0 +1,84 @@ +"""File-related datamodels: FileItem, FilePreview, FileData.""" + +from typing import Dict, Any, Optional, Union +from pydantic import BaseModel, Field +from modules.shared.attributeUtils import register_model_labels, ModelMixin +from modules.shared.timezoneUtils import get_utc_timestamp +import uuid +import base64 + + +class FileItem(BaseModel, ModelMixin): + id: str = Field(default_factory=lambda: str(uuid.uuid4()), description="Primary key", frontend_type="text", frontend_readonly=True, frontend_required=False) + mandateId: str = Field(description="ID of the mandate this file belongs to", frontend_type="text", frontend_readonly=True, frontend_required=False) + fileName: str = Field(description="Name of the file", frontend_type="text", frontend_readonly=False, frontend_required=True) + mimeType: str = Field(description="MIME type of the file", frontend_type="text", frontend_readonly=True, frontend_required=False) + fileHash: str = Field(description="Hash of the file", frontend_type="text", frontend_readonly=True, frontend_required=False) + fileSize: int = Field(description="Size of the file in bytes", frontend_type="integer", frontend_readonly=True, frontend_required=False) + creationDate: float = Field(default_factory=get_utc_timestamp, description="Date when the file was created (UTC timestamp in seconds)", frontend_type="timestamp", frontend_readonly=True, frontend_required=False) + + def to_dict(self) -> Dict[str, Any]: + return super().to_dict() + + +register_model_labels( + "FileItem", + {"en": "File Item", "fr": "Élément de fichier"}, + { + "id": {"en": "ID", "fr": "ID"}, + "mandateId": {"en": "Mandate ID", "fr": "ID du mandat"}, + "fileName": {"en": "fileName", "fr": "Nom de fichier"}, + "mimeType": {"en": "MIME Type", "fr": "Type MIME"}, + "fileHash": {"en": "File Hash", "fr": "Hash du fichier"}, + "fileSize": {"en": "File Size", "fr": "Taille du fichier"}, + "creationDate": {"en": "Creation Date", "fr": "Date de création"}, + }, +) + + +class FilePreview(BaseModel, ModelMixin): + content: Union[str, bytes] = Field(description="File content (text or binary)") + mimeType: str = Field(description="MIME type of the file") + fileName: str = Field(description="Original fileName") + isText: bool = Field(description="Whether the content is text (True) or binary (False)") + encoding: Optional[str] = Field(None, description="Text encoding if content is text") + size: int = Field(description="Size of the content in bytes") + + def to_dict(self) -> Dict[str, Any]: + data = super().to_dict() + if isinstance(data.get("content"), bytes): + data["content"] = base64.b64encode(data["content"]).decode("utf-8") + return data + + +register_model_labels( + "FilePreview", + {"en": "File Preview", "fr": "Aperçu du fichier"}, + { + "content": {"en": "Content", "fr": "Contenu"}, + "mimeType": {"en": "MIME Type", "fr": "Type MIME"}, + "fileName": {"en": "fileName", "fr": "Nom de fichier"}, + "isText": {"en": "Is Text", "fr": "Est du texte"}, + "encoding": {"en": "Encoding", "fr": "Encodage"}, + "size": {"en": "Size", "fr": "Taille"}, + }, +) + + +class FileData(BaseModel, ModelMixin): + id: str = Field(default_factory=lambda: str(uuid.uuid4()), description="Primary key") + data: str = Field(description="File data content") + base64Encoded: bool = Field(description="Whether the data is base64 encoded") + + +register_model_labels( + "FileData", + {"en": "File Data", "fr": "Données de fichier"}, + { + "id": {"en": "ID", "fr": "ID"}, + "data": {"en": "Data", "fr": "Données"}, + "base64Encoded": {"en": "Base64 Encoded", "fr": "Encodé en Base64"}, + }, +) + + diff --git a/modules/datamodels/datamodelNeutralizer.py b/modules/datamodels/datamodelNeutralizer.py new file mode 100644 index 00000000..475b1146 --- /dev/null +++ b/modules/datamodels/datamodelNeutralizer.py @@ -0,0 +1,56 @@ +"""Neutralizer models: DataNeutraliserConfig and DataNeutralizerAttributes.""" + +import uuid +from typing import Optional +from pydantic import BaseModel, Field +from modules.shared.attributeUtils import register_model_labels, ModelMixin + + +class DataNeutraliserConfig(BaseModel, ModelMixin): + id: str = Field(default_factory=lambda: str(uuid.uuid4()), description="Unique ID of the configuration", frontend_type="text", frontend_readonly=True, frontend_required=False) + mandateId: str = Field(description="ID of the mandate this configuration belongs to", frontend_type="text", frontend_readonly=True, frontend_required=True) + userId: str = Field(description="ID of the user who created this configuration", frontend_type="text", frontend_readonly=True, frontend_required=True) + enabled: bool = Field(default=True, description="Whether data neutralization is enabled", frontend_type="checkbox", frontend_readonly=False, frontend_required=False) + namesToParse: str = Field(default="", description="Multiline list of names to parse for neutralization", frontend_type="textarea", frontend_readonly=False, frontend_required=False) + sharepointSourcePath: str = Field(default="", description="SharePoint path to read files for neutralization", frontend_type="text", frontend_readonly=False, frontend_required=False) + sharepointTargetPath: str = Field(default="", description="SharePoint path to store neutralized files", frontend_type="text", frontend_readonly=False, frontend_required=False) + + +register_model_labels( + "DataNeutraliserConfig", + {"en": "Data Neutralization Config", "fr": "Configuration de neutralisation des données"}, + { + "id": {"en": "ID", "fr": "ID"}, + "mandateId": {"en": "Mandate ID", "fr": "ID de mandat"}, + "userId": {"en": "User ID", "fr": "ID utilisateur"}, + "enabled": {"en": "Enabled", "fr": "Activé"}, + "namesToParse": {"en": "Names to Parse", "fr": "Noms à analyser"}, + "sharepointSourcePath": {"en": "Source Path", "fr": "Chemin source"}, + "sharepointTargetPath": {"en": "Target Path", "fr": "Chemin cible"}, + }, +) + + +class DataNeutralizerAttributes(BaseModel, ModelMixin): + id: str = Field(default_factory=lambda: str(uuid.uuid4()), description="Unique ID of the attribute mapping (used as UID in neutralized files)", frontend_type="text", frontend_readonly=True, frontend_required=False) + mandateId: str = Field(description="ID of the mandate this attribute belongs to", frontend_type="text", frontend_readonly=True, frontend_required=True) + userId: str = Field(description="ID of the user who created this attribute", frontend_type="text", frontend_readonly=True, frontend_required=True) + originalText: str = Field(description="Original text that was neutralized", frontend_type="text", frontend_readonly=True, frontend_required=True) + fileId: Optional[str] = Field(default=None, description="ID of the file this attribute belongs to", frontend_type="text", frontend_readonly=True, frontend_required=False) + patternType: str = Field(description="Type of pattern that matched (email, phone, name, etc.)", frontend_type="text", frontend_readonly=True, frontend_required=True) + + +register_model_labels( + "DataNeutralizerAttributes", + {"en": "Neutralized Data Attribute", "fr": "Attribut de données neutralisées"}, + { + "id": {"en": "ID", "fr": "ID"}, + "mandateId": {"en": "Mandate ID", "fr": "ID de mandat"}, + "userId": {"en": "User ID", "fr": "ID utilisateur"}, + "originalText": {"en": "Original Text", "fr": "Texte original"}, + "fileId": {"en": "File ID", "fr": "ID de fichier"}, + "patternType": {"en": "Pattern Type", "fr": "Type de modèle"}, + }, +) + + diff --git a/modules/datamodels/datamodelSecurity.py b/modules/datamodels/datamodelSecurity.py new file mode 100644 index 00000000..ff6a3f6f --- /dev/null +++ b/modules/datamodels/datamodelSecurity.py @@ -0,0 +1,87 @@ +"""Security models: Token and AuthEvent.""" + +from typing import Optional +from pydantic import BaseModel, Field +from modules.shared.attributeUtils import register_model_labels, ModelMixin +from modules.shared.timezoneUtils import get_utc_timestamp +from .datamodelUam import AuthAuthority +from enum import Enum +import uuid + + +class TokenStatus(str, Enum): + ACTIVE = "active" + REVOKED = "revoked" + + +class Token(BaseModel, ModelMixin): + id: Optional[str] = None + userId: str + authority: AuthAuthority + connectionId: Optional[str] = Field(None, description="ID of the connection this token belongs to") + tokenAccess: str + tokenType: str = "bearer" + expiresAt: float = Field(description="When the token expires (UTC timestamp in seconds)") + tokenRefresh: Optional[str] = None + createdAt: Optional[float] = Field(None, description="When the token was created (UTC timestamp in seconds)") + status: TokenStatus = Field(default=TokenStatus.ACTIVE, description="Token status: active/revoked") + revokedAt: Optional[float] = Field(None, description="When the token was revoked (UTC timestamp in seconds)") + revokedBy: Optional[str] = Field(None, description="User ID who revoked the token (admin/self)") + reason: Optional[str] = Field(None, description="Optional revocation reason") + sessionId: Optional[str] = Field(None, description="Logical session grouping for logout revocation") + mandateId: Optional[str] = Field(None, description="Mandate ID for tenant scoping of the token") + + class Config: + use_enum_values = True + + +register_model_labels( + "Token", + {"en": "Token", "fr": "Jeton"}, + { + "id": {"en": "ID", "fr": "ID"}, + "userId": {"en": "User ID", "fr": "ID utilisateur"}, + "authority": {"en": "Authority", "fr": "Autorité"}, + "connectionId": {"en": "Connection ID", "fr": "ID de connexion"}, + "tokenAccess": {"en": "Access Token", "fr": "Jeton d'accès"}, + "tokenType": {"en": "Token Type", "fr": "Type de jeton"}, + "expiresAt": {"en": "Expires At", "fr": "Expire le"}, + "tokenRefresh": {"en": "Refresh Token", "fr": "Jeton de rafraîchissement"}, + "createdAt": {"en": "Created At", "fr": "Créé le"}, + "status": {"en": "Status", "fr": "Statut"}, + "revokedAt": {"en": "Revoked At", "fr": "Révoqué le"}, + "revokedBy": {"en": "Revoked By", "fr": "Révoqué par"}, + "reason": {"en": "Reason", "fr": "Raison"}, + "sessionId": {"en": "Session ID", "fr": "ID de session"}, + "mandateId": {"en": "Mandate ID", "fr": "ID de mandat"}, + }, +) + + +class AuthEvent(BaseModel, ModelMixin): + id: str = Field(default_factory=lambda: str(uuid.uuid4()), description="Unique ID of the auth event", frontend_type="text", frontend_readonly=True, frontend_required=False) + userId: str = Field(description="ID of the user this event belongs to", frontend_type="text", frontend_readonly=True, frontend_required=True) + eventType: str = Field(description="Type of authentication event (e.g., 'login', 'logout', 'token_refresh')", frontend_type="text", frontend_readonly=True, frontend_required=True) + timestamp: float = Field(default_factory=get_utc_timestamp, description="Unix timestamp when the event occurred", frontend_type="datetime", frontend_readonly=True, frontend_required=True) + ipAddress: Optional[str] = Field(default=None, description="IP address from which the event originated", frontend_type="text", frontend_readonly=True, frontend_required=False) + userAgent: Optional[str] = Field(default=None, description="User agent string from the request", frontend_type="text", frontend_readonly=True, frontend_required=False) + success: bool = Field(default=True, description="Whether the authentication event was successful", frontend_type="boolean", frontend_readonly=True, frontend_required=True) + details: Optional[str] = Field(default=None, description="Additional details about the event", frontend_type="text", frontend_readonly=True, frontend_required=False) + + +register_model_labels( + "AuthEvent", + {"en": "Authentication Event", "fr": "Événement d'authentification"}, + { + "id": {"en": "ID", "fr": "ID"}, + "userId": {"en": "User ID", "fr": "ID utilisateur"}, + "eventType": {"en": "Event Type", "fr": "Type d'événement"}, + "timestamp": {"en": "Timestamp", "fr": "Horodatage"}, + "ipAddress": {"en": "IP Address", "fr": "Adresse IP"}, + "userAgent": {"en": "User Agent", "fr": "Agent utilisateur"}, + "success": {"en": "Success", "fr": "Succès"}, + "details": {"en": "Details", "fr": "Détails"}, + }, +) + + diff --git a/modules/interfaces/interfaceTicketModel.py b/modules/datamodels/datamodelTickets.py similarity index 51% rename from modules/interfaces/interfaceTicketModel.py rename to modules/datamodels/datamodelTickets.py index a7cff7ad..d11606c6 100644 --- a/modules/interfaces/interfaceTicketModel.py +++ b/modules/datamodels/datamodelTickets.py @@ -1,6 +1,6 @@ -"""Base class for ticket classes.""" +"""Ticket datamodels used across Jira/ClickUp connectors.""" -from typing import Any, Dict +from typing import Optional from pydantic import BaseModel, Field from abc import ABC, abstractmethod @@ -10,17 +10,14 @@ class TicketFieldAttribute(BaseModel): field: str = Field(description="Ticket field ID/key") -class Task(BaseModel): - # A very flexible approach for now. Might want to be more strict in the future. - data: Dict[str, Any] = Field(default_factory=dict, description="Task data") - - class TicketBase(ABC): @abstractmethod async def read_attributes(self) -> list[TicketFieldAttribute]: ... @abstractmethod - async def read_tasks(self, limit: int = 0) -> list[Task]: ... + async def read_tasks(self, *, limit: int = 0) -> list[dict]: ... @abstractmethod - async def write_tasks(self, tasklist: list[Task]) -> None: ... + async def write_tasks(self, tasklist: list[dict]) -> None: ... + + diff --git a/modules/datamodels/datamodelUam.py b/modules/datamodels/datamodelUam.py new file mode 100644 index 00000000..283ff882 --- /dev/null +++ b/modules/datamodels/datamodelUam.py @@ -0,0 +1,154 @@ +"""UAM models: User, Mandate, UserConnection.""" + +import uuid +from typing import Optional +from enum import Enum +from pydantic import BaseModel, Field, EmailStr +from modules.shared.attributeUtils import register_model_labels, ModelMixin +from modules.shared.timezoneUtils import get_utc_timestamp + + +class AuthAuthority(str, Enum): + LOCAL = "local" + GOOGLE = "google" + MSFT = "msft" + + +class UserPrivilege(str, Enum): + SYSADMIN = "sysadmin" + ADMIN = "admin" + USER = "user" + + +class ConnectionStatus(str, Enum): + ACTIVE = "active" + EXPIRED = "expired" + REVOKED = "revoked" + PENDING = "pending" + + +class Mandate(BaseModel, ModelMixin): + id: str = Field(default_factory=lambda: str(uuid.uuid4()), description="Unique ID of the mandate", frontend_type="text", frontend_readonly=True, frontend_required=False) + name: str = Field(description="Name of the mandate", frontend_type="text", frontend_readonly=False, frontend_required=True) + language: str = Field(default="en", description="Default language of the mandate", frontend_type="select", frontend_readonly=False, frontend_required=True, frontend_options=[ + {"value": "de", "label": {"en": "Deutsch", "fr": "Allemand"}}, + {"value": "en", "label": {"en": "English", "fr": "Anglais"}}, + {"value": "fr", "label": {"en": "Français", "fr": "Français"}}, + {"value": "it", "label": {"en": "Italiano", "fr": "Italien"}}, + ]) + enabled: bool = Field(default=True, description="Indicates whether the mandate is enabled", frontend_type="checkbox", frontend_readonly=False, frontend_required=False) + + +register_model_labels( + "Mandate", + {"en": "Mandate", "fr": "Mandat"}, + { + "id": {"en": "ID", "fr": "ID"}, + "name": {"en": "Name", "fr": "Nom"}, + "language": {"en": "Language", "fr": "Langue"}, + "enabled": {"en": "Enabled", "fr": "Activé"}, + }, +) + + +class UserConnection(BaseModel, ModelMixin): + id: str = Field(default_factory=lambda: str(uuid.uuid4()), description="Unique ID of the connection", frontend_type="text", frontend_readonly=True, frontend_required=False) + userId: str = Field(description="ID of the user this connection belongs to", frontend_type="text", frontend_readonly=True, frontend_required=False) + authority: AuthAuthority = Field(description="Authentication authority", frontend_type="select", frontend_readonly=True, frontend_required=False, frontend_options=[ + {"value": "local", "label": {"en": "Local", "fr": "Local"}}, + {"value": "google", "label": {"en": "Google", "fr": "Google"}}, + {"value": "msft", "label": {"en": "Microsoft", "fr": "Microsoft"}}, + ]) + externalId: str = Field(description="User ID in the external system", frontend_type="text", frontend_readonly=True, frontend_required=False) + externalUsername: str = Field(description="Username in the external system", frontend_type="text", frontend_readonly=False, frontend_required=False) + externalEmail: Optional[EmailStr] = Field(None, description="Email in the external system", frontend_type="email", frontend_readonly=False, frontend_required=False) + status: ConnectionStatus = Field(default=ConnectionStatus.ACTIVE, description="Connection status", frontend_type="select", frontend_readonly=False, frontend_required=False, frontend_options=[ + {"value": "active", "label": {"en": "Active", "fr": "Actif"}}, + {"value": "inactive", "label": {"en": "Inactive", "fr": "Inactif"}}, + {"value": "expired", "label": {"en": "Expired", "fr": "Expiré"}}, + {"value": "pending", "label": {"en": "Pending", "fr": "En attente"}}, + ]) + connectedAt: float = Field(default_factory=get_utc_timestamp, description="When the connection was established (UTC timestamp in seconds)", frontend_type="timestamp", frontend_readonly=True, frontend_required=False) + lastChecked: float = Field(default_factory=get_utc_timestamp, description="When the connection was last verified (UTC timestamp in seconds)", frontend_type="timestamp", frontend_readonly=True, frontend_required=False) + expiresAt: Optional[float] = Field(None, description="When the connection expires (UTC timestamp in seconds)", frontend_type="timestamp", frontend_readonly=True, frontend_required=False) + tokenStatus: Optional[str] = Field(None, description="Current token status: active, expired, none", frontend_type="select", frontend_readonly=True, frontend_required=False, frontend_options=[ + {"value": "active", "label": {"en": "Active", "fr": "Actif"}}, + {"value": "expired", "label": {"en": "Expired", "fr": "Expiré"}}, + {"value": "none", "label": {"en": "None", "fr": "Aucun"}}, + ]) + tokenExpiresAt: Optional[float] = Field(None, description="When the current token expires (UTC timestamp in seconds)", frontend_type="timestamp", frontend_readonly=True, frontend_required=False) + + +register_model_labels( + "UserConnection", + {"en": "User Connection", "fr": "Connexion utilisateur"}, + { + "id": {"en": "ID", "fr": "ID"}, + "userId": {"en": "User ID", "fr": "ID utilisateur"}, + "authority": {"en": "Authority", "fr": "Autorité"}, + "externalId": {"en": "External ID", "fr": "ID externe"}, + "externalUsername": {"en": "External Username", "fr": "Nom d'utilisateur externe"}, + "externalEmail": {"en": "External Email", "fr": "Email externe"}, + "status": {"en": "Status", "fr": "Statut"}, + "connectedAt": {"en": "Connected At", "fr": "Connecté le"}, + "lastChecked": {"en": "Last Checked", "fr": "Dernière vérification"}, + "expiresAt": {"en": "Expires At", "fr": "Expire le"}, + "tokenStatus": {"en": "Connection Status", "fr": "Statut de connexion"}, + "tokenExpiresAt": {"en": "Expires At", "fr": "Expire le"}, + }, +) + + +class User(BaseModel, ModelMixin): + id: str = Field(default_factory=lambda: str(uuid.uuid4()), description="Unique ID of the user", frontend_type="text", frontend_readonly=True, frontend_required=False) + username: str = Field(description="Username for login", frontend_type="text", frontend_readonly=False, frontend_required=True) + email: Optional[EmailStr] = Field(None, description="Email address of the user", frontend_type="email", frontend_readonly=False, frontend_required=True) + fullName: Optional[str] = Field(None, description="Full name of the user", frontend_type="text", frontend_readonly=False, frontend_required=False) + language: str = Field(default="en", description="Preferred language of the user", frontend_type="select", frontend_readonly=False, frontend_required=True, frontend_options=[ + {"value": "de", "label": {"en": "Deutsch", "fr": "Allemand"}}, + {"value": "en", "label": {"en": "English", "fr": "Anglais"}}, + {"value": "fr", "label": {"en": "Français", "fr": "Français"}}, + {"value": "it", "label": {"en": "Italiano", "fr": "Italien"}}, + ]) + enabled: bool = Field(default=True, description="Indicates whether the user is enabled", frontend_type="checkbox", frontend_readonly=False, frontend_required=False) + privilege: UserPrivilege = Field(default=UserPrivilege.USER, description="Permission level", frontend_type="select", frontend_readonly=False, frontend_required=True, frontend_options=[ + {"value": "user", "label": {"en": "User", "fr": "Utilisateur"}}, + {"value": "admin", "label": {"en": "Admin", "fr": "Administrateur"}}, + {"value": "sysadmin", "label": {"en": "SysAdmin", "fr": "Administrateur système"}}, + ]) + authenticationAuthority: AuthAuthority = Field(default=AuthAuthority.LOCAL, description="Primary authentication authority", frontend_type="select", frontend_readonly=True, frontend_required=False, frontend_options=[ + {"value": "local", "label": {"en": "Local", "fr": "Local"}}, + {"value": "google", "label": {"en": "Google", "fr": "Google"}}, + {"value": "msft", "label": {"en": "Microsoft", "fr": "Microsoft"}}, + ]) + mandateId: Optional[str] = Field(None, description="ID of the mandate this user belongs to", frontend_type="text", frontend_readonly=True, frontend_required=False) + + +register_model_labels( + "User", + {"en": "User", "fr": "Utilisateur"}, + { + "id": {"en": "ID", "fr": "ID"}, + "username": {"en": "Username", "fr": "Nom d'utilisateur"}, + "email": {"en": "Email", "fr": "Email"}, + "fullName": {"en": "Full Name", "fr": "Nom complet"}, + "language": {"en": "Language", "fr": "Langue"}, + "enabled": {"en": "Enabled", "fr": "Activé"}, + "privilege": {"en": "Privilege", "fr": "Privilège"}, + "authenticationAuthority": {"en": "Auth Authority", "fr": "Autorité d'authentification"}, + "mandateId": {"en": "Mandate ID", "fr": "ID de mandat"}, + }, +) + + +class UserInDB(User): + hashedPassword: Optional[str] = Field(None, description="Hash of the user password") + + +register_model_labels( + "UserInDB", + {"en": "User Access", "fr": "Accès de l'utilisateur"}, + {"hashedPassword": {"en": "Password hash", "fr": "Hachage de mot de passe"}}, +) + + diff --git a/modules/datamodels/datamodelUtils.py b/modules/datamodels/datamodelUtils.py new file mode 100644 index 00000000..82a888e7 --- /dev/null +++ b/modules/datamodels/datamodelUtils.py @@ -0,0 +1,26 @@ +"""Utility datamodels: Prompt.""" + +from pydantic import BaseModel, Field +from modules.shared.attributeUtils import register_model_labels, ModelMixin +import uuid + + +class Prompt(BaseModel, ModelMixin): + id: str = Field(default_factory=lambda: str(uuid.uuid4()), description="Primary key", frontend_type="text", frontend_readonly=True, frontend_required=False) + mandateId: str = Field(description="ID of the mandate this prompt belongs to", frontend_type="text", frontend_readonly=True, frontend_required=False) + content: str = Field(description="Content of the prompt", frontend_type="textarea", frontend_readonly=False, frontend_required=True) + name: str = Field(description="Name of the prompt", frontend_type="text", frontend_readonly=False, frontend_required=True) + + +register_model_labels( + "Prompt", + {"en": "Prompt", "fr": "Invite"}, + { + "id": {"en": "ID", "fr": "ID"}, + "mandateId": {"en": "Mandate ID", "fr": "ID du mandat"}, + "content": {"en": "Content", "fr": "Contenu"}, + "name": {"en": "Name", "fr": "Nom"}, + }, +) + + diff --git a/modules/datamodels/datamodelVoice.py b/modules/datamodels/datamodelVoice.py new file mode 100644 index 00000000..3fc69cd8 --- /dev/null +++ b/modules/datamodels/datamodelVoice.py @@ -0,0 +1,43 @@ +"""Voice settings datamodel.""" + +from typing import Dict, Any, Optional +from pydantic import BaseModel, Field +from modules.shared.attributeUtils import register_model_labels, ModelMixin +from modules.shared.timezoneUtils import get_utc_timestamp +import uuid + + +class VoiceSettings(BaseModel, ModelMixin): + id: str = Field(default_factory=lambda: str(uuid.uuid4()), description="Primary key", frontend_type="text", frontend_readonly=True, frontend_required=False) + userId: str = Field(description="ID of the user these settings belong to", frontend_type="text", frontend_readonly=True, frontend_required=True) + mandateId: str = Field(description="ID of the mandate these settings belong to", frontend_type="text", frontend_readonly=True, frontend_required=False) + sttLanguage: str = Field(default="de-DE", description="Speech-to-Text language", frontend_type="select", frontend_readonly=False, frontend_required=True) + ttsLanguage: str = Field(default="de-DE", description="Text-to-Speech language", frontend_type="select", frontend_readonly=False, frontend_required=True) + ttsVoice: str = Field(default="de-DE-KatjaNeural", description="Text-to-Speech voice", frontend_type="select", frontend_readonly=False, frontend_required=True) + translationEnabled: bool = Field(default=True, description="Whether translation is enabled", frontend_type="checkbox", frontend_readonly=False, frontend_required=False) + targetLanguage: str = Field(default="en-US", description="Target language for translation", frontend_type="select", frontend_readonly=False, frontend_required=False) + creationDate: float = Field(default_factory=get_utc_timestamp, description="Date when the settings were created (UTC timestamp in seconds)", frontend_type="timestamp", frontend_readonly=True, frontend_required=False) + lastModified: float = Field(default_factory=get_utc_timestamp, description="Date when the settings were last modified (UTC timestamp in seconds)", frontend_type="timestamp", frontend_readonly=True, frontend_required=False) + + def to_dict(self) -> Dict[str, Any]: + return super().to_dict() + + +register_model_labels( + "VoiceSettings", + {"en": "Voice Settings", "fr": "Paramètres vocaux"}, + { + "id": {"en": "ID", "fr": "ID"}, + "userId": {"en": "User ID", "fr": "ID utilisateur"}, + "mandateId": {"en": "Mandate ID", "fr": "ID du mandat"}, + "sttLanguage": {"en": "STT Language", "fr": "Langue STT"}, + "ttsLanguage": {"en": "TTS Language", "fr": "Langue TTS"}, + "ttsVoice": {"en": "TTS Voice", "fr": "Voix TTS"}, + "translationEnabled": {"en": "Translation Enabled", "fr": "Traduction activée"}, + "targetLanguage": {"en": "Target Language", "fr": "Langue cible"}, + "creationDate": {"en": "Creation Date", "fr": "Date de création"}, + "lastModified": {"en": "Last Modified", "fr": "Dernière modification"}, + }, +) + + diff --git a/modules/interfaces/interfaceWebModel.py b/modules/datamodels/datamodelWeb.py similarity index 69% rename from modules/interfaces/interfaceWebModel.py rename to modules/datamodels/datamodelWeb.py index 4f030e4e..3ebf7e58 100644 --- a/modules/interfaces/interfaceWebModel.py +++ b/modules/datamodels/datamodelWeb.py @@ -1,36 +1,20 @@ -"""Base class for web classes.""" +"""Web-related modules.datamodels (search, crawl, scrape).""" from abc import ABC, abstractmethod -from modules.interfaces.interfaceChatModel import ActionDocument, ActionResult from pydantic import BaseModel, Field, HttpUrl from typing import List, Optional, Literal from modules.shared.configuration import APP_CONFIG +from modules.datamodels.datamodelWorkflow import ActionDocument, ActionResult -# Configuration loading functions -def get_web_search_max_query_length() -> int: - """Get maximum query length from configuration""" - return int(APP_CONFIG.get("Web_Search_MAX_QUERY_LENGTH", "400")) - - -def get_web_search_max_results() -> int: - """Get maximum search results from configuration""" - return int(APP_CONFIG.get("Web_Search_MAX_RESULTS", "20")) - - -def get_web_search_min_results() -> int: - """Get minimum search results from configuration""" - return int(APP_CONFIG.get("Web_Search_MIN_RESULTS", "1")) - - -# --- Web search --- - -# query -> list of URLs +WEB_SEARCH_MAX_QUERY_LENGTH: int = int(APP_CONFIG.get("Web_Search_MAX_QUERY_LENGTH", "400")) +WEB_SEARCH_MAX_RESULTS: int = int(APP_CONFIG.get("Web_Search_MAX_RESULTS", "20")) +WEB_SEARCH_MIN_RESULTS: int = int(APP_CONFIG.get("Web_Search_MIN_RESULTS", "1")) class WebSearchRequest(BaseModel): - query: str = Field(min_length=1, max_length=get_web_search_max_query_length()) - max_results: int = Field(ge=get_web_search_min_results(), le=get_web_search_max_results()) + query: str = Field(min_length=1, max_length=WEB_SEARCH_MAX_QUERY_LENGTH) + max_results: int = Field(ge=WEB_SEARCH_MIN_RESULTS, le=WEB_SEARCH_MAX_RESULTS) # Tavily tuning options search_depth: Optional[Literal["basic", "advanced"]] = Field(default=None) time_range: Optional[Literal["d", "w", "m", "y"]] = Field( @@ -52,10 +36,11 @@ class WebSearchResultItem(BaseModel): class WebSearchDocumentData(BaseModel): - """Complete search results document""" + """Complete search (and scrape) results document""" - query: str = Field(min_length=1, max_length=get_web_search_max_query_length()) - results: List[WebSearchResultItem] + query: str = Field(min_length=1, max_length=WEB_SEARCH_MAX_QUERY_LENGTH) + # Allow both WebSearchResultItem and WebScrapeResultItem to be stored here + results: List[object] total_count: int @@ -74,8 +59,6 @@ class WebSearchBase(ABC): # --- Web crawl --- -# list of URLs -> list of extracted HTML content - class WebCrawlRequest(BaseModel): urls: List[HttpUrl] @@ -116,12 +99,10 @@ class WebCrawlBase(ABC): # --- Web scrape --- -# scrape -> list of extracted text; combines web search and crawl in one step - class WebScrapeRequest(BaseModel): - query: str = Field(min_length=1, max_length=get_web_search_max_query_length()) - max_results: int = Field(ge=get_web_search_min_results(), le=get_web_search_max_results()) + query: str = Field(min_length=1, max_length=WEB_SEARCH_MAX_QUERY_LENGTH) + max_results: int = Field(ge=WEB_SEARCH_MIN_RESULTS, le=WEB_SEARCH_MAX_RESULTS) # Pass-through search options search_depth: Optional[Literal["basic", "advanced"]] = Field(default=None) time_range: Optional[Literal["d", "w", "m", "y"]] = Field(default=None) @@ -143,16 +124,8 @@ class WebScrapeResultItem(BaseModel): content: str -class WebScrapeDocumentData(BaseModel): - """Complete scrape results document""" - - query: str = Field(min_length=1, max_length=get_web_search_max_query_length()) - results: List[WebScrapeResultItem] - total_count: int - - class WebScrapeActionDocument(ActionDocument): - documentData: WebScrapeDocumentData = Field( + documentData: WebSearchDocumentData = Field( description="The data extracted from scraped URLs" ) @@ -164,3 +137,5 @@ class WebScrapeActionResult(ActionResult): class WebScrapeBase(ABC): @abstractmethod async def scrape(self, request: WebScrapeRequest) -> WebScrapeActionResult: ... + + diff --git a/modules/datamodels/datamodelWorkflow.py b/modules/datamodels/datamodelWorkflow.py new file mode 100644 index 00000000..f4554def --- /dev/null +++ b/modules/datamodels/datamodelWorkflow.py @@ -0,0 +1,446 @@ +"""Workflow-related base datamodels and step/task structures.""" + +from typing import List, Dict, Any, Optional +from pydantic import BaseModel, Field +from modules.shared.attributeUtils import register_model_labels, ModelMixin + + +class ActionDocument(BaseModel, ModelMixin): + """Clear document structure for action results""" + documentName: str = Field(description="Name of the document") + documentData: Any = Field(description="Content/data of the document") + mimeType: str = Field(description="MIME type of the document") + + +register_model_labels( + "ActionDocument", + {"en": "Action Document", "fr": "Document d'action"}, + { + "documentName": {"en": "Document Name", "fr": "Nom du document"}, + "documentData": {"en": "Document Data", "fr": "Données du document"}, + "mimeType": {"en": "MIME Type", "fr": "Type MIME"}, + }, +) + + +class ActionResult(BaseModel, ModelMixin): + """Clean action result with documents as primary output + + IMPORTANT: Action methods should NOT set resultLabel in their return value. + The resultLabel is managed by the action handler using the action's execResultLabel + from the action plan. This ensures consistent document routing throughout the workflow. + """ + + success: bool = Field(description="Whether execution succeeded") + error: Optional[str] = Field(None, description="Error message if failed") + documents: List[ActionDocument] = Field(default_factory=list, description="Document outputs") + resultLabel: Optional[str] = Field(None, description="Label for document routing (set by action handler, not by action methods)") + + @classmethod + def isSuccess(cls, documents: List[ActionDocument] = None) -> "ActionResult": + return cls(success=True, documents=documents or []) + + @classmethod + def isFailure(cls, error: str, documents: List[ActionDocument] = None) -> "ActionResult": + return cls(success=False, documents=documents or [], error=error) + + +register_model_labels( + "ActionResult", + {"en": "Action Result", "fr": "Résultat de l'action"}, + { + "success": {"en": "Success", "fr": "Succès"}, + "error": {"en": "Error", "fr": "Erreur"}, + "documents": {"en": "Documents", "fr": "Documents"}, + "resultLabel": {"en": "Result Label", "fr": "Étiquette du résultat"}, + }, +) + + +# ===== Additional workflow models migrated from interfaceChatModel ===== + + +class ActionSelection(BaseModel, ModelMixin): + method: str = Field(description="Method to execute (e.g., web, document, ai)") + name: str = Field(description="Action name within the method (e.g., search, extract)") + + +register_model_labels( + "ActionSelection", + {"en": "Action Selection", "fr": "Sélection d'action"}, + { + "method": {"en": "Method", "fr": "Méthode"}, + "name": {"en": "Action Name", "fr": "Nom de l'action"}, + }, +) + + +class ActionParameters(BaseModel, ModelMixin): + parameters: Dict[str, Any] = Field(default_factory=dict, description="Parameters to execute the selected action") + + +register_model_labels( + "ActionParameters", + {"en": "Action Parameters", "fr": "Paramètres d'action"}, + { + "parameters": {"en": "Parameters", "fr": "Paramètres"}, + }, +) + + +class ObservationPreview(BaseModel, ModelMixin): + name: str = Field(description="Document name or URL label") + mime: str = Field(description="MIME type or kind") + snippet: str = Field(description="Short snippet or summary") + + +register_model_labels( + "ObservationPreview", + {"en": "Observation Preview", "fr": "Aperçu d'observation"}, + { + "name": {"en": "Name", "fr": "Nom"}, + "mime": {"en": "MIME", "fr": "MIME"}, + "snippet": {"en": "Snippet", "fr": "Extrait"}, + }, +) + + +class Observation(BaseModel, ModelMixin): + success: bool = Field(description="Action execution success flag") + resultLabel: str = Field(description="Deterministic label for produced documents") + documentsCount: int = Field(description="Number of produced documents") + previews: List[ObservationPreview] = Field(default_factory=list, description="Compact previews of outputs") + notes: List[str] = Field(default_factory=list, description="Short notes or key facts") + + +register_model_labels( + "Observation", + {"en": "Observation", "fr": "Observation"}, + { + "success": {"en": "Success", "fr": "Succès"}, + "resultLabel": {"en": "Result Label", "fr": "Étiquette du résultat"}, + "documentsCount": {"en": "Documents Count", "fr": "Nombre de documents"}, + "previews": {"en": "Previews", "fr": "Aperçus"}, + "notes": {"en": "Notes", "fr": "Notes"}, + }, +) + + +class TaskStatus(str): + PENDING = "pending" + RUNNING = "running" + COMPLETED = "completed" + FAILED = "failed" + CANCELLED = "cancelled" + + +register_model_labels( + "TaskStatus", + {"en": "Task Status", "fr": "Statut de la tâche"}, + { + "PENDING": {"en": "Pending", "fr": "En attente"}, + "RUNNING": {"en": "Running", "fr": "En cours"}, + "COMPLETED": {"en": "Completed", "fr": "Terminé"}, + "FAILED": {"en": "Failed", "fr": "Échec"}, + "CANCELLED": {"en": "Cancelled", "fr": "Annulé"}, + }, +) + + +class DocumentExchange(BaseModel, ModelMixin): + documentsLabel: str = Field(description="Label for the set of documents") + documents: List[str] = Field(default_factory=list, description="List of document references") + + +register_model_labels( + "DocumentExchange", + {"en": "Document Exchange", "fr": "Échange de documents"}, + { + "documentsLabel": {"en": "Documents Label", "fr": "Label des documents"}, + "documents": {"en": "Documents", "fr": "Documents"}, + }, +) + + +class TaskAction(BaseModel, ModelMixin): + id: str = Field(..., description="Action ID") + execMethod: str = Field(..., description="Method to execute") + execAction: str = Field(..., description="Action to perform") + execParameters: Dict[str, Any] = Field(default_factory=dict, description="Action parameters") + execResultLabel: Optional[str] = Field(None, description="Label for the set of result documents") + expectedDocumentFormats: Optional[List[Dict[str, str]]] = Field(None, description="Expected document formats (optional)") + userMessage: Optional[str] = Field(None, description="User-friendly message in user's language") + status: TaskStatus = Field(default=TaskStatus.PENDING, description="Action status") + error: Optional[str] = Field(None, description="Error message if action failed") + retryCount: int = Field(default=0, description="Number of retries attempted") + retryMax: int = Field(default=3, description="Maximum number of retries") + processingTime: Optional[float] = Field(None, description="Processing time in seconds") + timestamp: float = Field(..., description="When the action was executed (UTC timestamp in seconds)") + result: Optional[str] = Field(None, description="Result of the action") + + +register_model_labels( + "TaskAction", + {"en": "Task Action", "fr": "Action de tâche"}, + { + "id": {"en": "Action ID", "fr": "ID de l'action"}, + "execMethod": {"en": "Method", "fr": "Méthode"}, + "execAction": {"en": "Action", "fr": "Action"}, + "execParameters": {"en": "Parameters", "fr": "Paramètres"}, + "execResultLabel": {"en": "Result Label", "fr": "Label du résultat"}, + "expectedDocumentFormats": {"en": "Expected Document Formats", "fr": "Formats de documents attendus"}, + "userMessage": {"en": "User Message", "fr": "Message utilisateur"}, + "status": {"en": "Status", "fr": "Statut"}, + "error": {"en": "Error", "fr": "Erreur"}, + "retryCount": {"en": "Retry Count", "fr": "Nombre de tentatives"}, + "retryMax": {"en": "Max Retries", "fr": "Tentatives max"}, + "processingTime": {"en": "Processing Time", "fr": "Temps de traitement"}, + "timestamp": {"en": "Timestamp", "fr": "Horodatage"}, + "result": {"en": "Result", "fr": "Résultat"}, + }, +) + + +class TaskResult(BaseModel, ModelMixin): + taskId: str = Field(..., description="Task ID") + status: TaskStatus = Field(default=TaskStatus.PENDING, description="Task status") + success: bool = Field(..., description="Whether the task was successful") + feedback: Optional[str] = Field(None, description="Task feedback message") + error: Optional[str] = Field(None, description="Error message if task failed") + + +register_model_labels( + "TaskResult", + {"en": "Task Result", "fr": "Résultat de tâche"}, + { + "taskId": {"en": "Task ID", "fr": "ID de la tâche"}, + "status": {"en": "Status", "fr": "Statut"}, + "success": {"en": "Success", "fr": "Succès"}, + "feedback": {"en": "Feedback", "fr": "Retour"}, + "error": {"en": "Error", "fr": "Erreur"}, + }, +) + + +class TaskItem(BaseModel, ModelMixin): + id: str = Field(..., description="Task ID") + workflowId: str = Field(..., description="Workflow ID") + userInput: str = Field(..., description="User input that triggered the task") + status: TaskStatus = Field(default=TaskStatus.PENDING, description="Task status") + error: Optional[str] = Field(None, description="Error message if task failed") + startedAt: Optional[float] = Field(None, description="When the task started (UTC timestamp in seconds)") + finishedAt: Optional[float] = Field(None, description="When the task finished (UTC timestamp in seconds)") + actionList: List[TaskAction] = Field(default_factory=list, description="List of actions to execute") + retryCount: int = Field(default=0, description="Number of retries attempted") + retryMax: int = Field(default=3, description="Maximum number of retries") + rollbackOnFailure: bool = Field(default=True, description="Whether to rollback on failure") + dependencies: List[str] = Field(default_factory=list, description="List of task IDs this task depends on") + feedback: Optional[str] = Field(None, description="Task feedback message") + processingTime: Optional[float] = Field(None, description="Total processing time in seconds") + resultLabels: Optional[Dict[str, Any]] = Field(default_factory=dict, description="Map of result labels to their values") + + +register_model_labels( + "TaskItem", + {"en": "Task", "fr": "Tâche"}, + { + "id": {"en": "Task ID", "fr": "ID de la tâche"}, + "workflowId": {"en": "Workflow ID", "fr": "ID du workflow"}, + "userInput": {"en": "User Input", "fr": "Entrée utilisateur"}, + "status": {"en": "Status", "fr": "Statut"}, + "error": {"en": "Error", "fr": "Erreur"}, + "startedAt": {"en": "Started At", "fr": "Démarré à"}, + "finishedAt": {"en": "Finished At", "fr": "Terminé à"}, + "actionList": {"en": "Actions", "fr": "Actions"}, + "retryCount": {"en": "Retry Count", "fr": "Nombre de tentatives"}, + "retryMax": {"en": "Max Retries", "fr": "Tentatives max"}, + "processingTime": {"en": "Processing Time", "fr": "Temps de traitement"}, + }, +) + + +class TaskStep(BaseModel, ModelMixin): + id: str + objective: str + dependencies: Optional[list[str]] = Field(default_factory=list) + success_criteria: Optional[list[str]] = Field(default_factory=list) + estimated_complexity: Optional[str] = None + userMessage: Optional[str] = Field(None, description="User-friendly message in user's language") + + +register_model_labels( + "TaskStep", + {"en": "Task Step", "fr": "Étape de tâche"}, + { + "id": {"en": "ID", "fr": "ID"}, + "objective": {"en": "Objective", "fr": "Objectif"}, + "dependencies": {"en": "Dependencies", "fr": "Dépendances"}, + "success_criteria": {"en": "Success Criteria", "fr": "Critères de succès"}, + "estimated_complexity": {"en": "Estimated Complexity", "fr": "Complexité estimée"}, + "userMessage": {"en": "User Message", "fr": "Message utilisateur"}, + }, +) + + +class TaskHandover(BaseModel, ModelMixin): + taskId: str = Field(description="Target task ID") + sourceTask: Optional[str] = Field(None, description="Source task ID") + inputDocuments: List[DocumentExchange] = Field(default_factory=list, description="Available input documents") + outputDocuments: List[DocumentExchange] = Field(default_factory=list, description="Produced output documents") + context: Dict[str, Any] = Field(default_factory=dict, description="Task context") + previousResults: List[str] = Field(default_factory=list, description="Previous result summaries") + improvements: List[str] = Field(default_factory=list, description="Improvement suggestions") + workflowSummary: Optional[str] = Field(None, description="Summarized workflow context") + messageHistory: List[str] = Field(default_factory=list, description="Key message summaries") + timestamp: float = Field(..., description="When the handover was created (UTC timestamp in seconds)") + handoverType: str = Field(default="task", description="Type of handover: task, phase, or workflow") + + +register_model_labels( + "TaskHandover", + {"en": "Task Handover", "fr": "Transfert de tâche"}, + { + "taskId": {"en": "Task ID", "fr": "ID de la tâche"}, + "sourceTask": {"en": "Source Task", "fr": "Tâche source"}, + "inputDocuments": {"en": "Input Documents", "fr": "Documents d'entrée"}, + "outputDocuments": {"en": "Output Documents", "fr": "Documents de sortie"}, + "context": {"en": "Context", "fr": "Contexte"}, + "previousResults": {"en": "Previous Results", "fr": "Résultats précédents"}, + "improvements": {"en": "Improvements", "fr": "Améliorations"}, + "workflowSummary": {"en": "Workflow Summary", "fr": "Résumé du workflow"}, + "messageHistory": {"en": "Message History", "fr": "Historique des messages"}, + "timestamp": {"en": "Timestamp", "fr": "Horodatage"}, + "handoverType": {"en": "Handover Type", "fr": "Type de transfert"}, + }, +) + + +class TaskContext(BaseModel, ModelMixin): + task_step: TaskStep + workflow: Optional['ChatWorkflow'] = None + workflow_id: Optional[str] = None + available_documents: Optional[str] = "No documents available" + available_connections: Optional[list[str]] = Field(default_factory=list) + previous_results: Optional[list[str]] = Field(default_factory=list) + previous_handover: Optional[TaskHandover] = None + improvements: Optional[list[str]] = Field(default_factory=list) + retry_count: Optional[int] = 0 + previous_action_results: Optional[list] = Field(default_factory=list) + previous_review_result: Optional[dict] = None + is_regeneration: Optional[bool] = False + failure_patterns: Optional[list[str]] = Field(default_factory=list) + failed_actions: Optional[list] = Field(default_factory=list) + successful_actions: Optional[list] = Field(default_factory=list) + criteria_progress: Optional[dict] = None + + def getDocumentReferences(self) -> List[str]: + docs = [] + if self.previous_handover: + for doc_exchange in self.previous_handover.inputDocuments: + docs.extend(doc_exchange.documents) + return list(set(docs)) + + def addImprovement(self, improvement: str) -> None: + if improvement not in (self.improvements or []): + if self.improvements is None: + self.improvements = [] + self.improvements.append(improvement) + + +class ReviewContext(BaseModel, ModelMixin): + task_step: TaskStep + task_actions: Optional[list] = Field(default_factory=list) + action_results: Optional[list] = Field(default_factory=list) + step_result: Optional[dict] = Field(default_factory=dict) + workflow_id: Optional[str] = None + previous_results: Optional[list[str]] = Field(default_factory=list) + + +class ReviewResult(BaseModel, ModelMixin): + status: str + reason: Optional[str] = None + improvements: Optional[list[str]] = Field(default_factory=list) + quality_score: Optional[int] = 5 + missing_outputs: Optional[list[str]] = Field(default_factory=list) + met_criteria: Optional[list[str]] = Field(default_factory=list) + unmet_criteria: Optional[list[str]] = Field(default_factory=list) + confidence: Optional[float] = 0.5 + userMessage: Optional[str] = Field(None, description="User-friendly message in user's language") + + +register_model_labels( + "ReviewResult", + {"en": "Review Result", "fr": "Résultat de l'évaluation"}, + { + "status": {"en": "Status", "fr": "Statut"}, + "reason": {"en": "Reason", "fr": "Raison"}, + "improvements": {"en": "Improvements", "fr": "Améliorations"}, + "quality_score": {"en": "Quality Score", "fr": "Score de qualité"}, + "missing_outputs": {"en": "Missing Outputs", "fr": "Sorties manquantes"}, + "met_criteria": {"en": "Met Criteria", "fr": "Critères respectés"}, + "unmet_criteria": {"en": "Unmet Criteria", "fr": "Critères non respectés"}, + "confidence": {"en": "Confidence", "fr": "Confiance"}, + "userMessage": {"en": "User Message", "fr": "Message utilisateur"}, + }, +) + + +class TaskPlan(BaseModel, ModelMixin): + overview: str + tasks: list[TaskStep] + userMessage: Optional[str] = Field(None, description="Overall user-friendly message for the task plan") + + +register_model_labels( + "TaskPlan", + {"en": "Task Plan", "fr": "Plan de tâches"}, + { + "overview": {"en": "Overview", "fr": "Aperçu"}, + "tasks": {"en": "Tasks", "fr": "Tâches"}, + "userMessage": {"en": "User Message", "fr": "Message utilisateur"}, + }, +) + + +class WorkflowResult(BaseModel, ModelMixin): + status: str + completed_tasks: int + total_tasks: int + execution_time: float + final_results_count: int + error: Optional[str] = None + phase: Optional[str] = None + + +register_model_labels( + "WorkflowResult", + {"en": "Workflow Result", "fr": "Résultat du workflow"}, + { + "status": {"en": "Status", "fr": "Statut"}, + "completed_tasks": {"en": "Completed Tasks", "fr": "Tâches terminées"}, + "total_tasks": {"en": "Total Tasks", "fr": "Total des tâches"}, + "execution_time": {"en": "Execution Time", "fr": "Temps d'exécution"}, + "final_results_count": {"en": "Final Results Count", "fr": "Nombre de résultats finaux"}, + "error": {"en": "Error", "fr": "Erreur"}, + "phase": {"en": "Phase", "fr": "Phase"}, + }, +) + + +class UserInputRequest(BaseModel, ModelMixin): + prompt: str = Field(description="Prompt for the user") + listFileId: List[str] = Field(default_factory=list, description="List of file IDs") + userLanguage: str = Field(default="en", description="User's preferred language") + + +register_model_labels( + "UserInputRequest", + {"en": "User Input Request", "fr": "Demande de saisie utilisateur"}, + { + "prompt": {"en": "Prompt", "fr": "Invite"}, + "listFileId": {"en": "File IDs", "fr": "IDs des fichiers"}, + "userLanguage": {"en": "User Language", "fr": "Langue de l'utilisateur"}, + }, +) + + diff --git a/modules/features/chatPlayground/mainChatPlayground.py b/modules/features/chatPlayground/mainChatPlayground.py index 6fa9ab1a..60e53420 100644 --- a/modules/features/chatPlayground/mainChatPlayground.py +++ b/modules/features/chatPlayground/mainChatPlayground.py @@ -2,8 +2,9 @@ import logging import asyncio from typing import Optional -from modules.interfaces.interfaceAppModel import User -from modules.interfaces.interfaceChatModel import ChatWorkflow, UserInputRequest +from modules.datamodels.datamodelUam import User +from modules.datamodels.datamodelChat import ChatWorkflow +from modules.datamodels.datamodelWorkflow import UserInputRequest from modules.shared.timezoneUtils import get_utc_timestamp logger = logging.getLogger(__name__) diff --git a/modules/features/neutralizePlayground/mainNeutralizePlayground.py b/modules/features/neutralizePlayground/mainNeutralizePlayground.py index 256b50c2..9c3219a5 100644 --- a/modules/features/neutralizePlayground/mainNeutralizePlayground.py +++ b/modules/features/neutralizePlayground/mainNeutralizePlayground.py @@ -1,7 +1,8 @@ import logging from typing import Any, Dict, List, Optional -from modules.interfaces.interfaceAppModel import User, DataNeutralizerAttributes, DataNeutraliserConfig +from modules.datamodels.datamodelUam import User +from modules.datamodels.datamodelNeutralizer import DataNeutralizerAttributes, DataNeutraliserConfig from modules.services.serviceNeutralization.mainServiceNeutralization import NeutralizationService logger = logging.getLogger(__name__) @@ -161,7 +162,7 @@ class SharepointProcessor: async def _getSharepointConnection(self, sharepointPath: str = None): try: - from modules.interfaces.interfaceAppModel import UserConnection + from modules.datamodels.datamodelUam import UserConnection connections = self.service.app_interface.db.getRecordset( UserConnection, recordFilter={"userId": self.service.app_interface.userId} diff --git a/modules/features/syncDelta/mainSyncDelta.py b/modules/features/syncDelta/mainSyncDelta.py index 2c115671..2250a15a 100644 --- a/modules/features/syncDelta/mainSyncDelta.py +++ b/modules/features/syncDelta/mainSyncDelta.py @@ -850,13 +850,13 @@ try: misfire_grace_time=1800, ) logger.info("Registered DG ticket sync via EventManagement (every 20 minutes)") - - # Run initial sync - #import asyncio - #asyncio.create_task(scheduled_sync()) - #logger.info("Initial sync scheduled") else: logger.info(f"Skipping DG scheduler registration for ticket sync in env: {APP_ENV_TYPE}") - + + # Run initial sync + #import asyncio + #asyncio.create_task(scheduled_sync()) + #logger.info("Initial sync scheduled") + except Exception as e: logger.error(f"Failed to register DG ticket sync: {str(e)}") diff --git a/modules/interfaces/interfaceAiObjects.py b/modules/interfaces/interfaceAiObjects.py index 6f59eace..8626f277 100644 --- a/modules/interfaces/interfaceAiObjects.py +++ b/modules/interfaces/interfaceAiObjects.py @@ -3,7 +3,7 @@ from typing import Dict, Any, List from modules.connectors.connectorAiOpenai import AiOpenai from modules.connectors.connectorAiAnthropic import AiAnthropic -from modules.interfaces.interfaceAiModel import AiCallOptions, AiCallRequest, AiCallResponse +from modules.datamodels.datamodelAi import AiCallOptions, AiCallRequest, AiCallResponse logger = logging.getLogger(__name__) diff --git a/modules/interfaces/interfaceAppAccess.py b/modules/interfaces/interfaceAppAccess.py index d91dddc1..5520152a 100644 --- a/modules/interfaces/interfaceAppAccess.py +++ b/modules/interfaces/interfaceAppAccess.py @@ -5,7 +5,8 @@ Access control for the Application. import logging from typing import Dict, Any, List, Optional from datetime import datetime -from modules.interfaces.interfaceAppModel import UserPrivilege, User, UserInDB, AuthEvent, Mandate +from modules.datamodels.datamodelUam import UserPrivilege, User, UserInDB, Mandate +from modules.datamodels.datamodelSecurity import AuthEvent from modules.shared.timezoneUtils import get_utc_now # Configure logger diff --git a/modules/interfaces/interfaceAppModel.py b/modules/interfaces/interfaceAppModel.py deleted file mode 100644 index 8d58325a..00000000 --- a/modules/interfaces/interfaceAppModel.py +++ /dev/null @@ -1,566 +0,0 @@ -""" -Models for User Service -""" - -import uuid -from pydantic import BaseModel, Field, EmailStr -from typing import List, Dict, Any, Optional -from datetime import datetime -from enum import Enum -from modules.shared.attributeUtils import register_model_labels, AttributeDefinition, ModelMixin -from modules.shared.timezoneUtils import get_utc_timestamp - -class AuthAuthority(str, Enum): - """Authentication authority enum""" - LOCAL = "local" - GOOGLE = "google" - MSFT = "msft" - -class UserPrivilege(str, Enum): - """User privilege levels""" - SYSADMIN = "sysadmin" - ADMIN = "admin" - USER = "user" - -class ConnectionStatus(str, Enum): - """Connection status""" - ACTIVE = "active" - EXPIRED = "expired" - REVOKED = "revoked" - PENDING = "pending" - -class TokenStatus(str, Enum): - """Status of an issued gateway JWT access token""" - ACTIVE = "active" - REVOKED = "revoked" - -class Mandate(BaseModel, ModelMixin): - """Data model for a mandate""" - id: str = Field( - default_factory=lambda: str(uuid.uuid4()), - description="Unique ID of the mandate", - frontend_type="text", - frontend_readonly=True, - frontend_required=False - ) - name: str = Field( - description="Name of the mandate", - frontend_type="text", - frontend_readonly=False, - frontend_required=True - ) - language: str = Field( - default="en", - description="Default language of the mandate", - frontend_type="select", - frontend_readonly=False, - frontend_required=True, - frontend_options=[ - {"value": "de", "label": {"en": "Deutsch", "fr": "Allemand"}}, - {"value": "en", "label": {"en": "English", "fr": "Anglais"}}, - {"value": "fr", "label": {"en": "Français", "fr": "Français"}}, - {"value": "it", "label": {"en": "Italiano", "fr": "Italien"}} - ] - ) - enabled: bool = Field( - default=True, - description="Indicates whether the mandate is enabled", - frontend_type="checkbox", - frontend_readonly=False, - frontend_required=False - ) - -# Register labels for Mandate -register_model_labels( - "Mandate", - {"en": "Mandate", "fr": "Mandat"}, - { - "id": {"en": "ID", "fr": "ID"}, - "name": {"en": "Name", "fr": "Nom"}, - "language": {"en": "Language", "fr": "Langue"}, - "enabled": {"en": "Enabled", "fr": "Activé"} - } -) - -class UserConnection(BaseModel, ModelMixin): - """Data model for a user's connection to an external service""" - id: str = Field( - default_factory=lambda: str(uuid.uuid4()), - description="Unique ID of the connection", - frontend_type="text", - frontend_readonly=True, - frontend_required=False - ) - userId: str = Field( - description="ID of the user this connection belongs to", - frontend_type="text", - frontend_readonly=True, - frontend_required=False - ) - authority: AuthAuthority = Field( - description="Authentication authority", - frontend_type="select", - frontend_readonly=True, - frontend_required=False, - frontend_options=[ - {"value": "local", "label": {"en": "Local", "fr": "Local"}}, - {"value": "google", "label": {"en": "Google", "fr": "Google"}}, - {"value": "msft", "label": {"en": "Microsoft", "fr": "Microsoft"}} - ] - ) - externalId: str = Field( - description="User ID in the external system", - frontend_type="text", - frontend_readonly=True, - frontend_required=False - ) - externalUsername: str = Field( - description="Username in the external system", - frontend_type="text", - frontend_readonly=False, - frontend_required=False - ) - externalEmail: Optional[EmailStr] = Field( - None, - description="Email in the external system", - frontend_type="email", - frontend_readonly=False, - frontend_required=False - ) - status: ConnectionStatus = Field( - default=ConnectionStatus.ACTIVE, - description="Connection status", - frontend_type="select", - frontend_readonly=False, - frontend_required=False, - frontend_options=[ - {"value": "active", "label": {"en": "Active", "fr": "Actif"}}, - {"value": "inactive", "label": {"en": "Inactive", "fr": "Inactif"}}, - {"value": "expired", "label": {"en": "Expired", "fr": "Expiré"}}, - {"value": "pending", "label": {"en": "Pending", "fr": "En attente"}} - ] - ) - connectedAt: float = Field( - default_factory=get_utc_timestamp, - description="When the connection was established (UTC timestamp in seconds)", - frontend_type="timestamp", - frontend_readonly=True, - frontend_required=False - ) - lastChecked: float = Field( - default_factory=get_utc_timestamp, - description="When the connection was last verified (UTC timestamp in seconds)", - frontend_type="timestamp", - frontend_readonly=True, - frontend_required=False - ) - expiresAt: Optional[float] = Field( - None, - description="When the connection expires (UTC timestamp in seconds)", - frontend_type="timestamp", - frontend_readonly=True, - frontend_required=False - ) - tokenStatus: Optional[str] = Field( - None, - description="Current token status: active, expired, none", - frontend_type="select", - frontend_readonly=True, - frontend_required=False, - frontend_options=[ - {"value": "active", "label": {"en": "Active", "fr": "Actif"}}, - {"value": "expired", "label": {"en": "Expired", "fr": "Expiré"}}, - {"value": "none", "label": {"en": "None", "fr": "Aucun"}} - ] - ) - tokenExpiresAt: Optional[float] = Field( - None, - description="When the current token expires (UTC timestamp in seconds)", - frontend_type="timestamp", - frontend_readonly=True, - frontend_required=False - ) - -# Register labels for UserConnection -register_model_labels( - "UserConnection", - {"en": "User Connection", "fr": "Connexion utilisateur"}, - { - "id": {"en": "ID", "fr": "ID"}, - "userId": {"en": "User ID", "fr": "ID utilisateur"}, - "authority": {"en": "Authority", "fr": "Autorité"}, - "externalId": {"en": "External ID", "fr": "ID externe"}, - "externalUsername": {"en": "External Username", "fr": "Nom d'utilisateur externe"}, - "externalEmail": {"en": "External Email", "fr": "Email externe"}, - "status": {"en": "Status", "fr": "Statut"}, - "connectedAt": {"en": "Connected At", "fr": "Connecté le"}, - "lastChecked": {"en": "Last Checked", "fr": "Dernière vérification"}, - "expiresAt": {"en": "Expires At", "fr": "Expire le"}, - "tokenStatus": {"en": "Connection Status", "fr": "Statut de connexion"}, - "tokenExpiresAt": {"en": "Expires At", "fr": "Expire le"} - } -) - - - -class User(BaseModel, ModelMixin): - """Data model for a user""" - id: str = Field( - default_factory=lambda: str(uuid.uuid4()), - description="Unique ID of the user", - frontend_type="text", - frontend_readonly=True, - frontend_required=False - ) - username: str = Field( - description="Username for login", - frontend_type="text", - frontend_readonly=False, - frontend_required=True - ) - email: Optional[EmailStr] = Field( - None, - description="Email address of the user", - frontend_type="email", - frontend_readonly=False, - frontend_required=True - ) - fullName: Optional[str] = Field( - None, - description="Full name of the user", - frontend_type="text", - frontend_readonly=False, - frontend_required=False - ) - language: str = Field( - default="en", - description="Preferred language of the user", - frontend_type="select", - frontend_readonly=False, - frontend_required=True, - frontend_options=[ - {"value": "de", "label": {"en": "Deutsch", "fr": "Allemand"}}, - {"value": "en", "label": {"en": "English", "fr": "Anglais"}}, - {"value": "fr", "label": {"en": "Français", "fr": "Français"}}, - {"value": "it", "label": {"en": "Italiano", "fr": "Italien"}} - ] - ) - enabled: bool = Field( - default=True, - description="Indicates whether the user is enabled", - frontend_type="checkbox", - frontend_readonly=False, - frontend_required=False - ) - privilege: UserPrivilege = Field( - default=UserPrivilege.USER, - description="Permission level", - frontend_type="select", - frontend_readonly=False, - frontend_required=True, - frontend_options=[ - {"value": "user", "label": {"en": "User", "fr": "Utilisateur"}}, - {"value": "admin", "label": {"en": "Admin", "fr": "Administrateur"}}, - {"value": "sysadmin", "label": {"en": "SysAdmin", "fr": "Administrateur système"}} - ] - ) - authenticationAuthority: AuthAuthority = Field( - default=AuthAuthority.LOCAL, - description="Primary authentication authority", - frontend_type="select", - frontend_readonly=True, - frontend_required=False, - frontend_options=[ - {"value": "local", "label": {"en": "Local", "fr": "Local"}}, - {"value": "google", "label": {"en": "Google", "fr": "Google"}}, - {"value": "msft", "label": {"en": "Microsoft", "fr": "Microsoft"}} - ] - ) - mandateId: Optional[str] = Field( - None, - description="ID of the mandate this user belongs to", - frontend_type="text", - frontend_readonly=True, - frontend_required=False - ) - -# Register labels for User -register_model_labels( - "User", - {"en": "User", "fr": "Utilisateur"}, - { - "id": {"en": "ID", "fr": "ID"}, - "username": {"en": "Username", "fr": "Nom d'utilisateur"}, - "email": {"en": "Email", "fr": "Email"}, - "fullName": {"en": "Full Name", "fr": "Nom complet"}, - "language": {"en": "Language", "fr": "Langue"}, - "enabled": {"en": "Enabled", "fr": "Activé"}, - "privilege": {"en": "Privilege", "fr": "Privilège"}, - "authenticationAuthority": {"en": "Auth Authority", "fr": "Autorité d'authentification"}, - "mandateId": {"en": "Mandate ID", "fr": "ID de mandat"} - } -) - -class UserInDB(User): - """Extended user class with password hash""" - hashedPassword: Optional[str] = Field(None, description="Hash of the user password") - -# Register labels for UserInDB -register_model_labels( - "UserInDB", - {"en": "User Access", "fr": "Accès de l'utilisateur"}, - { - "hashedPassword": {"en": "Password hash", "fr": "Hachage de mot de passe"} - } -) - -# Token Models -class Token(BaseModel, ModelMixin): - """Token model for all authentication types""" - id: Optional[str] = None - userId: str - authority: AuthAuthority - connectionId: Optional[str] = Field(None, description="ID of the connection this token belongs to") - tokenAccess: str - tokenType: str = "bearer" - expiresAt: float = Field(description="When the token expires (UTC timestamp in seconds)") - tokenRefresh: Optional[str] = None - createdAt: Optional[float] = Field(None, description="When the token was created (UTC timestamp in seconds)") - # Revocation and session tracking (for LOCAL gateway JWTs) - status: TokenStatus = Field(default=TokenStatus.ACTIVE, description="Token status: active/revoked") - revokedAt: Optional[float] = Field(None, description="When the token was revoked (UTC timestamp in seconds)") - revokedBy: Optional[str] = Field(None, description="User ID who revoked the token (admin/self)") - reason: Optional[str] = Field(None, description="Optional revocation reason") - sessionId: Optional[str] = Field(None, description="Logical session grouping for logout revocation") - mandateId: Optional[str] = Field(None, description="Mandate ID for tenant scoping of the token") - - class Config: - useEnumValues = True - -# Register labels for Token -register_model_labels( - "Token", - {"en": "Token", "fr": "Jeton"}, - { - "id": {"en": "ID", "fr": "ID"}, - "userId": {"en": "User ID", "fr": "ID utilisateur"}, - "authority": {"en": "Authority", "fr": "Autorité"}, - "connectionId": {"en": "Connection ID", "fr": "ID de connexion"}, - "tokenAccess": {"en": "Access Token", "fr": "Jeton d'accès"}, - "tokenType": {"en": "Token Type", "fr": "Type de jeton"}, - "expiresAt": {"en": "Expires At", "fr": "Expire le"}, - "tokenRefresh": {"en": "Refresh Token", "fr": "Jeton de rafraîchissement"}, - "createdAt": {"en": "Created At", "fr": "Créé le"}, - "status": {"en": "Status", "fr": "Statut"}, - "revokedAt": {"en": "Revoked At", "fr": "Révoqué le"}, - "revokedBy": {"en": "Revoked By", "fr": "Révoqué par"}, - "reason": {"en": "Reason", "fr": "Raison"}, - "sessionId": {"en": "Session ID", "fr": "ID de session"}, - "mandateId": {"en": "Mandate ID", "fr": "ID de mandat"} - } -) - -class LocalToken(Token): - """Local authentication token model""" - pass - -class GoogleToken(Token): - """Google OAuth token model""" - pass - -class MsftToken(Token): - """Microsoft OAuth token model""" - pass - -class AuthEvent(BaseModel, ModelMixin): - """Data model for authentication events""" - id: str = Field( - default_factory=lambda: str(uuid.uuid4()), - description="Unique ID of the auth event", - frontend_type="text", - frontend_readonly=True, - frontend_required=False - ) - userId: str = Field( - description="ID of the user this event belongs to", - frontend_type="text", - frontend_readonly=True, - frontend_required=True - ) - eventType: str = Field( - description="Type of authentication event (e.g., 'login', 'logout', 'token_refresh')", - frontend_type="text", - frontend_readonly=True, - frontend_required=True - ) - timestamp: float = Field( - default_factory=get_utc_timestamp, - description="Unix timestamp when the event occurred", - frontend_type="datetime", - frontend_readonly=True, - frontend_required=True - ) - ipAddress: Optional[str] = Field( - default=None, - description="IP address from which the event originated", - frontend_type="text", - frontend_readonly=True, - frontend_required=False - ) - userAgent: Optional[str] = Field( - default=None, - description="User agent string from the request", - frontend_type="text", - frontend_readonly=True, - frontend_required=False - ) - success: bool = Field( - default=True, - description="Whether the authentication event was successful", - frontend_type="boolean", - frontend_readonly=True, - frontend_required=True - ) - details: Optional[str] = Field( - default=None, - description="Additional details about the event", - frontend_type="text", - frontend_readonly=True, - frontend_required=False - ) - -# Register labels for AuthEvent -register_model_labels( - "AuthEvent", - {"en": "Authentication Event", "fr": "Événement d'authentification"}, - { - "id": {"en": "ID", "fr": "ID"}, - "userId": {"en": "User ID", "fr": "ID utilisateur"}, - "eventType": {"en": "Event Type", "fr": "Type d'événement"}, - "timestamp": {"en": "Timestamp", "fr": "Horodatage"}, - "ipAddress": {"en": "IP Address", "fr": "Adresse IP"}, - "userAgent": {"en": "User Agent", "fr": "Agent utilisateur"}, - "success": {"en": "Success", "fr": "Succès"}, - "details": {"en": "Details", "fr": "Détails"} - } -) - -class DataNeutraliserConfig(BaseModel, ModelMixin): - """Data model for data neutralization configuration""" - id: str = Field( - default_factory=lambda: str(uuid.uuid4()), - description="Unique ID of the configuration", - frontend_type="text", - frontend_readonly=True, - frontend_required=False - ) - mandateId: str = Field( - description="ID of the mandate this configuration belongs to", - frontend_type="text", - frontend_readonly=True, - frontend_required=True - ) - userId: str = Field( - description="ID of the user who created this configuration", - frontend_type="text", - frontend_readonly=True, - frontend_required=True - ) - enabled: bool = Field( - default=True, - description="Whether data neutralization is enabled", - frontend_type="checkbox", - frontend_readonly=False, - frontend_required=False - ) - namesToParse: str = Field( - default="", - description="Multiline list of names to parse for neutralization", - frontend_type="textarea", - frontend_readonly=False, - frontend_required=False - ) - sharepointSourcePath: str = Field( - default="", - description="SharePoint path to read files for neutralization", - frontend_type="text", - frontend_readonly=False, - frontend_required=False - ) - sharepointTargetPath: str = Field( - default="", - description="SharePoint path to store neutralized files", - frontend_type="text", - frontend_readonly=False, - frontend_required=False - ) - -# Register labels for DataNeutraliserConfig -register_model_labels( - "DataNeutraliserConfig", - {"en": "Data Neutralization Config", "fr": "Configuration de neutralisation des données"}, - { - "id": {"en": "ID", "fr": "ID"}, - "mandateId": {"en": "Mandate ID", "fr": "ID de mandat"}, - "userId": {"en": "User ID", "fr": "ID utilisateur"}, - "enabled": {"en": "Enabled", "fr": "Activé"}, - "namesToParse": {"en": "Names to Parse", "fr": "Noms à analyser"}, - "sharepointSourcePath": {"en": "Source Path", "fr": "Chemin source"}, - "sharepointTargetPath": {"en": "Target Path", "fr": "Chemin cible"} - } -) - -class DataNeutralizerAttributes(BaseModel, ModelMixin): - """Data model for neutralized data attributes mapping""" - id: str = Field( - default_factory=lambda: str(uuid.uuid4()), - description="Unique ID of the attribute mapping (used as UID in neutralized files)", - frontend_type="text", - frontend_readonly=True, - frontend_required=False - ) - mandateId: str = Field( - description="ID of the mandate this attribute belongs to", - frontend_type="text", - frontend_readonly=True, - frontend_required=True - ) - userId: str = Field( - description="ID of the user who created this attribute", - frontend_type="text", - frontend_readonly=True, - frontend_required=True - ) - originalText: str = Field( - description="Original text that was neutralized", - frontend_type="text", - frontend_readonly=True, - frontend_required=True - ) - fileId: Optional[str] = Field( - default=None, - description="ID of the file this attribute belongs to", - frontend_type="text", - frontend_readonly=True, - frontend_required=False - ) - patternType: str = Field( - description="Type of pattern that matched (email, phone, name, etc.)", - frontend_type="text", - frontend_readonly=True, - frontend_required=True - ) - -# Register labels for DataNeutralizerAttributes -register_model_labels( - "DataNeutralizerAttributes", - {"en": "Neutralized Data Attribute", "fr": "Attribut de données neutralisées"}, - { - "id": {"en": "ID", "fr": "ID"}, - "mandateId": {"en": "Mandate ID", "fr": "ID de mandat"}, - "userId": {"en": "User ID", "fr": "ID utilisateur"}, - "originalText": {"en": "Original Text", "fr": "Texte original"}, - "fileId": {"en": "File ID", "fr": "ID de fichier"}, - "patternType": {"en": "Pattern Type", "fr": "Type de modèle"} - } -) diff --git a/modules/interfaces/interfaceAppObjects.py b/modules/interfaces/interfaceAppObjects.py index 9320b102..a56e9344 100644 --- a/modules/interfaces/interfaceAppObjects.py +++ b/modules/interfaces/interfaceAppObjects.py @@ -17,12 +17,12 @@ from modules.connectors.connectorDbPostgre import DatabaseConnector from modules.shared.configuration import APP_CONFIG from modules.shared.timezoneUtils import get_utc_now, get_utc_timestamp from modules.interfaces.interfaceAppAccess import AppAccess -from modules.interfaces.interfaceAppModel import ( - User, Mandate, UserInDB, UserConnection, - AuthAuthority, UserPrivilege, - ConnectionStatus, Token, AuthEvent, TokenStatus, - DataNeutraliserConfig, DataNeutralizerAttributes +from modules.datamodels.datamodelUam import ( + User, Mandate, UserInDB, UserConnection, + AuthAuthority, UserPrivilege, ConnectionStatus, ) +from modules.datamodels.datamodelSecurity import Token, AuthEvent, TokenStatus +from modules.datamodels.datamodelNeutralizer import DataNeutraliserConfig, DataNeutralizerAttributes logger = logging.getLogger(__name__) diff --git a/modules/interfaces/interfaceChatAccess.py b/modules/interfaces/interfaceChatAccess.py index 0b4055dc..f37c8e96 100644 --- a/modules/interfaces/interfaceChatAccess.py +++ b/modules/interfaces/interfaceChatAccess.py @@ -4,8 +4,8 @@ Handles user access management and permission checks. """ from typing import Dict, Any, List, Optional -from modules.interfaces.interfaceAppModel import User, UserPrivilege -from modules.interfaces.interfaceChatModel import ChatWorkflow, ChatMessage, ChatLog, ChatStat, ChatDocument +from modules.datamodels.datamodelUam import User, UserPrivilege +from modules.datamodels.datamodelChat import ChatWorkflow, ChatMessage, ChatLog, ChatStat, ChatDocument class ChatAccess: """ diff --git a/modules/interfaces/interfaceChatObjects.py b/modules/interfaces/interfaceChatObjects.py index a79b2284..04dcd4cd 100644 --- a/modules/interfaces/interfaceChatObjects.py +++ b/modules/interfaces/interfaceChatObjects.py @@ -12,10 +12,10 @@ from typing import Dict, Any, List, Optional, Union, get_origin, get_args import asyncio from modules.interfaces.interfaceChatAccess import ChatAccess -from modules.interfaces.interfaceChatModel import ( - TaskStatus, UserInputRequest, ChatDocument, TaskItem, ChatStat, ChatLog, ChatMessage, ChatWorkflow, TaskAction, TaskResult, ActionResult -) -from modules.interfaces.interfaceAppModel import User +from modules.datamodels.datamodelWorkflow import UserInputRequest, TaskAction, TaskResult +from modules.datamodels.datamodelWorkflow import TaskItem, TaskStatus, ActionResult +from modules.datamodels.datamodelChat import ChatDocument, ChatStat, ChatLog, ChatMessage, ChatWorkflow +from modules.datamodels.datamodelUam import User # DYNAMIC PART: Connectors to the Interface from modules.connectors.connectorDbPostgre import DatabaseConnector diff --git a/modules/interfaces/interfaceComponentAccess.py b/modules/interfaces/interfaceComponentAccess.py index e1792c5e..3d259789 100644 --- a/modules/interfaces/interfaceComponentAccess.py +++ b/modules/interfaces/interfaceComponentAccess.py @@ -5,9 +5,11 @@ Handles user access management and permission checks. import logging from typing import Dict, Any, List, Optional -from modules.interfaces.interfaceAppModel import User, UserInDB -from modules.interfaces.interfaceComponentModel import Prompt, FileItem, FileData, VoiceSettings -from modules.interfaces.interfaceChatModel import ChatWorkflow, ChatMessage, ChatLog +from modules.datamodels.datamodelUam import User, UserInDB +from modules.datamodels.datamodelUtils import Prompt +from modules.datamodels.datamodelFiles import FileItem, FileData +from modules.datamodels.datamodelVoice import VoiceSettings +from modules.datamodels.datamodelChat import ChatWorkflow, ChatMessage, ChatLog # Configure logger logger = logging.getLogger(__name__) diff --git a/modules/interfaces/interfaceComponentModel.py b/modules/interfaces/interfaceComponentModel.py deleted file mode 100644 index 27dbf48b..00000000 --- a/modules/interfaces/interfaceComponentModel.py +++ /dev/null @@ -1,264 +0,0 @@ -""" -Service Management model classes for the service management system. -Updated to match the Entity Relation Diagram structure. -""" - -from pydantic import BaseModel, Field -from typing import List, Dict, Any, Optional, Union -from datetime import datetime -import uuid - -# Import for label registration -from modules.shared.attributeUtils import register_model_labels, ModelMixin -from modules.shared.timezoneUtils import get_utc_timestamp - -# CORE MODELS - -class FileItem(BaseModel, ModelMixin): - """Data model for a file item""" - id: str = Field( - default_factory=lambda: str(uuid.uuid4()), - description="Primary key", - frontend_type="text", - frontend_readonly=True, - frontend_required=False - ) - mandateId: str = Field( - description="ID of the mandate this file belongs to", - frontend_type="text", - frontend_readonly=True, - frontend_required=False - ) - fileName: str = Field( - description="Name of the file", - frontend_type="text", - frontend_readonly=False, - frontend_required=True - ) - mimeType: str = Field( - description="MIME type of the file", - frontend_type="text", - frontend_readonly=True, - frontend_required=False - ) - fileHash: str = Field( - description="Hash of the file", - frontend_type="text", - frontend_readonly=True, - frontend_required=False - ) - fileSize: int = Field( - description="Size of the file in bytes", - frontend_type="integer", - frontend_readonly=True, - frontend_required=False - ) - creationDate: float = Field( - default_factory=get_utc_timestamp, - description="Date when the file was created (UTC timestamp in seconds)", - frontend_type="timestamp", - frontend_readonly=True, - frontend_required=False - ) - - def to_dict(self) -> Dict[str, Any]: - """Convert model to dictionary""" - return super().to_dict() - -# Register labels for FileItem -register_model_labels( - "FileItem", - {"en": "File Item", "fr": "Élément de fichier"}, - { - "id": {"en": "ID", "fr": "ID"}, - "mandateId": {"en": "Mandate ID", "fr": "ID du mandat"}, - "fileName": {"en": "fileName", "fr": "Nom de fichier"}, - "mimeType": {"en": "MIME Type", "fr": "Type MIME"}, - "fileHash": {"en": "File Hash", "fr": "Hash du fichier"}, - "fileSize": {"en": "File Size", "fr": "Taille du fichier"}, - "creationDate": {"en": "Creation Date", "fr": "Date de création"} - } -) - -class FilePreview(BaseModel, ModelMixin): - """Data model for file preview""" - content: Union[str, bytes] = Field(description="File content (text or binary)") - mimeType: str = Field(description="MIME type of the file") - fileName: str = Field(description="Original fileName") - isText: bool = Field(description="Whether the content is text (True) or binary (False)") - encoding: Optional[str] = Field(None, description="Text encoding if content is text") - size: int = Field(description="Size of the content in bytes") - - def to_dict(self) -> Dict[str, Any]: - """Convert model to dictionary with proper content handling""" - data = super().to_dict() - # Convert bytes to base64 string if content is binary - if isinstance(data.get("content"), bytes): - import base64 - data["content"] = base64.b64encode(data["content"]).decode('utf-8') - return data - -# Register labels for FilePreview -register_model_labels( - "FilePreview", - {"en": "File Preview", "fr": "Aperçu du fichier"}, - { - "content": {"en": "Content", "fr": "Contenu"}, - "mimeType": {"en": "MIME Type", "fr": "Type MIME"}, - "fileName": {"en": "fileName", "fr": "Nom de fichier"}, - "isText": {"en": "Is Text", "fr": "Est du texte"}, - "encoding": {"en": "Encoding", "fr": "Encodage"}, - "size": {"en": "Size", "fr": "Taille"} - } -) - -class FileData(BaseModel, ModelMixin): - """Data model for file data""" - id: str = Field(default_factory=lambda: str(uuid.uuid4()), description="Primary key") - data: str = Field(description="File data content") - base64Encoded: bool = Field(description="Whether the data is base64 encoded") - -# Register labels for FileData -register_model_labels( - "FileData", - {"en": "File Data", "fr": "Données de fichier"}, - { - "id": {"en": "ID", "fr": "ID"}, - "data": {"en": "Data", "fr": "Données"}, - "base64Encoded": {"en": "Base64 Encoded", "fr": "Encodé en Base64"} - } -) - -class Prompt(BaseModel, ModelMixin): - """Data model for a prompt""" - id: str = Field( - default_factory=lambda: str(uuid.uuid4()), - description="Primary key", - frontend_type="text", - frontend_readonly=True, - frontend_required=False - ) - mandateId: str = Field( - description="ID of the mandate this prompt belongs to", - frontend_type="text", - frontend_readonly=True, - frontend_required=False - ) - content: str = Field( - description="Content of the prompt", - frontend_type="textarea", - frontend_readonly=False, - frontend_required=True - ) - name: str = Field( - description="Name of the prompt", - frontend_type="text", - frontend_readonly=False, - frontend_required=True - ) - -# Register labels for Prompt -register_model_labels( - "Prompt", - {"en": "Prompt", "fr": "Invite"}, - { - "id": {"en": "ID", "fr": "ID"}, - "mandateId": {"en": "Mandate ID", "fr": "ID du mandat"}, - "content": {"en": "Content", "fr": "Contenu"}, - "name": {"en": "Name", "fr": "Nom"} - } -) - -class VoiceSettings(BaseModel, ModelMixin): - """Data model for voice service settings per user""" - id: str = Field( - default_factory=lambda: str(uuid.uuid4()), - description="Primary key", - frontend_type="text", - frontend_readonly=True, - frontend_required=False - ) - userId: str = Field( - description="ID of the user these settings belong to", - frontend_type="text", - frontend_readonly=True, - frontend_required=True - ) - mandateId: str = Field( - description="ID of the mandate these settings belong to", - frontend_type="text", - frontend_readonly=True, - frontend_required=False - ) - sttLanguage: str = Field( - default="de-DE", - description="Speech-to-Text language", - frontend_type="select", - frontend_readonly=False, - frontend_required=True - ) - ttsLanguage: str = Field( - default="de-DE", - description="Text-to-Speech language", - frontend_type="select", - frontend_readonly=False, - frontend_required=True - ) - ttsVoice: str = Field( - default="de-DE-KatjaNeural", - description="Text-to-Speech voice", - frontend_type="select", - frontend_readonly=False, - frontend_required=True - ) - translationEnabled: bool = Field( - default=True, - description="Whether translation is enabled", - frontend_type="checkbox", - frontend_readonly=False, - frontend_required=False - ) - targetLanguage: str = Field( - default="en-US", - description="Target language for translation", - frontend_type="select", - frontend_readonly=False, - frontend_required=False - ) - creationDate: float = Field( - default_factory=get_utc_timestamp, - description="Date when the settings were created (UTC timestamp in seconds)", - frontend_type="timestamp", - frontend_readonly=True, - frontend_required=False - ) - lastModified: float = Field( - default_factory=get_utc_timestamp, - description="Date when the settings were last modified (UTC timestamp in seconds)", - frontend_type="timestamp", - frontend_readonly=True, - frontend_required=False - ) - - def to_dict(self) -> Dict[str, Any]: - """Convert model to dictionary""" - return super().to_dict() - -# Register labels for VoiceSettings -register_model_labels( - "VoiceSettings", - {"en": "Voice Settings", "fr": "Paramètres vocaux"}, - { - "id": {"en": "ID", "fr": "ID"}, - "userId": {"en": "User ID", "fr": "ID utilisateur"}, - "mandateId": {"en": "Mandate ID", "fr": "ID du mandat"}, - "sttLanguage": {"en": "STT Language", "fr": "Langue STT"}, - "ttsLanguage": {"en": "TTS Language", "fr": "Langue TTS"}, - "ttsVoice": {"en": "TTS Voice", "fr": "Voix TTS"}, - "translationEnabled": {"en": "Translation Enabled", "fr": "Traduction activée"}, - "targetLanguage": {"en": "Target Language", "fr": "Langue cible"}, - "creationDate": {"en": "Creation Date", "fr": "Date de création"}, - "lastModified": {"en": "Last Modified", "fr": "Dernière modification"} - } -) - diff --git a/modules/interfaces/interfaceComponentObjects.py b/modules/interfaces/interfaceComponentObjects.py index 7e467869..87ec954d 100644 --- a/modules/interfaces/interfaceComponentObjects.py +++ b/modules/interfaces/interfaceComponentObjects.py @@ -11,10 +11,10 @@ from typing import Dict, Any, List, Optional, Union import hashlib from modules.interfaces.interfaceComponentAccess import ComponentAccess -from modules.interfaces.interfaceComponentModel import ( - FilePreview, Prompt, FileItem, FileData, VoiceSettings -) -from modules.interfaces.interfaceAppModel import User, Mandate +from modules.datamodels.datamodelFiles import FilePreview, FileItem, FileData +from modules.datamodels.datamodelUtils import Prompt +from modules.datamodels.datamodelVoice import VoiceSettings +from modules.datamodels.datamodelUam import User, Mandate # DYNAMIC PART: Connectors to the Interface from modules.connectors.connectorDbPostgre import DatabaseConnector diff --git a/modules/interfaces/interfaceTicketObjects.py b/modules/interfaces/interfaceTicketObjects.py index 727aa6bc..8a5e851d 100644 --- a/modules/interfaces/interfaceTicketObjects.py +++ b/modules/interfaces/interfaceTicketObjects.py @@ -1,8 +1,6 @@ from typing import Any, Optional from datetime import datetime, timezone -from modules.interfaces.interfaceTicketModel import TicketBase, Task - # Module-level factory to create TicketInterface by connector type async def createTicketInterfaceByType( *, @@ -28,18 +26,22 @@ async def createTicketInterfaceByType( class TicketInterface: - def __init__(self, *, connector_ticket: TicketBase, task_sync_definition: dict): + def __init__(self, *, connector_ticket, task_sync_definition: dict): self.connector_ticket = connector_ticket self.task_sync_definition = task_sync_definition async def exportTicketsAsList(self) -> list[dict]: - tickets = await self.connector_ticket.read_tasks(limit=0) - transformed_tasks = self._transformTasks(tickets, includePut=True) - data_list = [task.data for task in transformed_tasks] - return self._filterEmptyRecords(data_list) + tickets: list[dict] = await self.connector_ticket.read_tasks(limit=0) + transformed_tasks = self._transformTicketRecords(tickets, includePut=True) + # Return plain dictionaries filtered by presence of ID + rows: list[dict] = [] + for task in transformed_tasks: + if isinstance(task, dict) and task.get("ID"): + rows.append(task) + return rows async def importListToTickets(self, records: list[dict]) -> None: - updates: list[Task] = [] + updates: list[dict] = [] for row in records: task_id = row.get("ID") if not task_id: @@ -53,15 +55,15 @@ class TicketInterface: field_id = field_path[1] fields[field_id] = value if fields: - updates.append(Task(data={"ID": task_id, "fields": fields})) + updates.append({"ID": task_id, "fields": fields}) if updates: await self.connector_ticket.write_tasks(updates) - def _transformTasks( - self, tasks: list[Task], includePut: bool = False - ) -> list[Task]: + def _transformTicketRecords( + self, tasks: list[dict], includePut: bool = False + ) -> list[dict]: """Transforms tasks according to the task_sync_definition.""" - transformed_tasks = [] + transformed_tasks: list[dict] = [] for task in tasks: transformed_data = {} @@ -73,12 +75,10 @@ class TicketInterface: # Get the right fields if direction == "get" or includePut: - value = self._extractFieldValue(task.data, field_path, field_name) + value = self._extractFieldValue(task, field_path, field_name) transformed_data[field_name] = value - # Create new Task with transformed data - transformed_task = Task(data=transformed_data) - transformed_tasks.append(transformed_task) + transformed_tasks.append(transformed_data) return transformed_tasks @@ -173,14 +173,5 @@ class TicketInterface: return any(keyword in field_name.lower() for keyword in date_keywords) def _filterEmptyRecords(self, records: list[dict]) -> list[dict]: - """Remove records that are missing an ID. - - Purposefully only filter by presence of 'ID' to avoid dropping - valid rows with many empty optional fields. - """ - filtered: list[dict] = [] - for row in records: - if isinstance(row, dict) and row.get("ID"): - filtered.append(row) - return filtered + return [row for row in records if isinstance(row, dict) and row.get("ID")] diff --git a/modules/interfaces/interfaceWebObjects.py b/modules/interfaces/interfaceWebObjects.py index 023eb539..69ef0c59 100644 --- a/modules/interfaces/interfaceWebObjects.py +++ b/modules/interfaces/interfaceWebObjects.py @@ -1,113 +1,51 @@ -from typing import Optional -import json -import csv -import io -from modules.interfaces.interfaceWebModel import ( - WebCrawlActionResult, - WebSearchActionResult, - WebSearchRequest, - WebCrawlRequest, - WebScrapeActionResult, - WebScrapeRequest, - WebCrawlDocumentData, - WebScrapeDocumentData, - WebSearchDocumentData, -) - from dataclasses import dataclass -from modules.connectors.connectorWebTavily import ConnectorTavily -from modules.interfaces.interfaceChatModel import ActionDocument +from modules.datamodels.datamodelWeb import ( + WebCrawlActionResult, + WebCrawlActionDocument, + WebCrawlDocumentData, + WebCrawlRequest, + WebCrawlResultItem, + WebScrapeActionResult, + WebScrapeActionDocument, + WebSearchDocumentData as WebScrapeDocumentData, + WebScrapeRequest, + WebScrapeResultItem, + WebSearchActionResult, + WebSearchActionDocument, + WebSearchDocumentData, + WebSearchRequest, + WebSearchResultItem, +) +from modules.connectors.connectorWebTavily import ConnectorWeb +from modules.datamodels.datamodelWorkflow import ActionDocument + @dataclass(slots=True) class WebInterface: - connectorWebTavily: ConnectorTavily + connectorWebTavily: ConnectorWeb def __post_init__(self) -> None: if self.connectorWebTavily is None: raise TypeError( "connectorWebTavily must be provided. " - "Use `await WebInterface.create()` or pass a ConnectorTavily." + "Use `await WebInterface.create()` or pass a ConnectorWeb." ) @classmethod async def create(cls) -> "WebInterface": - connectorWebTavily = await ConnectorTavily.create() + connectorWebTavily = await ConnectorWeb.create() return WebInterface(connectorWebTavily=connectorWebTavily) + # Methods + async def search(self, web_search_request: WebSearchRequest) -> WebSearchActionResult: - # NOTE: Add connectors here - return await self.connectorWebTavily.search_urls(web_search_request) + return await self.connectorWebTavily.search(web_search_request) async def crawl(self, web_crawl_request: WebCrawlRequest) -> WebCrawlActionResult: - # NOTE: Add connectors here - return await self.connectorWebTavily.crawl_urls(web_crawl_request) + return await self.connectorWebTavily.crawl(web_crawl_request) async def scrape(self, web_scrape_request: WebScrapeRequest) -> WebScrapeActionResult: - # NOTE: Add connectors here return await self.connectorWebTavily.scrape(web_scrape_request) - def convert_web_result_to_json(self, web_result) -> str: - """Convert WebCrawlActionResult or WebScrapeActionResult to proper JSON format""" - if not web_result.success or not web_result.documents: - return json.dumps({"success": web_result.success, "error": web_result.error}) - - # Extract the document data and convert to dict - document_data = web_result.documents[0].documentData - - # Convert Pydantic model to dict - result_dict = { - "success": web_result.success, - "results": [ - { - "url": str(result.url), - "content": result.content - } - for result in document_data.results - ], - "total_count": document_data.total_count - } - - # Add type-specific fields - if hasattr(document_data, 'urls'): - # WebCrawlDocumentData has urls field - result_dict["urls"] = [str(url) for url in document_data.urls] - elif hasattr(document_data, 'query'): - # WebScrapeDocumentData has query field - result_dict["query"] = document_data.query - - return json.dumps(result_dict, indent=2, ensure_ascii=False) - - def convert_web_search_result_to_csv(self, web_search_result: WebSearchActionResult) -> str: - """Convert WebSearchActionResult to CSV format with url and title columns""" - if not web_search_result.success or not web_search_result.documents: - return "" - - output = io.StringIO() - writer = csv.writer(output, delimiter=';') - - # Write header - writer.writerow(['url', 'title']) - - # Write data rows - document_data = web_search_result.documents[0].documentData - for result in document_data.results: - writer.writerow([str(result.url), result.title]) - - return output.getvalue() - - def create_json_action_document(self, json_content: str, document_name: str) -> ActionDocument: - """Create an ActionDocument with JSON content""" - return ActionDocument( - documentName=document_name, - documentData=json_content, - mimeType="application/json" - ) - - def create_csv_action_document(self, csv_content: str, document_name: str) -> ActionDocument: - """Create an ActionDocument with CSV content""" - return ActionDocument( - documentName=document_name, - documentData=csv_content, - mimeType="text/csv" - ) \ No newline at end of file + # Helpers moved to MethodWeb \ No newline at end of file diff --git a/modules/routes/routeAdmin.py b/modules/routes/routeAdmin.py index 4ddfcf84..15659cd4 100644 --- a/modules/routes/routeAdmin.py +++ b/modules/routes/routeAdmin.py @@ -10,7 +10,7 @@ from datetime import datetime from modules.shared.configuration import APP_CONFIG from modules.security.auth import limiter, getCurrentUser -from modules.interfaces.interfaceAppModel import User +from modules.datamodels.datamodelUam import User from modules.interfaces.interfaceAppObjects import getRootInterface # Static folder setup - using absolute path from app root diff --git a/modules/routes/routeAttributes.py b/modules/routes/routeAttributes.py index 45a759e1..cf6095c9 100644 --- a/modules/routes/routeAttributes.py +++ b/modules/routes/routeAttributes.py @@ -11,7 +11,7 @@ import logging from modules.security.auth import limiter, getCurrentUser # Import the attribute definition and helper functions -from modules.interfaces.interfaceAppModel import User +from modules.datamodels.datamodelUam import User from modules.shared.attributeUtils import getModelClasses, getModelAttributeDefinitions, AttributeResponse, AttributeDefinition # Configure logger diff --git a/modules/routes/routeChatPlayground.py b/modules/routes/routeChatPlayground.py index 186e65a8..b7d7238f 100644 --- a/modules/routes/routeChatPlayground.py +++ b/modules/routes/routeChatPlayground.py @@ -16,11 +16,9 @@ import modules.interfaces.interfaceChatObjects as interfaceChatObjects from modules.interfaces.interfaceChatObjects import getInterface # Import models -from modules.interfaces.interfaceChatModel import ( - ChatWorkflow, - UserInputRequest -) -from modules.interfaces.interfaceAppModel import User +from modules.datamodels.datamodelChat import ChatWorkflow +from modules.datamodels.datamodelWorkflow import UserInputRequest +from modules.datamodels.datamodelUam import User # Import workflow control functions from modules.features.chatPlayground.mainChatPlayground import chatStart, chatStop diff --git a/modules/routes/routeDataConnections.py b/modules/routes/routeDataConnections.py index 30c5601f..79075e64 100644 --- a/modules/routes/routeDataConnections.py +++ b/modules/routes/routeDataConnections.py @@ -15,7 +15,8 @@ from datetime import datetime import logging import json -from modules.interfaces.interfaceAppModel import User, UserConnection, AuthAuthority, ConnectionStatus, Token +from modules.datamodels.datamodelUam import User, UserConnection, AuthAuthority, ConnectionStatus +from modules.datamodels.datamodelSecurity import Token from modules.security.auth import getCurrentUser, limiter from modules.interfaces.interfaceAppObjects import getInterface, getRootInterface from modules.shared.timezoneUtils import get_utc_timestamp diff --git a/modules/routes/routeDataFiles.py b/modules/routes/routeDataFiles.py index 3f11342b..f75d8585 100644 --- a/modules/routes/routeDataFiles.py +++ b/modules/routes/routeDataFiles.py @@ -15,9 +15,9 @@ from modules.security.auth import limiter, getCurrentUser # Import interfaces import modules.interfaces.interfaceComponentObjects as interfaceComponentObjects -from modules.interfaces.interfaceComponentModel import FileItem, FilePreview +from modules.datamodels.datamodelFiles import FileItem, FilePreview from modules.shared.attributeUtils import getModelAttributeDefinitions, AttributeResponse, AttributeDefinition -from modules.interfaces.interfaceAppModel import User +from modules.datamodels.datamodelUam import User # Configure logger logger = logging.getLogger(__name__) diff --git a/modules/routes/routeDataMandates.py b/modules/routes/routeDataMandates.py index a6be8e8d..6c847569 100644 --- a/modules/routes/routeDataMandates.py +++ b/modules/routes/routeDataMandates.py @@ -21,7 +21,7 @@ import modules.interfaces.interfaceAppObjects as interfaceAppObjects from modules.shared.attributeUtils import getModelAttributeDefinitions, AttributeResponse, AttributeDefinition # Import the model classes -from modules.interfaces.interfaceAppModel import Mandate, User +from modules.datamodels.datamodelUam import Mandate, User # Configure logger logger = logging.getLogger(__name__) diff --git a/modules/routes/routeDataNeutralization.py b/modules/routes/routeDataNeutralization.py index 71b1db25..61e8c25d 100644 --- a/modules/routes/routeDataNeutralization.py +++ b/modules/routes/routeDataNeutralization.py @@ -6,7 +6,8 @@ import logging from modules.security.auth import limiter, getCurrentUser # Import interfaces -from modules.interfaces.interfaceAppModel import User, DataNeutraliserConfig, DataNeutralizerAttributes +from modules.datamodels.datamodelUam import User +from modules.datamodels.datamodelNeutralizer import DataNeutraliserConfig, DataNeutralizerAttributes from modules.features.neutralizePlayground.mainNeutralizePlayground import NeutralizationPlayground # Configure logger diff --git a/modules/routes/routeDataPrompts.py b/modules/routes/routeDataPrompts.py index c2e58771..b652fc6a 100644 --- a/modules/routes/routeDataPrompts.py +++ b/modules/routes/routeDataPrompts.py @@ -13,9 +13,9 @@ from modules.security.auth import limiter, getCurrentUser # Import interfaces import modules.interfaces.interfaceComponentObjects as interfaceComponentObjects -from modules.interfaces.interfaceComponentModel import Prompt +from modules.datamodels.datamodelUtils import Prompt from modules.shared.attributeUtils import getModelAttributeDefinitions, AttributeResponse, AttributeDefinition -from modules.interfaces.interfaceAppModel import User +from modules.datamodels.datamodelUam import User # Configure logger logger = logging.getLogger(__name__) diff --git a/modules/routes/routeDataUsers.py b/modules/routes/routeDataUsers.py index 01fb41f1..559783db 100644 --- a/modules/routes/routeDataUsers.py +++ b/modules/routes/routeDataUsers.py @@ -18,7 +18,8 @@ import modules.interfaces.interfaceAppObjects as interfaceAppObjects from modules.security.auth import getCurrentUser, limiter, getCurrentUser # Import the attribute definition and helper functions -from modules.interfaces.interfaceAppModel import User, AttributeDefinition +from modules.datamodels.datamodelUam import User +from modules.shared.attributeUtils import AttributeDefinition from modules.shared.attributeUtils import getModelAttributeDefinitions, AttributeResponse # Configure logger @@ -177,7 +178,7 @@ async def reset_user_password( # SECURITY: Automatically revoke all tokens for the user after password reset try: - from modules.interfaces.interfaceAppModel import AuthAuthority + from modules.datamodels.datamodelUam import AuthAuthority revoked_count = appInterface.revokeTokensByUser( userId=userId, authority=None, # Revoke all authorities @@ -253,7 +254,7 @@ async def change_password( # SECURITY: Automatically revoke all tokens for the user after password change try: - from modules.interfaces.interfaceAppModel import AuthAuthority + from modules.datamodels.datamodelUam import AuthAuthority revoked_count = appInterface.revokeTokensByUser( userId=str(currentUser.id), authority=None, # Revoke all authorities diff --git a/modules/routes/routeSecurityAdmin.py b/modules/routes/routeSecurityAdmin.py index a7203965..da837223 100644 --- a/modules/routes/routeSecurityAdmin.py +++ b/modules/routes/routeSecurityAdmin.py @@ -6,7 +6,8 @@ import logging from modules.security.auth import getCurrentUser, limiter from modules.interfaces.interfaceAppObjects import getInterface, getRootInterface -from modules.interfaces.interfaceAppModel import User, UserInDB, AuthAuthority, Token +from modules.datamodels.datamodelUam import User, UserInDB, AuthAuthority +from modules.datamodels.datamodelSecurity import Token from modules.shared.configuration import APP_CONFIG logger = logging.getLogger(__name__) diff --git a/modules/routes/routeSecurityGoogle.py b/modules/routes/routeSecurityGoogle.py index bf87259e..20ff17b0 100644 --- a/modules/routes/routeSecurityGoogle.py +++ b/modules/routes/routeSecurityGoogle.py @@ -13,7 +13,8 @@ import httpx from modules.shared.configuration import APP_CONFIG from modules.interfaces.interfaceAppObjects import getInterface, getRootInterface -from modules.interfaces.interfaceAppModel import AuthAuthority, User, Token, ConnectionStatus, UserConnection +from modules.datamodels.datamodelUam import AuthAuthority, User, ConnectionStatus, UserConnection +from modules.datamodels.datamodelSecurity import Token from modules.security.auth import getCurrentUser, limiter from modules.shared.attributeUtils import ModelMixin from modules.shared.timezoneUtils import get_utc_now, create_expiration_timestamp, get_utc_timestamp @@ -169,7 +170,7 @@ async def login( try: if connectionId: rootInterface = getRootInterface() - from modules.interfaces.interfaceAppModel import UserConnection + from modules.datamodels.datamodelUam import UserConnection records = rootInterface.db.getRecordset(UserConnection, recordFilter={"id": connectionId}) if records: record = records[0] @@ -208,7 +209,7 @@ async def auth_callback(code: str, state: str, request: Request) -> HTMLResponse """Handle Google OAuth callback""" try: # Import Token at function level to avoid scoping issues - from modules.interfaces.interfaceAppModel import Token + from modules.datamodels.datamodelSecurity import Token # Parse state state_data = json.loads(state) @@ -469,7 +470,7 @@ async def auth_callback(code: str, state: str, request: Request) -> HTMLResponse connection.externalEmail = user_info.get("email") # Update connection record directly - from modules.interfaces.interfaceAppModel import UserConnection + from modules.datamodels.datamodelUam import UserConnection rootInterface.db.recordModify(UserConnection, connection_id, connection.to_dict()) diff --git a/modules/routes/routeSecurityLocal.py b/modules/routes/routeSecurityLocal.py index 017e7e90..5a1cf864 100644 --- a/modules/routes/routeSecurityLocal.py +++ b/modules/routes/routeSecurityLocal.py @@ -16,7 +16,8 @@ from pydantic import BaseModel from modules.security.auth import getCurrentUser, limiter, SECRET_KEY, ALGORITHM from modules.security.jwtService import createAccessToken, createRefreshToken, setAccessTokenCookie, setRefreshTokenCookie from modules.interfaces.interfaceAppObjects import getInterface, getRootInterface -from modules.interfaces.interfaceAppModel import User, UserInDB, AuthAuthority, UserPrivilege, Token +from modules.datamodels.datamodelUam import User, UserInDB, AuthAuthority, UserPrivilege +from modules.datamodels.datamodelSecurity import Token from modules.shared.attributeUtils import ModelMixin # Configure logger @@ -56,7 +57,7 @@ async def login( rootInterface = getRootInterface() # Get default mandate ID - from modules.interfaces.interfaceAppModel import Mandate + from modules.datamodels.datamodelUam import Mandate defaultMandateId = rootInterface.getInitialId(Mandate) if not defaultMandateId: raise HTTPException( @@ -197,7 +198,7 @@ async def register_user( appInterface = getRootInterface() # Get default mandate ID - from modules.interfaces.interfaceAppModel import Mandate + from modules.datamodels.datamodelUam import Mandate defaultMandateId = appInterface.getInitialId(Mandate) if not defaultMandateId: raise HTTPException( @@ -210,7 +211,7 @@ async def register_user( # Create user with local authentication # Set safe default privilege level for new registrations - from modules.interfaces.interfaceAppModel import UserPrivilege + from modules.datamodels.datamodelUam import UserPrivilege user = appInterface.createUser( username=userData.username, password=password, diff --git a/modules/routes/routeSecurityMsft.py b/modules/routes/routeSecurityMsft.py index 2b73db59..3f77100e 100644 --- a/modules/routes/routeSecurityMsft.py +++ b/modules/routes/routeSecurityMsft.py @@ -13,7 +13,8 @@ import httpx from modules.shared.configuration import APP_CONFIG from modules.interfaces.interfaceAppObjects import getInterface, getRootInterface -from modules.interfaces.interfaceAppModel import AuthAuthority, User, Token, ConnectionStatus, UserConnection +from modules.datamodels.datamodelUam import AuthAuthority, User, ConnectionStatus, UserConnection +from modules.datamodels.datamodelSecurity import Token from modules.security.auth import getCurrentUser, limiter from modules.security.jwtService import createAccessToken from modules.shared.attributeUtils import ModelMixin diff --git a/modules/routes/routeVoiceGoogle.py b/modules/routes/routeVoiceGoogle.py index 2b68299b..83fa1afc 100644 --- a/modules/routes/routeVoiceGoogle.py +++ b/modules/routes/routeVoiceGoogle.py @@ -10,7 +10,7 @@ from fastapi.responses import Response from typing import Optional, Dict, Any from modules.connectors.connectorGoogleSpeech import ConnectorGoogleSpeech from modules.security.auth import getCurrentUser -from modules.interfaces.interfaceAppModel import User +from modules.datamodels.datamodelUam import User from modules.interfaces.interfaceComponentObjects import getInterface logger = logging.getLogger(__name__) diff --git a/modules/routes/routeWorkflows.py b/modules/routes/routeWorkflows.py index 7b9dd8f9..57e596ea 100644 --- a/modules/routes/routeWorkflows.py +++ b/modules/routes/routeWorkflows.py @@ -19,15 +19,15 @@ import modules.interfaces.interfaceChatObjects as interfaceChatObjects from modules.interfaces.interfaceChatObjects import getInterface # Import models -from modules.interfaces.interfaceChatModel import ( +from modules.datamodels.datamodelChat import ( ChatWorkflow, ChatMessage, ChatLog, ChatStat, - ChatDocument + ChatDocument, ) from modules.shared.attributeUtils import getModelAttributeDefinitions, AttributeResponse -from modules.interfaces.interfaceAppModel import User +from modules.datamodels.datamodelUam import User from modules.shared.timezoneUtils import get_utc_timestamp diff --git a/modules/security/auth.py b/modules/security/auth.py index a457c60c..67b5dd2c 100644 --- a/modules/security/auth.py +++ b/modules/security/auth.py @@ -16,7 +16,8 @@ from slowapi.util import get_remote_address from modules.shared.configuration import APP_CONFIG from modules.shared.timezoneUtils import get_utc_now, get_utc_timestamp from modules.interfaces.interfaceAppObjects import getRootInterface -from modules.interfaces.interfaceAppModel import User, AuthAuthority, Token +from modules.datamodels.datamodelUam import User, AuthAuthority +from modules.datamodels.datamodelSecurity import Token # Get Config Data SECRET_KEY = APP_CONFIG.get("APP_JWT_KEY_SECRET") diff --git a/modules/security/tokenManager.py b/modules/security/tokenManager.py index 92fa747f..0918b4ea 100644 --- a/modules/security/tokenManager.py +++ b/modules/security/tokenManager.py @@ -8,7 +8,8 @@ import httpx from datetime import datetime from typing import Optional, Dict, Any, Callable -from modules.interfaces.interfaceAppModel import Token, AuthAuthority +from modules.datamodels.datamodelSecurity import Token +from modules.datamodels.datamodelUam import AuthAuthority from modules.shared.configuration import APP_CONFIG from modules.shared.timezoneUtils import get_utc_timestamp, create_expiration_timestamp diff --git a/modules/security/tokenRefreshService.py b/modules/security/tokenRefreshService.py index 5dcef46a..27351403 100644 --- a/modules/security/tokenRefreshService.py +++ b/modules/security/tokenRefreshService.py @@ -10,7 +10,8 @@ import logging from typing import Optional, Dict, Any, List from datetime import datetime, timedelta from modules.interfaces.interfaceAppObjects import getInterface -from modules.interfaces.interfaceAppModel import User, UserConnection, AuthAuthority, Token +from modules.datamodels.datamodelUam import User, UserConnection, AuthAuthority +from modules.datamodels.datamodelSecurity import Token from modules.shared.timezoneUtils import get_utc_timestamp from modules.shared.auditLogger import audit_logger diff --git a/modules/services/__init__.py b/modules/services/__init__.py index 174f2ba9..e8bbf75e 100644 --- a/modules/services/__init__.py +++ b/modules/services/__init__.py @@ -1,7 +1,7 @@ from typing import Any -from modules.interfaces.interfaceAppModel import User -from modules.interfaces.interfaceChatModel import ChatWorkflow +from modules.datamodels.datamodelUam import User +from modules.datamodels.datamodelChat import ChatWorkflow class PublicService: """Lightweight proxy exposing only public callable attributes of a target. @@ -75,6 +75,9 @@ class Services: from .serviceWorkflow.mainServiceWorkflow import WorkflowService self.workflow = PublicService(WorkflowService(self)) + + from .serviceWeb.mainServiceWeb import WebService + self.web = PublicService(WebService(self)) def getInterface(user: User, workflow: ChatWorkflow) -> Services: diff --git a/modules/services/serviceAi/mainServiceAi.py b/modules/services/serviceAi/mainServiceAi.py index 834e39a0..0ee4c776 100644 --- a/modules/services/serviceAi/mainServiceAi.py +++ b/modules/services/serviceAi/mainServiceAi.py @@ -1,9 +1,9 @@ import logging from typing import Dict, Any, List, Optional, Tuple -from modules.interfaces.interfaceChatModel import ChatDocument +from modules.datamodels.datamodelChat import ChatDocument from modules.services.serviceDocument.mainServiceDocumentExtraction import DocumentExtractionService -from modules.interfaces.interfaceAiModel import AiCallRequest, AiCallOptions +from modules.datamodels.datamodelAi import AiCallRequest, AiCallOptions from modules.interfaces.interfaceAiObjects import AiObjects @@ -15,8 +15,6 @@ logger = logging.getLogger(__name__) class AiService: """Centralized AI service orchestrating documents, model selection and failover. - - The concrete connector instances (OpenAI/Anthropic) are injected by the interface layer. """ def __init__(self, serviceCenter=None) -> None: @@ -60,10 +58,6 @@ class AiService: logger.error(f"Error in centralized AI call: {str(e)}") return f"Error: {str(e)}" - # Model selection now handled by interface AiObjects - - # Cost estimation handled by interface for model selection - async def _processDocumentsForAi( self, documents: List[ChatDocument], @@ -113,8 +107,6 @@ class AiService: return "\n\n---\n\n".join(processedContents) - # Prompt/context optimization (compression) handled by interface - async def _compressContent(self, content: str, targetSize: int, contentType: str) -> str: if len(content.encode("utf-8")) <= targetSize: return content @@ -136,8 +128,3 @@ class AiService: logger.warning(f"AI compression failed, using truncation: {str(e)}") return content[:targetSize] + "... [truncated]" - # Failover logic now centralized in interface via model selection; service delegates a single call - - # Fallback selection moved to interface; service doesn't select models directly - - diff --git a/modules/services/serviceDocument/mainServiceDocumentExtraction.py b/modules/services/serviceDocument/mainServiceDocumentExtraction.py index 3f1c9ef9..7b977f35 100644 --- a/modules/services/serviceDocument/mainServiceDocumentExtraction.py +++ b/modules/services/serviceDocument/mainServiceDocumentExtraction.py @@ -17,11 +17,8 @@ from modules.services.serviceDocument.documentUtility import ( convertDocumentDataToString ) -from modules.interfaces.interfaceChatModel import ( - ExtractedContent, - ContentItem, - ContentMetadata -) +from modules.datamodels.datamodelWorkflow import ExtractedContent +from modules.datamodels.datamodelChat import ContentItem, ContentMetadata from modules.services.serviceNeutralization.mainServiceNeutralization import NeutralizationService from modules.shared.configuration import APP_CONFIG @@ -1424,7 +1421,7 @@ class DocumentExtractionService: Original prompt: {prompt} """ - from modules.interfaces.interfaceChatModel import ChatDocument + from modules.datamodels.datamodelChat import ChatDocument image_doc = ChatDocument(fileData=chunk, fileName="image", mimeType=mimeType) # Use direct import to avoid circular dependency from modules.services.serviceAi.mainServiceAi import AiService diff --git a/modules/services/serviceNeutralization/mainServiceNeutralization.py b/modules/services/serviceNeutralization/mainServiceNeutralization.py index d3f7ea20..6817b5bc 100644 --- a/modules/services/serviceNeutralization/mainServiceNeutralization.py +++ b/modules/services/serviceNeutralization/mainServiceNeutralization.py @@ -11,7 +11,7 @@ import re import json from typing import Dict, List, Any, Optional -from modules.interfaces.interfaceAppModel import DataNeutraliserConfig, DataNeutralizerAttributes +from modules.datamodels.datamodelNeutralizer import DataNeutraliserConfig, DataNeutralizerAttributes # Import all necessary classes and functions for neutralization from modules.services.serviceNeutralization.subProcessCommon import CommonUtils, NeutralizationResult, NeutralizationAttribute diff --git a/modules/services/serviceWeb/mainServiceWeb.py b/modules/services/serviceWeb/mainServiceWeb.py new file mode 100644 index 00000000..5a9fdd9d --- /dev/null +++ b/modules/services/serviceWeb/mainServiceWeb.py @@ -0,0 +1,49 @@ +import logging +from typing import Optional, List + +from modules.datamodels.datamodelWeb import ( + WebSearchRequest, + WebCrawlRequest, + WebScrapeRequest, + WebSearchActionResult, + WebCrawlActionResult, + WebScrapeActionResult, +) +from modules.interfaces.interfaceWebObjects import WebInterface + + +logger = logging.getLogger(__name__) + + +class WebService: + """Centralized Web service providing wrappers around web interface actions. + """ + + def __init__(self, serviceCenter=None) -> None: + self.serviceCenter = serviceCenter + + async def webSearch(self, request: WebSearchRequest) -> WebSearchActionResult: + try: + web_interface = await WebInterface.create() + return await web_interface.search(request) + except Exception as e: + logger.error(f"Error in webSearch: {str(e)}") + raise + + async def webCrawl(self, request: WebCrawlRequest) -> WebCrawlActionResult: + try: + web_interface = await WebInterface.create() + return await web_interface.crawl(request) + except Exception as e: + logger.error(f"Error in webCrawl: {str(e)}") + raise + + async def webScrape(self, request: WebScrapeRequest) -> WebScrapeActionResult: + try: + web_interface = await WebInterface.create() + return await web_interface.scrape(request) + except Exception as e: + logger.error(f"Error in webScrape: {str(e)}") + raise + + diff --git a/modules/services/serviceWorkflow/mainServiceWorkflow.py b/modules/services/serviceWorkflow/mainServiceWorkflow.py index a9440993..5908f33f 100644 --- a/modules/services/serviceWorkflow/mainServiceWorkflow.py +++ b/modules/services/serviceWorkflow/mainServiceWorkflow.py @@ -1,8 +1,9 @@ import logging import uuid from typing import Dict, Any, List, Optional -from modules.interfaces.interfaceAppModel import User, UserConnection -from modules.interfaces.interfaceChatModel import ChatDocument, ChatMessage, ExtractedContent +from modules.datamodels.datamodelUam import User, UserConnection +from modules.datamodels.datamodelChat import ChatDocument, ChatMessage +from modules.datamodels.datamodelChat import ExtractedContent from modules.services.serviceDocument.mainServiceDocumentExtraction import DocumentExtractionService from modules.services.serviceDocument.documentUtility import getFileExtension, getMimeTypeFromExtension, detectContentTypeFromData from modules.shared.timezoneUtils import get_utc_timestamp diff --git a/modules/workflows/methods/methodAi.py b/modules/workflows/methods/methodAi.py index 811df305..228fcf5e 100644 --- a/modules/workflows/methods/methodAi.py +++ b/modules/workflows/methods/methodAi.py @@ -8,7 +8,7 @@ from typing import Dict, Any, List, Optional from datetime import datetime, UTC from modules.workflows.methods.methodBase import MethodBase, action -from modules.interfaces.interfaceChatModel import ActionResult +from modules.datamodels.datamodelWorkflow import ActionResult from modules.shared.timezoneUtils import get_utc_timestamp logger = logging.getLogger(__name__) diff --git a/modules/workflows/methods/methodDocument.py b/modules/workflows/methods/methodDocument.py index fe41dc60..59b7a1a3 100644 --- a/modules/workflows/methods/methodDocument.py +++ b/modules/workflows/methods/methodDocument.py @@ -9,7 +9,7 @@ from typing import Dict, Any, List, Optional from datetime import datetime, UTC from modules.workflows.methods.methodBase import MethodBase, action -from modules.interfaces.interfaceChatModel import ActionResult +from modules.datamodels.datamodelWorkflow import ActionResult, ChatDocument from modules.shared.timezoneUtils import get_utc_timestamp logger = logging.getLogger(__name__) @@ -768,12 +768,11 @@ SOURCE DOCUMENT CONTENT: # Build ChatDocument list from chatDocuments documents = [] try: - from modules.interfaces.interfaceChatModel import ChatDocument as ChatDoc for d in validDocuments: try: data = self.service.getFileData(d.fileId) if hasattr(d, 'fileId') else None if data: - documents.append(ChatDoc(fileData=data, fileName=d.fileName, mimeType=d.mimeType)) + documents.append(ChatDocument(fileData=data, fileName=d.fileName, mimeType=d.mimeType)) except Exception: continue except Exception: diff --git a/modules/workflows/methods/methodOutlook.py b/modules/workflows/methods/methodOutlook.py index 99766697..9fda6c04 100644 --- a/modules/workflows/methods/methodOutlook.py +++ b/modules/workflows/methods/methodOutlook.py @@ -1,90 +1,22 @@ """ Microsoft Outlook Email Operations Module - -This module provides actions for composing and sending emails via Microsoft Outlook using the Microsoft Graph API. - -ACTION CONTRACT DEFINITION: -========================== - -1. COMPOSE EMAIL ACTION (composeEmail): - ==================================== - - Purpose: Use AI to compose professional email content - - Input Parameters: - - context (str): Email context/requirements - - recipient (str, optional): Recipient information - - attachments (List[str], optional): Available documents to reference - - tone (str, optional): Email tone (formal, casual, etc.) - - expectedDocumentFormats (list, optional): Ignored - always produces JSON - - Output Contract: - The action produces a JSON document with this EXACT structure: - { - "context": "original context", - "recipient": "recipient info", - "tone": "email tone", - "timestamp": "ISO timestamp", - "usage": "usage description", - "to": ["email@example.com"], - "subject": "Email subject", - "body": "Email body content", - "cc": [], - "bcc": [], - "attachments": ["docItem:uuid:fileName.pdf"] - } - - Key Points: - - Email fields (to, subject, body, cc, bcc, attachments) are at ROOT LEVEL - - NOT wrapped in a "composedEmail" field - - Always produces .json format regardless of expectedDocumentFormats - - AI response is validated and parsed before output - -2. SEND EMAIL ACTION (sendEmail): - ============================== - - Purpose: Send the composed email via Outlook (creates draft) - - Input Parameters: - - connectionReference (str): Microsoft connection reference - - composedEmail (str): Reference to composed email document (docItem:...) - - expectedDocumentFormats (list, optional): Expected output formats - - Input Contract: - The composedEmail document MUST have this EXACT structure: - { - "to": ["email@example.com"], - "subject": "Email subject", - "body": "Email body content", - "cc": [], - "bcc": [], - "attachments": ["docItem:uuid:fileName.pdf"] - } - - Key Points: - - Email fields must be at ROOT LEVEL - - NOT wrapped in a nested structure - - Reads file content from database using fileId - - Creates email draft in Outlook Drafts folder - - Returns success/failure status - -DATA FLOW: -========== -composeEmail → JSON Document → sendEmail → Outlook Draft - -The contract ensures that composeEmail outputs exactly what sendEmail expects to consume. """ +import base64 +import re import logging from typing import Dict, Any, List, Optional from datetime import datetime, UTC import json import uuid +import requests from modules.workflows.methods.methodBase import MethodBase, action -from modules.interfaces.interfaceChatModel import ActionResult -from modules.interfaces.interfaceAppModel import ConnectionStatus +from modules.datamodels.datamodelWorkflow import ActionResult, ChatDocument +from modules.datamodels.datamodelUam import ConnectionStatus from modules.shared.timezoneUtils import get_utc_timestamp +from modules.services import getInterface as getServices +from modules.security.tokenManager import TokenManager logger = logging.getLogger(__name__) @@ -97,7 +29,6 @@ class MethodOutlook(MethodBase): self.name = "outlook" self.description = "Handle Microsoft Outlook email operations" # Centralized services interface (for AI) - from modules.services import getInterface as getServices self.services = getServices(self.service.user, self.service.workflow) def _format_timestamp_for_filename(self) -> str: @@ -120,7 +51,6 @@ class MethodOutlook(MethodBase): logger.debug(f"Found connection: {userConnection.id}, status: {userConnection.status.value}, authority: {userConnection.authority.value}") # Get a fresh token for this specific connection - from modules.security.tokenManager import TokenManager token = TokenManager().getFreshToken(self.service.interfaceApp, userConnection.id) if not token: logger.error(f"Token not found for connection: {userConnection.id}") @@ -156,8 +86,6 @@ class MethodOutlook(MethodBase): Check if the current connection has the necessary permissions for Outlook operations. """ try: - import requests - graph_url = "https://graph.microsoft.com/v1.0" headers = { "Authorization": f"Bearer {connection['accessToken']}", @@ -215,7 +143,6 @@ class MethodOutlook(MethodBase): # For basic text search, ensure it's safe for contains() filter # Remove any characters that might break the OData filter syntax - import re # Remove or escape characters that could break OData filter syntax safe_query = re.sub(r'[\\\'"]', '', clean_query) @@ -322,8 +249,6 @@ class MethodOutlook(MethodBase): This is needed for proper filtering when using advanced search queries """ try: - import requests - graph_url = "https://graph.microsoft.com/v1.0" headers = { "Authorization": f"Bearer {connection['accessToken']}", @@ -412,8 +337,6 @@ class MethodOutlook(MethodBase): # Read emails using Microsoft Graph API try: - import requests - # Microsoft Graph API endpoint for messages graph_url = "https://graph.microsoft.com/v1.0" headers = { @@ -619,7 +542,6 @@ class MethodOutlook(MethodBase): # Parse the email data (should be JSON) if isinstance(email_data, str): - import json try: # First try to parse as direct JSON parsed_email_data = json.loads(email_data) @@ -629,10 +551,6 @@ class MethodOutlook(MethodBase): logger.error(f"JSON parsing error: {str(e)}") logger.error(f"Content that failed to parse: {repr(email_data[:500])}") - # If that fails, try to extract JSON from HTML content - - import re - # Look for JSON content within HTML tags or as a script json_pattern = r'\{[^{}]*"to"[^{}]*"subject"[^{}]*"body"[^{}]*\}' json_match = re.search(json_pattern, email_data, re.DOTALL) @@ -686,8 +604,6 @@ class MethodOutlook(MethodBase): # Create email draft using Microsoft Graph API try: - import requests - # Microsoft Graph API endpoint for creating draft messages graph_url = "https://graph.microsoft.com/v1.0" headers = { @@ -740,7 +656,6 @@ class MethodOutlook(MethodBase): file_content = self.service.getFileData(file_id) if file_content: # Convert to base64 for Graph API - import base64 if isinstance(file_content, bytes): content_bytes = file_content else: @@ -903,8 +818,6 @@ class MethodOutlook(MethodBase): # Search emails using Microsoft Graph API try: - import requests - # Microsoft Graph API endpoint for searching messages graph_url = "https://graph.microsoft.com/v1.0" headers = { @@ -1087,8 +1000,6 @@ class MethodOutlook(MethodBase): # List drafts using Microsoft Graph API try: - import requests - # Microsoft Graph API endpoint for messages graph_url = "https://graph.microsoft.com/v1.0" headers = { @@ -1208,8 +1119,6 @@ class MethodOutlook(MethodBase): # Find drafts using Microsoft Graph API try: - import requests - # Microsoft Graph API endpoint for messages graph_url = "https://graph.microsoft.com/v1.0" headers = { @@ -1301,8 +1210,6 @@ class MethodOutlook(MethodBase): This is a helper method to identify which folder a draft is in """ try: - import requests - graph_url = "https://graph.microsoft.com/v1.0" headers = { "Authorization": f"Bearer {connection['accessToken']}", @@ -1347,8 +1254,6 @@ class MethodOutlook(MethodBase): # Check Drafts folder directly try: - import requests - # Microsoft Graph API endpoint for messages graph_url = "https://graph.microsoft.com/v1.0" headers = { @@ -1613,12 +1518,11 @@ class MethodOutlook(MethodBase): documents = [] try: if composition_documents: - from modules.interfaces.interfaceChatModel import ChatDocument as ChatDoc for d in composition_documents: try: data = self.service.getFileData(d.fileId) if hasattr(d, 'fileId') else None if data: - documents.append(ChatDoc(fileData=data, fileName=d.fileName, mimeType=d.mimeType)) + documents.append(ChatDocument(fileData=data, fileName=d.fileName, mimeType=d.mimeType)) except Exception: continue except Exception: @@ -1642,7 +1546,6 @@ class MethodOutlook(MethodBase): # Parse the AI response to ensure it's valid JSON try: - import json # Clean the response and parse as JSON cleaned_response = composed_email.strip() if cleaned_response.startswith('```json'): diff --git a/modules/workflows/methods/methodSharepoint.py b/modules/workflows/methods/methodSharepoint.py index 35999e17..b5cffd67 100644 --- a/modules/workflows/methods/methodSharepoint.py +++ b/modules/workflows/methods/methodSharepoint.py @@ -14,7 +14,7 @@ import aiohttp import asyncio from modules.workflows.methods.methodBase import MethodBase, action -from modules.interfaces.interfaceChatModel import ActionResult +from modules.datamodels.datamodelWorkflow import ActionResult from modules.shared.timezoneUtils import get_utc_timestamp logger = logging.getLogger(__name__) diff --git a/modules/workflows/methods/methodWeb.py b/modules/workflows/methods/methodWeb.py index 8d31a1ca..bb75d5f3 100644 --- a/modules/workflows/methods/methodWeb.py +++ b/modules/workflows/methods/methodWeb.py @@ -1,11 +1,11 @@ import logging import csv import io +import json as _json from typing import Any, Dict from modules.workflows.methods.methodBase import MethodBase, action -from modules.interfaces.interfaceChatModel import ActionResult, ActionDocument -from modules.interfaces.interfaceWebObjects import WebInterface -from modules.interfaces.interfaceWebModel import ( +from modules.datamodels.datamodelWorkflow import ActionResult, ActionDocument +from modules.datamodels.datamodelWeb import ( WebSearchRequest, WebCrawlRequest, WebScrapeRequest, @@ -64,15 +64,16 @@ class MethodWeb(MethodBase): include_raw_content=parameters.get("includeRawContent"), ) - # Perform request - web_interface = await WebInterface.create() - web_search_result = await web_interface.search(web_search_request) + # Perform request via centralized service wrappers + web_search_result = await self.services.web.webSearch(web_search_request) # Convert search results to CSV format (generic) if web_search_result.success and web_search_result.documents: - csv_content = web_interface.convert_web_search_result_to_csv(web_search_result) - csv_document = web_interface.create_csv_action_document( - csv_content, f"web_search_results.csv" + csv_content = self._convert_web_result_to_csv(web_search_result) + csv_document = ActionDocument( + documentName=f"web_search_results.csv", + documentData=csv_content, + mimeType="text/csv" ) return ActionResult(success=True, documents=[csv_document]) else: @@ -254,9 +255,8 @@ class MethodWeb(MethodBase): format=fmt, ) - # Perform request - web_interface = await WebInterface.create() - web_crawl_result = await web_interface.crawl(web_crawl_request) + # Perform request via centralized service wrappers + web_crawl_result = await self.services.web.webCrawl(web_crawl_request) # Convert and enrich with concise summaries per URL for better context if web_crawl_result.success: @@ -310,10 +310,12 @@ class MethodWeb(MethodBase): json_content = _json.dumps(payload, ensure_ascii=False, indent=2) except Exception: # Fallback to original conversion - json_content = web_interface.convert_web_result_to_json(web_crawl_result) + json_content = self._convert_web_result_to_json(web_crawl_result) - json_document = web_interface.create_json_action_document( - json_content, f"web_crawl_results.json" + json_document = ActionDocument( + documentName=f"web_crawl_results.json", + documentData=json_content, + mimeType="application/json" ) return ActionResult(success=True, documents=[json_document]) else: @@ -383,16 +385,16 @@ class MethodWeb(MethodBase): format=fmt, ) - # Perform request - web_interface = await WebInterface.create() - web_scrape_result = await web_interface.scrape(web_scrape_request) + # Perform request via centralized service wrappers + web_scrape_result = await self.services.web.webScrape(web_scrape_request) # Convert to proper JSON format if web_scrape_result.success: - json_content = web_interface.convert_web_result_to_json(web_scrape_result) - json_document = web_interface.create_json_action_document( - json_content, - f"web_scrape_results.json" + json_content = self._convert_web_result_to_json(web_scrape_result) + json_document = ActionDocument( + documentName=f"web_scrape_results.json", + documentData=json_content, + mimeType="application/json" ) return ActionResult( success=True, @@ -403,3 +405,36 @@ class MethodWeb(MethodBase): except Exception as e: return ActionResult(success=False, error=str(e)) + + # Helpers + def _convert_web_result_to_json(self, web_result): + if not getattr(web_result, 'success', False) or not getattr(web_result, 'documents', None): + return _json.dumps({"success": getattr(web_result, 'success', False), "error": getattr(web_result, 'error', None)}) + document_data = web_result.documents[0].documentData + result_dict = { + "success": True, + "results": [ + { + "url": str(getattr(result, 'url', "")), + "content": getattr(result, 'content', "") + } + for result in getattr(document_data, 'results', []) + ], + "total_count": getattr(document_data, 'total_count', 0) + } + if hasattr(document_data, 'urls'): + result_dict["urls"] = [str(url) for url in getattr(document_data, 'urls', [])] + elif hasattr(document_data, 'query'): + result_dict["query"] = getattr(document_data, 'query', None) + return _json.dumps(result_dict, indent=2, ensure_ascii=False) + + def _convert_web_result_to_csv(self, web_search_result): + if not getattr(web_search_result, 'success', False) or not getattr(web_search_result, 'documents', None): + return "" + output = io.StringIO() + writer = csv.writer(output, delimiter=';') + writer.writerow(['url', 'title']) + document_data = web_search_result.documents[0].documentData + for result in getattr(document_data, 'results', []): + writer.writerow([str(getattr(result, 'url', "")), getattr(result, 'title', "")]) + return output.getvalue() diff --git a/modules/workflows/processing/executionState.py b/modules/workflows/processing/executionState.py index 1d9b1963..3c506326 100644 --- a/modules/workflows/processing/executionState.py +++ b/modules/workflows/processing/executionState.py @@ -4,7 +4,8 @@ import logging from typing import List from datetime import datetime, UTC -from modules.interfaces.interfaceChatModel import TaskStep, ActionResult +from modules.datamodels.datamodelWorkflow import TaskStep +from modules.datamodels.datamodelWorkflow import ActionResult logger = logging.getLogger(__name__) diff --git a/modules/workflows/processing/handlingTasks.py b/modules/workflows/processing/handlingTasks.py index f37022be..db796e50 100644 --- a/modules/workflows/processing/handlingTasks.py +++ b/modules/workflows/processing/handlingTasks.py @@ -7,9 +7,9 @@ import json import time from typing import Dict, Any, Optional, List, Union from datetime import datetime, UTC -from modules.interfaces.interfaceChatModel import ( - TaskStatus, TaskStep, TaskContext, TaskAction, ReviewResult, TaskPlan, WorkflowResult, TaskResult, ReviewContext, ActionResult -) +from modules.datamodels.datamodelWorkflow import (TaskStep, TaskContext, ReviewResult, TaskPlan, WorkflowResult, TaskResult, ReviewContext) +from modules.datamodels.datamodelWorkflow import TaskStatus, ActionResult +from modules.datamodels.datamodelChat import ChatWorkflow, ChatMessage, ChatDocument from modules.interfaces.interfaceAppObjects import getInterface as getAppObjects from modules.shared.timezoneUtils import get_utc_timestamp from modules.workflows.processing.executionState import TaskExecutionState @@ -73,7 +73,7 @@ class HandlingTasks: # Create proper context object for task planning # For task planning, we need to create a minimal TaskStep since TaskContext requires it - from modules.interfaces.interfaceChatModel import TaskStep + from modules.datamodels.datamodelWorkflow import TaskStep planning_task_step = TaskStep( id="planning", objective=userInput, diff --git a/modules/workflows/processing/promptFactory.py b/modules/workflows/processing/promptFactory.py index 1df21183..9c4bf63b 100644 --- a/modules/workflows/processing/promptFactory.py +++ b/modules/workflows/processing/promptFactory.py @@ -7,7 +7,8 @@ import importlib import pkgutil import inspect from typing import Any, Dict, List -from modules.interfaces.interfaceChatModel import TaskContext, ReviewContext, ChatDocument, DocumentExchange +from modules.datamodels.datamodelWorkflow import TaskContext, ReviewContext, DocumentExchange +from modules.datamodels.datamodelChat import ChatDocument from modules.services.serviceDocument.documentUtility import getFileExtension from modules.workflows.methods.methodBase import MethodBase diff --git a/modules/workflows/workflowManager.py b/modules/workflows/workflowManager.py index 597725d7..caa5f0b0 100644 --- a/modules/workflows/workflowManager.py +++ b/modules/workflows/workflowManager.py @@ -6,10 +6,12 @@ import asyncio from modules.interfaces.interfaceAppObjects import User -from modules.interfaces.interfaceChatModel import (UserInputRequest, ChatMessage, ChatWorkflow, TaskItem, TaskStatus, ChatDocument) +from modules.datamodels.datamodelWorkflow import UserInputRequest +from modules.datamodels.datamodelChat import ChatMessage, ChatWorkflow, ChatDocument +from modules.datamodels.datamodelWorkflow import TaskItem, TaskStatus from modules.interfaces.interfaceChatObjects import ChatObjects from modules.workflows.processing.handlingTasks import HandlingTasks, WorkflowStoppedException -from modules.interfaces.interfaceChatModel import WorkflowResult +from modules.datamodels.datamodelWorkflow import WorkflowResult from modules.shared.timezoneUtils import get_utc_timestamp import uuid @@ -239,7 +241,7 @@ class WorkflowManager: logger.info(f"Task {current_task_index}/{total_tasks}: {task_step.objective}") # Build TaskContext (mode-specific behavior is inside HandlingTasks) - from modules.interfaces.interfaceChatModel import TaskContext + from modules.datamodels.datamodelWorkflow import TaskContext task_context = TaskContext( task_step=task_step, workflow=workflow, diff --git a/static/favicon.ico b/static/favicon.ico deleted file mode 100644 index a11777cc471a4344702741ab1c8a588998b1311a..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 3870 zcma);c{J4h9>;%nil|2-o+rCuEF-(I%-F}ijC~o(k~HKAkr0)!FCj~d>`RtpD?8b; zXOC1OD!V*IsqUwzbMF1)-gEDD=A573Z-&G7^LoAC9|WO7Xc0Cx1g^Zu0u_SjAPB3vGa^W|sj)80f#V0@M_CAZTIO(t--xg= z!sii`1giyH7EKL_+Wi0ab<)&E_0KD!3Rp2^HNB*K2@PHCs4PWSA32*-^7d{9nH2_E zmC{C*N*)(vEF1_aMamw2A{ZH5aIDqiabnFdJ|y0%aS|64E$`s2ccV~3lR!u<){eS` z#^Mx6o(iP1Ix%4dv`t@!&Za-K@mTm#vadc{0aWDV*_%EiGK7qMC_(`exc>-$Gb9~W!w_^{*pYRm~G zBN{nA;cm^w$VWg1O^^<6vY`1XCD|s_zv*g*5&V#wv&s#h$xlUilPe4U@I&UXZbL z0)%9Uj&@yd03n;!7do+bfixH^FeZ-Ema}s;DQX2gY+7g0s(9;`8GyvPY1*vxiF&|w z>!vA~GA<~JUqH}d;DfBSi^IT*#lrzXl$fNpq0_T1tA+`A$1?(gLb?e#0>UELvljtQ zK+*74m0jn&)5yk8mLBv;=@}c{t0ztT<v;Avck$S6D`Z)^c0(jiwKhQsn|LDRY&w(Fmi91I7H6S;b0XM{e zXp0~(T@k_r-!jkLwd1_Vre^v$G4|kh4}=Gi?$AaJ)3I+^m|Zyj#*?Kp@w(lQdJZf4 z#|IJW5z+S^e9@(6hW6N~{pj8|NO*>1)E=%?nNUAkmv~OY&ZV;m-%?pQ_11)hAr0oAwILrlsGawpxx4D43J&K=n+p3WLnlDsQ$b(9+4 z?mO^hmV^F8MV{4Lx>(Q=aHhQ1){0d*(e&s%G=i5rq3;t{JC zmgbn5Nkl)t@fPH$v;af26lyhH!k+#}_&aBK4baYPbZy$5aFx4}ka&qxl z$=Rh$W;U)>-=S-0=?7FH9dUAd2(q#4TCAHky!$^~;Dz^j|8_wuKc*YzfdAht@Q&ror?91Dm!N03=4=O!a)I*0q~p0g$Fm$pmr$ zb;wD;STDIi$@M%y1>p&_>%?UP($15gou_ue1u0!4(%81;qcIW8NyxFEvXpiJ|H4wz z*mFT(qVx1FKufG11hByuX%lPk4t#WZ{>8ka2efjY`~;AL6vWyQKpJun2nRiZYDij$ zP>4jQXPaP$UC$yIVgGa)jDV;F0l^n(V=HMRB5)20V7&r$jmk{UUIe zVjKroK}JAbD>B`2cwNQ&GDLx8{pg`7hbA~grk|W6LgiZ`8y`{Iq0i>t!3p2}MS6S+ zO_ruKyAElt)rdS>CtF7j{&6rP-#c=7evGMt7B6`7HG|-(WL`bDUAjyn+k$mx$CH;q2Dz4x;cPP$hW=`pFfLO)!jaCL@V2+F)So3}vg|%O*^T1j>C2lx zsURO-zIJC$^$g2byVbRIo^w>UxK}74^TqUiRR#7s_X$e)$6iYG1(PcW7un-va-S&u zHk9-6Zn&>T==A)lM^D~bk{&rFzCi35>UR!ZjQkdSiNX*-;l4z9j*7|q`TBl~Au`5& z+c)*8?#-tgUR$Zd%Q3bs96w6k7q@#tUn`5rj+r@_sAVVLqco|6O{ILX&U-&-cbVa3 zY?ngHR@%l{;`ri%H*0EhBWrGjv!LE4db?HEWb5mu*t@{kv|XwK8?npOshmzf=vZA@ zVSN9sL~!sn?r(AK)Q7Jk2(|M67Uy3I{eRy z_l&Y@A>;vjkWN5I2xvFFTLX0i+`{qz7C_@bo`ZUzDugfq4+>a3?1v%)O+YTd6@Ul7 zAfLfm=nhZ`)P~&v90$&UcF+yXm9sq!qCx3^9gzIcO|Y(js^Fj)Rvq>nQAHI92ap=P z10A4@prk+AGWCb`2)dQYFuR$|H6iDE8p}9a?#nV2}LBCoCf(Xi2@szia7#gY>b|l!-U`c}@ zLdhvQjc!BdLJvYvzzzngnw51yRYCqh4}$oRCy-z|v3Hc*d|?^Wj=l~18*E~*cR_kU z{XsxM1i{V*4GujHQ3DBpl2w4FgFR48Nma@HPgnyKoIEY-MqmMeY=I<%oG~l!f<+FN z1ZY^;10j4M4#HYXP zw5eJpA_y(>uLQ~OucgxDLuf}fVs272FaMxhn4xnDGIyLXnw>Xsd^J8XhcWIwIoQ9} z%FoSJTAGW(SRGwJwb=@pY7r$uQRK3Zd~XbxU)ts!4XsJrCycrWSI?e!IqwqIR8+Jh zlRjZ`UO1I!BtJR_2~7AbkbSm%XQqxEPkz6BTGWx8e}nQ=w7bZ|eVP4?*Tb!$(R)iC z9)&%bS*u(lXqzitAN)Oo=&Ytn>%Hzjc<5liuPi>zC_nw;Z0AE3Y$Jao_Q90R-gl~5 z_xAb2J%eArrC1CN4G$}-zVvCqF1;H;abAu6G*+PDHSYFx@Tdbfox*uEd3}BUyYY-l zTfEsOqsi#f9^FoLO;ChK<554qkri&Av~SIM*{fEYRE?vH7pTAOmu2pz3X?Wn*!ROX ztd54huAk&mFBemMooL33RV-*1f0Q3_(7hl$<#*|WF9P!;r;4_+X~k~uKEqdzZ$5Al zV63XN@)j$FN#cCD;ek1R#l zv%pGrhB~KWgoCj%GT?%{@@o(AJGt*PG#l3i>lhmb_twKH^EYvacVY-6bsCl5*^~L0 zonm@lk2UvvTKr2RS%}T>^~EYqdL1q4nD%0n&Xqr^cK^`J5W;lRRB^R-O8b&HENO||mo0xaD+S=I8RTlIfVgqN@SXDr2&-)we--K7w= zJVU8?Z+7k9dy;s;^gDkQa`0nz6N{T?(A&Iz)2!DEecLyRa&FI!id#5Z7B*O2=PsR0 zEvc|8{NS^)!d)MDX(97Xw}m&kEO@5jqRaDZ!+%`wYOI<23q|&js`&o4xvjP7D_xv@ z5hEwpsp{HezI9!~6O{~)lLR@oF7?J7i>1|5a~UuoN=q&6N}EJPV_GD`&M*v8Y`^2j zKII*d_@Fi$+i*YEW+Hbzn{iQk~yP z>7N{S4)r*!NwQ`(qcN#8SRQsNK6>{)X12nbF`*7#ecO7I)Q$uZsV+xS4E7aUn+U(K baj7?x%VD!5Cxk2YbYLNVeiXvvpMCWYo=by@ diff --git a/tests/__init__.py b/tests/__init__.py deleted file mode 100644 index 4ede8e6d..00000000 --- a/tests/__init__.py +++ /dev/null @@ -1 +0,0 @@ -# noqa diff --git a/tests/fixtures/__init__.py b/tests/fixtures/__init__.py deleted file mode 100644 index e69de29b..00000000 diff --git a/tests/fixtures/tavily_responses.py b/tests/fixtures/tavily_responses.py deleted file mode 100644 index ab94d353..00000000 --- a/tests/fixtures/tavily_responses.py +++ /dev/null @@ -1,71 +0,0 @@ -"""Sample tavily responses for patching responses in tests.""" - -RESPONSE_SEARCH_HOW_OLD_IS_EARTH_NO_ANSWER = { - "query": "How old is the earth", - "follow_up_questions": None, - "answer": None, - "images": [], - "results": [ - { - "url": "https://en.wikipedia.org/wiki/Age_of_Earth", - "title": "Age of Earth - Wikipedia", - "content": 'Scientific dating of the age of Earth The **age of Earth** is estimated to be 4.54 ± 0.05 billion years. In 1862, the physicist William Thomson, 1st Baron Kelvin published calculations that fixed the age of Earth at between 20 million and 400 million years. This suggested that it might be possible to measure the age of Earth by determining the relative proportions of radioactive materials in geological samples. Holmes published *The Age of the Earth, an Introduction to Geological Ideas* in 1927 in which he presented a range of 1.6 to 3.0 billion years. "The age of the Earth and the invention of geological time".', - "score": 0.8775715, - "raw_content": None, - }, - { - "url": "https://answersingenesis.org/age-of-the-earth/how-old-earth/?srsltid=AfmBOorqG4wgNP3fQ457C11mdj7kVx0IcByShaqH3wwc1VivvrqvJnCF", - "title": "How Old Is the Earth? | Answers in Genesis", - "content": "If you ask this question of most scientifically literate people, they will answer that the earth is about 4.54 billion years old.", - "score": 0.8703443, - "raw_content": None, - }, - { - "url": "https://sites.nd.edu/james-applewhite/2020/03/22/age-of-our-earth/", - "title": "Age of Our Earth: 6000 or 4.5 billion years old? - Notre Dame Sites", - "content": "If the Earth is only 6,000 years old, why does radiometric dating techniques used by geologists suggest the age is around much older? Each technique demonstrates the earth is much older than 6,000 years old and when combined with the various different techniques of relative dating using rock strata and formations, it becomes apparent that we have solid scientific evidence that the earth is much older than what AIG thinks. With this, as they try to discount radiometric dating as evidence since we were not around back then, they invalidate their own argument as they suggest that we should accept the words of the Bible as evidence.", - "score": 0.7975099, - "raw_content": None, - }, - { - "url": "https://www.tomorrowsworld.org/magazines/2013/march-april/how-old-is-the-earth", - "title": "How Old Is the Earth? | Tomorrow's World", - "content": "Was it billions of years ago—close to the scientists' estimate of a 4.5 billion-year-old Earth? Or was it earlier or later? On these details, the Bible is", - "score": 0.78944516, - "raw_content": None, - }, - { - "url": "https://www.planetary.org/articles/how-old-is-the-earth", - "title": "How old is the Earth? | The Planetary Society", - "content": "Skip to main content Community Account Renew Search * Become A Member * Renew Back To Main Menu Learn how our members and community are changing the worlds. Back To Main Menu * ### The Planetary Report Back To Main Menu + Become A Member + Action Center + Renew Membership Back To Main Menu Back To Main Menu + Become A Member + Renew Membership * Take Action * Member Community * Account Center * Search Public Education Specialist, The Planetary Society Along with other planets, the Earth was born in the early days of the Solar System, which first started forming about 4.6 billion years ago. thanks to techniques including radiometric dating of rocks and minerals,", - "score": 0.7756902, - "raw_content": None, - }, - ], - "response_time": 0.96, - "request_id": "3c36cccd-0918-49fd-bd1c-23c62ba7ec2d", -} - - -RESPONSE_EXTRACT_HOW_OLD_IS_EARTH_NO_ANSWER = { - "results": [ - { - "url": "https://en.wikipedia.org/wiki/Age_of_Earth", - "raw_content": 'Jump to content\nAge of Earth\n\nAfrikaans\nالعربية\nAzərbaycanca\nবাংলা\nБеларуская\nБългарски\nCatalà\nČeština\nDansk\nΕλληνικά\nEspañol\nEsperanto\nEuskara\nفارسی\nFrançais\nGalego\n한국어\nՀայերեն\nBahasa Indonesia\nItaliano\nעברית\nҚазақша\nLatina\nLëtzebuergesch\nLietuvių\nBahasa Melayu\nNederlands\n日本語\nپښتو\nPortuguês\nRomnă\nРусский\nSimple English\nSlovenčina\nSlovenščina\nСрпски / srpski\nSrpskohrvatski / српскохрватски\nSvenska\nTürkçe\nУкраїнська\nاردو\nTiếng Việt\n文言\nYorùbá\n粵語\n中文\n\nEdit links\nFrom Wikipedia, the free encyclopedia\nScientific dating of the age of Earth\nThe age of Earth is estimated to be 4.54 ± 0.05 billion years. This age represents the final stages of Earth\'s accretion and planetary differentiation. Age estimates are based on evidence from radiometric age-dating of meteoritic material—consistent with the radiometric ages of the oldest-known terrestrial material and lunar samples—and astrophysical accretion models consistent with observations of planet formation in protoplanetary disks.\nFollowing the development of radiometric dating in the early 20th century, measurements of lead in uranium-rich minerals showed that some were in excess of a billion years old. The oldest such minerals analyzed to date—small crystals of zircon from the Jack Hills of Western Australia—are at least 4.404 billion years old. Calcium–aluminium-rich inclusions—the oldest known solid constituents within meteorites that are formed within the Solar System—are 4.5673 ± 0.00016 billion years old giving a lower limit for the age of the Solar System.\nIt is hypothesized that the accretion of Earth began soon after the formation of the calcium-aluminium-rich inclusions. Because the duration of this accretion process is not yet adequately constrained—predictions from different accretion models range from around 30 million to 100 million years—the difference between the age of Earth and of the oldest rocks is difficult to determine. It can also be difficult to determine the exact age of the oldest rocks on Earth, exposed at the surface, as they are aggregates of minerals of possibly different ages.\nDevelopment of modern geologic concepts\n| | | |\n --- \n| Life timeline | | |\n| This box: view talk edit | | |\n| −4500 — – — – −4000 — – — – −3500 — – — – −3000 — – — – −2500 — – — – −2000 — – — – −1500 — – — – −1000 — – — – −500 — – — – 0 — | Water Single-celled life Photosynthesis Multicellular life Plants Arthropods Molluscs Flowers Dinosaurs Mammals Birds Primates Hadean Archean Proterozoic Phanerozoic | | | | --- | | ← | Earth formed | | | | --- | | ← | Earliest water | | | | --- | | ← | LUCA | | | | --- | | ← | Earliest fossils | | | | --- | | ← | Atmospheric oxygen | | | | --- | | ← | Sexual reproduction | | | | --- | | ← | Earliest fungi | | | | --- | | ← | Neoproterozoic oxygenation event | | | | --- | | ← | Ediacaran biota | | | | --- | | ← | Cambrian explosion | | | | --- | | ← | Earliest tetrapods | | | | --- | | ← | Earliest hominoid | |\n| (million years ago) | | |\nMain article: History of geology\nFurther information: Relative dating\nStudies of strata—the layering of rocks and soil—gave naturalists an appreciation that Earth may have been through many changes during its existence. These layers often contained fossilized remains of unknown creatures, leading some to interpret a progression of organisms from layer to layer.\nNicolas Steno in the 17th century was one of the first naturalists to appreciate the connection between fossil remains and strata. His observations led him to formulate important stratigraphic concepts (i.e., the "law of superposition" and the "principle of original horizontality"). In the 1790s, William Smith hypothesized that if two layers of rock at widely differing locations contained similar fossils, then it was very plausible that the layers were the same age. Smith\'s nephew and student, John Phillips, later calculated by such means that Earth was about 96 million years old.\nIn the mid-18th century, the naturalist Mikhail Lomonosov suggested that Earth had been created separately from, and several hundred thousand years before, the rest of the universe.[citation needed] Lomonosov\'s ideas were mostly speculative.[citation needed] In 1779 the Comte du Buffon tried to obtain a value for the age of Earth using an experiment: he created a small globe that resembled Earth in composition and then measured its rate of cooling. This led him to estimate that Earth was about 75,000 years old. Even earlier, in 1687, in his Principia, the mathematician and physicist Isaac Newton was the first to calculate the age of the Earth by experiment, coming to a conclusion of 50,000 years.\nOther naturalists used these hypotheses to construct a history of Earth, though their timelines were inexact as they did not know how long it took to lay down stratigraphic layers. In 1830, geologist Charles Lyell, developing ideas found in James Hutton\'s works, popularized the concept that the features of Earth were in perpetual change, eroding and reforming continuously, and the rate of this change was roughly constant. This was a challenge to the traditional view, which saw the history of Earth as dominated by intermittent catastrophes. Many naturalists were influenced by Lyell to become "uniformitarians" who believed that changes were constant and uniform.[citation needed]\nEarly calculations\nFurther information: William Thomson, 1st Baron Kelvin § Age of the Earth: geology\nIn 1862, the physicist William Thomson, 1st Baron Kelvin published calculations that fixed the age of Earth at between 20 million and 400 million years. He assumed that Earth had formed as a completely molten object, and determined the amount of time it would take for the near-surface temperature gradient to decrease to its present value. His calculations did not account for heat produced via radioactive decay (a then unknown process) or, more significantly, convection inside Earth, which allows the temperature in the upper mantle to remain high much longer, maintaining a high thermal gradient in the crust much longer. Even more constraining were Thomson\'s estimates of the age of the Sun, which were based on estimates of its thermal output and a theory that the Sun obtains its energy from gravitational collapse; Thomson estimated that the Sun is about 20 million years old.\nGeologists such as Lyell had difficulty accepting such a short age for Earth. For biologists, even 100 million years seemed much too short to be plausible. In Charles Darwin\'s theory of evolution, the process of random heritable variation with cumulative selection requires great durations of time, and Darwin stated that Thomson\'s estimates did not appear to provide enough time. According to modern biology, the total evolutionary history from the beginning of life to today has taken place since 3.5 to 3.8 billion years ago, the amount of time which passed since the last universal ancestor of all living organisms as shown by geological dating.\nIn a lecture in 1869, Darwin\'s great advocate, Thomas Henry Huxley, attacked Thomson\'s calculations, suggesting they appeared precise in themselves but were based on faulty assumptions. The physicist Hermann von Helmholtz (in 1856) and astronomer Simon Newcomb (in 1892) contributed their own calculations of 22 and 18 million years, respectively, to the debate: they independently calculated the amount of time it would take for the Sun to condense down to its current diameter and brightness from the nebula of gas and dust from which it was born. Their values were consistent with Thomson\'s calculations. However, they assumed that the Sun was only glowing from the heat of its gravitational contraction. The process of solar nuclear fusion was not yet known to science.\nIn 1892, Thomson was ennobled as Lord Kelvin in appreciation of his many scientific accomplishments. In 1895 John Perry challenged Kelvin\'s figure on the basis of his assumptions on conductivity, and Oliver Heaviside entered the dialogue, considering it "a vehicle to display the ability of his operator method to solve problems of astonishing complexity." Other scientists backed up Kelvin\'s figures. Darwin\'s son, the astronomer George H. Darwin, proposed that Earth and Moon had broken apart in their early days when they were both molten. He calculated the amount of time it would have taken for tidal friction to give Earth its current 24-hour day. His value of 56 million years was additional evidence that Thomson was on the right track. The last estimate Kelvin gave, in 1897, was: "that it was more than 20 and less than 40 million year old, and probably much nearer 20 than 40". In 1899 and 1900, John Joly calculated the rate at which the oceans should have accumulated salt from erosion processes and determined that the oceans were about 80 to 100 million years old.\nRadiometric dating\nMain article: Radiometric dating\nOverview\nBy their chemical nature, rock minerals contain certain elements and not others; but in rocks containing radioactive isotopes, the process of radioactive decay generates exotic elements over time. By measuring the concentration of the stable end product of the decay, coupled with knowledge of the half life and initial concentration of the decaying element, the age of the rock can be calculated. Typical radioactive end products are argon from decay of potassium-40, and lead from decay of uranium and thorium. If the rock becomes molten, as happens in Earth\'s mantle, such nonradioactive end products typically escape or are redistributed. Thus the age of the oldest terrestrial rock gives a minimum for the age of Earth, assuming that no rock has been intact for longer than Earth itself.\nConvective mantle and radioactivity\nThe discovery of radioactivity introduced another factor in the calculation. After Henri Becquerel\'s initial discovery in 1896, Marie and Pierre Curie discovered the radioactive elements polonium and radium in 1898; and in 1903, Pierre Curie and Albert Laborde announced that radium produces enough heat to melt its own weight in ice in less than an hour. Geologists quickly realized that this upset the assumptions underlying most calculations of the age of Earth. These had assumed that the original heat of Earth and the Sun had dissipated steadily into space, but radioactive decay meant that this heat had been continually replenished. George Darwin and John Joly were the first to point this out, in 1903.\nInvention of radiometric dating\nRadioactivity, which had overthrown the old calculations, yielded a bonus by providing a basis for new calculations, in the form of radiometric dating.\nErnest Rutherford and Frederick Soddy jointly had continued their work on radioactive materials and concluded that radioactivity was caused by a spontaneous transmutation of atomic elements. In radioactive decay, an element breaks down into another, lighter element, releasing alpha, beta, or gamma radiation in the process. They also determined that a particular isotope of a radioactive element decays into another element at a distinctive rate. This rate is given in terms of a "half-life", or the amount of time it takes half of a mass of that radioactive material to break down into its "decay product".\nSome radioactive materials have short half-lives; some have long half-lives. Uranium and thorium have long half-lives and so persist in Earth\'s crust, but radioactive elements with short half-lives have generally disappeared. This suggested that it might be possible to measure the age of Earth by determining the relative proportions of radioactive materials in geological samples. In reality, radioactive elements do not always decay into nonradioactive ("stable") elements directly, instead, decaying into other radioactive elements that have their own half-lives and so on, until they reach a stable element. These "decay chains", such as the uranium-radium and thorium series, were known within a few years of the discovery of radioactivity and provided a basis for constructing techniques of radiometric dating.\nThe pioneers of radioactivity were chemist Bertram B. Boltwood and physicist Rutherford. Boltwood had conducted studies of radioactive materials as a consultant, and when Rutherford lectured at Yale in 1904, Boltwood was inspired to describe the relationships between elements in various decay series. Late in 1904, Rutherford took the first step toward radiometric dating by suggesting that the alpha particles released by radioactive decay could be trapped in a rocky material as helium atoms. At the time, Rutherford was only guessing at the relationship between alpha particles and helium atoms, but he would prove the connection four years later.\nSoddy and Sir William Ramsay had just determined the rate at which radium produces alpha particles, and Rutherford proposed that he could determine the age of a rock sample by measuring its concentration of helium. He dated a rock in his possession to an age of 40 million years by this technique. Rutherford wrote of addressing a meeting of the Royal Institution in 1904:\n\nI came into the room, which was half dark, and presently spotted Lord Kelvin in the audience and realized that I was in trouble at the last part of my speech dealing with the age of the Earth, where my views conflicted with his. To my relief, Kelvin fell fast asleep, but as I came to the important point, I saw the old bird sit up, open an eye, and cock a baleful glance at me! Then a sudden inspiration came, and I said, "Lord Kelvin had limited the age of the Earth, provided no new source was discovered. That prophetic utterance refers to what we are now considering tonight, radium!" Behold! the old boy beamed upon me.\n\nRutherford assumed that the rate of decay of radium as determined by Ramsay and Soddy was accurate and that helium did not escape from the sample over time. Rutherford\'s scheme was inaccurate, but it was a useful first step. Boltwood focused on the end products of decay series. In 1905, he suggested that lead was the final stable product of the decay of radium. It was already known that radium was an intermediate product of the decay of uranium. Rutherford joined in, outlining a decay process in which radium emitted five alpha particles through various intermediate products to end up with lead, and speculated that the radium–lead decay chain could be used to date rock samples. Boltwood did the legwork and by the end of 1905 had provided dates for 26 separate rock samples, ranging from 92 to 570 million years. He did not publish these results, which was fortunate because they were flawed by measurement errors and poor estimates of the half-life of radium. Boltwood refined his work and finally published the results in 1907.\nBoltwood\'s paper pointed out that samples taken from comparable layers of strata had similar lead-to-uranium ratios, and that samples from older layers had a higher proportion of lead, except where there was evidence that lead had leached out of the sample. His studies were flawed by the fact that the decay series of thorium was not understood, which led to incorrect results for samples that contained both uranium and thorium. However, his calculations were far more accurate than any that had been performed to that time. Refinements in the technique would later give ages for Boltwood\'s 26 samples of 410 million to 2.2 billion years.\nArthur Holmes establishes radiometric dating\nAlthough Boltwood published his paper in a prominent geological journal, the geological community had little interest in radioactivity.[citation needed] Boltwood gave up work on radiometric dating and went on to investigate other decay series. Rutherford remained mildly curious about the issue of the age of Earth but did little work on it.\nRobert Strutt tinkered with Rutherford\'s helium method until 1910 and then ceased. However, Strutt\'s student Arthur Holmes became interested in radiometric dating and continued to work on it after everyone else had given up. Holmes focused on lead dating because he regarded the helium method as unpromising. He performed measurements on rock samples and concluded in 1911 that the oldest (a sample from Ceylon) was about 1.6 billion years old. These calculations were not particularly trustworthy. For example, he assumed that the samples had contained only uranium and no lead when they were formed.\nMore important research was published in 1913. It showed that elements generally exist in multiple variants with different masses, or "isotopes". In the 1930s, isotopes would be shown to have nuclei with differing numbers of the neutral particles known as "neutrons". In that same year, other research was published establishing the rules for radioactive decay, allowing more precise identification of decay series.\nMany geologists felt these new discoveries made radiometric dating so complicated as to be worthless.[citation needed] Holmes felt that they gave him tools to improve his techniques, and he plodded ahead with his research, publishing before and after the First World War. His work was generally ignored until the 1920s, though in 1917 Joseph Barrell, a professor of geology at Yale, redrew geological history as it was understood at the time to conform to Holmes\'s findings in radiometric dating. Barrell\'s research determined that the layers of strata had not all been laid down at the same rate, and so current rates of geological change could not be used to provide accurate timelines of the history of Earth.[citation needed]\nHolmes\' persistence finally began to pay off in 1921, when the speakers at the yearly meeting of the British Association for the Advancement of Science came to a rough consensus that Earth was a few billion years old and that radiometric dating was credible. Holmes published The Age of the Earth, an Introduction to Geological Ideas in 1927 in which he presented a range of 1.6 to 3.0 billion years. No great push to embrace radiometric dating followed, however, and the die-hards in the geological community stubbornly resisted. They had never cared for attempts by physicists to intrude in their domain, and had successfully ignored them so far. The growing weight of evidence finally tilted the balance in 1931, when the National Research Council of the US National Academy of Sciences decided to resolve the question of the age of Earth by appointing a committee to investigate.\nHolmes, being one of the few people who was trained in radiometric dating techniques, was a committee member and in fact wrote most of the final report. Thus, Holmes\' report concluded that radioactive dating was the only reliable means of pinning down a geologic time scale. Questions of bias were deflected by the great and exacting detail of the report. It described the methods used, the care with which measurements were made, and their error bars and limitations.[citation needed]\nModern radiometric dating\nRadiometric dating continues to be the predominant way scientists date geologic time scales. Techniques for radioactive dating have been tested and fine-tuned on an ongoing basis since the 1960s. Forty or so different dating techniques have been utilized to date, working on a wide variety of materials. Dates for the same sample using these different techniques are in very close agreement on the age of the material.[citation needed] Possible contamination problems do exist, but they have been studied and dealt with by careful investigation, leading to sample preparation procedures being minimized to limit the chance of contamination.[citation needed]\nUse of meteorites\nAn age of 4.55 ± 0.07 billion years, very close to today\'s accepted age, was determined by Clair Cameron Patterson using uranium–lead isotope dating (specifically lead–lead dating) on several meteorites including the Canyon Diablo meteorite and published in 1956. The quoted age of Earth is derived, in part, from the Canyon Diablo meteorite for several important reasons and is built upon a modern understanding of cosmochemistry built up over decades of research.\nMost geological samples from Earth are unable to give a direct date of the formation of Earth from the solar nebula because Earth has undergone differentiation into the core, mantle, and crust, and this has then undergone a long history of mixing and unmixing of these sample reservoirs by plate tectonics, weathering and hydrothermal circulation.\nAll of these processes may adversely affect isotopic dating mechanisms because the sample cannot always be assumed to have remained as a closed system, by which it is meant that either the parent or daughter nuclide (a species of atom characterised by the number of neutrons and protons an atom contains) or an intermediate daughter nuclide may have been partially removed from the sample, which will skew the resulting isotopic date. To mitigate this effect it is usual to date several minerals in the same sample, to provide an isochron. Alternatively, more than one dating system may be used on a sample to check the date.\nSome meteorites are furthermore considered to represent the primitive material from which the accreting solar disk was formed. Some have behaved as closed systems (for some isotopic systems) soon after the solar disk and the planets formed.[citation needed] To date, these assumptions are supported by much scientific observation and repeated isotopic dates, and it is certainly a more robust hypothesis than that which assumes a terrestrial rock has retained its original composition.\nNevertheless, ancient Archaean lead ores of galena have been used to date the formation of Earth as these represent the earliest formed lead-only minerals on the planet and record the earliest homogeneous lead–lead isotope systems on the planet. These have returned age dates of 4.54 billion years with a precision of as little as 1% margin for error.\nStatistics for several meteorites that have undergone isochron dating are as follows:\n| 1. St. Severin (ordinary chondrite) | | | |\n --- --- |\n| | 1. | Pb-Pb isochron | 4.543 ± 0.019 billion years |\n| | 2. | Sm-Nd isochron | 4.55 ± 0.33 billion years |\n| | 3. | Rb-Sr isochron | 4.51 ± 0.15 billion years |\n| | 4. | Re-Os isochron | 4.68 ± 0.15 billion years |\n| 2. Juvinas (basaltic achondrite) | | | |\n| | 1. | Pb-Pb isochron | 4.556 ± 0.012 billion years |\n| | 2. | Pb-Pb isochron | 4.540 ± 0.001 billion years |\n| | 3. | Sm-Nd isochron | 4.56 ± 0.08 billion years |\n| | 4. | Rb-Sr isochron | 4.50 ± 0.07 billion years |\n| 3. Allende (carbonaceous chondrite) | | | |\n| | 1. | Pb-Pb isochron | 4.553 ± 0.004 billion years |\n| | 2. | Ar-Ar age spectrum | 4.52 ± 0.02 billion years |\n| | 3. | Ar-Ar age spectrum | 4.55 ± 0.03 billion years |\n| | 4. | Ar-Ar age spectrum | 4.56 ± 0.05 billion years |\nCanyon Diablo meteorite\nFurther information: Age of the Solar System and Canyon Diablo (meteorite)\nThe Canyon Diablo meteorite was used because it is both large and representative of a particularly rare type of meteorite that contains sulfide minerals (particularly troilite, FeS), metallic nickel-iron alloys, plus silicate minerals. This is important because the presence of the three mineral phases allows investigation of isotopic dates using samples that provide a great separation in concentrations between parent and daughter nuclides. This is particularly true of uranium and lead. Lead is strongly chalcophilic and is found in the sulfide at a much greater concentration than in the silicate, versus uranium. Because of this segregation in the parent and daughter nuclides during the formation of the meteorite, this allowed a much more precise date of the formation of the solar disk and hence the planets than ever before.\nThe age determined from the Canyon Diablo meteorite has been confirmed by hundreds of other age determinations, from both terrestrial samples and other meteorites. The meteorite samples, however, show a spread from 4.53 to 4.58 billion years ago. This is interpreted as the duration of formation of the solar nebula and its collapse into the solar disk to form the Sun and the planets. This 50 million year time span allows for accretion of the planets from the original solar dust and meteorites.\nThe Moon, as another extraterrestrial body that has not undergone plate tectonics and that has no atmosphere, provides quite precise age dates from the samples returned from the Apollo missions. Rocks returned from the Moon have been dated at a maximum of 4.51 billion years old. Martian meteorites that have landed upon Earth have also been dated to around 4.5 billion years old by lead–lead dating. Lunar samples, since they have not been disturbed by weathering, plate tectonics or material moved by organisms, can also provide dating by direct electron microscope examination of cosmic ray tracks. The accumulation of dislocations generated by high energy cosmic ray particle impacts provides another confirmation of the isotopic dates. Cosmic ray dating is only useful on material that has not been melted, since melting erases the crystalline structure of the material, and wipes away the tracks left by the particles.\nSee also\n\nWorld portal\n\nAge of the universe\n\nCreation myth\nGeochronology\nHistory of Earth\nNatural history\nOldest dated rocks\nTimeline of natural history\n\nReferences\n\n^ "Age of the Earth". U.S. Geological Survey. 1997. Archived from the original on 23 December 2005. Retrieved 2006-01-10.\n^ Dalrymple, G. Brent (2001). "The age of the Earth in the twentieth century: a problem (mostly) solved". Special Publications, Geological Society of London. 190 (1): 205–221. Bibcode:2001GSLSP.190..205D. doi:10.1144/GSL.SP.2001.190.01.14. S2CID 130092094.\n^ Manhesa, Gérard; Allègre, Claude J.; Dupréa, Bernard & Hamelin, Bruno (1980). "Lead isotope study of basic-ultrabasic layered complexes: Speculations about the age of the earth and primitive mantle characteristics". Earth and Planetary Science Letters. 47 (3): 370–382. Bibcode:1980E&PSL..47..370M. doi:10.1016/0012-821X(80)90024-2.\n^ Braterman, Paul S. (2013). "How Science Figured Out the Age of Earth". Scientific American. Archived from the original on 2016-04-12.\n^ a b Mezger, K.; Schönbächler, M.; Bouvier, A. (2020-03-04). "Accretion of the Earth—Missing Components?". Space Science Reviews. 216 (2): 27. doi:10.1007/s11214-020-00649-y. hdl:20.500.11850/405628. ISSN 1572-9672.\n^ Hedman, Matthew (2007). "9: Meteorites and the Age of the Solar System". The Age of Everything. University of Chicago Press. pp. 142–162. ISBN 9780226322940. Archived from the original on 2018-02-14.\n^ a b Wilde, S. A.; Valley, J. W.; Peck, W. H.; Graham C. M. (2001-01-11). "Evidence from detrital zircons for the existence of continental crust and oceans on the Earth 4.4 Gyr ago". Nature. 409 (6817): 175–178. Bibcode:2001Natur.409..175W. doi:10.1038/35051550. PMID 11196637. S2CID 4319774.\n^ Barboni, Melanie; Boehnke, Patrick; Keller, Brenhin; Kohl, Issaku E.; Schoene, Blair; Young, Edward D.; McKeegan, Kevin D. (2017-01-06). "Early formation of the Moon 4.51 billion years ago". Science Advances. 3 (1): e1602365. Bibcode:2017SciA....3E2365B. doi:10.1126/sciadv.1602365. ISSN 2375-2548. PMC 5226643. PMID 28097222.\n^ Halliday, Alex N.; Canup, Robin M. (2022-11-29). "The accretion of planet Earth". Nature Reviews Earth & Environment. 4 (1): 19–35. doi:10.1038/s43017-022-00370-0. ISSN 2662-138X.\n^ Pfalzner, S; Davies, M B; Gounelle, M; Johansen, A; Münker, C; Lacerda, P; Zwart, S Portegies; Testi, L; Trieloff, M; Veras, D (2015-06-01). "The formation of the solar system". Physica Scripta. 90 (6): 068001. arXiv:1501.03101. doi:10.1088/0031-8949/90/6/068001. ISSN 0031-8949.\n^ a b c Boltwood, B. B. (1907). "On the ultimate disintegration products of the radio-active elements. Part II. The disintegration products of uranium". American Journal of Science. 23 (134): 77–88. doi:10.2475/ajs.s4-23.134.78. S2CID 131688682.\n For the abstract, see: Chemical Abstracts Service, American Chemical Society (1907). Chemical Abstracts. New York, London: American Chemical Society. p. 817. Retrieved 2008-12-19.\n^ Valley, John W.; Peck, William H.; Kin, Elizabeth M. (1999). "Zircons Are Forever" (PDF). The Outcrop, Geology Alumni Newsletter. University of Wisconsin-Madison. pp. 34–35. Archived (PDF) from the original on 2009-02-26. Retrieved 2008-12-22.\n^ Wyche, S.; Nelson, D. R.; Riganti, A. (2004). "4350–3130 Ma detrital zircons in the Southern Cross Granite–Greenstone Terrane, Western Australia: implications for the early evolution of the Yilgarn Craton". Australian Journal of Earth Sciences. 51 (1): 31–45. Bibcode:2004AuJES..51...31W. doi:10.1046/j.1400-0952.2003.01042.x.\n^ Amelin, Yuri; Kaltenbach, Angela; Iizuka, Tsuyoshi; Stirling, Claudine H.; Ireland, Trevor R.; Petaev, Michail; Jacobsen, Stein B. (2010-12-01). "U–Pb chronology of the Solar System\'s oldest solids with variable 238U/235U". Earth and Planetary Science Letters. 300 (3): 343–350. doi:10.1016/j.epsl.2010.10.015. hdl:1885/21305. ISSN 0012-821X.\n^ Connelly, James N.; Bizzarro, Martin; Krot, Alexander N.; Nordlund, Åke; Wielandt, Daniel; Ivanova, Marina A. (2012-11-02). "The Absolute Chronology and Thermal Processing of Solids in the Solar Protoplanetary Disk". Science. 338 (6107): 651–655. doi:10.1126/science.1226919.\n^ Sossi, Paolo A.; Stotz, Ingo L.; Jacobson, Seth A.; Morbidelli, Alessandro; O’Neill, Hugh St C. (2022-07-07). "Stochastic accretion of the Earth". Nature Astronomy. 6 (8): 951–960. doi:10.1038/s41550-022-01702-2. ISSN 2397-3366. PMC 7613298.\n^ Lyell, Charles, Sir (1866). Elements of Geology; or, The Ancient Changes of the Earth and its Inhabitants as Illustrated by Geological Monuments (Sixth ed.). New York: D. Appleton and company. Retrieved 2008-12-19.{{cite book}}: CS1 maint: multiple names: authors list (link)\n^ a b Stiebing, William H. (1994). Uncovering the Past. Oxford University Press US. ISBN 978-0-19-508921-9.\n^ a b Brookfield, Michael E. (2004). Principles of Stratigraphy. Blackwell Publishing. p. 116. ISBN 978-1-4051-1164-5.\n^ Fuller, J. G. C. M. (2007-07-17). "Smith\'s other debt, John Strachey, William Smith and the strata of England 1719–1801". Geoscientist. The Geological Society. Archived from the original on 24 November 2008. Retrieved 2008-12-19.\n^ Burchfield, Joe D. (1998). "The age of the Earth and the invention of geological time". Geological Society, London, Special Publications. 143 (1): 137–143. Bibcode:1998GSLSP.143..137B. CiteSeerX 10.1.1.557.2702. doi:10.1144/GSL.SP.1998.143.01.12. S2CID 129443412.\n^ BUFFON, GEORGES LOUIS LECLERC (2022). HISTOIRE NATURELLE, GA (C)NA (C)RALE ET PARTICULIARE, : introduction a l\'histoire... des mina (c)raux (classic reprint). [S.l.]: FORGOTTEN BOOKS. ISBN 978-0-265-92735-9. OCLC 1354275595.\n^ Merrill, Ronald T. (2010). Our Magnetic Earth: The Science of Geomagnetism. Chicago: University of Chicago Press. p. 86. ISBN 978-0-226-52053-7.\n^ Simms, D. L. (2004). "Newton\'s Contribution to the Science of Heat". Annals of Science. 61 (1): 33–77. doi:10.1080/00033790210123810. ISSN 0003-3790.\n^ a b England, P.; Molnar, P.; Righter, F. (January 2007). "John Perry\'s neglected critique of Kelvin\'s age for the Earth: A missed opportunity in geodynamics". GSA Today. 17 (1): 4–9. Bibcode:2007GSAT...17R...4E. doi:10.1130/GSAT01701A.1.\n^ Dalrymple (1994) pp. 14–17, 38\n^ Burchfield, Joe D. (1990-05-15). Lord Kelvin and the Age of the Earth. University of Chicago Press. pp. 69 ff. ISBN 9780226080437. Archived from the original on 2018-02-14.\n^ Stacey, Frank D. (2000). "Kelvin\'s age of the Earth paradox revisited". Journal of Geophysical Research. 105 (B6): 13155–13158. Bibcode:2000JGR...10513155S. doi:10.1029/2000JB900028.\n^ Origin of Species, Charles Darwin, 1872 edition, page 286\n^ Borenstein, Seth (November 13, 2013). "Oldest fossil found: Meet your microbial mom". Excite. Yonkers, NY: Mindspark Interactive Network. Associated Press. Archived from the original on June 29, 2015. Retrieved 2015-03-02.)\n^ a b c Dalrymple (1994) pp. 14–17\n^ Paul J. Nahin (1985) Oliver Heaviside, Fractional Operators, and the Age of the Earth, IEEE Transactions on Education E-28(2): 94–104, link from IEEE Explore\n^ Dalrymple (1994) pp. 14, 43\n^ a b c Nichols, Gary (2009). "21.2 Radiometric Dating". Sedimentology and Stratigraphy. John Wiley & Sons. pp. 325–327. ISBN 978-1405193795.\n^ Henri Becquerel (1896). "Sur les radiations émises par phosphorescence". Comptes Rendus. 122: 420–421.\n^ Comptes Rendus 122: 420 (1896), translated by Carmen Giunta. Accessed 12 April 2021.\n^ Henri Becquerel (1896). "Sur les radiations invisibles émises par les corps phosphorescents". Comptes Rendus. 122: 501–503.\n^ Comptes Rendus 122: 501–503 (1896), translated by Carmen Giunta. Accessed 12 April 2021.\n^ Curie, Pierre; Curie, Marie & Bémont, Gustave (1898). "Sur une nouvelle substance fortement radio-active, contenue dans la pechblende (On a new, strongly radioactive substance contained in pitchblende)". Comptes Rendus. 127: 1215–1217. Archived from the original on 6 August 2009. Retrieved 12 April 2021.\n^ Curie, Pierre; Laborde, Albert (1903). "Sur la chaleur dégagée spontanément par les sels de radium". Comptes Rendus. 136: 673–675.\n^ Joly, John (1909). Radioactivity and Geology: An Account of the Influence of Radioactive Energy on Terrestrial History (1st ed.). London, UK: Archibald Constable & Co., ltd. p. 36. Reprinted by BookSurge Publishing (2004) ISBN 1-4021-3577-7.\n^ Rutherford, E. (1906). Radioactive Transformations. London: Charles Scribner\'s Sons. Reprinted by Juniper Grove (2007) ISBN 978-1-60355-054-3.\n^ Eve, Arthur Stewart (1939). Rutherford: Being the life and letters of the Rt. Hon. Lord Rutherford, O. M.. Cambridge: Cambridge University Press.\n^ Dalrymple (1994) p. 74\n^ The Age of the Earth Debate Badash, L Scientific American 1989 esp p95 Archived 2016-11-05 at the Wayback Machine\n^ Dalrymple (1994) pp. 77–78\n^ Patterson, Claire (1956). "Age of meteorites and the earth" (PDF). Geochimica et Cosmochimica Acta. 10 (4): 230–237. Bibcode:1956GeCoA..10..230P. doi:10.1016/0016-7037(56)90036-9. Archived (PDF) from the original on 2010-06-21. Retrieved 2009-07-07.\n^ Carlson, R. W.; Tera, F. (December 1–3, 1998). "Lead–Lead Constraints on the Timescale of Early Planetary Differentiation" (PDF). Conference Proceedings, Origin of the Earth and Moon. Houston, Texas: Lunar and Planetary Institute. p. 6. Archived (PDF) from the original on 16 December 2008. Retrieved 2008-12-22.\n^ Dalrymple (1994) pp. 310–341\n^ Dalrymple, Brent G. (2004). Ancient Earth, Ancient Skies: The Age of the Earth and Its Cosmic Surroundings. Stanford University Press. pp. 147, 169. ISBN 978-0-8047-4933-6.\n^ Terada, K.; Sano, Y. (May 20–24, 2001). "In-situ ion microprobe U-Pb dating of phosphates in H-chondrites" (PDF). Proceedings, Eleventh Annual V. M. Goldschmidt Conference. Hot Springs, Virginia: Lunar and Planetary Institute. Bibcode:2001eag..conf.3306T. Archived (PDF) from the original on 16 December 2008. Retrieved 2008-12-22.\n\nBibliography\n\nDalrymple, G. Brent (1994-02-01). The Age of the Earth. Stanford University Press. ISBN 978-0-8047-2331-2.\n\nFurther reading\n\nBaadsgaard, H.; Lerbekmo, J.F.; Wijbrans, J.R., 1993. Multimethod radiometric age for a bentonite near the top of the Baculites reesidei Zone of southwestern Saskatchewan (Campanian-Maastrichtian stage boundary?). Canadian Journal of Earth Sciences, v.30, p. 769–775.\nBaadsgaard, H. and Lerbekmo, J.F., 1988. A radiometric age for the Cretaceous-Tertiary boundary based on K-Ar, Rb-Sr, and U-Pb ages of bentonites from Alberta, Saskatchewan, and Montana. Canadian Journal of Earth Sciences, v.25, p. 1088–1097.\nEberth, D.A. and Braman, D., 1990. Stratigraphy, sedimentology, and vertebrate paleontology of the Judith River Formation (Campanian) near Muddy Lake, west-central Saskatchewan. Bulletin of Canadian Petroleum Geology, v.38, no.4, p. 387–406.\nGoodwin, M.B. and Deino, A.L., 1989. The first radiometric ages from the Judith River Formation (Upper Cretaceous), Hill County, Montana. Canadian Journal of Earth Sciences, v.26, p. 1384–1391.\nGradstein, F. M.; Agterberg, F.P.; Ogg, J.G.; Hardenbol, J.; van Veen, P.; Thierry, J. and Zehui Huang., 1995. A Triassic, Jurassic and Cretaceous time scale. IN: Bergren, W. A.; Kent, D.V.; Aubry, M-P. and Hardenbol, J. (eds.), Geochronology, Time Scales, and Global Stratigraphic Correlation. Society of Economic Paleontologists and Mineralogists, Special Publication No. 54, p. 95–126.\nHarland, W.B., Cox, A.V.; Llewellyn, P.G.; Pickton, C.A.G.; Smith, A.G.; and Walters, R., 1982. A Geologic Time Scale: 1982 edition. Cambridge University Press: Cambridge, 131p.\nHarland, W.B.; Armstrong, R.L.; Cox, A.V.; Craig, L.E.; Smith, A.G.; Smith, D.G., 1990. A Geologic Time Scale, 1989 edition. Cambridge University Press: Cambridge, p. 1–263. ISBN 0-521-38765-5\nHarper, C.W. Jr (1980). "Relative age inference in paleontology". Lethaia. 13 (3): 239–248. Bibcode:1980Letha..13..239H. doi:10.1111/j.1502-3931.1980.tb00638.x.\nObradovich, J.D., 1993. A Cretaceous time scale. IN: Caldwell, W.G.E. and Kauffman, E.G. (eds.). Evolution of the Western Interior Basin. Geological Association of Canada, Special Paper 39, p. 379–396.\nPalmer, Allison R (1983). "The Decade of North American Geology 1983 Geologic Time Scale". Geology. 11 (9): 503–504. Bibcode:1983Geo....11..503P. doi:10.1130/0091-7613(1983)11<503:tdonag>2.0.co;2.\nPowell, James Lawrence, 2001, Mysteries of Terra Firma: the Age and Evolution of the Earth, Simon & Schuster, ISBN 0-684-87282-X\n\nExternal links\n\nThe Age of the Earth by Chris Stassen (TalkOrigins.org)\nUSGS preface on the Age of the Earth\nNASA exposition on the age of Martian meteorites\nAgeing the Earth on In Our Time at the BBC\nPre-1900 Non-Religious Estimates of the Age of the Earth\n\n| | |\n --- |\n| Outline History | |\n| Atmosphere | Atmosphere of Earth Prebiotic atmosphere Troposphere Stratosphere Mesosphere Thermosphere Exosphere Weather |\n| Climate | Climate system Energy balance Climate change Climate variability and change Climatology Paleoclimatology |\n| Continents | Africa Antarctica Asia Australia Europe North America South America |\n| Culture and society | List of sovereign states + dependent territories In culture Earth Day Flag Symbol World economy Etymology World history Time zones World |\n| Environment | Biome Biosphere Biogeochemical cycles Ecology Ecosystem Human impact on the environment Evolutionary history of life Nature |\n| Geodesy | Cartography + Computer cartography Earth\'s orbit Geodetic astronomy Geomatics Gravity Navigation Remote Sensing Geopositioning Virtual globe |\n| Geophysics | Earth structure Fluid dynamics Geomagnetism Magnetosphere Mineral physics Seismology Plate tectonics Signal processing Tomography |\n| Geology | Age of Earth Earth science Extremes on Earth Future Geological history + Geologic time scale Geologic record History of Earth |\n| Oceans | Antarctic/Southern Ocean Arctic Ocean Atlantic Ocean Indian Ocean Pacific Ocean Oceanography |\n| Planetary science | The Moon Evolution of the Solar System Geology of solar terrestrial planets Location in the Universe Solar System |\n| | |\n| Authority control databases | |\n --- |\n| National | United States Israel |\n| Other | Yale LUX |\nRetrieved from "\nCategories:\n\nGeochronology\nHistory of Earth science\nGeology theories\n\nHidden categories:\n\nCS1 maint: multiple names: authors list\nWebarchive template wayback links\nArticles with short description\nShort description is different from Wikidata\nWikipedia pages semi-protected against vandalism\nAll articles with unsourced statements\nArticles with unsourced statements from February 2023\nArticles with unsourced statements from March 2015', - "images": [], - }, - { - "url": "https://www.planetary.org/articles/how-old-is-the-earth", - "raw_content": "Skip to main content\nCommunity Account Renew Search\nJoin\n\nBecome A Member\nRenew\nGift Membership\nKids Membership\nOther Ways to Give\n\nDonate\nJoin\nDonate\nBack To Main Menu\n\nWhat We Do\n\nExplore Worlds\n\nFind Life\nDefend Earth\n\nHow We Work\n\nEducation & Public Outreach\n\nSpace Policy & Advocacy\nScience & Technology\nGlobal Collaboration\n\nOur Results\n\nOur Impact\nLearn how our members and community are changing the worlds.\n + LightSail\nOur citizen-funded spacecraft successfully demonstrated solar sailing for CubeSats.\n\nBack To Main Menu\n\nSpace Topics\n\nPlanets & Other Worlds\n\nSpace Missions\nNight Sky\nSpace Policy\nFor Kids\n\nLearn\n\nArticles\n\nPlanetary Radio\nSpace Images\nVideos\nCourses\n\nThe Planetary Report\n\n#### Solar Maximum\nOur dynamic star's reach throughout the Solar System.\nBack To Main Menu\n\nGet Involved\n\nBecome A Member\nMembership programs for explorers of all ages.\n + Email Signup\nGet updates and weekly tools to learn, share, and advocate for space exploration.\n + Action Center\nVolunteer as a space advocate.\n ### Support Our Mission\n\nRenew Membership\n\nSociety Projects\nTravel\nOther Ways to Give\nStore\n\nThe Planetary Fund\n\nAccelerate progress in our three core enterprises — Explore Worlds, Find Life, and Defend Earth. You can support the entire fund, or designate a core enterprise of your choice.\nGive Today\nBack To Main Menu\n\nAbout Us\n\nOverview\n\nStrategic Framework\nNews & Press\nCareers\nContact Us\nOur Story\n\nThe Planetary Society\n\n#### Our Vision\nKnow the Cosmos and our place within it.\n#### Our Mission\nEmpowering the world's citizens to advance space science and exploration.\nBack To Main Menu\n\nMembership\n\nBecome A Member\n\nRenew Membership\nGift Membership\nKids Membership\nOther Ways to Give\n\nContact Us\n\nOur Work\n\nExplore Space\nTake Action\nAbout\nMembership\nMember Community\nAccount Center\n\nSearch\n\n“Exploration is in our nature.” - Carl Sagan\n\nHow old is the Earth?\nWritten by\nKate Howells\nPublic Education Specialist, The Planetary Society \nNovember 14, 2023\nThe Earth is thought to be about 4.54 billion years old. Along with other planets, the Earth was born in the early days of the Solar System, which first started forming about 4.6 billion years ago.\nHow did the Earth form?\nThe Solar System formed about 4.6 billion years ago from material in a massive, rotating cloud of gas and dust called the solar nebula. Gravity caused this cloud to collapse in on itself, spin, and flatten into a disk shape. Most of the material in that cloud was pulled toward the center, forming the protostar that would eventually become our Sun. The rest of the material began to come together into clumps called planetesimals. These in turn gradually came together with other planetesimals, forming larger bodies called protoplanets. Earth started as one of these protoplanets, likely about 4.5 billion years ago.\nThe Earth’s history\nAs the proto-Earth grew, heavier elements within it began to sink toward the center, forming the core, and lighter elements rose to the surface. This process, called differentiation, likely took place over tens of millions of years.\nDuring these early stages a Mars-sized protoplanet, often referred to as Theia, collided with the young Earth, ejecting material from both protoplanets into space. Some of this material fell back to Earth, but some of the material eventually coalesced in orbit around Earth to form the Moon.\nThe Earth continued to experience impacts throughout its early life, though none as dramatic as the collision with Theia. During a period called the Late Heavy Bombardment, which likely happened between 4.1 and 3.8 billion years ago, there was an increased rate of asteroid and comet impacts in the inner Solar System. The Late Heavy Bombardment had major geological consequences, including causing Earth’s crust to melt and differentiate and shaping the early atmosphere and oceans. Although geological activity has erased the craters from this time on Earth, they are preserved on the Moon. These are some of the craters you can see from Earth.\nBy about 4.3 billion years ago, the Earth's surface had cooled enough for water vapor in the atmosphere to condense on the surface, leading to the formation of oceans. Volcanic activity, which was more widespread at the time, released gasses that shaped the early atmosphere. Life emerged around 3.5 to 4 billion years ago in the form of simple, single-celled organisms.\nThe Earth has probably been as we know it today — with recognizable continents, oceans, a hospitable climate, and diverse life — for the past few hundred million years. But it continues to evolve through its own gradual tectonic and volcanic activity, and through the more rapid effects of climate change.\nHow do scientists determine the age of the Earth?\nScientists have been able to piece together our planet’s timeline\nthanks to techniques including radiometric dating of rocks and minerals,\nexamining layers of sedimentary rock, and studying the Earth's magnetic\nfield.\nThe most precise method is radiometric dating,\nwhich measures the decay of radioactive isotopes in rocks. Because\ngeologists know how long these isotopes take to decay, they can\ndetermine a rock’s age by looking at the ratio of parent (pre-decay) and\ndaughter (post-decay) isotopes in a sample.\nOne challenge with dating the Earth via rocks is that most of the\noriginal rocks that formed on our planet at the earliest stages of its\ncreation have likely been recycled into the mantle since then. Because\nof this, geologists also learn about the history of the Solar System by\nstudying rocks from beyond Earth, including meteorites that were formed\nbillions of years before falling to Earth, meteorites of Earth material\nthat have been found on the Moon,\nand asteroids that have coasted through space undisturbed for billions\nof years without undergoing any major composition-altering change. The\nasteroid Bennu,\nfor example, is thought to have formed in the first 10 million years of\nthe Solar System’s history. By studying the samples returned to Earth\nby the OSIRIS-REx mission, scientists can learn a lot about the early Solar System.\nEarth's First Line of Defense\nSupport the team of astronomers defending Earth with a gift today.\nDonate", - "images": [], - }, - { - "url": "https://answersingenesis.org/age-of-the-earth/how-old-earth/?srsltid=AfmBOoqSX0LqvRa1nZM5V8YjVoWspP8t9WHAhFQRrQUEVoHW8DukYZf4", - "raw_content": "Published Time: Sept. 1, 2018, 6 a.m.\nHow Old Is the Earth? | Answers in Genesis \n\nInternet Explorer is no longer supported. Try downloading another browser like Chrome or Firefox.\n\nCart\nAccount\nUnited States / English\n\nIf you already have an account, Sign in.\nView Cart\n×\nUnited States / English\n\nAnswers\nStore\nEvents\nVideos\nKids\nEducation\nDonate\n\nSubscribe\n\nAnswers in Genesis\nAnswers\nScience\nGeology\nAge of the Earth\nHow Old Is the Earth?\n\nHow Old Is the Earth?\nIs the earth 4.54 billion years old?\nby Dr. Danny R. Faulkner on September 1, 2018; last featured November 26, 2023\nFeatured in Answers Magazine\nAudio Version\nShare\n\nWatch the video on YouTube.\nIf you ask this question of most scientifically literate people, they will answer that the earth is about 4.54 billion years old. But if you ask biblically literate people, many will answer that the earth is little more than 6,000 years old. Why the huge difference? We look at the same world but come to different conclusions because our worldviews are different.\n Latest Answers -------------- Stay up to date each week with top articles, blogs, news, videos, and more. Sign Up Now\nTo fully understand the issue, we must look beyond this earth. Literally. Let me explain.\nThe Biblical Date\nHow do we arrive at the biblical date? The genealogies of Genesis 5 and 11 make it clear that Abraham lived about 2,000 years after creation. And we know from chronologies found elsewhere in the Bible that Abraham lived about 2,000 years before the birth of Jesus Christ. Furthermore, we know that Jesus’ ministry was about 2,000 years ago. Summing these lengths of time, we get about 6,000 years (technically just a little more).\nWhat Most Scientists Think\nHow do many scientists arrive at the age of 4.54 billion years? They rely on radiometric dating, though the story is a bit more complicated than it sounds. Some rocks contain trace amounts of radioactive atoms. Those radioactive atoms decay into stable atoms over time. By knowing the decay rate and measuring the amount of both kinds of atoms in a rock, scientists can compute the amount of time it took to produce the stable atoms.\nSome assumptions are involved, however. Were some of the stable atoms present in the rock to begin with? Did some of either type of atom leave or enter the rock during the time being measured for decay? To make matters worse, measuring the age of a rock by different kinds of radioactive atoms (such as uranium or rubidium) often yields very different ages. There are many examples of such discordant ages.\nBut even if we accept these ages as correct, there are many other assumptions that cause even more problems. You see, we never find rocks on earth that date back 4.54 billion years. The earth is a very dynamic place, with volcanic eruptions and tectonic plate movements that constantly recycle old rocks into new rocks. When rocks are recycled this way, it is believed that their radiometric dates are reset.\nSo we wouldn’t expect to find the original “primordial” rocks on earth. Instead, scientists must look to other bodies in the solar system that are less active geologically. The search for primordial rocks was one of the scientific reasons we sent men to the moon a half-century ago. Scientists thought that since the moon has far less geological activity than the earth, its rocks would be older.\nIndeed, the moon’s rocks generally yield old radiometric dates, but even they don’t yield dates of 4.54 billion years. Why?\nWhile the moon is far less active than the earth, that hasn’t always been the case. Most scientists think that the moon was very active early in its history. So while moon rocks have relatively old radiometric dates, they aren’t primordial either. To find truly primordial rocks, planetary scientists think they must look at meteorites, debris that has fallen onto the earth’s surface from somewhere else in the solar system. The 4.54-billion-year age of the earth comes from radiometric dating of meteorites.\nHow can they know these are the earliest rocks? They have a theory that the whole solar system formed at the same time, around 4.54 billion years ago. This means the sun and planets would be about the same age. Material that didn’t become part of the sun supposedly coalesced into larger and larger pieces in outer space, eventually forming planets and their satellites, or moons. But many of the pieces never formed into planets or satellites. Fragments of these pieces are thought to be the origin of meteorites. Since meteorites didn’t form into planets, they must have avoided the geological process that reset radiometric ages on earth. This is particularly true of the carbonaceous chondrites.\nBut note all the unproven assumptions. Evolutionary assumptions at that.\nThe Real Agenda\nFor decades, scientists who believe the earth is billions of years old have said that radiometric dates are their reason for believing so. But this hasn’t always been the case. The methods for radiometric dating were developed only a hundred years ago. Prior to that, many scientists already believed the earth was billions of years old, not based upon radiometric dates but the assumption that modern life evolved from nonlife. Evolutionists recognize that we can’t see planets and life evolving before our eyes. They say it requires great time; so the earth must be very old.\nThere is a good lesson here. In the late 1800s, many scientists concluded that the earth must be at least 100 million years old because that was considered the minimum time necessary for evolution to account for the earth’s biology and geology. The need for time drives the claims of ancient dates.\nLord Kelvin, one of the most significant scientists of the 1800s, tested the then-popular age of 100 million years and produced two quantitative tests that showed the earth and sun could be no more than about one-third of this age. Yet his evolutionary colleagues persisted in their belief despite Lord Kelvin’s objections. Since then, many critics have noted that his objections have been explained to their satisfaction. But that misses the point. Many of Kelvin’s colleagues believed in great age despite the evidence, not because of it.\nToday many scientists continue to believe in a 4.54-billion-year-old earth, which evolution requires. They will continue to choose to believe that age, even though solid scientific reasons are available to doubt those dates.\nOur Job\nIt is the job of creation scientists to reevaluate scientific claims using their biblical worldview. Scientists have found many evidences that the earth is far younger than 4.54 billion years—even as young as 6,000 years—but these are usually swept under the rug.\n\nScientists have found many evidences that the earth is far younger than 4.54 billion years, but these are usually swept under the rug.\n\nCreationist literature (including this magazine) is filled with examples. They include the composition of the earth’s atmosphere and seawater, which would be much different if helium had been escaping the atmosphere and salt had been accumulating in the ocean for millions of years. Also, the moon’s tidal interaction with the earth is causing the moon to spiral outward, which limits how long it has been in orbit. (Just go online and search for “evidences of a young earth” for details and more examples.)\nMost importantly, we need to point people to the importance of starting in the right place—God’s Word—when interpreting the evidence.\nDr. Danny R. Faulkner joined the staff of Answers in Genesis after more than 26 years as professor of physics and astronomy at the University of South Carolina Lancaster. He has written numerous articles in astronomical journals, and he is the author of Universe by Design.\nRelated Videos\nWhy Shouldn’t Christians Accept Millions of Years?\n\nAnswers Magazine\nSeptember–October 2018\n\nEven as skepticism spreads around the globe, the creation movement is flourishing. Meet some of the new generation of creation scientists.\nBrowse IssueSubscribe\nRecommended Resources\n\nUniverse by Design$18.99 \nThe Heavens Declare Set$39.99 Sale \n\nAge of the Earth$12.99 \n\nScience\n\nWhat Is Science?\nAstronomy\nBiology\nChemistry\nEnvironmental Science\nFossils\nGenetics\nGeology\nHuman Body\nMathematics\nPhysics\n\nNewsletter\nGet the latest answers emailed to you.\nBy submitting this form, you accept our Privacy Policy and will be given an opportunity to receive emails from Answers in Genesis regarding our latest news, resources, and events.\nThank You!\nThank you for signing up to receive email newsletters from Answers in Genesis.\nYou can also sign up for our free print newsletter (US only).\nFinish your subscription\nYou're almost done! Please follow the instructions we emailed you in order to finish subscribing.\nYou can also sign up for our free print newsletter (US only).\nWhoops!\nYour newsletter signup did not work out. Please refresh the page and try again.\nSupport the creation/gospel message by donating or getting involved! \n\nAnswers in Genesis is an apologetics ministry, dedicated to helping Christians defend their faith and proclaim the good news of Jesus Christ.\nLearn more\n\nCustomer Service 800.778.3390\nAvailable Monday–Friday | 9 AM–5 PM ET\n© 2025 Answers in Genesis", - "images": [], - }, - ], - "failed_results": [], - "response_time": 5.16, - "request_id": "81c3ce85-3014-4a81-b342-fd9f2a9fba32", -} diff --git a/tests/methods/__init__.py b/tests/methods/__init__.py deleted file mode 100644 index e69de29b..00000000 diff --git a/tests/methods/test_method_web.py b/tests/methods/test_method_web.py deleted file mode 100644 index 0d1509e2..00000000 --- a/tests/methods/test_method_web.py +++ /dev/null @@ -1,248 +0,0 @@ -"""Tests for method web.py""" - -import json -import logging - -import pytest -from unittest.mock import patch -from modules.workflows.methods.methodWeb import MethodWeb -from tests.fixtures.tavily_responses import ( - RESPONSE_SEARCH_HOW_OLD_IS_EARTH_NO_ANSWER, - RESPONSE_EXTRACT_HOW_OLD_IS_EARTH_NO_ANSWER, -) - -logger = logging.getLogger(__name__) - - -@pytest.mark.asyncio -@pytest.mark.expensive -async def test_method_web_search_live(): - """Tests method web search with live API calls.""" - - logger.info("=" * 50) - logger.info("==> Test: Method Web Search Live") - - method_web = MethodWeb(serviceCenter=None) - - # Actual request - action_result = await method_web.search( - {"query": "How old is the earth", "maxResults": 5} - ) - - # Evaluate results - assert action_result.success - assert len(action_result.documents) > 0 - - logger.info(f"Action result success status: {action_result.success}") - logger.info(f"Action result error: {action_result.error}") - logger.info(f"Action result label: {action_result.resultLabel}") - - logger.info("Documents:") - for doc in action_result.documents: - logger.info(f" - Document Name: {doc.documentName}") - logger.info(f" --> Document Mime Type: {doc.mimeType}") - logger.info(f" --> Document Data: {doc.documentData}") - - -@pytest.mark.asyncio -async def test_method_web_search_dummy(): - """Tests method web search with dummy response data - no external API calls.""" - - logger.info("=" * 50) - logger.info("==> Test: Method Web Search Dummy") - - method_web = MethodWeb(serviceCenter=None) - - # Mock the Tavily API response - with patch( - "tavily.AsyncTavilyClient.search", - return_value=RESPONSE_SEARCH_HOW_OLD_IS_EARTH_NO_ANSWER, - ) as mock_client: - action_result = await method_web.search( - {"query": "How old is the earth", "maxResults": 5} - ) - mock_client.assert_called_once() - - # Evaluate results - assert action_result.success - assert len(action_result.documents) > 0 - - logger.info(f"Action result success status: {action_result.success}") - logger.info(f"Action result error: {action_result.error}") - logger.info(f"Action result label: {action_result.resultLabel}") - - logger.info("Documents:") - for doc in action_result.documents: - logger.info(f" - Document Name: {doc.documentName}") - logger.info(f" --> Document Mime Type: {doc.mimeType}") - logger.info(f" --> Document Data: {doc.documentData}") - - -@pytest.mark.asyncio -@pytest.mark.expensive -async def test_method_web_crawl_live(): - """Tests method web crawl with live API calls.""" - - logger.info("=" * 50) - logger.info("==> Test: Method Web Crawl Live") - - method_web = MethodWeb(serviceCenter=None) - - # Create mock document data with URLs from search results - search_results_json = { - "documentData": { - "results": [ - {"url": "https://en.wikipedia.org/wiki/Age_of_Earth"}, - {"url": "https://www.planetary.org/articles/how-old-is-the-earth"}, - ] - } - } - - # Mock the service center methods - with patch.object(method_web, "service") as mock_service: - mock_service.getChatDocumentsFromDocumentList.return_value = [ - type("MockDoc", (), {"fileId": "test-file-id", "fileName": "test-search-results.json"})() - ] - mock_service.getFileData.return_value = json.dumps(search_results_json).encode( - "utf-8" - ) - - # Actual request - action_result = await method_web.crawl({"documentList": "test-document-list-ref"}) - - # Evaluate results - assert action_result.success - assert len(action_result.documents) > 0 - - logger.info(f"Action result success status: {action_result.success}") - logger.info(f"Action result error: {action_result.error}") - logger.info(f"Action result label: {action_result.resultLabel}") - - logger.info("Documents:") - for doc in action_result.documents: - logger.info(f" - Document Name: {doc.documentName}") - logger.info(f" --> Document Mime Type: {doc.mimeType}") - logger.info(f" --> Document Data: {doc.documentData}") - - -@pytest.mark.asyncio -async def test_method_web_crawl_dummy(): - """Tests method web crawl with dummy response data - no external API calls.""" - - logger.info("=" * 50) - logger.info("==> Test: Method Web Crawl Dummy") - - method_web = MethodWeb(serviceCenter=None) - - # Create mock document data with URLs from search results - search_results_json = { - "documentData": { - "results": [ - {"url": "https://en.wikipedia.org/wiki/Age_of_Earth"}, - {"url": "https://www.planetary.org/articles/how-old-is-the-earth"}, - ] - } - } - - # Mock both the service center and Tavily API - with ( - patch.object(method_web, "service") as mock_service, - patch( - "tavily.AsyncTavilyClient.extract", - return_value=RESPONSE_EXTRACT_HOW_OLD_IS_EARTH_NO_ANSWER, - ) as mock_client, - ): - mock_service.getChatDocumentsFromDocumentList.return_value = [ - type("MockDoc", (), {"fileId": "test-file-id", "fileName": "test-search-results.json"})() - ] - mock_service.getFileData.return_value = json.dumps(search_results_json).encode( - "utf-8" - ) - - action_result = await method_web.crawl({"documentList": "test-document-list-ref"}) - mock_client.assert_called_once() - - # Evaluate results - assert action_result.success - assert len(action_result.documents) > 0 - - logger.info(f"Action result success status: {action_result.success}") - logger.info(f"Action result error: {action_result.error}") - logger.info(f"Action result label: {action_result.resultLabel}") - - logger.info("Documents:") - for doc in action_result.documents: - logger.info(f" - Document Name: {doc.documentName}") - logger.info(f" --> Document Mime Type: {doc.mimeType}") - logger.info(f" --> Document Data: {doc.documentData}") - - -@pytest.mark.asyncio -@pytest.mark.expensive -async def test_method_web_scrape_live(): - """Tests method web scrape with live API calls.""" - - logger.info("=" * 50) - logger.info("==> Test: Method Web Scrape Live") - - method_web = MethodWeb(serviceCenter=None) - - # Actual request - action_result = await method_web.scrape( - {"query": "How old is the earth", "maxResults": 3} - ) - - # Evaluate results - assert action_result.success - assert len(action_result.documents) > 0 - - logger.info(f"Action result success status: {action_result.success}") - logger.info(f"Action result error: {action_result.error}") - logger.info(f"Action result label: {action_result.resultLabel}") - - logger.info("Documents:") - for doc in action_result.documents: - logger.info(f" - Document Name: {doc.documentName}") - logger.info(f" --> Document Mime Type: {doc.mimeType}") - logger.info(f" --> Document Data: {doc.documentData}") - - -@pytest.mark.asyncio -async def test_method_web_scrape_dummy(): - """Tests method web scrape with dummy response data - no external API calls.""" - - logger.info("=" * 50) - logger.info("==> Test: Method Web Scrape Dummy") - - method_web = MethodWeb(serviceCenter=None) - - # Mock both Tavily API responses (search + extract) - with ( - patch( - "tavily.AsyncTavilyClient.search", - return_value=RESPONSE_SEARCH_HOW_OLD_IS_EARTH_NO_ANSWER, - ) as mock_search, - patch( - "tavily.AsyncTavilyClient.extract", - return_value=RESPONSE_EXTRACT_HOW_OLD_IS_EARTH_NO_ANSWER, - ) as mock_extract, - ): - action_result = await method_web.scrape( - {"query": "How old is the earth", "maxResults": 3} - ) - mock_search.assert_called_once() - mock_extract.assert_called_once() - - # Evaluate results - assert action_result.success - assert len(action_result.documents) > 0 - - logger.info(f"Action result success status: {action_result.success}") - logger.info(f"Action result error: {action_result.error}") - logger.info(f"Action result label: {action_result.resultLabel}") - - logger.info("Documents:") - for doc in action_result.documents: - logger.info(f" - Document Name: {doc.documentName}") - logger.info(f" --> Document Mime Type: {doc.mimeType}") - logger.info(f" --> Document Data: {doc.documentData}") diff --git a/tests/test_graph_search.py b/tests/test_graph_search.py deleted file mode 100644 index 981aa778..00000000 --- a/tests/test_graph_search.py +++ /dev/null @@ -1,311 +0,0 @@ -#!/usr/bin/env python3 -""" -Simple test script for Microsoft Graph Search API -Tests folder search queries directly -""" - -import requests -import json -import sys -import os - -# Add the gateway modules to the path -sys.path.append(os.path.dirname(os.path.abspath(__file__))) - -def test_graph_folders_direct(access_token): - """Test direct Microsoft Graph API call to list folders""" - print("🔍 Testing direct Graph API folder listing...") - - # Try to list folders from the main site - need to get site ID first - # Let's try to find the site by name first - url = "https://graph.microsoft.com/v1.0/sites/pcuster.sharepoint.com:/sites/SSSRESYNachfolge:/drive/root/children" - - headers = { - "Authorization": f"Bearer {access_token}", - "Content-Type": "application/json" - } - - try: - response = requests.get(url, headers=headers) - - if response.status_code == 200: - data = response.json() - items = data.get('value', []) - print(f"✅ SUCCESS - Found {len(items)} items in root") - - folders = [] - files = [] - - for item in items: - if 'folder' in item: - folders.append(item) - elif 'file' in item: - files.append(item) - - print(f" 📁 Folders: {len(folders)}") - print(f" 📄 Files: {len(files)}") - - if folders: - print("\n📁 FOLDERS found:") - for i, folder in enumerate(folders[:5], 1): - name = folder.get('name', 'No name') - web_url = folder.get('webUrl', 'No URL') - print(f" {i}. {name}") - print(f" URL: {web_url}") - print() - - else: - print(f"❌ ERROR - Status {response.status_code}") - print(f"Error: {response.text[:200]}") - - except Exception as e: - print(f"Exception: {str(e)}") - -def test_graph_search(access_token, query_string): - """Test a Microsoft Graph Search API query and show resulting paths""" - - url = "https://graph.microsoft.com/v1.0/search/query" - - headers = { - "Authorization": f"Bearer {access_token}", - "Content-Type": "application/json" - } - - payload = { - "requests": [ - { - "entityTypes": ["driveItem"], - "query": { - "queryString": query_string - }, - "from": 0, - "size": 50 - } - ] - } - - print(f"Testing: {query_string}") - print("-" * 50) - - try: - response = requests.post(url, headers=headers, json=payload) - - if response.status_code == 200: - data = response.json() - - # Extract useful info - if "value" in data and len(data["value"]) > 0: - hits = data["value"][0].get("hitsContainers", []) - if hits: - total = hits[0].get("total", 0) - results = hits[0].get("hits", []) - print(f"✅ SUCCESS - Found {total} results") - - # First, let's see what types of results we're getting - print(f"📊 Analyzing {len(results)} results...") - - # Count different types of results with better detection - file_count = 0 - folder_count = 0 - other_count = 0 - - # Debug: Let's see what the actual resource structure looks like - if results: - print("🔍 DEBUG: First result structure:") - first_result = results[0] - print(f" Keys: {list(first_result.keys())}") - if 'resource' in first_result: - resource = first_result['resource'] - print(f" Resource keys: {list(resource.keys())}") - if 'folder' in resource: - print(f" Folder info: {resource['folder']}") - if 'file' in resource: - print(f" File info: {resource['file']}") - print() - - for result in results: - resource = result.get('resource', {}) - - # Better detection logic - is_folder = False - is_file = False - - # Check for explicit folder/file indicators - if 'folder' in resource: - is_folder = True - elif 'file' in resource: - is_file = True - else: - # Try to detect by URL pattern or other indicators - web_url = resource.get('webUrl', '') - name = resource.get('name', '') - - # Check if URL ends with a file extension (likely a file) - if '.' in name and any(name.lower().endswith(ext) for ext in ['.pdf', '.docx', '.xlsx', '.pptx', '.txt', '.cs', '.py', '.js', '.html', '.css']): - is_file = True - # Check if URL has no file extension and looks like a folder path - elif '.' not in name and ('/' in web_url or '\\' in web_url): - is_folder = True - - if is_folder: - folder_count += 1 - elif is_file: - file_count += 1 - else: - other_count += 1 - - print(f" 📄 Files: {file_count}") - print(f" 📁 Folders: {folder_count}") - print(f" ❓ Other: {other_count}") - print() - - # Show sample results regardless of type - print(f"📋 Sample results (showing first 5):") - for i, result in enumerate(results[:5], 1): - resource = result.get('resource', {}) - web_url = resource.get('webUrl', 'No URL') - name = resource.get('name', 'No name') - - # Determine type using same logic as counting - is_folder = False - is_file = False - - if 'folder' in resource: - is_folder = True - elif 'file' in resource: - is_file = True - else: - # Try to detect by URL pattern or other indicators - web_url = resource.get('webUrl', '') - name = resource.get('name', '') - - # Check if URL ends with a file extension (likely a file) - if '.' in name and any(name.lower().endswith(ext) for ext in ['.pdf', '.docx', '.xlsx', '.pptx', '.txt', '.cs', '.py', '.js', '.html', '.css']): - is_file = True - # Check if URL has no file extension and looks like a folder path - elif '.' not in name and ('/' in web_url or '\\' in web_url): - is_folder = True - - if is_folder: - item_type = "📁 FOLDER" - elif is_file: - file_info = resource.get('file', {}) - mime_type = file_info.get('mimeType', 'Unknown type') if file_info else 'Detected by extension' - item_type = f"📄 FILE ({mime_type})" - else: - item_type = "❓ UNKNOWN" - - # Extract path from webUrl - if '/sites/SSSRESYNachfolge/' in web_url: - path_part = web_url.split('/sites/SSSRESYNachfolge/')[-1] - path_with_backslashes = path_part.replace('/', '\\') - display_path = f"\\{path_with_backslashes}" - else: - display_path = web_url - - print(f" {i}. {item_type} - {name}") - print(f" Path: {display_path}") - print(f" URL: {web_url}") - print() - - if len(results) > 5: - print(f" ... and {len(results) - 5} more results") - - # Now filter and show only FOLDER results if any exist - folder_results = [] - for result in results: - resource = result.get('resource', {}) - - # Use the same detection logic as counting - is_folder = False - if 'folder' in resource: - is_folder = True - else: - # Try to detect by URL pattern or other indicators - web_url = resource.get('webUrl', '') - name = resource.get('name', '') - - # Check if URL has no file extension and looks like a folder path - if '.' not in name and ('/' in web_url or '\\' in web_url): - is_folder = True - - if is_folder: - folder_results.append(result) - - if folder_results: - print(f"\n📁 FOLDER DETAILS ({len(folder_results)} folders found):") - for i, result in enumerate(folder_results, 1): - web_url = result.get('resource', {}).get('webUrl', 'No URL') - name = result.get('resource', {}).get('name', 'No name') - - if '/sites/SSSRESYNachfolge/' in web_url: - path_part = web_url.split('/sites/SSSRESYNachfolge/')[-1] - path_with_backslashes = path_part.replace('/', '\\') - folder_path = f"\\{path_with_backslashes}" - else: - folder_path = web_url - - print(f" {i}. 📁 {name}") - print(f" Path: {folder_path}") - print(f" URL: {web_url}") - print() - else: - print(f"\n❌ No folders found in results - all {total} results are files or other types") - else: - print("❌ SUCCESS but no hits containers found") - else: - print("❌ SUCCESS but no value array in response") - - else: - print(f"❌ ERROR - Status {response.status_code}") - error_text = response.text[:200] + "..." if len(response.text) > 200 else response.text - print(f"Error: {error_text}") - - except Exception as e: - print(f"Exception: {str(e)}") - -def main(): - """Main test function""" - - # Use the access token from the database - access_token = "eyJ0eXAiOiJKV1QiLCJub25jZSI6IkxwTjBjTXo2SGlja2ZPLUpnekRwTFE1QktfQmVOWHBwRWZ2UzZBMDh2REUiLCJhbGciOiJSUzI1NiIsIng1dCI6IkpZaEFjVFBNWl9MWDZEQmxPV1E3SG4wTmVYRSIsImtpZCI6IkpZaEFjVFBNWl9MWDZEQmxPV1E3SG4wTmVYRSJ9.eyJhdWQiOiIwMDAwMDAwMy0wMDAwLTAwMDAtYzAwMC0wMDAwMDAwMDAwMDAiLCJpc3MiOiJodHRwczovL3N0cy53aW5kb3dzLm5ldC82YTUxYWFlYi0yNDY3LTQxODYtOTUwNC0yYTA1YWVkYzU5MWYvIiwiaWF0IjoxNzU3MDEwNTc0LCJuYmYiOjE3NTcwMTA1NzQsImV4cCI6MTc1NzAxNTQ1MSwiYWNjdCI6MCwiYWNyIjoiMSIsImFjcnMiOlsicDEiXSwiYWlvIjoiQVpRQWEvOFpBQUFBcU0xNVFOMkhaQld5QXNsbStiT0QzbzRuU1RhUzg5bGdTV3ZUQVZvYVhqcUhlT1VaNFE1aEh0bE51WUdxelEvM0tDRnZlZktycU1HTUp2VmlVaWVibUhjbnBtL0FaRFA1Sk1YNnI4c1FCSVdLVTZPY29sUUNuOWpvcVZLb1VIOFl3WTJhM3picTlkeGdqVC94dU5NaCtKcXhMV1JMdEUrUjBZeGl0c3J0QXhpd0pRaGZmalIzK0xPSGtmVkxhOExaIiwiYW1yIjpbInB3ZCIsIm1mYSJdLCJhcHBfZGlzcGxheW5hbWUiOiJQb3dlck9uIEFwcCIsImFwcGlkIjoiYzdlNzExMmQtNjFkYy00ZjNhLThjZDMtMDhjYzRjZDc1MDRjIiwiYXBwaWRhY3IiOiIxIiwiZmFtaWx5X25hbWUiOiJNb3RzY2giLCJnaXZlbl9uYW1lIjoiUGF0cmljayIsImlkdHlwIjoidXNlciIsImlwYWRkciI6IjE3OC4xOTcuMjE4LjQ4IiwibmFtZSI6IlBhdHJpY2sgTW90c2NoIiwib2lkIjoiN2QwOGFhYjktYTE3MC00OTc1LTg4OTgtYmM3ZTBhOTU0ODhlIiwicGxhdGYiOiIzIiwicHVpZCI6IjEwMDM3RkZFOENERDZBODIiLCJyaCI6IjEuQVFzQTY2cFJhbWNraGtHVkJDb0ZydHhaSHdNQUFBQUFBQUFBd0FBQUFBQUFBQUNFQURBTEFBLiIsInNjcCI6IkZpbGVzLlJlYWRXcml0ZS5BbGwgTWFpbC5SZWFkV3JpdGUgTWFpbC5SZWFkV3JpdGUuU2hhcmVkIE1haWwuU2VuZCBvcGVuaWQgcHJvZmlsZSBTaXRlcy5SZWFkV3JpdGUuQWxsIFVzZXIuUmVhZCBlbWFpbCIsInNpZCI6IjAwNmY5Mjk5LTY3ZDUtYmU3Zi1kYWI4LWQwYTBlZTI1MTBkNiIsInNpZ25pbl9zdGF0ZSI6WyJrbXNpIl0sInN1YiI6IklnMGlwM3hhZGJMaXVLemJGZ3dWaE5JTV9Eekcwd3B4aUVGYjJKWXVjbjQiLCJ0ZW5hbnRfcmVnaW9uX3Njb3BlIjoiRVUiLCJ0aWQiOiI2YTUxYWFlYi0yNDY3LTQxODYtOTUwNC0yYTA1YWVkYzU5MWYiLCJ1bmlxdWVfbmFtZSI6InAubW90c2NoQHZhbHVlb24uY2giLCJ1cG4iOiJwLm1vdHNjaEB2YWx1ZW9uLmNoIiwidXRpIjoieTh5ZGhEcWRDMG1nVTBpLV94azFBUSIsInZlciI6IjEuMCIsIndpZHMiOlsiOWI4OTVkOTItMmNkMy00NGM3LTlkMDItYTZhYzJkNWVhNWMzIiwiY2YxYzM4ZTUtMzYyMS00MDA0LWE3Y2ItODc5NjI0ZGNlZDdjIiwiMTU4YzA0N2EtYzkwNy00NTU2LWI3ZWYtNDQ2NTUxYTZiNWY3IiwiODkyYzU4NDItYTlhNi00NjNhLTgwNDEtNzJhYTA4Y2EzY2Y2IiwiOWYwNjIwNGQtNzNjMS00ZDRjLTg4MGEtNmVkYjkwNjA2ZmQ4IiwiYjc5ZmJmNGQtM2VmOS00Njg5LTgxNDMtNzZiMTk0ZTg1NTA5Il0sInhtc19mdGQiOiIwcEZ4RVctQnl6Y3M5UW5HdXNDbU1Ka1V4MHNQWlEzOUkzWUwxRGZJdnpzQmMzZGxaR1Z1WXkxa2MyMXoiLCJ4bXNfaWRyZWwiOiIxIDI0IiwieG1zX3N0Ijp7InN1YiI6IlIydkQwRzFtbWFZUkM3SllXY0lTWlcyS0RQZ05CakJMRmw2ZUxBQl9QVU0ifSwieG1zX3RjZHQiOjE0MTgyMTQ1MDEsInhtc190ZGJyIjoiRVUifQ.JYEWH2YxBrgWSn-9WN3BixJ91q19RGd0U7HgiiLpmwKUicft8zrovO8wKVU5rkly6CBcEO_eGAvyqQHSjFLHXKGDrutrFVdLTLB0vUu3J1Lkw31CiJF_y6Y3r2VytOF8evcYwh_Ye-5eoAxIr5avR8j_T51RPkLG53QSJ-tA5utDgHGWa65T5-mmeZxI-ThYxfyLori1uS8TSchJBdwrWwv8pkklHn6lZrFfgiuviRjLrOOLVUL_fzIod_eOKjo31YHhUzfm-QD3vvQkqnWNcdQ4D0UaTxKW291fHFafQZ9SkH9m0BD9nn56QBqijUBhvA8qMZC_cObb3DpR0GR_xA" - - print("=" * 60) - print("Microsoft Graph API Test Suite") - print("=" * 60) - - # First test: Direct folder listing (should work better than search) - print("\nTEST 0: Direct Graph API folder listing") - test_graph_folders_direct(access_token) - - # Test different query types to find both files and folders - test_queries = [ - # Test 1: Test with Venus folder (empty folder created for testing) - "Venus", - - # Test 2: Folder-specific searches for Venus - "kind:folder AND Venus", - - # Test 3: Original specific query (found 8 results - all files) - "Druckersteuerung AND Eskalation AND Logobject", - - # Test 4: Broader folder-focused queries - "Druckersteuerung", - "Eskalation", - "Logobject", - - # Test 5: Folder-specific searches - "kind:folder AND Druckersteuerung", - "kind:folder AND Eskalation", - - # Test 6: General folder search to see what folders exist - "kind:folder", - ] - - for i, query in enumerate(test_queries, 1): - print(f"\nTEST {i}: {query}") - test_graph_search(access_token, query) - print() - -if __name__ == "__main__": - main() diff --git a/tests/test_neutralizer/apprun.py b/tests/test_neutralizer/apprun.py deleted file mode 100644 index 609124f6..00000000 --- a/tests/test_neutralizer/apprun.py +++ /dev/null @@ -1,263 +0,0 @@ -""" -DSGVO-konformer Daten-Neutralisierer für KI-Agentensysteme -Unterstützt TXT, JSON, CSV, Excel und Word-Dateien -Mehrsprachig: DE, EN, FR, IT -""" - -import os -import shutil -import logging -import csv -import json -import pandas as pd -from datetime import datetime -from pathlib import Path -from neutralizer import DataAnonymizer -import traceback - -# Define directories -SCRIPT_DIR = Path(__file__).parent -INPUT_DIR = SCRIPT_DIR / 'input' -OUTPUT_DIR = SCRIPT_DIR / 'output' -LOG_DIR = SCRIPT_DIR / 'logs' -LOG_MAPPING = LOG_DIR / 'log_mapping.csv' -LOG_REPLACEMENTS = LOG_DIR / 'log_replacements.csv' - -# Configure logging -logging.basicConfig( - level=logging.DEBUG, - format='%(asctime)s - %(name)s - %(levelname)s - %(message)s', - handlers=[ - logging.StreamHandler() - ] -) -logger = logging.getLogger(__name__) - -def setup_directories(): - """Setup output and log directories""" - # Close any existing log handlers - for handler in logger.handlers[:]: - handler.close() - logger.removeHandler(handler) - - # Clear and recreate output directory - output_dir = Path("output") - if output_dir.exists(): - shutil.rmtree(output_dir) - output_dir.mkdir() - logger.info(f"Output directory '{output_dir}' created") - - # Clear and recreate logs directory - log_dir = Path("logs") - if log_dir.exists(): - shutil.rmtree(log_dir) - log_dir.mkdir() - logger.info(f"Log directory '{log_dir}' created") - - # Create log files - mapping_log = log_dir / "log_mapping.csv" - replacements_log = log_dir / "log_replacements.csv" - - # Create headers for mapping log - with open(mapping_log, 'w', newline='', encoding='utf-8') as f: - writer = csv.writer(f) - writer.writerow(['timestamp', 'success', 'file_name', 'replaced_fields', 'content_type', 'headers', 'row_count']) - - # Create headers for replacements log - with open(replacements_log, 'w', newline='', encoding='utf-8') as f: - writer = csv.writer(f) - writer.writerow(['timestamp', 'success', 'file_name', 'original', 'replacement']) - - # Reconfigure logging with new log file - for handler in logger.handlers[:]: - handler.close() - logger.removeHandler(handler) - - file_handler = logging.FileHandler(LOG_DIR / 'app.log', encoding='utf-8') - file_handler.setFormatter(logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s')) - logger.addHandler(file_handler) - - return output_dir, log_dir - -def log_mapping(log_dir: Path, file_name: str, success: bool, replaced_fields: list): - """Log mapping information""" - try: - with open(log_dir / "log_mapping.csv", 'a', newline='', encoding='utf-8') as f: - writer = csv.writer(f) - writer.writerow([ - datetime.now().isoformat(), - file_name, - success, - ';'.join(replaced_fields) if replaced_fields else '', - 'unknown' - ]) - except Exception as e: - logger.error(f"Error logging mapping: {str(e)}") - -def log_replacements(log_dir: Path, file_name: str, mapping: dict): - """Log replacement information""" - try: - with open(log_dir / "log_replacements.csv", 'a', newline='', encoding='utf-8') as f: - writer = csv.writer(f) - for original, replacement in mapping.items(): - writer.writerow([ - datetime.now().isoformat(), - file_name, - original, - replacement - ]) - except Exception as e: - logger.error(f"Error logging replacements: {str(e)}") - -def save_anonymized_data(data: any, output_path: Path, file_type: str): - """Save anonymized data to file""" - try: - if file_type == '.csv': - if isinstance(data, pd.DataFrame): - data.to_csv(output_path, index=False, encoding='utf-8') - else: - raise ValueError("Data must be a DataFrame for CSV output") - elif file_type == '.json': - if isinstance(data, pd.DataFrame): - data.to_json(output_path, orient='records', force_ascii=False) - else: - with open(output_path, 'w', encoding='utf-8') as f: - json.dump(data, f, ensure_ascii=False, indent=2) - elif file_type == '.xml': - if isinstance(data, str): - with open(output_path, 'w', encoding='utf-8') as f: - f.write(data) - else: - raise ValueError("Data must be a string for XML output") - elif file_type in ['.txt', '.docx']: - if isinstance(data, str): - with open(output_path, 'w', encoding='utf-8') as f: - f.write(data) - else: - raise ValueError("Data must be a string for text output") - else: - raise ValueError(f"Unsupported file type: {file_type}") - except Exception as e: - logger.error(f"Error saving anonymized data to {output_path}: {str(e)}") - raise - -def read_file_content(file_path: Path) -> tuple[str, str]: - """Read file content and determine content type""" - try: - file_type = file_path.suffix.lower() - if file_type == '.docx': - import docx - doc = docx.Document(file_path) - content = '\n'.join(para.text for para in doc.paragraphs) - else: - with open(file_path, 'r', encoding='utf-8') as f: - content = f.read() - return content, file_type - except Exception as e: - logger.error(f"Error reading file {file_path}: {str(e)}") - raise - -def process_file(file_path: Path, anonymizer: DataAnonymizer) -> bool: - """Process a single file and save anonymized version""" - try: - # Read file content - content = file_path.read_text(encoding='utf-8') - - # Process content - result = anonymizer.process_content(content, file_path.suffix[1:]) - - if result.data is None: - logger.error(f"Failed to process {file_path.name}") - return False - - # Save anonymized content with neutralized_ prefix - output_path = OUTPUT_DIR / f"neutralized_{file_path.name}" - if file_path.suffix.lower() == '.csv': - result.data.to_csv(output_path, index=False) - elif file_path.suffix.lower() == '.json': - with open(output_path, 'w', encoding='utf-8') as f: - json.dump(result.data, f, indent=2) - elif file_path.suffix.lower() == '.xml': - with open(output_path, 'w', encoding='utf-8') as f: - f.write(result.data) - else: - # For text files, preserve original whitespace - with open(output_path, 'w', encoding='utf-8') as f: - f.write(result.data) - - # Log processing details - timestamp = datetime.now().isoformat() - success = True - - # Create detailed log entry - log_entry = { - 'timestamp': timestamp, - 'success': success, - 'file_name': file_path.name, - 'replaced_fields': ';'.join(set(result.replaced_fields)), # Use set to remove duplicates - 'content_type': result.processed_info.get('type', 'unknown') - } - - # Add type-specific details - if result.processed_info['type'] == 'table': - log_entry.update({ - 'headers': ';'.join(result.processed_info['headers']), - 'row_count': result.processed_info['row_count'] - }) - elif result.processed_info['type'] == 'text': - tables = result.processed_info.get('tables', []) - text_sections = result.processed_info.get('text_sections', []) - log_entry.update({ - 'headers': '', # Empty for text files - 'row_count': sum(s['length'] for s in text_sections) # Total text length - }) - - # Write to log file - with open(LOG_MAPPING, 'a', newline='', encoding='utf-8') as f: - writer = csv.DictWriter(f, fieldnames=log_entry.keys()) - if f.tell() == 0: # Write header if file is empty - writer.writeheader() - writer.writerow(log_entry) - - # Log replacements - for original, replacement in result.mapping.items(): - with open(LOG_REPLACEMENTS, 'a', newline='', encoding='utf-8') as f: - writer = csv.writer(f) - writer.writerow([timestamp, success, file_path.name, original, replacement]) - - return True - - except Exception as e: - logger.error(f"Error processing {file_path.name}: {str(e)}") - logger.debug(traceback.format_exc()) - return False - -def main(): - # Setup directories - setup_directories() - - # Initialize anonymizer - anonymizer = DataAnonymizer() - - # Process all files - logger.info("Starting file processing...") - testdata_dir = Path("testdata") - - for file_path in testdata_dir.glob("*.*"): - try: - logger.info(f"Processing file: {file_path.name}") - - # Process file - if process_file(file_path, anonymizer): - logger.info(f"Anonymization completed for {file_path.name}") - else: - logger.error(f"Error processing {file_path.name}") - - except Exception as e: - logger.error(f"Error processing {file_path.name}: {str(e)}") - continue - - logger.info("Processing completed!") - -if __name__ == "__main__": - main() \ No newline at end of file diff --git a/tests/test_neutralizer/logs/log_mapping.csv b/tests/test_neutralizer/logs/log_mapping.csv deleted file mode 100644 index 0816acc3..00000000 --- a/tests/test_neutralizer/logs/log_mapping.csv +++ /dev/null @@ -1,17 +0,0 @@ -timestamp,success,file_name,replaced_fields,content_type,headers,row_count -2025-06-07T18:12:08.019661,True,Case.md,,text,,0 -2025-06-07T18:12:08.040653,True,customers.csv,ahv_number;phone;credit_card;address;name;iban;email,table,id;name;email;phone;address;iban;credit_card;ahv_number,5 -2025-06-07T18:12:08.064201,True,cv_lara_meier.txt,,text,,0 -2025-06-07T18:12:08.080961,True,employees.csv,bank_account;first_name;last_name;office_address;phone;uid_number;email,table,employee_id;first_name;last_name;email;phone;department;office_address;uid_number;bank_account,5 -2025-06-07T18:12:08.101660,True,english.txt,,text,,0 -2025-06-07T18:12:08.125634,True,example.json,,json -2025-06-07T18:12:08.157397,True,example.xml,,xml -2025-06-07T18:12:08.178030,True,french.txt,,text,,0 -2025-06-07T18:12:08.201071,True,german.txt,,text,,0 -2025-06-07T18:12:08.228652,True,geschaeftsstrategie.txt,,text,,0 -2025-06-07T18:12:08.255652,True,geschäfte.csv,datum;kundenname;kundenemail;zahlungsdetails;lieferadresse,table,geschäft_id;datum;kundenname;kundenemail;betrag;zahlungsmethode;zahlungsdetails;lieferadresse,5 -2025-06-07T18:12:08.284172,True,italian.txt,,text,,0 -2025-06-07T18:12:08.314527,True,kunden.csv,kreditkarte;vorname;adresse;telefon;iban;steuernummer;email;nachname,table,kunden_id;vorname;nachname;email;telefon;adresse;iban;kreditkarte;steuernummer,5 -2025-06-07T18:12:08.349750,True,mitarbeiter.csv,vorname;büroadresse;telefon;steuernummer;email;sozialversicherungsnummer;nachname,table,mitarbeiter_id;vorname;nachname;email;telefon;abteilung;büroadresse;steuernummer;sozialversicherungsnummer,5 -2025-06-07T18:12:08.389324,True,swiss.txt,,text,,0 -2025-06-07T18:12:08.431916,True,transactions.csv,date;customer_email;payment_details;customer_name;shipping_address,table,transaction_id;date;customer_name;customer_email;amount;payment_method;payment_details;shipping_address,5 diff --git a/tests/test_neutralizer/logs/log_replacements.csv b/tests/test_neutralizer/logs/log_replacements.csv deleted file mode 100644 index ce5a4702..00000000 --- a/tests/test_neutralizer/logs/log_replacements.csv +++ /dev/null @@ -1,1947 +0,0 @@ -timestamp,success,file_name,original,replacement -2025-06-07T18:12:08.019661,True,Case.md,000-180.000,[PHONE_1] -2025-06-07T18:12:08.019661,True,Case.md,2027 Bewertung,[ADDRESS_2] -2025-06-07T18:12:08.040653,True,customers.csv,000-180.000,[PHONE_1] -2025-06-07T18:12:08.040653,True,customers.csv,2027 Bewertung,[ADDRESS_2] -2025-06-07T18:12:08.040653,True,customers.csv,Max Mustermann,[NAME_3] -2025-06-07T18:12:08.040653,True,customers.csv,Peter Schmid,[NAME_4] -2025-06-07T18:12:08.040653,True,customers.csv,Marie Dupont,[NAME_5] -2025-06-07T18:12:08.040653,True,customers.csv,Marco Rossi,[NAME_6] -2025-06-07T18:12:08.040653,True,customers.csv,John Smith,[NAME_7] -2025-06-07T18:12:08.040653,True,customers.csv,max.mustermann@beispiel.de,[EMAIL_8] -2025-06-07T18:12:08.040653,True,customers.csv,peter.schmid@beispiel.ch,[EMAIL_9] -2025-06-07T18:12:08.040653,True,customers.csv,marie.dupont@exemple.fr,[EMAIL_10] -2025-06-07T18:12:08.040653,True,customers.csv,marco.rossi@esempio.it,[EMAIL_11] -2025-06-07T18:12:08.040653,True,customers.csv,john.smith@example.com,[EMAIL_12] -2025-06-07T18:12:08.040653,True,customers.csv,+49 30 12345678,[PHONE_13] -2025-06-07T18:12:08.040653,True,customers.csv,+41 44 123 45 67,[PHONE_14] -2025-06-07T18:12:08.040653,True,customers.csv,+33 1 23 45 67 89,[PHONE_15] -2025-06-07T18:12:08.040653,True,customers.csv,+39 02 1234 5678,[PHONE_16] -2025-06-07T18:12:08.040653,True,customers.csv,+44 20 1234 5678,[PHONE_17] -2025-06-07T18:12:08.040653,True,customers.csv,Musterstraße 123 12345 Berlin,[ADDRESS_18] -2025-06-07T18:12:08.040653,True,customers.csv,Bahnhofstrasse 1 8001 Zürich,[ADDRESS_19] -2025-06-07T18:12:08.040653,True,customers.csv,123 Rue de Paris 75001 Paris,[ADDRESS_20] -2025-06-07T18:12:08.040653,True,customers.csv,Via Roma 123 20100 Milano,[ADDRESS_21] -2025-06-07T18:12:08.040653,True,customers.csv,123 High Street London SW1A 1AA,[ADDRESS_22] -2025-06-07T18:12:08.040653,True,customers.csv,DE89 3704 0044 0532 0130 00,[IBAN_23] -2025-06-07T18:12:08.040653,True,customers.csv,CH93 0076 7000 E529 3557 7,[IBAN_24] -2025-06-07T18:12:08.040653,True,customers.csv,FR76 3000 6000 0112 3456 7890 189,[IBAN_25] -2025-06-07T18:12:08.040653,True,customers.csv,IT60 X054 2811 1010 0000 0123 456,[IBAN_26] -2025-06-07T18:12:08.040653,True,customers.csv,GB29 NWBK 6016 1331 9268 19,[IBAN_27] -2025-06-07T18:12:08.040653,True,customers.csv,4532 1234 5678 9012,[IBAN_28] -2025-06-07T18:12:08.040653,True,customers.csv,4532 1234 5678 9013,[IBAN_29] -2025-06-07T18:12:08.040653,True,customers.csv,4532 1234 5678 9014,[IBAN_30] -2025-06-07T18:12:08.040653,True,customers.csv,4532 1234 5678 9015,[IBAN_31] -2025-06-07T18:12:08.040653,True,customers.csv,4532 1234 5678 9016,[IBAN_32] -2025-06-07T18:12:08.040653,True,customers.csv,nan,[SSN_33] -2025-06-07T18:12:08.040653,True,customers.csv,756.1234.5678.90,[SSN_34] -2025-06-07T18:12:08.040653,True,customers.csv, ,[SSN_35] -2025-06-07T18:12:08.064201,True,cv_lara_meier.txt,000-180.000,[PHONE_1] -2025-06-07T18:12:08.064201,True,cv_lara_meier.txt,2027 Bewertung,[ADDRESS_2] -2025-06-07T18:12:08.064201,True,cv_lara_meier.txt,Max Mustermann,[NAME_3] -2025-06-07T18:12:08.064201,True,cv_lara_meier.txt,Peter Schmid,[NAME_4] -2025-06-07T18:12:08.064201,True,cv_lara_meier.txt,Marie Dupont,[NAME_5] -2025-06-07T18:12:08.064201,True,cv_lara_meier.txt,Marco Rossi,[NAME_6] -2025-06-07T18:12:08.064201,True,cv_lara_meier.txt,John Smith,[NAME_7] -2025-06-07T18:12:08.064201,True,cv_lara_meier.txt,max.mustermann@beispiel.de,[EMAIL_8] -2025-06-07T18:12:08.064201,True,cv_lara_meier.txt,peter.schmid@beispiel.ch,[EMAIL_9] -2025-06-07T18:12:08.064201,True,cv_lara_meier.txt,marie.dupont@exemple.fr,[EMAIL_10] -2025-06-07T18:12:08.064201,True,cv_lara_meier.txt,marco.rossi@esempio.it,[EMAIL_11] -2025-06-07T18:12:08.064201,True,cv_lara_meier.txt,john.smith@example.com,[EMAIL_12] -2025-06-07T18:12:08.064201,True,cv_lara_meier.txt,+49 30 12345678,[PHONE_13] -2025-06-07T18:12:08.064201,True,cv_lara_meier.txt,+41 44 123 45 67,[PHONE_14] -2025-06-07T18:12:08.064201,True,cv_lara_meier.txt,+33 1 23 45 67 89,[PHONE_15] -2025-06-07T18:12:08.064201,True,cv_lara_meier.txt,+39 02 1234 5678,[PHONE_16] -2025-06-07T18:12:08.064201,True,cv_lara_meier.txt,+44 20 1234 5678,[PHONE_17] -2025-06-07T18:12:08.064201,True,cv_lara_meier.txt,Musterstraße 123 12345 Berlin,[ADDRESS_18] -2025-06-07T18:12:08.064201,True,cv_lara_meier.txt,Bahnhofstrasse 1 8001 Zürich,[ADDRESS_19] -2025-06-07T18:12:08.064201,True,cv_lara_meier.txt,123 Rue de Paris 75001 Paris,[ADDRESS_20] -2025-06-07T18:12:08.064201,True,cv_lara_meier.txt,Via Roma 123 20100 Milano,[ADDRESS_21] -2025-06-07T18:12:08.064201,True,cv_lara_meier.txt,123 High Street London SW1A 1AA,[ADDRESS_22] -2025-06-07T18:12:08.064201,True,cv_lara_meier.txt,DE89 3704 0044 0532 0130 00,[IBAN_23] -2025-06-07T18:12:08.064201,True,cv_lara_meier.txt,CH93 0076 7000 E529 3557 7,[IBAN_24] -2025-06-07T18:12:08.064201,True,cv_lara_meier.txt,FR76 3000 6000 0112 3456 7890 189,[IBAN_25] -2025-06-07T18:12:08.064201,True,cv_lara_meier.txt,IT60 X054 2811 1010 0000 0123 456,[IBAN_26] -2025-06-07T18:12:08.064201,True,cv_lara_meier.txt,GB29 NWBK 6016 1331 9268 19,[IBAN_27] -2025-06-07T18:12:08.064201,True,cv_lara_meier.txt,4532 1234 5678 9012,[IBAN_28] -2025-06-07T18:12:08.064201,True,cv_lara_meier.txt,4532 1234 5678 9013,[IBAN_29] -2025-06-07T18:12:08.064201,True,cv_lara_meier.txt,4532 1234 5678 9014,[IBAN_30] -2025-06-07T18:12:08.064201,True,cv_lara_meier.txt,4532 1234 5678 9015,[IBAN_31] -2025-06-07T18:12:08.064201,True,cv_lara_meier.txt,4532 1234 5678 9016,[IBAN_32] -2025-06-07T18:12:08.064201,True,cv_lara_meier.txt,nan,[SSN_33] -2025-06-07T18:12:08.064201,True,cv_lara_meier.txt,756.1234.5678.90,[SSN_34] -2025-06-07T18:12:08.064201,True,cv_lara_meier.txt, ,[SSN_35] -2025-06-07T18:12:08.064201,True,cv_lara_meier.txt,sarah.weber@techsolutions.ch,[EMAIL_36] -2025-06-07T18:12:08.064201,True,cv_lara_meier.txt,+41 44 987 65,[PHONE_37] -2025-06-07T18:12:08.064201,True,cv_lara_meier.txt,"Dr. Sarah Weber -TechSolutions",[NAME_38] -2025-06-07T18:12:08.064201,True,cv_lara_meier.txt,hans.mueller@ethz.ch,[EMAIL_39] -2025-06-07T18:12:08.064201,True,cv_lara_meier.txt,+41 44 123 45,[PHONE_40] -2025-06-07T18:12:08.064201,True,cv_lara_meier.txt,l.meier@digitalsystems.ch,[EMAIL_41] -2025-06-07T18:12:08.064201,True,cv_lara_meier.txt,+41 44 456 78,[PHONE_42] -2025-06-07T18:12:08.064201,True,cv_lara_meier.txt,987.654.321,[SSN_43] -2025-06-07T18:12:08.064201,True,cv_lara_meier.txt,CHE-987.654.321,[SSN_44] -2025-06-07T18:12:08.064201,True,cv_lara_meier.txt,8002 Zürich,[ADDRESS_45] -2025-06-07T18:12:08.064201,True,cv_lara_meier.txt,Musterstrasse 123,[ADDRESS_46] -2025-06-07T18:12:08.064201,True,cv_lara_meier.txt,lara.meier@techsolutions.ch,[EMAIL_47] -2025-06-07T18:12:08.064201,True,cv_lara_meier.txt,123.456.789,[SSN_48] -2025-06-07T18:12:08.064201,True,cv_lara_meier.txt,CHE-123.456.789,[SSN_49] -2025-06-07T18:12:08.064201,True,cv_lara_meier.txt,8004 Zürich,[ADDRESS_50] -2025-06-07T18:12:08.064201,True,cv_lara_meier.txt,Industriestrasse 100,[ADDRESS_51] -2025-06-07T18:12:08.064201,True,cv_lara_meier.txt,lara.meier@example.ch,[EMAIL_52] -2025-06-07T18:12:08.064201,True,cv_lara_meier.txt,8001 Zürich,[ADDRESS_53] -2025-06-07T18:12:08.064201,True,cv_lara_meier.txt,Bahnhofstrasse 45,[ADDRESS_54] -2025-06-07T18:12:08.064201,True,cv_lara_meier.txt,"1990 -Adresse",[ADDRESS_55] -2025-06-07T18:12:08.080961,True,employees.csv,000-180.000,[PHONE_1] -2025-06-07T18:12:08.080961,True,employees.csv,2027 Bewertung,[ADDRESS_2] -2025-06-07T18:12:08.080961,True,employees.csv,Max Mustermann,[NAME_3] -2025-06-07T18:12:08.080961,True,employees.csv,Peter Schmid,[NAME_4] -2025-06-07T18:12:08.080961,True,employees.csv,Marie Dupont,[NAME_5] -2025-06-07T18:12:08.080961,True,employees.csv,Marco Rossi,[NAME_6] -2025-06-07T18:12:08.080961,True,employees.csv,John Smith,[NAME_7] -2025-06-07T18:12:08.080961,True,employees.csv,max.mustermann@beispiel.de,[EMAIL_8] -2025-06-07T18:12:08.080961,True,employees.csv,peter.schmid@beispiel.ch,[EMAIL_9] -2025-06-07T18:12:08.080961,True,employees.csv,marie.dupont@exemple.fr,[EMAIL_10] -2025-06-07T18:12:08.080961,True,employees.csv,marco.rossi@esempio.it,[EMAIL_11] -2025-06-07T18:12:08.080961,True,employees.csv,john.smith@example.com,[EMAIL_12] -2025-06-07T18:12:08.080961,True,employees.csv,+49 30 12345678,[PHONE_13] -2025-06-07T18:12:08.080961,True,employees.csv,+41 44 123 45 67,[PHONE_14] -2025-06-07T18:12:08.080961,True,employees.csv,+33 1 23 45 67 89,[PHONE_15] -2025-06-07T18:12:08.080961,True,employees.csv,+39 02 1234 5678,[PHONE_16] -2025-06-07T18:12:08.080961,True,employees.csv,+44 20 1234 5678,[PHONE_17] -2025-06-07T18:12:08.080961,True,employees.csv,Musterstraße 123 12345 Berlin,[ADDRESS_18] -2025-06-07T18:12:08.080961,True,employees.csv,Bahnhofstrasse 1 8001 Zürich,[ADDRESS_19] -2025-06-07T18:12:08.080961,True,employees.csv,123 Rue de Paris 75001 Paris,[ADDRESS_20] -2025-06-07T18:12:08.080961,True,employees.csv,Via Roma 123 20100 Milano,[ADDRESS_21] -2025-06-07T18:12:08.080961,True,employees.csv,123 High Street London SW1A 1AA,[ADDRESS_22] -2025-06-07T18:12:08.080961,True,employees.csv,DE89 3704 0044 0532 0130 00,[IBAN_23] -2025-06-07T18:12:08.080961,True,employees.csv,CH93 0076 7000 E529 3557 7,[IBAN_24] -2025-06-07T18:12:08.080961,True,employees.csv,FR76 3000 6000 0112 3456 7890 189,[IBAN_25] -2025-06-07T18:12:08.080961,True,employees.csv,IT60 X054 2811 1010 0000 0123 456,[IBAN_26] -2025-06-07T18:12:08.080961,True,employees.csv,GB29 NWBK 6016 1331 9268 19,[IBAN_27] -2025-06-07T18:12:08.080961,True,employees.csv,4532 1234 5678 9012,[IBAN_28] -2025-06-07T18:12:08.080961,True,employees.csv,4532 1234 5678 9013,[IBAN_29] -2025-06-07T18:12:08.080961,True,employees.csv,4532 1234 5678 9014,[IBAN_30] -2025-06-07T18:12:08.080961,True,employees.csv,4532 1234 5678 9015,[IBAN_31] -2025-06-07T18:12:08.080961,True,employees.csv,4532 1234 5678 9016,[IBAN_32] -2025-06-07T18:12:08.080961,True,employees.csv,nan,[SSN_33] -2025-06-07T18:12:08.080961,True,employees.csv,756.1234.5678.90,[SSN_34] -2025-06-07T18:12:08.080961,True,employees.csv, ,[SSN_35] -2025-06-07T18:12:08.080961,True,employees.csv,sarah.weber@techsolutions.ch,[EMAIL_36] -2025-06-07T18:12:08.080961,True,employees.csv,+41 44 987 65,[PHONE_37] -2025-06-07T18:12:08.080961,True,employees.csv,"Dr. Sarah Weber -TechSolutions",[NAME_38] -2025-06-07T18:12:08.080961,True,employees.csv,hans.mueller@ethz.ch,[EMAIL_39] -2025-06-07T18:12:08.080961,True,employees.csv,+41 44 123 45,[PHONE_40] -2025-06-07T18:12:08.080961,True,employees.csv,l.meier@digitalsystems.ch,[EMAIL_41] -2025-06-07T18:12:08.080961,True,employees.csv,+41 44 456 78,[PHONE_42] -2025-06-07T18:12:08.080961,True,employees.csv,987.654.321,[SSN_43] -2025-06-07T18:12:08.080961,True,employees.csv,CHE-987.654.321,[SSN_44] -2025-06-07T18:12:08.080961,True,employees.csv,8002 Zürich,[ADDRESS_45] -2025-06-07T18:12:08.080961,True,employees.csv,Musterstrasse 123,[ADDRESS_46] -2025-06-07T18:12:08.080961,True,employees.csv,lara.meier@techsolutions.ch,[EMAIL_47] -2025-06-07T18:12:08.080961,True,employees.csv,123.456.789,[SSN_48] -2025-06-07T18:12:08.080961,True,employees.csv,CHE-123.456.789,[SSN_49] -2025-06-07T18:12:08.080961,True,employees.csv,8004 Zürich,[ADDRESS_50] -2025-06-07T18:12:08.080961,True,employees.csv,Industriestrasse 100,[ADDRESS_51] -2025-06-07T18:12:08.080961,True,employees.csv,lara.meier@example.ch,[EMAIL_52] -2025-06-07T18:12:08.080961,True,employees.csv,8001 Zürich,[ADDRESS_53] -2025-06-07T18:12:08.080961,True,employees.csv,Bahnhofstrasse 45,[ADDRESS_54] -2025-06-07T18:12:08.080961,True,employees.csv,"1990 -Adresse",[ADDRESS_55] -2025-06-07T18:12:08.080961,True,employees.csv,Hans,[NAME_56] -2025-06-07T18:12:08.080961,True,employees.csv,Thomas,[NAME_57] -2025-06-07T18:12:08.080961,True,employees.csv,Sophie,[NAME_58] -2025-06-07T18:12:08.080961,True,employees.csv,Luca,[NAME_59] -2025-06-07T18:12:08.080961,True,employees.csv,Emma,[NAME_60] -2025-06-07T18:12:08.080961,True,employees.csv,Müller,[NAME_61] -2025-06-07T18:12:08.080961,True,employees.csv,Weber,[NAME_62] -2025-06-07T18:12:08.080961,True,employees.csv,Martin,[NAME_63] -2025-06-07T18:12:08.080961,True,employees.csv,Ferrari,[NAME_64] -2025-06-07T18:12:08.080961,True,employees.csv,Wilson,[NAME_65] -2025-06-07T18:12:08.080961,True,employees.csv,hans.mueller@firma.de,[EMAIL_66] -2025-06-07T18:12:08.080961,True,employees.csv,thomas.weber@firma.ch,[EMAIL_67] -2025-06-07T18:12:08.080961,True,employees.csv,sophie.martin@entreprise.fr,[EMAIL_68] -2025-06-07T18:12:08.080961,True,employees.csv,luca.ferrari@azienda.it,[EMAIL_69] -2025-06-07T18:12:08.080961,True,employees.csv,emma.wilson@company.com,[EMAIL_70] -2025-06-07T18:12:08.080961,True,employees.csv,+49 89 12345678,[PHONE_71] -2025-06-07T18:12:08.080961,True,employees.csv,+41 44 234 56 78,[PHONE_72] -2025-06-07T18:12:08.080961,True,employees.csv,+33 1 34 56 78 90,[PHONE_73] -2025-06-07T18:12:08.080961,True,employees.csv,+39 02 2345 6789,[PHONE_74] -2025-06-07T18:12:08.080961,True,employees.csv,+44 20 2345 6789,[PHONE_75] -2025-06-07T18:12:08.080961,True,employees.csv,Hauptstraße 1 80331 München,[ADDRESS_76] -2025-06-07T18:12:08.080961,True,employees.csv,Bahnhofstrasse 2 8001 Zürich,[ADDRESS_77] -2025-06-07T18:12:08.080961,True,employees.csv,15 Avenue des Champs-Élysées 75008 Paris,[ADDRESS_78] -2025-06-07T18:12:08.080961,True,employees.csv,Via Monte Napoleone 8 20121 Milano,[ADDRESS_79] -2025-06-07T18:12:08.080961,True,employees.csv,25 Old Street London EC1V 9HL,[ADDRESS_80] -2025-06-07T18:12:08.080961,True,employees.csv,01-234567-8,[IBAN_81] -2025-06-07T18:12:08.080961,True,employees.csv,GB29 NWBK 6016 1331 9268 19 ,[IBAN_82] -2025-06-07T18:12:08.101660,True,english.txt,000-180.000,[PHONE_1] -2025-06-07T18:12:08.101660,True,english.txt,2027 Bewertung,[ADDRESS_2] -2025-06-07T18:12:08.101660,True,english.txt,Max Mustermann,[NAME_3] -2025-06-07T18:12:08.101660,True,english.txt,Peter Schmid,[NAME_4] -2025-06-07T18:12:08.101660,True,english.txt,Marie Dupont,[NAME_5] -2025-06-07T18:12:08.101660,True,english.txt,Marco Rossi,[NAME_6] -2025-06-07T18:12:08.101660,True,english.txt,John Smith,[NAME_7] -2025-06-07T18:12:08.101660,True,english.txt,max.mustermann@beispiel.de,[EMAIL_8] -2025-06-07T18:12:08.101660,True,english.txt,peter.schmid@beispiel.ch,[EMAIL_9] -2025-06-07T18:12:08.101660,True,english.txt,marie.dupont@exemple.fr,[EMAIL_10] -2025-06-07T18:12:08.101660,True,english.txt,marco.rossi@esempio.it,[EMAIL_11] -2025-06-07T18:12:08.101660,True,english.txt,john.smith@example.com,[EMAIL_12] -2025-06-07T18:12:08.101660,True,english.txt,+49 30 12345678,[PHONE_13] -2025-06-07T18:12:08.101660,True,english.txt,+41 44 123 45 67,[PHONE_14] -2025-06-07T18:12:08.101660,True,english.txt,+33 1 23 45 67 89,[PHONE_15] -2025-06-07T18:12:08.101660,True,english.txt,+39 02 1234 5678,[PHONE_16] -2025-06-07T18:12:08.101660,True,english.txt,+44 20 1234 5678,[PHONE_17] -2025-06-07T18:12:08.101660,True,english.txt,Musterstraße 123 12345 Berlin,[ADDRESS_18] -2025-06-07T18:12:08.101660,True,english.txt,Bahnhofstrasse 1 8001 Zürich,[ADDRESS_19] -2025-06-07T18:12:08.101660,True,english.txt,123 Rue de Paris 75001 Paris,[ADDRESS_20] -2025-06-07T18:12:08.101660,True,english.txt,Via Roma 123 20100 Milano,[ADDRESS_21] -2025-06-07T18:12:08.101660,True,english.txt,123 High Street London SW1A 1AA,[ADDRESS_22] -2025-06-07T18:12:08.101660,True,english.txt,DE89 3704 0044 0532 0130 00,[IBAN_23] -2025-06-07T18:12:08.101660,True,english.txt,CH93 0076 7000 E529 3557 7,[IBAN_24] -2025-06-07T18:12:08.101660,True,english.txt,FR76 3000 6000 0112 3456 7890 189,[IBAN_25] -2025-06-07T18:12:08.101660,True,english.txt,IT60 X054 2811 1010 0000 0123 456,[IBAN_26] -2025-06-07T18:12:08.101660,True,english.txt,GB29 NWBK 6016 1331 9268 19,[IBAN_27] -2025-06-07T18:12:08.101660,True,english.txt,4532 1234 5678 9012,[IBAN_28] -2025-06-07T18:12:08.101660,True,english.txt,4532 1234 5678 9013,[IBAN_29] -2025-06-07T18:12:08.101660,True,english.txt,4532 1234 5678 9014,[IBAN_30] -2025-06-07T18:12:08.101660,True,english.txt,4532 1234 5678 9015,[IBAN_31] -2025-06-07T18:12:08.101660,True,english.txt,4532 1234 5678 9016,[IBAN_32] -2025-06-07T18:12:08.101660,True,english.txt,nan,[SSN_33] -2025-06-07T18:12:08.101660,True,english.txt,756.1234.5678.90,[SSN_34] -2025-06-07T18:12:08.101660,True,english.txt, ,[SSN_35] -2025-06-07T18:12:08.101660,True,english.txt,sarah.weber@techsolutions.ch,[EMAIL_36] -2025-06-07T18:12:08.101660,True,english.txt,+41 44 987 65,[PHONE_37] -2025-06-07T18:12:08.101660,True,english.txt,"Dr. Sarah Weber -TechSolutions",[NAME_38] -2025-06-07T18:12:08.101660,True,english.txt,hans.mueller@ethz.ch,[EMAIL_39] -2025-06-07T18:12:08.101660,True,english.txt,+41 44 123 45,[PHONE_40] -2025-06-07T18:12:08.101660,True,english.txt,l.meier@digitalsystems.ch,[EMAIL_41] -2025-06-07T18:12:08.101660,True,english.txt,+41 44 456 78,[PHONE_42] -2025-06-07T18:12:08.101660,True,english.txt,987.654.321,[SSN_43] -2025-06-07T18:12:08.101660,True,english.txt,CHE-987.654.321,[SSN_44] -2025-06-07T18:12:08.101660,True,english.txt,8002 Zürich,[ADDRESS_45] -2025-06-07T18:12:08.101660,True,english.txt,Musterstrasse 123,[ADDRESS_46] -2025-06-07T18:12:08.101660,True,english.txt,lara.meier@techsolutions.ch,[EMAIL_47] -2025-06-07T18:12:08.101660,True,english.txt,123.456.789,[SSN_48] -2025-06-07T18:12:08.101660,True,english.txt,CHE-123.456.789,[SSN_49] -2025-06-07T18:12:08.101660,True,english.txt,8004 Zürich,[ADDRESS_50] -2025-06-07T18:12:08.101660,True,english.txt,Industriestrasse 100,[ADDRESS_51] -2025-06-07T18:12:08.101660,True,english.txt,lara.meier@example.ch,[EMAIL_52] -2025-06-07T18:12:08.101660,True,english.txt,8001 Zürich,[ADDRESS_53] -2025-06-07T18:12:08.101660,True,english.txt,Bahnhofstrasse 45,[ADDRESS_54] -2025-06-07T18:12:08.101660,True,english.txt,"1990 -Adresse",[ADDRESS_55] -2025-06-07T18:12:08.101660,True,english.txt,Hans,[NAME_56] -2025-06-07T18:12:08.101660,True,english.txt,Thomas,[NAME_57] -2025-06-07T18:12:08.101660,True,english.txt,Sophie,[NAME_58] -2025-06-07T18:12:08.101660,True,english.txt,Luca,[NAME_59] -2025-06-07T18:12:08.101660,True,english.txt,Emma,[NAME_60] -2025-06-07T18:12:08.101660,True,english.txt,Müller,[NAME_61] -2025-06-07T18:12:08.101660,True,english.txt,Weber,[NAME_62] -2025-06-07T18:12:08.101660,True,english.txt,Martin,[NAME_63] -2025-06-07T18:12:08.101660,True,english.txt,Ferrari,[NAME_64] -2025-06-07T18:12:08.101660,True,english.txt,Wilson,[NAME_65] -2025-06-07T18:12:08.101660,True,english.txt,hans.mueller@firma.de,[EMAIL_66] -2025-06-07T18:12:08.101660,True,english.txt,thomas.weber@firma.ch,[EMAIL_67] -2025-06-07T18:12:08.101660,True,english.txt,sophie.martin@entreprise.fr,[EMAIL_68] -2025-06-07T18:12:08.101660,True,english.txt,luca.ferrari@azienda.it,[EMAIL_69] -2025-06-07T18:12:08.101660,True,english.txt,emma.wilson@company.com,[EMAIL_70] -2025-06-07T18:12:08.101660,True,english.txt,+49 89 12345678,[PHONE_71] -2025-06-07T18:12:08.101660,True,english.txt,+41 44 234 56 78,[PHONE_72] -2025-06-07T18:12:08.101660,True,english.txt,+33 1 34 56 78 90,[PHONE_73] -2025-06-07T18:12:08.101660,True,english.txt,+39 02 2345 6789,[PHONE_74] -2025-06-07T18:12:08.101660,True,english.txt,+44 20 2345 6789,[PHONE_75] -2025-06-07T18:12:08.101660,True,english.txt,Hauptstraße 1 80331 München,[ADDRESS_76] -2025-06-07T18:12:08.101660,True,english.txt,Bahnhofstrasse 2 8001 Zürich,[ADDRESS_77] -2025-06-07T18:12:08.101660,True,english.txt,15 Avenue des Champs-Élysées 75008 Paris,[ADDRESS_78] -2025-06-07T18:12:08.101660,True,english.txt,Via Monte Napoleone 8 20121 Milano,[ADDRESS_79] -2025-06-07T18:12:08.101660,True,english.txt,25 Old Street London EC1V 9HL,[ADDRESS_80] -2025-06-07T18:12:08.101660,True,english.txt,01-234567-8,[IBAN_81] -2025-06-07T18:12:08.101660,True,english.txt,GB29 NWBK 6016 1331 9268 19 ,[IBAN_82] -2025-06-07T18:12:08.101660,True,english.txt,"9012 - -Best",[ADDRESS_83] -2025-06-07T18:12:08.101660,True,english.txt,"5678 -Address",[ADDRESS_84] -2025-06-07T18:12:08.101660,True,english.txt,contact@example.com,[EMAIL_85] -2025-06-07T18:12:08.125634,True,example.json,000-180.000,[PHONE_1] -2025-06-07T18:12:08.125634,True,example.json,2027 Bewertung,[ADDRESS_2] -2025-06-07T18:12:08.125634,True,example.json,Max Mustermann,[NAME_3] -2025-06-07T18:12:08.125634,True,example.json,Peter Schmid,[NAME_4] -2025-06-07T18:12:08.125634,True,example.json,Marie Dupont,[NAME_5] -2025-06-07T18:12:08.125634,True,example.json,Marco Rossi,[NAME_6] -2025-06-07T18:12:08.125634,True,example.json,John Smith,[NAME_7] -2025-06-07T18:12:08.125634,True,example.json,max.mustermann@beispiel.de,[EMAIL_8] -2025-06-07T18:12:08.125634,True,example.json,peter.schmid@beispiel.ch,[EMAIL_9] -2025-06-07T18:12:08.125634,True,example.json,marie.dupont@exemple.fr,[EMAIL_10] -2025-06-07T18:12:08.125634,True,example.json,marco.rossi@esempio.it,[EMAIL_11] -2025-06-07T18:12:08.125634,True,example.json,john.smith@example.com,[EMAIL_12] -2025-06-07T18:12:08.125634,True,example.json,+49 30 12345678,[PHONE_13] -2025-06-07T18:12:08.125634,True,example.json,+41 44 123 45 67,[PHONE_14] -2025-06-07T18:12:08.125634,True,example.json,+33 1 23 45 67 89,[PHONE_15] -2025-06-07T18:12:08.125634,True,example.json,+39 02 1234 5678,[PHONE_16] -2025-06-07T18:12:08.125634,True,example.json,+44 20 1234 5678,[PHONE_17] -2025-06-07T18:12:08.125634,True,example.json,Musterstraße 123 12345 Berlin,[ADDRESS_18] -2025-06-07T18:12:08.125634,True,example.json,Bahnhofstrasse 1 8001 Zürich,[ADDRESS_19] -2025-06-07T18:12:08.125634,True,example.json,123 Rue de Paris 75001 Paris,[ADDRESS_20] -2025-06-07T18:12:08.125634,True,example.json,Via Roma 123 20100 Milano,[ADDRESS_21] -2025-06-07T18:12:08.125634,True,example.json,123 High Street London SW1A 1AA,[ADDRESS_22] -2025-06-07T18:12:08.125634,True,example.json,DE89 3704 0044 0532 0130 00,[IBAN_23] -2025-06-07T18:12:08.125634,True,example.json,CH93 0076 7000 E529 3557 7,[IBAN_24] -2025-06-07T18:12:08.125634,True,example.json,FR76 3000 6000 0112 3456 7890 189,[IBAN_25] -2025-06-07T18:12:08.125634,True,example.json,IT60 X054 2811 1010 0000 0123 456,[IBAN_26] -2025-06-07T18:12:08.125634,True,example.json,GB29 NWBK 6016 1331 9268 19,[IBAN_27] -2025-06-07T18:12:08.125634,True,example.json,4532 1234 5678 9012,[IBAN_28] -2025-06-07T18:12:08.125634,True,example.json,4532 1234 5678 9013,[IBAN_29] -2025-06-07T18:12:08.125634,True,example.json,4532 1234 5678 9014,[IBAN_30] -2025-06-07T18:12:08.125634,True,example.json,4532 1234 5678 9015,[IBAN_31] -2025-06-07T18:12:08.125634,True,example.json,4532 1234 5678 9016,[IBAN_32] -2025-06-07T18:12:08.125634,True,example.json,nan,[SSN_33] -2025-06-07T18:12:08.125634,True,example.json,756.1234.5678.90,[SSN_34] -2025-06-07T18:12:08.125634,True,example.json, ,[SSN_35] -2025-06-07T18:12:08.125634,True,example.json,sarah.weber@techsolutions.ch,[EMAIL_36] -2025-06-07T18:12:08.125634,True,example.json,+41 44 987 65,[PHONE_37] -2025-06-07T18:12:08.125634,True,example.json,"Dr. Sarah Weber -TechSolutions",[NAME_38] -2025-06-07T18:12:08.125634,True,example.json,hans.mueller@ethz.ch,[EMAIL_39] -2025-06-07T18:12:08.125634,True,example.json,+41 44 123 45,[PHONE_40] -2025-06-07T18:12:08.125634,True,example.json,l.meier@digitalsystems.ch,[EMAIL_41] -2025-06-07T18:12:08.125634,True,example.json,+41 44 456 78,[PHONE_42] -2025-06-07T18:12:08.125634,True,example.json,987.654.321,[SSN_43] -2025-06-07T18:12:08.125634,True,example.json,CHE-987.654.321,[SSN_44] -2025-06-07T18:12:08.125634,True,example.json,8002 Zürich,[ADDRESS_45] -2025-06-07T18:12:08.125634,True,example.json,Musterstrasse 123,[ADDRESS_46] -2025-06-07T18:12:08.125634,True,example.json,lara.meier@techsolutions.ch,[EMAIL_47] -2025-06-07T18:12:08.125634,True,example.json,123.456.789,[SSN_48] -2025-06-07T18:12:08.125634,True,example.json,CHE-123.456.789,[SSN_49] -2025-06-07T18:12:08.125634,True,example.json,8004 Zürich,[ADDRESS_50] -2025-06-07T18:12:08.125634,True,example.json,Industriestrasse 100,[ADDRESS_51] -2025-06-07T18:12:08.125634,True,example.json,lara.meier@example.ch,[EMAIL_52] -2025-06-07T18:12:08.125634,True,example.json,8001 Zürich,[ADDRESS_53] -2025-06-07T18:12:08.125634,True,example.json,Bahnhofstrasse 45,[ADDRESS_54] -2025-06-07T18:12:08.125634,True,example.json,"1990 -Adresse",[ADDRESS_55] -2025-06-07T18:12:08.125634,True,example.json,Hans,[NAME_56] -2025-06-07T18:12:08.125634,True,example.json,Thomas,[NAME_57] -2025-06-07T18:12:08.125634,True,example.json,Sophie,[NAME_58] -2025-06-07T18:12:08.125634,True,example.json,Luca,[NAME_59] -2025-06-07T18:12:08.125634,True,example.json,Emma,[NAME_60] -2025-06-07T18:12:08.125634,True,example.json,Müller,[NAME_61] -2025-06-07T18:12:08.125634,True,example.json,Weber,[NAME_62] -2025-06-07T18:12:08.125634,True,example.json,Martin,[NAME_63] -2025-06-07T18:12:08.125634,True,example.json,Ferrari,[NAME_64] -2025-06-07T18:12:08.125634,True,example.json,Wilson,[NAME_65] -2025-06-07T18:12:08.125634,True,example.json,hans.mueller@firma.de,[EMAIL_66] -2025-06-07T18:12:08.125634,True,example.json,thomas.weber@firma.ch,[EMAIL_67] -2025-06-07T18:12:08.125634,True,example.json,sophie.martin@entreprise.fr,[EMAIL_68] -2025-06-07T18:12:08.125634,True,example.json,luca.ferrari@azienda.it,[EMAIL_69] -2025-06-07T18:12:08.125634,True,example.json,emma.wilson@company.com,[EMAIL_70] -2025-06-07T18:12:08.125634,True,example.json,+49 89 12345678,[PHONE_71] -2025-06-07T18:12:08.125634,True,example.json,+41 44 234 56 78,[PHONE_72] -2025-06-07T18:12:08.125634,True,example.json,+33 1 34 56 78 90,[PHONE_73] -2025-06-07T18:12:08.125634,True,example.json,+39 02 2345 6789,[PHONE_74] -2025-06-07T18:12:08.125634,True,example.json,+44 20 2345 6789,[PHONE_75] -2025-06-07T18:12:08.125634,True,example.json,Hauptstraße 1 80331 München,[ADDRESS_76] -2025-06-07T18:12:08.125634,True,example.json,Bahnhofstrasse 2 8001 Zürich,[ADDRESS_77] -2025-06-07T18:12:08.125634,True,example.json,15 Avenue des Champs-Élysées 75008 Paris,[ADDRESS_78] -2025-06-07T18:12:08.125634,True,example.json,Via Monte Napoleone 8 20121 Milano,[ADDRESS_79] -2025-06-07T18:12:08.125634,True,example.json,25 Old Street London EC1V 9HL,[ADDRESS_80] -2025-06-07T18:12:08.125634,True,example.json,01-234567-8,[IBAN_81] -2025-06-07T18:12:08.125634,True,example.json,GB29 NWBK 6016 1331 9268 19 ,[IBAN_82] -2025-06-07T18:12:08.125634,True,example.json,"9012 - -Best",[ADDRESS_83] -2025-06-07T18:12:08.125634,True,example.json,"5678 -Address",[ADDRESS_84] -2025-06-07T18:12:08.125634,True,example.json,contact@example.com,[EMAIL_85] -2025-06-07T18:12:08.125634,True,example.json,max.mustermann@example.com,[EMAIL_86] -2025-06-07T18:12:08.125634,True,example.json,+49 123 4567890,[PHONE_87] -2025-06-07T18:12:08.125634,True,example.json,2024-03-15,[DATE_88] -2025-06-07T18:12:08.125634,True,example.json,4111 1111 1111 1111,[IBAN_89] -2025-06-07T18:12:08.125634,True,example.json,Tech Solutions GmbH,[NAME_90] -2025-06-07T18:12:08.125634,True,example.json,Dr. Anna Schmidt,NAME_91 -2025-06-07T18:12:08.125634,True,example.json,anna.schmidt@techsolutions.de,[EMAIL_92] -2025-06-07T18:12:08.157397,True,example.xml,000-180.000,[PHONE_1] -2025-06-07T18:12:08.157397,True,example.xml,2027 Bewertung,[ADDRESS_2] -2025-06-07T18:12:08.157397,True,example.xml,Max Mustermann,[NAME_3] -2025-06-07T18:12:08.157397,True,example.xml,Peter Schmid,[NAME_4] -2025-06-07T18:12:08.157397,True,example.xml,Marie Dupont,[NAME_5] -2025-06-07T18:12:08.157397,True,example.xml,Marco Rossi,[NAME_6] -2025-06-07T18:12:08.157397,True,example.xml,John Smith,[NAME_7] -2025-06-07T18:12:08.157397,True,example.xml,max.mustermann@beispiel.de,[EMAIL_8] -2025-06-07T18:12:08.157397,True,example.xml,peter.schmid@beispiel.ch,[EMAIL_9] -2025-06-07T18:12:08.157397,True,example.xml,marie.dupont@exemple.fr,[EMAIL_10] -2025-06-07T18:12:08.157397,True,example.xml,marco.rossi@esempio.it,[EMAIL_11] -2025-06-07T18:12:08.157397,True,example.xml,john.smith@example.com,[EMAIL_12] -2025-06-07T18:12:08.157397,True,example.xml,+49 30 12345678,[PHONE_13] -2025-06-07T18:12:08.157397,True,example.xml,+41 44 123 45 67,[PHONE_14] -2025-06-07T18:12:08.157397,True,example.xml,+33 1 23 45 67 89,[PHONE_15] -2025-06-07T18:12:08.157397,True,example.xml,+39 02 1234 5678,[PHONE_16] -2025-06-07T18:12:08.157397,True,example.xml,+44 20 1234 5678,[PHONE_17] -2025-06-07T18:12:08.157397,True,example.xml,Musterstraße 123 12345 Berlin,[ADDRESS_18] -2025-06-07T18:12:08.157397,True,example.xml,Bahnhofstrasse 1 8001 Zürich,[ADDRESS_19] -2025-06-07T18:12:08.157397,True,example.xml,123 Rue de Paris 75001 Paris,[ADDRESS_20] -2025-06-07T18:12:08.157397,True,example.xml,Via Roma 123 20100 Milano,[ADDRESS_21] -2025-06-07T18:12:08.157397,True,example.xml,123 High Street London SW1A 1AA,[ADDRESS_22] -2025-06-07T18:12:08.157397,True,example.xml,DE89 3704 0044 0532 0130 00,[IBAN_23] -2025-06-07T18:12:08.157397,True,example.xml,CH93 0076 7000 E529 3557 7,[IBAN_24] -2025-06-07T18:12:08.157397,True,example.xml,FR76 3000 6000 0112 3456 7890 189,[IBAN_25] -2025-06-07T18:12:08.157397,True,example.xml,IT60 X054 2811 1010 0000 0123 456,[IBAN_26] -2025-06-07T18:12:08.157397,True,example.xml,GB29 NWBK 6016 1331 9268 19,[IBAN_27] -2025-06-07T18:12:08.157397,True,example.xml,4532 1234 5678 9012,[IBAN_28] -2025-06-07T18:12:08.157397,True,example.xml,4532 1234 5678 9013,[IBAN_29] -2025-06-07T18:12:08.157397,True,example.xml,4532 1234 5678 9014,[IBAN_30] -2025-06-07T18:12:08.157397,True,example.xml,4532 1234 5678 9015,[IBAN_31] -2025-06-07T18:12:08.157397,True,example.xml,4532 1234 5678 9016,[IBAN_32] -2025-06-07T18:12:08.157397,True,example.xml,nan,[SSN_33] -2025-06-07T18:12:08.157397,True,example.xml,756.1234.5678.90,[SSN_34] -2025-06-07T18:12:08.157397,True,example.xml, ,[SSN_35] -2025-06-07T18:12:08.157397,True,example.xml,sarah.weber@techsolutions.ch,[EMAIL_36] -2025-06-07T18:12:08.157397,True,example.xml,+41 44 987 65,[PHONE_37] -2025-06-07T18:12:08.157397,True,example.xml,"Dr. Sarah Weber -TechSolutions",[NAME_38] -2025-06-07T18:12:08.157397,True,example.xml,hans.mueller@ethz.ch,[EMAIL_39] -2025-06-07T18:12:08.157397,True,example.xml,+41 44 123 45,[PHONE_40] -2025-06-07T18:12:08.157397,True,example.xml,l.meier@digitalsystems.ch,[EMAIL_41] -2025-06-07T18:12:08.157397,True,example.xml,+41 44 456 78,[PHONE_42] -2025-06-07T18:12:08.157397,True,example.xml,987.654.321,[SSN_43] -2025-06-07T18:12:08.157397,True,example.xml,CHE-987.654.321,[SSN_44] -2025-06-07T18:12:08.157397,True,example.xml,8002 Zürich,[ADDRESS_45] -2025-06-07T18:12:08.157397,True,example.xml,Musterstrasse 123,[ADDRESS_46] -2025-06-07T18:12:08.157397,True,example.xml,lara.meier@techsolutions.ch,[EMAIL_47] -2025-06-07T18:12:08.157397,True,example.xml,123.456.789,[SSN_48] -2025-06-07T18:12:08.157397,True,example.xml,CHE-123.456.789,[SSN_49] -2025-06-07T18:12:08.157397,True,example.xml,8004 Zürich,[ADDRESS_50] -2025-06-07T18:12:08.157397,True,example.xml,Industriestrasse 100,[ADDRESS_51] -2025-06-07T18:12:08.157397,True,example.xml,lara.meier@example.ch,[EMAIL_52] -2025-06-07T18:12:08.157397,True,example.xml,8001 Zürich,[ADDRESS_53] -2025-06-07T18:12:08.157397,True,example.xml,Bahnhofstrasse 45,[ADDRESS_54] -2025-06-07T18:12:08.157397,True,example.xml,"1990 -Adresse",[ADDRESS_55] -2025-06-07T18:12:08.157397,True,example.xml,Hans,[NAME_56] -2025-06-07T18:12:08.157397,True,example.xml,Thomas,[NAME_57] -2025-06-07T18:12:08.157397,True,example.xml,Sophie,[NAME_58] -2025-06-07T18:12:08.157397,True,example.xml,Luca,[NAME_59] -2025-06-07T18:12:08.157397,True,example.xml,Emma,[NAME_60] -2025-06-07T18:12:08.157397,True,example.xml,Müller,[NAME_61] -2025-06-07T18:12:08.157397,True,example.xml,Weber,[NAME_62] -2025-06-07T18:12:08.157397,True,example.xml,Martin,[NAME_63] -2025-06-07T18:12:08.157397,True,example.xml,Ferrari,[NAME_64] -2025-06-07T18:12:08.157397,True,example.xml,Wilson,[NAME_65] -2025-06-07T18:12:08.157397,True,example.xml,hans.mueller@firma.de,[EMAIL_66] -2025-06-07T18:12:08.157397,True,example.xml,thomas.weber@firma.ch,[EMAIL_67] -2025-06-07T18:12:08.157397,True,example.xml,sophie.martin@entreprise.fr,[EMAIL_68] -2025-06-07T18:12:08.157397,True,example.xml,luca.ferrari@azienda.it,[EMAIL_69] -2025-06-07T18:12:08.157397,True,example.xml,emma.wilson@company.com,[EMAIL_70] -2025-06-07T18:12:08.157397,True,example.xml,+49 89 12345678,[PHONE_71] -2025-06-07T18:12:08.157397,True,example.xml,+41 44 234 56 78,[PHONE_72] -2025-06-07T18:12:08.157397,True,example.xml,+33 1 34 56 78 90,[PHONE_73] -2025-06-07T18:12:08.157397,True,example.xml,+39 02 2345 6789,[PHONE_74] -2025-06-07T18:12:08.157397,True,example.xml,+44 20 2345 6789,[PHONE_75] -2025-06-07T18:12:08.157397,True,example.xml,Hauptstraße 1 80331 München,[ADDRESS_76] -2025-06-07T18:12:08.157397,True,example.xml,Bahnhofstrasse 2 8001 Zürich,[ADDRESS_77] -2025-06-07T18:12:08.157397,True,example.xml,15 Avenue des Champs-Élysées 75008 Paris,[ADDRESS_78] -2025-06-07T18:12:08.157397,True,example.xml,Via Monte Napoleone 8 20121 Milano,[ADDRESS_79] -2025-06-07T18:12:08.157397,True,example.xml,25 Old Street London EC1V 9HL,[ADDRESS_80] -2025-06-07T18:12:08.157397,True,example.xml,01-234567-8,[IBAN_81] -2025-06-07T18:12:08.157397,True,example.xml,GB29 NWBK 6016 1331 9268 19 ,[IBAN_82] -2025-06-07T18:12:08.157397,True,example.xml,"9012 - -Best",[ADDRESS_83] -2025-06-07T18:12:08.157397,True,example.xml,"5678 -Address",[ADDRESS_84] -2025-06-07T18:12:08.157397,True,example.xml,contact@example.com,[EMAIL_85] -2025-06-07T18:12:08.157397,True,example.xml,max.mustermann@example.com,[EMAIL_86] -2025-06-07T18:12:08.157397,True,example.xml,+49 123 4567890,[PHONE_87] -2025-06-07T18:12:08.157397,True,example.xml,2024-03-15,[DATE_88] -2025-06-07T18:12:08.157397,True,example.xml,4111 1111 1111 1111,[IBAN_89] -2025-06-07T18:12:08.157397,True,example.xml,Tech Solutions GmbH,[NAME_90] -2025-06-07T18:12:08.157397,True,example.xml,Dr. Anna Schmidt,NAME_91 -2025-06-07T18:12:08.157397,True,example.xml,anna.schmidt@techsolutions.de,[EMAIL_92] -2025-06-07T18:12:08.157397,True,example.xml,Dr. Thomas Weber,[NAME_93] -2025-06-07T18:12:08.157397,True,example.xml,thomas.weber@company.de,[EMAIL_94] -2025-06-07T18:12:08.157397,True,example.xml,maria.schmidt@company.de,[EMAIL_95] -2025-06-07T18:12:08.157397,True,example.xml,+49 40 98765432,[PHONE_96] -2025-06-07T18:12:08.157397,True,example.xml,DE27 3704 0044 0532 0130 01,[PHONE_97] -2025-06-07T18:12:08.157397,True,example.xml,info@techinnovations.de,[EMAIL_98] -2025-06-07T18:12:08.157397,True,example.xml,+49 89 12345679,[PHONE_99] -2025-06-07T18:12:08.157397,True,example.xml,DE89 3704 0044 0532 0130 02,[PHONE_100] -2025-06-07T18:12:08.178030,True,french.txt,000-180.000,[PHONE_1] -2025-06-07T18:12:08.178030,True,french.txt,2027 Bewertung,[ADDRESS_2] -2025-06-07T18:12:08.178030,True,french.txt,Max Mustermann,[NAME_3] -2025-06-07T18:12:08.178030,True,french.txt,Peter Schmid,[NAME_4] -2025-06-07T18:12:08.178030,True,french.txt,Marie Dupont,[NAME_5] -2025-06-07T18:12:08.178030,True,french.txt,Marco Rossi,[NAME_6] -2025-06-07T18:12:08.178030,True,french.txt,John Smith,[NAME_7] -2025-06-07T18:12:08.178030,True,french.txt,max.mustermann@beispiel.de,[EMAIL_8] -2025-06-07T18:12:08.178030,True,french.txt,peter.schmid@beispiel.ch,[EMAIL_9] -2025-06-07T18:12:08.178030,True,french.txt,marie.dupont@exemple.fr,[EMAIL_10] -2025-06-07T18:12:08.178030,True,french.txt,marco.rossi@esempio.it,[EMAIL_11] -2025-06-07T18:12:08.178030,True,french.txt,john.smith@example.com,[EMAIL_12] -2025-06-07T18:12:08.178030,True,french.txt,+49 30 12345678,[PHONE_13] -2025-06-07T18:12:08.178030,True,french.txt,+41 44 123 45 67,[PHONE_14] -2025-06-07T18:12:08.178030,True,french.txt,+33 1 23 45 67 89,[PHONE_15] -2025-06-07T18:12:08.178030,True,french.txt,+39 02 1234 5678,[PHONE_16] -2025-06-07T18:12:08.178030,True,french.txt,+44 20 1234 5678,[PHONE_17] -2025-06-07T18:12:08.178030,True,french.txt,Musterstraße 123 12345 Berlin,[ADDRESS_18] -2025-06-07T18:12:08.178030,True,french.txt,Bahnhofstrasse 1 8001 Zürich,[ADDRESS_19] -2025-06-07T18:12:08.178030,True,french.txt,123 Rue de Paris 75001 Paris,[ADDRESS_20] -2025-06-07T18:12:08.178030,True,french.txt,Via Roma 123 20100 Milano,[ADDRESS_21] -2025-06-07T18:12:08.178030,True,french.txt,123 High Street London SW1A 1AA,[ADDRESS_22] -2025-06-07T18:12:08.178030,True,french.txt,DE89 3704 0044 0532 0130 00,[IBAN_23] -2025-06-07T18:12:08.178030,True,french.txt,CH93 0076 7000 E529 3557 7,[IBAN_24] -2025-06-07T18:12:08.178030,True,french.txt,FR76 3000 6000 0112 3456 7890 189,[IBAN_25] -2025-06-07T18:12:08.178030,True,french.txt,IT60 X054 2811 1010 0000 0123 456,[IBAN_26] -2025-06-07T18:12:08.178030,True,french.txt,GB29 NWBK 6016 1331 9268 19,[IBAN_27] -2025-06-07T18:12:08.178030,True,french.txt,4532 1234 5678 9012,[IBAN_28] -2025-06-07T18:12:08.178030,True,french.txt,4532 1234 5678 9013,[IBAN_29] -2025-06-07T18:12:08.178030,True,french.txt,4532 1234 5678 9014,[IBAN_30] -2025-06-07T18:12:08.178030,True,french.txt,4532 1234 5678 9015,[IBAN_31] -2025-06-07T18:12:08.178030,True,french.txt,4532 1234 5678 9016,[IBAN_32] -2025-06-07T18:12:08.178030,True,french.txt,nan,[SSN_33] -2025-06-07T18:12:08.178030,True,french.txt,756.1234.5678.90,[SSN_34] -2025-06-07T18:12:08.178030,True,french.txt, ,[SSN_35] -2025-06-07T18:12:08.178030,True,french.txt,sarah.weber@techsolutions.ch,[EMAIL_36] -2025-06-07T18:12:08.178030,True,french.txt,+41 44 987 65,[PHONE_37] -2025-06-07T18:12:08.178030,True,french.txt,"Dr. Sarah Weber -TechSolutions",[NAME_38] -2025-06-07T18:12:08.178030,True,french.txt,hans.mueller@ethz.ch,[EMAIL_39] -2025-06-07T18:12:08.178030,True,french.txt,+41 44 123 45,[PHONE_40] -2025-06-07T18:12:08.178030,True,french.txt,l.meier@digitalsystems.ch,[EMAIL_41] -2025-06-07T18:12:08.178030,True,french.txt,+41 44 456 78,[PHONE_42] -2025-06-07T18:12:08.178030,True,french.txt,987.654.321,[SSN_43] -2025-06-07T18:12:08.178030,True,french.txt,CHE-987.654.321,[SSN_44] -2025-06-07T18:12:08.178030,True,french.txt,8002 Zürich,[ADDRESS_45] -2025-06-07T18:12:08.178030,True,french.txt,Musterstrasse 123,[ADDRESS_46] -2025-06-07T18:12:08.178030,True,french.txt,lara.meier@techsolutions.ch,[EMAIL_47] -2025-06-07T18:12:08.178030,True,french.txt,123.456.789,[SSN_48] -2025-06-07T18:12:08.178030,True,french.txt,CHE-123.456.789,[SSN_49] -2025-06-07T18:12:08.178030,True,french.txt,8004 Zürich,[ADDRESS_50] -2025-06-07T18:12:08.178030,True,french.txt,Industriestrasse 100,[ADDRESS_51] -2025-06-07T18:12:08.178030,True,french.txt,lara.meier@example.ch,[EMAIL_52] -2025-06-07T18:12:08.178030,True,french.txt,8001 Zürich,[ADDRESS_53] -2025-06-07T18:12:08.178030,True,french.txt,Bahnhofstrasse 45,[ADDRESS_54] -2025-06-07T18:12:08.178030,True,french.txt,"1990 -Adresse",[ADDRESS_55] -2025-06-07T18:12:08.178030,True,french.txt,Hans,[NAME_56] -2025-06-07T18:12:08.178030,True,french.txt,Thomas,[NAME_57] -2025-06-07T18:12:08.178030,True,french.txt,Sophie,[NAME_58] -2025-06-07T18:12:08.178030,True,french.txt,Luca,[NAME_59] -2025-06-07T18:12:08.178030,True,french.txt,Emma,[NAME_60] -2025-06-07T18:12:08.178030,True,french.txt,Müller,[NAME_61] -2025-06-07T18:12:08.178030,True,french.txt,Weber,[NAME_62] -2025-06-07T18:12:08.178030,True,french.txt,Martin,[NAME_63] -2025-06-07T18:12:08.178030,True,french.txt,Ferrari,[NAME_64] -2025-06-07T18:12:08.178030,True,french.txt,Wilson,[NAME_65] -2025-06-07T18:12:08.178030,True,french.txt,hans.mueller@firma.de,[EMAIL_66] -2025-06-07T18:12:08.178030,True,french.txt,thomas.weber@firma.ch,[EMAIL_67] -2025-06-07T18:12:08.178030,True,french.txt,sophie.martin@entreprise.fr,[EMAIL_68] -2025-06-07T18:12:08.178030,True,french.txt,luca.ferrari@azienda.it,[EMAIL_69] -2025-06-07T18:12:08.178030,True,french.txt,emma.wilson@company.com,[EMAIL_70] -2025-06-07T18:12:08.178030,True,french.txt,+49 89 12345678,[PHONE_71] -2025-06-07T18:12:08.178030,True,french.txt,+41 44 234 56 78,[PHONE_72] -2025-06-07T18:12:08.178030,True,french.txt,+33 1 34 56 78 90,[PHONE_73] -2025-06-07T18:12:08.178030,True,french.txt,+39 02 2345 6789,[PHONE_74] -2025-06-07T18:12:08.178030,True,french.txt,+44 20 2345 6789,[PHONE_75] -2025-06-07T18:12:08.178030,True,french.txt,Hauptstraße 1 80331 München,[ADDRESS_76] -2025-06-07T18:12:08.178030,True,french.txt,Bahnhofstrasse 2 8001 Zürich,[ADDRESS_77] -2025-06-07T18:12:08.178030,True,french.txt,15 Avenue des Champs-Élysées 75008 Paris,[ADDRESS_78] -2025-06-07T18:12:08.178030,True,french.txt,Via Monte Napoleone 8 20121 Milano,[ADDRESS_79] -2025-06-07T18:12:08.178030,True,french.txt,25 Old Street London EC1V 9HL,[ADDRESS_80] -2025-06-07T18:12:08.178030,True,french.txt,01-234567-8,[IBAN_81] -2025-06-07T18:12:08.178030,True,french.txt,GB29 NWBK 6016 1331 9268 19 ,[IBAN_82] -2025-06-07T18:12:08.178030,True,french.txt,"9012 - -Best",[ADDRESS_83] -2025-06-07T18:12:08.178030,True,french.txt,"5678 -Address",[ADDRESS_84] -2025-06-07T18:12:08.178030,True,french.txt,contact@example.com,[EMAIL_85] -2025-06-07T18:12:08.178030,True,french.txt,max.mustermann@example.com,[EMAIL_86] -2025-06-07T18:12:08.178030,True,french.txt,+49 123 4567890,[PHONE_87] -2025-06-07T18:12:08.178030,True,french.txt,2024-03-15,[DATE_88] -2025-06-07T18:12:08.178030,True,french.txt,4111 1111 1111 1111,[IBAN_89] -2025-06-07T18:12:08.178030,True,french.txt,Tech Solutions GmbH,[NAME_90] -2025-06-07T18:12:08.178030,True,french.txt,Dr. Anna Schmidt,NAME_91 -2025-06-07T18:12:08.178030,True,french.txt,anna.schmidt@techsolutions.de,[EMAIL_92] -2025-06-07T18:12:08.178030,True,french.txt,Dr. Thomas Weber,[NAME_93] -2025-06-07T18:12:08.178030,True,french.txt,thomas.weber@company.de,[EMAIL_94] -2025-06-07T18:12:08.178030,True,french.txt,maria.schmidt@company.de,[EMAIL_95] -2025-06-07T18:12:08.178030,True,french.txt,+49 40 98765432,[PHONE_96] -2025-06-07T18:12:08.178030,True,french.txt,DE27 3704 0044 0532 0130 01,[PHONE_97] -2025-06-07T18:12:08.178030,True,french.txt,info@techinnovations.de,[EMAIL_98] -2025-06-07T18:12:08.178030,True,french.txt,+49 89 12345679,[PHONE_99] -2025-06-07T18:12:08.178030,True,french.txt,DE89 3704 0044 0532 0130 02,[PHONE_100] -2025-06-07T18:12:08.178030,True,french.txt,"9012 - -Cordialement",[ADDRESS_101] -2025-06-07T18:12:08.178030,True,french.txt,0112 3456 7890,[PHONE_102] -2025-06-07T18:12:08.178030,True,french.txt,+33 1 23 45,[PHONE_103] -2025-06-07T18:12:08.178030,True,french.txt,contact@exemple.fr,[EMAIL_104] -2025-06-07T18:12:08.201071,True,german.txt,000-180.000,[PHONE_1] -2025-06-07T18:12:08.201071,True,german.txt,2027 Bewertung,[ADDRESS_2] -2025-06-07T18:12:08.201071,True,german.txt,Max Mustermann,[NAME_3] -2025-06-07T18:12:08.201071,True,german.txt,Peter Schmid,[NAME_4] -2025-06-07T18:12:08.201071,True,german.txt,Marie Dupont,[NAME_5] -2025-06-07T18:12:08.201071,True,german.txt,Marco Rossi,[NAME_6] -2025-06-07T18:12:08.201071,True,german.txt,John Smith,[NAME_7] -2025-06-07T18:12:08.201071,True,german.txt,max.mustermann@beispiel.de,[EMAIL_8] -2025-06-07T18:12:08.201071,True,german.txt,peter.schmid@beispiel.ch,[EMAIL_9] -2025-06-07T18:12:08.201071,True,german.txt,marie.dupont@exemple.fr,[EMAIL_10] -2025-06-07T18:12:08.201071,True,german.txt,marco.rossi@esempio.it,[EMAIL_11] -2025-06-07T18:12:08.201071,True,german.txt,john.smith@example.com,[EMAIL_12] -2025-06-07T18:12:08.201071,True,german.txt,+49 30 12345678,[PHONE_13] -2025-06-07T18:12:08.201071,True,german.txt,+41 44 123 45 67,[PHONE_14] -2025-06-07T18:12:08.201071,True,german.txt,+33 1 23 45 67 89,[PHONE_15] -2025-06-07T18:12:08.201071,True,german.txt,+39 02 1234 5678,[PHONE_16] -2025-06-07T18:12:08.201071,True,german.txt,+44 20 1234 5678,[PHONE_17] -2025-06-07T18:12:08.201071,True,german.txt,Musterstraße 123 12345 Berlin,[ADDRESS_18] -2025-06-07T18:12:08.201071,True,german.txt,Bahnhofstrasse 1 8001 Zürich,[ADDRESS_19] -2025-06-07T18:12:08.201071,True,german.txt,123 Rue de Paris 75001 Paris,[ADDRESS_20] -2025-06-07T18:12:08.201071,True,german.txt,Via Roma 123 20100 Milano,[ADDRESS_21] -2025-06-07T18:12:08.201071,True,german.txt,123 High Street London SW1A 1AA,[ADDRESS_22] -2025-06-07T18:12:08.201071,True,german.txt,DE89 3704 0044 0532 0130 00,[IBAN_23] -2025-06-07T18:12:08.201071,True,german.txt,CH93 0076 7000 E529 3557 7,[IBAN_24] -2025-06-07T18:12:08.201071,True,german.txt,FR76 3000 6000 0112 3456 7890 189,[IBAN_25] -2025-06-07T18:12:08.201071,True,german.txt,IT60 X054 2811 1010 0000 0123 456,[IBAN_26] -2025-06-07T18:12:08.201071,True,german.txt,GB29 NWBK 6016 1331 9268 19,[IBAN_27] -2025-06-07T18:12:08.201071,True,german.txt,4532 1234 5678 9012,[IBAN_28] -2025-06-07T18:12:08.201071,True,german.txt,4532 1234 5678 9013,[IBAN_29] -2025-06-07T18:12:08.201071,True,german.txt,4532 1234 5678 9014,[IBAN_30] -2025-06-07T18:12:08.201071,True,german.txt,4532 1234 5678 9015,[IBAN_31] -2025-06-07T18:12:08.201071,True,german.txt,4532 1234 5678 9016,[IBAN_32] -2025-06-07T18:12:08.201071,True,german.txt,nan,[SSN_33] -2025-06-07T18:12:08.201071,True,german.txt,756.1234.5678.90,[SSN_34] -2025-06-07T18:12:08.201071,True,german.txt, ,[SSN_35] -2025-06-07T18:12:08.201071,True,german.txt,sarah.weber@techsolutions.ch,[EMAIL_36] -2025-06-07T18:12:08.201071,True,german.txt,+41 44 987 65,[PHONE_37] -2025-06-07T18:12:08.201071,True,german.txt,"Dr. Sarah Weber -TechSolutions",[NAME_38] -2025-06-07T18:12:08.201071,True,german.txt,hans.mueller@ethz.ch,[EMAIL_39] -2025-06-07T18:12:08.201071,True,german.txt,+41 44 123 45,[PHONE_40] -2025-06-07T18:12:08.201071,True,german.txt,l.meier@digitalsystems.ch,[EMAIL_41] -2025-06-07T18:12:08.201071,True,german.txt,+41 44 456 78,[PHONE_42] -2025-06-07T18:12:08.201071,True,german.txt,987.654.321,[SSN_43] -2025-06-07T18:12:08.201071,True,german.txt,CHE-987.654.321,[SSN_44] -2025-06-07T18:12:08.201071,True,german.txt,8002 Zürich,[ADDRESS_45] -2025-06-07T18:12:08.201071,True,german.txt,Musterstrasse 123,[ADDRESS_46] -2025-06-07T18:12:08.201071,True,german.txt,lara.meier@techsolutions.ch,[EMAIL_47] -2025-06-07T18:12:08.201071,True,german.txt,123.456.789,[SSN_48] -2025-06-07T18:12:08.201071,True,german.txt,CHE-123.456.789,[SSN_49] -2025-06-07T18:12:08.201071,True,german.txt,8004 Zürich,[ADDRESS_50] -2025-06-07T18:12:08.201071,True,german.txt,Industriestrasse 100,[ADDRESS_51] -2025-06-07T18:12:08.201071,True,german.txt,lara.meier@example.ch,[EMAIL_52] -2025-06-07T18:12:08.201071,True,german.txt,8001 Zürich,[ADDRESS_53] -2025-06-07T18:12:08.201071,True,german.txt,Bahnhofstrasse 45,[ADDRESS_54] -2025-06-07T18:12:08.201071,True,german.txt,"1990 -Adresse",[ADDRESS_55] -2025-06-07T18:12:08.201071,True,german.txt,Hans,[NAME_56] -2025-06-07T18:12:08.201071,True,german.txt,Thomas,[NAME_57] -2025-06-07T18:12:08.201071,True,german.txt,Sophie,[NAME_58] -2025-06-07T18:12:08.201071,True,german.txt,Luca,[NAME_59] -2025-06-07T18:12:08.201071,True,german.txt,Emma,[NAME_60] -2025-06-07T18:12:08.201071,True,german.txt,Müller,[NAME_61] -2025-06-07T18:12:08.201071,True,german.txt,Weber,[NAME_62] -2025-06-07T18:12:08.201071,True,german.txt,Martin,[NAME_63] -2025-06-07T18:12:08.201071,True,german.txt,Ferrari,[NAME_64] -2025-06-07T18:12:08.201071,True,german.txt,Wilson,[NAME_65] -2025-06-07T18:12:08.201071,True,german.txt,hans.mueller@firma.de,[EMAIL_66] -2025-06-07T18:12:08.201071,True,german.txt,thomas.weber@firma.ch,[EMAIL_67] -2025-06-07T18:12:08.201071,True,german.txt,sophie.martin@entreprise.fr,[EMAIL_68] -2025-06-07T18:12:08.201071,True,german.txt,luca.ferrari@azienda.it,[EMAIL_69] -2025-06-07T18:12:08.201071,True,german.txt,emma.wilson@company.com,[EMAIL_70] -2025-06-07T18:12:08.201071,True,german.txt,+49 89 12345678,[PHONE_71] -2025-06-07T18:12:08.201071,True,german.txt,+41 44 234 56 78,[PHONE_72] -2025-06-07T18:12:08.201071,True,german.txt,+33 1 34 56 78 90,[PHONE_73] -2025-06-07T18:12:08.201071,True,german.txt,+39 02 2345 6789,[PHONE_74] -2025-06-07T18:12:08.201071,True,german.txt,+44 20 2345 6789,[PHONE_75] -2025-06-07T18:12:08.201071,True,german.txt,Hauptstraße 1 80331 München,[ADDRESS_76] -2025-06-07T18:12:08.201071,True,german.txt,Bahnhofstrasse 2 8001 Zürich,[ADDRESS_77] -2025-06-07T18:12:08.201071,True,german.txt,15 Avenue des Champs-Élysées 75008 Paris,[ADDRESS_78] -2025-06-07T18:12:08.201071,True,german.txt,Via Monte Napoleone 8 20121 Milano,[ADDRESS_79] -2025-06-07T18:12:08.201071,True,german.txt,25 Old Street London EC1V 9HL,[ADDRESS_80] -2025-06-07T18:12:08.201071,True,german.txt,01-234567-8,[IBAN_81] -2025-06-07T18:12:08.201071,True,german.txt,GB29 NWBK 6016 1331 9268 19 ,[IBAN_82] -2025-06-07T18:12:08.201071,True,german.txt,"9012 - -Best",[ADDRESS_83] -2025-06-07T18:12:08.201071,True,german.txt,"5678 -Address",[ADDRESS_84] -2025-06-07T18:12:08.201071,True,german.txt,contact@example.com,[EMAIL_85] -2025-06-07T18:12:08.201071,True,german.txt,max.mustermann@example.com,[EMAIL_86] -2025-06-07T18:12:08.201071,True,german.txt,+49 123 4567890,[PHONE_87] -2025-06-07T18:12:08.201071,True,german.txt,2024-03-15,[DATE_88] -2025-06-07T18:12:08.201071,True,german.txt,4111 1111 1111 1111,[IBAN_89] -2025-06-07T18:12:08.201071,True,german.txt,Tech Solutions GmbH,[NAME_90] -2025-06-07T18:12:08.201071,True,german.txt,Dr. Anna Schmidt,NAME_91 -2025-06-07T18:12:08.201071,True,german.txt,anna.schmidt@techsolutions.de,[EMAIL_92] -2025-06-07T18:12:08.201071,True,german.txt,Dr. Thomas Weber,[NAME_93] -2025-06-07T18:12:08.201071,True,german.txt,thomas.weber@company.de,[EMAIL_94] -2025-06-07T18:12:08.201071,True,german.txt,maria.schmidt@company.de,[EMAIL_95] -2025-06-07T18:12:08.201071,True,german.txt,+49 40 98765432,[PHONE_96] -2025-06-07T18:12:08.201071,True,german.txt,DE27 3704 0044 0532 0130 01,[PHONE_97] -2025-06-07T18:12:08.201071,True,german.txt,info@techinnovations.de,[EMAIL_98] -2025-06-07T18:12:08.201071,True,german.txt,+49 89 12345679,[PHONE_99] -2025-06-07T18:12:08.201071,True,german.txt,DE89 3704 0044 0532 0130 02,[PHONE_100] -2025-06-07T18:12:08.201071,True,german.txt,"9012 - -Cordialement",[ADDRESS_101] -2025-06-07T18:12:08.201071,True,german.txt,0112 3456 7890,[PHONE_102] -2025-06-07T18:12:08.201071,True,german.txt,+33 1 23 45,[PHONE_103] -2025-06-07T18:12:08.201071,True,german.txt,contact@exemple.fr,[EMAIL_104] -2025-06-07T18:12:08.201071,True,german.txt,"9012 - -Mit",[ADDRESS_105] -2025-06-07T18:12:08.201071,True,german.txt,0532 0130 00,[PHONE_106] -2025-06-07T18:12:08.201071,True,german.txt,0044 0532 0130,[PHONE_107] -2025-06-07T18:12:08.201071,True,german.txt,030-12345678,[PHONE_108] -2025-06-07T18:12:08.228652,True,geschaeftsstrategie.txt,000-180.000,[PHONE_1] -2025-06-07T18:12:08.228652,True,geschaeftsstrategie.txt,2027 Bewertung,[ADDRESS_2] -2025-06-07T18:12:08.228652,True,geschaeftsstrategie.txt,Max Mustermann,[NAME_3] -2025-06-07T18:12:08.228652,True,geschaeftsstrategie.txt,Peter Schmid,[NAME_4] -2025-06-07T18:12:08.228652,True,geschaeftsstrategie.txt,Marie Dupont,[NAME_5] -2025-06-07T18:12:08.228652,True,geschaeftsstrategie.txt,Marco Rossi,[NAME_6] -2025-06-07T18:12:08.228652,True,geschaeftsstrategie.txt,John Smith,[NAME_7] -2025-06-07T18:12:08.228652,True,geschaeftsstrategie.txt,max.mustermann@beispiel.de,[EMAIL_8] -2025-06-07T18:12:08.228652,True,geschaeftsstrategie.txt,peter.schmid@beispiel.ch,[EMAIL_9] -2025-06-07T18:12:08.228652,True,geschaeftsstrategie.txt,marie.dupont@exemple.fr,[EMAIL_10] -2025-06-07T18:12:08.228652,True,geschaeftsstrategie.txt,marco.rossi@esempio.it,[EMAIL_11] -2025-06-07T18:12:08.228652,True,geschaeftsstrategie.txt,john.smith@example.com,[EMAIL_12] -2025-06-07T18:12:08.228652,True,geschaeftsstrategie.txt,+49 30 12345678,[PHONE_13] -2025-06-07T18:12:08.228652,True,geschaeftsstrategie.txt,+41 44 123 45 67,[PHONE_14] -2025-06-07T18:12:08.228652,True,geschaeftsstrategie.txt,+33 1 23 45 67 89,[PHONE_15] -2025-06-07T18:12:08.228652,True,geschaeftsstrategie.txt,+39 02 1234 5678,[PHONE_16] -2025-06-07T18:12:08.228652,True,geschaeftsstrategie.txt,+44 20 1234 5678,[PHONE_17] -2025-06-07T18:12:08.228652,True,geschaeftsstrategie.txt,Musterstraße 123 12345 Berlin,[ADDRESS_18] -2025-06-07T18:12:08.228652,True,geschaeftsstrategie.txt,Bahnhofstrasse 1 8001 Zürich,[ADDRESS_19] -2025-06-07T18:12:08.228652,True,geschaeftsstrategie.txt,123 Rue de Paris 75001 Paris,[ADDRESS_20] -2025-06-07T18:12:08.228652,True,geschaeftsstrategie.txt,Via Roma 123 20100 Milano,[ADDRESS_21] -2025-06-07T18:12:08.228652,True,geschaeftsstrategie.txt,123 High Street London SW1A 1AA,[ADDRESS_22] -2025-06-07T18:12:08.228652,True,geschaeftsstrategie.txt,DE89 3704 0044 0532 0130 00,[IBAN_23] -2025-06-07T18:12:08.228652,True,geschaeftsstrategie.txt,CH93 0076 7000 E529 3557 7,[IBAN_24] -2025-06-07T18:12:08.228652,True,geschaeftsstrategie.txt,FR76 3000 6000 0112 3456 7890 189,[IBAN_25] -2025-06-07T18:12:08.228652,True,geschaeftsstrategie.txt,IT60 X054 2811 1010 0000 0123 456,[IBAN_26] -2025-06-07T18:12:08.228652,True,geschaeftsstrategie.txt,GB29 NWBK 6016 1331 9268 19,[IBAN_27] -2025-06-07T18:12:08.228652,True,geschaeftsstrategie.txt,4532 1234 5678 9012,[IBAN_28] -2025-06-07T18:12:08.228652,True,geschaeftsstrategie.txt,4532 1234 5678 9013,[IBAN_29] -2025-06-07T18:12:08.228652,True,geschaeftsstrategie.txt,4532 1234 5678 9014,[IBAN_30] -2025-06-07T18:12:08.228652,True,geschaeftsstrategie.txt,4532 1234 5678 9015,[IBAN_31] -2025-06-07T18:12:08.228652,True,geschaeftsstrategie.txt,4532 1234 5678 9016,[IBAN_32] -2025-06-07T18:12:08.228652,True,geschaeftsstrategie.txt,nan,[SSN_33] -2025-06-07T18:12:08.228652,True,geschaeftsstrategie.txt,756.1234.5678.90,[SSN_34] -2025-06-07T18:12:08.228652,True,geschaeftsstrategie.txt, ,[SSN_35] -2025-06-07T18:12:08.228652,True,geschaeftsstrategie.txt,sarah.weber@techsolutions.ch,[EMAIL_36] -2025-06-07T18:12:08.228652,True,geschaeftsstrategie.txt,+41 44 987 65,[PHONE_37] -2025-06-07T18:12:08.228652,True,geschaeftsstrategie.txt,"Dr. Sarah Weber -TechSolutions",[NAME_38] -2025-06-07T18:12:08.228652,True,geschaeftsstrategie.txt,hans.mueller@ethz.ch,[EMAIL_39] -2025-06-07T18:12:08.228652,True,geschaeftsstrategie.txt,+41 44 123 45,[PHONE_40] -2025-06-07T18:12:08.228652,True,geschaeftsstrategie.txt,l.meier@digitalsystems.ch,[EMAIL_41] -2025-06-07T18:12:08.228652,True,geschaeftsstrategie.txt,+41 44 456 78,[PHONE_42] -2025-06-07T18:12:08.228652,True,geschaeftsstrategie.txt,987.654.321,[SSN_43] -2025-06-07T18:12:08.228652,True,geschaeftsstrategie.txt,CHE-987.654.321,[SSN_44] -2025-06-07T18:12:08.228652,True,geschaeftsstrategie.txt,8002 Zürich,[ADDRESS_45] -2025-06-07T18:12:08.228652,True,geschaeftsstrategie.txt,Musterstrasse 123,[ADDRESS_46] -2025-06-07T18:12:08.228652,True,geschaeftsstrategie.txt,lara.meier@techsolutions.ch,[EMAIL_47] -2025-06-07T18:12:08.228652,True,geschaeftsstrategie.txt,123.456.789,[SSN_48] -2025-06-07T18:12:08.228652,True,geschaeftsstrategie.txt,CHE-123.456.789,[SSN_49] -2025-06-07T18:12:08.228652,True,geschaeftsstrategie.txt,8004 Zürich,[ADDRESS_50] -2025-06-07T18:12:08.228652,True,geschaeftsstrategie.txt,Industriestrasse 100,[ADDRESS_51] -2025-06-07T18:12:08.228652,True,geschaeftsstrategie.txt,lara.meier@example.ch,[EMAIL_52] -2025-06-07T18:12:08.228652,True,geschaeftsstrategie.txt,8001 Zürich,[ADDRESS_53] -2025-06-07T18:12:08.228652,True,geschaeftsstrategie.txt,Bahnhofstrasse 45,[ADDRESS_54] -2025-06-07T18:12:08.228652,True,geschaeftsstrategie.txt,"1990 -Adresse",[ADDRESS_55] -2025-06-07T18:12:08.228652,True,geschaeftsstrategie.txt,Hans,[NAME_56] -2025-06-07T18:12:08.228652,True,geschaeftsstrategie.txt,Thomas,[NAME_57] -2025-06-07T18:12:08.228652,True,geschaeftsstrategie.txt,Sophie,[NAME_58] -2025-06-07T18:12:08.228652,True,geschaeftsstrategie.txt,Luca,[NAME_59] -2025-06-07T18:12:08.228652,True,geschaeftsstrategie.txt,Emma,[NAME_60] -2025-06-07T18:12:08.228652,True,geschaeftsstrategie.txt,Müller,[NAME_61] -2025-06-07T18:12:08.228652,True,geschaeftsstrategie.txt,Weber,[NAME_62] -2025-06-07T18:12:08.228652,True,geschaeftsstrategie.txt,Martin,[NAME_63] -2025-06-07T18:12:08.228652,True,geschaeftsstrategie.txt,Ferrari,[NAME_64] -2025-06-07T18:12:08.228652,True,geschaeftsstrategie.txt,Wilson,[NAME_65] -2025-06-07T18:12:08.228652,True,geschaeftsstrategie.txt,hans.mueller@firma.de,[EMAIL_66] -2025-06-07T18:12:08.228652,True,geschaeftsstrategie.txt,thomas.weber@firma.ch,[EMAIL_67] -2025-06-07T18:12:08.228652,True,geschaeftsstrategie.txt,sophie.martin@entreprise.fr,[EMAIL_68] -2025-06-07T18:12:08.228652,True,geschaeftsstrategie.txt,luca.ferrari@azienda.it,[EMAIL_69] -2025-06-07T18:12:08.228652,True,geschaeftsstrategie.txt,emma.wilson@company.com,[EMAIL_70] -2025-06-07T18:12:08.228652,True,geschaeftsstrategie.txt,+49 89 12345678,[PHONE_71] -2025-06-07T18:12:08.228652,True,geschaeftsstrategie.txt,+41 44 234 56 78,[PHONE_72] -2025-06-07T18:12:08.228652,True,geschaeftsstrategie.txt,+33 1 34 56 78 90,[PHONE_73] -2025-06-07T18:12:08.228652,True,geschaeftsstrategie.txt,+39 02 2345 6789,[PHONE_74] -2025-06-07T18:12:08.228652,True,geschaeftsstrategie.txt,+44 20 2345 6789,[PHONE_75] -2025-06-07T18:12:08.228652,True,geschaeftsstrategie.txt,Hauptstraße 1 80331 München,[ADDRESS_76] -2025-06-07T18:12:08.228652,True,geschaeftsstrategie.txt,Bahnhofstrasse 2 8001 Zürich,[ADDRESS_77] -2025-06-07T18:12:08.228652,True,geschaeftsstrategie.txt,15 Avenue des Champs-Élysées 75008 Paris,[ADDRESS_78] -2025-06-07T18:12:08.228652,True,geschaeftsstrategie.txt,Via Monte Napoleone 8 20121 Milano,[ADDRESS_79] -2025-06-07T18:12:08.228652,True,geschaeftsstrategie.txt,25 Old Street London EC1V 9HL,[ADDRESS_80] -2025-06-07T18:12:08.228652,True,geschaeftsstrategie.txt,01-234567-8,[IBAN_81] -2025-06-07T18:12:08.228652,True,geschaeftsstrategie.txt,GB29 NWBK 6016 1331 9268 19 ,[IBAN_82] -2025-06-07T18:12:08.228652,True,geschaeftsstrategie.txt,"9012 - -Best",[ADDRESS_83] -2025-06-07T18:12:08.228652,True,geschaeftsstrategie.txt,"5678 -Address",[ADDRESS_84] -2025-06-07T18:12:08.228652,True,geschaeftsstrategie.txt,contact@example.com,[EMAIL_85] -2025-06-07T18:12:08.228652,True,geschaeftsstrategie.txt,max.mustermann@example.com,[EMAIL_86] -2025-06-07T18:12:08.228652,True,geschaeftsstrategie.txt,+49 123 4567890,[PHONE_87] -2025-06-07T18:12:08.228652,True,geschaeftsstrategie.txt,2024-03-15,[DATE_88] -2025-06-07T18:12:08.228652,True,geschaeftsstrategie.txt,4111 1111 1111 1111,[IBAN_89] -2025-06-07T18:12:08.228652,True,geschaeftsstrategie.txt,Tech Solutions GmbH,[NAME_90] -2025-06-07T18:12:08.228652,True,geschaeftsstrategie.txt,Dr. Anna Schmidt,NAME_91 -2025-06-07T18:12:08.228652,True,geschaeftsstrategie.txt,anna.schmidt@techsolutions.de,[EMAIL_92] -2025-06-07T18:12:08.228652,True,geschaeftsstrategie.txt,Dr. Thomas Weber,[NAME_93] -2025-06-07T18:12:08.228652,True,geschaeftsstrategie.txt,thomas.weber@company.de,[EMAIL_94] -2025-06-07T18:12:08.228652,True,geschaeftsstrategie.txt,maria.schmidt@company.de,[EMAIL_95] -2025-06-07T18:12:08.228652,True,geschaeftsstrategie.txt,+49 40 98765432,[PHONE_96] -2025-06-07T18:12:08.228652,True,geschaeftsstrategie.txt,DE27 3704 0044 0532 0130 01,[PHONE_97] -2025-06-07T18:12:08.228652,True,geschaeftsstrategie.txt,info@techinnovations.de,[EMAIL_98] -2025-06-07T18:12:08.228652,True,geschaeftsstrategie.txt,+49 89 12345679,[PHONE_99] -2025-06-07T18:12:08.228652,True,geschaeftsstrategie.txt,DE89 3704 0044 0532 0130 02,[PHONE_100] -2025-06-07T18:12:08.228652,True,geschaeftsstrategie.txt,"9012 - -Cordialement",[ADDRESS_101] -2025-06-07T18:12:08.228652,True,geschaeftsstrategie.txt,0112 3456 7890,[PHONE_102] -2025-06-07T18:12:08.228652,True,geschaeftsstrategie.txt,+33 1 23 45,[PHONE_103] -2025-06-07T18:12:08.228652,True,geschaeftsstrategie.txt,contact@exemple.fr,[EMAIL_104] -2025-06-07T18:12:08.228652,True,geschaeftsstrategie.txt,"9012 - -Mit",[ADDRESS_105] -2025-06-07T18:12:08.228652,True,geschaeftsstrategie.txt,0532 0130 00,[PHONE_106] -2025-06-07T18:12:08.228652,True,geschaeftsstrategie.txt,0044 0532 0130,[PHONE_107] -2025-06-07T18:12:08.228652,True,geschaeftsstrategie.txt,030-12345678,[PHONE_108] -2025-06-07T18:12:08.228652,True,geschaeftsstrategie.txt,sarah.mueller@techsolutions.ch,[EMAIL_109] -2025-06-07T18:12:08.228652,True,geschaeftsstrategie.txt,thomas.weber@techsolutions.ch,[EMAIL_110] -2025-06-07T18:12:08.228652,True,geschaeftsstrategie.txt,info@techsolutions.ch,[EMAIL_111] -2025-06-07T18:12:08.228652,True,geschaeftsstrategie.txt,"2026 -TechSolutions",[ADDRESS_112] -2025-06-07T18:12:08.255652,True,geschäfte.csv,000-180.000,[PHONE_1] -2025-06-07T18:12:08.255652,True,geschäfte.csv,2027 Bewertung,[ADDRESS_2] -2025-06-07T18:12:08.255652,True,geschäfte.csv,Max Mustermann,[NAME_3] -2025-06-07T18:12:08.255652,True,geschäfte.csv,Peter Schmid,[NAME_4] -2025-06-07T18:12:08.255652,True,geschäfte.csv,Marie Dupont,[NAME_5] -2025-06-07T18:12:08.255652,True,geschäfte.csv,Marco Rossi,[NAME_6] -2025-06-07T18:12:08.255652,True,geschäfte.csv,John Smith,[NAME_7] -2025-06-07T18:12:08.255652,True,geschäfte.csv,max.mustermann@beispiel.de,[EMAIL_8] -2025-06-07T18:12:08.255652,True,geschäfte.csv,peter.schmid@beispiel.ch,[EMAIL_9] -2025-06-07T18:12:08.255652,True,geschäfte.csv,marie.dupont@exemple.fr,[EMAIL_10] -2025-06-07T18:12:08.255652,True,geschäfte.csv,marco.rossi@esempio.it,[EMAIL_11] -2025-06-07T18:12:08.255652,True,geschäfte.csv,john.smith@example.com,[EMAIL_12] -2025-06-07T18:12:08.255652,True,geschäfte.csv,+49 30 12345678,[PHONE_13] -2025-06-07T18:12:08.255652,True,geschäfte.csv,+41 44 123 45 67,[PHONE_14] -2025-06-07T18:12:08.255652,True,geschäfte.csv,+33 1 23 45 67 89,[PHONE_15] -2025-06-07T18:12:08.255652,True,geschäfte.csv,+39 02 1234 5678,[PHONE_16] -2025-06-07T18:12:08.255652,True,geschäfte.csv,+44 20 1234 5678,[PHONE_17] -2025-06-07T18:12:08.255652,True,geschäfte.csv,Musterstraße 123 12345 Berlin,[ADDRESS_18] -2025-06-07T18:12:08.255652,True,geschäfte.csv,Bahnhofstrasse 1 8001 Zürich,[ADDRESS_19] -2025-06-07T18:12:08.255652,True,geschäfte.csv,123 Rue de Paris 75001 Paris,[ADDRESS_20] -2025-06-07T18:12:08.255652,True,geschäfte.csv,Via Roma 123 20100 Milano,[ADDRESS_21] -2025-06-07T18:12:08.255652,True,geschäfte.csv,123 High Street London SW1A 1AA,[ADDRESS_22] -2025-06-07T18:12:08.255652,True,geschäfte.csv,DE89 3704 0044 0532 0130 00,[IBAN_23] -2025-06-07T18:12:08.255652,True,geschäfte.csv,CH93 0076 7000 E529 3557 7,[IBAN_24] -2025-06-07T18:12:08.255652,True,geschäfte.csv,FR76 3000 6000 0112 3456 7890 189,[IBAN_25] -2025-06-07T18:12:08.255652,True,geschäfte.csv,IT60 X054 2811 1010 0000 0123 456,[IBAN_26] -2025-06-07T18:12:08.255652,True,geschäfte.csv,GB29 NWBK 6016 1331 9268 19,[IBAN_27] -2025-06-07T18:12:08.255652,True,geschäfte.csv,4532 1234 5678 9012,[IBAN_28] -2025-06-07T18:12:08.255652,True,geschäfte.csv,4532 1234 5678 9013,[IBAN_29] -2025-06-07T18:12:08.255652,True,geschäfte.csv,4532 1234 5678 9014,[IBAN_30] -2025-06-07T18:12:08.255652,True,geschäfte.csv,4532 1234 5678 9015,[IBAN_31] -2025-06-07T18:12:08.255652,True,geschäfte.csv,4532 1234 5678 9016,[IBAN_32] -2025-06-07T18:12:08.255652,True,geschäfte.csv,nan,[SSN_33] -2025-06-07T18:12:08.255652,True,geschäfte.csv,756.1234.5678.90,[SSN_34] -2025-06-07T18:12:08.255652,True,geschäfte.csv, ,[SSN_35] -2025-06-07T18:12:08.255652,True,geschäfte.csv,sarah.weber@techsolutions.ch,[EMAIL_36] -2025-06-07T18:12:08.255652,True,geschäfte.csv,+41 44 987 65,[PHONE_37] -2025-06-07T18:12:08.255652,True,geschäfte.csv,"Dr. Sarah Weber -TechSolutions",[NAME_38] -2025-06-07T18:12:08.255652,True,geschäfte.csv,hans.mueller@ethz.ch,[EMAIL_39] -2025-06-07T18:12:08.255652,True,geschäfte.csv,+41 44 123 45,[PHONE_40] -2025-06-07T18:12:08.255652,True,geschäfte.csv,l.meier@digitalsystems.ch,[EMAIL_41] -2025-06-07T18:12:08.255652,True,geschäfte.csv,+41 44 456 78,[PHONE_42] -2025-06-07T18:12:08.255652,True,geschäfte.csv,987.654.321,[SSN_43] -2025-06-07T18:12:08.255652,True,geschäfte.csv,CHE-987.654.321,[SSN_44] -2025-06-07T18:12:08.255652,True,geschäfte.csv,8002 Zürich,[ADDRESS_45] -2025-06-07T18:12:08.255652,True,geschäfte.csv,Musterstrasse 123,[ADDRESS_46] -2025-06-07T18:12:08.255652,True,geschäfte.csv,lara.meier@techsolutions.ch,[EMAIL_47] -2025-06-07T18:12:08.255652,True,geschäfte.csv,123.456.789,[SSN_48] -2025-06-07T18:12:08.255652,True,geschäfte.csv,CHE-123.456.789,[SSN_49] -2025-06-07T18:12:08.255652,True,geschäfte.csv,8004 Zürich,[ADDRESS_50] -2025-06-07T18:12:08.255652,True,geschäfte.csv,Industriestrasse 100,[ADDRESS_51] -2025-06-07T18:12:08.255652,True,geschäfte.csv,lara.meier@example.ch,[EMAIL_52] -2025-06-07T18:12:08.255652,True,geschäfte.csv,8001 Zürich,[ADDRESS_53] -2025-06-07T18:12:08.255652,True,geschäfte.csv,Bahnhofstrasse 45,[ADDRESS_54] -2025-06-07T18:12:08.255652,True,geschäfte.csv,"1990 -Adresse",[ADDRESS_55] -2025-06-07T18:12:08.255652,True,geschäfte.csv,Hans,[NAME_56] -2025-06-07T18:12:08.255652,True,geschäfte.csv,Thomas,[NAME_57] -2025-06-07T18:12:08.255652,True,geschäfte.csv,Sophie,[NAME_58] -2025-06-07T18:12:08.255652,True,geschäfte.csv,Luca,[NAME_59] -2025-06-07T18:12:08.255652,True,geschäfte.csv,Emma,[NAME_60] -2025-06-07T18:12:08.255652,True,geschäfte.csv,Müller,[NAME_61] -2025-06-07T18:12:08.255652,True,geschäfte.csv,Weber,[NAME_62] -2025-06-07T18:12:08.255652,True,geschäfte.csv,Martin,[NAME_63] -2025-06-07T18:12:08.255652,True,geschäfte.csv,Ferrari,[NAME_64] -2025-06-07T18:12:08.255652,True,geschäfte.csv,Wilson,[NAME_65] -2025-06-07T18:12:08.255652,True,geschäfte.csv,hans.mueller@firma.de,[EMAIL_66] -2025-06-07T18:12:08.255652,True,geschäfte.csv,thomas.weber@firma.ch,[EMAIL_67] -2025-06-07T18:12:08.255652,True,geschäfte.csv,sophie.martin@entreprise.fr,[EMAIL_68] -2025-06-07T18:12:08.255652,True,geschäfte.csv,luca.ferrari@azienda.it,[EMAIL_69] -2025-06-07T18:12:08.255652,True,geschäfte.csv,emma.wilson@company.com,[EMAIL_70] -2025-06-07T18:12:08.255652,True,geschäfte.csv,+49 89 12345678,[PHONE_71] -2025-06-07T18:12:08.255652,True,geschäfte.csv,+41 44 234 56 78,[PHONE_72] -2025-06-07T18:12:08.255652,True,geschäfte.csv,+33 1 34 56 78 90,[PHONE_73] -2025-06-07T18:12:08.255652,True,geschäfte.csv,+39 02 2345 6789,[PHONE_74] -2025-06-07T18:12:08.255652,True,geschäfte.csv,+44 20 2345 6789,[PHONE_75] -2025-06-07T18:12:08.255652,True,geschäfte.csv,Hauptstraße 1 80331 München,[ADDRESS_76] -2025-06-07T18:12:08.255652,True,geschäfte.csv,Bahnhofstrasse 2 8001 Zürich,[ADDRESS_77] -2025-06-07T18:12:08.255652,True,geschäfte.csv,15 Avenue des Champs-Élysées 75008 Paris,[ADDRESS_78] -2025-06-07T18:12:08.255652,True,geschäfte.csv,Via Monte Napoleone 8 20121 Milano,[ADDRESS_79] -2025-06-07T18:12:08.255652,True,geschäfte.csv,25 Old Street London EC1V 9HL,[ADDRESS_80] -2025-06-07T18:12:08.255652,True,geschäfte.csv,01-234567-8,[IBAN_81] -2025-06-07T18:12:08.255652,True,geschäfte.csv,GB29 NWBK 6016 1331 9268 19 ,[IBAN_82] -2025-06-07T18:12:08.255652,True,geschäfte.csv,"9012 - -Best",[ADDRESS_83] -2025-06-07T18:12:08.255652,True,geschäfte.csv,"5678 -Address",[ADDRESS_84] -2025-06-07T18:12:08.255652,True,geschäfte.csv,contact@example.com,[EMAIL_85] -2025-06-07T18:12:08.255652,True,geschäfte.csv,max.mustermann@example.com,[EMAIL_86] -2025-06-07T18:12:08.255652,True,geschäfte.csv,+49 123 4567890,[PHONE_87] -2025-06-07T18:12:08.255652,True,geschäfte.csv,2024-03-15,[DATE_88] -2025-06-07T18:12:08.255652,True,geschäfte.csv,4111 1111 1111 1111,[IBAN_89] -2025-06-07T18:12:08.255652,True,geschäfte.csv,Tech Solutions GmbH,[NAME_90] -2025-06-07T18:12:08.255652,True,geschäfte.csv,Dr. Anna Schmidt,NAME_91 -2025-06-07T18:12:08.255652,True,geschäfte.csv,anna.schmidt@techsolutions.de,[EMAIL_92] -2025-06-07T18:12:08.255652,True,geschäfte.csv,Dr. Thomas Weber,[NAME_93] -2025-06-07T18:12:08.255652,True,geschäfte.csv,thomas.weber@company.de,[EMAIL_94] -2025-06-07T18:12:08.255652,True,geschäfte.csv,maria.schmidt@company.de,[EMAIL_95] -2025-06-07T18:12:08.255652,True,geschäfte.csv,+49 40 98765432,[PHONE_96] -2025-06-07T18:12:08.255652,True,geschäfte.csv,DE27 3704 0044 0532 0130 01,[PHONE_97] -2025-06-07T18:12:08.255652,True,geschäfte.csv,info@techinnovations.de,[EMAIL_98] -2025-06-07T18:12:08.255652,True,geschäfte.csv,+49 89 12345679,[PHONE_99] -2025-06-07T18:12:08.255652,True,geschäfte.csv,DE89 3704 0044 0532 0130 02,[PHONE_100] -2025-06-07T18:12:08.255652,True,geschäfte.csv,"9012 - -Cordialement",[ADDRESS_101] -2025-06-07T18:12:08.255652,True,geschäfte.csv,0112 3456 7890,[PHONE_102] -2025-06-07T18:12:08.255652,True,geschäfte.csv,+33 1 23 45,[PHONE_103] -2025-06-07T18:12:08.255652,True,geschäfte.csv,contact@exemple.fr,[EMAIL_104] -2025-06-07T18:12:08.255652,True,geschäfte.csv,"9012 - -Mit",[ADDRESS_105] -2025-06-07T18:12:08.255652,True,geschäfte.csv,0532 0130 00,[PHONE_106] -2025-06-07T18:12:08.255652,True,geschäfte.csv,0044 0532 0130,[PHONE_107] -2025-06-07T18:12:08.255652,True,geschäfte.csv,030-12345678,[PHONE_108] -2025-06-07T18:12:08.255652,True,geschäfte.csv,sarah.mueller@techsolutions.ch,[EMAIL_109] -2025-06-07T18:12:08.255652,True,geschäfte.csv,thomas.weber@techsolutions.ch,[EMAIL_110] -2025-06-07T18:12:08.255652,True,geschäfte.csv,info@techsolutions.ch,[EMAIL_111] -2025-06-07T18:12:08.255652,True,geschäfte.csv,"2026 -TechSolutions",[ADDRESS_112] -2025-06-07T18:12:08.255652,True,geschäfte.csv,Anna Schmidt,[DATE_113] -2025-06-07T18:12:08.255652,True,geschäfte.csv,Franz Huber,[DATE_114] -2025-06-07T18:12:08.255652,True,geschäfte.csv,Pierre Dubois,[DATE_115] -2025-06-07T18:12:08.255652,True,geschäfte.csv,Giovanni Bianchi,[DATE_116] -2025-06-07T18:12:08.255652,True,geschäfte.csv,William Brown,[DATE_117] -2025-06-07T18:12:08.255652,True,geschäfte.csv,anna.schmidt@kunde.de,[NAME_118] -2025-06-07T18:12:08.255652,True,geschäfte.csv,franz.huber@kunde.de,[NAME_119] -2025-06-07T18:12:08.255652,True,geschäfte.csv,pierre.dubois@kunde.de,[NAME_120] -2025-06-07T18:12:08.255652,True,geschäfte.csv,giovanni.bianchi@kunde.de,[NAME_121] -2025-06-07T18:12:08.255652,True,geschäfte.csv,william.brown@kunde.de,[NAME_122] -2025-06-07T18:12:08.255652,True,geschäfte.csv,1250,[EMAIL_123] -2025-06-07T18:12:08.255652,True,geschäfte.csv,890,[EMAIL_124] -2025-06-07T18:12:08.255652,True,geschäfte.csv,2340,[EMAIL_125] -2025-06-07T18:12:08.255652,True,geschäfte.csv,1750,[EMAIL_126] -2025-06-07T18:12:08.255652,True,geschäfte.csv,3200,[EMAIL_127] -2025-06-07T18:12:08.255652,True,geschäfte.csv,DE02 5001 0517 5407 3249 31,[IBAN_128] -2025-06-07T18:12:08.255652,True,geschäfte.csv,DE27 2005 0550 1045 1862 37,[IBAN_129] -2025-06-07T18:12:08.255652,True,geschäfte.csv,DE02 5001 0517 5407 3249 32,[IBAN_130] -2025-06-07T18:12:08.255652,True,geschäfte.csv,DE02 5001 0517 5407 3249 33,[IBAN_131] -2025-06-07T18:12:08.255652,True,geschäfte.csv,Kirchstraße 10 10115 Berlin,[ADDRESS_132] -2025-06-07T18:12:08.255652,True,geschäfte.csv,Seefeldstraße 5 10117 Berlin,[ADDRESS_133] -2025-06-07T18:12:08.255652,True,geschäfte.csv,15 Rue de la Paix 10115 Berlin,[ADDRESS_134] -2025-06-07T18:12:08.255652,True,geschäfte.csv,Via della Spiga 20 10115 Berlin,[ADDRESS_135] -2025-06-07T18:12:08.255652,True,geschäfte.csv,42 Oxford Street 10115 Berlin ,[ADDRESS_136] -2025-06-07T18:12:08.284172,True,italian.txt,000-180.000,[PHONE_1] -2025-06-07T18:12:08.284172,True,italian.txt,2027 Bewertung,[ADDRESS_2] -2025-06-07T18:12:08.284172,True,italian.txt,Max Mustermann,[NAME_3] -2025-06-07T18:12:08.284172,True,italian.txt,Peter Schmid,[NAME_4] -2025-06-07T18:12:08.284172,True,italian.txt,Marie Dupont,[NAME_5] -2025-06-07T18:12:08.284172,True,italian.txt,Marco Rossi,[NAME_6] -2025-06-07T18:12:08.284172,True,italian.txt,John Smith,[NAME_7] -2025-06-07T18:12:08.284172,True,italian.txt,max.mustermann@beispiel.de,[EMAIL_8] -2025-06-07T18:12:08.284172,True,italian.txt,peter.schmid@beispiel.ch,[EMAIL_9] -2025-06-07T18:12:08.284172,True,italian.txt,marie.dupont@exemple.fr,[EMAIL_10] -2025-06-07T18:12:08.284172,True,italian.txt,marco.rossi@esempio.it,[EMAIL_11] -2025-06-07T18:12:08.284172,True,italian.txt,john.smith@example.com,[EMAIL_12] -2025-06-07T18:12:08.284172,True,italian.txt,+49 30 12345678,[PHONE_13] -2025-06-07T18:12:08.284172,True,italian.txt,+41 44 123 45 67,[PHONE_14] -2025-06-07T18:12:08.284172,True,italian.txt,+33 1 23 45 67 89,[PHONE_15] -2025-06-07T18:12:08.284172,True,italian.txt,+39 02 1234 5678,[PHONE_16] -2025-06-07T18:12:08.284172,True,italian.txt,+44 20 1234 5678,[PHONE_17] -2025-06-07T18:12:08.284172,True,italian.txt,Musterstraße 123 12345 Berlin,[ADDRESS_18] -2025-06-07T18:12:08.284172,True,italian.txt,Bahnhofstrasse 1 8001 Zürich,[ADDRESS_19] -2025-06-07T18:12:08.284172,True,italian.txt,123 Rue de Paris 75001 Paris,[ADDRESS_20] -2025-06-07T18:12:08.284172,True,italian.txt,Via Roma 123 20100 Milano,[ADDRESS_21] -2025-06-07T18:12:08.284172,True,italian.txt,123 High Street London SW1A 1AA,[ADDRESS_22] -2025-06-07T18:12:08.284172,True,italian.txt,DE89 3704 0044 0532 0130 00,[IBAN_23] -2025-06-07T18:12:08.284172,True,italian.txt,CH93 0076 7000 E529 3557 7,[IBAN_24] -2025-06-07T18:12:08.284172,True,italian.txt,FR76 3000 6000 0112 3456 7890 189,[IBAN_25] -2025-06-07T18:12:08.284172,True,italian.txt,IT60 X054 2811 1010 0000 0123 456,[IBAN_26] -2025-06-07T18:12:08.284172,True,italian.txt,GB29 NWBK 6016 1331 9268 19,[IBAN_27] -2025-06-07T18:12:08.284172,True,italian.txt,4532 1234 5678 9012,[IBAN_28] -2025-06-07T18:12:08.284172,True,italian.txt,4532 1234 5678 9013,[IBAN_29] -2025-06-07T18:12:08.284172,True,italian.txt,4532 1234 5678 9014,[IBAN_30] -2025-06-07T18:12:08.284172,True,italian.txt,4532 1234 5678 9015,[IBAN_31] -2025-06-07T18:12:08.284172,True,italian.txt,4532 1234 5678 9016,[IBAN_32] -2025-06-07T18:12:08.284172,True,italian.txt,nan,[SSN_33] -2025-06-07T18:12:08.284172,True,italian.txt,756.1234.5678.90,[SSN_34] -2025-06-07T18:12:08.284172,True,italian.txt, ,[SSN_35] -2025-06-07T18:12:08.284172,True,italian.txt,sarah.weber@techsolutions.ch,[EMAIL_36] -2025-06-07T18:12:08.284172,True,italian.txt,+41 44 987 65,[PHONE_37] -2025-06-07T18:12:08.284172,True,italian.txt,"Dr. Sarah Weber -TechSolutions",[NAME_38] -2025-06-07T18:12:08.284172,True,italian.txt,hans.mueller@ethz.ch,[EMAIL_39] -2025-06-07T18:12:08.284172,True,italian.txt,+41 44 123 45,[PHONE_40] -2025-06-07T18:12:08.284172,True,italian.txt,l.meier@digitalsystems.ch,[EMAIL_41] -2025-06-07T18:12:08.284172,True,italian.txt,+41 44 456 78,[PHONE_42] -2025-06-07T18:12:08.284172,True,italian.txt,987.654.321,[SSN_43] -2025-06-07T18:12:08.284172,True,italian.txt,CHE-987.654.321,[SSN_44] -2025-06-07T18:12:08.284172,True,italian.txt,8002 Zürich,[ADDRESS_45] -2025-06-07T18:12:08.284172,True,italian.txt,Musterstrasse 123,[ADDRESS_46] -2025-06-07T18:12:08.284172,True,italian.txt,lara.meier@techsolutions.ch,[EMAIL_47] -2025-06-07T18:12:08.284172,True,italian.txt,123.456.789,[SSN_48] -2025-06-07T18:12:08.284172,True,italian.txt,CHE-123.456.789,[SSN_49] -2025-06-07T18:12:08.284172,True,italian.txt,8004 Zürich,[ADDRESS_50] -2025-06-07T18:12:08.284172,True,italian.txt,Industriestrasse 100,[ADDRESS_51] -2025-06-07T18:12:08.284172,True,italian.txt,lara.meier@example.ch,[EMAIL_52] -2025-06-07T18:12:08.284172,True,italian.txt,8001 Zürich,[ADDRESS_53] -2025-06-07T18:12:08.284172,True,italian.txt,Bahnhofstrasse 45,[ADDRESS_54] -2025-06-07T18:12:08.284172,True,italian.txt,"1990 -Adresse",[ADDRESS_55] -2025-06-07T18:12:08.284172,True,italian.txt,Hans,[NAME_56] -2025-06-07T18:12:08.284172,True,italian.txt,Thomas,[NAME_57] -2025-06-07T18:12:08.284172,True,italian.txt,Sophie,[NAME_58] -2025-06-07T18:12:08.284172,True,italian.txt,Luca,[NAME_59] -2025-06-07T18:12:08.284172,True,italian.txt,Emma,[NAME_60] -2025-06-07T18:12:08.284172,True,italian.txt,Müller,[NAME_61] -2025-06-07T18:12:08.284172,True,italian.txt,Weber,[NAME_62] -2025-06-07T18:12:08.284172,True,italian.txt,Martin,[NAME_63] -2025-06-07T18:12:08.284172,True,italian.txt,Ferrari,[NAME_64] -2025-06-07T18:12:08.284172,True,italian.txt,Wilson,[NAME_65] -2025-06-07T18:12:08.284172,True,italian.txt,hans.mueller@firma.de,[EMAIL_66] -2025-06-07T18:12:08.284172,True,italian.txt,thomas.weber@firma.ch,[EMAIL_67] -2025-06-07T18:12:08.284172,True,italian.txt,sophie.martin@entreprise.fr,[EMAIL_68] -2025-06-07T18:12:08.284172,True,italian.txt,luca.ferrari@azienda.it,[EMAIL_69] -2025-06-07T18:12:08.284172,True,italian.txt,emma.wilson@company.com,[EMAIL_70] -2025-06-07T18:12:08.284172,True,italian.txt,+49 89 12345678,[PHONE_71] -2025-06-07T18:12:08.284172,True,italian.txt,+41 44 234 56 78,[PHONE_72] -2025-06-07T18:12:08.284172,True,italian.txt,+33 1 34 56 78 90,[PHONE_73] -2025-06-07T18:12:08.284172,True,italian.txt,+39 02 2345 6789,[PHONE_74] -2025-06-07T18:12:08.284172,True,italian.txt,+44 20 2345 6789,[PHONE_75] -2025-06-07T18:12:08.284172,True,italian.txt,Hauptstraße 1 80331 München,[ADDRESS_76] -2025-06-07T18:12:08.284172,True,italian.txt,Bahnhofstrasse 2 8001 Zürich,[ADDRESS_77] -2025-06-07T18:12:08.284172,True,italian.txt,15 Avenue des Champs-Élysées 75008 Paris,[ADDRESS_78] -2025-06-07T18:12:08.284172,True,italian.txt,Via Monte Napoleone 8 20121 Milano,[ADDRESS_79] -2025-06-07T18:12:08.284172,True,italian.txt,25 Old Street London EC1V 9HL,[ADDRESS_80] -2025-06-07T18:12:08.284172,True,italian.txt,01-234567-8,[IBAN_81] -2025-06-07T18:12:08.284172,True,italian.txt,GB29 NWBK 6016 1331 9268 19 ,[IBAN_82] -2025-06-07T18:12:08.284172,True,italian.txt,"9012 - -Best",[ADDRESS_83] -2025-06-07T18:12:08.284172,True,italian.txt,"5678 -Address",[ADDRESS_84] -2025-06-07T18:12:08.284172,True,italian.txt,contact@example.com,[EMAIL_85] -2025-06-07T18:12:08.284172,True,italian.txt,max.mustermann@example.com,[EMAIL_86] -2025-06-07T18:12:08.284172,True,italian.txt,+49 123 4567890,[PHONE_87] -2025-06-07T18:12:08.284172,True,italian.txt,2024-03-15,[DATE_88] -2025-06-07T18:12:08.284172,True,italian.txt,4111 1111 1111 1111,[IBAN_89] -2025-06-07T18:12:08.284172,True,italian.txt,Tech Solutions GmbH,[NAME_90] -2025-06-07T18:12:08.284172,True,italian.txt,Dr. Anna Schmidt,NAME_91 -2025-06-07T18:12:08.284172,True,italian.txt,anna.schmidt@techsolutions.de,[EMAIL_92] -2025-06-07T18:12:08.284172,True,italian.txt,Dr. Thomas Weber,[NAME_93] -2025-06-07T18:12:08.284172,True,italian.txt,thomas.weber@company.de,[EMAIL_94] -2025-06-07T18:12:08.284172,True,italian.txt,maria.schmidt@company.de,[EMAIL_95] -2025-06-07T18:12:08.284172,True,italian.txt,+49 40 98765432,[PHONE_96] -2025-06-07T18:12:08.284172,True,italian.txt,DE27 3704 0044 0532 0130 01,[PHONE_97] -2025-06-07T18:12:08.284172,True,italian.txt,info@techinnovations.de,[EMAIL_98] -2025-06-07T18:12:08.284172,True,italian.txt,+49 89 12345679,[PHONE_99] -2025-06-07T18:12:08.284172,True,italian.txt,DE89 3704 0044 0532 0130 02,[PHONE_100] -2025-06-07T18:12:08.284172,True,italian.txt,"9012 - -Cordialement",[ADDRESS_101] -2025-06-07T18:12:08.284172,True,italian.txt,0112 3456 7890,[PHONE_102] -2025-06-07T18:12:08.284172,True,italian.txt,+33 1 23 45,[PHONE_103] -2025-06-07T18:12:08.284172,True,italian.txt,contact@exemple.fr,[EMAIL_104] -2025-06-07T18:12:08.284172,True,italian.txt,"9012 - -Mit",[ADDRESS_105] -2025-06-07T18:12:08.284172,True,italian.txt,0532 0130 00,[PHONE_106] -2025-06-07T18:12:08.284172,True,italian.txt,0044 0532 0130,[PHONE_107] -2025-06-07T18:12:08.284172,True,italian.txt,030-12345678,[PHONE_108] -2025-06-07T18:12:08.284172,True,italian.txt,sarah.mueller@techsolutions.ch,[EMAIL_109] -2025-06-07T18:12:08.284172,True,italian.txt,thomas.weber@techsolutions.ch,[EMAIL_110] -2025-06-07T18:12:08.284172,True,italian.txt,info@techsolutions.ch,[EMAIL_111] -2025-06-07T18:12:08.284172,True,italian.txt,"2026 -TechSolutions",[ADDRESS_112] -2025-06-07T18:12:08.284172,True,italian.txt,Anna Schmidt,[DATE_113] -2025-06-07T18:12:08.284172,True,italian.txt,Franz Huber,[DATE_114] -2025-06-07T18:12:08.284172,True,italian.txt,Pierre Dubois,[DATE_115] -2025-06-07T18:12:08.284172,True,italian.txt,Giovanni Bianchi,[DATE_116] -2025-06-07T18:12:08.284172,True,italian.txt,William Brown,[DATE_117] -2025-06-07T18:12:08.284172,True,italian.txt,anna.schmidt@kunde.de,[NAME_118] -2025-06-07T18:12:08.284172,True,italian.txt,franz.huber@kunde.de,[NAME_119] -2025-06-07T18:12:08.284172,True,italian.txt,pierre.dubois@kunde.de,[NAME_120] -2025-06-07T18:12:08.284172,True,italian.txt,giovanni.bianchi@kunde.de,[NAME_121] -2025-06-07T18:12:08.284172,True,italian.txt,william.brown@kunde.de,[NAME_122] -2025-06-07T18:12:08.284172,True,italian.txt,1250,[EMAIL_123] -2025-06-07T18:12:08.284172,True,italian.txt,890,[EMAIL_124] -2025-06-07T18:12:08.284172,True,italian.txt,2340,[EMAIL_125] -2025-06-07T18:12:08.284172,True,italian.txt,1750,[EMAIL_126] -2025-06-07T18:12:08.284172,True,italian.txt,3200,[EMAIL_127] -2025-06-07T18:12:08.284172,True,italian.txt,DE02 5001 0517 5407 3249 31,[IBAN_128] -2025-06-07T18:12:08.284172,True,italian.txt,DE27 2005 0550 1045 1862 37,[IBAN_129] -2025-06-07T18:12:08.284172,True,italian.txt,DE02 5001 0517 5407 3249 32,[IBAN_130] -2025-06-07T18:12:08.284172,True,italian.txt,DE02 5001 0517 5407 3249 33,[IBAN_131] -2025-06-07T18:12:08.284172,True,italian.txt,Kirchstraße 10 10115 Berlin,[ADDRESS_132] -2025-06-07T18:12:08.284172,True,italian.txt,Seefeldstraße 5 10117 Berlin,[ADDRESS_133] -2025-06-07T18:12:08.284172,True,italian.txt,15 Rue de la Paix 10115 Berlin,[ADDRESS_134] -2025-06-07T18:12:08.284172,True,italian.txt,Via della Spiga 20 10115 Berlin,[ADDRESS_135] -2025-06-07T18:12:08.284172,True,italian.txt,42 Oxford Street 10115 Berlin ,[ADDRESS_136] -2025-06-07T18:12:08.284172,True,italian.txt,"9012 - -Cordiali",[ADDRESS_137] -2025-06-07T18:12:08.284172,True,italian.txt,0000 0123 456,[PHONE_138] -2025-06-07T18:12:08.284172,True,italian.txt,"5678 -Indirizzo",[ADDRESS_139] -2025-06-07T18:12:08.284172,True,italian.txt,02 1234 5678,[PHONE_140] -2025-06-07T18:12:08.284172,True,italian.txt,info@esempio.it,[EMAIL_141] -2025-06-07T18:12:08.314527,True,kunden.csv,000-180.000,[PHONE_1] -2025-06-07T18:12:08.314527,True,kunden.csv,2027 Bewertung,[ADDRESS_2] -2025-06-07T18:12:08.314527,True,kunden.csv,Max Mustermann,[NAME_3] -2025-06-07T18:12:08.314527,True,kunden.csv,Peter Schmid,[NAME_4] -2025-06-07T18:12:08.314527,True,kunden.csv,Marie Dupont,[NAME_5] -2025-06-07T18:12:08.314527,True,kunden.csv,Marco Rossi,[NAME_6] -2025-06-07T18:12:08.314527,True,kunden.csv,John Smith,[NAME_7] -2025-06-07T18:12:08.314527,True,kunden.csv,max.mustermann@beispiel.de,[EMAIL_8] -2025-06-07T18:12:08.314527,True,kunden.csv,peter.schmid@beispiel.ch,[EMAIL_9] -2025-06-07T18:12:08.314527,True,kunden.csv,marie.dupont@exemple.fr,[EMAIL_10] -2025-06-07T18:12:08.314527,True,kunden.csv,marco.rossi@esempio.it,[EMAIL_11] -2025-06-07T18:12:08.314527,True,kunden.csv,john.smith@example.com,[EMAIL_12] -2025-06-07T18:12:08.314527,True,kunden.csv,+49 30 12345678,[PHONE_13] -2025-06-07T18:12:08.314527,True,kunden.csv,+41 44 123 45 67,[PHONE_14] -2025-06-07T18:12:08.314527,True,kunden.csv,+33 1 23 45 67 89,[PHONE_15] -2025-06-07T18:12:08.314527,True,kunden.csv,+39 02 1234 5678,[PHONE_16] -2025-06-07T18:12:08.314527,True,kunden.csv,+44 20 1234 5678,[PHONE_17] -2025-06-07T18:12:08.314527,True,kunden.csv,Musterstraße 123 12345 Berlin,[ADDRESS_18] -2025-06-07T18:12:08.314527,True,kunden.csv,Bahnhofstrasse 1 8001 Zürich,[ADDRESS_19] -2025-06-07T18:12:08.314527,True,kunden.csv,123 Rue de Paris 75001 Paris,[ADDRESS_20] -2025-06-07T18:12:08.314527,True,kunden.csv,Via Roma 123 20100 Milano,[ADDRESS_21] -2025-06-07T18:12:08.314527,True,kunden.csv,123 High Street London SW1A 1AA,[ADDRESS_22] -2025-06-07T18:12:08.314527,True,kunden.csv,DE89 3704 0044 0532 0130 00,[IBAN_23] -2025-06-07T18:12:08.314527,True,kunden.csv,CH93 0076 7000 E529 3557 7,[IBAN_24] -2025-06-07T18:12:08.314527,True,kunden.csv,FR76 3000 6000 0112 3456 7890 189,[IBAN_25] -2025-06-07T18:12:08.314527,True,kunden.csv,IT60 X054 2811 1010 0000 0123 456,[IBAN_26] -2025-06-07T18:12:08.314527,True,kunden.csv,GB29 NWBK 6016 1331 9268 19,[IBAN_27] -2025-06-07T18:12:08.314527,True,kunden.csv,4532 1234 5678 9012,[IBAN_28] -2025-06-07T18:12:08.314527,True,kunden.csv,4532 1234 5678 9013,[IBAN_29] -2025-06-07T18:12:08.314527,True,kunden.csv,4532 1234 5678 9014,[IBAN_30] -2025-06-07T18:12:08.314527,True,kunden.csv,4532 1234 5678 9015,[IBAN_31] -2025-06-07T18:12:08.314527,True,kunden.csv,4532 1234 5678 9016,[IBAN_32] -2025-06-07T18:12:08.314527,True,kunden.csv,nan,[SSN_33] -2025-06-07T18:12:08.314527,True,kunden.csv,756.1234.5678.90,[SSN_34] -2025-06-07T18:12:08.314527,True,kunden.csv, ,[SSN_35] -2025-06-07T18:12:08.314527,True,kunden.csv,sarah.weber@techsolutions.ch,[EMAIL_36] -2025-06-07T18:12:08.314527,True,kunden.csv,+41 44 987 65,[PHONE_37] -2025-06-07T18:12:08.314527,True,kunden.csv,"Dr. Sarah Weber -TechSolutions",[NAME_38] -2025-06-07T18:12:08.314527,True,kunden.csv,hans.mueller@ethz.ch,[EMAIL_39] -2025-06-07T18:12:08.314527,True,kunden.csv,+41 44 123 45,[PHONE_40] -2025-06-07T18:12:08.314527,True,kunden.csv,l.meier@digitalsystems.ch,[EMAIL_41] -2025-06-07T18:12:08.314527,True,kunden.csv,+41 44 456 78,[PHONE_42] -2025-06-07T18:12:08.314527,True,kunden.csv,987.654.321,[SSN_43] -2025-06-07T18:12:08.314527,True,kunden.csv,CHE-987.654.321,[SSN_44] -2025-06-07T18:12:08.314527,True,kunden.csv,8002 Zürich,[ADDRESS_45] -2025-06-07T18:12:08.314527,True,kunden.csv,Musterstrasse 123,[ADDRESS_46] -2025-06-07T18:12:08.314527,True,kunden.csv,lara.meier@techsolutions.ch,[EMAIL_47] -2025-06-07T18:12:08.314527,True,kunden.csv,123.456.789,[SSN_48] -2025-06-07T18:12:08.314527,True,kunden.csv,CHE-123.456.789,[SSN_49] -2025-06-07T18:12:08.314527,True,kunden.csv,8004 Zürich,[ADDRESS_50] -2025-06-07T18:12:08.314527,True,kunden.csv,Industriestrasse 100,[ADDRESS_51] -2025-06-07T18:12:08.314527,True,kunden.csv,lara.meier@example.ch,[EMAIL_52] -2025-06-07T18:12:08.314527,True,kunden.csv,8001 Zürich,[ADDRESS_53] -2025-06-07T18:12:08.314527,True,kunden.csv,Bahnhofstrasse 45,[ADDRESS_54] -2025-06-07T18:12:08.314527,True,kunden.csv,"1990 -Adresse",[ADDRESS_55] -2025-06-07T18:12:08.314527,True,kunden.csv,Hans,[NAME_56] -2025-06-07T18:12:08.314527,True,kunden.csv,Thomas,[NAME_57] -2025-06-07T18:12:08.314527,True,kunden.csv,Sophie,[NAME_58] -2025-06-07T18:12:08.314527,True,kunden.csv,Luca,[NAME_59] -2025-06-07T18:12:08.314527,True,kunden.csv,Emma,[NAME_60] -2025-06-07T18:12:08.314527,True,kunden.csv,Müller,[NAME_61] -2025-06-07T18:12:08.314527,True,kunden.csv,Weber,[NAME_62] -2025-06-07T18:12:08.314527,True,kunden.csv,Martin,[NAME_63] -2025-06-07T18:12:08.314527,True,kunden.csv,Ferrari,[NAME_64] -2025-06-07T18:12:08.314527,True,kunden.csv,Wilson,[NAME_65] -2025-06-07T18:12:08.314527,True,kunden.csv,hans.mueller@firma.de,[EMAIL_66] -2025-06-07T18:12:08.314527,True,kunden.csv,thomas.weber@firma.ch,[EMAIL_67] -2025-06-07T18:12:08.314527,True,kunden.csv,sophie.martin@entreprise.fr,[EMAIL_68] -2025-06-07T18:12:08.314527,True,kunden.csv,luca.ferrari@azienda.it,[EMAIL_69] -2025-06-07T18:12:08.314527,True,kunden.csv,emma.wilson@company.com,[EMAIL_70] -2025-06-07T18:12:08.314527,True,kunden.csv,+49 89 12345678,[PHONE_71] -2025-06-07T18:12:08.314527,True,kunden.csv,+41 44 234 56 78,[PHONE_72] -2025-06-07T18:12:08.314527,True,kunden.csv,+33 1 34 56 78 90,[PHONE_73] -2025-06-07T18:12:08.314527,True,kunden.csv,+39 02 2345 6789,[PHONE_74] -2025-06-07T18:12:08.314527,True,kunden.csv,+44 20 2345 6789,[PHONE_75] -2025-06-07T18:12:08.314527,True,kunden.csv,Hauptstraße 1 80331 München,[ADDRESS_76] -2025-06-07T18:12:08.314527,True,kunden.csv,Bahnhofstrasse 2 8001 Zürich,[ADDRESS_77] -2025-06-07T18:12:08.314527,True,kunden.csv,15 Avenue des Champs-Élysées 75008 Paris,[ADDRESS_78] -2025-06-07T18:12:08.314527,True,kunden.csv,Via Monte Napoleone 8 20121 Milano,[ADDRESS_79] -2025-06-07T18:12:08.314527,True,kunden.csv,25 Old Street London EC1V 9HL,[ADDRESS_80] -2025-06-07T18:12:08.314527,True,kunden.csv,01-234567-8,[IBAN_81] -2025-06-07T18:12:08.314527,True,kunden.csv,GB29 NWBK 6016 1331 9268 19 ,[IBAN_82] -2025-06-07T18:12:08.314527,True,kunden.csv,"9012 - -Best",[ADDRESS_83] -2025-06-07T18:12:08.314527,True,kunden.csv,"5678 -Address",[ADDRESS_84] -2025-06-07T18:12:08.314527,True,kunden.csv,contact@example.com,[EMAIL_85] -2025-06-07T18:12:08.314527,True,kunden.csv,max.mustermann@example.com,[EMAIL_86] -2025-06-07T18:12:08.314527,True,kunden.csv,+49 123 4567890,[PHONE_87] -2025-06-07T18:12:08.314527,True,kunden.csv,2024-03-15,[DATE_88] -2025-06-07T18:12:08.314527,True,kunden.csv,4111 1111 1111 1111,[IBAN_89] -2025-06-07T18:12:08.314527,True,kunden.csv,Tech Solutions GmbH,[NAME_90] -2025-06-07T18:12:08.314527,True,kunden.csv,Dr. Anna Schmidt,NAME_91 -2025-06-07T18:12:08.314527,True,kunden.csv,anna.schmidt@techsolutions.de,[EMAIL_92] -2025-06-07T18:12:08.314527,True,kunden.csv,Dr. Thomas Weber,[NAME_93] -2025-06-07T18:12:08.314527,True,kunden.csv,thomas.weber@company.de,[EMAIL_94] -2025-06-07T18:12:08.314527,True,kunden.csv,maria.schmidt@company.de,[EMAIL_95] -2025-06-07T18:12:08.314527,True,kunden.csv,+49 40 98765432,[PHONE_96] -2025-06-07T18:12:08.314527,True,kunden.csv,DE27 3704 0044 0532 0130 01,[PHONE_97] -2025-06-07T18:12:08.314527,True,kunden.csv,info@techinnovations.de,[EMAIL_98] -2025-06-07T18:12:08.314527,True,kunden.csv,+49 89 12345679,[PHONE_99] -2025-06-07T18:12:08.314527,True,kunden.csv,DE89 3704 0044 0532 0130 02,[PHONE_100] -2025-06-07T18:12:08.314527,True,kunden.csv,"9012 - -Cordialement",[ADDRESS_101] -2025-06-07T18:12:08.314527,True,kunden.csv,0112 3456 7890,[PHONE_102] -2025-06-07T18:12:08.314527,True,kunden.csv,+33 1 23 45,[PHONE_103] -2025-06-07T18:12:08.314527,True,kunden.csv,contact@exemple.fr,[EMAIL_104] -2025-06-07T18:12:08.314527,True,kunden.csv,"9012 - -Mit",[ADDRESS_105] -2025-06-07T18:12:08.314527,True,kunden.csv,0532 0130 00,[PHONE_106] -2025-06-07T18:12:08.314527,True,kunden.csv,0044 0532 0130,[PHONE_107] -2025-06-07T18:12:08.314527,True,kunden.csv,030-12345678,[PHONE_108] -2025-06-07T18:12:08.314527,True,kunden.csv,sarah.mueller@techsolutions.ch,[EMAIL_109] -2025-06-07T18:12:08.314527,True,kunden.csv,thomas.weber@techsolutions.ch,[EMAIL_110] -2025-06-07T18:12:08.314527,True,kunden.csv,info@techsolutions.ch,[EMAIL_111] -2025-06-07T18:12:08.314527,True,kunden.csv,"2026 -TechSolutions",[ADDRESS_112] -2025-06-07T18:12:08.314527,True,kunden.csv,Anna Schmidt,[DATE_113] -2025-06-07T18:12:08.314527,True,kunden.csv,Franz Huber,[DATE_114] -2025-06-07T18:12:08.314527,True,kunden.csv,Pierre Dubois,[DATE_115] -2025-06-07T18:12:08.314527,True,kunden.csv,Giovanni Bianchi,[DATE_116] -2025-06-07T18:12:08.314527,True,kunden.csv,William Brown,[DATE_117] -2025-06-07T18:12:08.314527,True,kunden.csv,anna.schmidt@kunde.de,[NAME_118] -2025-06-07T18:12:08.314527,True,kunden.csv,franz.huber@kunde.de,[NAME_119] -2025-06-07T18:12:08.314527,True,kunden.csv,pierre.dubois@kunde.de,[NAME_120] -2025-06-07T18:12:08.314527,True,kunden.csv,giovanni.bianchi@kunde.de,[NAME_121] -2025-06-07T18:12:08.314527,True,kunden.csv,william.brown@kunde.de,[NAME_122] -2025-06-07T18:12:08.314527,True,kunden.csv,1250,[EMAIL_123] -2025-06-07T18:12:08.314527,True,kunden.csv,890,[EMAIL_124] -2025-06-07T18:12:08.314527,True,kunden.csv,2340,[EMAIL_125] -2025-06-07T18:12:08.314527,True,kunden.csv,1750,[EMAIL_126] -2025-06-07T18:12:08.314527,True,kunden.csv,3200,[EMAIL_127] -2025-06-07T18:12:08.314527,True,kunden.csv,DE02 5001 0517 5407 3249 31,[IBAN_128] -2025-06-07T18:12:08.314527,True,kunden.csv,DE27 2005 0550 1045 1862 37,[IBAN_129] -2025-06-07T18:12:08.314527,True,kunden.csv,DE02 5001 0517 5407 3249 32,[IBAN_130] -2025-06-07T18:12:08.314527,True,kunden.csv,DE02 5001 0517 5407 3249 33,[IBAN_131] -2025-06-07T18:12:08.314527,True,kunden.csv,Kirchstraße 10 10115 Berlin,[ADDRESS_132] -2025-06-07T18:12:08.314527,True,kunden.csv,Seefeldstraße 5 10117 Berlin,[ADDRESS_133] -2025-06-07T18:12:08.314527,True,kunden.csv,15 Rue de la Paix 10115 Berlin,[ADDRESS_134] -2025-06-07T18:12:08.314527,True,kunden.csv,Via della Spiga 20 10115 Berlin,[ADDRESS_135] -2025-06-07T18:12:08.314527,True,kunden.csv,42 Oxford Street 10115 Berlin ,[ADDRESS_136] -2025-06-07T18:12:08.314527,True,kunden.csv,"9012 - -Cordiali",[ADDRESS_137] -2025-06-07T18:12:08.314527,True,kunden.csv,0000 0123 456,[PHONE_138] -2025-06-07T18:12:08.314527,True,kunden.csv,"5678 -Indirizzo",[ADDRESS_139] -2025-06-07T18:12:08.314527,True,kunden.csv,02 1234 5678,[PHONE_140] -2025-06-07T18:12:08.314527,True,kunden.csv,info@esempio.it,[EMAIL_141] -2025-06-07T18:12:08.314527,True,kunden.csv,Michael,[NAME_142] -2025-06-07T18:12:08.314527,True,kunden.csv,Sabine,[NAME_143] -2025-06-07T18:12:08.314527,True,kunden.csv,Petra,[NAME_144] -2025-06-07T18:12:08.314527,True,kunden.csv,Klaus,[NAME_145] -2025-06-07T18:12:08.314527,True,kunden.csv,Schmidt,[NAME_146] -2025-06-07T18:12:08.314527,True,kunden.csv,Fischer,[NAME_147] -2025-06-07T18:12:08.314527,True,kunden.csv,Wagner,[NAME_148] -2025-06-07T18:12:08.314527,True,kunden.csv,michael.schmidt@kunde.de,[EMAIL_149] -2025-06-07T18:12:08.314527,True,kunden.csv,sabine.weber@kunde.de,[EMAIL_150] -2025-06-07T18:12:08.314527,True,kunden.csv,thomas.mueller@kunde.de,[EMAIL_151] -2025-06-07T18:12:08.314527,True,kunden.csv,petra.fischer@kunde.de,[EMAIL_152] -2025-06-07T18:12:08.314527,True,kunden.csv,klaus.wagner@kunde.de,[EMAIL_153] -2025-06-07T18:12:08.314527,True,kunden.csv,+49 89 23456789,[PHONE_154] -2025-06-07T18:12:08.314527,True,kunden.csv,+49 40 34567890,[PHONE_155] -2025-06-07T18:12:08.314527,True,kunden.csv,+49 69 45678901,[PHONE_156] -2025-06-07T18:12:08.314527,True,kunden.csv,+49 211 56789012,[PHONE_157] -2025-06-07T18:12:08.314527,True,kunden.csv,Hauptstraße 45 80331 München,[ADDRESS_158] -2025-06-07T18:12:08.314527,True,kunden.csv,Neue Straße 78 20095 Hamburg,[ADDRESS_159] -2025-06-07T18:12:08.314527,True,kunden.csv,Frankfurter Ring 12 60313 Frankfurt,[ADDRESS_160] -2025-06-07T18:12:08.314527,True,kunden.csv,Königsallee 92 40212 Düsseldorf,[ADDRESS_161] -2025-06-07T18:12:08.314527,True,kunden.csv,12/345/67890,[SSN_162] -2025-06-07T18:12:08.314527,True,kunden.csv,98/765/43210,[SSN_163] -2025-06-07T18:12:08.314527,True,kunden.csv,45/678/90123,[SSN_164] -2025-06-07T18:12:08.314527,True,kunden.csv,34/567/89012,[SSN_165] -2025-06-07T18:12:08.314527,True,kunden.csv,23/456/78901 ,[SSN_166] -2025-06-07T18:12:08.349750,True,mitarbeiter.csv,000-180.000,[PHONE_1] -2025-06-07T18:12:08.349750,True,mitarbeiter.csv,2027 Bewertung,[ADDRESS_2] -2025-06-07T18:12:08.349750,True,mitarbeiter.csv,Max Mustermann,[NAME_3] -2025-06-07T18:12:08.349750,True,mitarbeiter.csv,Peter Schmid,[NAME_4] -2025-06-07T18:12:08.349750,True,mitarbeiter.csv,Marie Dupont,[NAME_5] -2025-06-07T18:12:08.349750,True,mitarbeiter.csv,Marco Rossi,[NAME_6] -2025-06-07T18:12:08.349750,True,mitarbeiter.csv,John Smith,[NAME_7] -2025-06-07T18:12:08.349750,True,mitarbeiter.csv,max.mustermann@beispiel.de,[EMAIL_8] -2025-06-07T18:12:08.349750,True,mitarbeiter.csv,peter.schmid@beispiel.ch,[EMAIL_9] -2025-06-07T18:12:08.349750,True,mitarbeiter.csv,marie.dupont@exemple.fr,[EMAIL_10] -2025-06-07T18:12:08.349750,True,mitarbeiter.csv,marco.rossi@esempio.it,[EMAIL_11] -2025-06-07T18:12:08.349750,True,mitarbeiter.csv,john.smith@example.com,[EMAIL_12] -2025-06-07T18:12:08.349750,True,mitarbeiter.csv,+49 30 12345678,[PHONE_13] -2025-06-07T18:12:08.349750,True,mitarbeiter.csv,+41 44 123 45 67,[PHONE_14] -2025-06-07T18:12:08.349750,True,mitarbeiter.csv,+33 1 23 45 67 89,[PHONE_15] -2025-06-07T18:12:08.349750,True,mitarbeiter.csv,+39 02 1234 5678,[PHONE_16] -2025-06-07T18:12:08.349750,True,mitarbeiter.csv,+44 20 1234 5678,[PHONE_17] -2025-06-07T18:12:08.349750,True,mitarbeiter.csv,Musterstraße 123 12345 Berlin,[ADDRESS_18] -2025-06-07T18:12:08.349750,True,mitarbeiter.csv,Bahnhofstrasse 1 8001 Zürich,[ADDRESS_19] -2025-06-07T18:12:08.349750,True,mitarbeiter.csv,123 Rue de Paris 75001 Paris,[ADDRESS_20] -2025-06-07T18:12:08.349750,True,mitarbeiter.csv,Via Roma 123 20100 Milano,[ADDRESS_21] -2025-06-07T18:12:08.349750,True,mitarbeiter.csv,123 High Street London SW1A 1AA,[ADDRESS_22] -2025-06-07T18:12:08.349750,True,mitarbeiter.csv,DE89 3704 0044 0532 0130 00,[IBAN_23] -2025-06-07T18:12:08.349750,True,mitarbeiter.csv,CH93 0076 7000 E529 3557 7,[IBAN_24] -2025-06-07T18:12:08.349750,True,mitarbeiter.csv,FR76 3000 6000 0112 3456 7890 189,[IBAN_25] -2025-06-07T18:12:08.349750,True,mitarbeiter.csv,IT60 X054 2811 1010 0000 0123 456,[IBAN_26] -2025-06-07T18:12:08.349750,True,mitarbeiter.csv,GB29 NWBK 6016 1331 9268 19,[IBAN_27] -2025-06-07T18:12:08.349750,True,mitarbeiter.csv,4532 1234 5678 9012,[IBAN_28] -2025-06-07T18:12:08.349750,True,mitarbeiter.csv,4532 1234 5678 9013,[IBAN_29] -2025-06-07T18:12:08.349750,True,mitarbeiter.csv,4532 1234 5678 9014,[IBAN_30] -2025-06-07T18:12:08.349750,True,mitarbeiter.csv,4532 1234 5678 9015,[IBAN_31] -2025-06-07T18:12:08.349750,True,mitarbeiter.csv,4532 1234 5678 9016,[IBAN_32] -2025-06-07T18:12:08.349750,True,mitarbeiter.csv,nan,[SSN_33] -2025-06-07T18:12:08.349750,True,mitarbeiter.csv,756.1234.5678.90,[SSN_34] -2025-06-07T18:12:08.349750,True,mitarbeiter.csv, ,[SSN_35] -2025-06-07T18:12:08.349750,True,mitarbeiter.csv,sarah.weber@techsolutions.ch,[EMAIL_36] -2025-06-07T18:12:08.349750,True,mitarbeiter.csv,+41 44 987 65,[PHONE_37] -2025-06-07T18:12:08.349750,True,mitarbeiter.csv,"Dr. Sarah Weber -TechSolutions",[NAME_38] -2025-06-07T18:12:08.349750,True,mitarbeiter.csv,hans.mueller@ethz.ch,[EMAIL_39] -2025-06-07T18:12:08.349750,True,mitarbeiter.csv,+41 44 123 45,[PHONE_40] -2025-06-07T18:12:08.349750,True,mitarbeiter.csv,l.meier@digitalsystems.ch,[EMAIL_41] -2025-06-07T18:12:08.349750,True,mitarbeiter.csv,+41 44 456 78,[PHONE_42] -2025-06-07T18:12:08.349750,True,mitarbeiter.csv,987.654.321,[SSN_43] -2025-06-07T18:12:08.349750,True,mitarbeiter.csv,CHE-987.654.321,[SSN_44] -2025-06-07T18:12:08.349750,True,mitarbeiter.csv,8002 Zürich,[ADDRESS_45] -2025-06-07T18:12:08.349750,True,mitarbeiter.csv,Musterstrasse 123,[ADDRESS_46] -2025-06-07T18:12:08.349750,True,mitarbeiter.csv,lara.meier@techsolutions.ch,[EMAIL_47] -2025-06-07T18:12:08.349750,True,mitarbeiter.csv,123.456.789,[SSN_48] -2025-06-07T18:12:08.349750,True,mitarbeiter.csv,CHE-123.456.789,[SSN_49] -2025-06-07T18:12:08.349750,True,mitarbeiter.csv,8004 Zürich,[ADDRESS_50] -2025-06-07T18:12:08.349750,True,mitarbeiter.csv,Industriestrasse 100,[ADDRESS_51] -2025-06-07T18:12:08.349750,True,mitarbeiter.csv,lara.meier@example.ch,[EMAIL_52] -2025-06-07T18:12:08.349750,True,mitarbeiter.csv,8001 Zürich,[ADDRESS_53] -2025-06-07T18:12:08.349750,True,mitarbeiter.csv,Bahnhofstrasse 45,[ADDRESS_54] -2025-06-07T18:12:08.349750,True,mitarbeiter.csv,"1990 -Adresse",[ADDRESS_55] -2025-06-07T18:12:08.349750,True,mitarbeiter.csv,Hans,[NAME_56] -2025-06-07T18:12:08.349750,True,mitarbeiter.csv,Thomas,[NAME_57] -2025-06-07T18:12:08.349750,True,mitarbeiter.csv,Sophie,[NAME_58] -2025-06-07T18:12:08.349750,True,mitarbeiter.csv,Luca,[NAME_59] -2025-06-07T18:12:08.349750,True,mitarbeiter.csv,Emma,[NAME_60] -2025-06-07T18:12:08.349750,True,mitarbeiter.csv,Müller,[NAME_61] -2025-06-07T18:12:08.349750,True,mitarbeiter.csv,Weber,[NAME_62] -2025-06-07T18:12:08.349750,True,mitarbeiter.csv,Martin,[NAME_63] -2025-06-07T18:12:08.349750,True,mitarbeiter.csv,Ferrari,[NAME_64] -2025-06-07T18:12:08.349750,True,mitarbeiter.csv,Wilson,[NAME_65] -2025-06-07T18:12:08.349750,True,mitarbeiter.csv,hans.mueller@firma.de,[EMAIL_66] -2025-06-07T18:12:08.349750,True,mitarbeiter.csv,thomas.weber@firma.ch,[EMAIL_67] -2025-06-07T18:12:08.349750,True,mitarbeiter.csv,sophie.martin@entreprise.fr,[EMAIL_68] -2025-06-07T18:12:08.349750,True,mitarbeiter.csv,luca.ferrari@azienda.it,[EMAIL_69] -2025-06-07T18:12:08.349750,True,mitarbeiter.csv,emma.wilson@company.com,[EMAIL_70] -2025-06-07T18:12:08.349750,True,mitarbeiter.csv,+49 89 12345678,[PHONE_71] -2025-06-07T18:12:08.349750,True,mitarbeiter.csv,+41 44 234 56 78,[PHONE_72] -2025-06-07T18:12:08.349750,True,mitarbeiter.csv,+33 1 34 56 78 90,[PHONE_73] -2025-06-07T18:12:08.349750,True,mitarbeiter.csv,+39 02 2345 6789,[PHONE_74] -2025-06-07T18:12:08.349750,True,mitarbeiter.csv,+44 20 2345 6789,[PHONE_75] -2025-06-07T18:12:08.349750,True,mitarbeiter.csv,Hauptstraße 1 80331 München,[ADDRESS_76] -2025-06-07T18:12:08.349750,True,mitarbeiter.csv,Bahnhofstrasse 2 8001 Zürich,[ADDRESS_77] -2025-06-07T18:12:08.349750,True,mitarbeiter.csv,15 Avenue des Champs-Élysées 75008 Paris,[ADDRESS_78] -2025-06-07T18:12:08.349750,True,mitarbeiter.csv,Via Monte Napoleone 8 20121 Milano,[ADDRESS_79] -2025-06-07T18:12:08.349750,True,mitarbeiter.csv,25 Old Street London EC1V 9HL,[ADDRESS_80] -2025-06-07T18:12:08.349750,True,mitarbeiter.csv,01-234567-8,[IBAN_81] -2025-06-07T18:12:08.349750,True,mitarbeiter.csv,GB29 NWBK 6016 1331 9268 19 ,[IBAN_82] -2025-06-07T18:12:08.349750,True,mitarbeiter.csv,"9012 - -Best",[ADDRESS_83] -2025-06-07T18:12:08.349750,True,mitarbeiter.csv,"5678 -Address",[ADDRESS_84] -2025-06-07T18:12:08.349750,True,mitarbeiter.csv,contact@example.com,[EMAIL_85] -2025-06-07T18:12:08.349750,True,mitarbeiter.csv,max.mustermann@example.com,[EMAIL_86] -2025-06-07T18:12:08.349750,True,mitarbeiter.csv,+49 123 4567890,[PHONE_87] -2025-06-07T18:12:08.349750,True,mitarbeiter.csv,2024-03-15,[DATE_88] -2025-06-07T18:12:08.349750,True,mitarbeiter.csv,4111 1111 1111 1111,[IBAN_89] -2025-06-07T18:12:08.349750,True,mitarbeiter.csv,Tech Solutions GmbH,[NAME_90] -2025-06-07T18:12:08.349750,True,mitarbeiter.csv,Dr. Anna Schmidt,NAME_91 -2025-06-07T18:12:08.349750,True,mitarbeiter.csv,anna.schmidt@techsolutions.de,[EMAIL_92] -2025-06-07T18:12:08.349750,True,mitarbeiter.csv,Dr. Thomas Weber,[NAME_93] -2025-06-07T18:12:08.349750,True,mitarbeiter.csv,thomas.weber@company.de,[EMAIL_94] -2025-06-07T18:12:08.349750,True,mitarbeiter.csv,maria.schmidt@company.de,[EMAIL_95] -2025-06-07T18:12:08.349750,True,mitarbeiter.csv,+49 40 98765432,[PHONE_96] -2025-06-07T18:12:08.349750,True,mitarbeiter.csv,DE27 3704 0044 0532 0130 01,[PHONE_97] -2025-06-07T18:12:08.349750,True,mitarbeiter.csv,info@techinnovations.de,[EMAIL_98] -2025-06-07T18:12:08.349750,True,mitarbeiter.csv,+49 89 12345679,[PHONE_99] -2025-06-07T18:12:08.349750,True,mitarbeiter.csv,DE89 3704 0044 0532 0130 02,[PHONE_100] -2025-06-07T18:12:08.349750,True,mitarbeiter.csv,"9012 - -Cordialement",[ADDRESS_101] -2025-06-07T18:12:08.349750,True,mitarbeiter.csv,0112 3456 7890,[PHONE_102] -2025-06-07T18:12:08.349750,True,mitarbeiter.csv,+33 1 23 45,[PHONE_103] -2025-06-07T18:12:08.349750,True,mitarbeiter.csv,contact@exemple.fr,[EMAIL_104] -2025-06-07T18:12:08.349750,True,mitarbeiter.csv,"9012 - -Mit",[ADDRESS_105] -2025-06-07T18:12:08.349750,True,mitarbeiter.csv,0532 0130 00,[PHONE_106] -2025-06-07T18:12:08.349750,True,mitarbeiter.csv,0044 0532 0130,[PHONE_107] -2025-06-07T18:12:08.349750,True,mitarbeiter.csv,030-12345678,[PHONE_108] -2025-06-07T18:12:08.349750,True,mitarbeiter.csv,sarah.mueller@techsolutions.ch,[EMAIL_109] -2025-06-07T18:12:08.349750,True,mitarbeiter.csv,thomas.weber@techsolutions.ch,[EMAIL_110] -2025-06-07T18:12:08.349750,True,mitarbeiter.csv,info@techsolutions.ch,[EMAIL_111] -2025-06-07T18:12:08.349750,True,mitarbeiter.csv,"2026 -TechSolutions",[ADDRESS_112] -2025-06-07T18:12:08.349750,True,mitarbeiter.csv,Anna Schmidt,[DATE_113] -2025-06-07T18:12:08.349750,True,mitarbeiter.csv,Franz Huber,[DATE_114] -2025-06-07T18:12:08.349750,True,mitarbeiter.csv,Pierre Dubois,[DATE_115] -2025-06-07T18:12:08.349750,True,mitarbeiter.csv,Giovanni Bianchi,[DATE_116] -2025-06-07T18:12:08.349750,True,mitarbeiter.csv,William Brown,[DATE_117] -2025-06-07T18:12:08.349750,True,mitarbeiter.csv,anna.schmidt@kunde.de,[NAME_118] -2025-06-07T18:12:08.349750,True,mitarbeiter.csv,franz.huber@kunde.de,[NAME_119] -2025-06-07T18:12:08.349750,True,mitarbeiter.csv,pierre.dubois@kunde.de,[NAME_120] -2025-06-07T18:12:08.349750,True,mitarbeiter.csv,giovanni.bianchi@kunde.de,[NAME_121] -2025-06-07T18:12:08.349750,True,mitarbeiter.csv,william.brown@kunde.de,[NAME_122] -2025-06-07T18:12:08.349750,True,mitarbeiter.csv,1250,[EMAIL_123] -2025-06-07T18:12:08.349750,True,mitarbeiter.csv,890,[EMAIL_124] -2025-06-07T18:12:08.349750,True,mitarbeiter.csv,2340,[EMAIL_125] -2025-06-07T18:12:08.349750,True,mitarbeiter.csv,1750,[EMAIL_126] -2025-06-07T18:12:08.349750,True,mitarbeiter.csv,3200,[EMAIL_127] -2025-06-07T18:12:08.349750,True,mitarbeiter.csv,DE02 5001 0517 5407 3249 31,[IBAN_128] -2025-06-07T18:12:08.349750,True,mitarbeiter.csv,DE27 2005 0550 1045 1862 37,[IBAN_129] -2025-06-07T18:12:08.349750,True,mitarbeiter.csv,DE02 5001 0517 5407 3249 32,[IBAN_130] -2025-06-07T18:12:08.349750,True,mitarbeiter.csv,DE02 5001 0517 5407 3249 33,[IBAN_131] -2025-06-07T18:12:08.349750,True,mitarbeiter.csv,Kirchstraße 10 10115 Berlin,[ADDRESS_132] -2025-06-07T18:12:08.349750,True,mitarbeiter.csv,Seefeldstraße 5 10117 Berlin,[ADDRESS_133] -2025-06-07T18:12:08.349750,True,mitarbeiter.csv,15 Rue de la Paix 10115 Berlin,[ADDRESS_134] -2025-06-07T18:12:08.349750,True,mitarbeiter.csv,Via della Spiga 20 10115 Berlin,[ADDRESS_135] -2025-06-07T18:12:08.349750,True,mitarbeiter.csv,42 Oxford Street 10115 Berlin ,[ADDRESS_136] -2025-06-07T18:12:08.349750,True,mitarbeiter.csv,"9012 - -Cordiali",[ADDRESS_137] -2025-06-07T18:12:08.349750,True,mitarbeiter.csv,0000 0123 456,[PHONE_138] -2025-06-07T18:12:08.349750,True,mitarbeiter.csv,"5678 -Indirizzo",[ADDRESS_139] -2025-06-07T18:12:08.349750,True,mitarbeiter.csv,02 1234 5678,[PHONE_140] -2025-06-07T18:12:08.349750,True,mitarbeiter.csv,info@esempio.it,[EMAIL_141] -2025-06-07T18:12:08.349750,True,mitarbeiter.csv,Michael,[NAME_142] -2025-06-07T18:12:08.349750,True,mitarbeiter.csv,Sabine,[NAME_143] -2025-06-07T18:12:08.349750,True,mitarbeiter.csv,Petra,[NAME_144] -2025-06-07T18:12:08.349750,True,mitarbeiter.csv,Klaus,[NAME_145] -2025-06-07T18:12:08.349750,True,mitarbeiter.csv,Schmidt,[NAME_146] -2025-06-07T18:12:08.349750,True,mitarbeiter.csv,Fischer,[NAME_147] -2025-06-07T18:12:08.349750,True,mitarbeiter.csv,Wagner,[NAME_148] -2025-06-07T18:12:08.349750,True,mitarbeiter.csv,michael.schmidt@kunde.de,[EMAIL_149] -2025-06-07T18:12:08.349750,True,mitarbeiter.csv,sabine.weber@kunde.de,[EMAIL_150] -2025-06-07T18:12:08.349750,True,mitarbeiter.csv,thomas.mueller@kunde.de,[EMAIL_151] -2025-06-07T18:12:08.349750,True,mitarbeiter.csv,petra.fischer@kunde.de,[EMAIL_152] -2025-06-07T18:12:08.349750,True,mitarbeiter.csv,klaus.wagner@kunde.de,[EMAIL_153] -2025-06-07T18:12:08.349750,True,mitarbeiter.csv,+49 89 23456789,[PHONE_154] -2025-06-07T18:12:08.349750,True,mitarbeiter.csv,+49 40 34567890,[PHONE_155] -2025-06-07T18:12:08.349750,True,mitarbeiter.csv,+49 69 45678901,[PHONE_156] -2025-06-07T18:12:08.349750,True,mitarbeiter.csv,+49 211 56789012,[PHONE_157] -2025-06-07T18:12:08.349750,True,mitarbeiter.csv,Hauptstraße 45 80331 München,[ADDRESS_158] -2025-06-07T18:12:08.349750,True,mitarbeiter.csv,Neue Straße 78 20095 Hamburg,[ADDRESS_159] -2025-06-07T18:12:08.349750,True,mitarbeiter.csv,Frankfurter Ring 12 60313 Frankfurt,[ADDRESS_160] -2025-06-07T18:12:08.349750,True,mitarbeiter.csv,Königsallee 92 40212 Düsseldorf,[ADDRESS_161] -2025-06-07T18:12:08.349750,True,mitarbeiter.csv,12/345/67890,[SSN_162] -2025-06-07T18:12:08.349750,True,mitarbeiter.csv,98/765/43210,[SSN_163] -2025-06-07T18:12:08.349750,True,mitarbeiter.csv,45/678/90123,[SSN_164] -2025-06-07T18:12:08.349750,True,mitarbeiter.csv,34/567/89012,[SSN_165] -2025-06-07T18:12:08.349750,True,mitarbeiter.csv,23/456/78901 ,[SSN_166] -2025-06-07T18:12:08.349750,True,mitarbeiter.csv,Andreas,[NAME_167] -2025-06-07T18:12:08.349750,True,mitarbeiter.csv,Monika,[NAME_168] -2025-06-07T18:12:08.349750,True,mitarbeiter.csv,Frank,[NAME_169] -2025-06-07T18:12:08.349750,True,mitarbeiter.csv,Susanne,[NAME_170] -2025-06-07T18:12:08.349750,True,mitarbeiter.csv,Dieter,[NAME_171] -2025-06-07T18:12:08.349750,True,mitarbeiter.csv,Becker,[NAME_172] -2025-06-07T18:12:08.349750,True,mitarbeiter.csv,Hoffmann,[NAME_173] -2025-06-07T18:12:08.349750,True,mitarbeiter.csv,Schäfer,[NAME_174] -2025-06-07T18:12:08.349750,True,mitarbeiter.csv,Koch,[NAME_175] -2025-06-07T18:12:08.349750,True,mitarbeiter.csv,Richter,[NAME_176] -2025-06-07T18:12:08.349750,True,mitarbeiter.csv,andreas.becker@firma.de,[EMAIL_177] -2025-06-07T18:12:08.349750,True,mitarbeiter.csv,monika.hoffmann@firma.de,[EMAIL_178] -2025-06-07T18:12:08.349750,True,mitarbeiter.csv,frank.schaefer@firma.de,[EMAIL_179] -2025-06-07T18:12:08.349750,True,mitarbeiter.csv,susanne.koch@firma.de,[EMAIL_180] -2025-06-07T18:12:08.349750,True,mitarbeiter.csv,dieter.richter@firma.de,[EMAIL_181] -2025-06-07T18:12:08.349750,True,mitarbeiter.csv,+49 30 98765432,[PHONE_182] -2025-06-07T18:12:08.349750,True,mitarbeiter.csv,+49 89 87654321,[PHONE_183] -2025-06-07T18:12:08.349750,True,mitarbeiter.csv,+49 40 76543210,[PHONE_184] -2025-06-07T18:12:08.349750,True,mitarbeiter.csv,+49 69 65432109,[PHONE_185] -2025-06-07T18:12:08.349750,True,mitarbeiter.csv,+49 211 54321098,[PHONE_186] -2025-06-07T18:12:08.349750,True,mitarbeiter.csv,Hauptstraße 1 10115 Berlin,[ADDRESS_187] -2025-06-07T18:12:08.349750,True,mitarbeiter.csv,Leopoldstraße 45 80802 München,[ADDRESS_188] -2025-06-07T18:12:08.349750,True,mitarbeiter.csv,Neuer Wall 78 20354 Hamburg,[ADDRESS_189] -2025-06-07T18:12:08.349750,True,mitarbeiter.csv,Mainzer Landstraße 12 60329 Frankfurt,[ADDRESS_190] -2025-06-07T18:12:08.349750,True,mitarbeiter.csv,23/456/78901,[SSN_191] -2025-06-07T18:12:08.349750,True,mitarbeiter.csv,12 345678 901,[SSN_192] -2025-06-07T18:12:08.349750,True,mitarbeiter.csv,98 765432 102,[SSN_193] -2025-06-07T18:12:08.349750,True,mitarbeiter.csv,45 678901 203,[SSN_194] -2025-06-07T18:12:08.349750,True,mitarbeiter.csv,34 567890 304,[SSN_195] -2025-06-07T18:12:08.349750,True,mitarbeiter.csv,23 456789 405 ,[SSN_196] -2025-06-07T18:12:08.389324,True,swiss.txt,000-180.000,[PHONE_1] -2025-06-07T18:12:08.389324,True,swiss.txt,2027 Bewertung,[ADDRESS_2] -2025-06-07T18:12:08.389324,True,swiss.txt,Max Mustermann,[NAME_3] -2025-06-07T18:12:08.389324,True,swiss.txt,Peter Schmid,[NAME_4] -2025-06-07T18:12:08.389324,True,swiss.txt,Marie Dupont,[NAME_5] -2025-06-07T18:12:08.389324,True,swiss.txt,Marco Rossi,[NAME_6] -2025-06-07T18:12:08.389324,True,swiss.txt,John Smith,[NAME_7] -2025-06-07T18:12:08.389324,True,swiss.txt,max.mustermann@beispiel.de,[EMAIL_8] -2025-06-07T18:12:08.389324,True,swiss.txt,peter.schmid@beispiel.ch,[EMAIL_9] -2025-06-07T18:12:08.389324,True,swiss.txt,marie.dupont@exemple.fr,[EMAIL_10] -2025-06-07T18:12:08.389324,True,swiss.txt,marco.rossi@esempio.it,[EMAIL_11] -2025-06-07T18:12:08.389324,True,swiss.txt,john.smith@example.com,[EMAIL_12] -2025-06-07T18:12:08.389324,True,swiss.txt,+49 30 12345678,[PHONE_13] -2025-06-07T18:12:08.389324,True,swiss.txt,+41 44 123 45 67,[PHONE_14] -2025-06-07T18:12:08.389324,True,swiss.txt,+33 1 23 45 67 89,[PHONE_15] -2025-06-07T18:12:08.389324,True,swiss.txt,+39 02 1234 5678,[PHONE_16] -2025-06-07T18:12:08.389324,True,swiss.txt,+44 20 1234 5678,[PHONE_17] -2025-06-07T18:12:08.389324,True,swiss.txt,Musterstraße 123 12345 Berlin,[ADDRESS_18] -2025-06-07T18:12:08.389324,True,swiss.txt,Bahnhofstrasse 1 8001 Zürich,[ADDRESS_19] -2025-06-07T18:12:08.389324,True,swiss.txt,123 Rue de Paris 75001 Paris,[ADDRESS_20] -2025-06-07T18:12:08.389324,True,swiss.txt,Via Roma 123 20100 Milano,[ADDRESS_21] -2025-06-07T18:12:08.389324,True,swiss.txt,123 High Street London SW1A 1AA,[ADDRESS_22] -2025-06-07T18:12:08.389324,True,swiss.txt,DE89 3704 0044 0532 0130 00,[IBAN_23] -2025-06-07T18:12:08.389324,True,swiss.txt,CH93 0076 7000 E529 3557 7,[IBAN_24] -2025-06-07T18:12:08.389324,True,swiss.txt,FR76 3000 6000 0112 3456 7890 189,[IBAN_25] -2025-06-07T18:12:08.389324,True,swiss.txt,IT60 X054 2811 1010 0000 0123 456,[IBAN_26] -2025-06-07T18:12:08.389324,True,swiss.txt,GB29 NWBK 6016 1331 9268 19,[IBAN_27] -2025-06-07T18:12:08.389324,True,swiss.txt,4532 1234 5678 9012,[IBAN_28] -2025-06-07T18:12:08.389324,True,swiss.txt,4532 1234 5678 9013,[IBAN_29] -2025-06-07T18:12:08.389324,True,swiss.txt,4532 1234 5678 9014,[IBAN_30] -2025-06-07T18:12:08.389324,True,swiss.txt,4532 1234 5678 9015,[IBAN_31] -2025-06-07T18:12:08.389324,True,swiss.txt,4532 1234 5678 9016,[IBAN_32] -2025-06-07T18:12:08.389324,True,swiss.txt,nan,[SSN_33] -2025-06-07T18:12:08.389324,True,swiss.txt,756.1234.5678.90,[SSN_34] -2025-06-07T18:12:08.389324,True,swiss.txt, ,[SSN_35] -2025-06-07T18:12:08.389324,True,swiss.txt,sarah.weber@techsolutions.ch,[EMAIL_36] -2025-06-07T18:12:08.389324,True,swiss.txt,+41 44 987 65,[PHONE_37] -2025-06-07T18:12:08.389324,True,swiss.txt,"Dr. Sarah Weber -TechSolutions",[NAME_38] -2025-06-07T18:12:08.389324,True,swiss.txt,hans.mueller@ethz.ch,[EMAIL_39] -2025-06-07T18:12:08.389324,True,swiss.txt,+41 44 123 45,[PHONE_40] -2025-06-07T18:12:08.389324,True,swiss.txt,l.meier@digitalsystems.ch,[EMAIL_41] -2025-06-07T18:12:08.389324,True,swiss.txt,+41 44 456 78,[PHONE_42] -2025-06-07T18:12:08.389324,True,swiss.txt,987.654.321,[SSN_43] -2025-06-07T18:12:08.389324,True,swiss.txt,CHE-987.654.321,[SSN_44] -2025-06-07T18:12:08.389324,True,swiss.txt,8002 Zürich,[ADDRESS_45] -2025-06-07T18:12:08.389324,True,swiss.txt,Musterstrasse 123,[ADDRESS_46] -2025-06-07T18:12:08.389324,True,swiss.txt,lara.meier@techsolutions.ch,[EMAIL_47] -2025-06-07T18:12:08.389324,True,swiss.txt,123.456.789,[SSN_48] -2025-06-07T18:12:08.389324,True,swiss.txt,CHE-123.456.789,[SSN_49] -2025-06-07T18:12:08.389324,True,swiss.txt,8004 Zürich,[ADDRESS_50] -2025-06-07T18:12:08.389324,True,swiss.txt,Industriestrasse 100,[ADDRESS_51] -2025-06-07T18:12:08.389324,True,swiss.txt,lara.meier@example.ch,[EMAIL_52] -2025-06-07T18:12:08.389324,True,swiss.txt,8001 Zürich,[ADDRESS_53] -2025-06-07T18:12:08.389324,True,swiss.txt,Bahnhofstrasse 45,[ADDRESS_54] -2025-06-07T18:12:08.389324,True,swiss.txt,"1990 -Adresse",[ADDRESS_55] -2025-06-07T18:12:08.389324,True,swiss.txt,Hans,[NAME_56] -2025-06-07T18:12:08.389324,True,swiss.txt,Thomas,[NAME_57] -2025-06-07T18:12:08.389324,True,swiss.txt,Sophie,[NAME_58] -2025-06-07T18:12:08.389324,True,swiss.txt,Luca,[NAME_59] -2025-06-07T18:12:08.389324,True,swiss.txt,Emma,[NAME_60] -2025-06-07T18:12:08.389324,True,swiss.txt,Müller,[NAME_61] -2025-06-07T18:12:08.389324,True,swiss.txt,Weber,[NAME_62] -2025-06-07T18:12:08.389324,True,swiss.txt,Martin,[NAME_63] -2025-06-07T18:12:08.389324,True,swiss.txt,Ferrari,[NAME_64] -2025-06-07T18:12:08.389324,True,swiss.txt,Wilson,[NAME_65] -2025-06-07T18:12:08.389324,True,swiss.txt,hans.mueller@firma.de,[EMAIL_66] -2025-06-07T18:12:08.389324,True,swiss.txt,thomas.weber@firma.ch,[EMAIL_67] -2025-06-07T18:12:08.389324,True,swiss.txt,sophie.martin@entreprise.fr,[EMAIL_68] -2025-06-07T18:12:08.389324,True,swiss.txt,luca.ferrari@azienda.it,[EMAIL_69] -2025-06-07T18:12:08.389324,True,swiss.txt,emma.wilson@company.com,[EMAIL_70] -2025-06-07T18:12:08.389324,True,swiss.txt,+49 89 12345678,[PHONE_71] -2025-06-07T18:12:08.389324,True,swiss.txt,+41 44 234 56 78,[PHONE_72] -2025-06-07T18:12:08.389324,True,swiss.txt,+33 1 34 56 78 90,[PHONE_73] -2025-06-07T18:12:08.389324,True,swiss.txt,+39 02 2345 6789,[PHONE_74] -2025-06-07T18:12:08.389324,True,swiss.txt,+44 20 2345 6789,[PHONE_75] -2025-06-07T18:12:08.389324,True,swiss.txt,Hauptstraße 1 80331 München,[ADDRESS_76] -2025-06-07T18:12:08.389324,True,swiss.txt,Bahnhofstrasse 2 8001 Zürich,[ADDRESS_77] -2025-06-07T18:12:08.389324,True,swiss.txt,15 Avenue des Champs-Élysées 75008 Paris,[ADDRESS_78] -2025-06-07T18:12:08.389324,True,swiss.txt,Via Monte Napoleone 8 20121 Milano,[ADDRESS_79] -2025-06-07T18:12:08.389324,True,swiss.txt,25 Old Street London EC1V 9HL,[ADDRESS_80] -2025-06-07T18:12:08.389324,True,swiss.txt,01-234567-8,[IBAN_81] -2025-06-07T18:12:08.389324,True,swiss.txt,GB29 NWBK 6016 1331 9268 19 ,[IBAN_82] -2025-06-07T18:12:08.389324,True,swiss.txt,"9012 - -Best",[ADDRESS_83] -2025-06-07T18:12:08.389324,True,swiss.txt,"5678 -Address",[ADDRESS_84] -2025-06-07T18:12:08.389324,True,swiss.txt,contact@example.com,[EMAIL_85] -2025-06-07T18:12:08.389324,True,swiss.txt,max.mustermann@example.com,[EMAIL_86] -2025-06-07T18:12:08.389324,True,swiss.txt,+49 123 4567890,[PHONE_87] -2025-06-07T18:12:08.389324,True,swiss.txt,2024-03-15,[DATE_88] -2025-06-07T18:12:08.389324,True,swiss.txt,4111 1111 1111 1111,[IBAN_89] -2025-06-07T18:12:08.389324,True,swiss.txt,Tech Solutions GmbH,[NAME_90] -2025-06-07T18:12:08.389324,True,swiss.txt,Dr. Anna Schmidt,NAME_91 -2025-06-07T18:12:08.389324,True,swiss.txt,anna.schmidt@techsolutions.de,[EMAIL_92] -2025-06-07T18:12:08.389324,True,swiss.txt,Dr. Thomas Weber,[NAME_93] -2025-06-07T18:12:08.389324,True,swiss.txt,thomas.weber@company.de,[EMAIL_94] -2025-06-07T18:12:08.389324,True,swiss.txt,maria.schmidt@company.de,[EMAIL_95] -2025-06-07T18:12:08.389324,True,swiss.txt,+49 40 98765432,[PHONE_96] -2025-06-07T18:12:08.389324,True,swiss.txt,DE27 3704 0044 0532 0130 01,[PHONE_97] -2025-06-07T18:12:08.389324,True,swiss.txt,info@techinnovations.de,[EMAIL_98] -2025-06-07T18:12:08.389324,True,swiss.txt,+49 89 12345679,[PHONE_99] -2025-06-07T18:12:08.389324,True,swiss.txt,DE89 3704 0044 0532 0130 02,[PHONE_100] -2025-06-07T18:12:08.389324,True,swiss.txt,"9012 - -Cordialement",[ADDRESS_101] -2025-06-07T18:12:08.389324,True,swiss.txt,0112 3456 7890,[PHONE_102] -2025-06-07T18:12:08.389324,True,swiss.txt,+33 1 23 45,[PHONE_103] -2025-06-07T18:12:08.389324,True,swiss.txt,contact@exemple.fr,[EMAIL_104] -2025-06-07T18:12:08.389324,True,swiss.txt,"9012 - -Mit",[ADDRESS_105] -2025-06-07T18:12:08.389324,True,swiss.txt,0532 0130 00,[PHONE_106] -2025-06-07T18:12:08.389324,True,swiss.txt,0044 0532 0130,[PHONE_107] -2025-06-07T18:12:08.389324,True,swiss.txt,030-12345678,[PHONE_108] -2025-06-07T18:12:08.389324,True,swiss.txt,sarah.mueller@techsolutions.ch,[EMAIL_109] -2025-06-07T18:12:08.389324,True,swiss.txt,thomas.weber@techsolutions.ch,[EMAIL_110] -2025-06-07T18:12:08.389324,True,swiss.txt,info@techsolutions.ch,[EMAIL_111] -2025-06-07T18:12:08.389324,True,swiss.txt,"2026 -TechSolutions",[ADDRESS_112] -2025-06-07T18:12:08.389324,True,swiss.txt,Anna Schmidt,[DATE_113] -2025-06-07T18:12:08.389324,True,swiss.txt,Franz Huber,[DATE_114] -2025-06-07T18:12:08.389324,True,swiss.txt,Pierre Dubois,[DATE_115] -2025-06-07T18:12:08.389324,True,swiss.txt,Giovanni Bianchi,[DATE_116] -2025-06-07T18:12:08.389324,True,swiss.txt,William Brown,[DATE_117] -2025-06-07T18:12:08.389324,True,swiss.txt,anna.schmidt@kunde.de,[NAME_118] -2025-06-07T18:12:08.389324,True,swiss.txt,franz.huber@kunde.de,[NAME_119] -2025-06-07T18:12:08.389324,True,swiss.txt,pierre.dubois@kunde.de,[NAME_120] -2025-06-07T18:12:08.389324,True,swiss.txt,giovanni.bianchi@kunde.de,[NAME_121] -2025-06-07T18:12:08.389324,True,swiss.txt,william.brown@kunde.de,[NAME_122] -2025-06-07T18:12:08.389324,True,swiss.txt,1250,[EMAIL_123] -2025-06-07T18:12:08.389324,True,swiss.txt,890,[EMAIL_124] -2025-06-07T18:12:08.389324,True,swiss.txt,2340,[EMAIL_125] -2025-06-07T18:12:08.389324,True,swiss.txt,1750,[EMAIL_126] -2025-06-07T18:12:08.389324,True,swiss.txt,3200,[EMAIL_127] -2025-06-07T18:12:08.389324,True,swiss.txt,DE02 5001 0517 5407 3249 31,[IBAN_128] -2025-06-07T18:12:08.389324,True,swiss.txt,DE27 2005 0550 1045 1862 37,[IBAN_129] -2025-06-07T18:12:08.389324,True,swiss.txt,DE02 5001 0517 5407 3249 32,[IBAN_130] -2025-06-07T18:12:08.389324,True,swiss.txt,DE02 5001 0517 5407 3249 33,[IBAN_131] -2025-06-07T18:12:08.389324,True,swiss.txt,Kirchstraße 10 10115 Berlin,[ADDRESS_132] -2025-06-07T18:12:08.389324,True,swiss.txt,Seefeldstraße 5 10117 Berlin,[ADDRESS_133] -2025-06-07T18:12:08.389324,True,swiss.txt,15 Rue de la Paix 10115 Berlin,[ADDRESS_134] -2025-06-07T18:12:08.389324,True,swiss.txt,Via della Spiga 20 10115 Berlin,[ADDRESS_135] -2025-06-07T18:12:08.389324,True,swiss.txt,42 Oxford Street 10115 Berlin ,[ADDRESS_136] -2025-06-07T18:12:08.389324,True,swiss.txt,"9012 - -Cordiali",[ADDRESS_137] -2025-06-07T18:12:08.389324,True,swiss.txt,0000 0123 456,[PHONE_138] -2025-06-07T18:12:08.389324,True,swiss.txt,"5678 -Indirizzo",[ADDRESS_139] -2025-06-07T18:12:08.389324,True,swiss.txt,02 1234 5678,[PHONE_140] -2025-06-07T18:12:08.389324,True,swiss.txt,info@esempio.it,[EMAIL_141] -2025-06-07T18:12:08.389324,True,swiss.txt,Michael,[NAME_142] -2025-06-07T18:12:08.389324,True,swiss.txt,Sabine,[NAME_143] -2025-06-07T18:12:08.389324,True,swiss.txt,Petra,[NAME_144] -2025-06-07T18:12:08.389324,True,swiss.txt,Klaus,[NAME_145] -2025-06-07T18:12:08.389324,True,swiss.txt,Schmidt,[NAME_146] -2025-06-07T18:12:08.389324,True,swiss.txt,Fischer,[NAME_147] -2025-06-07T18:12:08.389324,True,swiss.txt,Wagner,[NAME_148] -2025-06-07T18:12:08.389324,True,swiss.txt,michael.schmidt@kunde.de,[EMAIL_149] -2025-06-07T18:12:08.389324,True,swiss.txt,sabine.weber@kunde.de,[EMAIL_150] -2025-06-07T18:12:08.389324,True,swiss.txt,thomas.mueller@kunde.de,[EMAIL_151] -2025-06-07T18:12:08.389324,True,swiss.txt,petra.fischer@kunde.de,[EMAIL_152] -2025-06-07T18:12:08.389324,True,swiss.txt,klaus.wagner@kunde.de,[EMAIL_153] -2025-06-07T18:12:08.389324,True,swiss.txt,+49 89 23456789,[PHONE_154] -2025-06-07T18:12:08.389324,True,swiss.txt,+49 40 34567890,[PHONE_155] -2025-06-07T18:12:08.389324,True,swiss.txt,+49 69 45678901,[PHONE_156] -2025-06-07T18:12:08.389324,True,swiss.txt,+49 211 56789012,[PHONE_157] -2025-06-07T18:12:08.389324,True,swiss.txt,Hauptstraße 45 80331 München,[ADDRESS_158] -2025-06-07T18:12:08.389324,True,swiss.txt,Neue Straße 78 20095 Hamburg,[ADDRESS_159] -2025-06-07T18:12:08.389324,True,swiss.txt,Frankfurter Ring 12 60313 Frankfurt,[ADDRESS_160] -2025-06-07T18:12:08.389324,True,swiss.txt,Königsallee 92 40212 Düsseldorf,[ADDRESS_161] -2025-06-07T18:12:08.389324,True,swiss.txt,12/345/67890,[SSN_162] -2025-06-07T18:12:08.389324,True,swiss.txt,98/765/43210,[SSN_163] -2025-06-07T18:12:08.389324,True,swiss.txt,45/678/90123,[SSN_164] -2025-06-07T18:12:08.389324,True,swiss.txt,34/567/89012,[SSN_165] -2025-06-07T18:12:08.389324,True,swiss.txt,23/456/78901 ,[SSN_166] -2025-06-07T18:12:08.389324,True,swiss.txt,Andreas,[NAME_167] -2025-06-07T18:12:08.389324,True,swiss.txt,Monika,[NAME_168] -2025-06-07T18:12:08.389324,True,swiss.txt,Frank,[NAME_169] -2025-06-07T18:12:08.389324,True,swiss.txt,Susanne,[NAME_170] -2025-06-07T18:12:08.389324,True,swiss.txt,Dieter,[NAME_171] -2025-06-07T18:12:08.389324,True,swiss.txt,Becker,[NAME_172] -2025-06-07T18:12:08.389324,True,swiss.txt,Hoffmann,[NAME_173] -2025-06-07T18:12:08.389324,True,swiss.txt,Schäfer,[NAME_174] -2025-06-07T18:12:08.389324,True,swiss.txt,Koch,[NAME_175] -2025-06-07T18:12:08.389324,True,swiss.txt,Richter,[NAME_176] -2025-06-07T18:12:08.389324,True,swiss.txt,andreas.becker@firma.de,[EMAIL_177] -2025-06-07T18:12:08.389324,True,swiss.txt,monika.hoffmann@firma.de,[EMAIL_178] -2025-06-07T18:12:08.389324,True,swiss.txt,frank.schaefer@firma.de,[EMAIL_179] -2025-06-07T18:12:08.389324,True,swiss.txt,susanne.koch@firma.de,[EMAIL_180] -2025-06-07T18:12:08.389324,True,swiss.txt,dieter.richter@firma.de,[EMAIL_181] -2025-06-07T18:12:08.389324,True,swiss.txt,+49 30 98765432,[PHONE_182] -2025-06-07T18:12:08.389324,True,swiss.txt,+49 89 87654321,[PHONE_183] -2025-06-07T18:12:08.389324,True,swiss.txt,+49 40 76543210,[PHONE_184] -2025-06-07T18:12:08.389324,True,swiss.txt,+49 69 65432109,[PHONE_185] -2025-06-07T18:12:08.389324,True,swiss.txt,+49 211 54321098,[PHONE_186] -2025-06-07T18:12:08.389324,True,swiss.txt,Hauptstraße 1 10115 Berlin,[ADDRESS_187] -2025-06-07T18:12:08.389324,True,swiss.txt,Leopoldstraße 45 80802 München,[ADDRESS_188] -2025-06-07T18:12:08.389324,True,swiss.txt,Neuer Wall 78 20354 Hamburg,[ADDRESS_189] -2025-06-07T18:12:08.389324,True,swiss.txt,Mainzer Landstraße 12 60329 Frankfurt,[ADDRESS_190] -2025-06-07T18:12:08.389324,True,swiss.txt,23/456/78901,[SSN_191] -2025-06-07T18:12:08.389324,True,swiss.txt,12 345678 901,[SSN_192] -2025-06-07T18:12:08.389324,True,swiss.txt,98 765432 102,[SSN_193] -2025-06-07T18:12:08.389324,True,swiss.txt,45 678901 203,[SSN_194] -2025-06-07T18:12:08.389324,True,swiss.txt,34 567890 304,[SSN_195] -2025-06-07T18:12:08.389324,True,swiss.txt,23 456789 405 ,[SSN_196] -2025-06-07T18:12:08.389324,True,swiss.txt,01-234567,[PHONE_197] -2025-06-07T18:12:08.389324,True,swiss.txt,Bahnhofstrasse 1,[ADDRESS_198] -2025-06-07T18:12:08.389324,True,swiss.txt,info@beispiel.ch,[EMAIL_199] -2025-06-07T18:12:08.389324,True,swiss.txt,"Herr Schmid - -Vielen Dank",[NAME_200] -2025-06-07T18:12:08.431916,True,transactions.csv,000-180.000,[PHONE_1] -2025-06-07T18:12:08.431916,True,transactions.csv,2027 Bewertung,[ADDRESS_2] -2025-06-07T18:12:08.431916,True,transactions.csv,Max Mustermann,[NAME_3] -2025-06-07T18:12:08.431916,True,transactions.csv,Peter Schmid,[NAME_4] -2025-06-07T18:12:08.431916,True,transactions.csv,Marie Dupont,[NAME_5] -2025-06-07T18:12:08.431916,True,transactions.csv,Marco Rossi,[NAME_6] -2025-06-07T18:12:08.431916,True,transactions.csv,John Smith,[NAME_7] -2025-06-07T18:12:08.431916,True,transactions.csv,max.mustermann@beispiel.de,[EMAIL_8] -2025-06-07T18:12:08.431916,True,transactions.csv,peter.schmid@beispiel.ch,[EMAIL_9] -2025-06-07T18:12:08.431916,True,transactions.csv,marie.dupont@exemple.fr,[EMAIL_10] -2025-06-07T18:12:08.431916,True,transactions.csv,marco.rossi@esempio.it,[EMAIL_11] -2025-06-07T18:12:08.431916,True,transactions.csv,john.smith@example.com,[EMAIL_12] -2025-06-07T18:12:08.431916,True,transactions.csv,+49 30 12345678,[PHONE_13] -2025-06-07T18:12:08.431916,True,transactions.csv,+41 44 123 45 67,[PHONE_14] -2025-06-07T18:12:08.431916,True,transactions.csv,+33 1 23 45 67 89,[PHONE_15] -2025-06-07T18:12:08.431916,True,transactions.csv,+39 02 1234 5678,[PHONE_16] -2025-06-07T18:12:08.431916,True,transactions.csv,+44 20 1234 5678,[PHONE_17] -2025-06-07T18:12:08.431916,True,transactions.csv,Musterstraße 123 12345 Berlin,[ADDRESS_18] -2025-06-07T18:12:08.431916,True,transactions.csv,Bahnhofstrasse 1 8001 Zürich,[ADDRESS_19] -2025-06-07T18:12:08.431916,True,transactions.csv,123 Rue de Paris 75001 Paris,[ADDRESS_20] -2025-06-07T18:12:08.431916,True,transactions.csv,Via Roma 123 20100 Milano,[ADDRESS_21] -2025-06-07T18:12:08.431916,True,transactions.csv,123 High Street London SW1A 1AA,[ADDRESS_22] -2025-06-07T18:12:08.431916,True,transactions.csv,DE89 3704 0044 0532 0130 00,[IBAN_23] -2025-06-07T18:12:08.431916,True,transactions.csv,CH93 0076 7000 E529 3557 7,[IBAN_24] -2025-06-07T18:12:08.431916,True,transactions.csv,FR76 3000 6000 0112 3456 7890 189,[IBAN_25] -2025-06-07T18:12:08.431916,True,transactions.csv,IT60 X054 2811 1010 0000 0123 456,[IBAN_26] -2025-06-07T18:12:08.431916,True,transactions.csv,GB29 NWBK 6016 1331 9268 19,[IBAN_27] -2025-06-07T18:12:08.431916,True,transactions.csv,4532 1234 5678 9012,[IBAN_28] -2025-06-07T18:12:08.431916,True,transactions.csv,4532 1234 5678 9013,[IBAN_29] -2025-06-07T18:12:08.431916,True,transactions.csv,4532 1234 5678 9014,[IBAN_30] -2025-06-07T18:12:08.431916,True,transactions.csv,4532 1234 5678 9015,[IBAN_31] -2025-06-07T18:12:08.431916,True,transactions.csv,4532 1234 5678 9016,[IBAN_32] -2025-06-07T18:12:08.431916,True,transactions.csv,nan,[SSN_33] -2025-06-07T18:12:08.431916,True,transactions.csv,756.1234.5678.90,[SSN_34] -2025-06-07T18:12:08.431916,True,transactions.csv, ,[SSN_35] -2025-06-07T18:12:08.431916,True,transactions.csv,sarah.weber@techsolutions.ch,[EMAIL_36] -2025-06-07T18:12:08.431916,True,transactions.csv,+41 44 987 65,[PHONE_37] -2025-06-07T18:12:08.431916,True,transactions.csv,"Dr. Sarah Weber -TechSolutions",[NAME_38] -2025-06-07T18:12:08.431916,True,transactions.csv,hans.mueller@ethz.ch,[EMAIL_39] -2025-06-07T18:12:08.431916,True,transactions.csv,+41 44 123 45,[PHONE_40] -2025-06-07T18:12:08.431916,True,transactions.csv,l.meier@digitalsystems.ch,[EMAIL_41] -2025-06-07T18:12:08.431916,True,transactions.csv,+41 44 456 78,[PHONE_42] -2025-06-07T18:12:08.431916,True,transactions.csv,987.654.321,[SSN_43] -2025-06-07T18:12:08.431916,True,transactions.csv,CHE-987.654.321,[SSN_44] -2025-06-07T18:12:08.431916,True,transactions.csv,8002 Zürich,[ADDRESS_45] -2025-06-07T18:12:08.431916,True,transactions.csv,Musterstrasse 123,[ADDRESS_46] -2025-06-07T18:12:08.431916,True,transactions.csv,lara.meier@techsolutions.ch,[EMAIL_47] -2025-06-07T18:12:08.431916,True,transactions.csv,123.456.789,[SSN_48] -2025-06-07T18:12:08.431916,True,transactions.csv,CHE-123.456.789,[SSN_49] -2025-06-07T18:12:08.431916,True,transactions.csv,8004 Zürich,[ADDRESS_50] -2025-06-07T18:12:08.431916,True,transactions.csv,Industriestrasse 100,[ADDRESS_51] -2025-06-07T18:12:08.431916,True,transactions.csv,lara.meier@example.ch,[EMAIL_52] -2025-06-07T18:12:08.431916,True,transactions.csv,8001 Zürich,[ADDRESS_53] -2025-06-07T18:12:08.431916,True,transactions.csv,Bahnhofstrasse 45,[ADDRESS_54] -2025-06-07T18:12:08.431916,True,transactions.csv,"1990 -Adresse",[ADDRESS_55] -2025-06-07T18:12:08.431916,True,transactions.csv,Hans,[NAME_56] -2025-06-07T18:12:08.431916,True,transactions.csv,Thomas,[NAME_57] -2025-06-07T18:12:08.431916,True,transactions.csv,Sophie,[NAME_58] -2025-06-07T18:12:08.431916,True,transactions.csv,Luca,[NAME_59] -2025-06-07T18:12:08.431916,True,transactions.csv,Emma,[NAME_60] -2025-06-07T18:12:08.431916,True,transactions.csv,Müller,[NAME_61] -2025-06-07T18:12:08.431916,True,transactions.csv,Weber,[NAME_62] -2025-06-07T18:12:08.431916,True,transactions.csv,Martin,[NAME_63] -2025-06-07T18:12:08.431916,True,transactions.csv,Ferrari,[NAME_64] -2025-06-07T18:12:08.431916,True,transactions.csv,Wilson,[NAME_65] -2025-06-07T18:12:08.431916,True,transactions.csv,hans.mueller@firma.de,[EMAIL_66] -2025-06-07T18:12:08.431916,True,transactions.csv,thomas.weber@firma.ch,[EMAIL_67] -2025-06-07T18:12:08.431916,True,transactions.csv,sophie.martin@entreprise.fr,[EMAIL_68] -2025-06-07T18:12:08.431916,True,transactions.csv,luca.ferrari@azienda.it,[EMAIL_69] -2025-06-07T18:12:08.431916,True,transactions.csv,emma.wilson@company.com,[EMAIL_70] -2025-06-07T18:12:08.431916,True,transactions.csv,+49 89 12345678,[PHONE_71] -2025-06-07T18:12:08.431916,True,transactions.csv,+41 44 234 56 78,[PHONE_72] -2025-06-07T18:12:08.431916,True,transactions.csv,+33 1 34 56 78 90,[PHONE_73] -2025-06-07T18:12:08.431916,True,transactions.csv,+39 02 2345 6789,[PHONE_74] -2025-06-07T18:12:08.431916,True,transactions.csv,+44 20 2345 6789,[PHONE_75] -2025-06-07T18:12:08.431916,True,transactions.csv,Hauptstraße 1 80331 München,[ADDRESS_76] -2025-06-07T18:12:08.431916,True,transactions.csv,Bahnhofstrasse 2 8001 Zürich,[ADDRESS_77] -2025-06-07T18:12:08.431916,True,transactions.csv,15 Avenue des Champs-Élysées 75008 Paris,[ADDRESS_78] -2025-06-07T18:12:08.431916,True,transactions.csv,Via Monte Napoleone 8 20121 Milano,[ADDRESS_79] -2025-06-07T18:12:08.431916,True,transactions.csv,25 Old Street London EC1V 9HL,[ADDRESS_80] -2025-06-07T18:12:08.431916,True,transactions.csv,01-234567-8,[IBAN_81] -2025-06-07T18:12:08.431916,True,transactions.csv,GB29 NWBK 6016 1331 9268 19 ,[IBAN_82] -2025-06-07T18:12:08.431916,True,transactions.csv,"9012 - -Best",[ADDRESS_83] -2025-06-07T18:12:08.431916,True,transactions.csv,"5678 -Address",[ADDRESS_84] -2025-06-07T18:12:08.431916,True,transactions.csv,contact@example.com,[EMAIL_85] -2025-06-07T18:12:08.431916,True,transactions.csv,max.mustermann@example.com,[EMAIL_86] -2025-06-07T18:12:08.431916,True,transactions.csv,+49 123 4567890,[PHONE_87] -2025-06-07T18:12:08.431916,True,transactions.csv,2024-03-15,[DATE_88] -2025-06-07T18:12:08.431916,True,transactions.csv,4111 1111 1111 1111,[IBAN_89] -2025-06-07T18:12:08.431916,True,transactions.csv,Tech Solutions GmbH,[NAME_90] -2025-06-07T18:12:08.431916,True,transactions.csv,Dr. Anna Schmidt,NAME_91 -2025-06-07T18:12:08.431916,True,transactions.csv,anna.schmidt@techsolutions.de,[EMAIL_92] -2025-06-07T18:12:08.431916,True,transactions.csv,Dr. Thomas Weber,[NAME_93] -2025-06-07T18:12:08.431916,True,transactions.csv,thomas.weber@company.de,[EMAIL_94] -2025-06-07T18:12:08.431916,True,transactions.csv,maria.schmidt@company.de,[EMAIL_95] -2025-06-07T18:12:08.431916,True,transactions.csv,+49 40 98765432,[PHONE_96] -2025-06-07T18:12:08.431916,True,transactions.csv,DE27 3704 0044 0532 0130 01,[PHONE_97] -2025-06-07T18:12:08.431916,True,transactions.csv,info@techinnovations.de,[EMAIL_98] -2025-06-07T18:12:08.431916,True,transactions.csv,+49 89 12345679,[PHONE_99] -2025-06-07T18:12:08.431916,True,transactions.csv,DE89 3704 0044 0532 0130 02,[PHONE_100] -2025-06-07T18:12:08.431916,True,transactions.csv,"9012 - -Cordialement",[ADDRESS_101] -2025-06-07T18:12:08.431916,True,transactions.csv,0112 3456 7890,[PHONE_102] -2025-06-07T18:12:08.431916,True,transactions.csv,+33 1 23 45,[PHONE_103] -2025-06-07T18:12:08.431916,True,transactions.csv,contact@exemple.fr,[EMAIL_104] -2025-06-07T18:12:08.431916,True,transactions.csv,"9012 - -Mit",[ADDRESS_105] -2025-06-07T18:12:08.431916,True,transactions.csv,0532 0130 00,[PHONE_106] -2025-06-07T18:12:08.431916,True,transactions.csv,0044 0532 0130,[PHONE_107] -2025-06-07T18:12:08.431916,True,transactions.csv,030-12345678,[PHONE_108] -2025-06-07T18:12:08.431916,True,transactions.csv,sarah.mueller@techsolutions.ch,[EMAIL_109] -2025-06-07T18:12:08.431916,True,transactions.csv,thomas.weber@techsolutions.ch,[EMAIL_110] -2025-06-07T18:12:08.431916,True,transactions.csv,info@techsolutions.ch,[EMAIL_111] -2025-06-07T18:12:08.431916,True,transactions.csv,"2026 -TechSolutions",[ADDRESS_112] -2025-06-07T18:12:08.431916,True,transactions.csv,Anna Schmidt,[DATE_113] -2025-06-07T18:12:08.431916,True,transactions.csv,Franz Huber,[DATE_114] -2025-06-07T18:12:08.431916,True,transactions.csv,Pierre Dubois,[DATE_115] -2025-06-07T18:12:08.431916,True,transactions.csv,Giovanni Bianchi,[DATE_116] -2025-06-07T18:12:08.431916,True,transactions.csv,William Brown,[DATE_117] -2025-06-07T18:12:08.431916,True,transactions.csv,anna.schmidt@kunde.de,[NAME_118] -2025-06-07T18:12:08.431916,True,transactions.csv,franz.huber@kunde.de,[NAME_119] -2025-06-07T18:12:08.431916,True,transactions.csv,pierre.dubois@kunde.de,[NAME_120] -2025-06-07T18:12:08.431916,True,transactions.csv,giovanni.bianchi@kunde.de,[NAME_121] -2025-06-07T18:12:08.431916,True,transactions.csv,william.brown@kunde.de,[NAME_122] -2025-06-07T18:12:08.431916,True,transactions.csv,1250,[EMAIL_123] -2025-06-07T18:12:08.431916,True,transactions.csv,890,[EMAIL_124] -2025-06-07T18:12:08.431916,True,transactions.csv,2340,[EMAIL_125] -2025-06-07T18:12:08.431916,True,transactions.csv,1750,[EMAIL_126] -2025-06-07T18:12:08.431916,True,transactions.csv,3200,[EMAIL_127] -2025-06-07T18:12:08.431916,True,transactions.csv,DE02 5001 0517 5407 3249 31,[IBAN_128] -2025-06-07T18:12:08.431916,True,transactions.csv,DE27 2005 0550 1045 1862 37,[IBAN_129] -2025-06-07T18:12:08.431916,True,transactions.csv,DE02 5001 0517 5407 3249 32,[IBAN_130] -2025-06-07T18:12:08.431916,True,transactions.csv,DE02 5001 0517 5407 3249 33,[IBAN_131] -2025-06-07T18:12:08.431916,True,transactions.csv,Kirchstraße 10 10115 Berlin,[ADDRESS_132] -2025-06-07T18:12:08.431916,True,transactions.csv,Seefeldstraße 5 10117 Berlin,[ADDRESS_133] -2025-06-07T18:12:08.431916,True,transactions.csv,15 Rue de la Paix 10115 Berlin,[ADDRESS_134] -2025-06-07T18:12:08.431916,True,transactions.csv,Via della Spiga 20 10115 Berlin,[ADDRESS_135] -2025-06-07T18:12:08.431916,True,transactions.csv,42 Oxford Street 10115 Berlin ,[ADDRESS_136] -2025-06-07T18:12:08.431916,True,transactions.csv,"9012 - -Cordiali",[ADDRESS_137] -2025-06-07T18:12:08.431916,True,transactions.csv,0000 0123 456,[PHONE_138] -2025-06-07T18:12:08.431916,True,transactions.csv,"5678 -Indirizzo",[ADDRESS_139] -2025-06-07T18:12:08.431916,True,transactions.csv,02 1234 5678,[PHONE_140] -2025-06-07T18:12:08.431916,True,transactions.csv,info@esempio.it,[EMAIL_141] -2025-06-07T18:12:08.431916,True,transactions.csv,Michael,[NAME_142] -2025-06-07T18:12:08.431916,True,transactions.csv,Sabine,[NAME_143] -2025-06-07T18:12:08.431916,True,transactions.csv,Petra,[NAME_144] -2025-06-07T18:12:08.431916,True,transactions.csv,Klaus,[NAME_145] -2025-06-07T18:12:08.431916,True,transactions.csv,Schmidt,[NAME_146] -2025-06-07T18:12:08.431916,True,transactions.csv,Fischer,[NAME_147] -2025-06-07T18:12:08.431916,True,transactions.csv,Wagner,[NAME_148] -2025-06-07T18:12:08.431916,True,transactions.csv,michael.schmidt@kunde.de,[EMAIL_149] -2025-06-07T18:12:08.431916,True,transactions.csv,sabine.weber@kunde.de,[EMAIL_150] -2025-06-07T18:12:08.431916,True,transactions.csv,thomas.mueller@kunde.de,[EMAIL_151] -2025-06-07T18:12:08.431916,True,transactions.csv,petra.fischer@kunde.de,[EMAIL_152] -2025-06-07T18:12:08.431916,True,transactions.csv,klaus.wagner@kunde.de,[EMAIL_153] -2025-06-07T18:12:08.431916,True,transactions.csv,+49 89 23456789,[PHONE_154] -2025-06-07T18:12:08.431916,True,transactions.csv,+49 40 34567890,[PHONE_155] -2025-06-07T18:12:08.431916,True,transactions.csv,+49 69 45678901,[PHONE_156] -2025-06-07T18:12:08.431916,True,transactions.csv,+49 211 56789012,[PHONE_157] -2025-06-07T18:12:08.431916,True,transactions.csv,Hauptstraße 45 80331 München,[ADDRESS_158] -2025-06-07T18:12:08.431916,True,transactions.csv,Neue Straße 78 20095 Hamburg,[ADDRESS_159] -2025-06-07T18:12:08.431916,True,transactions.csv,Frankfurter Ring 12 60313 Frankfurt,[ADDRESS_160] -2025-06-07T18:12:08.431916,True,transactions.csv,Königsallee 92 40212 Düsseldorf,[ADDRESS_161] -2025-06-07T18:12:08.431916,True,transactions.csv,12/345/67890,[SSN_162] -2025-06-07T18:12:08.431916,True,transactions.csv,98/765/43210,[SSN_163] -2025-06-07T18:12:08.431916,True,transactions.csv,45/678/90123,[SSN_164] -2025-06-07T18:12:08.431916,True,transactions.csv,34/567/89012,[SSN_165] -2025-06-07T18:12:08.431916,True,transactions.csv,23/456/78901 ,[SSN_166] -2025-06-07T18:12:08.431916,True,transactions.csv,Andreas,[NAME_167] -2025-06-07T18:12:08.431916,True,transactions.csv,Monika,[NAME_168] -2025-06-07T18:12:08.431916,True,transactions.csv,Frank,[NAME_169] -2025-06-07T18:12:08.431916,True,transactions.csv,Susanne,[NAME_170] -2025-06-07T18:12:08.431916,True,transactions.csv,Dieter,[NAME_171] -2025-06-07T18:12:08.431916,True,transactions.csv,Becker,[NAME_172] -2025-06-07T18:12:08.431916,True,transactions.csv,Hoffmann,[NAME_173] -2025-06-07T18:12:08.431916,True,transactions.csv,Schäfer,[NAME_174] -2025-06-07T18:12:08.431916,True,transactions.csv,Koch,[NAME_175] -2025-06-07T18:12:08.431916,True,transactions.csv,Richter,[NAME_176] -2025-06-07T18:12:08.431916,True,transactions.csv,andreas.becker@firma.de,[EMAIL_177] -2025-06-07T18:12:08.431916,True,transactions.csv,monika.hoffmann@firma.de,[EMAIL_178] -2025-06-07T18:12:08.431916,True,transactions.csv,frank.schaefer@firma.de,[EMAIL_179] -2025-06-07T18:12:08.431916,True,transactions.csv,susanne.koch@firma.de,[EMAIL_180] -2025-06-07T18:12:08.431916,True,transactions.csv,dieter.richter@firma.de,[EMAIL_181] -2025-06-07T18:12:08.431916,True,transactions.csv,+49 30 98765432,[PHONE_182] -2025-06-07T18:12:08.431916,True,transactions.csv,+49 89 87654321,[PHONE_183] -2025-06-07T18:12:08.431916,True,transactions.csv,+49 40 76543210,[PHONE_184] -2025-06-07T18:12:08.431916,True,transactions.csv,+49 69 65432109,[PHONE_185] -2025-06-07T18:12:08.431916,True,transactions.csv,+49 211 54321098,[PHONE_186] -2025-06-07T18:12:08.431916,True,transactions.csv,Hauptstraße 1 10115 Berlin,[ADDRESS_187] -2025-06-07T18:12:08.431916,True,transactions.csv,Leopoldstraße 45 80802 München,[ADDRESS_188] -2025-06-07T18:12:08.431916,True,transactions.csv,Neuer Wall 78 20354 Hamburg,[ADDRESS_189] -2025-06-07T18:12:08.431916,True,transactions.csv,Mainzer Landstraße 12 60329 Frankfurt,[ADDRESS_190] -2025-06-07T18:12:08.431916,True,transactions.csv,23/456/78901,[SSN_191] -2025-06-07T18:12:08.431916,True,transactions.csv,12 345678 901,[SSN_192] -2025-06-07T18:12:08.431916,True,transactions.csv,98 765432 102,[SSN_193] -2025-06-07T18:12:08.431916,True,transactions.csv,45 678901 203,[SSN_194] -2025-06-07T18:12:08.431916,True,transactions.csv,34 567890 304,[SSN_195] -2025-06-07T18:12:08.431916,True,transactions.csv,23 456789 405 ,[SSN_196] -2025-06-07T18:12:08.431916,True,transactions.csv,01-234567,[PHONE_197] -2025-06-07T18:12:08.431916,True,transactions.csv,Bahnhofstrasse 1,[ADDRESS_198] -2025-06-07T18:12:08.431916,True,transactions.csv,info@beispiel.ch,[EMAIL_199] -2025-06-07T18:12:08.431916,True,transactions.csv,"Herr Schmid - -Vielen Dank",[NAME_200] -2025-06-07T18:12:08.431916,True,transactions.csv,franz.huber@kunde.ch,[EMAIL_201] -2025-06-07T18:12:08.431916,True,transactions.csv,pierre.dubois@client.fr,[EMAIL_202] -2025-06-07T18:12:08.431916,True,transactions.csv,giovanni.bianchi@cliente.it,[EMAIL_203] -2025-06-07T18:12:08.431916,True,transactions.csv,william.brown@customer.com,[EMAIL_204] -2025-06-07T18:12:08.431916,True,transactions.csv,Seefeldstrasse 5 8008 Zürich,[ADDRESS_205] -2025-06-07T18:12:08.431916,True,transactions.csv,15 Rue de la Paix 75002 Paris,[ADDRESS_206] -2025-06-07T18:12:08.431916,True,transactions.csv,Via della Spiga 20 20121 Milano,[ADDRESS_207] -2025-06-07T18:12:08.431916,True,transactions.csv,42 Oxford Street London W1D 2BJ ,[ADDRESS_208] diff --git a/tests/test_neutralizer/neutralizer.py b/tests/test_neutralizer/neutralizer.py deleted file mode 100644 index 0691cd88..00000000 --- a/tests/test_neutralizer/neutralizer.py +++ /dev/null @@ -1,334 +0,0 @@ -""" -DSGVO-konformer Daten-Neutralisierer für KI-Agentensysteme -Unterstützt TXT, JSON, CSV, Excel und Word-Dateien -Mehrsprachig: DE, EN, FR, IT -""" - -import re -import json -import pandas as pd -from typing import Dict, List, Tuple, Any, Union, Optional -from dataclasses import dataclass -import logging -import traceback -import xml.etree.ElementTree as ET -from io import StringIO -from patterns import Pattern, HeaderPatterns, DataPatterns, get_pattern_for_header, find_patterns_in_text, TextTablePatterns - -# Configure logging -logger = logging.getLogger(__name__) - -@dataclass -class TableData: - """Repräsentiert Tabellendaten""" - headers: List[str] - rows: List[List[str]] - source_type: str # 'csv', 'json', 'xml', 'text_table' - -@dataclass -class PlainText: - """Repräsentiert normalen Text""" - content: str - source_type: str # 'txt', 'docx', 'text_plain' - -@dataclass -class ProcessResult: - """Result of content processing""" - data: Any - mapping: Dict[str, str] - replaced_fields: List[str] - processed_info: Dict[str, Any] # Additional processing information - -class DataAnonymizer: - """Hauptklasse für die Datenanonymisierung""" - - def __init__(self): - """Initialize the anonymizer with patterns""" - self.header_patterns = HeaderPatterns.patterns - self.data_patterns = DataPatterns.patterns - self.replaced_fields = set() - self.mapping = {} - self.processing_info = [] - - def _normalize_whitespace(self, text: str) -> str: - """Normalize whitespace in text""" - text = re.sub(r'\s+', ' ', text) - text = text.replace('\r\n', '\n').replace('\r', '\n') - return text.strip() - - def _is_table_line(self, line: str) -> bool: - """Check if a line represents a table row""" - return bool(re.match(r'^\s*[^:]+:\s*[^:]+$', line) or - re.match(r'^\s*[^\t]+\t[^\t]+$', line)) - - def _extract_tables_from_text(self, content: str) -> Tuple[List[TableData], List[PlainText]]: - """ - Extract tables and plain text from content - - Args: - content: Content to process - - Returns: - Tuple of (list of tables, list of plain text sections) - """ - tables = [] - plain_texts = [] - - # Process the entire content as plain text - plain_texts.append(PlainText(content=content, source_type='text_plain')) - - return tables, plain_texts - - def _anonymize_table(self, table: TableData) -> TableData: - """Anonymize table data""" - try: - anonymized_table = TableData( - headers=table.headers.copy(), - rows=[row.copy() for row in table.rows], - source_type=table.source_type - ) - - for i, header in enumerate(anonymized_table.headers): - pattern = get_pattern_for_header(header, self.header_patterns) - if pattern: - for row in anonymized_table.rows: - if row[i] is not None: - original = str(row[i]) - if original not in self.mapping: - self.mapping[original] = pattern.replacement_template.format(len(self.mapping) + 1) - row[i] = self.mapping[original] - - return anonymized_table - - except Exception as e: - logger.error(f"Error anonymizing table: {str(e)}") - logger.debug(traceback.format_exc()) - raise - - def _anonymize_plain_text(self, text: PlainText) -> PlainText: - """Anonymize plain text content""" - try: - # Process the entire text at once instead of line by line - current_text = text.content - - # Find all matches in the entire text - matches = find_patterns_in_text(current_text, self.data_patterns) - - # Process matches in reverse order to avoid position shifting - for match in sorted(matches, key=lambda x: x[2], reverse=True): - pattern_name, matched_text, start, end = match - - # Skip if the matched text is already a placeholder - if re.match(r'\[[A-Z_]+\d+\]', matched_text): - continue - - # Find the pattern that matched - pattern = next((p for p in self.data_patterns if p.name == pattern_name), None) - if pattern: - # Use the pattern's replacement template - if matched_text not in self.mapping: - self.mapping[matched_text] = pattern.replacement_template.format(len(self.mapping) + 1) - replacement = self.mapping[matched_text] - - # Replace the matched text while preserving surrounding whitespace - current_text = current_text[:start] + replacement + current_text[end:] - - return PlainText(content=current_text, source_type=text.source_type) - - except Exception as e: - logger.error(f"Error anonymizing plain text: {str(e)}") - logger.debug(traceback.format_exc()) - raise - - def _anonymize_json_value(self, value: Any, key: str = None) -> Any: - """ - Recursively anonymize JSON values based on their keys and content - - Args: - value: Value to anonymize - key: Key name (if part of a key-value pair) - - Returns: - Anonymized value - """ - if isinstance(value, dict): - return {k: self._anonymize_json_value(v, k) for k, v in value.items()} - elif isinstance(value, list): - return [self._anonymize_json_value(item) for item in value] - elif isinstance(value, str): - # Check if this is a key we should process - if key: - pattern = get_pattern_for_header(key, self.header_patterns) - if pattern: - if value not in self.mapping: - self.mapping[value] = pattern.replacement_template.format(len(self.mapping) + 1) - return self.mapping[value] - - # Check if the value itself matches any patterns - matches = find_patterns_in_text(value, self.data_patterns) - if matches: - # Use the first match's pattern - pattern_name = matches[0][0] - if value not in self.mapping: - self.mapping[value] = f"{pattern_name.upper()}_{len(self.mapping) + 1}" - return self.mapping[value] - - return value - else: - return value - - def _anonymize_xml_element(self, element: ET.Element, indent: str = '') -> str: - """ - Recursively process XML element and return formatted string - - Args: - element: XML element to process - indent: Current indentation level - - Returns: - Formatted XML string - """ - # Process attributes - processed_attrs = {} - for attr_name, attr_value in element.attrib.items(): - # Check if attribute name matches any header patterns - pattern = get_pattern_for_header(attr_name, self.header_patterns) - if pattern: - if attr_value not in self.mapping: - self.mapping[attr_value] = pattern.replacement_template.format(len(self.mapping) + 1) - processed_attrs[attr_name] = self.mapping[attr_value] - else: - # Check if attribute value matches any data patterns - matches = find_patterns_in_text(attr_value, self.data_patterns) - if matches: - pattern_name = matches[0][0] - pattern = next((p for p in self.data_patterns if p.name == pattern_name), None) - if pattern: - if attr_value not in self.mapping: - self.mapping[attr_value] = pattern.replacement_template.format(len(self.mapping) + 1) - processed_attrs[attr_name] = self.mapping[attr_value] - else: - processed_attrs[attr_name] = attr_value - else: - processed_attrs[attr_name] = attr_value - - attrs = ' '.join(f'{k}="{v}"' for k, v in processed_attrs.items()) - attrs = f' {attrs}' if attrs else '' - - # Process text content - text = element.text.strip() if element.text and element.text.strip() else '' - if text: - # Check if text matches any patterns - matches = find_patterns_in_text(text, self.data_patterns) - if matches: - pattern_name = matches[0][0] - pattern = next((p for p in self.data_patterns if p.name == pattern_name), None) - if pattern: - if text not in self.mapping: - self.mapping[text] = pattern.replacement_template.format(len(self.mapping) + 1) - text = self.mapping[text] - - # Process child elements - children = [] - for child in element: - child_str = self._anonymize_xml_element(child, indent + ' ') - children.append(child_str) - - # Build element string - if not children and not text: - return f"{indent}<{element.tag}{attrs}/>" - elif not children: - return f"{indent}<{element.tag}{attrs}>{text}" - else: - result = [f"{indent}<{element.tag}{attrs}>"] - if text: - result.append(f"{indent} {text}") - result.extend(children) - result.append(f"{indent}") - return '\n'.join(result) - - def process_content(self, content: str, content_type: str) -> ProcessResult: - """ - Process content and return anonymized data - - Args: - content: Content to process - content_type: Type of content ('csv', 'json', 'xml', 'text') - - Returns: - ProcessResult: Contains anonymized data, mapping, replaced fields and processing info - """ - try: - replaced_fields = [] - processed_info = {} - - if content_type in ['csv', 'json', 'xml']: - # Handle as table - if content_type == 'csv': - df = pd.read_csv(StringIO(content), encoding='utf-8') - table = TableData( - headers=df.columns.tolist(), - rows=df.values.tolist(), - source_type='csv' - ) - processed_info['type'] = 'table' - processed_info['headers'] = table.headers - processed_info['row_count'] = len(table.rows) - elif content_type == 'json': - data = json.loads(content) - # Process JSON recursively - result = self._anonymize_json_value(data) - processed_info['type'] = 'json' - return ProcessResult(result, self.mapping, replaced_fields, processed_info) - else: # xml - root = ET.fromstring(content) - # Process XML recursively with proper formatting - result = self._anonymize_xml_element(root) - processed_info['type'] = 'xml' - return ProcessResult(result, self.mapping, replaced_fields, processed_info) - - if not table.rows: - return ProcessResult(None, self.mapping, [], processed_info) - - anonymized_table = self._anonymize_table(table) - - # Track replaced fields - for i, header in enumerate(anonymized_table.headers): - for orig_row, anon_row in zip(table.rows, anonymized_table.rows): - if anon_row[i] != orig_row[i]: - replaced_fields.append(header) - - # Convert back to original format - if content_type == 'csv': - result = pd.DataFrame(anonymized_table.rows, columns=anonymized_table.headers) - elif content_type == 'json': - if len(anonymized_table.headers) == 1 and anonymized_table.headers[0] == 'value': - result = anonymized_table.rows[0][0] - else: - result = dict(zip(anonymized_table.headers, anonymized_table.rows[0])) - else: # xml - result = ET.tostring(root, encoding='unicode') - - return ProcessResult(result, self.mapping, replaced_fields, processed_info) - else: - # Handle as text - # First, identify what needs to be replaced using table detection - tables, plain_texts = self._extract_tables_from_text(content) - processed_info['type'] = 'text' - processed_info['tables'] = [{'headers': t.headers, 'row_count': len(t.rows)} for t in tables] - - # Process plain text sections - anonymized_texts = [self._anonymize_plain_text(text) for text in plain_texts] - - # Combine all processed content - result = content - for text, anonymized_text in zip(plain_texts, anonymized_texts): - if text.content != anonymized_text.content: - result = result.replace(text.content, anonymized_text.content) - - return ProcessResult(result, self.mapping, replaced_fields, processed_info) - - except Exception as e: - logger.error(f"Error processing content: {str(e)}") - logger.debug(traceback.format_exc()) - return ProcessResult(None, self.mapping, [], {'type': 'error', 'error': str(e)}) \ No newline at end of file diff --git a/tests/test_neutralizer/output/neutralized_Case.md b/tests/test_neutralizer/output/neutralized_Case.md deleted file mode 100644 index e05a856e..00000000 --- a/tests/test_neutralizer/output/neutralized_Case.md +++ /dev/null @@ -1,608 +0,0 @@ - -# Bewertung der PowerOn AI Platform - -Basierend auf dem nachstehenden Q&A ergibt sich nachfolgende Bewertung des Softwareprodukts "PowerOn AI Platform". - -## Aktueller Wert per Juni 2025 - -1. **Technischer Wert des Codes**: - - Professionelle, modulare Codebasis (~50.000-60.000 LOC) - - Moderne Architektur mit innovativen Komponenten - - 5 Personenmonate Entwicklung x CHF 15.000/PM = CHF 75.000 - - Zusätzlicher Wert durch Enterprise-Ready Architektur: CHF 50.000 - -2. **Bisherige Investitionen**: - - Hardware/Software: CHF 20.000 - - Expertise-Premium (30+ Jahre Erfahrung): CHF 25.000 - -3. **IP und Innovationswert**: - - Multi-Agent Workflow-System - - Modulare Architecture mit Alleinstellungsmerkmalen - - Geschätzter Wert: CHF 100.000 - -4. **Marktpotenzial-Faktor**: - - Adressierbares Marktvolumen von CHF 500-700 Mio. - - Wachstumsmarkt (25-30% jährlich) - - Frühphasen-Multiplikator: 2x - -**Aktueller Gesamtwert (Juni 2025)**: **CHF 500.000** - -## Prognostizierte Wertentwicklung - -### Ende 2025 -- Abschluss der technischen Entwicklung -- Erste Pilotprojekte mit 3-5 Referenzkunden -- Validierung des Produkts am Markt -- **Geschätzter Wert Ende 2025**: **CHF 1,2 Millionen** - (Steigerung durch Marktvalidierung und Risikoreduktion) - -### Ende 2026 -- 20-30 Kunden -- ARR: CHF 0,5-0,8 Mio. -- Etablierung im DACH-Markt -- **Geschätzter Wert Ende 2026**: **CHF 4-5 Millionen** - (Bewertungsmultiplikator von 6-8x ARR für wachstumsstarke SaaS) - -### Ende 2027 -- 70-90 Kunden -- ARR: CHF 2-2,5 Mio. -- Erweiterung der Produktpalette -- **Geschätzter Wert Ende 2027**: **CHF 12-15 Millionen** - (Bewertungsmultiplikator von 6x ARR) - -### Ende 2028 -- 150+ Kunden -- ARR: CHF 4,5 Mio. -- Internationale Expansion -- **Geschätzter Wert Ende 2028**: **CHF 25-30 Millionen** - (Bewertungsmultiplikator von 5,5-6,5x ARR für etablierte SaaS) - -## Schlüsselfaktoren und Risikobeurteilung für die Wertentwicklung - -1. **Erfolgreiche Markteinführung**: Der Übergang von Entwicklung zu erfolgreicher Pilotphase ist kritisch für die Wertentwicklung 2025-2026. - -2. **Skalierung des Vertriebs**: Die Fähigkeit, die Kundenakquisition gemäss der Prognose zu skalieren, ist entscheidend für die 2026-[ADDRESS_2]. - -3. **Kapitaleffizienz**: Die effiziente Nutzung des Kapitals (CHF 750.000-1.050.000) für die nächste Entwicklungsphase wird die Bewertung massgeblich beeinflussen. - -4. **Marktdynamik**: Die Entwicklung des KI-Marktes und regulatorische Änderungen können sowohl positive als auch negative Auswirkungen haben. - -Diese Bewertung basiert auf der Annahme, dass die Meilensteine wie geplant erreicht werden und keine signifikanten externen Faktoren die Marktentwicklung negativ beeinflussen. - - -# Kriterienkatalog zur Softwarebewertung - -## Teil 1: Technische Bewertung (Code-basiert) - -1. Wie umfangreich ist die Codebasis (LOC, Module, Komponenten)? -2. Welche Programmiersprachen und Frameworks wurden verwendet? -3. Wie hoch ist die Codequalität und -konsistenz (saubere Architektur, Dokumentation, Tests)? -4. Gibt es innovative Algorithmen oder patentierbare technische Lösungen? -5. Wie modular und wartbar ist die Software gestaltet? -6. Wie robust ist die Fehlerbehandlung und Sicherheitsarchitektur? -7. Wie skalierbar ist die technische Infrastruktur? -8. Gibt es technische Schulden, die zukünftige Entwicklungen behindern könnten? - -### 1. Umfang der Codebasis -- **Frontend**: Modulare JavaScript-Struktur mit ca. 15 Hauptmodulen -- **Backend**: Python/FastAPI mit ca. 15 Hauptmodulen -- **Hauptkomponenten**: - - Frontend: Workflow, UI, Koordination, Datenmanagement - - Backend: Gateway, Agent Service, Connectors, Workflow Manager -- **Geschätzte LOC**: ~50,000-60,000 Zeilen Code - -### 2. Programmiersprachen und Frameworks -- **Frontend**: - - JavaScript (ES6+) - - Modulares System mit ES6-Import/Export - - Vanilla JS ohne externe Frameworks -- **Backend**: - - Python 3.x - - FastAPI für REST-API - - Asyncio für asynchrone Verarbeitung - -### 3. Codequalität und -konsistenz -- **Saubere Architektur**: - - Klare Trennung Frontend/Backend - - Modulare Struktur mit definierten Verantwortlichkeiten - - State Machine Pattern für Workflow-Management -- **Dokumentation**: - - Ausführliche JSDoc/Python-Docstrings - - Architekturdiagramme (Mermaid) - - Technische Spezifikationen -- **Tests**: - - Automatisierte Modultests - - Manuell Integrationstests - - Benutzertests über Tickets in Clickup - -### 4. Innovative Algorithmen/patentierbare Lösungen -- Multi-Agent Workflow-System mit spezialisierten Agenten -- Modulares Agent-Registry-System -- State Machine für Workflow-Koordination -- Dynamische Agenten-Integration - -### 5. Modularität und Wartbarkeit -- **Hohe Modularität**: - - Klare Trennung der Verantwortlichkeiten - - Plug-and-Play Agent-System - - Erweiterbare Connector-Architektur -- **Wartbarkeit**: - - Konsistente Codestruktur - - Klare Namenskonventionen - - Dokumentierte Schnittstellen - -### 6. Fehlerbehandlung und Sicherheit -- **Robuste Fehlerbehandlung**: - - State Machine für Workflow-Status - - Exception Handling auf allen Ebenen - - Logging-System für Debugging -- **Sicherheitsarchitektur**: - - Multi-Tenant-Architektur - - Authentifizierung/Autorisierung - - Mandantenverwaltung - -### 7. Skalierbarkeit -- **Horizontale Skalierbarkeit**: - - Modulare Architektur - - Asynchrone Verarbeitung - - Connector-System für externe Dienste -- **Vertikale Skalierbarkeit**: - - Workflow-Parallelisierung - - Agent-Pooling - - Caching-Mechanismen - -### 8. Technische Schulden -- **Potenzielle Verbesserungsbereiche**: - - Test-Coverage nicht sichtbar - - Eventuell fehlende Performance-Optimierungen - - Dokumentation für Zielgruppen noch unvollständig -- **Keine kritischen Blockierer identifiziert** - -### Fazit der technischen Bewertung -Die Codebasis zeigt eine professionelle, gut strukturierte Enterprise-Anwendung mit klarer Architektur und modernen Best Practices. Die modulare Struktur und die saubere Implementierung der State Machine für Workflow-Management sind besonders hervorzuheben. Die Anwendung ist technisch reif und zeigt ein hohes Mass an Professionalität in der Implementierung. - - - -## Teil 2: Bewertung der bisherigen Aufwände - -### 1. Entwicklungsaufwände -- **Patrick**: - - ValueOn AG: 80 Stunden - - Private Entwicklung: 650 Stunden - - **Gesamt**: 730 Stunden (ca. 4.5 Personenmonate) -- **Ida**: - - ValueOn AG: 60 Stunden - - **Gesamt**: 60 Stunden (ca. 0.4 Personenmonate) -- **Gesamtaufwand**: ~5 Personenmonate - -### 2. Qualifikationen und Erfahrungslevel -- **Patrick**: - - Business Consultant - - Software Architect - - Full Stack Developer - - 30+ Jahre Berufserfahrung - - Experte für Enterprise-Architekturen -- **Ida**: - - Business Analyst - - Project Manager - - Scrum Master - - Erfahrung in agiler Entwicklung - - Expertise in Prozessoptimierung - -### 3. Spezifische Fachkenntnisse -- **Patrick**: - - Umfassende Markt- und Business-Erfahrung (30 Jahre) - - Expertise in Software-Architektur - - Full Stack Entwicklung - - Azure Cloud-Integration - - KI/ML Integration - - Enterprise-Systeme -- **Ida**: - - Projektmanagement - - Agile Methoden - - Business Analysis - - Prozessoptimierung - - Qualitätssicherung - -### 4. Finanzielle Investitionen -- **Hardware**: CHF 10,000 - - Entwicklungsserver - - Testumgebungen - - Entwicklungshardware -- **Software & Lizenzen**: CHF 10,000 - - Entwicklungstools - - Cloud-Services - - KI-API-Zugänge -- **Gesamt**: CHF 20,000 - -### 5. Externe Dienstleister -- **Aktueller Status**: Keine externen Dienstleister -- **Vorteile**: - - Volle Kontrolle über Entwicklung - - Tiefes Verständnis der Architektur - - Schnelle Entscheidungswege - - Kosteneffizienz - -### 6. Schlüsselkomponenten-Entwicklung -- **Patrick**: - - Frontend-Architektur - - Backend-System - - Workflow-Engine - - Agent-System - - Connector-Framework - - Datenmanagement - - Sicherheitsarchitektur -- **Verantwortlichkeiten**: - - Systemarchitektur - - Technische Leitung - - Code-Review - - Qualitätssicherung - -### 7. Nicht-monetäre Ressourcen -- **Dominic**: - - Umfangreiches Sales & Marketing Netzwerk - - Marktzugang - - Branchenkontakte -- **ValueOn AG**: - - Infrastruktur - - Rechtlicher Rahmen - - Geschäftsprozesse -- **Netzwerke**: - - Technologie-Partner - - Potenzielle Kunden - - Branchenexperten - -### 8. Finanzielle Risiken -- **ValueOn AG**: - - Bereitstellung von Infrastruktur - - Personelle Ressourcen - - Rechtlicher Rahmen -- **Private Investitionen**: - - Entwicklungszeit - - Hardware/Software - - Cloud-Services - -### Fazit bisherige Aufwände -Die bisherigen Aufwände zeigen ein ausgewogenes Verhältnis zwischen technischer Expertise und Business-Know-how. Die private Investition von 730 Stunden durch Patrick demonstriert ein hohes Engagement und tiefes Verständnis der Technologie. Die Kombination aus technischer Expertise, Business-Erfahrung und Marktzugang bildet eine solide Grundlage für die weitere Entwicklung. Die bisherigen Investitionen sind effizient eingesetzt worden, mit Fokus auf kritische Kernkomponenten und skalierbare Architektur. - -## Teil 3: Markt- und Geschäftspotenzial - -### 1. Adressierbarer Gesamtmarkt und Wachstumspotenzial -- **Gesamtmarktvolumen 2025**: - - KI-Markt: $190 Mrd. - - Business Process Automation: $19,6 Mrd. - - Enterprise Knowledge Management: $43 Mrd. -- **Adressierbarer Markt (SAM)**: - - Initial: Mittlerer Markt in DACH (Professional Services, Finanzdienstleistungen, Gesundheitswesen) - - Geschätztes SAM: CHF 500-700 Mio. -- **Wachstumspotenzial**: - - Jährliches Marktwachstum: 25-30% - - Erweiterung auf internationale Märkte - - Branchenspezifische Lösungen - -### 2. Alleinstellungsmerkmale -1. **Technologische Vorteile**: - - Proprietäre Multi-Agent-Technologie - - Modellunabhängige KI-Integration - - Enterprise-Ready Architektur - - Fortschrittliche Workflow-Orchestrierung - -2. **Funktionale Vorteile**: - - Nahtlose Integration verschiedener KI-Modelle - - Robuste Fehlerbehandlung - - Skalierbare Multi-Tenant-Architektur - - Umfassende Enterprise-Features - -### 3. Kunden und Pilotprojekte -- **Aktueller Status**: - - In Entwicklung - - Erste Referenzkunden in Planung - - Fokus auf mittelständische Unternehmen -- **Pilotphase**: - - Geplant für Q3/Q4 2025 - - 3-5 Schlüsselreferenzkunden - - Branchenspezifische Templates - -### 4. Geschäftsmodell -- **Hauptmodell**: SaaS (Software as a Service) -- **Preismodell**: - - Basis: Pro-Benutzer/Monat Abonnement - - Zusätzlich: Nutzungsbasierte Abrechnung - - Enterprise-Lizenzen für grössere Kunden -- **Erwartete Margen**: 75-85% nach Skalierung - -### 5. Preisgestaltung -- **Wettbewerbsvergleich**: - - Unterhalb Enterprise-Lösungen - - Über Standard-BPA-Tools - - Flexiblere Preisgestaltung als Konkurrenz -- **Preisstruktur**: - - Basis-Abonnement: CHF 50-100 pro Benutzer/Monat - - Nutzungsbasierte Komponente: CHF 0.10-0.50 pro Verarbeitungseinheit - - Enterprise-Pakete: Individuelle Preisgestaltung - -### 6. Umsatzpotenziale -- **Jahr 1 (2026)**: - - Ziel: 20-30 Kunden - - Erwartetes ARR: CHF 0.5-0.8 Mio. -- **Jahr 2 (2027)**: - - Ziel: 70-90 Kunden - - Erwartetes ARR: CHF 2-2.5 Mio. -- **Jahr 3 (2028)**: - - Ziel: 150+ Kunden - - Erwartetes ARR: CHF 4.5 Mio. - -### 7. Akquisitionskosten -- **Customer Acquisition Cost (CAC)**: - - Erwarteter CAC: CHF 15,000-20,000 - - Payback-Zeit: 12-18 Monate -- **Kostenstruktur**: - - 30% Vertrieb und Marketing - - Fokus auf effiziente Akquisition - - Skaleneffekte ab 50+ Kunden - -### 8. Regulatorische Herausforderungen -- **Datenschutz**: - - DSGVO-Konformität - - Datensicherheit - - Mandantentrennung -- **KI-Regulierung**: - - EU AI Act - - Transparenzpflichten - - Qualitätssicherung -- **Branchenspezifische Regulierung**: - - Finanzdienstleistungen - - Gesundheitswesen - - Professional Services - -### Fazit Markt- und Geschäftspotenzial -Die PowerOn AI Platform adressiert einen wachsenden Markt mit klaren Alleinstellungsmerkmalen. Das Geschäftsmodell ist skalierbar und die Preisgestaltung wettbewerbsfähig. Die regulatorischen Herausforderungen sind bekannt und adressierbar. Die Umsatzprognosen sind konservativ kalkuliert und basieren auf realistischen Marktannahmen. - -## Teil 4: Skalierungs- und Zukunftspotenzial - -### 1. Ressourcen für Support und Weiterentwicklung -- **Entwicklungsteam**: - - 2-3 Full-Stack Entwickler, KI-unterstützte Entwicklung - - 1 DevOps Engineer - - 1 QA Engineer -- **Support-Team**: - - 1-2 Support Engineers - - 1 Technical Account Manager -- **Infrastruktur**: - - Cloud-basierte Skalierung (Azure) - - Automatisierte Deployment-Pipeline - - Monitoring und Logging-Systeme - -### 2. Skalierbarkeit -- **Technische Skalierbarkeit**: - - Horizontale Skalierung durch Multi-Tenant-Architektur - - Vertikale Skalierung durch Agent-Pooling - - Automatische Lastverteilung -- **Skalierungszeitrahmen**: - - 2x Nutzer: Sofort möglich - - 5x Nutzer: 1-2 Monate Vorbereitung - - 10x Nutzer: 3-4 Monate mit Infrastruktur-Erweiterung - -### 3. Kapitalbedarf -- **Nächste Entwicklungsphase (12 Monate)**: - - Entwicklung: CHF 400,000-500,000 - - Marketing & Sales: CHF 100,000-200,000 - - Infrastruktur: CHF 50,000-100,000 - - Betrieb & Support: CHF 100,000-150,000 - - **Gesamt**: CHF 750,000-1,050,000 - -### 4. Schlüsselpersonen -- **Technische Leitung**: - - Patrick (Software Architect, Full Stack Developer) - - Verantwortlich für: Architektur, Entwicklung, Technische Strategie -- **Business & Operations**: - - Ida (Business Analyst, Project Manager) - - Verantwortlich für: Projektmanagement, Business Analysis -- **Sales & Marketing**: - - Dominic (Sales & Marketing) - - Verantwortlich für: Marktentwicklung, Kundenakquisition - -### 5. Exit-Strategien -- **Strategische Übernahme**: - - Enterprise Software Anbieter - - KI/ML Plattform Betreiber - - Business Process Automation Unternehmen -- **IPO-Potenzial**: - - Ab CHF 50 Mio. ARR - - Zeitrahmen: 5-7 Jahre -- **Extraktion aus ValueOn AG**: - - Vergütung der Aufwände - - Anrechnung des Mehrwerts für Schlüsselpersonen - - Beschaffung des notwendigen Kapitals - -### 6. Strategische Partnerschaften -- **Technologie-Partner**: - - KI-Provider (OpenAI, Anthropic) - - Cloud-Provider (Azure) - - Enterprise Software Anbieter -- **Vertriebspartner**: - - Systemhäuser - - Beratungsunternehmen - - Branchenspezialisten -- **Forschungspartner**: - - Universitäten - - Forschungsinstitute - - KI-Labore - -### 7. Geplante Erweiterungen -- **Kurzfristig (12 Monate)**: - - Erweiterte Agent-Typen - - Branchenspezifische Templates - - API-Erweiterungen -- **Mittelfristig (24 Monate)**: - - Agentenmarktplatz - - Proprietäre KI-Modelle - - Erweiterte Analytics -- **Langfristig (36+ Monate)**: - - KI-Middleware für Unternehmen - - Branchenlösungen - - Internationale Expansion - -### 8. Langfristige Vision -- **Technologische Vision**: - - Führende Multi-Agent KI-Plattform - - Standard für Enterprise Workflow Automation - - Innovationstreiber in der KI-Integration -- **Marktvision**: - - Globaler Marktführer in Nischenbereichen - - Branchenstandard für bestimmte Anwendungsfälle - - Referenz für KI-gestützte Prozessoptimierung -- **Geschäftsvision**: - - Nachhaltiges Wachstum - - Profitables Geschäftsmodell - - Führende Position in ausgewählten Märkten - -### Fazit Skalierungs- und Zukunftspotenzial -Die PowerOn AI Platform verfügt über ein solides Skalierungspotenzial sowohl technisch als auch geschäftlich. Die modulare Architektur ermöglicht schnelles Wachstum, während die klare Vision und die strategischen Partnerschaften den langfristigen Erfolg unterstützen. Die Kapitalanforderungen sind realistisch kalkuliert und die Exit-Strategien bieten verschiedene Optionen für die Zukunft. - - -# Exitplan: PowerOn AI Platform als eigenständige AG - -## Bewertung und Ausgangssituation - -**Aktueller Wert (Juni 2025)**: CHF 500.000 -**Angepasster Gründungswert**: CHF 800.000 (berücksichtigt den strategischen Wert der Produktvision, welche bereits als innerer Wert im Produkt enthalten ist) - -## Strukturierung der PowerOn AG - -### 1. Aktienstruktur bei Gründung - -**Gesamtes Aktienkapital**: 1.000.000 Aktien (Nennwert CHF 0,10) -**Firmenvaluation bei Gründung**: CHF 800.000 - -#### Verteilung der initialen Aktien: - -- **Patrick**: - - Eingebrachte Leistung: Entwicklung, technische Expertise und essenzielles Gesamtkonzept - - **Aktienanteil**: 35% (350.000 Aktien) - -- **Dominic**: - - Eingebrachte Leistung: Netzwerk, Sales & Marketing Expertise und Vision - - **Aktienanteil**: 15% (150.000 Aktien) - -- **ValueOn AG**: - - Eingebrachte Leistung: Infrastruktur, rechtlicher Rahmen, Arbeitszeit, Übertragung von IP - - **Aktienanteil**: 25% (250.000 Aktien) - -- **Reservierter Anteil für Mitarbeiter-Pool**: - - **Aktienanteil**: 10% (100.000 Aktien) - -- **Reserviert für Investoren (erste Runde)**: - - **Aktienanteil**: 15% (150.000 Aktien) - -### 2. Kapitalbedarfsplanung (18 Monate) - -| Kategorie | Betrag (CHF) | -|-----------|--------------| -| Entwicklung | 550.000 | -| Marketing & Sales | 250.000 | -| Infrastruktur | 100.000 | -| Betrieb & Support | 200.000 | -| **Gesamtbedarf** | **1.100.000** | - -**Kapitalbeschaffungsstrategie**: -- **Erste Finanzierungsrunde**: CHF 1.000.000 (für 18 Monate) -- **Sicherheitspuffer**: CHF 100.000 (aus Umsätzen/zukünftigen Einnahmen) - -### 3. Investitionskonditionen - -**Pre-Money Bewertung**: CHF 800.000 -**Investitionsvolumen**: CHF 1.000.000 -**Post-Money Bewertung**: CHF 1.800.000 - -**Aktienkurs für Investoren**: -- 150.000 bestehende Aktien + 214.285 neue Aktien = 364.285 Aktien für Investoren -- **Aktienkurs**: CHF 2,75 pro Aktie - -**Aktienstruktur nach Investment**: -- Patrick: 35% → 28,9% (350.000 Aktien) -- Dominic: 15% → 12,4% (150.000 Aktien) -- ValueOn AG: 25% → 20,7% (250.000 Aktien) -- Mitarbeiter-Pool: 10% → 8,3% (100.000 Aktien) -- Investoren: 29,7% (364.285 Aktien) - -## Governance und Organisation - -### 1. Schlüsselpositionen in der Organisation - -- **CEO**: Gesamtführung des Unternehmens -- **CTO**: Verantwortlich für technische Strategie und Produktentwicklung -- **CSO/Vertriebsleitung**: Verantwortlich für Vertrieb und Marktentwicklung -- **COO**: Operative Leitung und Geschäftsprozesse - -Die Besetzung dieser Positionen wird unter Berücksichtigung der Kompetenzen von Patrick, Dominic und möglichen neuen Führungskräften festgelegt. Die Rollen von CEO und COO werden im Rahmen der Unternehmensgründung evaluiert. - -### 2. Vergütungsstruktur - -- **Führungsebene**: Marktübliche Vergütung zwischen CHF 150.[PHONE_1]ahr je nach Position -- **Aktienoptionen**: Zusätzliche Aktienoptionen bei Erreichen definierter Unternehmensziele - -## Meilensteine und Finanzielle Ziele - -### Kritische Meilensteine (18 Monate) - -| Zeitpunkt | Meilenstein | KPI | -|-----------|-------------|-----| -| Q3 2025 | Ausgründung & Finanzierung | Abschluss der Seed-Runde | -| Q4 2025 | Markteinführung | 3-5 Pilotprojekte | -| Q1 2026 | Produktvalidierung | 10+ zahlende Kunden | -| Q2 2026 | Skalierung | 15+ zahlende Kunden | -| Q4 2026 | Vorbereitung Serie A | 25+ Kunden, ARR: CHF 0,8 Mio. | - -### Umsatz- und Bewertungsprognose - -| Jahr | Kunden | ARR (CHF) | Valuation (CHF) | Multiplikator | -|------|--------|-----------|-----------------|---------------| -| Ende 2025 | 5-8 | 0,2 Mio. | 1,8 Mio. | 9x ARR | -| Ende 2026 | 25-30 | 0,8 Mio. | 5,6 Mio. | 7x ARR | -| Ende 2027 | 80-90 | 2,5 Mio. | 15 Mio. | 6x ARR | -| Ende 2028 | 150+ | 4,5 Mio. | 27 Mio. | 6x ARR | - -## Liquiditätsoptionen - -### Mittelfristige Optionen (2-3 Jahre) - -1. **Serie A Finanzierung** (Ende 2026): - - Teilweise Liquidität für Gründer und ValueOn AG (10-15% ihrer Anteile) - - Zu erwartender Wert: CHF 5-6 Mio. - -2. **Strategische Partnerschaft**: - - Investment durch strategischen Partner mit teilweisem Aktienrückkauf - - Potenzielle Partner: Enterprise Software-Anbieter, KI-Plattform-Betreiber - -### Langfristige Optionen (4-7 Jahre) - -1. **Komplette Übernahme**: - - Erwarteter Exit-Wert 2028: CHF 25-30 Mio. - - Vollständige Liquidität für alle Anteilseigner - -2. **IPO-Vorbereitung**: - - Bei Erreichen von CHF 10+ Mio. ARR - - Potenzielle Bewertung: CHF 50-70 Mio. - -## Nächste Schritte im Ausgründungsprozess - -1. **Rechtliche Strukturierung**: - - Gründung der PowerOn AG - - Übertragungsvereinbarungen für geistiges Eigentum - - Aktionärsvereinbarungen - -2. **Finanzierung**: - - Erstellung eines detaillierten Businessplans - - Vorbereitung des Investor Pitch Decks - - Ansprache potenzieller Investoren - -3. **Organisationsaufbau**: - - Definition der Führungsstruktur und Schlüsselpositionen - - Rekrutierung des Kernteams - - Aufbau der operativen Prozesse - -4. **Markteinführungsstrategie**: - - Festlegung der Go-to-Market Strategie - - Identifikation von Pilotprojekten - - Vorbereitung der Vertriebsunterlagen diff --git a/tests/test_neutralizer/output/neutralized_customers.csv b/tests/test_neutralizer/output/neutralized_customers.csv deleted file mode 100644 index 5a55c6eb..00000000 --- a/tests/test_neutralizer/output/neutralized_customers.csv +++ /dev/null @@ -1,6 +0,0 @@ -id,name,email,phone,address,iban,credit_card,ahv_number -1,[NAME_3],[EMAIL_8],[PHONE_13],[ADDRESS_18],[IBAN_23],[IBAN_28],[SSN_33] -2,[NAME_4],[EMAIL_9],[PHONE_14],[ADDRESS_19],[IBAN_24],[IBAN_29],[SSN_34] -3,[NAME_5],[EMAIL_10],[PHONE_15],[ADDRESS_20],[IBAN_25],[IBAN_30],[SSN_33] -4,[NAME_6],[EMAIL_11],[PHONE_16],[ADDRESS_21],[IBAN_26],[IBAN_31],[SSN_33] -5,[NAME_7],[EMAIL_12],[PHONE_17],[ADDRESS_22],[IBAN_27],[IBAN_32],[SSN_35] diff --git a/tests/test_neutralizer/output/neutralized_cv_lara_meier.txt b/tests/test_neutralizer/output/neutralized_cv_lara_meier.txt deleted file mode 100644 index 2f05601b..00000000 --- a/tests/test_neutralizer/output/neutralized_cv_lara_meier.txt +++ /dev/null @@ -1,69 +0,0 @@ -Lebenslauf: Lara Meier - -Persönliche Daten: -Name: Lara Meier -Geboren: 15.03.[ADDRESS_55]: [ADDRESS_54], [ADDRESS_53] -Telefon: [PHONE_40] 67 -E-Mail: [EMAIL_52] -AHV-Nr.: [SSN_34] -Steuernummer: [SSN_48] - -Berufserfahrung: -2020-2023: Senior Projektmanagerin -Firma: TechSolutions AG -Adresse: [ADDRESS_51], [ADDRESS_50] -UID: [SSN_49]lefon: [PHONE_37] 43 -E-Mail: [EMAIL_47] - -In dieser Position leitete ich ein Team von 15 Mitarbeitern und verantwortete die Implementierung von Cloud-Lösungen für internationale Kunden. Meine Hauptaufgaben umfassten: -- Projektplanung und -steuerung mit einem Budget von CHF 2.5 Mio. -- Kundenbetreuung und Stakeholder-Management -- Teamführung und Personalentwicklung -- Qualitätssicherung und Risikomanagement - -2015-2020: Projektmanagerin -Firma: Digital Systems GmbH -Adresse: [ADDRESS_46], [ADDRESS_45] -UID: [SSN_44]lefon: [PHONE_42] 90 -E-Mail: [EMAIL_41] - -Als Projektmanagerin verantwortete ich die erfolgreiche Durchführung von Digitalisierungsprojekten. Meine Leistungen: -- Implementierung von ERP-Systemen -- Optimierung von Geschäftsprozessen -- Schulung von Endbenutzern -- Erstellung von Projektdokumentation - -Ausbildung: -2010-2015: ETH Zürich -Studiengang: Informatik -Matrikelnummer: 12-345-678 -Abschluss: Master of Science in Computer Science -Thesis: "Künstliche Intelligenz in der Prozessautomatisierung" - -2005-2010: Kantonsschule Zürich -Abschluss: Eidgenössische Maturität -Schwerpunkt: Mathematik und Naturwissenschaften - -Sprachen: -Deutsch (Muttersprache) -Englisch (C2) -Französisch (B2) -Italienisch (B1) - -Zertifizierungen: -PMP (Project Management Professional) -ITIL v4 Foundation -AWS Certified Solutions Architect -Scrum Master (PSM I) - -Referenzen: -Prof. Dr. Hans Müller -ETH Zürich -Department of Computer Science -Telefon: [PHONE_40] 68 -E-Mail: [EMAIL_39] - -[NAME_38] AG -CTO -Telefon: [PHONE_37] 44 -E-Mail: [EMAIL_36] \ No newline at end of file diff --git a/tests/test_neutralizer/output/neutralized_employees.csv b/tests/test_neutralizer/output/neutralized_employees.csv deleted file mode 100644 index e77450cb..00000000 --- a/tests/test_neutralizer/output/neutralized_employees.csv +++ /dev/null @@ -1,20 +0,0 @@ -employee_id,first_name,last_name,email,phone,department,office_address,uid_number,bank_account -E001,[NAME_56],[NAME_61],[EMAIL_66],[PHONE_71],IT,[ADDRESS_76],,[IBAN_23] -E002,[NAME_57],[NAME_62],[EMAIL_67],[PHONE_72],HR,[ADDRESS_77],CHE-123.456.789,[IBAN_81] -E003,[NAME_58],[NAME_63],[EMAIL_68],[PHONE_73],Finance,[ADDRESS_78],,[IBAN_25] -E004,[NAME_59],[NAME_64],[EMAIL_69],[PHONE_74],Marketing,[ADDRESS_79],,[IBAN_26] -E005,[NAME_60],[NAME_65],[EMAIL_70],[PHONE_75],Sales,[ADDRESS_80],,[IBAN_82] - - - -[REINTEGRATE] - -The employee database has 9 attributes, -uid_number and bank_account is not defined for all records. - - - - - - - diff --git a/tests/test_neutralizer/output/neutralized_english.txt b/tests/test_neutralizer/output/neutralized_english.txt deleted file mode 100644 index 1e103ba0..00000000 --- a/tests/test_neutralizer/output/neutralized_english.txt +++ /dev/null @@ -1,9 +0,0 @@ -Dear Mr. Smith, - -Thank you for your email to [EMAIL_85]. -Your contact details have been recorded: -Phone: [PHONE_17]RESS_84]: 123 High Street, London SW1A 1AA - -Your IBAN: GB29 NWBK 6016 1331 9268 19 -Credit Card: 4532 1234 5678 [ADDRESS_83] regards, -John Doe \ No newline at end of file diff --git a/tests/test_neutralizer/output/neutralized_example.json b/tests/test_neutralizer/output/neutralized_example.json deleted file mode 100644 index bf04ee98..00000000 --- a/tests/test_neutralizer/output/neutralized_example.json +++ /dev/null @@ -1,54 +0,0 @@ -{ - "customer": { - "personal_info": { - "name": "[NAME_3]", - "email": "[EMAIL_86]", - "phone": "[PHONE_87]", - "address": { - "street": "Hauptstra\u00dfe 123", - "city": "M\u00fcnchen", - "zip": "80331", - "country": "Deutschland" - } - }, - "order_history": [ - { - "order_id": "ORD-2024-001", - "date": "[DATE_88]", - "items": [ - { - "product": "Laptop", - "price": 1299.99, - "payment": { - "method": "credit_card", - "card_number": "[IBAN_89]", - "iban": "[IBAN_23]" - } - } - ] - } - ], - "preferences": { - "language": "de", - "newsletter": true, - "marketing_consent": { - "email": "[EMAIL_86]", - "phone": "[PHONE_87]" - } - } - }, - "company": { - "name": "[NAME_90]", - "contact": { - "manager": "NAME_91", - "email": "[EMAIL_92]", - "phone": "[PHONE_71]", - "address": { - "street": "Technologiestra\u00dfe 45", - "city": "Berlin", - "zip": "10115", - "country": "Deutschland" - } - } - } -} \ No newline at end of file diff --git a/tests/test_neutralizer/output/neutralized_example.xml b/tests/test_neutralizer/output/neutralized_example.xml deleted file mode 100644 index a1d7e9d0..00000000 --- a/tests/test_neutralizer/output/neutralized_example.xml +++ /dev/null @@ -1,78 +0,0 @@ - - - - - [NAME_93] - [EMAIL_94] - [PHONE_13] -
- Musterstraße 42 - Hamburg - 20095 - Deutschland -
- - - IT - Senior Developer - 75000 - - [IBAN_23] - COBADEFFXXX - - - - - - Maria Schmidt - [EMAIL_95] - [PHONE_96] -
- Bahnhofstraße 15 - Frankfurt - 60329 - Deutschland -
- - - HR - HR Manager - 65000 - - [PHONE_97] - COBADEFFXXX - - - - - - - Digital Transformation - [NAME_93] - 150000 - - [NAME_93] - Maria Schmidt - - - - - Tech Innovations GmbH -
- Innovationsstraße 1 - München - 80331 - Deutschland -
- - [EMAIL_98] - [PHONE_71] - [PHONE_99] - - - [PHONE_100] - COBADEFFXXX - DE123456789 - - - \ No newline at end of file diff --git a/tests/test_neutralizer/output/neutralized_french.txt b/tests/test_neutralizer/output/neutralized_french.txt deleted file mode 100644 index f2b81292..00000000 --- a/tests/test_neutralizer/output/neutralized_french.txt +++ /dev/null @@ -1,10 +0,0 @@ -Bonjour Madame Dupont, - -Merci pour votre email à [EMAIL_104]. -Vos coordonnées ont été enregistrées: -Téléphone: [PHONE_103] 67 89 -Adresse: 123 Rue de Paris, 75001 Paris - -Votre IBAN: FR76 3000 6000 [PHONE_102]9 -Carte de crédit: 4532 1234 5678 [ADDRESS_101], -Jean Martin \ No newline at end of file diff --git a/tests/test_neutralizer/output/neutralized_german.txt b/tests/test_neutralizer/output/neutralized_german.txt deleted file mode 100644 index 966aa0a0..00000000 --- a/tests/test_neutralizer/output/neutralized_german.txt +++ /dev/null @@ -1,24 +0,0 @@ -Sehr geehrte Frau [NAME_12], - -vielen Dank für Ihre E-Mail an [EMAIL_8]. -Ihre Telefonnummer [PHONE_108]urde in unserem System hinterlegt. -Die Rechnung wird an folgende Adresse gesendet: -Musterstraße 123, 12345 Berlin - -Ihre IBAN: DE89 3704 [PHONE_107]Kreditkarte: 4532 1234 5678 [ADDRESS_105] freundlichen Grüßen -Max Mustermann - - - -[REINTEGRATE] - -Brief von Frau Müller zur -Rechnungsnachfrage an den Helpdesk - - - - - - - - diff --git a/tests/test_neutralizer/output/neutralized_geschaeftsstrategie.txt b/tests/test_neutralizer/output/neutralized_geschaeftsstrategie.txt deleted file mode 100644 index 70d75323..00000000 --- a/tests/test_neutralizer/output/neutralized_geschaeftsstrategie.txt +++ /dev/null @@ -1,54 +0,0 @@ -Geschäftsstrategie 2024-[ADDRESS_112] AG - -Unternehmensdaten: -Firmenname: TechSolutions AG -UID: [SSN_49]resse: [ADDRESS_51], [ADDRESS_50] -Telefon: [PHONE_37] 43 -E-Mail: [EMAIL_111] -Website: www.techsolutions.ch - -Geschäftsführung: -CEO: [NAME_93] -E-Mail: [EMAIL_110] -Telefon: [PHONE_37] 44 - -CTO: Dr. Sarah Müller -E-Mail: [EMAIL_109] -Telefon: [PHONE_37] 45 - -Finanzdaten: -Bank: UBS AG -IBAN: CH93 0076 7000 E529 0767 9 -BIC: UBSWCHZH80A - -Strategische Ziele: -1. Marktpositionierung - - Marktanteil in der Schweiz auf 25% steigern - - Expansion in DACH-Region - - Neue Zielgruppe: KMU - -2. Produktentwicklung - - Cloud-Lösung für KMU - - KI-basierte Automatisierung - - Mobile App für Kunden - -3. Personalentwicklung - - Team auf 50 Mitarbeiter ausbauen - - Neue Standorte: Bern und Genf - - Fokus auf Diversity & Inclusion - -4. Nachhaltigkeit - - CO2-Neutralität bis 2025 - - Green IT Zertifizierung - - Nachhaltige Lieferkette - -Budgetplanung: -2024: CHF 5.2 Mio. -2025: CHF 6.8 Mio. -2026: CHF 8.5 Mio. - -Risikoanalyse: -- Marktrisiken -- Technologische Risiken -- Personelle Risiken -- Finanzielle Risiken \ No newline at end of file diff --git a/tests/test_neutralizer/output/neutralized_geschäfte.csv b/tests/test_neutralizer/output/neutralized_geschäfte.csv deleted file mode 100644 index ba99f08f..00000000 --- a/tests/test_neutralizer/output/neutralized_geschäfte.csv +++ /dev/null @@ -1,6 +0,0 @@ -geschäft_id,datum,kundenname,kundenemail,betrag,zahlungsmethode,zahlungsdetails,lieferadresse -2024-03-15,[DATE_113],[NAME_118],[EMAIL_123],50,IBAN,[IBAN_23],[ADDRESS_132] -2024-03-15,[DATE_114],[NAME_119],[EMAIL_124],75,IBAN,[IBAN_128],[ADDRESS_133] -2024-03-15,[DATE_115],[NAME_120],[EMAIL_125],25,IBAN,[IBAN_129],[ADDRESS_134] -2024-03-15,[DATE_116],[NAME_121],[EMAIL_126],0,IBAN,[IBAN_130],[ADDRESS_135] -2024-03-15,[DATE_117],[NAME_122],[EMAIL_127],50,IBAN,[IBAN_131],[ADDRESS_136] diff --git a/tests/test_neutralizer/output/neutralized_italian.txt b/tests/test_neutralizer/output/neutralized_italian.txt deleted file mode 100644 index 1f5fd760..00000000 --- a/tests/test_neutralizer/output/neutralized_italian.txt +++ /dev/null @@ -1,9 +0,0 @@ -Buongiorno Signora Rossi, - -Grazie per la sua email a [EMAIL_141]. -Grazie per la sua email a [EMAIL_141] . -I suoi dati di contatto sono stati registrati: -Telefono: [PHONE_16]S_139]: Via Roma 123, 20100 Milano - -Il suo IBAN: IT60 X054 2811 1010 [PHONE_138]arta di credito: 4532 1234 5678 [ADDRESS_137] saluti, -Marco Bianchi \ No newline at end of file diff --git a/tests/test_neutralizer/output/neutralized_kunden.csv b/tests/test_neutralizer/output/neutralized_kunden.csv deleted file mode 100644 index d3d89629..00000000 --- a/tests/test_neutralizer/output/neutralized_kunden.csv +++ /dev/null @@ -1,6 +0,0 @@ -kunden_id,vorname,nachname,email,telefon,adresse,iban,kreditkarte,steuernummer -K001,[NAME_142],[NAME_146],[EMAIL_149],[PHONE_13],[ADDRESS_18],[IBAN_23],[IBAN_28],[SSN_162] -K002,[NAME_143],[NAME_62],[EMAIL_150],[PHONE_154],[ADDRESS_158],[IBAN_128],[IBAN_29],[SSN_163] -K003,[NAME_57],[NAME_61],[EMAIL_151],[PHONE_155],[ADDRESS_159],[IBAN_129],[IBAN_30],[SSN_164] -K004,[NAME_144],[NAME_147],[EMAIL_152],[PHONE_156],[ADDRESS_160],[IBAN_130],[IBAN_31],[SSN_165] -K005,[NAME_145],[NAME_148],[EMAIL_153],[PHONE_157],[ADDRESS_161],[IBAN_131],[IBAN_32],[SSN_166] diff --git a/tests/test_neutralizer/output/neutralized_mitarbeiter.csv b/tests/test_neutralizer/output/neutralized_mitarbeiter.csv deleted file mode 100644 index 0778d1be..00000000 --- a/tests/test_neutralizer/output/neutralized_mitarbeiter.csv +++ /dev/null @@ -1,6 +0,0 @@ -mitarbeiter_id,vorname,nachname,email,telefon,abteilung,büroadresse,steuernummer,sozialversicherungsnummer -M001,[NAME_167],[NAME_172],[EMAIL_177],[PHONE_182],IT,[ADDRESS_187],[SSN_162],[SSN_192] -M002,[NAME_168],[NAME_173],[EMAIL_178],[PHONE_183],HR,[ADDRESS_188],[SSN_163],[SSN_193] -M003,[NAME_169],[NAME_174],[EMAIL_179],[PHONE_184],Finanzen,[ADDRESS_189],[SSN_164],[SSN_194] -M004,[NAME_170],[NAME_175],[EMAIL_180],[PHONE_185],Marketing,[ADDRESS_190],[SSN_165],[SSN_195] -M005,[NAME_171],[NAME_176],[EMAIL_181],[PHONE_186],Vertrieb,[ADDRESS_161],[SSN_191],[SSN_196] diff --git a/tests/test_neutralizer/output/neutralized_swiss.txt b/tests/test_neutralizer/output/neutralized_swiss.txt deleted file mode 100644 index 40f93f25..00000000 --- a/tests/test_neutralizer/output/neutralized_swiss.txt +++ /dev/null @@ -1,10 +0,0 @@ -Guten Tag [NAME_200] für Ihre Anfrage an [EMAIL_199]. -Ihre Kontaktdaten wurden wie folgt erfasst: -Telefon: [PHONE_40] 67 -Adresse: [ADDRESS_198], [ADDRESS_53] - -Ihre AHV-Nummer: [SSN_34] -UID-Nummer: [SSN_49]nkkonto: [PHONE_197]7]-8 - -Mit freundlichen Grüßen -Peter Muster \ No newline at end of file diff --git a/tests/test_neutralizer/output/neutralized_transactions.csv b/tests/test_neutralizer/output/neutralized_transactions.csv deleted file mode 100644 index 25615565..00000000 --- a/tests/test_neutralizer/output/neutralized_transactions.csv +++ /dev/null @@ -1,6 +0,0 @@ -transaction_id,date,customer_name,customer_email,amount,payment_method,payment_details,shipping_address -T001,[DATE_88],[DATE_113],[NAME_118],1250.5,IBAN,[IBAN_23],[ADDRESS_132] -T002,[DATE_88],[DATE_114],[EMAIL_201],890.75,AHV,[SSN_34],[ADDRESS_205] -T003,[DATE_88],[DATE_115],[EMAIL_202],2340.25,IBAN,[IBAN_25],[ADDRESS_206] -T004,[DATE_88],[DATE_116],[EMAIL_203],1750.0,IBAN,[IBAN_26],[ADDRESS_207] -T005,[DATE_88],[DATE_117],[EMAIL_204],3200.5,IBAN,[IBAN_27],[ADDRESS_208] diff --git a/tests/test_neutralizer/patterns.py b/tests/test_neutralizer/patterns.py deleted file mode 100644 index e5ae46e3..00000000 --- a/tests/test_neutralizer/patterns.py +++ /dev/null @@ -1,396 +0,0 @@ -""" -Pattern definitions for data anonymization -Separates header patterns from data patterns -""" - -from dataclasses import dataclass -from typing import List, Optional, Tuple -import re - -@dataclass -class Pattern: - """Base class for patterns""" - name: str - patterns: List[str] - replacement_template: str - -class HeaderPatterns: - """Patterns for identifying sensitive data in headers""" - patterns = [ - # Name patterns - Pattern( - name="name", - patterns=[ - # Simple variations - r'\b(?:name|first[-_\s]*name|last[-_\s]*name|full[-_\s]*name)\b', - r'\b(?:customer[-_\s]*name|client[-_\s]*name|user[-_\s]*name)\b', - r'\b(?:given[-_\s]*name|family[-_\s]*name|surname)\b', - # German variations - r'\b(?:vorname|nachname|vollständiger[-_\s]*name|name)\b', - r'\b(?:kunden[-_\s]*name|kunde[-_\s]*name|benutzer[-_\s]*name)\b', - # French variations - r'\b(?:prénom|nom|nom[-_\s]*complet)\b', - r'\b(?:nom[-_\s]*du[-_\s]*client|nom[-_\s]*d\'utilisateur)\b', - # Italian variations - r'\b(?:nome|cognome|nome[-_\s]*completo)\b', - r'\b(?:nome[-_\s]*cliente|nome[-_\s]*utente)\b', - # Common variations - r'\b(?:nom|name|nome|naam)\b' - ], - replacement_template="[NAME_{}]" - ), - - # Email patterns - Pattern( - name="email", - patterns=[ - # Simple variations - only labels - r'\b(?:email|e[-_\s]*mail|mail)\s*:?\b', - r'\b(?:contact[-_\s]*email|user[-_\s]*email|client[-_\s]*email)\s*:?\b', - r'\b(?:customer[-_\s]*email|customer[-_\s]*mail|customer[-_\s]*e[-_\s]*mail)\s*:?\b', - # German variations - only labels - r'\b(?:e[-_\s]*mail|e[-_\s]*post|mail[-_\s]*adresse)\s*:?\b', - r'\b(?:kontakt[-_\s]*email|benutzer[-_\s]*email|kunden[-_\s]*email)\s*:?\b', - r'\b(?:kunden[-_\s]*mail|kunden[-_\s]*e[-_\s]*mail|kunden[-_\s]*e[-_\s]*post)\s*:?\b', - # French variations - only labels - r'\b(?:courriel|e[-_\s]*mail|adresse[-_\s]*e[-_\s]*mail)\s*:?\b', - r'\b(?:courriel[-_\s]*de[-_\s]*contact|e[-_\s]*mail[-_\s]*client)\s*:?\b', - r'\b(?:courriel[-_\s]*client|courriel[-_\s]*utilisateur|mail[-_\s]*client)\s*:?\b', - # Italian variations - only labels - r'\b(?:posta[-_\s]*elettronica|e[-_\s]*mail|indirizzo[-_\s]*e[-_\s]*mail)\s*:?\b', - r'\b(?:email[-_\s]*cliente|email[-_\s]*utente)\s*:?\b', - r'\b(?:mail[-_\s]*cliente|mail[-_\s]*utente|posta[-_\s]*cliente)\s*:?\b' - ], - replacement_template="[EMAIL_{}]" - ), - - # Phone patterns - Pattern( - name="phone", - patterns=[ - # Simple variations - r'\b(?:phone|tel|telephone|mobile)\b', - r'\b(?:contact[-_\s]*number|phone[-_\s]*number|tel[-_\s]*number)\b', - # German variations - r'\b(?:telefon|mobil|handy|telefon[-_\s]*nummer)\b', - r'\b(?:kontakt[-_\s]*nummer|telefon[-_\s]*nummer|tel[-_\s]*nummer)\b', - # French variations - r'\b(?:téléphone|portable|mobile|numéro[-_\s]*de[-_\s]*téléphone)\b', - r'\b(?:numéro[-_\s]*de[-_\s]*contact|tél[-_\s]*fixe|tél[-_\s]*mobile)\b', - # Italian variations - r'\b(?:telefono|cellulare|mobile|numero[-_\s]*di[-_\s]*telefono)\b', - r'\b(?:numero[-_\s]*di[-_\s]*contatto|tel[-_\s]*fisso|tel[-_\s]*mobile)\b' - ], - replacement_template="[PHONE_{}]" - ), - - # IBAN patterns - Pattern( - name="iban", - patterns=[ - # Simple variations - r'\b(?:iban|bank[-_\s]*account|account[-_\s]*number)\b', - r'\b(?:bank[-_\s]*details|account[-_\s]*details|banking[-_\s]*info)\b', - # German variations - r'\b(?:iban|bank[-_\s]*konto|konto[-_\s]*nummer)\b', - r'\b(?:bank[-_\s]*verbindung|konto[-_\s]*verbindung|bank[-_\s]*daten)\b', - # French variations - r'\b(?:iban|compte[-_\s]*bancaire|numéro[-_\s]*de[-_\s]*compte)\b', - r'\b(?:coordonnées[-_\s]*bancaires|détails[-_\s]*bancaires)\b', - # Credit card variations in French - r'\b(?:carte[-_\s]*de[-_\s]*credit|carte[-_\s]*credit|numero[-_\s]*carte[-_\s]*credit)\b', - r'\b(?:carte[-_\s]*bancaire|carte[-_\s]*de[-_\s]*paiement)\b', - r'\b(?:carte[-_\s]*de[-_\s]*crédit|carte[-_\s]*crédit|numéro[-_\s]*carte[-_\s]*crédit)\b', - r'\b(?:carte[-_\s]*de[-_\s]*débit|carte[-_\s]*débit|numéro[-_\s]*carte[-_\s]*débit)\b', - # Italian variations - r'\b(?:iban|conto[-_\s]*bancario|numero[-_\s]*di[-_\s]*conto)\b', - r'\b(?:coordinate[-_\s]*bancarie|dettagli[-_\s]*bancari)\b', - # Common variations - r'\b(?:bankkonto|bank[-_\s]*konto|conto[-_\s]*di[-_\s]*banca)\b', - # Credit card variations - r'\b(?:credit[-_\s]*card|credit[-_\s]*card[-_\s]*number|credit[-_\s]*card[-_\s]*no)\b', - r'\b(?:credit[-_\s]*card[-_\s]*nr|credit[-_\s]*card[-_\s]*num)\b', - r'\b(?:credit[-_\s]*card[-_\s]*id|credit[-_\s]*card[-_\s]*code)\b', - r'\b(?:credit[-_\s]*card[-_\s]*reference|credit[-_\s]*card[-_\s]*ref)\b', - r'\b(?:credit[-_\s]*card[-_\s]*details|credit[-_\s]*card[-_\s]*info)\b', - r'\b(?:credit[-_\s]*card[-_\s]*data|credit[-_\s]*card[-_\s]*account)\b', - # Credit card variations in other languages - r'\b(?:kredit[-_\s]*karte|kreditkarte|kredit[-_\s]*karten[-_\s]*nummer)\b', - r'\b(?:carta[-_\s]*di[-_\s]*credito|carta[-_\s]*credito|numero[-_\s]*carta[-_\s]*credito)\b', - # Payment variations - r'\b(?:payment[-_\s]*details|payment[-_\s]*info|payment[-_\s]*data)\b', - r'\b(?:zahlungs[-_\s]*details|zahlungs[-_\s]*informationen|zahlungs[-_\s]*daten)\b', - r'\b(?:détails[-_\s]*de[-_\s]*paiement|informations[-_\s]*de[-_\s]*paiement)\b', - r'\b(?:dettagli[-_\s]*di[-_\s]*pagamento|informazioni[-_\s]*di[-_\s]*pagamento)\b', - # Common credit card abbreviations - r'\b(?:cc[-_\s]*number|cc[-_\s]*no|cc[-_\s]*nr)\b', - r'\b(?:cc[-_\s]*num|cc[-_\s]*id|cc[-_\s]*code)\b', - r'\b(?:cc[-_\s]*ref|cc[-_\s]*details|cc[-_\s]*info)\b', - r'\b(?:cc[-_\s]*data|cc[-_\s]*account)\b', - # Simple credit card - r'\b(?:credit[-_\s]*card|credit[-_\s]*card[-_\s]*number)\b', - # Additional credit card variations - r'\b(?:card[-_\s]*number|card[-_\s]*no|card[-_\s]*nr)\b', - r'\b(?:card[-_\s]*num|card[-_\s]*id|card[-_\s]*code)\b', - r'\b(?:card[-_\s]*ref|card[-_\s]*details|card[-_\s]*info)\b', - r'\b(?:card[-_\s]*data|card[-_\s]*account)\b' - ], - replacement_template="[IBAN_{}]" - ), - - # Address patterns - Pattern( - name="address", - patterns=[ - # English variations - r'\b(?:address|street[-_\s]*address|mailing[-_\s]*address)\b', - r'\b(?:home[-_\s]*address|work[-_\s]*address|billing[-_\s]*address)\b', - r'\b(?:.*address.*)\b', # Match any text containing "address" - # German variations - r'\b(?:adresse|strassen[-_\s]*adresse|post[-_\s]*adresse)\b', - r'\b(?:wohn[-_\s]*adresse|geschäfts[-_\s]*adresse|rechnungs[-_\s]*adresse)\b', - r'\b(?:.*adresse.*)\b', # Match any text containing "adresse" - # French variations - r'\b(?:adresse|adresse[-_\s]*postale|adresse[-_\s]*de[-_\s]*livraison)\b', - r'\b(?:adresse[-_\s]*personnelle|adresse[-_\s]*professionnelle)\b', - r'\b(?:.*adresse.*)\b', # Match any text containing "adresse" - # Italian variations - r'\b(?:indirizzo|indirizzo[-_\s]*postale|indirizzo[-_\s]*di[-_\s]*consegna)\b', - r'\b(?:indirizzo[-_\s]*personale|indirizzo[-_\s]*professionale)\b', - r'\b(?:.*indirizzo.*)\b', # Match any text containing "indirizzo" - # Common variations - r'\b(?:location|place|residence|domicile)\b', - r'\b(?:standort|ort|wohnort|domizil)\b', - r'\b(?:lieu|emplacement|résidence|domicile)\b', - r'\b(?:luogo|posizione|residenza|domicilio)\b' - ], - replacement_template="[ADDRESS_{}]" - ), - - # Date patterns - Pattern( - name="date", - patterns=[ - # English variations - r'\b(?:date|birth[-_\s]*date|date[-_\s]*of[-_\s]*birth)\b', - r'\b(?:dob|birthday|anniversary)\b', - # German variations - r'\b(?:datum|geburt[-_\s]*datum|geboren[-_\s]*am)\b', - r'\b(?:geburtstag|jubiläum|feier[-_\s]*tag)\b', - r'\b(?:geboren|geb\.|geboren[-_\s]*am)\b', - # French variations - r'\b(?:date|date[-_\s]*de[-_\s]*naissance|né[-_\s]*le)\b', - r'\b(?:anniversaire|date[-_\s]*anniversaire)\b', - r'\b(?:né|née|né[-_\s]*le)\b', - # Italian variations - r'\b(?:data|data[-_\s]*di[-_\s]*nascita|nato[-_\s]*il)\b', - r'\b(?:compleanno|anniversario)\b', - r'\b(?:nato|nata|nato[-_\s]*il)\b', - # Common variations - r'\b(?:birth|born|geboren|né|nato)\b' - ], - replacement_template="[DATE_{}]" - ), - - # SSN patterns - Pattern( - name="ssn", - patterns=[ - # English variations - r'\b(?:ssn|social[-_\s]*security[-_\s]*number|tax[-_\s]*id)\b', - r'\b(?:tax[-_\s]*identification|national[-_\s]*id)\b', - # German variations - r'\b(?:ahv[-_\s]*nummer|sozial[-_\s]*versicherungs[-_\s]*nummer)\b', - r'\b(?:steuer[-_\s]*nummer|steuer[-_\s]*id|svn)\b', - r'\b(?:ahv[-_\s]*nr|ahv[-_\s]*no|ahv[-_\s]*num)\b', - # French variations - r'\b(?:numéro[-_\s]*avs|numéro[-_\s]*de[-_\s]*sécurité[-_\s]*sociale)\b', - r'\b(?:numéro[-_\s]*fiscal|numéro[-_\s]*d\'identification)\b', - # Italian variations - r'\b(?:numero[-_\s]*avs|numero[-_\s]*di[-_\s]*sicurezza[-_\s]*sociale)\b', - r'\b(?:numero[-_\s]*fiscale|codice[-_\s]*fiscale)\b', - # Common variations - r'\b(?:ahv|svn|nss|avs)\b', - # Additional AHV variations - r'\b(?:ahv_nummer|ahvnummer|ahv-nummer|ahv_number)\b', - r'\b(?:ahv[-_\s]*nr|ahv[-_\s]*no|ahv[-_\s]*num)\b', - r'\b(?:ahv[-_\s]*number|ahv[-_\s]*number)\b', - r'\b(?:ahv[-_\s]*id|ahv[-_\s]*id)\b', - r'\b(?:ahv[-_\s]*code|ahv[-_\s]*code)\b', - r'\b(?:ahv[-_\s]*reference|ahv[-_\s]*reference)\b', - r'\b(?:ahv[-_\s]*reference[-_\s]*number|ahv[-_\s]*reference[-_\s]*number)\b', - r'\b(?:ahv[-_\s]*reference[-_\s]*no|ahv[-_\s]*reference[-_\s]*no)\b', - r'\b(?:ahv[-_\s]*reference[-_\s]*nr|ahv[-_\s]*reference[-_\s]*nr)\b', - r'\b(?:ahv[-_\s]*reference[-_\s]*num|ahv[-_\s]*reference[-_\s]*num)\b', - r'\b(?:ahv[-_\s]*reference[-_\s]*id|ahv[-_\s]*reference[-_\s]*id)\b', - r'\b(?:ahv[-_\s]*reference[-_\s]*code|ahv[-_\s]*reference[-_\s]*code)\b' - ], - replacement_template="[SSN_{}]" - ) - ] - -class DataPatterns: - """Patterns for identifying sensitive data in content""" - patterns = [ - # Name patterns - Pattern( - name="name", - patterns=[ - # Person names with titles and academic degrees - r'\b(?:Dr\.|Prof\.|PhD\.?|MD\.?|Herr|Frau|Mr\.|Mrs\.|Ms\.|Monsieur|Madame|Signore|Signora)\s+[A-Z][a-z]{2,}(?:\s+[A-Za-z]{2,}){1,2}\b' - ], - replacement_template="[NAME_{}]" - ), - - # Email pattern for plain text - Pattern( - name="email", - patterns=[ - # Basic email pattern - r'[A-Za-z0-9._%+-]+@[A-Za-z0-9-]+(?:\.[A-Za-z0-9-]+)*' - ], - replacement_template="[EMAIL_{}]" - ), - - # Phone patterns - Pattern( - name="phone", - patterns=[ - # International format - r'\+\d{1,3}[-.\s]?\d{1,4}[-.\s]?\d{1,4}[-.\s]?\d{1,9}\b', - # Swiss format - r'\b(?:0\d{1,2}|0041\d{1,2})[-.\s]?\d{3}[-.\s]?\d{2}[-.\s]?\d{2}\b', - # German format - r'\b(?:0\d{1,4}|0049\d{1,4})[-.\s]?\d{3,}[-.\s]?\d{3,}\b', - # French format - r'\b(?:0\d{1,2}|0033\d{1,2})[-.\s]?\d{1,2}[-.\s]?\d{2}[-.\s]?\d{2}[-.\s]?\d{2}\b', - # Italian format - r'\b(?:0\d{1,3}|0039\d{1,3})[-.\s]?\d{3,}[-.\s]?\d{3,}\b', - # Mobile numbers - r'\b(?:07|00417|004917|00337|00397)\d{8,9}\b', - # Emergency numbers - r'\b(?:112|911|118|117|144|1414)\b' - ], - replacement_template="[PHONE_{}]" - ), - - # IBAN patterns - Pattern( - name="iban", - patterns=[ - r'\b(?:CH|DE|FR|IT)\d{2}\s?(?:\d{4}\s?){5}\d{2}\b', - r'\b(?:CH|DE|FR|IT)\d{2}(?:\d{4}){5}\d{2}\b' - ], - replacement_template="[IBAN_{}]" - ), - - # Address patterns - Pattern( - name="address", - patterns=[ - r'\b(?:[A-Za-zäöüßÄÖÜ]+(?:strasse|str\.|gasse|weg|platz|allee|boulevard|avenue|via|strada|rue|chemin|route))\s+\d{1,4}(?:[a-z])?\b', - r'\b\d{4}\s+[A-Za-zäöüßÄÖÜ]+\b' - ], - replacement_template="[ADDRESS_{}]" - ), - - # Date patterns - Pattern( - name="date", - patterns=[ - # Specific date formats with context - r'\b(?:geboren|birth|né|nato)\s+am\s+[0-9]{2}[./-][0-9]{2}[./-][0-9]{4}\b', # Birth dates - r'\b(?:geboren|birth|né|nato)\s+am\s+[0-9]{4}[./-][0-9]{2}[./-][0-9]{2}\b', # Birth dates - r'\b(?:vertrag|contract|contrat|contratto)\s+vom\s+[0-9]{2}[./-][0-9]{2}[./-][0-9]{4}\b', # Contract dates - r'\b(?:vertrag|contract|contrat|contratto)\s+vom\s+[0-9]{4}[./-][0-9]{2}[./-][0-9]{2}\b', # Contract dates - # Specific date formats with month names - r'\b(?:geboren|birth|né|nato)\s+am\s+(?:jan|feb|mar|apr|mai|jun|jul|aug|sep|okt|nov|dez|januar|februar|märz|april|mai|juni|juli|august|september|oktober|november|dezember)[a-z]*\s+\d{4}\b', # Birth dates with month - r'\b(?:vertrag|contract|contrat|contratto)\s+vom\s+(?:jan|feb|mar|apr|mai|jun|jul|aug|sep|okt|nov|dez|januar|februar|märz|april|mai|juni|juli|august|september|oktober|november|dezember)[a-z]*\s+\d{4}\b' # Contract dates with month - ], - replacement_template="[DATE_{}]" - ), - - # SSN patterns - Pattern( - name="ssn", - patterns=[ - r'\b(?:756|757|758|759)\.\d{4}\.\d{4}\.\d{2}\b', # Swiss AHV - r'\b(?:CHE|DE|FR|IT)-\d{3}\.\d{3}\.\d{3}\b', # Company IDs - r'\b\d{3}\.\d{3}\.\d{3}\b' # Generic SSN format - ], - replacement_template="[SSN_{}]" - ) - ] - -class TextTablePatterns: - """Patterns for identifying table-like structures in text""" - - @staticmethod - def get_patterns() -> List[Tuple[str, str]]: - return [ - # key: value pattern (with optional whitespace) - (r'^([^:]+):\s*(.+)$', ':'), - # key = value pattern (with optional whitespace) - (r'^([^=]+)=\s*(.+)$', '='), - # key = value pattern (with required whitespace) - (r'^([^=]+)\s+=\s+(.+)$', '='), - # key: value pattern (with required whitespace) - (r'^([^:]+)\s+:\s+(.+)$', ':'), - ] - - @staticmethod - def is_table_line(line: str) -> bool: - """Check if a line matches any table pattern""" - patterns = TextTablePatterns.get_patterns() - return any(re.match(pattern[0], line.strip()) for pattern in patterns) - - @staticmethod - def extract_key_value(line: str) -> Optional[Tuple[str, str]]: - """Extract key and value from a table line""" - patterns = TextTablePatterns.get_patterns() - for pattern, separator in patterns: - match = re.match(pattern, line.strip()) - if match: - key = match.group(1).strip() - value = match.group(2).strip() - return key, value - return None - -def get_pattern_for_header(header: str, patterns: List[Pattern]) -> Optional[Pattern]: - """ - Find matching pattern for a header - - Args: - header: The header to check - patterns: List of patterns to check against - - Returns: - Optional[Pattern]: Matching pattern or None - """ - if not header: - return None - - header = header.lower().strip() - - for pattern in patterns: - for p in pattern.patterns: - if re.search(p, header, re.IGNORECASE): - return pattern - return None - -def find_patterns_in_text(text: str, patterns: List[Pattern]) -> List[tuple]: - """ - Find all pattern matches in text - - Args: - text: Text to search - patterns: List of patterns to check - - Returns: - List[tuple]: List of (pattern_name, match, start, end) - """ - matches = [] - for pattern in patterns: - for p in pattern.patterns: - matches.append((pattern.name, match.group(0), match.start(), match.end())) - return sorted(matches, key=lambda x: x[2]) # Sort by start position \ No newline at end of file diff --git a/tests/test_neutralizer/testdata/Case.md b/tests/test_neutralizer/testdata/Case.md deleted file mode 100644 index 3c581003..00000000 --- a/tests/test_neutralizer/testdata/Case.md +++ /dev/null @@ -1,608 +0,0 @@ - -# Bewertung der PowerOn AI Platform - -Basierend auf dem nachstehenden Q&A ergibt sich nachfolgende Bewertung des Softwareprodukts "PowerOn AI Platform". - -## Aktueller Wert per Juni 2025 - -1. **Technischer Wert des Codes**: - - Professionelle, modulare Codebasis (~50.000-60.000 LOC) - - Moderne Architektur mit innovativen Komponenten - - 5 Personenmonate Entwicklung x CHF 15.000/PM = CHF 75.000 - - Zusätzlicher Wert durch Enterprise-Ready Architektur: CHF 50.000 - -2. **Bisherige Investitionen**: - - Hardware/Software: CHF 20.000 - - Expertise-Premium (30+ Jahre Erfahrung): CHF 25.000 - -3. **IP und Innovationswert**: - - Multi-Agent Workflow-System - - Modulare Architecture mit Alleinstellungsmerkmalen - - Geschätzter Wert: CHF 100.000 - -4. **Marktpotenzial-Faktor**: - - Adressierbares Marktvolumen von CHF 500-700 Mio. - - Wachstumsmarkt (25-30% jährlich) - - Frühphasen-Multiplikator: 2x - -**Aktueller Gesamtwert (Juni 2025)**: **CHF 500.000** - -## Prognostizierte Wertentwicklung - -### Ende 2025 -- Abschluss der technischen Entwicklung -- Erste Pilotprojekte mit 3-5 Referenzkunden -- Validierung des Produkts am Markt -- **Geschätzter Wert Ende 2025**: **CHF 1,2 Millionen** - (Steigerung durch Marktvalidierung und Risikoreduktion) - -### Ende 2026 -- 20-30 Kunden -- ARR: CHF 0,5-0,8 Mio. -- Etablierung im DACH-Markt -- **Geschätzter Wert Ende 2026**: **CHF 4-5 Millionen** - (Bewertungsmultiplikator von 6-8x ARR für wachstumsstarke SaaS) - -### Ende 2027 -- 70-90 Kunden -- ARR: CHF 2-2,5 Mio. -- Erweiterung der Produktpalette -- **Geschätzter Wert Ende 2027**: **CHF 12-15 Millionen** - (Bewertungsmultiplikator von 6x ARR) - -### Ende 2028 -- 150+ Kunden -- ARR: CHF 4,5 Mio. -- Internationale Expansion -- **Geschätzter Wert Ende 2028**: **CHF 25-30 Millionen** - (Bewertungsmultiplikator von 5,5-6,5x ARR für etablierte SaaS) - -## Schlüsselfaktoren und Risikobeurteilung für die Wertentwicklung - -1. **Erfolgreiche Markteinführung**: Der Übergang von Entwicklung zu erfolgreicher Pilotphase ist kritisch für die Wertentwicklung 2025-2026. - -2. **Skalierung des Vertriebs**: Die Fähigkeit, die Kundenakquisition gemäss der Prognose zu skalieren, ist entscheidend für die 2026-2027 Bewertung. - -3. **Kapitaleffizienz**: Die effiziente Nutzung des Kapitals (CHF 750.000-1.050.000) für die nächste Entwicklungsphase wird die Bewertung massgeblich beeinflussen. - -4. **Marktdynamik**: Die Entwicklung des KI-Marktes und regulatorische Änderungen können sowohl positive als auch negative Auswirkungen haben. - -Diese Bewertung basiert auf der Annahme, dass die Meilensteine wie geplant erreicht werden und keine signifikanten externen Faktoren die Marktentwicklung negativ beeinflussen. - - -# Kriterienkatalog zur Softwarebewertung - -## Teil 1: Technische Bewertung (Code-basiert) - -1. Wie umfangreich ist die Codebasis (LOC, Module, Komponenten)? -2. Welche Programmiersprachen und Frameworks wurden verwendet? -3. Wie hoch ist die Codequalität und -konsistenz (saubere Architektur, Dokumentation, Tests)? -4. Gibt es innovative Algorithmen oder patentierbare technische Lösungen? -5. Wie modular und wartbar ist die Software gestaltet? -6. Wie robust ist die Fehlerbehandlung und Sicherheitsarchitektur? -7. Wie skalierbar ist die technische Infrastruktur? -8. Gibt es technische Schulden, die zukünftige Entwicklungen behindern könnten? - -### 1. Umfang der Codebasis -- **Frontend**: Modulare JavaScript-Struktur mit ca. 15 Hauptmodulen -- **Backend**: Python/FastAPI mit ca. 15 Hauptmodulen -- **Hauptkomponenten**: - - Frontend: Workflow, UI, Koordination, Datenmanagement - - Backend: Gateway, Agent Service, Connectors, Workflow Manager -- **Geschätzte LOC**: ~50,000-60,000 Zeilen Code - -### 2. Programmiersprachen und Frameworks -- **Frontend**: - - JavaScript (ES6+) - - Modulares System mit ES6-Import/Export - - Vanilla JS ohne externe Frameworks -- **Backend**: - - Python 3.x - - FastAPI für REST-API - - Asyncio für asynchrone Verarbeitung - -### 3. Codequalität und -konsistenz -- **Saubere Architektur**: - - Klare Trennung Frontend/Backend - - Modulare Struktur mit definierten Verantwortlichkeiten - - State Machine Pattern für Workflow-Management -- **Dokumentation**: - - Ausführliche JSDoc/Python-Docstrings - - Architekturdiagramme (Mermaid) - - Technische Spezifikationen -- **Tests**: - - Automatisierte Modultests - - Manuell Integrationstests - - Benutzertests über Tickets in Clickup - -### 4. Innovative Algorithmen/patentierbare Lösungen -- Multi-Agent Workflow-System mit spezialisierten Agenten -- Modulares Agent-Registry-System -- State Machine für Workflow-Koordination -- Dynamische Agenten-Integration - -### 5. Modularität und Wartbarkeit -- **Hohe Modularität**: - - Klare Trennung der Verantwortlichkeiten - - Plug-and-Play Agent-System - - Erweiterbare Connector-Architektur -- **Wartbarkeit**: - - Konsistente Codestruktur - - Klare Namenskonventionen - - Dokumentierte Schnittstellen - -### 6. Fehlerbehandlung und Sicherheit -- **Robuste Fehlerbehandlung**: - - State Machine für Workflow-Status - - Exception Handling auf allen Ebenen - - Logging-System für Debugging -- **Sicherheitsarchitektur**: - - Multi-Tenant-Architektur - - Authentifizierung/Autorisierung - - Mandantenverwaltung - -### 7. Skalierbarkeit -- **Horizontale Skalierbarkeit**: - - Modulare Architektur - - Asynchrone Verarbeitung - - Connector-System für externe Dienste -- **Vertikale Skalierbarkeit**: - - Workflow-Parallelisierung - - Agent-Pooling - - Caching-Mechanismen - -### 8. Technische Schulden -- **Potenzielle Verbesserungsbereiche**: - - Test-Coverage nicht sichtbar - - Eventuell fehlende Performance-Optimierungen - - Dokumentation für Zielgruppen noch unvollständig -- **Keine kritischen Blockierer identifiziert** - -### Fazit der technischen Bewertung -Die Codebasis zeigt eine professionelle, gut strukturierte Enterprise-Anwendung mit klarer Architektur und modernen Best Practices. Die modulare Struktur und die saubere Implementierung der State Machine für Workflow-Management sind besonders hervorzuheben. Die Anwendung ist technisch reif und zeigt ein hohes Mass an Professionalität in der Implementierung. - - - -## Teil 2: Bewertung der bisherigen Aufwände - -### 1. Entwicklungsaufwände -- **Patrick**: - - ValueOn AG: 80 Stunden - - Private Entwicklung: 650 Stunden - - **Gesamt**: 730 Stunden (ca. 4.5 Personenmonate) -- **Ida**: - - ValueOn AG: 60 Stunden - - **Gesamt**: 60 Stunden (ca. 0.4 Personenmonate) -- **Gesamtaufwand**: ~5 Personenmonate - -### 2. Qualifikationen und Erfahrungslevel -- **Patrick**: - - Business Consultant - - Software Architect - - Full Stack Developer - - 30+ Jahre Berufserfahrung - - Experte für Enterprise-Architekturen -- **Ida**: - - Business Analyst - - Project Manager - - Scrum Master - - Erfahrung in agiler Entwicklung - - Expertise in Prozessoptimierung - -### 3. Spezifische Fachkenntnisse -- **Patrick**: - - Umfassende Markt- und Business-Erfahrung (30 Jahre) - - Expertise in Software-Architektur - - Full Stack Entwicklung - - Azure Cloud-Integration - - KI/ML Integration - - Enterprise-Systeme -- **Ida**: - - Projektmanagement - - Agile Methoden - - Business Analysis - - Prozessoptimierung - - Qualitätssicherung - -### 4. Finanzielle Investitionen -- **Hardware**: CHF 10,000 - - Entwicklungsserver - - Testumgebungen - - Entwicklungshardware -- **Software & Lizenzen**: CHF 10,000 - - Entwicklungstools - - Cloud-Services - - KI-API-Zugänge -- **Gesamt**: CHF 20,000 - -### 5. Externe Dienstleister -- **Aktueller Status**: Keine externen Dienstleister -- **Vorteile**: - - Volle Kontrolle über Entwicklung - - Tiefes Verständnis der Architektur - - Schnelle Entscheidungswege - - Kosteneffizienz - -### 6. Schlüsselkomponenten-Entwicklung -- **Patrick**: - - Frontend-Architektur - - Backend-System - - Workflow-Engine - - Agent-System - - Connector-Framework - - Datenmanagement - - Sicherheitsarchitektur -- **Verantwortlichkeiten**: - - Systemarchitektur - - Technische Leitung - - Code-Review - - Qualitätssicherung - -### 7. Nicht-monetäre Ressourcen -- **Dominic**: - - Umfangreiches Sales & Marketing Netzwerk - - Marktzugang - - Branchenkontakte -- **ValueOn AG**: - - Infrastruktur - - Rechtlicher Rahmen - - Geschäftsprozesse -- **Netzwerke**: - - Technologie-Partner - - Potenzielle Kunden - - Branchenexperten - -### 8. Finanzielle Risiken -- **ValueOn AG**: - - Bereitstellung von Infrastruktur - - Personelle Ressourcen - - Rechtlicher Rahmen -- **Private Investitionen**: - - Entwicklungszeit - - Hardware/Software - - Cloud-Services - -### Fazit bisherige Aufwände -Die bisherigen Aufwände zeigen ein ausgewogenes Verhältnis zwischen technischer Expertise und Business-Know-how. Die private Investition von 730 Stunden durch Patrick demonstriert ein hohes Engagement und tiefes Verständnis der Technologie. Die Kombination aus technischer Expertise, Business-Erfahrung und Marktzugang bildet eine solide Grundlage für die weitere Entwicklung. Die bisherigen Investitionen sind effizient eingesetzt worden, mit Fokus auf kritische Kernkomponenten und skalierbare Architektur. - -## Teil 3: Markt- und Geschäftspotenzial - -### 1. Adressierbarer Gesamtmarkt und Wachstumspotenzial -- **Gesamtmarktvolumen 2025**: - - KI-Markt: $190 Mrd. - - Business Process Automation: $19,6 Mrd. - - Enterprise Knowledge Management: $43 Mrd. -- **Adressierbarer Markt (SAM)**: - - Initial: Mittlerer Markt in DACH (Professional Services, Finanzdienstleistungen, Gesundheitswesen) - - Geschätztes SAM: CHF 500-700 Mio. -- **Wachstumspotenzial**: - - Jährliches Marktwachstum: 25-30% - - Erweiterung auf internationale Märkte - - Branchenspezifische Lösungen - -### 2. Alleinstellungsmerkmale -1. **Technologische Vorteile**: - - Proprietäre Multi-Agent-Technologie - - Modellunabhängige KI-Integration - - Enterprise-Ready Architektur - - Fortschrittliche Workflow-Orchestrierung - -2. **Funktionale Vorteile**: - - Nahtlose Integration verschiedener KI-Modelle - - Robuste Fehlerbehandlung - - Skalierbare Multi-Tenant-Architektur - - Umfassende Enterprise-Features - -### 3. Kunden und Pilotprojekte -- **Aktueller Status**: - - In Entwicklung - - Erste Referenzkunden in Planung - - Fokus auf mittelständische Unternehmen -- **Pilotphase**: - - Geplant für Q3/Q4 2025 - - 3-5 Schlüsselreferenzkunden - - Branchenspezifische Templates - -### 4. Geschäftsmodell -- **Hauptmodell**: SaaS (Software as a Service) -- **Preismodell**: - - Basis: Pro-Benutzer/Monat Abonnement - - Zusätzlich: Nutzungsbasierte Abrechnung - - Enterprise-Lizenzen für grössere Kunden -- **Erwartete Margen**: 75-85% nach Skalierung - -### 5. Preisgestaltung -- **Wettbewerbsvergleich**: - - Unterhalb Enterprise-Lösungen - - Über Standard-BPA-Tools - - Flexiblere Preisgestaltung als Konkurrenz -- **Preisstruktur**: - - Basis-Abonnement: CHF 50-100 pro Benutzer/Monat - - Nutzungsbasierte Komponente: CHF 0.10-0.50 pro Verarbeitungseinheit - - Enterprise-Pakete: Individuelle Preisgestaltung - -### 6. Umsatzpotenziale -- **Jahr 1 (2026)**: - - Ziel: 20-30 Kunden - - Erwartetes ARR: CHF 0.5-0.8 Mio. -- **Jahr 2 (2027)**: - - Ziel: 70-90 Kunden - - Erwartetes ARR: CHF 2-2.5 Mio. -- **Jahr 3 (2028)**: - - Ziel: 150+ Kunden - - Erwartetes ARR: CHF 4.5 Mio. - -### 7. Akquisitionskosten -- **Customer Acquisition Cost (CAC)**: - - Erwarteter CAC: CHF 15,000-20,000 - - Payback-Zeit: 12-18 Monate -- **Kostenstruktur**: - - 30% Vertrieb und Marketing - - Fokus auf effiziente Akquisition - - Skaleneffekte ab 50+ Kunden - -### 8. Regulatorische Herausforderungen -- **Datenschutz**: - - DSGVO-Konformität - - Datensicherheit - - Mandantentrennung -- **KI-Regulierung**: - - EU AI Act - - Transparenzpflichten - - Qualitätssicherung -- **Branchenspezifische Regulierung**: - - Finanzdienstleistungen - - Gesundheitswesen - - Professional Services - -### Fazit Markt- und Geschäftspotenzial -Die PowerOn AI Platform adressiert einen wachsenden Markt mit klaren Alleinstellungsmerkmalen. Das Geschäftsmodell ist skalierbar und die Preisgestaltung wettbewerbsfähig. Die regulatorischen Herausforderungen sind bekannt und adressierbar. Die Umsatzprognosen sind konservativ kalkuliert und basieren auf realistischen Marktannahmen. - -## Teil 4: Skalierungs- und Zukunftspotenzial - -### 1. Ressourcen für Support und Weiterentwicklung -- **Entwicklungsteam**: - - 2-3 Full-Stack Entwickler, KI-unterstützte Entwicklung - - 1 DevOps Engineer - - 1 QA Engineer -- **Support-Team**: - - 1-2 Support Engineers - - 1 Technical Account Manager -- **Infrastruktur**: - - Cloud-basierte Skalierung (Azure) - - Automatisierte Deployment-Pipeline - - Monitoring und Logging-Systeme - -### 2. Skalierbarkeit -- **Technische Skalierbarkeit**: - - Horizontale Skalierung durch Multi-Tenant-Architektur - - Vertikale Skalierung durch Agent-Pooling - - Automatische Lastverteilung -- **Skalierungszeitrahmen**: - - 2x Nutzer: Sofort möglich - - 5x Nutzer: 1-2 Monate Vorbereitung - - 10x Nutzer: 3-4 Monate mit Infrastruktur-Erweiterung - -### 3. Kapitalbedarf -- **Nächste Entwicklungsphase (12 Monate)**: - - Entwicklung: CHF 400,000-500,000 - - Marketing & Sales: CHF 100,000-200,000 - - Infrastruktur: CHF 50,000-100,000 - - Betrieb & Support: CHF 100,000-150,000 - - **Gesamt**: CHF 750,000-1,050,000 - -### 4. Schlüsselpersonen -- **Technische Leitung**: - - Patrick (Software Architect, Full Stack Developer) - - Verantwortlich für: Architektur, Entwicklung, Technische Strategie -- **Business & Operations**: - - Ida (Business Analyst, Project Manager) - - Verantwortlich für: Projektmanagement, Business Analysis -- **Sales & Marketing**: - - Dominic (Sales & Marketing) - - Verantwortlich für: Marktentwicklung, Kundenakquisition - -### 5. Exit-Strategien -- **Strategische Übernahme**: - - Enterprise Software Anbieter - - KI/ML Plattform Betreiber - - Business Process Automation Unternehmen -- **IPO-Potenzial**: - - Ab CHF 50 Mio. ARR - - Zeitrahmen: 5-7 Jahre -- **Extraktion aus ValueOn AG**: - - Vergütung der Aufwände - - Anrechnung des Mehrwerts für Schlüsselpersonen - - Beschaffung des notwendigen Kapitals - -### 6. Strategische Partnerschaften -- **Technologie-Partner**: - - KI-Provider (OpenAI, Anthropic) - - Cloud-Provider (Azure) - - Enterprise Software Anbieter -- **Vertriebspartner**: - - Systemhäuser - - Beratungsunternehmen - - Branchenspezialisten -- **Forschungspartner**: - - Universitäten - - Forschungsinstitute - - KI-Labore - -### 7. Geplante Erweiterungen -- **Kurzfristig (12 Monate)**: - - Erweiterte Agent-Typen - - Branchenspezifische Templates - - API-Erweiterungen -- **Mittelfristig (24 Monate)**: - - Agentenmarktplatz - - Proprietäre KI-Modelle - - Erweiterte Analytics -- **Langfristig (36+ Monate)**: - - KI-Middleware für Unternehmen - - Branchenlösungen - - Internationale Expansion - -### 8. Langfristige Vision -- **Technologische Vision**: - - Führende Multi-Agent KI-Plattform - - Standard für Enterprise Workflow Automation - - Innovationstreiber in der KI-Integration -- **Marktvision**: - - Globaler Marktführer in Nischenbereichen - - Branchenstandard für bestimmte Anwendungsfälle - - Referenz für KI-gestützte Prozessoptimierung -- **Geschäftsvision**: - - Nachhaltiges Wachstum - - Profitables Geschäftsmodell - - Führende Position in ausgewählten Märkten - -### Fazit Skalierungs- und Zukunftspotenzial -Die PowerOn AI Platform verfügt über ein solides Skalierungspotenzial sowohl technisch als auch geschäftlich. Die modulare Architektur ermöglicht schnelles Wachstum, während die klare Vision und die strategischen Partnerschaften den langfristigen Erfolg unterstützen. Die Kapitalanforderungen sind realistisch kalkuliert und die Exit-Strategien bieten verschiedene Optionen für die Zukunft. - - -# Exitplan: PowerOn AI Platform als eigenständige AG - -## Bewertung und Ausgangssituation - -**Aktueller Wert (Juni 2025)**: CHF 500.000 -**Angepasster Gründungswert**: CHF 800.000 (berücksichtigt den strategischen Wert der Produktvision, welche bereits als innerer Wert im Produkt enthalten ist) - -## Strukturierung der PowerOn AG - -### 1. Aktienstruktur bei Gründung - -**Gesamtes Aktienkapital**: 1.000.000 Aktien (Nennwert CHF 0,10) -**Firmenvaluation bei Gründung**: CHF 800.000 - -#### Verteilung der initialen Aktien: - -- **Patrick**: - - Eingebrachte Leistung: Entwicklung, technische Expertise und essenzielles Gesamtkonzept - - **Aktienanteil**: 35% (350.000 Aktien) - -- **Dominic**: - - Eingebrachte Leistung: Netzwerk, Sales & Marketing Expertise und Vision - - **Aktienanteil**: 15% (150.000 Aktien) - -- **ValueOn AG**: - - Eingebrachte Leistung: Infrastruktur, rechtlicher Rahmen, Arbeitszeit, Übertragung von IP - - **Aktienanteil**: 25% (250.000 Aktien) - -- **Reservierter Anteil für Mitarbeiter-Pool**: - - **Aktienanteil**: 10% (100.000 Aktien) - -- **Reserviert für Investoren (erste Runde)**: - - **Aktienanteil**: 15% (150.000 Aktien) - -### 2. Kapitalbedarfsplanung (18 Monate) - -| Kategorie | Betrag (CHF) | -|-----------|--------------| -| Entwicklung | 550.000 | -| Marketing & Sales | 250.000 | -| Infrastruktur | 100.000 | -| Betrieb & Support | 200.000 | -| **Gesamtbedarf** | **1.100.000** | - -**Kapitalbeschaffungsstrategie**: -- **Erste Finanzierungsrunde**: CHF 1.000.000 (für 18 Monate) -- **Sicherheitspuffer**: CHF 100.000 (aus Umsätzen/zukünftigen Einnahmen) - -### 3. Investitionskonditionen - -**Pre-Money Bewertung**: CHF 800.000 -**Investitionsvolumen**: CHF 1.000.000 -**Post-Money Bewertung**: CHF 1.800.000 - -**Aktienkurs für Investoren**: -- 150.000 bestehende Aktien + 214.285 neue Aktien = 364.285 Aktien für Investoren -- **Aktienkurs**: CHF 2,75 pro Aktie - -**Aktienstruktur nach Investment**: -- Patrick: 35% → 28,9% (350.000 Aktien) -- Dominic: 15% → 12,4% (150.000 Aktien) -- ValueOn AG: 25% → 20,7% (250.000 Aktien) -- Mitarbeiter-Pool: 10% → 8,3% (100.000 Aktien) -- Investoren: 29,7% (364.285 Aktien) - -## Governance und Organisation - -### 1. Schlüsselpositionen in der Organisation - -- **CEO**: Gesamtführung des Unternehmens -- **CTO**: Verantwortlich für technische Strategie und Produktentwicklung -- **CSO/Vertriebsleitung**: Verantwortlich für Vertrieb und Marktentwicklung -- **COO**: Operative Leitung und Geschäftsprozesse - -Die Besetzung dieser Positionen wird unter Berücksichtigung der Kompetenzen von Patrick, Dominic und möglichen neuen Führungskräften festgelegt. Die Rollen von CEO und COO werden im Rahmen der Unternehmensgründung evaluiert. - -### 2. Vergütungsstruktur - -- **Führungsebene**: Marktübliche Vergütung zwischen CHF 150.000-180.000/Jahr je nach Position -- **Aktienoptionen**: Zusätzliche Aktienoptionen bei Erreichen definierter Unternehmensziele - -## Meilensteine und Finanzielle Ziele - -### Kritische Meilensteine (18 Monate) - -| Zeitpunkt | Meilenstein | KPI | -|-----------|-------------|-----| -| Q3 2025 | Ausgründung & Finanzierung | Abschluss der Seed-Runde | -| Q4 2025 | Markteinführung | 3-5 Pilotprojekte | -| Q1 2026 | Produktvalidierung | 10+ zahlende Kunden | -| Q2 2026 | Skalierung | 15+ zahlende Kunden | -| Q4 2026 | Vorbereitung Serie A | 25+ Kunden, ARR: CHF 0,8 Mio. | - -### Umsatz- und Bewertungsprognose - -| Jahr | Kunden | ARR (CHF) | Valuation (CHF) | Multiplikator | -|------|--------|-----------|-----------------|---------------| -| Ende 2025 | 5-8 | 0,2 Mio. | 1,8 Mio. | 9x ARR | -| Ende 2026 | 25-30 | 0,8 Mio. | 5,6 Mio. | 7x ARR | -| Ende 2027 | 80-90 | 2,5 Mio. | 15 Mio. | 6x ARR | -| Ende 2028 | 150+ | 4,5 Mio. | 27 Mio. | 6x ARR | - -## Liquiditätsoptionen - -### Mittelfristige Optionen (2-3 Jahre) - -1. **Serie A Finanzierung** (Ende 2026): - - Teilweise Liquidität für Gründer und ValueOn AG (10-15% ihrer Anteile) - - Zu erwartender Wert: CHF 5-6 Mio. - -2. **Strategische Partnerschaft**: - - Investment durch strategischen Partner mit teilweisem Aktienrückkauf - - Potenzielle Partner: Enterprise Software-Anbieter, KI-Plattform-Betreiber - -### Langfristige Optionen (4-7 Jahre) - -1. **Komplette Übernahme**: - - Erwarteter Exit-Wert 2028: CHF 25-30 Mio. - - Vollständige Liquidität für alle Anteilseigner - -2. **IPO-Vorbereitung**: - - Bei Erreichen von CHF 10+ Mio. ARR - - Potenzielle Bewertung: CHF 50-70 Mio. - -## Nächste Schritte im Ausgründungsprozess - -1. **Rechtliche Strukturierung**: - - Gründung der PowerOn AG - - Übertragungsvereinbarungen für geistiges Eigentum - - Aktionärsvereinbarungen - -2. **Finanzierung**: - - Erstellung eines detaillierten Businessplans - - Vorbereitung des Investor Pitch Decks - - Ansprache potenzieller Investoren - -3. **Organisationsaufbau**: - - Definition der Führungsstruktur und Schlüsselpositionen - - Rekrutierung des Kernteams - - Aufbau der operativen Prozesse - -4. **Markteinführungsstrategie**: - - Festlegung der Go-to-Market Strategie - - Identifikation von Pilotprojekten - - Vorbereitung der Vertriebsunterlagen diff --git a/tests/test_neutralizer/testdata/customers.csv b/tests/test_neutralizer/testdata/customers.csv deleted file mode 100644 index bebbfa11..00000000 --- a/tests/test_neutralizer/testdata/customers.csv +++ /dev/null @@ -1,6 +0,0 @@ -id,name,email,phone,address,iban,credit_card,ahv_number -1,Max Mustermann,max.mustermann@beispiel.de,+49 30 12345678,Musterstraße 123 12345 Berlin,DE89 3704 0044 0532 0130 00,4532 1234 5678 9012, -2,Peter Schmid,peter.schmid@beispiel.ch,+41 44 123 45 67,Bahnhofstrasse 1 8001 Zürich,CH93 0076 7000 E529 3557 7,4532 1234 5678 9013,756.1234.5678.90 -3,Marie Dupont,marie.dupont@exemple.fr,+33 1 23 45 67 89,123 Rue de Paris 75001 Paris,FR76 3000 6000 0112 3456 7890 189,4532 1234 5678 9014, -4,Marco Rossi,marco.rossi@esempio.it,+39 02 1234 5678,Via Roma 123 20100 Milano,IT60 X054 2811 1010 0000 0123 456,4532 1234 5678 9015, -5,John Smith,john.smith@example.com,+44 20 1234 5678,123 High Street London SW1A 1AA,GB29 NWBK 6016 1331 9268 19,4532 1234 5678 9016, \ No newline at end of file diff --git a/tests/test_neutralizer/testdata/cv_lara_meier.txt b/tests/test_neutralizer/testdata/cv_lara_meier.txt deleted file mode 100644 index 8aec5d3b..00000000 --- a/tests/test_neutralizer/testdata/cv_lara_meier.txt +++ /dev/null @@ -1,73 +0,0 @@ -Lebenslauf: Lara Meier - -Persönliche Daten: -Name: Lara Meier -Geboren: 15.03.1990 -Adresse: Bahnhofstrasse 45, 8001 Zürich -Telefon: +41 44 123 45 67 -E-Mail: lara.meier@example.ch -AHV-Nr.: 756.1234.5678.90 -Steuernummer: 123.456.789 - -Berufserfahrung: -2020-2023: Senior Projektmanagerin -Firma: TechSolutions AG -Adresse: Industriestrasse 100, 8004 Zürich -UID: CHE-123.456.789 -Telefon: +41 44 987 65 43 -E-Mail: lara.meier@techsolutions.ch - -In dieser Position leitete ich ein Team von 15 Mitarbeitern und verantwortete die Implementierung von Cloud-Lösungen für internationale Kunden. Meine Hauptaufgaben umfassten: -- Projektplanung und -steuerung mit einem Budget von CHF 2.5 Mio. -- Kundenbetreuung und Stakeholder-Management -- Teamführung und Personalentwicklung -- Qualitätssicherung und Risikomanagement - -2015-2020: Projektmanagerin -Firma: Digital Systems GmbH -Adresse: Musterstrasse 123, 8002 Zürich -UID: CHE-987.654.321 -Telefon: +41 44 456 78 90 -E-Mail: l.meier@digitalsystems.ch - -Als Projektmanagerin verantwortete ich die erfolgreiche Durchführung von Digitalisierungsprojekten. Meine Leistungen: -- Implementierung von ERP-Systemen -- Optimierung von Geschäftsprozessen -- Schulung von Endbenutzern -- Erstellung von Projektdokumentation - -Ausbildung: -2010-2015: ETH Zürich -Studiengang: Informatik -Matrikelnummer: 12-345-678 -Abschluss: Master of Science in Computer Science -Thesis: "Künstliche Intelligenz in der Prozessautomatisierung" - -2005-2010: Kantonsschule Zürich -Abschluss: Eidgenössische Maturität -Schwerpunkt: Mathematik und Naturwissenschaften - -Sprachen: -Deutsch (Muttersprache) -Englisch (C2) -Französisch (B2) -Italienisch (B1) - -Zertifizierungen: -PMP (Project Management Professional) -ITIL v4 Foundation -AWS Certified Solutions Architect -Scrum Master (PSM I) - -Referenzen: -Prof. Dr. Hans Müller -ETH Zürich -Department of Computer Science -Telefon: +41 44 123 45 68 -E-Mail: hans.mueller@ethz.ch - -Dr. Sarah Weber -TechSolutions AG -CTO -Telefon: +41 44 987 65 44 -E-Mail: sarah.weber@techsolutions.ch \ No newline at end of file diff --git a/tests/test_neutralizer/testdata/employees.csv b/tests/test_neutralizer/testdata/employees.csv deleted file mode 100644 index 080ccdbc..00000000 --- a/tests/test_neutralizer/testdata/employees.csv +++ /dev/null @@ -1,6 +0,0 @@ -employee_id,first_name,last_name,email,phone,department,office_address,uid_number,bank_account -E001,Hans,Müller,hans.mueller@firma.de,+49 89 12345678,IT,Hauptstraße 1 80331 München,,DE89 3704 0044 0532 0130 00 -E002,Thomas,Weber,thomas.weber@firma.ch,+41 44 234 56 78,HR,Bahnhofstrasse 2 8001 Zürich,CHE-123.456.789,01-234567-8 -E003,Sophie,Martin,sophie.martin@entreprise.fr,+33 1 34 56 78 90,Finance,15 Avenue des Champs-Élysées 75008 Paris,,FR76 3000 6000 0112 3456 7890 189 -E004,Luca,Ferrari,luca.ferrari@azienda.it,+39 02 2345 6789,Marketing,Via Monte Napoleone 8 20121 Milano,,IT60 X054 2811 1010 0000 0123 456 -E005,Emma,Wilson,emma.wilson@company.com,+44 20 2345 6789,Sales,25 Old Street London EC1V 9HL,,GB29 NWBK 6016 1331 9268 19 \ No newline at end of file diff --git a/tests/test_neutralizer/testdata/english.txt b/tests/test_neutralizer/testdata/english.txt deleted file mode 100644 index d813b5eb..00000000 --- a/tests/test_neutralizer/testdata/english.txt +++ /dev/null @@ -1,12 +0,0 @@ -Dear Mr. Smith, - -Thank you for your email to contact@example.com. -Your contact details have been recorded: -Phone: +44 20 1234 5678 -Address: 123 High Street, London SW1A 1AA - -Your IBAN: GB29 NWBK 6016 1331 9268 19 -Credit Card: 4532 1234 5678 9012 - -Best regards, -John Doe \ No newline at end of file diff --git a/tests/test_neutralizer/testdata/example.json b/tests/test_neutralizer/testdata/example.json deleted file mode 100644 index e268aa8e..00000000 --- a/tests/test_neutralizer/testdata/example.json +++ /dev/null @@ -1,54 +0,0 @@ -{ - "customer": { - "personal_info": { - "name": "Max Mustermann", - "email": "max.mustermann@example.com", - "phone": "+49 123 4567890", - "address": { - "street": "Hauptstraße 123", - "city": "München", - "zip": "80331", - "country": "Deutschland" - } - }, - "order_history": [ - { - "order_id": "ORD-2024-001", - "date": "2024-03-15", - "items": [ - { - "product": "Laptop", - "price": 1299.99, - "payment": { - "method": "credit_card", - "card_number": "4111 1111 1111 1111", - "iban": "DE89 3704 0044 0532 0130 00" - } - } - ] - } - ], - "preferences": { - "language": "de", - "newsletter": true, - "marketing_consent": { - "email": "max.mustermann@example.com", - "phone": "+49 123 4567890" - } - } - }, - "company": { - "name": "Tech Solutions GmbH", - "contact": { - "manager": "Dr. Anna Schmidt", - "email": "anna.schmidt@techsolutions.de", - "phone": "+49 89 12345678", - "address": { - "street": "Technologiestraße 45", - "city": "Berlin", - "zip": "10115", - "country": "Deutschland" - } - } - } -} \ No newline at end of file diff --git a/tests/test_neutralizer/testdata/example.xml b/tests/test_neutralizer/testdata/example.xml deleted file mode 100644 index 37837e53..00000000 --- a/tests/test_neutralizer/testdata/example.xml +++ /dev/null @@ -1,79 +0,0 @@ - - - - - - Dr. Thomas Weber - thomas.weber@company.de - +49 30 12345678 -
- Musterstraße 42 - Hamburg - 20095 - Deutschland -
- - - IT - Senior Developer - 75000 - - DE89 3704 0044 0532 0130 00 - COBADEFFXXX - - - - - - Maria Schmidt - maria.schmidt@company.de - +49 40 98765432 -
- Bahnhofstraße 15 - Frankfurt - 60329 - Deutschland -
- - - HR - HR Manager - 65000 - - DE27 3704 0044 0532 0130 01 - COBADEFFXXX - - - - - - - Digital Transformation - Dr. Thomas Weber - 150000 - - Dr. Thomas Weber - Maria Schmidt - - - - - Tech Innovations GmbH -
- Innovationsstraße 1 - München - 80331 - Deutschland -
- - info@techinnovations.de - +49 89 12345678 - +49 89 12345679 - - - DE89 3704 0044 0532 0130 02 - COBADEFFXXX - DE123456789 - - - \ No newline at end of file diff --git a/tests/test_neutralizer/testdata/french.txt b/tests/test_neutralizer/testdata/french.txt deleted file mode 100644 index 4ff7ab61..00000000 --- a/tests/test_neutralizer/testdata/french.txt +++ /dev/null @@ -1,12 +0,0 @@ -Bonjour Madame Dupont, - -Merci pour votre email à contact@exemple.fr. -Vos coordonnées ont été enregistrées: -Téléphone: +33 1 23 45 67 89 -Adresse: 123 Rue de Paris, 75001 Paris - -Votre IBAN: FR76 3000 6000 0112 3456 7890 189 -Carte de crédit: 4532 1234 5678 9012 - -Cordialement, -Jean Martin \ No newline at end of file diff --git a/tests/test_neutralizer/testdata/german.txt b/tests/test_neutralizer/testdata/german.txt deleted file mode 100644 index eb9f4ed4..00000000 --- a/tests/test_neutralizer/testdata/german.txt +++ /dev/null @@ -1,12 +0,0 @@ -Sehr geehrte Frau Müller, - -vielen Dank für Ihre E-Mail an max.mustermann@beispiel.de. -Ihre Telefonnummer 030-12345678 wurde in unserem System hinterlegt. -Die Rechnung wird an folgende Adresse gesendet: -Musterstraße 123, 12345 Berlin - -Ihre IBAN: DE89 3704 0044 0532 0130 00 -Kreditkarte: 4532 1234 5678 9012 - -Mit freundlichen Grüßen -Max Mustermann \ No newline at end of file diff --git a/tests/test_neutralizer/testdata/geschaeftsstrategie.txt b/tests/test_neutralizer/testdata/geschaeftsstrategie.txt deleted file mode 100644 index 4c8215b9..00000000 --- a/tests/test_neutralizer/testdata/geschaeftsstrategie.txt +++ /dev/null @@ -1,56 +0,0 @@ -Geschäftsstrategie 2024-2026 -TechSolutions AG - -Unternehmensdaten: -Firmenname: TechSolutions AG -UID: CHE-123.456.789 -Adresse: Industriestrasse 100, 8004 Zürich -Telefon: +41 44 987 65 43 -E-Mail: info@techsolutions.ch -Website: www.techsolutions.ch - -Geschäftsführung: -CEO: Dr. Thomas Weber -E-Mail: thomas.weber@techsolutions.ch -Telefon: +41 44 987 65 44 - -CTO: Dr. Sarah Müller -E-Mail: sarah.mueller@techsolutions.ch -Telefon: +41 44 987 65 45 - -Finanzdaten: -Bank: UBS AG -IBAN: CH93 0076 7000 E529 0767 9 -BIC: UBSWCHZH80A - -Strategische Ziele: -1. Marktpositionierung - - Marktanteil in der Schweiz auf 25% steigern - - Expansion in DACH-Region - - Neue Zielgruppe: KMU - -2. Produktentwicklung - - Cloud-Lösung für KMU - - KI-basierte Automatisierung - - Mobile App für Kunden - -3. Personalentwicklung - - Team auf 50 Mitarbeiter ausbauen - - Neue Standorte: Bern und Genf - - Fokus auf Diversity & Inclusion - -4. Nachhaltigkeit - - CO2-Neutralität bis 2025 - - Green IT Zertifizierung - - Nachhaltige Lieferkette - -Budgetplanung: -2024: CHF 5.2 Mio. -2025: CHF 6.8 Mio. -2026: CHF 8.5 Mio. - -Risikoanalyse: -- Marktrisiken -- Technologische Risiken -- Personelle Risiken -- Finanzielle Risiken \ No newline at end of file diff --git a/tests/test_neutralizer/testdata/geschäfte.csv b/tests/test_neutralizer/testdata/geschäfte.csv deleted file mode 100644 index 4a1377e2..00000000 --- a/tests/test_neutralizer/testdata/geschäfte.csv +++ /dev/null @@ -1,6 +0,0 @@ -geschäft_id,datum,kundenname,kundenemail,betrag,zahlungsmethode,zahlungsdetails,lieferadresse -G001,2024-03-15,Anna Schmidt,anna.schmidt@kunde.de,1250,50,IBAN,DE89 3704 0044 0532 0130 00,Kirchstraße 10 10115 Berlin -G002,2024-03-15,Franz Huber,franz.huber@kunde.de,890,75,IBAN,DE02 5001 0517 5407 3249 31,Seefeldstraße 5 10117 Berlin -G003,2024-03-15,Pierre Dubois,pierre.dubois@kunde.de,2340,25,IBAN,DE27 2005 0550 1045 1862 37,15 Rue de la Paix 10115 Berlin -G004,2024-03-15,Giovanni Bianchi,giovanni.bianchi@kunde.de,1750,00,IBAN,DE02 5001 0517 5407 3249 32,Via della Spiga 20 10115 Berlin -G005,2024-03-15,William Brown,william.brown@kunde.de,3200,50,IBAN,DE02 5001 0517 5407 3249 33,42 Oxford Street 10115 Berlin \ No newline at end of file diff --git a/tests/test_neutralizer/testdata/italian.txt b/tests/test_neutralizer/testdata/italian.txt deleted file mode 100644 index 20cd6e92..00000000 --- a/tests/test_neutralizer/testdata/italian.txt +++ /dev/null @@ -1,13 +0,0 @@ -Buongiorno Signora Rossi, - -Grazie per la sua email a info@esempio.it. -Grazie per la sua email a info@esempio.it . -I suoi dati di contatto sono stati registrati: -Telefono: +39 02 1234 5678 -Indirizzo: Via Roma 123, 20100 Milano - -Il suo IBAN: IT60 X054 2811 1010 0000 0123 456 -Carta di credito: 4532 1234 5678 9012 - -Cordiali saluti, -Marco Bianchi \ No newline at end of file diff --git a/tests/test_neutralizer/testdata/kunden.csv b/tests/test_neutralizer/testdata/kunden.csv deleted file mode 100644 index 3393ac11..00000000 --- a/tests/test_neutralizer/testdata/kunden.csv +++ /dev/null @@ -1,6 +0,0 @@ -kunden_id,vorname,nachname,email,telefon,adresse,iban,kreditkarte,steuernummer -K001,Michael,Schmidt,michael.schmidt@kunde.de,+49 30 12345678,Musterstraße 123 12345 Berlin,DE89 3704 0044 0532 0130 00,4532 1234 5678 9012,12/345/67890 -K002,Sabine,Weber,sabine.weber@kunde.de,+49 89 23456789,Hauptstraße 45 80331 München,DE02 5001 0517 5407 3249 31,4532 1234 5678 9013,98/765/43210 -K003,Thomas,Müller,thomas.mueller@kunde.de,+49 40 34567890,Neue Straße 78 20095 Hamburg,DE27 2005 0550 1045 1862 37,4532 1234 5678 9014,45/678/90123 -K004,Petra,Fischer,petra.fischer@kunde.de,+49 69 45678901,Frankfurter Ring 12 60313 Frankfurt,DE02 5001 0517 5407 3249 32,4532 1234 5678 9015,34/567/89012 -K005,Klaus,Wagner,klaus.wagner@kunde.de,+49 211 56789012,Königsallee 92 40212 Düsseldorf,DE02 5001 0517 5407 3249 33,4532 1234 5678 9016,23/456/78901 \ No newline at end of file diff --git a/tests/test_neutralizer/testdata/mitarbeiter.csv b/tests/test_neutralizer/testdata/mitarbeiter.csv deleted file mode 100644 index 4e15441c..00000000 --- a/tests/test_neutralizer/testdata/mitarbeiter.csv +++ /dev/null @@ -1,6 +0,0 @@ -mitarbeiter_id,vorname,nachname,email,telefon,abteilung,büroadresse,steuernummer,sozialversicherungsnummer -M001,Andreas,Becker,andreas.becker@firma.de,+49 30 98765432,IT,Hauptstraße 1 10115 Berlin,12/345/67890,12 345678 901 -M002,Monika,Hoffmann,monika.hoffmann@firma.de,+49 89 87654321,HR,Leopoldstraße 45 80802 München,98/765/43210,98 765432 102 -M003,Frank,Schäfer,frank.schaefer@firma.de,+49 40 76543210,Finanzen,Neuer Wall 78 20354 Hamburg,45/678/90123,45 678901 203 -M004,Susanne,Koch,susanne.koch@firma.de,+49 69 65432109,Marketing,Mainzer Landstraße 12 60329 Frankfurt,34/567/89012,34 567890 304 -M005,Dieter,Richter,dieter.richter@firma.de,+49 211 54321098,Vertrieb,Königsallee 92 40212 Düsseldorf,23/456/78901,23 456789 405 \ No newline at end of file diff --git a/tests/test_neutralizer/testdata/swiss.txt b/tests/test_neutralizer/testdata/swiss.txt deleted file mode 100644 index 9455e4a1..00000000 --- a/tests/test_neutralizer/testdata/swiss.txt +++ /dev/null @@ -1,13 +0,0 @@ -Guten Tag Herr Schmid - -Vielen Dank für Ihre Anfrage an info@beispiel.ch. -Ihre Kontaktdaten wurden wie folgt erfasst: -Telefon: +41 44 123 45 67 -Adresse: Bahnhofstrasse 1, 8001 Zürich - -Ihre AHV-Nummer: 756.1234.5678.90 -UID-Nummer: CHE-123.456.789 -Bankkonto: 01-234567-8 - -Mit freundlichen Grüßen -Peter Muster \ No newline at end of file diff --git a/tests/test_neutralizer/testdata/transactions.csv b/tests/test_neutralizer/testdata/transactions.csv deleted file mode 100644 index e362f340..00000000 --- a/tests/test_neutralizer/testdata/transactions.csv +++ /dev/null @@ -1,6 +0,0 @@ -transaction_id,date,customer_name,customer_email,amount,payment_method,payment_details,shipping_address -T001,2024-03-15,Anna Schmidt,anna.schmidt@kunde.de,1250.50,IBAN,DE89 3704 0044 0532 0130 00,Kirchstraße 10 10115 Berlin -T002,2024-03-15,Franz Huber,franz.huber@kunde.ch,890.75,AHV,756.1234.5678.90,Seefeldstrasse 5 8008 Zürich -T003,2024-03-15,Pierre Dubois,pierre.dubois@client.fr,2340.25,IBAN,FR76 3000 6000 0112 3456 7890 189,15 Rue de la Paix 75002 Paris -T004,2024-03-15,Giovanni Bianchi,giovanni.bianchi@cliente.it,1750.00,IBAN,IT60 X054 2811 1010 0000 0123 456,Via della Spiga 20 20121 Milano -T005,2024-03-15,William Brown,william.brown@customer.com,3200.50,IBAN,GB29 NWBK 6016 1331 9268 19,42 Oxford Street London W1D 2BJ \ No newline at end of file diff --git a/tests/test_neutralizer/zdocu.html b/tests/test_neutralizer/zdocu.html deleted file mode 100644 index 7194d23f..00000000 --- a/tests/test_neutralizer/zdocu.html +++ /dev/null @@ -1,157 +0,0 @@ - - - - - Daten-Neutralisierer Logik - - - -

Daten-Neutralisierer Logik

-

Der Daten-Neutralisierer unterstützt die Anonymisierung von sensiblen Daten in verschiedenen Dateiformaten (TXT, JSON, CSV, XML, DOCX) und Sprachen (DE, EN, FR, IT).

- -

1. Unterstützte Dateiformate

-
    -
  • CSV-Dateien
    • -
  • JSON-Dateien
    • -
  • XML-Dateien
    • -
  • Text-Dateien (TXT)
    • -
  • Word-Dokumente (DOCX)
    • -
- -

2. Neutralisierungsmuster

- -
-

Personenbezogene Daten

- -
-

Vornamen

-

Deutsch: vorname, vornamen, rufname, taufname

-

Englisch: first name, given name, forename, personal name

-

Französisch: prénom, prénoms, nom de baptême

-

Italienisch: nome, nome di battesimo

-

Ersetzung: NAME_{uuid}

-
- -
-

Nachnamen

-

Deutsch: nachname, nachnamen, familienname, familiennamen, zuname, zunamen

-

Englisch: last name, family name, surname, second name

-

Französisch: nom de famille, nom, noms

-

Italienisch: cognome, cognomi

-

Ersetzung: NAME_{uuid}

-
- -
-

Namen mit Titel

-

Muster: Herr/Frau/Mr./Mrs./Ms./Monsieur/Madame/Signore/Signora/Dr./Prof.

-

Ersetzung: NAME_{uuid}

-
-
- -
-

Kontaktdaten

- -
-

E-Mail-Adressen

-

Muster: [A-Za-z0-9._%+-]+@[A-Za-z0-9.-]+\.[A-Z|a-z]{2,}

-

Ersetzung: EMAIL_{uuid}

-
- -
-

Telefonnummern

-

Allgemein: (+XXX)? (XXX) XXX-XXXX

-

Deutsch: (+49|0049|0) XXX XXXXXXX

-

Schweiz: (+41|0041|0) XXX XXX XX XX

-

Ersetzung: PHONE_{uuid}

-
-
- -
-

Finanzielle Daten

- -
-

IBAN

-

Muster: [A-Z]{2}\d{2}[A-Z0-9]{4}\d{7}([A-Z0-9]?){0,16}

-

Ersetzung: IBAN_{uuid}

-
- -
-

Kreditkartennummern

-

Muster: XXXX-XXXX-XXXX-XXXX

-

Ersetzung: CREDITCARD_{uuid}

-
-
- -
-

Adressdaten

- -
-

Vollständige Adressen

-

Muster: Straßenname + Hausnummer + PLZ + Stadt

-

Ersetzung: ADDRESS_{uuid}

-
- -
-

Städte und Orte

-

Muster: Stadtnamen mit typischen Endungen (-stadt, -dorf, -berg, etc.)

-

Ersetzung: CITY_{uuid}

-
-
- -

3. Verarbeitungslogik

-
    -
  1. Datei wird basierend auf dem Dateityp eingelesen
    2. -
  2. Bei Tabellendaten (CSV, JSON, XML): -
      -
    • Header werden analysiert und Spaltentypen identifiziert
      • -
    • Daten in den Spalten werden entsprechend dem Spaltentyp neutralisiert
      • -
    -
    3. -
  3. Bei Textdaten: -
      -
    • Text wird nach Tabellenstrukturen analysiert
      • -
    • Gefundene Tabellen werden wie oben verarbeitet
      • -
    • Verbleibender Text wird nach Mustern durchsucht
      • -
    -
    4. -
  4. Alle Ersetzungen werden in einer Mapping-Datei protokolliert
    5. -
- -

4. Logging und Tracking

-
    -
  • Mapping-Log: Protokolliert alle durchgeführten Ersetzungen
    • -
  • Replacements-Log: Speichert Original- und anonymisierte Werte
    • -
  • Alle Aktionen werden mit Zeitstempel und Dateiname dokumentiert
    • -
- - \ No newline at end of file