From d9ee4d960573ad7cbd9bb2db73ccd24dba992711 Mon Sep 17 00:00:00 2001 From: ValueOn AG Date: Sun, 12 Oct 2025 14:52:42 +0200 Subject: [PATCH] full ai center integration test and fix with extraction and generation engine --- env_dev.20251012_121418.backup | 90 ++++ env_dev.env | 6 +- env_int.20251012_121418.backup | 90 ++++ env_int.env | 6 +- env_prod.20251012_121418.backup | 90 ++++ env_prod.env | 6 +- modules/connectors/connectorAiAnthropic.py | 39 +- modules/services/serviceAi/mainServiceAi.py | 209 +++++---- .../extractors/extractorImage.py | 32 ++ .../services/serviceExtraction/subPipeline.py | 40 +- .../services/serviceExtraction/subRegistry.py | 9 +- .../mainServiceGeneration.py | 12 +- .../serviceGeneration/renderers/registry.py | 8 +- .../renderers/rendererBaseTemplate.py | 15 +- .../renderers/rendererDocx.py | 4 +- .../renderers/rendererPdf.py | 71 ++- .../renderers/rendererXlsx.py | 41 +- .../serviceGeneration/subPromptBuilder.py | 35 +- .../mainServiceNeutralization.py | 2 +- .../mainServiceSharepoint.py | 2 +- .../serviceTicket/mainServiceTicket.py | 2 +- .../services/serviceUtils/mainServiceUtils.py | 42 +- .../serviceWorkflow/mainServiceWorkflow.py | 32 +- .../shared/promptGenerationActionsReact.py | 2 +- requirements.txt | 1 + test_document_processing.py | 104 ++++- tool_security_encrypt_all_env_files.py | 422 ++++++++++++++++++ 27 files changed, 1158 insertions(+), 254 deletions(-) create mode 100644 env_dev.20251012_121418.backup create mode 100644 env_int.20251012_121418.backup create mode 100644 env_prod.20251012_121418.backup create mode 100644 tool_security_encrypt_all_env_files.py diff --git a/env_dev.20251012_121418.backup b/env_dev.20251012_121418.backup new file mode 100644 index 00000000..9ebbb93b --- /dev/null +++ b/env_dev.20251012_121418.backup @@ -0,0 +1,90 @@ +# Development Environment Configuration + +# System Configuration +APP_ENV_TYPE = dev +APP_ENV_LABEL = Development Instance Patrick +APP_API_URL = http://localhost:8000 +APP_KEY_SYSVAR = D:/Athi/Local/Web/poweron/local/key.txt +APP_INIT_PASS_ADMIN_SECRET = DEV_ENC:Z0FBQUFBQm8xSUpEeFFtRGtQeVUtcjlrU3dab1ZxUm9WSks0MlJVYUtERFlqUElHemZrOGNENk1tcmJNX3Vxc01UMDhlNU40VzZZRVBpUGNmT3podzZrOGhOeEJIUEt4eVlSWG5UYXA3d09DVXlLT21Kb1JYSUU9 +APP_INIT_PASS_EVENT_SECRET = DEV_ENC:Z0FBQUFBQm8xSUpERzZjNm56WGVBdjJTeG5Udjd6OGQwUVotYXUzQjJ1YVNyVXVBa3NZVml3ODU0MVNkZjhWWmJwNUFkc19BcHlHMTU1Q3BRcHU0cDBoZkFlR2l6UEZQU3d2U3MtMDh5UDZteGFoQ0EyMUE1ckE9 + +# PostgreSQL Storage (new) +DB_APP_HOST=localhost +DB_APP_DATABASE=poweron_app +DB_APP_USER=poweron_dev +DB_APP_PASSWORD_SECRET = DEV_ENC:Z0FBQUFBQm8xSUpEcUIxNEFfQ2xnS0RrSC1KNnUxTlVvTGZoMHgzaEI4Z3NlVzVROTVLak5Ubi1vaEZubFZaMTFKMGd6MXAxekN2d2NvMy1hRjg2UVhybktlcFA5anZ1WjFlQmZhcXdwaGhWdzRDc3ExeUhzWTg9 +DB_APP_PORT=5432 + +# PostgreSQL Storage (new) +DB_CHAT_HOST=localhost +DB_CHAT_DATABASE=poweron_chat +DB_CHAT_USER=poweron_dev +DB_CHAT_PASSWORD_SECRET = DEV_ENC:Z0FBQUFBQm8xSUpERFNzNVhoalpCR0QxYXAwdEpXWXVVOTdZdWtqWW5FNXFGcFl2amNYLWYwYl9STXltRlFxLWNzVWlMVnNYdXk0RklnRExFT0FaQjg2aGswNnhhSGhCN29KN2VEb2FlUV9NTlV3b0tLelplSVU9 +DB_CHAT_PORT=5432 + +# PostgreSQL Storage (new) +DB_MANAGEMENT_HOST=localhost +DB_MANAGEMENT_DATABASE=poweron_management +DB_MANAGEMENT_USER=poweron_dev +DB_MANAGEMENT_PASSWORD_SECRET = DEV_ENC:Z0FBQUFBQm8xSUpEUldqSTVpUnFqdGhITDYzT3RScGlMYVdTMmZhOXdudDRCc3dhdllOd3l6MS1vWHY2MjVsTUF1Sk9saEJOSk9ONUlBZjQwb2c2T1gtWWJhcXFzVVVXd01xc0U0b0lJX0JyVDRxaDhNS01JcWs9 +DB_MANAGEMENT_PORT=5432 + +# Security Configuration +APP_JWT_KEY_SECRET = DEV_ENC:Z0FBQUFBQm8xSUpERjlrSktmZHVuQnJ1VVJDdndLaUcxZGJsT2ZlUFRlcFdOZ001RnlzM2FhLWhRV2tjWWFhaWQwQ3hkcUFvbThMcndxSjFpYTdfRV9OZGhTcksxbXFTZWg5MDZvOHpCVXBHcDJYaHlJM0tyNWRZckZsVHpQcmxTZHJoZUs1M3lfU2ljRnJaTmNSQ0w0X085OXI0QW80M2xfQnJqZmZ6VEh3TUltX0xzeE42SGtZPQ== +APP_TOKEN_EXPIRY=300 + +# CORS Configuration +APP_ALLOWED_ORIGINS=http://localhost:8080,https://playground.poweron-center.net + +# Logging configuration +APP_LOGGING_LOG_LEVEL = DEBUG +APP_LOGGING_LOG_DIR = D:/Athi/Local/Web/poweron/local/logs +APP_LOGGING_FORMAT = %(asctime)s - %(levelname)s - %(name)s - %(message)s +APP_LOGGING_DATE_FORMAT = %Y-%m-%d %H:%M:%S +APP_LOGGING_CONSOLE_ENABLED = True +APP_LOGGING_FILE_ENABLED = True +APP_LOGGING_ROTATION_SIZE = 10485760 +APP_LOGGING_BACKUP_COUNT = 5 + +# Service Redirects +Service_MSFT_REDIRECT_URI = http://localhost:8000/api/msft/auth/callback +Service_GOOGLE_REDIRECT_URI = http://localhost:8000/api/google/auth/callback + +# OpenAI configuration +Connector_AiOpenai_API_URL = https://api.openai.com/v1/chat/completions +Connector_AiOpenai_API_SECRET = DEV_ENC:Z0FBQUFBQm8xSUpEajBuZmtYTVdqLTBpQm9KZ2pCXzRCV3VhZzlYTEhKb1FqWXNrV3lyb25uZUN1WVVQUEY3dGYtejludV9MNGlKeVREanZGOGloV09mY2ttQ3k5SjBFOGFac2ZQTkNKNUZWVnRINVQyeWhsR2wyYnVrRDNzV2NqSHB0ajQ4UWtGeGZtbmR0Q3VvS0hDZlphVmpSc2Z6RG5nPT0= +Connector_AiOpenai_MODEL_NAME = gpt-4o +Connector_AiOpenai_TEMPERATURE = 0.2 +Connector_AiOpenai_MAX_TOKENS = 2000 + +# Anthropic configuration +Connector_AiAnthropic_API_URL = https://api.anthropic.com/v1/messages +Connector_AiAnthropic_API_SECRET = DEV_ENC:Z0FBQUFBQm8xSUpENmFBWG16STFQUVZxNzZZRzRLYTA4X3lRanF1VkF4cU45OExNMzlsQmdISGFxTUxud1dXODBKcFhMVG9KNjdWVnlTTFFROVc3NDlsdlNHLUJXeG41NDBHaXhHR0VHVWl5UW9RNkVWbmlhakRKVW5pM0R4VHk0LUw0TV9LdkljNHdBLXJua21NQkl2b3l4UkVkMGN1YjBrMmJEeWtMay1jbmxrYWJNbUV0aktCXzU1djR2d2RSQXZORTNwcG92ZUVvVGMtQzQzTTVncEZTRGRtZUFIZWQ0dz09 +Connector_AiAnthropic_MODEL_NAME = claude-3-5-sonnet-20241022 +Connector_AiAnthropic_TEMPERATURE = 0.2 +Connector_AiAnthropic_MAX_TOKENS = 2000 + +# Perplexity AI configuration +Connector_AiPerplexity_API_URL = https://api.perplexity.ai/chat/completions +Connector_AiPerplexity_API_SECRET = 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 +Connector_AiPerplexity_MODEL_NAME = sonar +Connector_AiPerplexity_TEMPERATURE = 0.2 +Connector_AiPerplexity_MAX_TOKENS = 2000 + +# Agent Mail configuration +Service_MSFT_CLIENT_ID = c7e7112d-61dc-4f3a-8cd3-08cc4cd7504c +Service_MSFT_CLIENT_SECRET = DEV_ENC:Z0FBQUFBQm8xSUpEQk4xYnpmbnItUEU3dHU4eHB5dzVYay1WT012RTRLUWJDTlBILVY5dC1FX3VMNjZmLThrbDRFNWFSNGprY3RRTlpYNGlubVBpNnY3MjNJcGtzVk9PMzRacl9LUlM2RU5vTVVZWHJvaUhWSHVfc1pNR0pfQmI5SEprOG5KdlB1QnQ= +Service_MSFT_TENANT_ID = common + +# Google Service configuration +Service_GOOGLE_CLIENT_ID = 354925410565-aqs2b2qaiqmm73qpjnel6al8eid78uvg.apps.googleusercontent.com +Service_GOOGLE_CLIENT_SECRET = DEV_ENC:Z0FBQUFBQm8xSUpETDJhbGVQMHlFQzNPVFI1ZzBMa3pNMGlQUHhaQm10eVl1bFlSeTBybzlTOWE2MURXQ0hkRlo0NlNGbHQxWEl1OVkxQnVKYlhhOXR1cUF4T3k0WDdscktkY1oyYllRTmdDTWpfbUdwWGtSd1JvNlYxeTBJdEtaaS1vYnItcW0yaFM= + +# Tavily Web Search configuration +Connector_WebTavily_API_KEY_SECRET = DEV_ENC:Z0FBQUFBQm8xSUpEQTdnUHMwd2pIaXNtMmtCTFREd0pyQXRKb1F5eGtHSnkyOGZiUnlBOFc0b3Vzcndrc3ViRm1nMDJIOEZKYWxqdWNkZGh5N0Z4R0JlQmxXSG5pVnJUR2VYckZhMWNMZ1FNeXJ3enJLVlpiblhOZTNleUg3ZzZyUzRZanFSeDlVMkI= + +# Google Cloud Speech Services configuration +Connector_GoogleSpeech_API_KEY_SECRET = DEV_ENC:Z0FBQUFBQm8xSUpETk5FWWM3Q0JKMzhIYTlyMkhuNjA4NlF4dk82U2NScHhTVGY3UG83NkhfX3RrcWVtWWcyLXRjU1dTT21zWEl6YWRMMUFndXpsUnJOeHh3QThsNDZKRXROTzdXRUdsT0JZajZJNVlfb0gtMXkwWm9DOERPVnpjU0pyUEZfOGJsUnprT3ltMVVhalUyUm9hMUFtZEtHUnJqOGZ4dEZjZm5SWVVTckVCWnY1UkdVSHVmUlgwbnAyc0xDQW84R3ViSko5OHVCVWZRUVNiaG1pVFB6X3EwS0FPd2dUYjhiSmRjcXh2WEZiXzI4SFZqT21tbDduUWRyVWdFZXpmcVM5ZDR0VWtzZnF5UER6cGwwS2JlLV9CSTZ0Z0IyQ1h0YW9TcmhRTXZEckp4bWhmTkt6UTNYMk4zVkpnbUJmaDIxZnoyR2dWTEYwTUFEV0w2eUdUUGpoZk9XRkt4RVF1Z1NPdUpBeTcyWV9PY1Ffd2s0ZEdVekxGekhoeEl4TmNqaXYtbUJuSVdycFducERWdWtZajZnX011Q2w4eE9VMTBqQ1ZxRmdScWhXY1E3WWhzX1JZcHhxam9FbDVPN3Q1MWtrMUZuTUg3LVFQVHp1T1hpQWNDMzEzekVJWk9ybl91YUVjSkFob1VaMi1ONEtuMnRSOEg1S3QybUMwbVZDejItajBLTjM2Zy1hNzZQMW5LLVVDVGdFWm5BZUxNeEFnUkZzU3dxV0lCUlc0LWo4b05GczVpOGZSV2ZxbFBwUml6OU5tYjdnTks3Y3hrVEZVTHlmc1NPdFh4WE5pWldEZklOQUxBbjBpMTlkX3FFQVJ6c2NSZGdzTThycE92VW82enZKamhiRGFnU25aZGlHZHhZd2lUUmhuTVptNjhoWVlJQkxIOEkzbzJNMjZCZFJyM25tdXBnQ2ZWaHV3b2p6UWJpdk9xUEhBc1dyTlNmeF9wbm5yYUhHV01UZnVXWDFlNzBkdXlWUWhvcmJpSmljbmE3LUpUZEg4VzRwZ2JVSjdYUm1sODViQXVxUzdGTmZFbVpiN2V1YW5XV3U4b2VRWmxldGVGVHZsSldoekhVLU9wZ2V0cGZIYkNqM2pXVGctQVAyUm4xTHhpd1VVLXFhcnVEV21Rby1hbTlqTl84TjVveHdYTExUVkhHQ0ltaTB2WXJnY1NQVE5PbWg3ejgySElYc1JSTlQ3NDlFUWR6STZVUjVqaXFRN200NF9LY1ljQ0R2UldlWUtKY1NQVnJ4QXRyYTBGSWVuenhyM0Z0cWtndTd1eG8xRzY5a2dNZ1hkQm5MV3BHVzA2N1QwUkd6WlRGYTZQOUhnVWQ2S0Y5U0s1dXFNVXh5Q2pLWVUxSUQ2MlR1ak52NmRIZ2hlYTk1SGZGWS1RV3hWVU9rR3d1Rk9MLS11REZXbzhqMHpsSm1HYW1jMUNLT29YOHZsRWNaLTVvOFpmT3l3MHVwaERTT0dNLWFjcGRYZ25qT2szTkVFUnRFR3JWYS1aNXFIRnMyalozTlQzNFF2NXJLVHVPVF9zdTF6ZjlkbzJ4RFc2ZENmNFFxZDZzTzhfMUl0bW96V0lPZkh1dXFYZlEteFBlSG84Si1FNS1TTi1OMkFnX2pOYW8xY3MxMVJnVC02MDUyaXZfMEVHWDQtVlRpcENmV0h3V0dCWEFRS2prQXdNRlQ5dnRFVHU0Q1dNTmh0SlBCaU55bFMydWM1TTFFLW96ODBnV3dNZHFZTWZhRURYSHlrdzF3RlRuWDBoQUhSOUJWemtRM3pxcDJFbGJoaTJ3ZktRTlJxbXltaHBoZXVJVDlxS3cxNWo2c0ZBV0NzaUstRWdsMW1xLXFkanZGYUFiU0tSLXFQa0tkcDFoMV9kak41ZjQ0R214UmtOR1ZBanRuemY3Mmw1SkZ5aDZodGIzT3N2aV85MW9kcld6c0g0ZDgtTWo3b3Y3VjJCRnR2U2tMVm9rUXNVRnVHbzZXVTZ6RmI2RkNmajBfMWVnODVFbnpkT0oyci15czJHU0p1cUowTGZJMzVnd3hIRjQyTVhKOGRkcFRKdVpyQ3Yzd01Jb1lSajFmV0paeEV0cjk1SmpmdWpDVFJMUmMtUFctOGhaTmlKQXNRVlVUNlhJemxudHZCR056SVlBb3NOTEYxRTRLaFlVd2d3TWtxVlB6ZEtQLTkxOGMyY3N0a2pYRFUweDBNaGhja2xSSklPOUZla1dKTWRNbG8tUGdSNEV5cW90OWlOZFlIUExBd3U2b2hyS1owbXVMM3p0Qm41cUtzWUxYNzB1N3JpUTNBSGdsT0NuamNTb1lIbXR4MG1sakNPVkxBUXRLVE1xX0YxWDhOcERIY1lTQVFqS01CaXZKNllFaXlIR0JsM1pKMmV1OUo3TGI1WkRaVnYxUTl1LTM0SU1qN1V1b0RCT0x0VHNLTmNLZnk1S0MxYnBBcm03WnVua0xqaEhGUzhOU253ZkppRzdudXBSVlMxeFVOSWxtZ1o2RVBSQUhEUEFuQ1hxSVZMME4yWUtaU3VyRGo3RkUyRUNjT0pNcE1BdE1ZRzdXVl8ydUtXZjdMdHdEVW4teHUtTi1HSGliLUxud21TX0NtcGVkRFBHNkZ1WTlNczR4OUJfUVluc1BoV09oWS1scUdsNnB5d1U5M1huX3k4QzAyNldtb2hybktYN2xKZ1NTNWFsaWwzV3pCRVhkaGR5eTNlV1d6ZzFfaFZTT0E4UjRpQ3pKdEZxUlJ6UFZXM3laUndyWEk2NlBXLUpoajVhZzVwQXpWVzUtVjVNZFBwdWdQa3AxZC1KdGdqNnhibjN4dmFYb2cxcEVwc1g5R09zRUdINUZtOE5QRjVUU0dpZy1QVl9odnFtVDNuWFZLSURtMXlSMlhRNTBWSVFJbEdOOWpfVWV0SmdRWDdlUXZZWE8xRUxDN1I0aEN6MHYwNzM1cmpJS0ZpMnBYWkxfb3FsbEV1VnlqWGxqdVJ6SHlwSjAzRlMycTBaQ295NXNnZERpUnJQcjhrUUd3bkI4bDVzRmxQblhkaFJPTTdISnVUQmhET3BOMTM4bjVvUEc2VmZhb2lrR1FyTUl2RWNEeGg0U0dsNnV6eU5zOUxiNDY5SXBxR0hBS00wOTgyWTFnWkQyaEtLVUloT3ZxZGh0RWVGRmJzenFsaUtfZENQM0JzdkVVeTdXR3hUSmJST1NBMUI1NkVFWncwNW5JZVVLX1p1RXdqVnFfQWpvQ08yQjZhN1NkTkpTSnUxOVRXZXE0WFEtZWxhZW1NNXYtQ2sya0VGLURmS01lMkctNVY3c2ZhN0ZGRFgwWHlabTFkeS1hcUZ1dDZ3cnpPQ3hha2IzVE11M0pqbklmU0diczBqTFBNZC1QZGp6VzNTSnJVSjJoWkJUQjVORG4tYUJmMEJtSUNUdVpEaGt6OTM3TjFOdVhXUHItZjRtZ25nU3NhZC1sVTVXNTRDTmxZbnlfeHNsdkpuMXhUYnE1MnpVQ0ZOclRWM1M4eHdXTzRXbFRZZVQtTS1iRVdXVWZMSGotcWg3MUxUYTFnSEEtanBCRHlZRUNIdGdpUFhsYjdYUndCZnRITzhMZVJ1dHFoVlVNb0duVjlxd0U4OGRuQVV3MG90R0hiYW5MWkxWVklzbWFRNzBfSUNrdzc5bVdtTXg0dExEYnRCaDI3c1I4TWFwLXZKR0wxSjRZYjZIV3ZqZjNqTWhFT0RGSDVMc1A1UzY2bDBiMGFSUy1fNVRQRzRJWDVydUpqb1ZfSHNVbldVeUN2YlAxSW5WVDdxVzJ1WHpLeUdmb0xWMDNHN05oQzY3YnhvUUdhS2xaOHNidkVvbTZtSHFlblhOYmwyR3NQdVJDRUdxREhWdF9ZcXhwUWxHc2hyLW5vUGhIUVhJNUNhY0hFU0ptVnI0TFVhZDE1TFBBUEstSkRoZWJ5MHJhUmZrR1ZrRlFtRGpxS1pOMmFMQjBsdjluY3FiYUU4eGJVVXlZVEpuNWdHVVhJMGtwaTdZR2NDbXd2eHpOQ09SeTV6N1BaVUpsR1pQVDBZcElJUUt6VnVpQmxSYnE4Y1BCWV9IRWdVV0p3enBGVHItdnBGN3NyNWFBWmkySnByWThsbDliSlExQmp3LVlBaDIyZXp6UnR6cU9rTzJmTDBlSVpON0tiWllMdm1oME1zTFl2S2ZYYllhQlY2VHNZRGtHUDY4U1lIVExLZTU4VzZxSTZrZHl1ZTBDc0g4SjI4WGYyZHV1bm9wQ3R2Z09ld1ZmUkN5alJGeHZKSHl1bWhQVXpNMzdjblpLcUhfSm02Qlh5S1FVN3lIcHl0NnlRPT0= + +# Feature SyncDelta JIRA configuration +Feature_SyncDelta_JIRA_DELTA_TOKEN_SECRET = DEV_ENC:Z0FBQUFBQm8xSUpEbm0yRUJ6VUJKbUwyRW5kMnRaNW4wM2YxMkJUTXVXZUdmdVRCaUZIVHU2TTV2RWZLRmUtZkcwZE4yRUNlNDQ0aUJWYjNfdVg5YjV5c2JwMHhoUUYxZWdkeS11bXR0eGxRLWRVaVU3cUVQZWJlNDRtY1lWUDdqeDVFSlpXS0VFX21WajlRS3lHQjc0bS11akkybWV3QUFlR2hNWUNYLUdiRjZuN2dQODdDSExXWG1Dd2ZGclI2aUhlSWhETVZuY3hYdnhkb2c2LU1JTFBvWFpTNmZtMkNVOTZTejJwbDI2eGE0OS1xUlIwQnlCSmFxRFNCeVJNVzlOMDhTR1VUamx4RDRyV3p6Tk9qVHBrWWdySUM3TVRaYjd3N0JHMFhpdzFhZTNDLTFkRVQ2RVE4U19COXRhRWtNc0NVOHRqUS1CRDFpZ19xQmtFLU9YSDU3TXBZQXpVcld3PT0= diff --git a/env_dev.env b/env_dev.env index 9ebbb93b..44089293 100644 --- a/env_dev.env +++ b/env_dev.env @@ -66,7 +66,7 @@ Connector_AiAnthropic_MAX_TOKENS = 2000 # Perplexity AI configuration Connector_AiPerplexity_API_URL = https://api.perplexity.ai/chat/completions -Connector_AiPerplexity_API_SECRET = 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 +Connector_AiPerplexity_API_SECRET = DEV_ENC:Z0FBQUFBQm82Mzk2Q1MwZ0dNcUVBcUtuRDJIcTZkMXVvYnpjM3JEMzJiT1NKSHljX282ZDIyZTJYc09VSTdVNXAtOWU2UXp5S193NTk5dHJsWlFjRjhWektFOG1DVGY4ZUhHTXMzS0RPN1lNcF9nSlVWbW5BZ1hkZDVTejl6bVZNRFVvX29xamJidWRFMmtjQmkyRUQ2RUh6UTN1aWNPSUJBPT0= Connector_AiPerplexity_MODEL_NAME = sonar Connector_AiPerplexity_TEMPERATURE = 0.2 Connector_AiPerplexity_MAX_TOKENS = 2000 @@ -88,3 +88,7 @@ Connector_GoogleSpeech_API_KEY_SECRET = DEV_ENC:Z0FBQUFBQm8xSUpETk5FWWM3Q0JKMzhI # Feature SyncDelta JIRA configuration Feature_SyncDelta_JIRA_DELTA_TOKEN_SECRET = DEV_ENC:Z0FBQUFBQm8xSUpEbm0yRUJ6VUJKbUwyRW5kMnRaNW4wM2YxMkJUTXVXZUdmdVRCaUZIVHU2TTV2RWZLRmUtZkcwZE4yRUNlNDQ0aUJWYjNfdVg5YjV5c2JwMHhoUUYxZWdkeS11bXR0eGxRLWRVaVU3cUVQZWJlNDRtY1lWUDdqeDVFSlpXS0VFX21WajlRS3lHQjc0bS11akkybWV3QUFlR2hNWUNYLUdiRjZuN2dQODdDSExXWG1Dd2ZGclI2aUhlSWhETVZuY3hYdnhkb2c2LU1JTFBvWFpTNmZtMkNVOTZTejJwbDI2eGE0OS1xUlIwQnlCSmFxRFNCeVJNVzlOMDhTR1VUamx4RDRyV3p6Tk9qVHBrWWdySUM3TVRaYjd3N0JHMFhpdzFhZTNDLTFkRVQ2RVE4U19COXRhRWtNc0NVOHRqUS1CRDFpZ19xQmtFLU9YSDU3TXBZQXpVcld3PT0= + +# Debug Configuration +APP_DEBUG_CHAT_WORKFLOW_ENABLED = True +APP_DEBUG_CHAT_WORKFLOW_DIR = ./test-chat \ No newline at end of file diff --git a/env_int.20251012_121418.backup b/env_int.20251012_121418.backup new file mode 100644 index 00000000..4a0f3e39 --- /dev/null +++ b/env_int.20251012_121418.backup @@ -0,0 +1,90 @@ +# Integration Environment Configuration + +# System Configuration +APP_ENV_TYPE = int +APP_ENV_LABEL = Integration Instance +APP_API_URL = https://gateway-int.poweron-center.net +APP_KEY_SYSVAR = CONFIG_KEY +APP_INIT_PASS_ADMIN_SECRET = INT_ENC:Z0FBQUFBQm8xSVRjWm41MWZ4TUZGaVlrX3pWZWNwakJsY3Facm0wLVZDd1VKeTFoZEVZQnItcEdUUnVJS1NXeDBpM2xKbGRsYmxOSmRhc29PZjJSU2txQjdLbUVrTTE1NEJjUXBHbV9NOVJWZUR3QlJkQnJvTEU9 +APP_INIT_PASS_EVENT_SECRET = INT_ENC:Z0FBQUFBQm8xSVRjdmtrakgxa0djekZVNGtTZV8wM2I5UUpCZllveVBMWXROYk5yS3BiV3JEelJSM09VYTRONHpnY3VtMGxDRk5JTEZSRFhtcDZ0RVRmZ1RicTFhb3c5dVZRQ1o4SmlkLVpPTW5MMTU2eTQ0Vkk9 + +# PostgreSQL Storage (new) +DB_APP_HOST=gateway-int-server.postgres.database.azure.com +DB_APP_DATABASE=poweron_app +DB_APP_USER=heeshkdlby +DB_APP_PASSWORD_SECRET = INT_ENC:Z0FBQUFBQm8xSVRjb2dka2pnN0tUbW1EU0w1Rk1jNERKQ0Z1U3JkVDhuZWZDM0g5M0kwVDE5VHdubkZna3gtZVAxTnl4MDdrR1c1ZXJ3ejJHYkZvcGUwbHJaajBGOWJob0EzRXVHc0JnZkJyNGhHZTZHOXBxd2c9 +DB_APP_PORT=5432 + +# PostgreSQL Storage (new) +DB_CHAT_HOST=gateway-int-server.postgres.database.azure.com +DB_CHAT_DATABASE=poweron_chat +DB_CHAT_USER=heeshkdlby +DB_CHAT_PASSWORD_SECRET = INT_ENC:Z0FBQUFBQm8xSVRjczYzOUtTa21MMGJVTUQ5UmFfdWc3YlhCbWZOeXFaNEE1QzdJV3BLVjhnalBkLVVCMm5BZzdxdlFXQXc2RHYzLWtPSFZkZE1iWG9rQ1NkVWlpRnF5TURVbnl1cm9iYXlSMGYxd1BGYVc0VDA9 +DB_CHAT_PORT=5432 + +# PostgreSQL Storage (new) +DB_MANAGEMENT_HOST=gateway-int-server.postgres.database.azure.com +DB_MANAGEMENT_DATABASE=poweron_management +DB_MANAGEMENT_USER=heeshkdlby +DB_MANAGEMENT_PASSWORD_SECRET = INT_ENC:Z0FBQUFBQm8xSVRjTnJKNlJMNmEwQ0Y5dVNrR3pkZk9SQXVvLTRTNW9lQ1g3TTE5cFhBNTd5UENqWW9qdWd3NWNseWhnUHJveDJyd1Z3X1czS3VuZnAwZHBXYVNQWlZsRy12ME42NndEVlR5X3ZPdFBNNmhLYm89 +DB_MANAGEMENT_PORT=5432 + +# Security Configuration +APP_JWT_KEY_SECRET = INT_ENC:Z0FBQUFBQm8xSVRjNUctb2RwU25iR3ZnanBOdHZhWUtIajZ1RnZzTEp4aDR0MktWRjNoeVBrY1Npd1R0VE9YVHp3M2w1cXRzbUxNaU82QUJvaDNFeVQyN05KblRWblBvbWtoT0VXbkNBbDQ5OHhwSUFnaDZGRG10Vmgtdm1YUkRsYUhFMzRVZURmSFlDTFIzVWg4MXNueDZyMGc5aVpFdWRxY3dkTExGM093ZTVUZVl5LUhGWnlRPQ== +APP_TOKEN_EXPIRY=300 + +# CORS Configuration +APP_ALLOWED_ORIGINS=http://localhost:8080,https://playground.poweron-center.net,https://playground-int.poweron-center.net,http://localhost:5176,https://nyla.poweron-center.net, https://nyla-int.poweron-center.net + +# Logging configuration +APP_LOGGING_LOG_LEVEL = DEBUG +APP_LOGGING_LOG_DIR = /home/site/wwwroot/ +APP_LOGGING_FORMAT = %(asctime)s - %(levelname)s - %(name)s - %(message)s +APP_LOGGING_DATE_FORMAT = %Y-%m-%d %H:%M:%S +APP_LOGGING_CONSOLE_ENABLED = True +APP_LOGGING_FILE_ENABLED = True +APP_LOGGING_ROTATION_SIZE = 10485760 +APP_LOGGING_BACKUP_COUNT = 5 + +# Service Redirects +Service_MSFT_REDIRECT_URI = https://gateway-int.poweron-center.net/api/msft/auth/callback +Service_GOOGLE_REDIRECT_URI = https://gateway-int.poweron-center.net/api/google/auth/callback + +# OpenAI configuration +Connector_AiOpenai_API_URL = https://api.openai.com/v1/chat/completions +Connector_AiOpenai_API_SECRET = INT_ENC:Z0FBQUFBQm8xSVRjSDBNYkptSkQxTUotYVVpZVNZc0dxNGNwSEtkOEE0T3RZWjROTEhSRlRXdlZmQUxxZ0w3Y0xOV2JNV19LNF9yTUZiU1pUNG15U2VDUDdSVlI4VlpnR3JXVFFtcXBaTEZiaUtSclVFd0lCZG1rWVhra1dfWTVQOTBEYUU0MjByYVNEMTFmeXNOcmpUT216MmJKdlVPeW5nPT0= +Connector_AiOpenai_MODEL_NAME = gpt-4o +Connector_AiOpenai_TEMPERATURE = 0.2 +Connector_AiOpenai_MAX_TOKENS = 2000 + +# Anthropic configuration +Connector_AiAnthropic_API_URL = https://api.anthropic.com/v1/messages +Connector_AiAnthropic_API_SECRET = INT_ENC:Z0FBQUFBQm8xSVRjT1ZlRWVJdVZMT3ljSFJDcFdxRFBRVkZhS204NnN5RDBlQ0tpenhTM0FFVktuWW9mWHNwRWx2dHB0eDBSZ0JFQnZKWlp6c01pVGREWHd1eGpERnU0Q2xhaks1clQ1ZXVsdnd2ZzhpNXNQS1BhY3FjSkdkVEhHalNaRGR4emhpakZncnpDQUVxOHVXQzVUWmtQc0FsYmFwTF9TSG5FOUFtWk5Ick1NcHFvY2s1T1c2WXlRUFFJZnh6TWhuaVpMYmppcDR0QUx0a0R6RXlwbGRYb1R4dzJkUT09 +Connector_AiAnthropic_MODEL_NAME = claude-3-5-sonnet-20241022 +Connector_AiAnthropic_TEMPERATURE = 0.2 +Connector_AiAnthropic_MAX_TOKENS = 2000 + +# Perplexity AI configuration +Connector_AiPerplexity_API_URL = https://api.perplexity.ai/chat/completions +Connector_AiPerplexity_API_SECRET = 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 +Connector_AiPerplexity_MODEL_NAME = sonar +Connector_AiPerplexity_TEMPERATURE = 0.2 +Connector_AiPerplexity_MAX_TOKENS = 2000 + +# Agent Mail configuration +Service_MSFT_CLIENT_ID = c7e7112d-61dc-4f3a-8cd3-08cc4cd7504c +Service_MSFT_CLIENT_SECRET = INT_ENC:Z0FBQUFBQm8xSVRjNzB2M3ZjaE1SVE9ON2FKam9yVURxcHl1Ym5VNVUtS0MyWUpNVXVlaWpWS2U3VVd3em9vQl9lcnVYay03bS04YjNBbDZZNTB4eUtjT3ppQjJjY3dOT0FNLW9LeDhIUU5iaTNqNURUWE5La3kzaHNGcU9yNVI0YjhWZTZRRFktcTk= +Service_MSFT_TENANT_ID = common + +# Google Service configuration +Service_GOOGLE_CLIENT_ID = 354925410565-aqs2b2qaiqmm73qpjnel6al8eid78uvg.apps.googleusercontent.com +Service_GOOGLE_CLIENT_SECRET = INT_ENC:Z0FBQUFBQm8xSVRjNThGeVRNd3hacThtRnE0bzlDa0JPUWQyaEd6QjlFckdsMGZjRlRfUks2bXV3aDdVRTF3LVRlZVY5WjVzSXV4ZGNnX002RDl3dkNYdGFzZkxVUW01My1wTHRCanVCLUozZEx4TlduQlB5MnpvNTR2SGlvbFl1YkhzTEtsSi1SOEo= + +# Tavily Web Search configuration +Connector_WebTavily_API_KEY_SECRET = INT_ENC:Z0FBQUFBQm8xSVRkdkJMTDY0akhXNzZDWHVYSEt1cDZoOWEzSktneHZEV2JndTNmWlNSMV9KbFNIZmQzeVlrNE5qUEIwcUlBSGM1a0hOZ3J6djIyOVhnZzI3M1dIUkdicl9FVXF3RGktMmlEYmhnaHJfWTdGUkktSXVUSGdQMC1vSEV6VE8zR2F1SVk= + +# Google Cloud Speech Services configuration +Connector_GoogleSpeech_API_KEY_SECRET = INT_ENC:Z0FBQUFBQm8xSVRkNmVXZ1pWcHcydTF2MXF0ZGJoWHBydF85bTczTktiaEJ3Wk1vMW1mZVhDSG1yd0ZxR2ZuSGJTX0N3MWptWXFJTkNTWjh1SUVVTXI4UDVzcGdLMkU5SHJ2TUpkRlRoRWdnSldtYjNTQkh4UDJHY2xmdTdZQ1ZiMTZZcGZxS3RzaHdjV3dtVkZUcEpJcWx0b2xuQVR6ZmpoVFZPY1hNMTV2SnhDaC1IZEh4UUpLTy1ILXA4RG1zamJTbUJ4X0t2M2NkdzJPbEJxSmFpRzV3WC0wZThoVzlxcmpHZ3ZkLVlVY3REZk1vV19WQ05BOWN6cnJ4MWNYYnNiQ0FQSUVnUlpfM3BhMnlsVlZUOG5wM3pzM1lSN1UzWlZKUXRLczlHbjI1LTFvSUJ4SlVXMy1BNk43bE5Hb0RfTTVlWk9oZnFIaVg0SW5pbm9EcXRTTzU1RFlYY3dTcnpKWWNyNjN5T1BGZ0FmX253cEFncmhvZVRuM05KYzhkOEhFMFJsc2NBSEwzZVZ1R0JMOGxsekVwUE55alZaRXFrdzNWWVNGWXNmbnhKeWhQSFo2VXBTUlRPeHdvdVdncEFuOWgydEtsSUFneUN6cGVaTnBSdjNCdVJseGJFdmlMc203UFhLVlYyTENkaGg2dVN6Z2xwT1ZmTmN5bVZGUkM3ZWcyVkt2ckFUVVd3WFFwYnJjNVRobEh2SkVJbXRwUUpEOFJKQ1NUc0Q4NHNqUFhPSDh5cTV6MEcwSDEwRUJCQ2JiTTJlOE5nd3pMMkJaQ1dVYjMwZVVWWnlETmp2dkZ3aXEtQ29WNkxZTFkzYUkxdTlQUU1OTnhWWU12YU9MVnJQa1d2ZjRtUlhneTNubEMxTmp1eUNPOThSMlB3Y1F0T2tCdFNsNFlKalZPV25yR2QycVBUb096RmZ1V0FTaGsxLV9FWDBmenBIOXpMdGpLcUc0TWRoY2hlMFhYTzlET1ZRekw0ZHNwUVBQdVJBX2h6Q2ZzWVZJWTNybTJiekp3WmhmWF9SUFBXQzlqUjctcVlHWWVMZWVQallzR0JGTVF0WmtnWlg1aTM1bFprNVExZXY5dnNvWF93UjhwbkJ3RzNXaVJ2d2RRU3JJVlBvaVh4eTlBRUtqWkJia3dJQVVBV2Nqdm9FUTRUVW1TaHp2ZUwxT0N2ZndxQ2Nka1RYWXF0LWxIWFE0dTFQcVhncFFPM0hFdUUtYlFnemx3WkF4bjA1aDFULUdrZlVZbEJtRGRCdjJyVkdJSXozd0I0dF9zbWhOeHFqRDA4T1NVaWR5cjBwSVgwbllPU294NjZGTnM1bFhIdGpNQUxFOENWd3FCbGpSRFRmRXotQnU0N2lCVEU5RGF6Qi10S2U2NGdadDlrRjZtVE5oZkw5ZWFjXzhCTmxXQzNFTFgxRXVYY3J3YkxnbnlBSm9PY3h4MlM1NVFQbVNDRW5Ld1dvNWMxSmdoTXJuaE1pT2VFeXYwWXBHZ29MZDVlN2lwUUNIeGNCVVdQVi1rRXdJMWFncUlPTXR0MmZVQ1l0d09mZTdzWGFBWUJMUFd3b0RSOU8zeER2UWpNdzAxS0ZJWnB5S3FJdU9wUDJnTTNwMWw3VFVqVXQ3ZGZnU1RkUktkc0NhUHJ0SGFxZ0lVWDEzYjNtU2JfMGNWM1Y0dHlCTzNESEdENC1jUWF5MVppRzR1QlBNSUJySjFfRi1ENHEwcmJ4S3hQUFpXVHA0TG9DZWdoUlo5WnNSM1lCZm1KbEs2ak1yUUU4Wk9JcVJGUkJwc0NvUkMyTjhoTWxtZmVQeDREZVRKZkhYN2duLVNTeGZzdFdBVnhEandJSXB5QjM0azF0ckI3Tk1wSzFhNGVOUVRrNjU0cG9JQ29pN09xOFkwR1lMTlktaGp4TktxdTVtTnNEcldsV2pEZm5nQWpJc2hxY0hjQnVSWUR5VVdaUXBHWUloTzFZUC1oNzJ4UjZ1dnpLcDJxWEZtQlNIMWkzZ0hXWXdKeC1iLXdZWVJhcU04VFlpMU5pd2ZIdTdCdkVWVFVBdmJuRk16bEFFQTh4alBrcTV2RzliT2hGdTVPOXlRMjFuZktiRTZIamQ1VFVqS0hRTXhxcU1mdkgyQ1NjQmZfcjl4c3NJd0RIeDVMZUFBbHJqdEJxWWl3aWdGUEQxR3ZnMkNGdVB4RUxkZi1xOVlFQXh1NjRfbkFEaEJ5TVZlUGFrWVhSTVRPeGxqNlJDTHNsRWRrei1pYjhnUmZrb3BvWkQ2QXBzYjFHNXZoWU1LSExhLWtlYlJTZlJmYUM5Y1Rhb1pkMVYyWTByM3NTS0VXMG1ybm1BTVN2QXRYaXZqX2dKSkZrajZSS2cyVlNOQnd5Y29zMlVyaWlNbTJEb3FuUFFtbWNTNVpZTktUenFZSl91cVFXZjRkQUZyYmtPczU2S1RKQ19ONGFOTHlwX2hOOEE1UHZEVjhnT0xxRjMxTEE4SHhRbmlmTkZwVXJBdlJDbU5oZS05SzI4QVhEWDZaN2ZiSlFwUGRXSnB5TE9MZV9ia3pYcmZVa1dicG5FMHRXUFZXMWJQVDAwOEdDQzJmZEl0ZDhUOEFpZXZWWXl5Q2xwSmFienNCMldlb2NKb2ZRYV9KbUdHRzNUcjU1VUFhMzk1a2J6dDVuNTl6NTdpM0hGa3k0UWVtbF9pdDVsQVp2cndDLUU5dnNYOF9CLS0ySXhBSFdCSnpqV010bllBb3U0cEZZYVF5R2tSNFM5NlRhdS1fb1NqbDBKMkw0V2N0VEZhNExtQlR3ckZ3cVlCeHVXdXJ6X0s4cEtsaG5rVUxCN2RRbHQxTmcyVFBqYUxyOHJzeFBXVUJaRHpXbUoxdHZzMFBzQk1UTUFvX1pGNFNMNDFvZWdTdEUtMUNKMXNIeVlvQk1CeEdpZVdmN0tsSDVZZHJXSGt5c2o2MHdwSTZIMVBhRzM1eU43Q2FtcVNidExxczNJeUx5U2RuUG5EeHpCTlg2SV9WNk1ET3BRNXFuc0pNWlVvZUYtY21oRGtJSmwxQ09QbHBUV3BuS3B5NE9RVkhfellqZjJUQ0diSV94QlhQWmdaaC1TRWxsMUVWSXB0aE1McFZDZDNwQUVKZ2t5cXRTXzlRZVJwN0pZSnJSV21XMlh0TzFRVEl0c2I4QjBxOGRCYkNxek04a011X1lrb2poQ3h2LUhKTGJiUlhneHp5QWFBcE5nMElkNTVzM3JGOWtUQ19wNVBTaVVHUHFDNFJnNXJaWDNBSkMwbi1WbTdtSnFySkhNQl9ZQjZrR2xDcXhTRExhMmNHcGlyWjR3ZU9SSjRZd1l4ZjVPeHNiYk53SW5SYnZPTzNkd1lnZmFseV9tQ3BxM3lNYVBHT0J0elJnMTByZ3VHemxta0tVQzZZRllmQ2VLZ1ZCNDhUUTc3LWNCZXBMekFwWW1fQkQ1NktzNGFMYUdYTU0xbXprY1FONUNlUHNMY3h2NFJMMmhNa3VNdzF4TVFWQk9odnJUMjFJMVd3Z2N6Sms5aEM2SWlWZFViZ0JWTEpUWWM5NmIzOS1oQmRqdkt1NUUycFlVcUxERUZGbnZqTUxIYnJmMDBHZDEzbnJsWEEzSUo3UmNPUDg1dnRUU1FzcWtjTWZwUG9zM0JTY3RqMDdST2UxcXFTM0d0bGkwdFhnMk5LaUlxNWx3V1pLaVlLUFJXZzBzVl9Ia1V1OHdYUEFWOU50UndycGtCdzM0Q0NQamp2VTNqbFBLaGhsbUk5dUI5MjU5OHVySk1oY0drUWtXUloyVVRvOWJmbUVYRzFVeWNQczh2NXJCeVppRlZiWDNJaDhOSmRmX2lURTNVS3NXQXFZT1QtUmdvMWJoVWYxU3lqUUJhbzEyX3I3TXhwbm9wc1FoQ1ZUTlNBRjMyQTBTY2tzbHZ3RFUtTjVxQ0o1QXRTVks2WENwMGZCRGstNU1jN3FhUFJCQThyaFhhMVRsbnlSRXNGRmt3Yk01X21ldmV3bTItWm1JaGpZQWZROEFtT1d1UUtPQlhYVVFqT2NxLUxQenJHX3JfMEdscDRiMXcyZ1ZmU3NFMzVoelZJaDlvT0ZoRGQ2bmtlM0M5ZHlCd2ZMbnRZRkZUWHVBUEx4czNfTmtMckh5eXZrZFBzOEItOGRYOEhsMzBhZ0xlOWFjZzgteVBsdnpPT1pYdUxnbFNXYnhKaVB6QUxVdUJCOFpvU2x2c1FHZV94MDBOVWJhYkxISkswc0U5UmdPWFJLXzZNYklHTjN1QzRKaldKdEVHb0pOU284N3c2LXZGMGVleEZ5NGZ6OGV1dm1tM0J0aTQ3VFlNOEJrdEh3PT0= + +# Feature SyncDelta JIRA configuration +Feature_SyncDelta_JIRA_DELTA_TOKEN_SECRET = INT_ENC:Z0FBQUFBQm8xSVRkTUNsWm4wX0p6eXFDZmJ4dFdHNEs1MV9MUzdrb3RzeC1jVWVYZ0REWHRyZkFiaGZLcUQtTXFBZzZkNzRmQ0gxbEhGbUNlVVFfR1JEQTc0aldkZkgyWnBOcjdlUlZxR0tDTEdKRExULXAyUEtsVmNTMkRKU1BJNnFiM0hlMXo4YndMcHlRMExtZDQ3Zm9vNFhMcEZCcHpBPT0= diff --git a/env_int.env b/env_int.env index 4a0f3e39..c05249dd 100644 --- a/env_int.env +++ b/env_int.env @@ -66,7 +66,7 @@ Connector_AiAnthropic_MAX_TOKENS = 2000 # Perplexity AI configuration Connector_AiPerplexity_API_URL = https://api.perplexity.ai/chat/completions -Connector_AiPerplexity_API_SECRET = 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 +Connector_AiPerplexity_API_SECRET = INT_ENC:Z0FBQUFBQm82Mzk2UWZJdUFhSW8yc3RKc0tKRXphd0xWMkZOVlFpSGZ4SGhFWnk0cTF5VjlKQVZjdS1QSWdkS0pUSWw4OFU5MjUxdTVQel9aeWVIZTZ5TXRuVmFkZG0zWEdTOGdHMHpsTzI0TGlWYURKU1Q0VVpKTlhxUk5FTmN6SUJScDZ3ZldIaUJZcWpaQVRiSEpyQm9tRTNDWk9KTnZBPT0= Connector_AiPerplexity_MODEL_NAME = sonar Connector_AiPerplexity_TEMPERATURE = 0.2 Connector_AiPerplexity_MAX_TOKENS = 2000 @@ -88,3 +88,7 @@ Connector_GoogleSpeech_API_KEY_SECRET = INT_ENC:Z0FBQUFBQm8xSVRkNmVXZ1pWcHcydTF2 # Feature SyncDelta JIRA configuration Feature_SyncDelta_JIRA_DELTA_TOKEN_SECRET = INT_ENC:Z0FBQUFBQm8xSVRkTUNsWm4wX0p6eXFDZmJ4dFdHNEs1MV9MUzdrb3RzeC1jVWVYZ0REWHRyZkFiaGZLcUQtTXFBZzZkNzRmQ0gxbEhGbUNlVVFfR1JEQTc0aldkZkgyWnBOcjdlUlZxR0tDTEdKRExULXAyUEtsVmNTMkRKU1BJNnFiM0hlMXo4YndMcHlRMExtZDQ3Zm9vNFhMcEZCcHpBPT0= + +# Debug Configuration +APP_DEBUG_CHAT_WORKFLOW_ENABLED = FALSE +APP_DEBUG_CHAT_WORKFLOW_DIR = ./test-chat \ No newline at end of file diff --git a/env_prod.20251012_121418.backup b/env_prod.20251012_121418.backup new file mode 100644 index 00000000..c1ba8086 --- /dev/null +++ b/env_prod.20251012_121418.backup @@ -0,0 +1,90 @@ +# Production Environment Configuration + +# System Configuration +APP_ENV_TYPE = prod +APP_ENV_LABEL = Production Instance +APP_API_URL = https://gateway.poweron-center.net +APP_KEY_SYSVAR = CONFIG_KEY +APP_INIT_PASS_ADMIN_SECRET = PROD_ENC:Z0FBQUFBQm8xSU5pSXoyVEVwNDZ6cmthQTROUkxGUjh1UWF2UU5zaWRuX3p2aHJCVFo2NEstR0RqdnQ5clZmeVliRlhHZGFHTlhZV2dzMmRPZFVEemVlSHd5VHR3cmpNUXRaRlhZSFZ6d1dsX2Y5Zl9lOXdYdEU9 +APP_INIT_PASS_EVENT_SECRET = PROD_ENC:Z0FBQUFBQm8xSU5peGNMWExjWGZxQ2VndXVOSUVGcWhQTWd0N3d0blU3bGJvNjgzNVVNNktCQnZlTEtVckV5RUtQMjMwRTBkdmxEMlZwX0k1M1hlOFFNY3hjaWsyd2JmRGl2UWxfSXEwenVnQ3NmaTlxckp2VXM9 + +# PostgreSQL Storage (new) +DB_APP_HOST=gateway-prod-server.postgres.database.azure.com +DB_APP_DATABASE=poweron_app +DB_APP_USER=gzxxmcrdhn +DB_APP_PASSWORD_SECRET = PROD_ENC:Z0FBQUFBQm8xSU5pVmtwYWZQakdWZnJPamVlRWJPa0tnc3daSVVHejVrQ0x1VFZZbHhVSkk0S2tFWl92T2NwWURBMU9UbFROMHZ2TkNKZFlEWjhJZDZ0bnFndC1oYjhNRW1VLWpEYnlDNEJwcGVKckpUVlp6YTg9 +DB_APP_PORT=5432 + +# PostgreSQL Storage (new) +DB_CHAT_HOST=gateway-prod-server.postgres.database.azure.com +DB_CHAT_DATABASE=poweron_chat +DB_CHAT_USER=gzxxmcrdhn +DB_CHAT_PASSWORD_SECRET = PROD_ENC:Z0FBQUFBQm8xSU5pZVZnTzBPTDY1Q3c2U1pDV0lxbXhoWnlYSXRDWVhIeGJwSkdNMzMxR2h5a1FRN00xcWtYUE4ySGpqRllSaGM5SmRZZk9Bd2trVDJNZDdWcEFIbTJtel91MHpsazlTQnRsV2docGdBc0RVeEU9 +DB_CHAT_PORT=5432 + +# PostgreSQL Storage (new) +DB_MANAGEMENT_HOST=gateway-prod-server.postgres.database.azure.com +DB_MANAGEMENT_DATABASE=poweron_management +DB_MANAGEMENT_USER=gzxxmcrdhn +DB_MANAGEMENT_PASSWORD_SECRET = PROD_ENC:Z0FBQUFBQm8xSU5pQXdaRnVEQUx2MmU5ck9XZzNfaGVoRXlYMlVjSVM5dWNTekhmR2VYNkd6WVhELUlkLWdFWWRWQ1JJLWZ4WUNwclZVRlg3ZHBCS0xwM1laNklTaEs1czFDRTMxYlV2TWNueEJlTHFyNEt4aVk9 +DB_MANAGEMENT_PORT=5432 + +# Security Configuration +APP_JWT_KEY_SECRET = PROD_ENC:Z0FBQUFBQm8xSU5pY3JfX1R3cEJhTjAzZGx2amtRSE4yVzZhMmY3a3FHam9BdzBxVWd5R0FRSW1KbmNGS3JDMktKTWptZm4wYmZZZTVDQkh3NVlxSW1MZEdiVWdORng4dm0xV08wZDh0YlBNQTdEbmlnVWduMzNWY1RPX1BqaGtnOTc2ZWNBTnNnd1AtaTNRUExpRThVdzNmdVFHM2hkTjFjcW0ya2szMWNaT3VDeDhXMlJ1NDM4PQ== +APP_TOKEN_EXPIRY=300 + +# CORS Configuration +APP_ALLOWED_ORIGINS=http://localhost:8080,https://playground.poweron-center.net,https://playground-int.poweron-center.net,http://localhost:5176,https://nyla.poweron-center.net,https://nyla-int.poweron-center.net + +# Logging configuration +APP_LOGGING_LOG_LEVEL = DEBUG +APP_LOGGING_LOG_DIR = /home/site/wwwroot/ +APP_LOGGING_FORMAT = %(asctime)s - %(levelname)s - %(name)s - %(message)s +APP_LOGGING_DATE_FORMAT = %Y-%m-%d %H:%M:%S +APP_LOGGING_CONSOLE_ENABLED = True +APP_LOGGING_FILE_ENABLED = True +APP_LOGGING_ROTATION_SIZE = 10485760 +APP_LOGGING_BACKUP_COUNT = 5 + +# Service Redirects +Service_MSFT_REDIRECT_URI = https://gateway-prod.poweron-center.net/api/msft/auth/callback +Service_GOOGLE_REDIRECT_URI = https://gateway-prod.poweron-center.net/api/google/auth/callback + +# OpenAI configuration +Connector_AiOpenai_API_URL = https://api.openai.com/v1/chat/completions +Connector_AiOpenai_API_SECRET = PROD_ENC:Z0FBQUFBQm8xSU5pU05XM2hMaExPMnpYeFpwRVhyYl9JZmRITmlmRDlWOUJSSWE4NTFLZUptSkJhNlEycHBLZmh3WFA2ZmU5VmxHZks1UUNVOUZnckZNdXZ2MTY2dFg1Nl8yWDRrcTRlT0tHYkhyRGZINTEzU25iYVFRMzJGeUZIdlc4LU9GbmpQYmtmU3lJT2VVZ1UzLVd3R25ZQ092SUVnPT0= +Connector_AiOpenai_MODEL_NAME = gpt-4o +Connector_AiOpenai_TEMPERATURE = 0.2 +Connector_AiOpenai_MAX_TOKENS = 2000 + +# Anthropic configuration +Connector_AiAnthropic_API_URL = https://api.anthropic.com/v1/messages +Connector_AiAnthropic_API_SECRET = PROD_ENC:Z0FBQUFBQm8xSU5pNTA1RkZ3UllCOXVsNVZzbkw2Rkl1TWxCZ0wwWEVXUm9ReUhBcVl1cGFUdW9FRVh4elVxR0x3NVRxZkc4SkxHVFdzSU1YNG5Rb0FqSHJhdElwWm1iLWdubTVDcUl3UkVjVHNoU0xLa0ZTSFlfTlJUVXg4cVVwUWdlVDBTSFU5SnBzS0ZnVjlQcmtiNzV2UTNMck1IakZ0OWlubUtlWDZnMk4yX2JsZ1U4Wm1yT29fM2d2NVBNOWNBbWtTRWNyQ2tZNjhwSVF6bG5SU3dTenR2MzA3Z19NUT09 +Connector_AiAnthropic_MODEL_NAME = claude-3-5-sonnet-20241022 +Connector_AiAnthropic_TEMPERATURE = 0.2 +Connector_AiAnthropic_MAX_TOKENS = 2000 + +# Perplexity AI configuration +Connector_AiPerplexity_API_URL = https://api.perplexity.ai/chat/completions +Connector_AiPerplexity_API_SECRET = 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 +Connector_AiPerplexity_MODEL_NAME = sonar +Connector_AiPerplexity_TEMPERATURE = 0.2 +Connector_AiPerplexity_MAX_TOKENS = 2000 + +# Agent Mail configuration +Service_MSFT_CLIENT_ID = c7e7112d-61dc-4f3a-8cd3-08cc4cd7504c +Service_MSFT_CLIENT_SECRET = PROD_ENC:Z0FBQUFBQm8xSU5pVEhHdlZHU3FNMmhuRGVwaGc3YzIxSjlZNzBCQjlOV2pSYVNXb0t1ZnVwQzZsQzY4cHMtVlZtNF85OEVaV1BMTzdXMmpzaGZpaG1DalJ0bkNPMHA5ZUcwZjNDdGk1TFdxYTJSZnVrVmhhZ2VRUEZxbjJOOGFhWk9EYlY3dmRVTnI= +Service_MSFT_TENANT_ID = common + +# Google Service configuration +Service_GOOGLE_CLIENT_ID = 354925410565-aqs2b2qaiqmm73qpjnel6al8eid78uvg.apps.googleusercontent.com +Service_GOOGLE_CLIENT_SECRET = PROD_ENC:Z0FBQUFBQm8xSU5pV2JEV0lNUXhwa1VTUGh2RWcyYnJHSFQyTmdBOEhwRkJWc3MwOFZlcHJGUmlGOVVFbG1XalNyUXVuaExESy1xeFNIQlRiSFVIWTB6Rm1fNFg0OHZZSkF4ZlBIcFZDMjZHcFRERXJ0WlVFclhHa29Za1BqWGxsM05NZGFRc1BLZnE= + +# Tavily Web Search configuration +Connector_WebTavily_API_KEY_SECRET = PROD_ENC:Z0FBQUFBQm8xSU5pMjhJNS1CZFJubUlkN3ZrTUoxR0Y1QzJFWEJSMk0wQkI0UndqOW1UelVieWhGaTVBcHoxRXo1VjRzVVRROHFIeHMyS3Q5cDZCeUlEMzE1ZlhVTmNveFk5VmFQMm80NTRyVW1TZHVsR3dUN0RtMnd4LW1VWlpqOXJPeXZBTmg4OEM= + +# Google Cloud Speech Services configuration +Connector_GoogleSpeech_API_KEY_SECRET = PROD_ENC:Z0FBQUFBQm8xSU5pNjlJdmFMeERXUUQzR0duRUY4cGRZRzdwQlpnVFAzSzQ5cHZNRnVUZ0xWd3dQMHR3QjVsdF92NmdUQlJGRk1RcG1RYWZzcE9RbEhjQmR5Yk5Ud3ZKTW5jbmpEVGJ2ZkxVeVJpcUxaT2lNREFXaks5WHg5aVlHcXlUZldMdnZGYklHWjlJOWJ6Wm5RSkNmdm5feENjS1E0QUVXTTE5SW5sNFBEeTJ1RjRmVm9SQUNIYmF2U1U2dklsbTVlWFpCcHMwTFF1SUg5NmNfcWhQRFlpeWt0U19HMXNuUHd2RFdrVl9XdUFaY0hWdVBPYWlybU1CdGlCN1A0RzZBbi1IUVJ1TWMxTE9Ea09sTURhcDFZb1JIUW1zUFJybW15MDcxOUtfVXA2N0xwMnFrczA1YTJaN05pRHhOYWNzMjVmUHdhbVdlemF3TEIzN0pJaVo3bGJBMXJnZmNYTXVJVDdmYkRXWTlBT2F2NmN4eTlteUI1SlJTOXc2WWFWUTBCZTJBVHRLVDhEVjBFeHE0Nmk1YkxYd3N3RXgtVUdGdlZFSmk4dHM0QjFmbktsQTctbmJMT0MtMDlKS1pUR0pELXBxckhULUUycjlBZmVJQjFrM0xEUm50U2ZabExtVjZ1WWZ1WnlobUZIOVlndjNydUZfczJUWVVRZURTd1lYazllaER4VU10cXUyVS1ZNG9Ha2hnbTAzOEpGMklFSWpWeVV5eFB2UlVWYmJJakZnOVM2R2lJSXRSM3VzVEZZNUVpNmVjRzdXRUJsT2hzcjhZWERFeGV5c1dFQVM3dkhGY2Q3ckNBRDZCcVdhZnZkdzM3QVNpODZYWE81TEIyZGUycldkSVRvbm5hR3Jib2UzOEtXdUpHQ2FyWDQtMDdQbC1ycEdfUzdXd0U2dHFIVjhoRDJ0YkNsWUpva1dzOGNPdXRpZjVwUldtT3FVN3RrZUhTN3JfX1M3LU9PaXZELWkzRmtMbjgxZGZ6ZjVJNW9RZW1nM2hqUXo4Z2I5Z2tSVTVMdUNLblRxOGQ1Y3F4SGZIbWo4YkFBV3FIbjB6LUxGNHdsQWgxQUM4bzVrblBObFFfVWNaQ3QwejQ1eGFlSXVIcXlyVEZEdzVKNV9pd2o4RW1UVjlqb3VMWnF0V1JTcWF1R0RjdUNjM2lLUHRqZDl2WWtXUnhmbVdxeHA3REFHTkdkMjM4LTllajBWQnd3RHlFSVdiUThfQnduOVFJdmR6OUVGN1lOYjBqclhadHozX21kRzlUT2EtWVBkYWFRSjRGdW80dmlEUTVrVjhWbjJYNGtCeGNtNzRHQXJsRlZyWjBYdHltVDM2MV9IT0RFT2dLLTVBREtsS09HdUxrODRLcEQ1TmRoVDh6WmgybGc5MzgtbmJSYThQd3FFaUcxbmg3eE95RkJVX2hHM20wT1k2c21qd24wSkFWNGROaklQeHZrc21PdTVsdHVxR0pxd3Ztb1NQVHEtd25URHRNa1pqa3BLdVdkTnNFeDNManJST0dOb1RWM2hqekxFTlFSZkd6TlZBY1VQT1NFOVlDQzlPQWVlVXQ4MW0wdGkzd0Myam1lSWE2aEtVVTVNc3N3dENpa1BWRl9ZQ3daYllONWRmRUF0THpleFRmdWRqTFM2aldmLUFuZzFGdkFQNHR6d21SdzRGQ0Q4cU8yV0xGUTVUY01TZlYxSzZ4cmtfUGZvVDhmYmNBX1pibTVTcl9lenJoME9KSnBucUxPRU1PRXBmLWFENEgwRWZOU0RvRDlvQk9ueVp0dXJrUVgtQUk5VldVbV9MS19PYmlua3liWl80Z2hMcFRnTXBkZDA3enIxRWFzaU56TEZKa0hPQUtNY0dCY1pnQ2V3Zml6ZFczWFBESUlLd3BSVEs5ZXlGLUpINDRsd1NBVjBkR1dvbE8wLWZBeEhFQ0hvY3E5UGJsTDdteGdSRjBIZTRobXpsd29PMmhKQkxXY3Znd2FMdWtZU1VkQlVRZXlSZ3FaVnNqcXpwR3N3SktOTDA3aUZIcE9TR1VDcXdaTDhQX2E5VDlwckoyX0xlNmFQcnoydEkwc0s1S08yaVlsM0pwYktUVWl3LU5hQzF2UVZNSm9ZR3QyQWdrUXB2a25QNzhkVEFOYmZ0b1BmTXRCMmVQZTAtYzdOeUlBYlNINlZNZW1nUTFfSV92UlJiWGt6Qms1c1hBc3kzZkVRMzEwNVJDOS1JeVg4YWtVeUJyOTZPQ0FnSUs1Z25sMlY0S1V1c0dIWEpuX2pMQmZ4Z29SY1U0bVZscXNWcjJwRy1UZEFYSXBzQURGblRTelBybU5BeDF6N3hZLXZwSHBkMmlzbHZWN2JkU3hRcE0zQ0hna3QwYWlJX3hBdGcxUHdGRE55cndUNHRvbXU5VTRMRmZDRjhvXzIwajI1Y0RCcmR2OV94cS1XYkNwalNHS2lObHlkNGZBbklycnZMSlJYVnlfakRXb1ZfWUo2MGxzYUNIektYeENGTkUzMUJXRE9WRHRrY2o5UFJHckZza2RQbjNPUkstbG9GZG4yNmxKeEdtbHo4WDZFc0lvT01wZkxuN29ycXl3X1hTN1prRGdvWG9hRFYwNzBwVVpuMW0wQlZYbGZxZjFQUHp2XzBQT3Fqa3lzejVKZmJDMG0wRzhqWV9HY1dxaXB2VFNQUzV2LUJSOXRFRUllak83cUI3RGUtYVBJakF1YUVOV0otT1BxUHJqS0NLdFVHc0tsT2RGcWd6UTU4Yi1kc0JZS1VPT1NXSlc3TDM5ZDVEZlRDOURZU1hMT0YxZ25ndVBUaG1VcGsxWFZSS1RxT1ZZTU1vclZjVU5iYmZMd0VBTXlvdTE0YjdoclZ6ZnNKMmE2Yy1ORmNCMnJNX3dwcVJSN2RSd2d6aENLRXQyTjhkcDlLTFVZMHBydFowNTJoZm1mVHNRVHI1YjhTNnl1Vll4dFZhenZfa0dybk9KYVh6LUluSUo0djUzRFNEdzBoVGt5UU9tMlg5UnBLbk9WaEhoU2txY2tUSXJmemlmNEExb3Q1blI5bE9adHluWVI3NXZQNUtXdmpra05aNy15dTBXdlVqcXhteFVqSXFxNnlQR2FGeVNONkx3NVpQUk1FNk5yTUY4T1hQV1FCdm9PYzdFTGl4QXZkODltSlprbGJ6cWREcEM1VlNwN3V5aWdWYXNkekk4X3U0cjJjZ1k2X190cmNnMlpMQVlLdExxM3pFNkZudVFKci1CalE1U3kzdmotQ01LV0ZzWnp0VUxRblhkdlN6VG1MWHNQdGlrNmF4RnFtd0c3UXNqZFVRZTRFMGl1NFU5T2k3VEpjZXA1U052VkJtdUhDWEpTaDRGQnM0SDQwY2IxdDVNbUtELTQ0R0s0OHpfTHdFOHZ0VmRMTC1FUVpPSkJ4QXRWNnl5MURUdjVyUk53emRwbDBxUnloUmlheXhKY3RBUG1mX3JxM2w0VlZvcE40b2ROeG15NS01RFlvUHdoYllLNVhCZUNEd0dwQnFCLVdZU0RhVEFzR2gxTVpub3FGRnl4VDNiSVZrTnpMQUlxeGJGQzh5WlNZR2NKbklHRVRTaVJ2REduN0hXaGo5MHFGb1FOa0U5TUFwQ09zOXVWMnRRNVlJWmZpaTUxLWFIeWR0UEFtaVNDX1k5Q1p3Y2V4ckVXQVBRYzV1eGwwMWd0SE15WUxiYzUyLTUzTGlyTUhZUDFlRTFjcFpieWQwU0pxRWJXSE53Nkd5aHp5T28wZVd6Z1phLTQ4TmgxU3hvNHpySzExUk5WZlFFS3VpOXNHMDdZU0gzSGxYUlU4WmgwNUlPdlhQcUI0cGtITmQ4SlByczN0THUxNHc0a21vUEp6S1hLNnFRNmFfdlpmUWpJQ1VNYXVEOW1abzlsd2RoRG5pVXRVbjBKV2RFTGFEa3ZYTHByOTJjalc1b3hTWkFmS2RPdVlTUTVkRkpSTnZsMWtnYWZEUm1SR3lBemdON2xiN3pkZlNfX2NSYU5wWHNybHh4V0lnNHJjQ2NON1hiRHMycUdmNC1kay13bUE0OTBPN0xmNDA1NlQxVmRySEJvM1VUN2Y2Sl9KX2pZVHRPWEdfR2RYNUoxY01Va3pXb2VBd3lZb3BSXzU5NVJfWlhEYXFSVDJrUnFHWG42RVZJUVQ2RlJWUEkyQnRnREI3eHNiRERiQ3FUczJsRTBDZ3pUUGZPcjExZUFKc21QUWxVYVBmV2hPZXRGd3lJX3ZTczhCVG1jWFVwanhIZHlyTTdiR2c5cTBVSXBRV1U4ZExtWWdub1pTSHU0cU5aYWJVWmExbXI0MjE3WUVnPT0= + +# Feature SyncDelta JIRA configuration +Feature_SyncDelta_JIRA_DELTA_TOKEN_SECRET = PROD_ENC:Z0FBQUFBQm8xSU5pTDhnTVNzRUhScU8wYnZsZk52bHFkSWxLc18xQmtCeC1HbnNwTzVBbXRNTmQzRjZYaGE2MVlCNGtnWDk1T2I5VXVKNHpKU1VRbXEyN2tRWUJnU2ltZE5qZ3lmNEF6Z1hMTTEwZkk2NUNBYjhmVTJEcWpRUW9HNEVpSGFWdjBWQXQ3eUtHUTFJS3U5QWpaeno0RFNhMUxnPT0= diff --git a/env_prod.env b/env_prod.env index c1ba8086..2269e9bc 100644 --- a/env_prod.env +++ b/env_prod.env @@ -66,7 +66,7 @@ Connector_AiAnthropic_MAX_TOKENS = 2000 # Perplexity AI configuration Connector_AiPerplexity_API_URL = https://api.perplexity.ai/chat/completions -Connector_AiPerplexity_API_SECRET = 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 +Connector_AiPerplexity_API_SECRET = PROD_ENC:Z0FBQUFBQm82Mzk2Q1FGRkJEUkI4LXlQbHYzT2RkdVJEcmM4WGdZTWpJTEhoeUF1NW5LUVpJdDBYN3k1WFN4a2FQSWJSQmd0U0xJbzZDTmFFN05FcXl0Z3V1OEpsZjYydV94TXVjVjVXRTRYSWdLMkd5XzZIbFV6emRCZHpuOUpQeThadE5xcDNDVGV1RHJrUEN0c1BBYXctZFNWcFRuVXhRPT0= Connector_AiPerplexity_MODEL_NAME = sonar Connector_AiPerplexity_TEMPERATURE = 0.2 Connector_AiPerplexity_MAX_TOKENS = 2000 @@ -88,3 +88,7 @@ Connector_GoogleSpeech_API_KEY_SECRET = PROD_ENC:Z0FBQUFBQm8xSU5pNjlJdmFMeERXUUQ # Feature SyncDelta JIRA configuration Feature_SyncDelta_JIRA_DELTA_TOKEN_SECRET = PROD_ENC:Z0FBQUFBQm8xSU5pTDhnTVNzRUhScU8wYnZsZk52bHFkSWxLc18xQmtCeC1HbnNwTzVBbXRNTmQzRjZYaGE2MVlCNGtnWDk1T2I5VXVKNHpKU1VRbXEyN2tRWUJnU2ltZE5qZ3lmNEF6Z1hMTTEwZkk2NUNBYjhmVTJEcWpRUW9HNEVpSGFWdjBWQXQ3eUtHUTFJS3U5QWpaeno0RFNhMUxnPT0= + +# Debug Configuration +APP_DEBUG_CHAT_WORKFLOW_ENABLED = FALSE +APP_DEBUG_CHAT_WORKFLOW_DIR = ./test-chat \ No newline at end of file diff --git a/modules/connectors/connectorAiAnthropic.py b/modules/connectors/connectorAiAnthropic.py index 1bcfe289..e7eb07a2 100644 --- a/modules/connectors/connectorAiAnthropic.py +++ b/modules/connectors/connectorAiAnthropic.py @@ -1,5 +1,6 @@ import logging import httpx +import os from typing import Dict, Any, List, Union from fastapi import HTTPException from modules.shared.configuration import APP_CONFIG @@ -147,6 +148,11 @@ class AiAnthropic: # Direct content as string (in older API versions) content = anthropicResponse["content"] + # Debug logging for empty responses + if not content or content.strip() == "": + logger.warning(f"Anthropic API returned empty content. Full response: {anthropicResponse}") + content = "[Anthropic API returned empty response]" + # Return in OpenAI format return { "id": anthropicResponse.get("id", ""), @@ -182,14 +188,27 @@ class AiAnthropic: The analysis response as text """ try: + # Debug logging + logger.info(f"callAiImage called with imageData type: {type(imageData)}, length: {len(imageData) if imageData else 0}, mimeType: {mimeType}") + # Distinguish between file path and binary data if isinstance(imageData, str): - # It's a file path - import filehandling only when needed - from modules import agentserviceFilemanager as fileHandler - base64Data, autoMimeType = fileHandler.encodeFileToBase64(imageData) - mimeType = mimeType or autoMimeType + # Check if it's base64 encoded data or a file path + if len(imageData) > 100 and not os.path.exists(imageData): + # It's likely base64 encoded data + logger.info("Treating imageData as base64 encoded string") + base64Data = imageData + if not mimeType: + mimeType = "image/png" + else: + # It's a file path - import filehandling only when needed + logger.info(f"Treating imageData as file path: {imageData}") + from modules import agentserviceFilemanager as fileHandler + base64Data, autoMimeType = fileHandler.encodeFileToBase64(imageData) + mimeType = mimeType or autoMimeType else: # It's binary data + logger.info("Treating imageData as binary data") import base64 base64Data = base64.b64encode(imageData).decode('utf-8') # MIME type must be specified for binary data @@ -216,8 +235,16 @@ class AiAnthropic: # Use the existing callAiBasic function with the Vision model response = await self.callAiBasic(messages) - # Extract and return content - return response["choices"][0]["message"]["content"] + # Extract and return content with proper error handling + try: + content = response["choices"][0]["message"]["content"] + if content is None or content.strip() == "": + return "[AI returned empty response for image analysis]" + return content + except (KeyError, IndexError, TypeError) as e: + logger.error(f"Error extracting content from AI response: {str(e)}") + logger.error(f"Response structure: {response}") + return f"[Error extracting AI response: {str(e)}]" except Exception as e: logger.error(f"Error during image analysis: {str(e)}", exc_info=True) diff --git a/modules/services/serviceAi/mainServiceAi.py b/modules/services/serviceAi/mainServiceAi.py index 5f24e158..031f2ea4 100644 --- a/modules/services/serviceAi/mainServiceAi.py +++ b/modules/services/serviceAi/mainServiceAi.py @@ -33,7 +33,7 @@ class AiService: Args: serviceCenter: Service center instance for accessing other services """ - self.serviceCenter = serviceCenter + self.services = serviceCenter # Only depend on interfaces self.aiObjects = None # Will be initialized in create() self._extractionService = None # Lazy initialization @@ -43,7 +43,7 @@ class AiService: """Lazy initialization of extraction service.""" if self._extractionService is None: logger.info("Lazy initializing ExtractionService...") - self._extractionService = ExtractionService() + self._extractionService = ExtractionService(self.services) return self._extractionService async def _ensureAiObjectsInitialized(self): @@ -63,42 +63,6 @@ class AiService: logger.info("AiObjects.create() completed") return instance - def _createImageAnalysisPrompt(self, userPrompt: str, generate_json: bool = False) -> str: - """Create a safe image analysis prompt based on the user's original request.""" - # Escape the user prompt to prevent string formatting issues - safeUserPrompt = userPrompt.replace("'", "\\'").replace('"', '\\"').replace('\n', ' ').replace('\r', ' ') - - if generate_json: - return f"""Based on the user's request: '{safeUserPrompt}', analyze this image and extract all content as structured JSON. - -You must respond with valid JSON only in this exact format: -{{ - "metadata": {{ - "title": "Image Content", - "version": "1.0" - }}, - "sections": [ - {{ - "id": "image_content", - "type": "paragraph", - "data": {{ - "text": "Detailed description of everything visible in the image" - }} - }} - ] -}} - -Analyze the image completely and describe everything you see including: -- All text content -- Tables or structured data -- Logos, graphics, and visual elements -- Layout and formatting -- Any other relevant information - -Return only the JSON structure with actual content from the image. Do not include any text before or after the JSON.""" - else: - return f"Based on the user's request: '{safeUserPrompt}', \n\nAnalyze this image completely. Provide a detailed description of everything you see. Include all text, logos, colors, fonts, layout, objects, and visual elements. This is a document processing system that requires complete image analysis for all content types including logos, graphics, simple images, flags, and animations. You must analyze and describe everything visible in the image without any disclaimers or refusals." - # AI Image Analysis async def readImage( self, @@ -109,8 +73,15 @@ Return only the JSON structure with actual content from the image. Do not includ ) -> str: """Call AI for image analysis using interface.callImage().""" try: - print(f"πŸ” readImage called with prompt: '{prompt[:100]}...', imageData type: {type(imageData)}, length: {len(imageData) if imageData else 0}, mimeType: {mimeType}") - logger.info(f"readImage called with prompt: '{prompt[:100]}...', imageData type: {type(imageData)}, length: {len(imageData) if imageData else 0}, mimeType: {mimeType}") + # Check if imageData is valid + if not imageData: + error_msg = "No image data provided" + self.services.utils.debugLogToFile(f"Error in AI image analysis: {error_msg}", "AI_SERVICE") + logger.error(f"Error in AI image analysis: {error_msg}") + return f"Error: {error_msg}" + + self.services.utils.debugLogToFile(f"readImage called with prompt, imageData type: {type(imageData)}, length: {len(imageData) if imageData else 0}, mimeType: {mimeType}", "AI_SERVICE") + logger.info(f"readImage called with prompt, imageData type: {type(imageData)}, length: {len(imageData) if imageData else 0}, mimeType: {mimeType}") # Always use IMAGE_ANALYSIS operation type for image processing if options is None: @@ -119,14 +90,25 @@ Return only the JSON structure with actual content from the image. Do not includ # Override the operation type to ensure image analysis options.operationType = OperationType.IMAGE_ANALYSIS - print(f"πŸ” Calling aiObjects.callImage with operationType: {options.operationType}") + self.services.utils.debugLogToFile(f"Calling aiObjects.callImage with operationType: {options.operationType}", "AI_SERVICE") logger.info(f"Calling aiObjects.callImage with operationType: {options.operationType}") result = await self.aiObjects.callImage(prompt, imageData, mimeType, options) - print(f"πŸ” callImage returned: {result[:200]}..." if len(result) > 200 else result) + + # Debug the result + self.services.utils.debugLogToFile(f"Raw AI result type: {type(result)}, value: {repr(result)}", "AI_SERVICE") + + # Check if result is valid + if not result or (isinstance(result, str) and not result.strip()): + error_msg = f"No response from AI image analysis (result: {repr(result)})" + self.services.utils.debugLogToFile(f"Error in AI image analysis: {error_msg}", "AI_SERVICE") + logger.error(f"Error in AI image analysis: {error_msg}") + return f"Error: {error_msg}" + + self.services.utils.debugLogToFile(f"callImage returned: {result[:200]}..." if len(result) > 200 else result, "AI_SERVICE") logger.info(f"callImage returned: {result[:200]}..." if len(result) > 200 else result) return result except Exception as e: - print(f"πŸ” Error in AI image analysis: {str(e)}") + self.services.utils.debugLogToFile(f"Error in AI image analysis: {str(e)}", "AI_SERVICE") logger.error(f"Error in AI image analysis: {str(e)}") return f"Error: {str(e)}" @@ -562,7 +544,7 @@ Return only the JSON structure with actual content from the image. Do not includ }, } - logger.debug(f"Per-chunk extraction options: {extractionOptions}") + logger.debug(f"Per-chunk extraction options: prompt length={len(extractionOptions.get('prompt', ''))} chars, operationType={extractionOptions.get('operationType')}") try: # Extract content with chunking @@ -620,7 +602,7 @@ Return only the JSON structure with actual content from the image. Do not includ }, } - logger.debug(f"Per-chunk extraction options (JSON mode): {extractionOptions}") + logger.debug(f"Per-chunk extraction options (JSON mode): prompt length={len(extractionOptions.get('prompt', ''))} chars, operationType={extractionOptions.get('operationType')}") try: # Extract content with chunking @@ -695,17 +677,37 @@ Return only the JSON structure with actual content from the image. Do not includ ) # Debug logging - print(f"πŸ” Chunk {chunk_index}: document_mime_type={document_mime_type}, part.mimeType={part.mimeType}, part.typeGroup={part.typeGroup}, is_image={is_image}") + self.services.utils.debugLogToFile(f"Chunk {chunk_index}: document_mime_type={document_mime_type}, part.mimeType={part.mimeType}, part.typeGroup={part.typeGroup}, is_image={is_image}", "AI_SERVICE") logger.info(f"Chunk {chunk_index}: document_mime_type={document_mime_type}, part.mimeType={part.mimeType}, part.typeGroup={part.typeGroup}, is_image={is_image}") if is_image: # Use the same extraction prompt for image analysis (contains table JSON format) - ai_result = await self.readImage( - prompt=prompt, - imageData=part.data, - mimeType=part.mimeType, - options=options - ) + self.services.utils.debugLogToFile(f"Processing image chunk {chunk_index}: mimeType={part.mimeType}, data_length={len(part.data) if part.data else 0}", "AI_SERVICE") + + # Check if image data is available + if not part.data: + error_msg = f"No image data available for chunk {chunk_index}" + logger.warning(error_msg) + ai_result = f"Error: {error_msg}" + else: + try: + ai_result = await self.readImage( + prompt=prompt, + imageData=part.data, + mimeType=part.mimeType, + options=options + ) + + self.services.utils.debugLogToFile(f"Image analysis result for chunk {chunk_index}: length={len(ai_result) if ai_result else 0}, preview={ai_result[:200] if ai_result else 'None'}...", "AI_SERVICE") + + # Check if result is empty or None + if not ai_result or not ai_result.strip(): + logger.warning(f"Image chunk {chunk_index} returned empty response from AI") + ai_result = "No content detected in image" + + except Exception as e: + logger.error(f"Error processing image chunk {chunk_index}: {str(e)}") + ai_result = f"Error analyzing image: {str(e)}" # If generating JSON, clean image analysis result if generate_json: @@ -715,43 +717,63 @@ Return only the JSON structure with actual content from the image. Do not includ # Clean the response - remove markdown code blocks if present cleaned_result = ai_result.strip() + + # Remove various markdown patterns if cleaned_result.startswith('```json'): - # Remove ```json from start and ``` from end cleaned_result = re.sub(r'^```json\s*', '', cleaned_result) cleaned_result = re.sub(r'\s*```$', '', cleaned_result) elif cleaned_result.startswith('```'): - # Remove ``` from start and end cleaned_result = re.sub(r'^```\s*', '', cleaned_result) cleaned_result = re.sub(r'\s*```$', '', cleaned_result) + # Remove any leading/trailing text that's not JSON + # Look for the first { and last } to extract JSON + first_brace = cleaned_result.find('{') + last_brace = cleaned_result.rfind('}') + + if first_brace != -1 and last_brace != -1 and last_brace > first_brace: + cleaned_result = cleaned_result[first_brace:last_brace + 1] + + # Additional cleaning for common AI response issues + cleaned_result = cleaned_result.strip() + # Validate JSON json.loads(cleaned_result) ai_result = cleaned_result # Use cleaned version + self.services.utils.debugLogToFile(f"Image chunk {chunk_index} JSON validation successful", "AI_SERVICE") except json.JSONDecodeError as e: logger.warning(f"Image chunk {chunk_index} returned invalid JSON: {str(e)}") - # Create fallback JSON + logger.warning(f"Raw response was: '{ai_result[:500]}...'") + + # Create fallback JSON with the actual response content (not the error message) + # Use the original AI response content, not the error message + fallback_content = ai_result if ai_result and ai_result.strip() else "No content detected" + + self.services.utils.debugLogToFile(f"IMAGE FALLBACK CONTENT PREVIEW: '{fallback_content[:200]}...'", "AI_SERVICE") + ai_result = json.dumps({ - "metadata": {"title": "Error Section"}, + "metadata": {"title": f"Image Analysis - Chunk {chunk_index}"}, "sections": [{ - "id": f"error_section_{chunk_index}", + "id": f"image_section_{chunk_index}", "type": "paragraph", - "data": {"text": f"Error parsing JSON: {str(e)}"} + "data": {"text": fallback_content} }] }) + self.services.utils.debugLogToFile(f"Created fallback JSON for image chunk {chunk_index} with actual content", "AI_SERVICE") elif part.typeGroup in ("container", "binary"): # Handle ALL container and binary content generically - let AI process any document type - print(f"πŸ” DEBUG: Chunk {chunk_index}: typeGroup={part.typeGroup}, mimeType={part.mimeType}, data_length={len(part.data) if part.data else 0}") + self.services.utils.debugLogToFile(f"DEBUG: Chunk {chunk_index}: typeGroup={part.typeGroup}, mimeType={part.mimeType}, data_length={len(part.data) if part.data else 0}", "AI_SERVICE") if part.mimeType and part.data and len(part.data.strip()) > 0: # Process any document container as text content request_options = options if options is not None else AiCallOptions() request_options.operationType = OperationType.GENERAL - print(f"πŸ” EXTRACTION CONTAINER CHUNK {chunk_index}: Processing {part.mimeType} container as text with generate_json={generate_json}") + self.services.utils.debugLogToFile(f"EXTRACTION CONTAINER CHUNK {chunk_index}: Processing {part.mimeType} container as text with generate_json={generate_json}", "AI_SERVICE") logger.info(f"Chunk {chunk_index}: Processing {part.mimeType} container as text with generate_json={generate_json}") # Log extraction prompt and context - print(f"πŸ” EXTRACTION PROMPT: {prompt}") - print(f"πŸ” EXTRACTION CONTEXT LENGTH: {len(part.data) if part.data else 0} characters") + self.services.utils.debugLogToFile(f"EXTRACTION PROMPT: {prompt}", "AI_SERVICE") + self.services.utils.debugLogToFile(f"EXTRACTION CONTEXT LENGTH: {len(part.data) if part.data else 0} characters", "AI_SERVICE") request = AiCallRequest( prompt=prompt, @@ -762,7 +784,7 @@ Return only the JSON structure with actual content from the image. Do not includ ai_result = response.content # Log extraction response - print(f"πŸ” EXTRACTION RESPONSE LENGTH: {len(ai_result) if ai_result else 0} characters") + self.services.utils.debugLogToFile(f"EXTRACTION RESPONSE LENGTH: {len(ai_result) if ai_result else 0} characters", "AI_SERVICE") # Save full extraction prompt and response to debug file try: @@ -786,33 +808,52 @@ Return only the JSON structure with actual content from the image. Do not includ # Clean the response - remove markdown code blocks if present cleaned_result = ai_result.strip() + + # Remove various markdown patterns if cleaned_result.startswith('```json'): - # Remove ```json from start and ``` from end cleaned_result = re.sub(r'^```json\s*', '', cleaned_result) cleaned_result = re.sub(r'\s*```$', '', cleaned_result) elif cleaned_result.startswith('```'): - # Remove ``` from start and end cleaned_result = re.sub(r'^```\s*', '', cleaned_result) cleaned_result = re.sub(r'\s*```$', '', cleaned_result) + # Remove any leading/trailing text that's not JSON + # Look for the first { and last } to extract JSON + first_brace = cleaned_result.find('{') + last_brace = cleaned_result.rfind('}') + + if first_brace != -1 and last_brace != -1 and last_brace > first_brace: + cleaned_result = cleaned_result[first_brace:last_brace + 1] + + # Additional cleaning for common AI response issues + cleaned_result = cleaned_result.strip() + # Validate JSON json.loads(cleaned_result) ai_result = cleaned_result # Use cleaned version except json.JSONDecodeError as e: logger.warning(f"Container chunk {chunk_index} ({part.mimeType}) returned invalid JSON: {str(e)}") - # Create fallback JSON + logger.warning(f"Raw response was: '{ai_result[:500]}...'") + + # Create fallback JSON with the actual response content (not the error message) + # Use the original AI response content, not the error message + fallback_content = ai_result if ai_result and ai_result.strip() else "No content detected" + + self.services.utils.debugLogToFile(f"FALLBACK CONTENT PREVIEW: '{fallback_content[:200]}...'", "AI_SERVICE") + ai_result = json.dumps({ - "metadata": {"title": "Error Section"}, + "metadata": {"title": f"Document Analysis - Chunk {chunk_index}"}, "sections": [{ - "id": f"error_section_{chunk_index}", + "id": f"analysis_section_{chunk_index}", "type": "paragraph", - "data": {"text": f"Error parsing JSON: {str(e)}"} + "data": {"text": fallback_content} }] }) + self.services.utils.debugLogToFile(f"Created fallback JSON for container chunk {chunk_index} with actual content", "AI_SERVICE") else: # Skip empty or invalid container/binary content - don't create a result - print(f"πŸ” DEBUG: Chunk {chunk_index}: Skipping empty container - mimeType={part.mimeType}, data_length={len(part.data) if part.data else 0}") + self.services.utils.debugLogToFile(f"DEBUG: Chunk {chunk_index}: Skipping empty container - mimeType={part.mimeType}, data_length={len(part.data) if part.data else 0}", "AI_SERVICE") # Return None to indicate this chunk should be completely skipped return None else: @@ -820,12 +861,11 @@ Return only the JSON structure with actual content from the image. Do not includ request_options = options if options is not None else AiCallOptions() # FIXED: Set operation type to general for text processing request_options.operationType = OperationType.GENERAL - print(f"πŸ” EXTRACTION CHUNK {chunk_index}: Calling aiObjects.call with operationType={request_options.operationType}, generate_json={generate_json}") + self.services.utils.debugLogToFile(f"EXTRACTION CHUNK {chunk_index}: Calling aiObjects.call with operationType={request_options.operationType}, generate_json={generate_json}", "AI_SERVICE") logger.info(f"Chunk {chunk_index}: Calling aiObjects.call with operationType={request_options.operationType}, generate_json={generate_json}") - # Log extraction prompt and context - print(f"πŸ” EXTRACTION PROMPT: {prompt}") - print(f"πŸ” EXTRACTION CONTEXT LENGTH: {len(part.data) if part.data else 0} characters") + # Log extraction context length + self.services.utils.debugLogToFile(f"EXTRACTION CONTEXT LENGTH: {len(part.data) if part.data else 0} characters", "AI_SERVICE") request = AiCallRequest( prompt=prompt, @@ -835,10 +875,10 @@ Return only the JSON structure with actual content from the image. Do not includ response = await self.aiObjects.call(request) ai_result = response.content - # Log extraction response - print(f"πŸ” EXTRACTION RESPONSE LENGTH: {len(ai_result) if ai_result else 0} characters") + # Log extraction response length + self.services.utils.debugLogToFile(f"EXTRACTION RESPONSE LENGTH: {len(ai_result) if ai_result else 0} characters", "AI_SERVICE") - # Save full extraction prompt and response to debug file + # Save extraction response to debug file (without verbose prompt) try: import os from datetime import datetime, UTC @@ -846,8 +886,6 @@ Return only the JSON structure with actual content from the image. Do not includ debug_root = "./test-chat/ai" os.makedirs(debug_root, exist_ok=True) with open(os.path.join(debug_root, f"{ts}_extraction_chunk_{chunk_index}.txt"), "w", encoding="utf-8") as f: - f.write(f"EXTRACTION PROMPT:\n{prompt}\n\n") - f.write(f"EXTRACTION CONTEXT:\n{part.data if part.data else 'No context'}\n\n") f.write(f"EXTRACTION RESPONSE:\n{ai_result if ai_result else 'No response'}\n") except Exception: pass @@ -929,9 +967,9 @@ Return only the JSON structure with actual content from the image. Do not includ max_concurrent = options.maxParallelChunks logger.info(f"Processing {len(chunks_to_process)} chunks with max concurrency: {max_concurrent}") - print(f"πŸ” DEBUG: Chunks to process: {len(chunks_to_process)}") + self.services.utils.debugLogToFile(f"DEBUG: Chunks to process: {len(chunks_to_process)}", "AI_SERVICE") for i, chunk_info in enumerate(chunks_to_process): - print(f"πŸ” DEBUG: Chunk {i}: typeGroup={chunk_info['part'].typeGroup}, mimeType={chunk_info['part'].mimeType}, data_length={len(chunk_info['part'].data) if chunk_info['part'].data else 0}") + self.services.utils.debugLogToFile(f"DEBUG: Chunk {i}: typeGroup={chunk_info['part'].typeGroup}, mimeType={chunk_info['part'].mimeType}, data_length={len(chunk_info['part'].data) if chunk_info['part'].data else 0}", "AI_SERVICE") # Create semaphore for concurrency control semaphore = asyncio.Semaphore(max_concurrent) @@ -942,9 +980,9 @@ Return only the JSON structure with actual content from the image. Do not includ # Process all chunks in parallel with concurrency control tasks = [process_with_semaphore(chunk_info) for chunk_info in chunks_to_process] - print(f"πŸ” DEBUG: Created {len(tasks)} tasks for parallel processing") + self.services.utils.debugLogToFile(f"DEBUG: Created {len(tasks)} tasks for parallel processing", "AI_SERVICE") chunk_results = await asyncio.gather(*tasks, return_exceptions=True) - print(f"πŸ” DEBUG: Got {len(chunk_results)} results from parallel processing") + self.services.utils.debugLogToFile(f"DEBUG: Got {len(chunk_results)} results from parallel processing", "AI_SERVICE") # Handle any exceptions in the gather itself processed_results = [] @@ -1626,8 +1664,7 @@ Return only the JSON structure with actual content from the image. Do not includ # Get log directory from configuration via service center if possible logDir = None try: - if self.serviceCenter and hasattr(self.serviceCenter, 'utils'): - logDir = self.serviceCenter.utils.configGet("APP_LOGGING_LOG_DIR", "./") + logDir = self.services.utils.configGet("APP_LOGGING_LOG_DIR", "./") except Exception: pass if not logDir: @@ -1800,7 +1837,7 @@ Return only the JSON structure with actual content from the image. Do not includ try: # Get format-specific extraction prompt from generation service from modules.services.serviceGeneration.mainServiceGeneration import GenerationService - generation_service = GenerationService(self.serviceCenter) + generation_service = GenerationService(self.services) # Use default title if not provided if not title: diff --git a/modules/services/serviceExtraction/extractors/extractorImage.py b/modules/services/serviceExtraction/extractors/extractorImage.py index 22327f50..3f94459c 100644 --- a/modules/services/serviceExtraction/extractors/extractorImage.py +++ b/modules/services/serviceExtraction/extractors/extractorImage.py @@ -1,10 +1,13 @@ from typing import Any, Dict, List import base64 +import logging from ..subUtils import makeId from modules.datamodels.datamodelExtraction import ContentPart from ..subRegistry import Extractor +logger = logging.getLogger(__name__) + class ImageExtractor(Extractor): def detect(self, fileName: str, mimeType: str, headBytes: bytes) -> bool: @@ -12,6 +15,35 @@ class ImageExtractor(Extractor): def extract(self, fileBytes: bytes, context: Dict[str, Any]) -> List[ContentPart]: mimeType = context.get("mimeType") or "image/unknown" + fileName = context.get("fileName", "") + + # Convert GIF to PNG during extraction + if mimeType.lower() == "image/gif": + try: + from PIL import Image + import io + + # Open GIF and convert to PNG + with Image.open(io.BytesIO(fileBytes)) as img: + # Convert to RGB (removes animation) + if img.mode in ('RGBA', 'LA', 'P'): + img = img.convert('RGB') + + # Save as PNG in memory + png_buffer = io.BytesIO() + img.save(png_buffer, format='PNG') + png_data = png_buffer.getvalue() + + # Update mimeType and fileBytes + mimeType = "image/png" + fileBytes = png_data + + logger.info(f"GIF converted to PNG during extraction: {fileName}, original={len(fileBytes)} bytes, converted={len(png_data)} bytes") + + except Exception as e: + logger.warning(f"GIF conversion failed during extraction for {fileName}: {str(e)}, using original") + # Keep original GIF data if conversion fails + return [ContentPart( id=makeId(), parentId=None, diff --git a/modules/services/serviceExtraction/subPipeline.py b/modules/services/serviceExtraction/subPipeline.py index 515fd293..382bd74d 100644 --- a/modules/services/serviceExtraction/subPipeline.py +++ b/modules/services/serviceExtraction/subPipeline.py @@ -85,7 +85,7 @@ def runExtraction(extractorRegistry: ExtractorRegistry, chunkerRegistry: Chunker chunk_parts = [p for p in parts if p.metadata.get("chunk", False)] logger.debug(f"runExtraction: Preserving {len(chunk_parts)} chunks from merging") - print(f"πŸ” DEBUG: runExtraction - non_chunk_parts: {len(non_chunk_parts)}, chunk_parts: {len(chunk_parts)}") + logger.debug(f"runExtraction - non_chunk_parts: {len(non_chunk_parts)}, chunk_parts: {len(chunk_parts)}") # Apply intelligent merging for small text parts if non_chunk_parts: @@ -99,7 +99,7 @@ def runExtraction(extractorRegistry: ExtractorRegistry, chunkerRegistry: Chunker parts = non_chunk_parts + chunk_parts logger.debug(f"runExtraction: Final parts after merging: {len(parts)} (chunks: {len(chunk_parts)})") - print(f"πŸ” DEBUG: runExtraction - Final parts: {len(parts)} (chunks: {len(chunk_parts)})") + logger.debug(f"runExtraction - Final parts: {len(parts)} (chunks: {len(chunk_parts)})") # DEBUG: dump parts and chunks to files TODO TO REMOVE try: base_dir = "./test-chat/ai" @@ -154,22 +154,22 @@ def poolAndLimit(parts: List[ContentPart], chunkerRegistry: ChunkerRegistry, opt kept: List[ContentPart] = [] remaining: List[ContentPart] = [] - print(f"πŸ” DEBUG: Starting poolAndLimit with {len(parts)} parts, maxSize={maxSize}") + logger.debug(f"Starting poolAndLimit with {len(parts)} parts, maxSize={maxSize}") for i, p in enumerate(parts): size = int(p.metadata.get("size", 0) or 0) # Show first 50 characters of text content for debugging content_preview = p.data[:50].replace('\n', '\\n') if p.data else "" - print(f"πŸ” DEBUG: Part {i}: {p.typeGroup} - {size} bytes - '{content_preview}...' (current: {current})") + logger.debug(f"Part {i}: {p.typeGroup} - {size} bytes - '{content_preview}...' (current: {current})") if current + size <= maxSize: kept.append(p) current += size - print(f"πŸ” DEBUG: Part {i} kept (total: {current})") + logger.debug(f"Part {i} kept (total: {current})") else: remaining.append(p) - print(f"πŸ” DEBUG: Part {i} moved to remaining") + logger.debug(f"Part {i} moved to remaining") - print(f"πŸ” DEBUG: Kept: {len(kept)}, Remaining: {len(remaining)}") + logger.debug(f"Kept: {len(kept)}, Remaining: {len(remaining)}") # If we have remaining parts and chunking is allowed, try chunking if remaining and chunkAllowed: @@ -177,15 +177,15 @@ def poolAndLimit(parts: List[ContentPart], chunkerRegistry: ChunkerRegistry, opt logger.debug(f"Remaining parts to chunk: {len(remaining)}") logger.debug(f"Max size limit: {maxSize} bytes") logger.debug(f"Current size used: {current} bytes") - print(f"πŸ” DEBUG: Chunking {len(remaining)} remaining parts") + logger.debug(f"Chunking {len(remaining)} remaining parts") for p in remaining: if p.typeGroup in ("text", "table", "structure", "image", "container", "binary"): logger.debug(f"Chunking {p.typeGroup} part: {len(p.data)} chars") - print(f"πŸ” DEBUG: Chunking {p.typeGroup} part with {len(p.data)} chars") + logger.debug(f"Chunking {p.typeGroup} part with {len(p.data)} chars") chunks = chunkerRegistry.resolve(p.typeGroup).chunk(p, options) logger.debug(f"Created {len(chunks)} chunks") - print(f"πŸ” DEBUG: Created {len(chunks)} chunks") + logger.debug(f"Created {len(chunks)} chunks") chunks_added = 0 for ch in chunks: @@ -229,7 +229,7 @@ def poolAndLimit(parts: List[ContentPart], chunkerRegistry: ChunkerRegistry, opt kept = non_chunk_parts + chunk_parts logger.debug(f"Final parts after merging: {len(kept)} (chunks: {len(chunk_parts)})") - print(f"πŸ” DEBUG: Final parts after merging: {len(kept)} (chunks: {len(chunk_parts)})") + logger.debug(f"Final parts after merging: {len(kept)} (chunks: {len(chunk_parts)})") # Re-check size after merging totalSize = sum(int(p.metadata.get("size", 0) or 0) for p in kept) @@ -237,13 +237,13 @@ def poolAndLimit(parts: List[ContentPart], chunkerRegistry: ChunkerRegistry, opt # Apply size limit to merged parts kept = _applySizeLimit(kept, maxSize) - print(f"πŸ” DEBUG: poolAndLimit returning {len(kept)} parts") + logger.debug(f"poolAndLimit returning {len(kept)} parts") return kept def _applyMerging(parts: List[ContentPart], strategy: Dict[str, Any]) -> List[ContentPart]: """Apply merging strategy to parts with intelligent token-aware merging.""" - print(f"πŸ” DEBUG: _applyMerging called with {len(parts)} parts") + logger.debug(f"_applyMerging called with {len(parts)} parts") # Check if intelligent merging is enabled if strategy.get("useIntelligentMerging", False): @@ -256,7 +256,7 @@ def _applyMerging(parts: List[ContentPart], strategy: Dict[str, Any]) -> List[Co # Calculate and log optimization stats stats = subMerger.calculate_optimization_stats(parts, merged) logger.info(f"🧠 Intelligent merging stats: {stats}") - print(f"πŸ” DEBUG: Intelligent merging: {stats['original_ai_calls']} β†’ {stats['optimized_ai_calls']} calls ({stats['reduction_percent']}% reduction)") + logger.debug(f"Intelligent merging: {stats['original_ai_calls']} β†’ {stats['optimized_ai_calls']} calls ({stats['reduction_percent']}% reduction)") return merged @@ -271,29 +271,29 @@ def _applyMerging(parts: List[ContentPart], strategy: Dict[str, Any]) -> List[Co structureParts = [p for p in parts if p.typeGroup == "structure"] otherParts = [p for p in parts if p.typeGroup not in ("text", "table", "structure")] - print(f"πŸ” DEBUG: Grouped - text: {len(textParts)}, table: {len(tableParts)}, structure: {len(structureParts)}, other: {len(otherParts)}") + logger.debug(f"Grouped - text: {len(textParts)}, table: {len(tableParts)}, structure: {len(structureParts)}, other: {len(otherParts)}") merged: List[ContentPart] = [] if textParts: textMerged = textMerger.merge(textParts, strategy) - print(f"πŸ” DEBUG: TextMerger merged {len(textParts)} parts into {len(textMerged)} parts") + logger.debug(f"TextMerger merged {len(textParts)} parts into {len(textMerged)} parts") merged.extend(textMerged) if tableParts: tableMerged = tableMerger.merge(tableParts, strategy) - print(f"πŸ” DEBUG: TableMerger merged {len(tableParts)} parts into {len(tableMerged)} parts") + logger.debug(f"TableMerger merged {len(tableParts)} parts into {len(tableMerged)} parts") merged.extend(tableMerged) if structureParts: # For now, treat structure like text structureMerged = textMerger.merge(structureParts, strategy) - print(f"πŸ” DEBUG: StructureMerger merged {len(structureParts)} parts into {len(structureMerged)} parts") + logger.debug(f"StructureMerger merged {len(structureParts)} parts into {len(structureMerged)} parts") merged.extend(structureMerged) if otherParts: otherMerged = defaultMerger.merge(otherParts, strategy) - print(f"πŸ” DEBUG: DefaultMerger merged {len(otherParts)} parts into {len(otherMerged)} parts") + logger.debug(f"DefaultMerger merged {len(otherParts)} parts into {len(otherMerged)} parts") merged.extend(otherMerged) - print(f"πŸ” DEBUG: _applyMerging returning {len(merged)} parts") + logger.debug(f"_applyMerging returning {len(merged)} parts") return merged diff --git a/modules/services/serviceExtraction/subRegistry.py b/modules/services/serviceExtraction/subRegistry.py index 7f4b9c11..ae994bbf 100644 --- a/modules/services/serviceExtraction/subRegistry.py +++ b/modules/services/serviceExtraction/subRegistry.py @@ -1,7 +1,10 @@ from typing import Any, Dict, Optional +import logging from modules.datamodels.datamodelExtraction import ContentPart +logger = logging.getLogger(__name__) + class Extractor: def detect(self, fileName: str, mimeType: str, headBytes: bytes) -> bool: @@ -64,9 +67,9 @@ class ExtractorRegistry: self.register("ppt", PptxExtractor()) # fallback self.setFallback(BinaryExtractor()) - print(f"βœ… ExtractorRegistry: Successfully registered {len(self._map)} extractors") + logger.info(f"ExtractorRegistry: Successfully registered {len(self._map)} extractors") except Exception as e: - print(f"❌ ExtractorRegistry: Failed to register extractors: {str(e)}") + logger.error(f"ExtractorRegistry: Failed to register extractors: {str(e)}") import traceback traceback.print_exc() @@ -105,7 +108,7 @@ class ChunkerRegistry: self.register("container", TextChunker()) self.register("binary", TextChunker()) except Exception as e: - print(f"❌ ChunkerRegistry: Failed to register chunkers: {str(e)}") + logger.error(f"ChunkerRegistry: Failed to register chunkers: {str(e)}") import traceback traceback.print_exc() diff --git a/modules/services/serviceGeneration/mainServiceGeneration.py b/modules/services/serviceGeneration/mainServiceGeneration.py index 2d3aa21f..0380455e 100644 --- a/modules/services/serviceGeneration/mainServiceGeneration.py +++ b/modules/services/serviceGeneration/mainServiceGeneration.py @@ -18,7 +18,7 @@ logger = logging.getLogger(__name__) class GenerationService: def __init__(self, serviceCenter=None): # Directly use interfaces from the provided service center (no self.service calls) - self.serviceCenter = serviceCenter + self.services = serviceCenter self.interfaceDbComponent = getattr(serviceCenter, 'interfaceDbComponent', None) if serviceCenter else None self.interfaceDbChat = getattr(serviceCenter, 'interfaceDbChat', None) if serviceCenter else None self.workflow = getattr(serviceCenter, 'workflow', None) if serviceCenter else None @@ -346,7 +346,8 @@ class GenerationService: outputFormat=outputFormat, userPrompt=userPrompt, title=title, - aiService=aiService + aiService=aiService, + services=self.services ) except Exception as e: logger.warning(f"Failed to generate AI-based generation prompt: {str(e)}, using user prompt") @@ -395,7 +396,8 @@ class GenerationService: renderer=renderer, userPrompt=userPrompt, title=title, - aiService=aiService + aiService=aiService, + services=self.services ) logger.info(f"Generated {outputFormat}-specific extraction prompt: {len(extractionPrompt)} characters") @@ -409,14 +411,14 @@ class GenerationService: """Get the appropriate renderer for the specified format using auto-discovery.""" try: from .renderers.registry import get_renderer - renderer = get_renderer(output_format) + renderer = get_renderer(output_format, services=self.services) if renderer: return renderer # Fallback to text renderer if no specific renderer found logger.warning(f"No renderer found for format {output_format}, falling back to text") - fallback_renderer = get_renderer('text') + fallback_renderer = get_renderer('text', services=self.services) if fallback_renderer: return fallback_renderer diff --git a/modules/services/serviceGeneration/renderers/registry.py b/modules/services/serviceGeneration/renderers/registry.py index 6843c114..909cfb2c 100644 --- a/modules/services/serviceGeneration/renderers/registry.py +++ b/modules/services/serviceGeneration/renderers/registry.py @@ -92,7 +92,7 @@ class RendererRegistry: except Exception as e: logger.error(f"Error registering renderer {renderer_class.__name__}: {str(e)}") - def get_renderer(self, output_format: str) -> Optional[BaseRenderer]: + def get_renderer(self, output_format: str, services=None) -> Optional[BaseRenderer]: """Get a renderer instance for the specified format.""" if not self._discovered: self.discover_renderers() @@ -109,7 +109,7 @@ class RendererRegistry: if renderer_class: try: - return renderer_class() + return renderer_class(services=services) except Exception as e: logger.error(f"Error creating renderer instance for {format_name}: {str(e)}") return None @@ -144,9 +144,9 @@ class RendererRegistry: # Global registry instance _registry = RendererRegistry() -def get_renderer(output_format: str) -> Optional[BaseRenderer]: +def get_renderer(output_format: str, services=None) -> Optional[BaseRenderer]: """Get a renderer instance for the specified format.""" - return _registry.get_renderer(output_format) + return _registry.get_renderer(output_format, services) def get_supported_formats() -> List[str]: """Get list of all supported formats.""" diff --git a/modules/services/serviceGeneration/renderers/rendererBaseTemplate.py b/modules/services/serviceGeneration/renderers/rendererBaseTemplate.py index 4c6b7001..ed10c01b 100644 --- a/modules/services/serviceGeneration/renderers/rendererBaseTemplate.py +++ b/modules/services/serviceGeneration/renderers/rendererBaseTemplate.py @@ -12,8 +12,9 @@ logger = logging.getLogger(__name__) class BaseRenderer(ABC): """Base class for all format renderers.""" - def __init__(self): + def __init__(self, services=None): self.logger = logger + self.services = services # Add services attribute @classmethod def get_supported_formats(cls) -> List[str]: @@ -313,7 +314,6 @@ class BaseRenderer(ABC): Dict with styling definitions """ # DEBUG: Show which renderer is calling this method - print(f"πŸ” BASE TEMPLATE _get_ai_styles called by: {self.__class__.__name__}") if not ai_service: return default_styles @@ -361,11 +361,8 @@ class BaseRenderer(ABC): self.logger.warning(f"AI styling returned invalid JSON: {json_error}") # Use print instead of logger to avoid truncation - print(f"πŸ” FULL AI RESPONSE THAT FAILED TO PARSE:") - print("=" * 100) - print(result) - print("=" * 100) - print(f"πŸ” RESPONSE LENGTH: {len(result)} characters") + self.services.utils.debugLogToFile(f"FULL AI RESPONSE THAT FAILED TO PARSE: {result}", "RENDERER") + self.services.utils.debugLogToFile(f"RESPONSE LENGTH: {len(result)} characters", "RENDERER") self.logger.warning(f"Raw content that failed to parse: {result}") @@ -446,10 +443,6 @@ class BaseRenderer(ABC): schema_json = json.dumps(style_schema, indent=4) # DEBUG: Show the schema being sent - print(f"πŸ” AI STYLE SCHEMA FOR {format_name.upper()}:") - print("=" * 80) - print(schema_json) - print("=" * 80) return f"""You are a professional document styling expert. Generate a complete JSON styling configuration for {format_name.upper()} documents. diff --git a/modules/services/serviceGeneration/renderers/rendererDocx.py b/modules/services/serviceGeneration/renderers/rendererDocx.py index b972fc07..0e0417f9 100644 --- a/modules/services/serviceGeneration/renderers/rendererDocx.py +++ b/modules/services/serviceGeneration/renderers/rendererDocx.py @@ -42,7 +42,7 @@ class RendererDocx(BaseRenderer): async def render(self, extracted_content: Dict[str, Any], title: str, user_prompt: str = None, ai_service=None) -> Tuple[str, str]: """Render extracted JSON content to DOCX format using AI-analyzed styling.""" - print(f"πŸ” DOCX RENDER CALLED: title={title}, user_prompt={user_prompt[:50] if user_prompt else 'None'}...") + self.services.utils.debugLogToFile(f"DOCX RENDER CALLED: title={title}, user_prompt={user_prompt[:50] if user_prompt else 'None'}...", "DOCX_RENDERER") try: if not DOCX_AVAILABLE: # Fallback to HTML if python-docx not available @@ -68,10 +68,8 @@ class RendererDocx(BaseRenderer): doc = Document() # Get AI-generated styling definitions - print(f"πŸ” ABOUT TO CALL AI STYLING: user_prompt={user_prompt[:50] if user_prompt else 'None'}...") self.logger.info(f"About to call AI styling with user_prompt: {user_prompt[:100] if user_prompt else 'None'}...") styles = await self._get_docx_styles(user_prompt, ai_service) - print(f"πŸ” AI STYLING RESULT: {type(styles)}") # Apply basic document setup self._setup_basic_document_styles(doc) diff --git a/modules/services/serviceGeneration/renderers/rendererPdf.py b/modules/services/serviceGeneration/renderers/rendererPdf.py index 1c5f0739..8dd1917d 100644 --- a/modules/services/serviceGeneration/renderers/rendererPdf.py +++ b/modules/services/serviceGeneration/renderers/rendererPdf.py @@ -103,11 +103,11 @@ class RendererPdf(BaseRenderer): # Process each section sections = json_content.get("sections", []) - print(f"πŸ” PDF SECTIONS TO PROCESS: {len(sections)} sections") + self.services.utils.debugLogToFile(f"PDF SECTIONS TO PROCESS: {len(sections)} sections", "PDF_RENDERER") for i, section in enumerate(sections): - print(f"πŸ” PDF SECTION {i}: type={section.get('type', 'unknown')}, id={section.get('id', 'unknown')}") + self.services.utils.debugLogToFile(f"PDF SECTION {i}: type={section.get('type', 'unknown')}, id={section.get('id', 'unknown')}", "PDF_RENDERER") section_elements = self._render_json_section(section, styles) - print(f"πŸ” PDF SECTION {i} ELEMENTS: {len(section_elements)} elements") + self.services.utils.debugLogToFile(f"PDF SECTION {i} ELEMENTS: {len(section_elements)} elements", "PDF_RENDERER") story.extend(section_elements) # Build PDF @@ -139,40 +139,15 @@ class RendererPdf(BaseRenderer): style_template = self._create_ai_style_template("pdf", user_prompt, style_schema) - # DEBUG: Show which method is being called - print(f"πŸ” PDF RENDERER: Calling base template _get_ai_styles") - # Use base template method like DOCX does (this works!) styles = await self._get_ai_styles(ai_service, style_template, self._get_default_pdf_styles()) - # DEBUG: Check what we got from AI styling - print(f"πŸ” PDF AI STYLING RESULT: {type(styles)}") if styles is None: - print(f"πŸ” PDF AI STYLING RETURNED NONE!") return self._get_default_pdf_styles() - elif isinstance(styles, dict): - print(f"πŸ” PDF AI STYLING KEYS: {list(styles.keys())}") - print(f"πŸ” PDF AI STYLING CONTENT:") - for key, value in styles.items(): - print(f" {key}: {value}") - # Check specific colors - print(f"πŸ” PDF TITLE COLOR FROM AI: {styles.get('title', {}).get('color', 'NOT_FOUND')}") - print(f"πŸ” PDF HEADING1 COLOR FROM AI: {styles.get('heading1', {}).get('color', 'NOT_FOUND')}") - print(f"πŸ” PDF PARAGRAPH COLOR FROM AI: {styles.get('paragraph', {}).get('color', 'NOT_FOUND')}") - else: - print(f"πŸ” PDF AI STYLING VALUE: {styles}") # Convert colors to PDF format after getting styles - print(f"πŸ” PDF BEFORE COLOR CONVERSION:") - for key, value in styles.items(): - print(f" {key}: {value}") - styles = self._convert_colors_format(styles) - print(f"πŸ” PDF AFTER COLOR CONVERSION:") - for key, value in styles.items(): - print(f" {key}: {value}") - # Validate and fix contrast issues return self._validate_pdf_styles_contrast(styles) @@ -255,11 +230,8 @@ class RendererPdf(BaseRenderer): self.logger.warning(f"AI styling returned invalid JSON: {json_error}") # Use print instead of logger to avoid truncation - print(f"πŸ” FULL AI RESPONSE THAT FAILED TO PARSE:") - print("=" * 100) - print(result) - print("=" * 100) - print(f"πŸ” RESPONSE LENGTH: {len(result)} characters") + self.services.utils.debugLogToFile(f"FULL AI RESPONSE THAT FAILED TO PARSE: {result}", "PDF_RENDERER") + self.services.utils.debugLogToFile(f"RESPONSE LENGTH: {len(result)} characters", "PDF_RENDERER") self.logger.warning(f"Raw content that failed to parse: {result}") @@ -399,8 +371,8 @@ class RendererPdf(BaseRenderer): # DEBUG: Show what color and spacing is being used for title title_color = title_style_def.get("color", "#1F4E79") title_space_after = title_style_def.get("space_after", 30) - print(f"πŸ” PDF TITLE COLOR: {title_color} -> {self._hex_to_color(title_color)}") - print(f"πŸ” PDF TITLE SPACE_AFTER: {title_space_after}") + self.services.utils.debugLogToFile(f"PDF TITLE COLOR: {title_color} -> {self._hex_to_color(title_color)}", "PDF_RENDERER") + self.services.utils.debugLogToFile(f"PDF TITLE SPACE_AFTER: {title_space_after}", "PDF_RENDERER") return ParagraphStyle( 'CustomTitle', @@ -441,12 +413,35 @@ class RendererPdf(BaseRenderer): def _get_alignment(self, align: str) -> int: """Convert alignment string to reportlab alignment constant.""" + if not align or not isinstance(align, str): + return TA_LEFT + align_map = { "center": TA_CENTER, "left": TA_LEFT, - "justify": TA_JUSTIFY + "justify": TA_JUSTIFY, + "right": TA_LEFT, # ReportLab doesn't have TA_RIGHT, use LEFT as fallback + "0": TA_LEFT, # Handle numeric strings + "1": TA_CENTER, + "2": TA_JUSTIFY } - return align_map.get(align.lower(), TA_LEFT) + return align_map.get(align.lower().strip(), TA_LEFT) + + def _get_table_alignment(self, align: str) -> str: + """Convert alignment string to ReportLab table alignment string.""" + if not align or not isinstance(align, str): + return 'LEFT' + + align_map = { + "center": 'CENTER', + "left": 'LEFT', + "justify": 'LEFT', # Tables don't support justify, use LEFT + "right": 'RIGHT', + "0": 'LEFT', # Handle numeric strings + "1": 'CENTER', + "2": 'LEFT' # Tables don't support justify, use LEFT + } + return align_map.get(align.lower().strip(), 'LEFT') def _hex_to_color(self, hex_color: str) -> colors.Color: """Convert hex color to reportlab color.""" @@ -518,7 +513,7 @@ class RendererPdf(BaseRenderer): table_style = [ ('BACKGROUND', (0, 0), (-1, 0), self._hex_to_color(table_header_style.get("background", "#4F4F4F"))), ('TEXTCOLOR', (0, 0), (-1, 0), self._hex_to_color(table_header_style.get("text_color", "#FFFFFF"))), - ('ALIGN', (0, 0), (-1, -1), self._get_alignment(table_cell_style.get("align", "left"))), + ('ALIGN', (0, 0), (-1, -1), self._get_table_alignment(table_cell_style.get("align", "left"))), ('FONTNAME', (0, 0), (-1, 0), 'Helvetica-Bold' if table_header_style.get("bold", True) else 'Helvetica'), ('FONTSIZE', (0, 0), (-1, 0), table_header_style.get("font_size", 12)), ('BOTTOMPADDING', (0, 0), (-1, 0), 12), diff --git a/modules/services/serviceGeneration/renderers/rendererXlsx.py b/modules/services/serviceGeneration/renderers/rendererXlsx.py index 9885988d..90487bcd 100644 --- a/modules/services/serviceGeneration/renderers/rendererXlsx.py +++ b/modules/services/serviceGeneration/renderers/rendererXlsx.py @@ -202,8 +202,8 @@ class RendererXlsx(BaseRenderer): """Generate Excel content from structured JSON document using AI-generated styling.""" try: # Debug output - print(f"πŸ” EXCEL JSON CONTENT TYPE: {type(json_content)}") - print(f"πŸ” EXCEL JSON CONTENT KEYS: {list(json_content.keys()) if isinstance(json_content, dict) else 'Not a dict'}") + self.services.utils.debugLogToFile(f"EXCEL JSON CONTENT TYPE: {type(json_content)}", "EXCEL_RENDERER") + self.services.utils.debugLogToFile(f"EXCEL JSON CONTENT KEYS: {list(json_content.keys()) if isinstance(json_content, dict) else 'Not a dict'}", "EXCEL_RENDERER") # Get AI-generated styling definitions styles = await self._get_excel_styles(user_prompt, ai_service) @@ -223,7 +223,7 @@ class RendererXlsx(BaseRenderer): # Create sheets based on content sheets = self._create_excel_sheets(wb, json_content, styles) - print(f"πŸ” EXCEL SHEETS CREATED: {list(sheets.keys()) if sheets else 'None'}") + self.services.utils.debugLogToFile(f"EXCEL SHEETS CREATED: {list(sheets.keys()) if sheets else 'None'}", "EXCEL_RENDERER") # Populate sheets with content self._populate_excel_sheets(sheets, json_content, styles) @@ -235,12 +235,12 @@ class RendererXlsx(BaseRenderer): # Convert to base64 excel_bytes = buffer.getvalue() - print(f"πŸ” EXCEL BYTES LENGTH: {len(excel_bytes)}") + self.services.utils.debugLogToFile(f"EXCEL BYTES LENGTH: {len(excel_bytes)}", "EXCEL_RENDERER") try: excel_base64 = base64.b64encode(excel_bytes).decode('utf-8') - print(f"πŸ” EXCEL BASE64 LENGTH: {len(excel_base64)}") + self.services.utils.debugLogToFile(f"EXCEL BASE64 LENGTH: {len(excel_base64)}", "EXCEL_RENDERER") except Exception as b64_error: - print(f"πŸ” BASE64 ENCODING ERROR: {b64_error}") + self.services.utils.debugLogToFile(f"BASE64 ENCODING ERROR: {b64_error}", "EXCEL_RENDERER") raise return excel_base64 @@ -285,10 +285,6 @@ class RendererXlsx(BaseRenderer): import json import re - # Debug output - print(f"πŸ” AI STYLING RESPONSE TYPE: {type(response)}") - print(f"πŸ” AI STYLING RESPONSE LENGTH: {len(response.content) if response and hasattr(response, 'content') and response.content else 0}") - # Clean and parse JSON result = response.content.strip() if response and response.content else "" @@ -301,23 +297,20 @@ class RendererXlsx(BaseRenderer): json_match = re.search(r'```json\s*\n(.*?)\n```', result, re.DOTALL) if json_match: result = json_match.group(1).strip() - print(f"πŸ” EXTRACTED JSON FROM MARKDOWN: {result[:100]}...") + self.services.utils.debugLogToFile(f"EXTRACTED JSON FROM MARKDOWN: {result[:100]}...", "EXCEL_RENDERER") elif result.startswith('```json'): result = re.sub(r'^```json\s*', '', result) result = re.sub(r'\s*```$', '', result) - print(f"πŸ” CLEANED JSON FROM MARKDOWN: {result[:100]}...") + self.services.utils.debugLogToFile(f"CLEANED JSON FROM MARKDOWN: {result[:100]}...", "EXCEL_RENDERER") elif result.startswith('```'): result = re.sub(r'^```\s*', '', result) result = re.sub(r'\s*```$', '', result) - print(f"πŸ” CLEANED JSON FROM GENERIC MARKDOWN: {result[:100]}...") + self.services.utils.debugLogToFile(f"CLEANED JSON FROM GENERIC MARKDOWN: {result[:100]}...", "EXCEL_RENDERER") # Try to parse JSON try: styles = json.loads(result) - print(f"πŸ” AI STYLING PARSED KEYS: {list(styles.keys()) if isinstance(styles, dict) else 'Not a dict'}") except json.JSONDecodeError as json_error: - print(f"πŸ” AI STYLING JSON ERROR: {json_error}") - print(f"πŸ” AI STYLING RAW RESULT: {result[:200]}...") self.logger.warning(f"AI styling returned invalid JSON: {json_error}, using defaults") return default_styles @@ -352,23 +345,19 @@ class RendererXlsx(BaseRenderer): def _convert_colors_format(self, styles: Dict[str, Any]) -> Dict[str, Any]: """Convert hex colors to aRGB format for Excel compatibility.""" try: - print(f"πŸ” CONVERTING COLORS IN STYLES: {styles}") + self.services.utils.debugLogToFile(f"CONVERTING COLORS IN STYLES: {styles}", "EXCEL_RENDERER") for style_name, style_config in styles.items(): if isinstance(style_config, dict): for prop, value in style_config.items(): if isinstance(value, str) and value.startswith('#') and len(value) == 7: # Convert #RRGGBB to #AARRGGBB (add FF alpha channel) - old_value = value styles[style_name][prop] = f"FF{value[1:]}" - print(f"πŸ” CONVERTED COLOR: {old_value} β†’ {styles[style_name][prop]}") elif isinstance(value, str) and value.startswith('#') and len(value) == 9: - print(f"πŸ” COLOR ALREADY aRGB: {value}") + pass # Already aRGB format elif isinstance(value, str) and value.startswith('#'): - print(f"πŸ” UNEXPECTED COLOR FORMAT: {value} (length: {len(value)})") - print(f"πŸ” FINAL CONVERTED STYLES: {styles}") + pass # Unexpected format, keep as is return styles except Exception as e: - print(f"πŸ” COLOR CONVERSION ERROR: {e}") return styles def _validate_excel_styles_contrast(self, styles: Dict[str, Any]) -> Dict[str, Any]: @@ -426,7 +415,7 @@ class RendererXlsx(BaseRenderer): # Get sheet names from AI styles or generate based on content sheet_names = styles.get("sheet_names", self._generate_sheet_names_from_content(json_content)) - print(f"πŸ” EXCEL SHEET NAMES: {sheet_names}") + self.services.utils.debugLogToFile(f"EXCEL SHEET NAMES: {sheet_names}", "EXCEL_RENDERER") # Create sheets for i, sheet_name in enumerate(sheet_names): @@ -562,15 +551,11 @@ class RendererXlsx(BaseRenderer): # Safety check for title style title_style = styles.get("title", {"font_size": 16, "bold": True, "color": "#FF1F4E79", "align": "center"}) - print(f"πŸ” EXCEL TITLE STYLE: {title_style}") - print(f"πŸ” EXCEL TITLE COLOR: {title_style['color']} (type: {type(title_style['color'])}, length: {len(title_style['color']) if isinstance(title_style['color'], str) else 'not string'})") try: safe_color = self._get_safe_color(title_style["color"]) sheet['A1'].font = Font(size=title_style["font_size"], bold=title_style["bold"], color=safe_color) sheet['A1'].alignment = Alignment(horizontal=title_style["align"]) - print(f"πŸ” EXCEL TITLE FONT CREATED SUCCESSFULLY with color: {safe_color}") except Exception as font_error: - print(f"πŸ” EXCEL TITLE FONT ERROR: {font_error}") # Try with a safe color sheet['A1'].font = Font(size=title_style["font_size"], bold=title_style["bold"], color="FF000000") sheet['A1'].alignment = Alignment(horizontal=title_style["align"]) diff --git a/modules/services/serviceGeneration/subPromptBuilder.py b/modules/services/serviceGeneration/subPromptBuilder.py index f7054adb..00b9be5c 100644 --- a/modules/services/serviceGeneration/subPromptBuilder.py +++ b/modules/services/serviceGeneration/subPromptBuilder.py @@ -21,7 +21,8 @@ async def buildExtractionPrompt( renderer: _RendererLike, userPrompt: str, title: str, - aiService=None + aiService=None, + services=None ) -> str: """ Build the final extraction prompt by combining: @@ -35,7 +36,7 @@ async def buildExtractionPrompt( """ # Parse user prompt to separate extraction intent from generation format using AI - extractionIntent = await _parseExtractionIntent(userPrompt, outputFormat, aiService) + extractionIntent = await _parseExtractionIntent(userPrompt, outputFormat, aiService, services) # Import JSON schema for structured output from .subJsonSchema import get_document_subJsonSchema @@ -95,6 +96,14 @@ Content Types to Extract: 3. Headings: Extract with appropriate levels 4. Paragraphs: Extract as structured text 5. Code: Extract code blocks with language identification +6. Images: Analyze images and describe all visible content including text, tables, logos, graphics, layout, and visual elements + +Image Analysis Requirements: +- If you cannot analyze an image for any reason, explain why in the JSON response +- Describe everything you see in the image +- Include all text content, tables, logos, graphics, layout, and visual elements +- If the image is too small, corrupted, or unclear, explain this +- Always provide feedback - never return empty responses Return only the JSON structure with actual data from the documents. Do not include any text before or after the JSON. """.strip() @@ -103,7 +112,7 @@ Return only the JSON structure with actual data from the documents. Do not inclu finalPrompt = genericIntro # Debug output - print(f"πŸ” EXTRACTION INTENT: {extractionIntent}") + services.utils.debugLogToFile(f"EXTRACTION INTENT: Processed", "PROMPT_BUILDER") # Save full extraction prompt to debug file try: @@ -125,7 +134,8 @@ async def buildGenerationPrompt( outputFormat: str, userPrompt: str, title: str, - aiService=None + aiService=None, + services=None ) -> str: """ Use AI to build the generation prompt based on user intent and format requirements. @@ -140,7 +150,7 @@ async def buildGenerationPrompt( safeUserPrompt = userPrompt.replace('"', '\\"').replace("'", "\\'").replace('\n', ' ').replace('\r', ' ') # Debug output - print(f"πŸ” GENERATION PROMPT REQUEST: buildGenerationPrompt called with outputFormat='{outputFormat}', title='{title}'") + services.utils.debugLogToFile(f"GENERATION PROMPT REQUEST: buildGenerationPrompt called with outputFormat='{outputFormat}', title='{title}'", "PROMPT_BUILDER") # AI call to generate the appropriate generation prompt generationPromptRequest = f""" @@ -165,7 +175,7 @@ Return only the generation prompt, starting with "Generate a {outputFormat} docu """ # Call AI service to generate the prompt - print(f"πŸ” GENERATION PROMPT REQUEST: Calling AI for generation prompt...") + services.utils.debugLogToFile("GENERATION PROMPT REQUEST: Calling AI for generation prompt...", "PROMPT_BUILDER") # Import and set proper options for AI call from modules.datamodels.datamodelAi import AiCallRequest, AiCallOptions, OperationType @@ -175,7 +185,6 @@ Return only the generation prompt, starting with "Generate a {outputFormat} docu request = AiCallRequest(prompt=generationPromptRequest, context="", options=request_options) response = await aiService.aiObjects.call(request) result = response.content if response else "" - print(f"πŸ” GENERATION PROMPT AI RESPONSE: '{result}'") # Replace the placeholder that the AI created with actual format rules if result: @@ -183,7 +192,7 @@ Return only the generation prompt, starting with "Generate a {outputFormat} docu result = result.replace("PLACEHOLDER_FOR_FORMAT_RULES", formatRules) # Debug output - print(f"πŸ” GENERATION PROMPT FINAL: {result if result else 'None'}") + services.utils.debugLogToFile(f"GENERATION PROMPT: Generated successfully", "PROMPT_BUILDER") # Save full generation prompt and AI response to debug file try: @@ -203,7 +212,7 @@ Return only the generation prompt, starting with "Generate a {outputFormat} docu except Exception as e: # Fallback on any error - preserve user prompt for language instructions - print(f"πŸ” DEBUG: AI generation prompt failed: {str(e)}") + services.utils.debugLogToFile(f"DEBUG: AI generation prompt failed: {str(e)}", "PROMPT_BUILDER") return f"Generate a comprehensive {outputFormat} document titled '{title}' based on the extracted content. User requirements: {userPrompt}" @@ -222,7 +231,7 @@ def _getFormatRules(outputFormat: str) -> str: """.strip() -async def _parseExtractionIntent(userPrompt: str, outputFormat: str, aiService=None) -> str: +async def _parseExtractionIntent(userPrompt: str, outputFormat: str, aiService=None, services=None) -> str: """ Use AI to extract the core content intention from the user prompt. Focus on WHAT the user wants to extract, not HOW to format it. @@ -250,7 +259,7 @@ Do not include formatting instructions, file types, or output methods. """ # Call AI service to extract intention - print(f"πŸ” DEBUG: Calling AI for extraction intent...") + services.utils.debugLogToFile("DEBUG: Calling AI for extraction intent...", "PROMPT_BUILDER") # Import and set proper options for AI call from modules.datamodels.datamodelAi import AiCallRequest, AiCallOptions, OperationType @@ -260,13 +269,13 @@ Do not include formatting instructions, file types, or output methods. request = AiCallRequest(prompt=extractionPrompt, context="", options=request_options) response = await aiService.aiObjects.call(request) result = response.content if response else "" - print(f"πŸ” DEBUG: AI extraction intent result: '{result}'") + services.utils.debugLogToFile(f"DEBUG: Extraction intent processed", "PROMPT_BUILDER") return result if result else f"Extract all relevant content from the document according to the user's requirements: {userPrompt}" except Exception as e: # Fallback on any error - preserve user prompt for language instructions - print(f"πŸ” DEBUG: AI extraction intent failed: {str(e)}") + services.utils.debugLogToFile(f"DEBUG: AI extraction intent failed: {str(e)}", "PROMPT_BUILDER") return f"Extract all relevant content from the document according to the user's requirements: {userPrompt}" diff --git a/modules/services/serviceNeutralization/mainServiceNeutralization.py b/modules/services/serviceNeutralization/mainServiceNeutralization.py index a76cf397..c48939f6 100644 --- a/modules/services/serviceNeutralization/mainServiceNeutralization.py +++ b/modules/services/serviceNeutralization/mainServiceNeutralization.py @@ -32,7 +32,7 @@ class NeutralizationService: serviceCenter: Service center instance for accessing other services NamesToParse: List of names to parse and replace (case-insensitive) """ - self.serviceCenter = serviceCenter + self.services = serviceCenter self.interfaceDbApp = serviceCenter.interfaceDbApp # Initialize anonymization processors diff --git a/modules/services/serviceSharepoint/mainServiceSharepoint.py b/modules/services/serviceSharepoint/mainServiceSharepoint.py index 0b692b4b..66a6a7cf 100644 --- a/modules/services/serviceSharepoint/mainServiceSharepoint.py +++ b/modules/services/serviceSharepoint/mainServiceSharepoint.py @@ -21,7 +21,7 @@ class SharepointService: Use setAccessTokenFromConnection() method to configure the access token before making API calls. """ - self.serviceCenter = serviceCenter + self.services = serviceCenter self.access_token = None self.base_url = "https://graph.microsoft.com/v1.0" diff --git a/modules/services/serviceTicket/mainServiceTicket.py b/modules/services/serviceTicket/mainServiceTicket.py index c94ee1eb..3f1f982b 100644 --- a/modules/services/serviceTicket/mainServiceTicket.py +++ b/modules/services/serviceTicket/mainServiceTicket.py @@ -16,7 +16,7 @@ class TicketService: Args: serviceCenter: Service center instance for accessing other services """ - self.serviceCenter = serviceCenter + self.services = serviceCenter async def _createTicketInterfaceByType( self, diff --git a/modules/services/serviceUtils/mainServiceUtils.py b/modules/services/serviceUtils/mainServiceUtils.py index 90a18daa..8379382a 100644 --- a/modules/services/serviceUtils/mainServiceUtils.py +++ b/modules/services/serviceUtils/mainServiceUtils.py @@ -4,6 +4,7 @@ Provides centralized access to configuration, events, and other utilities. """ import logging +import os from typing import Any, Optional, Dict, Callable from modules.shared.configuration import APP_CONFIG from modules.shared.eventManagement import eventManager @@ -139,4 +140,43 @@ class UtilsService: return TokenManager().getFreshToken(connectionId) except Exception as e: logger.error(f"Error getting fresh token for connection {connectionId}: {str(e)}") - return None \ No newline at end of file + return None + + def debugLogToFile(self, message: str, context: str = "DEBUG"): + """ + Log debug message to file if debug logging is enabled. + + Args: + message: Debug message to log + context: Context identifier for the debug message + """ + try: + # Check if debug logging is enabled + debug_enabled = self.configGet("APP_DEBUG_CHAT_WORKFLOW_ENABLED", False) + if not debug_enabled: + return + + # Get debug directory + debug_dir = self.configGet("APP_DEBUG_CHAT_WORKFLOW_DIR", "./test-chat") + if not os.path.isabs(debug_dir): + # If relative path, make it relative to the gateway directory + gateway_dir = os.path.dirname(os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))) + debug_dir = os.path.join(gateway_dir, debug_dir) + + # Ensure debug directory exists + os.makedirs(debug_dir, exist_ok=True) + + # Create debug file path + debug_file = os.path.join(debug_dir, "debug_workflow.log") + + # Format the debug entry + timestamp = self.getUtcTimestamp() + debug_entry = f"[{timestamp}] [{context}] {message}\n" + + # Write to debug file + with open(debug_file, "a", encoding="utf-8") as f: + f.write(debug_entry) + + except Exception as e: + # Don't log debug errors to avoid recursion + pass \ No newline at end of file diff --git a/modules/services/serviceWorkflow/mainServiceWorkflow.py b/modules/services/serviceWorkflow/mainServiceWorkflow.py index c26de8c3..1bb5607c 100644 --- a/modules/services/serviceWorkflow/mainServiceWorkflow.py +++ b/modules/services/serviceWorkflow/mainServiceWorkflow.py @@ -16,7 +16,7 @@ class WorkflowService: """Service class containing methods for document processing, chat operations, and workflow management""" def __init__(self, serviceCenter): - self.serviceCenter = serviceCenter + self.services = serviceCenter self.user = serviceCenter.user self.workflow = serviceCenter.workflow self.interfaceDbChat = serviceCenter.interfaceDbChat @@ -79,7 +79,7 @@ class WorkflowService: """Get ChatDocuments from a list of document references using all three formats.""" try: # Get the current workflow from services (same pattern as setWorkflowContext) - workflow = getattr(self.serviceCenter, 'currentWorkflow', None) or self.workflow + workflow = getattr(self.services, 'currentWorkflow', None) or self.workflow if not workflow: logger.error("No workflow available for document list resolution") return [] @@ -241,7 +241,8 @@ class WorkflowService: token_status = f"error: {str(e)}" # Build enhanced reference with state information - base_ref = f"connection:{connection.authority.value}:{connection.externalUsername}:{connection.id}" + # Format: connection:msft: (without UUID) + base_ref = f"connection:{connection.authority.value}:{connection.externalUsername}" state_info = f" [status:{connection.status.value}, token:{token_status}]" logger.debug(f"getConnectionReferenceFromUserConnection: Built reference: {base_ref + state_info}") @@ -264,26 +265,25 @@ class WorkflowService: return None def getUserConnectionFromConnectionReference(self, connectionReference: str) -> Optional[UserConnection]: - """Get UserConnection from reference string (handles both old and enhanced formats)""" + """Get UserConnection from reference string (handles new format without UUID)""" try: - # Parse reference format: connection:{authority}:{username}:{id} [status:..., token:...] + # Parse reference format: connection:{authority}:{username} [status:..., token:...] # Remove state information if present base_reference = connectionReference.split(' [')[0] parts = base_reference.split(':') - if len(parts) != 4 or parts[0] != "connection": + if len(parts) != 3 or parts[0] != "connection": return None authority = parts[1] username = parts[2] - conn_id = parts[3] # Get user connections through AppObjects interface user_connections = self.interfaceDbApp.getUserConnections(self.user.id) - # Find matching connection + # Find matching connection by authority and username (no UUID needed) for conn in user_connections: - if str(conn.id) == conn_id and conn.authority.value == authority and conn.externalUsername == username: + if conn.authority.value == authority and conn.externalUsername == username: return conn return None @@ -419,7 +419,7 @@ class WorkflowService: """Set current workflow context for document generation and routing""" try: # Get the current workflow from services - workflow = getattr(self.serviceCenter, 'currentWorkflow', None) or self.workflow + workflow = getattr(self.services, 'currentWorkflow', None) or self.workflow if not workflow: logger.error("No workflow available for context setting") return @@ -530,7 +530,7 @@ class WorkflowService: """Get document count for task planning (matching old handlingTasks.py logic)""" try: # Get the current workflow from services - workflow = getattr(self.serviceCenter, 'currentWorkflow', None) or self.workflow + workflow = getattr(self.services, 'currentWorkflow', None) or self.workflow if not workflow: return "No documents available" @@ -552,7 +552,7 @@ class WorkflowService: """Get workflow history context for task planning (matching old handlingTasks.py logic)""" try: # Get the current workflow from services - workflow = getattr(self.serviceCenter, 'currentWorkflow', None) or self.workflow + workflow = getattr(self.services, 'currentWorkflow', None) or self.workflow if not workflow: return "No previous round context available" @@ -832,14 +832,14 @@ class WorkflowService: """Get connection reference list (matching old handlingTasks.py logic)""" try: # Get connections from the database using the same logic as the old system - if hasattr(self.serviceCenter, 'interfaceDbApp') and hasattr(self.serviceCenter, 'user'): - userId = self.serviceCenter.user.id - connections = self.serviceCenter.interfaceDbApp.getUserConnections(userId) + if hasattr(self.services, 'interfaceDbApp') and hasattr(self.services, 'user'): + userId = self.services.user.id + connections = self.services.interfaceDbApp.getUserConnections(userId) if connections: # Format connections as reference strings using the same pattern as the old system connectionRefs = [] for conn in connections: - # Create reference string in format: connection:{authority}:{username}:{id} [status:..., token:...] + # Create reference string in format: connection:{authority}:{username} [status:..., token:...] # This matches the format expected by getUserConnectionFromConnectionReference() ref = self.getConnectionReferenceFromUserConnection(conn) connectionRefs.append(ref) diff --git a/modules/workflows/processing/shared/promptGenerationActionsReact.py b/modules/workflows/processing/shared/promptGenerationActionsReact.py index 7d3591d3..fc5ca265 100644 --- a/modules/workflows/processing/shared/promptGenerationActionsReact.py +++ b/modules/workflows/processing/shared/promptGenerationActionsReact.py @@ -64,7 +64,7 @@ REPLY: Return ONLY a JSON object with the following structure (no comments, no e EXAMPLE how to assign references from AVAILABLE_DOCUMENTS_INDEX and AVAILABLE_CONNECTIONS_INDEX: "requiredInputDocuments": ["docList:msg_47a7a578-e8f2-4ba8-ac66-0dbff40605e0:round8_task1_action1_results","docItem:5d8b7aee-b546-4487-b6a8-835c86f7b186:AI_Generated_Document_20251006-104256.docx"], -"requiredConnection": "connection:msft:p.motsch@valueon.ch:1ae8b8e5-128b-49b8-b1cb-7c632669eeae", +"requiredConnection": "connection:msft:p.motsch@valueon.ch", RULES: 1. Use EXACT action names from AVAILABLE_METHODS diff --git a/requirements.txt b/requirements.txt index f5a1a2dc..0ff02f00 100644 --- a/requirements.txt +++ b/requirements.txt @@ -41,6 +41,7 @@ markdown ## Web Scraping & HTTP beautifulsoup4==4.12.2 # Required for HTML/XML parsing requests==2.31.0 +requests-oauthlib==1.3.1 # Required for Google OAuth2Session chardet>=5.0.0 # FΓΌr Zeichensatzerkennung bei Webinhalten aiohttp>=3.8.0 # Required for SharePoint operations (async HTTP) selenium>=4.15.0 # Required for web automation and JavaScript-heavy pages diff --git a/test_document_processing.py b/test_document_processing.py index c16add06..777b0ddf 100644 --- a/test_document_processing.py +++ b/test_document_processing.py @@ -20,9 +20,15 @@ from modules.services.serviceAi.mainServiceAi import AiService from modules.services.serviceGeneration.mainServiceGeneration import GenerationService # Set up logging -logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(name)s - %(levelname)s - %(message)s') +logging.basicConfig(level=logging.DEBUG, format='%(asctime)s - %(name)s - %(levelname)s - %(message)s') logger = logging.getLogger(__name__) +# Set all module loggers to DEBUG level +logging.getLogger('modules.services.serviceAi.mainServiceAi').setLevel(logging.DEBUG) +logging.getLogger('modules.services.serviceGeneration.mainServiceGeneration').setLevel(logging.DEBUG) +logging.getLogger('modules.services.serviceGeneration.subPromptBuilder').setLevel(logging.DEBUG) +logging.getLogger('modules.services.serviceExtraction.mainServiceExtraction').setLevel(logging.DEBUG) + async def process_documents_and_generate_summary(): """Process documents using the main AI service with intelligent chunk integration.""" @@ -86,9 +92,50 @@ async def process_documents_and_generate_summary(): db_interface_module.getInterface = lambda: TestDbInterface(file_data_map) logger.info("πŸ”§ Database interface mocked successfully") + # Create a mock service center with utils + class MockServiceCenter: + def __init__(self): + self.utils = MockUtils() + + class MockUtils: + def debugLogToFile(self, message, label): + logger.debug(f"[{label}] {message}") + print(f"DEBUG [{label}]: {message}") # Also print to console for visibility + + # Only write to debug file if debug logging is enabled (matching real implementation) + debug_enabled = self.configGet("APP_DEBUG_CHAT_WORKFLOW_ENABLED", False) + if debug_enabled: + try: + import os + from datetime import datetime, UTC + debug_dir = self.configGet("APP_DEBUG_CHAT_WORKFLOW_DIR", "./test-chat") + if not os.path.isabs(debug_dir): + # If relative path, make it relative to the gateway directory + gateway_dir = os.path.dirname(os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))) + debug_dir = os.path.join(gateway_dir, debug_dir) + + os.makedirs(debug_dir, exist_ok=True) + debug_file = os.path.join(debug_dir, "debug_workflow.log") + timestamp = datetime.now(UTC).strftime("%Y-%m-%d %H:%M:%S.%f")[:-3] + debug_entry = f"[{timestamp}] [{label}] {message}\n" + with open(debug_file, "a", encoding="utf-8") as f: + f.write(debug_entry) + except Exception: + pass # Don't fail on debug logging errors + + def configGet(self, key, default): + # Return debug settings + if key == "APP_DEBUG_CHAT_WORKFLOW_ENABLED": + return True + elif key == "APP_DEBUG_CHAT_WORKFLOW_DIR": + return "./test-chat" + return default + + mock_service_center = MockServiceCenter() + # Initialize the main AI service - let it handle everything logger.info("πŸ”§ Initializing main AI service...") - ai_service = await AiService.create() + ai_service = await AiService.create(mock_service_center) # Create test documents - the AI service will handle file access internally documents = [] @@ -152,9 +199,9 @@ async def process_documents_and_generate_summary(): # Run a single end-to-end test to avoid the loop issue logger.info("πŸ§ͺ Running single end-to-end test...") - # userPrompt = "Analyze these documents and create a comprehensive DOCX summary document including: 1) Document types and purposes, 2) Key information and main points, 3) Important details and numbers, 4) Notable sections, 5) Overall assessment and recommendations." + userPrompt = "Analyze these documents and create a comprehensive summary for all input documents, each input document in a separate chapter summarized in 10-20 sentences." - userPrompt = "Analyze these documents and create a fitting image for the content" + # userPrompt = "Analyze these documents and create a fitting image for the content" # userPrompt = "Extract the table from file and produce 2 lists in excel. one list with all entries, one list only with entries that are yellow highlighted." @@ -168,7 +215,7 @@ async def process_documents_and_generate_summary(): prompt=userPrompt, documents=documents, options=ai_options, - outputFormat="txt", + outputFormat="docx", title="Formulaire" ) @@ -299,16 +346,30 @@ async def process_documents_and_generate_summary(): logger.info(f"βœ… Document saved as text: {output_path} ({len(doc_data)} characters)") elif file_ext in ['.png', '.jpg', '.jpeg']: # Image formats - decode from base64 - doc_bytes = base64.b64decode(doc_data) - with open(output_path, 'wb') as f: - f.write(doc_bytes) - logger.info(f"βœ… Image saved: {output_path} ({len(doc_bytes)} bytes)") + try: + doc_bytes = base64.b64decode(doc_data) + with open(output_path, 'wb') as f: + f.write(doc_bytes) + logger.info(f"βœ… Image saved: {output_path} ({len(doc_bytes)} bytes)") + except Exception as e: + logger.warning(f"⚠️ Failed to decode image as base64: {e}") + # Save as text if base64 decoding fails + with open(output_path, 'w', encoding='utf-8') as f: + f.write(doc_data) + logger.info(f"βœ… Image saved as text (fallback): {output_path}") else: # Other binary formats - decode from base64 - doc_bytes = base64.b64decode(doc_data) - with open(output_path, 'wb') as f: - f.write(doc_bytes) - logger.info(f"βœ… Document saved as binary: {output_path} ({len(doc_bytes)} bytes)") + try: + doc_bytes = base64.b64decode(doc_data) + with open(output_path, 'wb') as f: + f.write(doc_bytes) + logger.info(f"βœ… Document saved as binary: {output_path} ({len(doc_bytes)} bytes)") + except Exception as e: + logger.warning(f"⚠️ Failed to decode document as base64: {e}") + # Save as text if base64 decoding fails + with open(output_path, 'w', encoding='utf-8') as f: + f.write(doc_data) + logger.info(f"βœ… Document saved as text (fallback): {output_path}") # Also save raw content as text content = response.get('content', '') @@ -420,6 +481,23 @@ async def process_documents_and_generate_summary(): logger.info(f"βœ… Comprehensive test report saved: {report_path}") + # Show debug file locations + debug_files = [] + try: + debug_dir = Path("test-chat") + if debug_dir.exists(): + debug_files.extend(list(debug_dir.glob("*.log"))) + debug_files.extend(list(debug_dir.glob("ai/*.txt"))) + + if debug_files: + logger.info("πŸ“ Debug files created:") + for debug_file in debug_files: + logger.info(f" - {debug_file}") + else: + logger.info("πŸ“ No debug files found in test-chat directory") + except Exception as e: + logger.warning(f"Could not list debug files: {e}") + # Restore original database interface db_interface_module.getInterface = original_get_interface diff --git a/tool_security_encrypt_all_env_files.py b/tool_security_encrypt_all_env_files.py new file mode 100644 index 00000000..e3c40bea --- /dev/null +++ b/tool_security_encrypt_all_env_files.py @@ -0,0 +1,422 @@ +#!/usr/bin/env python3 +""" +Tool for encrypting all *_SECRET variables in all environment files. + +This tool automatically processes all three environment files (dev, int, prod) +and encrypts any unencrypted *_SECRET variables using the appropriate encryption +keys for each environment. + +Usage: + # Encrypt all secrets in all environment files + python tool_security_encrypt_all_env_files.py + + # Dry run - show what would be changed without making changes + python tool_security_encrypt_all_env_files.py --dry-run + + # Skip backup creation + python tool_security_encrypt_all_env_files.py --no-backup + + # Process only specific environment files + python tool_security_encrypt_all_env_files.py --files env_dev.env env_prod.env +""" + +import sys +import os +import argparse +import shutil +from pathlib import Path +from datetime import datetime +from typing import List, Dict, Any + +# Add the modules directory to the Python path +current_dir = Path(__file__).parent +modules_dir = current_dir / 'modules' +if modules_dir.exists(): + sys.path.insert(0, str(modules_dir)) +else: + print(f"Error: Modules directory not found: {modules_dir}") + print(f"Make sure you're running this script from the gateway directory") + sys.exit(1) + +# Import encryption functions +try: + from modules.shared.configuration import encrypt_value +except ImportError as e: + print(f"Error: Could not import encryption functions from shared.configuration: {e}") + print(f"Make sure you're running this script from the gateway directory") + print(f"Modules directory: {modules_dir}") + sys.exit(1) + +def get_env_type_from_file(file_path: Path) -> str: + """ + Read the APP_ENV_TYPE from the environment file. + + Args: + file_path: Path to the environment file + + Returns: + str: The environment type (dev, int, prod) or 'dev' as default + """ + if not file_path.exists(): + return 'dev' + + try: + with open(file_path, 'r', encoding='utf-8') as f: + for line in f: + line = line.strip() + if line.startswith('APP_ENV_TYPE') and '=' in line: + _, value = line.split('=', 1) + return value.strip().lower() + except Exception as e: + print(f"Warning: Could not read APP_ENV_TYPE from {file_path}: {e}") + + return 'dev' + +def is_any_encrypted_value(value: str) -> bool: + """ + Check if a value has any encryption prefix (DEV_ENC:, INT_ENC:, PROD_ENC:, etc.). + + Args: + value: The value to check + + Returns: + bool: True if the value has any encryption prefix, False otherwise + """ + if not value or not isinstance(value, str): + return False + + # Check for any environment-specific encryption prefixes + return (value.startswith('DEV_ENC:') or + value.startswith('INT_ENC:') or + value.startswith('PROD_ENC:') or + value.startswith('TEST_ENC:') or + value.startswith('STAGING_ENC:')) + +def find_secret_keys_in_file(file_path: Path) -> list: + """ + Find all *_SECRET keys in an environment file that are not encrypted. + + Args: + file_path: Path to the environment file + + Returns: + list: List of tuples (line_number, key, value, full_line) + """ + secret_keys = [] + + if not file_path.exists(): + return secret_keys + + # Get the environment type from the file itself + file_env_type = get_env_type_from_file(file_path) + + try: + with open(file_path, 'r', encoding='utf-8') as f: + lines = f.readlines() + + i = 0 + while i < len(lines): + line = lines[i].strip() + + # Skip empty lines and comments + if not line or line.startswith('#'): + i += 1 + continue + + # Check if line contains a key-value pair + if '=' in line: + key, value = line.split('=', 1) + key = key.strip() + value = value.strip() + + # Check if it's a secret key and not already encrypted with ANY prefix + if key.endswith('_SECRET') and value and not is_any_encrypted_value(value): + # Check if value starts with { (JSON object) + if value.startswith('{'): + # Collect all lines until we find the closing } + json_lines = [value] + start_line = i + 1 + i += 1 + brace_count = value.count('{') - value.count('}') + + while i < len(lines) and brace_count > 0: + json_lines.append(lines[i].rstrip('\n')) + brace_count += lines[i].count('{') - lines[i].count('}') + i += 1 + + # Join all lines and create the full JSON value + full_json_value = '\n'.join(json_lines) + secret_keys.append((start_line, key, full_json_value, line)) + i -= 1 # Adjust for the loop increment + else: + # Single line value + secret_keys.append((i + 1, key, value, line)) + # Check if it's a secret key with multiline JSON (value is just "{") + elif key.endswith('_SECRET') and value == '{' and not is_any_encrypted_value(value): + # Collect all lines until we find the closing } + json_lines = [value] + start_line = i + 1 + i += 1 + brace_count = 1 # We already have one opening brace + + while i < len(lines) and brace_count > 0: + json_lines.append(lines[i].rstrip('\n')) + brace_count += lines[i].count('{') - lines[i].count('}') + i += 1 + + # Join all lines and create the full JSON value + full_json_value = '\n'.join(json_lines) + secret_keys.append((start_line, key, full_json_value, line)) + i -= 1 # Adjust for the loop increment + + i += 1 + + except Exception as e: + print(f"Error reading {file_path}: {e}") + + return secret_keys + +def backup_file(file_path: Path) -> Path: + """ + Create a backup of the file before modification. + + Args: + file_path: Path to the file to backup + + Returns: + Path: Path to the backup file + """ + timestamp = datetime.now().strftime("%Y%m%d_%H%M%S") + backup_path = file_path.with_suffix(f'.{timestamp}.backup') + shutil.copy2(file_path, backup_path) + return backup_path + +def encrypt_all_secrets_in_file(file_path: Path, dry_run: bool = False, create_backup: bool = True) -> Dict[str, Any]: + """ + Encrypt all non-encrypted secrets in a file. + + Args: + file_path: Path to the environment file + dry_run: If True, only show what would be changed + create_backup: If True, create a backup before modifying + + Returns: + dict: Results of the encryption process + """ + # Get the environment type from the file itself + file_env_type = get_env_type_from_file(file_path) + + results = { + 'file': str(file_path), + 'env_type': file_env_type, + 'secrets_found': 0, + 'secrets_encrypted': 0, + 'errors': [], + 'backup_created': None + } + + # Find all secret keys + secret_keys = find_secret_keys_in_file(file_path) + results['secrets_found'] = len(secret_keys) + + if not secret_keys: + print(f" βœ… No unencrypted secrets found - all values already have encryption prefixes") + return results + + print(f" Found {len(secret_keys)} non-encrypted secrets") + + if dry_run: + print(" [DRY RUN] Would encrypt the following secrets:") + for line_num, key, value, full_line in secret_keys: + print(f" Line {line_num}: {key} = {value[:50]}{'...' if len(value) > 50 else ''}") + return results + + # Create backup if requested + if create_backup: + try: + backup_path = backup_file(file_path) + results['backup_created'] = str(backup_path) + print(f" πŸ“‹ Backup created: {backup_path.name}") + except Exception as e: + results['errors'].append(f"Failed to create backup: {e}") + print(f" ⚠️ Warning: Could not create backup: {e}") + + # Read the file content + try: + with open(file_path, 'r', encoding='utf-8') as f: + lines = f.readlines() + except Exception as e: + results['errors'].append(f"Failed to read file: {e}") + return results + + # Process each secret key + for line_num, key, value, full_line in secret_keys: + try: + print(f" πŸ” Encrypting {key}...") + + # Encrypt the value using the environment type from the file + encrypted_value = encrypt_value(value, file_env_type) + + # Replace the line in the file content + new_line = f"{key} = {encrypted_value}\n" + lines[line_num - 1] = new_line + + # If this was a multiline JSON, we need to remove the remaining lines + if value.startswith('{') and '\n' in value: + # Count how many lines the original JSON spanned + json_lines = value.split('\n') + lines_to_remove = len(json_lines) - 1 # -1 because we already replaced the first line + + # Remove the remaining lines + for i in range(line_num, line_num + lines_to_remove): + if i < len(lines): + lines[i] = "" + + results['secrets_encrypted'] += 1 + print(f" βœ“ Encrypted successfully") + + except Exception as e: + error_msg = f"Failed to encrypt {key}: {e}" + results['errors'].append(error_msg) + print(f" βœ— {error_msg}") + + # Write the modified content back to the file + if results['secrets_encrypted'] > 0: + try: + with open(file_path, 'w', encoding='utf-8') as f: + f.writelines(lines) + print(f" πŸ’Ύ File updated successfully") + except Exception as e: + results['errors'].append(f"Failed to write file: {e}") + print(f" βœ— Failed to write file: {e}") + + return results + +def process_all_env_files(env_files: List[str] = None, dry_run: bool = False, create_backup: bool = True) -> Dict[str, Any]: + """ + Process all environment files and encrypt unencrypted secrets. + + Args: + env_files: List of specific files to process (if None, processes all three default files) + dry_run: If True, only show what would be changed + create_backup: If True, create backups before modifying + + Returns: + dict: Summary of all processing results + """ + # Default environment files if none specified + if env_files is None: + env_files = ['env_dev.env', 'env_int.env', 'env_prod.env'] + + # Convert to Path objects and check if they exist + env_paths = [] + for env_file in env_files: + env_path = Path(env_file) + if not env_path.exists(): + print(f"⚠️ Warning: Environment file not found: {env_file}") + continue + env_paths.append(env_path) + + if not env_paths: + print("❌ No valid environment files found to process") + return {'total_files': 0, 'total_secrets_found': 0, 'total_secrets_encrypted': 0, 'total_errors': 0, 'files': []} + + print("πŸ” PowerOn Batch Secret Encryption Tool") + print("=" * 60) + print("⚠️ IMPORTANT: The tool will read APP_ENV_TYPE from each file itself") + print("⚠️ Each file will be processed with its own environment-specific encryption") + print() + + if dry_run: + print("πŸ” DRY RUN MODE - No changes will be made") + print() + + # Process each file + all_results = [] + total_secrets_found = 0 + total_secrets_encrypted = 0 + total_errors = 0 + + for env_path in env_paths: + print(f"\nπŸ“ Processing {env_path.name}:") + results = encrypt_all_secrets_in_file(env_path, dry_run, create_backup) + all_results.append(results) + + total_secrets_found += results['secrets_found'] + total_secrets_encrypted += results['secrets_encrypted'] + total_errors += len(results['errors']) + + # Summary + print("\n" + "=" * 60) + print("πŸ“Š SUMMARY") + print("=" * 60) + print(f"Files processed: {len(env_paths)}") + print(f"Total secrets found: {total_secrets_found}") + + if not dry_run: + print(f"Total secrets encrypted: {total_secrets_encrypted}") + print(f"Total errors: {total_errors}") + + if total_errors == 0 and total_secrets_encrypted > 0: + print("\nπŸŽ‰ All secrets encrypted successfully!") + elif total_errors > 0: + print(f"\n⚠️ Completed with {total_errors} errors") + else: + print("\nβœ… No secrets needed encryption") + else: + print(f"Secrets that would be encrypted: {total_secrets_found}") + + # Show backup information + backups_created = [r['backup_created'] for r in all_results if r['backup_created']] + if backups_created: + print(f"\nπŸ“‹ Backups created: {len(backups_created)}") + for backup in backups_created: + print(f" - {Path(backup).name}") + + # Show errors if any + all_errors = [] + for results in all_results: + all_errors.extend(results['errors']) + + if all_errors: + print(f"\n❌ Errors encountered:") + for error in all_errors: + print(f" - {error}") + + return { + 'total_files': len(env_paths), + 'total_secrets_found': total_secrets_found, + 'total_secrets_encrypted': total_secrets_encrypted, + 'total_errors': total_errors, + 'files': all_results + } + +def main(): + parser = argparse.ArgumentParser(description='Encrypt all *_SECRET variables in all environment files') + parser.add_argument('--files', '-f', nargs='+', + help='Specific environment files to process (default: all three env files)') + parser.add_argument('--dry-run', action='store_true', + help='Show what would be changed without making changes') + parser.add_argument('--no-backup', action='store_true', + help='Skip creating backup files') + + args = parser.parse_args() + + try: + results = process_all_env_files( + env_files=args.files, + dry_run=args.dry_run, + create_backup=not args.no_backup + ) + + # Return appropriate exit code + if results['total_errors'] > 0: + return 1 + return 0 + + except Exception as e: + print(f"Error: {e}") + return 1 + +if __name__ == '__main__': + sys.exit(main())