From 8259c198e74f53e0e76ecb2a5f6ad1412cac432f Mon Sep 17 00:00:00 2001 From: ValueOn AG Date: Fri, 12 Sep 2025 19:38:13 +0200 Subject: [PATCH] DELTA Jira sync excel operational --- force_reauth.py | 44 ++ modules/connectors/connectorSharepoint.py | 8 +- modules/interfaces/interfaceTicketObjects.py | 468 ++++++++++++++++++- modules/routes/routeSecurityMsft.py | 4 +- modules/services/serviceDeltaSync.py | 77 ++- test_excel_fix.py | 77 +++ 6 files changed, 652 insertions(+), 26 deletions(-) create mode 100644 force_reauth.py create mode 100644 test_excel_fix.py diff --git a/force_reauth.py b/force_reauth.py new file mode 100644 index 00000000..71a6df0c --- /dev/null +++ b/force_reauth.py @@ -0,0 +1,44 @@ +#!/usr/bin/env python3 +""" +Script to force Microsoft re-authentication for SharePoint access. +This will disconnect the existing connection and provide a new login URL. +""" + +import requests +import json + +# Configuration +BASE_URL = "http://localhost:8000" # Adjust if your server runs on different port +CONNECTION_ID = "cc62583d-3a68-44b6-8283-726725916a7e" # From the logs + +def force_reauth(): + """Force Microsoft re-authentication by disconnecting and providing new login URL.""" + + print("šŸ”„ Forcing Microsoft re-authentication for SharePoint access...") + + # Step 1: Disconnect existing connection + print(f"1. Disconnecting connection {CONNECTION_ID}...") + disconnect_url = f"{BASE_URL}/api/connections/{CONNECTION_ID}/disconnect" + + try: + response = requests.post(disconnect_url) + if response.status_code == 200: + print("āœ… Connection disconnected successfully") + else: + print(f"āŒ Failed to disconnect: {response.status_code} - {response.text}") + return + except Exception as e: + print(f"āŒ Error disconnecting: {e}") + return + + # Step 2: Get new login URL + print("2. Getting new Microsoft login URL...") + login_url = f"{BASE_URL}/api/msft/login?state=connection&connectionId={CONNECTION_ID}" + + print(f"\nšŸ”— Please visit this URL to re-authenticate with SharePoint permissions:") + print(f" {login_url}") + print("\nAfter re-authentication, the JIRA sync should work with SharePoint access.") + print("\nNote: The new token will include Sites.ReadWrite.All and Files.ReadWrite.All scopes.") + +if __name__ == "__main__": + force_reauth() diff --git a/modules/connectors/connectorSharepoint.py b/modules/connectors/connectorSharepoint.py index 89bdffbe..f501e710 100644 --- a/modules/connectors/connectorSharepoint.py +++ b/modules/connectors/connectorSharepoint.py @@ -406,8 +406,12 @@ class ConnectorSharepoint: logger.info(f"File copied: {source_file} -> {dest_file}") except Exception as e: - logger.error(f"Error copying file: {str(e)}") - raise + # Provide more specific error information + error_msg = str(e) + if "itemNotFound" in error_msg or "404" in error_msg: + raise Exception(f"Source file not found (404): {source_path} - {error_msg}") + else: + raise Exception(f"Error copying file: {error_msg}") async def download_file_by_path(self, site_id: str, file_path: str) -> Optional[bytes]: """Download a file by its path within a site.""" diff --git a/modules/interfaces/interfaceTicketObjects.py b/modules/interfaces/interfaceTicketObjects.py index 991c9da0..d1c9389a 100644 --- a/modules/interfaces/interfaceTicketObjects.py +++ b/modules/interfaces/interfaceTicketObjects.py @@ -2,6 +2,7 @@ from dataclasses import dataclass from io import BytesIO, StringIO from typing import Any import pandas as pd +import openpyxl from modules.shared.timezoneUtils import get_utc_now from modules.connectors.connectorSharepoint import ConnectorSharepoint @@ -48,13 +49,21 @@ class TicketSharepointSyncInterface: timestamp = get_utc_now().strftime("%Y%m%d_%H%M%S") backup_filename = f"backup_{timestamp}_{self.sync_file}" - await self.connector_sharepoint.copy_file_async( - site_id=self.site_id, - source_folder=self.sync_folder, - source_file=self.sync_file, - dest_folder=self.backup_folder, - dest_file=backup_filename, - ) + try: + await self.connector_sharepoint.copy_file_async( + site_id=self.site_id, + source_folder=self.sync_folder, + source_file=self.sync_file, + dest_folder=self.backup_folder, + dest_file=backup_filename, + ) + except Exception as e: + # If the source file doesn't exist (404 error), that's okay for first-time sync + if "itemNotFound" in str(e) or "404" in str(e) or "could not be found" in str(e): + raise Exception(f"Source file does not exist - no backup needed: {self.sync_file}") + else: + # Re-raise other errors + raise async def sync_from_jira_to_csv(self): """Syncs tasks from JIRA to a CSV file in SharePoint.""" @@ -369,6 +378,296 @@ class TicketSharepointSyncInterface: # Write audit log to SharePoint await self._write_audit_log(audit_log, "csv_to_jira") + async def sync_from_jira_to_excel(self): + """Syncs tasks from JIRA to an Excel file in SharePoint.""" + start_time = get_utc_now() + audit_log = [] + + audit_log.append("=== JIRA TO EXCEL SYNC STARTED ===") + audit_log.append(f"Start Time: {start_time.strftime('%Y-%m-%d %H:%M:%S')}") + audit_log.append(f"Sync File: {self.sync_file}") + audit_log.append(f"Sync Folder: {self.sync_folder}") + audit_log.append("") + + try: + # 1. Read JIRA tickets + audit_log.append("Step 1: Reading JIRA tickets...") + tickets = await self.connector_ticket.read_tasks(limit=0) + audit_log.append(f"JIRA issues read: {len(tickets)}") + audit_log.append("") + + # 2. Transform tasks according to task_sync_definition + audit_log.append("Step 2: Transforming JIRA data...") + transformed_tasks = self._transform_tasks(tickets, include_put=True) + jira_data = [task.data for task in transformed_tasks] + audit_log.append(f"JIRA issues transformed: {len(jira_data)}") + audit_log.append("") + + # 3. Create JIRA export file in audit folder + audit_log.append("Step 3: Creating JIRA export file...") + try: + timestamp = get_utc_now().strftime("%Y%m%d_%H%M%S") + jira_export_filename = f"jira_export_{timestamp}.xlsx" + # Use default headers for JIRA export + jira_export_content = self._create_excel_content(jira_data, {"header1": "JIRA Export", "header2": "Raw Data"}) + await self.connector_sharepoint.upload_file( + site_id=self.site_id, + folder_path=self.audit_folder, + file_name=jira_export_filename, + content=jira_export_content, + ) + audit_log.append(f"JIRA export file created: {jira_export_filename}") + except Exception as e: + audit_log.append(f"Failed to create JIRA export file: {str(e)}") + audit_log.append("") + + # 4. Create backup of existing Excel file (if it exists) + audit_log.append("Step 4: Creating backup...") + backup_created = False + try: + await self.create_backup() + backup_created = True + audit_log.append("Backup created successfully") + except Exception as e: + audit_log.append( + f"Backup creation failed (file might not exist): {str(e)}" + ) + audit_log.append("") + + # 5. Try to read existing Excel file from SharePoint + audit_log.append("Step 5: Reading existing Excel file...") + existing_data = [] + existing_file_found = False + existing_headers = {"header1": "Header 1", "header2": "Header 2"} + try: + file_path = f"{self.sync_folder}/{self.sync_file}" + excel_content = await self.connector_sharepoint.download_file_by_path( + site_id=self.site_id, file_path=file_path + ) + + # Parse Excel file with 4-row structure + existing_data, existing_headers = self._parse_excel_content(excel_content) + existing_file_found = True + audit_log.append( + f"Existing Excel file found with {len(existing_data)} records" + ) + audit_log.append(f"Preserved headers: Header1='{existing_headers['header1']}', Header2='{existing_headers['header2']}'") + except Exception as e: + audit_log.append(f"No existing Excel file found or read error: {str(e)}") + audit_log.append("") + + # 6. Merge JIRA data with existing data and track changes + audit_log.append("Step 6: Merging JIRA data with existing data...") + merged_data, change_details = self._merge_jira_with_existing_detailed( + jira_data, existing_data + ) + + # Log detailed changes + audit_log.append(f"Total records after merge: {len(merged_data)}") + audit_log.append(f"Records updated: {change_details['updated']}") + audit_log.append(f"Records added: {change_details['added']}") + audit_log.append(f"Records unchanged: {change_details['unchanged']}") + audit_log.append("") + + # Log individual changes + if change_details["changes"]: + audit_log.append("DETAILED CHANGES:") + for change in change_details["changes"]: + audit_log.append(f"- {change}") + audit_log.append("") + + # 7. Create Excel with 4-row structure and write to SharePoint + audit_log.append("Step 7: Writing updated Excel to SharePoint...") + excel_content = self._create_excel_content(merged_data, existing_headers) + await self.connector_sharepoint.upload_file( + site_id=self.site_id, + folder_path=self.sync_folder, + file_name=self.sync_file, + content=excel_content, + ) + audit_log.append("Excel file successfully written to SharePoint") + audit_log.append("") + + # Success summary + end_time = get_utc_now() + duration = (end_time - start_time).total_seconds() + audit_log.append("=== SYNC COMPLETED SUCCESSFULLY ===") + audit_log.append(f"End Time: {end_time.strftime('%Y-%m-%d %H:%M:%S')}") + audit_log.append(f"Duration: {duration:.2f} seconds") + audit_log.append(f"Total JIRA issues processed: {len(jira_data)}") + audit_log.append(f"Total records in final Excel: {len(merged_data)}") + + except Exception as e: + # Error handling + end_time = get_utc_now() + duration = (end_time - start_time).total_seconds() + audit_log.append("") + audit_log.append("=== SYNC FAILED ===") + audit_log.append(f"Error Time: {end_time.strftime('%Y-%m-%d %H:%M:%S')}") + audit_log.append(f"Duration before failure: {duration:.2f} seconds") + audit_log.append(f"Error: {str(e)}") + raise + finally: + # Write audit log to SharePoint + await self._write_audit_log(audit_log, "jira_to_excel") + + async def sync_from_excel_to_jira(self): + """Syncs tasks from an Excel file in SharePoint to JIRA.""" + start_time = get_utc_now() + audit_log = [] + + audit_log.append("=== EXCEL TO JIRA SYNC STARTED ===") + audit_log.append(f"Start Time: {start_time.strftime('%Y-%m-%d %H:%M:%S')}") + audit_log.append(f"Sync File: {self.sync_file}") + audit_log.append(f"Sync Folder: {self.sync_folder}") + audit_log.append("") + + try: + # 1. Read Excel file from SharePoint + audit_log.append("Step 1: Reading Excel file from SharePoint...") + try: + file_path = f"{self.sync_folder}/{self.sync_file}" + excel_content = await self.connector_sharepoint.download_file_by_path( + site_id=self.site_id, file_path=file_path + ) + # Parse Excel file with 4-row structure + excel_data, _ = self._parse_excel_content(excel_content) + audit_log.append( + f"Excel file read successfully with {len(excel_data)} records" + ) + except Exception as e: + audit_log.append(f"Failed to read Excel file: {str(e)}") + audit_log.append("Excel to JIRA sync aborted - no file to process") + return + audit_log.append("") + + # 2. Read current JIRA data for comparison + audit_log.append("Step 2: Reading current JIRA data for comparison...") + try: + current_jira_tasks = await self.connector_ticket.read_tasks(limit=0) + current_jira_data = self._transform_tasks( + current_jira_tasks, include_put=True + ) + jira_lookup = { + task.data.get("ID"): task.data for task in current_jira_data + } + audit_log.append(f"Current JIRA data read: {len(jira_lookup)} tasks") + except Exception as e: + audit_log.append(f"Failed to read current JIRA data: {str(e)}") + raise + audit_log.append("") + + # 3. Detect actual changes in "put" fields + audit_log.append("Step 3: Detecting changes in 'put' fields...") + actual_changes = {} + records_with_changes = 0 + total_changes = 0 + + for row in excel_data: + task_id = row.get("ID") + if not task_id or task_id not in jira_lookup: + continue + + current_jira_task = jira_lookup[task_id] + task_changes = {} + + for field_name, field_config in self.task_sync_definition.items(): + if field_config[0] == "put": # Only process "put" fields + excel_value = row.get(field_name, "") + jira_value = current_jira_task.get(field_name, "") + + # Convert None to empty string for comparison + excel_value = "" if excel_value is None else str(excel_value).strip() + jira_value = ( + "" if jira_value is None else str(jira_value).strip() + ) + + # Include if values are different (allow empty strings to clear fields like the reference does) + if excel_value != jira_value: + task_changes[field_name] = excel_value + + if task_changes: + actual_changes[task_id] = task_changes + records_with_changes += 1 + total_changes += len(task_changes) + + audit_log.append(f"Records with actual changes: {records_with_changes}") + audit_log.append(f"Total field changes detected: {total_changes}") + audit_log.append("") + + # Log detailed changes + if actual_changes: + audit_log.append("DETAILED CHANGES TO APPLY TO JIRA:") + for task_id, changes in actual_changes.items(): + change_list = [ + f"{field}: '{value}'" for field, value in changes.items() + ] + audit_log.append(f"- Task ID {task_id}: {', '.join(change_list)}") + audit_log.append("") + + # 4. Update JIRA tasks with actual changes + if actual_changes: + audit_log.append("Step 4: Updating JIRA tasks...") + + # Convert to Task objects for the connector + tasks_to_update = [] + for task_id, changes in actual_changes.items(): + # Create task data structure expected by JIRA connector + # Build the nested fields structure that JIRA expects + fields = {} + for field_name, new_value in changes.items(): + # Map back to JIRA field structure using task_sync_definition + field_config = self.task_sync_definition[field_name] + field_path = field_config[1] + + # Extract the JIRA field ID from the path + # For "put" fields, the path is like ['fields', 'customfield_10067'] + if len(field_path) >= 2 and field_path[0] == "fields": + jira_field_id = field_path[1] + fields[jira_field_id] = new_value + + if fields: + task_data = {"ID": task_id, "fields": fields} + task = Task(data=task_data) + tasks_to_update.append(task) + + # Write tasks back to JIRA + try: + await self.connector_ticket.write_tasks(tasks_to_update) + audit_log.append( + f"Successfully updated {len(tasks_to_update)} JIRA tasks" + ) + except Exception as e: + audit_log.append(f"Failed to update JIRA tasks: {str(e)}") + raise + else: + audit_log.append("Step 4: No changes to apply to JIRA") + audit_log.append("") + + # Success summary + end_time = get_utc_now() + duration = (end_time - start_time).total_seconds() + audit_log.append("=== SYNC COMPLETED SUCCESSFULLY ===") + audit_log.append(f"End Time: {end_time.strftime('%Y-%m-%d %H:%M:%S')}") + audit_log.append(f"Duration: {duration:.2f} seconds") + audit_log.append(f"Total Excel records processed: {len(excel_data)}") + audit_log.append(f"Records with actual changes: {records_with_changes}") + audit_log.append(f"JIRA tasks updated: {len(actual_changes)}") + + except Exception as e: + # Error handling + end_time = get_utc_now() + duration = (end_time - start_time).total_seconds() + audit_log.append("") + audit_log.append("=== SYNC FAILED ===") + audit_log.append(f"Error Time: {end_time.strftime('%Y-%m-%d %H:%M:%S')}") + audit_log.append(f"Duration before failure: {duration:.2f} seconds") + audit_log.append(f"Error: {str(e)}") + raise + finally: + # Write audit log to SharePoint + await self._write_audit_log(audit_log, "excel_to_jira") + def _transform_tasks( self, tasks: list[Task], include_put: bool = False ) -> list[Task]: @@ -656,3 +955,158 @@ class TicketSharepointSyncInterface: csv_text = StringIO() final_df.to_csv(csv_text, index=False, header=False, quoting=1, escapechar='\\') return csv_text.getvalue().encode("utf-8") + + def _create_excel_content(self, data: list[dict], existing_headers: dict = None) -> bytes: + """Create Excel content with 4-row structure matching reference code.""" + # Get current timestamp for header + timestamp = get_utc_now().strftime("%Y-%m-%d %H:%M:%S UTC") + + # Use existing headers if provided, otherwise use defaults + if existing_headers is None: + existing_headers = {"header1": "Header 1", "header2": "Header 2"} + + if not data: + # Build an empty table with the expected columns from schema + cols = list(self.task_sync_definition.keys()) + + df = pd.DataFrame(columns=cols) + + # Parse existing headers to extract individual columns + import csv as csv_module + header1_text = existing_headers.get("header1", "Header 1") + header2_text = existing_headers.get("header2", "Header 2") + + # Parse the existing header rows + header1_reader = csv_module.reader([header1_text]) + header2_reader = csv_module.reader([header2_text]) + header1_row = next(header1_reader, []) + header2_row = next(header2_reader, []) + + # Row 1: Use existing header1 or default + if len(header1_row) >= len(cols): + header_row1_data = header1_row[:len(cols)] + else: + header_row1_data = header1_row + [""] * (len(cols) - len(header1_row)) + header_row1 = pd.DataFrame([header_row1_data], columns=cols) + + # Row 2: Use existing header2 and add timestamp to second column + if len(header2_row) >= len(cols): + header_row2_data = header2_row[:len(cols)] + else: + header_row2_data = header2_row + [""] * (len(cols) - len(header2_row)) + if len(header_row2_data) > 1: + header_row2_data[1] = timestamp + header_row2 = pd.DataFrame([header_row2_data], columns=cols) + + # Row 3: table headers + table_headers = pd.DataFrame([cols], columns=cols) + + final_df = pd.concat( + [header_row1, header_row2, table_headers, df], ignore_index=True + ) + + # Create Excel file in memory + excel_buffer = BytesIO() + final_df.to_excel(excel_buffer, index=False, header=False, engine='openpyxl') + return excel_buffer.getvalue() + + # Create DataFrame from data + df = pd.DataFrame(data) + + # Force all columns to be object (string) type to preserve empty cells + for column in df.columns: + df[column] = df[column].astype("object") + df[column] = df[column].fillna("") + + # Clean data: replace actual line breaks with \n and escape quotes + for column in df.columns: + df[column] = df[column].astype(str).str.replace('\n', '\\n', regex=False) + df[column] = df[column].str.replace('"', '""', regex=False) + + # Create the 4-row structure + # Parse existing headers to extract individual columns + import csv as csv_module + header1_text = existing_headers.get("header1", "Header 1") + header2_text = existing_headers.get("header2", "Header 2") + + # Parse the existing header rows + header1_reader = csv_module.reader([header1_text]) + header2_reader = csv_module.reader([header2_text]) + header1_row = next(header1_reader, []) + header2_row = next(header2_reader, []) + + # Row 1: Use existing header1 or default + if len(header1_row) >= len(df.columns): + header_row1_data = header1_row[:len(df.columns)] + else: + header_row1_data = header1_row + [""] * (len(df.columns) - len(header1_row)) + header_row1 = pd.DataFrame([header_row1_data], columns=df.columns) + + # Row 2: Use existing header2 and add timestamp to second column + if len(header2_row) >= len(df.columns): + header_row2_data = header2_row[:len(df.columns)] + else: + header_row2_data = header2_row + [""] * (len(df.columns) - len(header2_row)) + if len(header_row2_data) > 1: + header_row2_data[1] = timestamp + header_row2 = pd.DataFrame([header_row2_data], columns=df.columns) + + # Row 3: Table headers (column names) + table_headers = pd.DataFrame([df.columns.tolist()], columns=df.columns) + + # Concatenate all rows: header1 + header2 + table_headers + data + final_df = pd.concat( + [header_row1, header_row2, table_headers, df], ignore_index=True + ) + + # Create Excel file in memory + excel_buffer = BytesIO() + final_df.to_excel(excel_buffer, index=False, header=False, engine='openpyxl') + return excel_buffer.getvalue() + + def _parse_excel_content(self, excel_content: bytes) -> tuple[list[dict], dict]: + """Parse Excel content with 4-row structure and return data and headers.""" + try: + # Load Excel file from bytes + df = pd.read_excel( + BytesIO(excel_content), + engine='openpyxl', + header=None + ) + + # Extract the 4 parts: + # Row 1: Static header row 1 + header_row1 = df.iloc[0:1].copy() + + # Row 2: Static header row 2 + header_row2 = df.iloc[1:2].copy() + + # Row 3: Table headers + table_headers = df.iloc[2:3].copy() + + # Row 4+: Data rows + df_data = df.iloc[3:].copy() + # Set column names from row 3 + df_data.columns = table_headers.iloc[0] + # Reset index to start from 0 + df_data = df_data.reset_index(drop=True) + + # Force all columns to be object (string) type and handle NaN values + for column in df_data.columns: + df_data[column] = df_data[column].astype('object') + # Fill NaN values with empty string to keep cells empty + df_data[column] = df_data[column].fillna('') + + # Convert DataFrame to list of dictionaries + data = df_data.to_dict(orient='records') + + # Extract headers as strings (like CSV version) + headers = { + "header1": ",".join([str(x) if pd.notna(x) else "" for x in header_row1.iloc[0].tolist()]), + "header2": ",".join([str(x) if pd.notna(x) else "" for x in header_row2.iloc[0].tolist()]) + } + + return data, headers + + except Exception as e: + raise Exception(f"Failed to parse Excel content: {str(e)}") diff --git a/modules/routes/routeSecurityMsft.py b/modules/routes/routeSecurityMsft.py index 1d4d8f10..d14a0555 100644 --- a/modules/routes/routeSecurityMsft.py +++ b/modules/routes/routeSecurityMsft.py @@ -44,7 +44,9 @@ SCOPES = [ "Mail.ReadWrite", # Read and write mail "Mail.Send", # Send mail "Mail.ReadWrite.Shared", # Access shared mailboxes - "User.Read" # Read user profile + "User.Read", # Read user profile + "Sites.ReadWrite.All", # Read and write all SharePoint sites + "Files.ReadWrite.All" # Read and write all files ] @router.get("/login") diff --git a/modules/services/serviceDeltaSync.py b/modules/services/serviceDeltaSync.py index 4e0a0874..a45200cf 100644 --- a/modules/services/serviceDeltaSync.py +++ b/modules/services/serviceDeltaSync.py @@ -25,17 +25,29 @@ APP_ENV_TYPE = APP_CONFIG.get("APP_ENV_TYPE", "dev") class ManagerSyncDelta: - """Manages JIRA to SharePoint synchronization for Delta Group.""" + """Manages JIRA to SharePoint synchronization for Delta Group. + + Supports two sync modes: + - CSV mode: Uses CSV files for synchronization (default) + - Excel mode: Uses Excel (.xlsx) files for synchronization + + To change sync mode, use the set_sync_mode() method or modify SYNC_MODE class variable. + """ SHAREPOINT_SITE_ID = "02830618-4029-4dc8-8d3d-f5168f282249" SHAREPOINT_SITE_NAME = "SteeringBPM" SHAREPOINT_SITE_PATH = "SteeringBPM" - SHAREPOINT_MAIN_FOLDER = "/sites/SteeringBPM/Freigegebene Dokumente/General/50 Docs hosted by SELISE" - SHAREPOINT_BACKUP_FOLDER = "/sites/SteeringBPM/Freigegebene Dokumente/General/50 Docs hosted by SELISE/SyncHistory" - SHAREPOINT_AUDIT_FOLDER = "/sites/SteeringBPM/Freigegebene Dokumente/General/50 Docs hosted by SELISE/SyncHistory" + SHAREPOINT_HOSTNAME = "deltasecurityag.sharepoint.com" + SHAREPOINT_MAIN_FOLDER = "/General/50 Docs hosted by SELISE" + SHAREPOINT_BACKUP_FOLDER = "/General/50 Docs hosted by SELISE/SyncHistory" + SHAREPOINT_AUDIT_FOLDER = "/General/50 Docs hosted by SELISE/SyncHistory" SHAREPOINT_USER_ID = "patrick.motsch@delta.ch" - # Fixed filename for the main CSV file (like original synchronizer) - SYNC_FILE_NAME = "DELTAgroup x SELISE Ticket Exchange List.csv" + # Sync mode: "csv" or "xlsx" + SYNC_MODE = "xlsx" # Can be "csv" or "xlsx" + + # File names for different sync modes + SYNC_FILE_CSV = "DELTAgroup x SELISE Ticket Exchange List.csv" + SYNC_FILE_XLSX = "DELTAgroup x SELISE Ticket Exchange List.xlsx" # JIRA connection parameters (hardcoded for Delta Group) JIRA_USERNAME = "p.motsch@valueon.ch" @@ -65,6 +77,30 @@ class ManagerSyncDelta: self.jira_connector = None self.sharepoint_connector = None self.target_site = None + + def get_sync_file_name(self) -> str: + """Get the appropriate sync file name based on the sync mode.""" + if self.SYNC_MODE == "xlsx": + return self.SYNC_FILE_XLSX + else: # Default to CSV + return self.SYNC_FILE_CSV + + def set_sync_mode(self, mode: str) -> bool: + """Set the sync mode to either 'csv' or 'xlsx'. + + Args: + mode: Either 'csv' or 'xlsx' + + Returns: + bool: True if mode was set successfully, False if invalid mode + """ + if mode.lower() in ["csv", "xlsx"]: + self.SYNC_MODE = mode.lower() + logger.info(f"Sync mode changed to: {self.SYNC_MODE}") + return True + else: + logger.error(f"Invalid sync mode: {mode}. Must be 'csv' or 'xlsx'") + return False async def initialize_connectors(self) -> bool: """Initialize JIRA and SharePoint connectors.""" @@ -159,33 +195,42 @@ class ManagerSyncDelta: async def sync_jira_to_sharepoint(self) -> bool: """Perform the main JIRA to SharePoint synchronization using sophisticated sync logic.""" try: - logger.info("Starting JIRA to SharePoint synchronization") + logger.info(f"Starting JIRA to SharePoint synchronization (Mode: {self.SYNC_MODE})") # Initialize connectors if not await self.initialize_connectors(): logger.error("Failed to initialize connectors") return False + # Get the appropriate sync file name based on mode + sync_file_name = self.get_sync_file_name() + logger.info(f"Using sync file: {sync_file_name}") + # Create the sophisticated sync interface sync_interface = await TicketSharepointSyncInterface.create( connector_ticket=self.jira_connector, connector_sharepoint=self.sharepoint_connector, task_sync_definition=self.TASK_SYNC_DEFINITION, sync_folder=self.SHAREPOINT_MAIN_FOLDER, - sync_file=self.SYNC_FILE_NAME, + sync_file=sync_file_name, backup_folder=self.SHAREPOINT_BACKUP_FOLDER, audit_folder=self.SHAREPOINT_AUDIT_FOLDER, site_id=self.target_site['id'] ) - # Perform the sophisticated sync - logger.info("Performing JIRA to CSV sync...") - await sync_interface.sync_from_jira_to_csv() - # TODO: Uncomment when CSV to JIRA sync is implemented - #logger.info("Performing CSV to JIRA sync...") - #await sync_interface.sync_from_csv_to_jira() + # Perform the sophisticated sync based on mode + if self.SYNC_MODE == "xlsx": + logger.info("Performing JIRA to Excel sync...") + await sync_interface.sync_from_jira_to_excel() + logger.info("Performing Excel to JIRA sync...") + await sync_interface.sync_from_excel_to_jira() + else: # CSV mode (default) + logger.info("Performing JIRA to CSV sync...") + await sync_interface.sync_from_jira_to_csv() + logger.info("Performing CSV to JIRA sync...") + await sync_interface.sync_from_csv_to_jira() - logger.info("JIRA to SharePoint synchronization completed successfully") + logger.info(f"JIRA to SharePoint synchronization completed successfully (Mode: {self.SYNC_MODE})") return True except Exception as e: @@ -205,7 +250,7 @@ async def perform_sync_jira_delta_group() -> bool: """ try: #TODO: ADAPT to prod - if APP_ENV_TYPE != "dev": + if APP_ENV_TYPE != "dev" and APP_ENV_TYPE != "prod": logger.info("JIRA to SharePoint synchronization: TASK to run only in PROD") return True diff --git a/test_excel_fix.py b/test_excel_fix.py new file mode 100644 index 00000000..17a57070 --- /dev/null +++ b/test_excel_fix.py @@ -0,0 +1,77 @@ +#!/usr/bin/env python3 +""" +Test script to verify the Excel header parsing fix +""" + +import sys +import os +import pandas as pd +from io import BytesIO + +# Add the gateway modules to the path +sys.path.append(os.path.join(os.path.dirname(__file__), 'modules')) + +from modules.interfaces.interfaceTicketObjects import TicketSharepointSyncInterface + +def test_excel_header_parsing(): + """Test the Excel header parsing fix""" + print("=== Testing Excel Header Parsing Fix ===\n") + + # Create a mock interface instance + interface = TicketSharepointSyncInterface( + connector_ticket=None, + connector_sharepoint=None, + task_sync_definition={ + "ID": ["get", ["id"]], + "Summary": ["get", ["fields", "summary"]], + "Status": ["get", ["fields", "status", "name"]], + "Assignee": ["put", ["fields", "assignee", "displayName"]] + }, + sync_folder="test", + sync_file="test.xlsx", + backup_folder="backup", + audit_folder="audit", + site_id="test" + ) + + # Test data + test_data = [ + {"ID": "TEST-1", "Summary": "Test Issue 1", "Status": "Open", "Assignee": "John Doe"}, + {"ID": "TEST-2", "Summary": "Test Issue 2", "Status": "Closed", "Assignee": "Jane Smith"}, + ] + + # Create Excel content + print("1. Creating Excel content...") + excel_content = interface._create_excel_content(test_data) + print(f" āœ“ Created Excel content: {len(excel_content)} bytes") + + # Parse it back + print("2. Parsing Excel content...") + try: + parsed_data, parsed_headers = interface._parse_excel_content(excel_content) + print(f" āœ“ Parsed Excel content: {len(parsed_data)} records") + print(f" āœ“ Headers type: header1={type(parsed_headers['header1'])}, header2={type(parsed_headers['header2'])}") + print(f" āœ“ Headers content: header1='{parsed_headers['header1']}', header2='{parsed_headers['header2']}'") + + # Test creating content with the parsed headers + print("3. Testing round-trip with parsed headers...") + new_excel_content = interface._create_excel_content(test_data, parsed_headers) + print(f" āœ“ Created new Excel content: {len(new_excel_content)} bytes") + + # Parse the new content + final_data, final_headers = interface._parse_excel_content(new_excel_content) + print(f" āœ“ Final parse successful: {len(final_data)} records") + print(f" āœ“ Final headers: header1='{final_headers['header1']}', header2='{final_headers['header2']}'") + + print("\nāœ… All tests passed! The header parsing fix works correctly.") + return True + + except Exception as e: + print(f" āœ— Error during parsing: {e}") + import traceback + traceback.print_exc() + return False + +if __name__ == "__main__": + success = test_excel_header_parsing() + exit(0 if success else 1)