From bacf2a96867060cf12ab8d282d89a76f82fb99bc Mon Sep 17 00:00:00 2001 From: Christopher Gondek Date: Wed, 3 Sep 2025 12:19:07 +0200 Subject: [PATCH 01/12] feat: add TicketInterface; add CRUD connector JIRA --- modules/connectors/connectorTicketJira.py | 240 +++++++++++++++++++ modules/interfaces/interfaceTicketModel.py | 26 ++ modules/interfaces/interfaceTicketObjects.py | 10 + 3 files changed, 276 insertions(+) create mode 100644 modules/connectors/connectorTicketJira.py create mode 100644 modules/interfaces/interfaceTicketModel.py create mode 100644 modules/interfaces/interfaceTicketObjects.py diff --git a/modules/connectors/connectorTicketJira.py b/modules/connectors/connectorTicketJira.py new file mode 100644 index 00000000..f2edd200 --- /dev/null +++ b/modules/connectors/connectorTicketJira.py @@ -0,0 +1,240 @@ +"""Jira connector for CRUD operations.""" + +from dataclasses import dataclass +import logging +import aiohttp +import json +from typing import Optional + +from modules.interfaces.interfaceTicketModel import ( + TicketBase, + TicketFieldAttribute, + Task, +) + + +logger = logging.getLogger(__name__) + + +@dataclass +class ConnectorTicketJira(TicketBase): + jira_username: str + jira_api_token: str + jira_url: str + project_code: str + issue_type: str + + @classmethod + async def create( + cls, + *, + jira_username: str, + jira_api_token: str, + jira_url: str, + project_code: str, + issue_type: str, + ): + return ConnectorTicketJira( + jira_username=jira_username, + jira_api_token=jira_api_token, + jira_url=jira_url, + project_code=project_code, + issue_type=issue_type, + ) + + async def read_attributes(self) -> list[TicketFieldAttribute]: + """ + Read field attributes from Jira by querying for a single issue + and extracting the field mappings. + + Returns: + list[TicketFieldAttribute]: List of field attributes with names and IDs + """ + jql_query = f"project={self.project_code} AND issuetype={self.issue_type}" + + # Prepare the request URL and parameters + url = f"{self.jira_url}/rest/api/2/search" + params = {"jql": jql_query, "maxResults": 1, "expand": "names"} + + # Prepare authentication + auth = aiohttp.BasicAuth(self.jira_username, self.jira_api_token) + + try: + async with aiohttp.ClientSession() as session: + async with session.get(url, params=params, auth=auth) as response: + if response.status != 200: + error_text = await response.text() + logger.error( + f"Jira API request failed with status {response.status}: {error_text}" + ) + raise Exception( + f"Jira API request failed with status {response.status}" + ) + + data = await response.json() + + # Extract issues and field names + issues = data.get("issues", []) + field_names = data.get("names", {}) + + if not issues: + logger.warning(f"No issues found for query: {jql_query}") + return [] + + # Extract field attributes from the first issue + attributes = [] + issue = issues[0] + fields = issue.get("fields", {}) + + for field_id, value in fields.items(): + field_name = field_names.get(field_id, field_id) + attributes.append( + TicketFieldAttribute(field_name=field_name, field=field_id) + ) + + logger.info( + f"Successfully retrieved {len(attributes)} field attributes from Jira" + ) + return attributes + + except aiohttp.ClientError as e: + logger.error(f"HTTP client error while fetching Jira attributes: {str(e)}") + raise Exception(f"Failed to connect to Jira: {str(e)}") + except json.JSONDecodeError as e: + logger.error(f"Failed to parse Jira API response: {str(e)}") + raise Exception(f"Invalid response from Jira API: {str(e)}") + except Exception as e: + logger.error(f"Unexpected error while fetching Jira attributes: {str(e)}") + raise + + async def read_tasks(self, *, limit: int = 0) -> list[Task]: + """ + Read tasks from Jira with pagination support. + + Args: + limit: Maximum number of tasks to retrieve. 0 means no limit. + + Returns: + list[Task]: List of tasks with their data + """ + jql_query = f"project={self.project_code} AND issuetype={self.issue_type}" + + # Initialize variables for pagination + start_at = 0 + max_results = 50 + total = 1 # Initialize with a value greater than 0 to enter the loop + tasks = [] + + # Prepare authentication + auth = aiohttp.BasicAuth(self.jira_username, self.jira_api_token) + url = f"{self.jira_url}/rest/api/2/search" + + try: + async with aiohttp.ClientSession() as session: + while start_at < total and (limit == 0 or len(tasks) < limit): + # Prepare request parameters + params = { + "jql": jql_query, + "startAt": start_at, + "maxResults": max_results, + } + + headers = {"Content-Type": "application/json"} + + async with session.get( + url, params=params, auth=auth, headers=headers + ) as response: + if response.status != 200: + error_text = await response.text() + logger.error( + f"Failed to fetch tasks from Jira. Status code: {response.status}, Response: {error_text}" + ) + break + + data = await response.json() + issues = data.get("issues", []) + total = data.get("total", 0) + + for issue in issues: + # Create task with all issue data + task_data = { + "id": issue.get("id"), + "key": issue.get("key"), + "fields": issue.get("fields", {}), + "self": issue.get("self"), + "expand": issue.get("expand", ""), + } + + task = Task(data=task_data) + tasks.append(task) + + # Check limit + if limit > 0 and len(tasks) >= limit: + break + + start_at += max_results + logger.debug(f"Issues packages reading: {len(tasks)}") + + logger.info(f"JIRA issues read: {len(tasks)}") + return tasks + + except aiohttp.ClientError as e: + logger.error(f"HTTP client error while fetching Jira tasks: {str(e)}") + raise Exception(f"Failed to connect to Jira: {str(e)}") + except json.JSONDecodeError as e: + logger.error(f"Failed to parse Jira API response: {str(e)}") + raise Exception(f"Invalid response from Jira API: {str(e)}") + except Exception as e: + logger.error(f"Unexpected error while fetching Jira tasks: {str(e)}") + raise + + async def write_tasks(self, tasklist: list[Task]) -> None: + """ + Write/update tasks to Jira. + + Args: + tasklist: List of Task objects containing task data to update + """ + headers = {"Accept": "application/json", "Content-Type": "application/json"} + auth = aiohttp.BasicAuth(self.jira_username, self.jira_api_token) + + try: + async with aiohttp.ClientSession() as session: + for task in tasklist: + task_data = task.data + task_id = task_data.get("id") or task_data.get("key") + + if not task_id: + logger.warning("Task missing ID or key, skipping update") + continue + + # Extract fields to update from task data + fields = task_data.get("fields", {}) + + if not fields: + logger.debug(f"No fields to update for task {task_id}") + continue + + # Prepare update data + update_data = {"fields": fields} + + # Make the update request + url = f"{self.jira_url}/rest/api/2/issue/{task_id}" + + async with session.put( + url, json=update_data, headers=headers, auth=auth + ) as response: + if response.status == 204: + logger.info(f"JIRA task {task_id} updated successfully.") + else: + error_text = await response.text() + logger.error( + f"JIRA failed to update task {task_id}: {response.status} - {error_text}" + ) + + except aiohttp.ClientError as e: + logger.error(f"HTTP client error while updating Jira tasks: {str(e)}") + raise Exception(f"Failed to connect to Jira: {str(e)}") + except Exception as e: + logger.error(f"Unexpected error while updating Jira tasks: {str(e)}") + raise diff --git a/modules/interfaces/interfaceTicketModel.py b/modules/interfaces/interfaceTicketModel.py new file mode 100644 index 00000000..151eabac --- /dev/null +++ b/modules/interfaces/interfaceTicketModel.py @@ -0,0 +1,26 @@ +"""Base class for ticket classes.""" + +from typing import Any, Dict +from pydantic import BaseModel, Field +from abc import ABC, abstractmethod + + +class TicketFieldAttribute(BaseModel): + field_name: str = Field(description="Human-readable field name") + field: str = Field(description="JIRA field ID/key") + + +class Task(BaseModel): + # A very flexible approach for now. Might want to be more strict in the future. + data: Dict[str, Any] = Field(default_factory=dict, description="Task data") + + +class TicketBase(ABC): + @abstractmethod + async def read_attributes(self) -> list[TicketFieldAttribute]: ... + + @abstractmethod + async def read_tasks(self) -> list[Task]: ... + + @abstractmethod + async def write_tasks(self, tasklist: list[Task]) -> None: ... diff --git a/modules/interfaces/interfaceTicketObjects.py b/modules/interfaces/interfaceTicketObjects.py new file mode 100644 index 00000000..e1ac75a8 --- /dev/null +++ b/modules/interfaces/interfaceTicketObjects.py @@ -0,0 +1,10 @@ +from dataclasses import dataclass + + +SUPPORTED_SYSTEMS = ["jira"] + + +@dataclass(slots=True) +class TicketInterface: + # TODO: user must create instance of Ticket connector + ticketConnector = None From f1f6bd210b8fc9ebbbfadec41b445ea2c063522e Mon Sep 17 00:00:00 2001 From: Christopher Gondek Date: Wed, 3 Sep 2025 17:16:42 +0200 Subject: [PATCH 02/12] feat: add sharepoint; jira connections --- modules/connectors/connectorSharepoint.py | 55 +++++++++++++++++++ modules/interfaces/interfaceTicketObjects.py | 46 +++++++++++++++- modules/routes/routeJira.py | 58 ++++++++++++++++++++ requirements.txt | 1 + 4 files changed, 157 insertions(+), 3 deletions(-) create mode 100644 modules/connectors/connectorSharepoint.py create mode 100644 modules/routes/routeJira.py diff --git a/modules/connectors/connectorSharepoint.py b/modules/connectors/connectorSharepoint.py new file mode 100644 index 00000000..33e220a0 --- /dev/null +++ b/modules/connectors/connectorSharepoint.py @@ -0,0 +1,55 @@ +"""Connector for CRUD sharepoint operations.""" + +from dataclasses import dataclass +from office365.sharepoint.client_context import ClientContext + + +@dataclass +class ConnectorSharepoint: + ctx: ClientContext + + @classmethod + async def create(cls, ctx: ClientContext) -> "ConnectorSharepoint": + """Creates an instance of the Sharepoint connector. + + Params: + ctx: The ClientContext instance. + + Returns: + ConnectorSharepoint: An instance of the Sharepoint connector. + """ + return cls(ctx=ctx) + + @classmethod + def get_client_context_from_username_password( + cls, site_url: str, username: str, password: str + ) -> ClientContext: + """Creates a ClientContext instance from username and password. + + Params: + site_url: The URL of the SharePoint site. + username: The username for authentication. + password: The password for authentication. + + Returns: + ClientContext: An instance of the ClientContext. + """ + return ClientContext(site_url).with_user_credentials(username, password) + + @classmethod + def get_client_context_from_app( + cls, site_url: str, client_id: str, client_secret: str + ) -> ClientContext: + """Creates a ClientContext instance from client ID and client secret. + + Params: + site_url: The URL of the SharePoint site. + client_id: The client ID for authentication. + client_secret: The client secret for authentication. + + Returns: + ClientContext: An instance of the ClientContext. + """ + return ClientContext(site_url).with_client_credentials( + client_id=client_id, client_secret=client_secret + ) diff --git a/modules/interfaces/interfaceTicketObjects.py b/modules/interfaces/interfaceTicketObjects.py index e1ac75a8..3eb7a2bd 100644 --- a/modules/interfaces/interfaceTicketObjects.py +++ b/modules/interfaces/interfaceTicketObjects.py @@ -1,10 +1,50 @@ from dataclasses import dataclass +from shareplum import Site +from shareplum import Office365 +from shareplum.site import Version + +from modules.interfaces.interfaceTicketModel import TicketBase SUPPORTED_SYSTEMS = ["jira"] @dataclass(slots=True) -class TicketInterface: - # TODO: user must create instance of Ticket connector - ticketConnector = None +class TicketSharepointSyncInterface: + ticketConnector: TicketBase + task_sync_definition: dict + + # TODO: shareplum instance + + @classmethod + async def create( + cls, + ticket_connector: TicketBase, + ) -> "TicketSharepointSyncInterface": + instance = cls() + instance.ticketConnector = ticket_connector + return instance + + # TODO: 1. Read JIRA tickets + # TODO: 2. Transform tasks according to task_sync_definition (get_task_object) l. 79ff + + # TODO: 3. Create export file: Save transformed tasks to a timestamped export file in sharepoint + # - maybe not needed? + + # TODO: 4. Backup current main sync file + + # TODO: 5. Compare JIRA data (export file) with current main sync file and update line by line + # - update GET only + # - important so that we don't overwrite the changes from SELISE in the main sync file + + # TODO: 6. Take PUT changes from the main sync file and write it back to JIRA. + + # TODO: Write file to sharepoint folder + # TODO: Remove file from sharepoint folder + # TODO: Rename file in sharepoint folder + + # Next steps: + # - Complete connectorSharepoint CRUD-ish + # - pytest sharepoint connector + # - pytest JIRA connector + # - connect logic here... diff --git a/modules/routes/routeJira.py b/modules/routes/routeJira.py new file mode 100644 index 00000000..0a1b8195 --- /dev/null +++ b/modules/routes/routeJira.py @@ -0,0 +1,58 @@ +# Configure logger +import logging +from fastapi import APIRouter + +from modules.connectors.connectorTicketJira import ConnectorTicketJira + + +logger = logging.getLogger(__name__) + +router = APIRouter( + prefix="/api/users", + tags=["Manage Users"], +) + + +@router.post("/sync/delta-group") +async def sync_jira(): + logger.info("Syncing Jira issues...") + # Implement synchronization logic here + + jira_username = None + jira_api_token = None + sharepoint_client_id = None + sharepoint_client_secret = None + jira_url = "https://deltasecurity.atlassian.net" + project_code = "DCS" + issue_type = "Task" + task_sync_definition = { + # key=excel-header, [get:jira>excel | put: excel>jira, jira-xml-field-list] + "ID": ["get", ["key"]], + "Module Category": ["get", ["fields", "customfield_10058", "value"]], + "Summary": ["get", ["fields", "summary"]], + "Description": ["get", ["fields", "description"]], + "References": ["get", ["fields", "customfield_10066"]], + "Priority": ["get", ["fields", "priority", "name"]], + "Issue Status": ["get", ["fields", "customfield_10062"]], + "Assignee": ["get", ["fields", "assignee", "displayName"]], + "Issue Created": ["get", ["fields", "created"]], + "Due Date": ["get", ["fields", "duedate"]], + "DELTA Comments": ["get", ["fields", "customfield_10060"]], + "SELISE Ticket References": ["put", ["fields", "customfield_10067"]], + "SELISE Status Values": ["put", ["fields", "customfield_10065"]], + "SELISE Comments": ["put", ["fields", "customfield_10064"]], + } + + # Create the jira connector instance + jira_connector = ConnectorTicketJira( + jira_username=jira_username, + jira_api_token=jira_api_token, + jira_url=jira_url, + project_code=project_code, + issue_type=issue_type, + ) + + # Read the JIRA tickets + jira_attributes = await jira_connector.read_tasks(limit=0) + + return {"message": "Jira issues synchronized successfully"} diff --git a/requirements.txt b/requirements.txt index 783db728..75f2d078 100644 --- a/requirements.txt +++ b/requirements.txt @@ -43,6 +43,7 @@ chardet>=5.0.0 # Für Zeichensatzerkennung bei Webinhalten aiohttp>=3.8.0 # Required for SharePoint operations (async HTTP) selenium>=4.15.0 # Required for web automation and JavaScript-heavy pages tavily-python==0.7.11 # Tavily SDK +Office365-REST-Python-Client==2.6.2 # Easy Sharepoint integration ## Image Processing Pillow>=10.0.0 # Für Bildverarbeitung (als PIL importiert) From 1a7ca4fa13a445ff26ae5e78a9f696011c04cfb1 Mon Sep 17 00:00:00 2001 From: Christopher Gondek Date: Fri, 5 Sep 2025 11:58:25 +0200 Subject: [PATCH 03/12] feat: finish routejira implementation (untested) --- modules/connectors/connectorSharepoint.py | 126 +++++ modules/connectors/connectorTicketJira.py | 21 +- modules/interfaces/interfaceTicketModel.py | 2 +- modules/interfaces/interfaceTicketObjects.py | 551 ++++++++++++++++++- modules/routes/routeJira.py | 65 ++- 5 files changed, 712 insertions(+), 53 deletions(-) diff --git a/modules/connectors/connectorSharepoint.py b/modules/connectors/connectorSharepoint.py index 33e220a0..0b1c8370 100644 --- a/modules/connectors/connectorSharepoint.py +++ b/modules/connectors/connectorSharepoint.py @@ -1,7 +1,13 @@ """Connector for CRUD sharepoint operations.""" +import asyncio +from concurrent.futures import ThreadPoolExecutor from dataclasses import dataclass +from datetime import datetime +from io import BytesIO +from typing import Optional from office365.sharepoint.client_context import ClientContext +from office365.sharepoint.files.file import File @dataclass @@ -53,3 +59,123 @@ class ConnectorSharepoint: return ClientContext(site_url).with_client_credentials( client_id=client_id, client_secret=client_secret ) + + def copy_file( + self, *, source_folder: str, source_file: str, dest_folder: str, dest_file: str + ) -> bool: + """Copy a file from one SharePoint location to another. + + Params: + source_folder: Source folder path (server-relative) + source_file: Source file name + dest_folder: Destination folder path (server-relative) + dest_file: Destination file name + + Returns: + bool: True if successful, False otherwise + """ + source_path = f"{source_folder.rstrip('/')}/{source_file}" + dest_path = f"{dest_folder.rstrip('/')}/{dest_file}" + + source_file_obj = self.ctx.web.get_file_by_server_relative_url(source_path) + source_file_obj.copyto(dest_path).execute_query() + return True + + async def copy_file_async( + self, *, source_folder: str, source_file: str, dest_folder: str, dest_file: str + ) -> bool: + """Copy a file from one SharePoint location to another (async version). + + Params: + source_folder: Source folder path (server-relative) + source_file: Source file name + dest_folder: Destination folder path (server-relative) + dest_file: Destination file name + + Returns: + bool: True if successful, False otherwise + """ + loop = asyncio.get_event_loop() + with ThreadPoolExecutor() as executor: + return await loop.run_in_executor( + executor, + lambda: self.copy_file( + source_folder=source_folder, + source_file=source_file, + dest_folder=dest_folder, + dest_file=dest_file, + ), + ) + + def read_file(self, *, folder_path: str, file_name: str) -> bytes: + """Read a file from SharePoint and return its content as bytes. + + Params: + folder_path: Folder path (server-relative) + file_name: File name + + Returns: + bytes: File content as bytes + """ + file_path = f"{folder_path.rstrip('/')}/{file_name}" + response = File.open_binary(self.ctx, file_path) + return response.content + + async def read_file_async(self, *, folder_path: str, file_name: str) -> bytes: + """Read a file from SharePoint and return its content as bytes (async version). + + Params: + folder_path: Folder path (server-relative) + file_name: File name + + Returns: + bytes: File content as bytes + """ + loop = asyncio.get_event_loop() + with ThreadPoolExecutor() as executor: + return await loop.run_in_executor( + executor, + lambda: self.read_file(folder_path=folder_path, file_name=file_name), + ) + + def overwrite_file( + self, *, folder_path: str, file_name: str, content: bytes + ) -> bool: + """Write content to a SharePoint file, overwriting if it exists. + + Params: + folder_path: Target folder path (server-relative) + file_name: Target file name + content: File content as bytes + + Returns: + bool: True if successful, False otherwise + """ + target_folder = self.ctx.web.get_folder_by_server_relative_url(folder_path) + buffer = BytesIO(content) + target_folder.files.upload(buffer, file_name).execute_query() + return True + + async def overwrite_file_async( + self, *, folder_path: str, file_name: str, content: bytes + ) -> bool: + """Write content to a SharePoint file, overwriting if it exists (async version). + + Params: + folder_path: Target folder path (server-relative) + file_name: Target file name + content: File content as bytes + + Returns: + bool: True if successful, False otherwise + """ + loop = asyncio.get_event_loop() + with ThreadPoolExecutor() as executor: + return await loop.run_in_executor( + executor, + lambda: self.overwrite_file( + folder_path=folder_path, + file_name=file_name, + content=content, + ), + ) diff --git a/modules/connectors/connectorTicketJira.py b/modules/connectors/connectorTicketJira.py index f2edd200..93020f2c 100644 --- a/modules/connectors/connectorTicketJira.py +++ b/modules/connectors/connectorTicketJira.py @@ -4,7 +4,6 @@ from dataclasses import dataclass import logging import aiohttp import json -from typing import Optional from modules.interfaces.interfaceTicketModel import ( TicketBase, @@ -156,16 +155,9 @@ class ConnectorTicketJira(TicketBase): total = data.get("total", 0) for issue in issues: - # Create task with all issue data - task_data = { - "id": issue.get("id"), - "key": issue.get("key"), - "fields": issue.get("fields", {}), - "self": issue.get("self"), - "expand": issue.get("expand", ""), - } - - task = Task(data=task_data) + # Store the raw JIRA issue data directly + # This matches what the reference implementation expects + task = Task(data=issue) tasks.append(task) # Check limit @@ -202,13 +194,18 @@ class ConnectorTicketJira(TicketBase): async with aiohttp.ClientSession() as session: for task in tasklist: task_data = task.data - task_id = task_data.get("id") or task_data.get("key") + task_id = ( + task_data.get("ID") + or task_data.get("id") + or task_data.get("key") + ) if not task_id: logger.warning("Task missing ID or key, skipping update") continue # Extract fields to update from task data + # The task data should contain the field updates in a "fields" key fields = task_data.get("fields", {}) if not fields: diff --git a/modules/interfaces/interfaceTicketModel.py b/modules/interfaces/interfaceTicketModel.py index 151eabac..98329a7b 100644 --- a/modules/interfaces/interfaceTicketModel.py +++ b/modules/interfaces/interfaceTicketModel.py @@ -20,7 +20,7 @@ class TicketBase(ABC): async def read_attributes(self) -> list[TicketFieldAttribute]: ... @abstractmethod - async def read_tasks(self) -> list[Task]: ... + async def read_tasks(self, limit: int = 0) -> list[Task]: ... @abstractmethod async def write_tasks(self, tasklist: list[Task]) -> None: ... diff --git a/modules/interfaces/interfaceTicketObjects.py b/modules/interfaces/interfaceTicketObjects.py index 3eb7a2bd..cebb043f 100644 --- a/modules/interfaces/interfaceTicketObjects.py +++ b/modules/interfaces/interfaceTicketObjects.py @@ -1,50 +1,541 @@ from dataclasses import dataclass -from shareplum import Site -from shareplum import Office365 -from shareplum.site import Version +from io import BytesIO +from typing import Any +import pandas as pd +from modules.shared.timezoneUtils import get_utc_now + +from modules.connectors.connectorSharepoint import ConnectorSharepoint from modules.interfaces.interfaceTicketModel import TicketBase - - -SUPPORTED_SYSTEMS = ["jira"] +from modules.interfaces.interfaceTicketModel import Task @dataclass(slots=True) class TicketSharepointSyncInterface: - ticketConnector: TicketBase + connector_ticket: TicketBase + connector_sharepoint: ConnectorSharepoint task_sync_definition: dict - - # TODO: shareplum instance + sync_folder: str + sync_file: str + backup_folder: str + audit_folder: str @classmethod async def create( cls, - ticket_connector: TicketBase, + connector_ticket: TicketBase, + connector_sharepoint: ConnectorSharepoint, + task_sync_definition: dict, + sync_folder: str, + sync_file: str, + backup_folder: str, + audit_folder: str, ) -> "TicketSharepointSyncInterface": - instance = cls() - instance.ticketConnector = ticket_connector - return instance + return cls( + connector_ticket=connector_ticket, + connector_sharepoint=connector_sharepoint, + task_sync_definition=task_sync_definition, + sync_folder=sync_folder, + sync_file=sync_file, + backup_folder=backup_folder, + audit_folder=audit_folder, + ) - # TODO: 1. Read JIRA tickets - # TODO: 2. Transform tasks according to task_sync_definition (get_task_object) l. 79ff + async def create_backup(self): + """Creates a backup of the current sync file in the backup folder.""" + timestamp = get_utc_now().strftime("%Y%m%d_%H%M%S") + backup_filename = f"backup_{timestamp}_{self.sync_file}" - # TODO: 3. Create export file: Save transformed tasks to a timestamped export file in sharepoint - # - maybe not needed? + await self.connector_sharepoint.copy_file_async( + source_folder=self.sync_folder, + source_file=self.sync_file, + dest_folder=self.backup_folder, + dest_file=backup_filename, + ) - # TODO: 4. Backup current main sync file + async def sync_from_jira_to_csv(self): + """Syncs tasks from JIRA to a CSV file in SharePoint.""" + start_time = get_utc_now() + audit_log = [] - # TODO: 5. Compare JIRA data (export file) with current main sync file and update line by line - # - update GET only - # - important so that we don't overwrite the changes from SELISE in the main sync file + audit_log.append("=== JIRA TO CSV SYNC STARTED ===") + audit_log.append(f"Start Time: {start_time.strftime('%Y-%m-%d %H:%M:%S')}") + audit_log.append(f"Sync File: {self.sync_file}") + audit_log.append(f"Sync Folder: {self.sync_folder}") + audit_log.append("") - # TODO: 6. Take PUT changes from the main sync file and write it back to JIRA. + try: + # 1. Read JIRA tickets + audit_log.append("Step 1: Reading JIRA tickets...") + tickets = await self.connector_ticket.read_tasks(limit=0) + audit_log.append(f"JIRA issues read: {len(tickets)}") + audit_log.append("") - # TODO: Write file to sharepoint folder - # TODO: Remove file from sharepoint folder - # TODO: Rename file in sharepoint folder + # 2. Transform tasks according to task_sync_definition + audit_log.append("Step 2: Transforming JIRA data...") + transformed_tasks = self._transform_tasks(tickets) + jira_data = [task.data for task in transformed_tasks] + audit_log.append(f"JIRA issues transformed: {len(jira_data)}") + audit_log.append("") - # Next steps: - # - Complete connectorSharepoint CRUD-ish - # - pytest sharepoint connector - # - pytest JIRA connector - # - connect logic here... + # 3. Create JIRA export file in audit folder + audit_log.append("Step 3: Creating JIRA export file...") + try: + timestamp = get_utc_now().strftime("%Y%m%d_%H%M%S") + jira_export_filename = f"jira_export_{timestamp}.csv" + jira_export_content = self._create_csv_content(jira_data) + await self.connector_sharepoint.overwrite_file_async( + folder_path=self.audit_folder, + file_name=jira_export_filename, + content=jira_export_content, + ) + audit_log.append(f"JIRA export file created: {jira_export_filename}") + except Exception as e: + audit_log.append(f"Failed to create JIRA export file: {str(e)}") + audit_log.append("") + + # 4. Create backup of existing sync file (if it exists) + audit_log.append("Step 4: Creating backup...") + backup_created = False + try: + await self.create_backup() + backup_created = True + audit_log.append("Backup created successfully") + except Exception as e: + audit_log.append( + f"Backup creation failed (file might not exist): {str(e)}" + ) + audit_log.append("") + + # 5. Try to read existing CSV file from SharePoint + audit_log.append("Step 5: Reading existing CSV file...") + existing_data = [] + existing_file_found = False + try: + csv_content = await self.connector_sharepoint.read_file_async( + folder_path=self.sync_folder, file_name=self.sync_file + ) + df_existing = pd.read_csv( + BytesIO(csv_content), skiprows=2 + ) # Skip header rows + existing_data = df_existing.to_dict("records") + existing_file_found = True + audit_log.append( + f"Existing CSV file found with {len(existing_data)} records" + ) + except Exception as e: + audit_log.append(f"No existing CSV file found or read error: {str(e)}") + audit_log.append("") + + # 6. Merge JIRA data with existing data and track changes + audit_log.append("Step 6: Merging JIRA data with existing data...") + merged_data, change_details = self._merge_jira_with_existing_detailed( + jira_data, existing_data + ) + + # Log detailed changes + audit_log.append(f"Total records after merge: {len(merged_data)}") + audit_log.append(f"Records updated: {change_details['updated']}") + audit_log.append(f"Records added: {change_details['added']}") + audit_log.append(f"Records unchanged: {change_details['unchanged']}") + audit_log.append("") + + # Log individual changes + if change_details["changes"]: + audit_log.append("DETAILED CHANGES:") + for change in change_details["changes"]: + audit_log.append(f"- {change}") + audit_log.append("") + + # 7. Create CSV with 4-row structure and write to SharePoint + audit_log.append("Step 7: Writing updated CSV to SharePoint...") + csv_content = self._create_csv_content(merged_data) + await self.connector_sharepoint.overwrite_file_async( + folder_path=self.sync_folder, + file_name=self.sync_file, + content=csv_content, + ) + audit_log.append("CSV file successfully written to SharePoint") + audit_log.append("") + + # Success summary + end_time = get_utc_now() + duration = (end_time - start_time).total_seconds() + audit_log.append("=== SYNC COMPLETED SUCCESSFULLY ===") + audit_log.append(f"End Time: {end_time.strftime('%Y-%m-%d %H:%M:%S')}") + audit_log.append(f"Duration: {duration:.2f} seconds") + audit_log.append(f"Total JIRA issues processed: {len(jira_data)}") + audit_log.append(f"Total records in final CSV: {len(merged_data)}") + + except Exception as e: + # Error handling + end_time = get_utc_now() + duration = (end_time - start_time).total_seconds() + audit_log.append("") + audit_log.append("=== SYNC FAILED ===") + audit_log.append(f"Error Time: {end_time.strftime('%Y-%m-%d %H:%M:%S')}") + audit_log.append(f"Duration before failure: {duration:.2f} seconds") + audit_log.append(f"Error: {str(e)}") + raise + finally: + # Write audit log to SharePoint + await self._write_audit_log(audit_log, "jira_to_csv") + + async def sync_from_csv_to_jira(self): + """Syncs tasks from a CSV file in SharePoint to JIRA.""" + start_time = get_utc_now() + audit_log = [] + + audit_log.append("=== CSV TO JIRA SYNC STARTED ===") + audit_log.append(f"Start Time: {start_time.strftime('%Y-%m-%d %H:%M:%S')}") + audit_log.append(f"Sync File: {self.sync_file}") + audit_log.append(f"Sync Folder: {self.sync_folder}") + audit_log.append("") + + try: + # 1. Read CSV file from SharePoint + audit_log.append("Step 1: Reading CSV file from SharePoint...") + try: + csv_content = await self.connector_sharepoint.read_file_async( + folder_path=self.sync_folder, file_name=self.sync_file + ) + df = pd.read_csv(BytesIO(csv_content), skiprows=2) # Skip header rows + csv_data = df.to_dict("records") + audit_log.append( + f"CSV file read successfully with {len(csv_data)} records" + ) + except Exception as e: + audit_log.append(f"Failed to read CSV file: {str(e)}") + audit_log.append("CSV to JIRA sync aborted - no file to process") + return + audit_log.append("") + + # 2. Read current JIRA data for comparison + audit_log.append("Step 2: Reading current JIRA data for comparison...") + try: + current_jira_tasks = await self.connector_ticket.read_tasks(limit=0) + current_jira_data = self._transform_tasks(current_jira_tasks) + jira_lookup = { + task.data.get("ID"): task.data for task in current_jira_data + } + audit_log.append(f"Current JIRA data read: {len(jira_lookup)} tasks") + except Exception as e: + audit_log.append(f"Failed to read current JIRA data: {str(e)}") + raise + audit_log.append("") + + # 3. Detect actual changes in "put" fields + audit_log.append("Step 3: Detecting changes in 'put' fields...") + actual_changes = {} + records_with_changes = 0 + total_changes = 0 + + for row in csv_data: + task_id = row.get("ID") + if not task_id or task_id not in jira_lookup: + continue + + current_jira_task = jira_lookup[task_id] + task_changes = {} + + for field_name, field_config in self.task_sync_definition.items(): + if field_config[0] == "put": # Only process "put" fields + csv_value = row.get(field_name, "") + jira_value = current_jira_task.get(field_name, "") + + # Convert None to empty string for comparison + csv_value = "" if csv_value is None else str(csv_value).strip() + jira_value = ( + "" if jira_value is None else str(jira_value).strip() + ) + + # Only include if values are different and CSV has non-empty value + if csv_value != jira_value and csv_value: + task_changes[field_name] = csv_value + + if task_changes: + actual_changes[task_id] = task_changes + records_with_changes += 1 + total_changes += len(task_changes) + + audit_log.append(f"Records with actual changes: {records_with_changes}") + audit_log.append(f"Total field changes detected: {total_changes}") + audit_log.append("") + + # Log detailed changes + if actual_changes: + audit_log.append("DETAILED CHANGES TO APPLY TO JIRA:") + for task_id, changes in actual_changes.items(): + change_list = [ + f"{field}: '{value}'" for field, value in changes.items() + ] + audit_log.append(f"- Task ID {task_id}: {', '.join(change_list)}") + audit_log.append("") + + # 4. Update JIRA tasks with actual changes + if actual_changes: + audit_log.append("Step 4: Updating JIRA tasks...") + + # Convert to Task objects for the connector + tasks_to_update = [] + for task_id, changes in actual_changes.items(): + # Create task data structure expected by JIRA connector + # Build the nested fields structure that JIRA expects + fields = {} + for field_name, new_value in changes.items(): + # Map back to JIRA field structure using task_sync_definition + field_config = self.task_sync_definition[field_name] + field_path = field_config[1] + + # Extract the JIRA field ID from the path + # For "put" fields, the path is like ['fields', 'customfield_10067'] + if len(field_path) >= 2 and field_path[0] == "fields": + jira_field_id = field_path[1] + fields[jira_field_id] = new_value + + if fields: + task_data = {"ID": task_id, "fields": fields} + task = Task(data=task_data) + tasks_to_update.append(task) + + # Write tasks back to JIRA + try: + await self.connector_ticket.write_tasks(tasks_to_update) + audit_log.append( + f"Successfully updated {len(tasks_to_update)} JIRA tasks" + ) + except Exception as e: + audit_log.append(f"Failed to update JIRA tasks: {str(e)}") + raise + else: + audit_log.append("Step 4: No changes to apply to JIRA") + audit_log.append("") + + # Success summary + end_time = get_utc_now() + duration = (end_time - start_time).total_seconds() + audit_log.append("=== SYNC COMPLETED SUCCESSFULLY ===") + audit_log.append(f"End Time: {end_time.strftime('%Y-%m-%d %H:%M:%S')}") + audit_log.append(f"Duration: {duration:.2f} seconds") + audit_log.append(f"Total CSV records processed: {len(csv_data)}") + audit_log.append(f"Records with actual changes: {records_with_changes}") + audit_log.append(f"JIRA tasks updated: {len(actual_changes)}") + + except Exception as e: + # Error handling + end_time = get_utc_now() + duration = (end_time - start_time).total_seconds() + audit_log.append("") + audit_log.append("=== SYNC FAILED ===") + audit_log.append(f"Error Time: {end_time.strftime('%Y-%m-%d %H:%M:%S')}") + audit_log.append(f"Duration before failure: {duration:.2f} seconds") + audit_log.append(f"Error: {str(e)}") + raise + finally: + # Write audit log to SharePoint + await self._write_audit_log(audit_log, "csv_to_jira") + + def _transform_tasks(self, tasks: list[Task]) -> list[Task]: + """Transforms tasks according to the task_sync_definition.""" + transformed_tasks = [] + + for task in tasks: + transformed_data = {} + + # Process each field in the sync definition + for field_name, field_config in self.task_sync_definition.items(): + direction = field_config[0] # "get" or "put" + field_path = field_config[1] # List of keys to navigate + + # Only process "get" fields (JIRA → CSV) + if direction == "get": + # Extract value using the field path + value = self._extract_field_value(task.data, field_path) + transformed_data[field_name] = value + + # Create new Task with transformed data + transformed_task = Task(data=transformed_data) + transformed_tasks.append(transformed_task) + + return transformed_tasks + + def _extract_field_value(self, issue_data: dict, field_path: list[str]) -> Any: + """Extract field value from JIRA issue data using field path.""" + value = issue_data + try: + for key in field_path: + if value is not None: + value = value[key] + + if value is None: + return None + + # Handle complex objects that have a 'value' field (like custom field options) + if isinstance(value, dict) and "value" in value: + value = value["value"] + # Handle lists of objects with 'value' fields + elif ( + isinstance(value, list) + and len(value) > 0 + and isinstance(value[0], dict) + and "value" in value[0] + ): + value = value[0]["value"] + + return value + except (KeyError, TypeError): + return None + + def _merge_jira_with_existing( + self, jira_data: list[dict], existing_data: list[dict] + ) -> list[dict]: + """Merge JIRA data with existing CSV data, updating only 'get' fields.""" + # Create a lookup for existing data by ID + existing_lookup = {row.get("ID"): row for row in existing_data if row.get("ID")} + + merged_data = [] + for jira_row in jira_data: + jira_id = jira_row.get("ID") + if jira_id and jira_id in existing_lookup: + # Update existing row with JIRA data (only 'get' fields) + existing_row = existing_lookup[jira_id].copy() + for field_name, field_config in self.task_sync_definition.items(): + if field_config[0] == "get": # Only update 'get' fields + existing_row[field_name] = jira_row.get(field_name) + merged_data.append(existing_row) + # Remove from lookup to track processed items + del existing_lookup[jira_id] + else: + # New row from JIRA + merged_data.append(jira_row) + + # Add any remaining existing rows that weren't in JIRA data + merged_data.extend(existing_lookup.values()) + + return merged_data + + def _merge_jira_with_existing_detailed( + self, jira_data: list[dict], existing_data: list[dict] + ) -> tuple[list[dict], dict]: + """Merge JIRA data with existing CSV data and track detailed changes.""" + # Create a lookup for existing data by ID + existing_lookup = {row.get("ID"): row for row in existing_data if row.get("ID")} + + merged_data = [] + changes = [] + updated_count = 0 + added_count = 0 + unchanged_count = 0 + + for jira_row in jira_data: + jira_id = jira_row.get("ID") + if jira_id and jira_id in existing_lookup: + # Update existing row with JIRA data (only 'get' fields) + existing_row = existing_lookup[jira_id].copy() + row_changes = [] + + for field_name, field_config in self.task_sync_definition.items(): + if field_config[0] == "get": # Only update 'get' fields + old_value = existing_row.get(field_name, "") + new_value = jira_row.get(field_name, "") + + # Convert None to empty string for comparison + old_value = "" if old_value is None else str(old_value) + new_value = "" if new_value is None else str(new_value) + + if old_value != new_value: + row_changes.append( + f"{field_name}: '{old_value}' → '{new_value}'" + ) + + existing_row[field_name] = jira_row.get(field_name) + + merged_data.append(existing_row) + + if row_changes: + updated_count += 1 + changes.append( + f"Row ID {jira_id} updated: {', '.join(row_changes)}" + ) + else: + unchanged_count += 1 + + # Remove from lookup to track processed items + del existing_lookup[jira_id] + else: + # New row from JIRA + merged_data.append(jira_row) + added_count += 1 + changes.append(f"Row ID {jira_id} added as new record") + + # Add any remaining existing rows that weren't in JIRA data + for remaining_row in existing_lookup.values(): + merged_data.append(remaining_row) + unchanged_count += 1 + + change_details = { + "updated": updated_count, + "added": added_count, + "unchanged": unchanged_count, + "changes": changes, + } + + return merged_data, change_details + + async def _write_audit_log(self, audit_log: list[str], operation_type: str): + """Write audit log to SharePoint.""" + try: + timestamp = get_utc_now().strftime("%Y%m%d_%H%M%S") + audit_filename = f"audit_{operation_type}_{timestamp}.log" + + # Convert audit log to bytes + audit_content = "\n".join(audit_log).encode("utf-8") + + # Write to SharePoint + await self.connector_sharepoint.overwrite_file_async( + folder_path=self.audit_folder, + file_name=audit_filename, + content=audit_content, + ) + except Exception as e: + # If audit logging fails, we don't want to break the main sync process + # Just log the error (this could be enhanced with fallback logging) + print(f"Failed to write audit log: {str(e)}") + + def _create_csv_content(self, data: list[dict]) -> bytes: + """Create CSV content with 4-row structure matching reference code.""" + if not data: + return b"" + + # Create DataFrame from data + df = pd.DataFrame(data) + + # Force all columns to be object (string) type to preserve empty cells + for column in df.columns: + df[column] = df[column].astype("object") + df[column] = df[column].fillna("") + + # Create the 4-row structure + # Row 1: Static header row 1 + header_row1 = pd.DataFrame( + [["Header 1"] + [""] * (len(df.columns) - 1)], columns=df.columns + ) + + # Row 2: Static header row 2 with timestamp + timestamp = get_utc_now().strftime("%Y-%m-%d %H:%M:%S") + header_row2 = pd.DataFrame( + [[f"{timestamp}"] + [""] * (len(df.columns) - 1)], columns=df.columns + ) + + # Row 3: Table headers (column names) + table_headers = pd.DataFrame([df.columns.tolist()], columns=df.columns) + + # Concatenate all rows: header1 + header2 + table_headers + data + final_df = pd.concat( + [header_row1, header_row2, table_headers, df], ignore_index=True + ) + + # Convert to CSV bytes + csv_buffer = BytesIO() + final_df.to_csv(csv_buffer, index=False, header=False) + return csv_buffer.getvalue() diff --git a/modules/routes/routeJira.py b/modules/routes/routeJira.py index 0a1b8195..66564e55 100644 --- a/modules/routes/routeJira.py +++ b/modules/routes/routeJira.py @@ -3,28 +3,41 @@ import logging from fastapi import APIRouter from modules.connectors.connectorTicketJira import ConnectorTicketJira - +from modules.connectors.connectorSharepoint import ConnectorSharepoint +from modules.interfaces.interfaceTicketObjects import TicketSharepointSyncInterface logger = logging.getLogger(__name__) router = APIRouter( - prefix="/api/users", - tags=["Manage Users"], + prefix="/api/jira", + tags=["JIRA Sync"], ) @router.post("/sync/delta-group") async def sync_jira(): logger.info("Syncing Jira issues...") - # Implement synchronization logic here - jira_username = None - jira_api_token = None + # Sharepoint connection parameters sharepoint_client_id = None sharepoint_client_secret = None + sharepoint_site_url = None + + # Jira connection parameters + jira_username = None + jira_api_token = None jira_url = "https://deltasecurity.atlassian.net" project_code = "DCS" issue_type = "Task" + + # Basic validation (credentials will be added later) + if not all([sharepoint_client_id, sharepoint_client_secret, sharepoint_site_url]): + logger.warning("SharePoint credentials not configured - sync will fail") + + if not all([jira_username, jira_api_token]): + logger.warning("JIRA credentials not configured - sync will fail") + + # Define the task sync definition task_sync_definition = { # key=excel-header, [get:jira>excel | put: excel>jira, jira-xml-field-list] "ID": ["get", ["key"]], @@ -43,8 +56,14 @@ async def sync_jira(): "SELISE Comments": ["put", ["fields", "customfield_10064"]], } + # SharePoint file configuration + sync_folder = "Shared Documents/TicketSync" + sync_file = "delta_group_selise_ticket_exchange_list.csv" + backup_folder = "Shared Documents/TicketSync/Backups" + audit_folder = "Shared Documents/TicketSync/AuditLogs" + # Create the jira connector instance - jira_connector = ConnectorTicketJira( + jira_connector = await ConnectorTicketJira.create( jira_username=jira_username, jira_api_token=jira_api_token, jira_url=jira_url, @@ -52,7 +71,33 @@ async def sync_jira(): issue_type=issue_type, ) - # Read the JIRA tickets - jira_attributes = await jira_connector.read_tasks(limit=0) + # Create the sharepoint connector instance + ctx = ConnectorSharepoint.get_client_context_from_app( + site_url=sharepoint_site_url, + client_id=sharepoint_client_id, + client_secret=sharepoint_client_secret, + ) + sharepoint_connector = await ConnectorSharepoint.create(ctx=ctx) - return {"message": "Jira issues synchronized successfully"} + # Create the sync interface instance + sync_interface = await TicketSharepointSyncInterface.create( + connector_ticket=jira_connector, + connector_sharepoint=sharepoint_connector, + task_sync_definition=task_sync_definition, + sync_folder=sync_folder, + sync_file=sync_file, + backup_folder=backup_folder, + audit_folder=audit_folder, + ) + + # Create a backup of the current sync file + await sync_interface.create_backup() + + # Sync from JIRA to CSV in Sharepoint + await sync_interface.sync_from_jira_to_csv() + + # Sync from CSV in Sharepoint to JIRA + await sync_interface.sync_from_csv_to_jira() + + # Return a response + return {"status": "Sync completed"} From fe459731cdb31077b6295271d0a9137843a1de01 Mon Sep 17 00:00:00 2001 From: Christopher Gondek Date: Fri, 5 Sep 2025 14:23:18 +0200 Subject: [PATCH 04/12] fix: include put for csv -> jira --- modules/interfaces/interfaceTicketObjects.py | 12 ++++++++---- modules/routes/routeJira.py | 6 +++--- 2 files changed, 11 insertions(+), 7 deletions(-) diff --git a/modules/interfaces/interfaceTicketObjects.py b/modules/interfaces/interfaceTicketObjects.py index cebb043f..d7bed987 100644 --- a/modules/interfaces/interfaceTicketObjects.py +++ b/modules/interfaces/interfaceTicketObjects.py @@ -214,7 +214,9 @@ class TicketSharepointSyncInterface: audit_log.append("Step 2: Reading current JIRA data for comparison...") try: current_jira_tasks = await self.connector_ticket.read_tasks(limit=0) - current_jira_data = self._transform_tasks(current_jira_tasks) + current_jira_data = self._transform_tasks( + current_jira_tasks, include_put=True + ) jira_lookup = { task.data.get("ID"): task.data for task in current_jira_data } @@ -335,7 +337,9 @@ class TicketSharepointSyncInterface: # Write audit log to SharePoint await self._write_audit_log(audit_log, "csv_to_jira") - def _transform_tasks(self, tasks: list[Task]) -> list[Task]: + def _transform_tasks( + self, tasks: list[Task], include_put: bool = False + ) -> list[Task]: """Transforms tasks according to the task_sync_definition.""" transformed_tasks = [] @@ -347,8 +351,8 @@ class TicketSharepointSyncInterface: direction = field_config[0] # "get" or "put" field_path = field_config[1] # List of keys to navigate - # Only process "get" fields (JIRA → CSV) - if direction == "get": + # Get the right fields + if direction == "get" or include_put: # Extract value using the field path value = self._extract_field_value(task.data, field_path) transformed_data[field_name] = value diff --git a/modules/routes/routeJira.py b/modules/routes/routeJira.py index 66564e55..f812da7e 100644 --- a/modules/routes/routeJira.py +++ b/modules/routes/routeJira.py @@ -57,10 +57,10 @@ async def sync_jira(): } # SharePoint file configuration - sync_folder = "Shared Documents/TicketSync" + sync_folder = "/sites//Shared Documents/TicketSync" sync_file = "delta_group_selise_ticket_exchange_list.csv" - backup_folder = "Shared Documents/TicketSync/Backups" - audit_folder = "Shared Documents/TicketSync/AuditLogs" + backup_folder = "/sites//Shared Documents/TicketSync/Backups" + audit_folder = "/sites//Shared Documents/TicketSync/AuditLogs" # Create the jira connector instance jira_connector = await ConnectorTicketJira.create( From 9837bc1a19ed429605d5347bbeae050a75624189 Mon Sep 17 00:00:00 2001 From: Christopher Gondek Date: Fri, 5 Sep 2025 14:41:05 +0200 Subject: [PATCH 05/12] fix: remove redundant backup --- modules/routes/routeJira.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/modules/routes/routeJira.py b/modules/routes/routeJira.py index f812da7e..847010ec 100644 --- a/modules/routes/routeJira.py +++ b/modules/routes/routeJira.py @@ -90,9 +90,6 @@ async def sync_jira(): audit_folder=audit_folder, ) - # Create a backup of the current sync file - await sync_interface.create_backup() - # Sync from JIRA to CSV in Sharepoint await sync_interface.sync_from_jira_to_csv() From e1618c9ffb6d84b32afa3fa3b6bb4455d8ad6425 Mon Sep 17 00:00:00 2001 From: Christopher Gondek Date: Fri, 5 Sep 2025 14:48:23 +0200 Subject: [PATCH 06/12] fix: row 2 formatting --- modules/interfaces/interfaceTicketObjects.py | 5 ++--- modules/routes/routeJira.py | 17 ++++++++++++----- 2 files changed, 14 insertions(+), 8 deletions(-) diff --git a/modules/interfaces/interfaceTicketObjects.py b/modules/interfaces/interfaceTicketObjects.py index d7bed987..d55daad3 100644 --- a/modules/interfaces/interfaceTicketObjects.py +++ b/modules/interfaces/interfaceTicketObjects.py @@ -525,10 +525,9 @@ class TicketSharepointSyncInterface: [["Header 1"] + [""] * (len(df.columns) - 1)], columns=df.columns ) - # Row 2: Static header row 2 with timestamp - timestamp = get_utc_now().strftime("%Y-%m-%d %H:%M:%S") + # Row 2: Static header row 2 with strict compatibility header_row2 = pd.DataFrame( - [[f"{timestamp}"] + [""] * (len(df.columns) - 1)], columns=df.columns + [["Header 2"] + [""] * (len(df.columns) - 1)], columns=df.columns ) # Row 3: Table headers (column names) diff --git a/modules/routes/routeJira.py b/modules/routes/routeJira.py index 847010ec..31bcbe73 100644 --- a/modules/routes/routeJira.py +++ b/modules/routes/routeJira.py @@ -15,8 +15,18 @@ router = APIRouter( @router.post("/sync/delta-group") -async def sync_jira(): - logger.info("Syncing Jira issues...") +async def sync_jira_delta_group(): + """Endpoint to trigger JIRA-SharePoint sync for Delta Group project.""" + + logger.info("Received request to sync JIRA Delta Group project") + await perform_sync_jira_delta_group() + + # Return a response + return {"status": "Sync completed"} + + +async def perform_sync_jira_delta_group(): + logger.info("Syncing Jira issues for Delta Group...") # Sharepoint connection parameters sharepoint_client_id = None @@ -95,6 +105,3 @@ async def sync_jira(): # Sync from CSV in Sharepoint to JIRA await sync_interface.sync_from_csv_to_jira() - - # Return a response - return {"status": "Sync completed"} From 949a3c97aee16ee5762474b06d611252661d33c8 Mon Sep 17 00:00:00 2001 From: Christopher Gondek Date: Fri, 5 Sep 2025 14:57:13 +0200 Subject: [PATCH 07/12] fix: fail fast; include put in jira -> CSV --- modules/interfaces/interfaceTicketObjects.py | 2 +- modules/routes/routeJira.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/modules/interfaces/interfaceTicketObjects.py b/modules/interfaces/interfaceTicketObjects.py index d55daad3..94cf8427 100644 --- a/modules/interfaces/interfaceTicketObjects.py +++ b/modules/interfaces/interfaceTicketObjects.py @@ -73,7 +73,7 @@ class TicketSharepointSyncInterface: # 2. Transform tasks according to task_sync_definition audit_log.append("Step 2: Transforming JIRA data...") - transformed_tasks = self._transform_tasks(tickets) + transformed_tasks = self._transform_tasks(tickets, include_put=False) jira_data = [task.data for task in transformed_tasks] audit_log.append(f"JIRA issues transformed: {len(jira_data)}") audit_log.append("") diff --git a/modules/routes/routeJira.py b/modules/routes/routeJira.py index 31bcbe73..e49c3815 100644 --- a/modules/routes/routeJira.py +++ b/modules/routes/routeJira.py @@ -42,10 +42,10 @@ async def perform_sync_jira_delta_group(): # Basic validation (credentials will be added later) if not all([sharepoint_client_id, sharepoint_client_secret, sharepoint_site_url]): - logger.warning("SharePoint credentials not configured - sync will fail") + raise ValueError("SharePoint credentials not configured") if not all([jira_username, jira_api_token]): - logger.warning("JIRA credentials not configured - sync will fail") + raise ValueError("JIRA credentials not configured") # Define the task sync definition task_sync_definition = { From b4481dc92fcc55bfb74282a95679f42328a5543d Mon Sep 17 00:00:00 2001 From: Christopher Gondek Date: Fri, 5 Sep 2025 15:05:20 +0200 Subject: [PATCH 08/12] fix: populate put columns on first run --- modules/interfaces/interfaceTicketObjects.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/interfaces/interfaceTicketObjects.py b/modules/interfaces/interfaceTicketObjects.py index 94cf8427..4ae4051e 100644 --- a/modules/interfaces/interfaceTicketObjects.py +++ b/modules/interfaces/interfaceTicketObjects.py @@ -73,7 +73,7 @@ class TicketSharepointSyncInterface: # 2. Transform tasks according to task_sync_definition audit_log.append("Step 2: Transforming JIRA data...") - transformed_tasks = self._transform_tasks(tickets, include_put=False) + transformed_tasks = self._transform_tasks(tickets, include_put=True) jira_data = [task.data for task in transformed_tasks] audit_log.append(f"JIRA issues transformed: {len(jira_data)}") audit_log.append("") From 4b9b80563236e295c860ea649e451cde6eb85dc9 Mon Sep 17 00:00:00 2001 From: Christopher Gondek Date: Fri, 5 Sep 2025 15:26:33 +0200 Subject: [PATCH 09/12] fix: minor fixes --- modules/connectors/connectorSharepoint.py | 3 +-- modules/interfaces/interfaceTicketObjects.py | 14 +++++++------- 2 files changed, 8 insertions(+), 9 deletions(-) diff --git a/modules/connectors/connectorSharepoint.py b/modules/connectors/connectorSharepoint.py index 0b1c8370..b5eaa703 100644 --- a/modules/connectors/connectorSharepoint.py +++ b/modules/connectors/connectorSharepoint.py @@ -152,8 +152,7 @@ class ConnectorSharepoint: bool: True if successful, False otherwise """ target_folder = self.ctx.web.get_folder_by_server_relative_url(folder_path) - buffer = BytesIO(content) - target_folder.files.upload(buffer, file_name).execute_query() + target_folder.upload_file(file_name, content).execute_query() return True async def overwrite_file_async( diff --git a/modules/interfaces/interfaceTicketObjects.py b/modules/interfaces/interfaceTicketObjects.py index 4ae4051e..d65e3457 100644 --- a/modules/interfaces/interfaceTicketObjects.py +++ b/modules/interfaces/interfaceTicketObjects.py @@ -1,5 +1,5 @@ from dataclasses import dataclass -from io import BytesIO +from io import BytesIO, StringIO from typing import Any import pandas as pd from modules.shared.timezoneUtils import get_utc_now @@ -251,8 +251,8 @@ class TicketSharepointSyncInterface: "" if jira_value is None else str(jira_value).strip() ) - # Only include if values are different and CSV has non-empty value - if csv_value != jira_value and csv_value: + # Include if values are different (allow empty strings to clear fields like the reference does) + if csv_value != jira_value: task_changes[field_name] = csv_value if task_changes: @@ -538,7 +538,7 @@ class TicketSharepointSyncInterface: [header_row1, header_row2, table_headers, df], ignore_index=True ) - # Convert to CSV bytes - csv_buffer = BytesIO() - final_df.to_csv(csv_buffer, index=False, header=False) - return csv_buffer.getvalue() + # Convert to CSV bytes (write text, then encode) + csv_text = StringIO() + final_df.to_csv(csv_text, index=False, header=False) + return csv_text.getvalue().encode("utf-8") From e02b250a5123de2285639b2c8d720dfb75df0483 Mon Sep 17 00:00:00 2001 From: Christopher Gondek Date: Fri, 5 Sep 2025 20:51:10 +0200 Subject: [PATCH 10/12] feat: connect router; add hourly scheduling --- app.py | 3 +++ modules/routes/routeJira.py | 36 +++++++++++++++++++++++++++++++++++- 2 files changed, 38 insertions(+), 1 deletion(-) diff --git a/app.py b/app.py index 4740357b..7622d20e 100644 --- a/app.py +++ b/app.py @@ -211,3 +211,6 @@ app.include_router(msftRouter) from modules.routes.routeSecurityGoogle import router as googleRouter app.include_router(googleRouter) + +from modules.routes.routeJira import router as jiraRouter +app.include_router(jiraRouter) \ No newline at end of file diff --git a/modules/routes/routeJira.py b/modules/routes/routeJira.py index e49c3815..7874b181 100644 --- a/modules/routes/routeJira.py +++ b/modules/routes/routeJira.py @@ -1,16 +1,50 @@ # Configure logger import logging -from fastapi import APIRouter +from fastapi import APIRouter, FastAPI +from contextlib import asynccontextmanager +from zoneinfo import ZoneInfo + from modules.connectors.connectorTicketJira import ConnectorTicketJira from modules.connectors.connectorSharepoint import ConnectorSharepoint from modules.interfaces.interfaceTicketObjects import TicketSharepointSyncInterface +from apscheduler.schedulers.asyncio import AsyncIOScheduler +from apscheduler.triggers.cron import CronTrigger + + logger = logging.getLogger(__name__) + +scheduler = AsyncIOScheduler(timezone=ZoneInfo("Europe/Zurich")) + + +@asynccontextmanager +async def router_lifespan(app: FastAPI): + # start scheduler when this router is mounted + scheduler.add_job( + perform_sync_jira_delta_group, + CronTrigger(minute="0"), # run at the top of every hour + id="jira_delta_group_sync", + replace_existing=True, + coalesce=True, + max_instances=1, + misfire_grace_time=1800, + ) + scheduler.start() + logger.info("APScheduler started (jira_delta_group_sync hourly)") + try: + yield + finally: + if scheduler.running: + scheduler.shutdown(wait=False) + logger.info("APScheduler stopped") + + router = APIRouter( prefix="/api/jira", tags=["JIRA Sync"], + lifespan=router_lifespan, ) From 24f2e7718b2d3851526bcd71f0bfd8b9bd857731 Mon Sep 17 00:00:00 2001 From: Christopher Gondek Date: Fri, 5 Sep 2025 21:06:19 +0200 Subject: [PATCH 11/12] chore: updated requirements --- requirements.txt | 3 +++ 1 file changed, 3 insertions(+) diff --git a/requirements.txt b/requirements.txt index 75f2d078..385a652e 100644 --- a/requirements.txt +++ b/requirements.txt @@ -74,6 +74,9 @@ chardet>=4.0.0 # For encoding detection pytest>=8.0.0 pytest-asyncio>=0.21.0 +## For Scheduling / Repeated Tasks +APScheduler==3.11.0 + ## Missing Dependencies for IPython and other tools decorator>=5.0.0 jedi>=0.16 From 2f0f87ea8cccbb7e2f475c0a5d7ab7701e1d74f1 Mon Sep 17 00:00:00 2001 From: Christopher Gondek Date: Fri, 5 Sep 2025 21:20:00 +0200 Subject: [PATCH 12/12] feat: harden against empty jira --- modules/interfaces/interfaceTicketObjects.py | 23 +++++++++++++++++++- 1 file changed, 22 insertions(+), 1 deletion(-) diff --git a/modules/interfaces/interfaceTicketObjects.py b/modules/interfaces/interfaceTicketObjects.py index d65e3457..3df6464f 100644 --- a/modules/interfaces/interfaceTicketObjects.py +++ b/modules/interfaces/interfaceTicketObjects.py @@ -509,7 +509,28 @@ class TicketSharepointSyncInterface: def _create_csv_content(self, data: list[dict]) -> bytes: """Create CSV content with 4-row structure matching reference code.""" if not data: - return b"" + # Build an empty table with the expected columns from schema + cols = list(self.task_sync_definition.keys()) + + df = pd.DataFrame(columns=cols) + + # Row 1 & 2: keep your current banner lines + header_row1 = pd.DataFrame( + [["Header 1"] + [""] * (len(cols) - 1)], columns=cols + ) + header_row2 = pd.DataFrame( + [["Header 2"] + [""] * (len(cols) - 1)], columns=cols + ) + + # Row 3: table headers + table_headers = pd.DataFrame([cols], columns=cols) + + final_df = pd.concat( + [header_row1, header_row2, table_headers, df], ignore_index=True + ) + csv_text = StringIO() + final_df.to_csv(csv_text, index=False, header=False) + return csv_text.getvalue().encode("utf-8") # Create DataFrame from data df = pd.DataFrame(data)