1099 lines
49 KiB
Python
1099 lines
49 KiB
Python
"""
|
|
JIRA operations method module.
|
|
Handles JIRA ticket operations including connection, export, import, and data processing.
|
|
"""
|
|
|
|
import logging
|
|
import json
|
|
import io
|
|
import pandas as pd
|
|
import csv as csv_module
|
|
from io import StringIO, BytesIO
|
|
from typing import Dict, Any, List, Optional
|
|
from datetime import datetime, UTC
|
|
|
|
from modules.workflows.methods.methodBase import MethodBase, action
|
|
from modules.datamodels.datamodelChat import ActionResult, ActionDocument
|
|
from modules.datamodels.datamodelDocref import DocumentReferenceList
|
|
from modules.shared.configuration import APP_CONFIG
|
|
|
|
logger = logging.getLogger(__name__)
|
|
|
|
class MethodJira(MethodBase):
|
|
"""JIRA operations methods."""
|
|
|
|
def __init__(self, services):
|
|
super().__init__(services)
|
|
self.name = "jira"
|
|
self.description = "JIRA operations methods"
|
|
# Store connections in memory (keyed by connectionId)
|
|
self._connections: Dict[str, Any] = {}
|
|
|
|
def _convertAdfToText(self, adfData):
|
|
"""Convert Atlassian Document Format (ADF) to plain text.
|
|
|
|
Based on Atlassian Document Format specification for JIRA fields.
|
|
Handles paragraphs, lists, text formatting, and other ADF node types.
|
|
|
|
Args:
|
|
adfData: ADF object or None
|
|
|
|
Returns:
|
|
str: Plain text content, or empty string if None/invalid
|
|
"""
|
|
if not adfData or not isinstance(adfData, dict):
|
|
return ""
|
|
|
|
if adfData.get("type") != "doc":
|
|
return str(adfData) if adfData else ""
|
|
|
|
content = adfData.get("content", [])
|
|
if not isinstance(content, list):
|
|
return ""
|
|
|
|
def extractTextFromContent(contentList, listLevel=0):
|
|
"""Recursively extract text from ADF content with proper formatting."""
|
|
textParts = []
|
|
listCounter = 1
|
|
|
|
for item in contentList:
|
|
if not isinstance(item, dict):
|
|
continue
|
|
|
|
itemType = item.get("type", "")
|
|
|
|
if itemType == "text":
|
|
# Extract text content, preserving formatting
|
|
text = item.get("text", "")
|
|
marks = item.get("marks", [])
|
|
|
|
# Handle text formatting (bold, italic, etc.)
|
|
if marks:
|
|
for mark in marks:
|
|
if mark.get("type") == "strong":
|
|
text = f"**{text}**"
|
|
elif mark.get("type") == "em":
|
|
text = f"*{text}*"
|
|
elif mark.get("type") == "code":
|
|
text = f"`{text}`"
|
|
elif mark.get("type") == "link":
|
|
attrs = mark.get("attrs", {})
|
|
href = attrs.get("href", "")
|
|
if href:
|
|
text = f"[{text}]({href})"
|
|
|
|
textParts.append(text)
|
|
|
|
elif itemType == "hardBreak":
|
|
textParts.append("\n")
|
|
|
|
elif itemType == "paragraph":
|
|
paragraphContent = item.get("content", [])
|
|
if paragraphContent:
|
|
paragraphText = extractTextFromContent(paragraphContent, listLevel)
|
|
if paragraphText.strip():
|
|
textParts.append(paragraphText)
|
|
|
|
elif itemType == "bulletList":
|
|
listContent = item.get("content", [])
|
|
for listItem in listContent:
|
|
if listItem.get("type") == "listItem":
|
|
listItemContent = listItem.get("content", [])
|
|
for listParagraph in listItemContent:
|
|
if listParagraph.get("type") == "paragraph":
|
|
listParagraphContent = listParagraph.get("content", [])
|
|
if listParagraphContent:
|
|
indent = " " * listLevel
|
|
bulletText = extractTextFromContent(listParagraphContent, listLevel + 1)
|
|
if bulletText.strip():
|
|
textParts.append(f"{indent}• {bulletText}")
|
|
|
|
elif itemType == "orderedList":
|
|
listContent = item.get("content", [])
|
|
for listItem in listContent:
|
|
if listItem.get("type") == "listItem":
|
|
listItemContent = listItem.get("content", [])
|
|
for listParagraph in listItemContent:
|
|
if listParagraph.get("type") == "paragraph":
|
|
listParagraphContent = listParagraph.get("content", [])
|
|
if listParagraphContent:
|
|
indent = " " * listLevel
|
|
orderedText = extractTextFromContent(listParagraphContent, listLevel + 1)
|
|
if orderedText.strip():
|
|
textParts.append(f"{indent}{listCounter}. {orderedText}")
|
|
listCounter += 1
|
|
|
|
elif itemType == "listItem":
|
|
# Handle nested list items
|
|
listItemContent = item.get("content", [])
|
|
if listItemContent:
|
|
textParts.append(extractTextFromContent(listItemContent, listLevel))
|
|
|
|
elif itemType == "embedCard":
|
|
# Handle embedded content (videos, etc.)
|
|
attrs = item.get("attrs", {})
|
|
url = attrs.get("url", "")
|
|
if url:
|
|
textParts.append(f"[Embedded Content: {url}]")
|
|
|
|
elif itemType == "codeBlock":
|
|
# Handle code blocks
|
|
codeContent = item.get("content", [])
|
|
if codeContent:
|
|
codeText = extractTextFromContent(codeContent, listLevel)
|
|
if codeText.strip():
|
|
textParts.append(f"```\n{codeText}\n```")
|
|
|
|
elif itemType == "blockquote":
|
|
# Handle blockquotes
|
|
quoteContent = item.get("content", [])
|
|
if quoteContent:
|
|
quoteText = extractTextFromContent(quoteContent, listLevel)
|
|
if quoteText.strip():
|
|
textParts.append(f"> {quoteText}")
|
|
|
|
elif itemType == "heading":
|
|
# Handle headings
|
|
headingContent = item.get("content", [])
|
|
if headingContent:
|
|
headingText = extractTextFromContent(headingContent, listLevel)
|
|
if headingText.strip():
|
|
level = item.get("attrs", {}).get("level", 1)
|
|
textParts.append(f"{'#' * level} {headingText}")
|
|
|
|
elif itemType == "rule":
|
|
# Handle horizontal rules
|
|
textParts.append("---")
|
|
|
|
else:
|
|
# Handle unknown types by trying to extract content
|
|
if "content" in item:
|
|
contentText = extractTextFromContent(item.get("content", []), listLevel)
|
|
if contentText.strip():
|
|
textParts.append(contentText)
|
|
|
|
return "\n".join(textParts)
|
|
|
|
result = extractTextFromContent(content)
|
|
return result.strip()
|
|
|
|
def _getDocumentData(self, documentReference: str) -> Any:
|
|
"""Get document data from a document reference (string or document object)."""
|
|
try:
|
|
if isinstance(documentReference, str):
|
|
# Get document from workflow
|
|
documentList = DocumentReferenceList.from_string_list([documentReference])
|
|
chatDocuments = self.services.chat.getChatDocumentsFromDocumentList(documentList)
|
|
if not chatDocuments or len(chatDocuments) == 0:
|
|
return None
|
|
document = chatDocuments[0]
|
|
return document.documentData
|
|
else:
|
|
# Assume it's already a document object
|
|
return documentReference.documentData if hasattr(documentReference, 'documentData') else documentReference
|
|
except Exception as e:
|
|
logger.error(f"Error getting document data: {str(e)}")
|
|
return None
|
|
|
|
def _parseJsonFromDocument(self, documentReference: str) -> Optional[Dict[str, Any]]:
|
|
"""Parse JSON from a document reference."""
|
|
data = self._getDocumentData(documentReference)
|
|
if data is None:
|
|
return None
|
|
|
|
if isinstance(data, str):
|
|
try:
|
|
return json.loads(data)
|
|
except json.JSONDecodeError:
|
|
return None
|
|
elif isinstance(data, dict):
|
|
return data
|
|
else:
|
|
return None
|
|
|
|
@action
|
|
async def connectJira(self, parameters: Dict[str, Any]) -> ActionResult:
|
|
"""
|
|
Connect to JIRA instance and create ticket interface.
|
|
|
|
Parameters:
|
|
- apiUsername (str, required): JIRA API username/email
|
|
- apiTokenConfigKey (str, required): APP_CONFIG key name for JIRA API token
|
|
- apiUrl (str, required): JIRA instance URL (e.g., https://example.atlassian.net)
|
|
- projectCode (str, required): JIRA project code (e.g., "DCS")
|
|
- issueType (str, required): JIRA issue type (e.g., "Task")
|
|
- taskSyncDefinition (str or dict, required): Field mapping definition as JSON string or dict
|
|
|
|
Returns:
|
|
- ActionResult with ActionDocument containing connection ID
|
|
"""
|
|
try:
|
|
apiUsername = parameters.get("apiUsername")
|
|
if not apiUsername:
|
|
return ActionResult.isFailure(error="apiUsername parameter is required")
|
|
|
|
apiTokenConfigKey = parameters.get("apiTokenConfigKey")
|
|
if not apiTokenConfigKey:
|
|
return ActionResult.isFailure(error="apiTokenConfigKey parameter is required")
|
|
|
|
apiUrl = parameters.get("apiUrl")
|
|
if not apiUrl:
|
|
return ActionResult.isFailure(error="apiUrl parameter is required")
|
|
|
|
projectCode = parameters.get("projectCode")
|
|
if not projectCode:
|
|
return ActionResult.isFailure(error="projectCode parameter is required")
|
|
|
|
issueType = parameters.get("issueType")
|
|
if not issueType:
|
|
return ActionResult.isFailure(error="issueType parameter is required")
|
|
|
|
taskSyncDefinitionParam = parameters.get("taskSyncDefinition")
|
|
if not taskSyncDefinitionParam:
|
|
return ActionResult.isFailure(error="taskSyncDefinition parameter is required")
|
|
|
|
# Parse taskSyncDefinition
|
|
if isinstance(taskSyncDefinitionParam, str):
|
|
try:
|
|
taskSyncDefinition = json.loads(taskSyncDefinitionParam)
|
|
except json.JSONDecodeError as e:
|
|
return ActionResult.isFailure(error=f"taskSyncDefinition is not valid JSON: {str(e)}")
|
|
elif isinstance(taskSyncDefinitionParam, dict):
|
|
taskSyncDefinition = taskSyncDefinitionParam
|
|
else:
|
|
return ActionResult.isFailure(error=f"taskSyncDefinition must be a dict or JSON string, got {type(taskSyncDefinitionParam)}")
|
|
|
|
# Get API token from APP_CONFIG
|
|
apiToken = APP_CONFIG.get(apiTokenConfigKey)
|
|
if not apiToken:
|
|
errorMsg = f"{apiTokenConfigKey} not found in APP_CONFIG"
|
|
logger.error(errorMsg)
|
|
return ActionResult.isFailure(error=errorMsg)
|
|
|
|
# Create ticket interface
|
|
syncInterface = await self.services.ticket.connectTicket(
|
|
taskSyncDefinition=taskSyncDefinition,
|
|
connectorType="Jira",
|
|
connectorParams={
|
|
"apiUsername": apiUsername,
|
|
"apiToken": apiToken,
|
|
"apiUrl": apiUrl,
|
|
"projectCode": projectCode,
|
|
"ticketType": issueType,
|
|
},
|
|
)
|
|
|
|
# Store connection with unique ID
|
|
import uuid
|
|
connectionId = str(uuid.uuid4())
|
|
self._connections[connectionId] = {
|
|
"interface": syncInterface,
|
|
"taskSyncDefinition": taskSyncDefinition,
|
|
"apiUrl": apiUrl,
|
|
"projectCode": projectCode,
|
|
}
|
|
|
|
logger.info(f"JIRA connection established: {connectionId} (Project: {projectCode})")
|
|
|
|
# Generate filename
|
|
workflowContext = self.services.chat.getWorkflowContext() if hasattr(self.services, 'chat') else None
|
|
filename = self._generateMeaningfulFileName(
|
|
"jira_connection",
|
|
"json",
|
|
workflowContext,
|
|
"connectJira"
|
|
)
|
|
|
|
# Create connection info document
|
|
connectionInfo = {
|
|
"connectionId": connectionId,
|
|
"apiUrl": apiUrl,
|
|
"projectCode": projectCode,
|
|
"issueType": issueType,
|
|
}
|
|
|
|
validationMetadata = self._createValidationMetadata(
|
|
"connectJira",
|
|
connectionId=connectionId,
|
|
apiUrl=apiUrl,
|
|
projectCode=projectCode
|
|
)
|
|
|
|
document = ActionDocument(
|
|
documentName=filename,
|
|
documentData=json.dumps(connectionInfo, indent=2),
|
|
mimeType="application/json",
|
|
validationMetadata=validationMetadata
|
|
)
|
|
|
|
return ActionResult.isSuccess(documents=[document])
|
|
|
|
except Exception as e:
|
|
errorMsg = f"Error connecting to JIRA: {str(e)}"
|
|
logger.error(errorMsg)
|
|
return ActionResult.isFailure(error=errorMsg)
|
|
|
|
@action
|
|
async def exportTicketsAsJson(self, parameters: Dict[str, Any]) -> ActionResult:
|
|
"""
|
|
Export tickets from JIRA as JSON list.
|
|
|
|
Parameters:
|
|
- connectionId (str, required): Connection ID from connectJira action result
|
|
- taskSyncDefinition (str or dict, optional): Field mapping definition (if not provided, uses stored definition)
|
|
|
|
Returns:
|
|
- ActionResult with ActionDocument containing list of tickets as JSON
|
|
"""
|
|
try:
|
|
connectionIdParam = parameters.get("connectionId")
|
|
if not connectionIdParam:
|
|
return ActionResult.isFailure(error="connectionId parameter is required")
|
|
|
|
# Get connection ID from document if it's a reference
|
|
connectionId = None
|
|
if isinstance(connectionIdParam, str):
|
|
# Try to parse from document reference
|
|
connectionInfo = self._parseJsonFromDocument(connectionIdParam)
|
|
if connectionInfo and "connectionId" in connectionInfo:
|
|
connectionId = connectionInfo["connectionId"]
|
|
else:
|
|
# Assume it's the connection ID directly
|
|
connectionId = connectionIdParam
|
|
|
|
if not connectionId or connectionId not in self._connections:
|
|
return ActionResult.isFailure(error=f"Connection ID {connectionIdParam} not found. Ensure connectJira was called first.")
|
|
|
|
connection = self._connections[connectionId]
|
|
syncInterface = connection["interface"]
|
|
|
|
# Export tickets
|
|
dataList = await syncInterface.exportTicketsAsList()
|
|
|
|
logger.info(f"Exported {len(dataList)} tickets from JIRA")
|
|
|
|
# Generate filename
|
|
workflowContext = self.services.chat.getWorkflowContext() if hasattr(self.services, 'chat') else None
|
|
filename = self._generateMeaningfulFileName(
|
|
"jira_tickets_export",
|
|
"json",
|
|
workflowContext,
|
|
"exportTicketsAsJson"
|
|
)
|
|
|
|
validationMetadata = self._createValidationMetadata(
|
|
"exportTicketsAsJson",
|
|
connectionId=connectionId,
|
|
ticketCount=len(dataList)
|
|
)
|
|
|
|
document = ActionDocument(
|
|
documentName=filename,
|
|
documentData=json.dumps(dataList, indent=2, ensure_ascii=False),
|
|
mimeType="application/json",
|
|
validationMetadata=validationMetadata
|
|
)
|
|
|
|
return ActionResult.isSuccess(documents=[document])
|
|
|
|
except Exception as e:
|
|
errorMsg = f"Error exporting tickets from JIRA: {str(e)}"
|
|
logger.error(errorMsg)
|
|
return ActionResult.isFailure(error=errorMsg)
|
|
|
|
@action
|
|
async def importTicketsFromJson(self, parameters: Dict[str, Any]) -> ActionResult:
|
|
"""
|
|
Import ticket data from JSON back to JIRA.
|
|
|
|
Parameters:
|
|
- connectionId (str, required): Connection ID from connectJira action result
|
|
- ticketData (str, required): Document reference containing ticket data as JSON
|
|
- taskSyncDefinition (str or dict, optional): Field mapping definition (if not provided, uses stored definition)
|
|
|
|
Returns:
|
|
- ActionResult with ActionDocument containing import result with counts
|
|
"""
|
|
try:
|
|
connectionIdParam = parameters.get("connectionId")
|
|
if not connectionIdParam:
|
|
return ActionResult.isFailure(error="connectionId parameter is required")
|
|
|
|
ticketDataParam = parameters.get("ticketData")
|
|
if not ticketDataParam:
|
|
return ActionResult.isFailure(error="ticketData parameter is required")
|
|
|
|
# Get connection ID from document if it's a reference
|
|
connectionId = None
|
|
if isinstance(connectionIdParam, str):
|
|
connectionInfo = self._parseJsonFromDocument(connectionIdParam)
|
|
if connectionInfo and "connectionId" in connectionInfo:
|
|
connectionId = connectionInfo["connectionId"]
|
|
else:
|
|
connectionId = connectionIdParam
|
|
|
|
if not connectionId or connectionId not in self._connections:
|
|
return ActionResult.isFailure(error=f"Connection ID {connectionIdParam} not found. Ensure connectJira was called first.")
|
|
|
|
connection = self._connections[connectionId]
|
|
syncInterface = connection["interface"]
|
|
|
|
# Get ticket data from document
|
|
ticketDataJson = self._parseJsonFromDocument(ticketDataParam)
|
|
if ticketDataJson is None:
|
|
return ActionResult.isFailure(error="Could not parse ticket data from document reference")
|
|
|
|
# Ensure it's a list
|
|
if not isinstance(ticketDataJson, list):
|
|
return ActionResult.isFailure(error="ticketData must be a JSON array")
|
|
|
|
# Import tickets
|
|
await syncInterface.importListToTickets(ticketDataJson)
|
|
|
|
logger.info(f"Imported {len(ticketDataJson)} tickets to JIRA")
|
|
|
|
# Generate filename
|
|
workflowContext = self.services.chat.getWorkflowContext() if hasattr(self.services, 'chat') else None
|
|
filename = self._generateMeaningfulFileName(
|
|
"jira_import_result",
|
|
"json",
|
|
workflowContext,
|
|
"importTicketsFromJson"
|
|
)
|
|
|
|
importResult = {
|
|
"imported": len(ticketDataJson),
|
|
"connectionId": connectionId,
|
|
}
|
|
|
|
validationMetadata = self._createValidationMetadata(
|
|
"importTicketsFromJson",
|
|
connectionId=connectionId,
|
|
importedCount=len(ticketDataJson)
|
|
)
|
|
|
|
document = ActionDocument(
|
|
documentName=filename,
|
|
documentData=json.dumps(importResult, indent=2),
|
|
mimeType="application/json",
|
|
validationMetadata=validationMetadata
|
|
)
|
|
|
|
return ActionResult.isSuccess(documents=[document])
|
|
|
|
except Exception as e:
|
|
errorMsg = f"Error importing tickets to JIRA: {str(e)}"
|
|
logger.error(errorMsg)
|
|
return ActionResult.isFailure(error=errorMsg)
|
|
|
|
@action
|
|
async def mergeTicketData(self, parameters: Dict[str, Any]) -> ActionResult:
|
|
"""
|
|
Merge JIRA export data with existing SharePoint data.
|
|
|
|
Parameters:
|
|
- jiraData (str, required): Document reference containing JIRA ticket data as JSON array
|
|
- existingData (str, required): Document reference containing existing SharePoint data as JSON array
|
|
- taskSyncDefinition (str or dict, required): Field mapping definition
|
|
- idField (str, optional): Field name to use as ID for merging (default: "ID")
|
|
|
|
Returns:
|
|
- ActionResult with ActionDocument containing merged data and merge details
|
|
"""
|
|
try:
|
|
jiraDataParam = parameters.get("jiraData")
|
|
if not jiraDataParam:
|
|
return ActionResult.isFailure(error="jiraData parameter is required")
|
|
|
|
existingDataParam = parameters.get("existingData")
|
|
if not existingDataParam:
|
|
return ActionResult.isFailure(error="existingData parameter is required")
|
|
|
|
taskSyncDefinitionParam = parameters.get("taskSyncDefinition")
|
|
if not taskSyncDefinitionParam:
|
|
return ActionResult.isFailure(error="taskSyncDefinition parameter is required")
|
|
|
|
idField = parameters.get("idField", "ID")
|
|
|
|
# Parse taskSyncDefinition
|
|
if isinstance(taskSyncDefinitionParam, str):
|
|
try:
|
|
taskSyncDefinition = json.loads(taskSyncDefinitionParam)
|
|
except json.JSONDecodeError as e:
|
|
return ActionResult.isFailure(error=f"taskSyncDefinition is not valid JSON: {str(e)}")
|
|
elif isinstance(taskSyncDefinitionParam, dict):
|
|
taskSyncDefinition = taskSyncDefinitionParam
|
|
else:
|
|
return ActionResult.isFailure(error=f"taskSyncDefinition must be a dict or JSON string, got {type(taskSyncDefinitionParam)}")
|
|
|
|
# Get data from documents
|
|
jiraDataJson = self._parseJsonFromDocument(jiraDataParam)
|
|
if jiraDataJson is None or not isinstance(jiraDataJson, list):
|
|
return ActionResult.isFailure(error="Could not parse jiraData as JSON array")
|
|
|
|
existingDataJson = self._parseJsonFromDocument(existingDataParam)
|
|
if existingDataJson is None or not isinstance(existingDataJson, list):
|
|
# Empty existing data is OK
|
|
existingDataJson = []
|
|
|
|
# Perform merge
|
|
existingLookup = {row.get(idField): row for row in existingDataJson if row.get(idField)}
|
|
mergedData: List[dict] = []
|
|
changes: List[str] = []
|
|
updatedCount = addedCount = unchangedCount = 0
|
|
|
|
for jiraRow in jiraDataJson:
|
|
jiraId = jiraRow.get(idField)
|
|
if jiraId and jiraId in existingLookup:
|
|
existingRow = existingLookup[jiraId].copy()
|
|
rowChanges: List[str] = []
|
|
|
|
for fieldName, fieldConfig in taskSyncDefinition.items():
|
|
if fieldConfig[0] == 'get':
|
|
oldValue = "" if existingRow.get(fieldName) is None else str(existingRow.get(fieldName))
|
|
newValue = "" if jiraRow.get(fieldName) is None else str(jiraRow.get(fieldName))
|
|
|
|
# Convert ADF data to readable text for logging
|
|
if isinstance(newValue, dict) and newValue.get("type") == "doc":
|
|
newValueReadable = self._convertAdfToText(newValue)
|
|
if oldValue != newValueReadable:
|
|
rowChanges.append(f"{fieldName}: '{oldValue[:100]}...' -> '{newValueReadable[:100]}...'")
|
|
elif oldValue != newValue:
|
|
# Truncate long values for logging
|
|
oldTruncated = oldValue[:100] + "..." if len(oldValue) > 100 else oldValue
|
|
newTruncated = newValue[:100] + "..." if len(newValue) > 100 else newValue
|
|
rowChanges.append(f"{fieldName}: '{oldTruncated}' -> '{newTruncated}'")
|
|
|
|
existingRow[fieldName] = jiraRow.get(fieldName)
|
|
|
|
mergedData.append(existingRow)
|
|
if rowChanges:
|
|
updatedCount += 1
|
|
changes.append(f"Row ID {jiraId} updated: {', '.join(rowChanges)}")
|
|
else:
|
|
unchangedCount += 1
|
|
del existingLookup[jiraId]
|
|
else:
|
|
mergedData.append(jiraRow)
|
|
addedCount += 1
|
|
changes.append(f"Row ID {jiraId} added as new record")
|
|
|
|
# Add remaining existing rows
|
|
for remaining in existingLookup.values():
|
|
mergedData.append(remaining)
|
|
unchangedCount += 1
|
|
|
|
mergeDetails = {
|
|
"updated": updatedCount,
|
|
"added": addedCount,
|
|
"unchanged": unchangedCount,
|
|
"changes": changes
|
|
}
|
|
|
|
logger.info(f"Merged ticket data: {updatedCount} updated, {addedCount} added, {unchangedCount} unchanged")
|
|
|
|
# Generate filename
|
|
workflowContext = self.services.chat.getWorkflowContext() if hasattr(self.services, 'chat') else None
|
|
filename = self._generateMeaningfulFileName(
|
|
"merged_ticket_data",
|
|
"json",
|
|
workflowContext,
|
|
"mergeTicketData"
|
|
)
|
|
|
|
result = {
|
|
"data": mergedData,
|
|
"mergeDetails": mergeDetails
|
|
}
|
|
|
|
validationMetadata = self._createValidationMetadata(
|
|
"mergeTicketData",
|
|
updated=updatedCount,
|
|
added=addedCount,
|
|
unchanged=unchangedCount
|
|
)
|
|
|
|
document = ActionDocument(
|
|
documentName=filename,
|
|
documentData=json.dumps(result, indent=2, ensure_ascii=False),
|
|
mimeType="application/json",
|
|
validationMetadata=validationMetadata
|
|
)
|
|
|
|
return ActionResult.isSuccess(documents=[document])
|
|
|
|
except Exception as e:
|
|
errorMsg = f"Error merging ticket data: {str(e)}"
|
|
logger.error(errorMsg)
|
|
return ActionResult.isFailure(error=errorMsg)
|
|
|
|
@action
|
|
async def parseCsvContent(self, parameters: Dict[str, Any]) -> ActionResult:
|
|
"""
|
|
Parse CSV content with custom headers.
|
|
|
|
Parameters:
|
|
- csvContent (str, required): Document reference containing CSV file content as bytes
|
|
- skipRows (int, optional): Number of header rows to skip (default: 2)
|
|
- hasCustomHeaders (bool, optional): Whether CSV has custom header rows (default: true)
|
|
|
|
Returns:
|
|
- ActionResult with ActionDocument containing parsed data and headers as JSON
|
|
"""
|
|
try:
|
|
csvContentParam = parameters.get("csvContent")
|
|
if not csvContentParam:
|
|
return ActionResult.isFailure(error="csvContent parameter is required")
|
|
|
|
skipRows = parameters.get("skipRows", 2)
|
|
hasCustomHeaders = parameters.get("hasCustomHeaders", True)
|
|
|
|
# Get CSV content from document
|
|
csvBytes = self._getDocumentData(csvContentParam)
|
|
if csvBytes is None:
|
|
return ActionResult.isFailure(error="Could not get CSV content from document reference")
|
|
|
|
# Convert to bytes if needed
|
|
if isinstance(csvBytes, str):
|
|
csvBytes = csvBytes.encode('utf-8')
|
|
elif not isinstance(csvBytes, bytes):
|
|
return ActionResult.isFailure(error="CSV content must be bytes or string")
|
|
|
|
# Parse headers if hasCustomHeaders
|
|
headers = {"header1": "Header 1", "header2": "Header 2"}
|
|
if hasCustomHeaders:
|
|
csvLines = csvBytes.decode('utf-8').split('\n')
|
|
if len(csvLines) >= 2:
|
|
headers["header1"] = csvLines[0].rstrip('\r\n')
|
|
headers["header2"] = csvLines[1].rstrip('\r\n')
|
|
|
|
# Parse CSV data
|
|
df = pd.read_csv(
|
|
io.BytesIO(csvBytes),
|
|
skiprows=skipRows,
|
|
quoting=1,
|
|
escapechar='\\',
|
|
on_bad_lines='skip',
|
|
engine='python'
|
|
)
|
|
|
|
# Convert to dict records
|
|
for column in df.columns:
|
|
df[column] = df[column].astype('object').fillna('')
|
|
data = df.to_dict(orient='records')
|
|
|
|
logger.info(f"Parsed CSV: {len(data)} rows, {len(df.columns)} columns")
|
|
|
|
# Generate filename
|
|
workflowContext = self.services.chat.getWorkflowContext() if hasattr(self.services, 'chat') else None
|
|
filename = self._generateMeaningfulFileName(
|
|
"parsed_csv_data",
|
|
"json",
|
|
workflowContext,
|
|
"parseCsvContent"
|
|
)
|
|
|
|
result = {
|
|
"data": data,
|
|
"headers": headers,
|
|
"rowCount": len(data),
|
|
"columnCount": len(df.columns)
|
|
}
|
|
|
|
validationMetadata = self._createValidationMetadata(
|
|
"parseCsvContent",
|
|
rowCount=len(data),
|
|
columnCount=len(df.columns),
|
|
skipRows=skipRows
|
|
)
|
|
|
|
document = ActionDocument(
|
|
documentName=filename,
|
|
documentData=json.dumps(result, indent=2, ensure_ascii=False),
|
|
mimeType="application/json",
|
|
validationMetadata=validationMetadata
|
|
)
|
|
|
|
return ActionResult.isSuccess(documents=[document])
|
|
|
|
except Exception as e:
|
|
errorMsg = f"Error parsing CSV content: {str(e)}"
|
|
logger.error(errorMsg)
|
|
return ActionResult.isFailure(error=errorMsg)
|
|
|
|
@action
|
|
async def parseExcelContent(self, parameters: Dict[str, Any]) -> ActionResult:
|
|
"""
|
|
Parse Excel content with custom headers.
|
|
|
|
Parameters:
|
|
- excelContent (str, required): Document reference containing Excel file content as bytes
|
|
- skipRows (int, optional): Number of header rows to skip (default: 3)
|
|
- hasCustomHeaders (bool, optional): Whether Excel has custom header rows (default: true)
|
|
|
|
Returns:
|
|
- ActionResult with ActionDocument containing parsed data and headers as JSON
|
|
"""
|
|
try:
|
|
excelContentParam = parameters.get("excelContent")
|
|
if not excelContentParam:
|
|
return ActionResult.isFailure(error="excelContent parameter is required")
|
|
|
|
skipRows = parameters.get("skipRows", 3)
|
|
hasCustomHeaders = parameters.get("hasCustomHeaders", True)
|
|
|
|
# Get Excel content from document
|
|
excelBytes = self._getDocumentData(excelContentParam)
|
|
if excelBytes is None:
|
|
return ActionResult.isFailure(error="Could not get Excel content from document reference")
|
|
|
|
# Convert to bytes if needed
|
|
if isinstance(excelBytes, str):
|
|
excelBytes = excelBytes.encode('latin-1') # Excel might have binary data
|
|
elif not isinstance(excelBytes, bytes):
|
|
return ActionResult.isFailure(error="Excel content must be bytes or string")
|
|
|
|
# Parse Excel
|
|
df = pd.read_excel(BytesIO(excelBytes), engine='openpyxl', header=None)
|
|
|
|
# Extract headers if hasCustomHeaders
|
|
headers = {"header1": "Header 1", "header2": "Header 2"}
|
|
if hasCustomHeaders and len(df) >= 3:
|
|
headerRow1 = df.iloc[0:1].copy()
|
|
headerRow2 = df.iloc[1:2].copy()
|
|
tableHeaders = df.iloc[2:3].copy()
|
|
dfData = df.iloc[skipRows:].copy()
|
|
dfData.columns = tableHeaders.iloc[0]
|
|
|
|
headers = {
|
|
"header1": ",".join([str(x) if pd.notna(x) else "" for x in headerRow1.iloc[0].tolist()]),
|
|
"header2": ",".join([str(x) if pd.notna(x) else "" for x in headerRow2.iloc[0].tolist()]),
|
|
}
|
|
else:
|
|
# No custom headers, use standard parsing
|
|
if skipRows > 0:
|
|
dfData = df.iloc[skipRows:].copy()
|
|
if len(df) > skipRows:
|
|
dfData.columns = df.iloc[skipRows-1]
|
|
else:
|
|
dfData = df.copy()
|
|
|
|
# Reset index and clean data
|
|
dfData = dfData.reset_index(drop=True)
|
|
for column in dfData.columns:
|
|
dfData[column] = dfData[column].astype('object').fillna('')
|
|
|
|
data = dfData.to_dict(orient='records')
|
|
|
|
logger.info(f"Parsed Excel: {len(data)} rows, {len(dfData.columns)} columns")
|
|
|
|
# Generate filename
|
|
workflowContext = self.services.chat.getWorkflowContext() if hasattr(self.services, 'chat') else None
|
|
filename = self._generateMeaningfulFileName(
|
|
"parsed_excel_data",
|
|
"json",
|
|
workflowContext,
|
|
"parseExcelContent"
|
|
)
|
|
|
|
result = {
|
|
"data": data,
|
|
"headers": headers,
|
|
"rowCount": len(data),
|
|
"columnCount": len(dfData.columns)
|
|
}
|
|
|
|
validationMetadata = self._createValidationMetadata(
|
|
"parseExcelContent",
|
|
rowCount=len(data),
|
|
columnCount=len(dfData.columns),
|
|
skipRows=skipRows
|
|
)
|
|
|
|
document = ActionDocument(
|
|
documentName=filename,
|
|
documentData=json.dumps(result, indent=2, ensure_ascii=False),
|
|
mimeType="application/json",
|
|
validationMetadata=validationMetadata
|
|
)
|
|
|
|
return ActionResult.isSuccess(documents=[document])
|
|
|
|
except Exception as e:
|
|
errorMsg = f"Error parsing Excel content: {str(e)}"
|
|
logger.error(errorMsg)
|
|
return ActionResult.isFailure(error=errorMsg)
|
|
|
|
@action
|
|
async def createCsvContent(self, parameters: Dict[str, Any]) -> ActionResult:
|
|
"""
|
|
Create CSV content with custom headers.
|
|
|
|
Parameters:
|
|
- data (str, required): Document reference containing data as JSON (with "data" field from mergeTicketData)
|
|
- headers (str, optional): Document reference containing headers JSON (from parseCsvContent/parseExcelContent)
|
|
- columns (str or list, optional): List of column names (if not provided, extracted from taskSyncDefinition or data)
|
|
- taskSyncDefinition (str or dict, optional): Field mapping definition (used to extract column names if columns not provided)
|
|
|
|
Returns:
|
|
- ActionResult with ActionDocument containing CSV content as bytes
|
|
"""
|
|
try:
|
|
dataParam = parameters.get("data")
|
|
if not dataParam:
|
|
return ActionResult.isFailure(error="data parameter is required")
|
|
|
|
headersParam = parameters.get("headers")
|
|
columnsParam = parameters.get("columns")
|
|
taskSyncDefinitionParam = parameters.get("taskSyncDefinition")
|
|
|
|
# Get data from document
|
|
dataJson = self._parseJsonFromDocument(dataParam)
|
|
if dataJson is None:
|
|
return ActionResult.isFailure(error="Could not parse data from document reference")
|
|
|
|
# Extract data array if wrapped in object
|
|
if isinstance(dataJson, dict) and "data" in dataJson:
|
|
dataList = dataJson["data"]
|
|
elif isinstance(dataJson, list):
|
|
dataList = dataJson
|
|
else:
|
|
return ActionResult.isFailure(error="Data must be a JSON array or object with 'data' field")
|
|
|
|
# Get headers
|
|
headers = {"header1": "Header 1", "header2": "Header 2"}
|
|
if headersParam:
|
|
headersJson = self._parseJsonFromDocument(headersParam)
|
|
if headersJson and isinstance(headersJson, dict) and "headers" in headersJson:
|
|
headers = headersJson["headers"]
|
|
elif headersJson and isinstance(headersJson, dict):
|
|
headers = headersJson
|
|
|
|
# Get columns
|
|
if columnsParam:
|
|
if isinstance(columnsParam, str):
|
|
try:
|
|
columns = json.loads(columnsParam) if columnsParam.startswith('[') or columnsParam.startswith('{') else columnsParam.split(',')
|
|
except:
|
|
columns = columnsParam.split(',')
|
|
elif isinstance(columnsParam, list):
|
|
columns = columnsParam
|
|
else:
|
|
columns = None
|
|
elif taskSyncDefinitionParam:
|
|
# Extract columns from taskSyncDefinition
|
|
if isinstance(taskSyncDefinitionParam, str):
|
|
taskSyncDefinition = json.loads(taskSyncDefinitionParam)
|
|
else:
|
|
taskSyncDefinition = taskSyncDefinitionParam
|
|
columns = list(taskSyncDefinition.keys())
|
|
elif dataList and len(dataList) > 0:
|
|
columns = list(dataList[0].keys())
|
|
else:
|
|
columns = []
|
|
|
|
# Create DataFrame
|
|
if not dataList:
|
|
df = pd.DataFrame(columns=columns)
|
|
else:
|
|
df = pd.DataFrame(dataList)
|
|
# Ensure all columns exist
|
|
for col in columns:
|
|
if col not in df.columns:
|
|
df[col] = ""
|
|
# Reorder columns
|
|
df = df[columns]
|
|
|
|
# Clean data
|
|
for column in df.columns:
|
|
df[column] = df[column].astype("object").fillna("")
|
|
df[column] = df[column].astype(str).str.replace('\n', '\\n', regex=False).str.replace('"', '""', regex=False)
|
|
|
|
# Create headers with timestamp
|
|
timestamp = datetime.fromtimestamp(self.services.utils.timestampGetUtc(), UTC).strftime("%Y-%m-%d %H:%M:%S UTC")
|
|
header1Row = next(csv_module.reader([headers.get("header1", "Header 1")]), [])
|
|
header2Row = next(csv_module.reader([headers.get("header2", "Header 2")]), [])
|
|
if len(header2Row) > 1:
|
|
header2Row[1] = timestamp
|
|
|
|
headerRow1 = pd.DataFrame([header1Row + [""] * (len(df.columns) - len(header1Row))], columns=df.columns)
|
|
headerRow2 = pd.DataFrame([header2Row + [""] * (len(df.columns) - len(header2Row))], columns=df.columns)
|
|
tableHeaders = pd.DataFrame([df.columns.tolist()], columns=df.columns)
|
|
finalDf = pd.concat([headerRow1, headerRow2, tableHeaders, df], ignore_index=True)
|
|
|
|
# Convert to CSV bytes
|
|
out = StringIO()
|
|
finalDf.to_csv(out, index=False, header=False, quoting=1, escapechar='\\')
|
|
csvBytes = out.getvalue().encode('utf-8')
|
|
|
|
logger.info(f"Created CSV content: {len(dataList)} rows, {len(columns)} columns")
|
|
|
|
# Generate filename
|
|
workflowContext = self.services.chat.getWorkflowContext() if hasattr(self.services, 'chat') else None
|
|
filename = self._generateMeaningfulFileName(
|
|
"ticket_sync",
|
|
"csv",
|
|
workflowContext,
|
|
"createCsvContent"
|
|
)
|
|
|
|
validationMetadata = self._createValidationMetadata(
|
|
"createCsvContent",
|
|
rowCount=len(dataList),
|
|
columnCount=len(columns)
|
|
)
|
|
|
|
# Store as base64 for document
|
|
import base64
|
|
csvBase64 = base64.b64encode(csvBytes).decode('utf-8')
|
|
|
|
document = ActionDocument(
|
|
documentName=filename,
|
|
documentData=csvBase64,
|
|
mimeType="application/octet-stream",
|
|
validationMetadata=validationMetadata
|
|
)
|
|
|
|
return ActionResult.isSuccess(documents=[document])
|
|
|
|
except Exception as e:
|
|
errorMsg = f"Error creating CSV content: {str(e)}"
|
|
logger.error(errorMsg)
|
|
return ActionResult.isFailure(error=errorMsg)
|
|
|
|
@action
|
|
async def createExcelContent(self, parameters: Dict[str, Any]) -> ActionResult:
|
|
"""
|
|
Create Excel content with custom headers.
|
|
|
|
Parameters:
|
|
- data (str, required): Document reference containing data as JSON (with "data" field from mergeTicketData)
|
|
- headers (str, optional): Document reference containing headers JSON (from parseExcelContent)
|
|
- columns (str or list, optional): List of column names (if not provided, extracted from taskSyncDefinition or data)
|
|
- taskSyncDefinition (str or dict, optional): Field mapping definition (used to extract column names if columns not provided)
|
|
|
|
Returns:
|
|
- ActionResult with ActionDocument containing Excel content as bytes
|
|
"""
|
|
try:
|
|
dataParam = parameters.get("data")
|
|
if not dataParam:
|
|
return ActionResult.isFailure(error="data parameter is required")
|
|
|
|
headersParam = parameters.get("headers")
|
|
columnsParam = parameters.get("columns")
|
|
taskSyncDefinitionParam = parameters.get("taskSyncDefinition")
|
|
|
|
# Get data from document
|
|
dataJson = self._parseJsonFromDocument(dataParam)
|
|
if dataJson is None:
|
|
return ActionResult.isFailure(error="Could not parse data from document reference")
|
|
|
|
# Extract data array if wrapped in object
|
|
if isinstance(dataJson, dict) and "data" in dataJson:
|
|
dataList = dataJson["data"]
|
|
elif isinstance(dataJson, list):
|
|
dataList = dataJson
|
|
else:
|
|
return ActionResult.isFailure(error="Data must be a JSON array or object with 'data' field")
|
|
|
|
# Get headers
|
|
headers = {"header1": "Header 1", "header2": "Header 2"}
|
|
if headersParam:
|
|
headersJson = self._parseJsonFromDocument(headersParam)
|
|
if headersJson and isinstance(headersJson, dict) and "headers" in headersJson:
|
|
headers = headersJson["headers"]
|
|
elif headersJson and isinstance(headersJson, dict):
|
|
headers = headersJson
|
|
|
|
# Get columns
|
|
if columnsParam:
|
|
if isinstance(columnsParam, str):
|
|
try:
|
|
columns = json.loads(columnsParam) if columnsParam.startswith('[') or columnsParam.startswith('{') else columnsParam.split(',')
|
|
except:
|
|
columns = columnsParam.split(',')
|
|
elif isinstance(columnsParam, list):
|
|
columns = columnsParam
|
|
else:
|
|
columns = None
|
|
elif taskSyncDefinitionParam:
|
|
# Extract columns from taskSyncDefinition
|
|
if isinstance(taskSyncDefinitionParam, str):
|
|
taskSyncDefinition = json.loads(taskSyncDefinitionParam)
|
|
else:
|
|
taskSyncDefinition = taskSyncDefinitionParam
|
|
columns = list(taskSyncDefinition.keys())
|
|
elif dataList and len(dataList) > 0:
|
|
columns = list(dataList[0].keys())
|
|
else:
|
|
columns = []
|
|
|
|
# Create DataFrame
|
|
if not dataList:
|
|
df = pd.DataFrame(columns=columns)
|
|
else:
|
|
df = pd.DataFrame(dataList)
|
|
# Ensure all columns exist
|
|
for col in columns:
|
|
if col not in df.columns:
|
|
df[col] = ""
|
|
# Reorder columns
|
|
df = df[columns]
|
|
|
|
# Clean data
|
|
for column in df.columns:
|
|
df[column] = df[column].astype("object").fillna("")
|
|
df[column] = df[column].astype(str).str.replace('\n', '\\n', regex=False).str.replace('"', '""', regex=False)
|
|
|
|
# Create headers with timestamp
|
|
timestamp = datetime.fromtimestamp(self.services.utils.timestampGetUtc(), UTC).strftime("%Y-%m-%d %H:%M:%S UTC")
|
|
header1Row = next(csv_module.reader([headers.get("header1", "Header 1")]), [])
|
|
header2Row = next(csv_module.reader([headers.get("header2", "Header 2")]), [])
|
|
if len(header2Row) > 1:
|
|
header2Row[1] = timestamp
|
|
|
|
headerRow1 = pd.DataFrame([header1Row + [""] * (len(df.columns) - len(header1Row))], columns=df.columns)
|
|
headerRow2 = pd.DataFrame([header2Row + [""] * (len(df.columns) - len(header2Row))], columns=df.columns)
|
|
tableHeaders = pd.DataFrame([df.columns.tolist()], columns=df.columns)
|
|
finalDf = pd.concat([headerRow1, headerRow2, tableHeaders, df], ignore_index=True)
|
|
|
|
# Convert to Excel bytes
|
|
buf = BytesIO()
|
|
finalDf.to_excel(buf, index=False, header=False, engine='openpyxl')
|
|
excelBytes = buf.getvalue()
|
|
|
|
logger.info(f"Created Excel content: {len(dataList)} rows, {len(columns)} columns")
|
|
|
|
# Generate filename
|
|
workflowContext = self.services.chat.getWorkflowContext() if hasattr(self.services, 'chat') else None
|
|
filename = self._generateMeaningfulFileName(
|
|
"ticket_sync",
|
|
"xlsx",
|
|
workflowContext,
|
|
"createExcelContent"
|
|
)
|
|
|
|
validationMetadata = self._createValidationMetadata(
|
|
"createExcelContent",
|
|
rowCount=len(dataList),
|
|
columnCount=len(columns)
|
|
)
|
|
|
|
# Store as base64 for document
|
|
import base64
|
|
excelBase64 = base64.b64encode(excelBytes).decode('utf-8')
|
|
|
|
document = ActionDocument(
|
|
documentName=filename,
|
|
documentData=excelBase64,
|
|
mimeType="application/vnd.openxmlformats-officedocument.spreadsheetml.sheet",
|
|
validationMetadata=validationMetadata
|
|
)
|
|
|
|
return ActionResult.isSuccess(documents=[document])
|
|
|
|
except Exception as e:
|
|
errorMsg = f"Error creating Excel content: {str(e)}"
|
|
logger.error(errorMsg)
|
|
return ActionResult.isFailure(error=errorMsg)
|