{value}")
+ elif runType == "link":
+ href = htmlLib.escape(run.get("href", ""))
+ parts.append(f'{value}')
+ elif runType == "image":
+ b64 = run.get("base64Data", "")
+ mime = run.get("mimeType", "image/png")
+ alt = value
+ if b64:
+ parts.append(f'{self._renderInlineRuns(inlineRuns)}
') + continue text = content.get("text", "") elif isinstance(content, str): text = content @@ -495,7 +667,8 @@ class RendererHtml(BaseRenderer): # Table header htmlParts.append('tags return '\n'.join(f'
{text}
' for text in texts) return "" elif isinstance(paragraphData, str): return f'{paragraphData}
' elif isinstance(paragraphData, dict): - # Handle nested content structure: element.content vs element.text # Extract from nested content structure content = paragraphData.get("content", {}) if isinstance(content, dict): + # New format: inlineRuns + inlineRuns = content.get("inlineRuns") + if inlineRuns and isinstance(inlineRuns, list): + return f'{self._renderInlineRuns(inlineRuns)}
' text = content.get("text", "") elif isinstance(content, str): text = content diff --git a/modules/serviceCenter/services/serviceGeneration/renderers/rendererImage.py b/modules/serviceCenter/services/serviceGeneration/renderers/rendererImage.py index 2aff559f..8141b798 100644 --- a/modules/serviceCenter/services/serviceGeneration/renderers/rendererImage.py +++ b/modules/serviceCenter/services/serviceGeneration/renderers/rendererImage.py @@ -43,7 +43,7 @@ class RendererImage(BaseRenderer): """ return ["image"] - async def render(self, extractedContent: Dict[str, Any], title: str, userPrompt: str = None, aiService=None) -> List[RenderedDocument]: + async def render(self, extractedContent: Dict[str, Any], title: str, userPrompt: str = None, aiService=None, *, style: Dict[str, Any] = None) -> List[RenderedDocument]: """Render extracted JSON content to image format using AI image generation.""" try: # Generate AI image from content diff --git a/modules/serviceCenter/services/serviceGeneration/renderers/rendererJson.py b/modules/serviceCenter/services/serviceGeneration/renderers/rendererJson.py index 076210bc..470d4543 100644 --- a/modules/serviceCenter/services/serviceGeneration/renderers/rendererJson.py +++ b/modules/serviceCenter/services/serviceGeneration/renderers/rendererJson.py @@ -42,7 +42,7 @@ class RendererJson(BaseRenderer): # Return all types except image return [st for st in supportedSectionTypes if st != "image"] - async def render(self, extractedContent: Dict[str, Any], title: str, userPrompt: str = None, aiService=None) -> List[RenderedDocument]: + async def render(self, extractedContent: Dict[str, Any], title: str, userPrompt: str = None, aiService=None, *, style: Dict[str, Any] = None) -> List[RenderedDocument]: """Render extracted JSON content to JSON format.""" try: # The extracted content should already be JSON from the AI diff --git a/modules/serviceCenter/services/serviceGeneration/renderers/rendererMarkdown.py b/modules/serviceCenter/services/serviceGeneration/renderers/rendererMarkdown.py index a3b8b5b3..552266e9 100644 --- a/modules/serviceCenter/services/serviceGeneration/renderers/rendererMarkdown.py +++ b/modules/serviceCenter/services/serviceGeneration/renderers/rendererMarkdown.py @@ -40,7 +40,7 @@ class RendererMarkdown(BaseRenderer): from modules.datamodels.datamodelJson import supportedSectionTypes return [st for st in supportedSectionTypes if st != "image"] - async def render(self, extractedContent: Dict[str, Any], title: str, userPrompt: str = None, aiService=None) -> List[RenderedDocument]: + async def render(self, extractedContent: Dict[str, Any], title: str, userPrompt: str = None, aiService=None, *, style: Dict[str, Any] = None) -> List[RenderedDocument]: """Render extracted JSON content to Markdown format.""" try: # Generate markdown from JSON structure diff --git a/modules/serviceCenter/services/serviceGeneration/renderers/rendererPdf.py b/modules/serviceCenter/services/serviceGeneration/renderers/rendererPdf.py index df2aff10..7913a246 100644 --- a/modules/serviceCenter/services/serviceGeneration/renderers/rendererPdf.py +++ b/modules/serviceCenter/services/serviceGeneration/renderers/rendererPdf.py @@ -106,17 +106,17 @@ class RendererPdf(BaseRenderer): from modules.datamodels.datamodelJson import supportedSectionTypes return list(supportedSectionTypes) - async def render(self, extractedContent: Dict[str, Any], title: str, userPrompt: str = None, aiService=None) -> List[RenderedDocument]: + async def render(self, extractedContent: Dict[str, Any], title: str, userPrompt: str = None, aiService=None, *, style: Dict[str, Any] = None) -> List[RenderedDocument]: """Render extracted JSON content to PDF format using AI-analyzed styling.""" try: if not REPORTLAB_AVAILABLE: # Fallback to HTML if reportlab not available from .rendererHtml import RendererHtml html_renderer = RendererHtml() - return await html_renderer.render(extractedContent, title, userPrompt, aiService) + return await html_renderer.render(extractedContent, title, userPrompt, aiService, style=style) # Generate PDF using AI-analyzed styling - pdf_content = await self._generatePdfFromJson(extractedContent, title, userPrompt, aiService) + pdf_content = await self._generatePdfFromJson(extractedContent, title, userPrompt, aiService, unifiedStyle=style) # Extract metadata for document type and other info metadata = extractedContent.get("metadata", {}) if extractedContent else {} @@ -163,11 +163,28 @@ class RendererPdf(BaseRenderer): ) ] - async def _generatePdfFromJson(self, json_content: Dict[str, Any], title: str, userPrompt: str = None, aiService=None) -> str: + async def _generatePdfFromJson(self, json_content: Dict[str, Any], title: str, userPrompt: str = None, aiService=None, *, unifiedStyle: Dict[str, Any] = None) -> str: """Generate PDF content from structured JSON document using AI-generated styling.""" try: - # Get style set: use styles from metadata if available, otherwise enhance with AI - styles = await self._getStyleSet(json_content, userPrompt, aiService) + # Get style set from unified style or legacy approach + if unifiedStyle: + styles = self._convertUnifiedStyleToInternal(unifiedStyle) + self._unifiedStyle = unifiedStyle + for level in range(1, 7): + hKey = f"heading{level}" + if hKey not in styles: + styles[hKey] = self._defaultHeadingStyleDef(level) + else: + styles[hKey].setdefault("space_after", 12) + styles[hKey].setdefault("space_before", 12) + styles["paragraph"].setdefault("space_after", 6) + styles["paragraph"].setdefault("line_height", unifiedStyle["paragraph"].get("lineSpacing", 1.2)) + styles["bullet_list"].setdefault("space_after", 3) + styles["code_block"].setdefault("space_after", 6) + styles["code_block"].setdefault("align", "left") + else: + styles = await self._getStyleSet(json_content, userPrompt, aiService) + self._unifiedStyle = None # Validate JSON structure if not self._validateJsonStructure(json_content): @@ -179,15 +196,13 @@ class RendererPdf(BaseRenderer): # Create a buffer to hold the PDF buffer = io.BytesIO() - # Create PDF document - doc = SimpleDocTemplate( - buffer, - pagesize=A4, - rightMargin=72, - leftMargin=72, - topMargin=72, - bottomMargin=18 - ) + # Create PDF document with unified page margins or defaults + pageCfg = unifiedStyle["page"] if unifiedStyle else None + if pageCfg: + m = pageCfg["marginsPt"] + doc = SimpleDocTemplate(buffer, pagesize=A4, rightMargin=m["right"], leftMargin=m["left"], topMargin=m["top"], bottomMargin=m["bottom"]) + else: + doc = SimpleDocTemplate(buffer, pagesize=A4, rightMargin=72, leftMargin=72, topMargin=72, bottomMargin=18) # Build PDF content (no cover page — body starts on page 1; filename still uses `title`) story = [] @@ -232,13 +247,28 @@ class RendererPdf(BaseRenderer): removed = False for idx, flowable in enumerate(story): fRepr = repr(flowable) + if "Image" in fRepr and hasattr(flowable, 'drawWidth') and hasattr(flowable, 'drawHeight'): + from reportlab.platypus import Image as ReportLabImage + if isinstance(flowable, ReportLabImage): + frameH = 650.0 + frameW = 450.0 + if flowable.drawHeight > frameH or flowable.drawWidth > frameW: + scaleW = frameW / flowable.drawWidth if flowable.drawWidth > frameW else 1.0 + scaleH = frameH / flowable.drawHeight if flowable.drawHeight > frameH else 1.0 + s = min(scaleW, scaleH) * 0.9 + flowable.drawWidth = flowable.drawWidth * s + flowable.drawHeight = flowable.drawHeight * s + flowable._width = flowable.drawWidth + flowable._height = flowable.drawHeight + removed = True + break if "Table" in fRepr and hasattr(flowable, '_cellvalues'): try: nRows = len(flowable._cellvalues) nCols = len(flowable._cellvalues[0]) if flowable._cellvalues else 0 if nRows == 1 and nCols == 1: errPara = Paragraph( - "[Code block omitted — content too large for PDF page]", + "[Code block omitted - content too large for PDF page]", self._createNormalStyle({}), ) story[idx] = errPara @@ -609,6 +639,31 @@ class RendererPdf(BaseRenderer): .replace(">", ">") ) + def _renderInlineRunsToPdfXml(self, runs: list) -> str: + """Convert inline runs to ReportLab Paragraph XML.""" + parts = [] + us = getattr(self, '_unifiedStyle', None) + monoFont = us["fonts"]["monospace"] if us else "Courier" + for run in runs: + runType = run.get("type", "text") + value = self._escapeReportlabXml(run.get("value", "")) + if runType == "text": + parts.append(value) + elif runType == "bold": + parts.append(f"{value}") + elif runType == "italic": + parts.append(f"{value}") + elif runType == "code": + parts.append(f'{value}') + elif runType == "link": + href = self._escapeReportlabXml(run.get("href", "")) + parts.append(f'{value}') + elif runType == "image": + parts.append(f"[Image: {value}]") + else: + parts.append(value) + return "".join(parts) + def _applyInlineMarkdownToEscapedPlain(self, text: str) -> str: """Escape XML then apply bold/italic to a segment with no `code` spans (code is handled separately).""" if not text: @@ -744,10 +799,10 @@ class RendererPdf(BaseRenderer): return [] headers = content.get("headers", []) rows = content.get("rows", []) - + if not headers or not rows: return [] - + numCols = len(headers) colWidth = _PDF_CONTENT_WIDTH_PT / max(numCols, 1) colWidths = [colWidth] * numCols @@ -755,8 +810,12 @@ class RendererPdf(BaseRenderer): hdrPs = self._createTableCellParagraphStyle(styles, header=True, tableStyleKey="table_header") cellPs = self._createTableCellParagraphStyle(styles, header=False, tableStyleKey="table_cell") - def _cellPara(val, ps): - return self._paragraphFromInlineMarkdown(str(val) if val is not None else "", ps) + def _cellPara(cell, ps): + runs = self._inlineRunsForCell(cell) + if isinstance(cell, list): + xml = self._renderInlineRunsToPdfXml(runs) + return Paragraph(_wrapEmojiSpansInXml(xml), ps) + return self._paragraphFromInlineMarkdown(str(cell) if cell is not None else "", ps) headerRow = [_cellPara(h, hdrPs) for h in headers] bodyRows = [] @@ -786,7 +845,7 @@ class RendererPdf(BaseRenderer): ] table.setStyle(TableStyle(table_style)) return [table, Spacer(1, 12)] - + except Exception as e: self.logger.warning(f"Error rendering table: {str(e)}") return [] @@ -794,32 +853,29 @@ class RendererPdf(BaseRenderer): def _renderJsonBulletList(self, list_data: Dict[str, Any], styles: Dict[str, Any]) -> List[Any]: """Render a JSON bullet list to PDF elements using AI-generated styles.""" try: - # Extract from nested content structure content = list_data.get("content", {}) if not isinstance(content, dict): return [] items = content.get("items", []) - bullet_style_def = styles.get("bullet_list", {}) - + bulletStyleDef = styles.get("bullet_list", {}) + normalStyle = self._createNormalStyle(styles) + elements = [] for item in items: - if isinstance(item, str): - elements.append( - Paragraph(f"• {self._markdownInlineToReportlabXml(item)}", self._createNormalStyle(styles)) - ) + runs = self._inlineRunsForListItem(item) + if isinstance(item, list): + xml = self._renderInlineRunsToPdfXml(runs) + elements.append(Paragraph(f"\u2022 {_wrapEmojiSpansInXml(xml)}", normalStyle)) + elif isinstance(item, str): + elements.append(Paragraph(f"\u2022 {self._markdownInlineToReportlabXml(item)}", normalStyle)) elif isinstance(item, dict) and "text" in item: - elements.append( - Paragraph( - f"• {self._markdownInlineToReportlabXml(item['text'])}", - self._createNormalStyle(styles), - ) - ) - + elements.append(Paragraph(f"\u2022 {self._markdownInlineToReportlabXml(item['text'])}", normalStyle)) + if elements: - elements.append(Spacer(1, bullet_style_def.get("space_after", 3))) - + elements.append(Spacer(1, bulletStyleDef.get("space_after", 3))) + return elements - + except Exception as e: self.logger.warning(f"Error rendering bullet list: {str(e)}") return [] @@ -848,20 +904,27 @@ class RendererPdf(BaseRenderer): def _renderJsonParagraph(self, paragraph_data: Dict[str, Any], styles: Dict[str, Any]) -> List[Any]: """Render a JSON paragraph to PDF elements using AI-generated styles.""" try: - # Extract from nested content structure content = paragraph_data.get("content", {}) - if isinstance(content, dict): - text = content.get("text", "") - elif isinstance(content, str): - text = content - else: - text = "" - + if isinstance(content, str): + content = {"text": content} + if not isinstance(content, dict): + return [] + + normalStyle = self._createNormalStyle(styles) + + if "inlineRuns" in content: + runs = self._inlineRunsFromContent(content) + xml = self._renderInlineRunsToPdfXml(runs) + if xml: + return [Paragraph(_wrapEmojiSpansInXml(xml), normalStyle)] + return [] + + text = content.get("text", "") if text: - return [self._paragraphFromInlineMarkdown(text, self._createNormalStyle(styles))] - + return [self._paragraphFromInlineMarkdown(text, normalStyle)] + return [] - + except Exception as e: self.logger.warning(f"Error rendering paragraph: {str(e)}") return [] @@ -1030,20 +1093,18 @@ class RendererPdf(BaseRenderer): pilImage = PILImage.open(imageStream) originalWidth, originalHeight = pilImage.size - # Calculate available page dimensions (A4 with margins: 72pt left/right, 72pt top, 18pt bottom) pageWidth = A4[0] # 595.27 points pageHeight = A4[1] # 841.89 points - leftMargin = 72 - rightMargin = 72 - topMargin = 72 - bottomMargin = 18 - - # Use actual frame dimensions from SimpleDocTemplate - # Frame is smaller than page minus margins due to internal spacing - # From error message: frame is 439.27559055118115 x 739.8897637795277 - # Use conservative values with safety margin - availableWidth = 430.0 # Slightly smaller than frame width for safety - availableHeight = 730.0 # Slightly smaller than frame height for safety + # Use page dimensions minus margins with generous safety buffer + # A4 = 595.27 x 841.89 pt; frame = page - margins - internal padding + _us = getattr(self, '_unifiedStyle', None) or {} + _pageMgn = (_us.get('page') or {}).get('marginsPt') or {} + marginTop = _pageMgn.get('top', 60) + marginBottom = _pageMgn.get('bottom', 60) + marginLeft = _pageMgn.get('left', 60) + marginRight = _pageMgn.get('right', 60) + availableWidth = pageWidth - marginLeft - marginRight - 20 # 20pt safety + availableHeight = pageHeight - marginTop - marginBottom - 80 # 80pt safety for header/footer # Convert original image size from pixels to points # PIL provides size in pixels, need to convert to points diff --git a/modules/serviceCenter/services/serviceGeneration/renderers/rendererPptx.py b/modules/serviceCenter/services/serviceGeneration/renderers/rendererPptx.py index 3bdff7f1..49ee8048 100644 --- a/modules/serviceCenter/services/serviceGeneration/renderers/rendererPptx.py +++ b/modules/serviceCenter/services/serviceGeneration/renderers/rendererPptx.py @@ -59,7 +59,7 @@ class RendererPptx(BaseRenderer): from modules.datamodels.datamodelJson import supportedSectionTypes return list(supportedSectionTypes) - async def render(self, extractedContent: Dict[str, Any], title: str, userPrompt: str = None, aiService=None) -> List[RenderedDocument]: + async def render(self, extractedContent: Dict[str, Any], title: str, userPrompt: str = None, aiService=None, *, style: Dict[str, Any] = None) -> List[RenderedDocument]: """ Render content as PowerPoint presentation from JSON data. @@ -68,7 +68,7 @@ class RendererPptx(BaseRenderer): title: Title for the presentation userPrompt: User prompt for AI styling aiService: AI service for styling - **kwargs: Additional rendering options + style: Unified style dict from pipeline (preferred over AI-generated styles) Returns: Base64-encoded PowerPoint presentation as string @@ -81,8 +81,19 @@ class RendererPptx(BaseRenderer): from pptx.dml.color import RGBColor import re - # Get style set: use styles from metadata if available, otherwise enhance with AI - styles = await self._getStyleSet(extractedContent, userPrompt, aiService) + # Get style set: prefer unified style, then metadata, then AI-enhanced + if style: + internalStyle = self._convertUnifiedStyleToInternal(style) + defaultPptx = self._getDefaultStyleSet() + for key in ("slide_size", "content_per_slide", "design_theme", "color_scheme", "background_style", "accent_colors", "professional_grade", "executive_ready"): + internalStyle[key] = defaultPptx.get(key) + internalStyle["heading"] = internalStyle["heading1"] + internalStyle["subheading"] = internalStyle["heading2"] + styles = internalStyle + self._unifiedStyle = style + else: + styles = await self._getStyleSet(extractedContent, userPrompt, aiService) + self._unifiedStyle = None # Create new presentation prs = Presentation() @@ -910,15 +921,17 @@ JSON ONLY. NO OTHER TEXT.""" # Extract from nested content structure content = paragraph_data.get("content", {}) if isinstance(content, dict): - text = content.get("text", "") + if content.get("inlineRuns"): + text = "".join(r.get("value", "") for r in content["inlineRuns"]) + else: + text = content.get("text", "") elif isinstance(content, str): text = content else: text = "" if text: - # Limit paragraph length based on content density - max_length = 200 # Default limit + max_length = 200 if len(text) > max_length: text = text[:max_length] + "..." @@ -1303,6 +1316,32 @@ JSON ONLY. NO OTHER TEXT.""" r.text = text[pos:] _applyBase(r) + def _renderInlineRunsPptx(self, runs, paragraph, fontSize=None, fontColor=None): + """Process InlineRun dicts into pptx text runs.""" + from pptx.util import Pt + paragraph.text = "" + us = getattr(self, '_unifiedStyle', None) + monoFont = us["fonts"]["monospace"] if us else "Courier New" + for run in runs: + runType = run.get("type", "text") + value = run.get("value", "") + r = paragraph.add_run() + r.text = value + if fontSize: + r.font.size = fontSize + if fontColor: + r.font.color.rgb = fontColor + if runType == "bold": + r.font.bold = True + elif runType == "italic": + r.font.italic = True + elif runType == "code": + r.font.name = monoFont + if fontSize and hasattr(fontSize, 'pt'): + r.font.size = Pt(max(8, int(fontSize.pt * 0.85))) + elif runType == "link": + r.font.underline = True + def _addTableToSlide(self, slide, element: Dict[str, Any], styles: Dict[str, Any], top: float = None, max_width: float = None) -> None: """Add a PowerPoint table to slide.""" try: @@ -1374,7 +1413,8 @@ JSON ONLY. NO OTHER TEXT.""" cell = table.cell(0, col_idx) # Clear existing text and set new text cell.text_frame.clear() - header_text = str(header) if header else "" + cellRuns = self._inlineRunsForCell(header) + header_text = "".join(r.get("value", "") for r in cellRuns) cell.text = header_text # Ensure paragraph exists @@ -1420,7 +1460,8 @@ JSON ONLY. NO OTHER TEXT.""" cell = table.cell(row_idx, col_idx) # Clear existing text and set new text cell.text_frame.clear() - cell_text = str(cell_data) if cell_data is not None else "" + cellRuns = self._inlineRunsForCell(cell_data) + cell_text = "".join(r.get("value", "") for r in cellRuns) cell.text = cell_text # Ensure paragraph exists @@ -1462,9 +1503,8 @@ JSON ONLY. NO OTHER TEXT.""" fontColor = RGBColor(*self._getSafeColor(listStyle.get("color", (47, 47, 47)))) for item in items: - itemText = item.get("text", "") if isinstance(item, dict) else str(item) - if not itemText or not itemText.strip(): - continue + runs = self._inlineRunsForListItem(item) + isNewFormat = isinstance(item, list) p = text_frame.add_paragraph() p.level = 0 @@ -1472,21 +1512,33 @@ JSON ONLY. NO OTHER TEXT.""" p.space_before = Pt(2) p.space_after = Pt(2) - # Consistent bullet prefix - self._addMarkdownInlineRuns(p, f" • {itemText}", fontSize=fontSize, fontColor=fontColor, fontBold=False) + if isNewFormat: + bulletRuns = [{"type": "text", "value": " \u2022 "}] + runs + self._renderInlineRunsPptx(bulletRuns, p, fontSize=fontSize, fontColor=fontColor) + else: + itemText = item.get("text", "") if isinstance(item, dict) else str(item) + if not itemText or not itemText.strip(): + continue + self._addMarkdownInlineRuns(p, f" \u2022 {itemText}", fontSize=fontSize, fontColor=fontColor, fontBold=False) - # Subitems + # Subitems (only for dict-style items) if isinstance(item, dict): for sub in item.get("subitems", []): - subText = sub.get("text", "") if isinstance(sub, dict) else str(sub) - if not subText: - continue + subRuns = self._inlineRunsForListItem(sub) + isSubNew = isinstance(sub, list) sp = text_frame.add_paragraph() sp.level = 0 sp.alignment = PP_ALIGN.LEFT sp.space_before = Pt(1) sp.space_after = Pt(1) - self._addMarkdownInlineRuns(sp, f" – {subText}", fontSize=fontSize, fontColor=fontColor, fontBold=False) + if isSubNew: + subBulletRuns = [{"type": "text", "value": " \u2013 "}] + subRuns + self._renderInlineRunsPptx(subBulletRuns, sp, fontSize=fontSize, fontColor=fontColor) + else: + subText = sub.get("text", "") if isinstance(sub, dict) else str(sub) + if not subText: + continue + self._addMarkdownInlineRuns(sp, f" \u2013 {subText}", fontSize=fontSize, fontColor=fontColor, fontBold=False) except Exception as e: logger.warning(f"Error adding bullet list to slide: {str(e)}") @@ -1540,42 +1592,53 @@ JSON ONLY. NO OTHER TEXT.""" # Extract from nested content structure content = element.get("content", {}) if isinstance(content, dict): + inlineRuns = self._inlineRunsFromContent(content) + hasInlineRuns = content.get("inlineRuns") is not None text = content.get("text", "") elif isinstance(content, str): text = content + inlineRuns = [{"type": "text", "value": text}] if text else [] + hasInlineRuns = False else: text = "" + inlineRuns = [] + hasInlineRuns = False - if text: - p = text_frame.add_paragraph() - p.level = 0 - - try: - if hasattr(p, 'paragraph_format'): - p.paragraph_format.bullet.type = None - except (AttributeError, TypeError): - pass - - paragraph_style = styles.get("paragraph", {}) - base_font_size = paragraph_style.get("font_size", 14) - calculated_size = max(10, int(base_font_size * font_size_multiplier)) - fSize = Pt(calculated_size) - fColor = RGBColor(*self._getSafeColor(paragraph_style.get("color", (47, 47, 47)))) - fBold = paragraph_style.get("bold", False) + if not inlineRuns and not text: + return + + p = text_frame.add_paragraph() + p.level = 0 + + try: + if hasattr(p, 'paragraph_format'): + p.paragraph_format.bullet.type = None + except (AttributeError, TypeError): + pass + + paragraph_style = styles.get("paragraph", {}) + base_font_size = paragraph_style.get("font_size", 14) + calculated_size = max(10, int(base_font_size * font_size_multiplier)) + fSize = Pt(calculated_size) + fColor = RGBColor(*self._getSafeColor(paragraph_style.get("color", (47, 47, 47)))) + fBold = paragraph_style.get("bold", False) + + if hasInlineRuns: + self._renderInlineRunsPptx(inlineRuns, p, fontSize=fSize, fontColor=fColor) + else: self._addMarkdownInlineRuns(p, text, fontSize=fSize, fontColor=fColor, fontBold=fBold) - - # Add proper spacing - p.space_before = Pt(6) # Space before paragraph - p.space_after = Pt(6) # Space after paragraph - p.line_spacing = 1.2 # Line spacing for readability - - align = paragraph_style.get("align", "left") - if align == "center": - p.alignment = PP_ALIGN.CENTER - elif align == "right": - p.alignment = PP_ALIGN.RIGHT - else: - p.alignment = PP_ALIGN.LEFT + + p.space_before = Pt(6) + p.space_after = Pt(6) + p.line_spacing = 1.2 + + align = paragraph_style.get("align", "left") + if align == "center": + p.alignment = PP_ALIGN.CENTER + elif align == "right": + p.alignment = PP_ALIGN.RIGHT + else: + p.alignment = PP_ALIGN.LEFT except Exception as e: logger.warning(f"Error adding paragraph to slide: {str(e)}") diff --git a/modules/serviceCenter/services/serviceGeneration/renderers/rendererText.py b/modules/serviceCenter/services/serviceGeneration/renderers/rendererText.py index 15a7161c..94400df9 100644 --- a/modules/serviceCenter/services/serviceGeneration/renderers/rendererText.py +++ b/modules/serviceCenter/services/serviceGeneration/renderers/rendererText.py @@ -76,7 +76,7 @@ class RendererText(BaseRenderer): # Text renderer accepts all types except images return [st for st in supportedSectionTypes if st != "image"] - async def render(self, extractedContent: Dict[str, Any], title: str, userPrompt: str = None, aiService=None) -> List[RenderedDocument]: + async def render(self, extractedContent: Dict[str, Any], title: str, userPrompt: str = None, aiService=None, *, style: Dict[str, Any] = None) -> List[RenderedDocument]: """Render extracted JSON content to plain text format.""" try: # Generate text from JSON structure diff --git a/modules/serviceCenter/services/serviceGeneration/renderers/rendererXlsx.py b/modules/serviceCenter/services/serviceGeneration/renderers/rendererXlsx.py index 79f5688c..3c6fdd5e 100644 --- a/modules/serviceCenter/services/serviceGeneration/renderers/rendererXlsx.py +++ b/modules/serviceCenter/services/serviceGeneration/renderers/rendererXlsx.py @@ -68,17 +68,17 @@ class RendererXlsx(BaseRenderer): from modules.datamodels.datamodelJson import supportedSectionTypes return list(supportedSectionTypes) - async def render(self, extractedContent: Dict[str, Any], title: str, userPrompt: str = None, aiService=None) -> List[RenderedDocument]: + async def render(self, extractedContent: Dict[str, Any], title: str, userPrompt: str = None, aiService=None, *, style: Dict[str, Any] = None) -> List[RenderedDocument]: """Render extracted JSON content to Excel format using AI-analyzed styling.""" try: if not OPENPYXL_AVAILABLE: # Fallback to CSV if openpyxl not available from .rendererCsv import RendererCsv csvRenderer = RendererCsv() - return await csvRenderer.render(extractedContent, title, userPrompt, aiService) + return await csvRenderer.render(extractedContent, title, userPrompt, aiService, style=style) # Generate Excel using AI-analyzed styling - excelContent = await self._generateExcelFromJson(extractedContent, title, userPrompt, aiService) + excelContent = await self._generateExcelFromJson(extractedContent, title, userPrompt, aiService, style=style) # Extract metadata for document type and other info metadata = extractedContent.get("metadata", {}) if extractedContent else {} @@ -298,15 +298,22 @@ class RendererXlsx(BaseRenderer): except Exception as e: self.logger.warning(f"Could not populate analysis sheet: {str(e)}") - async def _generateExcelFromJson(self, jsonContent: Dict[str, Any], title: str, userPrompt: str = None, aiService=None) -> str: + async def _generateExcelFromJson(self, jsonContent: Dict[str, Any], title: str, userPrompt: str = None, aiService=None, *, style: Dict[str, Any] = None) -> str: """Generate Excel content from structured JSON document using AI-generated styling.""" try: # Debug output self.services.utils.debugLogToFile(f"EXCEL JSON CONTENT TYPE: {type(jsonContent)}", "EXCEL_RENDERER") self.services.utils.debugLogToFile(f"EXCEL JSON CONTENT KEYS: {list(jsonContent.keys()) if isinstance(jsonContent, dict) else 'Not a dict'}", "EXCEL_RENDERER") - # Get style set: use styles from metadata if available, otherwise enhance with AI - styles = await self._getStyleSet(jsonContent, userPrompt, aiService) + # Store unified style for use by inline-run helpers + self._unifiedStyle = style + + # Get style set: prefer unified style, fall back to legacy approach + if style: + styles = self._convertUnifiedStyleToInternal(style) + styles = self._convertColorsFormat(styles) + else: + styles = await self._getStyleSet(jsonContent, userPrompt, aiService) # Validate JSON structure (standardized schema: {metadata: {...}, documents: [{sections: [...]}]}) if not self._validateJsonStructure(jsonContent): @@ -511,6 +518,10 @@ class RendererXlsx(BaseRenderer): "code_block": {"font": "Courier New", "font_size": 10, "color": "FF2F2F2F", "background": "FFF5F5F5"} } + def _renderInlineRuns(self, runs: list) -> str: + """Flatten inline runs to plain text for Excel cells.""" + return "".join(r.get("value", "") for r in runs) + async def _getAiStylesWithExcelColors(self, aiService, styleTemplate: str, defaultStyles: Dict[str, Any]) -> Dict[str, Any]: """Get AI styles with proper Excel color conversion.""" if not aiService: @@ -1206,7 +1217,9 @@ class RendererXlsx(BaseRenderer): # Add headers with formatting - OPTIMIZED: use cached style objects for col, header in enumerate(headers, 1): - sanitized_header = self._sanitizeCellValue(header) + runs = self._inlineRunsForCell(header) + headerText = self._renderInlineRuns(runs) + sanitized_header = self._sanitizeCellValue(headerText) cell = sheet.cell(row=headerRow, column=col, value=sanitized_header) # Apply styling with fallbacks - use pre-calculated objects @@ -1272,7 +1285,9 @@ class RendererXlsx(BaseRenderer): cell_values = cell_values[:header_count] for col, cell_value in enumerate(cell_values, 1): - sanitized_value = self._sanitizeCellValue(cell_value) + runs = self._inlineRunsForCell(cell_value) + cellText = self._renderInlineRuns(runs) + sanitized_value = self._sanitizeCellValue(cellText) cell = sheet.cell(row=startRow, column=col, value=sanitized_value) # Apply styling with fallbacks - use pre-calculated objects @@ -1311,20 +1326,20 @@ class RendererXlsx(BaseRenderer): def _addListToExcel(self, sheet, element: Dict[str, Any], styles: Dict[str, Any], startRow: int) -> int: """Add a list element to Excel sheet. Expects nested content structure.""" try: - # Extract from nested content structure content = element.get("content", {}) if not isinstance(content, dict): return startRow - list_items = content.get("items") or [] - # Ensure list_items is a list - if not isinstance(list_items, list): - list_items = [] + listItems = content.get("items") or [] + if not isinstance(listItems, list): + listItems = [] - list_style = styles.get("bullet_list", {}) - for item in list_items: - sheet.cell(row=startRow, column=1, value=f"• {item}") - if list_style.get("color"): - sheet.cell(row=startRow, column=1).font = Font(color=self._getSafeColor(list_style["color"])) + listStyle = styles.get("bullet_list", {}) + for item in listItems: + runs = self._inlineRunsForListItem(item) + text = self._renderInlineRuns(runs) + sheet.cell(row=startRow, column=1, value=f"\u2022 {text}") + if listStyle.get("color"): + sheet.cell(row=startRow, column=1).font = Font(color=self._getSafeColor(listStyle["color"])) startRow += 1 return startRow @@ -1336,10 +1351,10 @@ class RendererXlsx(BaseRenderer): def _addParagraphToExcel(self, sheet, element: Dict[str, Any], styles: Dict[str, Any], startRow: int) -> int: """Add a paragraph element to Excel sheet. Expects nested content structure.""" try: - # Extract from nested content structure content = element.get("content", {}) if isinstance(content, dict): - text = content.get("text", "") + runs = self._inlineRunsFromContent(content) + text = self._renderInlineRuns(runs) elif isinstance(content, str): text = content else: diff --git a/modules/serviceCenter/services/serviceGeneration/styleDefaults.py b/modules/serviceCenter/services/serviceGeneration/styleDefaults.py new file mode 100644 index 00000000..b5a92641 --- /dev/null +++ b/modules/serviceCenter/services/serviceGeneration/styleDefaults.py @@ -0,0 +1,75 @@ +# Copyright (c) 2025 Patrick Motsch +# All rights reserved. +"""Default style definitions and style resolution for document rendering.""" + +from typing import Any, Dict + + +DEFAULT_STYLE: Dict[str, Any] = { + "fonts": { + "primary": "Calibri", + "monospace": "Consolas", + }, + "colors": { + "primary": "#1F3864", + "secondary": "#2C3E50", + "accent": "#2980B9", + "background": "#FFFFFF", + }, + "headings": { + "h1": {"sizePt": 24, "weight": "bold", "color": "#1F3864", "spaceBeforePt": 12, "spaceAfterPt": 6}, + "h2": {"sizePt": 18, "weight": "bold", "color": "#1F3864", "spaceBeforePt": 10, "spaceAfterPt": 4}, + "h3": {"sizePt": 14, "weight": "bold", "color": "#2C3E50", "spaceBeforePt": 8, "spaceAfterPt": 3}, + "h4": {"sizePt": 12, "weight": "bold", "color": "#2C3E50", "spaceBeforePt": 6, "spaceAfterPt": 2}, + }, + "paragraph": {"sizePt": 11, "lineSpacing": 1.15, "color": "#333333"}, + "table": { + "headerBg": "#1F3864", + "headerFg": "#FFFFFF", + "headerSizePt": 10, + "bodySizePt": 10, + "rowBandingEven": "#F2F6FC", + "rowBandingOdd": "#FFFFFF", + "borderColor": "#CBD5E1", + "borderWidthPt": 0.5, + }, + "list": {"bulletChar": "\u2022", "indentPt": 18, "sizePt": 11}, + "image": {"defaultWidthPt": 480, "maxWidthPt": 800, "alignment": "center"}, + "codeBlock": {"fontSizePt": 9, "background": "#F8F9FA", "borderColor": "#E2E8F0"}, + "page": { + "format": "A4", + "marginsPt": {"top": 60, "bottom": 60, "left": 60, "right": 60}, + "showPageNumbers": True, + "headerHeight": 30, + "footerHeight": 30, + "headerLogo": None, + "headerText": "", + "footerText": "", + }, +} + + +def _deepMerge(base: Dict[str, Any], override: Dict[str, Any]) -> Dict[str, Any]: + """Recursively merge override into base. Both dicts left unchanged; returns new dict.""" + result = {} + for key in base: + if key in override: + baseVal = base[key] + overVal = override[key] + if isinstance(baseVal, dict) and isinstance(overVal, dict): + result[key] = _deepMerge(baseVal, overVal) + else: + result[key] = overVal + else: + result[key] = base[key] + for key in override: + if key not in base: + result[key] = override[key] + return result + + +def resolveStyle(agentStyle: dict | None) -> Dict[str, Any]: + """Deep-merge DEFAULT_STYLE <- agentStyle. Returns fully resolved style dict.""" + if not agentStyle: + return dict(DEFAULT_STYLE) + return _deepMerge(DEFAULT_STYLE, agentStyle) diff --git a/modules/serviceCenter/services/serviceGeneration/subDocumentUtility.py b/modules/serviceCenter/services/serviceGeneration/subDocumentUtility.py index 8a3e7cea..594fbe02 100644 --- a/modules/serviceCenter/services/serviceGeneration/subDocumentUtility.py +++ b/modules/serviceCenter/services/serviceGeneration/subDocumentUtility.py @@ -9,11 +9,70 @@ from typing import Any, Dict logger = logging.getLogger(__name__) +def _parseInlineRuns(text: str) -> list: + """ + Parse inline markdown formatting into a list of InlineRun dicts. + Handles: images, links, bold, italic, inline code, plain text. + Uses a regex-based tokenizer that processes tokens left-to-right. + """ + if not text: + return [{"type": "text", "value": ""}] + + # Pattern order matters: images before links, bold before italic + _TOKEN_RE = re.compile( + r'!\[(?P[^`]+)`' # inline code
+ r'|\*\*(?P.+?)\*\*' # bold
+ r'|(?.+?)\*(?!\w)' # italic *x*
+ r'|(?.+?)_(?!\w)' # italic _x_
+ )
+
+ runs = []
+ lastEnd = 0
+
+ for m in _TOKEN_RE.finditer(text):
+ # Plain text before this match
+ if m.start() > lastEnd:
+ runs.append({"type": "text", "value": text[lastEnd:m.start()]})
+
+ if m.group("imgAlt") is not None or m.group("imgSrc") is not None:
+ alt = (m.group("imgAlt") or "").strip() or "Image"
+ src = (m.group("imgSrc") or "").strip()
+ widthStr = m.group("imgWidth")
+ run = {"type": "image", "value": alt}
+ if src.startswith("file:"):
+ run["fileId"] = src[5:]
+ else:
+ run["href"] = src
+ if widthStr:
+ run["widthPt"] = int(widthStr)
+ runs.append(run)
+ elif m.group("linkText") is not None:
+ runs.append({"type": "link", "value": m.group("linkText"), "href": m.group("linkHref")})
+ elif m.group("code") is not None:
+ runs.append({"type": "code", "value": m.group("code")})
+ elif m.group("bold") is not None:
+ runs.append({"type": "bold", "value": m.group("bold")})
+ elif m.group("italic1") is not None:
+ runs.append({"type": "italic", "value": m.group("italic1")})
+ elif m.group("italic2") is not None:
+ runs.append({"type": "italic", "value": m.group("italic2")})
+
+ lastEnd = m.end()
+
+ # Trailing plain text
+ if lastEnd < len(text):
+ runs.append({"type": "text", "value": text[lastEnd:]})
+
+ return runs if runs else [{"type": "text", "value": text}]
+
+
def markdownToDocumentJson(markdown: str, title: str, language: str = "de") -> Dict[str, Any]:
"""
- Convert markdown content to the standard document JSON format expected by renderReport.
- Supports headings, code blocks, tables, lists, images (file: refs), paragraphs.
- For plain text: wraps entire content in a single paragraph section.
+ Convert markdown content to the standard document JSON format with Inline-Run model.
+ Sections use inlineRuns (list of run dicts) instead of plain text strings.
+ Supports headings, code blocks, tables, lists, images, paragraphs.
"""
if not isinstance(markdown, str):
markdown = str(markdown) if markdown else ""
@@ -31,7 +90,7 @@ def markdownToDocumentJson(markdown: str, title: str, language: str = "de") -> D
while i < len(lines):
line = lines[i]
- # Headings
+ # Headings (plain text, no inline formatting)
headingMatch = re.match(r"^(#{1,6})\s+(.+)", line)
if headingMatch:
level = len(headingMatch.group(1))
@@ -43,7 +102,7 @@ def markdownToDocumentJson(markdown: str, title: str, language: str = "de") -> D
i += 1
continue
- # Fenced code blocks
+ # Fenced code blocks (no inline formatting)
codeMatch = re.match(r"^```(\w*)", line)
if codeMatch:
lang = codeMatch.group(1) or "text"
@@ -59,14 +118,14 @@ def markdownToDocumentJson(markdown: str, title: str, language: str = "de") -> D
})
continue
- # Tables
+ # Tables - cells are List[InlineRun]
tableMatch = re.match(r"^\|(.+)\|$", line)
if tableMatch and (i + 1) < len(lines) and re.match(r"^\|[\s\-:|]+\|$", lines[i + 1]):
- headerCells = [c.strip() for c in tableMatch.group(1).split("|")]
+ headerCells = [_parseInlineRuns(c.strip()) for c in tableMatch.group(1).split("|")]
i += 2
rows = []
while i < len(lines) and re.match(r"^\|(.+)\|$", lines[i]):
- rowCells = [c.strip() for c in lines[i][1:-1].split("|")]
+ rowCells = [_parseInlineRuns(c.strip()) for c in lines[i][1:-1].split("|")]
rows.append(rowCells)
i += 1
sections.append({
@@ -75,14 +134,14 @@ def markdownToDocumentJson(markdown: str, title: str, language: str = "de") -> D
})
continue
- # Bullet / numbered lists
+ # Bullet / numbered lists - items are List[List[InlineRun]]
listMatch = re.match(r"^(\s*)([-*+]|\d+[.)]) (.+)", line)
if listMatch:
isNumbered = bool(re.match(r"\d+[.)]", listMatch.group(2)))
items = []
while i < len(lines) and re.match(r"^(\s*)([-*+]|\d+[.)]) (.+)", lines[i]):
m = re.match(r"^(\s*)([-*+]|\d+[.)]) (.+)", lines[i])
- items.append({"text": m.group(3).strip()})
+ items.append(_parseInlineRuns(m.group(3).strip()))
i += 1
sections.append({
"id": _nextId(), "content_type": "bullet_list", "order": order,
@@ -95,46 +154,50 @@ def markdownToDocumentJson(markdown: str, title: str, language: str = "de") -> D
i += 1
continue
- # Images (simplified: store as paragraph with ref for now - full resolution needs Knowledge Store)
- imgMatch = re.match(r"^!\[([^\]]*)\]\(([^)]+)\)", line)
+ # Standalone image on its own line -> block-level image section
+ imgMatch = re.match(r"^!\[([^\]]*)\]\(([^)\"]+)(?:\s+\"(\d+)pt\")?\)\s*$", line)
if imgMatch:
altText = imgMatch.group(1).strip() or "Image"
src = imgMatch.group(2).strip()
+ widthStr = imgMatch.group(3)
fileId = src[5:] if src.startswith("file:") else ""
+ content = {
+ "altText": altText,
+ "base64Data": "",
+ "_fileRef": fileId,
+ "_srcUrl": src if not fileId else "",
+ }
+ if widthStr:
+ content["widthPt"] = int(widthStr)
sections.append({
"id": _nextId(), "content_type": "image", "order": order,
- "elements": [{
- "content": {
- "altText": altText,
- "base64Data": "",
- "_fileRef": fileId,
- "_srcUrl": src if not fileId else "",
- }
- }],
+ "elements": [{"content": content}],
})
i += 1
continue
- # Paragraph
+ # Paragraph - produces inlineRuns
paraLines = []
while i < len(lines) and lines[i].strip() and not re.match(
- r"^(#{1,6}\s|```|\|.+\||!\[|(\s*)([-*+]|\d+[.)]) )", lines[i]
+ r"^(#{1,6}\s|```|\|.+\||!\[[^\]]*\]\([^)]+\)\s*$|(\s*)([-*+]|\d+[.)]) )", lines[i]
):
paraLines.append(lines[i])
i += 1
if paraLines:
+ combinedText = " ".join(paraLines)
sections.append({
"id": _nextId(), "content_type": "paragraph", "order": order,
- "elements": [{"content": {"text": " ".join(paraLines)}}],
+ "elements": [{"content": {"inlineRuns": _parseInlineRuns(combinedText)}}],
})
continue
i += 1
if not sections:
+ fallbackText = markdown.strip() or "(empty)"
sections.append({
"id": _nextId(), "content_type": "paragraph", "order": order,
- "elements": [{"content": {"text": markdown.strip() or "(empty)"}}],
+ "elements": [{"content": {"inlineRuns": _parseInlineRuns(fallbackText)}}],
})
return {
diff --git a/modules/serviceCenter/services/serviceKnowledge/mainServiceKnowledge.py b/modules/serviceCenter/services/serviceKnowledge/mainServiceKnowledge.py
index dab8cc25..6698e164 100644
--- a/modules/serviceCenter/services/serviceKnowledge/mainServiceKnowledge.py
+++ b/modules/serviceCenter/services/serviceKnowledge/mainServiceKnowledge.py
@@ -2,9 +2,13 @@
# All rights reserved.
"""Knowledge service: 3-tier RAG with indexing, semantic search, and context building."""
+import hashlib
+import json
import logging
import re
-from typing import Any, Callable, Dict, List, Optional
+import time
+from dataclasses import dataclass, field
+from typing import Any, Callable, Dict, List, Optional, Union
from modules.datamodels.datamodelKnowledge import (
FileContentIndex, ContentChunk, WorkflowMemory,
@@ -20,6 +24,68 @@ DEFAULT_CHUNK_TOKENS = 400
DEFAULT_CONTEXT_BUDGET = 12000
+# =============================================================================
+# Ingestion façade (P0 of unified-knowledge-indexing concept)
+# =============================================================================
+
+@dataclass
+class IngestionJob:
+ """One request to add or refresh content in the unified knowledge store.
+
+ Callers from any lane (routes, feature hooks, agent tools, connector sync)
+ describe the work they want done via this object; idempotency, scope
+ resolution, and embedding are handled by KnowledgeService.requestIngestion.
+ """
+ sourceKind: str
+ sourceId: str
+ fileName: str
+ mimeType: str
+ userId: str
+ contentObjects: List[Dict[str, Any]] = field(default_factory=list)
+ featureInstanceId: str = ""
+ mandateId: str = ""
+ structure: Optional[Dict[str, Any]] = None
+ containerPath: Optional[str] = None
+ contentVersion: Optional[str] = None
+ provenance: Optional[Dict[str, Any]] = None
+ # Connector-driven neutralization: True when the user opted in via §2.6 preferences.
+ # For sourceKind == "file", _indexFileInternal resolves this from FileItem.neutralize instead.
+ neutralize: bool = False
+
+
+@dataclass
+class IngestionHandle:
+ """Result of requestIngestion. Stable across in-process and future queue impls."""
+ jobId: str
+ status: str
+ contentHash: str
+ fileId: str
+ index: Optional[FileContentIndex] = None
+ error: Optional[str] = None
+
+
+def _computeIngestionHash(contentObjects: List[Dict[str, Any]]) -> str:
+ """Deterministic SHA256 over (contentType, data) tuples in extractor order.
+
+ `contentObjectId` is intentionally excluded because extractors generate
+ fresh UUIDs per run (`uuid.uuid4()`), which would make the hash unstable
+ across re-extractions of the same source — defeating idempotency.
+ Order is preserved (no sort) because two different documents can share the
+ same multiset of parts but differ in arrangement (e.g. swapped pages).
+ Text whitespace is preserved intentionally because chunk boundaries
+ depend on it.
+ """
+ normalized = [
+ (
+ str(o.get("contentType", "text") or "text"),
+ o.get("data", "") or "",
+ )
+ for o in (contentObjects or [])
+ ]
+ payload = json.dumps(normalized, ensure_ascii=False, separators=(",", ":"))
+ return hashlib.sha256(payload.encode("utf-8")).hexdigest()
+
+
class KnowledgeService:
"""Service for Knowledge Store operations: indexing, retrieval, and context building."""
@@ -46,6 +112,224 @@ class KnowledgeService:
results = await self._embed([text])
return results[0] if results else []
+ # =========================================================================
+ # Ingestion façade (single entry point for all lanes)
+ # =========================================================================
+
+ async def requestIngestion(self, job: IngestionJob) -> IngestionHandle:
+ """Unified entry point for filling the knowledge corpus.
+
+ Applies idempotency based on a content hash (or caller-supplied
+ `contentVersion`) persisted in `FileContentIndex.structure._ingestion`.
+ Re-runs indexing only when the hash differs or the previous run did
+ not reach `indexed` state. Runs embedding synchronously for now
+ (callers already schedule background tasks where needed).
+ """
+ jobId = f"{job.sourceKind}:{job.sourceId}"
+ startMs = time.time()
+ contentHash = job.contentVersion or _computeIngestionHash(job.contentObjects)
+
+ # 1. Check for duplicate via existing FileContentIndex row.
+ existing = None
+ try:
+ existing = self._knowledgeDb.getFileContentIndex(job.sourceId)
+ except Exception:
+ existing = None
+
+ if existing:
+ existingStructure = (
+ existing.get("structure") if isinstance(existing, dict)
+ else getattr(existing, "structure", {})
+ ) or {}
+ existingMeta = existingStructure.get("_ingestion", {}) or {}
+ existingStatus = (
+ existing.get("status") if isinstance(existing, dict)
+ else getattr(existing, "status", "")
+ ) or ""
+ if existingMeta.get("hash") == contentHash and existingStatus == "indexed":
+ logger.info(
+ "ingestion.skipped.duplicate sourceKind=%s sourceId=%s hash=%s",
+ job.sourceKind, job.sourceId, contentHash[:12],
+ extra={
+ "event": "ingestion.skipped.duplicate",
+ "jobId": jobId,
+ "sourceKind": job.sourceKind,
+ "sourceId": job.sourceId,
+ "hash": contentHash,
+ "durationMs": int((time.time() - startMs) * 1000),
+ },
+ )
+ return IngestionHandle(
+ jobId=jobId,
+ status="duplicate",
+ contentHash=contentHash,
+ fileId=job.sourceId,
+ index=None,
+ )
+
+ # 2. Prepare ingestion metadata; stays in structure._ingestion so
+ # later connector revoke/purge can filter chunks by sourceKind /
+ # provenance.connectionId without a schema migration.
+ ingestionMeta = {
+ "hash": contentHash,
+ "sourceKind": job.sourceKind,
+ "sourceId": job.sourceId,
+ "contentVersion": job.contentVersion,
+ "indexedAt": getUtcTimestamp(),
+ "provenance": dict(job.provenance or {}),
+ }
+ structure = dict(job.structure or {})
+ structure["_ingestion"] = ingestionMeta
+
+ logger.info(
+ "ingestion.queued sourceKind=%s sourceId=%s objects=%d hash=%s",
+ job.sourceKind, job.sourceId, len(job.contentObjects or []), contentHash[:12],
+ extra={
+ "event": "ingestion.queued",
+ "jobId": jobId,
+ "sourceKind": job.sourceKind,
+ "sourceId": job.sourceId,
+ "hash": contentHash,
+ "objectCount": len(job.contentObjects or []),
+ },
+ )
+
+ # 3. Run real indexing.
+ try:
+ index = await self._indexFileInternal(
+ fileId=job.sourceId,
+ fileName=job.fileName,
+ mimeType=job.mimeType,
+ userId=job.userId,
+ featureInstanceId=job.featureInstanceId,
+ mandateId=job.mandateId,
+ contentObjects=job.contentObjects or [],
+ structure=structure,
+ containerPath=job.containerPath,
+ sourceKind=job.sourceKind,
+ connectionId=(job.provenance or {}).get("connectionId"),
+ neutralize=job.neutralize,
+ )
+ except Exception as exc:
+ logger.error(
+ "ingestion.failed sourceKind=%s sourceId=%s error=%s",
+ job.sourceKind, job.sourceId, exc,
+ exc_info=True,
+ extra={
+ "event": "ingestion.failed",
+ "jobId": jobId,
+ "sourceKind": job.sourceKind,
+ "sourceId": job.sourceId,
+ "hash": contentHash,
+ "error": str(exc),
+ "durationMs": int((time.time() - startMs) * 1000),
+ },
+ )
+ try:
+ self._knowledgeDb.updateFileStatus(job.sourceId, "failed")
+ except Exception:
+ pass
+ return IngestionHandle(
+ jobId=jobId,
+ status="failed",
+ contentHash=contentHash,
+ fileId=job.sourceId,
+ index=None,
+ error=str(exc),
+ )
+
+ logger.info(
+ "ingestion.indexed sourceKind=%s sourceId=%s objects=%d durationMs=%d",
+ job.sourceKind, job.sourceId, len(job.contentObjects or []),
+ int((time.time() - startMs) * 1000),
+ extra={
+ "event": "ingestion.indexed",
+ "jobId": jobId,
+ "sourceKind": job.sourceKind,
+ "sourceId": job.sourceId,
+ "hash": contentHash,
+ "objectCount": len(job.contentObjects or []),
+ "durationMs": int((time.time() - startMs) * 1000),
+ },
+ )
+ return IngestionHandle(
+ jobId=jobId,
+ status="indexed",
+ contentHash=contentHash,
+ fileId=job.sourceId,
+ index=index,
+ )
+
+ def purgeConnection(self, connectionId: str) -> Dict[str, int]:
+ """Delete every FileContentIndex + ContentChunk linked to a UserConnection.
+
+ Called on `connection.revoked` events so the knowledge corpus never
+ holds chunks the user has withdrawn access to. Returns deletion counts
+ for observability.
+ """
+ if not connectionId:
+ return {"indexRows": 0, "chunks": 0}
+ startMs = time.time()
+ result = self._knowledgeDb.deleteFileContentIndexByConnectionId(connectionId)
+ logger.info(
+ "ingestion.connection.purged connectionId=%s rows=%d chunks=%d durationMs=%d",
+ connectionId, result["indexRows"], result["chunks"],
+ int((time.time() - startMs) * 1000),
+ extra={
+ "event": "ingestion.connection.purged",
+ "connectionId": connectionId,
+ "indexRows": result["indexRows"],
+ "chunks": result["chunks"],
+ "durationMs": int((time.time() - startMs) * 1000),
+ },
+ )
+ return result
+
+ def getIngestionStatus(
+ self, handleOrJobId: Union[IngestionHandle, str]
+ ) -> Dict[str, Any]:
+ """Map a handle or `sourceKind:sourceId` jobId to a status snapshot."""
+ if isinstance(handleOrJobId, IngestionHandle):
+ sourceId = handleOrJobId.fileId
+ jobId = handleOrJobId.jobId
+ elif isinstance(handleOrJobId, str) and ":" in handleOrJobId:
+ jobId = handleOrJobId
+ sourceId = handleOrJobId.split(":", 1)[1]
+ else:
+ jobId = str(handleOrJobId)
+ sourceId = str(handleOrJobId)
+
+ row = None
+ try:
+ row = self._knowledgeDb.getFileContentIndex(sourceId)
+ except Exception:
+ row = None
+ if not row:
+ return {
+ "jobId": jobId,
+ "sourceId": sourceId,
+ "status": "unknown",
+ "contentHash": None,
+ }
+
+ structure = (
+ row.get("structure") if isinstance(row, dict)
+ else getattr(row, "structure", {})
+ ) or {}
+ meta = structure.get("_ingestion", {}) or {}
+ status = (
+ row.get("status") if isinstance(row, dict)
+ else getattr(row, "status", "")
+ ) or "unknown"
+ return {
+ "jobId": jobId,
+ "sourceId": sourceId,
+ "status": status,
+ "contentHash": meta.get("hash"),
+ "sourceKind": meta.get("sourceKind"),
+ "indexedAt": meta.get("indexedAt"),
+ }
+
# =========================================================================
# File Indexing (called after extraction, before embedding)
# =========================================================================
@@ -61,6 +345,57 @@ class KnowledgeService:
contentObjects: List[Dict[str, Any]] = None,
structure: Dict[str, Any] = None,
containerPath: str = None,
+ ) -> Optional[FileContentIndex]:
+ """Backward-compatible wrapper delegating to requestIngestion.
+
+ Existing callers that still invoke `indexFile` directly automatically
+ participate in the idempotency/metrics layer. New callers should
+ prefer `requestIngestion` so they can pass `sourceKind` and
+ `provenance` for connector revoke/purge later.
+ """
+ job = IngestionJob(
+ sourceKind="file",
+ sourceId=fileId,
+ fileName=fileName,
+ mimeType=mimeType,
+ userId=userId,
+ featureInstanceId=featureInstanceId,
+ mandateId=mandateId,
+ contentObjects=list(contentObjects or []),
+ structure=structure,
+ containerPath=containerPath,
+ )
+ handle = await self.requestIngestion(job)
+ if handle.index is not None:
+ return handle.index
+ if handle.status == "duplicate":
+ row = None
+ try:
+ row = self._knowledgeDb.getFileContentIndex(fileId)
+ except Exception:
+ row = None
+ if isinstance(row, dict):
+ try:
+ return FileContentIndex(**row)
+ except Exception:
+ return None
+ return row
+ return None
+
+ async def _indexFileInternal(
+ self,
+ fileId: str,
+ fileName: str,
+ mimeType: str,
+ userId: str,
+ featureInstanceId: str = "",
+ mandateId: str = "",
+ contentObjects: List[Dict[str, Any]] = None,
+ structure: Dict[str, Any] = None,
+ containerPath: str = None,
+ sourceKind: str = "file",
+ connectionId: Optional[str] = None,
+ neutralize: bool = False,
) -> FileContentIndex:
"""Index a file's content objects and create embeddings for text chunks.
@@ -83,39 +418,41 @@ class KnowledgeService:
"""
contentObjects = contentObjects or []
- # 1. Resolve scope fields from FileItem (Single Source of Truth)
- # FileItem lives in poweron_management; its scope/mandateId/featureInstanceId
- # are authoritative and must be mirrored onto the FileContentIndex.
+ # 1. Resolve scope fields from FileItem (Single Source of Truth) for
+ # uploaded files. Connector-sourced ingestion (sharepoint_item,
+ # outlook_message, ...) has no FileItem row — trust the caller's
+ # scope + ids directly.
resolvedScope = "personal"
resolvedMandateId = mandateId
resolvedFeatureInstanceId = featureInstanceId
resolvedUserId = userId
- _shouldNeutralize = False
- try:
- from modules.datamodels.datamodelFiles import FileItem as _FileItem
- _dbComponent = getattr(self._context, "interfaceDbComponent", None)
- _fileRecords = _dbComponent.getRecordset(_FileItem, recordFilter={"id": fileId}) if _dbComponent else []
- if not _fileRecords:
- from modules.interfaces.interfaceDbManagement import ComponentObjects
- _row = ComponentObjects().db._loadRecord(_FileItem, fileId)
- if _row:
- _fileRecords = [_row]
- if _fileRecords:
- _fileRecord = _fileRecords[0]
- _get = (lambda k, d=None: _fileRecord.get(k, d)) if isinstance(_fileRecord, dict) else (lambda k, d=None: getattr(_fileRecord, k, d))
- _shouldNeutralize = bool(_get("neutralize", False))
- _fileScope = _get("scope")
- if _fileScope:
- resolvedScope = _fileScope
- if not resolvedMandateId:
- resolvedMandateId = str(_get("mandateId", "") or "")
- if not resolvedFeatureInstanceId:
- resolvedFeatureInstanceId = str(_get("featureInstanceId", "") or "")
- _fileCreatedBy = _get("sysCreatedBy")
- if _fileCreatedBy:
- resolvedUserId = str(_fileCreatedBy)
- except Exception:
- pass
+ _shouldNeutralize = neutralize # caller-supplied flag (connector prefs / IngestionJob)
+ if sourceKind == "file":
+ try:
+ from modules.datamodels.datamodelFiles import FileItem as _FileItem
+ _dbComponent = getattr(self._context, "interfaceDbComponent", None)
+ _fileRecords = _dbComponent.getRecordset(_FileItem, recordFilter={"id": fileId}) if _dbComponent else []
+ if not _fileRecords:
+ from modules.interfaces.interfaceDbManagement import ComponentObjects
+ _row = ComponentObjects().db._loadRecord(_FileItem, fileId)
+ if _row:
+ _fileRecords = [_row]
+ if _fileRecords:
+ _fileRecord = _fileRecords[0]
+ _get = (lambda k, d=None: _fileRecord.get(k, d)) if isinstance(_fileRecord, dict) else (lambda k, d=None: getattr(_fileRecord, k, d))
+ _shouldNeutralize = bool(_get("neutralize", False)) # FileItem is authoritative for uploads
+ _fileScope = _get("scope")
+ if _fileScope:
+ resolvedScope = _fileScope
+ if not resolvedMandateId:
+ resolvedMandateId = str(_get("mandateId", "") or "")
+ if not resolvedFeatureInstanceId:
+ resolvedFeatureInstanceId = str(_get("featureInstanceId", "") or "")
+ _fileCreatedBy = _get("sysCreatedBy")
+ if _fileCreatedBy:
+ resolvedUserId = str(_fileCreatedBy)
+ except Exception:
+ pass
# 2. Create FileContentIndex with correct scope from the start
index = FileContentIndex(
@@ -124,6 +461,8 @@ class KnowledgeService:
featureInstanceId=resolvedFeatureInstanceId,
mandateId=resolvedMandateId,
scope=resolvedScope,
+ sourceKind=sourceKind,
+ connectionId=connectionId,
fileName=fileName,
mimeType=mimeType,
containerPath=containerPath,
@@ -300,7 +639,12 @@ class KnowledgeService:
Formatted context string for injection into the agent's system prompt.
"""
queryVector = await self._embedSingle(currentPrompt)
+ logger.debug(
+ "buildAgentContext.start userId=%s featureInstanceId=%s mandateId=%s isSysAdmin=%s prompt=%r",
+ userId, featureInstanceId, mandateId, isSysAdmin, (currentPrompt or "")[:120],
+ )
if not queryVector:
+ logger.debug("buildAgentContext.abort reason=no_query_vector")
return ""
builder = _ContextBuilder(budget=contextBudget)
@@ -327,9 +671,14 @@ class KnowledgeService:
featureInstanceId=featureInstanceId,
mandateId=mandateId,
limit=15,
- minScore=0.65,
+ minScore=0.35,
isSysAdmin=isSysAdmin,
)
+ logger.debug(
+ "buildAgentContext.layer1 instanceChunks=%d top_scores=%s",
+ len(instanceChunks),
+ [round(float(c.get("_score", 0) or 0), 3) for c in (instanceChunks or [])[:3]],
+ )
if instanceChunks:
builder.add(priority=1, label="Relevant Documents", items=instanceChunks, maxChars=4000)
@@ -338,7 +687,7 @@ class KnowledgeService:
queryVector=queryVector,
workflowId=workflowId,
limit=10,
- minScore=0.55,
+ minScore=0.35,
)
if roundMemories:
memItems = []
@@ -376,7 +725,7 @@ class KnowledgeService:
scope="mandate",
mandateId=mandateId,
limit=10,
- minScore=0.7,
+ minScore=0.35,
isSysAdmin=isSysAdmin,
)
if mandateChunks:
@@ -392,7 +741,12 @@ class KnowledgeService:
maxChars=500,
)
- return builder.build()
+ _result = builder.build()
+ logger.debug(
+ "buildAgentContext.done totalChars=%d userId=%s",
+ len(_result), userId,
+ )
+ return _result
# =========================================================================
# Workflow Memory
diff --git a/modules/serviceCenter/services/serviceKnowledge/subConnectorIngestConsumer.py b/modules/serviceCenter/services/serviceKnowledge/subConnectorIngestConsumer.py
new file mode 100644
index 00000000..97ac61d5
--- /dev/null
+++ b/modules/serviceCenter/services/serviceKnowledge/subConnectorIngestConsumer.py
@@ -0,0 +1,334 @@
+# Copyright (c) 2025 Patrick Motsch
+# All rights reserved.
+"""Connection-lifecycle consumer bridging OAuth events to ingestion jobs.
+
+Subscribes to `connection.established` and `connection.revoked` callbacks
+emitted by the OAuth callbacks / connection management routes and dispatches:
+
+- `connection.established` -> enqueue a `connection.bootstrap` BackgroundJob
+ that walks the connector and ingests all reachable items via
+ KnowledgeService.requestIngestion (file-like or virtual documents).
+- `connection.revoked` -> run `KnowledgeService.purgeConnection` synchronously
+ so the knowledge corpus releases the data before the UI confirms the revoke.
+
+The consumer is registered once at process boot (see `app.py` lifespan).
+It intentionally does NOT hold a per-user service context; each callback
+creates whatever context it needs from the UserConnection row itself.
+"""
+
+from __future__ import annotations
+
+import asyncio
+import logging
+from typing import Any, Dict, Optional
+
+from modules.interfaces.interfaceDbKnowledge import getInterface as getKnowledgeInterface
+from modules.shared.callbackRegistry import callbackRegistry
+from modules.serviceCenter.services.serviceBackgroundJobs import (
+ registerJobHandler,
+ startJob,
+)
+
+logger = logging.getLogger(__name__)
+
+BOOTSTRAP_JOB_TYPE = "connection.bootstrap"
+
+_registered = False
+
+
+def _onConnectionEstablished(
+ *,
+ connectionId: str,
+ authority: str,
+ userId: Optional[str] = None,
+ **kwargs: Any,
+) -> None:
+ """Fire-and-forget bootstrap enqueue for a freshly connected UserConnection."""
+ if not connectionId:
+ logger.warning("connection.established without connectionId; ignoring")
+ return
+ payload: Dict[str, Any] = {
+ "connectionId": connectionId,
+ "authority": (authority or "").lower(),
+ "userId": userId,
+ }
+ logger.info(
+ "ingestion.connection.bootstrap.queued connectionId=%s authority=%s",
+ connectionId, authority,
+ extra={
+ "event": "ingestion.connection.bootstrap.queued",
+ "connectionId": connectionId,
+ "authority": authority,
+ },
+ )
+
+ async def _enqueue() -> None:
+ try:
+ await startJob(
+ BOOTSTRAP_JOB_TYPE,
+ payload,
+ triggeredBy=userId,
+ )
+ except Exception as exc:
+ logger.error(
+ "ingestion.connection.bootstrap.enqueue_failed connectionId=%s error=%s",
+ connectionId, exc, exc_info=True,
+ )
+
+ try:
+ loop = asyncio.get_event_loop()
+ if loop.is_running():
+ loop.create_task(_enqueue())
+ else:
+ loop.run_until_complete(_enqueue())
+ except RuntimeError:
+ asyncio.run(_enqueue())
+
+
+def _onConnectionRevoked(
+ *,
+ connectionId: str,
+ authority: Optional[str] = None,
+ userId: Optional[str] = None,
+ reason: Optional[str] = None,
+ **kwargs: Any,
+) -> None:
+ """Run the knowledge purge synchronously so UI feedback is authoritative."""
+ if not connectionId:
+ logger.warning("connection.revoked without connectionId; ignoring")
+ return
+ try:
+ # Purge lives on the DB interface to avoid ServiceCenter/user-context
+ # plumbing here; the service method is a thin wrapper on top of this.
+ result = getKnowledgeInterface(None).deleteFileContentIndexByConnectionId(connectionId)
+ except Exception as exc:
+ logger.error(
+ "ingestion.connection.purged.failed connectionId=%s error=%s",
+ connectionId, exc, exc_info=True,
+ )
+ return
+ logger.info(
+ "ingestion.connection.purged connectionId=%s authority=%s reason=%s rows=%d chunks=%d",
+ connectionId, authority, reason,
+ result.get("indexRows", 0), result.get("chunks", 0),
+ extra={
+ "event": "ingestion.connection.purged",
+ "connectionId": connectionId,
+ "authority": authority,
+ "reason": reason,
+ "indexRows": result.get("indexRows", 0),
+ "chunks": result.get("chunks", 0),
+ },
+ )
+
+
+async def _bootstrapJobHandler(
+ job: Dict[str, Any],
+ progressCb,
+) -> Dict[str, Any]:
+ """Dispatch bootstrap by authority. Each authority runs its own sub-bootstraps."""
+ payload = job.get("payload") or {}
+ connectionId = payload.get("connectionId")
+ authority = (payload.get("authority") or "").lower()
+ if not connectionId:
+ raise ValueError("connection.bootstrap requires payload.connectionId")
+
+ progressCb(5, f"resolving {authority} connection")
+
+ # Defensive consent check: if the connection has since disabled knowledge ingestion
+ # (e.g. user toggled setting after the job was enqueued), skip all walkers.
+ try:
+ from modules.interfaces.interfaceDbApp import getRootInterface
+ _root = getRootInterface()
+ _conn = _root.getUserConnectionById(connectionId)
+ if _conn and not getattr(_conn, "knowledgeIngestionEnabled", True):
+ logger.info(
+ "ingestion.connection.bootstrap.skipped — consent disabled connectionId=%s",
+ connectionId,
+ extra={
+ "event": "ingestion.connection.bootstrap.skipped",
+ "connectionId": connectionId,
+ "authority": authority,
+ "reason": "consent_disabled",
+ },
+ )
+ return {"connectionId": connectionId, "authority": authority, "skipped": True, "reason": "consent_disabled"}
+ except Exception as _guardErr:
+ logger.debug("Could not load connection for consent guard: %s", _guardErr)
+
+ def _normalize(res: Any, label: str) -> Dict[str, Any]:
+ if isinstance(res, Exception):
+ logger.error(
+ "ingestion.connection.bootstrap.failed part=%s connectionId=%s error=%s",
+ label, connectionId, res, exc_info=res,
+ )
+ return {"error": str(res)}
+ return res or {}
+
+ if authority == "msft":
+ from modules.serviceCenter.services.serviceKnowledge.subConnectorSyncSharepoint import (
+ bootstrapSharepoint,
+ )
+ from modules.serviceCenter.services.serviceKnowledge.subConnectorSyncOutlook import (
+ bootstrapOutlook,
+ )
+
+ progressCb(10, "sharepoint + outlook")
+ spResult, olResult = await asyncio.gather(
+ bootstrapSharepoint(connectionId=connectionId, progressCb=progressCb),
+ bootstrapOutlook(connectionId=connectionId, progressCb=progressCb),
+ return_exceptions=True,
+ )
+ return {
+ "connectionId": connectionId,
+ "authority": authority,
+ "sharepoint": _normalize(spResult, "sharepoint"),
+ "outlook": _normalize(olResult, "outlook"),
+ }
+
+ if authority == "google":
+ from modules.serviceCenter.services.serviceKnowledge.subConnectorSyncGdrive import (
+ bootstrapGdrive,
+ )
+ from modules.serviceCenter.services.serviceKnowledge.subConnectorSyncGmail import (
+ bootstrapGmail,
+ )
+
+ progressCb(10, "drive + gmail")
+ gdResult, gmResult = await asyncio.gather(
+ bootstrapGdrive(connectionId=connectionId, progressCb=progressCb),
+ bootstrapGmail(connectionId=connectionId, progressCb=progressCb),
+ return_exceptions=True,
+ )
+ return {
+ "connectionId": connectionId,
+ "authority": authority,
+ "drive": _normalize(gdResult, "gdrive"),
+ "gmail": _normalize(gmResult, "gmail"),
+ }
+
+ if authority == "clickup":
+ from modules.serviceCenter.services.serviceKnowledge.subConnectorSyncClickup import (
+ bootstrapClickup,
+ )
+
+ progressCb(10, "clickup tasks")
+ cuResult = await bootstrapClickup(connectionId=connectionId, progressCb=progressCb)
+ return {
+ "connectionId": connectionId,
+ "authority": authority,
+ "clickup": _normalize(cuResult, "clickup"),
+ }
+
+ logger.info(
+ "ingestion.connection.bootstrap.skipped reason=unsupported_authority authority=%s connectionId=%s",
+ authority, connectionId,
+ extra={
+ "event": "ingestion.connection.bootstrap.skipped",
+ "authority": authority,
+ "connectionId": connectionId,
+ "reason": "unsupported_authority",
+ },
+ )
+ return {
+ "connectionId": connectionId,
+ "authority": authority,
+ "skipped": True,
+ "reason": "unsupported_authority",
+ }
+
+
+async def _scheduledDailyResync() -> None:
+ """Enqueue a connection.bootstrap job for every active knowledge connection.
+
+ Runs once per day (default 2 AM Europe/Zurich). Each job re-walks the
+ connector and hands new / changed items to KnowledgeService.requestIngestion.
+ Unchanged items are deduplicated by content-hash and skipped automatically.
+ """
+ try:
+ from modules.interfaces.interfaceDbApp import getRootInterface
+ rootInterface = getRootInterface()
+ connections = rootInterface.getActiveKnowledgeConnections()
+ except Exception as exc:
+ logger.error("knowledge.daily_resync: could not load connections: %s", exc, exc_info=True)
+ return
+
+ if not connections:
+ logger.info("knowledge.daily_resync: no active knowledge connections — nothing to do")
+ return
+
+ logger.info(
+ "knowledge.daily_resync: enqueuing bootstrap for %d connection(s)",
+ len(connections),
+ extra={"event": "knowledge.daily_resync.started", "count": len(connections)},
+ )
+
+ enqueued = 0
+ skipped = 0
+ for conn in connections:
+ connectionId = str(conn.id)
+ authority = conn.authority.value if hasattr(conn.authority, "value") else str(conn.authority)
+ userId = str(conn.userId)
+ payload: Dict[str, Any] = {
+ "connectionId": connectionId,
+ "authority": authority.lower(),
+ "userId": userId,
+ }
+ try:
+ await startJob(
+ BOOTSTRAP_JOB_TYPE,
+ payload,
+ triggeredBy="scheduler.daily_resync",
+ )
+ enqueued += 1
+ logger.debug(
+ "knowledge.daily_resync: queued connectionId=%s authority=%s",
+ connectionId, authority,
+ )
+ except Exception as exc:
+ skipped += 1
+ logger.error(
+ "knowledge.daily_resync: failed to enqueue connectionId=%s: %s",
+ connectionId, exc,
+ )
+
+ logger.info(
+ "knowledge.daily_resync: done — enqueued=%d skipped=%d",
+ enqueued, skipped,
+ extra={"event": "knowledge.daily_resync.done", "enqueued": enqueued, "skipped": skipped},
+ )
+
+
+def registerDailyResyncScheduler(*, hour: int = 2, minute: int = 0) -> None:
+ """Register the daily knowledge re-sync cron job. Idempotent.
+
+ Args:
+ hour: Hour of day to run (0–23, default 2 → 2 AM Europe/Zurich).
+ minute: Minute within the hour (default 0).
+ """
+ try:
+ from modules.shared.eventManagement import eventManager
+ eventManager.registerCron(
+ jobId="knowledge.daily_resync",
+ func=_scheduledDailyResync,
+ cronKwargs={"hour": str(hour), "minute": str(minute)},
+ )
+ logger.info(
+ "knowledge.daily_resync scheduler registered (daily %02d:%02d Europe/Zurich)",
+ hour, minute,
+ )
+ except Exception as exc:
+ logger.warning("knowledge.daily_resync scheduler registration failed (non-critical): %s", exc)
+
+
+def registerKnowledgeIngestionConsumer() -> None:
+ """Register callback subscribers + background job handler. Idempotent."""
+ global _registered
+ if _registered:
+ return
+ callbackRegistry.register("connection.established", _onConnectionEstablished)
+ callbackRegistry.register("connection.revoked", _onConnectionRevoked)
+ registerJobHandler(BOOTSTRAP_JOB_TYPE, _bootstrapJobHandler)
+ registerDailyResyncScheduler()
+ _registered = True
+ logger.info("KnowledgeIngestionConsumer registered (established/revoked + %s handler + daily resync)", BOOTSTRAP_JOB_TYPE)
diff --git a/modules/serviceCenter/services/serviceKnowledge/subConnectorPrefs.py b/modules/serviceCenter/services/serviceKnowledge/subConnectorPrefs.py
new file mode 100644
index 00000000..950400ce
--- /dev/null
+++ b/modules/serviceCenter/services/serviceKnowledge/subConnectorPrefs.py
@@ -0,0 +1,101 @@
+"""Per-connection knowledge ingestion preference helpers.
+
+Walkers call `loadConnectionPrefs(connectionId)` once at bootstrap start and
+receive a `ConnectionIngestionPrefs` dataclass they can pass down into their
+inner loops. All fields have safe defaults so walkers stay backward-compatible
+with connections that predate the §2.6 preference schema (knowledgePreferences
+is None).
+"""
+from __future__ import annotations
+
+import logging
+from dataclasses import dataclass, field
+from typing import Any, Dict, List, Optional
+
+logger = logging.getLogger(__name__)
+
+_DEFAULT_MAX_AGE_DAYS = 90
+_DEFAULT_MAIL_DEPTH = "full"
+_DEFAULT_CLICKUP_SCOPE = "title_description"
+
+
+@dataclass
+class ConnectionIngestionPrefs:
+ """Parsed per-connection preferences for knowledge ingestion walkers."""
+
+ # PII
+ neutralizeBeforeEmbed: bool = False
+
+ # Mail (Outlook + Gmail)
+ mailContentDepth: str = _DEFAULT_MAIL_DEPTH # "metadata" | "snippet" | "full"
+ mailIndexAttachments: bool = False
+
+ # Files (Drive / SharePoint / OneDrive)
+ filesIndexBinaries: bool = True
+ mimeAllowlist: List[str] = field(default_factory=list) # empty = all allowed
+
+ # ClickUp
+ clickupScope: str = _DEFAULT_CLICKUP_SCOPE # "titles" | "title_description" | "with_comments"
+ clickupIndexAttachments: bool = False
+
+ # Per-authority surface toggles (default everything on)
+ gmailEnabled: bool = True
+ driveEnabled: bool = True
+ sharepointEnabled: bool = True
+ outlookEnabled: bool = True
+
+ # Time window
+ maxAgeDays: int = _DEFAULT_MAX_AGE_DAYS # 0 = no limit
+
+
+def loadConnectionPrefs(connectionId: str) -> ConnectionIngestionPrefs:
+ """Load and parse per-connection preferences from the database.
+
+ Returns safe defaults for any missing or unparseable values so walkers
+ never fail due to missing preference data.
+ """
+ try:
+ from modules.interfaces.interfaceDbApp import getRootInterface
+ root = getRootInterface()
+ conn = root.getUserConnectionById(connectionId)
+ if not conn:
+ logger.debug("loadConnectionPrefs: connection %s not found, using defaults", connectionId)
+ return ConnectionIngestionPrefs()
+
+ raw: Optional[Dict[str, Any]] = getattr(conn, "knowledgePreferences", None)
+ if not raw or not isinstance(raw, dict):
+ return ConnectionIngestionPrefs()
+
+ def _bool(key: str, default: bool) -> bool:
+ v = raw.get(key)
+ return bool(v) if isinstance(v, bool) else default
+
+ def _str(key: str, allowed: List[str], default: str) -> str:
+ v = raw.get(key)
+ return v if v in allowed else default
+
+ def _int(key: str, default: int) -> int:
+ v = raw.get(key)
+ return int(v) if isinstance(v, int) else default
+
+ surface = raw.get("surfaceToggles") or {}
+ google_surf = surface.get("google") or {}
+ msft_surf = surface.get("msft") or {}
+
+ return ConnectionIngestionPrefs(
+ neutralizeBeforeEmbed=_bool("neutralizeBeforeEmbed", False),
+ mailContentDepth=_str("mailContentDepth", ["metadata", "snippet", "full"], _DEFAULT_MAIL_DEPTH),
+ mailIndexAttachments=_bool("mailIndexAttachments", False),
+ filesIndexBinaries=_bool("filesIndexBinaries", True),
+ mimeAllowlist=list(raw.get("mimeAllowlist") or []),
+ clickupScope=_str("clickupScope", ["titles", "title_description", "with_comments"], _DEFAULT_CLICKUP_SCOPE),
+ clickupIndexAttachments=_bool("clickupIndexAttachments", False),
+ gmailEnabled=bool(google_surf.get("gmail", True)),
+ driveEnabled=bool(google_surf.get("drive", True)),
+ sharepointEnabled=bool(msft_surf.get("sharepoint", True)),
+ outlookEnabled=bool(msft_surf.get("outlook", True)),
+ maxAgeDays=_int("maxAgeDays", _DEFAULT_MAX_AGE_DAYS),
+ )
+ except Exception as exc:
+ logger.warning("loadConnectionPrefs failed for %s, using defaults: %s", connectionId, exc)
+ return ConnectionIngestionPrefs()
diff --git a/modules/serviceCenter/services/serviceKnowledge/subConnectorSyncClickup.py b/modules/serviceCenter/services/serviceKnowledge/subConnectorSyncClickup.py
new file mode 100644
index 00000000..31ac9687
--- /dev/null
+++ b/modules/serviceCenter/services/serviceKnowledge/subConnectorSyncClickup.py
@@ -0,0 +1,512 @@
+# Copyright (c) 2025 Patrick Motsch
+# All rights reserved.
+"""ClickUp bootstrap for the unified knowledge ingestion lane.
+
+ClickUp tasks are ingested as *virtual documents* — we never download file
+bytes. Each task becomes a `sourceKind="clickup_task"` IngestionJob whose
+`contentObjects` carry a summary header (name + status + metadata) and the
+task description / text content so retrieval finds them without a live API
+call.
+
+Hierarchy traversal: workspace (team) → spaces → folders / folderless lists →
+tasks. We cap the fan-out with `maxWorkspaces` / `maxListsPerWorkspace` /
+`maxTasks` and skip tasks older than `maxAgeDays` (default 180 d).
+
+Idempotency: `date_updated` from the ClickUp task payload is a millisecond
+timestamp and strictly monotonic per revision — used as `contentVersion`.
+"""
+
+from __future__ import annotations
+
+import hashlib
+import logging
+import time
+from dataclasses import dataclass, field
+from datetime import datetime, timedelta, timezone
+from typing import Any, Callable, Dict, List, Optional
+
+logger = logging.getLogger(__name__)
+
+MAX_TASKS_DEFAULT = 500
+MAX_WORKSPACES_DEFAULT = 3
+MAX_LISTS_PER_WORKSPACE_DEFAULT = 20
+MAX_DESCRIPTION_CHARS_DEFAULT = 8000
+MAX_AGE_DAYS_DEFAULT = 180
+
+
+@dataclass
+class ClickupBootstrapLimits:
+ maxTasks: int = MAX_TASKS_DEFAULT
+ maxWorkspaces: int = MAX_WORKSPACES_DEFAULT
+ maxListsPerWorkspace: int = MAX_LISTS_PER_WORKSPACE_DEFAULT
+ maxDescriptionChars: int = MAX_DESCRIPTION_CHARS_DEFAULT
+ # Only ingest tasks updated within the last N days. None disables filter.
+ maxAgeDays: Optional[int] = MAX_AGE_DAYS_DEFAULT
+ # Include closed/archived tasks if they still meet the recency filter.
+ # ClickUp `closed` tasks often carry the most useful RAG context
+ # ("why was this shipped the way it was?").
+ includeClosed: bool = True
+ # Pass-through to IngestionJob.neutralize
+ neutralize: bool = False
+ # Content scope: "titles" | "title_description" | "with_comments"
+ clickupScope: str = "title_description"
+
+
+@dataclass
+class ClickupBootstrapResult:
+ connectionId: str
+ indexed: int = 0
+ skippedDuplicate: int = 0
+ skippedPolicy: int = 0
+ failed: int = 0
+ workspaces: int = 0
+ lists: int = 0
+ errors: List[str] = field(default_factory=list)
+
+
+def _syntheticTaskId(connectionId: str, taskId: str) -> str:
+ token = hashlib.sha256(f"{connectionId}:{taskId}".encode("utf-8")).hexdigest()[:16]
+ return f"cu:{connectionId[:8]}:{token}"
+
+
+def _truncate(value: Any, limit: int) -> str:
+ text = str(value or "").strip()
+ if not text:
+ return ""
+ if len(text) <= limit:
+ return text
+ return text[:limit].rstrip() + "\n[truncated]"
+
+
+def _isRecent(dateUpdatedMs: Any, maxAgeDays: Optional[int]) -> bool:
+ if not maxAgeDays:
+ return True
+ if not dateUpdatedMs:
+ return True
+ try:
+ ts = datetime.fromtimestamp(int(dateUpdatedMs) / 1000.0, tz=timezone.utc)
+ except Exception:
+ return True
+ cutoff = datetime.now(timezone.utc) - timedelta(days=maxAgeDays)
+ return ts >= cutoff
+
+
+def _buildContentObjects(task: Dict[str, Any], limits: ClickupBootstrapLimits) -> List[Dict[str, Any]]:
+ """Header (name/status/metadata) + optional description + text_content.
+
+ `limits.clickupScope` controls how much is embedded:
+ - "titles": task name + status metadata only
+ - "title_description": header + description / text_content (default)
+ - "with_comments": header + description + text_content
+ (comments themselves are not yet fetched in v1)
+ """
+ name = task.get("name") or f"Task {task.get('id', '')}"
+ status = ((task.get("status") or {}).get("status")) or ""
+ assignees = ", ".join(
+ filter(None, [
+ (a.get("username") or a.get("email") or "")
+ for a in (task.get("assignees") or [])
+ ])
+ )
+ tags = ", ".join(filter(None, [t.get("name", "") for t in (task.get("tags") or [])]))
+ listInfo = task.get("list") or {}
+ folderInfo = task.get("folder") or {}
+ spaceInfo = task.get("space") or {}
+ dueMs = task.get("due_date")
+ dueIso = ""
+ if dueMs:
+ try:
+ dueIso = datetime.fromtimestamp(int(dueMs) / 1000.0, tz=timezone.utc).strftime("%Y-%m-%d")
+ except Exception:
+ dueIso = ""
+
+ headerLines = [
+ f"Task: {name}",
+ f"Status: {status}" if status else "",
+ f"List: {listInfo.get('name', '')}" if listInfo else "",
+ f"Folder: {folderInfo.get('name', '')}" if folderInfo else "",
+ f"Space: {spaceInfo.get('name', '')}" if spaceInfo else "",
+ f"Assignees: {assignees}" if assignees else "",
+ f"Tags: {tags}" if tags else "",
+ f"Due: {dueIso}" if dueIso else "",
+ f"Url: {task.get('url', '')}" if task.get("url") else "",
+ ]
+ header = "\n".join(line for line in headerLines if line)
+
+ parts: List[Dict[str, Any]] = [{
+ "contentObjectId": "header",
+ "contentType": "text",
+ "data": header,
+ "contextRef": {"part": "header"},
+ }]
+
+ scope = getattr(limits, "clickupScope", "title_description")
+ if scope in ("title_description", "with_comments"):
+ description = _truncate(task.get("description"), limits.maxDescriptionChars)
+ if description:
+ parts.append({
+ "contentObjectId": "description",
+ "contentType": "text",
+ "data": description,
+ "contextRef": {"part": "description"},
+ })
+ # text_content is ClickUp's rendered-markdown version; include if it adds
+ # something beyond the plain description (common for bullet lists, checklists).
+ textContent = _truncate(task.get("text_content"), limits.maxDescriptionChars)
+ if textContent and textContent != description:
+ parts.append({
+ "contentObjectId": "text_content",
+ "contentType": "text",
+ "data": textContent,
+ "contextRef": {"part": "text_content"},
+ })
+ return parts
+
+
+async def bootstrapClickup(
+ connectionId: str,
+ *,
+ progressCb: Optional[Callable[[int, Optional[str]], None]] = None,
+ adapter: Any = None,
+ connection: Any = None,
+ knowledgeService: Any = None,
+ limits: Optional[ClickupBootstrapLimits] = None,
+) -> Dict[str, Any]:
+ """Walk workspaces → lists → tasks and ingest each task as a virtual doc."""
+ from modules.serviceCenter.services.serviceKnowledge.subConnectorPrefs import loadConnectionPrefs
+ prefs = loadConnectionPrefs(connectionId)
+
+ if not limits:
+ limits = ClickupBootstrapLimits(
+ maxAgeDays=prefs.maxAgeDays if prefs.maxAgeDays > 0 else None,
+ neutralize=prefs.neutralizeBeforeEmbed,
+ clickupScope=prefs.clickupScope,
+ )
+
+ startMs = time.time()
+ result = ClickupBootstrapResult(connectionId=connectionId)
+
+ logger.info(
+ "ingestion.connection.bootstrap.started part=clickup connectionId=%s",
+ connectionId,
+ extra={
+ "event": "ingestion.connection.bootstrap.started",
+ "part": "clickup",
+ "connectionId": connectionId,
+ },
+ )
+
+ if adapter is None or knowledgeService is None or connection is None:
+ adapter, connection, knowledgeService = await _resolveDependencies(connectionId)
+
+ mandateId = str(getattr(connection, "mandateId", "") or "") if connection is not None else ""
+ userId = str(getattr(connection, "userId", "") or "") if connection is not None else ""
+
+ svc = getattr(adapter, "_svc", None)
+ if svc is None:
+ result.errors.append("adapter missing _svc instance")
+ return _finalizeResult(connectionId, result, startMs)
+
+ try:
+ teamsResp = await svc.getAuthorizedTeams()
+ except Exception as exc:
+ logger.error("clickup team discovery failed for %s: %s", connectionId, exc, exc_info=True)
+ result.errors.append(f"teams: {exc}")
+ return _finalizeResult(connectionId, result, startMs)
+
+ teams = (teamsResp or {}).get("teams") or []
+ for team in teams[: limits.maxWorkspaces]:
+ if result.indexed + result.skippedDuplicate >= limits.maxTasks:
+ break
+ teamId = str(team.get("id", "") or "")
+ if not teamId:
+ continue
+ result.workspaces += 1
+ try:
+ await _walkTeam(
+ svc=svc,
+ knowledgeService=knowledgeService,
+ connectionId=connectionId,
+ mandateId=mandateId,
+ userId=userId,
+ team=team,
+ limits=limits,
+ result=result,
+ progressCb=progressCb,
+ )
+ except Exception as exc:
+ logger.error("clickup team %s walk failed: %s", teamId, exc, exc_info=True)
+ result.errors.append(f"team({teamId}): {exc}")
+
+ return _finalizeResult(connectionId, result, startMs)
+
+
+async def _resolveDependencies(connectionId: str):
+ from modules.interfaces.interfaceDbApp import getRootInterface
+ from modules.auth import TokenManager
+ from modules.connectors.providerClickup.connectorClickup import ClickupConnector
+ from modules.serviceCenter import getService
+ from modules.serviceCenter.context import ServiceCenterContext
+ from modules.security.rootAccess import getRootUser
+
+ rootInterface = getRootInterface()
+ connection = rootInterface.getUserConnectionById(connectionId)
+ if connection is None:
+ raise ValueError(f"UserConnection not found: {connectionId}")
+
+ token = TokenManager().getFreshToken(connectionId)
+ if not token or not token.tokenAccess:
+ raise ValueError(f"No valid token for connection {connectionId}")
+
+ provider = ClickupConnector(connection, token.tokenAccess)
+ adapter = provider.getServiceAdapter("clickup")
+
+ rootUser = getRootUser()
+ ctx = ServiceCenterContext(
+ user=rootUser,
+ mandate_id=str(getattr(connection, "mandateId", "") or ""),
+ )
+ knowledgeService = getService("knowledge", ctx)
+ return adapter, connection, knowledgeService
+
+
+async def _walkTeam(
+ *,
+ svc,
+ knowledgeService,
+ connectionId: str,
+ mandateId: str,
+ userId: str,
+ team: Dict[str, Any],
+ limits: ClickupBootstrapLimits,
+ result: ClickupBootstrapResult,
+ progressCb: Optional[Callable[[int, Optional[str]], None]],
+) -> None:
+ teamId = str(team.get("id", "") or "")
+ spacesResp = await svc.getSpaces(teamId)
+ spaces = (spacesResp or {}).get("spaces") or []
+
+ listsCollected: List[Dict[str, Any]] = []
+ for space in spaces:
+ if len(listsCollected) >= limits.maxListsPerWorkspace:
+ break
+ spaceId = str(space.get("id", "") or "")
+ if not spaceId:
+ continue
+
+ # Folderless lists directly under the space
+ folderless = await svc.getFolderlessLists(spaceId)
+ for lst in (folderless or {}).get("lists") or []:
+ if len(listsCollected) >= limits.maxListsPerWorkspace:
+ break
+ listsCollected.append({**lst, "_space": space})
+
+ # Lists inside folders
+ foldersResp = await svc.getFolders(spaceId)
+ for folder in (foldersResp or {}).get("folders") or []:
+ if len(listsCollected) >= limits.maxListsPerWorkspace:
+ break
+ folderId = str(folder.get("id", "") or "")
+ if not folderId:
+ continue
+ folderLists = await svc.getListsInFolder(folderId)
+ for lst in (folderLists or {}).get("lists") or []:
+ if len(listsCollected) >= limits.maxListsPerWorkspace:
+ break
+ listsCollected.append({**lst, "_space": space, "_folder": folder})
+
+ for lst in listsCollected:
+ if result.indexed + result.skippedDuplicate >= limits.maxTasks:
+ return
+ result.lists += 1
+ await _walkList(
+ svc=svc,
+ knowledgeService=knowledgeService,
+ connectionId=connectionId,
+ mandateId=mandateId,
+ userId=userId,
+ teamId=teamId,
+ lst=lst,
+ limits=limits,
+ result=result,
+ progressCb=progressCb,
+ )
+
+
+async def _walkList(
+ *,
+ svc,
+ knowledgeService,
+ connectionId: str,
+ mandateId: str,
+ userId: str,
+ teamId: str,
+ lst: Dict[str, Any],
+ limits: ClickupBootstrapLimits,
+ result: ClickupBootstrapResult,
+ progressCb: Optional[Callable[[int, Optional[str]], None]],
+) -> None:
+ listId = str(lst.get("id", "") or "")
+ if not listId:
+ return
+ page = 0
+ while result.indexed + result.skippedDuplicate < limits.maxTasks:
+ resp = await svc.getTasksInList(
+ listId,
+ page=page,
+ include_closed=limits.includeClosed,
+ subtasks=True,
+ )
+ if isinstance(resp, dict) and resp.get("error"):
+ logger.warning("clickup tasks list=%s page=%d error: %s", listId, page, resp.get("error"))
+ result.errors.append(f"list({listId}): {resp.get('error')}")
+ return
+ tasks = (resp or {}).get("tasks") or []
+ if not tasks:
+ return
+
+ for task in tasks:
+ if result.indexed + result.skippedDuplicate >= limits.maxTasks:
+ return
+ if not _isRecent(task.get("date_updated"), limits.maxAgeDays):
+ result.skippedPolicy += 1
+ continue
+ # Inject the list/folder/space metadata we already loaded.
+ task["list"] = task.get("list") or {"id": listId, "name": lst.get("name")}
+ task["folder"] = task.get("folder") or lst.get("_folder") or {}
+ task["space"] = task.get("space") or lst.get("_space") or {}
+ await _ingestTask(
+ knowledgeService=knowledgeService,
+ connectionId=connectionId,
+ mandateId=mandateId,
+ userId=userId,
+ teamId=teamId,
+ task=task,
+ limits=limits,
+ result=result,
+ progressCb=progressCb,
+ )
+
+ if len(tasks) < 100: # ClickUp page-size hint: fewer than 100 => last page
+ return
+ page += 1
+
+
+async def _ingestTask(
+ *,
+ knowledgeService,
+ connectionId: str,
+ mandateId: str,
+ userId: str,
+ teamId: str,
+ task: Dict[str, Any],
+ limits: ClickupBootstrapLimits,
+ result: ClickupBootstrapResult,
+ progressCb: Optional[Callable[[int, Optional[str]], None]],
+) -> None:
+ from modules.serviceCenter.services.serviceKnowledge.mainServiceKnowledge import IngestionJob
+
+ taskId = str(task.get("id", "") or "")
+ if not taskId:
+ result.skippedPolicy += 1
+ return
+ revision = str(task.get("date_updated") or task.get("date_created") or "")
+ name = task.get("name") or f"Task {taskId}"
+ syntheticId = _syntheticTaskId(connectionId, taskId)
+ fileName = f"{name[:80].strip() or taskId}.task.json"
+
+ contentObjects = _buildContentObjects(task, limits)
+
+ try:
+ handle = await knowledgeService.requestIngestion(
+ IngestionJob(
+ sourceKind="clickup_task",
+ sourceId=syntheticId,
+ fileName=fileName,
+ mimeType="application/vnd.clickup.task+json",
+ userId=userId,
+ mandateId=mandateId,
+ contentObjects=contentObjects,
+ contentVersion=revision or None,
+ neutralize=limits.neutralize,
+ provenance={
+ "connectionId": connectionId,
+ "authority": "clickup",
+ "service": "clickup",
+ "externalItemId": taskId,
+ "teamId": teamId,
+ "listId": ((task.get("list") or {}).get("id")),
+ "spaceId": ((task.get("space") or {}).get("id")),
+ "url": task.get("url"),
+ "status": ((task.get("status") or {}).get("status")),
+ "tier": limits.clickupScope,
+ },
+ )
+ )
+ except Exception as exc:
+ logger.error("clickup ingestion %s failed: %s", taskId, exc, exc_info=True)
+ result.failed += 1
+ result.errors.append(f"ingest({taskId}): {exc}")
+ return
+
+ if handle.status == "duplicate":
+ result.skippedDuplicate += 1
+ elif handle.status == "indexed":
+ result.indexed += 1
+ else:
+ result.failed += 1
+
+ if progressCb is not None and (result.indexed + result.skippedDuplicate) % 50 == 0:
+ processed = result.indexed + result.skippedDuplicate
+ try:
+ progressCb(
+ min(90, 10 + int(80 * processed / max(1, limits.maxTasks))),
+ f"clickup processed={processed}",
+ )
+ except Exception:
+ pass
+ logger.info(
+ "ingestion.connection.bootstrap.progress part=clickup processed=%d skippedDup=%d failed=%d",
+ processed, result.skippedDuplicate, result.failed,
+ extra={
+ "event": "ingestion.connection.bootstrap.progress",
+ "part": "clickup",
+ "connectionId": connectionId,
+ "processed": processed,
+ "skippedDup": result.skippedDuplicate,
+ "failed": result.failed,
+ },
+ )
+
+
+def _finalizeResult(connectionId: str, result: ClickupBootstrapResult, startMs: float) -> Dict[str, Any]:
+ durationMs = int((time.time() - startMs) * 1000)
+ logger.info(
+ "ingestion.connection.bootstrap.done part=clickup connectionId=%s indexed=%d skippedDup=%d skippedPolicy=%d failed=%d workspaces=%d lists=%d durationMs=%d",
+ connectionId,
+ result.indexed, result.skippedDuplicate, result.skippedPolicy,
+ result.failed, result.workspaces, result.lists, durationMs,
+ extra={
+ "event": "ingestion.connection.bootstrap.done",
+ "part": "clickup",
+ "connectionId": connectionId,
+ "indexed": result.indexed,
+ "skippedDup": result.skippedDuplicate,
+ "skippedPolicy": result.skippedPolicy,
+ "failed": result.failed,
+ "workspaces": result.workspaces,
+ "lists": result.lists,
+ "durationMs": durationMs,
+ },
+ )
+ return {
+ "connectionId": result.connectionId,
+ "indexed": result.indexed,
+ "skippedDuplicate": result.skippedDuplicate,
+ "skippedPolicy": result.skippedPolicy,
+ "failed": result.failed,
+ "workspaces": result.workspaces,
+ "lists": result.lists,
+ "durationMs": durationMs,
+ "errors": result.errors[:20],
+ }
diff --git a/modules/serviceCenter/services/serviceKnowledge/subConnectorSyncGdrive.py b/modules/serviceCenter/services/serviceKnowledge/subConnectorSyncGdrive.py
new file mode 100644
index 00000000..5e4e659b
--- /dev/null
+++ b/modules/serviceCenter/services/serviceKnowledge/subConnectorSyncGdrive.py
@@ -0,0 +1,443 @@
+# Copyright (c) 2025 Patrick Motsch
+# All rights reserved.
+"""Google Drive bootstrap for the unified knowledge ingestion lane.
+
+Mirrors the SharePoint pilot (see subConnectorSyncSharepoint.py). Walks the
+user's *My Drive* tree from the virtual `root` folder, downloads each
+file-like item via `DriveAdapter.download` (which handles native Google docs
+via export), runs the standard extraction pipeline and routes results through
+`KnowledgeService.requestIngestion` with `sourceKind="gdrive_item"` and
+`contentVersion = modifiedTime` (monotonic per-revision).
+"""
+
+from __future__ import annotations
+
+import hashlib
+import logging
+import time
+from dataclasses import dataclass, field
+from datetime import datetime, timedelta, timezone
+from typing import Any, Callable, Dict, List, Optional
+
+from modules.datamodels.datamodelExtraction import ExtractionOptions
+
+logger = logging.getLogger(__name__)
+
+MAX_ITEMS_DEFAULT = 500
+MAX_BYTES_DEFAULT = 200 * 1024 * 1024
+MAX_FILE_SIZE_DEFAULT = 25 * 1024 * 1024
+SKIP_MIME_PREFIXES_DEFAULT = ("video/", "audio/")
+MAX_DEPTH_DEFAULT = 4
+MAX_AGE_DAYS_DEFAULT = 365
+
+# Google Drive uses virtual mime-types for folders and non-downloadable assets.
+FOLDER_MIME = "application/vnd.google-apps.folder"
+
+
+@dataclass
+class GdriveBootstrapLimits:
+ maxItems: int = MAX_ITEMS_DEFAULT
+ maxBytes: int = MAX_BYTES_DEFAULT
+ maxFileSize: int = MAX_FILE_SIZE_DEFAULT
+ skipMimePrefixes: tuple = SKIP_MIME_PREFIXES_DEFAULT
+ maxDepth: int = MAX_DEPTH_DEFAULT
+ # Only ingest files modified within the last N days. None disables filter.
+ maxAgeDays: Optional[int] = MAX_AGE_DAYS_DEFAULT
+ # Pass-through to IngestionJob.neutralize
+ neutralize: bool = False
+ # Whether to skip binary/non-text files
+ filesIndexBinaries: bool = True
+
+
+@dataclass
+class GdriveBootstrapResult:
+ connectionId: str
+ indexed: int = 0
+ skippedDuplicate: int = 0
+ skippedPolicy: int = 0
+ failed: int = 0
+ bytesProcessed: int = 0
+ errors: List[str] = field(default_factory=list)
+
+
+def _syntheticFileId(connectionId: str, externalItemId: str) -> str:
+ token = hashlib.sha256(f"{connectionId}:{externalItemId}".encode("utf-8")).hexdigest()[:16]
+ return f"gd:{connectionId[:8]}:{token}"
+
+
+def _toContentObjects(extracted, fileName: str) -> List[Dict[str, Any]]:
+ parts = getattr(extracted, "parts", None) or []
+ out: List[Dict[str, Any]] = []
+ for part in parts:
+ data = getattr(part, "data", None) or ""
+ if not data or not str(data).strip():
+ continue
+ typeGroup = getattr(part, "typeGroup", "text") or "text"
+ contentType = "text"
+ if typeGroup == "image":
+ contentType = "image"
+ elif typeGroup in ("binary", "container"):
+ contentType = "other"
+ out.append({
+ "contentObjectId": getattr(part, "id", ""),
+ "contentType": contentType,
+ "data": data,
+ "contextRef": {
+ "containerPath": fileName,
+ "location": getattr(part, "label", None) or "file",
+ **(getattr(part, "metadata", None) or {}),
+ },
+ })
+ return out
+
+
+def _isRecent(modifiedIso: Optional[str], maxAgeDays: Optional[int]) -> bool:
+ if not maxAgeDays:
+ return True
+ if not modifiedIso:
+ # No timestamp -> be permissive (Drive native docs sometimes omit it on export).
+ return True
+ try:
+ # Google returns RFC 3339 with `Z` or offset; python 3.11+ parses both.
+ ts = datetime.fromisoformat(modifiedIso.replace("Z", "+00:00"))
+ except Exception:
+ return True
+ cutoff = datetime.now(timezone.utc) - timedelta(days=maxAgeDays)
+ if ts.tzinfo is None:
+ ts = ts.replace(tzinfo=timezone.utc)
+ return ts >= cutoff
+
+
+async def bootstrapGdrive(
+ connectionId: str,
+ *,
+ progressCb: Optional[Callable[[int, Optional[str]], None]] = None,
+ adapter: Any = None,
+ connection: Any = None,
+ knowledgeService: Any = None,
+ limits: Optional[GdriveBootstrapLimits] = None,
+ runExtractionFn: Optional[Callable[..., Any]] = None,
+) -> Dict[str, Any]:
+ """Walk My Drive starting from the virtual root folder."""
+ from modules.serviceCenter.services.serviceKnowledge.subConnectorPrefs import loadConnectionPrefs
+ prefs = loadConnectionPrefs(connectionId)
+
+ if not limits:
+ limits = GdriveBootstrapLimits(
+ maxAgeDays=prefs.maxAgeDays if prefs.maxAgeDays > 0 else None,
+ neutralize=prefs.neutralizeBeforeEmbed,
+ filesIndexBinaries=prefs.filesIndexBinaries,
+ )
+
+ startMs = time.time()
+ result = GdriveBootstrapResult(connectionId=connectionId)
+
+ logger.info(
+ "ingestion.connection.bootstrap.started part=gdrive connectionId=%s",
+ connectionId,
+ extra={
+ "event": "ingestion.connection.bootstrap.started",
+ "part": "gdrive",
+ "connectionId": connectionId,
+ },
+ )
+
+ if adapter is None or knowledgeService is None or connection is None:
+ adapter, connection, knowledgeService = await _resolveDependencies(connectionId)
+ if runExtractionFn is None:
+ from modules.serviceCenter.services.serviceExtraction.subPipeline import runExtraction
+ from modules.serviceCenter.services.serviceExtraction.subRegistry import (
+ ExtractorRegistry, ChunkerRegistry,
+ )
+ extractorRegistry = ExtractorRegistry()
+ chunkerRegistry = ChunkerRegistry()
+
+ def runExtractionFn(bytesData, name, mime, options): # type: ignore[no-redef]
+ return runExtraction(extractorRegistry, chunkerRegistry, bytesData, name, mime, options)
+
+ mandateId = str(getattr(connection, "mandateId", "") or "") if connection is not None else ""
+ userId = str(getattr(connection, "userId", "") or "") if connection is not None else ""
+
+ try:
+ await _walkFolder(
+ adapter=adapter,
+ knowledgeService=knowledgeService,
+ runExtractionFn=runExtractionFn,
+ connectionId=connectionId,
+ mandateId=mandateId,
+ userId=userId,
+ folderPath="/", # DriveAdapter.browse maps "" / "/" -> "root"
+ depth=0,
+ limits=limits,
+ result=result,
+ progressCb=progressCb,
+ )
+ except Exception as exc:
+ logger.error("gdrive walk failed for %s: %s", connectionId, exc, exc_info=True)
+ result.errors.append(f"walk: {exc}")
+
+ return _finalizeResult(connectionId, result, startMs)
+
+
+async def _resolveDependencies(connectionId: str):
+ from modules.interfaces.interfaceDbApp import getRootInterface
+ from modules.auth import TokenManager
+ from modules.connectors.providerGoogle.connectorGoogle import GoogleConnector
+ from modules.serviceCenter import getService
+ from modules.serviceCenter.context import ServiceCenterContext
+ from modules.security.rootAccess import getRootUser
+
+ rootInterface = getRootInterface()
+ connection = rootInterface.getUserConnectionById(connectionId)
+ if connection is None:
+ raise ValueError(f"UserConnection not found: {connectionId}")
+
+ token = TokenManager().getFreshToken(connectionId)
+ if not token or not token.tokenAccess:
+ raise ValueError(f"No valid token for connection {connectionId}")
+
+ provider = GoogleConnector(connection, token.tokenAccess)
+ adapter = provider.getServiceAdapter("drive")
+
+ rootUser = getRootUser()
+ ctx = ServiceCenterContext(
+ user=rootUser,
+ mandate_id=str(getattr(connection, "mandateId", "") or ""),
+ )
+ knowledgeService = getService("knowledge", ctx)
+ return adapter, connection, knowledgeService
+
+
+async def _walkFolder(
+ *,
+ adapter,
+ knowledgeService,
+ runExtractionFn,
+ connectionId: str,
+ mandateId: str,
+ userId: str,
+ folderPath: str,
+ depth: int,
+ limits: GdriveBootstrapLimits,
+ result: GdriveBootstrapResult,
+ progressCb: Optional[Callable[[int, Optional[str]], None]],
+) -> None:
+ if depth > limits.maxDepth:
+ return
+ try:
+ entries = await adapter.browse(folderPath)
+ except Exception as exc:
+ logger.warning("gdrive browse %s failed: %s", folderPath, exc)
+ result.errors.append(f"browse({folderPath}): {exc}")
+ return
+
+ for entry in entries:
+ if result.indexed + result.skippedDuplicate >= limits.maxItems:
+ return
+ if result.bytesProcessed >= limits.maxBytes:
+ return
+
+ entryPath = getattr(entry, "path", "") or ""
+ metadata = getattr(entry, "metadata", {}) or {}
+ mimeType = getattr(entry, "mimeType", None) or metadata.get("mimeType")
+
+ if getattr(entry, "isFolder", False) or mimeType == FOLDER_MIME:
+ await _walkFolder(
+ adapter=adapter,
+ knowledgeService=knowledgeService,
+ runExtractionFn=runExtractionFn,
+ connectionId=connectionId,
+ mandateId=mandateId,
+ userId=userId,
+ folderPath=entryPath,
+ depth=depth + 1,
+ limits=limits,
+ result=result,
+ progressCb=progressCb,
+ )
+ continue
+
+ effectiveMime = mimeType or "application/octet-stream"
+ if any(effectiveMime.startswith(prefix) for prefix in limits.skipMimePrefixes):
+ result.skippedPolicy += 1
+ continue
+ size = int(getattr(entry, "size", 0) or 0)
+ if size and size > limits.maxFileSize:
+ result.skippedPolicy += 1
+ continue
+ modifiedTime = metadata.get("modifiedTime")
+ if not _isRecent(modifiedTime, limits.maxAgeDays):
+ result.skippedPolicy += 1
+ continue
+
+ externalItemId = metadata.get("id") or entryPath
+ revision = modifiedTime
+
+ await _ingestOne(
+ adapter=adapter,
+ knowledgeService=knowledgeService,
+ runExtractionFn=runExtractionFn,
+ connectionId=connectionId,
+ mandateId=mandateId,
+ userId=userId,
+ entry=entry,
+ entryPath=entryPath,
+ mimeType=effectiveMime,
+ externalItemId=externalItemId,
+ revision=revision,
+ limits=limits,
+ result=result,
+ progressCb=progressCb,
+ )
+
+
+async def _ingestOne(
+ *,
+ adapter,
+ knowledgeService,
+ runExtractionFn,
+ connectionId: str,
+ mandateId: str,
+ userId: str,
+ entry,
+ entryPath: str,
+ mimeType: str,
+ externalItemId: str,
+ revision: Optional[str],
+ limits: GdriveBootstrapLimits,
+ result: GdriveBootstrapResult,
+ progressCb: Optional[Callable[[int, Optional[str]], None]],
+) -> None:
+ from modules.serviceCenter.services.serviceKnowledge.mainServiceKnowledge import IngestionJob
+
+ syntheticFileId = _syntheticFileId(connectionId, externalItemId)
+ fileName = getattr(entry, "name", "") or externalItemId
+
+ try:
+ downloaded = await adapter.download(entryPath)
+ except Exception as exc:
+ logger.warning("gdrive download %s failed: %s", entryPath, exc)
+ result.failed += 1
+ result.errors.append(f"download({entryPath}): {exc}")
+ return
+
+ # Adapter.download returns raw bytes today; guard DownloadResult shape too.
+ fileBytes: bytes
+ if isinstance(downloaded, (bytes, bytearray)):
+ fileBytes = bytes(downloaded)
+ else:
+ fileBytes = bytes(getattr(downloaded, "data", b"") or b"")
+ if getattr(downloaded, "mimeType", None):
+ mimeType = downloaded.mimeType # export may have changed the type
+ if not fileBytes:
+ result.failed += 1
+ return
+ if len(fileBytes) > limits.maxFileSize:
+ result.skippedPolicy += 1
+ return
+
+ result.bytesProcessed += len(fileBytes)
+
+ try:
+ extracted = runExtractionFn(
+ fileBytes, fileName, mimeType,
+ ExtractionOptions(mergeStrategy=None),
+ )
+ except Exception as exc:
+ logger.warning("gdrive extraction %s failed: %s", entryPath, exc)
+ result.failed += 1
+ result.errors.append(f"extract({entryPath}): {exc}")
+ return
+
+ contentObjects = _toContentObjects(extracted, fileName)
+ if not contentObjects:
+ result.skippedPolicy += 1
+ return
+
+ try:
+ handle = await knowledgeService.requestIngestion(
+ IngestionJob(
+ sourceKind="gdrive_item",
+ sourceId=syntheticFileId,
+ fileName=fileName,
+ mimeType=mimeType,
+ userId=userId,
+ mandateId=mandateId,
+ contentObjects=contentObjects,
+ contentVersion=revision,
+ neutralize=limits.neutralize,
+ provenance={
+ "connectionId": connectionId,
+ "authority": "google",
+ "service": "drive",
+ "externalItemId": externalItemId,
+ "entryPath": entryPath,
+ "tier": "body",
+ },
+ )
+ )
+ except Exception as exc:
+ logger.error("gdrive ingestion %s failed: %s", entryPath, exc, exc_info=True)
+ result.failed += 1
+ result.errors.append(f"ingest({entryPath}): {exc}")
+ return
+
+ if handle.status == "duplicate":
+ result.skippedDuplicate += 1
+ elif handle.status == "indexed":
+ result.indexed += 1
+ else:
+ result.failed += 1
+
+ if progressCb is not None and (result.indexed + result.skippedDuplicate) % 50 == 0:
+ processed = result.indexed + result.skippedDuplicate
+ try:
+ progressCb(
+ min(90, 10 + int(80 * processed / max(1, limits.maxItems))),
+ f"gdrive processed={processed}",
+ )
+ except Exception:
+ pass
+ logger.info(
+ "ingestion.connection.bootstrap.progress part=gdrive processed=%d skippedDup=%d failed=%d",
+ processed, result.skippedDuplicate, result.failed,
+ extra={
+ "event": "ingestion.connection.bootstrap.progress",
+ "part": "gdrive",
+ "connectionId": connectionId,
+ "processed": processed,
+ "skippedDup": result.skippedDuplicate,
+ "failed": result.failed,
+ },
+ )
+
+
+def _finalizeResult(connectionId: str, result: GdriveBootstrapResult, startMs: float) -> Dict[str, Any]:
+ durationMs = int((time.time() - startMs) * 1000)
+ logger.info(
+ "ingestion.connection.bootstrap.done part=gdrive connectionId=%s indexed=%d skippedDup=%d skippedPolicy=%d failed=%d bytes=%d durationMs=%d",
+ connectionId,
+ result.indexed, result.skippedDuplicate, result.skippedPolicy,
+ result.failed, result.bytesProcessed, durationMs,
+ extra={
+ "event": "ingestion.connection.bootstrap.done",
+ "part": "gdrive",
+ "connectionId": connectionId,
+ "indexed": result.indexed,
+ "skippedDup": result.skippedDuplicate,
+ "skippedPolicy": result.skippedPolicy,
+ "failed": result.failed,
+ "bytes": result.bytesProcessed,
+ "durationMs": durationMs,
+ },
+ )
+ return {
+ "connectionId": result.connectionId,
+ "indexed": result.indexed,
+ "skippedDuplicate": result.skippedDuplicate,
+ "skippedPolicy": result.skippedPolicy,
+ "failed": result.failed,
+ "bytesProcessed": result.bytesProcessed,
+ "durationMs": durationMs,
+ "errors": result.errors[:20],
+ }
diff --git a/modules/serviceCenter/services/serviceKnowledge/subConnectorSyncGmail.py b/modules/serviceCenter/services/serviceKnowledge/subConnectorSyncGmail.py
new file mode 100644
index 00000000..21fec83d
--- /dev/null
+++ b/modules/serviceCenter/services/serviceKnowledge/subConnectorSyncGmail.py
@@ -0,0 +1,606 @@
+# Copyright (c) 2025 Patrick Motsch
+# All rights reserved.
+"""Gmail bootstrap for the unified knowledge ingestion lane.
+
+Mirrors the Outlook pilot (see subConnectorSyncOutlook.py) but talks to Google
+Mail's REST API. Messages become `sourceKind="gmail_message"` virtual documents
+with header / snippet / cleaned body content-objects; attachments are optional
+child jobs with `sourceKind="gmail_attachment"`.
+
+Idempotency: Gmail's stable `historyId` (or `internalDate` as fallback) is
+passed as `contentVersion`, so rerunning the bootstrap yields
+`ingestion.skipped.duplicate` for unchanged messages.
+"""
+
+from __future__ import annotations
+
+import asyncio
+import base64
+import hashlib
+import logging
+import time
+from dataclasses import dataclass, field
+from datetime import datetime, timedelta, timezone
+from typing import Any, Callable, Dict, List, Optional
+
+from modules.serviceCenter.services.serviceKnowledge.subTextClean import cleanEmailBody
+
+logger = logging.getLogger(__name__)
+
+MAX_MESSAGES_DEFAULT = 500
+MAX_BODY_CHARS_DEFAULT = 8000
+MAX_ATTACHMENT_BYTES_DEFAULT = 10 * 1024 * 1024
+DEFAULT_LABELS = ("INBOX", "SENT")
+
+
+@dataclass
+class GmailBootstrapLimits:
+ maxMessages: int = MAX_MESSAGES_DEFAULT
+ labels: tuple = DEFAULT_LABELS
+ maxBodyChars: int = MAX_BODY_CHARS_DEFAULT
+ includeAttachments: bool = False
+ maxAttachmentBytes: int = MAX_ATTACHMENT_BYTES_DEFAULT
+ # Only fetch messages newer than N days. None disables filter.
+ maxAgeDays: Optional[int] = 90
+ # Content depth: "metadata" | "snippet" | "full"
+ mailContentDepth: str = "full"
+ # Pass-through to IngestionJob.neutralize
+ neutralize: bool = False
+
+
+@dataclass
+class GmailBootstrapResult:
+ connectionId: str
+ indexed: int = 0
+ skippedDuplicate: int = 0
+ skippedPolicy: int = 0
+ failed: int = 0
+ attachmentsIndexed: int = 0
+ errors: List[str] = field(default_factory=list)
+
+
+def _syntheticMessageId(connectionId: str, messageId: str) -> str:
+ token = hashlib.sha256(f"{connectionId}:{messageId}".encode("utf-8")).hexdigest()[:16]
+ return f"gm:{connectionId[:8]}:{token}"
+
+
+def _syntheticAttachmentId(connectionId: str, messageId: str, attachmentId: str) -> str:
+ token = hashlib.sha256(
+ f"{connectionId}:{messageId}:{attachmentId}".encode("utf-8")
+ ).hexdigest()[:16]
+ return f"ga:{connectionId[:8]}:{token}"
+
+
+def _decodeBase64Url(data: str) -> bytes:
+ if not data:
+ return b""
+ # Gmail uses URL-safe base64 without padding.
+ padding = 4 - (len(data) % 4)
+ if padding != 4:
+ data = data + ("=" * padding)
+ try:
+ return base64.urlsafe_b64decode(data)
+ except Exception:
+ return b""
+
+
+def _walkPayloadForBody(payload: Dict[str, Any]) -> Dict[str, str]:
+ """Return {"text": ..., "html": ...} by walking MIME parts.
+
+ Gmail `payload` is a tree of parts. We prefer `text/plain` for the cleaned
+ body, but capture `text/html` as a fallback so `cleanEmailBody` can strip
+ markup if plain is missing.
+ """
+ found: Dict[str, str] = {"text": "", "html": ""}
+
+ def _walk(part: Dict[str, Any]) -> None:
+ mime = (part.get("mimeType") or "").lower()
+ body = part.get("body") or {}
+ raw = body.get("data") or ""
+ if raw and mime.startswith("text/"):
+ decoded = _decodeBase64Url(raw).decode("utf-8", errors="replace")
+ key = "text" if mime == "text/plain" else ("html" if mime == "text/html" else "")
+ if key and not found[key]:
+ found[key] = decoded
+ for sub in part.get("parts") or []:
+ _walk(sub)
+
+ _walk(payload or {})
+ return found
+
+
+def _headerMap(payload: Dict[str, Any]) -> Dict[str, str]:
+ return {
+ (h.get("name") or "").lower(): (h.get("value") or "")
+ for h in (payload.get("headers") or [])
+ }
+
+
+def _buildContentObjects(
+ message: Dict[str, Any],
+ maxBodyChars: int,
+ mailContentDepth: str = "full",
+) -> List[Dict[str, Any]]:
+ """Build content objects for a Gmail message.
+
+ `mailContentDepth` controls how much is embedded:
+ - "metadata": header only (subject, from, to, date)
+ - "snippet": header + Gmail snippet (~155 chars, no full body)
+ - "full": header + snippet + cleaned full body (default)
+ """
+ payload = message.get("payload") or {}
+ headers = _headerMap(payload)
+ subject = headers.get("subject") or "(no subject)"
+ fromAddr = headers.get("from") or ""
+ toAddr = headers.get("to") or ""
+ ccAddr = headers.get("cc") or ""
+ date = headers.get("date") or ""
+ snippet = message.get("snippet") or ""
+
+ parts: List[Dict[str, Any]] = []
+ header = (
+ f"Subject: {subject}\n"
+ f"From: {fromAddr}\n"
+ f"To: {toAddr}\n"
+ + (f"Cc: {ccAddr}\n" if ccAddr else "")
+ + f"Date: {date}"
+ )
+ parts.append({
+ "contentObjectId": "header",
+ "contentType": "text",
+ "data": header,
+ "contextRef": {"part": "header"},
+ })
+ if mailContentDepth in ("snippet", "full") and snippet:
+ parts.append({
+ "contentObjectId": "snippet",
+ "contentType": "text",
+ "data": snippet,
+ "contextRef": {"part": "snippet"},
+ })
+ if mailContentDepth == "full":
+ bodies = _walkPayloadForBody(payload)
+ rawBody = bodies["text"] or bodies["html"]
+ cleanedBody = cleanEmailBody(rawBody, maxChars=maxBodyChars) if rawBody else ""
+ if cleanedBody:
+ parts.append({
+ "contentObjectId": "body",
+ "contentType": "text",
+ "data": cleanedBody,
+ "contextRef": {"part": "body"},
+ })
+ return parts
+
+
+async def bootstrapGmail(
+ connectionId: str,
+ *,
+ progressCb: Optional[Callable[[int, Optional[str]], None]] = None,
+ adapter: Any = None,
+ connection: Any = None,
+ knowledgeService: Any = None,
+ limits: Optional[GmailBootstrapLimits] = None,
+ googleGetFn: Optional[Callable[..., Any]] = None,
+) -> Dict[str, Any]:
+ """Enumerate Gmail labels (INBOX + SENT default) and ingest messages."""
+ from modules.serviceCenter.services.serviceKnowledge.subConnectorPrefs import loadConnectionPrefs
+ prefs = loadConnectionPrefs(connectionId)
+
+ if not limits:
+ limits = GmailBootstrapLimits(
+ includeAttachments=prefs.mailIndexAttachments,
+ maxAgeDays=prefs.maxAgeDays if prefs.maxAgeDays > 0 else None,
+ mailContentDepth=prefs.mailContentDepth,
+ neutralize=prefs.neutralizeBeforeEmbed,
+ )
+
+ startMs = time.time()
+ result = GmailBootstrapResult(connectionId=connectionId)
+
+ logger.info(
+ "ingestion.connection.bootstrap.started part=gmail connectionId=%s",
+ connectionId,
+ extra={
+ "event": "ingestion.connection.bootstrap.started",
+ "part": "gmail",
+ "connectionId": connectionId,
+ },
+ )
+
+ if adapter is None or knowledgeService is None or connection is None:
+ adapter, connection, knowledgeService = await _resolveDependencies(connectionId)
+
+ if googleGetFn is None:
+ from modules.connectors.providerGoogle.connectorGoogle import _googleGet as _defaultGet
+
+ token = getattr(adapter, "_token", "")
+
+ async def googleGetFn(url: str) -> Dict[str, Any]: # type: ignore[no-redef]
+ return await _defaultGet(token, url)
+
+ mandateId = str(getattr(connection, "mandateId", "") or "") if connection is not None else ""
+ userId = str(getattr(connection, "userId", "") or "") if connection is not None else ""
+
+ for labelId in limits.labels:
+ if result.indexed + result.skippedDuplicate >= limits.maxMessages:
+ break
+ try:
+ await _ingestLabel(
+ googleGetFn=googleGetFn,
+ knowledgeService=knowledgeService,
+ connectionId=connectionId,
+ mandateId=mandateId,
+ userId=userId,
+ labelId=labelId,
+ limits=limits,
+ result=result,
+ progressCb=progressCb,
+ )
+ except Exception as exc:
+ logger.error("gmail ingestion label %s failed: %s", labelId, exc, exc_info=True)
+ result.errors.append(f"label({labelId}): {exc}")
+
+ return _finalizeResult(connectionId, result, startMs)
+
+
+async def _resolveDependencies(connectionId: str):
+ from modules.interfaces.interfaceDbApp import getRootInterface
+ from modules.auth import TokenManager
+ from modules.connectors.providerGoogle.connectorGoogle import GoogleConnector
+ from modules.serviceCenter import getService
+ from modules.serviceCenter.context import ServiceCenterContext
+ from modules.security.rootAccess import getRootUser
+
+ rootInterface = getRootInterface()
+ connection = rootInterface.getUserConnectionById(connectionId)
+ if connection is None:
+ raise ValueError(f"UserConnection not found: {connectionId}")
+
+ token = TokenManager().getFreshToken(connectionId)
+ if not token or not token.tokenAccess:
+ raise ValueError(f"No valid token for connection {connectionId}")
+
+ provider = GoogleConnector(connection, token.tokenAccess)
+ adapter = provider.getServiceAdapter("gmail")
+
+ rootUser = getRootUser()
+ ctx = ServiceCenterContext(
+ user=rootUser,
+ mandate_id=str(getattr(connection, "mandateId", "") or ""),
+ )
+ knowledgeService = getService("knowledge", ctx)
+ return adapter, connection, knowledgeService
+
+
+async def _ingestLabel(
+ *,
+ googleGetFn,
+ knowledgeService,
+ connectionId: str,
+ mandateId: str,
+ userId: str,
+ labelId: str,
+ limits: GmailBootstrapLimits,
+ result: GmailBootstrapResult,
+ progressCb: Optional[Callable[[int, Optional[str]], None]],
+) -> None:
+ remaining = limits.maxMessages - (result.indexed + result.skippedDuplicate)
+ if remaining <= 0:
+ return
+
+ pageSize = min(100, remaining)
+ query = ""
+ if limits.maxAgeDays:
+ cutoff = datetime.now(timezone.utc) - timedelta(days=limits.maxAgeDays)
+ # Gmail uses YYYY/MM/DD.
+ query = f"after:{cutoff.strftime('%Y/%m/%d')}"
+
+ baseUrl = (
+ "https://gmail.googleapis.com/gmail/v1/users/me/messages"
+ f"?labelIds={labelId}&maxResults={pageSize}"
+ )
+ if query:
+ baseUrl = f"{baseUrl}&q={query}"
+
+ nextPageToken: Optional[str] = None
+ while (result.indexed + result.skippedDuplicate) < limits.maxMessages:
+ url = baseUrl if not nextPageToken else f"{baseUrl}&pageToken={nextPageToken}"
+ page = await googleGetFn(url)
+ if not isinstance(page, dict) or "error" in page:
+ err = (page or {}).get("error") if isinstance(page, dict) else "unknown"
+ logger.warning("gmail list page error for label %s: %s", labelId, err)
+ result.errors.append(f"list({labelId}): {err}")
+ return
+
+ messageStubs = page.get("messages") or []
+ for stub in messageStubs:
+ if result.indexed + result.skippedDuplicate >= limits.maxMessages:
+ break
+ msgId = stub.get("id")
+ if not msgId:
+ continue
+ detailUrl = (
+ f"https://gmail.googleapis.com/gmail/v1/users/me/messages/{msgId}?format=full"
+ )
+ detail = await googleGetFn(detailUrl)
+ if not isinstance(detail, dict) or "error" in detail:
+ result.failed += 1
+ continue
+ await _ingestMessage(
+ googleGetFn=googleGetFn,
+ knowledgeService=knowledgeService,
+ connectionId=connectionId,
+ mandateId=mandateId,
+ userId=userId,
+ labelId=labelId,
+ message=detail,
+ limits=limits,
+ result=result,
+ progressCb=progressCb,
+ )
+
+ nextPageToken = page.get("nextPageToken")
+ if not nextPageToken:
+ break
+
+
+async def _ingestMessage(
+ *,
+ googleGetFn,
+ knowledgeService,
+ connectionId: str,
+ mandateId: str,
+ userId: str,
+ labelId: str,
+ message: Dict[str, Any],
+ limits: GmailBootstrapLimits,
+ result: GmailBootstrapResult,
+ progressCb: Optional[Callable[[int, Optional[str]], None]],
+) -> None:
+ from modules.serviceCenter.services.serviceKnowledge.mainServiceKnowledge import IngestionJob
+
+ messageId = message.get("id")
+ if not messageId:
+ result.skippedPolicy += 1
+ return
+ revision = message.get("historyId") or message.get("internalDate")
+ headers = _headerMap(message.get("payload") or {})
+ subject = headers.get("subject") or "(no subject)"
+ syntheticId = _syntheticMessageId(connectionId, messageId)
+ fileName = f"{subject[:80].strip()}.eml" if subject else f"{messageId}.eml"
+
+ contentObjects = _buildContentObjects(
+ message, limits.maxBodyChars, mailContentDepth=limits.mailContentDepth
+ )
+ try:
+ handle = await knowledgeService.requestIngestion(
+ IngestionJob(
+ sourceKind="gmail_message",
+ sourceId=syntheticId,
+ fileName=fileName,
+ mimeType="message/rfc822",
+ userId=userId,
+ mandateId=mandateId,
+ contentObjects=contentObjects,
+ contentVersion=str(revision) if revision else None,
+ neutralize=limits.neutralize,
+ provenance={
+ "connectionId": connectionId,
+ "authority": "google",
+ "service": "gmail",
+ "externalItemId": messageId,
+ "label": labelId,
+ "threadId": message.get("threadId"),
+ "tier": limits.mailContentDepth,
+ },
+ )
+ )
+ except Exception as exc:
+ logger.error("gmail ingestion %s failed: %s", messageId, exc, exc_info=True)
+ result.failed += 1
+ result.errors.append(f"ingest({messageId}): {exc}")
+ return
+
+ if handle.status == "duplicate":
+ result.skippedDuplicate += 1
+ elif handle.status == "indexed":
+ result.indexed += 1
+ else:
+ result.failed += 1
+
+ if limits.includeAttachments:
+ try:
+ await _ingestAttachments(
+ googleGetFn=googleGetFn,
+ knowledgeService=knowledgeService,
+ connectionId=connectionId,
+ mandateId=mandateId,
+ userId=userId,
+ message=message,
+ parentSyntheticId=syntheticId,
+ limits=limits,
+ result=result,
+ )
+ except Exception as exc:
+ logger.warning("gmail attachments %s failed: %s", messageId, exc)
+ result.errors.append(f"attachments({messageId}): {exc}")
+
+ if progressCb is not None and (result.indexed + result.skippedDuplicate) % 50 == 0:
+ processed = result.indexed + result.skippedDuplicate
+ try:
+ progressCb(
+ min(90, 10 + int(80 * processed / max(1, limits.maxMessages))),
+ f"gmail processed={processed}",
+ )
+ except Exception:
+ pass
+ logger.info(
+ "ingestion.connection.bootstrap.progress part=gmail processed=%d skippedDup=%d failed=%d",
+ processed, result.skippedDuplicate, result.failed,
+ extra={
+ "event": "ingestion.connection.bootstrap.progress",
+ "part": "gmail",
+ "connectionId": connectionId,
+ "processed": processed,
+ "skippedDup": result.skippedDuplicate,
+ "failed": result.failed,
+ },
+ )
+
+ await asyncio.sleep(0)
+
+
+async def _ingestAttachments(
+ *,
+ googleGetFn,
+ knowledgeService,
+ connectionId: str,
+ mandateId: str,
+ userId: str,
+ message: Dict[str, Any],
+ parentSyntheticId: str,
+ limits: GmailBootstrapLimits,
+ result: GmailBootstrapResult,
+) -> None:
+ """Child ingestion jobs for file attachments. Skips inline images (cid: refs)."""
+ from modules.serviceCenter.services.serviceKnowledge.mainServiceKnowledge import IngestionJob
+ from modules.datamodels.datamodelExtraction import ExtractionOptions
+ from modules.serviceCenter.services.serviceExtraction.subPipeline import runExtraction
+ from modules.serviceCenter.services.serviceExtraction.subRegistry import (
+ ExtractorRegistry, ChunkerRegistry,
+ )
+
+ messageId = message.get("id") or ""
+
+ def _collectAttachmentStubs(part: Dict[str, Any], acc: List[Dict[str, Any]]) -> None:
+ filename = part.get("filename") or ""
+ body = part.get("body") or {}
+ attId = body.get("attachmentId")
+ if filename and attId:
+ acc.append({
+ "filename": filename,
+ "mimeType": part.get("mimeType") or "application/octet-stream",
+ "attachmentId": attId,
+ "size": int(body.get("size") or 0),
+ })
+ for sub in part.get("parts") or []:
+ _collectAttachmentStubs(sub, acc)
+
+ stubs: List[Dict[str, Any]] = []
+ _collectAttachmentStubs(message.get("payload") or {}, stubs)
+ if not stubs:
+ return
+
+ extractorRegistry = ExtractorRegistry()
+ chunkerRegistry = ChunkerRegistry()
+
+ for stub in stubs:
+ if stub["size"] and stub["size"] > limits.maxAttachmentBytes:
+ result.skippedPolicy += 1
+ continue
+ attUrl = (
+ f"https://gmail.googleapis.com/gmail/v1/users/me/messages/{messageId}"
+ f"/attachments/{stub['attachmentId']}"
+ )
+ detail = await googleGetFn(attUrl)
+ if not isinstance(detail, dict) or "error" in detail:
+ result.failed += 1
+ continue
+ rawBytes = _decodeBase64Url(detail.get("data") or "")
+ if not rawBytes:
+ continue
+ fileName = stub["filename"]
+ mimeType = stub["mimeType"]
+ syntheticId = _syntheticAttachmentId(connectionId, messageId, stub["attachmentId"])
+
+ try:
+ extracted = runExtraction(
+ extractorRegistry, chunkerRegistry,
+ rawBytes, fileName, mimeType,
+ ExtractionOptions(mergeStrategy=None),
+ )
+ except Exception as exc:
+ logger.warning("gmail attachment extract %s failed: %s", stub["attachmentId"], exc)
+ result.failed += 1
+ continue
+
+ contentObjects: List[Dict[str, Any]] = []
+ for part in getattr(extracted, "parts", None) or []:
+ data = getattr(part, "data", None) or ""
+ if not data or not str(data).strip():
+ continue
+ typeGroup = getattr(part, "typeGroup", "text") or "text"
+ contentType = "text"
+ if typeGroup == "image":
+ contentType = "image"
+ elif typeGroup in ("binary", "container"):
+ contentType = "other"
+ contentObjects.append({
+ "contentObjectId": getattr(part, "id", ""),
+ "contentType": contentType,
+ "data": data,
+ "contextRef": {
+ "containerPath": fileName,
+ "location": getattr(part, "label", None) or "attachment",
+ **(getattr(part, "metadata", None) or {}),
+ },
+ })
+ if not contentObjects:
+ result.skippedPolicy += 1
+ continue
+
+ try:
+ await knowledgeService.requestIngestion(
+ IngestionJob(
+ sourceKind="gmail_attachment",
+ sourceId=syntheticId,
+ fileName=fileName,
+ mimeType=mimeType,
+ userId=userId,
+ mandateId=mandateId,
+ contentObjects=contentObjects,
+ provenance={
+ "connectionId": connectionId,
+ "authority": "google",
+ "service": "gmail",
+ "parentId": parentSyntheticId,
+ "externalItemId": stub["attachmentId"],
+ "parentMessageId": messageId,
+ },
+ )
+ )
+ result.attachmentsIndexed += 1
+ except Exception as exc:
+ logger.warning("gmail attachment ingest %s failed: %s", stub["attachmentId"], exc)
+ result.failed += 1
+
+
+def _finalizeResult(connectionId: str, result: GmailBootstrapResult, startMs: float) -> Dict[str, Any]:
+ durationMs = int((time.time() - startMs) * 1000)
+ logger.info(
+ "ingestion.connection.bootstrap.done part=gmail connectionId=%s indexed=%d skippedDup=%d skippedPolicy=%d attachments=%d failed=%d durationMs=%d",
+ connectionId,
+ result.indexed, result.skippedDuplicate, result.skippedPolicy,
+ result.attachmentsIndexed, result.failed, durationMs,
+ extra={
+ "event": "ingestion.connection.bootstrap.done",
+ "part": "gmail",
+ "connectionId": connectionId,
+ "indexed": result.indexed,
+ "skippedDup": result.skippedDuplicate,
+ "skippedPolicy": result.skippedPolicy,
+ "attachmentsIndexed": result.attachmentsIndexed,
+ "failed": result.failed,
+ "durationMs": durationMs,
+ },
+ )
+ return {
+ "connectionId": result.connectionId,
+ "indexed": result.indexed,
+ "skippedDuplicate": result.skippedDuplicate,
+ "skippedPolicy": result.skippedPolicy,
+ "attachmentsIndexed": result.attachmentsIndexed,
+ "failed": result.failed,
+ "durationMs": durationMs,
+ "errors": result.errors[:20],
+ }
diff --git a/modules/serviceCenter/services/serviceKnowledge/subConnectorSyncOutlook.py b/modules/serviceCenter/services/serviceKnowledge/subConnectorSyncOutlook.py
new file mode 100644
index 00000000..64a3545f
--- /dev/null
+++ b/modules/serviceCenter/services/serviceKnowledge/subConnectorSyncOutlook.py
@@ -0,0 +1,576 @@
+# Copyright (c) 2025 Patrick Motsch
+# All rights reserved.
+"""Outlook bootstrap for the unified knowledge ingestion lane.
+
+Unlike SharePoint, Outlook messages are "virtual documents" — we never persist
+file bytes in the store. Each message becomes a `sourceKind="outlook_message"`
+IngestionJob whose `contentObjects` carry the header, snippet and cleaned body
+so retrieval can show a compact answer without fetching Graph again.
+
+Attachments are optional (`includeAttachments` limit flag) and enqueued as
+child jobs with `sourceKind="outlook_attachment"` + `provenance.parentId`.
+"""
+
+from __future__ import annotations
+
+import asyncio
+import hashlib
+import logging
+import time
+from dataclasses import dataclass, field
+from typing import Any, Callable, Dict, List, Optional
+
+from modules.serviceCenter.services.serviceKnowledge.subTextClean import cleanEmailBody
+
+logger = logging.getLogger(__name__)
+
+MAX_MESSAGES_DEFAULT = 500
+MAX_FOLDERS_DEFAULT = 5
+MAX_BODY_CHARS_DEFAULT = 8000
+MAX_ATTACHMENT_BYTES_DEFAULT = 10 * 1024 * 1024
+WELL_KNOWN_FOLDERS = ("inbox", "sentitems")
+
+
+@dataclass
+class OutlookBootstrapLimits:
+ maxMessages: int = MAX_MESSAGES_DEFAULT
+ maxFolders: int = MAX_FOLDERS_DEFAULT
+ maxBodyChars: int = MAX_BODY_CHARS_DEFAULT
+ includeAttachments: bool = False
+ maxAttachmentBytes: int = MAX_ATTACHMENT_BYTES_DEFAULT
+ # Only fetch messages newer than N days. None disables filter.
+ maxAgeDays: Optional[int] = 90
+ # Content depth: "metadata" | "snippet" | "full"
+ mailContentDepth: str = "full"
+ # Pass-through to IngestionJob.neutralize
+ neutralize: bool = False
+
+
+@dataclass
+class OutlookBootstrapResult:
+ connectionId: str
+ indexed: int = 0
+ skippedDuplicate: int = 0
+ skippedPolicy: int = 0
+ failed: int = 0
+ attachmentsIndexed: int = 0
+ errors: List[str] = field(default_factory=list)
+
+
+def _syntheticMessageId(connectionId: str, messageId: str) -> str:
+ token = hashlib.sha256(f"{connectionId}:{messageId}".encode("utf-8")).hexdigest()[:16]
+ return f"om:{connectionId[:8]}:{token}"
+
+
+def _syntheticAttachmentId(connectionId: str, messageId: str, attachmentId: str) -> str:
+ token = hashlib.sha256(
+ f"{connectionId}:{messageId}:{attachmentId}".encode("utf-8")
+ ).hexdigest()[:16]
+ return f"oa:{connectionId[:8]}:{token}"
+
+
+def _extractRecipient(recipient: Dict[str, Any]) -> str:
+ email = (recipient or {}).get("emailAddress") or {}
+ name = email.get("name") or ""
+ addr = email.get("address") or ""
+ if name and addr:
+ return f"{name} <{addr}>"
+ return addr or name
+
+
+def _joinRecipients(recipients: List[Dict[str, Any]]) -> str:
+ return ", ".join(filter(None, [_extractRecipient(r) for r in recipients or []]))
+
+
+def _buildContentObjects(
+ message: Dict[str, Any],
+ maxBodyChars: int,
+ mailContentDepth: str = "full",
+) -> List[Dict[str, Any]]:
+ """Build content objects for an Outlook message.
+
+ `mailContentDepth` mirrors the Gmail walker:
+ - "metadata": header only
+ - "snippet": header + bodyPreview (~255 chars)
+ - "full": header + snippet + cleaned body (default)
+ """
+ subject = message.get("subject") or "(no subject)"
+ fromAddr = _extractRecipient(message.get("from") or {})
+ toAddr = _joinRecipients(message.get("toRecipients") or [])
+ ccAddr = _joinRecipients(message.get("ccRecipients") or [])
+ received = message.get("receivedDateTime") or ""
+ snippet = message.get("bodyPreview") or ""
+
+ parts: List[Dict[str, Any]] = []
+ header = (
+ f"Subject: {subject}\n"
+ f"From: {fromAddr}\n"
+ f"To: {toAddr}\n"
+ + (f"Cc: {ccAddr}\n" if ccAddr else "")
+ + f"Date: {received}"
+ )
+ parts.append({
+ "contentObjectId": "header",
+ "contentType": "text",
+ "data": header,
+ "contextRef": {"part": "header"},
+ })
+ if mailContentDepth in ("snippet", "full") and snippet:
+ parts.append({
+ "contentObjectId": "snippet",
+ "contentType": "text",
+ "data": snippet,
+ "contextRef": {"part": "snippet"},
+ })
+ if mailContentDepth == "full":
+ body = message.get("body") or {}
+ bodyContent = body.get("content") or ""
+ cleanedBody = cleanEmailBody(bodyContent, maxChars=maxBodyChars) if bodyContent else ""
+ if cleanedBody:
+ parts.append({
+ "contentObjectId": "body",
+ "contentType": "text",
+ "data": cleanedBody,
+ "contextRef": {"part": "body"},
+ })
+ return parts
+
+
+async def bootstrapOutlook(
+ connectionId: str,
+ *,
+ progressCb: Optional[Callable[[int, Optional[str]], None]] = None,
+ adapter: Any = None,
+ connection: Any = None,
+ knowledgeService: Any = None,
+ limits: Optional[OutlookBootstrapLimits] = None,
+) -> Dict[str, Any]:
+ """Enumerate Outlook folders (inbox + sent by default) and ingest messages."""
+ from modules.serviceCenter.services.serviceKnowledge.subConnectorPrefs import loadConnectionPrefs
+ prefs = loadConnectionPrefs(connectionId)
+
+ if not limits:
+ limits = OutlookBootstrapLimits(
+ includeAttachments=prefs.mailIndexAttachments,
+ maxAgeDays=prefs.maxAgeDays if prefs.maxAgeDays > 0 else None,
+ mailContentDepth=prefs.mailContentDepth,
+ neutralize=prefs.neutralizeBeforeEmbed,
+ )
+
+ startMs = time.time()
+ result = OutlookBootstrapResult(connectionId=connectionId)
+
+ logger.info(
+ "ingestion.connection.bootstrap.started part=outlook connectionId=%s",
+ connectionId,
+ extra={
+ "event": "ingestion.connection.bootstrap.started",
+ "part": "outlook",
+ "connectionId": connectionId,
+ },
+ )
+
+ if adapter is None or knowledgeService is None or connection is None:
+ adapter, connection, knowledgeService = await _resolveDependencies(connectionId)
+
+ mandateId = str(getattr(connection, "mandateId", "") or "") if connection is not None else ""
+ userId = str(getattr(connection, "userId", "") or "") if connection is not None else ""
+
+ folderIds = await _selectFolderIds(adapter, limits)
+ for folderId in folderIds:
+ if result.indexed + result.skippedDuplicate >= limits.maxMessages:
+ break
+ try:
+ await _ingestFolder(
+ adapter=adapter,
+ knowledgeService=knowledgeService,
+ connectionId=connectionId,
+ mandateId=mandateId,
+ userId=userId,
+ folderId=folderId,
+ limits=limits,
+ result=result,
+ progressCb=progressCb,
+ )
+ except Exception as exc:
+ logger.error("outlook ingestion folder %s failed: %s", folderId, exc, exc_info=True)
+ result.errors.append(f"folder({folderId}): {exc}")
+
+ return _finalizeResult(connectionId, result, startMs)
+
+
+async def _resolveDependencies(connectionId: str):
+ from modules.interfaces.interfaceDbApp import getRootInterface
+ from modules.auth import TokenManager
+ from modules.connectors.providerMsft.connectorMsft import MsftConnector
+ from modules.serviceCenter import getService
+ from modules.serviceCenter.context import ServiceCenterContext
+ from modules.security.rootAccess import getRootUser
+
+ rootInterface = getRootInterface()
+ connection = rootInterface.getUserConnectionById(connectionId)
+ if connection is None:
+ raise ValueError(f"UserConnection not found: {connectionId}")
+
+ token = TokenManager().getFreshToken(connectionId)
+ if not token or not token.tokenAccess:
+ raise ValueError(f"No valid token for connection {connectionId}")
+
+ provider = MsftConnector(connection, token.tokenAccess)
+ adapter = provider.getServiceAdapter("outlook")
+
+ rootUser = getRootUser()
+ ctx = ServiceCenterContext(
+ user=rootUser,
+ mandate_id=str(getattr(connection, "mandateId", "") or ""),
+ )
+ knowledgeService = getService("knowledge", ctx)
+ return adapter, connection, knowledgeService
+
+
+async def _selectFolderIds(adapter, limits: OutlookBootstrapLimits) -> List[str]:
+ """Prefer well-known folders (inbox, sentitems); fall back to browse()."""
+ folderIds: List[str] = []
+ for wellKnown in WELL_KNOWN_FOLDERS:
+ if len(folderIds) >= limits.maxFolders:
+ break
+ try:
+ row = await adapter._graphGet(f"me/mailFolders/{wellKnown}")
+ except Exception:
+ row = None
+ if isinstance(row, dict) and "error" not in row and row.get("id"):
+ folderIds.append(row["id"])
+
+ if len(folderIds) < limits.maxFolders:
+ try:
+ entries = await adapter.browse("/")
+ except Exception:
+ entries = []
+ for entry in entries:
+ metadata = getattr(entry, "metadata", {}) or {}
+ fid = metadata.get("id")
+ if fid and fid not in folderIds:
+ folderIds.append(fid)
+ if len(folderIds) >= limits.maxFolders:
+ break
+ return folderIds
+
+
+async def _ingestFolder(
+ *,
+ adapter,
+ knowledgeService,
+ connectionId: str,
+ mandateId: str,
+ userId: str,
+ folderId: str,
+ limits: OutlookBootstrapLimits,
+ result: OutlookBootstrapResult,
+ progressCb: Optional[Callable[[int, Optional[str]], None]],
+) -> None:
+ remaining = limits.maxMessages - (result.indexed + result.skippedDuplicate)
+ if remaining <= 0:
+ return
+
+ pageSize = min(100, remaining)
+ select = (
+ "id,subject,from,toRecipients,ccRecipients,receivedDateTime,"
+ "bodyPreview,body,internetMessageId,hasAttachments,changeKey"
+ )
+ endpoint: Optional[str] = (
+ f"me/mailFolders/{folderId}/messages"
+ f"?$top={pageSize}&$orderby=receivedDateTime desc&$select={select}"
+ )
+
+ # Keep header-based age filter in Graph itself to avoid shipping ancient
+ # messages we'd discard client-side.
+ if limits.maxAgeDays:
+ from datetime import datetime, timezone, timedelta
+
+ cutoff = datetime.now(timezone.utc) - timedelta(days=limits.maxAgeDays)
+ cutoffIso = cutoff.strftime("%Y-%m-%dT%H:%M:%SZ")
+ endpoint = f"{endpoint}&$filter=receivedDateTime ge {cutoffIso}"
+
+ while endpoint and (result.indexed + result.skippedDuplicate) < limits.maxMessages:
+ try:
+ page = await adapter._graphGet(endpoint)
+ except Exception as exc:
+ logger.warning("outlook graph page failed for folder %s: %s", folderId, exc)
+ result.errors.append(f"graph({folderId}): {exc}")
+ return
+ if not isinstance(page, dict) or "error" in page:
+ err = (page or {}).get("error") if isinstance(page, dict) else "unknown"
+ logger.warning("outlook graph page error for folder %s: %s", folderId, err)
+ result.errors.append(f"graph({folderId}): {err}")
+ return
+
+ for message in page.get("value", []) or []:
+ if result.indexed + result.skippedDuplicate >= limits.maxMessages:
+ break
+ await _ingestMessage(
+ adapter=adapter,
+ knowledgeService=knowledgeService,
+ connectionId=connectionId,
+ mandateId=mandateId,
+ userId=userId,
+ message=message,
+ limits=limits,
+ result=result,
+ progressCb=progressCb,
+ )
+
+ nextLink = page.get("@odata.nextLink")
+ if not nextLink:
+ break
+ # Strip Graph base so adapter._graphGet accepts the relative path.
+ from modules.connectors.providerMsft.connectorMsft import _stripGraphBase
+
+ endpoint = _stripGraphBase(nextLink)
+
+
+async def _ingestMessage(
+ *,
+ adapter,
+ knowledgeService,
+ connectionId: str,
+ mandateId: str,
+ userId: str,
+ message: Dict[str, Any],
+ limits: OutlookBootstrapLimits,
+ result: OutlookBootstrapResult,
+ progressCb: Optional[Callable[[int, Optional[str]], None]],
+) -> None:
+ from modules.serviceCenter.services.serviceKnowledge.mainServiceKnowledge import IngestionJob
+
+ messageId = message.get("id")
+ if not messageId:
+ result.skippedPolicy += 1
+ return
+ revision = message.get("changeKey") or message.get("internetMessageId")
+ subject = message.get("subject") or "(no subject)"
+ syntheticId = _syntheticMessageId(connectionId, messageId)
+ fileName = f"{subject[:80].strip()}.eml" if subject else f"{messageId}.eml"
+
+ contentObjects = _buildContentObjects(
+ message, limits.maxBodyChars, mailContentDepth=limits.mailContentDepth
+ )
+ # Always at least the header is emitted, so `contentObjects` is non-empty.
+ try:
+ handle = await knowledgeService.requestIngestion(
+ IngestionJob(
+ sourceKind="outlook_message",
+ sourceId=syntheticId,
+ fileName=fileName,
+ mimeType="message/rfc822",
+ userId=userId,
+ mandateId=mandateId,
+ contentObjects=contentObjects,
+ contentVersion=revision,
+ neutralize=limits.neutralize,
+ provenance={
+ "connectionId": connectionId,
+ "authority": "msft",
+ "service": "outlook",
+ "externalItemId": messageId,
+ "internetMessageId": message.get("internetMessageId"),
+ "tier": limits.mailContentDepth,
+ },
+ )
+ )
+ except Exception as exc:
+ logger.error("outlook ingestion %s failed: %s", messageId, exc, exc_info=True)
+ result.failed += 1
+ result.errors.append(f"ingest({messageId}): {exc}")
+ return
+
+ if handle.status == "duplicate":
+ result.skippedDuplicate += 1
+ elif handle.status == "indexed":
+ result.indexed += 1
+ else:
+ result.failed += 1
+
+ if limits.includeAttachments and message.get("hasAttachments"):
+ try:
+ await _ingestAttachments(
+ adapter=adapter,
+ knowledgeService=knowledgeService,
+ connectionId=connectionId,
+ mandateId=mandateId,
+ userId=userId,
+ messageId=messageId,
+ parentSyntheticId=syntheticId,
+ limits=limits,
+ result=result,
+ )
+ except Exception as exc:
+ logger.warning("outlook attachments %s failed: %s", messageId, exc)
+ result.errors.append(f"attachments({messageId}): {exc}")
+
+ if progressCb is not None and (result.indexed + result.skippedDuplicate) % 50 == 0:
+ processed = result.indexed + result.skippedDuplicate
+ try:
+ progressCb(
+ min(90, 10 + int(80 * processed / max(1, limits.maxMessages))),
+ f"outlook processed={processed}",
+ )
+ except Exception:
+ pass
+ logger.info(
+ "ingestion.connection.bootstrap.progress part=outlook processed=%d skippedDup=%d failed=%d",
+ processed, result.skippedDuplicate, result.failed,
+ extra={
+ "event": "ingestion.connection.bootstrap.progress",
+ "part": "outlook",
+ "connectionId": connectionId,
+ "processed": processed,
+ "skippedDup": result.skippedDuplicate,
+ "failed": result.failed,
+ },
+ )
+
+ await asyncio.sleep(0)
+
+
+async def _ingestAttachments(
+ *,
+ adapter,
+ knowledgeService,
+ connectionId: str,
+ mandateId: str,
+ userId: str,
+ messageId: str,
+ parentSyntheticId: str,
+ limits: OutlookBootstrapLimits,
+ result: OutlookBootstrapResult,
+) -> None:
+ """Child ingestion jobs for file attachments (skip inline & oversized)."""
+ from modules.serviceCenter.services.serviceKnowledge.mainServiceKnowledge import IngestionJob
+ from modules.datamodels.datamodelExtraction import ExtractionOptions
+ from modules.serviceCenter.services.serviceExtraction.subPipeline import runExtraction
+ from modules.serviceCenter.services.serviceExtraction.subRegistry import (
+ ExtractorRegistry, ChunkerRegistry,
+ )
+ import base64
+
+ page = await adapter._graphGet(f"me/messages/{messageId}/attachments")
+ if not isinstance(page, dict) or "error" in page:
+ return
+
+ extractorRegistry = ExtractorRegistry()
+ chunkerRegistry = ChunkerRegistry()
+
+ for attachment in page.get("value", []) or []:
+ if attachment.get("@odata.type") != "#microsoft.graph.fileAttachment":
+ continue
+ if attachment.get("isInline"):
+ continue
+ size = int(attachment.get("size") or 0)
+ if size and size > limits.maxAttachmentBytes:
+ result.skippedPolicy += 1
+ continue
+ contentBytesB64 = attachment.get("contentBytes")
+ if not contentBytesB64:
+ continue
+ try:
+ rawBytes = base64.b64decode(contentBytesB64)
+ except Exception:
+ result.skippedPolicy += 1
+ continue
+ fileName = attachment.get("name") or "attachment"
+ mimeType = attachment.get("contentType") or "application/octet-stream"
+ attachmentId = attachment.get("id") or fileName
+ syntheticId = _syntheticAttachmentId(connectionId, messageId, attachmentId)
+
+ try:
+ extracted = runExtraction(
+ extractorRegistry, chunkerRegistry,
+ rawBytes, fileName, mimeType,
+ ExtractionOptions(mergeStrategy=None),
+ )
+ except Exception as exc:
+ logger.warning("outlook attachment extract %s failed: %s", attachmentId, exc)
+ result.failed += 1
+ continue
+
+ contentObjects: List[Dict[str, Any]] = []
+ for part in getattr(extracted, "parts", None) or []:
+ data = getattr(part, "data", None) or ""
+ if not data or not str(data).strip():
+ continue
+ typeGroup = getattr(part, "typeGroup", "text") or "text"
+ contentType = "text"
+ if typeGroup == "image":
+ contentType = "image"
+ elif typeGroup in ("binary", "container"):
+ contentType = "other"
+ contentObjects.append({
+ "contentObjectId": getattr(part, "id", ""),
+ "contentType": contentType,
+ "data": data,
+ "contextRef": {
+ "containerPath": fileName,
+ "location": getattr(part, "label", None) or "attachment",
+ **(getattr(part, "metadata", None) or {}),
+ },
+ })
+ if not contentObjects:
+ result.skippedPolicy += 1
+ continue
+
+ try:
+ await knowledgeService.requestIngestion(
+ IngestionJob(
+ sourceKind="outlook_attachment",
+ sourceId=syntheticId,
+ fileName=fileName,
+ mimeType=mimeType,
+ userId=userId,
+ mandateId=mandateId,
+ contentObjects=contentObjects,
+ neutralize=limits.neutralize,
+ provenance={
+ "connectionId": connectionId,
+ "authority": "msft",
+ "service": "outlook",
+ "parentId": parentSyntheticId,
+ "externalItemId": attachmentId,
+ "parentMessageId": messageId,
+ },
+ )
+ )
+ result.attachmentsIndexed += 1
+ except Exception as exc:
+ logger.warning("outlook attachment ingest %s failed: %s", attachmentId, exc)
+ result.failed += 1
+
+
+def _finalizeResult(connectionId: str, result: OutlookBootstrapResult, startMs: float) -> Dict[str, Any]:
+ durationMs = int((time.time() - startMs) * 1000)
+ logger.info(
+ "ingestion.connection.bootstrap.done part=outlook connectionId=%s indexed=%d skippedDup=%d skippedPolicy=%d attachments=%d failed=%d durationMs=%d",
+ connectionId,
+ result.indexed, result.skippedDuplicate, result.skippedPolicy,
+ result.attachmentsIndexed, result.failed, durationMs,
+ extra={
+ "event": "ingestion.connection.bootstrap.done",
+ "part": "outlook",
+ "connectionId": connectionId,
+ "indexed": result.indexed,
+ "skippedDup": result.skippedDuplicate,
+ "skippedPolicy": result.skippedPolicy,
+ "attachmentsIndexed": result.attachmentsIndexed,
+ "failed": result.failed,
+ "durationMs": durationMs,
+ },
+ )
+ return {
+ "connectionId": result.connectionId,
+ "indexed": result.indexed,
+ "skippedDuplicate": result.skippedDuplicate,
+ "skippedPolicy": result.skippedPolicy,
+ "attachmentsIndexed": result.attachmentsIndexed,
+ "failed": result.failed,
+ "durationMs": durationMs,
+ "errors": result.errors[:20],
+ }
diff --git a/modules/serviceCenter/services/serviceKnowledge/subConnectorSyncSharepoint.py b/modules/serviceCenter/services/serviceKnowledge/subConnectorSyncSharepoint.py
new file mode 100644
index 00000000..07fef7a8
--- /dev/null
+++ b/modules/serviceCenter/services/serviceKnowledge/subConnectorSyncSharepoint.py
@@ -0,0 +1,433 @@
+# Copyright (c) 2025 Patrick Motsch
+# All rights reserved.
+"""SharePoint bootstrap for the unified knowledge ingestion lane.
+
+Walks the SharePoint drive(s) reachable via a UserConnection, downloads each
+file-like item, runs the standard content extraction pipeline and hands the
+result to `KnowledgeService.requestIngestion`. Idempotency is provided by the
+ingestion façade itself; repeat bootstraps therefore produce
+`ingestion.skipped.duplicate` for every unchanged item because we pass the
+Graph `eTag` as `contentVersion`.
+"""
+
+from __future__ import annotations
+
+import asyncio
+import hashlib
+import logging
+import time
+from dataclasses import dataclass, field
+from typing import Any, Callable, Dict, List, Optional
+
+from modules.datamodels.datamodelExtraction import ExtractionOptions
+
+logger = logging.getLogger(__name__)
+
+MAX_ITEMS_DEFAULT = 500
+MAX_BYTES_DEFAULT = 200 * 1024 * 1024
+MAX_FILE_SIZE_DEFAULT = 25 * 1024 * 1024
+SKIP_MIME_PREFIXES_DEFAULT = ("video/", "audio/")
+MAX_DEPTH_DEFAULT = 4
+MAX_SITES_DEFAULT = 3
+
+
+@dataclass
+class SharepointBootstrapLimits:
+ maxItems: int = MAX_ITEMS_DEFAULT
+ maxBytes: int = MAX_BYTES_DEFAULT
+ maxFileSize: int = MAX_FILE_SIZE_DEFAULT
+ skipMimePrefixes: tuple = SKIP_MIME_PREFIXES_DEFAULT
+ maxDepth: int = MAX_DEPTH_DEFAULT
+ maxSites: int = MAX_SITES_DEFAULT
+ # Pass-through to IngestionJob.neutralize
+ neutralize: bool = False
+
+
+@dataclass
+class SharepointBootstrapResult:
+ connectionId: str
+ indexed: int = 0
+ skippedDuplicate: int = 0
+ skippedPolicy: int = 0
+ failed: int = 0
+ bytesProcessed: int = 0
+ errors: List[str] = field(default_factory=list)
+
+
+def _syntheticFileId(connectionId: str, externalItemId: str) -> str:
+ """Deterministic synthetic FileContentIndex id for a SharePoint item.
+
+ Stable across bootstraps → idempotency works; independent of file name so
+ moves/renames don't duplicate chunks.
+ """
+ token = hashlib.sha256(f"{connectionId}:{externalItemId}".encode("utf-8")).hexdigest()[:16]
+ return f"sp:{connectionId[:8]}:{token}"
+
+
+def _toContentObjects(extracted, fileName: str) -> List[Dict[str, Any]]:
+ """Translate ExtractionResult → content objects accepted by requestIngestion."""
+ parts = getattr(extracted, "parts", None) or []
+ out: List[Dict[str, Any]] = []
+ for part in parts:
+ data = getattr(part, "data", None) or ""
+ if not data or not str(data).strip():
+ continue
+ typeGroup = getattr(part, "typeGroup", "text") or "text"
+ contentType = "text"
+ if typeGroup == "image":
+ contentType = "image"
+ elif typeGroup in ("binary", "container"):
+ contentType = "other"
+ out.append({
+ "contentObjectId": getattr(part, "id", ""),
+ "contentType": contentType,
+ "data": data,
+ "contextRef": {
+ "containerPath": fileName,
+ "location": getattr(part, "label", None) or "file",
+ **(getattr(part, "metadata", None) or {}),
+ },
+ })
+ return out
+
+
+async def bootstrapSharepoint(
+ connectionId: str,
+ *,
+ progressCb: Optional[Callable[[int, Optional[str]], None]] = None,
+ adapter: Any = None,
+ connection: Any = None,
+ knowledgeService: Any = None,
+ limits: Optional[SharepointBootstrapLimits] = None,
+ runExtractionFn: Optional[Callable[..., Any]] = None,
+) -> Dict[str, Any]:
+ """Enumerate SharePoint drives and ingest every reachable file via the façade.
+
+ Parameters allow injection for tests; production callers pass only
+ `connectionId` (and optionally a progressCb) and everything else is
+ resolved against the registered services.
+ """
+ from modules.serviceCenter.services.serviceKnowledge.subConnectorPrefs import loadConnectionPrefs
+ prefs = loadConnectionPrefs(connectionId)
+
+ if not limits:
+ limits = SharepointBootstrapLimits(neutralize=prefs.neutralizeBeforeEmbed)
+
+ startMs = time.time()
+ result = SharepointBootstrapResult(connectionId=connectionId)
+
+ logger.info(
+ "ingestion.connection.bootstrap.started part=sharepoint connectionId=%s",
+ connectionId,
+ extra={
+ "event": "ingestion.connection.bootstrap.started",
+ "part": "sharepoint",
+ "connectionId": connectionId,
+ },
+ )
+
+ if adapter is None or knowledgeService is None or connection is None:
+ adapter, connection, knowledgeService = await _resolveDependencies(connectionId)
+ if runExtractionFn is None:
+ from modules.serviceCenter.services.serviceExtraction.subPipeline import runExtraction
+ from modules.serviceCenter.services.serviceExtraction.subRegistry import (
+ ExtractorRegistry, ChunkerRegistry,
+ )
+ extractorRegistry = ExtractorRegistry()
+ chunkerRegistry = ChunkerRegistry()
+
+ def runExtractionFn(bytesData, name, mime, options): # type: ignore[no-redef]
+ return runExtraction(extractorRegistry, chunkerRegistry, bytesData, name, mime, options)
+
+ mandateId = str(getattr(connection, "mandateId", "") or "") if connection is not None else ""
+ userId = str(getattr(connection, "userId", "") or "") if connection is not None else ""
+
+ try:
+ sites = await adapter.browse("/", limit=limits.maxSites)
+ except Exception as exc:
+ logger.error("sharepoint site discovery failed for %s: %s", connectionId, exc, exc_info=True)
+ result.errors.append(f"site_discovery: {exc}")
+ return _finalizeResult(connectionId, result, startMs)
+
+ for site in sites[: limits.maxSites]:
+ if result.indexed + result.skippedDuplicate >= limits.maxItems:
+ break
+ sitePath = getattr(site, "path", "") or ""
+ try:
+ await _walkFolder(
+ adapter=adapter,
+ knowledgeService=knowledgeService,
+ runExtractionFn=runExtractionFn,
+ connectionId=connectionId,
+ mandateId=mandateId,
+ userId=userId,
+ folderPath=sitePath,
+ depth=0,
+ limits=limits,
+ result=result,
+ progressCb=progressCb,
+ )
+ except Exception as exc:
+ logger.error("sharepoint walk failed for site %s: %s", sitePath, exc, exc_info=True)
+ result.errors.append(f"walk({sitePath}): {exc}")
+
+ return _finalizeResult(connectionId, result, startMs)
+
+
+async def _resolveDependencies(connectionId: str):
+ """Load connection, instantiate SharepointAdapter, and build a KnowledgeService.
+
+ Runs with root privileges: bootstrap is a system operation triggered by an
+ authenticated user via callback; it must not be gated by a per-user
+ service-center context.
+ """
+ from modules.interfaces.interfaceDbApp import getRootInterface
+ from modules.auth import TokenManager
+ from modules.connectors.providerMsft.connectorMsft import MsftConnector
+ from modules.serviceCenter import getService
+ from modules.serviceCenter.context import ServiceCenterContext
+ from modules.security.rootAccess import getRootUser
+
+ rootInterface = getRootInterface()
+ connection = rootInterface.getUserConnectionById(connectionId)
+ if connection is None:
+ raise ValueError(f"UserConnection not found: {connectionId}")
+
+ token = TokenManager().getFreshToken(connectionId)
+ if not token or not token.tokenAccess:
+ raise ValueError(f"No valid token for connection {connectionId}")
+
+ provider = MsftConnector(connection, token.tokenAccess)
+ adapter = provider.getServiceAdapter("sharepoint")
+
+ rootUser = getRootUser()
+ ctx = ServiceCenterContext(
+ user=rootUser,
+ mandate_id=str(getattr(connection, "mandateId", "") or ""),
+ )
+ knowledgeService = getService("knowledge", ctx)
+ return adapter, connection, knowledgeService
+
+
+async def _walkFolder(
+ *,
+ adapter,
+ knowledgeService,
+ runExtractionFn,
+ connectionId: str,
+ mandateId: str,
+ userId: str,
+ folderPath: str,
+ depth: int,
+ limits: SharepointBootstrapLimits,
+ result: SharepointBootstrapResult,
+ progressCb: Optional[Callable[[int, Optional[str]], None]],
+) -> None:
+ if depth > limits.maxDepth:
+ return
+ try:
+ entries = await adapter.browse(folderPath)
+ except Exception as exc:
+ logger.warning("sharepoint browse %s failed: %s", folderPath, exc)
+ result.errors.append(f"browse({folderPath}): {exc}")
+ return
+
+ for entry in entries:
+ if result.indexed + result.skippedDuplicate >= limits.maxItems:
+ return
+ if result.bytesProcessed >= limits.maxBytes:
+ return
+
+ entryPath = getattr(entry, "path", "") or ""
+ if getattr(entry, "isFolder", False):
+ await _walkFolder(
+ adapter=adapter,
+ knowledgeService=knowledgeService,
+ runExtractionFn=runExtractionFn,
+ connectionId=connectionId,
+ mandateId=mandateId,
+ userId=userId,
+ folderPath=entryPath,
+ depth=depth + 1,
+ limits=limits,
+ result=result,
+ progressCb=progressCb,
+ )
+ continue
+
+ mimeType = getattr(entry, "mimeType", None) or "application/octet-stream"
+ if any(mimeType.startswith(prefix) for prefix in limits.skipMimePrefixes):
+ result.skippedPolicy += 1
+ continue
+ size = int(getattr(entry, "size", 0) or 0)
+ if size and size > limits.maxFileSize:
+ result.skippedPolicy += 1
+ continue
+
+ metadata = getattr(entry, "metadata", {}) or {}
+ externalItemId = metadata.get("id") or entryPath
+ revision = metadata.get("revision") or metadata.get("lastModifiedDateTime")
+
+ await _ingestOne(
+ adapter=adapter,
+ knowledgeService=knowledgeService,
+ runExtractionFn=runExtractionFn,
+ connectionId=connectionId,
+ mandateId=mandateId,
+ userId=userId,
+ entry=entry,
+ entryPath=entryPath,
+ mimeType=mimeType,
+ externalItemId=externalItemId,
+ revision=revision,
+ limits=limits,
+ result=result,
+ progressCb=progressCb,
+ )
+
+
+async def _ingestOne(
+ *,
+ adapter,
+ knowledgeService,
+ runExtractionFn,
+ connectionId: str,
+ mandateId: str,
+ userId: str,
+ entry,
+ entryPath: str,
+ mimeType: str,
+ externalItemId: str,
+ revision: Optional[str],
+ limits: SharepointBootstrapLimits,
+ result: SharepointBootstrapResult,
+ progressCb: Optional[Callable[[int, Optional[str]], None]],
+) -> None:
+ from modules.serviceCenter.services.serviceKnowledge.mainServiceKnowledge import IngestionJob
+
+ syntheticFileId = _syntheticFileId(connectionId, externalItemId)
+ fileName = getattr(entry, "name", "") or externalItemId
+
+ try:
+ fileBytes = await adapter.download(entryPath)
+ except Exception as exc:
+ logger.warning("sharepoint download %s failed: %s", entryPath, exc)
+ result.failed += 1
+ result.errors.append(f"download({entryPath}): {exc}")
+ return
+ if not fileBytes:
+ result.failed += 1
+ return
+
+ result.bytesProcessed += len(fileBytes)
+
+ try:
+ extracted = runExtractionFn(
+ fileBytes, fileName, mimeType,
+ ExtractionOptions(mergeStrategy=None),
+ )
+ except Exception as exc:
+ logger.warning("sharepoint extraction %s failed: %s", entryPath, exc)
+ result.failed += 1
+ result.errors.append(f"extract({entryPath}): {exc}")
+ return
+
+ contentObjects = _toContentObjects(extracted, fileName)
+ if not contentObjects:
+ result.skippedPolicy += 1
+ return
+
+ provenance: Dict[str, Any] = {
+ "connectionId": connectionId,
+ "authority": "msft",
+ "service": "sharepoint",
+ "externalItemId": externalItemId,
+ "externalPath": entryPath,
+ "revision": revision,
+ }
+ try:
+ handle = await knowledgeService.requestIngestion(
+ IngestionJob(
+ sourceKind="sharepoint_item",
+ sourceId=syntheticFileId,
+ fileName=fileName,
+ mimeType=mimeType,
+ userId=userId,
+ mandateId=mandateId,
+ contentObjects=contentObjects,
+ contentVersion=revision,
+ neutralize=limits.neutralize,
+ provenance=provenance,
+ )
+ )
+ except Exception as exc:
+ logger.error("sharepoint ingestion %s failed: %s", entryPath, exc, exc_info=True)
+ result.failed += 1
+ result.errors.append(f"ingest({entryPath}): {exc}")
+ return
+
+ if handle.status == "duplicate":
+ result.skippedDuplicate += 1
+ elif handle.status == "indexed":
+ result.indexed += 1
+ else:
+ result.failed += 1
+ if handle.error:
+ result.errors.append(f"ingest({entryPath}): {handle.error}")
+
+ if progressCb is not None and (result.indexed + result.skippedDuplicate) % 50 == 0:
+ processed = result.indexed + result.skippedDuplicate
+ try:
+ progressCb(
+ min(90, 10 + int(80 * processed / max(1, limits.maxItems))),
+ f"sharepoint processed={processed}",
+ )
+ except Exception:
+ pass
+ logger.info(
+ "ingestion.connection.bootstrap.progress part=sharepoint processed=%d skippedDup=%d failed=%d",
+ processed, result.skippedDuplicate, result.failed,
+ extra={
+ "event": "ingestion.connection.bootstrap.progress",
+ "part": "sharepoint",
+ "connectionId": connectionId,
+ "processed": processed,
+ "skippedDup": result.skippedDuplicate,
+ "failed": result.failed,
+ },
+ )
+
+ # Yield so the event loop can interleave other tasks (download/extract are
+ # CPU-ish and extraction uses sync libs; cooperative scheduling prevents
+ # starving other workers).
+ await asyncio.sleep(0)
+
+
+def _finalizeResult(connectionId: str, result: SharepointBootstrapResult, startMs: float) -> Dict[str, Any]:
+ durationMs = int((time.time() - startMs) * 1000)
+ logger.info(
+ "ingestion.connection.bootstrap.done part=sharepoint connectionId=%s indexed=%d skippedDup=%d skippedPolicy=%d failed=%d durationMs=%d",
+ connectionId,
+ result.indexed, result.skippedDuplicate, result.skippedPolicy, result.failed,
+ durationMs,
+ extra={
+ "event": "ingestion.connection.bootstrap.done",
+ "part": "sharepoint",
+ "connectionId": connectionId,
+ "indexed": result.indexed,
+ "skippedDup": result.skippedDuplicate,
+ "skippedPolicy": result.skippedPolicy,
+ "failed": result.failed,
+ "durationMs": durationMs,
+ },
+ )
+ return {
+ "connectionId": result.connectionId,
+ "indexed": result.indexed,
+ "skippedDuplicate": result.skippedDuplicate,
+ "skippedPolicy": result.skippedPolicy,
+ "failed": result.failed,
+ "bytesProcessed": result.bytesProcessed,
+ "durationMs": durationMs,
+ "errors": result.errors[:20],
+ }
diff --git a/modules/serviceCenter/services/serviceKnowledge/subTextClean.py b/modules/serviceCenter/services/serviceKnowledge/subTextClean.py
new file mode 100644
index 00000000..2d352cfa
--- /dev/null
+++ b/modules/serviceCenter/services/serviceKnowledge/subTextClean.py
@@ -0,0 +1,107 @@
+# Copyright (c) 2025 Patrick Motsch
+# All rights reserved.
+"""Text normalisation utilities used by knowledge ingestion.
+
+The email body cleaning logic is intentionally regex-based and works on plain
+text after an HTML→text pass so we never store unsanitised HTML/JS in the
+knowledge store and retrieval stays robust (no extraneous markup tokens
+eating embedding budget).
+"""
+
+from __future__ import annotations
+
+import re
+from typing import Optional
+
+DEFAULT_MAX_CHARS = 8000
+
+
+_QUOTE_MARKER_PATTERNS = [
+ re.compile(r"^\s*(?:On\s.+?\swrote:)\s*$", re.MULTILINE | re.IGNORECASE),
+ re.compile(r"^\s*(?:Am\s.+?\sschrieb.+?:)\s*$", re.MULTILINE | re.IGNORECASE),
+ re.compile(r"^\s*-{2,}\s*Original\s*Message\s*-{2,}\s*$", re.MULTILINE | re.IGNORECASE),
+ re.compile(r"^\s*-{2,}\s*Urspr.+Nachricht\s*-{2,}\s*$", re.MULTILINE | re.IGNORECASE),
+ re.compile(r"^\s*From:\s+.+$", re.MULTILINE | re.IGNORECASE),
+ re.compile(r"^\s*Von:\s+.+$", re.MULTILINE | re.IGNORECASE),
+ re.compile(r"^\s*Sent:\s+.+$", re.MULTILINE | re.IGNORECASE),
+ re.compile(r"^\s*Gesendet:\s+.+$", re.MULTILINE | re.IGNORECASE),
+]
+
+_SIGNATURE_MARKERS = [
+ re.compile(r"^\s*-{2,}\s*$", re.MULTILINE),
+ re.compile(r"^\s*—\s*$", re.MULTILINE),
+ re.compile(r"^\s*Best regards\b.*$", re.MULTILINE | re.IGNORECASE),
+ re.compile(r"^\s*Kind regards\b.*$", re.MULTILINE | re.IGNORECASE),
+ re.compile(r"^\s*Mit freundlichen Gr[üu]ßen\b.*$", re.MULTILINE | re.IGNORECASE),
+ re.compile(r"^\s*Viele Gr[üu]ße\b.*$", re.MULTILINE | re.IGNORECASE),
+ re.compile(r"^\s*Best,\s*$", re.MULTILINE | re.IGNORECASE),
+]
+
+
+def _htmlToText(html: str) -> str:
+ """Prefer BeautifulSoup when available, fall back to regex."""
+ try:
+ from bs4 import BeautifulSoup # type: ignore
+
+ soup = BeautifulSoup(html, "html.parser")
+ for tag in soup(["script", "style", "head"]):
+ tag.decompose()
+ for br in soup.find_all(["br"]):
+ br.replace_with("\n")
+ for p in soup.find_all(["p", "div", "li", "tr"]):
+ p.append("\n")
+ text = soup.get_text()
+ except Exception:
+ # Minimal fallback: strip tags crudely.
+ text = re.sub(r"", "\n", html, flags=re.IGNORECASE)
+ text = re.sub(r"", "\n", text, flags=re.IGNORECASE)
+ text = re.sub(r"<[^>]+>", "", text)
+ # Collapse non-breaking + zero-width whitespace.
+ text = text.replace("\u00a0", " ").replace("\u200b", "")
+ return text
+
+
+def _stripQuotedThread(text: str) -> str:
+ """Remove reply-chain content so only the author's own contribution remains."""
+ earliest = len(text)
+ for pattern in _QUOTE_MARKER_PATTERNS:
+ match = pattern.search(text)
+ if match and match.start() < earliest:
+ earliest = match.start()
+ # Drop any block starting with "> " quoted lines (often Gmail/Thunderbird).
+ quotedBlock = re.search(r"^(?:\s*>.*\n?)+", text, re.MULTILINE)
+ if quotedBlock and quotedBlock.start() < earliest:
+ earliest = quotedBlock.start()
+ return text[:earliest].rstrip()
+
+
+def _stripSignature(text: str) -> str:
+ earliest = len(text)
+ for pattern in _SIGNATURE_MARKERS:
+ match = pattern.search(text)
+ if match and match.start() < earliest:
+ earliest = match.start()
+ return text[:earliest].rstrip()
+
+
+def _collapseWhitespace(text: str) -> str:
+ text = re.sub(r"[ \t]+", " ", text)
+ text = re.sub(r"\n{3,}", "\n\n", text)
+ return text.strip()
+
+
+def cleanEmailBody(html: str, maxChars: Optional[int] = DEFAULT_MAX_CHARS) -> str:
+ """Return a compact plain-text view of an email body suitable for embedding.
+
+ Steps: HTML → text, remove quoted reply chain, remove signature, collapse
+ whitespace, truncate to maxChars. Always returns a string (possibly empty).
+ """
+ if not html:
+ return ""
+ text = _htmlToText(html) if "<" in html and ">" in html else html
+ text = _stripQuotedThread(text)
+ text = _stripSignature(text)
+ text = _collapseWhitespace(text)
+ if maxChars and len(text) > maxChars:
+ text = text[:maxChars].rstrip() + "…"
+ return text
diff --git a/modules/workflows/automation2/executionEngine.py b/modules/workflows/automation2/executionEngine.py
index 1d0ca5c8..55a63281 100644
--- a/modules/workflows/automation2/executionEngine.py
+++ b/modules/workflows/automation2/executionEngine.py
@@ -302,6 +302,30 @@ async def _executeWithRetry(executor, node, context, maxRetries: int = 0, retryD
raise lastError
+def _substituteFeatureInstancePlaceholders(
+ graph: Dict[str, Any],
+ targetFeatureInstanceId: str,
+) -> Dict[str, Any]:
+ """Replace ``{{featureInstanceId}}`` placeholders in the serialised graph.
+
+ Works on the full JSON representation so that placeholders inside nested
+ parameter dicts, prompt strings, etc. are all caught. Already-resolved
+ concrete UUIDs (pre-baked by ``_copyTemplateWorkflows``) are left untouched
+ because the placeholder literal ``{{featureInstanceId}}`` will not match.
+ """
+ import json as _json
+ raw = _json.dumps(graph)
+ if "{{featureInstanceId}}" not in raw:
+ return graph
+ replaced = raw.replace("{{featureInstanceId}}", targetFeatureInstanceId)
+ logger.debug(
+ "_substituteFeatureInstancePlaceholders: resolved %d occurrence(s) -> %s",
+ raw.count("{{featureInstanceId}}"),
+ targetFeatureInstanceId,
+ )
+ return _json.loads(replaced)
+
+
async def executeGraph(
graph: Dict[str, Any],
services: Any,
@@ -315,6 +339,7 @@ async def executeGraph(
runId: Optional[str] = None,
run_envelope: Optional[Dict[str, Any]] = None,
label: Optional[str] = None,
+ targetFeatureInstanceId: Optional[str] = None,
) -> Dict[str, Any]:
"""
Execute automation2 graph. Returns { success, nodeOutputs, error?, stopped? }.
@@ -322,14 +347,16 @@ async def executeGraph(
pauses the run, and returns { success: False, paused: True, taskId, runId }.
For resume: pass initialNodeOutputs (with result for the human node) and startAfterNodeId.
For fresh runs: pass run_envelope (unified start payload for the start node); normalized with userId into context.runEnvelope.
+ targetFeatureInstanceId: resolves {{featureInstanceId}} placeholders in the graph JSON before execution.
"""
logger.info(
- "executeGraph start: instanceId=%s workflowId=%s userId=%s mandateId=%s resume=%s",
+ "executeGraph start: instanceId=%s workflowId=%s userId=%s mandateId=%s resume=%s targetInstance=%s",
instanceId,
workflowId,
userId,
mandateId,
startAfterNodeId is not None,
+ targetFeatureInstanceId,
)
from modules.workflows.processing.shared.methodDiscovery import discoverMethods
discoverMethods(services)
@@ -338,6 +365,9 @@ async def executeGraph(
materializeFeatureInstanceRefs,
)
+ if targetFeatureInstanceId:
+ graph = _substituteFeatureInstancePlaceholders(graph, targetFeatureInstanceId)
+
# Phase-5 Schicht-4: typed-ref envelopes are materialized FIRST so the
# subsequent connection-ref pass and validation see the canonical shape.
graph = materializeFeatureInstanceRefs(graph)
diff --git a/modules/workflows/automation2/executors/actionNodeExecutor.py b/modules/workflows/automation2/executors/actionNodeExecutor.py
index 6162aa2d..163ed3b2 100644
--- a/modules/workflows/automation2/executors/actionNodeExecutor.py
+++ b/modules/workflows/automation2/executors/actionNodeExecutor.py
@@ -377,7 +377,11 @@ class ActionNodeExecutor:
if nodeType.startswith("ai."):
out["prompt"] = promptText
out["response"] = extractedContext
- out["context"] = f"{promptText}\n\n{extractedContext}" if promptText and extractedContext else (extractedContext or promptText)
+ inputContext = resolvedParams.get("context")
+ if inputContext is not None:
+ out["context"] = inputContext if isinstance(inputContext, str) else json.dumps(inputContext, ensure_ascii=False, default=str)
+ else:
+ out["context"] = ""
# Structured output
if extractedContext:
try:
diff --git a/modules/workflows/methods/methodAi/_common.py b/modules/workflows/methods/methodAi/_common.py
new file mode 100644
index 00000000..9e77d431
--- /dev/null
+++ b/modules/workflows/methods/methodAi/_common.py
@@ -0,0 +1,18 @@
+# Copyright (c) 2025 Patrick Motsch
+# All rights reserved.
+
+"""Shared helpers for AI workflow actions."""
+
+
+def applyCommonAiParams(parameters: dict, request) -> None:
+ """Apply common AI parameters (requireNeutralization, allowedModels) from node to request."""
+ requireNeutralization = parameters.get("requireNeutralization")
+ if requireNeutralization is not None:
+ request.requireNeutralization = bool(requireNeutralization)
+
+ allowedModels = parameters.get("allowedModels")
+ if allowedModels and isinstance(allowedModels, list):
+ if not request.options:
+ from modules.datamodels.datamodelAi import AiCallOptions
+ request.options = AiCallOptions()
+ request.options.allowedModels = allowedModels
diff --git a/modules/workflows/methods/methodAi/actions/consolidate.py b/modules/workflows/methods/methodAi/actions/consolidate.py
index fa622507..7483507e 100644
--- a/modules/workflows/methods/methodAi/actions/consolidate.py
+++ b/modules/workflows/methods/methodAi/actions/consolidate.py
@@ -67,6 +67,8 @@ async def consolidate(self, parameters: Dict[str, Any]) -> ActionResult:
prompt=prompt,
options=AiCallOptions(operationType=OperationTypeEnum.DATA_ANALYSE),
)
+ from modules.workflows.methods.methodAi._common import applyCommonAiParams
+ applyCommonAiParams(parameters, req)
resp = await ai_service.callAi(req)
except (SubscriptionInactiveException, BillingContextError):
raise
diff --git a/modules/workflows/methods/methodAi/actions/convertDocument.py b/modules/workflows/methods/methodAi/actions/convertDocument.py
index 39d6e16f..b2ed908b 100644
--- a/modules/workflows/methods/methodAi/actions/convertDocument.py
+++ b/modules/workflows/methods/methodAi/actions/convertDocument.py
@@ -36,6 +36,10 @@ async def convertDocument(self, parameters: Dict[str, Any]) -> ActionResult:
}
if parentOperationId:
processParams["parentOperationId"] = parentOperationId
+ if parameters.get("allowedModels"):
+ processParams["allowedModels"] = parameters["allowedModels"]
+ if parameters.get("requireNeutralization") is not None:
+ processParams["requireNeutralization"] = parameters["requireNeutralization"]
return await self.process(processParams)
diff --git a/modules/workflows/methods/methodAi/actions/generateCode.py b/modules/workflows/methods/methodAi/actions/generateCode.py
index 313057a0..5ec6b51d 100644
--- a/modules/workflows/methods/methodAi/actions/generateCode.py
+++ b/modules/workflows/methods/methodAi/actions/generateCode.py
@@ -55,6 +55,16 @@ async def generateCode(self, parameters: Dict[str, Any]) -> ActionResult:
processingMode=ProcessingModeEnum.DETAILED
)
+ # Apply node-level AI params
+ allowedModels = parameters.get("allowedModels")
+ if allowedModels and isinstance(allowedModels, list):
+ options.allowedModels = allowedModels
+ requireNeutralization = parameters.get("requireNeutralization")
+ if requireNeutralization is not None:
+ _ctx = getattr(self.services, '_context', None)
+ if _ctx:
+ _ctx.requireNeutralization = bool(requireNeutralization)
+
# outputFormat: Optional - if None, formats determined from prompt by AI
aiResponse: AiResponse = await self.services.ai.callAiContent(
prompt=prompt,
diff --git a/modules/workflows/methods/methodAi/actions/generateDocument.py b/modules/workflows/methods/methodAi/actions/generateDocument.py
index 0709b924..18c158c1 100644
--- a/modules/workflows/methods/methodAi/actions/generateDocument.py
+++ b/modules/workflows/methods/methodAi/actions/generateDocument.py
@@ -59,6 +59,16 @@ async def generateDocument(self, parameters: Dict[str, Any]) -> ActionResult:
compressContext=False
)
+ # Apply node-level AI params
+ allowedModels = parameters.get("allowedModels")
+ if allowedModels and isinstance(allowedModels, list):
+ options.allowedModels = allowedModels
+ requireNeutralization = parameters.get("requireNeutralization")
+ if requireNeutralization is not None:
+ _ctx = getattr(self.services, '_context', None)
+ if _ctx:
+ _ctx.requireNeutralization = bool(requireNeutralization)
+
# outputFormat: Optional - if None, formats determined from prompt by AI
aiResponse: AiResponse = await self.services.ai.callAiContent(
prompt=prompt,
diff --git a/modules/workflows/methods/methodAi/actions/process.py b/modules/workflows/methods/methodAi/actions/process.py
index 63e0f33e..2af480e7 100644
--- a/modules/workflows/methods/methodAi/actions/process.py
+++ b/modules/workflows/methods/methodAi/actions/process.py
@@ -73,6 +73,49 @@ def _action_docs_to_content_parts(services, docs: List[Any]) -> List[ContentPart
logger.info(f"ai.process: Extracted {len(ec.parts)} parts from {name} (no persistence)")
return all_parts
+def _resolve_file_refs_to_content_parts(services, fileIdRefs) -> List[ContentPart]:
+ """Fetch files by ID from the file store and extract content.
+ Used ONLY for automation2 workflows where documents are file-store
+ references, not chat message attachments. In the agent/chat context,
+ ``DocumentItemReference`` holds ChatDocument IDs that must be resolved
+ via ``getChatDocumentsFromDocumentList`` instead."""
+ from modules.datamodels.datamodelExtraction import ExtractionOptions, MergeStrategy
+
+ mgmt = getattr(services, 'interfaceDbComponent', None)
+ extraction = getattr(services, 'extraction', None)
+ if not mgmt or not extraction:
+ logger.warning("_resolve_file_refs_to_content_parts: missing interfaceDbComponent or extraction service")
+ return []
+
+ allParts: List[ContentPart] = []
+ opts = ExtractionOptions(prompt="", mergeStrategy=MergeStrategy())
+ for ref in fileIdRefs:
+ fileId = ref.documentId
+ fileMeta = mgmt.getFile(fileId)
+ if not fileMeta:
+ logger.warning(f"_resolve_file_refs_to_content_parts: file {fileId} not found")
+ continue
+ fileData = mgmt.getFileData(fileId)
+ if not fileData:
+ logger.warning(f"_resolve_file_refs_to_content_parts: no data for file {fileId}")
+ continue
+ fileName = getattr(fileMeta, 'fileName', fileId)
+ mimeType = getattr(fileMeta, 'mimeType', 'application/octet-stream')
+ ec = extraction.extractContentFromBytes(
+ documentBytes=fileData,
+ fileName=fileName,
+ mimeType=mimeType,
+ documentId=fileId,
+ options=opts,
+ )
+ for p in ec.parts:
+ if p.data or getattr(p, "typeGroup", "") == "image":
+ p.metadata.setdefault("originalFileName", fileName)
+ allParts.append(p)
+ logger.info(f"_resolve_file_refs_to_content_parts: extracted {len(ec.parts)} parts from {fileName}")
+ return allParts
+
+
async def process(self, parameters: Dict[str, Any]) -> ActionResult:
operationId = None
try:
@@ -129,6 +172,25 @@ async def process(self, parameters: Dict[str, Any]) -> ActionResult:
f"ai.process: Coerced documentList ({type(documentListParam).__name__}) "
f"to DocumentReferenceList with {len(documentList.references)} references"
)
+
+ # DocumentItemReferences carry either file-store IDs (automation2)
+ # or ChatDocument IDs (agent context with docItem: refs).
+ # Route based on context: if a chat workflow with messages exists,
+ # let getChatDocumentsFromDocumentList handle them (it resolves
+ # docItem:uuid via workflow.messages). Otherwise fall through to
+ # the file-store path for automation2.
+ from modules.datamodels.datamodelDocref import DocumentItemReference
+ fileIdRefs = [r for r in documentList.references if isinstance(r, DocumentItemReference)]
+ if fileIdRefs:
+ chatService = getattr(self.services, 'chat', None)
+ workflow = getattr(chatService, '_workflow', None) if chatService else None
+ hasChatContext = workflow and getattr(workflow, 'messages', None)
+ if not hasChatContext:
+ extractedParts = _resolve_file_refs_to_content_parts(self.services, fileIdRefs)
+ if extractedParts:
+ inline_content_parts = (inline_content_parts or []) + extractedParts
+ remaining = [r for r in documentList.references if not isinstance(r, DocumentItemReference)]
+ documentList = DocumentReferenceList(references=remaining)
# Optional: if omitted, formats determined from prompt. Default "txt" is validation fallback only.
resultType = parameters.get("resultType")
@@ -157,7 +219,19 @@ async def process(self, parameters: Dict[str, Any]) -> ActionResult:
mimeMap = {"txt": "text/plain", "json": "application/json", "html": "text/html", "md": "text/markdown", "csv": "text/csv", "xml": "application/xml"}
output_mime_type = mimeMap.get(normalized_result_type, "text/plain") if normalized_result_type else "text/plain"
-
+
+ # Normalize context: workflow refs may resolve to dict/list instead of str
+ paramContext = parameters.get("context")
+ if paramContext is not None and not isinstance(paramContext, str):
+ try:
+ paramContext = json.dumps(paramContext, ensure_ascii=False, default=str)
+ parameters["context"] = paramContext
+ logger.info(f"ai.process: Serialized non-string context ({type(parameters.get('context')).__name__}) to JSON ({len(paramContext)} chars)")
+ except Exception as e:
+ logger.warning(f"ai.process: Failed to serialize context: {e}")
+ paramContext = str(paramContext)
+ parameters["context"] = paramContext
+
# Phase 7.3: Pass documentList and/or contentParts to AI service
contentParts: Optional[List[ContentPart]] = inline_content_parts
if "contentParts" in parameters and not inline_content_parts:
@@ -212,6 +286,9 @@ async def process(self, parameters: Dict[str, Any]) -> ActionResult:
)
)
+ from modules.workflows.methods.methodAi._common import applyCommonAiParams
+ applyCommonAiParams(parameters, request)
+
aiResponse_obj = await self.services.ai.callAi(request)
# Convert AiCallResponse to AiResponse format
@@ -243,6 +320,16 @@ async def process(self, parameters: Dict[str, Any]) -> ActionResult:
operationType=OperationTypeEnum.IMAGE_GENERATE if isImageGeneration else OperationTypeEnum.DATA_GENERATE
)
+ # Apply node-level AI params (allowedModels, requireNeutralization)
+ allowedModels = parameters.get("allowedModels")
+ if allowedModels and isinstance(allowedModels, list):
+ options.allowedModels = allowedModels
+ requireNeutralization = parameters.get("requireNeutralization")
+ if requireNeutralization is not None:
+ _ctx = getattr(self.services, '_context', None)
+ if _ctx:
+ _ctx.requireNeutralization = bool(requireNeutralization)
+
# Get generationIntent from parameters (required for DATA_GENERATE)
# Default to "document" if not provided (most common use case)
# For code generation, use ai.generateCode action or explicitly pass generationIntent="code"
diff --git a/modules/workflows/methods/methodAi/actions/summarizeDocument.py b/modules/workflows/methods/methodAi/actions/summarizeDocument.py
index e32c1965..4c2bb2bc 100644
--- a/modules/workflows/methods/methodAi/actions/summarizeDocument.py
+++ b/modules/workflows/methods/methodAi/actions/summarizeDocument.py
@@ -39,6 +39,10 @@ async def summarizeDocument(self, parameters: Dict[str, Any]) -> ActionResult:
}
if parentOperationId:
processParams["parentOperationId"] = parentOperationId
+ if parameters.get("allowedModels"):
+ processParams["allowedModels"] = parameters["allowedModels"]
+ if parameters.get("requireNeutralization") is not None:
+ processParams["requireNeutralization"] = parameters["requireNeutralization"]
return await self.process(processParams)
diff --git a/modules/workflows/methods/methodAi/actions/translateDocument.py b/modules/workflows/methods/methodAi/actions/translateDocument.py
index bb6f8437..dc0533a9 100644
--- a/modules/workflows/methods/methodAi/actions/translateDocument.py
+++ b/modules/workflows/methods/methodAi/actions/translateDocument.py
@@ -41,6 +41,10 @@ async def translateDocument(self, parameters: Dict[str, Any]) -> ActionResult:
processParams["resultType"] = resultType
if parentOperationId:
processParams["parentOperationId"] = parentOperationId
+ if parameters.get("allowedModels"):
+ processParams["allowedModels"] = parameters["allowedModels"]
+ if parameters.get("requireNeutralization") is not None:
+ processParams["requireNeutralization"] = parameters["requireNeutralization"]
return await self.process(processParams)
diff --git a/modules/workflows/methods/methodAi/methodAi.py b/modules/workflows/methods/methodAi/methodAi.py
index 5265f5c9..ecd60b12 100644
--- a/modules/workflows/methods/methodAi/methodAi.py
+++ b/modules/workflows/methods/methodAi/methodAi.py
@@ -56,6 +56,23 @@ class MethodAi(MethodBase):
required=False,
description="Document reference(s) in any format to use as input/context"
),
+ "context": WorkflowActionParameter(
+ name="context",
+ type="str",
+ frontendType=FrontendType.TEXTAREA,
+ required=False,
+ default="",
+ description="Additional context data (string or upstream-bound dict/list, e.g. accounting data) appended to the prompt. Non-string values are JSON-serialized."
+ ),
+ "documentTheme": WorkflowActionParameter(
+ name="documentTheme",
+ type="str",
+ frontendType=FrontendType.SELECT,
+ frontendOptions=["general", "finance", "legal", "technical", "hr"],
+ required=False,
+ default="general",
+ description="Style hint for the document renderer (e.g. finance, legal). Used by the AI agent to choose colors and layout."
+ ),
"resultType": WorkflowActionParameter(
name="resultType",
type="str",
diff --git a/modules/workflows/methods/methodClickup/actions/list_tasks.py b/modules/workflows/methods/methodClickup/actions/list_tasks.py
index 4caf9e31..9ae57f94 100644
--- a/modules/workflows/methods/methodClickup/actions/list_tasks.py
+++ b/modules/workflows/methods/methodClickup/actions/list_tasks.py
@@ -31,8 +31,30 @@ async def list_tasks(self, parameters: Dict[str, Any]) -> ActionResult:
page = int(parameters.get("page") or 0)
include_closed = bool(parameters.get("includeClosed", False))
+
+ dateFilters = {}
+ for key in ("dateCreatedGt", "dateCreatedLt", "dateUpdatedGt", "dateUpdatedLt"):
+ val = parameters.get(key)
+ if val is not None and str(val).strip():
+ try:
+ dateFilters[key] = int(val)
+ except (ValueError, TypeError):
+ pass
+
+ rawCustomFields = parameters.get("customFields")
+ customFields = None
+ if rawCustomFields:
+ if isinstance(rawCustomFields, str):
+ try:
+ customFields = json.loads(rawCustomFields)
+ except json.JSONDecodeError:
+ return ActionResult.isFailure(error="customFields must be valid JSON array")
+ elif isinstance(rawCustomFields, list):
+ customFields = rawCustomFields
+
data = await self.services.clickup.getTasksInList(
- list_id, page=page, include_closed=include_closed, subtasks=True
+ list_id, page=page, include_closed=include_closed, subtasks=True,
+ **dateFilters, customFields=customFields,
)
if isinstance(data, dict) and data.get("error"):
return ActionResult.isFailure(error=str(data.get("error")) + (data.get("body") or ""))
diff --git a/modules/workflows/methods/methodClickup/methodClickup.py b/modules/workflows/methods/methodClickup/methodClickup.py
index 17f42300..725929dd 100644
--- a/modules/workflows/methods/methodClickup/methodClickup.py
+++ b/modules/workflows/methods/methodClickup/methodClickup.py
@@ -66,6 +66,41 @@ class MethodClickup(MethodBase):
default=False,
description="Include closed tasks",
),
+ "dateCreatedGt": WorkflowActionParameter(
+ name="dateCreatedGt",
+ type="int",
+ frontendType=FrontendType.NUMBER,
+ required=False,
+ description="Filter: created after this Unix ms timestamp",
+ ),
+ "dateCreatedLt": WorkflowActionParameter(
+ name="dateCreatedLt",
+ type="int",
+ frontendType=FrontendType.NUMBER,
+ required=False,
+ description="Filter: created before this Unix ms timestamp",
+ ),
+ "dateUpdatedGt": WorkflowActionParameter(
+ name="dateUpdatedGt",
+ type="int",
+ frontendType=FrontendType.NUMBER,
+ required=False,
+ description="Filter: updated after this Unix ms timestamp",
+ ),
+ "dateUpdatedLt": WorkflowActionParameter(
+ name="dateUpdatedLt",
+ type="int",
+ frontendType=FrontendType.NUMBER,
+ required=False,
+ description="Filter: updated before this Unix ms timestamp",
+ ),
+ "customFields": WorkflowActionParameter(
+ name="customFields",
+ type="str",
+ frontendType=FrontendType.TEXTAREA,
+ required=False,
+ description='JSON array of custom field filters per ClickUp API, e.g. [{"field_id":"abc","operator":"=","value":"123"}]',
+ ),
},
execute=list_tasks.__get__(self, self.__class__),
),
diff --git a/modules/workflows/scheduler/mainScheduler.py b/modules/workflows/scheduler/mainScheduler.py
index bf2cd0fd..0dce2ec5 100644
--- a/modules/workflows/scheduler/mainScheduler.py
+++ b/modules/workflows/scheduler/mainScheduler.py
@@ -243,6 +243,7 @@ class WorkflowScheduler:
runEnv = normalize_run_envelope(runEnv, user_id=str(eventUser.id) if eventUser else None)
_wfLabel = wf.get("label") if isinstance(wf, dict) else getattr(wf, "label", None)
+ _targetInstanceId = wf.get("targetFeatureInstanceId") if isinstance(wf, dict) else getattr(wf, "targetFeatureInstanceId", None)
result = await executeGraph(
graph=wf["graph"],
@@ -254,6 +255,7 @@ class WorkflowScheduler:
automation2_interface=iface,
run_envelope=runEnv,
label=_wfLabel,
+ targetFeatureInstanceId=_targetInstanceId,
)
logger.info(
"WorkflowScheduler: executed workflow %s success=%s paused=%s",
diff --git a/scripts/_archive/README.md b/scripts/_archive/README.md
new file mode 100644
index 00000000..dba3deef
--- /dev/null
+++ b/scripts/_archive/README.md
@@ -0,0 +1,19 @@
+# Archived one-shot scripts
+
+Diese Scripts haben einmal eine konkrete Daten- oder Code-Migration ausgefuehrt
+und werden nicht mehr aktiv aufgerufen. Sie bleiben hier liegen, falls jemand
+spaeter auf einem alten DB-Dump oder einem alten Branch nochmal denselben Stand
+herstellen muss.
+
+KEIN aktives Tool. Nicht aus CI, nicht aus Docs verlinken. Bei Aufraeumarbeiten
+(z.B. nach 6 Monaten ohne Anwendung) loeschen.
+
+## Inhalt
+
+| Datei | Migrationsthema | Archiviert am | Begruendung |
+|-------|-----------------|---------------|-------------|
+| `check_orphan_featureinstance.py` | Vor-Ort-Check mit hardcoded FeatureInstance-/Mandate-UUIDs | 2026-04-29 | Ad-hoc fuer einen konkreten Vorfall |
+| `script_db_cleanup_duplicate_roles.py` | Cleanup doppelter Roles wegen `IS NULL`-Bug in `connectorDbPostgre` | 2026-04-29 | Bug ist laengst gefixt, Cleanup ueberall durchgelaufen |
+| `migrate_async_to_sync.py` | One-shot Codemod `async def` -> `def` fuer FastAPI-Routes | 2026-04-29 | Refactor abgeschlossen |
+| `i18n_rekey_plaintext_keys.py` | Frontend `t('dot.notation')` -> `t('Klartext')` Rekey | 2026-04-29 | Frontend-Migration abgeschlossen (siehe `wiki/c-work/4-done/2026-04-ui-i18n-dynamic-language-sets.md`) |
+| `script_db_migrate_accessrules_objectkeys.py` | AccessRule-Items: kurz -> vollqualifiziert (Navigation-API) | 2026-04-29 | Navigation-API live, MIGRATION_MAP nur fuer trustee+realestate hardcoded |
diff --git a/scripts/check_orphan_featureinstance.py b/scripts/_archive/check_orphan_featureinstance.py
similarity index 100%
rename from scripts/check_orphan_featureinstance.py
rename to scripts/_archive/check_orphan_featureinstance.py
diff --git a/scripts/i18n_rekey_plaintext_keys.py b/scripts/_archive/i18n_rekey_plaintext_keys.py
similarity index 100%
rename from scripts/i18n_rekey_plaintext_keys.py
rename to scripts/_archive/i18n_rekey_plaintext_keys.py
diff --git a/scripts/migrate_async_to_sync.py b/scripts/_archive/migrate_async_to_sync.py
similarity index 100%
rename from scripts/migrate_async_to_sync.py
rename to scripts/_archive/migrate_async_to_sync.py
diff --git a/scripts/script_db_cleanup_duplicate_roles.py b/scripts/_archive/script_db_cleanup_duplicate_roles.py
similarity index 100%
rename from scripts/script_db_cleanup_duplicate_roles.py
rename to scripts/_archive/script_db_cleanup_duplicate_roles.py
diff --git a/scripts/script_db_migrate_accessrules_objectkeys.py b/scripts/_archive/script_db_migrate_accessrules_objectkeys.py
similarity index 100%
rename from scripts/script_db_migrate_accessrules_objectkeys.py
rename to scripts/_archive/script_db_migrate_accessrules_objectkeys.py
diff --git a/scripts/_listMandates.py b/scripts/_listMandates.py
deleted file mode 100644
index cf3e9bd2..00000000
--- a/scripts/_listMandates.py
+++ /dev/null
@@ -1,25 +0,0 @@
-import sys
-from pathlib import Path
-sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
-import psycopg2, psycopg2.extras
-from modules.shared.configuration import APP_CONFIG
-
-c = psycopg2.connect(
- host=APP_CONFIG.get('DB_HOST','localhost'),
- user=APP_CONFIG.get('DB_USER'),
- password=APP_CONFIG.get('DB_PASSWORD_SECRET'),
- port=int(APP_CONFIG.get('DB_PORT',5432)),
- dbname='poweron_app',
-)
-cur = c.cursor(cursor_factory=psycopg2.extras.RealDictCursor)
-cur.execute('SELECT id, name, label, enabled, "deletedAt", "sysCreatedAt" FROM "Mandate" ORDER BY "sysCreatedAt"')
-print("All Mandates in poweron_app:")
-for r in cur.fetchall():
- print(f" id={r['id']} name={r['name']} label={r['label']} enabled={r['enabled']} deletedAt={r['deletedAt']}")
-
-cur.execute('SELECT COUNT(*) AS n FROM "FeatureInstance" WHERE "featureCode" = %s', ("redmine",))
-print(f"\nTotal redmine FeatureInstances in poweron_app: {cur.fetchone()['n']}")
-
-cur.execute('SELECT id, "mandateId", label, enabled FROM "FeatureInstance" WHERE "featureCode" = %s ORDER BY "sysCreatedAt"', ("redmine",))
-for r in cur.fetchall():
- print(f" fi={r['id']} mandate={r['mandateId']} label={r['label']} enabled={r['enabled']}")
diff --git a/scripts/script_db_audit_legacy_state.py b/scripts/script_db_audit_legacy_state.py
new file mode 100644
index 00000000..f51a132a
--- /dev/null
+++ b/scripts/script_db_audit_legacy_state.py
@@ -0,0 +1,382 @@
+#!/usr/bin/env python3
+"""Audit-Skript fuer Legacy-Bestaende vor Bootstrap-Cleanup (Plan C).
+
+Prueft fuer jede der 5 Bootstrap-Migrationsroutinen, ob noch Restbestand
+existiert. Wenn alle Checks 0 / GREEN liefern, kann die jeweilige Routine
+sicher aus ``interfaceBootstrap.py`` / ``interfaceDbKnowledge.py`` entfernt
+werden.
+
+Checks:
+ 1. Mandate.description != NULL und Mandate.label leer
+ -> _migrateMandateDescriptionToLabel
+ 2. Mandate.label leer ODER Mandate.name verstoesst gegen Slug-Regeln
+ -> _migrateMandateNameLabelSlugRules
+ 3. Mandate mit name='Root' und isSystem=False
+ -> initRootMandate Legacy-Zweig
+ 4. Role mit roleLabel='sysadmin' im Root-Mandat
+ -> _migrateAndDropSysAdminRole
+ 5. FileContentIndex mit leerem mandateId UND leerem featureInstanceId
+ -> aggregateMandateRagTotalBytes Fallback-Block
+
+Verwendung:
+ python -m scripts.script_db_audit_legacy_state # text-output
+ python -m scripts.script_db_audit_legacy_state --json # JSON-output
+ python -m scripts.script_db_audit_legacy_state --purge-rag-orphans
+ # loescht FileContentIndex-Rows ohne mandateId UND ohne featureInstanceId
+ # (Voraussetzung fuer Removal des aggregateMandateRagTotalBytes-Fallback)
+
+Exit-Code:
+ 0 alle Checks GREEN (Removal sicher)
+ 1 mind. ein Check RED (erst Daten bereinigen)
+ 2 Skript-Fehler (DB nicht erreichbar etc.)
+
+Lese-Zugriffe sind die Default. Schreibzugriffe NUR mit explizitem
+``--purge-*``-Flag.
+"""
+
+from __future__ import annotations
+
+import argparse
+import json
+import logging
+import os
+import sys
+from dataclasses import dataclass, field
+from typing import Any, Callable, Dict, List, Optional
+
+
+_gatewayDir = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
+if _gatewayDir not in sys.path:
+ sys.path.insert(0, _gatewayDir)
+
+from dotenv import load_dotenv
+
+_envPath = os.path.join(_gatewayDir, "env_dev.env")
+if os.path.exists(_envPath):
+ load_dotenv(_envPath)
+
+from modules.datamodels.datamodelUam import Mandate
+from modules.datamodels.datamodelRbac import Role
+from modules.datamodels.datamodelKnowledge import FileContentIndex
+from modules.security.rootAccess import getRootDbAppConnector
+from modules.interfaces.interfaceDbKnowledge import KnowledgeObjects
+from modules.shared.mandateNameUtils import isValidMandateName
+
+logging.basicConfig(level=logging.WARNING, format="%(message)s")
+logger = logging.getLogger(__name__)
+
+
+@dataclass
+class _CheckResult:
+ """Ergebnis eines einzelnen Audit-Checks."""
+
+ name: str
+ routine: str
+ location: str
+ count: int
+ status: str
+ samples: List[Dict[str, Any]] = field(default_factory=list)
+ error: Optional[str] = None
+
+ def toDict(self) -> Dict[str, Any]:
+ return {
+ "name": self.name,
+ "routine": self.routine,
+ "location": self.location,
+ "count": self.count,
+ "status": self.status,
+ "samples": self.samples,
+ "error": self.error,
+ }
+
+
+def _getAppDb():
+ return getRootDbAppConnector()
+
+
+def _getKnowledgeDb():
+ return KnowledgeObjects().db
+
+
+def _checkMandateDescription(db) -> _CheckResult:
+ """Mandate.description noch vorhanden und label leer?"""
+ rows = db.getRecordset(Mandate)
+ legacy = [
+ {
+ "id": r.get("id"),
+ "name": r.get("name"),
+ "description": str(r.get("description"))[:60] if r.get("description") else None,
+ "label": r.get("label"),
+ }
+ for r in rows
+ if r.get("description") and not r.get("label")
+ ]
+ return _CheckResult(
+ name="mandate-description-to-label",
+ routine="_migrateMandateDescriptionToLabel",
+ location="interfaces/interfaceBootstrap.py:422-445",
+ count=len(legacy),
+ status="GREEN" if not legacy else "RED",
+ samples=legacy[:5],
+ )
+
+
+def _checkMandateSlugRules(db) -> _CheckResult:
+ """Mandate.name verletzt Slug-Regeln ODER Mandate.label leer?"""
+ rows = db.getRecordset(Mandate)
+ legacy = []
+ seen: set[str] = set()
+ for r in sorted(rows, key=lambda x: str(x.get("id", ""))):
+ name = (r.get("name") or "").strip()
+ labelRaw = r.get("label")
+ labelEmpty = not (labelRaw or "").strip() if labelRaw is not None else True
+ nameInvalid = not isValidMandateName(name)
+ nameCollides = name in seen
+ if not nameInvalid and not nameCollides:
+ seen.add(name)
+ if labelEmpty or nameInvalid or nameCollides:
+ legacy.append(
+ {
+ "id": r.get("id"),
+ "name": name,
+ "label": r.get("label"),
+ "labelEmpty": labelEmpty,
+ "nameInvalid": nameInvalid,
+ "nameCollides": nameCollides,
+ }
+ )
+ return _CheckResult(
+ name="mandate-name-slug-rules",
+ routine="_migrateMandateNameLabelSlugRules",
+ location="interfaces/interfaceBootstrap.py:448-511",
+ count=len(legacy),
+ status="GREEN" if not legacy else "RED",
+ samples=legacy[:5],
+ )
+
+
+def _checkRootMandateLegacy(db) -> _CheckResult:
+ """Mandate mit name='Root' (case-sensitive) ODER isSystem=False fuer root?"""
+ legacyByName = db.getRecordset(Mandate, recordFilter={"name": "Root"})
+ rows = db.getRecordset(Mandate, recordFilter={"name": "root"})
+ legacyByFlag = [r for r in rows if not r.get("isSystem")]
+ combined = list(legacyByName) + legacyByFlag
+ samples = [
+ {
+ "id": r.get("id"),
+ "name": r.get("name"),
+ "isSystem": r.get("isSystem"),
+ }
+ for r in combined
+ ]
+ return _CheckResult(
+ name="root-mandate-legacy",
+ routine="initRootMandate-legacy-branch",
+ location="interfaces/interfaceBootstrap.py:406-412",
+ count=len(samples),
+ status="GREEN" if not samples else "RED",
+ samples=samples[:5],
+ )
+
+
+def _checkSysadminRole(db) -> _CheckResult:
+ """Legacy 'sysadmin'-Rolle im Root-Mandat?"""
+ rootMandates = db.getRecordset(Mandate, recordFilter={"name": "root", "isSystem": True})
+ if not rootMandates:
+ return _CheckResult(
+ name="sysadmin-role",
+ routine="_migrateAndDropSysAdminRole",
+ location="interfaces/interfaceBootstrap.py:840-932",
+ count=0,
+ status="GREEN",
+ samples=[],
+ error="kein Root-Mandat gefunden -- Check uebersprungen (kann nicht relevant sein)",
+ )
+ rootId = str(rootMandates[0].get("id"))
+ rows = db.getRecordset(
+ Role,
+ recordFilter={"roleLabel": "sysadmin", "mandateId": rootId, "featureInstanceId": None},
+ )
+ samples = [{"id": r.get("id"), "roleLabel": r.get("roleLabel")} for r in rows]
+ return _CheckResult(
+ name="sysadmin-role",
+ routine="_migrateAndDropSysAdminRole",
+ location="interfaces/interfaceBootstrap.py:840-932",
+ count=len(samples),
+ status="GREEN" if not samples else "RED",
+ samples=samples[:5],
+ )
+
+
+def _checkRagFallback(knowDb) -> _CheckResult:
+ """FileContentIndex-Rows ohne mandateId UND ohne featureInstanceId?"""
+ rows = knowDb.getRecordset(FileContentIndex)
+ legacy = [
+ {
+ "id": r.get("id"),
+ "fileName": r.get("fileName"),
+ "totalSize": r.get("totalSize"),
+ }
+ for r in rows
+ if not (r.get("mandateId") or "").strip() and not (r.get("featureInstanceId") or "").strip()
+ ]
+ return _CheckResult(
+ name="rag-fallback-orphan-index",
+ routine="aggregateMandateRagTotalBytes-fallback",
+ location="interfaces/interfaceDbKnowledge.py:609-635",
+ count=len(legacy),
+ status="GREEN" if not legacy else "RED",
+ samples=legacy[:5],
+ )
+
+
+def _runChecks() -> List[_CheckResult]:
+ appDb = _getAppDb()
+ knowDb = _getKnowledgeDb()
+
+ appChecks: List[Callable[[Any], _CheckResult]] = [
+ _checkMandateDescription,
+ _checkMandateSlugRules,
+ _checkRootMandateLegacy,
+ _checkSysadminRole,
+ ]
+
+ results: List[_CheckResult] = []
+ for fn in appChecks:
+ try:
+ results.append(fn(appDb))
+ except Exception as exc:
+ results.append(
+ _CheckResult(
+ name=fn.__name__,
+ routine="?",
+ location="?",
+ count=-1,
+ status="ERROR",
+ error=f"{type(exc).__name__}: {exc}",
+ )
+ )
+
+ try:
+ results.append(_checkRagFallback(knowDb))
+ except Exception as exc:
+ results.append(
+ _CheckResult(
+ name="rag-fallback-orphan-index",
+ routine="aggregateMandateRagTotalBytes-fallback",
+ location="interfaces/interfaceDbKnowledge.py:609-635",
+ count=-1,
+ status="ERROR",
+ error=f"{type(exc).__name__}: {exc}",
+ )
+ )
+
+ return results
+
+
+def _printText(results: List[_CheckResult]) -> None:
+ print("=" * 78)
+ print("BOOTSTRAP-MIGRATIONS LEGACY-STATE-AUDIT")
+ print("=" * 78)
+ for r in results:
+ marker = {
+ "GREEN": "[OK]",
+ "RED": "[!!]",
+ "ERROR": "[ERR]",
+ }.get(r.status, "[?]")
+ print(f"\n{marker} {r.name}")
+ print(f" Routine : {r.routine}")
+ print(f" Location: {r.location}")
+ print(f" Count : {r.count}")
+ print(f" Status : {r.status}")
+ if r.error:
+ print(f" Note : {r.error}")
+ if r.samples:
+ print(f" Samples : (max 5)")
+ for s in r.samples:
+ print(f" {s}")
+
+ print("\n" + "=" * 78)
+ greens = sum(1 for r in results if r.status == "GREEN")
+ reds = sum(1 for r in results if r.status == "RED")
+ errs = sum(1 for r in results if r.status == "ERROR")
+ print(f"SUMMARY: {greens} GREEN {reds} RED {errs} ERROR ({len(results)} total)")
+ if reds == 0 and errs == 0:
+ print("VERDICT: alle Migrationsroutinen koennen entfernt werden.")
+ elif errs > 0:
+ print("VERDICT: Audit unvollstaendig (Fehler) -- bitte Skript fixen.")
+ else:
+ print("VERDICT: erst Daten bereinigen, dann Routinen entfernen.")
+ print("=" * 78)
+
+
+def _purgeRagOrphans() -> int:
+ """Loescht alle FileContentIndex-Rows ohne mandateId UND ohne featureInstanceId.
+
+ Returns: Anzahl geloeschter Rows.
+ """
+ knowDb = _getKnowledgeDb()
+ rows = knowDb.getRecordset(FileContentIndex)
+ orphans = [
+ r for r in rows
+ if not (r.get("mandateId") or "").strip()
+ and not (r.get("featureInstanceId") or "").strip()
+ ]
+ if not orphans:
+ print("Keine RAG-Orphans gefunden -- nichts zu purgen.")
+ return 0
+
+ print(f"Purge {len(orphans)} RAG-Orphan(s):")
+ deleted = 0
+ for r in orphans:
+ rid = r.get("id")
+ try:
+ knowDb.recordDelete(FileContentIndex, str(rid))
+ deleted += 1
+ print(f" geloescht: {rid} {r.get('fileName')}")
+ except Exception as exc:
+ print(f" FEHLER {rid}: {type(exc).__name__}: {exc}", file=sys.stderr)
+ print(f"Purge abgeschlossen: {deleted}/{len(orphans)} geloescht.")
+ return deleted
+
+
+def main() -> int:
+ parser = argparse.ArgumentParser(
+ description="Audit-Skript fuer Legacy-Bestaende (Bootstrap-Cleanup Plan C)"
+ )
+ parser.add_argument("--json", action="store_true", help="JSON-Output statt Text")
+ parser.add_argument(
+ "--purge-rag-orphans",
+ action="store_true",
+ help="WRITE: loescht FileContentIndex-Rows ohne mandateId UND featureInstanceId",
+ )
+ args = parser.parse_args()
+
+ if args.purge_rag_orphans:
+ try:
+ _purgeRagOrphans()
+ except Exception as exc:
+ print(f"FATAL: Purge fehlgeschlagen -- {type(exc).__name__}: {exc}", file=sys.stderr)
+ return 2
+ print()
+
+ try:
+ results = _runChecks()
+ except Exception as exc:
+ print(f"FATAL: konnte Audit nicht starten -- {type(exc).__name__}: {exc}", file=sys.stderr)
+ return 2
+
+ if args.json:
+ print(json.dumps([r.toDict() for r in results], indent=2, default=str))
+ else:
+ _printText(results)
+
+ if any(r.status == "ERROR" for r in results):
+ return 2
+ if any(r.status == "RED" for r in results):
+ return 1
+ return 0
+
+
+if __name__ == "__main__":
+ sys.exit(main())
diff --git a/scripts/stage0_filefolder_schema_check.py b/scripts/stage0_filefolder_schema_check.py
new file mode 100644
index 00000000..861d8671
--- /dev/null
+++ b/scripts/stage0_filefolder_schema_check.py
@@ -0,0 +1,58 @@
+"""Stage 0: verify FileFolder table + FileItem.folderId column in management DB.
+
+Run from the gateway directory (same as uvicorn):
+ python -m scripts.stage0_filefolder_schema_check
+"""
+from modules.connectors.connectorDbPostgre import getCachedConnector
+from modules.shared.configuration import APP_CONFIG
+
+managementDatabase = "poweron_management"
+
+dbHost = APP_CONFIG.get("DB_HOST", "_no_config_default_data")
+dbUser = APP_CONFIG.get("DB_USER")
+dbPassword = APP_CONFIG.get("DB_PASSWORD_SECRET")
+dbPort = int(APP_CONFIG.get("DB_PORT", 5432))
+
+c = getCachedConnector(
+ dbHost=dbHost,
+ dbDatabase=managementDatabase,
+ dbUser=dbUser,
+ dbPassword=dbPassword,
+ dbPort=dbPort,
+ userId=None,
+)
+if not c or not c.connection:
+ print("STAGE0: DB_CONNECTION=none (check config.ini / .env)")
+ raise SystemExit(2)
+
+cur = c.connection.cursor()
+
+
+def _scalar(cur):
+ row = cur.fetchone()
+ if row is None:
+ return None
+ if isinstance(row, dict):
+ return next(iter(row.values()))
+ return row[0]
+
+
+cur.execute(
+ """
+ SELECT EXISTS (
+ SELECT 1 FROM information_schema.tables
+ WHERE table_name = 'FileFolder'
+ ) AS ok
+ """
+)
+print("STAGE0: FileFolder_table=", _scalar(cur))
+cur.execute(
+ """
+ SELECT EXISTS (
+ SELECT 1 FROM information_schema.columns
+ WHERE table_name = 'FileItem' AND column_name = 'folderId'
+ ) AS ok
+ """
+)
+print("STAGE0: FileItem_folderId_column=", _scalar(cur))
+cur.close()
diff --git a/tests/serviceAi/__init__.py b/tests/serviceAi/__init__.py
new file mode 100644
index 00000000..e69de29b
diff --git a/tests/serviceAi/test_allowed_models_whitelist.py b/tests/serviceAi/test_allowed_models_whitelist.py
new file mode 100644
index 00000000..4593afd9
--- /dev/null
+++ b/tests/serviceAi/test_allowed_models_whitelist.py
@@ -0,0 +1,14 @@
+# Copyright (c) 2025 Patrick Motsch
+# All rights reserved.
+import pytest
+from modules.datamodels.datamodelAi import AiCallOptions
+
+
+def test_allowed_models_field_exists():
+ opts = AiCallOptions(allowedModels=["gpt-5-mini", "claude-4-7-opus"])
+ assert opts.allowedModels == ["gpt-5-mini", "claude-4-7-opus"]
+
+
+def test_allowed_models_default_none():
+ opts = AiCallOptions()
+ assert opts.allowedModels is None
diff --git a/tests/serviceGeneration/__init__.py b/tests/serviceGeneration/__init__.py
new file mode 100644
index 00000000..e69de29b
diff --git a/tests/serviceGeneration/test_inline_image_paragraph.py b/tests/serviceGeneration/test_inline_image_paragraph.py
new file mode 100644
index 00000000..be0c5d19
--- /dev/null
+++ b/tests/serviceGeneration/test_inline_image_paragraph.py
@@ -0,0 +1,23 @@
+# Copyright (c) 2025 Patrick Motsch
+# All rights reserved.
+import pytest
+from modules.serviceCenter.services.serviceGeneration.subDocumentUtility import markdownToDocumentJson
+
+
+def test_inline_image_in_paragraph():
+ md = "Results show ![chart](file:abc \"200pt\") clearly."
+ result = markdownToDocumentJson(md, "Test")
+ runs = result["documents"][0]["sections"][0]["elements"][0]["content"]["inlineRuns"]
+ types = [r["type"] for r in runs]
+ assert "text" in types
+ assert "image" in types
+ imgRun = next(r for r in runs if r["type"] == "image")
+ assert imgRun.get("fileId") == "abc"
+
+
+def test_multiple_inline_images():
+ md = "A  B  C"
+ result = markdownToDocumentJson(md, "Test")
+ runs = result["documents"][0]["sections"][0]["elements"][0]["content"]["inlineRuns"]
+ images = [r for r in runs if r["type"] == "image"]
+ assert len(images) == 2
diff --git a/tests/serviceGeneration/test_md_to_json_consolidation.py b/tests/serviceGeneration/test_md_to_json_consolidation.py
new file mode 100644
index 00000000..83118374
--- /dev/null
+++ b/tests/serviceGeneration/test_md_to_json_consolidation.py
@@ -0,0 +1,71 @@
+# Copyright (c) 2025 Patrick Motsch
+# All rights reserved.
+import pytest
+from modules.serviceCenter.services.serviceGeneration.subDocumentUtility import markdownToDocumentJson
+
+
+def test_basic_paragraph():
+ result = markdownToDocumentJson("Hello world", "Test")
+ doc = result["documents"][0]
+ section = doc["sections"][0]
+ assert section["content_type"] == "paragraph"
+ assert section["elements"][0]["content"]["inlineRuns"][0] == {"type": "text", "value": "Hello world"}
+
+
+def test_inline_bold():
+ result = markdownToDocumentJson("This is **bold** text", "Test")
+ runs = result["documents"][0]["sections"][0]["elements"][0]["content"]["inlineRuns"]
+ assert any(r["type"] == "bold" and r["value"] == "bold" for r in runs)
+
+
+def test_inline_image():
+ result = markdownToDocumentJson("Text  more", "Test")
+ runs = result["documents"][0]["sections"][0]["elements"][0]["content"]["inlineRuns"]
+ assert any(r["type"] == "image" and r.get("fileId") == "abc123" for r in runs)
+
+
+def test_inline_link():
+ result = markdownToDocumentJson("Click [here](https://example.com)", "Test")
+ runs = result["documents"][0]["sections"][0]["elements"][0]["content"]["inlineRuns"]
+ assert any(r["type"] == "link" and r.get("href") == "https://example.com" for r in runs)
+
+
+def test_table_cells_are_inline_runs():
+ md = "| A | B |\n| --- | --- |\n| **x** | y |"
+ result = markdownToDocumentJson(md, "Test")
+ section = result["documents"][0]["sections"][0]
+ assert section["content_type"] == "table"
+ rows = section["elements"][0]["content"]["rows"]
+ assert isinstance(rows[0][0], list)
+
+
+def test_bullet_list_inline_runs():
+ md = "- Item **one**\n- Item two"
+ result = markdownToDocumentJson(md, "Test")
+ section = result["documents"][0]["sections"][0]
+ assert section["content_type"] == "bullet_list"
+ items = section["elements"][0]["content"]["items"]
+ assert isinstance(items[0], list)
+
+
+def test_standalone_image_block():
+ md = ""
+ result = markdownToDocumentJson(md, "Test")
+ section = result["documents"][0]["sections"][0]
+ assert section["content_type"] == "image"
+
+
+def test_heading_unchanged():
+ result = markdownToDocumentJson("# Title", "Test")
+ section = result["documents"][0]["sections"][0]
+ assert section["content_type"] == "heading"
+ assert section["elements"][0]["content"]["text"] == "Title"
+ assert section["elements"][0]["content"]["level"] == 1
+
+
+def test_code_block_unchanged():
+ md = "```python\nprint('hi')\n```"
+ result = markdownToDocumentJson(md, "Test")
+ section = result["documents"][0]["sections"][0]
+ assert section["content_type"] == "code_block"
+ assert section["elements"][0]["content"]["code"] == "print('hi')"
diff --git a/tests/serviceGeneration/test_style_resolver.py b/tests/serviceGeneration/test_style_resolver.py
new file mode 100644
index 00000000..6b2b649a
--- /dev/null
+++ b/tests/serviceGeneration/test_style_resolver.py
@@ -0,0 +1,39 @@
+# Copyright (c) 2025 Patrick Motsch
+# All rights reserved.
+import pytest
+from modules.serviceCenter.services.serviceGeneration.styleDefaults import resolveStyle, DEFAULT_STYLE
+
+
+def test_resolve_none_returns_defaults():
+ result = resolveStyle(None)
+ assert result == DEFAULT_STYLE
+
+
+def test_resolve_empty_returns_defaults():
+ result = resolveStyle({})
+ assert result == DEFAULT_STYLE
+
+
+def test_override_single_color():
+ result = resolveStyle({"colors": {"primary": "#FF0000"}})
+ assert result["colors"]["primary"] == "#FF0000"
+ assert result["colors"]["secondary"] == DEFAULT_STYLE["colors"]["secondary"]
+
+
+def test_override_nested_heading():
+ result = resolveStyle({"headings": {"h1": {"sizePt": 30}}})
+ assert result["headings"]["h1"]["sizePt"] == 30
+ assert result["headings"]["h1"]["weight"] == "bold"
+
+
+def test_override_font():
+ result = resolveStyle({"fonts": {"primary": "Arial"}})
+ assert result["fonts"]["primary"] == "Arial"
+ assert result["fonts"]["monospace"] == "Consolas"
+
+
+def test_full_style_passthrough():
+ custom = {"fonts": {"primary": "Helvetica", "monospace": "Monaco"}}
+ result = resolveStyle(custom)
+ assert result["fonts"]["primary"] == "Helvetica"
+ assert result["fonts"]["monospace"] == "Monaco"
diff --git a/tests/unit/bootstrap/test_mandateNameMigration.py b/tests/unit/bootstrap/test_mandateNameMigration.py
deleted file mode 100644
index d09a6846..00000000
--- a/tests/unit/bootstrap/test_mandateNameMigration.py
+++ /dev/null
@@ -1,133 +0,0 @@
-#!/usr/bin/env python3
-# Copyright (c) 2025 Patrick Motsch
-# All rights reserved.
-"""
-Unit tests for ``_migrateMandateNameLabelSlugRules`` in interfaceBootstrap.
-
-Covers:
-- legacy ``name``/``label`` rows get fixed (label fill, slug rename),
-- collisions across legacy rows resolve via -2/-3 suffixes in stable id order,
-- valid rows are left untouched (idempotency),
-- second invocation is a no-op.
-"""
-
-from typing import Any, Dict, List, Optional
-
-import pytest
-
-from modules.datamodels.datamodelUam import Mandate
-from modules.interfaces.interfaceBootstrap import _migrateMandateNameLabelSlugRules
-from modules.shared.mandateNameUtils import isValidMandateName
-
-
-class _FakeDb:
- """Minimal connector simulating getRecordset(Mandate)+recordModify(Mandate, id, data)."""
-
- def __init__(self, rows: List[Dict[str, Any]]):
- self.rows: List[Dict[str, Any]] = [dict(r) for r in rows]
- self.modifyCalls: List[Dict[str, Any]] = []
-
- def getRecordset(self, model, recordFilter: Optional[Dict[str, Any]] = None):
- if model is not Mandate:
- return []
- if not recordFilter:
- return [dict(r) for r in self.rows]
- out = []
- for r in self.rows:
- if all(r.get(k) == v for k, v in recordFilter.items()):
- out.append(dict(r))
- return out
-
- def recordModify(self, model, recordId: str, data: Dict[str, Any]):
- self.modifyCalls.append({"id": str(recordId), "data": dict(data)})
- for r in self.rows:
- if str(r.get("id")) == str(recordId):
- r.update(data)
- return r
- return None
-
-
-def _row(mid: str, name: Any, label: Any = None) -> Dict[str, Any]:
- return {"id": mid, "name": name, "label": label}
-
-
-class TestMigrationFillsLabel:
- def test_emptyLabelGetsNameAsLabel(self):
- db = _FakeDb([_row("a1", "good-name", None)])
- _migrateMandateNameLabelSlugRules(db)
- assert db.rows[0]["label"] == "good-name"
- assert db.rows[0]["name"] == "good-name"
-
- def test_emptyLabelAndEmptyNameFallsBackToMandate(self):
- db = _FakeDb([_row("a1", "", "")])
- _migrateMandateNameLabelSlugRules(db)
- assert db.rows[0]["label"] == "Mandate"
- assert isValidMandateName(db.rows[0]["name"])
-
-
-class TestMigrationRenamesInvalidNames:
- def test_invalidNameGetsSlugFromLabel(self):
- db = _FakeDb([_row("a1", "Home patrick", "Home Patrick")])
- _migrateMandateNameLabelSlugRules(db)
- assert db.rows[0]["name"] == "home-patrick"
- assert db.rows[0]["label"] == "Home Patrick"
-
- def test_umlautsTransliterated(self):
- db = _FakeDb([_row("a1", "Müller AG", "Müller AG")])
- _migrateMandateNameLabelSlugRules(db)
- assert db.rows[0]["name"] == "mueller-ag"
-
-
-class TestMigrationCollisions:
- def test_collisionsResolveByStableIdOrder(self):
- rows = [
- _row("z1", "Home patrick", "Home Patrick"),
- _row("a1", "home-patrick", "Home Patrick Two"),
- ]
- db = _FakeDb(rows)
- _migrateMandateNameLabelSlugRules(db)
- byId = {r["id"]: r for r in db.rows}
- assert byId["a1"]["name"] == "home-patrick"
- assert byId["z1"]["name"] == "home-patrick-2"
-
- def test_threeWayCollisionGetsThirdSuffix(self):
- rows = [
- _row("id-aaa", "home-patrick", "Home Patrick"),
- _row("id-bbb", "Home patrick", "Home Patrick"),
- _row("id-ccc", "home patrick", "Home Patrick"),
- ]
- db = _FakeDb(rows)
- _migrateMandateNameLabelSlugRules(db)
- names = sorted(r["name"] for r in db.rows)
- assert names == ["home-patrick", "home-patrick-2", "home-patrick-3"]
-
-
-class TestMigrationIdempotency:
- def test_secondRunIsNoop(self):
- rows = [
- _row("a1", "home-patrick", "Home Patrick"),
- _row("b1", "Home Müller", ""),
- ]
- db = _FakeDb(rows)
- _migrateMandateNameLabelSlugRules(db)
- assert all(isValidMandateName(r["name"]) for r in db.rows)
- firstChanges = list(db.modifyCalls)
- db.modifyCalls.clear()
- _migrateMandateNameLabelSlugRules(db)
- assert db.modifyCalls == [], (
- f"expected no further changes after first migration, got {db.modifyCalls}; "
- f"firstRun changes: {firstChanges}"
- )
-
- def test_validRowsLeftUntouched(self):
- rows = [_row("a1", "root", "Root"), _row("b1", "alpina-treuhand", "Alpina Treuhand AG")]
- db = _FakeDb(rows)
- _migrateMandateNameLabelSlugRules(db)
- assert db.modifyCalls == []
-
-
-class TestMigrationEmpty:
- def test_emptyDbDoesNothing(self):
- db = _FakeDb([])
- _migrateMandateNameLabelSlugRules(db)
- assert db.modifyCalls == []
diff --git a/tests/unit/interfaces/test_folderRbac.py b/tests/unit/interfaces/test_folderRbac.py
new file mode 100644
index 00000000..049f392d
--- /dev/null
+++ b/tests/unit/interfaces/test_folderRbac.py
@@ -0,0 +1,327 @@
+# Copyright (c) 2026 Patrick Motsch
+# All rights reserved.
+"""Unit tests for folder RBAC two-user matrix (ownership & scope visibility)."""
+
+import uuid
+import pytest
+from unittest.mock import Mock, patch, MagicMock
+from typing import Dict, Any, List, Optional
+
+from modules.datamodels.datamodelFiles import FileFolder, FileItem
+from modules.datamodels.datamodelUam import User, UserPermissions, AccessLevel
+from modules.interfaces.interfaceDbManagement import ComponentObjects, FileNotFoundError
+
+
+_MANDATE_ID = "mandate-test-1"
+_FEATURE_INSTANCE_ID = "fi-test-1"
+_USER_A = "user-a-id"
+_USER_B = "user-b-id"
+
+
+# ── Fakes & helpers ──────────────────────────────────────────────────────────
+
+class _FakeDb:
+ """In-memory database mock."""
+
+ def __init__(self):
+ self._tables: Dict[str, Dict[str, Dict[str, Any]]] = {}
+ self.connection = MagicMock()
+
+ def getRecordset(self, modelClass, recordFilter=None):
+ tableName = modelClass.__name__
+ records = list(self._tables.get(tableName, {}).values())
+ if not recordFilter:
+ return records
+ return [
+ r for r in records
+ if all(r.get(k) == v for k, v in recordFilter.items())
+ ]
+
+ def recordCreate(self, modelClass, data):
+ tableName = modelClass.__name__
+ self._tables.setdefault(tableName, {})
+ rec = data.model_dump() if hasattr(data, "model_dump") else dict(data)
+ rec.setdefault("id", str(uuid.uuid4()))
+ self._tables[tableName][rec["id"]] = rec
+ return rec
+
+ def recordModify(self, modelClass, recordId, updates):
+ tbl = self._tables.get(modelClass.__name__, {})
+ if recordId in tbl:
+ tbl[recordId].update(updates)
+ return True
+ return False
+
+ def recordDelete(self, modelClass, recordId):
+ tbl = self._tables.get(modelClass.__name__, {})
+ if recordId in tbl:
+ del tbl[recordId]
+ return True
+ return False
+
+ def updateContext(self, userId):
+ pass
+
+ def _ensure_connection(self):
+ pass
+
+ def _ensureTableExists(self, modelClass):
+ return True
+
+ def seed(self, modelClass, record: Dict[str, Any]):
+ tableName = modelClass.__name__
+ self._tables.setdefault(tableName, {})
+ self._tables[tableName][record["id"]] = dict(record)
+
+
+def _makeUser(userId, username="testuser"):
+ return User(id=userId, username=username, language="en")
+
+
+def _makeRbac(
+ createLevel=AccessLevel.ALL,
+ readLevel=AccessLevel.ALL,
+ updateLevel=AccessLevel.MY,
+ deleteLevel=AccessLevel.MY,
+):
+ """Default: regular user can read all, but write only own records."""
+ rbac = Mock()
+ perms = UserPermissions(
+ view=True,
+ read=readLevel,
+ create=createLevel,
+ update=updateLevel,
+ delete=deleteLevel,
+ )
+ rbac.getUserPermissions.return_value = perms
+ return rbac
+
+
+def _buildComponent(userId, fakeDb, rbac=None):
+ with patch.object(ComponentObjects, "__init__", lambda self: None):
+ comp = ComponentObjects()
+ comp.db = fakeDb
+ comp.currentUser = _makeUser(userId)
+ comp.userId = userId
+ comp.mandateId = _MANDATE_ID
+ comp.featureInstanceId = _FEATURE_INSTANCE_ID
+ comp.rbac = rbac or _makeRbac()
+ comp.userLanguage = "en"
+ return comp
+
+
+def _makeFolder(
+ folderId=None, name="Folder", parentId=None,
+ userId=_USER_A, scope="personal", neutralize=False,
+):
+ return {
+ "id": folderId or str(uuid.uuid4()),
+ "name": name,
+ "parentId": parentId,
+ "mandateId": _MANDATE_ID,
+ "featureInstanceId": _FEATURE_INSTANCE_ID,
+ "scope": scope,
+ "neutralize": neutralize,
+ "sysCreatedBy": userId,
+ "sysCreatedAt": 1700000000.0,
+ "sysModifiedAt": 1700000000.0,
+ "sysModifiedBy": None,
+ }
+
+
+def _makeFile(fileId=None, folderId=None, userId=_USER_A, scope="personal"):
+ return {
+ "id": fileId or str(uuid.uuid4()),
+ "fileName": "test.txt",
+ "mimeType": "text/plain",
+ "fileHash": "abc123",
+ "fileSize": 100,
+ "folderId": folderId,
+ "mandateId": _MANDATE_ID,
+ "featureInstanceId": _FEATURE_INSTANCE_ID,
+ "scope": scope,
+ "neutralize": False,
+ "sysCreatedBy": userId,
+ "sysCreatedAt": 1700000000.0,
+ "sysModifiedAt": 1700000000.0,
+ "sysModifiedBy": None,
+ "tags": None,
+ "description": None,
+ "status": None,
+ }
+
+
+def _scopeAwareMock(fakeDb):
+ """Side-effect for getRecordsetWithRBAC that simulates scope-based visibility.
+
+ Visibility rules:
+ - Owner (sysCreatedBy == currentUser.id) always sees the record
+ - scope='global' -> visible to everyone
+ - scope='mandate' -> visible when mandateId matches
+ - scope='featureInstance' -> visible when featureInstanceId matches
+ - scope='personal' -> owner only (already covered above)
+ """
+ def _fn(connector, modelClass, currentUser, recordFilter=None, **kwargs):
+ requestMandateId = kwargs.get("mandateId", _MANDATE_ID)
+ requestFiId = kwargs.get("featureInstanceId", _FEATURE_INSTANCE_ID)
+ allRecords = fakeDb.getRecordset(modelClass, recordFilter=recordFilter)
+ visible = []
+ for rec in allRecords:
+ if rec.get("sysCreatedBy") == currentUser.id:
+ visible.append(rec)
+ continue
+ scope = rec.get("scope", "personal")
+ if scope == "global":
+ visible.append(rec)
+ elif scope == "mandate" and rec.get("mandateId") == requestMandateId:
+ visible.append(rec)
+ elif scope == "featureInstance" and rec.get("featureInstanceId") == requestFiId:
+ visible.append(rec)
+ return visible
+ return _fn
+
+
+# ── Test class ───────────────────────────────────────────────────────────────
+
+@patch("modules.interfaces.interfaceDbManagement.getRecordsetWithRBAC")
+class TestFolderRbac:
+ """Two-user matrix: ownership, scope visibility, and write-access guards."""
+
+ # ── 1. Ownership visibility ───────────────────────────────────────────
+
+ def testUserAFolderInOwnTreeNotInUserBOwnTree(self, mockRbacGet):
+ """User A's personal folder appears in A's own tree, not in B's."""
+ fakeDb = _FakeDb()
+ fakeDb.seed(FileFolder, _makeFolder(folderId="fa-1", name="A-Folder", userId=_USER_A))
+ mockRbacGet.side_effect = _scopeAwareMock(fakeDb)
+
+ compA = _buildComponent(_USER_A, fakeDb)
+ ownA = compA.getOwnFolderTree()
+ assert any(f["id"] == "fa-1" for f in ownA)
+
+ compB = _buildComponent(_USER_B, fakeDb)
+ ownB = compB.getOwnFolderTree()
+ assert not any(f["id"] == "fa-1" for f in ownB)
+
+ # ── 2. Scope change -> shared visibility ──────────────────────────────
+
+ def testScopeChangeToMandateMakesVisibleToUserB(self, mockRbacGet):
+ """Changing scope from personal to mandate makes the folder appear
+ in User B's shared tree."""
+ fakeDb = _FakeDb()
+ fakeDb.seed(FileFolder, _makeFolder(folderId="fa-1", scope="personal", userId=_USER_A))
+ mockRbacGet.side_effect = _scopeAwareMock(fakeDb)
+
+ compB = _buildComponent(_USER_B, fakeDb)
+ sharedBefore = compB.getSharedFolderTree()
+ assert not any(f["id"] == "fa-1" for f in sharedBefore)
+
+ fakeDb.recordModify(FileFolder, "fa-1", {"scope": "mandate"})
+
+ sharedAfter = compB.getSharedFolderTree()
+ assert any(f["id"] == "fa-1" for f in sharedAfter)
+
+ # ── 3-7. Non-owner cannot mutate ──────────────────────────────────────
+
+ def testUserBCannotRenameFolderOfUserA(self, mockRbacGet):
+ fakeDb = _FakeDb()
+ fakeDb.seed(FileFolder, _makeFolder(folderId="fa-1", scope="mandate", userId=_USER_A))
+ mockRbacGet.side_effect = _scopeAwareMock(fakeDb)
+
+ compB = _buildComponent(_USER_B, fakeDb)
+ with pytest.raises(PermissionError):
+ compB.renameFolder("fa-1", "Hijacked")
+
+ def testUserBCannotMoveFolderOfUserA(self, mockRbacGet):
+ fakeDb = _FakeDb()
+ fakeDb.seed(FileFolder, _makeFolder(folderId="fa-1", scope="mandate", userId=_USER_A))
+ fakeDb.seed(FileFolder, _makeFolder(folderId="fb-1", scope="mandate", userId=_USER_B))
+ mockRbacGet.side_effect = _scopeAwareMock(fakeDb)
+
+ compB = _buildComponent(_USER_B, fakeDb)
+ with pytest.raises(PermissionError):
+ compB.moveFolder("fa-1", "fb-1")
+
+ def testUserBCannotDeleteFolderOfUserA(self, mockRbacGet):
+ fakeDb = _FakeDb()
+ fakeDb.seed(FileFolder, _makeFolder(folderId="fa-1", scope="mandate", userId=_USER_A))
+ mockRbacGet.side_effect = _scopeAwareMock(fakeDb)
+
+ compB = _buildComponent(_USER_B, fakeDb)
+ with pytest.raises(PermissionError):
+ compB.deleteFolderCascade("fa-1")
+
+ def testUserBCannotPatchScopeOnFolderOfUserA(self, mockRbacGet):
+ fakeDb = _FakeDb()
+ fakeDb.seed(FileFolder, _makeFolder(folderId="fa-1", scope="mandate", userId=_USER_A))
+ mockRbacGet.side_effect = _scopeAwareMock(fakeDb)
+
+ compB = _buildComponent(_USER_B, fakeDb)
+ with pytest.raises(PermissionError):
+ compB.patchFolderScope("fa-1", "personal")
+
+ def testUserBCannotPatchNeutralizeOnFolderOfUserA(self, mockRbacGet):
+ fakeDb = _FakeDb()
+ fakeDb.seed(FileFolder, _makeFolder(folderId="fa-1", scope="mandate", userId=_USER_A))
+ mockRbacGet.side_effect = _scopeAwareMock(fakeDb)
+
+ compB = _buildComponent(_USER_B, fakeDb)
+ with pytest.raises(PermissionError):
+ compB.patchFolderNeutralize("fa-1", True)
+
+ # ── 8. contextOrphan ──────────────────────────────────────────────────
+
+ def testContextOrphanWhenParentFolderNotShared(self, mockRbacGet):
+ """User A's parent folder is personal, child folder is mandate.
+ User B sees only the child, flagged as contextOrphan."""
+ fakeDb = _FakeDb()
+ fakeDb.seed(FileFolder, _makeFolder(
+ folderId="parent-f", name="Private Parent", userId=_USER_A, scope="personal",
+ ))
+ fakeDb.seed(FileFolder, _makeFolder(
+ folderId="child-f", name="Shared Child", userId=_USER_A,
+ parentId="parent-f", scope="mandate",
+ ))
+ mockRbacGet.side_effect = _scopeAwareMock(fakeDb)
+
+ compB = _buildComponent(_USER_B, fakeDb)
+ shared = compB.getSharedFolderTree()
+
+ assert len(shared) == 1
+ assert shared[0]["id"] == "child-f"
+ assert shared[0]["contextOrphan"] is True
+
+ # ── 9. Shared folder children visible ─────────────────────────────────
+
+ def testSharedFolderMakesChildrenVisible(self, mockRbacGet):
+ """When User A shares a folder tree (scope=mandate), all child folders
+ become visible in User B's shared tree."""
+ fakeDb = _FakeDb()
+ fakeDb.seed(FileFolder, _makeFolder(
+ folderId="root-f", name="Root", userId=_USER_A, scope="mandate",
+ ))
+ fakeDb.seed(FileFolder, _makeFolder(
+ folderId="child1-f", name="Child 1", userId=_USER_A,
+ parentId="root-f", scope="mandate",
+ ))
+ fakeDb.seed(FileFolder, _makeFolder(
+ folderId="child2-f", name="Child 2", userId=_USER_A,
+ parentId="root-f", scope="mandate",
+ ))
+ fakeDb.seed(FileFolder, _makeFolder(
+ folderId="grandchild-f", name="Grandchild", userId=_USER_A,
+ parentId="child1-f", scope="mandate",
+ ))
+ mockRbacGet.side_effect = _scopeAwareMock(fakeDb)
+
+ compB = _buildComponent(_USER_B, fakeDb)
+ shared = compB.getSharedFolderTree()
+
+ sharedIds = {f["id"] for f in shared}
+ assert sharedIds == {"root-f", "child1-f", "child2-f", "grandchild-f"}
+
+ byId = {f["id"]: f for f in shared}
+ assert byId["root-f"]["contextOrphan"] is False
+ assert byId["child1-f"]["contextOrphan"] is False
+ assert byId["child2-f"]["contextOrphan"] is False
+ assert byId["grandchild-f"]["contextOrphan"] is False
diff --git a/tests/unit/rbac/test_sysadmin_migration.py b/tests/unit/rbac/test_sysadmin_migration.py
deleted file mode 100644
index 8ca077bf..00000000
--- a/tests/unit/rbac/test_sysadmin_migration.py
+++ /dev/null
@@ -1,209 +0,0 @@
-# Copyright (c) 2025 Patrick Motsch
-# All rights reserved.
-"""
-Unit tests for the one-shot sysadmin role -> isPlatformAdmin migration.
-
-Covers acceptance criteria from
-``wiki/c-work/4-done/2026-04-sysadmin-authority-split.md``:
-
-- AC#4 -> Existing sysadmin role-holders are promoted to ``isPlatformAdmin=True``
- and the legacy role is removed (Role + UserMandateRole + AccessRules)
- when the gateway boots.
-- AC#10 -> The migration is idempotent and removes ALL artefacts (Role,
- AccessRules, UserMandateRole) of the legacy ``sysadmin`` role.
-
-Strategy: use an in-memory fake ``DatabaseConnector`` that records calls
-and returns deterministic recordsets for ``Role``/``UserMandateRole``/
-``UserMandate``/``UserInDB``/``AccessRule`` lookups.
-"""
-
-from __future__ import annotations
-
-from typing import Any, Dict, List
-from unittest.mock import Mock
-
-from modules.interfaces.interfaceBootstrap import _migrateAndDropSysAdminRole
-from modules.datamodels.datamodelMembership import UserMandate, UserMandateRole
-from modules.datamodels.datamodelRbac import AccessRule, Role
-from modules.datamodels.datamodelUam import UserInDB
-
-
-_ROOT_MANDATE_ID = "root-mandate-id"
-_SYSADMIN_ROLE_ID = "sysadmin-role-id"
-_USER_MANDATE_ID = "user-mandate-id"
-_USER_ID = "legacy-user-id"
-_UMR_ROW_ID = "umr-row-id"
-_ACCESS_RULE_ID = "access-rule-id"
-
-
-def _buildFakeDb(
- *,
- sysadminRoles: List[Dict[str, Any]],
- umRoleRows: List[Dict[str, Any]],
- userMandateRows: List[Dict[str, Any]],
- users: List[Dict[str, Any]],
- accessRules: List[Dict[str, Any]],
-) -> Mock:
- """Build a fake ``DatabaseConnector`` that maps model -> recordset."""
-
- deletes: List[tuple] = []
- modifies: List[tuple] = []
-
- def _getRecordset(model, recordFilter=None, **_): # noqa: ANN001
- recordFilter = recordFilter or {}
- if model is Role:
- label = recordFilter.get("roleLabel")
- mandateId = recordFilter.get("mandateId")
- if label == "sysadmin" and mandateId == _ROOT_MANDATE_ID:
- return list(sysadminRoles)
- return []
- if model is UserMandateRole:
- wanted = recordFilter.get("roleId")
- return [r for r in umRoleRows if r.get("roleId") == wanted]
- if model is UserMandate:
- wanted = recordFilter.get("id")
- return [r for r in userMandateRows if r.get("id") == wanted]
- if model is UserInDB:
- wanted = recordFilter.get("id")
- return [r for r in users if r.get("id") == wanted]
- if model is AccessRule:
- wanted = recordFilter.get("roleId")
- return [r for r in accessRules if r.get("roleId") == wanted]
- return []
-
- def _recordModify(model, recordId, payload): # noqa: ANN001
- modifies.append((model, recordId, payload))
- # Reflect the change so a subsequent migration call is idempotent.
- if model is UserInDB:
- for u in users:
- if u.get("id") == recordId:
- u.update(payload)
- return True
-
- def _recordDelete(model, recordId): # noqa: ANN001
- deletes.append((model, recordId))
- if model is UserMandateRole:
- umRoleRows[:] = [r for r in umRoleRows if r.get("id") != recordId]
- elif model is AccessRule:
- accessRules[:] = [r for r in accessRules if r.get("id") != recordId]
- elif model is Role:
- sysadminRoles[:] = [r for r in sysadminRoles if r.get("id") != recordId]
- return True
-
- db = Mock()
- db.getRecordset = Mock(side_effect=_getRecordset)
- db.recordModify = Mock(side_effect=_recordModify)
- db.recordDelete = Mock(side_effect=_recordDelete)
- db._modifies = modifies # exposed for assertions
- db._deletes = deletes
- return db
-
-
-def _seed():
- return {
- "sysadminRoles": [{"id": _SYSADMIN_ROLE_ID, "roleLabel": "sysadmin",
- "mandateId": _ROOT_MANDATE_ID}],
- "umRoleRows": [{"id": _UMR_ROW_ID, "roleId": _SYSADMIN_ROLE_ID,
- "userMandateId": _USER_MANDATE_ID}],
- "userMandateRows": [{"id": _USER_MANDATE_ID, "userId": _USER_ID,
- "mandateId": _ROOT_MANDATE_ID}],
- "users": [{"id": _USER_ID, "username": "legacy",
- "isSysAdmin": False, "isPlatformAdmin": False}],
- "accessRules": [{"id": _ACCESS_RULE_ID, "roleId": _SYSADMIN_ROLE_ID}],
- }
-
-
-# ---------------------------------------------------------------------------
-# AC #4 — promote + drop on first run
-# ---------------------------------------------------------------------------
-
-
-def testMigrationPromotesUserAndDropsArtefacts():
- """AC#4: legacy holder is promoted; Role+AccessRule+UMR are deleted."""
- seed = _seed()
- db = _buildFakeDb(**seed)
-
- _migrateAndDropSysAdminRole(db, _ROOT_MANDATE_ID)
-
- # User got isPlatformAdmin=True
- assert seed["users"][0]["isPlatformAdmin"] is True
- assert any(
- m[0] is UserInDB and m[2] == {"isPlatformAdmin": True}
- for m in db._modifies
- ), "Expected UserInDB.isPlatformAdmin promotion call"
-
- # All three artefact tables had their rows deleted.
- deletedModels = {m[0] for m in db._deletes}
- assert UserMandateRole in deletedModels, "UserMandateRole row not deleted"
- assert AccessRule in deletedModels, "AccessRule row not deleted"
- assert Role in deletedModels, "Sysadmin Role record not deleted"
-
- # And the seeded lists are empty after the migration.
- assert seed["umRoleRows"] == []
- assert seed["accessRules"] == []
- assert seed["sysadminRoles"] == []
-
-
-# ---------------------------------------------------------------------------
-# AC #10 — idempotent: a second run is a no-op
-# ---------------------------------------------------------------------------
-
-
-def testMigrationIsIdempotent():
- """AC#10: a second invocation finds no sysadmin role and exits silently."""
- seed = _seed()
- db = _buildFakeDb(**seed)
-
- _migrateAndDropSysAdminRole(db, _ROOT_MANDATE_ID)
- firstModifies = list(db._modifies)
- firstDeletes = list(db._deletes)
-
- _migrateAndDropSysAdminRole(db, _ROOT_MANDATE_ID)
-
- # No additional writes on the second call.
- assert db._modifies == firstModifies, (
- "Second migration call must not perform additional writes"
- )
- assert db._deletes == firstDeletes, (
- "Second migration call must not perform additional deletes"
- )
-
-
-def testMigrationSkipsAlreadyPromotedUsers():
- """If a user already has ``isPlatformAdmin=True``, no redundant write."""
- seed = _seed()
- seed["users"][0]["isPlatformAdmin"] = True # already promoted
- db = _buildFakeDb(**seed)
-
- _migrateAndDropSysAdminRole(db, _ROOT_MANDATE_ID)
-
- # No promotion write for an already-promoted user.
- promotionWrites = [
- m for m in db._modifies
- if m[0] is UserInDB and m[2].get("isPlatformAdmin") is True
- ]
- assert promotionWrites == [], (
- "Should not re-write isPlatformAdmin if user already has it"
- )
-
- # But role + access-rule cleanup still happens.
- deletedModels = {m[0] for m in db._deletes}
- assert Role in deletedModels
- assert AccessRule in deletedModels
- assert UserMandateRole in deletedModels
-
-
-def testMigrationOnEmptyDbIsNoop():
- """No legacy sysadmin role at all -> no calls, no errors."""
- db = _buildFakeDb(
- sysadminRoles=[],
- umRoleRows=[],
- userMandateRows=[],
- users=[],
- accessRules=[],
- )
-
- _migrateAndDropSysAdminRole(db, _ROOT_MANDATE_ID)
-
- assert db._modifies == []
- assert db._deletes == []
diff --git a/tests/unit/routes/test_folder_crud.py b/tests/unit/routes/test_folder_crud.py
new file mode 100644
index 00000000..86eaf480
--- /dev/null
+++ b/tests/unit/routes/test_folder_crud.py
@@ -0,0 +1,392 @@
+# Copyright (c) 2026 Patrick Motsch
+# All rights reserved.
+"""Unit tests for folder CRUD operations in ComponentObjects."""
+
+import uuid
+import pytest
+from unittest.mock import Mock, patch, MagicMock
+from typing import Dict, Any, List, Optional
+
+from modules.datamodels.datamodelFiles import FileFolder, FileItem
+from modules.datamodels.datamodelUam import User, UserPermissions, AccessLevel
+from modules.interfaces.interfaceDbManagement import ComponentObjects, FileNotFoundError
+
+
+_MANDATE_ID = "mandate-test-1"
+_FEATURE_INSTANCE_ID = "fi-test-1"
+_USER_ID = "user-a-id"
+
+
+# ── Fakes & helpers ──────────────────────────────────────────────────────────
+
+class _FakeDb:
+ """In-memory database mock that mimics DatabaseConnector for unit tests."""
+
+ def __init__(self):
+ self._tables: Dict[str, Dict[str, Dict[str, Any]]] = {}
+ self.connection = MagicMock()
+
+ def getRecordset(self, modelClass, recordFilter=None):
+ tableName = modelClass.__name__
+ records = list(self._tables.get(tableName, {}).values())
+ if not recordFilter:
+ return records
+ return [
+ r for r in records
+ if all(r.get(k) == v for k, v in recordFilter.items())
+ ]
+
+ def recordCreate(self, modelClass, data):
+ tableName = modelClass.__name__
+ self._tables.setdefault(tableName, {})
+ rec = data.model_dump() if hasattr(data, "model_dump") else dict(data)
+ rec.setdefault("id", str(uuid.uuid4()))
+ self._tables[tableName][rec["id"]] = rec
+ return rec
+
+ def recordModify(self, modelClass, recordId, updates):
+ tableName = modelClass.__name__
+ tbl = self._tables.get(tableName, {})
+ if recordId in tbl:
+ tbl[recordId].update(updates)
+ return True
+ return False
+
+ def recordDelete(self, modelClass, recordId):
+ tableName = modelClass.__name__
+ tbl = self._tables.get(tableName, {})
+ if recordId in tbl:
+ del tbl[recordId]
+ return True
+ return False
+
+ def updateContext(self, userId):
+ pass
+
+ def _ensure_connection(self):
+ pass
+
+ def _ensureTableExists(self, modelClass):
+ return True
+
+ def seed(self, modelClass, record: Dict[str, Any]):
+ tableName = modelClass.__name__
+ self._tables.setdefault(tableName, {})
+ self._tables[tableName][record["id"]] = dict(record)
+
+
+def _makeUser(userId=_USER_ID, username="testuser"):
+ return User(id=userId, username=username, language="en")
+
+
+def _makeRbac(
+ createLevel=AccessLevel.ALL,
+ readLevel=AccessLevel.ALL,
+ updateLevel=AccessLevel.ALL,
+ deleteLevel=AccessLevel.ALL,
+):
+ rbac = Mock()
+ perms = UserPermissions(
+ view=True,
+ read=readLevel,
+ create=createLevel,
+ update=updateLevel,
+ delete=deleteLevel,
+ )
+ rbac.getUserPermissions.return_value = perms
+ return rbac
+
+
+def _buildComponent(
+ userId=_USER_ID,
+ fakeDb=None,
+ rbac=None,
+ mandateId=_MANDATE_ID,
+ featureInstanceId=_FEATURE_INSTANCE_ID,
+):
+ """Construct a ComponentObjects with mocked internals (no real DB)."""
+ with patch.object(ComponentObjects, "__init__", lambda self: None):
+ comp = ComponentObjects()
+ comp.db = fakeDb or _FakeDb()
+ comp.currentUser = _makeUser(userId)
+ comp.userId = userId
+ comp.mandateId = mandateId
+ comp.featureInstanceId = featureInstanceId
+ comp.rbac = rbac or _makeRbac()
+ comp.userLanguage = "en"
+ return comp
+
+
+def _rbacFromFakeDb(fakeDb):
+ """Side-effect for getRecordsetWithRBAC that delegates to _FakeDb."""
+ def _fn(connector, modelClass, currentUser, recordFilter=None, **kwargs):
+ return fakeDb.getRecordset(modelClass, recordFilter=recordFilter)
+ return _fn
+
+
+def _makeFolder(
+ folderId=None, name="Folder", parentId=None,
+ userId=_USER_ID, scope="personal", neutralize=False,
+):
+ return {
+ "id": folderId or str(uuid.uuid4()),
+ "name": name,
+ "parentId": parentId,
+ "mandateId": _MANDATE_ID,
+ "featureInstanceId": _FEATURE_INSTANCE_ID,
+ "scope": scope,
+ "neutralize": neutralize,
+ "sysCreatedBy": userId,
+ "sysCreatedAt": 1700000000.0,
+ "sysModifiedAt": 1700000000.0,
+ "sysModifiedBy": None,
+ }
+
+
+def _makeFile(fileId=None, folderId=None, userId=_USER_ID, scope="personal"):
+ return {
+ "id": fileId or str(uuid.uuid4()),
+ "fileName": "test.txt",
+ "mimeType": "text/plain",
+ "fileHash": "abc123",
+ "fileSize": 100,
+ "folderId": folderId,
+ "mandateId": _MANDATE_ID,
+ "featureInstanceId": _FEATURE_INSTANCE_ID,
+ "scope": scope,
+ "neutralize": False,
+ "sysCreatedBy": userId,
+ "sysCreatedAt": 1700000000.0,
+ "sysModifiedAt": 1700000000.0,
+ "sysModifiedBy": None,
+ "tags": None,
+ "description": None,
+ "status": None,
+ }
+
+
+# ── Test class ───────────────────────────────────────────────────────────────
+
+@patch("modules.interfaces.interfaceDbManagement.getRecordsetWithRBAC")
+class TestFolderCrud:
+ """Tests for folder create / rename / move / delete / patch operations."""
+
+ # ── Create ────────────────────────────────────────────────────────────
+
+ def testCreateFolderHappyPath(self, mockRbacGet):
+ fakeDb = _FakeDb()
+ comp = _buildComponent(fakeDb=fakeDb)
+ mockRbacGet.side_effect = _rbacFromFakeDb(fakeDb)
+
+ result = comp.createFolder("Test Folder")
+
+ assert result["name"] == "Test Folder"
+ assert result["scope"] == "personal"
+ assert result["parentId"] is None
+ assert result["mandateId"] == _MANDATE_ID
+
+ def testCreateFolderWithParent(self, mockRbacGet):
+ fakeDb = _FakeDb()
+ fakeDb.seed(FileFolder, _makeFolder(folderId="parent-1", name="Parent"))
+ comp = _buildComponent(fakeDb=fakeDb)
+ mockRbacGet.side_effect = _rbacFromFakeDb(fakeDb)
+
+ result = comp.createFolder("Child Folder", parentId="parent-1")
+
+ assert result["name"] == "Child Folder"
+ assert result["parentId"] == "parent-1"
+
+ def testCreateFolderMissingNameNoInterfaceValidation(self, mockRbacGet):
+ """Interface does not validate empty name; the route layer returns 400."""
+ fakeDb = _FakeDb()
+ comp = _buildComponent(fakeDb=fakeDb)
+ mockRbacGet.side_effect = _rbacFromFakeDb(fakeDb)
+
+ result = comp.createFolder("")
+ assert result["name"] == ""
+
+ # ── Rename ────────────────────────────────────────────────────────────
+
+ def testRenameFolderHappyPath(self, mockRbacGet):
+ fakeDb = _FakeDb()
+ fakeDb.seed(FileFolder, _makeFolder(folderId="f-1", name="Old Name"))
+ comp = _buildComponent(fakeDb=fakeDb)
+ mockRbacGet.side_effect = _rbacFromFakeDb(fakeDb)
+
+ result = comp.renameFolder("f-1", "New Name")
+
+ assert result["name"] == "New Name"
+ assert fakeDb.getRecordset(FileFolder, {"id": "f-1"})[0]["name"] == "New Name"
+
+ def testRenameFolderNotFound(self, mockRbacGet):
+ fakeDb = _FakeDb()
+ comp = _buildComponent(fakeDb=fakeDb)
+ mockRbacGet.side_effect = _rbacFromFakeDb(fakeDb)
+
+ with pytest.raises(FileNotFoundError):
+ comp.renameFolder("nonexistent", "New Name")
+
+ # ── Move ──────────────────────────────────────────────────────────────
+
+ def testMoveFolderHappyPath(self, mockRbacGet):
+ fakeDb = _FakeDb()
+ fakeDb.seed(FileFolder, _makeFolder(folderId="f-1", name="Movable"))
+ fakeDb.seed(FileFolder, _makeFolder(folderId="t-1", name="Target"))
+ comp = _buildComponent(fakeDb=fakeDb)
+ mockRbacGet.side_effect = _rbacFromFakeDb(fakeDb)
+
+ result = comp.moveFolder("f-1", "t-1")
+
+ assert result["parentId"] == "t-1"
+ assert fakeDb.getRecordset(FileFolder, {"id": "f-1"})[0]["parentId"] == "t-1"
+
+ def testMoveFolderToRoot(self, mockRbacGet):
+ fakeDb = _FakeDb()
+ fakeDb.seed(FileFolder, _makeFolder(folderId="f-1", name="Nested", parentId="old"))
+ comp = _buildComponent(fakeDb=fakeDb)
+ mockRbacGet.side_effect = _rbacFromFakeDb(fakeDb)
+
+ result = comp.moveFolder("f-1", None)
+
+ assert result["parentId"] is None
+
+ def testMoveFolderCircularReference(self, mockRbacGet):
+ """A -> B -> C: moving A under C creates a cycle."""
+ fakeDb = _FakeDb()
+ fakeDb.seed(FileFolder, _makeFolder(folderId="a", name="A", parentId=None))
+ fakeDb.seed(FileFolder, _makeFolder(folderId="b", name="B", parentId="a"))
+ fakeDb.seed(FileFolder, _makeFolder(folderId="c", name="C", parentId="b"))
+ comp = _buildComponent(fakeDb=fakeDb)
+ mockRbacGet.side_effect = _rbacFromFakeDb(fakeDb)
+
+ with pytest.raises(ValueError, match="circular reference"):
+ comp.moveFolder("a", "c")
+
+ # ── Delete cascade ────────────────────────────────────────────────────
+
+ def testDeleteFolderCascade(self, mockRbacGet):
+ """Deleting root folder removes root + child + their files."""
+ fakeDb = _FakeDb()
+ fakeDb.seed(FileFolder, _makeFolder(folderId="root", name="Root"))
+ fakeDb.seed(FileFolder, _makeFolder(folderId="child", name="Child", parentId="root"))
+ fakeDb.seed(FileItem, _makeFile(fileId="file-1", folderId="root"))
+ fakeDb.seed(FileItem, _makeFile(fileId="file-2", folderId="child"))
+ comp = _buildComponent(fakeDb=fakeDb)
+ mockRbacGet.side_effect = _rbacFromFakeDb(fakeDb)
+
+ result = comp.deleteFolderCascade("root")
+
+ assert result["deletedFolders"] == 2
+ assert result["deletedFiles"] == 2
+
+ def testDeleteFolderNotFound(self, mockRbacGet):
+ fakeDb = _FakeDb()
+ comp = _buildComponent(fakeDb=fakeDb)
+ mockRbacGet.side_effect = _rbacFromFakeDb(fakeDb)
+
+ with pytest.raises(FileNotFoundError):
+ comp.deleteFolderCascade("nonexistent")
+
+ # ── Patch scope ───────────────────────────────────────────────────────
+
+ def testPatchScopeNoCascade(self, mockRbacGet):
+ """Change folder scope without cascading to files."""
+ fakeDb = _FakeDb()
+ fakeDb.seed(FileFolder, _makeFolder(folderId="f-1", scope="personal"))
+ fakeDb.seed(FileItem, _makeFile(fileId="file-1", folderId="f-1"))
+ comp = _buildComponent(fakeDb=fakeDb)
+ mockRbacGet.side_effect = _rbacFromFakeDb(fakeDb)
+
+ result = comp.patchFolderScope("f-1", "mandate", cascadeToFiles=False)
+
+ assert result["scope"] == "mandate"
+ assert result["filesUpdated"] == 0
+ assert fakeDb.getRecordset(FileFolder, {"id": "f-1"})[0]["scope"] == "mandate"
+ assert fakeDb.getRecordset(FileItem, {"id": "file-1"})[0]["scope"] == "personal"
+
+ def testPatchScopeWithCascade(self, mockRbacGet):
+ """cascadeToFiles=True updates only owned files in the folder."""
+ fakeDb = _FakeDb()
+ fakeDb.seed(FileFolder, _makeFolder(folderId="f-1", scope="personal"))
+ fakeDb.seed(FileItem, _makeFile(fileId="own-file", folderId="f-1"))
+ fakeDb.seed(FileItem, _makeFile(fileId="other-file", folderId="f-1", userId="user-b"))
+ comp = _buildComponent(fakeDb=fakeDb)
+ mockRbacGet.side_effect = _rbacFromFakeDb(fakeDb)
+
+ result = comp.patchFolderScope("f-1", "mandate", cascadeToFiles=True)
+
+ assert result["filesUpdated"] == 1
+ assert fakeDb.getRecordset(FileItem, {"id": "own-file"})[0]["scope"] == "mandate"
+ assert fakeDb.getRecordset(FileItem, {"id": "other-file"})[0]["scope"] == "personal"
+
+ def testPatchScopeInvalid(self, mockRbacGet):
+ fakeDb = _FakeDb()
+ fakeDb.seed(FileFolder, _makeFolder(folderId="f-1"))
+ comp = _buildComponent(fakeDb=fakeDb)
+ mockRbacGet.side_effect = _rbacFromFakeDb(fakeDb)
+
+ with pytest.raises(ValueError, match="Invalid scope"):
+ comp.patchFolderScope("f-1", "invalid_scope")
+
+ # ── Patch neutralize ──────────────────────────────────────────────────
+
+ def testPatchNeutralizeToggle(self, mockRbacGet):
+ fakeDb = _FakeDb()
+ fakeDb.seed(FileFolder, _makeFolder(folderId="f-1", neutralize=False))
+ fakeDb.seed(FileItem, _makeFile(fileId="file-1", folderId="f-1"))
+ comp = _buildComponent(fakeDb=fakeDb)
+ mockRbacGet.side_effect = _rbacFromFakeDb(fakeDb)
+
+ resultOn = comp.patchFolderNeutralize("f-1", True)
+ assert resultOn["neutralize"] is True
+ assert resultOn["filesUpdated"] == 1
+ assert fakeDb.getRecordset(FileFolder, {"id": "f-1"})[0]["neutralize"] is True
+ assert fakeDb.getRecordset(FileItem, {"id": "file-1"})[0]["neutralize"] is True
+
+ resultOff = comp.patchFolderNeutralize("f-1", False)
+ assert resultOff["neutralize"] is False
+ assert fakeDb.getRecordset(FileItem, {"id": "file-1"})[0]["neutralize"] is False
+
+ # ── Tree queries ──────────────────────────────────────────────────────
+
+ def testGetOwnFolderTree(self, mockRbacGet):
+ fakeDb = _FakeDb()
+ fakeDb.seed(FileFolder, _makeFolder(folderId="own-1", name="Mine"))
+ fakeDb.seed(FileFolder, _makeFolder(folderId="other-1", name="Theirs", userId="user-b"))
+ comp = _buildComponent(fakeDb=fakeDb)
+ mockRbacGet.side_effect = _rbacFromFakeDb(fakeDb)
+
+ result = comp.getOwnFolderTree()
+
+ assert len(result) == 1
+ assert result[0]["id"] == "own-1"
+
+ def testGetSharedFolderTreeWithContextOrphan(self, mockRbacGet):
+ fakeDb = _FakeDb()
+ fakeDb.seed(FileFolder, _makeFolder(folderId="own", name="Own"))
+ fakeDb.seed(FileFolder, _makeFolder(
+ folderId="shared-root", name="Shared Root", userId="user-b", scope="mandate",
+ ))
+ fakeDb.seed(FileFolder, _makeFolder(
+ folderId="shared-child", name="Shared Child", userId="user-b",
+ parentId="shared-root", scope="mandate",
+ ))
+ fakeDb.seed(FileFolder, _makeFolder(
+ folderId="orphan", name="Orphan", userId="user-b",
+ parentId="invisible-parent", scope="mandate",
+ ))
+ comp = _buildComponent(fakeDb=fakeDb)
+ mockRbacGet.side_effect = _rbacFromFakeDb(fakeDb)
+
+ result = comp.getSharedFolderTree()
+
+ ids = {r["id"] for r in result}
+ assert "own" not in ids
+ assert "shared-root" in ids
+ assert "shared-child" in ids
+ assert "orphan" in ids
+
+ byId = {r["id"]: r for r in result}
+ assert byId["shared-root"]["contextOrphan"] is False
+ assert byId["shared-child"]["contextOrphan"] is False
+ assert byId["orphan"]["contextOrphan"] is True
diff --git a/tests/unit/services/test_bootstrap_clickup.py b/tests/unit/services/test_bootstrap_clickup.py
new file mode 100644
index 00000000..87c08c3d
--- /dev/null
+++ b/tests/unit/services/test_bootstrap_clickup.py
@@ -0,0 +1,203 @@
+#!/usr/bin/env python3
+# Copyright (c) 2025 Patrick Motsch
+# All rights reserved.
+"""Bootstrap ClickUp tests with a fake service + knowledge service.
+
+Verifies:
+- Teams → spaces → lists (folderless + folder-based) → tasks traversal.
+- Each task produces a `requestIngestion` call with `sourceKind="clickup_task"`
+ and header + description content-objects.
+- `date_updated` is forwarded as contentVersion → idempotency.
+- Recency filter drops tasks older than `maxAgeDays`.
+- maxWorkspaces / maxListsPerWorkspace / maxTasks caps are respected.
+"""
+
+import asyncio
+import os
+import sys
+import time
+from types import SimpleNamespace
+
+sys.path.insert(0, os.path.join(os.path.dirname(__file__), "../../.."))
+
+from modules.serviceCenter.services.serviceKnowledge.subConnectorSyncClickup import (
+ bootstrapClickup,
+ ClickupBootstrapLimits,
+ _syntheticTaskId,
+)
+
+
+def _nowMs(offsetDays: int = 0) -> str:
+ return str(int((time.time() + offsetDays * 86400) * 1000))
+
+
+class _FakeClickupService:
+ """Records API calls; serves a canned 1-team / 1-space / 1-list / 2-task layout."""
+
+ def __init__(self, taskCount=2, oldTask=False):
+ self._taskCount = taskCount
+ self._oldTask = oldTask # when True, the second task is 400 days old
+ self.calls = []
+
+ async def getAuthorizedTeams(self):
+ self.calls.append(("getAuthorizedTeams",))
+ return {"teams": [{"id": "team-1", "name": "Acme"}]}
+
+ async def getSpaces(self, team_id: str):
+ self.calls.append(("getSpaces", team_id))
+ return {"spaces": [{"id": "space-1", "name": "Engineering"}]}
+
+ async def getFolderlessLists(self, space_id: str):
+ self.calls.append(("getFolderlessLists", space_id))
+ return {"lists": [{"id": "list-1", "name": "Sprint 1"}]}
+
+ async def getFolders(self, space_id: str):
+ self.calls.append(("getFolders", space_id))
+ return {"folders": [{"id": "folder-1", "name": "Subproject"}]}
+
+ async def getListsInFolder(self, folder_id: str):
+ self.calls.append(("getListsInFolder", folder_id))
+ return {"lists": [{"id": "list-2", "name": "Sub-tasks"}]}
+
+ async def getTasksInList(self, list_id: str, *, page=0, include_closed=False, subtasks=True):
+ self.calls.append(("getTasksInList", list_id, page, include_closed))
+ if page > 0:
+ return {"tasks": []}
+ tasks = []
+ for i in range(self._taskCount):
+ tid = f"{list_id}-task-{i}"
+ offsetDays = -400 if (self._oldTask and i == 1) else 0
+ tasks.append({
+ "id": tid,
+ "name": f"Task {i} of {list_id}",
+ "description": f"Plain description for task {i}",
+ "text_content": f"Rich content for task {i}",
+ "status": {"status": "open" if i == 0 else "closed"},
+ "assignees": [{"username": "alice"}],
+ "tags": [{"name": "urgent"}],
+ "date_updated": _nowMs(offsetDays),
+ "date_created": _nowMs(-1),
+ "url": f"https://app.clickup.com/t/{tid}",
+ })
+ return {"tasks": tasks}
+
+
+class _FakeKnowledgeService:
+ def __init__(self, duplicateIds=None):
+ self.calls = []
+ self._duplicates = duplicateIds or set()
+
+ async def requestIngestion(self, job):
+ self.calls.append(job)
+ status = "duplicate" if job.sourceId in self._duplicates else "indexed"
+ return SimpleNamespace(
+ jobId=job.sourceId, status=status, contentHash="h",
+ fileId=job.sourceId, index=None, error=None,
+ )
+
+
+def _adapter(svc):
+ return SimpleNamespace(_svc=svc)
+
+
+def test_bootstrap_walks_team_space_lists_and_tasks():
+ svc = _FakeClickupService(taskCount=2)
+ knowledge = _FakeKnowledgeService()
+ connection = SimpleNamespace(mandateId="m1", userId="u1")
+
+ async def _run():
+ return await bootstrapClickup(
+ connectionId="c1",
+ adapter=_adapter(svc),
+ connection=connection,
+ knowledgeService=knowledge,
+ limits=ClickupBootstrapLimits(maxAgeDays=None),
+ )
+
+ result = asyncio.run(_run())
+ # 2 lists (folderless list-1 + folder's list-2) × 2 tasks each = 4 tasks
+ assert result["indexed"] == 4
+ assert result["workspaces"] == 1
+ assert result["lists"] == 2
+ sourceIds = {c.sourceId for c in knowledge.calls}
+ assert len(sourceIds) == 4
+ for job in knowledge.calls:
+ assert job.sourceKind == "clickup_task"
+ assert job.mimeType == "application/vnd.clickup.task+json"
+ assert job.mandateId == "m1"
+ assert job.provenance["connectionId"] == "c1"
+ assert job.provenance["authority"] == "clickup"
+ assert job.provenance["teamId"] == "team-1"
+ assert job.contentVersion # numeric millisecond string
+ # At least the header content-object is present.
+ ids = [co["contentObjectId"] for co in job.contentObjects]
+ assert "header" in ids
+
+
+def test_bootstrap_reports_duplicates_on_second_run():
+ svc = _FakeClickupService(taskCount=1)
+ duplicates = {
+ _syntheticTaskId("c1", "list-1-task-0"),
+ _syntheticTaskId("c1", "list-2-task-0"),
+ }
+ knowledge = _FakeKnowledgeService(duplicateIds=duplicates)
+ connection = SimpleNamespace(mandateId="m1", userId="u1")
+
+ async def _run():
+ return await bootstrapClickup(
+ connectionId="c1",
+ adapter=_adapter(svc),
+ connection=connection,
+ knowledgeService=knowledge,
+ limits=ClickupBootstrapLimits(maxAgeDays=None),
+ )
+
+ result = asyncio.run(_run())
+ assert result["indexed"] == 0
+ assert result["skippedDuplicate"] == 2
+
+
+def test_bootstrap_skips_tasks_older_than_maxAgeDays():
+ svc = _FakeClickupService(taskCount=2, oldTask=True)
+ knowledge = _FakeKnowledgeService()
+ connection = SimpleNamespace(mandateId="m1", userId="u1")
+
+ async def _run():
+ return await bootstrapClickup(
+ connectionId="c1",
+ adapter=_adapter(svc),
+ connection=connection,
+ knowledgeService=knowledge,
+ limits=ClickupBootstrapLimits(maxAgeDays=180),
+ )
+
+ result = asyncio.run(_run())
+ # 2 lists × (1 recent + 1 skipped old) = 2 indexed + 2 skippedPolicy
+ assert result["indexed"] == 2
+ assert result["skippedPolicy"] == 2
+
+
+def test_bootstrap_maxTasks_caps_ingestion():
+ svc = _FakeClickupService(taskCount=2)
+ knowledge = _FakeKnowledgeService()
+ connection = SimpleNamespace(mandateId="m1", userId="u1")
+
+ async def _run():
+ return await bootstrapClickup(
+ connectionId="c1",
+ adapter=_adapter(svc),
+ connection=connection,
+ knowledgeService=knowledge,
+ limits=ClickupBootstrapLimits(maxAgeDays=None, maxTasks=3),
+ )
+
+ result = asyncio.run(_run())
+ assert result["indexed"] == 3
+
+
+if __name__ == "__main__":
+ test_bootstrap_walks_team_space_lists_and_tasks()
+ test_bootstrap_reports_duplicates_on_second_run()
+ test_bootstrap_skips_tasks_older_than_maxAgeDays()
+ test_bootstrap_maxTasks_caps_ingestion()
+ print("OK — bootstrapClickup tests passed")
diff --git a/tests/unit/services/test_bootstrap_gdrive.py b/tests/unit/services/test_bootstrap_gdrive.py
new file mode 100644
index 00000000..1b88677e
--- /dev/null
+++ b/tests/unit/services/test_bootstrap_gdrive.py
@@ -0,0 +1,225 @@
+#!/usr/bin/env python3
+# Copyright (c) 2025 Patrick Motsch
+# All rights reserved.
+"""Bootstrap Google Drive tests with a fake adapter + knowledge service.
+
+Verifies:
+- Drive walk traverses root → subfolders, respecting `maxDepth`.
+- Every file triggers `requestIngestion` with `sourceKind="gdrive_item"`.
+- Duplicate runs (same modifiedTime revision) report `skippedDuplicate`.
+- Provenance carries `authority="google"` and the Drive file id.
+- Recency filter skips files older than `maxAgeDays`.
+"""
+
+import asyncio
+import os
+import sys
+from dataclasses import dataclass
+from datetime import datetime, timedelta, timezone
+from types import SimpleNamespace
+from typing import Any, Dict, List, Optional
+
+sys.path.insert(0, os.path.join(os.path.dirname(__file__), "../../.."))
+
+from modules.serviceCenter.services.serviceKnowledge.subConnectorSyncGdrive import (
+ bootstrapGdrive,
+ GdriveBootstrapLimits,
+ _syntheticFileId,
+)
+
+
+@dataclass
+class _ExtEntry:
+ name: str
+ path: str
+ isFolder: bool = False
+ size: Optional[int] = None
+ mimeType: Optional[str] = None
+ metadata: Dict[str, Any] = None
+
+
+def _today_iso(offsetDays: int = 0) -> str:
+ return (datetime.now(timezone.utc) + timedelta(days=offsetDays)).strftime("%Y-%m-%dT%H:%M:%SZ")
+
+
+class _FakeDriveAdapter:
+ """Minimal DriveAdapter stand-in.
+
+ Layout:
+ "/" (root) → 2 files + 1 folder (sub)
+ "/sub_id" → 1 file
+ """
+
+ def __init__(self, recent_only: bool = True):
+ self.downloaded: List[str] = []
+ self._recent = _today_iso(0)
+ self._old = _today_iso(-400)
+ self._recent_only = recent_only
+
+ async def browse(self, path: str, filter=None, limit=None):
+ if path in ("/", "", "root"):
+ return [
+ _ExtEntry(
+ name="f1.txt", path="/f1", size=20,
+ mimeType="text/plain",
+ metadata={"id": "f1", "modifiedTime": self._recent},
+ ),
+ _ExtEntry(
+ name="f2.txt", path="/f2", size=20,
+ mimeType="text/plain",
+ metadata={"id": "f2", "modifiedTime": self._recent if self._recent_only else self._old},
+ ),
+ _ExtEntry(
+ name="Subfolder", path="/sub_id", isFolder=True,
+ mimeType="application/vnd.google-apps.folder",
+ metadata={"id": "sub_id", "modifiedTime": self._recent},
+ ),
+ ]
+ if path == "/sub_id":
+ return [
+ _ExtEntry(
+ name="f3.txt", path="/f3", size=20,
+ mimeType="text/plain",
+ metadata={"id": "f3", "modifiedTime": self._recent},
+ ),
+ ]
+ return []
+
+ async def download(self, path: str) -> bytes:
+ self.downloaded.append(path)
+ return path.encode("utf-8")
+
+
+class _FakeKnowledgeService:
+ def __init__(self, duplicateIds=None):
+ self.calls: List[SimpleNamespace] = []
+ self._duplicateIds = duplicateIds or set()
+
+ async def requestIngestion(self, job):
+ self.calls.append(job)
+ status = "duplicate" if job.sourceId in self._duplicateIds else "indexed"
+ return SimpleNamespace(
+ jobId=f"{job.sourceKind}:{job.sourceId}",
+ status=status, contentHash="h",
+ fileId=job.sourceId, index=None, error=None,
+ )
+
+
+def _fakeRunExtraction(data, name, mime, options):
+ return SimpleNamespace(
+ parts=[
+ SimpleNamespace(
+ id="p1",
+ data=data.decode("utf-8") if isinstance(data, bytes) else str(data),
+ typeGroup="text",
+ label="page:1",
+ metadata={"pageIndex": 0},
+ )
+ ]
+ )
+
+
+def test_bootstrap_walks_drive_and_subfolders():
+ adapter = _FakeDriveAdapter()
+ knowledge = _FakeKnowledgeService()
+ connection = SimpleNamespace(mandateId="m1", userId="u1")
+
+ async def _run():
+ return await bootstrapGdrive(
+ connectionId="c1",
+ adapter=adapter,
+ connection=connection,
+ knowledgeService=knowledge,
+ runExtractionFn=_fakeRunExtraction,
+ limits=GdriveBootstrapLimits(maxAgeDays=None),
+ )
+
+ result = asyncio.run(_run())
+ assert len(knowledge.calls) == 3
+ sourceIds = {c.sourceId for c in knowledge.calls}
+ assert sourceIds == {
+ _syntheticFileId("c1", "f1"),
+ _syntheticFileId("c1", "f2"),
+ _syntheticFileId("c1", "f3"),
+ }
+ assert result["indexed"] == 3
+ assert result["skippedDuplicate"] == 0
+ assert adapter.downloaded == ["/f1", "/f2", "/f3"]
+
+
+def test_bootstrap_reports_duplicates_on_second_run():
+ adapter = _FakeDriveAdapter()
+ duplicateIds = {
+ _syntheticFileId("c1", "f1"),
+ _syntheticFileId("c1", "f2"),
+ _syntheticFileId("c1", "f3"),
+ }
+ knowledge = _FakeKnowledgeService(duplicateIds=duplicateIds)
+ connection = SimpleNamespace(mandateId="m1", userId="u1")
+
+ async def _run():
+ return await bootstrapGdrive(
+ connectionId="c1",
+ adapter=adapter,
+ connection=connection,
+ knowledgeService=knowledge,
+ runExtractionFn=_fakeRunExtraction,
+ limits=GdriveBootstrapLimits(maxAgeDays=None),
+ )
+
+ result = asyncio.run(_run())
+ assert result["indexed"] == 0
+ assert result["skippedDuplicate"] == 3
+
+
+def test_bootstrap_skips_files_older_than_maxAgeDays():
+ adapter = _FakeDriveAdapter(recent_only=False) # f2 is 400 days old
+ knowledge = _FakeKnowledgeService()
+ connection = SimpleNamespace(mandateId="m1", userId="u1")
+
+ async def _run():
+ return await bootstrapGdrive(
+ connectionId="c1",
+ adapter=adapter,
+ connection=connection,
+ knowledgeService=knowledge,
+ runExtractionFn=_fakeRunExtraction,
+ limits=GdriveBootstrapLimits(maxAgeDays=180),
+ )
+
+ result = asyncio.run(_run())
+ assert result["indexed"] == 2 # f1, f3
+ assert result["skippedPolicy"] == 1 # f2 filtered out
+
+
+def test_bootstrap_passes_connection_provenance():
+ adapter = _FakeDriveAdapter()
+ knowledge = _FakeKnowledgeService()
+ connection = SimpleNamespace(mandateId="m1", userId="u1")
+
+ async def _run():
+ return await bootstrapGdrive(
+ connectionId="c1",
+ adapter=adapter,
+ connection=connection,
+ knowledgeService=knowledge,
+ runExtractionFn=_fakeRunExtraction,
+ limits=GdriveBootstrapLimits(maxAgeDays=None),
+ )
+
+ asyncio.run(_run())
+ for job in knowledge.calls:
+ assert job.sourceKind == "gdrive_item"
+ assert job.mandateId == "m1"
+ assert job.provenance["connectionId"] == "c1"
+ assert job.provenance["authority"] == "google"
+ assert job.provenance["service"] == "drive"
+ assert job.contentVersion # modifiedTime ISO string
+
+
+if __name__ == "__main__":
+ test_bootstrap_walks_drive_and_subfolders()
+ test_bootstrap_reports_duplicates_on_second_run()
+ test_bootstrap_skips_files_older_than_maxAgeDays()
+ test_bootstrap_passes_connection_provenance()
+ print("OK — bootstrapGdrive tests passed")
diff --git a/tests/unit/services/test_bootstrap_gmail.py b/tests/unit/services/test_bootstrap_gmail.py
new file mode 100644
index 00000000..4f7cfe4d
--- /dev/null
+++ b/tests/unit/services/test_bootstrap_gmail.py
@@ -0,0 +1,240 @@
+#!/usr/bin/env python3
+# Copyright (c) 2025 Patrick Motsch
+# All rights reserved.
+"""Bootstrap Gmail tests with a fake googleGet + knowledge service.
+
+Verifies:
+- Default labels (INBOX + SENT) are traversed.
+- Each message produces a requestIngestion call with sourceKind=gmail_message
+ and structured contentObjects (header / snippet / body).
+- Pagination via `nextPageToken` is followed.
+- historyId is forwarded as contentVersion → idempotency.
+- MIME body extraction walks nested parts (multipart/alternative).
+"""
+
+import asyncio
+import base64
+import os
+import sys
+from types import SimpleNamespace
+
+sys.path.insert(0, os.path.join(os.path.dirname(__file__), "../../.."))
+
+from modules.serviceCenter.services.serviceKnowledge.subConnectorSyncGmail import (
+ bootstrapGmail,
+ GmailBootstrapLimits,
+ _syntheticMessageId,
+ _buildContentObjects,
+ _walkPayloadForBody,
+)
+
+
+def _b64url(text: str) -> str:
+ return base64.urlsafe_b64encode(text.encode("utf-8")).decode("ascii").rstrip("=")
+
+
+def _msg(mid: str, subject: str = "Hi", body: str = "Hello world", historyId: str = "h1"):
+ return {
+ "id": mid,
+ "threadId": f"thread-{mid}",
+ "historyId": historyId,
+ "internalDate": "1700000000000",
+ "snippet": body[:120],
+ "payload": {
+ "headers": [
+ {"name": "Subject", "value": subject},
+ {"name": "From", "value": "Alice "},
+ {"name": "To", "value": "Bob "},
+ {"name": "Date", "value": "Tue, 01 Jan 2025 10:00:00 +0000"},
+ ],
+ "mimeType": "text/plain",
+ "body": {"data": _b64url(body), "size": len(body)},
+ "parts": [],
+ },
+ }
+
+
+class _FakeGoogleGet:
+ """Records URLs + returns the wired-up page or message response."""
+
+ def __init__(self, messages_by_label, paginated_label=None, page2=None):
+ self._messages = messages_by_label
+ self._paginated = paginated_label
+ self._page2 = page2 or []
+ self._served_first_page = set()
+ self.requested = []
+
+ async def __call__(self, url: str):
+ self.requested.append(url)
+ # List page: contains `/users/me/messages?labelIds=...`
+ if "/users/me/messages?" in url:
+ for label, msgs in self._messages.items():
+ if f"labelIds={label}" in url:
+ if (
+ label == self._paginated
+ and label not in self._served_first_page
+ ):
+ self._served_first_page.add(label)
+ return {
+ "messages": [{"id": m["id"]} for m in msgs],
+ "nextPageToken": "token-2",
+ }
+ if label == self._paginated and "pageToken=token-2" in url:
+ return {
+ "messages": [{"id": m["id"]} for m in self._page2],
+ }
+ return {"messages": [{"id": m["id"]} for m in msgs]}
+ return {"messages": []}
+ # Detail fetch: /users/me/messages/{id}?format=full
+ if "/users/me/messages/" in url and "format=full" in url:
+ msgId = url.split("/users/me/messages/")[-1].split("?")[0]
+ for msgs in self._messages.values():
+ for m in msgs:
+ if m["id"] == msgId:
+ return m
+ for m in self._page2:
+ if m["id"] == msgId:
+ return m
+ return {"error": "not found"}
+
+
+class _FakeKnowledgeService:
+ def __init__(self, duplicateIds=None):
+ self.calls = []
+ self._duplicates = duplicateIds or set()
+
+ async def requestIngestion(self, job):
+ self.calls.append(job)
+ status = "duplicate" if job.sourceId in self._duplicates else "indexed"
+ return SimpleNamespace(
+ jobId=job.sourceId, status=status, contentHash="h",
+ fileId=job.sourceId, index=None, error=None,
+ )
+
+
+def test_buildContentObjects_emits_header_snippet_body():
+ parts = _buildContentObjects(_msg("m1", body="Hello\nWorld"), maxBodyChars=8000)
+ ids = [p["contentObjectId"] for p in parts]
+ assert ids == ["header", "snippet", "body"]
+ header = parts[0]["data"]
+ assert "Subject: Hi" in header
+ assert "From: Alice " in header
+ assert "To: Bob " in header
+
+
+def test_walkPayloadForBody_prefers_plain_over_html():
+ payload = {
+ "mimeType": "multipart/alternative",
+ "parts": [
+ {"mimeType": "text/plain", "body": {"data": _b64url("plain body")}},
+ {"mimeType": "text/html", "body": {"data": _b64url("html body
")}},
+ ],
+ }
+ bodies = _walkPayloadForBody(payload)
+ assert bodies["text"] == "plain body"
+ assert bodies["html"] == "html body
"
+
+
+def test_walkPayloadForBody_falls_back_to_html():
+ payload = {
+ "mimeType": "multipart/alternative",
+ "parts": [
+ {"mimeType": "text/html", "body": {"data": _b64url("only html
")}},
+ ],
+ }
+ bodies = _walkPayloadForBody(payload)
+ assert bodies["text"] == ""
+ assert "only html" in bodies["html"]
+
+
+def test_bootstrap_gmail_indexes_messages_from_inbox_and_sent():
+ fake_get = _FakeGoogleGet({
+ "INBOX": [_msg("m1"), _msg("m2")],
+ "SENT": [_msg("m3")],
+ })
+ knowledge = _FakeKnowledgeService()
+ connection = SimpleNamespace(mandateId="m1", userId="u1")
+
+ async def _run():
+ return await bootstrapGmail(
+ connectionId="c1",
+ adapter=SimpleNamespace(_token="t"),
+ connection=connection,
+ knowledgeService=knowledge,
+ limits=GmailBootstrapLimits(maxAgeDays=None),
+ googleGetFn=fake_get,
+ )
+
+ result = asyncio.run(_run())
+ assert result["indexed"] == 3
+ sourceIds = {c.sourceId for c in knowledge.calls}
+ assert sourceIds == {
+ _syntheticMessageId("c1", "m1"),
+ _syntheticMessageId("c1", "m2"),
+ _syntheticMessageId("c1", "m3"),
+ }
+ for job in knowledge.calls:
+ assert job.sourceKind == "gmail_message"
+ assert job.mimeType == "message/rfc822"
+ assert job.provenance["connectionId"] == "c1"
+ assert job.provenance["authority"] == "google"
+ assert job.provenance["service"] == "gmail"
+ assert job.contentVersion == "h1"
+ assert any(co["contentObjectId"] == "header" for co in job.contentObjects)
+
+
+def test_bootstrap_gmail_follows_pagination():
+ fake_get = _FakeGoogleGet(
+ messages_by_label={"INBOX": [_msg("m1")], "SENT": []},
+ paginated_label="INBOX",
+ page2=[_msg("m2"), _msg("m3")],
+ )
+ knowledge = _FakeKnowledgeService()
+ connection = SimpleNamespace(mandateId="m1", userId="u1")
+
+ async def _run():
+ return await bootstrapGmail(
+ connectionId="c1",
+ adapter=SimpleNamespace(_token="t"),
+ connection=connection,
+ knowledgeService=knowledge,
+ limits=GmailBootstrapLimits(maxAgeDays=None),
+ googleGetFn=fake_get,
+ )
+
+ result = asyncio.run(_run())
+ assert result["indexed"] == 3
+
+
+def test_bootstrap_gmail_reports_duplicates():
+ fake_get = _FakeGoogleGet({"INBOX": [_msg("m1"), _msg("m2")], "SENT": []})
+ duplicates = {
+ _syntheticMessageId("c1", "m1"),
+ _syntheticMessageId("c1", "m2"),
+ }
+ knowledge = _FakeKnowledgeService(duplicateIds=duplicates)
+ connection = SimpleNamespace(mandateId="m1", userId="u1")
+
+ async def _run():
+ return await bootstrapGmail(
+ connectionId="c1",
+ adapter=SimpleNamespace(_token="t"),
+ connection=connection,
+ knowledgeService=knowledge,
+ limits=GmailBootstrapLimits(maxAgeDays=None),
+ googleGetFn=fake_get,
+ )
+
+ result = asyncio.run(_run())
+ assert result["indexed"] == 0
+ assert result["skippedDuplicate"] == 2
+
+
+if __name__ == "__main__":
+ test_buildContentObjects_emits_header_snippet_body()
+ test_walkPayloadForBody_prefers_plain_over_html()
+ test_walkPayloadForBody_falls_back_to_html()
+ test_bootstrap_gmail_indexes_messages_from_inbox_and_sent()
+ test_bootstrap_gmail_follows_pagination()
+ test_bootstrap_gmail_reports_duplicates()
+ print("OK — bootstrapGmail tests passed")
diff --git a/tests/unit/services/test_bootstrap_outlook.py b/tests/unit/services/test_bootstrap_outlook.py
new file mode 100644
index 00000000..26664eaa
--- /dev/null
+++ b/tests/unit/services/test_bootstrap_outlook.py
@@ -0,0 +1,190 @@
+#!/usr/bin/env python3
+# Copyright (c) 2025 Patrick Motsch
+# All rights reserved.
+"""Bootstrap Outlook tests with a fake adapter + knowledge service.
+
+Verifies:
+- Well-known folders (inbox, sentitems) are discovered via Graph.
+- Each message produces a `requestIngestion` call with sourceKind=outlook_message
+ and structured contentObjects (header / snippet / body).
+- Pagination via `@odata.nextLink` is followed.
+- changeKey is forwarded as contentVersion → idempotency.
+"""
+
+import asyncio
+import os
+import sys
+from types import SimpleNamespace
+
+sys.path.insert(0, os.path.join(os.path.dirname(__file__), "../../.."))
+
+from modules.serviceCenter.services.serviceKnowledge.subConnectorSyncOutlook import (
+ bootstrapOutlook,
+ OutlookBootstrapLimits,
+ _syntheticMessageId,
+ _buildContentObjects,
+)
+
+
+class _FakeOutlookAdapter:
+ def __init__(self, messages_by_folder, paginated_folder=None, page2=None):
+ self._folders = {"inbox": "INBOX-ID", "sentitems": "SENT-ID"}
+ self._messages = messages_by_folder
+ self._paginated_folder = paginated_folder
+ self._page2 = page2 or []
+ self.requested_endpoints = []
+
+ async def _graphGet(self, endpoint: str):
+ self.requested_endpoints.append(endpoint)
+ if endpoint.startswith("me/mailFolders/") and "/messages" not in endpoint:
+ wellKnown = endpoint.split("/")[-1]
+ fid = self._folders.get(wellKnown)
+ if not fid:
+ return {"error": "not found"}
+ return {"id": fid, "displayName": wellKnown}
+ # message page request: e.g. me/mailFolders/INBOX-ID/messages?...
+ for fid, messages in self._messages.items():
+ if f"me/mailFolders/{fid}/messages" in endpoint:
+ page = {"value": messages}
+ if fid == self._paginated_folder and "skiptoken" not in endpoint:
+ page["@odata.nextLink"] = (
+ "https://graph.microsoft.com/v1.0/"
+ f"me/mailFolders/{fid}/messages?$skiptoken=abc"
+ )
+ elif fid == self._paginated_folder and "skiptoken" in endpoint:
+ page = {"value": self._page2}
+ return page
+ return {"value": []}
+
+ async def browse(self, path):
+ return []
+
+
+class _FakeKnowledgeService:
+ def __init__(self, duplicateIds=None):
+ self.calls = []
+ self._duplicates = duplicateIds or set()
+
+ async def requestIngestion(self, job):
+ self.calls.append(job)
+ status = "duplicate" if job.sourceId in self._duplicates else "indexed"
+ return SimpleNamespace(
+ jobId=job.sourceId, status=status, contentHash="h",
+ fileId=job.sourceId, index=None, error=None,
+ )
+
+
+def _msg(mid: str, subject: str = "Hi", change: str = "ck1"):
+ return {
+ "id": mid,
+ "subject": subject,
+ "from": {"emailAddress": {"name": "Alice", "address": "a@x.com"}},
+ "toRecipients": [{"emailAddress": {"name": "Bob", "address": "b@x.com"}}],
+ "ccRecipients": [],
+ "receivedDateTime": "2025-01-01T10:00:00Z",
+ "bodyPreview": "Hello world",
+ "body": {"contentType": "text", "content": "Hello world\nThis is the body."},
+ "internetMessageId": f"<{mid}@local>",
+ "hasAttachments": False,
+ "changeKey": change,
+ }
+
+
+def test_buildContentObjects_emits_header_snippet_body():
+ parts = _buildContentObjects(_msg("m1"), maxBodyChars=8000)
+ ids = [p["contentObjectId"] for p in parts]
+ assert ids == ["header", "snippet", "body"]
+ header = parts[0]["data"]
+ assert "Subject: Hi" in header
+ assert "From: Alice " in header
+ assert "To: Bob " in header
+
+
+def test_bootstrap_outlook_indexes_messages_from_inbox_and_sent():
+ adapter = _FakeOutlookAdapter({
+ "INBOX-ID": [_msg("m1"), _msg("m2")],
+ "SENT-ID": [_msg("m3")],
+ })
+ knowledge = _FakeKnowledgeService()
+ connection = SimpleNamespace(mandateId="m1", userId="u1")
+
+ async def _run():
+ return await bootstrapOutlook(
+ connectionId="c1",
+ adapter=adapter,
+ connection=connection,
+ knowledgeService=knowledge,
+ limits=OutlookBootstrapLimits(maxAgeDays=None),
+ )
+
+ result = asyncio.run(_run())
+ assert result["indexed"] == 3
+ sourceIds = {c.sourceId for c in knowledge.calls}
+ assert sourceIds == {
+ _syntheticMessageId("c1", "m1"),
+ _syntheticMessageId("c1", "m2"),
+ _syntheticMessageId("c1", "m3"),
+ }
+ for job in knowledge.calls:
+ assert job.sourceKind == "outlook_message"
+ assert job.mimeType == "message/rfc822"
+ assert job.provenance["connectionId"] == "c1"
+ assert job.provenance["service"] == "outlook"
+ assert job.contentVersion == "ck1"
+ assert any(co["contentObjectId"] == "header" for co in job.contentObjects)
+
+
+def test_bootstrap_outlook_follows_pagination():
+ adapter = _FakeOutlookAdapter(
+ messages_by_folder={"INBOX-ID": [_msg("m1")], "SENT-ID": []},
+ paginated_folder="INBOX-ID",
+ page2=[_msg("m2"), _msg("m3")],
+ )
+ knowledge = _FakeKnowledgeService()
+ connection = SimpleNamespace(mandateId="m1", userId="u1")
+
+ async def _run():
+ return await bootstrapOutlook(
+ connectionId="c1",
+ adapter=adapter,
+ connection=connection,
+ knowledgeService=knowledge,
+ limits=OutlookBootstrapLimits(maxAgeDays=None),
+ )
+
+ result = asyncio.run(_run())
+ assert result["indexed"] == 3
+
+
+def test_bootstrap_outlook_reports_duplicates():
+ adapter = _FakeOutlookAdapter({
+ "INBOX-ID": [_msg("m1"), _msg("m2")],
+ "SENT-ID": [],
+ })
+ duplicates = {
+ _syntheticMessageId("c1", "m1"),
+ _syntheticMessageId("c1", "m2"),
+ }
+ knowledge = _FakeKnowledgeService(duplicateIds=duplicates)
+ connection = SimpleNamespace(mandateId="m1", userId="u1")
+
+ async def _run():
+ return await bootstrapOutlook(
+ connectionId="c1",
+ adapter=adapter,
+ connection=connection,
+ knowledgeService=knowledge,
+ limits=OutlookBootstrapLimits(maxAgeDays=None),
+ )
+
+ result = asyncio.run(_run())
+ assert result["indexed"] == 0
+ assert result["skippedDuplicate"] == 2
+
+
+if __name__ == "__main__":
+ test_buildContentObjects_emits_header_snippet_body()
+ test_bootstrap_outlook_indexes_messages_from_inbox_and_sent()
+ test_bootstrap_outlook_follows_pagination()
+ test_bootstrap_outlook_reports_duplicates()
+ print("OK — bootstrapOutlook tests passed")
diff --git a/tests/unit/services/test_bootstrap_sharepoint.py b/tests/unit/services/test_bootstrap_sharepoint.py
new file mode 100644
index 00000000..8b011357
--- /dev/null
+++ b/tests/unit/services/test_bootstrap_sharepoint.py
@@ -0,0 +1,209 @@
+#!/usr/bin/env python3
+# Copyright (c) 2025 Patrick Motsch
+# All rights reserved.
+"""Bootstrap SharePoint tests with a fake adapter + knowledge service.
+
+Verifies:
+- Every discovered file triggers `requestIngestion`.
+- Duplicate runs (same eTag revisions) report `skippedDuplicate`.
+- Synthetic fileIds are stable across runs so idempotency works end-to-end.
+"""
+
+import asyncio
+import os
+import sys
+from dataclasses import dataclass
+from types import SimpleNamespace
+from typing import Any, Dict, List, Optional
+
+sys.path.insert(0, os.path.join(os.path.dirname(__file__), "../../.."))
+
+from modules.serviceCenter.services.serviceKnowledge.subConnectorSyncSharepoint import (
+ bootstrapSharepoint,
+ _syntheticFileId,
+)
+
+
+@dataclass
+class _ExtEntry:
+ name: str
+ path: str
+ isFolder: bool = False
+ size: Optional[int] = None
+ mimeType: Optional[str] = None
+ metadata: Dict[str, Any] = None
+
+
+class _FakeSpAdapter:
+ """Minimal SharepointAdapter stand-in.
+
+ Layout:
+ "/" → 1 site
+ "/sites/site-1" → 2 files (f1, f2) + 1 folder (sub)
+ "/sites/site-1/sub" → 1 file (f3)
+ """
+
+ def __init__(self):
+ self.downloaded: List[str] = []
+
+ async def browse(self, path: str, filter=None, limit=None):
+ if path == "/":
+ return [
+ _ExtEntry(
+ name="Site 1",
+ path="/sites/site-1",
+ isFolder=True,
+ metadata={"id": "site-1"},
+ ),
+ ]
+ if path == "/sites/site-1":
+ return [
+ _ExtEntry(
+ name="f1.txt", path="/sites/site-1/f1.txt",
+ mimeType="text/plain", size=20,
+ metadata={"id": "f1", "revision": "etag-f1"},
+ ),
+ _ExtEntry(
+ name="f2.txt", path="/sites/site-1/f2.txt",
+ mimeType="text/plain", size=20,
+ metadata={"id": "f2", "revision": "etag-f2"},
+ ),
+ _ExtEntry(
+ name="sub", path="/sites/site-1/sub",
+ isFolder=True, metadata={"id": "sub"},
+ ),
+ ]
+ if path == "/sites/site-1/sub":
+ return [
+ _ExtEntry(
+ name="f3.txt", path="/sites/site-1/sub/f3.txt",
+ mimeType="text/plain", size=20,
+ metadata={"id": "f3", "revision": "etag-f3"},
+ ),
+ ]
+ return []
+
+ async def download(self, path: str) -> bytes:
+ self.downloaded.append(path)
+ return path.encode("utf-8")
+
+
+class _FakeKnowledgeService:
+ """Records requestIngestion calls and returns the scripted handles."""
+
+ def __init__(self, duplicateIds=None):
+ self.calls: List[SimpleNamespace] = []
+ self._duplicateIds = duplicateIds or set()
+
+ async def requestIngestion(self, job):
+ self.calls.append(job)
+ status = "duplicate" if job.sourceId in self._duplicateIds else "indexed"
+ return SimpleNamespace(
+ jobId=f"{job.sourceKind}:{job.sourceId}",
+ status=status,
+ contentHash="h",
+ fileId=job.sourceId,
+ index=None,
+ error=None,
+ )
+
+
+def _fakeRunExtraction(data, name, mime, options):
+ """Produce a single synthetic text part so `_toContentObjects` returns one."""
+ return SimpleNamespace(
+ parts=[
+ SimpleNamespace(
+ id="p1",
+ data=data.decode("utf-8") if isinstance(data, bytes) else str(data),
+ typeGroup="text",
+ label="page:1",
+ metadata={"pageIndex": 0},
+ )
+ ]
+ )
+
+
+def test_bootstrap_walks_sites_and_subfolders():
+ adapter = _FakeSpAdapter()
+ knowledge = _FakeKnowledgeService()
+ connection = SimpleNamespace(mandateId="m1", userId="u1")
+
+ async def _run():
+ return await bootstrapSharepoint(
+ connectionId="c1",
+ adapter=adapter,
+ connection=connection,
+ knowledgeService=knowledge,
+ runExtractionFn=_fakeRunExtraction,
+ )
+
+ result = asyncio.run(_run())
+ assert len(knowledge.calls) == 3
+ sourceIds = {c.sourceId for c in knowledge.calls}
+ assert sourceIds == {
+ _syntheticFileId("c1", "f1"),
+ _syntheticFileId("c1", "f2"),
+ _syntheticFileId("c1", "f3"),
+ }
+ assert result["indexed"] == 3
+ assert result["skippedDuplicate"] == 0
+ assert adapter.downloaded == [
+ "/sites/site-1/f1.txt",
+ "/sites/site-1/f2.txt",
+ "/sites/site-1/sub/f3.txt",
+ ]
+
+
+def test_bootstrap_reports_duplicates_on_second_run():
+ adapter = _FakeSpAdapter()
+ duplicateIds = {
+ _syntheticFileId("c1", "f1"),
+ _syntheticFileId("c1", "f2"),
+ _syntheticFileId("c1", "f3"),
+ }
+ knowledge = _FakeKnowledgeService(duplicateIds=duplicateIds)
+ connection = SimpleNamespace(mandateId="m1", userId="u1")
+
+ async def _run():
+ return await bootstrapSharepoint(
+ connectionId="c1",
+ adapter=adapter,
+ connection=connection,
+ knowledgeService=knowledge,
+ runExtractionFn=_fakeRunExtraction,
+ )
+
+ result = asyncio.run(_run())
+ assert result["indexed"] == 0
+ assert result["skippedDuplicate"] == 3
+
+
+def test_bootstrap_passes_connection_provenance():
+ adapter = _FakeSpAdapter()
+ knowledge = _FakeKnowledgeService()
+ connection = SimpleNamespace(mandateId="m1", userId="u1")
+
+ async def _run():
+ return await bootstrapSharepoint(
+ connectionId="c1",
+ adapter=adapter,
+ connection=connection,
+ knowledgeService=knowledge,
+ runExtractionFn=_fakeRunExtraction,
+ )
+
+ asyncio.run(_run())
+ for job in knowledge.calls:
+ assert job.sourceKind == "sharepoint_item"
+ assert job.mandateId == "m1"
+ assert job.provenance["connectionId"] == "c1"
+ assert job.provenance["authority"] == "msft"
+ assert job.provenance["service"] == "sharepoint"
+ assert job.contentVersion and job.contentVersion.startswith("etag-")
+
+
+if __name__ == "__main__":
+ test_bootstrap_walks_sites_and_subfolders()
+ test_bootstrap_reports_duplicates_on_second_run()
+ test_bootstrap_passes_connection_provenance()
+ print("OK — bootstrapSharepoint tests passed")
diff --git a/tests/unit/services/test_clean_email_body.py b/tests/unit/services/test_clean_email_body.py
new file mode 100644
index 00000000..a3ee01df
--- /dev/null
+++ b/tests/unit/services/test_clean_email_body.py
@@ -0,0 +1,110 @@
+#!/usr/bin/env python3
+# Copyright (c) 2025 Patrick Motsch
+# All rights reserved.
+"""Unit tests for cleanEmailBody.
+
+Covers: HTML→text normalisation, quoted-reply removal, signature removal,
+whitespace collapse and truncation. The utility is used during Outlook
+bootstrap; buggy cleaning would leak quoted threads / signatures into every
+embedding.
+"""
+
+import os
+import sys
+
+sys.path.insert(0, os.path.join(os.path.dirname(__file__), "../../.."))
+
+from modules.serviceCenter.services.serviceKnowledge.subTextClean import (
+ cleanEmailBody,
+)
+
+
+def test_strips_html_tags_and_scripts():
+ html = (
+ ""
+ "Hello world
"
+ ""
+ )
+ cleaned = cleanEmailBody(html)
+ assert "Hello" in cleaned
+ assert "world" in cleaned
+ assert "<" not in cleaned
+ assert "alert" not in cleaned
+
+
+def test_strips_quoted_reply_english():
+ body = (
+ "Actual answer from me.\n\n"
+ "On Mon, 1 Jan 2024 at 10:00, Someone wrote:\n"
+ "> Original question?\n"
+ "> Second line.\n"
+ )
+ cleaned = cleanEmailBody(body)
+ assert "Actual answer" in cleaned
+ assert "Original question" not in cleaned
+ assert "wrote:" not in cleaned
+
+
+def test_strips_quoted_reply_german():
+ body = (
+ "Meine Antwort.\n\n"
+ "Am 1. Januar 2024 um 10:00 schrieb Max Muster :\n"
+ "> Ursprüngliche Frage?\n"
+ )
+ cleaned = cleanEmailBody(body)
+ assert "Meine Antwort" in cleaned
+ assert "Ursprüngliche Frage" not in cleaned
+
+
+def test_strips_signature_after_dashes():
+ body = (
+ "Kurze Nachricht.\n"
+ "\n"
+ "--\n"
+ "Max Muster\n"
+ "Vorstand, Beispiel GmbH\n"
+ )
+ cleaned = cleanEmailBody(body)
+ assert "Kurze Nachricht" in cleaned
+ assert "Beispiel GmbH" not in cleaned
+
+
+def test_strips_signature_salutation_de():
+ body = (
+ "Die eigentliche Information steht hier.\n\n"
+ "Mit freundlichen Grüßen\n"
+ "Max Muster"
+ )
+ cleaned = cleanEmailBody(body)
+ assert "eigentliche Information" in cleaned
+ assert "Max Muster" not in cleaned
+
+
+def test_truncate_to_max_chars():
+ body = "abc " * 5000
+ cleaned = cleanEmailBody(body, maxChars=200)
+ assert len(cleaned) <= 201 # includes trailing ellipsis
+
+
+def test_empty_input_returns_empty_string():
+ assert cleanEmailBody("") == ""
+ assert cleanEmailBody(None) == "" # type: ignore[arg-type]
+
+
+def test_collapses_whitespace():
+ body = "A lot of spaces\n\n\n\nand blank lines"
+ cleaned = cleanEmailBody(body)
+ assert " " not in cleaned
+ assert "\n\n\n" not in cleaned
+
+
+if __name__ == "__main__":
+ test_strips_html_tags_and_scripts()
+ test_strips_quoted_reply_english()
+ test_strips_quoted_reply_german()
+ test_strips_signature_after_dashes()
+ test_strips_signature_salutation_de()
+ test_truncate_to_max_chars()
+ test_empty_input_returns_empty_string()
+ test_collapses_whitespace()
+ print("OK — cleanEmailBody tests passed")
diff --git a/tests/unit/services/test_connection_purge.py b/tests/unit/services/test_connection_purge.py
new file mode 100644
index 00000000..c32cb5b3
--- /dev/null
+++ b/tests/unit/services/test_connection_purge.py
@@ -0,0 +1,119 @@
+#!/usr/bin/env python3
+# Copyright (c) 2025 Patrick Motsch
+# All rights reserved.
+"""Purge tests for KnowledgeObjects.deleteFileContentIndexByConnectionId.
+
+Ensures that a `connection.revoked` event wipes every FileContentIndex + chunk
+linked to the given connectionId while leaving entries from other connections
+(or upload-files with connectionId=None) intact.
+"""
+
+import os
+import sys
+
+sys.path.insert(0, os.path.join(os.path.dirname(__file__), "../../.."))
+
+from modules.datamodels.datamodelKnowledge import FileContentIndex, ContentChunk
+from modules.interfaces.interfaceDbKnowledge import KnowledgeObjects
+
+
+class _FakeDb:
+ """Minimal in-memory stand-in for ``KnowledgeObjects.db``.
+
+ Supports just the subset of APIs that deleteFileContentIndexByConnectionId
+ touches: getRecordset(FileContentIndex|ContentChunk, ...) + recordDelete.
+ """
+
+ def __init__(self):
+ self.indexRows: dict = {}
+ self.chunks: dict = {}
+
+ def addIndex(self, row: dict) -> None:
+ self.indexRows[row["id"]] = row
+
+ def addChunk(self, row: dict) -> None:
+ self.chunks[row["id"]] = row
+
+ def getRecordset(self, modelClass, recordFilter=None, **_):
+ filter_ = recordFilter or {}
+ if modelClass is FileContentIndex:
+ rows = list(self.indexRows.values())
+ elif modelClass is ContentChunk:
+ rows = list(self.chunks.values())
+ else:
+ return []
+
+ def match(row):
+ for k, v in filter_.items():
+ if row.get(k) != v:
+ return False
+ return True
+
+ return [r for r in rows if match(r)]
+
+ def recordDelete(self, modelClass, recordId):
+ if modelClass is FileContentIndex:
+ return self.indexRows.pop(recordId, None) is not None
+ if modelClass is ContentChunk:
+ return self.chunks.pop(recordId, None) is not None
+ return False
+
+
+def _buildKnowledge():
+ """Instantiate KnowledgeObjects without triggering the real DB bootstrap."""
+ ko = KnowledgeObjects.__new__(KnowledgeObjects)
+ ko.currentUser = None
+ ko.userId = None
+ ko._scopeCache = {}
+ ko.db = _FakeDb()
+ return ko
+
+
+def test_purge_by_connection_removes_only_matching_rows():
+ ko = _buildKnowledge()
+ ko.db.addIndex({"id": "sp1", "connectionId": "cx", "mandateId": "m1", "sourceKind": "sharepoint_item"})
+ ko.db.addIndex({"id": "sp2", "connectionId": "cx", "mandateId": "m1", "sourceKind": "sharepoint_item"})
+ ko.db.addIndex({"id": "upload", "connectionId": None, "mandateId": "m1", "sourceKind": "file"})
+ ko.db.addIndex({"id": "other", "connectionId": "cy", "mandateId": "m1", "sourceKind": "outlook_message"})
+ ko.db.addChunk({"id": "c1", "fileId": "sp1"})
+ ko.db.addChunk({"id": "c2", "fileId": "sp1"})
+ ko.db.addChunk({"id": "c3", "fileId": "sp2"})
+ ko.db.addChunk({"id": "c4", "fileId": "upload"})
+ ko.db.addChunk({"id": "c5", "fileId": "other"})
+
+ result = ko.deleteFileContentIndexByConnectionId("cx")
+
+ assert result == {"indexRows": 2, "chunks": 3}
+ assert "sp1" not in ko.db.indexRows
+ assert "sp2" not in ko.db.indexRows
+ assert "upload" in ko.db.indexRows
+ assert "other" in ko.db.indexRows
+ assert set(ko.db.chunks.keys()) == {"c4", "c5"}
+
+
+def test_purge_with_empty_connection_id_is_a_noop():
+ ko = _buildKnowledge()
+ ko.db.addIndex({"id": "sp1", "connectionId": "cx"})
+ ko.db.addChunk({"id": "c1", "fileId": "sp1"})
+
+ result = ko.deleteFileContentIndexByConnectionId("")
+
+ assert result == {"indexRows": 0, "chunks": 0}
+ assert "sp1" in ko.db.indexRows
+
+
+def test_purge_unknown_connection_returns_zero():
+ ko = _buildKnowledge()
+ ko.db.addIndex({"id": "sp1", "connectionId": "cx"})
+
+ result = ko.deleteFileContentIndexByConnectionId("nope")
+
+ assert result == {"indexRows": 0, "chunks": 0}
+ assert "sp1" in ko.db.indexRows
+
+
+if __name__ == "__main__":
+ test_purge_by_connection_removes_only_matching_rows()
+ test_purge_with_empty_connection_id_is_a_noop()
+ test_purge_unknown_connection_returns_zero()
+ print("OK — connection-purge tests passed")
diff --git a/tests/unit/services/test_extraction_merge_strategy.py b/tests/unit/services/test_extraction_merge_strategy.py
new file mode 100644
index 00000000..784bb783
--- /dev/null
+++ b/tests/unit/services/test_extraction_merge_strategy.py
@@ -0,0 +1,124 @@
+#!/usr/bin/env python3
+# Copyright (c) 2025 Patrick Motsch
+# All rights reserved.
+"""Test that runExtraction preserves per-part granularity when mergeStrategy=None.
+
+The default MergeStrategy concatenates all text parts into a single ContentPart, which
+collapses multi-page documents into one blob. This destroys RAG retrieval because every
+document ends up as a single ContentChunk with a "blurred average" embedding.
+
+Ingestion pipelines (requestIngestion callers) MUST pass mergeStrategy=None to preserve
+per-page / per-section chunks.
+"""
+
+import os
+import sys
+
+sys.path.insert(0, os.path.join(os.path.dirname(__file__), "../../.."))
+
+from modules.datamodels.datamodelExtraction import (
+ ContentPart,
+ ExtractionOptions,
+ MergeStrategy,
+)
+from modules.serviceCenter.services.serviceExtraction.subPipeline import runExtraction
+from modules.serviceCenter.services.serviceExtraction.subRegistry import (
+ ChunkerRegistry,
+ Extractor,
+ ExtractorRegistry,
+)
+
+
+class _FakeMultiPagePdfExtractor(Extractor):
+ """Emits one text ContentPart per simulated page."""
+
+ def __init__(self, pageCount: int = 10):
+ self.pageCount = pageCount
+
+ def detect(self, fileName: str, mimeType: str, headBytes: bytes) -> bool:
+ return mimeType == "application/pdf"
+
+ def getSupportedExtensions(self):
+ return [".pdf"]
+
+ def getSupportedMimeTypes(self):
+ return ["application/pdf"]
+
+ def extract(self, fileBytes: bytes, context):
+ return [
+ ContentPart(
+ id=f"page-{i}",
+ parentId=None,
+ label=f"page_{i + 1}",
+ typeGroup="text",
+ mimeType="text/plain",
+ data=f"Page {i + 1} content — distinct semantic anchor #{i}",
+ metadata={"pageIndex": i, "size": 64},
+ )
+ for i in range(self.pageCount)
+ ]
+
+
+def _buildRegistry(pageCount: int) -> ExtractorRegistry:
+ registry = ExtractorRegistry()
+ fake = _FakeMultiPagePdfExtractor(pageCount)
+ registry.register("application/pdf", fake)
+ registry.register("pdf", fake)
+ return registry
+
+
+def test_default_options_merge_all_text_parts_into_one():
+ """Regression safeguard: default ExtractionOptions still merges (legacy behaviour).
+
+ Non-ingestion callers (AI processing, summarization) rely on this default.
+ """
+ registry = _buildRegistry(pageCount=5)
+ extracted = runExtraction(
+ registry, ChunkerRegistry(), b"", "sample.pdf", "application/pdf",
+ ExtractionOptions(),
+ )
+ textParts = [p for p in extracted.parts if p.typeGroup == "text"]
+ assert len(textParts) == 1, (
+ f"Default options should merge all text parts into one, got {len(textParts)}"
+ )
+ assert "Page 1" in textParts[0].data and "Page 5" in textParts[0].data, (
+ "Merged text should contain content from all pages"
+ )
+ print("test_default_options_merge_all_text_parts_into_one [PASS]")
+
+
+def test_merge_none_preserves_all_text_parts():
+ """Core fix: mergeStrategy=None preserves per-page granularity for RAG ingestion."""
+ registry = _buildRegistry(pageCount=500)
+ extracted = runExtraction(
+ registry, ChunkerRegistry(), b"", "sample.pdf", "application/pdf",
+ ExtractionOptions(mergeStrategy=None),
+ )
+ textParts = [p for p in extracted.parts if p.typeGroup == "text"]
+ assert len(textParts) == 500, (
+ f"mergeStrategy=None should preserve all 500 text parts, got {len(textParts)}"
+ )
+ assert textParts[0].label == "page_1"
+ assert textParts[-1].label == "page_500"
+ print("test_merge_none_preserves_all_text_parts [PASS]")
+
+
+def test_explicit_merge_strategy_still_merges():
+ """Callers can still opt in to merging by passing an explicit MergeStrategy."""
+ registry = _buildRegistry(pageCount=3)
+ extracted = runExtraction(
+ registry, ChunkerRegistry(), b"", "sample.pdf", "application/pdf",
+ ExtractionOptions(mergeStrategy=MergeStrategy()),
+ )
+ textParts = [p for p in extracted.parts if p.typeGroup == "text"]
+ assert len(textParts) == 1, (
+ f"Explicit MergeStrategy should merge, got {len(textParts)} parts"
+ )
+ print("test_explicit_merge_strategy_still_merges [PASS]")
+
+
+if __name__ == "__main__":
+ test_default_options_merge_all_text_parts_into_one()
+ test_merge_none_preserves_all_text_parts()
+ test_explicit_merge_strategy_still_merges()
+ print("\nAll merge-strategy tests passed.")
diff --git a/tests/unit/services/test_ingestion_hash_stability.py b/tests/unit/services/test_ingestion_hash_stability.py
new file mode 100644
index 00000000..df25a4f0
--- /dev/null
+++ b/tests/unit/services/test_ingestion_hash_stability.py
@@ -0,0 +1,81 @@
+#!/usr/bin/env python3
+# Copyright (c) 2025 Patrick Motsch
+# All rights reserved.
+"""Test that _computeIngestionHash is stable across re-extractions of the same source.
+
+Extractors generate fresh contentObjectIds (uuid.uuid4()) per run. The ingestion
+hash MUST therefore be derived from content (contentType + data + order) only —
+otherwise idempotency (AC4) silently fails: every re-extraction looks "new" and
+triggers full re-embedding.
+"""
+
+import os
+import sys
+import uuid
+
+sys.path.insert(0, os.path.join(os.path.dirname(__file__), "../../.."))
+
+from modules.serviceCenter.services.serviceKnowledge.mainServiceKnowledge import (
+ _computeIngestionHash,
+)
+
+
+def _makeObjects(seed: str = "alpha"):
+ """Build a synthetic contentObjects list as routeDataFiles._autoIndexFile would."""
+ return [
+ {
+ "contentObjectId": str(uuid.uuid4()),
+ "contentType": "text",
+ "data": f"Page 1 of {seed}",
+ },
+ {
+ "contentObjectId": str(uuid.uuid4()),
+ "contentType": "text",
+ "data": f"Page 2 of {seed}",
+ },
+ {
+ "contentObjectId": str(uuid.uuid4()),
+ "contentType": "binary",
+ "data": "",
+ },
+ ]
+
+
+def test_hash_stable_across_uuid_regeneration():
+ """Same content + different contentObjectIds → same hash."""
+ a = _makeObjects("alpha")
+ b = _makeObjects("alpha") # identical data, fresh UUIDs
+ assert [o["contentObjectId"] for o in a] != [o["contentObjectId"] for o in b]
+ assert _computeIngestionHash(a) == _computeIngestionHash(b)
+
+
+def test_hash_changes_when_data_changes():
+ a = _makeObjects("alpha")
+ b = _makeObjects("beta")
+ assert _computeIngestionHash(a) != _computeIngestionHash(b)
+
+
+def test_hash_is_order_sensitive():
+ """Reordered pages produce a different hash (different document)."""
+ a = _makeObjects("alpha")
+ b = list(reversed(a))
+ assert _computeIngestionHash(a) != _computeIngestionHash(b)
+
+
+def test_hash_distinguishes_text_vs_binary_with_same_payload():
+ a = [{"contentObjectId": "x", "contentType": "text", "data": "hello"}]
+ b = [{"contentObjectId": "x", "contentType": "binary", "data": "hello"}]
+ assert _computeIngestionHash(a) != _computeIngestionHash(b)
+
+
+def test_hash_handles_empty_input():
+ assert _computeIngestionHash([]) == _computeIngestionHash([])
+
+
+if __name__ == "__main__":
+ test_hash_stable_across_uuid_regeneration()
+ test_hash_changes_when_data_changes()
+ test_hash_is_order_sensitive()
+ test_hash_distinguishes_text_vs_binary_with_same_payload()
+ test_hash_handles_empty_input()
+ print("OK — all 5 ingestion-hash stability tests passed")
diff --git a/tests/unit/services/test_knowledge_ingest_consumer.py b/tests/unit/services/test_knowledge_ingest_consumer.py
new file mode 100644
index 00000000..6b27a6e8
--- /dev/null
+++ b/tests/unit/services/test_knowledge_ingest_consumer.py
@@ -0,0 +1,235 @@
+#!/usr/bin/env python3
+# Copyright (c) 2025 Patrick Motsch
+# All rights reserved.
+"""Unit tests for KnowledgeIngestionConsumer event dispatch.
+
+- `connection.established` → enqueue a `connection.bootstrap` job.
+- `connection.revoked` → synchronous purge via KnowledgeObjects.
+"""
+
+import asyncio
+import os
+import sys
+import types
+
+sys.path.insert(0, os.path.join(os.path.dirname(__file__), "../../.."))
+
+from modules.serviceCenter.services.serviceKnowledge import subConnectorIngestConsumer as consumer
+
+
+def _resetRegistration(monkeypatch):
+ """Force the module-level guard to register fresh in each test."""
+ monkeypatch.setattr(consumer, "_registered", False)
+
+
+def test_onConnectionEstablished_enqueues_bootstrap(monkeypatch):
+ startedJobs = []
+
+ async def _fakeStartJob(jobType, payload, **kwargs):
+ startedJobs.append({"jobType": jobType, "payload": payload, "kwargs": kwargs})
+ return "job-1"
+
+ monkeypatch.setattr(consumer, "startJob", _fakeStartJob)
+ consumer._onConnectionEstablished(
+ connectionId="c1", authority="msft", userId="u1"
+ )
+ # Drain pending tasks created by the consumer.
+ loop = asyncio.new_event_loop()
+ try:
+ asyncio.set_event_loop(loop)
+ # If the consumer created a Task on a closed loop the fake startJob
+ # was still called synchronously via asyncio.run — in either case we
+ # check the recorded call.
+ finally:
+ loop.close()
+
+ assert len(startedJobs) == 1
+ assert startedJobs[0]["jobType"] == consumer.BOOTSTRAP_JOB_TYPE
+ assert startedJobs[0]["payload"]["connectionId"] == "c1"
+ assert startedJobs[0]["payload"]["authority"] == "msft"
+ assert startedJobs[0]["kwargs"]["triggeredBy"] == "u1"
+
+
+def test_onConnectionEstablished_ignores_missing_id(monkeypatch):
+ called = []
+
+ async def _fakeStartJob(*a, **kw):
+ called.append(1)
+ return "x"
+
+ monkeypatch.setattr(consumer, "startJob", _fakeStartJob)
+ consumer._onConnectionEstablished(connectionId="", authority="msft")
+ assert called == []
+
+
+def test_onConnectionRevoked_runs_sync_purge(monkeypatch):
+ class _FakeKnowledge:
+ def __init__(self):
+ self.calls = []
+
+ def deleteFileContentIndexByConnectionId(self, cid):
+ self.calls.append(cid)
+ return {"indexRows": 2, "chunks": 5}
+
+ fakeKnow = _FakeKnowledge()
+
+ def _fakeGetInterface(_user=None):
+ return fakeKnow
+
+ monkeypatch.setattr(consumer, "getKnowledgeInterface", _fakeGetInterface)
+ consumer._onConnectionRevoked(
+ connectionId="c1", authority="msft", userId="u1", reason="disconnected"
+ )
+ assert fakeKnow.calls == ["c1"]
+
+
+def test_onConnectionRevoked_ignores_missing_id(monkeypatch):
+ seen = []
+
+ def _fakeGetInterface(_user=None):
+ class _K:
+ def deleteFileContentIndexByConnectionId(self, cid):
+ seen.append(cid)
+ return {"indexRows": 0, "chunks": 0}
+
+ return _K()
+
+ monkeypatch.setattr(consumer, "getKnowledgeInterface", _fakeGetInterface)
+ consumer._onConnectionRevoked(connectionId="")
+ assert seen == []
+
+
+def test_bootstrap_job_skips_unsupported_authority(monkeypatch):
+ async def _run():
+ result = await consumer._bootstrapJobHandler(
+ {"payload": {"connectionId": "c1", "authority": "slack"}},
+ lambda *_: None,
+ )
+ return result
+
+ result = asyncio.run(_run())
+ assert result["skipped"] is True
+ assert result["authority"] == "slack"
+ assert result["reason"] == "unsupported_authority"
+
+
+def test_bootstrap_job_dispatches_msft_parts(monkeypatch):
+ calls = {"sp": 0, "ol": 0}
+
+ async def _fakeSp(connectionId, progressCb=None):
+ calls["sp"] += 1
+ return {"indexed": 1}
+
+ async def _fakeOl(connectionId, progressCb=None):
+ calls["ol"] += 1
+ return {"indexed": 2}
+
+ fakeSharepoint = types.ModuleType("subConnectorSyncSharepoint")
+ fakeSharepoint.bootstrapSharepoint = _fakeSp
+ fakeOutlook = types.ModuleType("subConnectorSyncOutlook")
+ fakeOutlook.bootstrapOutlook = _fakeOl
+ monkeypatch.setitem(
+ sys.modules,
+ "modules.serviceCenter.services.serviceKnowledge.subConnectorSyncSharepoint",
+ fakeSharepoint,
+ )
+ monkeypatch.setitem(
+ sys.modules,
+ "modules.serviceCenter.services.serviceKnowledge.subConnectorSyncOutlook",
+ fakeOutlook,
+ )
+
+ async def _run():
+ return await consumer._bootstrapJobHandler(
+ {"payload": {"connectionId": "c1", "authority": "msft"}},
+ lambda *_: None,
+ )
+
+ result = asyncio.run(_run())
+ assert calls == {"sp": 1, "ol": 1}
+ assert result["sharepoint"] == {"indexed": 1}
+ assert result["outlook"] == {"indexed": 2}
+
+
+def test_bootstrap_job_dispatches_google_parts(monkeypatch):
+ calls = {"gd": 0, "gm": 0}
+
+ async def _fakeGd(connectionId, progressCb=None):
+ calls["gd"] += 1
+ return {"indexed": 7}
+
+ async def _fakeGm(connectionId, progressCb=None):
+ calls["gm"] += 1
+ return {"indexed": 11}
+
+ fakeGdrive = types.ModuleType("subConnectorSyncGdrive")
+ fakeGdrive.bootstrapGdrive = _fakeGd
+ fakeGmail = types.ModuleType("subConnectorSyncGmail")
+ fakeGmail.bootstrapGmail = _fakeGm
+ monkeypatch.setitem(
+ sys.modules,
+ "modules.serviceCenter.services.serviceKnowledge.subConnectorSyncGdrive",
+ fakeGdrive,
+ )
+ monkeypatch.setitem(
+ sys.modules,
+ "modules.serviceCenter.services.serviceKnowledge.subConnectorSyncGmail",
+ fakeGmail,
+ )
+
+ async def _run():
+ return await consumer._bootstrapJobHandler(
+ {"payload": {"connectionId": "c1", "authority": "google"}},
+ lambda *_: None,
+ )
+
+ result = asyncio.run(_run())
+ assert calls == {"gd": 1, "gm": 1}
+ assert result["drive"] == {"indexed": 7}
+ assert result["gmail"] == {"indexed": 11}
+
+
+def test_bootstrap_job_dispatches_clickup_part(monkeypatch):
+ calls = {"cu": 0}
+
+ async def _fakeCu(connectionId, progressCb=None):
+ calls["cu"] += 1
+ return {"indexed": 4}
+
+ fakeClickup = types.ModuleType("subConnectorSyncClickup")
+ fakeClickup.bootstrapClickup = _fakeCu
+ monkeypatch.setitem(
+ sys.modules,
+ "modules.serviceCenter.services.serviceKnowledge.subConnectorSyncClickup",
+ fakeClickup,
+ )
+
+ async def _run():
+ return await consumer._bootstrapJobHandler(
+ {"payload": {"connectionId": "c1", "authority": "clickup"}},
+ lambda *_: None,
+ )
+
+ result = asyncio.run(_run())
+ assert calls == {"cu": 1}
+ assert result["clickup"] == {"indexed": 4}
+
+
+if __name__ == "__main__":
+ # Usable without pytest fixtures for a quick smoke run.
+ class _MP:
+ def __init__(self):
+ self.undos = []
+
+ def setattr(self, target, name_or_value, value=None):
+ if value is None:
+ # target is an object, name_or_value is value → no, original signature
+ raise SystemExit("use pytest monkeypatch in CLI")
+ self.undos.append((target, name_or_value, getattr(target, name_or_value)))
+ setattr(target, name_or_value, value)
+
+ def setitem(self, mapping, key, value):
+ self.undos.append((mapping, key, mapping.get(key)))
+ mapping[key] = value
+
+ print("Run via pytest: pytest tests/unit/services/test_knowledge_ingest_consumer.py")
diff --git a/tests/unit/services/test_p1d_consent_prefs.py b/tests/unit/services/test_p1d_consent_prefs.py
new file mode 100644
index 00000000..e00b0dfc
--- /dev/null
+++ b/tests/unit/services/test_p1d_consent_prefs.py
@@ -0,0 +1,298 @@
+#!/usr/bin/env python3
+"""Unit tests for P1d: consent gating, preference parsing, and walker behaviour.
+
+Tests
+-----
+1. Bootstrap runner skips when ``knowledgeIngestionEnabled=False``.
+2. ``loadConnectionPrefs`` returns safe defaults when preferences are absent.
+3. ``loadConnectionPrefs`` maps all §2.6 keys correctly from a full prefs dict.
+4. Gmail walker passes ``neutralize=True`` and ``mailContentDepth`` to IngestionJob.
+5. Gmail walker produces only a header content-object when depth="metadata".
+6. ClickUp walker skips description when scope="titles".
+"""
+
+from __future__ import annotations
+
+import asyncio
+import os
+import sys
+import types
+import unittest
+from typing import Any, Dict, Optional
+from unittest.mock import AsyncMock, MagicMock, patch
+
+sys.path.insert(0, os.path.join(os.path.dirname(__file__), "../../.."))
+
+
+# ---------------------------------------------------------------------------
+# 1. Bootstrap runner consent gate
+# ---------------------------------------------------------------------------
+
+class TestBootstrapConsentGate(unittest.TestCase):
+ """_bootstrapJobHandler must no-op when knowledgeIngestionEnabled is False."""
+
+ def _makeJob(self, connectionId="c-test", authority="google"):
+ return {"payload": {"connectionId": connectionId, "authority": authority}}
+
+ def _makeConn(self, enabled: bool):
+ conn = MagicMock()
+ conn.knowledgeIngestionEnabled = enabled
+ return conn
+
+ def test_skips_when_consent_disabled(self):
+ from modules.serviceCenter.services.serviceKnowledge import subConnectorIngestConsumer as sut
+
+ fake_root = MagicMock()
+ fake_root.getUserConnectionById.return_value = self._makeConn(False)
+
+ with patch("modules.interfaces.interfaceDbApp.getRootInterface", return_value=fake_root):
+ result = asyncio.get_event_loop().run_until_complete(
+ sut._bootstrapJobHandler(self._makeJob(), lambda *a: None)
+ )
+
+ assert result.get("skipped") is True
+ assert result.get("reason") == "consent_disabled"
+ fake_root.getUserConnectionById.assert_called_once_with("c-test")
+
+ def test_proceeds_when_consent_enabled(self):
+ """When consent is enabled, the handler should call at least one walker."""
+ from modules.serviceCenter.services.serviceKnowledge import subConnectorIngestConsumer as sut
+
+ fake_root = MagicMock()
+ fake_root.getUserConnectionById.return_value = self._makeConn(True)
+
+ # Patch the inner walker so it doesn't do real I/O.
+ async def _fakeBootstrap(**kwargs):
+ return {"indexed": 0}
+
+ with (
+ patch("modules.interfaces.interfaceDbApp.getRootInterface", return_value=fake_root),
+ patch(
+ "modules.serviceCenter.services.serviceKnowledge.subConnectorSyncGdrive.bootstrapGdrive",
+ new=AsyncMock(return_value={"indexed": 0}),
+ ),
+ patch(
+ "modules.serviceCenter.services.serviceKnowledge.subConnectorSyncGmail.bootstrapGmail",
+ new=AsyncMock(return_value={"indexed": 0}),
+ ),
+ ):
+ result = asyncio.get_event_loop().run_until_complete(
+ sut._bootstrapJobHandler(self._makeJob(authority="google"), lambda *a: None)
+ )
+
+ # Should not have 'skipped' at the top level.
+ assert result.get("skipped") is not True
+ assert result.get("authority") == "google"
+
+
+# ---------------------------------------------------------------------------
+# 2 + 3. loadConnectionPrefs
+# ---------------------------------------------------------------------------
+
+class TestLoadConnectionPrefs(unittest.TestCase):
+ def _makeConn(self, prefs: Optional[Dict[str, Any]]):
+ conn = MagicMock()
+ conn.knowledgePreferences = prefs
+ return conn
+
+ def _mockRoot(self, prefs):
+ root = MagicMock()
+ root.getUserConnectionById.return_value = self._makeConn(prefs)
+ return root
+
+ def test_returns_safe_defaults_when_prefs_none(self):
+ from modules.serviceCenter.services.serviceKnowledge.subConnectorPrefs import (
+ ConnectionIngestionPrefs,
+ loadConnectionPrefs,
+ )
+
+ with patch("modules.interfaces.interfaceDbApp.getRootInterface", return_value=self._mockRoot(None)):
+ prefs = loadConnectionPrefs("x")
+
+ assert prefs.neutralizeBeforeEmbed is False
+ assert prefs.mailContentDepth == "full"
+ assert prefs.mailIndexAttachments is False
+ assert prefs.maxAgeDays == 90
+ assert prefs.clickupScope == "title_description"
+ assert prefs.gmailEnabled is True
+ assert prefs.driveEnabled is True
+
+ def test_maps_all_keys(self):
+ from modules.serviceCenter.services.serviceKnowledge.subConnectorPrefs import loadConnectionPrefs
+
+ raw = {
+ "neutralizeBeforeEmbed": True,
+ "mailContentDepth": "metadata",
+ "mailIndexAttachments": True,
+ "filesIndexBinaries": False,
+ "clickupScope": "with_comments",
+ "maxAgeDays": 30,
+ "surfaceToggles": {
+ "google": {"gmail": False, "drive": True},
+ "msft": {"sharepoint": False, "outlook": True},
+ },
+ }
+
+ with patch("modules.interfaces.interfaceDbApp.getRootInterface", return_value=self._mockRoot(raw)):
+ prefs = loadConnectionPrefs("x")
+
+ assert prefs.neutralizeBeforeEmbed is True
+ assert prefs.mailContentDepth == "metadata"
+ assert prefs.mailIndexAttachments is True
+ assert prefs.filesIndexBinaries is False
+ assert prefs.clickupScope == "with_comments"
+ assert prefs.maxAgeDays == 30
+ assert prefs.gmailEnabled is False
+ assert prefs.driveEnabled is True
+ assert prefs.sharepointEnabled is False
+ assert prefs.outlookEnabled is True
+
+ def test_invalid_depth_falls_back_to_default(self):
+ from modules.serviceCenter.services.serviceKnowledge.subConnectorPrefs import loadConnectionPrefs
+
+ raw = {"mailContentDepth": "everything_please"}
+
+ with patch("modules.interfaces.interfaceDbApp.getRootInterface", return_value=self._mockRoot(raw)):
+ prefs = loadConnectionPrefs("x")
+
+ assert prefs.mailContentDepth == "full"
+
+
+# ---------------------------------------------------------------------------
+# 4. Gmail walker passes neutralize + mailContentDepth to IngestionJob
+# ---------------------------------------------------------------------------
+
+class TestGmailWalkerPrefs(unittest.TestCase):
+ def _make_message(self, *, subject="Test", snippet="hello", body_text="full body"):
+ import base64
+ encoded = base64.urlsafe_b64encode(body_text.encode()).decode()
+ return {
+ "id": "msg-1",
+ "historyId": "h-42",
+ "threadId": "t-1",
+ "snippet": snippet,
+ "payload": {
+ "mimeType": "multipart/alternative",
+ "headers": [
+ {"name": "Subject", "value": subject},
+ {"name": "From", "value": "alice@example.com"},
+ {"name": "To", "value": "bob@example.com"},
+ {"name": "Date", "value": "Mon, 20 Apr 2026 10:00:00 +0000"},
+ ],
+ "parts": [
+ {
+ "mimeType": "text/plain",
+ "body": {"data": encoded},
+ }
+ ],
+ },
+ }
+
+ def test_neutralize_flag_forwarded(self):
+ from modules.serviceCenter.services.serviceKnowledge.subConnectorSyncGmail import (
+ GmailBootstrapLimits,
+ _ingestMessage,
+ GmailBootstrapResult,
+ )
+ from modules.serviceCenter.services.serviceKnowledge.mainServiceKnowledge import IngestionJob
+
+ captured_jobs = []
+
+ async def fake_requestIngestion(job: IngestionJob):
+ captured_jobs.append(job)
+ return MagicMock(status="indexed", error=None)
+
+ ks = MagicMock()
+ ks.requestIngestion = fake_requestIngestion
+
+ limits = GmailBootstrapLimits(neutralize=True, mailContentDepth="full")
+ result = GmailBootstrapResult(connectionId="c-1")
+
+ asyncio.get_event_loop().run_until_complete(
+ _ingestMessage(
+ googleGetFn=AsyncMock(return_value={}),
+ knowledgeService=ks,
+ connectionId="c-1",
+ mandateId="",
+ userId="u-1",
+ labelId="INBOX",
+ message=self._make_message(),
+ limits=limits,
+ result=result,
+ progressCb=None,
+ )
+ )
+
+ assert len(captured_jobs) == 1
+ assert captured_jobs[0].neutralize is True
+
+ def test_metadata_depth_yields_only_header(self):
+ from modules.serviceCenter.services.serviceKnowledge.subConnectorSyncGmail import (
+ _buildContentObjects,
+ )
+
+ message = self._make_message(snippet="hi", body_text="should be excluded")
+ parts = _buildContentObjects(message, maxBodyChars=4000, mailContentDepth="metadata")
+ ids = [p["contentObjectId"] for p in parts]
+ assert ids == ["header"]
+
+ def test_snippet_depth_yields_header_and_snippet(self):
+ from modules.serviceCenter.services.serviceKnowledge.subConnectorSyncGmail import (
+ _buildContentObjects,
+ )
+
+ message = self._make_message(snippet="hi", body_text="should be excluded")
+ parts = _buildContentObjects(message, maxBodyChars=4000, mailContentDepth="snippet")
+ ids = [p["contentObjectId"] for p in parts]
+ assert "header" in ids
+ assert "snippet" in ids
+ assert "body" not in ids
+
+
+# ---------------------------------------------------------------------------
+# 5. ClickUp walker respects clickupScope="titles"
+# ---------------------------------------------------------------------------
+
+class TestClickupWalkerScope(unittest.TestCase):
+ def _make_task(self):
+ return {
+ "id": "task-1",
+ "name": "Ship feature X",
+ "date_updated": "1713888000000",
+ "description": "This should be omitted",
+ "text_content": "Also omitted",
+ "status": {"status": "open"},
+ "assignees": [],
+ "tags": [],
+ "list": {"name": "Backlog"},
+ "folder": {},
+ "space": {"name": "Engineering"},
+ }
+
+ def test_titles_scope_omits_description(self):
+ from modules.serviceCenter.services.serviceKnowledge.subConnectorSyncClickup import (
+ ClickupBootstrapLimits,
+ _buildContentObjects,
+ )
+
+ limits = ClickupBootstrapLimits(clickupScope="titles")
+ parts = _buildContentObjects(self._make_task(), limits)
+ ids = [p["contentObjectId"] for p in parts]
+ assert ids == ["header"]
+ assert "description" not in ids
+
+ def test_with_description_scope_includes_description(self):
+ from modules.serviceCenter.services.serviceKnowledge.subConnectorSyncClickup import (
+ ClickupBootstrapLimits,
+ _buildContentObjects,
+ )
+
+ limits = ClickupBootstrapLimits(clickupScope="title_description")
+ parts = _buildContentObjects(self._make_task(), limits)
+ ids = [p["contentObjectId"] for p in parts]
+ assert "header" in ids
+ assert "description" in ids
+
+
+if __name__ == "__main__":
+ unittest.main()
diff --git a/tests/unit/workflows/test_automation2_graphUtils.py b/tests/unit/workflows/test_automation2_graphUtils.py
index ff5df2cc..5ea7126a 100644
--- a/tests/unit/workflows/test_automation2_graphUtils.py
+++ b/tests/unit/workflows/test_automation2_graphUtils.py
@@ -66,6 +66,17 @@ class TestResolveParameterReferences:
value = "Land: {{n1.country}}"
assert resolveParameterReferences(value, node_outputs) == "Land: CH"
+ def test_legacy_string_template_loop_current_item_nested(self):
+ """Same shape as executionEngine sets on loop node id during body iteration."""
+ node_outputs = {
+ "loop93": {
+ "currentItem": {"subject": "Hello", "body": {"content": "World"}},
+ "currentIndex": 0,
+ },
+ }
+ value = "Subj: {{loop93.currentItem.subject}} Body: {{loop93.currentItem.body.content}}"
+ assert resolveParameterReferences(value, node_outputs) == "Subj: Hello Body: World"
+
class TestWildcardIteration:
"""Phase-4 typed Bindings-Resolver: ``*`` segment iterates over a list.