315 lines
11 KiB
Python
315 lines
11 KiB
Python
# Copyright (c) 2026 Patrick Motsch
|
|
# All rights reserved.
|
|
"""Incremental Redmine -> ``poweron_redmine`` mirror sync.
|
|
|
|
Strategy:
|
|
- **Full sync** when ``RedmineInstanceConfig.lastSyncAt`` is None or
|
|
``force=True`` is requested. Pulls every issue with ``status_id=*``
|
|
(open + closed) for the configured project, paginated.
|
|
- **Incremental sync** otherwise. Pulls only issues whose ``updated_on``
|
|
is greater than ``lastSyncAt - overlapSeconds`` (default 1h overlap to
|
|
catch clock skew and missed updates).
|
|
- Each issue is upserted into ``RedmineTicketMirror`` (looked up by
|
|
``(featureInstanceId, redmineId)``).
|
|
- The full set of relations attached to each issue replaces any existing
|
|
relation rows for that issue in ``RedmineRelationMirror``.
|
|
|
|
Concurrency: a per-instance ``asyncio.Lock`` prevents two concurrent
|
|
syncs for the same feature instance.
|
|
|
|
After every successful sync the in-memory stats cache is invalidated for
|
|
the instance.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import asyncio
|
|
import logging
|
|
import time
|
|
from typing import Any, Dict, List, Optional
|
|
|
|
from modules.connectors.connectorTicketsRedmine import RedmineApiError
|
|
from modules.datamodels.datamodelUam import User
|
|
from modules.features.redmine.datamodelRedmine import (
|
|
RedmineInstanceConfig,
|
|
RedmineRelationMirror,
|
|
RedmineSyncResultDto,
|
|
RedmineSyncStatusDto,
|
|
RedmineTicketMirror,
|
|
)
|
|
from modules.features.redmine.interfaceFeatureRedmine import getInterface
|
|
from modules.features.redmine.serviceRedmineStatsCache import _getStatsCache
|
|
|
|
logger = logging.getLogger(__name__)
|
|
|
|
|
|
_INCREMENTAL_OVERLAP_SECONDS = 60 * 60 # 1h overlap on incremental syncs
|
|
_DEFAULT_PAGE_SIZE = 100
|
|
_MAX_PAGES_SAFETY = 5000 # 500k tickets safety cap
|
|
|
|
_locks: Dict[str, asyncio.Lock] = {}
|
|
|
|
|
|
def _lockFor(featureInstanceId: str) -> asyncio.Lock:
|
|
if featureInstanceId not in _locks:
|
|
_locks[featureInstanceId] = asyncio.Lock()
|
|
return _locks[featureInstanceId]
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Public API
|
|
# ---------------------------------------------------------------------------
|
|
|
|
async def runSync(
|
|
currentUser: User,
|
|
mandateId: Optional[str],
|
|
featureInstanceId: str,
|
|
*,
|
|
force: bool = False,
|
|
pageSize: int = _DEFAULT_PAGE_SIZE,
|
|
) -> RedmineSyncResultDto:
|
|
"""Run a (full or incremental) sync for the given feature instance."""
|
|
iface = getInterface(currentUser, mandateId=mandateId, featureInstanceId=featureInstanceId)
|
|
connector = iface.resolveConnector(featureInstanceId)
|
|
cfg = iface.getConfig(featureInstanceId)
|
|
if not connector or not cfg:
|
|
raise RuntimeError(
|
|
f"Redmine instance {featureInstanceId} is not configured or inactive"
|
|
)
|
|
|
|
async with _lockFor(featureInstanceId):
|
|
started = time.monotonic()
|
|
full = force or cfg.lastSyncAt is None
|
|
updated_from_iso: Optional[str] = None
|
|
if not full and cfg.lastSyncAt is not None:
|
|
cursor_epoch = max(0.0, cfg.lastSyncAt - _INCREMENTAL_OVERLAP_SECONDS)
|
|
updated_from_iso = time.strftime(
|
|
"%Y-%m-%dT%H:%M:%SZ", time.gmtime(cursor_epoch)
|
|
)
|
|
|
|
try:
|
|
issues = await connector.listAllIssues(
|
|
statusId="*",
|
|
updatedOnFrom=updated_from_iso,
|
|
pageSize=pageSize,
|
|
maxPages=_MAX_PAGES_SAFETY,
|
|
include=["relations"],
|
|
)
|
|
except RedmineApiError as e:
|
|
iface.recordSyncFailure(featureInstanceId, str(e))
|
|
raise
|
|
|
|
tickets_upserted = 0
|
|
relations_upserted = 0
|
|
now_epoch = time.time()
|
|
|
|
for issue in issues:
|
|
tickets_upserted += _upsertTicket(iface, featureInstanceId, mandateId, issue, now_epoch)
|
|
relations_upserted += _replaceRelations(iface, featureInstanceId, issue, now_epoch)
|
|
|
|
duration_ms = int((time.monotonic() - started) * 1000)
|
|
iface.recordSyncSuccess(
|
|
featureInstanceId,
|
|
full=full,
|
|
ticketsUpserted=tickets_upserted,
|
|
durationMs=duration_ms,
|
|
lastSyncAt=now_epoch,
|
|
)
|
|
_getStatsCache().invalidateInstance(featureInstanceId)
|
|
|
|
return RedmineSyncResultDto(
|
|
instanceId=featureInstanceId,
|
|
full=full,
|
|
ticketsUpserted=tickets_upserted,
|
|
relationsUpserted=relations_upserted,
|
|
durationMs=duration_ms,
|
|
lastSyncAt=now_epoch,
|
|
)
|
|
|
|
|
|
def getSyncStatus(
|
|
currentUser: User,
|
|
mandateId: Optional[str],
|
|
featureInstanceId: str,
|
|
) -> RedmineSyncStatusDto:
|
|
iface = getInterface(currentUser, mandateId=mandateId, featureInstanceId=featureInstanceId)
|
|
cfg = iface.getConfig(featureInstanceId)
|
|
ticket_count = iface.countMirroredTickets(featureInstanceId)
|
|
relation_count = iface.countMirroredRelations(featureInstanceId)
|
|
return RedmineSyncStatusDto(
|
|
instanceId=featureInstanceId,
|
|
lastSyncAt=cfg.lastSyncAt if cfg else None,
|
|
lastFullSyncAt=cfg.lastFullSyncAt if cfg else None,
|
|
lastSyncDurationMs=cfg.lastSyncDurationMs if cfg else None,
|
|
lastSyncTicketCount=cfg.lastSyncTicketCount if cfg else None,
|
|
lastSyncErrorAt=cfg.lastSyncErrorAt if cfg else None,
|
|
lastSyncErrorMessage=cfg.lastSyncErrorMessage if cfg else None,
|
|
mirroredTicketCount=ticket_count,
|
|
mirroredRelationCount=relation_count,
|
|
)
|
|
|
|
|
|
async def upsertSingleTicket(
|
|
currentUser: User,
|
|
mandateId: Optional[str],
|
|
featureInstanceId: str,
|
|
issueId: int,
|
|
) -> int:
|
|
"""Re-fetch one issue from Redmine and upsert it into the mirror.
|
|
|
|
Used by the write paths in ``serviceRedmine`` so the mirror stays
|
|
consistent after every create / update without a full sync.
|
|
Returns the number of relation rows replaced.
|
|
"""
|
|
iface = getInterface(currentUser, mandateId=mandateId, featureInstanceId=featureInstanceId)
|
|
connector = iface.resolveConnector(featureInstanceId)
|
|
if not connector:
|
|
raise RuntimeError("Redmine instance not configured")
|
|
issue = await connector.getIssue(int(issueId), includeRelations=True)
|
|
now_epoch = time.time()
|
|
_upsertTicket(iface, featureInstanceId, mandateId, issue, now_epoch)
|
|
relations_upserted = _replaceRelations(iface, featureInstanceId, issue, now_epoch)
|
|
_getStatsCache().invalidateInstance(featureInstanceId)
|
|
return relations_upserted
|
|
|
|
|
|
def deleteMirroredTicket(
|
|
currentUser: User,
|
|
mandateId: Optional[str],
|
|
featureInstanceId: str,
|
|
issueId: int,
|
|
) -> bool:
|
|
"""Drop a ticket and its relations from the mirror after a successful Redmine DELETE."""
|
|
iface = getInterface(currentUser, mandateId=mandateId, featureInstanceId=featureInstanceId)
|
|
deleted = iface.deleteMirroredTicket(featureInstanceId, int(issueId))
|
|
iface.deleteMirroredRelationsForIssue(featureInstanceId, int(issueId))
|
|
_getStatsCache().invalidateInstance(featureInstanceId)
|
|
return deleted
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Per-issue upsert helpers (sync, run inside the per-instance lock)
|
|
# ---------------------------------------------------------------------------
|
|
|
|
def _upsertTicket(
|
|
iface,
|
|
featureInstanceId: str,
|
|
mandateId: Optional[str],
|
|
issue: Dict[str, Any],
|
|
nowEpoch: float,
|
|
) -> int:
|
|
redmine_id = issue.get("id")
|
|
if redmine_id is None:
|
|
return 0
|
|
statuses_lookup = (iface.getConfig(featureInstanceId).schemaCache or {}).get("statuses") or []
|
|
is_closed = _statusIsClosed(issue.get("status") or {}, statuses_lookup)
|
|
record = _ticketRecordFromIssue(issue, featureInstanceId, mandateId, is_closed, nowEpoch)
|
|
iface.upsertMirroredTicket(featureInstanceId, int(redmine_id), record)
|
|
return 1
|
|
|
|
|
|
def _replaceRelations(
|
|
iface,
|
|
featureInstanceId: str,
|
|
issue: Dict[str, Any],
|
|
nowEpoch: float,
|
|
) -> int:
|
|
issue_id = issue.get("id")
|
|
relations = issue.get("relations") or []
|
|
if issue_id is None:
|
|
return 0
|
|
iface.deleteMirroredRelationsForIssue(featureInstanceId, int(issue_id))
|
|
inserted = 0
|
|
for r in relations:
|
|
rid = r.get("id")
|
|
if rid is None:
|
|
continue
|
|
iface.insertMirroredRelation(
|
|
featureInstanceId,
|
|
{
|
|
"featureInstanceId": featureInstanceId,
|
|
"redmineRelationId": int(rid),
|
|
"issueId": int(r.get("issue_id") or 0),
|
|
"issueToId": int(r.get("issue_to_id") or 0),
|
|
"relationType": str(r.get("relation_type") or "relates"),
|
|
"delay": r.get("delay"),
|
|
"syncedAt": nowEpoch,
|
|
},
|
|
)
|
|
inserted += 1
|
|
return inserted
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Pure helpers
|
|
# ---------------------------------------------------------------------------
|
|
|
|
def _statusIsClosed(status: Dict[str, Any], statusesLookup: List[Dict[str, Any]]) -> bool:
|
|
"""Best-effort: prefer the schemaCache; fall back to inspecting the
|
|
raw issue (Redmine sets ``is_closed`` on the status object only when
|
|
explicitly requested)."""
|
|
sid = status.get("id")
|
|
if sid is None:
|
|
return False
|
|
for s in statusesLookup:
|
|
if s.get("id") == sid:
|
|
return bool(s.get("isClosed"))
|
|
return bool(status.get("is_closed"))
|
|
|
|
|
|
def _parseRedmineDateToEpoch(value: Optional[str]) -> Optional[float]:
|
|
if not value:
|
|
return None
|
|
try:
|
|
from datetime import datetime
|
|
s = value.replace("Z", "+00:00")
|
|
return datetime.fromisoformat(s).timestamp()
|
|
except Exception:
|
|
return None
|
|
|
|
|
|
def _ticketRecordFromIssue(
|
|
issue: Dict[str, Any],
|
|
featureInstanceId: str,
|
|
mandateId: Optional[str],
|
|
isClosed: bool,
|
|
nowEpoch: float,
|
|
) -> Dict[str, Any]:
|
|
tracker = issue.get("tracker") or {}
|
|
status = issue.get("status") or {}
|
|
priority = issue.get("priority") or {}
|
|
assigned = issue.get("assigned_to") or {}
|
|
author = issue.get("author") or {}
|
|
parent = issue.get("parent") or {}
|
|
fixed_version = issue.get("fixed_version") or {}
|
|
created_on = issue.get("created_on")
|
|
updated_on = issue.get("updated_on")
|
|
|
|
return {
|
|
"featureInstanceId": featureInstanceId,
|
|
"mandateId": mandateId,
|
|
"redmineId": int(issue.get("id")),
|
|
"subject": str(issue.get("subject") or ""),
|
|
"description": str(issue.get("description") or ""),
|
|
"trackerId": tracker.get("id"),
|
|
"trackerName": tracker.get("name"),
|
|
"statusId": status.get("id"),
|
|
"statusName": status.get("name"),
|
|
"isClosed": bool(isClosed),
|
|
"priorityId": priority.get("id"),
|
|
"priorityName": priority.get("name"),
|
|
"assignedToId": assigned.get("id"),
|
|
"assignedToName": assigned.get("name"),
|
|
"authorId": author.get("id"),
|
|
"authorName": author.get("name"),
|
|
"parentId": parent.get("id"),
|
|
"fixedVersionId": fixed_version.get("id"),
|
|
"fixedVersionName": fixed_version.get("name"),
|
|
"createdOn": created_on,
|
|
"updatedOn": updated_on,
|
|
"createdOnTs": _parseRedmineDateToEpoch(created_on),
|
|
"updatedOnTs": _parseRedmineDateToEpoch(updated_on),
|
|
"customFields": list(issue.get("custom_fields") or []),
|
|
"raw": issue,
|
|
"syncedAt": nowEpoch,
|
|
}
|