db connection pooling and rag limit transparency
Some checks failed
Deploy Nyla Frontend to Integration / build-and-deploy (push) Failing after 17s
Some checks failed
Deploy Nyla Frontend to Integration / build-and-deploy (push) Failing after 17s
This commit is contained in:
parent
bb441f5268
commit
f37774ff36
3 changed files with 79 additions and 5 deletions
|
|
@ -348,6 +348,9 @@ export interface RagDataSourceDto {
|
||||||
ragIndexEnabled: boolean;
|
ragIndexEnabled: boolean;
|
||||||
neutralize: boolean;
|
neutralize: boolean;
|
||||||
lastIndexed: number | null;
|
lastIndexed: number | null;
|
||||||
|
/** Distinct files indexed for this DataSource (one row per source document). */
|
||||||
|
fileCount: number;
|
||||||
|
/** Embedding-sized text fragments (one per ContentChunk row, ~400 tokens each). */
|
||||||
chunkCount: number;
|
chunkCount: number;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
@ -358,6 +361,7 @@ export interface RagConnectionDto {
|
||||||
knowledgeIngestionEnabled: boolean;
|
knowledgeIngestionEnabled: boolean;
|
||||||
preferences: KnowledgePreferences;
|
preferences: KnowledgePreferences;
|
||||||
dataSources: RagDataSourceDto[];
|
dataSources: RagDataSourceDto[];
|
||||||
|
totalFiles: number;
|
||||||
totalChunks: number;
|
totalChunks: number;
|
||||||
runningJobs: { jobId: string; progress: number; progressMessage: string }[];
|
runningJobs: { jobId: string; progress: number; progressMessage: string }[];
|
||||||
lastError?: { jobId: string; errorMessage: string; finishedAt: number | null } | null;
|
lastError?: { jobId: string; errorMessage: string; finishedAt: number | null } | null;
|
||||||
|
|
@ -369,12 +373,17 @@ export interface RagConnectionDto {
|
||||||
skippedPolicy: number;
|
skippedPolicy: number;
|
||||||
failed: number;
|
failed: number;
|
||||||
durationMs: number;
|
durationMs: number;
|
||||||
|
/** Name of the first budget that bit (e.g. "maxBytes", "maxItems", "maxTasks"); null if walk completed naturally. */
|
||||||
|
stoppedAtLimit?: string | null;
|
||||||
|
/** Effective limits used by the walker, for showing the value next to the limit name. */
|
||||||
|
limits?: Record<string, number>;
|
||||||
|
bytesProcessed?: number;
|
||||||
} | null;
|
} | null;
|
||||||
}
|
}
|
||||||
|
|
||||||
export interface RagInventoryDto {
|
export interface RagInventoryDto {
|
||||||
connections: RagConnectionDto[];
|
connections: RagConnectionDto[];
|
||||||
totals: { chunks: number; bytes?: number };
|
totals: { files: number; chunks: number; bytes?: number };
|
||||||
}
|
}
|
||||||
|
|
||||||
export interface RagActiveJobDto {
|
export interface RagActiveJobDto {
|
||||||
|
|
|
||||||
|
|
@ -220,6 +220,22 @@
|
||||||
opacity: 0.85;
|
opacity: 0.85;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/* Sync finished, but a hard limit (maxBytes/maxItems/...) cut the walk short.
|
||||||
|
Amber, not red — the data we DID index is valid; the user just needs to
|
||||||
|
know more would have been indexed without the limit. */
|
||||||
|
.partialBanner {
|
||||||
|
display: flex;
|
||||||
|
align-items: center;
|
||||||
|
gap: 8px;
|
||||||
|
padding: 8px 12px;
|
||||||
|
background: #fffbeb;
|
||||||
|
border: 1px solid #fcd34d;
|
||||||
|
border-radius: 6px;
|
||||||
|
margin-bottom: 8px;
|
||||||
|
font-size: 0.8125rem;
|
||||||
|
color: #92400e;
|
||||||
|
}
|
||||||
|
|
||||||
.reindexBtn {
|
.reindexBtn {
|
||||||
display: flex;
|
display: flex;
|
||||||
align-items: center;
|
align-items: center;
|
||||||
|
|
|
||||||
|
|
@ -139,6 +139,21 @@ export const RagInventoryPage: React.FC = () => {
|
||||||
return `${Math.floor(ms / 60000)}m ${Math.floor((ms % 60000) / 1000)}s`;
|
return `${Math.floor(ms / 60000)}m ${Math.floor((ms % 60000) / 1000)}s`;
|
||||||
}, []);
|
}, []);
|
||||||
|
|
||||||
|
/** Render the budget value next to its name. Bytes get MB units so the user
|
||||||
|
* immediately recognises the 200 MB default; everything else stays raw. */
|
||||||
|
const _formatLimit = useCallback((name: string, budget: number | undefined, bytesProcessed: number | undefined): string => {
|
||||||
|
if (budget == null) return name;
|
||||||
|
if (name === 'maxBytes') {
|
||||||
|
const mb = Math.round(budget / 1024 / 1024);
|
||||||
|
const procMb = bytesProcessed != null ? ` (${(bytesProcessed / 1024 / 1024).toFixed(0)} MB ${t('verarbeitet')})` : '';
|
||||||
|
return `${name}=${mb} MB${procMb}`;
|
||||||
|
}
|
||||||
|
if (name === 'maxFileSize') {
|
||||||
|
return `${name}=${Math.round(budget / 1024 / 1024)} MB`;
|
||||||
|
}
|
||||||
|
return `${name}=${budget}`;
|
||||||
|
}, [t]);
|
||||||
|
|
||||||
const scopeOptions = useMemo(() => {
|
const scopeOptions = useMemo(() => {
|
||||||
const opts: { value: string; label: string }[] = [
|
const opts: { value: string; label: string }[] = [
|
||||||
{ value: 'personal', label: t('Meine Verbindungen') },
|
{ value: 'personal', label: t('Meine Verbindungen') },
|
||||||
|
|
@ -193,7 +208,9 @@ export const RagInventoryPage: React.FC = () => {
|
||||||
{inventory && (
|
{inventory && (
|
||||||
<div className={styles.content}>
|
<div className={styles.content}>
|
||||||
<div className={styles.totals}>
|
<div className={styles.totals}>
|
||||||
<span className={styles.totalLabel}>{t('Total Chunks')}:</span>
|
<span className={styles.totalLabel}>{t('Total Dateien')}:</span>
|
||||||
|
<strong className={styles.totalValue}>{inventory.totals?.files ?? 0}</strong>
|
||||||
|
<span className={styles.totalLabel} title={t('Embedding-Fragmente (~400 Tokens), die der RAG-Retrieval trifft')}>{t('Total Chunks')}:</span>
|
||||||
<strong className={styles.totalValue}>{inventory.totals?.chunks ?? 0}</strong>
|
<strong className={styles.totalValue}>{inventory.totals?.chunks ?? 0}</strong>
|
||||||
{inventory.totals?.bytes != null && inventory.totals.bytes > 0 && (
|
{inventory.totals?.bytes != null && inventory.totals.bytes > 0 && (
|
||||||
<span className={styles.totalBytes}>{(inventory.totals.bytes / 1024 / 1024).toFixed(1)} MB</span>
|
<span className={styles.totalBytes}>{(inventory.totals.bytes / 1024 / 1024).toFixed(1)} MB</span>
|
||||||
|
|
@ -205,8 +222,13 @@ export const RagInventoryPage: React.FC = () => {
|
||||||
<div className={styles.connectionHeader}>
|
<div className={styles.connectionHeader}>
|
||||||
<span className={styles.authority}>{conn.authority}</span>
|
<span className={styles.authority}>{conn.authority}</span>
|
||||||
<span className={styles.email}>{conn.externalEmail}</span>
|
<span className={styles.email}>{conn.externalEmail}</span>
|
||||||
{conn.totalChunks > 0 && (
|
{(conn.totalFiles > 0 || conn.totalChunks > 0) && (
|
||||||
<span className={styles.connChunks}>{conn.totalChunks} chunks</span>
|
<span
|
||||||
|
className={styles.connChunks}
|
||||||
|
title={t('Embedding-Fragmente (~400 Tokens), die der RAG-Retrieval trifft')}
|
||||||
|
>
|
||||||
|
{t('{f} Dateien · {c} Chunks', { f: conn.totalFiles, c: conn.totalChunks })}
|
||||||
|
</span>
|
||||||
)}
|
)}
|
||||||
<button
|
<button
|
||||||
className={styles.consentToggle}
|
className={styles.consentToggle}
|
||||||
|
|
@ -261,6 +283,28 @@ export const RagInventoryPage: React.FC = () => {
|
||||||
s.skippedPolicy > 0 ? t('{n} übersprungen', { n: s.skippedPolicy }) : null,
|
s.skippedPolicy > 0 ? t('{n} übersprungen', { n: s.skippedPolicy }) : null,
|
||||||
s.failed > 0 ? t('{n} fehler', { n: s.failed }) : null,
|
s.failed > 0 ? t('{n} fehler', { n: s.failed }) : null,
|
||||||
].filter(Boolean).join(' · ');
|
].filter(Boolean).join(' · ');
|
||||||
|
|
||||||
|
const stop = s.stoppedAtLimit;
|
||||||
|
if (stop) {
|
||||||
|
const budget = s.limits?.[stop];
|
||||||
|
const limitText = _formatLimit(stop, budget, s.bytesProcessed);
|
||||||
|
return (
|
||||||
|
<div className={styles.partialBanner}>
|
||||||
|
<FaExclamationTriangle />
|
||||||
|
<span>
|
||||||
|
<strong>{t('Sync abgeschlossen, Korpus aber unvollständig')}</strong> ({_formatRelative(okAt)})
|
||||||
|
{' — '}
|
||||||
|
{t('Limit {l} erreicht', { l: limitText })}.
|
||||||
|
{stats && <> {stats}.</>}{' '}
|
||||||
|
{t('Weitere Dateien wurden NICHT indexiert. Limit erhöhen oder DataSource enger eingrenzen, dann erneut starten.')}
|
||||||
|
</span>
|
||||||
|
<button className={styles.reindexBtn} onClick={() => _handleReindex(conn.id)} title={t('Erneut indexieren')}>
|
||||||
|
<FaRedo size={12} /> {t('Erneut indexieren')}
|
||||||
|
</button>
|
||||||
|
</div>
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
return (
|
return (
|
||||||
<div className={styles.successBanner}>
|
<div className={styles.successBanner}>
|
||||||
<FaCheckCircle />
|
<FaCheckCircle />
|
||||||
|
|
@ -293,7 +337,12 @@ export const RagInventoryPage: React.FC = () => {
|
||||||
<div key={ds.id} className={`${styles.dsRow} ${ds.ragIndexEnabled ? styles.dsActive : ''}`}>
|
<div key={ds.id} className={`${styles.dsRow} ${ds.ragIndexEnabled ? styles.dsActive : ''}`}>
|
||||||
<span className={styles.dsLabel}>{ds.label || ds.path}</span>
|
<span className={styles.dsLabel}>{ds.label || ds.path}</span>
|
||||||
<span className={styles.dsType}>{ds.sourceType}</span>
|
<span className={styles.dsType}>{ds.sourceType}</span>
|
||||||
<span className={styles.dsChunks}>{ds.chunkCount} chunks</span>
|
<span
|
||||||
|
className={styles.dsChunks}
|
||||||
|
title={t('{f} indizierte Dateien · {c} Embedding-Chunks (~400 Tokens)', { f: ds.fileCount, c: ds.chunkCount })}
|
||||||
|
>
|
||||||
|
{ds.fileCount} {t('Dateien')} · {ds.chunkCount} {t('Chunks')}
|
||||||
|
</span>
|
||||||
<span className={styles.dsIndex}>{ds.ragIndexEnabled ? '\uD83E\uDDE0' : '\u2014'}</span>
|
<span className={styles.dsIndex}>{ds.ragIndexEnabled ? '\uD83E\uDDE0' : '\u2014'}</span>
|
||||||
</div>
|
</div>
|
||||||
))}
|
))}
|
||||||
|
|
|
||||||
Loading…
Reference in a new issue