feat: add registry-driven Kimi provider

This commit is contained in:
tommy0103
2026-07-20 22:43:23 +08:00
parent 21c3a1b9fc
commit 3ee44de4e5
39 changed files with 2240 additions and 732 deletions
+26 -109
View File
@@ -2,19 +2,17 @@
import { DB_PATH, openDb, openReadDb, openWriterLeaseDb, rebuildMemoryFts } from './db.ts';
import {
CLAUDE_DIR, CODEX_DIR, PROJECTS_DIR, fs, path, isDir, readLines,
inferProjectPath, discoverJsonlFiles, discoverCodexJsonlFiles, codexDbId, readCodexGuardianThreadInfo,
inferProjectPath, codexDbId,
} from './parsing.ts';
import { persist } from './persist.ts';
import {
createProviderIndexPlan,
indexProviderPlan,
writeProviderIndexMarkers,
} from './provider-indexing.ts';
import { nodeSqliteTransactionAdapter } from './tx.ts';
import { acquireWriterLease, writerLockPathFor } from './writer-lease.ts';
import { runRetryableWriteTransaction, isBeginBusyFailure, hasUnusableTransaction } from './write-coordinator.ts';
import {
CLAUDE_INPUT_TOKEN_SEMANTICS_MARKER,
parse as claudeParse,
} from './providers/claude.ts';
import { parse as codexParse } from './providers/codex.ts';
import type { Cursor, IndexRecord } from './providers/types.ts';
import type { ClaudeJsonlFile } from './parsing.ts';
import { createBuiltinProviderRegistry } from './providers/builtins.ts';
import type { NodeSqliteDb, SqliteRow } from './sqlite-types.ts';
const HISTORY_PATH = path.join(CLAUDE_DIR, 'history.jsonl');
@@ -37,14 +35,6 @@ function errorMessage(error: unknown): string {
}
function needsReindex(db: NodeSqliteDb, fp: string) {
const mt = fs.statSync(fp).mtimeMs;
const row = db.prepare('SELECT mtime, lines_processed FROM index_state WHERE jsonl_path = ?').get(fp);
if (!row) return { needed: true, skip: 0 };
return mt > row.mtime ? { needed: true, skip: row.lines_processed } : { needed: false, skip: 0 };
}
function indexCodexSessionIndex(db: NodeSqliteDb): void {
const indexPath = path.join(CODEX_DIR, 'session_index.jsonl');
if (!fs.existsSync(indexPath)) return;
@@ -78,27 +68,6 @@ function refreshSessionProjectPaths(db: NodeSqliteDb): void {
}
}
function indexSubagentMeta(db: NodeSqliteDb, fi: ClaudeJsonlFile): void {
if (!fi.isSubagent) return;
const mp = fi.path.replace('.jsonl', '.meta.json');
if (!fs.existsSync(mp)) return;
let meta: JsonRecord;
try {
meta = JSON.parse(fs.readFileSync(mp, 'utf8'));
} catch (e) {
process.stderr.write(`Warning: failed to read subagent meta ${mp}: ${errorMessage(e)}\n`);
return;
}
const tok = db.prepare('SELECT COALESCE(SUM(input_tokens),0)+COALESCE(SUM(output_tokens),0) as t FROM messages WHERE agent_id=?').get(fi.agentId);
const ts = db.prepare('SELECT MIN(timestamp) as t0, MAX(timestamp) as t1 FROM messages WHERE agent_id=?').get(fi.agentId);
const dur = ts?.t0 && ts?.t1 ? new Date(ts.t1).getTime() - new Date(ts.t0).getTime() : null;
if (fi.workflowRunId) {
db.prepare('INSERT OR REPLACE INTO workflow_agents (agent_id,run_id,session_id,agent_type,description) VALUES(?,?,?,?,?)').run(fi.agentId, fi.workflowRunId, fi.sessionId, meta.agentType||null, meta.description||null);
} else {
db.prepare('INSERT OR REPLACE INTO subagents VALUES(?,?,?,?,?,?,?)').run(fi.agentId, fi.sessionId, meta.toolUseId||null, meta.agentType||null, meta.description||null, dur, tok?.t||0);
}
}
function indexWorkflows(db: NodeSqliteDb): void {
if (!fs.existsSync(PROJECTS_DIR)) return;
let projects;
@@ -191,13 +160,6 @@ function inspectBuildOwnership({ force = false }: { force?: boolean } = {}) {
}
}
// A one-shot record stream that retracts a session, for routing guardian sweeps
// through persist (the single db writer) instead of deleting rows directly.
function* guardianDelete(sessionId: string): Generator<IndexRecord, Cursor> {
yield { kind: 'delete-session', sessionId };
return null;
}
function buildIndex({ force = false }: { force?: boolean } = {}) {
const ownership = inspectBuildOwnership({ force });
if (ownership.skip) return ownership;
@@ -213,17 +175,7 @@ function buildIndex({ force = false }: { force?: boolean } = {}) {
const db = openDb();
const txDb = nodeSqliteTransactionAdapter(db);
const claudeInputMarkerMissing = !db.prepare(
'SELECT jsonl_path FROM index_state WHERE jsonl_path = ?',
).get(CLAUDE_INPUT_TOKEN_SEMANTICS_MARKER);
const claudeInputSemanticsOutdated = claudeInputMarkerMissing && Boolean(db.prepare(`
SELECT 1 FROM messages
WHERE COALESCE(source, 'claude') = 'claude'
AND (input_tokens IS NOT NULL OR output_tokens IS NOT NULL)
LIMIT 1
`).get());
const skippedFiles: SkippedFile[] = [];
let claudeInputMigrationFailed = false;
try {
try {
if (force) {
@@ -246,56 +198,24 @@ function buildIndex({ force = false }: { force?: boolean } = {}) {
throw error;
}
const files = [
...discoverJsonlFiles(),
...discoverCodexJsonlFiles(),
];
for (const f of files) {
try {
runRetryableWriteTransaction(txDb, () => {
if (f.source === 'codex') {
// Codex goes through the pure adapter + shared persist (docs/adr/0001),
// full-reparse (countMode 'total') when the file changed. An unchanged
// file is not reparsed, but is still swept for stale guardian rows: a
// guardian/auto-review thread must never linger in the index, even if it
// was indexed before guardian detection removed it.
const { needed } = needsReindex(db, f.path);
if (needed) {
persist(db, { key: f.path, sessionId: '' }, codexParse({ key: f.path, sessionId: '' }, null));
} else {
const guardian = readCodexGuardianThreadInfo(f.path);
if (guardian) {
const sessionId = codexDbId(guardian.threadRawId);
if (sessionId) persist(db, { key: f.path, sessionId: '' }, guardianDelete(sessionId));
}
}
} else {
// Claude transcripts now go through the pure adapter + shared persist
// (docs/adr/0001). needsReindex keeps the "skip unchanged file" fast path;
// the cursor's line count drives incremental resume inside parse().
const { needed, skip } = needsReindex(db, f.path);
if (needed || claudeInputSemanticsOutdated) {
const unit = { key: f.path, sessionId: f.sessionId, project: f.project, isSubagent: f.isSubagent, agentId: f.agentId };
const cursor = !claudeInputSemanticsOutdated && skip > 0 ? `0:${skip}` : null;
persist(db, unit, claudeParse(unit, cursor));
}
indexSubagentMeta(db, f);
}
}, { label: `file:${f.path}` });
} catch (e) {
if (isBeginBusyFailure(e)) {
return { skip: true, reason: 'database_busy', skipped: skippedFiles.length, skippedFiles };
}
if (hasUnusableTransaction(e)) throw e;
if (claudeInputSemanticsOutdated && f.source !== 'codex') {
claudeInputMigrationFailed = true;
}
// A per-file failure is skippable: log and move on.
const error = e as { message?: unknown; obelisk?: unknown } | null;
const message = errorMessage(e);
skippedFiles.push({ path: f.path, error: message, diagnostics: error?.obelisk });
process.stderr.write(`Warning: failed to index ${f.path}: ${message}\n`);
}
const registry = createBuiltinProviderRegistry();
const providerPlan = createProviderIndexPlan(db, registry, { force });
const providerResult = indexProviderPlan({
db,
plan: providerPlan,
runTransaction: (label, work) => runRetryableWriteTransaction(txDb, work, { label }),
onError: (error, { provider, unit }) => {
if (isBeginBusyFailure(error)) return 'stop';
if (hasUnusableTransaction(error)) throw error;
const detail = error as { message?: unknown; obelisk?: unknown } | null;
const message = errorMessage(error);
skippedFiles.push({ path: unit.key, error: message, diagnostics: detail?.obelisk });
process.stderr.write(`Warning: failed to index ${provider.name} unit ${unit.key}: ${message}\n`);
return 'skip';
},
});
if (providerResult.stopped) {
return { skip: true, reason: 'database_busy', skipped: skippedFiles.length, skippedFiles };
}
// Finalize is one transaction and is NOT swallowed: a finalize failure fails
// the build (a half-finalized index would be inconsistent).
@@ -308,10 +228,7 @@ function buildIndex({ force = false }: { force?: boolean } = {}) {
db.exec("INSERT INTO messages_fts(messages_fts) VALUES('rebuild')");
rebuildMemoryFts(db);
db.prepare("INSERT OR REPLACE INTO index_state (jsonl_path, mtime, lines_processed) VALUES ('__last_build__', ?, 0)").run(Date.now());
if (!claudeInputSemanticsOutdated || !claudeInputMigrationFailed) {
db.prepare('INSERT OR REPLACE INTO index_state (jsonl_path, mtime, lines_processed) VALUES (?, ?, 0)')
.run(CLAUDE_INPUT_TOKEN_SEMANTICS_MARKER, Date.now());
}
writeProviderIndexMarkers(db, providerPlan, providerResult);
}, { label: 'finalize' });
} catch (error) {
if (isBeginBusyFailure(error)) {
+7 -7
View File
@@ -150,13 +150,13 @@ function inferProjectPath(project: string | null | undefined, observedCwds: unkn
return best?.path || legacyProjectPathFromSlug(project);
}
function discoverJsonlFiles(): ClaudeJsonlFile[] {
function discoverJsonlFiles(projectsDir = PROJECTS_DIR): ClaudeJsonlFile[] {
const files: ClaudeJsonlFile[] = [];
if (!fs.existsSync(PROJECTS_DIR)) return files;
if (!fs.existsSync(projectsDir)) return files;
let projects;
try { projects = fs.readdirSync(PROJECTS_DIR); } catch (e) { process.stderr.write(`Warning: cannot read projects dir: ${e instanceof Error ? e.message : String(e)}\n`); return files; }
try { projects = fs.readdirSync(projectsDir); } catch (e) { process.stderr.write(`Warning: cannot read projects dir: ${e instanceof Error ? e.message : String(e)}\n`); return files; }
for (const proj of projects) {
const projPath = path.join(PROJECTS_DIR, proj);
const projPath = path.join(projectsDir, proj);
if (!isDir(projPath)) continue;
let entries;
try { entries = fs.readdirSync(projPath); } catch { continue; }
@@ -192,9 +192,9 @@ function discoverJsonlFiles(): ClaudeJsonlFile[] {
return files;
}
function discoverCodexJsonlFiles(): CodexJsonlFile[] {
function discoverCodexJsonlFiles(sessionsDir = CODEX_SESSIONS_DIR): CodexJsonlFile[] {
const files: CodexJsonlFile[] = [];
if (!fs.existsSync(CODEX_SESSIONS_DIR)) return files;
if (!fs.existsSync(sessionsDir)) return files;
const walk = (dir: string): void => {
let entries;
try { entries = fs.readdirSync(dir, { withFileTypes: true }); } catch { return; }
@@ -207,7 +207,7 @@ function discoverCodexJsonlFiles(): CodexJsonlFile[] {
}
}
};
walk(CODEX_SESSIONS_DIR);
walk(sessionsDir);
return files;
}
+106
View File
@@ -0,0 +1,106 @@
import { persist } from './persist.ts';
import type { ProviderRegistry } from './providers/registry.ts';
import type { Cursor, IndexUnit, ProviderAdapter } from './providers/types.ts';
import type { SqliteDb } from './sqlite-types.ts';
export interface ProviderIndexItem {
readonly provider: ProviderAdapter;
readonly unit: IndexUnit;
readonly cursor: Cursor;
}
export interface ProviderIndexPlan {
readonly items: ProviderIndexItem[];
readonly pendingMarkers: ReadonlyMap<string, string>;
}
export interface ProviderIndexResult {
readonly committed: ProviderIndexItem[];
readonly failedProviders: ReadonlySet<string>;
readonly stopped?: { item: ProviderIndexItem; error: unknown };
}
export function storedProviderCursor(db: SqliteDb, key: string): Cursor {
const row = db.prepare('SELECT mtime, lines_processed FROM index_state WHERE jsonl_path = ?').get(key);
return row ? `${String(row.mtime)}:${String(row.lines_processed)}` : null;
}
function sourceAlreadyIndexed(db: SqliteDb, source: string): boolean {
return Boolean(db.prepare('SELECT 1 FROM sessions WHERE source = ? LIMIT 1').get(source));
}
export function createProviderIndexPlan(
db: SqliteDb,
registry: ProviderRegistry,
{ force = false, changedPaths }: { force?: boolean; changedPaths?: string[] } = {},
): ProviderIndexPlan {
const items: ProviderIndexItem[] = [];
const pendingMarkers = new Map<string, string>();
for (const provider of registry.list()) {
const marker = provider.indexVersionMarker;
const markerMissing = marker !== undefined && !db.prepare(
'SELECT jsonl_path FROM index_state WHERE jsonl_path = ?',
).get(marker);
if (markerMissing) pendingMarkers.set(provider.name, marker);
const fullReindex = force || (markerMissing && sourceAlreadyIndexed(db, provider.name));
const units = provider.discover({
lastCursor: fullReindex ? () => null : (key) => storedProviderCursor(db, key),
changedPaths: fullReindex ? undefined : changedPaths,
});
for (const unit of units) {
items.push({
provider,
unit,
cursor: fullReindex ? null : storedProviderCursor(db, unit.key),
});
}
}
return { items, pendingMarkers };
}
export function indexProviderPlan({
db,
plan,
runTransaction,
onCommitted = () => {},
onError,
}: {
db: SqliteDb;
plan: ProviderIndexPlan;
runTransaction: <T>(label: string, work: () => T) => T;
onCommitted?: (item: ProviderIndexItem, cursor: Cursor) => void;
onError: (error: unknown, item: ProviderIndexItem) => 'skip' | 'stop';
}): ProviderIndexResult {
const committed: ProviderIndexItem[] = [];
const failedProviders = new Set<string>();
for (const item of plan.items) {
try {
const cursor = runTransaction(`provider:${item.provider.name}:${item.unit.key}`, () => (
persist(db, item.unit, item.provider.parse(item.unit, item.cursor))
));
committed.push(item);
onCommitted(item, cursor);
} catch (error) {
failedProviders.add(item.provider.name);
if (onError(error, item) === 'stop') {
return { committed, failedProviders, stopped: { item, error } };
}
}
}
return { committed, failedProviders };
}
export function writeProviderIndexMarkers(
db: SqliteDb,
plan: ProviderIndexPlan,
result: ProviderIndexResult,
): void {
const write = db.prepare(
'INSERT OR REPLACE INTO index_state (jsonl_path, mtime, lines_processed) VALUES (?, ?, 0)',
);
for (const [provider, marker] of plan.pendingMarkers) {
if (!result.failedProviders.has(provider) && result.stopped === undefined) {
write.run(marker, Date.now());
}
}
}
+14
View File
@@ -0,0 +1,14 @@
import { createClaudeProvider } from './claude.ts';
import { createCodexProvider } from './codex.ts';
import { createKimiProvider } from './kimi.ts';
import { createProviderRegistry, type ProviderRegistry } from './registry.ts';
export type BuiltinProviderRoots = Readonly<Record<string, string | undefined>>;
export function createBuiltinProviderRegistry(roots: BuiltinProviderRoots = {}): ProviderRegistry {
return createProviderRegistry([
createClaudeProvider({ rootDir: roots['claude'] }),
createCodexProvider({ rootDir: roots['codex'] }),
createKimiProvider({ rootDir: roots['kimi'] }),
]);
}
+139 -7
View File
@@ -7,6 +7,8 @@
// (started_at/ended_at/message_count); persist merges them with any existing row.
import { createRequire } from 'node:module';
import { homedir } from 'node:os';
import { dirname, isAbsolute, join, normalize, relative } from 'node:path';
const require = createRequire(import.meta.url);
const fs = require('node:fs');
@@ -15,7 +17,15 @@ import {
filePath, trunc, truncJson, readLines, discoverJsonlFiles,
} from '../parsing.ts';
import type { Cursor, DiscoverContext, IndexRecord, IndexUnit, Provider } from './types.ts';
import type {
Cursor,
DiscoverContext,
IndexRecord,
IndexUnit,
ProviderAdapter,
RawLookup,
RawRecord,
} from './types.ts';
// Claude cursor encodes the file mtime and the number of lines already indexed:
// "<mtimeMs>:<linesProcessed>". mtime lets discovery detect change; lines lets
@@ -46,8 +56,32 @@ function totalInputTokens(usage: Record<string, unknown>): number | null {
return seen ? total : null;
}
export function discover(_ctx: DiscoverContext): IndexUnit[] {
return discoverJsonlFiles().map((f: any) => ({
function discoverAt(rootDir: string, ctx: DiscoverContext): IndexUnit[] {
const projectsDir = join(rootDir, 'projects');
const changedTranscriptPaths = new Set<string>();
const forcedPaths = new Set<string>();
for (const changedPath of ctx.changedPaths ?? []) {
const absolute = isAbsolute(changedPath)
? normalize(changedPath)
: normalize(join(projectsDir, changedPath));
const inside = relative(projectsDir, absolute);
if (!inside || inside.startsWith('..') || isAbsolute(inside)) continue;
if (absolute.toLowerCase().endsWith('.meta.json')) {
const transcript = absolute.slice(0, -'.meta.json'.length) + '.jsonl';
changedTranscriptPaths.add(transcript);
forcedPaths.add(transcript);
} else if (absolute.toLowerCase().endsWith('.jsonl')) {
changedTranscriptPaths.add(absolute);
}
}
return discoverJsonlFiles(projectsDir).filter((file) => {
const normalizedPath = normalize(file.path);
if (ctx.changedPaths !== undefined && !changedTranscriptPaths.has(normalizedPath)) return false;
const cursor = ctx.lastCursor(file.path);
return forcedPaths.has(normalizedPath)
|| cursor === null
|| Number(cursor.split(':')[0]) < fs.statSync(file.path).mtimeMs;
}).map((f: any) => ({
key: f.path,
sessionId: f.sessionId,
project: f.project,
@@ -57,6 +91,10 @@ export function discover(_ctx: DiscoverContext): IndexUnit[] {
}));
}
export function discover(ctx: DiscoverContext): IndexUnit[] {
return discoverAt(join(homedir(), '.claude'), ctx);
}
export function* parse(unit: IndexUnit, cursor: Cursor): Generator<IndexRecord, Cursor> {
const skip = cursorToSkip(cursor);
const mtime = fs.statSync(unit.key).mtimeMs;
@@ -70,15 +108,28 @@ export function* parse(unit: IndexUnit, cursor: Cursor): Generator<IndexRecord,
title: null as string | null,
n: 0,
};
const subagentStats = {
startedAt: null as string | null,
endedAt: null as string | null,
totalTokens: 0,
};
let lineNum = 0;
readLines(unit.key, (line: string) => {
lineNum++;
if (lineNum <= skip) return;
let obj: any;
try { obj = JSON.parse(line); } catch { return; }
const sid = unit.sessionId;
const ts = obj.timestamp || null;
const msg = obj.message || {};
const usage = msg.usage || {};
if (isSubagent && (obj.type === 'user' || obj.type === 'assistant')) {
if (ts && (!subagentStats.startedAt || ts < subagentStats.startedAt)) subagentStats.startedAt = ts;
if (ts && (!subagentStats.endedAt || ts > subagentStats.endedAt)) subagentStats.endedAt = ts;
subagentStats.totalTokens += (totalInputTokens(usage) ?? 0) + (usage.output_tokens ?? 0);
}
if (lineNum <= skip) return;
if (obj.type === 'ai-title' && obj.aiTitle) { sm.title = obj.aiTitle; return; }
if (obj.type === 'system' && obj.subtype === 'away_summary' && obj.content) {
@@ -97,11 +148,9 @@ export function* parse(unit: IndexUnit, cursor: Cursor): Generator<IndexRecord,
if (obj.version) sm.version = obj.version;
sm.n++;
const msg = obj.message || {};
const text = extractText(msg.content);
const contentType = extractContentType(msg.content);
const isMeta = extractMessageIsMeta(obj, text);
const usage = msg.usage || {};
const aid = isSubagent ? (unit.agentId ?? null) : (obj.agentId || null);
if (obj.uuid) {
@@ -132,6 +181,39 @@ export function* parse(unit: IndexUnit, cursor: Cursor): Generator<IndexRecord,
}
});
if (isSubagent && unit.agentId) {
const metaPath = unit.key.replace(/\.jsonl$/, '.meta.json');
if (fs.existsSync(metaPath)) {
try {
const meta = JSON.parse(fs.readFileSync(metaPath, 'utf8'));
const workflowRunId = (unit.meta as { workflowRunId?: string } | undefined)?.workflowRunId;
if (workflowRunId) {
records.push({
kind: 'workflow_agent',
agent_id: unit.agentId,
run_id: workflowRunId,
session_id: unit.sessionId,
agent_type: meta.agentType || null,
description: meta.description || null,
});
} else {
const started = subagentStats.startedAt ? new Date(subagentStats.startedAt).getTime() : null;
const ended = subagentStats.endedAt ? new Date(subagentStats.endedAt).getTime() : null;
records.push({
kind: 'subagent',
agent_id: unit.agentId,
session_id: unit.sessionId,
parent_tool_use_id: meta.toolUseId || null,
agent_type: meta.agentType || null,
description: meta.description || null,
duration_ms: started !== null && ended !== null ? ended - started : null,
total_tokens: subagentStats.totalTokens,
});
}
} catch { /* malformed optional subagent metadata */ }
}
}
// Subagent transcripts do not own a session row (matches indexJsonl).
if (!isSubagent) {
records.push({
@@ -146,4 +228,54 @@ export function* parse(unit: IndexUnit, cursor: Cursor): Generator<IndexRecord,
return `${mtime}:${lineNum}`;
}
export const claudeProvider: Provider = { name, discover, parse };
function rawClaude(input: RawLookup): RawRecord | null {
const mainPath = typeof input.session?.jsonl_path === 'string' ? input.session.jsonl_path : null;
if (mainPath === null) return null;
let sourcePath = mainPath;
if (input.agentId !== null) {
const runId = input.workflowAgent?.['run_id'];
sourcePath = typeof runId === 'string'
? join(dirname(mainPath), String(input.session?.id ?? ''), 'subagents', 'workflows', runId, `${input.agentId}.jsonl`)
: join(dirname(mainPath), String(input.session?.id ?? ''), 'subagents', `${input.agentId}.jsonl`);
}
if (!fs.existsSync(sourcePath)) return null;
let found: string | null = null;
readLines(sourcePath, (line: string) => {
if (!line.includes(input.messageUuid)) return;
try {
if (JSON.parse(line)?.uuid === input.messageUuid) {
found = line;
return false;
}
} catch { /* malformed source line */ }
});
const raw = found as string | null;
let messageText: string | null = null;
if (raw !== null) {
try {
const content = JSON.parse(raw)?.message?.content;
if (typeof content === 'string') messageText = content;
else if (Array.isArray(content)) {
const parts = content.map((part) => part?.text ?? part?.thinking).filter((part) => typeof part === 'string');
messageText = parts.length > 0 ? parts.join('\n') : null;
}
} catch { /* malformed source line */ }
}
return raw === null
? null
: { text: raw, totalLength: raw.length, offset: 0, limit: raw.length, hasMore: false, messageText };
}
export function createClaudeProvider({ rootDir = join(homedir(), '.claude') }: { rootDir?: string } = {}): ProviderAdapter {
return {
name,
descriptor: { id: name, name: 'Claude Code', vendor: 'Anthropic', defaultRoot: rootDir, color: '#d97757' },
indexVersionMarker: CLAUDE_INPUT_TOKEN_SEMANTICS_MARKER,
watchRoots: (configuredRoot) => [join(configuredRoot, 'projects')],
discover: (ctx) => discoverAt(rootDir, ctx),
parse,
raw: rawClaude,
};
}
export const claudeProvider = createClaudeProvider();
+117 -4
View File
@@ -9,6 +9,8 @@
// The per-line logic mirrors the original indexCodexJsonl.
import { createRequire } from 'node:module';
import { homedir } from 'node:os';
import { isAbsolute, join, normalize, relative } from 'node:path';
const require = createRequire(import.meta.url);
const fs = require('node:fs');
@@ -19,14 +21,62 @@ import {
codexIsGuardianThread, codexAgentNickname, codexAgentRole, codexUsage,
codexEventText, codexMessagePayloadText, codexVisibleMessageKey,
codexToolInput, codexToolOutput,
readCodexGuardianThreadInfo,
} from '../parsing.ts';
import type { Cursor, DiscoverContext, IndexRecord, IndexUnit, MessageRecord, Provider } from './types.ts';
import type {
Cursor,
DiscoverContext,
IndexRecord,
IndexUnit,
MessageRecord,
ProviderAdapter,
RawLookup,
RawRecord,
} from './types.ts';
export const name = 'codex';
export function discover(_ctx: DiscoverContext): IndexUnit[] {
return discoverCodexJsonlFiles().map((f: any) => ({ key: f.path, sessionId: '', meta: { source: 'codex' } }));
function discoverAt(rootDir: string, ctx: DiscoverContext): IndexUnit[] {
const sessionsDir = join(rootDir, 'sessions');
const changedFiles = new Set<string>();
for (const changedPath of ctx.changedPaths ?? []) {
const absolute = isAbsolute(changedPath)
? normalize(changedPath)
: normalize(join(sessionsDir, changedPath));
const inside = relative(sessionsDir, absolute);
if (!inside || inside.startsWith('..') || isAbsolute(inside)) continue;
if (absolute.toLowerCase().endsWith('.jsonl')) changedFiles.add(absolute);
}
return discoverCodexJsonlFiles(sessionsDir).flatMap((file) => {
if (ctx.changedPaths !== undefined && !changedFiles.has(normalize(file.path))) return [];
const cursor = ctx.lastCursor(file.path);
const guardian = readCodexGuardianThreadInfo(file.path);
if (cursor !== null && Number(cursor.split(':')[0]) >= fs.statSync(file.path).mtimeMs && guardian === null) {
return [];
}
let meta: any = null;
readLines(file.path, (line: string) => {
try {
const record = JSON.parse(line);
if (record?.type === 'session_meta' && record.payload?.id) {
meta = record.payload;
return false;
}
} catch { /* malformed source line */ }
});
const rawId = meta ? codexRawId(meta.id) : null;
const parentId = meta ? codexParentThreadId(meta) : null;
return [{
key: file.path,
sessionId: guardian === null ? codexDbId(parentId || rawId) ?? '' : '',
meta: { source: 'codex', guardian: guardian !== null },
}];
});
}
export function discover(ctx: DiscoverContext): IndexUnit[] {
return discoverAt(join(homedir(), '.codex'), ctx);
}
export function* parse(unit: IndexUnit, _cursor: Cursor): Generator<IndexRecord, Cursor> {
@@ -217,4 +267,67 @@ export function* parse(unit: IndexUnit, _cursor: Cursor): Generator<IndexRecord,
return outCursor;
}
export const codexProvider: Provider = { name, discover, parse };
function findCodexFile(rootDir: string, rawThreadId: string): string | null {
const stack = [join(rootDir, 'sessions')];
while (stack.length > 0) {
const current = stack.pop()!;
if (!fs.existsSync(current)) continue;
for (const entry of fs.readdirSync(current, { withFileTypes: true })) {
const path = join(current, entry.name);
if (entry.isDirectory()) stack.push(path);
else if (entry.isFile() && entry.name.endsWith(`${rawThreadId}.jsonl`)) return path;
}
}
return null;
}
function rawCodex(rootDir: string, input: RawLookup): RawRecord | null {
const match = /^codex:([^:]+):(\d+)$/.exec(input.messageUuid);
if (match === null) return null;
const path = input.agentId === null && typeof input.session?.jsonl_path === 'string'
? input.session.jsonl_path
: findCodexFile(rootDir, match[1]!);
if (path === null || !fs.existsSync(path)) return null;
let lineNumber = 0;
let found: string | null = null;
readLines(path, (line: string) => {
lineNumber++;
if (lineNumber !== Number(match[2])) return;
found = line;
return false;
});
const raw = found as string | null;
let messageText: string | null = null;
if (raw !== null) {
try {
const obj = JSON.parse(raw);
const payload = obj?.payload ?? {};
if (obj?.type === 'event_msg') {
messageText = typeof payload.message === 'string'
? payload.message
: typeof payload.text === 'string'
? payload.text
: null;
} else if (obj?.type === 'response_item' && payload.type === 'message' && Array.isArray(payload.content)) {
const parts = payload.content.map((part: any) => part?.text).filter((part: unknown) => typeof part === 'string');
messageText = parts.length > 0 ? parts.join('\n') : null;
}
} catch { /* malformed source line */ }
}
return raw === null
? null
: { text: raw, totalLength: raw.length, offset: 0, limit: raw.length, hasMore: false, messageText };
}
export function createCodexProvider({ rootDir = join(homedir(), '.codex') }: { rootDir?: string } = {}): ProviderAdapter {
return {
name,
descriptor: { id: name, name: 'Codex', vendor: 'OpenAI', defaultRoot: rootDir, color: '#10a37f' },
watchRoots: (configuredRoot) => [join(configuredRoot, 'sessions'), join(configuredRoot, 'session_index.jsonl')],
discover: (ctx) => discoverAt(rootDir, ctx),
parse,
raw: (input) => rawCodex(rootDir, input),
};
}
export const codexProvider = createCodexProvider();
+656
View File
@@ -0,0 +1,656 @@
import {
existsSync,
readdirSync,
readFileSync,
statSync,
} from 'node:fs';
import { homedir } from 'node:os';
import { basename, dirname, isAbsolute, join, normalize, relative, sep } from 'node:path';
import { filePath, projectSlugFromPath, trunc, truncJson } from '../parsing.ts';
import type {
Cursor,
DiscoverContext,
IndexRecord,
IndexUnit,
MessageRecord,
ProviderAdapter,
RawLookup,
RawRecord,
SubagentRecord,
SummaryRecord,
ToolCallRecord,
ToolResultRecord,
} from './types.ts';
type JsonRecord = Record<string, any>;
interface KimiWireFile {
readonly agentId: string;
readonly main: boolean;
readonly path: string;
}
interface KimiSessionUnitMeta {
readonly kind: 'session';
readonly sessionDir: string;
readonly statePath: string;
readonly wireFiles: readonly KimiWireFile[];
readonly currentCursor: Exclude<Cursor, null>;
}
interface LineRecord {
readonly line: number;
readonly record: JsonRecord;
}
interface ProjectedSession {
readonly messages: MessageRecord[];
readonly toolCalls: ToolCallRecord[];
readonly toolResults: ToolResultRecord[];
readonly summaries: SummaryRecord[];
readonly subagents: SubagentRecord[];
readonly durations: IndexRecord[];
readonly mainMessageCount: number;
readonly mainWirePath: string;
}
const SOURCE = 'kimi';
function defaultKimiRoot(): string {
return process.env['KIMI_CODE_HOME'] ?? join(homedir(), '.kimi-code');
}
function readState(path: string): JsonRecord {
try {
const parsed = JSON.parse(readFileSync(path, 'utf8')) as unknown;
return parsed !== null && typeof parsed === 'object' && !Array.isArray(parsed)
? parsed as JsonRecord
: {};
} catch {
return {};
}
}
function readWire(path: string): LineRecord[] {
const lines = readFileSync(path, 'utf8').split('\n');
const records: LineRecord[] = [];
for (let index = 0; index < lines.length; index++) {
const line = lines[index]!.endsWith('\r') ? lines[index]!.slice(0, -1) : lines[index]!;
if (line.length === 0) continue;
try {
records.push({ line: index + 1, record: JSON.parse(line) as JsonRecord });
} catch (error) {
if (index === lines.length - 1) break;
throw new Error(`wire.jsonl: corrupted line ${index + 1} in ${path}: ${String(error)}`, {
cause: error,
});
}
}
return records;
}
function listWireFiles(sessionDir: string): KimiWireFile[] {
const agentsDir = join(sessionDir, 'agents');
const files: KimiWireFile[] = [];
if (existsSync(agentsDir)) {
for (const entry of readdirSync(agentsDir, { withFileTypes: true })) {
if (!entry.isDirectory()) continue;
const path = join(agentsDir, entry.name, 'wire.jsonl');
if (existsSync(path)) files.push({ agentId: entry.name, main: entry.name === 'main', path });
}
}
if (!files.some((file) => file.main)) {
const legacy = join(sessionDir, 'wire.jsonl');
if (existsSync(legacy)) files.push({ agentId: 'main', main: true, path: legacy });
}
return files.sort((a, b) => Number(b.main) - Number(a.main) || a.agentId.localeCompare(b.agentId));
}
function fileLineCount(path: string): number {
const raw = readFileSync(path, 'utf8');
if (raw.length === 0) return 0;
const newlines = raw.match(/\n/g)?.length ?? 0;
return newlines + (raw.endsWith('\n') ? 0 : 1);
}
function cursorFor(statePath: string, wires: readonly KimiWireFile[]): Exclude<Cursor, null> {
const paths = [statePath, ...wires.map((wire) => wire.path)].filter(existsSync);
let maxMtime = 0;
let totalLines = 0;
for (const path of paths) {
maxMtime = Math.max(maxMtime, statSync(path).mtimeMs);
totalLines += fileLineCount(path);
}
return `${maxMtime}:${totalLines}`;
}
function normalizeTime(value: unknown): string | null {
if (typeof value === 'number' && Number.isFinite(value)) return new Date(value).toISOString();
if (typeof value === 'string' && !Number.isNaN(Date.parse(value))) {
return new Date(value).toISOString();
}
return null;
}
function contentParts(content: unknown): JsonRecord[] {
if (typeof content === 'string') return [{ type: 'text', text: content }];
return Array.isArray(content)
? content.filter((part): part is JsonRecord => part !== null && typeof part === 'object')
: [];
}
function partText(part: JsonRecord): string | null {
if (part.type === 'text' && typeof part.text === 'string') return trunc(part.text);
if (part.type === 'thinking' && typeof part.thinking === 'string') return trunc(part.thinking);
return null;
}
function messageText(content: unknown): string | null {
const parts = contentParts(content);
const text = parts.map(partText).filter((value): value is string => value !== null);
return text.length > 0 ? trunc(text.join('\n')) : null;
}
function messageContentType(content: unknown): string {
const types = new Set(contentParts(content).map((part) => String(part.type ?? 'unknown')));
return types.size === 1 ? [...types][0]! : 'unknown';
}
function namespacedSessionId(nativeId: string): string {
return `kimi:${nativeId}`;
}
function namespacedAgentId(sessionId: string, agentId: string): string {
return `${sessionId}:${agentId}`;
}
function namespacedEventId(sessionId: string, agentId: string, nativeId: unknown, line: number): string {
const suffix = typeof nativeId === 'string' && nativeId.length > 0 ? nativeId : `line-${line}`;
return `${sessionId}:${agentId}:${suffix}`;
}
function namespacedToolId(sessionId: string, agentId: string, nativeId: unknown): string {
return `${sessionId}:${agentId}:${String(nativeId)}`;
}
function numericField(record: JsonRecord, ...fields: string[]): number | null {
const value = fields.map((field) => record[field]).find((candidate) => (
typeof candidate === 'number' && Number.isFinite(candidate)
));
return typeof value === 'number' && Number.isFinite(value) ? value : null;
}
function inputUsage(usage: JsonRecord): number | null {
const normalized = numericField(usage, 'input_tokens', 'inputTokens');
if (normalized !== null) return normalized;
const fields = ['inputOther', 'inputCacheRead', 'inputCacheCreation'];
const values = fields.map((field) => numericField(usage, field));
return values.some((value) => value !== null)
? values.reduce<number>((sum, value) => sum + (value ?? 0), 0)
: null;
}
function outputUsage(usage: JsonRecord): number | null {
return numericField(usage, 'output_tokens', 'outputTokens', 'output');
}
function isRealUserMessage(message: JsonRecord): boolean {
if (message.role !== 'user') return false;
const origin = message.origin as JsonRecord | undefined;
if (origin === undefined || origin.kind === 'user') return true;
return (origin.kind === 'skill_activation' || origin.kind === 'plugin_command')
&& origin.trigger === 'user-slash';
}
function projectSession(meta: KimiSessionUnitMeta, sessionId: string, state: JsonRecord): ProjectedSession {
const cwd = typeof state.cwd === 'string'
? state.cwd
: typeof state.workDir === 'string'
? state.workDir
: null;
const messages: MessageRecord[] = [];
const toolCalls: ToolCallRecord[] = [];
const toolResults: ToolResultRecord[] = [];
const summaries: SummaryRecord[] = [];
const durations: IndexRecord[] = [];
const childParentCalls = new Map<string, string>();
let mainMessageCount = 0;
for (const wire of meta.wireFiles) {
const wireMessageStart = messages.length;
const records = readWire(wire.path);
const agentDbId = wire.main ? null : namespacedAgentId(sessionId, wire.agentId);
let previousUuid: string | null = null;
let model: string | null = null;
const stepStarts = new Map<string, number>();
const stepMessages = new Map<string, MessageRecord[]>();
const callMessageUuids = new Map<string, string>();
const injectionMessageUuids = new Set<string>();
const realUserMessageUuids = new Set<string>();
let undoFloor = wireMessageStart;
const resetOpenState = (): void => {
stepStarts.clear();
stepMessages.clear();
callMessageUuids.clear();
};
const applyUndo = (count: number): void => {
if (count <= 0) return;
const removedMessageUuids = new Set<string>();
let removedUserCount = 0;
for (let index = messages.length - 1; index >= undoFloor; index--) {
const message = messages[index]!;
if (injectionMessageUuids.has(message.uuid)) continue;
messages.splice(index, 1);
removedMessageUuids.add(message.uuid);
injectionMessageUuids.delete(message.uuid);
if (wire.main) mainMessageCount--;
if (realUserMessageUuids.delete(message.uuid)) {
removedUserCount++;
if (removedUserCount >= count) break;
}
}
const removedToolIds = new Set(
toolCalls.filter((call) => removedMessageUuids.has(call.message_uuid)).map((call) => call.id),
);
for (let index = toolCalls.length - 1; index >= 0; index--) {
if (removedMessageUuids.has(toolCalls[index]!.message_uuid)) toolCalls.splice(index, 1);
}
for (let index = toolResults.length - 1; index >= 0; index--) {
const result = toolResults[index]!;
if (removedMessageUuids.has(result.message_uuid) || removedToolIds.has(result.tool_use_id)) {
toolResults.splice(index, 1);
}
}
for (let index = durations.length - 1; index >= 0; index--) {
const duration = durations[index]!;
if (duration.kind === 'message-turn-duration' && removedMessageUuids.has(duration.uuid)) {
durations.splice(index, 1);
}
}
previousUuid = messages.slice(wireMessageStart).at(-1)?.uuid ?? null;
resetOpenState();
};
const pushMessage = (message: MessageRecord, stepUuid?: string): void => {
messages.push(message);
if (wire.main) mainMessageCount++;
previousUuid = message.uuid;
if (stepUuid !== undefined) {
const entries = stepMessages.get(stepUuid) ?? [];
entries.push(message);
stepMessages.set(stepUuid, entries);
}
};
for (const { line, record } of records) {
const timestamp = normalizeTime(record.time);
if (record.type === 'config.update') {
model = typeof record.modelAlias === 'string' ? record.modelAlias : model;
continue;
}
if (record.type === 'context.clear') {
undoFloor = messages.length;
resetOpenState();
continue;
}
if (record.type === 'context.undo') {
applyUndo(typeof record.count === 'number' ? record.count : 0);
continue;
}
if (record.type === 'context.append_message') {
const source = record.message as JsonRecord | undefined;
if (source === undefined || typeof source.role !== 'string') continue;
const uuid = namespacedEventId(sessionId, wire.agentId, source.id, line);
const origin = source.origin as JsonRecord | undefined;
const messageUuid = uuid;
pushMessage({
kind: 'message',
uuid,
session_id: sessionId,
type: source.role,
parent_uuid: previousUuid,
timestamp,
role: source.role,
text: messageText(source.content),
content_type: messageContentType(source.content),
is_meta: origin !== undefined && origin.kind !== 'user' ? 1 : 0,
model,
is_sidechain: wire.main ? 0 : 1,
agent_id: agentDbId,
input_tokens: null,
output_tokens: null,
cwd,
skill: null,
source: SOURCE,
});
if (origin?.kind === 'injection') injectionMessageUuids.add(messageUuid);
if (isRealUserMessage(source)) realUserMessageUuids.add(messageUuid);
if (Array.isArray(source.toolCalls)) {
for (const call of source.toolCalls) {
if (call === null || typeof call !== 'object' || typeof call.id !== 'string') continue;
const fn = call.function as JsonRecord | undefined;
const name = typeof fn?.name === 'string' ? fn.name : 'tool';
let args: unknown = fn?.arguments ?? {};
if (typeof args === 'string') {
try { args = JSON.parse(args); } catch { args = { raw: args }; }
}
const toolId = namespacedToolId(sessionId, wire.agentId, call.id);
toolCalls.push({
kind: 'tool_call',
id: toolId,
message_uuid: messageUuid,
session_id: sessionId,
name,
input_json: truncJson(args) ?? '{}',
file_path: filePath(name, args as JsonRecord | undefined),
});
callMessageUuids.set(call.id, messageUuid);
}
}
if (source.role === 'tool' && typeof source.toolCallId === 'string') {
const toolId = namespacedToolId(sessionId, wire.agentId, source.toolCallId);
toolResults.push({
kind: 'tool_result',
tool_use_id: toolId,
message_uuid: messageUuid,
session_id: sessionId,
content: messageText(source.content) ?? '',
file_path: null,
is_error: source.isError === true ? 1 : 0,
});
}
continue;
}
if (record.type === 'context.apply_compaction') {
const content = typeof record.contextSummary === 'string'
? record.contextSummary
: typeof record.summary === 'string'
? record.summary
: messageText((record.summary as JsonRecord | undefined)?.content);
if (content !== null) {
summaries.push({
kind: 'summary',
id: namespacedEventId(sessionId, wire.agentId, undefined, line),
session_id: sessionId,
timestamp,
source: 'compaction',
content,
});
}
undoFloor = messages.length;
resetOpenState();
continue;
}
if (record.type !== 'context.append_loop_event') continue;
const event = record.event as JsonRecord | undefined;
if (event === undefined || typeof event.type !== 'string') continue;
if (event.type === 'step.begin' && typeof event.uuid === 'string') {
stepStarts.set(event.uuid, typeof record.time === 'number' ? record.time : 0);
continue;
}
if (event.type === 'content.part' && typeof event.stepUuid === 'string') {
const part = event.part as JsonRecord | undefined;
if (part === undefined) continue;
pushMessage({
kind: 'message',
uuid: namespacedEventId(sessionId, wire.agentId, event.uuid, line),
session_id: sessionId,
type: 'assistant',
parent_uuid: previousUuid,
timestamp,
role: 'assistant',
text: partText(part),
content_type: typeof part.type === 'string' ? part.type : 'unknown',
is_meta: 0,
model,
is_sidechain: wire.main ? 0 : 1,
agent_id: agentDbId,
input_tokens: null,
output_tokens: null,
cwd,
skill: null,
source: SOURCE,
}, event.stepUuid);
continue;
}
if (event.type === 'tool.call' && typeof event.stepUuid === 'string' && event.toolCallId !== undefined) {
const uuid = namespacedEventId(sessionId, wire.agentId, event.uuid, line);
const toolId = namespacedToolId(sessionId, wire.agentId, event.toolCallId);
pushMessage({
kind: 'message', uuid, session_id: sessionId, type: 'assistant', parent_uuid: previousUuid,
timestamp, role: 'assistant', text: null, content_type: 'tool_use', is_meta: 0, model,
is_sidechain: wire.main ? 0 : 1, agent_id: agentDbId, input_tokens: null,
output_tokens: null, cwd, skill: null, source: SOURCE,
}, event.stepUuid);
toolCalls.push({
kind: 'tool_call',
id: toolId,
message_uuid: uuid,
session_id: sessionId,
name: String(event.name ?? 'tool'),
input_json: truncJson(event.args ?? {}) ?? '{}',
file_path: filePath(String(event.name ?? 'tool'), event.args as JsonRecord | undefined),
});
callMessageUuids.set(String(event.toolCallId), uuid);
continue;
}
if (event.type === 'tool.result' && event.toolCallId !== undefined) {
const nativeToolId = String(event.toolCallId);
const result = event.result as JsonRecord | undefined;
const output = result?.output;
const content = typeof output === 'string' ? trunc(output) : truncJson(output ?? '') ?? '';
const toolId = namespacedToolId(sessionId, wire.agentId, nativeToolId);
toolResults.push({
kind: 'tool_result',
tool_use_id: toolId,
message_uuid: callMessageUuids.get(nativeToolId) ?? '',
session_id: sessionId,
content,
file_path: null,
is_error: result?.isError === true ? 1 : 0,
});
const childId = typeof content === 'string' ? /^agent_id:\s*(\S+)/m.exec(content)?.[1] : undefined;
if (childId !== undefined) childParentCalls.set(childId, toolId);
continue;
}
if (event.type === 'step.end' && typeof event.uuid === 'string') {
const step = stepMessages.get(event.uuid) ?? [];
const last = step.at(-1);
if (last !== undefined) {
const usage = event.usage as JsonRecord | undefined;
if (usage !== undefined) {
last.input_tokens = inputUsage(usage);
last.output_tokens = outputUsage(usage);
}
const started = stepStarts.get(event.uuid);
if (started !== undefined && typeof record.time === 'number' && record.time >= started) {
durations.push({ kind: 'message-turn-duration', uuid: last.uuid, turn_duration_ms: record.time - started });
}
}
}
}
}
const agents = state.agents as JsonRecord | undefined;
const subagents: SubagentRecord[] = [];
if (agents !== undefined) {
for (const [agentId, candidate] of Object.entries(agents)) {
if (agentId === 'main' || candidate === null || typeof candidate !== 'object') continue;
const agent = candidate as JsonRecord;
const labels = agent.labels as JsonRecord | undefined;
subagents.push({
kind: 'subagent',
agent_id: namespacedAgentId(sessionId, agentId),
session_id: sessionId,
parent_tool_use_id: childParentCalls.get(agentId) ?? null,
agent_type: typeof labels?.profile === 'string'
? labels.profile
: typeof agent.type === 'string'
? agent.type
: null,
description: typeof agent.swarmItem === 'string' ? agent.swarmItem : null,
duration_ms: null,
total_tokens: null,
});
}
}
return {
messages,
toolCalls,
toolResults,
summaries,
subagents,
durations,
mainMessageCount,
mainWirePath: meta.wireFiles.find((wire) => wire.main)?.path ?? join(meta.sessionDir, 'wire.jsonl'),
};
}
function sessionDirectories(rootDir: string): string[] {
const sessionsDir = join(rootDir, 'sessions');
if (!existsSync(sessionsDir)) return [];
const result: string[] = [];
for (const workspace of readdirSync(sessionsDir, { withFileTypes: true })) {
if (!workspace.isDirectory()) continue;
const workspaceDir = join(sessionsDir, workspace.name);
for (const session of readdirSync(workspaceDir, { withFileTypes: true })) {
if (session.isDirectory()) result.push(join(workspaceDir, session.name));
}
}
return result.sort();
}
function changedSessionDirectories(rootDir: string, changedPaths: readonly string[]): Set<string> {
const sessionsDir = join(rootDir, 'sessions');
const result = new Set<string>();
for (const changedPath of changedPaths) {
const absolute = isAbsolute(changedPath)
? normalize(changedPath)
: normalize(join(sessionsDir, changedPath));
const inside = relative(sessionsDir, absolute);
if (!inside || inside.startsWith('..') || isAbsolute(inside)) continue;
const [workspaceId, sessionId] = inside.split(sep);
if (workspaceId && sessionId) result.add(join(sessionsDir, workspaceId, sessionId));
}
return result;
}
function rawFromWire(path: string, messageUuid: string): RawRecord | null {
if (!existsSync(path)) return null;
const fallbackLine = /:line-(\d+)$/.exec(messageUuid)?.[1];
const nativeId = messageUuid.split(':').at(-1);
const lines = readFileSync(path, 'utf8').split(/\r?\n/);
const line = fallbackLine !== undefined
? lines[Number(fallbackLine) - 1]
: lines.find((candidate) => nativeId !== undefined && candidate.includes(nativeId));
if (!line) return null;
let projectedText: string | null = null;
try {
const record = JSON.parse(line) as JsonRecord;
if (record.type === 'context.append_message') {
const content = (record.message as JsonRecord | undefined)?.content;
const parts = contentParts(content).map((part) => {
if (part.type === 'text' && typeof part.text === 'string') return part.text;
if (part.type === 'thinking' && typeof part.thinking === 'string') return part.thinking;
return null;
}).filter((part): part is string => part !== null);
projectedText = parts.length > 0 ? parts.join('\n') : null;
} else if (record.type === 'context.append_loop_event') {
const part = (record.event as JsonRecord | undefined)?.part as JsonRecord | undefined;
if (part?.type === 'text' && typeof part.text === 'string') projectedText = part.text;
if (part?.type === 'thinking' && typeof part.thinking === 'string') projectedText = part.thinking;
}
} catch { /* malformed torn source line */ }
return {
text: line,
totalLength: line.length,
offset: 0,
limit: line.length,
hasMore: false,
messageText: projectedText,
};
}
export function createKimiProvider({ rootDir = defaultKimiRoot() }: { rootDir?: string } = {}): ProviderAdapter {
const name = SOURCE;
return {
name,
descriptor: { id: name, name: 'Kimi Code', vendor: 'Moonshot AI', defaultRoot: rootDir, color: '#6d6afc' },
watchRoots: (configuredRoot) => [join(configuredRoot, 'sessions'), join(configuredRoot, 'session_index.jsonl')],
discover(ctx: DiscoverContext): IndexUnit[] {
const units: IndexUnit[] = [];
const changedSessions = ctx.changedPaths === undefined
? null
: changedSessionDirectories(rootDir, ctx.changedPaths);
for (const sessionDir of sessionDirectories(rootDir)) {
if (changedSessions !== null && !changedSessions.has(sessionDir)) continue;
const statePath = join(sessionDir, 'state.json');
const wireFiles = listWireFiles(sessionDir);
if (wireFiles.length === 0) continue;
const currentCursor = cursorFor(statePath, wireFiles);
if (changedSessions === null && ctx.lastCursor(sessionDir) === currentCursor) continue;
const state = readState(statePath);
const cwd = typeof state.cwd === 'string' ? state.cwd : typeof state.workDir === 'string' ? state.workDir : null;
units.push({
key: sessionDir,
sessionId: namespacedSessionId(basename(sessionDir)),
project: projectSlugFromPath(cwd) ?? undefined,
meta: { kind: 'session', sessionDir, statePath, wireFiles, currentCursor } satisfies KimiSessionUnitMeta,
});
}
return units;
},
*parse(unit: IndexUnit, _cursor: Cursor): Generator<IndexRecord, Cursor> {
const meta = unit.meta as KimiSessionUnitMeta;
const before = cursorFor(meta.statePath, meta.wireFiles);
const state = readState(meta.statePath);
const projected = projectSession(meta, unit.sessionId, state);
const after = cursorFor(meta.statePath, meta.wireFiles);
if (before !== after) throw new Error(`Kimi session changed while indexing: ${meta.sessionDir}`);
yield { kind: 'delete-session', sessionId: unit.sessionId };
yield {
kind: 'session',
id: unit.sessionId,
title: typeof state.title === 'string'
? state.title
: typeof state.lastPrompt === 'string'
? state.lastPrompt
: null,
project: unit.project ?? null,
started_at: normalizeTime(state.createdAt),
ended_at: normalizeTime(state.updatedAt),
git_branch: null,
version: null,
message_count: projected.mainMessageCount,
countMode: 'total',
jsonl_path: projected.mainWirePath,
source: SOURCE,
};
yield* projected.messages;
yield* projected.toolCalls;
yield* projected.toolResults;
yield* projected.summaries;
yield* projected.subagents;
yield* projected.durations;
return after;
},
raw(input: RawLookup): RawRecord | null {
const mainPath = typeof input.session?.jsonl_path === 'string' ? input.session.jsonl_path : null;
if (mainPath === null) return null;
if (input.agentId === null) return rawFromWire(mainPath, input.messageUuid);
const rawAgentId = input.agentId.split(':').at(-1);
if (rawAgentId === undefined) return null;
const sessionDir = basename(dirname(mainPath)) === 'main'
? dirname(dirname(dirname(mainPath)))
: dirname(mainPath);
return rawFromWire(join(sessionDir, 'agents', rawAgentId, 'wire.jsonl'), input.messageUuid);
},
};
}
export const kimiProvider = createKimiProvider();
+41
View File
@@ -0,0 +1,41 @@
import type {
ProviderAdapter,
ProviderDescriptor,
RawLookup,
RawRecord,
} from './types.ts';
export interface ProviderRegistry {
catalog(): ProviderDescriptor[];
get(source: string): ProviderAdapter | undefined;
list(): ProviderAdapter[];
watchRoots(configuredRoots?: Readonly<Record<string, string>>): string[];
raw(input: RawLookup): RawRecord | null;
}
export function createProviderRegistry(providers: readonly ProviderAdapter[]): ProviderRegistry {
const byId = new Map<string, ProviderAdapter>();
for (const provider of providers) {
const id = provider.descriptor.id;
if (provider.name !== id) {
throw new Error(`Provider name "${provider.name}" must match descriptor id "${id}"`);
}
if (byId.has(id)) throw new Error(`Duplicate provider id: ${id}`);
byId.set(id, provider);
}
const list = (): ProviderAdapter[] => [...byId.values()];
return {
catalog: () => list().map((provider) => ({ ...provider.descriptor })),
get: (source) => byId.get(source),
list,
watchRoots: (configuredRoots = {}) => [
...new Set(
list().flatMap((provider) =>
provider.watchRoots(configuredRoots[provider.name] ?? provider.descriptor.defaultRoot),
),
),
],
raw: (input) => byId.get(input.source)?.raw(input) ?? null,
};
}
+38 -1
View File
@@ -222,6 +222,43 @@ export interface Provider {
readonly name: string;
/** Discover units needing (re)indexing, using stored cursors to detect change. */
discover(ctx: DiscoverContext): IndexUnit[];
/** Stream records for one unit resuming from `cursor`; return the new cursor. */
/** Yield records for one unit resuming from `cursor`; return the new cursor. */
parse(unit: IndexUnit, cursor: Cursor): Generator<IndexRecord, Cursor>;
}
/** Serializable source metadata consumed by settings and renderer surfaces. */
export interface ProviderDescriptor {
readonly id: string;
readonly name: string;
readonly vendor: string;
readonly defaultRoot: string;
readonly color: string;
}
export interface RawLookup {
readonly source: string;
readonly messageUuid: string;
readonly session: Record<string, unknown> | null;
readonly agentId: string | null;
readonly subagent?: Record<string, unknown> | null;
readonly workflowAgent?: Record<string, unknown> | null;
}
export interface RawRecord {
readonly text: string;
readonly totalLength?: number;
readonly offset?: number;
readonly limit?: number;
readonly hasMore?: boolean;
/** Provider-projected full message body for renderer expansion. */
readonly messageText?: string | null;
}
/** Complete adapter interface used by every indexing and presentation caller. */
export interface ProviderAdapter extends Provider {
readonly descriptor: ProviderDescriptor;
/** Optional index semantics marker; absence forces one provider-owned replay. */
readonly indexVersionMarker?: string;
watchRoots(configuredRoot: string): string[];
raw(input: RawLookup): RawRecord | null;
}
+30 -70
View File
@@ -1,5 +1,7 @@
// Query and attune sandbox helpers for the Core package.
import { readLines, fs, path } from './db.ts';
import { fs, path } from './db.ts';
import { createBuiltinProviderRegistry } from './providers/builtins.ts';
import type { ProviderRegistry } from './providers/registry.ts';
import type { SqliteDb, SqliteRow } from './sqlite-types.ts';
type DbRow = SqliteRow;
@@ -100,7 +102,10 @@ function buildSafeFtsQuery(text: unknown): string {
.join(' ');
}
function createQueryApi(db: SqliteDb) {
function createQueryApi(
db: SqliteDb,
{ providerRegistry = createBuiltinProviderRegistry() }: { providerRegistry?: ProviderRegistry } = {},
) {
const q = (sql: string, ...p: any[]) => {
assertReadOnlySql(sql);
return db.prepare(sql).all(...p);
@@ -450,79 +455,34 @@ function createQueryApi(db: SqliteDb) {
};
};
const resolveJsonlPath = (messageUuid: string): string | null => {
const msg = db.prepare('SELECT session_id, agent_id, source FROM messages WHERE uuid=?').get(messageUuid);
if (!msg) return null;
if (msg.source === 'codex' || String(messageUuid).startsWith('codex:')) {
const match = /^codex:([^:]+):(\d+)$/.exec(String(messageUuid));
if (!match) return null;
const rawThreadId = match[1];
if (!msg.agent_id) {
const ses = db.prepare('SELECT jsonl_path FROM sessions WHERE id=?').get(msg.session_id);
if (ses?.jsonl_path) return ses.jsonl_path;
}
return db.prepare(`
SELECT jsonl_path FROM index_state
WHERE jsonl_path LIKE ? AND jsonl_path LIKE '%.jsonl'
ORDER BY length(jsonl_path) ASC
LIMIT 1
`).get(`%${rawThreadId}.jsonl`)?.jsonl_path || null;
}
if (msg.agent_id) {
const wa = db.prepare('SELECT agent_id, run_id, session_id FROM workflow_agents WHERE agent_id=?').get(msg.agent_id);
if (wa) {
const ses = db.prepare('SELECT jsonl_path FROM sessions WHERE id=?').get(wa.session_id);
if (ses) return path.join(path.dirname(ses.jsonl_path), wa.session_id, 'subagents', 'workflows', wa.run_id, wa.agent_id + '.jsonl');
}
const sa = db.prepare('SELECT agent_id, session_id FROM subagents WHERE agent_id=?').get(msg.agent_id);
if (sa) {
const ses = db.prepare('SELECT jsonl_path FROM sessions WHERE id=?').get(sa.session_id);
if (ses) return path.join(path.dirname(ses.jsonl_path), sa.session_id, 'subagents', sa.agent_id + '.jsonl');
}
} else {
const ses = db.prepare('SELECT jsonl_path FROM sessions WHERE id=?').get(msg.session_id);
if (ses) return ses.jsonl_path;
}
return null;
};
const findCodexRawLine = (jsonlPath: string | null, uuid: string): string | null => {
const match = /^codex:[^:]+:(\d+)$/.exec(String(uuid));
if (!match || !jsonlPath || !fs.existsSync(jsonlPath)) return null;
const targetLine = Number(match[1]);
let lineNum = 0;
let found = null;
readLines(jsonlPath, (line) => {
lineNum++;
if (lineNum !== targetLine) return;
found = line;
return false;
});
return found;
};
const findRawLine = (jsonlPath: string | null, uuid: string): string | null => {
if (!jsonlPath || !fs.existsSync(jsonlPath)) return null;
if (String(uuid).startsWith('codex:')) return findCodexRawLine(jsonlPath, uuid);
let found = null;
readLines(jsonlPath, (line) => {
if (!line.includes(uuid)) return;
try { const obj = JSON.parse(line); if (obj.uuid === uuid) { found = line; return false; } } catch { /* skip malformed JSONL lines */ }
});
return found;
};
const raw = (messageUuid: string, opts: { offset?: number; limit?: number } = {}) => {
const { offset = 0, limit = 10000 } = opts;
const jsonlPath = resolveJsonlPath(messageUuid);
const line = findRawLine(jsonlPath, messageUuid);
if (!line) return null;
const message = db.prepare('SELECT * FROM messages WHERE uuid=?').get(messageUuid);
if (!message) return null;
const session = db.prepare('SELECT * FROM sessions WHERE id=?').get(message.session_id) ?? null;
const subagent = message.agent_id
? db.prepare('SELECT * FROM subagents WHERE agent_id=?').get(message.agent_id) ?? null
: null;
const workflowAgent = message.agent_id
? db.prepare('SELECT * FROM workflow_agents WHERE agent_id=?').get(message.agent_id) ?? null
: null;
const source = message.source || session?.source || 'claude';
const record = providerRegistry.raw({
source,
messageUuid,
session,
agentId: message.agent_id || null,
subagent,
workflowAgent,
});
if (record === null) return null;
const totalLength = record.totalLength ?? record.text.length;
return {
text: line.slice(offset, offset + limit),
totalLength: line.length,
text: record.text.slice(offset, offset + limit),
totalLength,
offset,
limit,
hasMore: offset + limit < line.length,
hasMore: offset + limit < totalLength,
};
};