refactor: add canonical transcript assembly seam
This commit is contained in:
tommy0103
@@ -21,13 +21,14 @@ import {
|
||||
codexIsGuardianThread, codexAgentNickname, codexAgentRole, codexUsage,
|
||||
codexEventText, codexMessagePayloadText, codexVisibleMessageKey,
|
||||
codexToolInput, codexToolOutput,
|
||||
extractMessageIsMeta, isSkillInstructions,
|
||||
readCodexGuardianThreadInfo,
|
||||
} from '../parsing.ts';
|
||||
|
||||
import type {
|
||||
Cursor,
|
||||
DiscoverContext,
|
||||
IndexRecord,
|
||||
TranscriptRecord,
|
||||
IndexUnit,
|
||||
MessageRecord,
|
||||
ProviderAdapter,
|
||||
@@ -36,11 +37,40 @@ import type {
|
||||
} from './types.ts';
|
||||
|
||||
export const name = 'codex';
|
||||
const CODEX_CANONICAL_TRANSCRIPT_MARKER = '__codex_canonical_transcript_v2__';
|
||||
|
||||
const HIDDEN_CONTEXT_ENVELOPE_RE = /^\s*<(environment_context|codex_internal_context)\b[^>]*>[\s\S]*<\/\1>\s*$/;
|
||||
|
||||
function messageVisibility(role: string, text: string | null): 'visible' | 'hidden' {
|
||||
return role === 'user' && typeof text === 'string' && HIDDEN_CONTEXT_ENVELOPE_RE.test(text)
|
||||
? 'hidden'
|
||||
: 'visible';
|
||||
}
|
||||
|
||||
function discoverAt(rootDir: string, ctx: DiscoverContext): IndexUnit[] {
|
||||
const sessionsDir = join(rootDir, 'sessions');
|
||||
const sessionIndexPath = normalize(join(rootDir, 'session_index.jsonl'));
|
||||
const sessionIndex = new Map<string, { title: string; updatedAt: string | null }>();
|
||||
if (fs.existsSync(sessionIndexPath)) {
|
||||
readLines(sessionIndexPath, (line: string) => {
|
||||
try {
|
||||
const item = JSON.parse(line);
|
||||
if (item?.id && item?.thread_name) {
|
||||
sessionIndex.set(codexRawId(item.id) as string, {
|
||||
title: item.thread_name,
|
||||
updatedAt: item.updated_at || null,
|
||||
});
|
||||
}
|
||||
} catch { /* malformed session-index entry */ }
|
||||
});
|
||||
}
|
||||
const changedFiles = new Set<string>();
|
||||
let sessionIndexChanged = false;
|
||||
for (const changedPath of ctx.changedPaths ?? []) {
|
||||
const rootRelative = isAbsolute(changedPath)
|
||||
? normalize(changedPath)
|
||||
: normalize(join(rootDir, changedPath));
|
||||
if (rootRelative === sessionIndexPath) sessionIndexChanged = true;
|
||||
const absolute = isAbsolute(changedPath)
|
||||
? normalize(changedPath)
|
||||
: normalize(join(sessionsDir, changedPath));
|
||||
@@ -49,10 +79,10 @@ function discoverAt(rootDir: string, ctx: DiscoverContext): IndexUnit[] {
|
||||
if (absolute.toLowerCase().endsWith('.jsonl')) changedFiles.add(absolute);
|
||||
}
|
||||
return discoverCodexJsonlFiles(sessionsDir).flatMap((file) => {
|
||||
if (ctx.changedPaths !== undefined && !changedFiles.has(normalize(file.path))) return [];
|
||||
if (ctx.changedPaths !== undefined && !sessionIndexChanged && !changedFiles.has(normalize(file.path))) return [];
|
||||
const cursor = ctx.lastCursor(file.path);
|
||||
const guardian = readCodexGuardianThreadInfo(file.path);
|
||||
if (cursor !== null && Number(cursor.split(':')[0]) >= fs.statSync(file.path).mtimeMs && guardian === null) {
|
||||
if (!sessionIndexChanged && cursor !== null && Number(cursor.split(':')[0]) >= fs.statSync(file.path).mtimeMs && guardian === null) {
|
||||
return [];
|
||||
}
|
||||
let meta: any = null;
|
||||
@@ -67,10 +97,16 @@ function discoverAt(rootDir: string, ctx: DiscoverContext): IndexUnit[] {
|
||||
});
|
||||
const rawId = meta ? codexRawId(meta.id) : null;
|
||||
const parentId = meta ? codexParentThreadId(meta) : null;
|
||||
const indexed = rawId ? sessionIndex.get(rawId) : undefined;
|
||||
return [{
|
||||
key: file.path,
|
||||
sessionId: guardian === null ? codexDbId(parentId || rawId) ?? '' : '',
|
||||
meta: { source: 'codex', guardian: guardian !== null },
|
||||
meta: {
|
||||
source: 'codex',
|
||||
guardian: guardian !== null,
|
||||
indexedTitle: indexed?.title,
|
||||
indexedUpdatedAt: indexed?.updatedAt,
|
||||
},
|
||||
}];
|
||||
});
|
||||
}
|
||||
@@ -79,7 +115,7 @@ export function discover(ctx: DiscoverContext): IndexUnit[] {
|
||||
return discoverAt(join(homedir(), '.codex'), ctx);
|
||||
}
|
||||
|
||||
export function* parse(unit: IndexUnit, _cursor: Cursor): Generator<IndexRecord, Cursor> {
|
||||
export function* parse(unit: IndexUnit, _cursor: Cursor): Generator<TranscriptRecord, Cursor> {
|
||||
const mtime = fs.statSync(unit.key).mtimeMs;
|
||||
const records: { lineNum: number; obj: any }[] = [];
|
||||
let lineNum = 0;
|
||||
@@ -106,14 +142,19 @@ export function* parse(unit: IndexUnit, _cursor: Cursor): Generator<IndexRecord,
|
||||
const project = projectSlugFromPath(normalizeObservedCwd(meta.cwd));
|
||||
const lineUuid = (n: number): string => codexLineUuid(threadRawId, n) as string;
|
||||
|
||||
const out: IndexRecord[] = [];
|
||||
const out: TranscriptRecord[] = [];
|
||||
const msgByUuid = new Map<string, MessageRecord>();
|
||||
const indexedMeta = unit.meta as { indexedTitle?: string; indexedUpdatedAt?: string | null } | undefined;
|
||||
const initialTimestamp = (meta.timestamp || metaRecord.obj.timestamp || null) as string | null;
|
||||
const indexedUpdatedAt = indexedMeta?.indexedUpdatedAt ?? null;
|
||||
const sm = {
|
||||
started_at: (meta.timestamp || metaRecord.obj.timestamp || null) as string | null,
|
||||
ended_at: (meta.timestamp || metaRecord.obj.timestamp || null) as string | null,
|
||||
started_at: initialTimestamp,
|
||||
ended_at: indexedUpdatedAt && (!initialTimestamp || indexedUpdatedAt > initialTimestamp)
|
||||
? indexedUpdatedAt
|
||||
: initialTimestamp,
|
||||
git_branch: (meta.git?.branch || null) as string | null,
|
||||
version: (meta.cli_version || null) as string | null,
|
||||
title: null as string | null,
|
||||
title: indexedMeta?.indexedTitle ?? null,
|
||||
n: 0,
|
||||
lastMessageUuid: null as string | null,
|
||||
lastTextAssistantUuid: null as string | null,
|
||||
@@ -135,10 +176,14 @@ export function* parse(unit: IndexUnit, _cursor: Cursor): Generator<IndexRecord,
|
||||
const insertMessage = ({ uuid, type, role, text = null, contentType = 'text', timestamp, isMeta = 0 }: {
|
||||
uuid: string; type: string; role: string; text?: string | null; contentType?: string; timestamp: string | null; isMeta?: 0 | 1;
|
||||
}) => {
|
||||
const visibility = messageVisibility(role, text);
|
||||
const skillInstructions = role === 'user' && isSkillInstructions(text);
|
||||
const rec: MessageRecord = {
|
||||
kind: 'message', uuid, session_id: sessionId, type, parent_uuid: sm.lastMessageUuid,
|
||||
timestamp: timestamp || null, role, text: trunc(text), content_type: contentType,
|
||||
is_meta: isMeta, model: currentModel, is_sidechain: isSidechain, agent_id: agentId,
|
||||
timestamp: timestamp || null, role, text: trunc(text),
|
||||
content_type: skillInstructions ? 'skill_instructions' : contentType,
|
||||
is_meta: visibility === 'hidden' || skillInstructions ? 1 : (isMeta || extractMessageIsMeta({}, text)), visibility,
|
||||
model: currentModel, is_sidechain: isSidechain, agent_id: agentId,
|
||||
input_tokens: null, output_tokens: null, cwd: currentCwd, skill: null, source: 'codex',
|
||||
};
|
||||
out.push(rec);
|
||||
@@ -191,13 +236,13 @@ export function* parse(unit: IndexUnit, _cursor: Cursor): Generator<IndexRecord,
|
||||
}
|
||||
if (payload.type === 'collab_agent_spawn_end' && payload.call_id && payload.new_thread_id) {
|
||||
const uuid = insertMessage({ uuid: lineUuid(currentLine), type: 'assistant', role: 'assistant', text: null, contentType: 'tool_use', timestamp: ts });
|
||||
const toolId = codexCallId(payload.call_id) as string;
|
||||
const toolId = codexCallId(threadRawId, payload.call_id) as string;
|
||||
const description = payload.new_agent_nickname || payload.new_agent_role || 'Agent';
|
||||
const input = {
|
||||
description, subagent_type: payload.new_agent_role || 'Agent', prompt: payload.prompt || '',
|
||||
new_thread_id: payload.new_thread_id, model: payload.model || null, reasoning_effort: payload.reasoning_effort || null,
|
||||
};
|
||||
out.push({ kind: 'tool_call', id: toolId, message_uuid: uuid, session_id: sessionId, name: 'Agent', input_json: truncJson(input) as string, file_path: null });
|
||||
out.push({ kind: 'tool_call', id: toolId, message_uuid: uuid, session_id: sessionId, name: 'Agent', presentation: 'default', input_json: truncJson(input) as string, file_path: null });
|
||||
callMessageUuids.set(toolId, uuid);
|
||||
out.push({ kind: 'subagent', agent_id: codexDbId(payload.new_thread_id) as string, session_id: sessionId, parent_tool_use_id: toolId, agent_type: payload.new_agent_role || null, description });
|
||||
continue;
|
||||
@@ -235,13 +280,13 @@ export function* parse(unit: IndexUnit, _cursor: Cursor): Generator<IndexRecord,
|
||||
if (['function_call', 'custom_tool_call', 'tool_search_call', 'web_search_call'].includes(payload.type) && payload.call_id) {
|
||||
const uuid = insertMessage({ uuid: lineUuid(currentLine), type: 'assistant', role: 'assistant', text: null, contentType: 'tool_use', timestamp: ts });
|
||||
const name = payload.name || payload.tool || payload.type.replace(/_call$/, '');
|
||||
const toolId = codexCallId(payload.call_id) as string;
|
||||
out.push({ kind: 'tool_call', id: toolId, message_uuid: uuid, session_id: sessionId, name, input_json: truncJson(codexToolInput(payload)) as string, file_path: null });
|
||||
const toolId = codexCallId(threadRawId, payload.call_id) as string;
|
||||
out.push({ kind: 'tool_call', id: toolId, message_uuid: uuid, session_id: sessionId, name, presentation: name === 'Skill' ? 'skill' : 'default', input_json: truncJson(codexToolInput(payload)) as string, file_path: null });
|
||||
callMessageUuids.set(toolId, uuid);
|
||||
continue;
|
||||
}
|
||||
if (['function_call_output', 'custom_tool_call_output', 'tool_search_output'].includes(payload.type) && payload.call_id) {
|
||||
const toolId = codexCallId(payload.call_id) as string;
|
||||
const toolId = codexCallId(threadRawId, payload.call_id) as string;
|
||||
out.push({ kind: 'tool_result', tool_use_id: toolId, message_uuid: callMessageUuids.get(toolId) || '', session_id: sessionId, content: trunc(codexToolOutput(payload) || ''), file_path: null, is_error: payload.is_error ? 1 : 0 });
|
||||
}
|
||||
}
|
||||
@@ -309,8 +354,7 @@ function rawCodex(rootDir: string, input: RawLookup): RawRecord | null {
|
||||
? payload.text
|
||||
: null;
|
||||
} else if (obj?.type === 'response_item' && payload.type === 'message' && Array.isArray(payload.content)) {
|
||||
const parts = payload.content.map((part: any) => part?.text).filter((part: unknown) => typeof part === 'string');
|
||||
messageText = parts.length > 0 ? parts.join('\n') : null;
|
||||
messageText = codexMessagePayloadText(payload);
|
||||
}
|
||||
} catch { /* malformed source line */ }
|
||||
}
|
||||
@@ -323,6 +367,7 @@ export function createCodexProvider({ rootDir = join(homedir(), '.codex') }: { r
|
||||
return {
|
||||
name,
|
||||
descriptor: { id: name, name: 'Codex', vendor: 'OpenAI', defaultRoot: rootDir, color: '#10a37f' },
|
||||
indexVersionMarker: CODEX_CANONICAL_TRANSCRIPT_MARKER,
|
||||
watchRoots: (configuredRoot) => [join(configuredRoot, 'sessions'), join(configuredRoot, 'session_index.jsonl')],
|
||||
discover: (ctx) => discoverAt(rootDir, ctx),
|
||||
parse,
|
||||
|
||||
Reference in New Issue
Block a user