feat(providers): migrate codex indexing to adapter + persist, remove legacy indexers (Phase 5c)

Complete the skill-side provider migration: codex now goes through a pure adapter
and the shared persist layer, and the two original monolithic indexers are gone.

New:
- scripts/providers/codex.ts — pure codex adapter. Full-reparse (buffers the whole
  file) because the event_msg↔response_item dedup needs whole-file, bidirectional
  knowledge; emits SessionRecord with countMode 'total'. Handles guardian threads
  (→ delete-session), agent spawns/tool calls (→ tool_call/subagent), token_count
  (patched onto the message record) and task_complete (→ message-turn-duration).

Contract:
- SessionRecord.countMode ('total' | 'delta') tells persist whether to replace or
  accumulate message_count — claude is line-incremental (delta), codex full-reparse
  (total). SubagentRecord non-key fields are optional; persist merges them
  column-wise with COALESCE. MessageTurnDurationRecord.turn_duration_ms is nullable.

Orchestration:
- buildIndex's codex branch parses via the adapter and writes via persist. An
  unchanged file is skipped but still swept for stale guardian rows (routed through
  persist as a delete-session), preserving prior behavior.

Cleanup:
- Remove the now-unused indexJsonl, indexCodexJsonl, deleteCodexThreadRows and
  upsertCodexSubagent — their semantics now live in the adapters + persist.
  indexer.mjs drops from ~840 to 428 lines. Codex pure helpers stay exported for
  codex.ts and the guardian sweep (physical move deferred to the app-side reorg).
- Migrate the upsert drift test off indexJsonl to the claude.parse + persist path,
  keeping the rowid-stability and count-replace regression guards.

Tests: tests/codex-parse.test.mjs (record-stream golden: dedup, tools, token patch,
turn-duration, guardian→delete) and tests/codex-index.test.mjs (full buildIndex
path: fresh build + incremental full-reparse, total-count replace, no duplicates).

Verified equivalent on the real ~/.obelisk index: codex messages 82476 and
subagents 522 identical before/after, zero guardian leakage; real incremental
confirmed (touch a codex file → reparsed idempotently, unchanged files skipped).
lint + typecheck clean, 119/119.
This commit is contained in:
tommy0103
2026-07-08 20:43:54 +08:00
parent 1c346039f1
commit 0598c29aad
8 changed files with 474 additions and 469 deletions
+86
View File
@@ -0,0 +1,86 @@
// Phase 5c-2 golden test: pins the codex adapter's parse() record stream.
// Binding-independent (no database). Covers the event_msg↔response_item dedup,
// tool call/result, token patching, turn-duration, the 'total' session count,
// and guardian-thread → delete-session.
import { test } from 'node:test';
import assert from 'node:assert/strict';
import { mkdtempSync, writeFileSync } from 'node:fs';
import { tmpdir } from 'node:os';
import { join } from 'node:path';
import { parse } from '../scripts/providers/codex.ts';
function writeFixture(lines) {
const dir = mkdtempSync(join(tmpdir(), 'obelisk-codex-parse-'));
const path = join(dir, 'rollout.jsonl');
writeFileSync(path, lines.map(l => JSON.stringify(l)).join('\n') + '\n');
return path;
}
function drain(gen) {
const values = [];
let step = gen.next();
while (!step.done) { values.push(step.value); step = gen.next(); }
return { values, ret: step.value };
}
const META = { id: '019e8951-3e7d-7343-a3e3-05bff48a317d', cwd: '/proj', git: { branch: 'main' }, cli_version: '1.2', timestamp: '2026-06-10T10:00:00Z' };
test('codex parse() yields a deduped, tool-aware record stream with a total session', () => {
const path = writeFixture([
{ type: 'session_meta', timestamp: '2026-06-10T10:00:00Z', payload: META },
{ type: 'event_msg', timestamp: '2026-06-10T10:00:01Z', payload: { type: 'user_message', message: 'hello codex' } },
{ type: 'event_msg', timestamp: '2026-06-10T10:00:02Z', payload: { type: 'agent_message', message: 'hi there' } },
// Duplicate of the agent_message above — must be deduped (dropped).
{ type: 'response_item', timestamp: '2026-06-10T10:00:02Z', payload: { type: 'message', role: 'assistant', content: [{ type: 'output_text', text: 'hi there' }] } },
{ type: 'response_item', timestamp: '2026-06-10T10:00:03Z', payload: { type: 'function_call', call_id: 'call_1', name: 'shell', arguments: '{"cmd":"ls"}' } },
{ type: 'response_item', timestamp: '2026-06-10T10:00:04Z', payload: { type: 'function_call_output', call_id: 'call_1', output: 'file listing' } },
{ type: 'event_msg', payload: { type: 'token_count', info: { last_token_usage: { input_tokens: 100, output_tokens: 50 } } } },
{ type: 'event_msg', timestamp: '2026-06-10T10:00:05Z', payload: { type: 'task_complete', duration_ms: 1500 } },
]);
const { values } = drain(parse({ key: path, sessionId: '' }, null));
const byKind = k => values.filter(r => r.kind === k);
// Three messages: user, assistant text, assistant tool_use. The duplicate
// response_item 'hi there' was deduped.
const msgs = byKind('message');
assert.equal(msgs.length, 3);
assert.equal(msgs.filter(m => m.text === 'hi there').length, 1, 'agent_message deduped against response_item');
assert.equal(msgs.every(m => m.source === 'codex'), true);
// token_count patched the last text-assistant message's tokens.
const textAssistant = msgs.find(m => m.role === 'assistant' && m.content_type === 'text');
assert.equal(textAssistant.input_tokens, 100);
assert.equal(textAssistant.output_tokens, 50);
// Tool call + result.
assert.deepEqual(byKind('tool_call').map(t => ({ id: t.id, name: t.name })), [{ id: 'codex:call_1', name: 'shell' }]);
assert.equal(byKind('tool_result').length, 1);
assert.equal(byKind('tool_result')[0].tool_use_id, 'codex:call_1');
// task_complete → turn duration on the text-assistant message.
assert.deepEqual(byKind('message-turn-duration').map(d => d.turn_duration_ms), [1500]);
// One session record, full-reparse semantics.
const sessions = byKind('session');
assert.equal(sessions.length, 1);
assert.equal(sessions[0].source, 'codex');
assert.equal(sessions[0].countMode, 'total');
assert.equal(sessions[0].message_count, 3);
assert.equal(sessions[0].git_branch, 'main');
});
test('codex parse() retracts a guardian thread via delete-session and emits nothing else', () => {
const path = writeFixture([
{ type: 'session_meta', timestamp: '2026-06-10T10:00:00Z', payload: { ...META, source: { subagent: { other: 'guardian' } } } },
{ type: 'event_msg', timestamp: '2026-06-10T10:00:01Z', payload: { type: 'user_message', message: 'ignored' } },
]);
const { values } = drain(parse({ key: path, sessionId: '' }, null));
assert.equal(values.length, 1);
assert.equal(values[0].kind, 'delete-session');
assert.match(values[0].sessionId, /^codex:/);
});