feat(providers): migrate codex indexing to adapter + persist, remove legacy indexers (Phase 5c)
Complete the skill-side provider migration: codex now goes through a pure adapter
and the shared persist layer, and the two original monolithic indexers are gone.
New:
- scripts/providers/codex.ts — pure codex adapter. Full-reparse (buffers the whole
file) because the event_msg↔response_item dedup needs whole-file, bidirectional
knowledge; emits SessionRecord with countMode 'total'. Handles guardian threads
(→ delete-session), agent spawns/tool calls (→ tool_call/subagent), token_count
(patched onto the message record) and task_complete (→ message-turn-duration).
Contract:
- SessionRecord.countMode ('total' | 'delta') tells persist whether to replace or
accumulate message_count — claude is line-incremental (delta), codex full-reparse
(total). SubagentRecord non-key fields are optional; persist merges them
column-wise with COALESCE. MessageTurnDurationRecord.turn_duration_ms is nullable.
Orchestration:
- buildIndex's codex branch parses via the adapter and writes via persist. An
unchanged file is skipped but still swept for stale guardian rows (routed through
persist as a delete-session), preserving prior behavior.
Cleanup:
- Remove the now-unused indexJsonl, indexCodexJsonl, deleteCodexThreadRows and
upsertCodexSubagent — their semantics now live in the adapters + persist.
indexer.mjs drops from ~840 to 428 lines. Codex pure helpers stay exported for
codex.ts and the guardian sweep (physical move deferred to the app-side reorg).
- Migrate the upsert drift test off indexJsonl to the claude.parse + persist path,
keeping the rowid-stability and count-replace regression guards.
Tests: tests/codex-parse.test.mjs (record-stream golden: dedup, tools, token patch,
turn-duration, guardian→delete) and tests/codex-index.test.mjs (full buildIndex
path: fresh build + incremental full-reparse, total-count replace, no duplicates).
Verified equivalent on the real ~/.obelisk index: codex messages 82476 and
subagents 522 identical before/after, zero guardian leakage; real incremental
confirmed (touch a codex file → reparsed idempotently, unchanged files skipped).
lint + typecheck clean, 119/119.
This commit is contained in:
@@ -0,0 +1,86 @@
|
||||
// Phase 5c-2 golden test: pins the codex adapter's parse() record stream.
|
||||
// Binding-independent (no database). Covers the event_msg↔response_item dedup,
|
||||
// tool call/result, token patching, turn-duration, the 'total' session count,
|
||||
// and guardian-thread → delete-session.
|
||||
|
||||
import { test } from 'node:test';
|
||||
import assert from 'node:assert/strict';
|
||||
import { mkdtempSync, writeFileSync } from 'node:fs';
|
||||
import { tmpdir } from 'node:os';
|
||||
import { join } from 'node:path';
|
||||
|
||||
import { parse } from '../scripts/providers/codex.ts';
|
||||
|
||||
function writeFixture(lines) {
|
||||
const dir = mkdtempSync(join(tmpdir(), 'obelisk-codex-parse-'));
|
||||
const path = join(dir, 'rollout.jsonl');
|
||||
writeFileSync(path, lines.map(l => JSON.stringify(l)).join('\n') + '\n');
|
||||
return path;
|
||||
}
|
||||
|
||||
function drain(gen) {
|
||||
const values = [];
|
||||
let step = gen.next();
|
||||
while (!step.done) { values.push(step.value); step = gen.next(); }
|
||||
return { values, ret: step.value };
|
||||
}
|
||||
|
||||
const META = { id: '019e8951-3e7d-7343-a3e3-05bff48a317d', cwd: '/proj', git: { branch: 'main' }, cli_version: '1.2', timestamp: '2026-06-10T10:00:00Z' };
|
||||
|
||||
test('codex parse() yields a deduped, tool-aware record stream with a total session', () => {
|
||||
const path = writeFixture([
|
||||
{ type: 'session_meta', timestamp: '2026-06-10T10:00:00Z', payload: META },
|
||||
{ type: 'event_msg', timestamp: '2026-06-10T10:00:01Z', payload: { type: 'user_message', message: 'hello codex' } },
|
||||
{ type: 'event_msg', timestamp: '2026-06-10T10:00:02Z', payload: { type: 'agent_message', message: 'hi there' } },
|
||||
// Duplicate of the agent_message above — must be deduped (dropped).
|
||||
{ type: 'response_item', timestamp: '2026-06-10T10:00:02Z', payload: { type: 'message', role: 'assistant', content: [{ type: 'output_text', text: 'hi there' }] } },
|
||||
{ type: 'response_item', timestamp: '2026-06-10T10:00:03Z', payload: { type: 'function_call', call_id: 'call_1', name: 'shell', arguments: '{"cmd":"ls"}' } },
|
||||
{ type: 'response_item', timestamp: '2026-06-10T10:00:04Z', payload: { type: 'function_call_output', call_id: 'call_1', output: 'file listing' } },
|
||||
{ type: 'event_msg', payload: { type: 'token_count', info: { last_token_usage: { input_tokens: 100, output_tokens: 50 } } } },
|
||||
{ type: 'event_msg', timestamp: '2026-06-10T10:00:05Z', payload: { type: 'task_complete', duration_ms: 1500 } },
|
||||
]);
|
||||
|
||||
const { values } = drain(parse({ key: path, sessionId: '' }, null));
|
||||
const byKind = k => values.filter(r => r.kind === k);
|
||||
|
||||
// Three messages: user, assistant text, assistant tool_use. The duplicate
|
||||
// response_item 'hi there' was deduped.
|
||||
const msgs = byKind('message');
|
||||
assert.equal(msgs.length, 3);
|
||||
assert.equal(msgs.filter(m => m.text === 'hi there').length, 1, 'agent_message deduped against response_item');
|
||||
assert.equal(msgs.every(m => m.source === 'codex'), true);
|
||||
|
||||
// token_count patched the last text-assistant message's tokens.
|
||||
const textAssistant = msgs.find(m => m.role === 'assistant' && m.content_type === 'text');
|
||||
assert.equal(textAssistant.input_tokens, 100);
|
||||
assert.equal(textAssistant.output_tokens, 50);
|
||||
|
||||
// Tool call + result.
|
||||
assert.deepEqual(byKind('tool_call').map(t => ({ id: t.id, name: t.name })), [{ id: 'codex:call_1', name: 'shell' }]);
|
||||
assert.equal(byKind('tool_result').length, 1);
|
||||
assert.equal(byKind('tool_result')[0].tool_use_id, 'codex:call_1');
|
||||
|
||||
// task_complete → turn duration on the text-assistant message.
|
||||
assert.deepEqual(byKind('message-turn-duration').map(d => d.turn_duration_ms), [1500]);
|
||||
|
||||
// One session record, full-reparse semantics.
|
||||
const sessions = byKind('session');
|
||||
assert.equal(sessions.length, 1);
|
||||
assert.equal(sessions[0].source, 'codex');
|
||||
assert.equal(sessions[0].countMode, 'total');
|
||||
assert.equal(sessions[0].message_count, 3);
|
||||
assert.equal(sessions[0].git_branch, 'main');
|
||||
});
|
||||
|
||||
test('codex parse() retracts a guardian thread via delete-session and emits nothing else', () => {
|
||||
const path = writeFixture([
|
||||
{ type: 'session_meta', timestamp: '2026-06-10T10:00:00Z', payload: { ...META, source: { subagent: { other: 'guardian' } } } },
|
||||
{ type: 'event_msg', timestamp: '2026-06-10T10:00:01Z', payload: { type: 'user_message', message: 'ignored' } },
|
||||
]);
|
||||
|
||||
const { values } = drain(parse({ key: path, sessionId: '' }, null));
|
||||
|
||||
assert.equal(values.length, 1);
|
||||
assert.equal(values[0].kind, 'delete-session');
|
||||
assert.match(values[0].sessionId, /^codex:/);
|
||||
});
|
||||
Reference in New Issue
Block a user