tommy0103
0598c29aad
Complete the skill-side provider migration: codex now goes through a pure adapter
and the shared persist layer, and the two original monolithic indexers are gone.
New:
- scripts/providers/codex.ts — pure codex adapter. Full-reparse (buffers the whole
file) because the event_msg↔response_item dedup needs whole-file, bidirectional
knowledge; emits SessionRecord with countMode 'total'. Handles guardian threads
(→ delete-session), agent spawns/tool calls (→ tool_call/subagent), token_count
(patched onto the message record) and task_complete (→ message-turn-duration).
Contract:
- SessionRecord.countMode ('total' | 'delta') tells persist whether to replace or
accumulate message_count — claude is line-incremental (delta), codex full-reparse
(total). SubagentRecord non-key fields are optional; persist merges them
column-wise with COALESCE. MessageTurnDurationRecord.turn_duration_ms is nullable.
Orchestration:
- buildIndex's codex branch parses via the adapter and writes via persist. An
unchanged file is skipped but still swept for stale guardian rows (routed through
persist as a delete-session), preserving prior behavior.
Cleanup:
- Remove the now-unused indexJsonl, indexCodexJsonl, deleteCodexThreadRows and
upsertCodexSubagent — their semantics now live in the adapters + persist.
indexer.mjs drops from ~840 to 428 lines. Codex pure helpers stay exported for
codex.ts and the guardian sweep (physical move deferred to the app-side reorg).
- Migrate the upsert drift test off indexJsonl to the claude.parse + persist path,
keeping the rowid-stability and count-replace regression guards.
Tests: tests/codex-parse.test.mjs (record-stream golden: dedup, tools, token patch,
turn-duration, guardian→delete) and tests/codex-index.test.mjs (full buildIndex
path: fresh build + incremental full-reparse, total-count replace, no duplicates).
Verified equivalent on the real ~/.obelisk index: codex messages 82476 and
subagents 522 identical before/after, zero guardian leakage; real incremental
confirmed (touch a codex file → reparsed idempotently, unchanged files skipped).
lint + typecheck clean, 119/119.
79 lines
3.5 KiB
JavaScript
79 lines
3.5 KiB
JavaScript
// Phase 5c: exercises the full codex buildIndex path (discover → codex.parse →
|
|
// persist) for both a fresh full build and an incremental rebuild after append.
|
|
// Codex is full-reparse with countMode 'total', so growth must REPLACE the count
|
|
// (not accumulate) and upsert messages (no duplicates).
|
|
|
|
import { test } from 'node:test';
|
|
import assert from 'node:assert/strict';
|
|
import { createRequire } from 'node:module';
|
|
import { mkdtempSync, mkdirSync, writeFileSync, appendFileSync, utimesSync, statSync } from 'node:fs';
|
|
import { tmpdir } from 'node:os';
|
|
import { dirname, join, resolve } from 'node:path';
|
|
import { fileURLToPath } from 'node:url';
|
|
import { spawnSync } from 'node:child_process';
|
|
|
|
const repoRoot = resolve(dirname(fileURLToPath(import.meta.url)), '..');
|
|
const require = createRequire(import.meta.url);
|
|
const { DatabaseSync } = require('node:sqlite');
|
|
|
|
function runRuntime(args, home) {
|
|
return spawnSync(process.execPath, ['scripts/runtime.mjs', ...args], {
|
|
cwd: repoRoot, env: { ...process.env, HOME: home }, encoding: 'utf8',
|
|
});
|
|
}
|
|
|
|
const ID = '019ed000-0000-7000-8000-000000000001';
|
|
|
|
function metaLine() {
|
|
return JSON.stringify({ type: 'session_meta', timestamp: '2026-06-15T10:00:00Z', payload: { id: ID, timestamp: '2026-06-15T10:00:00Z', cwd: '/tmp/cdx', cli_version: '1.0' } });
|
|
}
|
|
function evt(type, message, ts) {
|
|
return JSON.stringify({ type: 'event_msg', timestamp: ts, payload: { type, message } });
|
|
}
|
|
|
|
function clearDebounce(home) {
|
|
const db = new DatabaseSync(join(home, '.obelisk', 'obelisk.sqlite'));
|
|
db.prepare("DELETE FROM index_state WHERE jsonl_path='__last_build__'").run();
|
|
db.close();
|
|
}
|
|
|
|
function codexCounts(home) {
|
|
writeFileSync(join(home, 'q.mjs'), `return {
|
|
sessions: sql("SELECT COUNT(*) c FROM sessions WHERE source='codex'")[0].c,
|
|
mc: sql("SELECT message_count FROM sessions WHERE source='codex'")[0]?.message_count ?? null,
|
|
msgs: sql("SELECT COUNT(*) c FROM messages WHERE source='codex'")[0].c,
|
|
hits: search('followup', { source: 'codex', limit: 5 }).length,
|
|
};`);
|
|
const r = runRuntime(['--query', join(home, 'q.mjs')], home);
|
|
assert.equal(r.status, 0, r.stderr || r.stdout);
|
|
return JSON.parse(r.stdout);
|
|
}
|
|
|
|
test('codex full build then incremental rebuild replaces the total count without duplicates', () => {
|
|
const home = mkdtempSync(join(tmpdir(), 'obelisk-codex-idx-'));
|
|
const dir = join(home, '.codex', 'sessions', '2026', '06', '15');
|
|
mkdirSync(dir, { recursive: true });
|
|
const jsonl = join(dir, `rollout-2026-06-15T10-00-00-${ID}.jsonl`);
|
|
|
|
// Full build: one user + one agent message.
|
|
writeFileSync(jsonl, [metaLine(), evt('user_message', 'codex hello', '2026-06-15T10:00:01Z'), evt('agent_message', 'codex reply', '2026-06-15T10:00:02Z')].join('\n') + '\n');
|
|
assert.equal(runRuntime(['--build'], home).status, 0);
|
|
|
|
let c = codexCounts(home);
|
|
assert.equal(c.sessions, 1, 'one codex session indexed');
|
|
assert.equal(c.mc, 2, 'two messages counted');
|
|
assert.equal(c.msgs, 2);
|
|
|
|
// Append a third message; bump mtime; incremental rebuild (full-reparse).
|
|
appendFileSync(jsonl, evt('user_message', 'codex followup', '2026-06-15T10:01:00Z') + '\n');
|
|
const t = statSync(jsonl).mtimeMs / 1000 + 10;
|
|
utimesSync(jsonl, t, t);
|
|
clearDebounce(home);
|
|
|
|
c = codexCounts(home);
|
|
// 'total' replace: 3, not 5 (2+3) and not a stale 2.
|
|
assert.equal(c.mc, 3, 'message_count replaced with the new total');
|
|
assert.equal(c.msgs, 3, 'exactly three messages, upserted (no duplicates)');
|
|
assert.equal(c.hits, 1, 'the appended message is searchable');
|
|
});
|