refactor(indexer): route claude indexing through adapter + persist (Phase 5b-2b)

buildIndex's claude branch now parses via providers/claude.ts and writes via
persist.ts instead of the inlined indexJsonl. Behavior is equivalent — the full
buildIndex integration suite (runtime.test.mjs) stays green, 116/116 — and a
force rebuild of the real ~/.obelisk index (327 sessions, 119k messages)
reproduced identical session counts with project_path fully populated.

codex, indexSubagentMeta, workflows, history and the project_path pass are
untouched. indexJsonl is now unused by buildIndex (kept for its drift test;
removed once codex is migrated).

Adds tests/incremental-index.test.mjs: verifies resume/accumulate through the
full buildIndex path (append new lines to an indexed session → incremental
build resumes from the cursor, message_count accumulates, no duplicates). The
30s shouldSkipBuild debounce is cleared in-test so the incremental run fires.
This commit is contained in:
tommy0103
2026-07-08 19:46:34 +08:00
parent 1d652d823f
commit 1c346039f1
2 changed files with 80 additions and 1 deletions
+10 -1
View File
@@ -1,4 +1,6 @@
import { CLAUDE_DIR, CODEX_DIR, openDb, rebuildMemoryFts, trunc, truncJson, extractText, extractContentType, extractMessageIsMeta, filePath, isDir, readLines, fs, path } from './db.mjs'; import { CLAUDE_DIR, CODEX_DIR, openDb, rebuildMemoryFts, trunc, truncJson, extractText, extractContentType, extractMessageIsMeta, filePath, isDir, readLines, fs, path } from './db.mjs';
import { persist } from './persist.ts';
import { parse as claudeParse } from './providers/claude.ts';
const PROJECTS_DIR = path.join(CLAUDE_DIR, 'projects'); const PROJECTS_DIR = path.join(CLAUDE_DIR, 'projects');
const HISTORY_PATH = path.join(CLAUDE_DIR, 'history.jsonl'); const HISTORY_PATH = path.join(CLAUDE_DIR, 'history.jsonl');
@@ -782,7 +784,14 @@ function buildIndex({ force = false } = {}) {
if (f.source === 'codex') { if (f.source === 'codex') {
indexCodexJsonl(db, f); indexCodexJsonl(db, f);
} else { } else {
indexJsonl(db, f); // Claude transcripts now go through the pure adapter + shared persist
// (docs/adr/0001). needsReindex keeps the "skip unchanged file" fast path;
// the cursor's line count drives incremental resume inside parse().
const { needed, skip } = needsReindex(db, f.path);
if (needed) {
const unit = { key: f.path, sessionId: f.sessionId, project: f.project, isSubagent: f.isSubagent, agentId: f.agentId };
persist(db, unit, claudeParse(unit, skip > 0 ? `0:${skip}` : null));
}
indexSubagentMeta(db, f); indexSubagentMeta(db, f);
} }
db.exec('COMMIT'); db.exec('COMMIT');
+70
View File
@@ -0,0 +1,70 @@
// Phase 5b-2b: verifies incremental (resume) indexing through the full rewired
// buildIndex path (needsReindex → cursor → claude.parse → persist). A force
// --build re-scans everything (skip=0) and never exercises resume, so this
// appends new lines to an already-indexed session and drives an incremental
// build. The 30s shouldSkipBuild debounce is cleared between steps (it would
// otherwise skip a build this soon after the previous one).
import { test } from 'node:test';
import assert from 'node:assert/strict';
import { createRequire } from 'node:module';
import { mkdtempSync, mkdirSync, writeFileSync, appendFileSync, utimesSync, statSync } from 'node:fs';
import { tmpdir } from 'node:os';
import { dirname, join, resolve } from 'node:path';
import { fileURLToPath } from 'node:url';
import { spawnSync } from 'node:child_process';
const repoRoot = resolve(dirname(fileURLToPath(import.meta.url)), '..');
const require = createRequire(import.meta.url);
const { DatabaseSync } = require('node:sqlite');
function runRuntime(args, home) {
return spawnSync(process.execPath, ['scripts/runtime.mjs', ...args], {
cwd: repoRoot, env: { ...process.env, HOME: home }, encoding: 'utf8',
});
}
function line(uuid, type, ts) {
return JSON.stringify({ uuid, type, timestamp: ts, cwd: '/tmp/proj', message: { role: type, content: `${type} ${uuid}` } });
}
function clearBuildDebounce(home) {
const db = new DatabaseSync(join(home, '.obelisk', 'obelisk.sqlite'));
db.prepare("DELETE FROM index_state WHERE jsonl_path='__last_build__'").run();
db.close();
}
function counts(home) {
writeFileSync(join(home, 'q.mjs'), "return { mc: sql(\"SELECT message_count FROM sessions WHERE id='sess'\")[0]?.message_count ?? null, n: sql('SELECT COUNT(*) c FROM messages')[0].c, lp: sql(\"SELECT lines_processed lp FROM index_state WHERE jsonl_path LIKE '%sess.jsonl'\")[0]?.lp ?? null };");
const r = runRuntime(['--query', join(home, 'q.mjs')], home);
assert.equal(r.status, 0, r.stderr || r.stdout);
return JSON.parse(r.stdout);
}
test('incremental buildIndex resumes from cursor and accumulates message_count', () => {
const home = mkdtempSync(join(tmpdir(), 'obelisk-incr-'));
const projDir = join(home, '.claude', 'projects', '-tmp-proj');
mkdirSync(projDir, { recursive: true });
const jsonl = join(projDir, 'sess.jsonl');
writeFileSync(jsonl, [line('u1', 'user', '2026-06-10T10:00:00Z'), line('a1', 'assistant', '2026-06-10T10:00:05Z')].join('\n') + '\n');
assert.equal(runRuntime(['--build'], home).status, 0);
const afterBuild = counts(home);
assert.equal(afterBuild.mc, 2, 'full build indexed both messages');
assert.equal(afterBuild.n, 2);
assert.equal(afterBuild.lp, 2, 'cursor recorded 2 lines processed');
// Append two messages; bump mtime so needsReindex detects the change.
appendFileSync(jsonl, [line('u2', 'user', '2026-06-10T10:01:00Z'), line('a2', 'assistant', '2026-06-10T10:01:05Z')].join('\n') + '\n');
const t = statSync(jsonl).mtimeMs / 1000 + 10;
utimesSync(jsonl, t, t);
clearBuildDebounce(home);
// Incremental build: resume at line 2, add exactly the two new messages.
// mc=4 proves resume+accumulate; 6 would mean re-count from stale base, 2 a miss.
const afterAppend = counts(home);
assert.equal(afterAppend.mc, 4, 'message_count accumulated to 4');
assert.equal(afterAppend.n, 4, 'exactly four messages, no duplicates');
assert.equal(afterAppend.lp, 4, 'cursor advanced to 4 lines');
});