refactor(indexer): route claude indexing through adapter + persist (Phase 5b-2b)
buildIndex's claude branch now parses via providers/claude.ts and writes via persist.ts instead of the inlined indexJsonl. Behavior is equivalent — the full buildIndex integration suite (runtime.test.mjs) stays green, 116/116 — and a force rebuild of the real ~/.obelisk index (327 sessions, 119k messages) reproduced identical session counts with project_path fully populated. codex, indexSubagentMeta, workflows, history and the project_path pass are untouched. indexJsonl is now unused by buildIndex (kept for its drift test; removed once codex is migrated). Adds tests/incremental-index.test.mjs: verifies resume/accumulate through the full buildIndex path (append new lines to an indexed session → incremental build resumes from the cursor, message_count accumulates, no duplicates). The 30s shouldSkipBuild debounce is cleared in-test so the incremental run fires.
This commit is contained in:
+10
-1
@@ -1,4 +1,6 @@
|
||||
import { CLAUDE_DIR, CODEX_DIR, openDb, rebuildMemoryFts, trunc, truncJson, extractText, extractContentType, extractMessageIsMeta, filePath, isDir, readLines, fs, path } from './db.mjs';
|
||||
import { persist } from './persist.ts';
|
||||
import { parse as claudeParse } from './providers/claude.ts';
|
||||
|
||||
const PROJECTS_DIR = path.join(CLAUDE_DIR, 'projects');
|
||||
const HISTORY_PATH = path.join(CLAUDE_DIR, 'history.jsonl');
|
||||
@@ -782,7 +784,14 @@ function buildIndex({ force = false } = {}) {
|
||||
if (f.source === 'codex') {
|
||||
indexCodexJsonl(db, f);
|
||||
} else {
|
||||
indexJsonl(db, f);
|
||||
// Claude transcripts now go through the pure adapter + shared persist
|
||||
// (docs/adr/0001). needsReindex keeps the "skip unchanged file" fast path;
|
||||
// the cursor's line count drives incremental resume inside parse().
|
||||
const { needed, skip } = needsReindex(db, f.path);
|
||||
if (needed) {
|
||||
const unit = { key: f.path, sessionId: f.sessionId, project: f.project, isSubagent: f.isSubagent, agentId: f.agentId };
|
||||
persist(db, unit, claudeParse(unit, skip > 0 ? `0:${skip}` : null));
|
||||
}
|
||||
indexSubagentMeta(db, f);
|
||||
}
|
||||
db.exec('COMMIT');
|
||||
|
||||
@@ -0,0 +1,70 @@
|
||||
// Phase 5b-2b: verifies incremental (resume) indexing through the full rewired
|
||||
// buildIndex path (needsReindex → cursor → claude.parse → persist). A force
|
||||
// --build re-scans everything (skip=0) and never exercises resume, so this
|
||||
// appends new lines to an already-indexed session and drives an incremental
|
||||
// build. The 30s shouldSkipBuild debounce is cleared between steps (it would
|
||||
// otherwise skip a build this soon after the previous one).
|
||||
|
||||
import { test } from 'node:test';
|
||||
import assert from 'node:assert/strict';
|
||||
import { createRequire } from 'node:module';
|
||||
import { mkdtempSync, mkdirSync, writeFileSync, appendFileSync, utimesSync, statSync } from 'node:fs';
|
||||
import { tmpdir } from 'node:os';
|
||||
import { dirname, join, resolve } from 'node:path';
|
||||
import { fileURLToPath } from 'node:url';
|
||||
import { spawnSync } from 'node:child_process';
|
||||
|
||||
const repoRoot = resolve(dirname(fileURLToPath(import.meta.url)), '..');
|
||||
const require = createRequire(import.meta.url);
|
||||
const { DatabaseSync } = require('node:sqlite');
|
||||
|
||||
function runRuntime(args, home) {
|
||||
return spawnSync(process.execPath, ['scripts/runtime.mjs', ...args], {
|
||||
cwd: repoRoot, env: { ...process.env, HOME: home }, encoding: 'utf8',
|
||||
});
|
||||
}
|
||||
|
||||
function line(uuid, type, ts) {
|
||||
return JSON.stringify({ uuid, type, timestamp: ts, cwd: '/tmp/proj', message: { role: type, content: `${type} ${uuid}` } });
|
||||
}
|
||||
|
||||
function clearBuildDebounce(home) {
|
||||
const db = new DatabaseSync(join(home, '.obelisk', 'obelisk.sqlite'));
|
||||
db.prepare("DELETE FROM index_state WHERE jsonl_path='__last_build__'").run();
|
||||
db.close();
|
||||
}
|
||||
|
||||
function counts(home) {
|
||||
writeFileSync(join(home, 'q.mjs'), "return { mc: sql(\"SELECT message_count FROM sessions WHERE id='sess'\")[0]?.message_count ?? null, n: sql('SELECT COUNT(*) c FROM messages')[0].c, lp: sql(\"SELECT lines_processed lp FROM index_state WHERE jsonl_path LIKE '%sess.jsonl'\")[0]?.lp ?? null };");
|
||||
const r = runRuntime(['--query', join(home, 'q.mjs')], home);
|
||||
assert.equal(r.status, 0, r.stderr || r.stdout);
|
||||
return JSON.parse(r.stdout);
|
||||
}
|
||||
|
||||
test('incremental buildIndex resumes from cursor and accumulates message_count', () => {
|
||||
const home = mkdtempSync(join(tmpdir(), 'obelisk-incr-'));
|
||||
const projDir = join(home, '.claude', 'projects', '-tmp-proj');
|
||||
mkdirSync(projDir, { recursive: true });
|
||||
const jsonl = join(projDir, 'sess.jsonl');
|
||||
|
||||
writeFileSync(jsonl, [line('u1', 'user', '2026-06-10T10:00:00Z'), line('a1', 'assistant', '2026-06-10T10:00:05Z')].join('\n') + '\n');
|
||||
assert.equal(runRuntime(['--build'], home).status, 0);
|
||||
|
||||
const afterBuild = counts(home);
|
||||
assert.equal(afterBuild.mc, 2, 'full build indexed both messages');
|
||||
assert.equal(afterBuild.n, 2);
|
||||
assert.equal(afterBuild.lp, 2, 'cursor recorded 2 lines processed');
|
||||
|
||||
// Append two messages; bump mtime so needsReindex detects the change.
|
||||
appendFileSync(jsonl, [line('u2', 'user', '2026-06-10T10:01:00Z'), line('a2', 'assistant', '2026-06-10T10:01:05Z')].join('\n') + '\n');
|
||||
const t = statSync(jsonl).mtimeMs / 1000 + 10;
|
||||
utimesSync(jsonl, t, t);
|
||||
clearBuildDebounce(home);
|
||||
|
||||
// Incremental build: resume at line 2, add exactly the two new messages.
|
||||
// mc=4 proves resume+accumulate; 6 would mean re-count from stale base, 2 a miss.
|
||||
const afterAppend = counts(home);
|
||||
assert.equal(afterAppend.mc, 4, 'message_count accumulated to 4');
|
||||
assert.equal(afterAppend.n, 4, 'exactly four messages, no duplicates');
|
||||
assert.equal(afterAppend.lp, 4, 'cursor advanced to 4 lines');
|
||||
});
|
||||
Reference in New Issue
Block a user