diff --git a/scripts/indexer.mjs b/scripts/indexer.mjs index b3c28c8..cfbbdb2 100644 --- a/scripts/indexer.mjs +++ b/scripts/indexer.mjs @@ -1,4 +1,6 @@ import { CLAUDE_DIR, CODEX_DIR, openDb, rebuildMemoryFts, trunc, truncJson, extractText, extractContentType, extractMessageIsMeta, filePath, isDir, readLines, fs, path } from './db.mjs'; +import { persist } from './persist.ts'; +import { parse as claudeParse } from './providers/claude.ts'; const PROJECTS_DIR = path.join(CLAUDE_DIR, 'projects'); const HISTORY_PATH = path.join(CLAUDE_DIR, 'history.jsonl'); @@ -782,7 +784,14 @@ function buildIndex({ force = false } = {}) { if (f.source === 'codex') { indexCodexJsonl(db, f); } else { - indexJsonl(db, f); + // Claude transcripts now go through the pure adapter + shared persist + // (docs/adr/0001). needsReindex keeps the "skip unchanged file" fast path; + // the cursor's line count drives incremental resume inside parse(). + const { needed, skip } = needsReindex(db, f.path); + if (needed) { + const unit = { key: f.path, sessionId: f.sessionId, project: f.project, isSubagent: f.isSubagent, agentId: f.agentId }; + persist(db, unit, claudeParse(unit, skip > 0 ? `0:${skip}` : null)); + } indexSubagentMeta(db, f); } db.exec('COMMIT'); diff --git a/tests/incremental-index.test.mjs b/tests/incremental-index.test.mjs new file mode 100644 index 0000000..1b03591 --- /dev/null +++ b/tests/incremental-index.test.mjs @@ -0,0 +1,70 @@ +// Phase 5b-2b: verifies incremental (resume) indexing through the full rewired +// buildIndex path (needsReindex → cursor → claude.parse → persist). A force +// --build re-scans everything (skip=0) and never exercises resume, so this +// appends new lines to an already-indexed session and drives an incremental +// build. The 30s shouldSkipBuild debounce is cleared between steps (it would +// otherwise skip a build this soon after the previous one). + +import { test } from 'node:test'; +import assert from 'node:assert/strict'; +import { createRequire } from 'node:module'; +import { mkdtempSync, mkdirSync, writeFileSync, appendFileSync, utimesSync, statSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { dirname, join, resolve } from 'node:path'; +import { fileURLToPath } from 'node:url'; +import { spawnSync } from 'node:child_process'; + +const repoRoot = resolve(dirname(fileURLToPath(import.meta.url)), '..'); +const require = createRequire(import.meta.url); +const { DatabaseSync } = require('node:sqlite'); + +function runRuntime(args, home) { + return spawnSync(process.execPath, ['scripts/runtime.mjs', ...args], { + cwd: repoRoot, env: { ...process.env, HOME: home }, encoding: 'utf8', + }); +} + +function line(uuid, type, ts) { + return JSON.stringify({ uuid, type, timestamp: ts, cwd: '/tmp/proj', message: { role: type, content: `${type} ${uuid}` } }); +} + +function clearBuildDebounce(home) { + const db = new DatabaseSync(join(home, '.obelisk', 'obelisk.sqlite')); + db.prepare("DELETE FROM index_state WHERE jsonl_path='__last_build__'").run(); + db.close(); +} + +function counts(home) { + writeFileSync(join(home, 'q.mjs'), "return { mc: sql(\"SELECT message_count FROM sessions WHERE id='sess'\")[0]?.message_count ?? null, n: sql('SELECT COUNT(*) c FROM messages')[0].c, lp: sql(\"SELECT lines_processed lp FROM index_state WHERE jsonl_path LIKE '%sess.jsonl'\")[0]?.lp ?? null };"); + const r = runRuntime(['--query', join(home, 'q.mjs')], home); + assert.equal(r.status, 0, r.stderr || r.stdout); + return JSON.parse(r.stdout); +} + +test('incremental buildIndex resumes from cursor and accumulates message_count', () => { + const home = mkdtempSync(join(tmpdir(), 'obelisk-incr-')); + const projDir = join(home, '.claude', 'projects', '-tmp-proj'); + mkdirSync(projDir, { recursive: true }); + const jsonl = join(projDir, 'sess.jsonl'); + + writeFileSync(jsonl, [line('u1', 'user', '2026-06-10T10:00:00Z'), line('a1', 'assistant', '2026-06-10T10:00:05Z')].join('\n') + '\n'); + assert.equal(runRuntime(['--build'], home).status, 0); + + const afterBuild = counts(home); + assert.equal(afterBuild.mc, 2, 'full build indexed both messages'); + assert.equal(afterBuild.n, 2); + assert.equal(afterBuild.lp, 2, 'cursor recorded 2 lines processed'); + + // Append two messages; bump mtime so needsReindex detects the change. + appendFileSync(jsonl, [line('u2', 'user', '2026-06-10T10:01:00Z'), line('a2', 'assistant', '2026-06-10T10:01:05Z')].join('\n') + '\n'); + const t = statSync(jsonl).mtimeMs / 1000 + 10; + utimesSync(jsonl, t, t); + clearBuildDebounce(home); + + // Incremental build: resume at line 2, add exactly the two new messages. + // mc=4 proves resume+accumulate; 6 would mean re-count from stale base, 2 a miss. + const afterAppend = counts(home); + assert.equal(afterAppend.mc, 4, 'message_count accumulated to 4'); + assert.equal(afterAppend.n, 4, 'exactly four messages, no duplicates'); + assert.equal(afterAppend.lp, 4, 'cursor advanced to 4 lines'); +});