fix(indexer): align claude message writes to canonical upsert semantics
scripts/indexer.mjs indexJsonl used INSERT OR REPLACE (rowid/FTS churn) and carried message_count forward on full re-scan; align to app's ON CONFLICT upsert + count reset so the two indexers no longer silently diverge. Regression test added.
This commit is contained in:
+23
-3
@@ -108,7 +108,27 @@ function indexJsonl(db, fi) {
|
||||
|
||||
const ins = {
|
||||
ses: db.prepare('INSERT OR REPLACE INTO sessions (id,title,project,project_path,started_at,ended_at,git_branch,version,message_count,jsonl_path,source) VALUES (?,?,?,?,?,?,?,?,?,?,?)'),
|
||||
msg: db.prepare('INSERT OR REPLACE INTO messages (uuid,session_id,type,parent_uuid,timestamp,role,text,content_type,is_meta,model,is_sidechain,agent_id,input_tokens,output_tokens,cwd,skill,source) VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)'),
|
||||
msg: db.prepare(`
|
||||
INSERT INTO messages (uuid,session_id,type,parent_uuid,timestamp,role,text,content_type,is_meta,model,is_sidechain,agent_id,input_tokens,output_tokens,cwd,skill,source)
|
||||
VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)
|
||||
ON CONFLICT(uuid) DO UPDATE SET
|
||||
session_id=excluded.session_id,
|
||||
type=excluded.type,
|
||||
parent_uuid=excluded.parent_uuid,
|
||||
timestamp=excluded.timestamp,
|
||||
role=excluded.role,
|
||||
text=excluded.text,
|
||||
content_type=excluded.content_type,
|
||||
is_meta=excluded.is_meta,
|
||||
model=excluded.model,
|
||||
is_sidechain=excluded.is_sidechain,
|
||||
agent_id=excluded.agent_id,
|
||||
input_tokens=excluded.input_tokens,
|
||||
output_tokens=excluded.output_tokens,
|
||||
cwd=excluded.cwd,
|
||||
skill=excluded.skill,
|
||||
source=excluded.source
|
||||
`),
|
||||
tc: db.prepare('INSERT OR REPLACE INTO tool_calls (id,message_uuid,session_id,name,input_json,file_path) VALUES (?,?,?,?,?,?)'),
|
||||
tr: db.prepare('INSERT OR REPLACE INTO tool_results (tool_use_id,message_uuid,session_id,content,file_path,is_error) VALUES (?,?,?,?,?,?)'),
|
||||
sum: db.prepare('INSERT OR REPLACE INTO summaries (id,session_id,timestamp,source,content) VALUES (?,?,?,?,?)'),
|
||||
@@ -122,7 +142,7 @@ function indexJsonl(db, fi) {
|
||||
git_branch: existing?.git_branch || null,
|
||||
version: existing?.version || null,
|
||||
title: existing?.title || null,
|
||||
n: existing?.message_count || 0,
|
||||
n: skip > 0 ? (existing?.message_count || 0) : 0,
|
||||
cwds: [],
|
||||
};
|
||||
|
||||
@@ -788,4 +808,4 @@ function buildIndex({ force = false } = {}) {
|
||||
db.close();
|
||||
}
|
||||
|
||||
export { buildIndex, inferProjectPath, refreshSessionProjectPaths, shouldSkipBuild };
|
||||
export { buildIndex, indexJsonl, inferProjectPath, refreshSessionProjectPaths, shouldSkipBuild };
|
||||
|
||||
@@ -0,0 +1,64 @@
|
||||
// Regression test for the indexer silent-drift fix.
|
||||
//
|
||||
// scripts/indexer.mjs and app/indexer.js had diverged in indexJsonl's message
|
||||
// write: scripts used INSERT OR REPLACE (churns rowid → FTS churn) and always
|
||||
// carried the previous message_count forward (inflating it on a full re-scan),
|
||||
// while app used ON CONFLICT DO UPDATE and reset the count when skip===0. app's
|
||||
// semantics are canonical; this pins them so the two cannot drift again and so
|
||||
// the Phase 5 provider-adapter merge inherits one known-correct behavior.
|
||||
|
||||
import { test } from 'node:test';
|
||||
import assert from 'node:assert/strict';
|
||||
import { createRequire } from 'node:module';
|
||||
import { mkdtempSync, writeFileSync } from 'node:fs';
|
||||
import { tmpdir } from 'node:os';
|
||||
import { join } from 'node:path';
|
||||
|
||||
import { indexJsonl } from '../scripts/indexer.mjs';
|
||||
|
||||
const require = createRequire(import.meta.url);
|
||||
const { DatabaseSync } = require('node:sqlite');
|
||||
const SCHEMA = require('node:fs').readFileSync(new URL('../scripts/schema.sql', import.meta.url), 'utf8');
|
||||
|
||||
function writeSessionJsonl() {
|
||||
const dir = mkdtempSync(join(tmpdir(), 'obelisk-drift-'));
|
||||
const jsonlPath = join(dir, 'sid-drift.jsonl');
|
||||
const lines = [
|
||||
{ uuid: 'u-1', type: 'user', timestamp: '2026-06-10T10:00:00Z', cwd: '/tmp/proj', message: { role: 'user', content: 'first question' } },
|
||||
{ uuid: 'a-1', type: 'assistant', timestamp: '2026-06-10T10:00:05Z', message: { role: 'assistant', model: 'claude-opus', content: 'first answer' } },
|
||||
{ uuid: 'u-2', type: 'user', timestamp: '2026-06-10T10:00:10Z', cwd: '/tmp/proj', message: { role: 'user', content: 'second question' } },
|
||||
];
|
||||
writeFileSync(jsonlPath, lines.map(l => JSON.stringify(l)).join('\n') + '\n');
|
||||
return jsonlPath;
|
||||
}
|
||||
|
||||
test('re-indexing a session upserts messages (stable rowid) and does not inflate message_count', () => {
|
||||
const db = new DatabaseSync(':memory:');
|
||||
db.exec(SCHEMA);
|
||||
const fi = { path: writeSessionJsonl(), sessionId: 'sid-drift', project: 'quiet-zero' };
|
||||
|
||||
indexJsonl(db, fi);
|
||||
|
||||
const countAfterFirst = db.prepare('SELECT message_count FROM sessions WHERE id=?').get('sid-drift').message_count;
|
||||
const rowidAfterFirst = db.prepare('SELECT rowid FROM messages WHERE uuid=?').get('u-1').rowid;
|
||||
const totalMessages = db.prepare('SELECT COUNT(*) AS c FROM messages').get().c;
|
||||
assert.equal(countAfterFirst, 3, 'three user/assistant messages counted');
|
||||
assert.equal(totalMessages, 3);
|
||||
|
||||
// Simulate a fresh full re-scan (force / lost index_state): skip resets to 0.
|
||||
db.prepare('DELETE FROM index_state').run();
|
||||
indexJsonl(db, fi);
|
||||
|
||||
const countAfterSecond = db.prepare('SELECT message_count FROM sessions WHERE id=?').get('sid-drift').message_count;
|
||||
const rowidAfterSecond = db.prepare('SELECT rowid FROM messages WHERE uuid=?').get('u-1').rowid;
|
||||
const totalAfterSecond = db.prepare('SELECT COUNT(*) AS c FROM messages').get().c;
|
||||
|
||||
// message_count is reset+recounted, not accumulated (would be 6 under the old bug).
|
||||
assert.equal(countAfterSecond, 3, 'message_count must not inflate on re-scan');
|
||||
// No duplicate rows.
|
||||
assert.equal(totalAfterSecond, 3);
|
||||
// Upsert preserves rowid; INSERT OR REPLACE would have churned it.
|
||||
assert.equal(rowidAfterSecond, rowidAfterFirst, 'upsert must preserve message rowid (no REPLACE churn)');
|
||||
|
||||
db.close();
|
||||
});
|
||||
Reference in New Issue
Block a user