feat(core): add first-class Pi session indexing (#23)
Pi cannot be read as another linear JSONL stream. Its history is a tree with a durable leaf, orphan roots, branch summaries, and two compaction forms, so the active context is something the format states rather than something line order implies. The adapter keeps those semantics inside itself and projects the result into the existing canonical tables. Sessions are keyed by (normalized header cwd, header id) rather than by path, because Pi's --session-id lookup is project-local: two projects may reuse an id, while a move or an identical copy is still one session. Discovery covers both layouts Pi writes and fingerprints each file by mtime, ctime, size and inode, so a rewrite that preserves mtime is not read as unchanged. Abandoned branches are preserved rather than dropped. Visibility becomes three-state -- visible, inactive, hidden -- and helpers return only visible rows until includeInactive asks for the superseded path, labeling every row so a caller knows which it holds. Usage counts all three, because an abandoned call still spent tokens; message_count reports only the visible transcript. A committed MIT-licensed oracle transcribed from Pi 0.83.0 pins the context algorithms, and a fixed-seed differential runs 512 generated sessions against it on every test run. Schema changes are additive.
This commit is contained in:
SaladDay
+287
-7
@@ -42,7 +42,8 @@ function searchDb() {
|
||||
CREATE TABLE messages (
|
||||
uuid TEXT PRIMARY KEY, session_id TEXT, text TEXT, role TEXT,
|
||||
timestamp TEXT, model TEXT, cwd TEXT, content_type TEXT,
|
||||
is_meta INTEGER DEFAULT 0, source TEXT DEFAULT 'claude'
|
||||
is_meta INTEGER DEFAULT 0, visibility TEXT DEFAULT 'visible',
|
||||
source TEXT DEFAULT 'claude'
|
||||
);
|
||||
CREATE VIRTUAL TABLE messages_fts USING fts5(
|
||||
uuid UNINDEXED, session_id UNINDEXED, text,
|
||||
@@ -54,13 +55,16 @@ function searchDb() {
|
||||
VALUES (?, ?, ?, ?)
|
||||
`).run('sid-search', 'Search session', 'quiet-zero', '2026-06-10T10:00:00Z');
|
||||
const insert = db.prepare(`
|
||||
INSERT INTO messages (uuid, session_id, text, role, timestamp, model, cwd, content_type, is_meta)
|
||||
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)
|
||||
INSERT INTO messages (uuid, session_id, text, role, timestamp, model, cwd, content_type, is_meta, visibility)
|
||||
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
|
||||
`);
|
||||
insert.run('msg-meta', 'sid-search', 'needle injected caveat', 'user', '2026-06-10T10:00:30Z', null, '/tmp/quiet-zero', 'text', 1);
|
||||
insert.run('msg-text', 'sid-search', 'needle visible reply', 'assistant', '2026-06-10T10:01:00Z', 'claude-opus', '/tmp/quiet-zero', 'text', 0);
|
||||
insert.run('msg-meta-near', 'sid-search', '<command-name>/exit</command-name>', 'user', '2026-06-10T10:01:30Z', null, '/tmp/quiet-zero', 'text', 1);
|
||||
insert.run('msg-thinking', 'sid-search', 'nearby reasoning trace', 'assistant', '2026-06-10T10:02:00Z', 'claude-opus', '/tmp/quiet-zero', 'thinking', 0);
|
||||
insert.run('msg-meta', 'sid-search', 'needle injected caveat', 'user', '2026-06-10T10:00:30Z', null, '/tmp/quiet-zero', 'text', 1, 'visible');
|
||||
insert.run('msg-text', 'sid-search', 'needle visible reply', 'assistant', '2026-06-10T10:01:00Z', 'claude-opus', '/tmp/quiet-zero', 'text', 0, 'visible');
|
||||
insert.run('msg-meta-near', 'sid-search', '<command-name>/exit</command-name>', 'user', '2026-06-10T10:01:30Z', null, '/tmp/quiet-zero', 'text', 1, 'visible');
|
||||
insert.run('msg-thinking', 'sid-search', 'nearby reasoning trace', 'assistant', '2026-06-10T10:02:00Z', 'claude-opus', '/tmp/quiet-zero', 'thinking', 0, 'visible');
|
||||
insert.run('msg-inactive', 'sid-search', 'needle superseded experiment', 'assistant', '2026-06-10T10:02:30Z', 'claude-opus', '/tmp/quiet-zero', 'text', 0, 'inactive');
|
||||
insert.run('msg-inactive-meta', 'sid-search', 'needle superseded injected', 'user', '2026-06-10T10:02:40Z', null, '/tmp/quiet-zero', 'text', 1, 'inactive');
|
||||
insert.run('msg-hidden', 'sid-search', 'needle abandoned branch', 'assistant', '2026-06-10T10:03:00Z', 'claude-opus', '/tmp/quiet-zero', 'text', 0, 'hidden');
|
||||
db.exec("INSERT INTO messages_fts(messages_fts) VALUES('rebuild')");
|
||||
return db;
|
||||
}
|
||||
@@ -102,16 +106,292 @@ test('search and thread omit meta messages by default and expose them on request
|
||||
const withMeta = api.search('injected', { includeMeta: true, limit: 5 });
|
||||
assert.equal(withMeta[0].message.uuid, 'msg-meta');
|
||||
assert.equal(withMeta[0].message.is_meta, 1);
|
||||
assert.deepEqual(api.search('abandoned', { includeMeta: true, limit: 5 }), []);
|
||||
|
||||
assert.deepEqual(api.thread('sid-search').map(m => m.uuid), ['msg-text', 'msg-thinking']);
|
||||
assert.deepEqual(
|
||||
api.thread('sid-search', { includeMeta: true }).map(m => m.uuid),
|
||||
['msg-meta', 'msg-text', 'msg-meta-near', 'msg-thinking'],
|
||||
);
|
||||
assert.deepEqual(
|
||||
api.thread('sid-search', { includeInactive: true }).map(m => [m.uuid, m.visibility]),
|
||||
[
|
||||
['msg-text', 'visible'],
|
||||
['msg-thinking', 'visible'],
|
||||
['msg-inactive', 'inactive'],
|
||||
],
|
||||
);
|
||||
|
||||
db.close();
|
||||
});
|
||||
|
||||
test('inactive search is opt-in, orthogonal to meta filtering, and always labeled', () => {
|
||||
const db = searchDb();
|
||||
const api = createQueryApi(db);
|
||||
|
||||
assert.deepEqual(api.search('superseded', { limit: 5 }), []);
|
||||
const inactive = api.search('superseded', { includeInactive: true, limit: 5 });
|
||||
assert.deepEqual(inactive.map(row => [row.message.uuid, row.message.visibility]), [
|
||||
['msg-inactive', 'inactive'],
|
||||
]);
|
||||
assert.equal(
|
||||
inactive[0].context.every(row => row.visibility === 'visible' || row.visibility === 'inactive'),
|
||||
true,
|
||||
);
|
||||
|
||||
const withMeta = api.search('superseded', {
|
||||
includeInactive: true,
|
||||
includeMeta: true,
|
||||
limit: 5,
|
||||
});
|
||||
assert.deepEqual(
|
||||
withMeta.map(row => [row.message.uuid, row.message.visibility]).sort(),
|
||||
[
|
||||
['msg-inactive', 'inactive'],
|
||||
['msg-inactive-meta', 'inactive'],
|
||||
],
|
||||
);
|
||||
assert.deepEqual(api.search('abandoned', { includeInactive: true, includeMeta: true }), []);
|
||||
db.close();
|
||||
});
|
||||
|
||||
test('context and trace reject hidden targets and omit hidden ancestors', () => {
|
||||
const db = new DatabaseSync(':memory:');
|
||||
db.exec(SCHEMA);
|
||||
db.prepare('INSERT INTO sessions (id,title,source) VALUES (?,?,?)')
|
||||
.run('sid-chain', 'Visibility chain', 'pi');
|
||||
const insert = db.prepare(`
|
||||
INSERT INTO messages (
|
||||
uuid,session_id,type,parent_uuid,role,text,timestamp,visibility,source
|
||||
) VALUES (?,?,?,?,?,?,?,?,?)
|
||||
`);
|
||||
insert.run('visible-root', 'sid-chain', 'user', null, 'user', 'root', '2026-08-02T10:00:00Z', 'visible', 'pi');
|
||||
insert.run('hidden-parent', 'sid-chain', 'assistant', 'visible-root', 'assistant', 'secret', '2026-08-02T10:00:01Z', 'hidden', 'pi');
|
||||
insert.run('visible-child', 'sid-chain', 'user', 'hidden-parent', 'user', 'continue', '2026-08-02T10:00:02Z', 'visible', 'pi');
|
||||
insert.run('inactive-child', 'sid-chain', 'assistant', 'visible-root', 'assistant', 'superseded', '2026-08-02T10:00:03Z', 'inactive', 'pi');
|
||||
|
||||
const api = createQueryApi(db);
|
||||
assert.equal(api.context('hidden-parent'), null);
|
||||
assert.equal(api.context('hidden-parent', { includeInactive: true }), null);
|
||||
assert.deepEqual(api.trace('hidden-parent'), []);
|
||||
assert.deepEqual(api.trace('hidden-parent', { includeInactive: true }), []);
|
||||
assert.equal(api.context('inactive-child'), null);
|
||||
assert.deepEqual(api.trace('inactive-child'), []);
|
||||
assert.deepEqual(
|
||||
api.context('inactive-child', { includeInactive: true }).parentChain
|
||||
.map(message => [message.uuid, message.visibility]),
|
||||
[['visible-root', 'visible']],
|
||||
);
|
||||
assert.deepEqual(
|
||||
api.trace('inactive-child', { includeInactive: true })
|
||||
.map(message => [message.uuid, message.visibility]),
|
||||
[
|
||||
['visible-root', 'visible'],
|
||||
['inactive-child', 'inactive'],
|
||||
],
|
||||
);
|
||||
assert.deepEqual(
|
||||
api.context('visible-child').parentChain.map(message => message.uuid),
|
||||
['visible-root'],
|
||||
);
|
||||
assert.deepEqual(
|
||||
api.trace('visible-child').map(message => message.uuid),
|
||||
['visible-root', 'visible-child'],
|
||||
);
|
||||
db.close();
|
||||
});
|
||||
|
||||
test('raw rejects hidden targets and labels explicitly included inactive evidence', () => {
|
||||
const db = searchDb();
|
||||
const providerRegistry = {
|
||||
raw: ({ messageUuid }) => ({
|
||||
text: `raw:${messageUuid}`,
|
||||
totalLength: `raw:${messageUuid}`.length,
|
||||
}),
|
||||
};
|
||||
const api = createQueryApi(db, { providerRegistry });
|
||||
|
||||
assert.equal(api.raw('msg-hidden'), null);
|
||||
assert.equal(api.raw('msg-hidden', { includeInactive: true }), null);
|
||||
assert.equal(api.raw('msg-inactive'), null);
|
||||
assert.deepEqual(
|
||||
api.raw('msg-inactive', { includeInactive: true }),
|
||||
{
|
||||
text: 'raw:msg-inactive',
|
||||
totalLength: 16,
|
||||
offset: 0,
|
||||
limit: 10000,
|
||||
hasMore: false,
|
||||
visibility: 'inactive',
|
||||
},
|
||||
);
|
||||
assert.equal(api.raw('msg-text').text, 'raw:msg-text');
|
||||
assert.equal(api.raw('msg-text').visibility, 'visible');
|
||||
db.close();
|
||||
});
|
||||
|
||||
test('failures nextMessages does not leak hidden branch messages', () => {
|
||||
const db = new DatabaseSync(':memory:');
|
||||
db.exec(SCHEMA);
|
||||
db.prepare('INSERT INTO sessions (id,title,source) VALUES (?,?,?)')
|
||||
.run('sid-failure', 'Failure branch', 'pi');
|
||||
const insertMessage = db.prepare(`
|
||||
INSERT INTO messages (uuid,session_id,type,role,text,timestamp,visibility,source)
|
||||
VALUES (?,?,?,?,?,?,?,?)
|
||||
`);
|
||||
insertMessage.run('failure-result', 'sid-failure', 'user', 'toolResult', 'failed', '2026-08-02T10:00:00Z', 'visible', 'pi');
|
||||
insertMessage.run('hidden-next', 'sid-failure', 'assistant', 'assistant', 'abandoned', '2026-08-02T10:00:01Z', 'hidden', 'pi');
|
||||
insertMessage.run('inactive-next', 'sid-failure', 'assistant', 'assistant', 'superseded', '2026-08-02T10:00:02Z', 'inactive', 'pi');
|
||||
insertMessage.run('visible-next', 'sid-failure', 'assistant', 'assistant', 'recovered', '2026-08-02T10:00:03Z', 'visible', 'pi');
|
||||
db.prepare(`
|
||||
INSERT INTO tool_calls (id,message_uuid,session_id,name,input_json)
|
||||
VALUES (?,?,?,?,?)
|
||||
`).run('call-failure', 'failure-result', 'sid-failure', 'read', '{}');
|
||||
db.prepare(`
|
||||
INSERT INTO tool_results (tool_use_id,message_uuid,session_id,content,is_error)
|
||||
VALUES (?,?,?,?,?)
|
||||
`).run('call-failure', 'failure-result', 'sid-failure', 'failed', 1);
|
||||
|
||||
const row = createQueryApi(db).failures('sid-failure')[0];
|
||||
assert.deepEqual(row.nextMessages.map(message => message.uuid), ['visible-next']);
|
||||
assert.equal(row.visibility, 'visible');
|
||||
assert.deepEqual(
|
||||
createQueryApi(db).failures({ sessionId: 'sid-failure', includeInactive: true })[0]
|
||||
.nextMessages.map(message => [message.uuid, message.visibility]),
|
||||
[
|
||||
['inactive-next', 'inactive'],
|
||||
['visible-next', 'visible'],
|
||||
],
|
||||
);
|
||||
db.close();
|
||||
});
|
||||
|
||||
test('failures gates both result and linked call message visibility', () => {
|
||||
const db = new DatabaseSync(':memory:');
|
||||
db.exec(SCHEMA);
|
||||
db.prepare('INSERT INTO sessions (id,title,source) VALUES (?,?,?)')
|
||||
.run('sid-edge-visibility', 'Tool edge visibility', 'pi');
|
||||
const insertMessage = db.prepare(`
|
||||
INSERT INTO messages (uuid,session_id,type,role,text,timestamp,visibility,source)
|
||||
VALUES (?,?,?,?,?,?,?,?)
|
||||
`);
|
||||
const insertCall = db.prepare(`
|
||||
INSERT INTO tool_calls (id,message_uuid,session_id,name,input_json,file_path)
|
||||
VALUES (?,?,?,?,?,?)
|
||||
`);
|
||||
const insertResult = db.prepare(`
|
||||
INSERT INTO tool_results (tool_use_id,message_uuid,session_id,content,is_error)
|
||||
VALUES (?,?,?,?,?)
|
||||
`);
|
||||
for (const [index, callVisibility] of ['visible', 'inactive', 'hidden'].entries()) {
|
||||
const callId = `call-${callVisibility}`;
|
||||
insertMessage.run(
|
||||
`message-${callVisibility}`,
|
||||
'sid-edge-visibility',
|
||||
'assistant',
|
||||
'assistant',
|
||||
null,
|
||||
`2026-08-02T10:00:0${index * 2}Z`,
|
||||
callVisibility,
|
||||
'pi',
|
||||
);
|
||||
insertMessage.run(
|
||||
`result-${callVisibility}`,
|
||||
'sid-edge-visibility',
|
||||
'user',
|
||||
'toolResult',
|
||||
`failed-${callVisibility}`,
|
||||
`2026-08-02T10:00:0${index * 2 + 1}Z`,
|
||||
'visible',
|
||||
'pi',
|
||||
);
|
||||
insertCall.run(
|
||||
callId,
|
||||
`message-${callVisibility}`,
|
||||
'sid-edge-visibility',
|
||||
'read',
|
||||
JSON.stringify({ path: `/${callVisibility}` }),
|
||||
`/${callVisibility}`,
|
||||
);
|
||||
insertResult.run(
|
||||
callId,
|
||||
`result-${callVisibility}`,
|
||||
'sid-edge-visibility',
|
||||
`failed-${callVisibility}`,
|
||||
1,
|
||||
);
|
||||
}
|
||||
const api = createQueryApi(db);
|
||||
|
||||
assert.deepEqual(
|
||||
api.failures('sid-edge-visibility').map(record => record.toolCall.id),
|
||||
['call-visible'],
|
||||
);
|
||||
assert.deepEqual(
|
||||
api.failures({ sessionId: 'sid-edge-visibility', includeInactive: true })
|
||||
.map(record => record.toolCall.id)
|
||||
.sort(),
|
||||
['call-inactive', 'call-visible'],
|
||||
);
|
||||
db.close();
|
||||
});
|
||||
|
||||
test('summaries, file history, and failures expose inactive rows only on request', () => {
|
||||
const db = new DatabaseSync(':memory:');
|
||||
db.exec(SCHEMA);
|
||||
db.prepare('INSERT INTO sessions (id,title,source) VALUES (?,?,?)')
|
||||
.run('sid-structured', 'Structured visibility', 'pi');
|
||||
const insertMessage = db.prepare(`
|
||||
INSERT INTO messages (uuid,session_id,type,role,text,timestamp,visibility,source)
|
||||
VALUES (?,?,?,?,?,?,?,?)
|
||||
`);
|
||||
const insertCall = db.prepare(`
|
||||
INSERT INTO tool_calls (id,message_uuid,session_id,name,input_json,file_path)
|
||||
VALUES (?,?,?,?,?,?)
|
||||
`);
|
||||
const insertResult = db.prepare(`
|
||||
INSERT INTO tool_results (tool_use_id,message_uuid,session_id,content,is_error)
|
||||
VALUES (?,?,?,?,?)
|
||||
`);
|
||||
const insertSummary = db.prepare(`
|
||||
INSERT INTO summaries (id,session_id,timestamp,source,content,visibility)
|
||||
VALUES (?,?,?,?,?,?)
|
||||
`);
|
||||
for (const [index, visibility] of ['visible', 'inactive', 'hidden'].entries()) {
|
||||
const suffix = visibility;
|
||||
const uuid = `message-${suffix}`;
|
||||
const callId = `call-${suffix}`;
|
||||
const timestamp = `2026-08-02T10:00:0${index}Z`;
|
||||
insertMessage.run(uuid, 'sid-structured', 'user', 'toolResult', suffix, timestamp, visibility, 'pi');
|
||||
insertCall.run(callId, uuid, 'sid-structured', 'read', '{}', '/tmp/visibility.ts');
|
||||
insertResult.run(callId, uuid, 'sid-structured', `failed-${suffix}`, 1);
|
||||
insertSummary.run(`summary-${suffix}`, 'sid-structured', timestamp, 'pi:branch_summary', suffix, visibility);
|
||||
}
|
||||
const api = createQueryApi(db);
|
||||
|
||||
assert.deepEqual(api.fileHistory('/tmp/visibility.ts').map(row => row.visibility), ['visible']);
|
||||
assert.deepEqual(
|
||||
api.fileHistory('/tmp/visibility.ts', { includeInactive: true }).map(row => row.visibility),
|
||||
['visible', 'inactive'],
|
||||
);
|
||||
assert.deepEqual(api.failures('sid-structured').map(row => row.visibility), ['visible']);
|
||||
assert.deepEqual(
|
||||
api.failures({ sessionId: 'sid-structured', includeInactive: true })
|
||||
.map(row => [row.visibility, row.result.visibility]),
|
||||
[
|
||||
['inactive', 'inactive'],
|
||||
['visible', 'visible'],
|
||||
],
|
||||
);
|
||||
assert.deepEqual(api.summaries('sid-structured').map(row => row.visibility), ['visible']);
|
||||
assert.deepEqual(
|
||||
api.summaries({ sessionId: 'sid-structured', includeInactive: true })
|
||||
.map(row => row.visibility),
|
||||
['inactive', 'visible'],
|
||||
);
|
||||
db.close();
|
||||
});
|
||||
|
||||
test('memories follows list-helper scalar opts and filters by query within scope', () => {
|
||||
const db = memoryDb();
|
||||
const api = createQueryApi(db);
|
||||
|
||||
Reference in New Issue
Block a user