feat(core): add first-class Pi session indexing (#23)
Pi cannot be read as another linear JSONL stream. Its history is a tree with a durable leaf, orphan roots, branch summaries, and two compaction forms, so the active context is something the format states rather than something line order implies. The adapter keeps those semantics inside itself and projects the result into the existing canonical tables. Sessions are keyed by (normalized header cwd, header id) rather than by path, because Pi's --session-id lookup is project-local: two projects may reuse an id, while a move or an identical copy is still one session. Discovery covers both layouts Pi writes and fingerprints each file by mtime, ctime, size and inode, so a rewrite that preserves mtime is not read as unchanged. Abandoned branches are preserved rather than dropped. Visibility becomes three-state -- visible, inactive, hidden -- and helpers return only visible rows until includeInactive asks for the superseded path, labeling every row so a caller knows which it holds. Usage counts all three, because an abandoned call still spent tokens; message_count reports only the visible transcript. A committed MIT-licensed oracle transcribed from Pi 0.83.0 pins the context algorithms, and a fixed-seed differential runs 512 generated sessions against it on every test run. Schema changes are additive.
This commit is contained in:
SaladDay
@@ -0,0 +1,5 @@
|
||||
{"type":"session","version":3,"id":"harness-checkpoint-fork","timestamp":"2026-08-02T10:17:40.694Z","cwd":"/tmp/pi-harness-project","parentSession":"/tmp/source-session.jsonl"}
|
||||
{"type":"compaction","id":"2bd121f3","parentId":"3582fbcc","timestamp":"2026-08-02T10:17:40.693Z","summary":"Harness compaction checkpoint","firstKeptEntryId":"bb37b0d7","tokensBefore":9001,"retainedTail":[{"role":"user","content":"Harness retained user turn","timestamp":1785664802000},{"role":"assistant","content":[{"type":"thinking","thinking":"retained reasoning"},{"type":"text","text":"Harness retained assistant turn"}],"api":"openai-responses","provider":"obelisk-probe","model":"probe-model","usage":{"input":101,"output":23,"cacheRead":17,"cacheWrite":5,"reasoning":7,"totalTokens":146,"cost":{"input":0,"output":0,"cacheRead":0,"cacheWrite":0,"total":0}},"stopReason":"stop","timestamp":1785664803000}]}
|
||||
{"type":"message","id":"f56a248d","parentId":"2bd121f3","timestamp":"2026-08-02T10:17:40.693Z","message":{"role":"user","content":"Harness post-compaction user turn","timestamp":1785664804000}}
|
||||
{"type":"message","id":"5b2c961b","parentId":"f56a248d","timestamp":"2026-08-02T10:17:40.693Z","message":{"role":"assistant","content":[{"type":"text","text":"Harness post-compaction assistant turn"}],"api":"openai-responses","provider":"obelisk-probe","model":"probe-model","usage":{"input":101,"output":23,"cacheRead":17,"cacheWrite":5,"reasoning":7,"totalTokens":146,"cost":{"input":0,"output":0,"cacheRead":0,"cacheWrite":0,"total":0}},"stopReason":"stop","timestamp":1785664805000}}
|
||||
{"type":"session_info","id":"4bb3c8bd","parentId":"5b2c961b","timestamp":"2026-08-02T10:17:40.694Z","name":"Checkpoint fork with retained tail"}
|
||||
+3
@@ -0,0 +1,3 @@
|
||||
{"type":"session","version":3,"id":"harness-null-leaf","timestamp":"2026-08-02T10:17:40.695Z","cwd":"/tmp/pi-harness-project"}
|
||||
{"type":"message","id":"be0488ce","parentId":null,"timestamp":"2026-08-02T10:17:40.695Z","message":{"role":"user","content":"This physical message is no longer active","timestamp":1785664806000}}
|
||||
{"type":"leaf","id":"ff4d594c","parentId":"be0488ce","timestamp":"2026-08-02T10:17:40.695Z","targetId":null}
|
||||
+11
@@ -0,0 +1,11 @@
|
||||
{"type":"session","version":3,"id":"harness-probe","timestamp":"2026-08-02T10:17:40.691Z","cwd":"/tmp/pi-harness-project","metadata":{"purpose":"obelisk-pi-adapter-probe"}}
|
||||
{"type":"message","id":"a49e0082","parentId":null,"timestamp":"2026-08-02T10:17:40.693Z","message":{"role":"user","content":"Harness first user turn","timestamp":1785664800000}}
|
||||
{"type":"message","id":"6edde419","parentId":"a49e0082","timestamp":"2026-08-02T10:17:40.693Z","message":{"role":"assistant","content":[{"type":"text","text":"Harness first assistant turn"}],"api":"openai-responses","provider":"obelisk-probe","model":"probe-model","usage":{"input":101,"output":23,"cacheRead":17,"cacheWrite":5,"reasoning":7,"totalTokens":146,"cost":{"input":0,"output":0,"cacheRead":0,"cacheWrite":0,"total":0}},"stopReason":"stop","timestamp":1785664801000}}
|
||||
{"type":"message","id":"bb37b0d7","parentId":"6edde419","timestamp":"2026-08-02T10:17:40.693Z","message":{"role":"user","content":"Harness retained user turn","timestamp":1785664802000}}
|
||||
{"type":"message","id":"79b4a046","parentId":"bb37b0d7","timestamp":"2026-08-02T10:17:40.693Z","message":{"role":"assistant","content":[{"type":"text","text":"Harness retained assistant turn"}],"api":"openai-responses","provider":"obelisk-probe","model":"probe-model","usage":{"input":101,"output":23,"cacheRead":17,"cacheWrite":5,"reasoning":7,"totalTokens":146,"cost":{"input":0,"output":0,"cacheRead":0,"cacheWrite":0,"total":0}},"stopReason":"stop","timestamp":1785664803000}}
|
||||
{"type":"active_tools_change","id":"3582fbcc","parentId":"79b4a046","timestamp":"2026-08-02T10:17:40.693Z","activeToolNames":["read","write"]}
|
||||
{"type":"compaction","id":"2bd121f3","parentId":"3582fbcc","timestamp":"2026-08-02T10:17:40.693Z","summary":"Harness compaction checkpoint","firstKeptEntryId":"bb37b0d7","tokensBefore":9001,"retainedTail":[{"role":"user","content":"Harness retained user turn","timestamp":1785664802000},{"role":"assistant","content":[{"type":"text","text":"Harness retained assistant turn"}],"api":"openai-responses","provider":"obelisk-probe","model":"probe-model","usage":{"input":101,"output":23,"cacheRead":17,"cacheWrite":5,"reasoning":7,"totalTokens":146,"cost":{"input":0,"output":0,"cacheRead":0,"cacheWrite":0,"total":0}},"stopReason":"stop","timestamp":1785664803000}]}
|
||||
{"type":"message","id":"f56a248d","parentId":"2bd121f3","timestamp":"2026-08-02T10:17:40.693Z","message":{"role":"user","content":"Harness post-compaction user turn","timestamp":1785664804000}}
|
||||
{"type":"message","id":"5b2c961b","parentId":"f56a248d","timestamp":"2026-08-02T10:17:40.693Z","message":{"role":"assistant","content":[{"type":"text","text":"Harness post-compaction assistant turn"}],"api":"openai-responses","provider":"obelisk-probe","model":"probe-model","usage":{"input":101,"output":23,"cacheRead":17,"cacheWrite":5,"reasoning":7,"totalTokens":146,"cost":{"input":0,"output":0,"cacheRead":0,"cacheWrite":0,"total":0}},"stopReason":"stop","timestamp":1785664805000}}
|
||||
{"type":"session_info","id":"116253e0","parentId":"5b2c961b","timestamp":"2026-08-02T10:17:40.695Z","name":"Source with durable leaf marker"}
|
||||
{"type":"leaf","id":"771ef412","parentId":"116253e0","timestamp":"2026-08-02T10:17:40.695Z","targetId":"a49e0082"}
|
||||
+169
@@ -0,0 +1,169 @@
|
||||
/*
|
||||
* Test-only transcription of the Pi 0.83.0 context algorithms:
|
||||
* https://github.com/earendil-works/pi/blob/v0.83.0/packages/coding-agent/src/core/session-manager.ts
|
||||
* https://github.com/earendil-works/pi/blob/v0.83.0/packages/agent/src/harness/session/session.ts
|
||||
*
|
||||
* MIT License
|
||||
*
|
||||
* Copyright (c) 2025 Mario Zechner
|
||||
*
|
||||
* Permission is hereby granted, free of charge, to any person obtaining a copy
|
||||
* of this software and associated documentation files (the "Software"), to deal
|
||||
* in the Software without restriction, including without limitation the rights
|
||||
* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
||||
* copies of the Software, and to permit persons to whom the Software is
|
||||
* furnished to do so, subject to the following conditions:
|
||||
*
|
||||
* The above copyright notice and this permission notice shall be included in all
|
||||
* copies or substantial portions of the Software.
|
||||
*
|
||||
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
||||
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
||||
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
||||
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
||||
* SOFTWARE.
|
||||
*/
|
||||
|
||||
export const PI_CONTEXT_ORACLE_VERSION = '0.83.0';
|
||||
|
||||
function buildEntryIndex(entries, byId) {
|
||||
if (byId) return byId;
|
||||
const index = new Map();
|
||||
for (const entry of entries) index.set(entry.id, entry);
|
||||
return index;
|
||||
}
|
||||
|
||||
export function buildCodingAgentSessionPath(entries, leafId, byId) {
|
||||
const index = buildEntryIndex(entries, byId);
|
||||
let leaf;
|
||||
if (leafId === null) return [];
|
||||
if (leafId) leaf = index.get(leafId);
|
||||
leaf ??= entries[entries.length - 1];
|
||||
if (!leaf) return [];
|
||||
|
||||
const path = [];
|
||||
let current = leaf;
|
||||
while (current) {
|
||||
path.push(current);
|
||||
current = current.parentId ? index.get(current.parentId) : undefined;
|
||||
}
|
||||
path.reverse();
|
||||
return path;
|
||||
}
|
||||
|
||||
// pi-agent-core 0.83.0 JsonlSessionStorage.getPathToRootOrCompaction().
|
||||
// The checked fixtures may contain synthetic orphan edges; those terminate the
|
||||
// path just as the coding-agent oracle above does instead of exercising the
|
||||
// storage API's separate invalid-session error.
|
||||
export function buildAgentCoreSessionPath(entries, leafId, byId) {
|
||||
const index = buildEntryIndex(entries, byId);
|
||||
if (leafId === null) return [];
|
||||
let current = leafId ? index.get(leafId) : entries[entries.length - 1];
|
||||
current ??= entries[entries.length - 1];
|
||||
if (!current) return [];
|
||||
|
||||
const path = [];
|
||||
let stopAtEntryId = null;
|
||||
while (current) {
|
||||
path.unshift(current);
|
||||
if (stopAtEntryId !== null && current.id === stopAtEntryId) break;
|
||||
if (current.type === 'compaction') {
|
||||
if (current.retainedTail) break;
|
||||
stopAtEntryId = current.firstKeptEntryId ?? null;
|
||||
}
|
||||
if (!current.parentId) break;
|
||||
current = index.get(current.parentId);
|
||||
}
|
||||
return path;
|
||||
}
|
||||
|
||||
export function buildCodingAgentContextEntries(entries, leafId, byId) {
|
||||
const path = buildCodingAgentSessionPath(entries, leafId, byId);
|
||||
let compaction = null;
|
||||
for (const entry of path) {
|
||||
if (entry.type === 'compaction') compaction = entry;
|
||||
}
|
||||
if (!compaction) return path;
|
||||
|
||||
const compactionIndex = path.findIndex(entry => entry.id === compaction.id);
|
||||
if (compactionIndex < 0) return path;
|
||||
|
||||
const contextEntries = [compaction];
|
||||
let foundFirstKept = false;
|
||||
for (let index = 0; index < compactionIndex; index++) {
|
||||
const entry = path[index];
|
||||
if (entry.id === compaction.firstKeptEntryId) foundFirstKept = true;
|
||||
if (foundFirstKept) contextEntries.push(entry);
|
||||
}
|
||||
contextEntries.push(...path.slice(compactionIndex + 1));
|
||||
return contextEntries;
|
||||
}
|
||||
|
||||
export function defaultAgentCoreContextEntryTransform(pathEntries) {
|
||||
let compaction = null;
|
||||
for (const entry of pathEntries) {
|
||||
if (entry.type === 'compaction') compaction = entry;
|
||||
}
|
||||
if (!compaction) return [...pathEntries];
|
||||
|
||||
const entries = [compaction];
|
||||
const compactionIndex = pathEntries.findIndex(entry => (
|
||||
entry.type === 'compaction' && entry.id === compaction.id
|
||||
));
|
||||
if (compaction.retainedTail) {
|
||||
for (let index = compactionIndex + 1; index < pathEntries.length; index++) {
|
||||
entries.push(pathEntries[index]);
|
||||
}
|
||||
return entries;
|
||||
}
|
||||
if (compaction.firstKeptEntryId) {
|
||||
let foundFirstKept = false;
|
||||
for (let index = 0; index < compactionIndex; index++) {
|
||||
const entry = pathEntries[index];
|
||||
if (entry.id === compaction.firstKeptEntryId) foundFirstKept = true;
|
||||
if (foundFirstKept) entries.push(entry);
|
||||
}
|
||||
}
|
||||
for (let index = compactionIndex + 1; index < pathEntries.length; index++) {
|
||||
entries.push(pathEntries[index]);
|
||||
}
|
||||
return entries;
|
||||
}
|
||||
|
||||
export function pi083LeafId(entries) {
|
||||
let leafId = null;
|
||||
for (const entry of entries) {
|
||||
leafId = entry.type === 'leaf' ? entry.targetId : entry.id;
|
||||
}
|
||||
return leafId;
|
||||
}
|
||||
|
||||
function evidence(kind, source, role, content) {
|
||||
return JSON.stringify({ kind, source, role, content });
|
||||
}
|
||||
|
||||
// The Pi oracles select the active context entries. Obelisk stores those entries
|
||||
// in durable physical order for its evidence timeline, while retainedTail
|
||||
// messages remain immediately after their owning compaction summary.
|
||||
export function projectCanonicalEvidence(physicalEntries, contextEntries) {
|
||||
const activeIds = new Set(contextEntries.map(entry => entry.id));
|
||||
return physicalEntries.filter(entry => activeIds.has(entry.id)).flatMap((entry) => {
|
||||
if (entry.type === 'message') {
|
||||
return [evidence('message', null, entry.message.role, entry.message.content)];
|
||||
}
|
||||
if (entry.type === 'branch_summary') {
|
||||
return [evidence('summary', 'pi:branch_summary', null, entry.summary)];
|
||||
}
|
||||
if (entry.type === 'compaction') {
|
||||
return [
|
||||
evidence('summary', 'pi:compaction', null, entry.summary),
|
||||
...(entry.retainedTail ?? []).map(message => (
|
||||
evidence('message', null, message.role, message.content)
|
||||
)),
|
||||
];
|
||||
}
|
||||
return [];
|
||||
});
|
||||
}
|
||||
+7
@@ -0,0 +1,7 @@
|
||||
{"type":"session","version":3,"id":"real-probe","timestamp":"2026-08-02T09:46:45.516Z","cwd":"/tmp/pi-project"}
|
||||
{"type":"session_info","id":"211e5f6a","parentId":null,"timestamp":"2026-08-02T09:46:45.517Z","name":"Real model probe"}
|
||||
{"type":"model_change","id":"ec3a3e01","parentId":"211e5f6a","timestamp":"2026-08-02T09:46:45.535Z","provider":"custom-openai","modelId":"gpt-probe"}
|
||||
{"type":"message","id":"a7d8022e","parentId":"ec3a3e01","timestamp":"2026-08-02T09:46:45.539Z","message":{"role":"user","content":[{"type":"text","text":"Reply with the probe marker"},{"type":"image","mimeType":"image/png","data":"QUJDRA=="}],"timestamp":1785664005538}}
|
||||
{"type":"message","id":"3fecb3df","parentId":"a7d8022e","timestamp":"2026-08-02T09:46:46.507Z","message":{"role":"assistant","content":[],"api":"openai-completions","provider":"custom-openai","model":"gpt-probe","usage":{"input":0,"output":0,"cacheRead":0,"cacheWrite":0,"totalTokens":0,"cost":{"input":0,"output":0,"cacheRead":0,"cacheWrite":0,"total":0}},"stopReason":"error","timestamp":1785664005566,"errorMessage":"404 not found"}}
|
||||
{"type":"message","id":"e40917d4","parentId":"3fecb3df","timestamp":"2026-08-02T09:47:57.481Z","message":{"role":"user","content":[{"type":"text","text":"Try the responses protocol"}],"timestamp":1785664077480}}
|
||||
{"type":"message","id":"57e279e0","parentId":"e40917d4","timestamp":"2026-08-02T09:47:59.459Z","message":{"role":"assistant","content":[{"type":"thinking","thinking":"","thinkingSignature":"opaque-probe-signature"},{"type":"text","text":"REAL_PI_PROBE_OK"}],"api":"openai-responses","provider":"custom-openai","model":"gpt-probe","usage":{"input":973,"output":20,"cacheRead":3840,"cacheWrite":5,"reasoning":9,"totalTokens":4838,"cost":{"input":0,"output":0,"cacheRead":0,"cacheWrite":0,"total":0}},"stopReason":"stop","timestamp":1785664077508}}
|
||||
Vendored
+8
@@ -0,0 +1,8 @@
|
||||
{"type":"session","version":3,"id":"tool-probe","timestamp":"2026-08-02T09:42:38.031Z","cwd":"/tmp/pi-project"}
|
||||
{"type":"session_info","id":"eacdf41f","parentId":null,"timestamp":"2026-08-02T09:42:38.032Z","name":"Tool probe"}
|
||||
{"type":"model_change","id":"fba5d21a","parentId":"eacdf41f","timestamp":"2026-08-02T09:42:38.041Z","provider":"obelisk-probe","modelId":"probe-model"}
|
||||
{"type":"thinking_level_change","id":"e32523fb","parentId":"fba5d21a","timestamp":"2026-08-02T09:42:38.041Z","thinkingLevel":"off"}
|
||||
{"type":"message","id":"7e0576ed","parentId":"e32523fb","timestamp":"2026-08-02T09:42:38.043Z","message":{"role":"user","content":[{"type":"text","text":"Read probe.txt and report it"}],"timestamp":1785663758043}}
|
||||
{"type":"message","id":"c2be25cd","parentId":"7e0576ed","timestamp":"2026-08-02T09:42:38.069Z","message":{"role":"assistant","content":[{"type":"toolCall","id":"call_obelisk_probe","name":"read","arguments":{"path":"probe.txt"}}],"api":"openai-completions","provider":"obelisk-probe","model":"probe-model","usage":{"input":14,"output":5,"cacheRead":3,"cacheWrite":0,"reasoning":2,"totalTokens":22,"cost":{"input":0,"output":0,"cacheRead":0,"cacheWrite":0,"total":0}},"stopReason":"toolUse","timestamp":1785663758058}}
|
||||
{"type":"message","id":"47e488af","parentId":"c2be25cd","timestamp":"2026-08-02T09:42:38.072Z","message":{"role":"toolResult","toolCallId":"call_obelisk_probe","toolName":"read","content":[{"type":"text","text":"real-pi-tool-result\n"}],"isError":false,"timestamp":1785663758072}}
|
||||
{"type":"message","id":"9db96a87","parentId":"47e488af","timestamp":"2026-08-02T09:42:38.075Z","message":{"role":"assistant","content":[{"type":"text","text":"The read tool returned real-pi-tool-result."}],"api":"openai-completions","provider":"obelisk-probe","model":"probe-model","usage":{"input":19,"output":9,"cacheRead":4,"cacheWrite":0,"reasoning":0,"totalTokens":32,"cost":{"input":0,"output":0,"cacheRead":0,"cacheWrite":0,"total":0}},"stopReason":"stop","timestamp":1785663758072}}
|
||||
Reference in New Issue
Block a user