feat(core): add first-class Pi session indexing (#23)

Pi cannot be read as another linear JSONL stream. Its history is a tree with
a durable leaf, orphan roots, branch summaries, and two compaction forms, so
the active context is something the format states rather than something line
order implies. The adapter keeps those semantics inside itself and projects
the result into the existing canonical tables.

Sessions are keyed by (normalized header cwd, header id) rather than by path,
because Pi's --session-id lookup is project-local: two projects may reuse an
id, while a move or an identical copy is still one session. Discovery covers
both layouts Pi writes and fingerprints each file by mtime, ctime, size and
inode, so a rewrite that preserves mtime is not read as unchanged.

Abandoned branches are preserved rather than dropped. Visibility becomes
three-state -- visible, inactive, hidden -- and helpers return only visible
rows until includeInactive asks for the superseded path, labeling every row
so a caller knows which it holds. Usage counts all three, because an
abandoned call still spent tokens; message_count reports only the visible
transcript.

A committed MIT-licensed oracle transcribed from Pi 0.83.0 pins the context
algorithms, and a fixed-seed differential runs 512 generated sessions against
it on every test run. Schema changes are additive.

packages/core/package.json

@@ -15,6 +15,7 @@
     "./providers/claude": "./dist/providers/claude.js",
     "./providers/codex": "./dist/providers/codex.js",
     "./providers/kimi": "./dist/providers/kimi.js",
+    "./providers/pi": "./dist/providers/pi.js",
     "./providers/registry": "./dist/providers/registry.js",
     "./providers/builtins": "./dist/providers/builtins.js",
     "./providers/types": "./dist/providers/types.js",

packages/core/src/core.ts

@@ -12,6 +12,11 @@ import { createContext, runInNewContext } from 'node:vm';
 
 import { DB_PATH, openDb, openReadDb, openWriterLeaseDb } from './db.ts';
 import { buildIndex, shouldSkipBuild } from './indexer.ts';
+import {
+  createConfiguredBuiltinProviderRuntime,
+  readPersistedProviderSettings,
+} from './provider-settings.ts';
+import type { ProviderRegistry } from './providers/registry.ts';
 import { createQueryApi, createAttuneApi } from './query.ts';
 import { acquireWriterLease, writerLockPathFor } from './writer-lease.ts';
 
@@ -19,6 +24,42 @@ export { buildIndex, DB_PATH };
 
 type SandboxApi = Record<string, unknown>;
 
+interface InventoryIssue {
+  provider?: unknown;
+  path?: unknown;
+  error?: unknown;
+}
+
+function reportIncompleteInventory(build: unknown): void {
+  if (build === null || typeof build !== 'object' || !('inventoryIssues' in build)) return;
+  const issues = (build as { inventoryIssues?: unknown }).inventoryIssues;
+  if (!Array.isArray(issues)) return;
+  for (const value of issues) {
+    const issue = value as InventoryIssue | null;
+    if (
+      issue !== null
+      && typeof issue.provider === 'string'
+      && typeof issue.path === 'string'
+      && typeof issue.error === 'string'
+    ) {
+      process.stderr.write(
+        `Warning: incomplete ${issue.provider} source inventory at ${issue.path}: ${issue.error}\n`,
+      );
+    }
+  }
+}
+
+function refreshQueryIndex(): ProviderRegistry {
+  const settings = readPersistedProviderSettings();
+  const providerRegistry = createConfiguredBuiltinProviderRuntime(settings.settings).registry;
+  if (!settings.ok) {
+    process.stderr.write(`Warning: ${settings.error}; index refresh skipped\n`);
+    return providerRegistry;
+  }
+  reportIncompleteInventory(buildIndex({ providerRegistry }));
+  return providerRegistry;
+}
+
 // Run a user-supplied CodeAct script inside the query/attune sandbox. The script
 // body runs as an async IIFE with a 30s timeout; its `return` value is resolved.
 function runInSandbox(api: SandboxApi, scriptContent: string): Promise<unknown> {
@@ -32,10 +73,10 @@ function runInSandbox(api: SandboxApi, scriptContent: string): Promise<unknown>
 
 // FTS search over indexed message text. Refreshes the index, then queries.
 export function searchText(text: string, opts?: Record<string, unknown>): unknown {
-  buildIndex();
+  const providerRegistry = refreshQueryIndex();
   const db = openReadDb();
   try {
-    return createQueryApi(db).search(text, opts);
+    return createQueryApi(db, { providerRegistry }).search(text, opts);
   } finally {
     db.close();
   }
@@ -43,10 +84,10 @@ export function searchText(text: string, opts?: Record<string, unknown>): unknow
 
 // Execute a read-only CodeAct query script and resolve its returned value.
 export async function executeQuery(scriptContent: string): Promise<unknown> {
-  buildIndex();
+  const providerRegistry = refreshQueryIndex();
   const db = openReadDb();
   try {
-    return await runInSandbox(createQueryApi(db), scriptContent);
+    return await runInSandbox(createQueryApi(db, { providerRegistry }), scriptContent);
   } finally {
     db.close();
   }
@@ -55,6 +96,7 @@ export async function executeQuery(scriptContent: string): Promise<unknown> {
 // Execute a memory-mutation CodeAct script (remember/forget only).
 export async function executeAttune(scriptContent: string): Promise<unknown> {
   const build = buildIndex() as { reason?: string } | undefined;
+  reportIncompleteInventory(build);
   if (build?.reason === 'daemon_active') {
     throw new Error('Obelisk daemon owns index writes; attune is read-only until the daemon stops');
   }

packages/core/src/indexer.ts

@@ -5,15 +5,22 @@ import { inferProjectPath } from './parsing.ts';
 import {
   createProviderIndexPlan,
   indexProviderPlan,
+  indexProviderPlanStrict,
+  ProviderIndexFailure,
   writeProviderIndexMarkers,
 } from './provider-indexing.ts';
 import { nodeSqliteTransactionAdapter } from './tx.ts';
 import { acquireWriterLease, writerLockPathFor } from './writer-lease.ts';
 import { runRetryableWriteTransaction, isBeginBusyFailure, hasUnusableTransaction } from './write-coordinator.ts';
-import { createBuiltinProviderRegistry } from './providers/builtins.ts';
+import {
+  createConfiguredBuiltinProviderRuntime,
+  readPersistedProviderSettings,
+} from './provider-settings.ts';
+import type { ProviderRegistry } from './providers/registry.ts';
 import type { NodeSqliteDb, SqliteRow } from './sqlite-types.ts';
 
 interface SkippedFile {
+  provider: string;
   path: string;
   error: string;
   diagnostics?: unknown;
@@ -24,6 +31,11 @@ interface BuildCheckOptions {
   ignoreRecentBuild?: boolean;
 }
 
+interface BuildIndexOptions {
+  force?: boolean;
+  providerRegistry?: ProviderRegistry;
+}
+
 function errorMessage(error: unknown): string {
   return error instanceof Error ? error.message : String(error);
 }
@@ -82,7 +94,7 @@ function inspectBuildOwnership({ force = false }: { force?: boolean } = {}) {
   }
 }
 
-function buildIndex({ force = false }: { force?: boolean } = {}) {
+function buildIndex({ force = false, providerRegistry }: BuildIndexOptions = {}) {
   const ownership = inspectBuildOwnership({ force });
   if (ownership.skip) return ownership;
   const lease = acquireWriterLease({
@@ -94,34 +106,100 @@ function buildIndex({ force = false }: { force?: boolean } = {}) {
     // Ownership may change between the first read and lease acquisition.
     const ownershipAfterLease = inspectBuildOwnership({ force });
     if (ownershipAfterLease.skip) return ownershipAfterLease;
+    let registry = providerRegistry;
+    if (registry === undefined) {
+      const settings = readPersistedProviderSettings();
+      if (!settings.ok) {
+        return { skip: true, reason: 'settings_unavailable', error: settings.error };
+      }
+      registry = createConfiguredBuiltinProviderRuntime(settings.settings).registry;
+    }
 
     const db = openDb();
     const txDb = nodeSqliteTransactionAdapter(db);
     const skippedFiles: SkippedFile[] = [];
     try {
-      try {
-        if (force) {
+      const providerPlan = createProviderIndexPlan(db, registry, { force });
+      const incompleteProviders = [...providerPlan.incompleteProviders].sort();
+      const inventoryIssues = [...providerPlan.inventoryIssues];
+      if (force && incompleteProviders.length > 0) {
+        return {
+          skip: false,
+          complete: false,
+          reason: 'incomplete_snapshot',
+          incompleteProviders,
+          inventoryIssues,
+          skipped: 0,
+          skippedFiles,
+        };
+      }
+
+      if (force) {
+        try {
           runRetryableWriteTransaction(txDb, () => {
-            db.prepare("DELETE FROM index_state WHERE jsonl_path != '__last_build__'").run();
-            // Clearing index_state alone re-indexes existing files but leaves rows for
-            // files that no longer exist on disk (stale sessions accumulate). A force
-            // build is a clean rebuild: drop every derived table, then re-index from the
-            // current files. `memories` is the durable, human-approved layer and is never
-            // cleared; messages_fts is repopulated by the 'rebuild' command in finalize.
+            // A force build publishes one complete source snapshot or nothing.
+            // The provider contract reserves no key prefix. A force snapshot
+            // recreates every unit cursor, provider marker, and system marker.
+            db.prepare('DELETE FROM index_state').run();
             for (const table of ['messages', 'tool_calls', 'tool_results', 'sessions', 'summaries', 'subagents', 'workflows', 'workflow_agents']) {
               db.prepare(`DELETE FROM ${table}`).run();
             }
-          }, { label: 'force-cleanup' });
+            const providerResult = indexProviderPlanStrict({
+              db,
+              plan: providerPlan,
+            });
+            refreshSessionProjectPaths(db);
+            db.exec("INSERT INTO messages_fts(messages_fts) VALUES('rebuild')");
+            rebuildMemoryFts(db);
+            db.prepare("INSERT OR REPLACE INTO index_state (jsonl_path, mtime, lines_processed) VALUES ('__last_build__', ?, 0)").run(Date.now());
+            writeProviderIndexMarkers(db, providerPlan, providerResult);
+          }, { label: 'force-rebuild' });
+        } catch (error) {
+          if (isBeginBusyFailure(error)) {
+            return {
+              skip: true,
+              complete: false,
+              reason: 'database_busy',
+              incompleteProviders,
+              inventoryIssues,
+              skipped: 0,
+              skippedFiles,
+            };
+          }
+          if (error instanceof ProviderIndexFailure) {
+            const skippedFile = {
+              provider: error.item.provider.name,
+              path: error.item.unit.key,
+              error: errorMessage(error.sourceError),
+              diagnostics: (error as { obelisk?: unknown }).obelisk
+                ?? (error.sourceError as { obelisk?: unknown } | null)?.obelisk,
+            };
+            skippedFiles.push(skippedFile);
+            process.stderr.write(
+              `Warning: failed to index ${error.item.provider.name} unit ${skippedFile.path}: ${skippedFile.error}\n`,
+            );
+            return {
+              skip: false,
+              complete: false,
+              reason: 'provider_failure',
+              incompleteProviders,
+              inventoryIssues,
+              skipped: skippedFiles.length,
+              skippedFiles,
+            };
+          }
+          throw error;
         }
-      } catch (error) {
-        if (isBeginBusyFailure(error)) {
-          return { skip: true, reason: 'database_busy', skipped: skippedFiles.length, skippedFiles };
-        }
-        throw error;
+        return {
+          skip: false,
+          complete: true,
+          incompleteProviders,
+          inventoryIssues,
+          skipped: 0,
+          skippedFiles,
+        };
       }
 
-      const registry = createBuiltinProviderRegistry();
-      const providerPlan = createProviderIndexPlan(db, registry, { force });
       const providerResult = indexProviderPlan({
         db,
         plan: providerPlan,
@@ -131,13 +209,26 @@ function buildIndex({ force = false }: { force?: boolean } = {}) {
           if (hasUnusableTransaction(error)) throw error;
           const detail = error as { message?: unknown; obelisk?: unknown } | null;
           const message = errorMessage(error);
-          skippedFiles.push({ path: unit.key, error: message, diagnostics: detail?.obelisk });
+          skippedFiles.push({
+            provider: provider.name,
+            path: unit.key,
+            error: message,
+            diagnostics: detail?.obelisk,
+          });
           process.stderr.write(`Warning: failed to index ${provider.name} unit ${unit.key}: ${message}\n`);
           return 'skip';
         },
       });
       if (providerResult.stopped) {
-        return { skip: true, reason: 'database_busy', skipped: skippedFiles.length, skippedFiles };
+        return {
+          skip: true,
+          complete: false,
+          reason: 'database_busy',
+          incompleteProviders,
+          inventoryIssues,
+          skipped: skippedFiles.length,
+          skippedFiles,
+        };
       }
       // Finalize is one transaction and is NOT swallowed: a finalize failure fails
       // the build (a half-finalized index would be inconsistent).
@@ -151,11 +242,26 @@ function buildIndex({ force = false }: { force?: boolean } = {}) {
         }, { label: 'finalize' });
       } catch (error) {
         if (isBeginBusyFailure(error)) {
-          return { skip: true, reason: 'database_busy', skipped: skippedFiles.length, skippedFiles };
+          return {
+            skip: true,
+            complete: false,
+            reason: 'database_busy',
+            incompleteProviders,
+            inventoryIssues,
+            skipped: skippedFiles.length,
+            skippedFiles,
+          };
         }
         throw error;
       }
-      return { skip: false, skipped: skippedFiles.length, skippedFiles };
+      return {
+        skip: false,
+        complete: providerResult.complete,
+        incompleteProviders,
+        inventoryIssues,
+        skipped: skippedFiles.length,
+        skippedFiles,
+      };
     } finally {
       db.close();
     }

packages/core/src/parsing.ts

@@ -6,6 +6,8 @@ import { closeSync, existsSync, openSync, readSync, readdirSync, statSync } from
 import { homedir } from 'node:os';
 import { isAbsolute, join, normalize } from 'node:path';
 
+import type { InventoryIssue } from './providers/types.ts';
+
 const CLAUDE_DIR = join(homedir(), '.claude');
 const CODEX_DIR = join(homedir(), '.codex');
 const PROJECTS_DIR = join(CLAUDE_DIR, 'projects');
@@ -15,6 +17,12 @@ const TEXT_LIMIT = 10000;
 type JsonRecord = Record<string, any>;
 type JsonValue = any;
 
+function sourceInventoryIssue(path: string, error: unknown): InventoryIssue {
+  return { path, error: error instanceof Error ? error.message : String(error) };
+}
+
+type DiscoveryIssueHandler = (issue: InventoryIssue) => void;
+
 export interface ClaudeJsonlFile {
   path: string;
   sessionId: string;
@@ -153,16 +161,19 @@ function inferProjectPath(project: string | null | undefined, observedCwds: unkn
   return best?.path || legacyProjectPathFromSlug(project);
 }
 
-function discoverJsonlFiles(projectsDir = PROJECTS_DIR): ClaudeJsonlFile[] {
+function discoverJsonlFiles(
+  projectsDir = PROJECTS_DIR,
+  reportIssue?: DiscoveryIssueHandler,
+): ClaudeJsonlFile[] {
   const files: ClaudeJsonlFile[] = [];
   if (!existsSync(projectsDir)) return files;
   let projects;
-  try { projects = readdirSync(projectsDir); } catch (e) { process.stderr.write(`Warning: cannot read projects dir: ${e instanceof Error ? e.message : String(e)}\n`); return files; }
+  try { projects = readdirSync(projectsDir); } catch (error) { reportIssue?.(sourceInventoryIssue(projectsDir, error)); return files; }
   for (const proj of projects) {
     const projPath = join(projectsDir, proj);
     if (!isDir(projPath)) continue;
     let entries;
-    try { entries = readdirSync(projPath); } catch { continue; }
+    try { entries = readdirSync(projPath); } catch (error) { reportIssue?.(sourceInventoryIssue(projPath, error)); continue; }
     for (const f of entries) {
       if (f.endsWith('.jsonl'))
         files.push({ path: join(projPath, f), sessionId: f.slice(0, -6), project: proj, isSubagent: false });
@@ -171,7 +182,7 @@ function discoverJsonlFiles(projectsDir = PROJECTS_DIR): ClaudeJsonlFile[] {
       const saDir = join(projPath, sd, 'subagents');
       if (!isDir(saDir)) continue;
       let saEntries;
-      try { saEntries = readdirSync(saDir); } catch { continue; }
+      try { saEntries = readdirSync(saDir); } catch (error) { reportIssue?.(sourceInventoryIssue(saDir, error)); continue; }
       for (const sf of saEntries) {
         if (sf.endsWith('.jsonl'))
           files.push({ path: join(saDir, sf), sessionId: sd, project: proj, isSubagent: true, agentId: sf.slice(0, -6) });
@@ -179,12 +190,12 @@ function discoverJsonlFiles(projectsDir = PROJECTS_DIR): ClaudeJsonlFile[] {
       const wfRoot = join(saDir, 'workflows');
       if (!isDir(wfRoot)) continue;
       let wfDirs;
-      try { wfDirs = readdirSync(wfRoot); } catch { continue; }
+      try { wfDirs = readdirSync(wfRoot); } catch (error) { reportIssue?.(sourceInventoryIssue(wfRoot, error)); continue; }
       for (const wfDir of wfDirs) {
         const wfPath = join(wfRoot, wfDir);
         if (!isDir(wfPath)) continue;
         let wfEntries;
-        try { wfEntries = readdirSync(wfPath); } catch { continue; }
+        try { wfEntries = readdirSync(wfPath); } catch (error) { reportIssue?.(sourceInventoryIssue(wfPath, error)); continue; }
         for (const wf of wfEntries) {
           if (wf.endsWith('.jsonl'))
             files.push({ path: join(wfPath, wf), sessionId: sd, project: proj, isSubagent: true, agentId: wf.slice(0, -6), workflowRunId: wfDir });
@@ -195,12 +206,15 @@ function discoverJsonlFiles(projectsDir = PROJECTS_DIR): ClaudeJsonlFile[] {
   return files;
 }
 
-function discoverCodexJsonlFiles(sessionsDir = CODEX_SESSIONS_DIR): CodexJsonlFile[] {
+function discoverCodexJsonlFiles(
+  sessionsDir = CODEX_SESSIONS_DIR,
+  reportIssue?: DiscoveryIssueHandler,
+): CodexJsonlFile[] {
   const files: CodexJsonlFile[] = [];
   if (!existsSync(sessionsDir)) return files;
   const walk = (dir: string): void => {
     let entries;
-    try { entries = readdirSync(dir, { withFileTypes: true }); } catch { return; }
+    try { entries = readdirSync(dir, { withFileTypes: true }); } catch (error) { reportIssue?.(sourceInventoryIssue(dir, error)); return; }
     for (const entry of entries) {
       const fp = join(dir, entry.name);
       if (entry.isDirectory()) {
@@ -369,7 +383,7 @@ export {
   CLAUDE_DIR, CODEX_DIR, PROJECTS_DIR, CODEX_SESSIONS_DIR, TEXT_LIMIT,
   trunc, truncJson, extractText, extractContentType, extractMessageIsMeta, isSkillInstructions, filePath, isDir, readLines,
   legacyProjectPathFromSlug, normalizeObservedCwd, projectSlugFromPath, inferProjectPath,
-  discoverJsonlFiles, discoverCodexJsonlFiles,
+  discoverJsonlFiles, discoverCodexJsonlFiles, sourceInventoryIssue,
   codexDbId, codexRawId, codexLineUuid, codexCallId, codexParentThreadId, codexIsGuardianThread,
   readCodexGuardianThreadInfo, codexAgentNickname, codexAgentRole, parseCodexJsonInput,
   codexUsage, codexEventText, codexMessagePayloadText, codexVisibleMessageKey, codexToolInput, codexToolOutput,

packages/core/src/persist.ts

@@ -33,7 +33,7 @@ function statements(db: SqliteDb) {
         cwd=excluded.cwd, skill=excluded.skill, source=excluded.source`),
     tc: db.prepare('INSERT OR REPLACE INTO tool_calls (id,message_uuid,session_id,name,presentation,input_json,file_path) VALUES (?,?,?,?,?,?,?)'),
     tr: db.prepare('INSERT OR REPLACE INTO tool_results (tool_use_id,message_uuid,session_id,content,file_path,is_error) VALUES (?,?,?,?,?,?)'),
-    sum: db.prepare('INSERT OR REPLACE INTO summaries (id,session_id,timestamp,source,content) VALUES (?,?,?,?,?)'),
+    sum: db.prepare('INSERT OR REPLACE INTO summaries (id,session_id,timestamp,source,content,visibility,input_tokens,output_tokens) VALUES (?,?,?,?,?,?,?,?)'),
     ses: db.prepare('INSERT OR REPLACE INTO sessions (id,title,project,project_path,started_at,ended_at,git_branch,version,message_count,jsonl_path,source) VALUES (?,?,?,?,?,?,?,?,?,?,?)'),
     sub: db.prepare(`
       INSERT INTO subagents (agent_id,session_id,parent_tool_use_id,agent_type,description,duration_ms,total_tokens)
@@ -65,7 +65,7 @@ function statements(db: SqliteDb) {
         tokens=COALESCE(excluded.tokens, workflow_agents.tokens),
         tool_calls=COALESCE(excluded.tool_calls, workflow_agents.tool_calls)`),
     turn: db.prepare('UPDATE messages SET turn_duration_ms=? WHERE uuid=?'),
-    idx: db.prepare('INSERT OR REPLACE INTO index_state (jsonl_path,mtime,lines_processed) VALUES (?,?,?)'),
+    idx: db.prepare('INSERT OR REPLACE INTO index_state (jsonl_path,mtime,lines_processed,cursor) VALUES (?,?,?,?)'),
     getSession: db.prepare('SELECT * FROM sessions WHERE id=?'),
   };
 }
@@ -99,7 +99,16 @@ export function persist(db: SqliteDb, unit: IndexUnit, gen: Generator<Transcript
         st.tr.run(r.tool_use_id, r.message_uuid, r.session_id, r.content, r.file_path, r.is_error);
         break;
       case 'summary':
-        st.sum.run(r.id, r.session_id, r.timestamp, r.source, r.content);
+        st.sum.run(
+          r.id,
+          r.session_id,
+          r.timestamp,
+          r.source,
+          r.content,
+          r.visibility ?? 'visible',
+          r.input_tokens ?? null,
+          r.output_tokens ?? null,
+        );
         break;
       case 'subagent':
         st.sub.run(r.agent_id, r.session_id, r.parent_tool_use_id ?? null, r.agent_type ?? null, r.description ?? null, r.duration_ms ?? null, r.total_tokens ?? null);
@@ -147,7 +156,7 @@ export function persist(db: SqliteDb, unit: IndexUnit, gen: Generator<Transcript
 
   if (cursor != null) {
     const [mtime, lines] = cursor.split(':');
-    st.idx.run(unit.key, Number(mtime), Number(lines));
+    st.idx.run(unit.key, Number(mtime), Number(lines), cursor);
   }
   return cursor;
 }

packages/core/src/provider-indexing.ts

@@ -1,52 +1,135 @@
 import { persist } from './persist.ts';
 import type { ProviderRegistry } from './providers/registry.ts';
-import type { Cursor, IndexUnit, ProviderAdapter } from './providers/types.ts';
+import type {
+  Cursor,
+  IndexedSession,
+  IndexUnit,
+  InventoryIssue,
+  ProviderAdapter,
+} from './providers/types.ts';
 import type { SqliteDb } from './sqlite-types.ts';
 
+export interface ProviderSessionProvenance extends IndexedSession {
+  readonly source: string;
+}
+
 export interface ProviderIndexItem {
   readonly provider: ProviderAdapter;
   readonly unit: IndexUnit;
   readonly cursor: Cursor;
 }
 
+export interface ProviderInventoryIssue extends InventoryIssue {
+  readonly provider: string;
+}
+
 export interface ProviderIndexPlan {
   readonly items: ProviderIndexItem[];
   readonly pendingMarkers: ReadonlyMap<string, string>;
+  readonly replayKeys: ReadonlyMap<string, readonly string[]>;
+  readonly incompleteProviders: ReadonlySet<string>;
+  readonly inventoryIssues: readonly ProviderInventoryIssue[];
 }
 
 export interface ProviderIndexResult {
   readonly committed: ProviderIndexItem[];
   readonly failedProviders: ReadonlySet<string>;
+  readonly failedItems: ProviderIndexItem[];
+  readonly complete: boolean;
   readonly stopped?: { item: ProviderIndexItem; error: unknown };
 }
 
-export function storedProviderCursor(db: SqliteDb, key: string): Cursor {
-  const row = db.prepare('SELECT mtime, lines_processed FROM index_state WHERE jsonl_path = ?').get(key);
-  return row ? `${String(row.mtime)}:${String(row.lines_processed)}` : null;
+export class ProviderIndexFailure extends Error {
+  readonly item: ProviderIndexItem;
+  readonly sourceError: unknown;
+
+  constructor(error: unknown, item: ProviderIndexItem) {
+    super(error instanceof Error ? error.message : String(error), { cause: error });
+    this.item = item;
+    this.sourceError = error;
+  }
 }
 
-function sourceAlreadyIndexed(db: SqliteDb, source: string): boolean {
-  return Boolean(db.prepare('SELECT 1 FROM sessions WHERE source = ? LIMIT 1').get(source));
+export function storedProviderCursor(db: SqliteDb, key: string): Cursor {
+  const row = db.prepare('SELECT mtime, lines_processed, cursor FROM index_state WHERE jsonl_path = ?').get(key);
+  if (!row) return null;
+  return typeof row.cursor === 'string'
+    ? row.cursor
+    : `${String(row.mtime)}:${String(row.lines_processed)}`;
+}
+
+export function readProviderSessionProvenance(db: SqliteDb): ProviderSessionProvenance[] {
+  return db.prepare(`
+    SELECT id, jsonl_path, COALESCE(source, 'claude') AS source
+    FROM sessions
+    WHERE jsonl_path IS NOT NULL
+      AND jsonl_path != ''
+  `).all().map((row) => ({
+    source: String(row.source),
+    sessionId: String(row.id),
+    jsonlPath: String(row.jsonl_path),
+  }));
 }
 
 export function createProviderIndexPlan(
   db: SqliteDb,
   registry: ProviderRegistry,
-  { force = false, changedPaths }: { force?: boolean; changedPaths?: string[] } = {},
+  {
+    force = false,
+    changedPaths,
+    priorSessions,
+  }: {
+    force?: boolean;
+    changedPaths?: string[];
+    priorSessions?: readonly ProviderSessionProvenance[];
+  } = {},
 ): ProviderIndexPlan {
   const items: ProviderIndexItem[] = [];
   const pendingMarkers = new Map<string, string>();
+  const replayKeys = new Map<string, readonly string[]>();
+  const incompleteProviders = new Set<string>();
+  const inventoryIssues: ProviderInventoryIssue[] = [];
+  const provenance = priorSessions ?? readProviderSessionProvenance(db);
   for (const provider of registry.list()) {
+    const indexedSessions = provenance
+      .filter((session) => session.source === provider.name)
+      .map(({ sessionId, jsonlPath }) => ({ sessionId, jsonlPath }));
     const marker = provider.indexVersionMarker;
     const markerMissing = marker !== undefined && !db.prepare(
       'SELECT jsonl_path FROM index_state WHERE jsonl_path = ?',
     ).get(marker);
-    if (markerMissing) pendingMarkers.set(provider.name, marker);
-    const fullReindex = force || (markerMissing && sourceAlreadyIndexed(db, provider.name));
+    const fullReindex = force || (markerMissing && indexedSessions.length > 0);
+    if (markerMissing && indexedSessions.length > 0) {
+      replayKeys.set(provider.name, [...new Set(indexedSessions.map((session) => session.jsonlPath))]);
+    }
+    let inventoryComplete = true;
+    let reportedIssue: InventoryIssue | undefined;
     const units = provider.discover({
       lastCursor: fullReindex ? () => null : (key) => storedProviderCursor(db, key),
       changedPaths: fullReindex ? undefined : changedPaths,
+      indexedSessions: () => indexedSessions,
+      reportIncompleteInventory: (issue) => {
+        inventoryComplete = false;
+        reportedIssue ??= issue;
+      },
     });
+    if (!inventoryComplete) {
+      incompleteProviders.add(provider.name);
+      inventoryIssues.push({
+        provider: provider.name,
+        ...(reportedIssue ?? {
+          path: provider.descriptor.defaultRoot,
+          error: 'Source inventory is incomplete',
+        }),
+      });
+    }
+    if (
+      marker !== undefined
+      && (force || markerMissing)
+      && (inventoryComplete || indexedSessions.length > 0)
+    ) {
+      pendingMarkers.set(provider.name, marker);
+    }
     for (const unit of units) {
       items.push({
         provider,
@@ -55,7 +138,7 @@ export function createProviderIndexPlan(
       });
     }
   }
-  return { items, pendingMarkers };
+  return { items, pendingMarkers, replayKeys, incompleteProviders, inventoryIssues };
 }
 
 export function indexProviderPlan({
@@ -73,6 +156,7 @@ export function indexProviderPlan({
 }): ProviderIndexResult {
   const committed: ProviderIndexItem[] = [];
   const failedProviders = new Set<string>();
+  const failedItems: ProviderIndexItem[] = [];
   for (const item of plan.items) {
     try {
       const cursor = runTransaction(`provider:${item.provider.name}:${item.unit.key}`, () => (
@@ -82,12 +166,45 @@ export function indexProviderPlan({
       onCommitted(item, cursor);
     } catch (error) {
       failedProviders.add(item.provider.name);
+      failedItems.push(item);
       if (onError(error, item) === 'stop') {
-        return { committed, failedProviders, stopped: { item, error } };
+        return {
+          committed,
+          failedProviders,
+          failedItems,
+          complete: false,
+          stopped: { item, error },
+        };
       }
     }
   }
-  return { committed, failedProviders };
+  return {
+    committed,
+    failedProviders,
+    failedItems,
+    complete: failedItems.length === 0 && plan.incompleteProviders.size === 0,
+  };
+}
+
+/** Index every planned unit inside a caller-owned transaction, failing as one snapshot. */
+export function indexProviderPlanStrict({
+  db,
+  plan,
+  onCommitted = () => {},
+}: {
+  db: SqliteDb;
+  plan: ProviderIndexPlan;
+  onCommitted?: (item: ProviderIndexItem, cursor: Cursor) => void;
+}): ProviderIndexResult {
+  return indexProviderPlan({
+    db,
+    plan,
+    runTransaction: (_label, work) => work(),
+    onCommitted,
+    onError: (error, item) => {
+      throw new ProviderIndexFailure(error, item);
+    },
+  });
 }
 
 export function writeProviderIndexMarkers(
@@ -95,12 +212,26 @@ export function writeProviderIndexMarkers(
   plan: ProviderIndexPlan,
   result: ProviderIndexResult,
 ): void {
+  if (result.stopped !== undefined) return;
+  const retry = db.prepare('DELETE FROM index_state WHERE jsonl_path = ?');
   const write = db.prepare(
     'INSERT OR REPLACE INTO index_state (jsonl_path, mtime, lines_processed) VALUES (?, ?, 0)',
   );
-  for (const [provider, marker] of plan.pendingMarkers) {
-    if (!result.failedProviders.has(provider) && result.stopped === undefined) {
-      write.run(marker, Date.now());
+  const committed = new Set(result.committed.map(
+    (item) => `${item.provider.name}\0${item.unit.key}`,
+  ));
+  // A marker records that replay was scheduled. Per-unit cursors record which
+  // known sources completed it, so an incomplete inventory retries only the
+  // missing or failed sources instead of every readable sibling.
+  for (const [provider, keys] of plan.replayKeys) {
+    for (const key of keys) {
+      if (!committed.has(`${provider}\0${key}`)) retry.run(key);
     }
   }
+  for (const item of result.failedItems) {
+    if (plan.pendingMarkers.has(item.provider.name)) retry.run(item.unit.key);
+  }
+  for (const marker of plan.pendingMarkers.values()) {
+    write.run(marker, Date.now());
+  }
 }

packages/core/src/provider-settings.ts

@@ -0,0 +1,135 @@
+import { existsSync, readFileSync } from 'node:fs';
+import { homedir } from 'node:os';
+import { isAbsolute, join, normalize } from 'node:path';
+
+import {
+  createBuiltinProviderRegistry,
+  type BuiltinProviderRoots,
+} from './providers/builtins.ts';
+import {
+  createProviderRegistry,
+  type ProviderRegistry,
+} from './providers/registry.ts';
+
+export type PersistedProviderSettings = Record<string, unknown> & {
+  providerRoots?: Record<string, unknown>;
+};
+
+export interface ProviderSettingsReadResult {
+  readonly ok: boolean;
+  readonly settings: PersistedProviderSettings;
+  readonly error?: string;
+}
+
+function configuredPath(value: unknown, homeDir: string): string | null {
+  if (typeof value !== 'string' || value.trim().length === 0) return null;
+  const trimmed = value.trim();
+  const expanded = trimmed === '~'
+    ? homeDir
+    : trimmed.startsWith('~/') || trimmed.startsWith('~\\')
+      ? join(homeDir, trimmed.slice(2))
+      : trimmed;
+  return isAbsolute(expanded) ? normalize(expanded) : null;
+}
+
+export function resolveProviderRoots(
+  registry: ProviderRegistry,
+  persisted: PersistedProviderSettings = {},
+  { homeDir = homedir() }: { homeDir?: string } = {},
+): Record<string, string> {
+  if (
+    persisted.providerRoots !== undefined
+    && persisted.providerRoots !== null
+    && (typeof persisted.providerRoots !== 'object' || Array.isArray(persisted.providerRoots))
+  ) return {};
+  const configured = (
+    persisted.providerRoots !== null
+    && typeof persisted.providerRoots === 'object'
+    && !Array.isArray(persisted.providerRoots)
+  ) ? persisted.providerRoots : {};
+  return Object.fromEntries(registry.catalog().flatMap((descriptor) => {
+    const modernKey = descriptor.id;
+    const legacyKey = `${descriptor.id}Dir`;
+    const hasModern = Object.prototype.hasOwnProperty.call(configured, modernKey)
+      && configured[modernKey] !== null;
+    const hasLegacy = Object.prototype.hasOwnProperty.call(persisted, legacyKey)
+      && persisted[legacyKey] !== null;
+    if (hasModern || hasLegacy) {
+      const explicit = configuredPath(
+        hasModern ? configured[modernKey] : persisted[legacyKey],
+        homeDir,
+      );
+      return explicit === null ? [] : [[descriptor.id, explicit]];
+    }
+    return descriptor.requiresExplicitRoot ? [] : [[descriptor.id, descriptor.defaultRoot]];
+  }));
+}
+
+export function readPersistedProviderSettings(
+  settingsPath = join(homedir(), '.obelisk', 'settings.json'),
+): ProviderSettingsReadResult {
+  if (!existsSync(settingsPath)) return { ok: true, settings: {} };
+  try {
+    const parsed = JSON.parse(readFileSync(settingsPath, 'utf8')) as unknown;
+    if (parsed === null || typeof parsed !== 'object' || Array.isArray(parsed)) {
+      return { ok: false, settings: {}, error: `Obelisk settings are not an object: ${settingsPath}` };
+    }
+    const roots = (parsed as PersistedProviderSettings).providerRoots;
+    if (
+      roots !== undefined
+      && roots !== null
+      && (typeof roots !== 'object' || Array.isArray(roots))
+    ) {
+      return { ok: false, settings: {}, error: `Obelisk providerRoots are not an object: ${settingsPath}` };
+    }
+    return { ok: true, settings: parsed as PersistedProviderSettings };
+  } catch (error) {
+    return {
+      ok: false,
+      settings: {},
+      error: `Unable to read Obelisk settings at ${settingsPath}: ${error instanceof Error ? error.message : String(error)}`,
+    };
+  }
+}
+
+export function createConfiguredBuiltinProviderRuntime(
+  persisted: PersistedProviderSettings = {},
+  {
+    homeDir = homedir(),
+    cwd = process.cwd(),
+    baseRoots = {},
+  }: {
+    homeDir?: string;
+    cwd?: string;
+    baseRoots?: BuiltinProviderRoots;
+  } = {},
+): { roots: Record<string, string>; registry: ProviderRegistry } {
+  const defaults = createBuiltinProviderRegistry(baseRoots, { cwd });
+  const roots = resolveProviderRoots(defaults, persisted, { homeDir });
+  const configured = createBuiltinProviderRegistry({ ...baseRoots, ...roots }, { cwd });
+  return {
+    roots,
+    registry: createProviderRegistry(configured.list().map((provider) => {
+      if (roots[provider.name] !== undefined) return provider;
+      const reason = provider.descriptor.rootResolutionReason
+        ?? `Configured ${provider.name} root must be absolute or start with ~`;
+      return {
+        ...provider,
+        descriptor: {
+          ...provider.descriptor,
+          requiresExplicitRoot: true,
+          rootResolutionReason: reason,
+        },
+        watchRoots: () => [],
+        discover: (ctx) => {
+          ctx.reportIncompleteInventory?.({
+            path: provider.descriptor.defaultRoot,
+            error: reason,
+          });
+          return [];
+        },
+        raw: () => null,
+      };
+    })),
+  };
+}

packages/core/src/providers/builtins.ts

@@ -1,14 +1,19 @@
 import { createClaudeProvider } from './claude.ts';
 import { createCodexProvider } from './codex.ts';
 import { createKimiProvider } from './kimi.ts';
+import { createPiProvider } from './pi.ts';
 import { createProviderRegistry, type ProviderRegistry } from './registry.ts';
 
 export type BuiltinProviderRoots = Readonly<Record<string, string | undefined>>;
 
-export function createBuiltinProviderRegistry(roots: BuiltinProviderRoots = {}): ProviderRegistry {
+export function createBuiltinProviderRegistry(
+  roots: BuiltinProviderRoots = {},
+  { cwd }: { cwd?: string } = {},
+): ProviderRegistry {
   return createProviderRegistry([
     createClaudeProvider({ rootDir: roots['claude'] }),
     createCodexProvider({ rootDir: roots['codex'] }),
     createKimiProvider({ rootDir: roots['kimi'] }),
+    createPiProvider({ rootDir: roots['pi'], cwd }),
   ]);
 }

packages/core/src/providers/claude.ts

@@ -12,7 +12,7 @@ import { dirname, isAbsolute, join, normalize, relative } from 'node:path';
 
 import {
   extractText, extractContentType, extractMessageIsMeta, isSkillInstructions,
-  filePath, trunc, truncJson, readLines, discoverJsonlFiles, isDir,
+  filePath, trunc, truncJson, readLines, discoverJsonlFiles, isDir, sourceInventoryIssue,
 } from '../parsing.ts';
 
 import type {
@@ -61,6 +61,9 @@ function totalInputTokens(usage: Record<string, unknown>): number | null {
 
 function discoverAt(rootDir: string, ctx: DiscoverContext): IndexUnit[] {
   const projectsDir = join(rootDir, 'projects');
+  if (!existsSync(projectsDir) && (ctx.indexedSessions?.().length ?? 0) > 0) {
+    ctx.reportIncompleteInventory?.({ path: projectsDir, error: 'Source folder is unavailable' });
+  }
   const historyPath = normalize(join(rootDir, 'history.jsonl'));
   const historyTitles = new Map<string, string>();
   if (existsSync(historyPath)) {
@@ -95,7 +98,7 @@ function discoverAt(rootDir: string, ctx: DiscoverContext): IndexUnit[] {
       changedWorkflowPaths.add(absolute);
     }
   }
-  const transcriptUnits = discoverJsonlFiles(projectsDir).filter((file) => {
+  const transcriptUnits = discoverJsonlFiles(projectsDir, ctx.reportIncompleteInventory).filter((file) => {
     const normalizedPath = normalize(file.path);
     if (ctx.changedPaths !== undefined && !historyChanged && !changedTranscriptPaths.has(normalizedPath)) return false;
     const cursor = ctx.lastCursor(file.path);
@@ -118,18 +121,27 @@ function discoverAt(rootDir: string, ctx: DiscoverContext): IndexUnit[] {
   const workflowUnits: IndexUnit[] = [];
   if (!existsSync(projectsDir)) return transcriptUnits;
   let projects: string[];
-  try { projects = readdirSync(projectsDir); } catch { return transcriptUnits; }
+  try { projects = readdirSync(projectsDir); } catch (error) {
+    ctx.reportIncompleteInventory?.(sourceInventoryIssue(projectsDir, error));
+    return transcriptUnits;
+  }
   for (const project of projects) {
     const projectPath = join(projectsDir, project);
     if (!isDir(projectPath)) continue;
     let sessionIds: string[];
-    try { sessionIds = readdirSync(projectPath); } catch { continue; }
+    try { sessionIds = readdirSync(projectPath); } catch (error) {
+      ctx.reportIncompleteInventory?.(sourceInventoryIssue(projectPath, error));
+      continue;
+    }
     for (const sessionId of sessionIds) {
       const workflowDir = join(projectPath, sessionId, 'workflows');
       if (!isDir(workflowDir)) continue;
       const mainTranscriptPath = join(projectPath, `${sessionId}.jsonl`);
       let files: string[];
-      try { files = readdirSync(workflowDir); } catch { continue; }
+      try { files = readdirSync(workflowDir); } catch (error) {
+        ctx.reportIncompleteInventory?.(sourceInventoryIssue(workflowDir, error));
+        continue;
+      }
       for (const file of files) {
         if (!file.endsWith('.json')) continue;
         const workflowPath = join(workflowDir, file);

packages/core/src/providers/codex.ts

@@ -47,6 +47,9 @@ function messageVisibility(role: string, text: string | null): 'visible' | 'hidd
 
 function discoverAt(rootDir: string, ctx: DiscoverContext): IndexUnit[] {
   const sessionsDir = join(rootDir, 'sessions');
+  if (!existsSync(sessionsDir) && (ctx.indexedSessions?.().length ?? 0) > 0) {
+    ctx.reportIncompleteInventory?.({ path: sessionsDir, error: 'Source folder is unavailable' });
+  }
   const sessionIndexPath = normalize(join(rootDir, 'session_index.jsonl'));
   const sessionIndex = new Map<string, { title: string; updatedAt: string | null }>();
   if (existsSync(sessionIndexPath)) {
@@ -76,7 +79,7 @@ function discoverAt(rootDir: string, ctx: DiscoverContext): IndexUnit[] {
     if (!inside || inside.startsWith('..') || isAbsolute(inside)) continue;
     if (absolute.toLowerCase().endsWith('.jsonl')) changedFiles.add(absolute);
   }
-  return discoverCodexJsonlFiles(sessionsDir).flatMap((file) => {
+  return discoverCodexJsonlFiles(sessionsDir, ctx.reportIncompleteInventory).flatMap((file) => {
     if (ctx.changedPaths !== undefined && !sessionIndexChanged && !changedFiles.has(normalize(file.path))) return [];
     const cursor = ctx.lastCursor(file.path);
     const guardian = readCodexGuardianThreadInfo(file.path);
@@ -187,7 +190,7 @@ export function* parse(unit: IndexUnit, _cursor: Cursor): Generator<TranscriptRe
     out.push(rec);
     msgByUuid.set(uuid, rec);
     sm.lastMessageUuid = uuid;
-    if (!agentId) sm.n++;
+    if (!agentId && visibility === 'visible') sm.n++;
     if (type === 'assistant' && contentType === 'text') sm.lastTextAssistantUuid = uuid;
     updateBounds(timestamp);
     return uuid;

packages/core/src/providers/kimi.ts

@@ -7,7 +7,7 @@ import {
 import { homedir } from 'node:os';
 import { basename, dirname, isAbsolute, join, normalize, relative, sep } from 'node:path';
 
-import { filePath, projectSlugFromPath, trunc, truncJson } from '../parsing.ts';
+import { filePath, projectSlugFromPath, sourceInventoryIssue, trunc, truncJson } from '../parsing.ts';
 import type {
   Cursor,
   DiscoverContext,
@@ -17,6 +17,7 @@ import type {
   ProviderAdapter,
   RawLookup,
   RawRecord,
+  InventoryIssue,
   SubagentRecord,
   SummaryRecord,
   ToolCallRecord,
@@ -572,14 +573,31 @@ function projectSession(meta: KimiSessionUnitMeta, sessionId: string, state: Jso
   };
 }
 
-function sessionDirectories(rootDir: string): string[] {
+function sessionDirectories(
+  rootDir: string,
+  reportIssue?: (issue: InventoryIssue) => void,
+): string[] {
   const sessionsDir = join(rootDir, 'sessions');
   if (!existsSync(sessionsDir)) return [];
   const result: string[] = [];
-  for (const workspace of readdirSync(sessionsDir, { withFileTypes: true })) {
+  let workspaces;
+  try {
+    workspaces = readdirSync(sessionsDir, { withFileTypes: true });
+  } catch (error) {
+    reportIssue?.(sourceInventoryIssue(sessionsDir, error));
+    return result;
+  }
+  for (const workspace of workspaces) {
     if (!workspace.isDirectory()) continue;
     const workspaceDir = join(sessionsDir, workspace.name);
-    for (const session of readdirSync(workspaceDir, { withFileTypes: true })) {
+    let sessions;
+    try {
+      sessions = readdirSync(workspaceDir, { withFileTypes: true });
+    } catch (error) {
+      reportIssue?.(sourceInventoryIssue(workspaceDir, error));
+      continue;
+    }
+    for (const session of sessions) {
       if (session.isDirectory()) result.push(join(workspaceDir, session.name));
     }
   }
@@ -649,10 +667,14 @@ export function createKimiProvider({ rootDir = defaultKimiRoot() }: { rootDir?:
     watchRoots: (configuredRoot) => [join(configuredRoot, 'sessions'), join(configuredRoot, 'session_index.jsonl')],
     discover(ctx: DiscoverContext): IndexUnit[] {
       const units: IndexUnit[] = [];
+      const sessionsDir = join(rootDir, 'sessions');
+      if (!existsSync(sessionsDir) && (ctx.indexedSessions?.().length ?? 0) > 0) {
+        ctx.reportIncompleteInventory?.({ path: sessionsDir, error: 'Source folder is unavailable' });
+      }
       const changedSessions = ctx.changedPaths === undefined
         ? null
         : changedSessionDirectories(rootDir, ctx.changedPaths);
-      for (const sessionDir of sessionDirectories(rootDir)) {
+      for (const sessionDir of sessionDirectories(rootDir, ctx.reportIncompleteInventory)) {
         if (changedSessions !== null && !changedSessions.has(sessionDir)) continue;
         const statePath = join(sessionDir, 'state.json');
         const wireFiles = listWireFiles(sessionDir);

packages/core/src/providers/types.ts

@@ -37,6 +37,20 @@ export interface IndexUnit {
   agentId?: string;
   /** Adapter-private payload, opaque to the orchestration. */
   meta?: unknown;
+  /** Previously indexed sessions this unit atomically supersedes or retracts. */
+  retractSessionIds?: readonly string[];
+}
+
+/** Read-only source provenance exposed to one provider during discovery. */
+export interface IndexedSession {
+  sessionId: string;
+  jsonlPath: string;
+}
+
+/** One source location that prevented a provider from certifying its inventory. */
+export interface InventoryIssue {
+  readonly path: string;
+  readonly error: string;
 }
 
 /** Context the orchestration provides to discovery. */
@@ -45,6 +59,10 @@ export interface DiscoverContext {
   lastCursor(key: string): Cursor;
   /** When set (daemon changed-path mode), restrict discovery to these paths. */
   changedPaths?: string[];
+  /** Sessions already indexed for this provider, keyed by their canonical source path. */
+  indexedSessions?(): readonly IndexedSession[];
+  /** Report that the source inventory could not be enumerated completely. */
+  reportIncompleteInventory?(issue?: InventoryIssue): void;
 }
 
 /** Canonical language emitted by every provider adapter. Persist serializes it;
@@ -62,7 +80,7 @@ export type TranscriptRecord =
   | MessageTurnDurationRecord
   | DeleteSessionRecord;
 
-export type MessageVisibility = 'visible' | 'hidden';
+export type MessageVisibility = 'visible' | 'inactive' | 'hidden';
 
 export interface MessageRecord {
   kind: 'message';
@@ -75,7 +93,10 @@ export interface MessageRecord {
   text: string | null;
   content_type: string | null;
   is_meta: 0 | 1;
-  /** Provider-normalized display eligibility. Assemblers never infer this from text. */
+  /**
+   * Provider-attested evidence state. `inactive` requires an explicit source
+   * supersession signal; assemblers never infer it from text or tree shape.
+   */
   visibility: MessageVisibility;
   model: string | null;
   is_sidechain: 0 | 1;
@@ -116,6 +137,11 @@ export interface SummaryRecord {
   timestamp: string | null;
   source: string;
   content: string;
+  /** Provider-attested evidence state. Aggregate accounting may include non-visible usage. */
+  visibility?: MessageVisibility;
+  /** Provider-normalized total input, including provider-reported cached input. */
+  input_tokens?: number | null;
+  output_tokens?: number | null;
 }
 
 // One codex subagent. Like workflow_agent, a row can be contributed by more than
@@ -202,7 +228,8 @@ export interface DeleteSessionRecord {
 // a line-incremental adapter (claude) yields only new messages ('delta', persist
 // accumulates onto the existing row); a full-reparse adapter (codex) yields every
 // message each run ('total', persist replaces). A 'delta' parse from an empty
-// cursor is equivalent to 'total'.
+// cursor is equivalent to 'total'. The count describes the standard visible
+// transcript surface; records behind inactive/hidden visibility do not contribute.
 export interface SessionRecord {
   kind: 'session';
   id: string;
@@ -238,6 +265,10 @@ export interface ProviderDescriptor {
   readonly vendor: string;
   readonly defaultRoot: string;
   readonly color: string;
+  /** The automatic root is ambiguous; callers must preserve omission until the user chooses one. */
+  readonly requiresExplicitRoot?: boolean;
+  /** User-facing explanation for an unavailable automatic root. */
+  readonly rootResolutionReason?: string;
 }
 
 export interface RawLookup {
@@ -245,6 +276,8 @@ export interface RawLookup {
   readonly messageUuid: string;
   readonly session: Record<string, unknown> | null;
   readonly agentId: string | null;
+  /** Cursor committed for this session's source unit, when available. */
+  readonly cursor?: Cursor;
   readonly subagent?: Record<string, unknown> | null;
   readonly workflowAgent?: Record<string, unknown> | null;
 }

packages/core/src/query.ts

@@ -18,6 +18,7 @@ interface QueryOptions extends Record<string, any> {
   branch?: string;
   source?: string;
   includeMeta?: boolean;
+  includeInactive?: boolean;
   query?: string;
   projectLimit?: number;
   memoryLimit?: number;
@@ -74,6 +75,34 @@ function buildWhere(opts: QueryOptions, aliases: ColumnAliases) {
 
 const BASH_EXIT_PAT = 'Exit code %';
 
+type QueryVisibility = 'visible' | 'inactive' | 'hidden';
+
+function normalizedVisibility(value: unknown): QueryVisibility {
+  if (value === null || value === undefined || value === 'visible') return 'visible';
+  if (value === 'inactive') return 'inactive';
+  return 'hidden';
+}
+
+function withVisibility(row: DbRow): DbRow {
+  return { ...row, visibility: normalizedVisibility(row.visibility) };
+}
+
+function isQueryableMessage(
+  row: DbRow | undefined,
+  includeInactive = false,
+): row is DbRow {
+  if (row === undefined) return false;
+  const visibility = normalizedVisibility(row.visibility);
+  return visibility === 'visible' || (includeInactive && visibility === 'inactive');
+}
+
+function visibilitySql(alias: string, includeInactive = false): string {
+  const column = `${alias}.visibility`;
+  return includeInactive
+    ? `COALESCE(${column},'visible') IN ('visible','inactive')`
+    : `COALESCE(${column},'visible')='visible'`;
+}
+
 function assertReadOnlySql(sql: unknown): void {
   const text = String(sql || '').trim();
   if (!/^(SELECT|WITH)\b/i.test(text)) {
@@ -120,7 +149,17 @@ function createQueryApi(
   };
 
   const search = (text: string, opts: QueryOptions = {}) => {
-    const { limit = 20, sessionId, project, after, before, cwd, source, includeMeta = false } = opts;
+    const {
+      limit = 20,
+      sessionId,
+      project,
+      after,
+      before,
+      cwd,
+      source,
+      includeMeta = false,
+      includeInactive = false,
+    } = opts;
     let where = 'WHERE mf.text MATCH ?';
     const filterParams: any[] = [];
     if (sessionId) { where += ' AND mf.session_id=?'; filterParams.push(sessionId); }
@@ -130,8 +169,10 @@ function createQueryApi(
     if (cwd)       { where += ' AND m.cwd LIKE ?';     filterParams.push(cwd); }
     if (source && source !== 'all') { where += " AND COALESCE(m.source, s.source, 'claude')=?"; filterParams.push(source); }
     if (!includeMeta) where += ' AND COALESCE(m.is_meta,0)=0';
+    where += ` AND ${visibilitySql('m', includeInactive)}`;
     const stmt = db.prepare(`
-      SELECT m.uuid,m.session_id,m.text,m.content_type,m.is_meta,m.role,m.timestamp,m.model,m.cwd,m.source as m_source,
+      SELECT m.uuid,m.session_id,m.text,m.content_type,m.is_meta,m.role,m.timestamp,m.model,m.cwd,
+             COALESCE(m.visibility,'visible') AS visibility,m.source as m_source,
              s.id as s_id,s.title as s_title,s.project as s_project,s.started_at as s_started,
              s.source as s_source,
              rank
@@ -151,11 +192,31 @@ function createQueryApi(
     return rows.map((r: DbRow) => {
       const metaClause = includeMeta ? '' : 'AND COALESCE(is_meta,0)=0';
       const ctx = db.prepare(
-        `SELECT uuid,text,content_type,is_meta,role,timestamp,model,COALESCE(source, 'claude') as source FROM messages WHERE session_id=? AND uuid!=? ${metaClause} ORDER BY ABS(JULIANDAY(timestamp)-JULIANDAY(?)) LIMIT 6`
-      ).all(r.session_id, r.uuid, r.timestamp).sort((a: DbRow, b: DbRow) => a.timestamp < b.timestamp ? -1 : 1);
+        `SELECT uuid,text,content_type,is_meta,role,timestamp,model,
+                COALESCE(visibility,'visible') AS visibility,
+                COALESCE(source, 'claude') as source
+         FROM messages
+         WHERE session_id=? AND uuid!=? ${metaClause}
+           AND ${visibilitySql('messages', includeInactive)}
+         ORDER BY ABS(JULIANDAY(timestamp)-JULIANDAY(?))
+         LIMIT 6`
+      ).all(r.session_id, r.uuid, r.timestamp)
+        .map(withVisibility)
+        .sort((a: DbRow, b: DbRow) => a.timestamp < b.timestamp ? -1 : 1);
       const sourceValue = r.m_source || r.s_source || 'claude';
       return {
-        message: { uuid: r.uuid, text: r.text, content_type: r.content_type, is_meta: r.is_meta || 0, role: r.role, timestamp: r.timestamp, model: r.model, cwd: r.cwd, source: sourceValue },
+        message: {
+          uuid: r.uuid,
+          text: r.text,
+          content_type: r.content_type,
+          is_meta: r.is_meta || 0,
+          role: r.role,
+          timestamp: r.timestamp,
+          model: r.model,
+          cwd: r.cwd,
+          visibility: normalizedVisibility(r.visibility),
+          source: sourceValue,
+        },
         session: { id: r.s_id, title: r.s_title, project: r.s_project, started_at: r.s_started, source: r.s_source || sourceValue },
         rank: r.rank,
         context: ctx,
@@ -163,33 +224,49 @@ function createQueryApi(
     });
   };
 
-  const context = (uuid: string) => {
+  const context = (uuid: string, opts: QueryOptions = {}) => {
+    const includeInactive = opts.includeInactive === true;
     const msg = db.prepare('SELECT * FROM messages WHERE uuid=?').get(uuid);
-    if (!msg) return null;
+    if (!isQueryableMessage(msg, includeInactive)) return null;
+    const message = withVisibility(msg);
     const session = db.prepare('SELECT * FROM sessions WHERE id=?').get(msg.session_id);
     const chain: DbRow[] = [];
     let cur: DbRow | undefined = msg;
-    while (cur?.parent_uuid) { cur = db.prepare('SELECT * FROM messages WHERE uuid=?').get(cur.parent_uuid); if (cur) chain.unshift(cur); }
+    while (cur?.parent_uuid) {
+      cur = db.prepare('SELECT * FROM messages WHERE uuid=?').get(cur.parent_uuid);
+      if (isQueryableMessage(cur, includeInactive)) chain.unshift(withVisibility(cur));
+    }
     const subagent = msg.agent_id ? db.prepare('SELECT * FROM subagents WHERE agent_id=?').get(msg.agent_id) : null;
     let workflow = null;
     if (msg.agent_id) {
       const wa = db.prepare('SELECT * FROM workflow_agents WHERE agent_id=?').get(msg.agent_id);
       if (wa) workflow = db.prepare('SELECT * FROM workflows WHERE run_id=?').get(wa.run_id);
     }
-    return { message: msg, parentChain: chain, session, subagent, workflow };
+    return { message, parentChain: chain, session, subagent, workflow };
   };
 
-  const trace = (uuid: string) => {
+  const trace = (uuid: string, opts: QueryOptions = {}) => {
+    const includeInactive = opts.includeInactive === true;
     const chain: DbRow[] = [];
     let cur = db.prepare('SELECT * FROM messages WHERE uuid=?').get(uuid);
-    while (cur) { chain.unshift(cur); cur = cur.parent_uuid ? db.prepare('SELECT * FROM messages WHERE uuid=?').get(cur.parent_uuid) : undefined; }
+    if (!isQueryableMessage(cur, includeInactive)) return chain;
+    while (cur) {
+      if (isQueryableMessage(cur, includeInactive)) chain.unshift(withVisibility(cur));
+      cur = cur.parent_uuid ? db.prepare('SELECT * FROM messages WHERE uuid=?').get(cur.parent_uuid) : undefined;
+    }
     return chain;
   };
 
   const thread = (sid: string, opts: QueryOptions = {}) => {
     const includeMeta = opts?.includeMeta === true;
+    const includeInactive = opts?.includeInactive === true;
     const metaClause = includeMeta ? '' : 'AND COALESCE(is_meta,0)=0';
-    return db.prepare(`SELECT * FROM messages WHERE session_id=? ${metaClause} ORDER BY timestamp`).all(sid);
+    return db.prepare(`
+      SELECT * FROM messages
+      WHERE session_id=? ${metaClause}
+        AND ${visibilitySql('messages', includeInactive)}
+      ORDER BY timestamp
+    `).all(sid).map(withVisibility);
   };
 
   const subagents = (optsOrSid?: QueryOptions | string) => {
@@ -228,37 +305,73 @@ function createQueryApi(
   };
 
   const fileHistory = (fp: string, opts: QueryOptions = {}) => {
-    const { limit = 200, after, before, source } = opts;
-    let where = 'tc.file_path=?';
+    const { limit = 200, after, before, source, includeInactive = false } = opts;
+    let where = `tc.file_path=? AND ${visibilitySql('m', includeInactive)}`;
     const params: any[] = [fp];
     if (after)  { where += ' AND m.timestamp > ?'; params.push(after); }
     if (before) { where += ' AND m.timestamp < ?'; params.push(before); }
     if (source && source !== 'all') { where += " AND COALESCE(s.source, 'claude') = ?"; params.push(source); }
     params.push(limit);
     return db.prepare(
-      `SELECT tc.*,s.title as s_title,s.project as s_project,m.timestamp as ts FROM tool_calls tc LEFT JOIN sessions s ON s.id=tc.session_id LEFT JOIN messages m ON m.uuid=tc.message_uuid WHERE ${where} ORDER BY m.timestamp LIMIT ?`
+      `SELECT tc.*,s.title as s_title,s.project as s_project,m.timestamp as ts,
+              COALESCE(m.visibility,'visible') AS visibility
+       FROM tool_calls tc
+       LEFT JOIN sessions s ON s.id=tc.session_id
+       LEFT JOIN messages m ON m.uuid=tc.message_uuid
+       WHERE ${where}
+       ORDER BY m.timestamp
+       LIMIT ?`
     ).all(...params).map((r: DbRow) => ({
       toolCall: { id: r.id, message_uuid: r.message_uuid, name: r.name, input_json: r.input_json },
       session: { id: r.session_id, title: r.s_title, project: r.s_project },
       timestamp: r.ts,
+      visibility: normalizedVisibility(r.visibility),
     }));
   };
 
   const failures = (optsOrSid?: QueryOptions | string) => {
     const opts = normalizeOpts(optsOrSid);
     const { limit = 50 } = opts;
+    const includeInactive = opts.includeInactive === true;
     const needsJoin = opts.project || opts.branch || opts.source;
     const { where, params: filterParams } = buildWhere(opts, { sessionId: 'tr.session_id', project: 's.project', timestamp: 'rm.timestamp', branch: 's.git_branch', source: 's.source' });
     const join = needsJoin ? 'LEFT JOIN sessions s ON s.id=tr.session_id' : '';
-    const errorCond = `(tr.is_error = 1 OR tr.content LIKE '${BASH_EXIT_PAT}')`;
+    const errorCond = [
+      `(tr.is_error = 1 OR tr.content LIKE '${BASH_EXIT_PAT}')`,
+      visibilitySql('rm', includeInactive),
+      visibilitySql('cm', includeInactive),
+    ].join(' AND ');
     const allParams = [...filterParams, limit];
-    const rows = db.prepare(`SELECT tr.* FROM tool_results tr ${join} LEFT JOIN messages rm ON rm.uuid=tr.message_uuid WHERE ${errorCond} AND ${where} ORDER BY rm.timestamp DESC LIMIT ?`).all(...allParams);
+    const rows = db.prepare(`
+      SELECT tr.*, COALESCE(rm.visibility,'visible') AS visibility
+      FROM tool_results tr
+      JOIN messages rm ON rm.uuid=tr.message_uuid
+      JOIN tool_calls tc ON tc.id=tr.tool_use_id
+      JOIN messages cm ON cm.uuid=tc.message_uuid
+      ${join}
+      WHERE ${errorCond} AND ${where}
+      ORDER BY rm.timestamp DESC
+      LIMIT ?
+    `).all(...allParams);
     return rows.map((r: DbRow) => {
       const tc = db.prepare('SELECT * FROM tool_calls WHERE id=?').get(r.tool_use_id);
       const session = db.prepare('SELECT * FROM sessions WHERE id=?').get(r.session_id);
-      const rm = db.prepare('SELECT * FROM messages WHERE uuid=?').get(r.message_uuid);
-      const next = rm?.timestamp ? db.prepare('SELECT * FROM messages WHERE session_id=? AND timestamp>? ORDER BY timestamp LIMIT 3').all(r.session_id, rm.timestamp) : [];
-      return { toolCall: tc, result: r, session, nextMessages: next };
+      const rmRow = db.prepare('SELECT * FROM messages WHERE uuid=?').get(r.message_uuid);
+      const rm = rmRow === undefined ? undefined : withVisibility(rmRow);
+      const next = rm?.timestamp ? db.prepare(`
+        SELECT * FROM messages
+        WHERE session_id=? AND timestamp>?
+          AND ${visibilitySql('messages', includeInactive)}
+        ORDER BY timestamp
+        LIMIT 3
+      `).all(r.session_id, rm.timestamp).map(withVisibility) : [];
+      return {
+        toolCall: tc,
+        result: withVisibility(r),
+        session,
+        nextMessages: next,
+        visibility: normalizedVisibility(r.visibility),
+      };
     });
   };
 
@@ -275,9 +388,17 @@ function createQueryApi(
   const summaries = (optsOrSid?: QueryOptions | string) => {
     const opts = normalizeOpts(optsOrSid);
     const { limit = 100 } = opts;
+    const includeInactive = opts.includeInactive === true;
     const { where, params } = buildWhere(opts, { sessionId: 'su.session_id', project: 's.project', timestamp: 'su.timestamp', branch: 's.git_branch', source: 's.source' });
     params.push(limit);
-    return db.prepare(`SELECT su.*, s.title as session_title, s.project FROM summaries su LEFT JOIN sessions s ON s.id=su.session_id WHERE ${where} ORDER BY su.timestamp DESC LIMIT ?`).all(...params);
+    return db.prepare(`
+      SELECT su.*, s.title as session_title, s.project
+      FROM summaries su
+      LEFT JOIN sessions s ON s.id=su.session_id
+      WHERE ${where} AND ${visibilitySql('su', includeInactive)}
+      ORDER BY su.timestamp DESC
+      LIMIT ?
+    `).all(...params).map(withVisibility);
   };
 
   const overview = (optsOrScalar?: QueryOptions | string | number) => {
@@ -456,10 +577,13 @@ function createQueryApi(
     };
   };
 
-  const raw = (messageUuid: string, opts: { offset?: number; limit?: number } = {}) => {
-    const { offset = 0, limit = 10000 } = opts;
+  const raw = (
+    messageUuid: string,
+    opts: { offset?: number; limit?: number; includeInactive?: boolean } = {},
+  ) => {
+    const { offset = 0, limit = 10000, includeInactive = false } = opts;
     const message = db.prepare('SELECT * FROM messages WHERE uuid=?').get(messageUuid);
-    if (!message) return null;
+    if (!isQueryableMessage(message, includeInactive)) return null;
     const session = db.prepare('SELECT * FROM sessions WHERE id=?').get(message.session_id) ?? null;
     const subagent = message.agent_id
       ? db.prepare('SELECT * FROM subagents WHERE agent_id=?').get(message.agent_id) ?? null
@@ -468,11 +592,15 @@ function createQueryApi(
       ? db.prepare('SELECT * FROM workflow_agents WHERE agent_id=?').get(message.agent_id) ?? null
       : null;
     const source = message.source || session?.source || 'claude';
+    const cursorRow = typeof session?.jsonl_path === 'string'
+      ? db.prepare('SELECT cursor FROM index_state WHERE jsonl_path=?').get(session.jsonl_path)
+      : undefined;
     const record = providerRegistry.raw({
       source,
       messageUuid,
       session,
       agentId: message.agent_id || null,
+      cursor: typeof cursorRow?.cursor === 'string' ? cursorRow.cursor : null,
       subagent,
       workflowAgent,
     });
@@ -484,6 +612,7 @@ function createQueryApi(
       offset,
       limit,
       hasMore: offset + limit < totalLength,
+      visibility: normalizedVisibility(message.visibility),
     };
   };
 

packages/core/src/schema-migrations.ts

@@ -8,6 +8,10 @@ const COLUMN_MIGRATIONS = [
   ['messages', 'source', "TEXT DEFAULT 'claude'"],
   ['tool_calls', 'presentation', "TEXT DEFAULT 'default'"],
   ['workflows', 'parent_tool_use_id', 'TEXT'],
+  ['index_state', 'cursor', 'TEXT'],
+  ['summaries', 'visibility', "TEXT DEFAULT 'visible'"],
+  ['summaries', 'input_tokens', 'INTEGER'],
+  ['summaries', 'output_tokens', 'INTEGER'],
   ['memories', 'anchors', 'TEXT'],
   ['memories', 'deleted_at', 'TEXT'],
   ['memories', 'deleted_reason', 'TEXT'],

packages/core/src/schema.sql

@@ -30,10 +30,11 @@ CREATE TABLE IF NOT EXISTS workflow_agents (
   phase TEXT, label TEXT, model TEXT, state TEXT,
   duration_ms INTEGER, tokens INTEGER, tool_calls INTEGER);
 CREATE TABLE IF NOT EXISTS index_state (
-  jsonl_path TEXT PRIMARY KEY, mtime REAL, lines_processed INTEGER);
+  jsonl_path TEXT PRIMARY KEY, mtime REAL, lines_processed INTEGER, cursor TEXT);
 CREATE TABLE IF NOT EXISTS summaries (
   id TEXT PRIMARY KEY, session_id TEXT, timestamp TEXT,
-  source TEXT, content TEXT);
+  source TEXT, content TEXT, visibility TEXT DEFAULT 'visible',
+  input_tokens INTEGER, output_tokens INTEGER);
 CREATE VIRTUAL TABLE IF NOT EXISTS messages_fts USING fts5(
   uuid UNINDEXED, session_id UNINDEXED, text, content=messages, content_rowid=rowid);
 CREATE TRIGGER IF NOT EXISTS messages_fts_ai AFTER INSERT ON messages BEGIN

packages/core/src/session-detail.ts

@@ -1,5 +1,6 @@
 import type {
   TranscriptRecord,
+  MessageVisibility,
   MessageRecord,
   SessionRecord,
   SummaryRecord,
@@ -163,6 +164,9 @@ export interface SessionSummaryRow {
   [key: string]: unknown;
   id: string | number;
   session_id?: string | null;
+  visibility?: string | null;
+  input_tokens?: number | null;
+  output_tokens?: number | null;
 }
 
 export interface SessionDetailRows {
@@ -180,6 +184,12 @@ function withoutKind<T extends { kind: string }>(record: T): WithoutKind<T> {
   return value;
 }
 
+function canonicalVisibility(value: unknown): MessageVisibility {
+  if (value === null || value === undefined || value === 'visible') return 'visible';
+  if (value === 'inactive') return 'inactive';
+  return 'hidden';
+}
+
 function assembleMessages(
   messages: SessionDetailMessage[],
   toolCalls: ToolCallRecord[],
@@ -187,8 +197,19 @@ function assembleMessages(
   subagents: Extract<TranscriptRecord, { kind: 'subagent' }>[],
   workflows: SessionDetailWorkflow[],
 ): AssembledMessage[] {
+  const visibleMessageUuids = new Set(messages.map((message) => message.uuid));
+  const visibleToolCalls = toolCalls.filter((toolCall) => visibleMessageUuids.has(toolCall.message_uuid));
+  const visibleToolResults = toolResults.filter((result) => visibleMessageUuids.has(result.message_uuid));
+  const visibleCallIds = new Set(visibleToolCalls.map((toolCall) => toolCall.id));
+  const attachedResultMessageUuids = new Set(
+    visibleToolResults
+      .filter((result) => visibleCallIds.has(result.tool_use_id))
+      .map((result) => result.message_uuid),
+  );
   const resultsByCallId = new Map<string, SessionDetailToolResult>();
-  for (const result of toolResults) resultsByCallId.set(result.tool_use_id, withoutKind(result));
+  for (const result of visibleToolResults) {
+    resultsByCallId.set(result.tool_use_id, withoutKind(result));
+  }
 
   const subagentsByCallId = new Map<string, Extract<TranscriptRecord, { kind: 'subagent' }>>();
   for (const subagent of subagents) {
@@ -201,7 +222,7 @@ function assembleMessages(
       .filter((workflow) => workflow.parent_tool_use_id)
       .map((workflow) => [workflow.parent_tool_use_id as string, workflow]),
   );
-  for (const toolCall of toolCalls) {
+  for (const toolCall of visibleToolCalls) {
     const call: AssembledToolCall = {
       id: toolCall.id,
       name: toolCall.name,
@@ -234,7 +255,10 @@ function assembleMessages(
   const output: AssembledMessage[] = [];
   for (let index = 0; index < raw.length; index++) {
     const message = raw[index];
-    if (message.content_type === 'tool_result') continue;
+    if (
+      message.content_type === 'tool_result'
+      && attachedResultMessageUuids.has(message.uuid)
+    ) continue;
 
     if (message.type === 'assistant' && message.content_type === 'thinking') {
       const thinkingParts = [message.text ?? ''];
@@ -272,7 +296,10 @@ function assembleMessages(
       let nextIndex = index + 1;
       while (nextIndex < raw.length) {
         const next = raw[nextIndex];
-        if (next.content_type === 'tool_result') {
+        if (
+          next.content_type === 'tool_result'
+          && attachedResultMessageUuids.has(next.uuid)
+        ) {
           nextIndex++;
           continue;
         }
@@ -303,7 +330,10 @@ function assembleMessages(
       let nextIndex = index + 1;
       while (nextIndex < raw.length) {
         const next = raw[nextIndex];
-        if (next.content_type === 'tool_result') {
+        if (
+          next.content_type === 'tool_result'
+          && attachedResultMessageUuids.has(next.uuid)
+        ) {
           nextIndex++;
           continue;
         }
@@ -359,7 +389,7 @@ function assembleTranscriptRecords(records: Iterable<TranscriptRecord>): Session
         };
         break;
       case 'message': {
-        if (record.visibility === 'hidden') break;
+        if (canonicalVisibility(record.visibility) !== 'visible') break;
         const message: SessionDetailMessage = {
           uuid: record.uuid,
           type: record.type || record.role,
@@ -401,6 +431,7 @@ function assembleTranscriptRecords(records: Iterable<TranscriptRecord>): Session
         } as WorkflowAgentRecord);
         break;
       case 'summary':
+        if (canonicalVisibility(record.visibility) !== 'visible') break;
         summaries.push(withoutKind(record));
         break;
       case 'message-turn-duration': {
@@ -487,7 +518,7 @@ function sessionDetailRecordsFromRows(input: SessionDetailRows): TranscriptRecor
       text: typeof message.text === 'string' ? message.text : null,
       content_type: typeof message.content_type === 'string' ? message.content_type : null,
       is_meta: message.is_meta ? 1 : 0,
-      visibility: message.visibility === 'hidden' ? 'hidden' : 'visible',
+      visibility: canonicalVisibility(message.visibility),
       model: typeof message.model === 'string' ? message.model : null,
       is_sidechain: message.is_sidechain ? 1 : 0,
       agent_id: typeof message.agent_id === 'string' ? message.agent_id : null,
@@ -583,6 +614,9 @@ function sessionDetailRecordsFromRows(input: SessionDetailRows): TranscriptRecor
       timestamp: typeof summary.timestamp === 'string' ? summary.timestamp : null,
       source: typeof summary.source === 'string' ? summary.source : '',
       content: typeof summary.content === 'string' ? summary.content : '',
+      visibility: canonicalVisibility(summary.visibility),
+      input_tokens: typeof summary.input_tokens === 'number' ? summary.input_tokens : null,
+      output_tokens: typeof summary.output_tokens === 'number' ? summary.output_tokens : null,
     });
   }
   return records;