fix: coordinate sqlite index writers

Implement the full ADR-0006 plan: three-layer separation of transaction correctness, retry policy, and cross-process writer coordination. Layer 1 — scripts/tx.ts (transaction correctness): - runWriteTransaction executes work exactly once; no internal retry. - BEGIN IMMEDIATE takes the write lock up front (avoids SQLITE_BUSY_SNAPSHOT). - Guarded rollback: checks inTransaction() via adapter before attempting ROLLBACK; never masks the primary exception. - WriteTxDiagnostics attached to errors: phase, code, label, rollbackSucceeded, rollbackError, transactionActive. - Binding adapters (betterSqliteTransactionAdapter, nodeSqliteTransactionAdapter) mapping better-sqlite3's `.inTransaction` and node:sqlite's `.isTransaction`. - configureConnection centralizes WAL + synchronous + busy_timeout. Layer 2 — scripts/write-coordinator.ts (retry policy): - runRetryableWriteTransaction: bounded retry with total time budget. - Only retries when the transaction confirmed ended (transactionActive=false) and the error is SQLITE_BUSY during work/commit phase. - BEGIN-phase BUSY = abort entire build (isBeginBusyFailure); the caller returns `{ deferred: true, reason: 'writer_busy' }` instead of waiting. - hasUnusableTransaction detects a still-active transaction after failure; aborts the build immediately, never retries. Layer 3 — scripts/writer-lease.ts (cross-process coordination): - acquireWriterLease: dedicated writer.lock.sqlite with busy_timeout=0 + BEGIN IMMEDIATE. Non-blocking attempt; bounded wait with retryDelayMs. - writerLockPathFor derives lock path from the target DB path. - Lease held for the entire build; released on completion or failure. - Lock DB uses DELETE journal (not WAL); crash/close auto-releases. - All consumers obey: skill acquires at build start (returns deferred if unavailable); app daemon (via worker) acquires for its build cycle. Build semantics changes: - affectedSessionIds updated only after successful commit. - BuildIndexResult gains skipped/skippedFiles for observability. - Skill finalize failure now fails the build (was silently warned). - Checkpoint changed to PASSIVE (TRUNCATE reserved for maintenance/exit). - Skill buildIndex returns { deferred, reason } on lease contention; indexer-service reschedules the build (deferredRetryMs) without publishing a heartbeat (so the build-deferred state is visible to cross-process arbitration). - Service publishes heartbeat immediately on start() for correct arbitration. Tests: - tests/write-transaction.test.mjs: single-shot execution, diagnostics propagation, auto-rolled-back transaction detected, rollback failure captured as metadata, BEGIN IMMEDIATE semantics. - tests/writer-lease.test.mjs: acquire/release, contention returns null, bounded wait with release during budget. - tests/app-writer-lease.test.mjs: better-sqlite3 adapter integration. - tests/app-rollback-guard.test.mjs: rewritten — transient BUSY recovered by coordinator, persistent BUSY skips file, begin-busy aborts build, live-transaction aborts build, phantom affectedSessionIds prevented. - tests/daemon-arbitration.test.mjs: skill defers to fresh app heartbeat, builds when heartbeat is stale. - tests/app-indexer-service.test.mjs: new cases for deferred-retry scheduling and immediate heartbeat on start. - app/tests/electron-concurrency.mjs + child: dual-child IPC structure for real better-sqlite3 contention (holder acquires lock → build child starts → delayed release → result collected; persistent contention bounded). ADR-0006 updated to reflect the implemented design. Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
2026-07-11 00:42:43 +08:00
parent 44029676d4
commit e3e61cc7ab
25 changed files with 2173 additions and 471 deletions
@@ -9,6 +9,7 @@ import { writeHeartbeat } from './indexer.ts';
 import { createIndexerService } from './indexer-service.ts';
 import { createWorkerBuildIndex } from './indexer-worker-client.ts';
 import { buildRecapExportQuery } from './recap-capture-query.ts';
+import { acquireWriterLease, writerLockPathFor } from '../../../scripts/writer-lease.ts';

 const __dirname = path.dirname(fileURLToPath(import.meta.url));

@@ -41,6 +42,16 @@ let db;
 let indexerService;
 let indexerWorker;

+type WriterLeaseMode = 'acquire' | 'caller-held';
+
+function acquireAppWriterLease(dbPath: string, waitMs = 0) {
+  return acquireWriterLease({
+    lockPath: writerLockPathFor(dbPath),
+    openDb: lockPath => new Database(lockPath),
+    waitMs,
+  });
+}
+
 function getConfiguredClaudeDir() {
  const persisted = loadPersistedSettings();
  return persisted.claudeDir || DEFAULT_CLAUDE_DIR;
@@ -60,15 +71,25 @@ function getPathsForClaudeDir(claudeDir = getConfiguredClaudeDir(), codexDir = g
  };
 }

-function migrateLegacyDbIfNeeded(paths = getPathsForClaudeDir()) {
+function migrateLegacyDbIfNeeded(
+  paths = getPathsForClaudeDir(),
+  { writerLeaseMode = 'acquire' }: { writerLeaseMode?: WriterLeaseMode } = {},
+) {
  if (fs.existsSync(paths.dbPath)) return;
  const legacyDbPath = path.join(paths.claudeDir, 'obelisk.sqlite');
  if (!fs.existsSync(legacyDbPath)) return;
+  const lease = writerLeaseMode === 'acquire' ? acquireAppWriterLease(paths.dbPath) : null;
+  if (writerLeaseMode === 'acquire' && !lease) return false;
  try {
+    if (fs.existsSync(paths.dbPath)) return true;
    fs.mkdirSync(path.dirname(paths.dbPath), { recursive: true });
    fs.copyFileSync(legacyDbPath, paths.dbPath);
+    return true;
  } catch (error) {
    console.warn?.(`Obelisk legacy DB migration skipped: ${(error as Error).message}`);
+    return false;
+  } finally {
+    lease?.release();
  }
 }

@@ -151,15 +172,40 @@ function closeDb() {
  db = null;
 }

-function openDb(dbPath = getPathsForClaudeDir().dbPath) {
+function openDb(
+  dbPath = getPathsForClaudeDir().dbPath,
+  { writerLeaseMode = 'acquire' }: { writerLeaseMode?: WriterLeaseMode } = {},
+) {
  closeDb();
  if (!fs.existsSync(dbPath)) return null;
  db = new Database(dbPath, { readonly: false });
-  db.pragma('journal_mode = WAL');
-  migrateDb(db);
+  db.pragma('busy_timeout = 5000');
+  const lease = writerLeaseMode === 'acquire' ? acquireAppWriterLease(dbPath) : null;
+  if (writerLeaseMode === 'caller-held' || lease) {
+    try {
+      db.pragma('journal_mode = WAL');
+      migrateDb(db);
+    } finally {
+      lease?.release();
+    }
+  }
  return db;
 }

+function runAppDbWrite(work: () => void): boolean {
+  if (!db) return false;
+  const lease = acquireAppWriterLease(getPathsForClaudeDir().dbPath, 250);
+  if (!lease) {
+    throw new Error('Obelisk index writer is busy; memory change was not applied');
+  }
+  try {
+    work();
+    return true;
+  } finally {
+    lease.release();
+  }
+}
+
 function notifyIndexUpdated(result: { affectedSessionIds?: unknown } = {}) {
  const affectedSessionIds = Array.isArray(result.affectedSessionIds)
    ? [...new Set(result.affectedSessionIds.filter(Boolean))]
@@ -200,8 +246,14 @@ function startIndexerService({ buildOnStart = false } = {}) {
        projectsDir: paths.projectsDir,
        dbPath: paths.dbPath,
      });
-      openDb(paths.dbPath);
-      notifyIndexUpdated(result);
+      if (result?.deferred) {
+        if (Array.isArray(result.affectedSessionIds) && result.affectedSessionIds.length) {
+          notifyIndexUpdated(result);
+        }
+      } else {
+        openDb(paths.dbPath);
+        notifyIndexUpdated(result);
+      }
      return result;
    },
    writeHeartbeat: () => writeHeartbeat({ dbPath: paths.dbPath }),
@@ -520,16 +572,16 @@ ipcMain.handle('db:readMemoryFile', (_, filePath) => {
 });

 ipcMain.handle('db:archiveMemory', (_, id, reason) => {
-  if (!db) return false;
-  db.prepare(`UPDATE memories SET deleted_at = ?, deleted_reason = ? WHERE id = ?`)
-    .run(new Date().toISOString(), reason || 'Archived via panel', id);
-  return true;
+  return runAppDbWrite(() => {
+    db.prepare(`UPDATE memories SET deleted_at = ?, deleted_reason = ? WHERE id = ?`)
+      .run(new Date().toISOString(), reason || 'Archived via panel', id);
+  });
 });

 ipcMain.handle('db:restoreMemory', (_, id) => {
-  if (!db) return false;
-  db.prepare(`UPDATE memories SET deleted_at = NULL, deleted_reason = NULL WHERE id = ?`).run(id);
-  return true;
+  return runAppDbWrite(() => {
+    db.prepare(`UPDATE memories SET deleted_at = NULL, deleted_reason = NULL WHERE id = ?`).run(id);
+  });
 });

 ipcMain.handle('db:getProjects', (_, opts = {}) => {
@@ -830,8 +882,27 @@ ipcMain.handle('settings:rebuildIndex', async () => {
    indexerWorker = createWorkerBuildIndex();
  }
  cleanupDbFiles(tempDbPath);
+  let writerLease: ReturnType<typeof acquireWriterLease> = null;
  try {
-    migrateLegacyDbIfNeeded(paths);
+    const writerLeasePath = writerLockPathFor(paths.dbPath);
+    writerLease = acquireWriterLease({
+      lockPath: writerLeasePath,
+      openDb: lockPath => new Database(lockPath),
+      waitMs: 2000,
+    });
+    if (!writerLease) {
+      return {
+        files: 0,
+        latestSourceMtime: 0,
+        affectedSessionIds: [],
+        ftsRebuilt: false,
+        skipped: 0,
+        skippedFiles: [],
+        deferred: true,
+        reason: 'writer_busy',
+      };
+    }
+    migrateLegacyDbIfNeeded(paths, { writerLeaseMode: 'caller-held' });
    const result = await indexerWorker.buildIndex({
      reason: 'manual-rebuild',
      force: true,
@@ -840,21 +911,30 @@ ipcMain.handle('settings:rebuildIndex', async () => {
      projectsDir: paths.projectsDir,
      dbPath: tempDbPath,
      preserveDbPath: fs.existsSync(paths.dbPath) ? paths.dbPath : null,
+      writerLeasePath,
+      writerLeaseMode: 'caller-held',
    });
+    if (result?.deferred) return result;
    closeDb();
    replaceDbWithTemp(tempDbPath, paths.dbPath);
-    openDb(paths.dbPath);
+    openDb(paths.dbPath, { writerLeaseMode: 'caller-held' });
    notifyIndexUpdated(result);
    return result;
  } finally {
-    cleanupDbFiles(tempDbPath);
-    if (!db) {
-      try {
-        openDb(paths.dbPath);
-      } catch (error) {
-        console.warn?.(`Obelisk DB reopen after rebuild failed: ${(error as Error).message}`);
+    try {
+      cleanupDbFiles(tempDbPath);
+      if (!db) {
+        try {
+          openDb(paths.dbPath, {
+            writerLeaseMode: writerLease ? 'caller-held' : 'acquire',
+          });
+        } catch (error) {
+          console.warn?.(`Obelisk DB reopen after rebuild failed: ${(error as Error).message}`);
+        }
      }
+    } finally {
+      writerLease?.release();
+      if (shouldRestartWatcher) startIndexerService({ buildOnStart: false });
    }
-    if (shouldRestartWatcher) startIndexerService({ buildOnStart: false });
  }
 });
@@ -8,6 +8,7 @@ const DEFAULT_DEBOUNCE_MS = 2000;
 const DEFAULT_STABILITY_MS = 500;
 const DEFAULT_HEARTBEAT_MS = 30000;
 const DEFAULT_WATCH_RETRY_MS = 5000;
+const DEFAULT_DEFERRED_RETRY_MS = 250;

 type TimerHandle = ReturnType<typeof setTimeout>;

@@ -22,6 +23,15 @@ interface Watcher {
  close(): unknown;
 }

+interface IndexerBuildResult {
+  deferred?: boolean;
+}
+
+type IndexerBuild = (args: {
+  reason?: string;
+  changedPaths?: string[];
+}) => IndexerBuildResult | void | Promise<IndexerBuildResult | void>;
+
 interface IndexerServiceOptions {
  projectsDir?: string;
  watchDirs?: string | string[];
@@ -29,8 +39,9 @@ interface IndexerServiceOptions {
  stabilityMs?: number;
  heartbeatMs?: number;
  watchRetryMs?: number;
-  buildIndex?: (args: { reason?: string; changedPaths?: string[] }) => unknown;
-  writeHeartbeat?: () => void;
+  deferredRetryMs?: number;
+  buildIndex?: IndexerBuild;
+  writeHeartbeat?: () => unknown;
  watchProjects?: (onChange: (changedPath: string) => void) => Watcher | null;
  chokidar?: any;
  timers?: Timers;
@@ -44,6 +55,7 @@ function createIndexerService({
  stabilityMs = DEFAULT_STABILITY_MS,
  heartbeatMs = DEFAULT_HEARTBEAT_MS,
  watchRetryMs = DEFAULT_WATCH_RETRY_MS,
+  deferredRetryMs = DEFAULT_DEFERRED_RETRY_MS,
  buildIndex,
  writeHeartbeat = () => {},
  watchProjects,
@@ -100,6 +112,7 @@ function createIndexerService({
  let stabilityTimer: TimerHandle | null = null;
  let heartbeatTimer: TimerHandle | null = null;
  let watchRetryTimer: TimerHandle | null = null;
+  let deferredRetryTimer: TimerHandle | null = null;
  let watcher: Watcher | null = null;
  let stopped = false;
  let running = false;
@@ -124,6 +137,15 @@ function createIndexerService({
    return paths;
  };

+  const publishHeartbeat = () => {
+    try {
+      return writeHeartbeat();
+    } catch (error) {
+      logger.warn?.(`Obelisk heartbeat failed: ${(error as Error).message}`);
+      return false;
+    }
+  };
+
  const runBuildNow = (reason = "manual", paths: string[] | undefined = undefined) => {
    addChangedPath(paths);
    if (stopped) return idlePromise;
@@ -135,8 +157,18 @@ function createIndexerService({
    pending = false;
    const buildChangedPaths = takeChangedPaths();
    idlePromise = (async () => {
-      await buildIndex({ reason, changedPaths: buildChangedPaths });
-      writeHeartbeat();
+      const result = await buildIndex({ reason, changedPaths: buildChangedPaths });
+      if (result?.deferred) {
+        addChangedPath(buildChangedPaths);
+        if (!stopped && !deferredRetryTimer) {
+          deferredRetryTimer = timers.setTimeout(() => {
+            deferredRetryTimer = null;
+            runBuildNow('writer-lease');
+          }, deferredRetryMs);
+        }
+        return;
+      }
+      publishHeartbeat();
    })()
      .catch((error) => {
        // A build in flight when the service is stopped (e.g. a manual rebuild
@@ -158,6 +190,8 @@ function createIndexerService({
    addChangedPath(changedPath);
    lastReason = reason;
    if (running) pending = true;
+    if (deferredRetryTimer) timers.clearTimeout(deferredRetryTimer);
+    deferredRetryTimer = null;
    if (buildTimer) timers.clearTimeout(buildTimer);
    if (stabilityTimer) timers.clearTimeout(stabilityTimer);
    buildTimer = timers.setTimeout(() => {
@@ -186,15 +220,12 @@ function createIndexerService({

  const start = ({ buildOnStart = true } = {}) => {
    stopped = false;
+    publishHeartbeat();
    if (buildOnStart) scheduleBuild('startup');
    startWatching();
    if (typeof timers.setInterval === 'function') {
      heartbeatTimer = timers.setInterval(() => {
-        try {
-          writeHeartbeat();
-        } catch (error) {
-          logger.warn?.(`Obelisk heartbeat failed: ${(error as Error).message}`);
-        }
+        publishHeartbeat();
      }, heartbeatMs);
    }
  };
@@ -208,6 +239,8 @@ function createIndexerService({
    stabilityTimer = null;
    if (watchRetryTimer) timers.clearTimeout(watchRetryTimer);
    watchRetryTimer = null;
+    if (deferredRetryTimer) timers.clearTimeout(deferredRetryTimer);
+    deferredRetryTimer = null;
    if (heartbeatTimer && typeof timers.clearInterval === 'function') timers.clearInterval(heartbeatTimer);
    heartbeatTimer = null;
    if (watcher?.close) watcher.close();
@@ -6,6 +6,9 @@ import Database from 'better-sqlite3';
 import { parse as claudeParse } from '../../../scripts/providers/claude.ts';
 import { parse as codexParse } from '../../../scripts/providers/codex.ts';
 import { persist } from '../../../scripts/persist.ts';
+import { runWriteTransaction, configureConnection, betterSqliteTransactionAdapter } from '../../../scripts/tx.ts';
+import { acquireWriterLease, writerLockPathFor } from '../../../scripts/writer-lease.ts';
+import { runRetryableWriteTransaction, isBeginBusyFailure, hasUnusableTransaction } from '../../../scripts/write-coordinator.ts';
 import {
  inferProjectPath,
  isDir,
@@ -35,15 +38,6 @@ interface FileInfo {
  source?: string;
 }

-// A rollback inside a catch must never throw over the real error. SQLite
-// auto-rolls back certain failures (SQLITE_BUSY, disk full, ...); a following
-// explicit ROLLBACK then throws "cannot rollback - no transaction is active",
-// which would both mask the true cause and turn a skippable per-file error into
-// a whole-build failure. Swallow only the rollback's own error.
-function safeRollback(db: { exec: (sql: string) => unknown }) {
-  try { db.exec('ROLLBACK'); } catch { /* no active transaction */ }
-}
-
 function resolveSchemaPath() {
  const candidates = [
    path.join(__dirname, 'schema.sql'),
@@ -63,8 +57,7 @@ function installSchema(db, schemaPath = resolveSchemaPath()) {
 function openIndexDb({ dbPath = DEFAULT_DB_PATH, schemaPath = resolveSchemaPath(), DatabaseImpl = Database }: { dbPath?: string; schemaPath?: string; DatabaseImpl?: new (dbPath: string) => any } = {}) {
  fs.mkdirSync(path.dirname(dbPath), { recursive: true });
  const db = new DatabaseImpl(dbPath);
-  db.pragma('journal_mode = WAL');
-  db.pragma('synchronous = NORMAL');
+  configureConnection(db, { busyTimeoutMs: 250 });
  installSchema(db, schemaPath);
  return db;
 }
@@ -138,7 +131,10 @@ function normalizeChangedPath(projectsDir, changedPath) {
 }

 function jsonlFileInfoFromPath(projectsDir, changedPath) {
-  const fp = normalizeChangedPath(projectsDir, changedPath);
+  let fp = normalizeChangedPath(projectsDir, changedPath);
+  if (fp?.toLowerCase().endsWith('.meta.json')) {
+    fp = fp.slice(0, -'.meta.json'.length) + '.jsonl';
+  }
  if (!fp || !fp.endsWith('.jsonl')) return null;
  if (!fs.existsSync(fp)) return null;
  const rel = path.relative(projectsDir, fp);
@@ -365,14 +361,16 @@ function indexCodexSessionIndex(db, { codexDir = DEFAULT_CODEX_DIR } = {}) {
  const indexPath = path.join(codexDir, 'session_index.jsonl');
  if (!fs.existsSync(indexPath)) return;
  readLines(indexPath, (line) => {
+    let item;
    try {
-      const item = JSON.parse(line);
-      if (!item.id || !item.thread_name) return;
-      db.prepare('UPDATE sessions SET title=COALESCE(title, ?), ended_at=COALESCE(ended_at, ?) WHERE id=? AND source=?')
-        .run(item.thread_name, item.updated_at || null, codexDbId(item.id), 'codex');
+      item = JSON.parse(line);
    } catch (error) {
      console.warn(`Warning: malformed Codex session index line: ${(error as Error).message}`);
+      return;
    }
+    if (!item.id || !item.thread_name) return;
+    db.prepare('UPDATE sessions SET title=COALESCE(title, ?), ended_at=COALESCE(ended_at, ?) WHERE id=? AND source=?')
+      .run(item.thread_name, item.updated_at || null, codexDbId(item.id), 'codex');
  });
 }

@@ -392,22 +390,25 @@ function refreshSessionProjectPaths(db) {
 }

 function indexSubagentMeta(db, fi) {
-  if (!fi.isSubagent) return;
+  if (!fi.isSubagent) return false;
  const mp = fi.path.replace('.jsonl', '.meta.json');
-  if (!fs.existsSync(mp)) return;
+  if (!fs.existsSync(mp)) return false;
+  let meta;
  try {
-    const meta = JSON.parse(fs.readFileSync(mp, 'utf8'));
-    const tok = db.prepare('SELECT COALESCE(SUM(input_tokens),0)+COALESCE(SUM(output_tokens),0) as t FROM messages WHERE agent_id=?').get(fi.agentId);
-    const ts = db.prepare('SELECT MIN(timestamp) as t0, MAX(timestamp) as t1 FROM messages WHERE agent_id=?').get(fi.agentId);
-    const dur = ts?.t0 && ts?.t1 ? new Date(ts.t1).getTime() - new Date(ts.t0).getTime() : null;
-    if (fi.workflowRunId) {
-      db.prepare('INSERT OR REPLACE INTO workflow_agents (agent_id,run_id,session_id,agent_type,description) VALUES(?,?,?,?,?)').run(fi.agentId, fi.workflowRunId, fi.sessionId, meta.agentType||null, meta.description||null);
-    } else {
-      db.prepare('INSERT OR REPLACE INTO subagents VALUES(?,?,?,?,?,?,?)').run(fi.agentId, fi.sessionId, meta.toolUseId||null, meta.agentType||null, meta.description||null, dur, tok?.t||0);
-    }
+    meta = JSON.parse(fs.readFileSync(mp, 'utf8'));
  } catch (error) {
    console.warn(`Warning: failed to read subagent meta ${mp}: ${(error as Error).message}`);
+    return false;
  }
+  const tok = db.prepare('SELECT COALESCE(SUM(input_tokens),0)+COALESCE(SUM(output_tokens),0) as t FROM messages WHERE agent_id=?').get(fi.agentId);
+  const ts = db.prepare('SELECT MIN(timestamp) as t0, MAX(timestamp) as t1 FROM messages WHERE agent_id=?').get(fi.agentId);
+  const dur = ts?.t0 && ts?.t1 ? new Date(ts.t1).getTime() - new Date(ts.t0).getTime() : null;
+  if (fi.workflowRunId) {
+    db.prepare('INSERT OR REPLACE INTO workflow_agents (agent_id,run_id,session_id,agent_type,description) VALUES(?,?,?,?,?)').run(fi.agentId, fi.workflowRunId, fi.sessionId, meta.agentType||null, meta.description||null);
+  } else {
+    db.prepare('INSERT OR REPLACE INTO subagents VALUES(?,?,?,?,?,?,?)').run(fi.agentId, fi.sessionId, meta.toolUseId||null, meta.agentType||null, meta.description||null, dur, tok?.t||0);
+  }
+  return true;
 }

 function indexWorkflows(db, { projectsDir = DEFAULT_PROJECTS_DIR } = {}) {
@@ -426,23 +427,25 @@ function indexWorkflows(db, { projectsDir = DEFAULT_PROJECTS_DIR } = {}) {
      try { wfFiles = fs.readdirSync(wd); } catch { continue; }
      for (const f of wfFiles) {
        if (!f.endsWith('.json')) continue;
+        let wf;
        try {
-          const wf = JSON.parse(fs.readFileSync(path.join(wd, f), 'utf8'));
-          if (!wf.runId) continue;
-          const ac = db.prepare('SELECT COUNT(*) as c FROM workflow_agents WHERE run_id=?').get(wf.runId);
-          db.prepare('INSERT OR REPLACE INTO workflows (run_id,session_id,task_id,script,result_json,timestamp,agent_count,duration_ms,total_tokens,status,workflow_name) VALUES(?,?,?,?,?,?,?,?,?,?,?)').run(
-            wf.runId, sd, wf.taskId||null, wf.script||null,
-            wf.result ? JSON.stringify(wf.result) : null, wf.timestamp||null, ac?.c||0,
-            wf.durationMs||null, wf.totalTokens||null, wf.status||null, wf.workflowName||null);
-          const progress = wf.workflowProgress || [];
-          for (const item of progress) {
-            if (item.type !== 'workflow_agent' || !item.agentId) continue;
-            db.prepare('UPDATE workflow_agents SET phase=?, label=?, model=?, state=?, duration_ms=?, tokens=?, tool_calls=? WHERE agent_id=?').run(
-              item.phaseTitle||null, item.label||null, item.model||null, item.state||null,
-              item.durationMs||null, item.tokens||null, item.toolCalls||null, 'agent-' + item.agentId);
-          }
+          wf = JSON.parse(fs.readFileSync(path.join(wd, f), 'utf8'));
        } catch (error) {
-          console.warn(`Warning: failed to index workflow ${f}: ${(error as Error).message}`);
+          console.warn(`Warning: failed to read workflow ${f}: ${(error as Error).message}`);
+          continue;
+        }
+        if (!wf.runId) continue;
+        const ac = db.prepare('SELECT COUNT(*) as c FROM workflow_agents WHERE run_id=?').get(wf.runId);
+        db.prepare('INSERT OR REPLACE INTO workflows (run_id,session_id,task_id,script,result_json,timestamp,agent_count,duration_ms,total_tokens,status,workflow_name) VALUES(?,?,?,?,?,?,?,?,?,?,?)').run(
+          wf.runId, sd, wf.taskId||null, wf.script||null,
+          wf.result ? JSON.stringify(wf.result) : null, wf.timestamp||null, ac?.c||0,
+          wf.durationMs||null, wf.totalTokens||null, wf.status||null, wf.workflowName||null);
+        const progress = wf.workflowProgress || [];
+        for (const item of progress) {
+          if (item.type !== 'workflow_agent' || !item.agentId) continue;
+          db.prepare('UPDATE workflow_agents SET phase=?, label=?, model=?, state=?, duration_ms=?, tokens=?, tool_calls=? WHERE agent_id=?').run(
+            item.phaseTitle||null, item.label||null, item.model||null, item.state||null,
+            item.durationMs||null, item.tokens||null, item.toolCalls||null, 'agent-' + item.agentId);
        }
      }
    }
@@ -452,12 +455,14 @@ function indexWorkflows(db, { projectsDir = DEFAULT_PROJECTS_DIR } = {}) {
 function indexHistory(db, { historyPath = DEFAULT_HISTORY_PATH } = {}) {
  if (!fs.existsSync(historyPath)) return;
  readLines(historyPath, (line) => {
+    let item;
    try {
-      const o = JSON.parse(line);
-      if (o.sessionId && o.title) db.prepare('UPDATE sessions SET title=? WHERE id=? AND title IS NULL').run(o.title, o.sessionId);
+      item = JSON.parse(line);
    } catch (error) {
      console.warn(`Warning: malformed history line: ${(error as Error).message}`);
+      return;
    }
+    if (item.sessionId && item.title) db.prepare('UPDATE sessions SET title=? WHERE id=? AND title IS NULL').run(item.title, item.sessionId);
  });
 }

@@ -466,9 +471,13 @@ function rebuildFts(db) {
  db.exec("INSERT INTO memories_fts(memories_fts) VALUES('rebuild')");
 }

-function checkpointDb(db) {
+// PASSIVE by default: it checkpoints what it can without blocking concurrent
+// readers/writers, so it is safe to run after every build. A blocking TRUNCATE
+// (which reclaims the -wal file but needs exclusive access and can contend with
+// the daemon + queries) is reserved for maintenance/exit — pass mode explicitly.
+function checkpointDb(db, mode = 'PASSIVE') {
  try {
-    db.pragma('wal_checkpoint(TRUNCATE)');
+    db.pragma(`wal_checkpoint(${mode})`);
  } catch {}
 }

@@ -497,13 +506,30 @@ function writeIndexMarker(db, key, value = Date.now()) {
  db.prepare('INSERT OR REPLACE INTO index_state (jsonl_path, mtime, lines_processed) VALUES (?, ?, 0)').run(key, value);
 }

-function writeHeartbeat({ dbPath = DEFAULT_DB_PATH, DatabaseImpl = Database } = {}) {
+function writeHeartbeat({
+  dbPath = DEFAULT_DB_PATH,
+  writerLeasePath = writerLockPathFor(dbPath),
+  DatabaseImpl = Database,
+  LockDatabaseImpl = DatabaseImpl,
+} = {}) {
  if (!fs.existsSync(dbPath)) return;
-  const db = new DatabaseImpl(dbPath);
+  const lease = acquireWriterLease({
+    lockPath: writerLeasePath,
+    openDb: lockPath => new LockDatabaseImpl(lockPath),
+  });
+  if (!lease) return false;
  try {
-    writeIndexMarker(db, '__app_heartbeat__');
+    const db = new DatabaseImpl(dbPath);
+    configureConnection(db, { busyTimeoutMs: 0 });
+    const txDb = betterSqliteTransactionAdapter(db);
+    try {
+      runWriteTransaction(txDb, () => writeIndexMarker(db, '__app_heartbeat__'), { label: 'heartbeat' });
+      return true;
+    } finally {
+      db.close();
+    }
  } finally {
-    db.close();
+    lease.release();
  }
 }

@@ -515,9 +541,19 @@ interface BuildIndexOptions {
  dbPath?: string;
  schemaPath?: string;
  DatabaseImpl?: new (dbPath: string) => any;
+  LockDatabaseImpl?: new (dbPath: string) => any;
  force?: boolean;
  changedPaths?: string[];
  preserveDbPath?: string | null;
+  writerLeasePath?: string;
+  writerLeaseWaitMs?: number;
+  writerLeaseMode?: 'acquire' | 'caller-held';
+}
+
+interface SkippedFile {
+  path: string;
+  error: string;
+  diagnostics?: unknown;
 }

 interface BuildIndexResult {
@@ -525,6 +561,27 @@ interface BuildIndexResult {
  latestSourceMtime: number;
  affectedSessionIds: string[];
  ftsRebuilt: boolean;
+  skipped: number;
+  skippedFiles: SkippedFile[];
+  deferred: boolean;
+  reason?: string;
+}
+
+function deferredBuildResult(
+  reason: string,
+  overrides: Partial<Omit<BuildIndexResult, 'deferred' | 'reason'>> = {},
+): BuildIndexResult {
+  return {
+    files: 0,
+    latestSourceMtime: 0,
+    affectedSessionIds: [],
+    ftsRebuilt: false,
+    skipped: 0,
+    skippedFiles: [],
+    ...overrides,
+    deferred: true,
+    reason,
+  };
 }

 function buildIndex({
@@ -535,90 +592,175 @@ function buildIndex({
  dbPath = DEFAULT_DB_PATH,
  schemaPath = resolveSchemaPath(),
  DatabaseImpl = Database,
+  LockDatabaseImpl = DatabaseImpl,
  force = false,
  changedPaths = undefined,
  preserveDbPath = null,
+  writerLeasePath = writerLockPathFor(dbPath),
+  writerLeaseWaitMs = 2000,
+  writerLeaseMode = 'acquire',
 }: BuildIndexOptions = {}): BuildIndexResult {
-  const db = openIndexDb({ dbPath, schemaPath, DatabaseImpl });
-  let messageFtsTriggersDropped = false;
-  if (preserveDbPath && path.resolve(preserveDbPath) !== path.resolve(dbPath)) {
-    copyMemoriesFromDb(db, preserveDbPath);
+  if (writerLeaseMode !== 'acquire' && writerLeaseMode !== 'caller-held') {
+    throw new Error(`Unknown writer lease mode: ${writerLeaseMode}`);
  }
-  const files = [
-    ...discoverJsonlFiles({ projectsDir, changedPaths: force ? undefined : changedPaths }),
-    ...discoverCodexJsonlFiles({ codexDir, changedPaths: force ? undefined : changedPaths }),
-  ];
-  const latestSourceMtime = files.reduce((latest, file) => {
-    try {
-      return Math.max(latest, fs.statSync(file.path).mtimeMs);
-    } catch {
-      return latest;
+  let lease: ReturnType<typeof acquireWriterLease> = null;
+  if (writerLeaseMode === 'acquire') {
+    lease = acquireWriterLease({
+      lockPath: writerLeasePath,
+      openDb: lockPath => new LockDatabaseImpl(lockPath),
+      waitMs: writerLeaseWaitMs,
+    });
+    if (!lease) {
+      return deferredBuildResult('writer_busy');
    }
-  }, 0);
-
+  }
  try {
-    if (force) {
-      dropMessageFtsTriggers(db);
-      messageFtsTriggersDropped = true;
-      db.prepare("DELETE FROM index_state WHERE substr(jsonl_path, 1, 2) != '__'").run();
-      db.prepare("DELETE FROM messages").run();
-      db.prepare("DELETE FROM tool_calls").run();
-      db.prepare("DELETE FROM tool_results").run();
-      db.prepare("DELETE FROM sessions").run();
-      db.prepare("DELETE FROM summaries").run();
-      db.prepare("DELETE FROM subagents").run();
-      db.prepare("DELETE FROM workflows").run();
-      db.prepare("DELETE FROM workflow_agents").run();
-    }
-    const affectedSessionIds = new Set<string>();
-    if (Array.isArray(changedPaths)) {
-      for (const changedPath of changedPaths) {
-        const sessionId = sessionIdFromChangedPath(projectsDir, changedPath);
-        if (sessionId) affectedSessionIds.add(sessionId);
-      }
-    }
-    for (const file of files) {
-      db.exec('BEGIN');
-      try {
-        const indexed = file.source === 'codex' ? indexCodexFile(db, file) : indexClaudeFile(db, file);
-        if (indexed?.sessionId) affectedSessionIds.add(indexed.sessionId);
-        if (file.source !== 'codex') indexSubagentMeta(db, file);
-        db.exec('COMMIT');
-      } catch (error) {
-        safeRollback(db);
-        console.warn(`Warning: failed to index ${file.path}: ${(error as Error).message}`);
-      }
-    }
-    db.exec('BEGIN');
-    let ftsRebuilt = false;
+    const db = openIndexDb({ dbPath, schemaPath, DatabaseImpl });
+    const txDb = betterSqliteTransactionAdapter(db);
+    let messageFtsTriggersDropped = false;
    try {
-      indexWorkflows(db, { projectsDir });
-      refreshSessionProjectPaths(db);
-      indexHistory(db, { historyPath });
-      indexCodexSessionIndex(db, { codexDir });
-      if (messageFtsTriggersDropped) installSchema(db, schemaPath);
-      ftsRebuilt = ensureFtsReady(db, { force });
-      writeIndexMarker(db, '__last_build__');
-      writeIndexMarker(db, '__app_heartbeat__');
-      writeIndexMarker(db, '__app_last_successful_build__');
-      writeIndexMarker(db, '__indexer_owner_app__');
-      if (latestSourceMtime) writeIndexMarker(db, '__last_source_mtime__', latestSourceMtime);
-      db.exec('COMMIT');
-    } catch (error) {
-      safeRollback(db);
-      throw error;
-    }
-    return { files: files.length, latestSourceMtime, affectedSessionIds: [...affectedSessionIds], ftsRebuilt };
-  } finally {
-    if (messageFtsTriggersDropped) {
-      try {
-        installSchema(db, schemaPath);
-      } catch (error) {
-        console.warn(`Warning: failed to restore message FTS triggers: ${(error as Error).message}`);
+      if (preserveDbPath && path.resolve(preserveDbPath) !== path.resolve(dbPath)) {
+        copyMemoriesFromDb(db, preserveDbPath);
      }
+      const files = [
+        ...discoverJsonlFiles({ projectsDir, changedPaths: force ? undefined : changedPaths }),
+        ...discoverCodexJsonlFiles({ codexDir, changedPaths: force ? undefined : changedPaths }),
+      ];
+      const latestSourceMtime = files.reduce((latest, file) => {
+        try {
+          return Math.max(latest, fs.statSync(file.path).mtimeMs);
+        } catch {
+          return latest;
+        }
+      }, 0);
+
+      try {
+        if (force) {
+          runRetryableWriteTransaction(txDb, () => {
+            dropMessageFtsTriggers(db);
+            db.prepare("DELETE FROM index_state WHERE substr(jsonl_path, 1, 2) != '__'").run();
+            db.prepare("DELETE FROM messages").run();
+            db.prepare("DELETE FROM tool_calls").run();
+            db.prepare("DELETE FROM tool_results").run();
+            db.prepare("DELETE FROM sessions").run();
+            db.prepare("DELETE FROM summaries").run();
+            db.prepare("DELETE FROM subagents").run();
+            db.prepare("DELETE FROM workflows").run();
+            db.prepare("DELETE FROM workflow_agents").run();
+          }, { label: 'force-cleanup' });
+          messageFtsTriggersDropped = true;
+        }
+      } catch (error) {
+        if (isBeginBusyFailure(error)) {
+          return deferredBuildResult('database_busy', {
+            files: files.length,
+            latestSourceMtime,
+          });
+        }
+        throw error;
+      }
+      const affectedSessionIds = new Set<string>();
+      const finalizeAffectedSessionIds = new Set<string>();
+      const changedMetaJsonlPaths = new Set<string>();
+      if (Array.isArray(changedPaths)) {
+        for (const changedPath of changedPaths) {
+          const sessionId = sessionIdFromChangedPath(projectsDir, changedPath);
+          const normalizedChangedPath = normalizeChangedPath(projectsDir, changedPath);
+          const isMetaChange = normalizedChangedPath?.toLowerCase().endsWith('.meta.json');
+          if (isMetaChange && normalizedChangedPath) {
+            changedMetaJsonlPaths.add(
+              normalizedChangedPath.slice(0, -'.meta.json'.length) + '.jsonl',
+            );
+          }
+          // Transcript files report their session only after their own transaction
+          // commits. Workflow changes are applied during finalize, so stage those
+          // IDs until the finalize transaction commits. Meta files map back to their
+          // transcript transaction and are reported only after that commit.
+          if (sessionId && !changedPath.toLowerCase().endsWith('.jsonl') && !isMetaChange) {
+            finalizeAffectedSessionIds.add(sessionId);
+          }
+        }
+      }
+      const skipped: SkippedFile[] = [];
+      for (const file of files) {
+        try {
+          // The write is committed before affectedSessionIds is updated, so a
+          // failed/rolled-back file never reports a phantom updated session.
+          const indexed = runRetryableWriteTransaction(txDb, () => {
+            const result = file.source === 'codex' ? indexCodexFile(db, file) : indexClaudeFile(db, file);
+            const metaIndexed = file.source !== 'codex' && indexSubagentMeta(db, file);
+            if (!result?.sessionId && metaIndexed && changedMetaJsonlPaths.has(file.path)) {
+              return { sessionId: file.sessionId, path: file.path };
+            }
+            return result;
+          }, { label: `file:${file.path}` });
+          if (indexed?.sessionId) affectedSessionIds.add(indexed.sessionId);
+        } catch (error) {
+          if (isBeginBusyFailure(error)) {
+            return deferredBuildResult('database_busy', {
+              files: files.length,
+              latestSourceMtime,
+              affectedSessionIds: [...affectedSessionIds],
+              skipped: skipped.length,
+              skippedFiles: skipped,
+            });
+          }
+          if (hasUnusableTransaction(error)) throw error;
+          skipped.push({ path: file.path, error: (error as Error).message, diagnostics: (error as { obelisk?: unknown }).obelisk });
+          console.warn(`Warning: failed to index ${file.path}: ${(error as Error).message}`);
+        }
+      }
+      let ftsRebuilt = false;
+      // Finalize is one transaction; a failure here fails the whole build (the
+      // index would otherwise be left inconsistent).
+      try {
+        runRetryableWriteTransaction(txDb, () => {
+          indexWorkflows(db, { projectsDir });
+          refreshSessionProjectPaths(db);
+          indexHistory(db, { historyPath });
+          indexCodexSessionIndex(db, { codexDir });
+          if (messageFtsTriggersDropped) installSchema(db, schemaPath);
+          ftsRebuilt = ensureFtsReady(db, { force });
+          writeIndexMarker(db, '__last_build__');
+          writeIndexMarker(db, '__app_last_successful_build__');
+          writeIndexMarker(db, '__indexer_owner_app__');
+          if (latestSourceMtime) writeIndexMarker(db, '__last_source_mtime__', latestSourceMtime);
+        }, { label: 'finalize' });
+      } catch (error) {
+        if (isBeginBusyFailure(error)) {
+          return deferredBuildResult('database_busy', {
+            files: files.length,
+            latestSourceMtime,
+            affectedSessionIds: [...affectedSessionIds],
+            skipped: skipped.length,
+            skippedFiles: skipped,
+          });
+        }
+        throw error;
+      }
+      for (const sessionId of finalizeAffectedSessionIds) affectedSessionIds.add(sessionId);
+      return {
+        files: files.length,
+        latestSourceMtime,
+        affectedSessionIds: [...affectedSessionIds],
+        ftsRebuilt,
+        skipped: skipped.length,
+        skippedFiles: skipped,
+        deferred: false,
+      };
+    } finally {
+      if (messageFtsTriggersDropped) {
+        try {
+          installSchema(db, schemaPath);
+        } catch (error) {
+          console.warn(`Warning: failed to restore message FTS triggers: ${(error as Error).message}`);
+        }
+      }
+      checkpointDb(db);
+      db.close();
    }
-    checkpointDb(db);
-    db.close();
+  } finally {
+    lease?.release();
  }
 }