refactor(core): move shared runtime into workspace package
This commit is contained in:
tommy0103
@@ -0,0 +1,29 @@
|
||||
{
|
||||
"name": "@obelisk/core",
|
||||
"version": "0.1.0",
|
||||
"private": true,
|
||||
"type": "module",
|
||||
"description": "Shared indexing and query core for Obelisk transports.",
|
||||
"exports": {
|
||||
".": "./dist/core.js",
|
||||
"./db": "./dist/db.js",
|
||||
"./indexer": "./dist/indexer.js",
|
||||
"./parsing": "./dist/parsing.js",
|
||||
"./persist": "./dist/persist.js",
|
||||
"./providers/claude": "./dist/providers/claude.js",
|
||||
"./providers/codex": "./dist/providers/codex.js",
|
||||
"./providers/types": "./dist/providers/types.js",
|
||||
"./query": "./dist/query.js",
|
||||
"./tx": "./dist/tx.js",
|
||||
"./write-coordinator": "./dist/write-coordinator.js",
|
||||
"./writer-lease": "./dist/writer-lease.js"
|
||||
},
|
||||
"files": [
|
||||
"dist",
|
||||
"src/schema.sql"
|
||||
],
|
||||
"scripts": {
|
||||
"build": "rm -rf dist && tsc -p tsconfig.build.json",
|
||||
"typecheck": "tsc --noEmit -p tsconfig.json"
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,91 @@
|
||||
// Obelisk Core package (see docs/adr/0003-core-typescript-esm-precompiled.md).
|
||||
//
|
||||
// The single shared implementation behind every transport. runtime.mjs (skill),
|
||||
// and later the CLI and MCP server, are thin shells over these four functions;
|
||||
// none of them re-implement retrieval or own the DB lifecycle.
|
||||
//
|
||||
// Authored in TypeScript with erasable-only syntax so Node can run it directly
|
||||
// via type stripping in development, while the skill artifact ships the tsc
|
||||
// output (Phase 6). The heavy internals (db/indexer/query) remain .mjs for now
|
||||
// and are migrated in later phases; Core is the typed seam over them.
|
||||
|
||||
import { createContext, runInNewContext } from 'node:vm';
|
||||
|
||||
import { DB_PATH, openDb, openReadDb, openWriterLeaseDb } from './db.mjs';
|
||||
import { buildIndex, shouldSkipBuild } from './indexer.mjs';
|
||||
import { createQueryApi, createAttuneApi } from './query.mjs';
|
||||
import { acquireWriterLease, writerLockPathFor } from './writer-lease.ts';
|
||||
|
||||
export { buildIndex, DB_PATH };
|
||||
|
||||
type SandboxApi = Record<string, unknown>;
|
||||
|
||||
// Run a user-supplied CodeAct script inside the query/attune sandbox. The script
|
||||
// body runs as an async IIFE with a 30s timeout; its `return` value is resolved.
|
||||
function runInSandbox(api: SandboxApi, scriptContent: string): Promise<unknown> {
|
||||
const sandbox = {
|
||||
...api, JSON, Math, Array, Object, Set, Map, Date, RegExp,
|
||||
parseInt, parseFloat, String, Number, Boolean, Error, Promise, console, setTimeout,
|
||||
};
|
||||
const ctx = createContext(sandbox);
|
||||
return runInNewContext(`(async()=>{${scriptContent}})()`, ctx, { timeout: 30000 });
|
||||
}
|
||||
|
||||
// FTS search over indexed message text. Refreshes the index, then queries.
|
||||
export function searchText(text: string, opts?: Record<string, unknown>): unknown {
|
||||
buildIndex();
|
||||
const db = openReadDb();
|
||||
try {
|
||||
return createQueryApi(db).search(text, opts);
|
||||
} finally {
|
||||
db.close();
|
||||
}
|
||||
}
|
||||
|
||||
// Execute a read-only CodeAct query script and resolve its returned value.
|
||||
export async function executeQuery(scriptContent: string): Promise<unknown> {
|
||||
buildIndex();
|
||||
const db = openReadDb();
|
||||
try {
|
||||
return await runInSandbox(createQueryApi(db), scriptContent);
|
||||
} finally {
|
||||
db.close();
|
||||
}
|
||||
}
|
||||
|
||||
// Execute a memory-mutation CodeAct script (remember/forget only).
|
||||
export async function executeAttune(scriptContent: string): Promise<unknown> {
|
||||
const build = buildIndex() as { reason?: string } | undefined;
|
||||
if (build?.reason === 'daemon_active') {
|
||||
throw new Error('Obelisk daemon owns index writes; attune is read-only until the daemon stops');
|
||||
}
|
||||
if (build?.reason === 'writer_busy' || build?.reason === 'database_busy') {
|
||||
throw new Error('Obelisk index writer is busy; attune was not applied');
|
||||
}
|
||||
const lease = acquireWriterLease({
|
||||
lockPath: writerLockPathFor(DB_PATH),
|
||||
openDb: openWriterLeaseDb,
|
||||
waitMs: 1000,
|
||||
});
|
||||
if (!lease) throw new Error('Obelisk index writer is busy; attune was not applied');
|
||||
try {
|
||||
// Close the heartbeat TOCTOU window after acquiring the hard lease.
|
||||
const ownershipDb = openReadDb();
|
||||
try {
|
||||
const ownership = shouldSkipBuild(ownershipDb, { ignoreRecentBuild: true });
|
||||
if (ownership.reason === 'daemon_active') {
|
||||
throw new Error('Obelisk daemon owns index writes; attune is read-only until the daemon stops');
|
||||
}
|
||||
} finally {
|
||||
ownershipDb.close();
|
||||
}
|
||||
const db = openDb();
|
||||
try {
|
||||
return await runInSandbox(createAttuneApi(db), scriptContent);
|
||||
} finally {
|
||||
db.close();
|
||||
}
|
||||
} finally {
|
||||
lease.release();
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,78 @@
|
||||
// node:sqlite lifecycle and migrations for the Core package.
|
||||
import { createRequire } from 'node:module';
|
||||
import { CLAUDE_DIR, CODEX_DIR, TEXT_LIMIT, trunc, truncJson, extractText, extractContentType, extractMessageIsMeta, filePath, isDir, readLines } from './parsing.mjs';
|
||||
import { configureConnection } from './tx.ts';
|
||||
const require = createRequire(import.meta.url);
|
||||
const fs = require('node:fs');
|
||||
const path = require('node:path');
|
||||
const os = require('node:os');
|
||||
const { DatabaseSync } = require('node:sqlite');
|
||||
|
||||
const OBELISK_DIR = path.join(os.homedir(), '.obelisk');
|
||||
const LEGACY_DB_PATH = path.join(CLAUDE_DIR, 'obelisk.sqlite');
|
||||
const DB_PATH = path.join(OBELISK_DIR, 'obelisk.sqlite');
|
||||
const SCHEMA = fs.readFileSync(new URL('./schema.sql', import.meta.url), 'utf8');
|
||||
|
||||
function migrateLegacyDbIfNeeded() {
|
||||
if (fs.existsSync(DB_PATH)) return;
|
||||
if (!fs.existsSync(LEGACY_DB_PATH)) return;
|
||||
fs.mkdirSync(path.dirname(DB_PATH), { recursive: true });
|
||||
fs.copyFileSync(LEGACY_DB_PATH, DB_PATH);
|
||||
}
|
||||
|
||||
function openDb() {
|
||||
migrateLegacyDbIfNeeded();
|
||||
fs.mkdirSync(path.dirname(DB_PATH), { recursive: true });
|
||||
const db = new DatabaseSync(DB_PATH);
|
||||
configureConnection(db, { busyTimeoutMs: 250 });
|
||||
migrateExistingColumns(db);
|
||||
db.exec(SCHEMA);
|
||||
migrateDb(db);
|
||||
return db;
|
||||
}
|
||||
|
||||
// Queries and daemon-arbitration checks must never migrate/configure the index.
|
||||
// The caller is responsible for ensuring the database exists first.
|
||||
function openReadDb() {
|
||||
const db = new DatabaseSync(DB_PATH, { readOnly: true });
|
||||
db.exec('PRAGMA busy_timeout=250');
|
||||
return db;
|
||||
}
|
||||
|
||||
function openWriterLeaseDb(lockPath) {
|
||||
return new DatabaseSync(lockPath);
|
||||
}
|
||||
|
||||
function ensureColumn(db, table, column, definition) {
|
||||
const columns = db.prepare(`PRAGMA table_info(${table})`).all().map(c => c.name);
|
||||
if (!columns.includes(column)) db.exec(`ALTER TABLE ${table} ADD COLUMN ${column} ${definition}`);
|
||||
}
|
||||
|
||||
function tableExists(db, table) {
|
||||
return Boolean(db.prepare("SELECT name FROM sqlite_master WHERE type='table' AND name=?").get(table));
|
||||
}
|
||||
|
||||
function migrateExistingColumns(db) {
|
||||
if (tableExists(db, 'sessions')) ensureColumn(db, 'sessions', 'source', "TEXT DEFAULT 'claude'");
|
||||
if (tableExists(db, 'messages')) {
|
||||
ensureColumn(db, 'messages', 'content_type', 'TEXT');
|
||||
ensureColumn(db, 'messages', 'is_meta', 'INTEGER DEFAULT 0');
|
||||
ensureColumn(db, 'messages', 'source', "TEXT DEFAULT 'claude'");
|
||||
}
|
||||
if (tableExists(db, 'memories')) {
|
||||
ensureColumn(db, 'memories', 'anchors', 'TEXT');
|
||||
ensureColumn(db, 'memories', 'deleted_at', 'TEXT');
|
||||
ensureColumn(db, 'memories', 'deleted_reason', 'TEXT');
|
||||
}
|
||||
}
|
||||
|
||||
function migrateDb(db) {
|
||||
migrateExistingColumns(db);
|
||||
}
|
||||
|
||||
function rebuildMemoryFts(db) {
|
||||
db.exec("INSERT INTO memories_fts(memories_fts) VALUES('rebuild')");
|
||||
}
|
||||
|
||||
|
||||
export { CLAUDE_DIR, CODEX_DIR, OBELISK_DIR, DB_PATH, TEXT_LIMIT, openDb, openReadDb, openWriterLeaseDb, rebuildMemoryFts, trunc, truncJson, extractText, extractContentType, extractMessageIsMeta, filePath, isDir, readLines, fs, path, os };
|
||||
@@ -0,0 +1,287 @@
|
||||
// Passive-pull indexing orchestration for the Core package.
|
||||
import { DB_PATH, openDb, openReadDb, openWriterLeaseDb, rebuildMemoryFts } from './db.mjs';
|
||||
import {
|
||||
CLAUDE_DIR, CODEX_DIR, PROJECTS_DIR, fs, path, isDir, readLines,
|
||||
inferProjectPath, discoverJsonlFiles, discoverCodexJsonlFiles, codexDbId, readCodexGuardianThreadInfo,
|
||||
} from './parsing.mjs';
|
||||
import { persist } from './persist.ts';
|
||||
import { nodeSqliteTransactionAdapter } from './tx.ts';
|
||||
import { acquireWriterLease, writerLockPathFor } from './writer-lease.ts';
|
||||
import { runRetryableWriteTransaction, isBeginBusyFailure, hasUnusableTransaction } from './write-coordinator.ts';
|
||||
import { parse as claudeParse } from './providers/claude.ts';
|
||||
import { parse as codexParse } from './providers/codex.ts';
|
||||
|
||||
const HISTORY_PATH = path.join(CLAUDE_DIR, 'history.jsonl');
|
||||
|
||||
|
||||
function needsReindex(db, fp) {
|
||||
const mt = fs.statSync(fp).mtimeMs;
|
||||
const row = db.prepare('SELECT mtime, lines_processed FROM index_state WHERE jsonl_path = ?').get(fp);
|
||||
if (!row) return { needed: true, skip: 0 };
|
||||
return mt > row.mtime ? { needed: true, skip: row.lines_processed } : { needed: false, skip: 0 };
|
||||
}
|
||||
|
||||
|
||||
function indexCodexSessionIndex(db) {
|
||||
const indexPath = path.join(CODEX_DIR, 'session_index.jsonl');
|
||||
if (!fs.existsSync(indexPath)) return;
|
||||
readLines(indexPath, (line) => {
|
||||
let item;
|
||||
try {
|
||||
item = JSON.parse(line);
|
||||
} catch (e) {
|
||||
process.stderr.write(`Warning: malformed Codex session index line: ${e.message}\n`);
|
||||
return;
|
||||
}
|
||||
if (!item.id || !item.thread_name) return;
|
||||
db.prepare('UPDATE sessions SET title=COALESCE(title, ?), ended_at=COALESCE(ended_at, ?) WHERE id=? AND source=?')
|
||||
.run(item.thread_name, item.updated_at || null, codexDbId(item.id), 'codex');
|
||||
});
|
||||
}
|
||||
|
||||
function refreshSessionProjectPaths(db) {
|
||||
const sessions = db.prepare('SELECT id, project FROM sessions').all();
|
||||
const cwdStmt = db.prepare(`
|
||||
SELECT cwd
|
||||
FROM messages
|
||||
WHERE session_id = ? AND cwd IS NOT NULL AND cwd != ''
|
||||
ORDER BY timestamp IS NULL, timestamp
|
||||
`);
|
||||
const update = db.prepare('UPDATE sessions SET project_path = ? WHERE id = ?');
|
||||
for (const session of sessions) {
|
||||
const cwds = cwdStmt.all(session.id).map(row => row.cwd);
|
||||
const projectPath = inferProjectPath(session.project, cwds);
|
||||
if (projectPath) update.run(projectPath, session.id);
|
||||
}
|
||||
}
|
||||
|
||||
function indexSubagentMeta(db, fi) {
|
||||
if (!fi.isSubagent) return;
|
||||
const mp = fi.path.replace('.jsonl', '.meta.json');
|
||||
if (!fs.existsSync(mp)) return;
|
||||
let meta;
|
||||
try {
|
||||
meta = JSON.parse(fs.readFileSync(mp, 'utf8'));
|
||||
} catch (e) {
|
||||
process.stderr.write(`Warning: failed to read subagent meta ${mp}: ${e.message}\n`);
|
||||
return;
|
||||
}
|
||||
const tok = db.prepare('SELECT COALESCE(SUM(input_tokens),0)+COALESCE(SUM(output_tokens),0) as t FROM messages WHERE agent_id=?').get(fi.agentId);
|
||||
const ts = db.prepare('SELECT MIN(timestamp) as t0, MAX(timestamp) as t1 FROM messages WHERE agent_id=?').get(fi.agentId);
|
||||
const dur = ts?.t0 && ts?.t1 ? new Date(ts.t1).getTime() - new Date(ts.t0).getTime() : null;
|
||||
if (fi.workflowRunId) {
|
||||
db.prepare('INSERT OR REPLACE INTO workflow_agents (agent_id,run_id,session_id,agent_type,description) VALUES(?,?,?,?,?)').run(fi.agentId, fi.workflowRunId, fi.sessionId, meta.agentType||null, meta.description||null);
|
||||
} else {
|
||||
db.prepare('INSERT OR REPLACE INTO subagents VALUES(?,?,?,?,?,?,?)').run(fi.agentId, fi.sessionId, meta.toolUseId||null, meta.agentType||null, meta.description||null, dur, tok?.t||0);
|
||||
}
|
||||
}
|
||||
|
||||
function indexWorkflows(db) {
|
||||
if (!fs.existsSync(PROJECTS_DIR)) return;
|
||||
let projects;
|
||||
try { projects = fs.readdirSync(PROJECTS_DIR); } catch { return; }
|
||||
for (const proj of projects) {
|
||||
const pp = path.join(PROJECTS_DIR, proj);
|
||||
if (!isDir(pp)) continue;
|
||||
let entries;
|
||||
try { entries = fs.readdirSync(pp); } catch { continue; }
|
||||
for (const sd of entries) {
|
||||
const wd = path.join(pp, sd, 'workflows');
|
||||
if (!isDir(wd)) continue;
|
||||
let wfFiles;
|
||||
try { wfFiles = fs.readdirSync(wd); } catch { continue; }
|
||||
for (const f of wfFiles) {
|
||||
if (!f.endsWith('.json')) continue;
|
||||
let wf;
|
||||
try {
|
||||
wf = JSON.parse(fs.readFileSync(path.join(wd, f), 'utf8'));
|
||||
} catch (e) {
|
||||
process.stderr.write(`Warning: failed to read workflow ${f}: ${e.message}\n`);
|
||||
continue;
|
||||
}
|
||||
if (!wf.runId) continue;
|
||||
const ac = db.prepare('SELECT COUNT(*) as c FROM workflow_agents WHERE run_id=?').get(wf.runId);
|
||||
db.prepare('INSERT OR REPLACE INTO workflows (run_id,session_id,task_id,script,result_json,timestamp,agent_count,duration_ms,total_tokens,status,workflow_name) VALUES(?,?,?,?,?,?,?,?,?,?,?)').run(
|
||||
wf.runId, sd, wf.taskId||null, wf.script||null,
|
||||
wf.result ? JSON.stringify(wf.result) : null, wf.timestamp||null, ac?.c||0,
|
||||
wf.durationMs||null, wf.totalTokens||null, wf.status||null, wf.workflowName||null);
|
||||
const progress = wf.workflowProgress || [];
|
||||
for (const item of progress) {
|
||||
if (item.type !== 'workflow_agent' || !item.agentId) continue;
|
||||
db.prepare('UPDATE workflow_agents SET phase=?, label=?, model=?, state=?, duration_ms=?, tokens=?, tool_calls=? WHERE agent_id=?').run(
|
||||
item.phaseTitle||null, item.label||null, item.model||null, item.state||null,
|
||||
item.durationMs||null, item.tokens||null, item.toolCalls||null, 'agent-' + item.agentId);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
function indexHistory(db) {
|
||||
if (!fs.existsSync(HISTORY_PATH)) return;
|
||||
readLines(HISTORY_PATH, (line) => {
|
||||
let item;
|
||||
try {
|
||||
item = JSON.parse(line);
|
||||
} catch (e) {
|
||||
process.stderr.write(`Warning: malformed history line: ${e.message}\n`);
|
||||
return;
|
||||
}
|
||||
if (item.sessionId && item.title) db.prepare('UPDATE sessions SET title=? WHERE id=? AND title IS NULL').run(item.title, item.sessionId);
|
||||
});
|
||||
}
|
||||
|
||||
const BUILD_DEBOUNCE_MS = 30000;
|
||||
const APP_HEARTBEAT_FRESH_MS = 60000;
|
||||
|
||||
function shouldSkipBuild(db, { now = Date.now(), ignoreRecentBuild = false } = {}) {
|
||||
const appHeartbeat = db.prepare("SELECT mtime FROM index_state WHERE jsonl_path='__app_heartbeat__'").get();
|
||||
if (appHeartbeat && now - appHeartbeat.mtime < APP_HEARTBEAT_FRESH_MS) {
|
||||
return { skip: true, reason: 'daemon_active' };
|
||||
}
|
||||
if (!ignoreRecentBuild) {
|
||||
const last = db.prepare("SELECT mtime FROM index_state WHERE jsonl_path='__last_build__'").get();
|
||||
if (last && now - last.mtime < BUILD_DEBOUNCE_MS) {
|
||||
return { skip: true, reason: 'recent_build' };
|
||||
}
|
||||
}
|
||||
return { skip: false };
|
||||
}
|
||||
|
||||
function isMissingIndexStateTable(error) {
|
||||
const message = error instanceof Error ? error.message : String(error);
|
||||
return /no such table:\s*(?:main\.)?index_state\b/i.test(message);
|
||||
}
|
||||
|
||||
function inspectBuildOwnership({ force = false } = {}) {
|
||||
if (!fs.existsSync(DB_PATH)) return { skip: false };
|
||||
const db = openReadDb();
|
||||
try {
|
||||
return shouldSkipBuild(db, { ignoreRecentBuild: force });
|
||||
} catch (error) {
|
||||
// A missing table means the write path must initialize a new/legacy index.
|
||||
// Any other read failure leaves daemon ownership unknown, so fail closed.
|
||||
if (isMissingIndexStateTable(error)) return { skip: false };
|
||||
throw error;
|
||||
} finally {
|
||||
db.close();
|
||||
}
|
||||
}
|
||||
|
||||
// A one-shot record stream that retracts a session, for routing guardian sweeps
|
||||
// through persist (the single db writer) instead of deleting rows directly.
|
||||
function* guardianDelete(sessionId) {
|
||||
yield { kind: 'delete-session', sessionId };
|
||||
return null;
|
||||
}
|
||||
|
||||
function buildIndex({ force = false } = {}) {
|
||||
const ownership = inspectBuildOwnership({ force });
|
||||
if (ownership.skip) return ownership;
|
||||
const lease = acquireWriterLease({
|
||||
lockPath: writerLockPathFor(DB_PATH),
|
||||
openDb: openWriterLeaseDb,
|
||||
});
|
||||
if (!lease) return { skip: true, reason: 'writer_busy' };
|
||||
try {
|
||||
// Ownership may change between the first read and lease acquisition.
|
||||
const ownershipAfterLease = inspectBuildOwnership({ force });
|
||||
if (ownershipAfterLease.skip) return ownershipAfterLease;
|
||||
|
||||
const db = openDb();
|
||||
const txDb = nodeSqliteTransactionAdapter(db);
|
||||
const skippedFiles = [];
|
||||
try {
|
||||
try {
|
||||
if (force) {
|
||||
runRetryableWriteTransaction(txDb, () => {
|
||||
db.prepare("DELETE FROM index_state WHERE jsonl_path != '__last_build__'").run();
|
||||
// Clearing index_state alone re-indexes existing files but leaves rows for
|
||||
// files that no longer exist on disk (stale sessions accumulate). A force
|
||||
// build is a clean rebuild: drop every derived table, then re-index from the
|
||||
// current files. `memories` is the durable, human-approved layer and is never
|
||||
// cleared; messages_fts is repopulated by the 'rebuild' command in finalize.
|
||||
for (const table of ['messages', 'tool_calls', 'tool_results', 'sessions', 'summaries', 'subagents', 'workflows', 'workflow_agents']) {
|
||||
db.prepare(`DELETE FROM ${table}`).run();
|
||||
}
|
||||
}, { label: 'force-cleanup' });
|
||||
}
|
||||
} catch (error) {
|
||||
if (isBeginBusyFailure(error)) {
|
||||
return { skip: true, reason: 'database_busy', skipped: skippedFiles.length, skippedFiles };
|
||||
}
|
||||
throw error;
|
||||
}
|
||||
|
||||
const files = [
|
||||
...discoverJsonlFiles(),
|
||||
...discoverCodexJsonlFiles(),
|
||||
];
|
||||
for (const f of files) {
|
||||
try {
|
||||
runRetryableWriteTransaction(txDb, () => {
|
||||
if (f.source === 'codex') {
|
||||
// Codex goes through the pure adapter + shared persist (docs/adr/0001),
|
||||
// full-reparse (countMode 'total') when the file changed. An unchanged
|
||||
// file is not reparsed, but is still swept for stale guardian rows: a
|
||||
// guardian/auto-review thread must never linger in the index, even if it
|
||||
// was indexed before guardian detection removed it.
|
||||
const { needed } = needsReindex(db, f.path);
|
||||
if (needed) {
|
||||
persist(db, { key: f.path, sessionId: '' }, codexParse({ key: f.path, sessionId: '' }, null));
|
||||
} else {
|
||||
const guardian = readCodexGuardianThreadInfo(f.path);
|
||||
if (guardian) {
|
||||
persist(db, { key: f.path, sessionId: '' }, guardianDelete(codexDbId(guardian.threadRawId)));
|
||||
}
|
||||
}
|
||||
} else {
|
||||
// Claude transcripts now go through the pure adapter + shared persist
|
||||
// (docs/adr/0001). needsReindex keeps the "skip unchanged file" fast path;
|
||||
// the cursor's line count drives incremental resume inside parse().
|
||||
const { needed, skip } = needsReindex(db, f.path);
|
||||
if (needed) {
|
||||
const unit = { key: f.path, sessionId: f.sessionId, project: f.project, isSubagent: f.isSubagent, agentId: f.agentId };
|
||||
persist(db, unit, claudeParse(unit, skip > 0 ? `0:${skip}` : null));
|
||||
}
|
||||
indexSubagentMeta(db, f);
|
||||
}
|
||||
}, { label: `file:${f.path}` });
|
||||
} catch (e) {
|
||||
if (isBeginBusyFailure(e)) {
|
||||
return { skip: true, reason: 'database_busy', skipped: skippedFiles.length, skippedFiles };
|
||||
}
|
||||
if (hasUnusableTransaction(e)) throw e;
|
||||
// A per-file failure is skippable: log and move on.
|
||||
skippedFiles.push({ path: f.path, error: e.message, diagnostics: e.obelisk });
|
||||
process.stderr.write(`Warning: failed to index ${f.path}: ${e.message}\n`);
|
||||
}
|
||||
}
|
||||
// Finalize is one transaction and is NOT swallowed: a finalize failure fails
|
||||
// the build (a half-finalized index would be inconsistent).
|
||||
try {
|
||||
runRetryableWriteTransaction(txDb, () => {
|
||||
indexWorkflows(db);
|
||||
refreshSessionProjectPaths(db);
|
||||
indexHistory(db);
|
||||
indexCodexSessionIndex(db);
|
||||
db.exec("INSERT INTO messages_fts(messages_fts) VALUES('rebuild')");
|
||||
rebuildMemoryFts(db);
|
||||
db.prepare("INSERT OR REPLACE INTO index_state (jsonl_path, mtime, lines_processed) VALUES ('__last_build__', ?, 0)").run(Date.now());
|
||||
}, { label: 'finalize' });
|
||||
} catch (error) {
|
||||
if (isBeginBusyFailure(error)) {
|
||||
return { skip: true, reason: 'database_busy', skipped: skippedFiles.length, skippedFiles };
|
||||
}
|
||||
throw error;
|
||||
}
|
||||
return { skip: false, skipped: skippedFiles.length, skippedFiles };
|
||||
} finally {
|
||||
db.close();
|
||||
}
|
||||
} finally {
|
||||
lease.release();
|
||||
}
|
||||
}
|
||||
|
||||
export { buildIndex, inferProjectPath, refreshSessionProjectPaths, shouldSkipBuild };
|
||||
@@ -0,0 +1,324 @@
|
||||
// Core's pure parse/discover helpers — node:sqlite-free by construction, so the compiled
|
||||
// providers can be consumed by the app (better-sqlite3 / a Node without
|
||||
// node:sqlite). Moved verbatim from db.mjs and indexer.mjs (Phase 5d-1); they use
|
||||
// only node:fs/path/os. Kept as .mjs (plain ESM) for a low-risk verbatim move.
|
||||
import { createRequire } from 'node:module';
|
||||
const require = createRequire(import.meta.url);
|
||||
const fs = require('node:fs');
|
||||
const path = require('node:path');
|
||||
const os = require('node:os');
|
||||
|
||||
const CLAUDE_DIR = path.join(os.homedir(), '.claude');
|
||||
const CODEX_DIR = path.join(os.homedir(), '.codex');
|
||||
const PROJECTS_DIR = path.join(CLAUDE_DIR, 'projects');
|
||||
const CODEX_SESSIONS_DIR = path.join(CODEX_DIR, 'sessions');
|
||||
const TEXT_LIMIT = 10000;
|
||||
|
||||
// ---- from db.mjs (message/text helpers) ----
|
||||
function trunc(s) {
|
||||
return typeof s === 'string' && s.length > TEXT_LIMIT ? s.slice(0, TEXT_LIMIT) : s;
|
||||
}
|
||||
|
||||
function truncJson(obj, limit = TEXT_LIMIT) {
|
||||
if (obj === null || obj === undefined) return null;
|
||||
const walk = (v) => {
|
||||
if (typeof v === 'string') return v.length > limit ? v.slice(0, limit) + '...[truncated]' : v;
|
||||
if (Array.isArray(v)) return v.map(walk);
|
||||
if (typeof v === 'object' && v !== null) {
|
||||
const out = {};
|
||||
for (const [k, val] of Object.entries(v)) out[k] = walk(val);
|
||||
return out;
|
||||
}
|
||||
return v;
|
||||
};
|
||||
return JSON.stringify(walk(obj));
|
||||
}
|
||||
|
||||
function extractText(content) {
|
||||
if (typeof content === 'string') return trunc(content);
|
||||
if (!Array.isArray(content)) return null;
|
||||
const parts = [];
|
||||
for (const b of content) {
|
||||
if (b.type === 'text' && b.text) parts.push(b.text);
|
||||
else if (b.type === 'thinking' && b.thinking) parts.push(b.thinking);
|
||||
}
|
||||
return parts.length ? trunc(parts.join('\n')) : null;
|
||||
}
|
||||
|
||||
function extractContentType(content) {
|
||||
if (typeof content === 'string') return 'text';
|
||||
if (!Array.isArray(content) || !content.length) return 'unknown';
|
||||
const types = new Set();
|
||||
let sawUnknown = false;
|
||||
for (const b of content) {
|
||||
if (!b || typeof b !== 'object') { sawUnknown = true; continue; }
|
||||
if (b.type === 'text') types.add('text');
|
||||
else if (b.type === 'thinking') types.add('thinking');
|
||||
else if (b.type === 'tool_use') types.add('tool_use');
|
||||
else if (b.type === 'tool_result') types.add('tool_result');
|
||||
else sawUnknown = true;
|
||||
}
|
||||
return !sawUnknown && types.size === 1 ? [...types][0] : 'unknown';
|
||||
}
|
||||
|
||||
const COMMAND_ENVELOPE_RE = /^\s*(<command-name>[^<]+<\/command-name>|<(?:task-notification|system-reminder)\b|<local-command(?:\b|-))/;
|
||||
|
||||
function extractMessageIsMeta(record, text = extractText(record?.message?.content)) {
|
||||
const msg = record?.message || {};
|
||||
if (record?.isMeta === true || msg.isMeta === true) return 1;
|
||||
return typeof text === 'string' && COMMAND_ENVELOPE_RE.test(text) ? 1 : 0;
|
||||
}
|
||||
|
||||
function filePath(name, input) {
|
||||
if (!input) return null;
|
||||
return ['Read', 'Edit', 'Write', 'NotebookEdit'].includes(name) ? (input.file_path || null) : null;
|
||||
}
|
||||
|
||||
function isDir(p) { try { return fs.statSync(p).isDirectory(); } catch { return false; } }
|
||||
|
||||
function readLines(filePath, callback) {
|
||||
const fd = fs.openSync(filePath, 'r');
|
||||
const bufSize = 64 * 1024;
|
||||
const buf = Buffer.alloc(bufSize);
|
||||
let remainder = '';
|
||||
let bytesRead;
|
||||
try {
|
||||
while ((bytesRead = fs.readSync(fd, buf, 0, bufSize)) > 0) {
|
||||
const chunk = remainder + buf.toString('utf8', 0, bytesRead);
|
||||
const lines = chunk.split('\n');
|
||||
remainder = lines.pop();
|
||||
for (const line of lines) {
|
||||
if (line && callback(line) === false) return;
|
||||
}
|
||||
}
|
||||
if (remainder) callback(remainder);
|
||||
} finally {
|
||||
fs.closeSync(fd);
|
||||
}
|
||||
}
|
||||
|
||||
// ---- from indexer.mjs (project-path + discovery helpers) ----
|
||||
function legacyProjectPathFromSlug(project) {
|
||||
if (!project) return null;
|
||||
return '/' + project.replace(/-/g, '/').replace(/^\//, '');
|
||||
}
|
||||
|
||||
function normalizeObservedCwd(cwd) {
|
||||
if (typeof cwd !== 'string' || !cwd.trim() || !path.isAbsolute(cwd)) return null;
|
||||
return path.normalize(cwd);
|
||||
}
|
||||
|
||||
function projectSlugFromPath(projectPath) {
|
||||
const normalized = normalizeObservedCwd(projectPath);
|
||||
if (!normalized) return null;
|
||||
return '-' + normalized.replace(/^[\\/]+/, '').replace(/[\\/]+/g, '-');
|
||||
}
|
||||
|
||||
function inferProjectPath(project, observedCwds = []) {
|
||||
const byPath = new Map();
|
||||
for (const cwd of observedCwds) {
|
||||
const normalized = normalizeObservedCwd(cwd);
|
||||
if (!normalized) continue;
|
||||
const current = byPath.get(normalized) || { path: normalized, count: 0, first: byPath.size };
|
||||
current.count++;
|
||||
byPath.set(normalized, current);
|
||||
}
|
||||
const best = [...byPath.values()].sort((a, b) => b.count - a.count || a.first - b.first)[0];
|
||||
return best?.path || legacyProjectPathFromSlug(project);
|
||||
}
|
||||
|
||||
function discoverJsonlFiles() {
|
||||
const files = [];
|
||||
if (!fs.existsSync(PROJECTS_DIR)) return files;
|
||||
let projects;
|
||||
try { projects = fs.readdirSync(PROJECTS_DIR); } catch (e) { process.stderr.write(`Warning: cannot read projects dir: ${e.message}\n`); return files; }
|
||||
for (const proj of projects) {
|
||||
const projPath = path.join(PROJECTS_DIR, proj);
|
||||
if (!isDir(projPath)) continue;
|
||||
let entries;
|
||||
try { entries = fs.readdirSync(projPath); } catch { continue; }
|
||||
for (const f of entries) {
|
||||
if (f.endsWith('.jsonl'))
|
||||
files.push({ path: path.join(projPath, f), sessionId: f.slice(0, -6), project: proj, isSubagent: false });
|
||||
}
|
||||
for (const sd of entries) {
|
||||
const saDir = path.join(projPath, sd, 'subagents');
|
||||
if (!isDir(saDir)) continue;
|
||||
let saEntries;
|
||||
try { saEntries = fs.readdirSync(saDir); } catch { continue; }
|
||||
for (const sf of saEntries) {
|
||||
if (sf.endsWith('.jsonl'))
|
||||
files.push({ path: path.join(saDir, sf), sessionId: sd, project: proj, isSubagent: true, agentId: sf.slice(0, -6) });
|
||||
}
|
||||
const wfRoot = path.join(saDir, 'workflows');
|
||||
if (!isDir(wfRoot)) continue;
|
||||
let wfDirs;
|
||||
try { wfDirs = fs.readdirSync(wfRoot); } catch { continue; }
|
||||
for (const wfDir of wfDirs) {
|
||||
const wfPath = path.join(wfRoot, wfDir);
|
||||
if (!isDir(wfPath)) continue;
|
||||
let wfEntries;
|
||||
try { wfEntries = fs.readdirSync(wfPath); } catch { continue; }
|
||||
for (const wf of wfEntries) {
|
||||
if (wf.endsWith('.jsonl'))
|
||||
files.push({ path: path.join(wfPath, wf), sessionId: sd, project: proj, isSubagent: true, agentId: wf.slice(0, -6), workflowRunId: wfDir });
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
return files;
|
||||
}
|
||||
|
||||
function discoverCodexJsonlFiles() {
|
||||
const files = [];
|
||||
if (!fs.existsSync(CODEX_SESSIONS_DIR)) return files;
|
||||
const walk = (dir) => {
|
||||
let entries;
|
||||
try { entries = fs.readdirSync(dir, { withFileTypes: true }); } catch { return; }
|
||||
for (const entry of entries) {
|
||||
const fp = path.join(dir, entry.name);
|
||||
if (entry.isDirectory()) {
|
||||
walk(fp);
|
||||
} else if (entry.isFile() && entry.name.endsWith('.jsonl')) {
|
||||
files.push({ path: fp, source: 'codex' });
|
||||
}
|
||||
}
|
||||
};
|
||||
walk(CODEX_SESSIONS_DIR);
|
||||
return files;
|
||||
}
|
||||
|
||||
// ---- from indexer.mjs (codex pure helpers) ----
|
||||
function codexDbId(id) {
|
||||
if (!id) return null;
|
||||
const raw = String(id).replace(/^codex:/, '');
|
||||
return `codex:${raw}`;
|
||||
}
|
||||
|
||||
function codexRawId(id) {
|
||||
return id ? String(id).replace(/^codex:/, '') : null;
|
||||
}
|
||||
|
||||
function codexLineUuid(threadId, lineNum) {
|
||||
return `codex:${codexRawId(threadId)}:${String(lineNum).padStart(6, '0')}`;
|
||||
}
|
||||
|
||||
function codexCallId(callId) {
|
||||
if (!callId) return null;
|
||||
return `codex:${String(callId).replace(/^codex:/, '')}`;
|
||||
}
|
||||
|
||||
function codexParentThreadId(meta) {
|
||||
const subagent = meta?.source?.subagent;
|
||||
return subagent?.thread_spawn?.parent_thread_id
|
||||
|| meta?.forked_from_id
|
||||
|| subagent?.parent_thread_id
|
||||
|| null;
|
||||
}
|
||||
|
||||
function codexIsGuardianThread(meta, records = []) {
|
||||
const subagent = meta?.source?.subagent;
|
||||
if (subagent?.other === 'guardian') return true;
|
||||
if (meta?.thread_source !== 'subagent') return false;
|
||||
return records.some(({ obj }) => obj?.payload?.model === 'codex-auto-review' || obj?.model === 'codex-auto-review');
|
||||
}
|
||||
|
||||
function readCodexGuardianThreadInfo(filePath) {
|
||||
const records = [];
|
||||
let metaRecord = null;
|
||||
let lineNum = 0;
|
||||
readLines(filePath, (line) => {
|
||||
lineNum++;
|
||||
let obj;
|
||||
try {
|
||||
obj = JSON.parse(line);
|
||||
} catch {
|
||||
return;
|
||||
}
|
||||
records.push({ lineNum, obj });
|
||||
if (obj?.type === 'session_meta' && obj.payload?.id) {
|
||||
metaRecord = { lineNum, obj };
|
||||
if (obj.payload?.source?.subagent?.other === 'guardian') return false;
|
||||
if (obj.payload?.thread_source !== 'subagent') return false;
|
||||
}
|
||||
if (metaRecord && codexIsGuardianThread(metaRecord.obj.payload, records)) return false;
|
||||
});
|
||||
const meta = metaRecord?.obj?.payload;
|
||||
if (!meta || !codexIsGuardianThread(meta, records)) return null;
|
||||
return { threadRawId: codexRawId(meta.id), lineNum };
|
||||
}
|
||||
|
||||
function codexAgentNickname(meta) {
|
||||
return meta?.agent_nickname
|
||||
|| meta?.source?.subagent?.thread_spawn?.agent_nickname
|
||||
|| null;
|
||||
}
|
||||
|
||||
function codexAgentRole(meta) {
|
||||
return meta?.agent_role
|
||||
|| meta?.source?.subagent?.thread_spawn?.agent_role
|
||||
|| null;
|
||||
}
|
||||
|
||||
function parseCodexJsonInput(value) {
|
||||
if (value === null || value === undefined || value === '') return {};
|
||||
if (typeof value !== 'string') return value;
|
||||
try { return JSON.parse(value); } catch { return value; }
|
||||
}
|
||||
|
||||
function codexUsage(payload) {
|
||||
const usage = payload?.info?.last_token_usage || payload?.info?.total_token_usage || payload?.last_token_usage || null;
|
||||
if (!usage) return {};
|
||||
return {
|
||||
inputTokens: usage.input_tokens ?? null,
|
||||
outputTokens: usage.output_tokens ?? null,
|
||||
};
|
||||
}
|
||||
|
||||
function codexEventText(payload) {
|
||||
if (typeof payload?.message === 'string') return payload.message;
|
||||
if (Array.isArray(payload?.text_elements) && payload.text_elements.length) {
|
||||
const parts = payload.text_elements.map(item => typeof item === 'string' ? item : item?.text).filter(Boolean);
|
||||
if (parts.length) return parts.join('\n');
|
||||
}
|
||||
if (typeof payload?.text === 'string') return payload.text;
|
||||
return null;
|
||||
}
|
||||
|
||||
function codexMessagePayloadText(payload) {
|
||||
if (!Array.isArray(payload?.content)) return null;
|
||||
const parts = [];
|
||||
for (const block of payload.content) {
|
||||
if (typeof block?.text === 'string') parts.push(block.text);
|
||||
}
|
||||
return parts.length ? parts.join('\n') : null;
|
||||
}
|
||||
|
||||
function codexVisibleMessageKey(role, text) {
|
||||
return `${role || ''}\u0000${text || ''}`;
|
||||
}
|
||||
|
||||
function codexToolInput(payload) {
|
||||
if (payload?.type === 'custom_tool_call') return parseCodexJsonInput(payload.input);
|
||||
if (payload?.type === 'tool_search_call') return parseCodexJsonInput(payload.arguments);
|
||||
if (payload?.type === 'web_search_call') return { action: payload.action || null };
|
||||
return parseCodexJsonInput(payload?.arguments);
|
||||
}
|
||||
|
||||
function codexToolOutput(payload) {
|
||||
if (typeof payload?.output === 'string') return payload.output;
|
||||
if (payload?.output !== undefined) return JSON.stringify(payload.output);
|
||||
if (payload?.tools !== undefined) return JSON.stringify(payload.tools);
|
||||
if (payload?.execution !== undefined) return JSON.stringify(payload.execution);
|
||||
return null;
|
||||
}
|
||||
|
||||
export {
|
||||
fs, path, os, CLAUDE_DIR, CODEX_DIR, PROJECTS_DIR, CODEX_SESSIONS_DIR, TEXT_LIMIT,
|
||||
trunc, truncJson, extractText, extractContentType, extractMessageIsMeta, filePath, isDir, readLines,
|
||||
legacyProjectPathFromSlug, normalizeObservedCwd, projectSlugFromPath, inferProjectPath,
|
||||
discoverJsonlFiles, discoverCodexJsonlFiles,
|
||||
codexDbId, codexRawId, codexLineUuid, codexCallId, codexParentThreadId, codexIsGuardianThread,
|
||||
readCodexGuardianThreadInfo, codexAgentNickname, codexAgentRole, parseCodexJsonInput,
|
||||
codexUsage, codexEventText, codexMessagePayloadText, codexVisibleMessageKey, codexToolInput, codexToolOutput,
|
||||
};
|
||||
@@ -0,0 +1,124 @@
|
||||
// Shared Core persist layer (see docs/adr/0001).
|
||||
//
|
||||
// Provider-agnostic and binding-agnostic: it consumes the IndexRecord stream
|
||||
// from any adapter's parse() and writes rows into the injected database handle
|
||||
// (node:sqlite for the skill/CLI, better-sqlite3 for the app — they share the
|
||||
// prepare/run/get API). It is the ONLY layer that touches the database and the
|
||||
// only place that knows the schema. Adapters stay pure.
|
||||
//
|
||||
// Write semantics are the canonical ones reconciled from the drift: messages
|
||||
// upsert via ON CONFLICT; sessions merge with any existing row (started_at MIN,
|
||||
// ended_at MAX, message_count reset-or-accumulate, fill-if-null for the rest);
|
||||
// turn-duration is a targeted UPDATE; delete-session cascades. The generator's
|
||||
// return value is the new cursor, persisted verbatim into index_state.
|
||||
|
||||
import type { Cursor, IndexRecord, IndexUnit } from './providers/types.ts';
|
||||
|
||||
const minStr = (a: string | null, b: string | null) => (a == null ? b : b == null ? a : a < b ? a : b);
|
||||
const maxStr = (a: string | null, b: string | null) => (a == null ? b : b == null ? a : a > b ? a : b);
|
||||
|
||||
function statements(db: any) {
|
||||
return {
|
||||
msg: db.prepare(`
|
||||
INSERT INTO messages (uuid,session_id,type,parent_uuid,timestamp,role,text,content_type,is_meta,model,is_sidechain,agent_id,input_tokens,output_tokens,cwd,skill,source)
|
||||
VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)
|
||||
ON CONFLICT(uuid) DO UPDATE SET
|
||||
session_id=excluded.session_id, type=excluded.type, parent_uuid=excluded.parent_uuid,
|
||||
timestamp=excluded.timestamp, role=excluded.role, text=excluded.text,
|
||||
content_type=excluded.content_type, is_meta=excluded.is_meta, model=excluded.model,
|
||||
is_sidechain=excluded.is_sidechain, agent_id=excluded.agent_id,
|
||||
input_tokens=excluded.input_tokens, output_tokens=excluded.output_tokens,
|
||||
cwd=excluded.cwd, skill=excluded.skill, source=excluded.source`),
|
||||
tc: db.prepare('INSERT OR REPLACE INTO tool_calls (id,message_uuid,session_id,name,input_json,file_path) VALUES (?,?,?,?,?,?)'),
|
||||
tr: db.prepare('INSERT OR REPLACE INTO tool_results (tool_use_id,message_uuid,session_id,content,file_path,is_error) VALUES (?,?,?,?,?,?)'),
|
||||
sum: db.prepare('INSERT OR REPLACE INTO summaries (id,session_id,timestamp,source,content) VALUES (?,?,?,?,?)'),
|
||||
ses: db.prepare('INSERT OR REPLACE INTO sessions (id,title,project,project_path,started_at,ended_at,git_branch,version,message_count,jsonl_path,source) VALUES (?,?,?,?,?,?,?,?,?,?,?)'),
|
||||
sub: db.prepare(`
|
||||
INSERT INTO subagents (agent_id,session_id,parent_tool_use_id,agent_type,description,duration_ms,total_tokens)
|
||||
VALUES (?,?,?,?,?,?,?)
|
||||
ON CONFLICT(agent_id) DO UPDATE SET
|
||||
session_id=excluded.session_id,
|
||||
parent_tool_use_id=COALESCE(excluded.parent_tool_use_id, subagents.parent_tool_use_id),
|
||||
agent_type=COALESCE(excluded.agent_type, subagents.agent_type),
|
||||
description=COALESCE(excluded.description, subagents.description),
|
||||
duration_ms=COALESCE(excluded.duration_ms, subagents.duration_ms),
|
||||
total_tokens=COALESCE(excluded.total_tokens, subagents.total_tokens)`),
|
||||
turn: db.prepare('UPDATE messages SET turn_duration_ms=? WHERE uuid=?'),
|
||||
idx: db.prepare('INSERT OR REPLACE INTO index_state (jsonl_path,mtime,lines_processed) VALUES (?,?,?)'),
|
||||
getSession: db.prepare('SELECT * FROM sessions WHERE id=?'),
|
||||
};
|
||||
}
|
||||
|
||||
// Cascade-delete every row belonging to a session/thread (guardian retraction).
|
||||
function deleteSession(db: any, sessionId: string) {
|
||||
db.prepare('DELETE FROM tool_results WHERE session_id=? OR message_uuid IN (SELECT uuid FROM messages WHERE session_id=? OR agent_id=?)').run(sessionId, sessionId, sessionId);
|
||||
db.prepare('DELETE FROM tool_calls WHERE session_id=? OR message_uuid IN (SELECT uuid FROM messages WHERE session_id=? OR agent_id=?)').run(sessionId, sessionId, sessionId);
|
||||
db.prepare('DELETE FROM messages WHERE session_id=? OR agent_id=?').run(sessionId, sessionId);
|
||||
db.prepare('DELETE FROM subagents WHERE agent_id=? OR session_id=?').run(sessionId, sessionId);
|
||||
db.prepare('DELETE FROM summaries WHERE session_id=?').run(sessionId);
|
||||
db.prepare('DELETE FROM sessions WHERE id=?').run(sessionId);
|
||||
}
|
||||
|
||||
// Consume one unit's record stream into the database and return the new cursor
|
||||
// (also written to index_state). `db` is any SQLite handle sharing prepare/run.
|
||||
export function persist(db: any, unit: IndexUnit, gen: Generator<IndexRecord, Cursor>): Cursor {
|
||||
const st = statements(db);
|
||||
|
||||
const write = (r: IndexRecord) => {
|
||||
switch (r.kind) {
|
||||
case 'message':
|
||||
st.msg.run(r.uuid, r.session_id, r.type, r.parent_uuid, r.timestamp, r.role, r.text, r.content_type, r.is_meta, r.model, r.is_sidechain, r.agent_id, r.input_tokens, r.output_tokens, r.cwd, r.skill, r.source);
|
||||
break;
|
||||
case 'tool_call':
|
||||
st.tc.run(r.id, r.message_uuid, r.session_id, r.name, r.input_json, r.file_path);
|
||||
break;
|
||||
case 'tool_result':
|
||||
st.tr.run(r.tool_use_id, r.message_uuid, r.session_id, r.content, r.file_path, r.is_error);
|
||||
break;
|
||||
case 'summary':
|
||||
st.sum.run(r.id, r.session_id, r.timestamp, r.source, r.content);
|
||||
break;
|
||||
case 'subagent':
|
||||
st.sub.run(r.agent_id, r.session_id, r.parent_tool_use_id ?? null, r.agent_type ?? null, r.description ?? null, r.duration_ms ?? null, r.total_tokens ?? null);
|
||||
break;
|
||||
case 'message-turn-duration':
|
||||
st.turn.run(r.turn_duration_ms, r.uuid);
|
||||
break;
|
||||
case 'session': {
|
||||
const prev = st.getSession.get(r.id);
|
||||
// 'delta' accumulates onto the existing count (line-incremental adapters);
|
||||
// 'total' replaces it (full-reparse adapters).
|
||||
const message_count = r.countMode === 'delta' ? (prev?.message_count || 0) + r.message_count : r.message_count;
|
||||
st.ses.run(
|
||||
r.id,
|
||||
r.title ?? prev?.title ?? null,
|
||||
r.project ?? prev?.project ?? null,
|
||||
prev?.project_path ?? null, // authoritative project_path is set by refreshSessionProjectPaths
|
||||
minStr(prev?.started_at ?? null, r.started_at),
|
||||
maxStr(prev?.ended_at ?? null, r.ended_at),
|
||||
r.git_branch ?? prev?.git_branch ?? null,
|
||||
r.version ?? prev?.version ?? null,
|
||||
message_count,
|
||||
r.jsonl_path,
|
||||
r.source,
|
||||
);
|
||||
break;
|
||||
}
|
||||
case 'delete-session':
|
||||
deleteSession(db, r.sessionId);
|
||||
break;
|
||||
default:
|
||||
throw new Error(`persist: unhandled record kind ${(r as { kind: string }).kind}`);
|
||||
}
|
||||
};
|
||||
|
||||
let step = gen.next();
|
||||
while (!step.done) { write(step.value); step = gen.next(); }
|
||||
const cursor = step.value;
|
||||
|
||||
if (cursor != null) {
|
||||
const [mtime, lines] = cursor.split(':');
|
||||
st.idx.run(unit.key, Number(mtime), Number(lines));
|
||||
}
|
||||
return cursor;
|
||||
}
|
||||
@@ -0,0 +1,131 @@
|
||||
// Claude Code provider adapter in Core (see docs/adr/0001).
|
||||
//
|
||||
// Pure: discovers Claude transcript files and parses one into a record stream.
|
||||
// It never touches the Obelisk database. The per-line logic mirrors the original
|
||||
// indexJsonl exactly, but yields IndexRecords instead of writing rows; the shared
|
||||
// persist layer consumes them. Session aggregates here reflect only THIS chunk
|
||||
// (started_at/ended_at/message_count); persist merges them with any existing row.
|
||||
|
||||
import { createRequire } from 'node:module';
|
||||
const require = createRequire(import.meta.url);
|
||||
const fs = require('node:fs');
|
||||
|
||||
import {
|
||||
extractText, extractContentType, extractMessageIsMeta,
|
||||
filePath, trunc, truncJson, readLines, discoverJsonlFiles,
|
||||
} from '../parsing.mjs';
|
||||
|
||||
import type { Cursor, DiscoverContext, IndexRecord, IndexUnit, Provider } from './types.ts';
|
||||
|
||||
// Claude cursor encodes the file mtime and the number of lines already indexed:
|
||||
// "<mtimeMs>:<linesProcessed>". mtime lets discovery detect change; lines lets
|
||||
// parse resume without reprocessing.
|
||||
function cursorToSkip(cursor: Cursor): number {
|
||||
if (!cursor) return 0;
|
||||
const n = Number(cursor.split(':')[1]);
|
||||
return Number.isFinite(n) ? n : 0;
|
||||
}
|
||||
|
||||
export const name = 'claude';
|
||||
|
||||
export function discover(_ctx: DiscoverContext): IndexUnit[] {
|
||||
return discoverJsonlFiles().map((f: any) => ({
|
||||
key: f.path,
|
||||
sessionId: f.sessionId,
|
||||
project: f.project,
|
||||
isSubagent: f.isSubagent,
|
||||
agentId: f.agentId,
|
||||
meta: f.workflowRunId ? { workflowRunId: f.workflowRunId } : undefined,
|
||||
}));
|
||||
}
|
||||
|
||||
export function* parse(unit: IndexUnit, cursor: Cursor): Generator<IndexRecord, Cursor> {
|
||||
const skip = cursorToSkip(cursor);
|
||||
const mtime = fs.statSync(unit.key).mtimeMs;
|
||||
const isSubagent = unit.isSubagent === true;
|
||||
const records: IndexRecord[] = [];
|
||||
const sm = {
|
||||
started_at: null as string | null,
|
||||
ended_at: null as string | null,
|
||||
git_branch: null as string | null,
|
||||
version: null as string | null,
|
||||
title: null as string | null,
|
||||
n: 0,
|
||||
};
|
||||
|
||||
let lineNum = 0;
|
||||
readLines(unit.key, (line: string) => {
|
||||
lineNum++;
|
||||
if (lineNum <= skip) return;
|
||||
let obj: any;
|
||||
try { obj = JSON.parse(line); } catch { return; }
|
||||
const sid = unit.sessionId;
|
||||
const ts = obj.timestamp || null;
|
||||
|
||||
if (obj.type === 'ai-title' && obj.aiTitle) { sm.title = obj.aiTitle; return; }
|
||||
if (obj.type === 'system' && obj.subtype === 'away_summary' && obj.content) {
|
||||
records.push({ kind: 'summary', id: obj.uuid || `${sid}-away-${ts}`, session_id: sid, timestamp: ts, source: 'away_summary', content: obj.content });
|
||||
return;
|
||||
}
|
||||
if (obj.type === 'system' && obj.subtype === 'turn_duration' && obj.parentUuid && obj.durationMs) {
|
||||
records.push({ kind: 'message-turn-duration', uuid: obj.parentUuid, turn_duration_ms: obj.durationMs });
|
||||
return;
|
||||
}
|
||||
if (obj.type !== 'user' && obj.type !== 'assistant') return;
|
||||
|
||||
if (ts && (!sm.started_at || ts < sm.started_at)) sm.started_at = ts;
|
||||
if (ts && (!sm.ended_at || ts > sm.ended_at)) sm.ended_at = ts;
|
||||
if (obj.gitBranch) sm.git_branch = obj.gitBranch;
|
||||
if (obj.version) sm.version = obj.version;
|
||||
sm.n++;
|
||||
|
||||
const msg = obj.message || {};
|
||||
const text = extractText(msg.content);
|
||||
const contentType = extractContentType(msg.content);
|
||||
const isMeta = extractMessageIsMeta(obj, text);
|
||||
const usage = msg.usage || {};
|
||||
const aid = isSubagent ? (unit.agentId ?? null) : (obj.agentId || null);
|
||||
|
||||
if (obj.uuid) {
|
||||
records.push({
|
||||
kind: 'message', uuid: obj.uuid, session_id: sid, type: obj.type,
|
||||
parent_uuid: obj.parentUuid || null, timestamp: ts, role: msg.role || obj.type,
|
||||
text, content_type: contentType, is_meta: (isMeta ? 1 : 0), model: msg.model || null,
|
||||
is_sidechain: obj.isSidechain ? 1 : 0, agent_id: aid,
|
||||
input_tokens: usage.input_tokens || null, output_tokens: usage.output_tokens || null,
|
||||
cwd: obj.cwd || null, skill: obj.attributionSkill || null, source: 'claude',
|
||||
});
|
||||
}
|
||||
|
||||
if (obj.type === 'assistant' && Array.isArray(msg.content)) {
|
||||
for (const b of msg.content) {
|
||||
if (b.type === 'tool_use' && b.id)
|
||||
records.push({ kind: 'tool_call', id: b.id, message_uuid: obj.uuid, session_id: sid, name: b.name, input_json: truncJson(b.input || {}) as string, file_path: filePath(b.name, b.input) });
|
||||
}
|
||||
}
|
||||
|
||||
if (obj.type === 'user' && Array.isArray(msg.content)) {
|
||||
for (const b of msg.content) {
|
||||
if (b.type !== 'tool_result' || !b.tool_use_id) continue;
|
||||
const rt = typeof b.content === 'string' ? b.content
|
||||
: Array.isArray(b.content) ? b.content.map((c: any) => c.text || '').join('\n') : '';
|
||||
records.push({ kind: 'tool_result', tool_use_id: b.tool_use_id, message_uuid: obj.uuid, session_id: sid, content: trunc(rt), file_path: obj.toolUseResult?.filePath || null, is_error: b.is_error ? 1 : 0 });
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
// Subagent transcripts do not own a session row (matches indexJsonl).
|
||||
if (!isSubagent) {
|
||||
records.push({
|
||||
kind: 'session', id: unit.sessionId, title: sm.title, project: unit.project || null,
|
||||
started_at: sm.started_at, ended_at: sm.ended_at, git_branch: sm.git_branch,
|
||||
version: sm.version, message_count: sm.n, countMode: skip > 0 ? 'delta' : 'total',
|
||||
jsonl_path: unit.key, source: 'claude',
|
||||
});
|
||||
}
|
||||
|
||||
yield* records;
|
||||
return `${mtime}:${lineNum}`;
|
||||
}
|
||||
|
||||
export const claudeProvider: Provider = { name, discover, parse };
|
||||
@@ -0,0 +1,220 @@
|
||||
// Codex provider adapter in Core (see docs/adr/0001).
|
||||
//
|
||||
// Pure: discovers Codex rollout files and parses one into a record stream. It
|
||||
// never touches the Obelisk database. Unlike claude, codex is a FULL-REPARSE
|
||||
// adapter: it buffers every line and re-emits every record on each run, because
|
||||
// the event_msg ↔ response_item dedup needs whole-file (bidirectional) knowledge
|
||||
// (the matching pair sits ±1 line apart but in either order). Hence the session
|
||||
// record uses countMode 'total' (persist replaces the count, never accumulates).
|
||||
// The per-line logic mirrors the original indexCodexJsonl.
|
||||
|
||||
import { createRequire } from 'node:module';
|
||||
const require = createRequire(import.meta.url);
|
||||
const fs = require('node:fs');
|
||||
|
||||
import {
|
||||
trunc, truncJson, readLines,
|
||||
discoverCodexJsonlFiles, normalizeObservedCwd, projectSlugFromPath,
|
||||
codexRawId, codexDbId, codexCallId, codexLineUuid, codexParentThreadId,
|
||||
codexIsGuardianThread, codexAgentNickname, codexAgentRole, codexUsage,
|
||||
codexEventText, codexMessagePayloadText, codexVisibleMessageKey,
|
||||
codexToolInput, codexToolOutput,
|
||||
} from '../parsing.mjs';
|
||||
|
||||
import type { Cursor, DiscoverContext, IndexRecord, IndexUnit, MessageRecord, Provider } from './types.ts';
|
||||
|
||||
export const name = 'codex';
|
||||
|
||||
export function discover(_ctx: DiscoverContext): IndexUnit[] {
|
||||
return discoverCodexJsonlFiles().map((f: any) => ({ key: f.path, sessionId: '', meta: { source: 'codex' } }));
|
||||
}
|
||||
|
||||
export function* parse(unit: IndexUnit, _cursor: Cursor): Generator<IndexRecord, Cursor> {
|
||||
const mtime = fs.statSync(unit.key).mtimeMs;
|
||||
const records: { lineNum: number; obj: any }[] = [];
|
||||
let lineNum = 0;
|
||||
readLines(unit.key, (line: string) => {
|
||||
lineNum++;
|
||||
try { records.push({ lineNum, obj: JSON.parse(line) }); } catch { /* skip malformed */ }
|
||||
});
|
||||
const outCursor = `${mtime}:${lineNum}`;
|
||||
|
||||
const metaRecord = records.find(r => r.obj?.type === 'session_meta' && r.obj.payload?.id);
|
||||
if (!metaRecord) return outCursor;
|
||||
|
||||
const meta = metaRecord.obj.payload;
|
||||
const threadRawId = codexRawId(meta.id) as string;
|
||||
if (codexIsGuardianThread(meta, records)) {
|
||||
yield { kind: 'delete-session', sessionId: codexDbId(threadRawId) as string };
|
||||
return outCursor;
|
||||
}
|
||||
|
||||
const parentRawId = codexParentThreadId(meta);
|
||||
const sessionId = codexDbId(parentRawId || threadRawId) as string;
|
||||
const agentId = (parentRawId ? codexDbId(threadRawId) : null) as string | null;
|
||||
const isSidechain: 0 | 1 = agentId ? 1 : 0;
|
||||
const project = projectSlugFromPath(normalizeObservedCwd(meta.cwd));
|
||||
const lineUuid = (n: number): string => codexLineUuid(threadRawId, n) as string;
|
||||
|
||||
const out: IndexRecord[] = [];
|
||||
const msgByUuid = new Map<string, MessageRecord>();
|
||||
const sm = {
|
||||
started_at: (meta.timestamp || metaRecord.obj.timestamp || null) as string | null,
|
||||
ended_at: (meta.timestamp || metaRecord.obj.timestamp || null) as string | null,
|
||||
git_branch: (meta.git?.branch || null) as string | null,
|
||||
version: (meta.cli_version || null) as string | null,
|
||||
title: null as string | null,
|
||||
n: 0,
|
||||
lastMessageUuid: null as string | null,
|
||||
lastTextAssistantUuid: null as string | null,
|
||||
totalInputTokens: 0,
|
||||
totalOutputTokens: 0,
|
||||
};
|
||||
|
||||
let currentCwd = normalizeObservedCwd(meta.cwd);
|
||||
let currentModel: string | null = null;
|
||||
const eventMessageKeys = new Set<string>();
|
||||
const callMessageUuids = new Map<string, string>();
|
||||
|
||||
const updateBounds = (ts: string | null) => {
|
||||
if (!ts) return;
|
||||
if (!sm.started_at || ts < sm.started_at) sm.started_at = ts;
|
||||
if (!sm.ended_at || ts > sm.ended_at) sm.ended_at = ts;
|
||||
};
|
||||
|
||||
const insertMessage = ({ uuid, type, role, text = null, contentType = 'text', timestamp, isMeta = 0 }: {
|
||||
uuid: string; type: string; role: string; text?: string | null; contentType?: string; timestamp: string | null; isMeta?: 0 | 1;
|
||||
}) => {
|
||||
const rec: MessageRecord = {
|
||||
kind: 'message', uuid, session_id: sessionId, type, parent_uuid: sm.lastMessageUuid,
|
||||
timestamp: timestamp || null, role, text: trunc(text), content_type: contentType,
|
||||
is_meta: isMeta, model: currentModel, is_sidechain: isSidechain, agent_id: agentId,
|
||||
input_tokens: null, output_tokens: null, cwd: currentCwd, skill: null, source: 'codex',
|
||||
};
|
||||
out.push(rec);
|
||||
msgByUuid.set(uuid, rec);
|
||||
sm.lastMessageUuid = uuid;
|
||||
if (!agentId) sm.n++;
|
||||
if (type === 'assistant' && contentType === 'text') sm.lastTextAssistantUuid = uuid;
|
||||
updateBounds(timestamp);
|
||||
return uuid;
|
||||
};
|
||||
|
||||
// First pass: collect visible event_msg keys so duplicate response_items drop.
|
||||
for (const { obj } of records) {
|
||||
if (obj?.type !== 'event_msg') continue;
|
||||
const payload = obj.payload || {};
|
||||
if (payload.type !== 'user_message' && payload.type !== 'agent_message') continue;
|
||||
const text = codexEventText(payload);
|
||||
if (text === null) continue;
|
||||
eventMessageKeys.add(codexVisibleMessageKey(payload.type === 'user_message' ? 'user' : 'assistant', text));
|
||||
}
|
||||
|
||||
for (const { lineNum: currentLine, obj } of records) {
|
||||
const ts = obj.timestamp || null;
|
||||
if (obj.type === 'session_meta') {
|
||||
if (obj.payload?.cwd) currentCwd = normalizeObservedCwd(obj.payload.cwd) || currentCwd;
|
||||
if (obj.payload?.git?.branch) sm.git_branch = obj.payload.git.branch;
|
||||
if (obj.payload?.cli_version) sm.version = obj.payload.cli_version;
|
||||
updateBounds(obj.payload?.timestamp || ts);
|
||||
continue;
|
||||
}
|
||||
if (obj.type === 'turn_context') {
|
||||
currentCwd = normalizeObservedCwd(obj.payload?.cwd) || currentCwd;
|
||||
currentModel = obj.payload?.model || currentModel;
|
||||
updateBounds(ts);
|
||||
continue;
|
||||
}
|
||||
if (obj.type === 'event_msg') {
|
||||
const payload = obj.payload || {};
|
||||
if (payload.type === 'user_message' || payload.type === 'agent_message' || payload.type === 'agent_reasoning') {
|
||||
const text = codexEventText(payload);
|
||||
if (text === null) continue;
|
||||
const isReasoning = payload.type === 'agent_reasoning';
|
||||
insertMessage({
|
||||
uuid: lineUuid(currentLine),
|
||||
type: payload.type === 'user_message' ? 'user' : 'assistant',
|
||||
role: payload.type === 'user_message' ? 'user' : 'assistant',
|
||||
text, contentType: isReasoning ? 'thinking' : 'text', timestamp: ts,
|
||||
});
|
||||
continue;
|
||||
}
|
||||
if (payload.type === 'collab_agent_spawn_end' && payload.call_id && payload.new_thread_id) {
|
||||
const uuid = insertMessage({ uuid: lineUuid(currentLine), type: 'assistant', role: 'assistant', text: null, contentType: 'tool_use', timestamp: ts });
|
||||
const toolId = codexCallId(payload.call_id) as string;
|
||||
const description = payload.new_agent_nickname || payload.new_agent_role || 'Agent';
|
||||
const input = {
|
||||
description, subagent_type: payload.new_agent_role || 'Agent', prompt: payload.prompt || '',
|
||||
new_thread_id: payload.new_thread_id, model: payload.model || null, reasoning_effort: payload.reasoning_effort || null,
|
||||
};
|
||||
out.push({ kind: 'tool_call', id: toolId, message_uuid: uuid, session_id: sessionId, name: 'Agent', input_json: truncJson(input) as string, file_path: null });
|
||||
callMessageUuids.set(toolId, uuid);
|
||||
out.push({ kind: 'subagent', agent_id: codexDbId(payload.new_thread_id) as string, session_id: sessionId, parent_tool_use_id: toolId, agent_type: payload.new_agent_role || null, description });
|
||||
continue;
|
||||
}
|
||||
if (payload.type === 'task_complete') {
|
||||
if (sm.lastTextAssistantUuid && payload.duration_ms !== undefined) {
|
||||
out.push({ kind: 'message-turn-duration', uuid: sm.lastTextAssistantUuid, turn_duration_ms: payload.duration_ms || null });
|
||||
}
|
||||
updateBounds(ts);
|
||||
continue;
|
||||
}
|
||||
if (payload.type === 'token_count') {
|
||||
const usage = codexUsage(payload);
|
||||
if (usage.inputTokens !== null) sm.totalInputTokens = usage.inputTokens;
|
||||
if (usage.outputTokens !== null) sm.totalOutputTokens = usage.outputTokens;
|
||||
if (sm.lastTextAssistantUuid && (usage.inputTokens !== null || usage.outputTokens !== null)) {
|
||||
const rec = msgByUuid.get(sm.lastTextAssistantUuid);
|
||||
if (rec) { rec.input_tokens = usage.inputTokens; rec.output_tokens = usage.outputTokens; }
|
||||
}
|
||||
continue;
|
||||
}
|
||||
if (payload.type === 'thread_name_updated' && payload.thread_name) sm.title = payload.thread_name;
|
||||
continue;
|
||||
}
|
||||
if (obj.type !== 'response_item') continue;
|
||||
const payload = obj.payload || {};
|
||||
if (payload.type === 'message' && payload.role !== 'developer') {
|
||||
const text = codexMessagePayloadText(payload);
|
||||
const role = payload.role || 'assistant';
|
||||
if (text !== null && !eventMessageKeys.has(codexVisibleMessageKey(role, text))) {
|
||||
insertMessage({ uuid: lineUuid(currentLine), type: role === 'user' ? 'user' : 'assistant', role, text, contentType: 'text', timestamp: ts });
|
||||
}
|
||||
continue;
|
||||
}
|
||||
if (['function_call', 'custom_tool_call', 'tool_search_call', 'web_search_call'].includes(payload.type) && payload.call_id) {
|
||||
const uuid = insertMessage({ uuid: lineUuid(currentLine), type: 'assistant', role: 'assistant', text: null, contentType: 'tool_use', timestamp: ts });
|
||||
const name = payload.name || payload.tool || payload.type.replace(/_call$/, '');
|
||||
const toolId = codexCallId(payload.call_id) as string;
|
||||
out.push({ kind: 'tool_call', id: toolId, message_uuid: uuid, session_id: sessionId, name, input_json: truncJson(codexToolInput(payload)) as string, file_path: null });
|
||||
callMessageUuids.set(toolId, uuid);
|
||||
continue;
|
||||
}
|
||||
if (['function_call_output', 'custom_tool_call_output', 'tool_search_output'].includes(payload.type) && payload.call_id) {
|
||||
const toolId = codexCallId(payload.call_id) as string;
|
||||
out.push({ kind: 'tool_result', tool_use_id: toolId, message_uuid: callMessageUuids.get(toolId) || '', session_id: sessionId, content: trunc(codexToolOutput(payload) || ''), file_path: null, is_error: payload.is_error ? 1 : 0 });
|
||||
}
|
||||
}
|
||||
|
||||
if (agentId) {
|
||||
const started = sm.started_at ? new Date(sm.started_at).getTime() : null;
|
||||
const ended = sm.ended_at ? new Date(sm.ended_at).getTime() : null;
|
||||
const tokenTotal = (sm.totalInputTokens || 0) + (sm.totalOutputTokens || 0);
|
||||
out.push({
|
||||
kind: 'subagent', agent_id: agentId, session_id: sessionId,
|
||||
agent_type: codexAgentRole(meta), description: codexAgentNickname(meta),
|
||||
duration_ms: started && ended ? ended - started : null, total_tokens: tokenTotal || null,
|
||||
});
|
||||
} else {
|
||||
out.push({
|
||||
kind: 'session', id: sessionId, title: sm.title, project,
|
||||
started_at: sm.started_at, ended_at: sm.ended_at, git_branch: sm.git_branch, version: sm.version,
|
||||
message_count: sm.n, countMode: 'total', jsonl_path: unit.key, source: 'codex',
|
||||
});
|
||||
}
|
||||
|
||||
yield* out;
|
||||
return outCursor;
|
||||
}
|
||||
|
||||
export const codexProvider: Provider = { name, discover, parse };
|
||||
@@ -0,0 +1,226 @@
|
||||
// Core provider contract (see docs/adr/0001).
|
||||
//
|
||||
// The indexing layer splits along two orthogonal axes:
|
||||
// - Provider axis: pure per-source adapters (claude, codex, later opencode,
|
||||
// pi, …) that discover their own work and parse it into records. A source is
|
||||
// NOT assumed to be a single JSONL file — an adapter may read a SQLite store,
|
||||
// a directory tree, etc. So discovery, change-detection, and resume cursoring
|
||||
// are all adapter-owned and format-specific.
|
||||
// - Persist axis: one shared, provider- and binding-agnostic orchestration
|
||||
// that consumes the records and writes them (index_state, FTS, upsert).
|
||||
//
|
||||
// This file defines only the shapes crossing that boundary. Record fields mirror
|
||||
// the columns in scripts/schema.sql; keep them in sync. Types only — no runtime
|
||||
// code — so consumers must import with `import type`.
|
||||
|
||||
// Opaque per-unit resume/watermark token. The orchestration stores it verbatim
|
||||
// (in index_state) and hands it back on the next run; ONLY the adapter that
|
||||
// produced it interprets it. A JSONL adapter might encode `"${mtime}:${lines}"`;
|
||||
// a SQLite-backed adapter might encode a rowid or timestamp high-water mark.
|
||||
export type Cursor = string | null;
|
||||
|
||||
// One unit of work an adapter has discovered. It is not necessarily a file: for
|
||||
// a file-based source `key` is the path; for a DB-backed source it might be
|
||||
// `"${dbPath}#${internalId}"`. `meta` carries adapter-private data (e.g. the
|
||||
// resolved file path or source handle) that the orchestration passes back to
|
||||
// parse() untouched.
|
||||
export interface IndexUnit {
|
||||
/** Stable identity used as the index_state cursor key. */
|
||||
key: string;
|
||||
/** Session id this unit indexes into. */
|
||||
sessionId: string;
|
||||
/** Project slug (dash-encoded path), when the source exposes one. */
|
||||
project?: string;
|
||||
/** Set for subagent transcripts, whose messages carry an agent id. */
|
||||
isSubagent?: boolean;
|
||||
agentId?: string;
|
||||
/** Adapter-private payload, opaque to the orchestration. */
|
||||
meta?: unknown;
|
||||
}
|
||||
|
||||
/** Context the orchestration provides to discovery. */
|
||||
export interface DiscoverContext {
|
||||
/** Look up the cursor persisted for a unit key on a previous run. */
|
||||
lastCursor(key: string): Cursor;
|
||||
/** When set (daemon changed-path mode), restrict discovery to these paths. */
|
||||
changedPaths?: string[];
|
||||
}
|
||||
|
||||
/** Discriminated union of everything an adapter's parse can emit. Each record
|
||||
* kind maps to one schema table (see scripts/schema.sql); `delete-session` is
|
||||
* the exception — a retraction op, not a table. Sources without a table
|
||||
* (history.jsonl, codex session_index.jsonl) are not records: adapters fold them
|
||||
* into the SessionRecord they already emit. */
|
||||
export type IndexRecord =
|
||||
| SessionRecord
|
||||
| MessageRecord
|
||||
| ToolCallRecord
|
||||
| ToolResultRecord
|
||||
| SummaryRecord
|
||||
| SubagentRecord
|
||||
| WorkflowRecord
|
||||
| WorkflowAgentRecord
|
||||
| MessageTurnDurationRecord
|
||||
| DeleteSessionRecord;
|
||||
|
||||
export interface MessageRecord {
|
||||
kind: 'message';
|
||||
uuid: string;
|
||||
session_id: string;
|
||||
type: string;
|
||||
parent_uuid: string | null;
|
||||
timestamp: string | null;
|
||||
role: string | null;
|
||||
text: string | null;
|
||||
content_type: string | null;
|
||||
is_meta: 0 | 1;
|
||||
model: string | null;
|
||||
is_sidechain: 0 | 1;
|
||||
agent_id: string | null;
|
||||
input_tokens: number | null;
|
||||
output_tokens: number | null;
|
||||
cwd: string | null;
|
||||
skill: string | null;
|
||||
source: string;
|
||||
}
|
||||
|
||||
export interface ToolCallRecord {
|
||||
kind: 'tool_call';
|
||||
id: string;
|
||||
message_uuid: string;
|
||||
session_id: string;
|
||||
name: string;
|
||||
input_json: string;
|
||||
file_path: string | null;
|
||||
}
|
||||
|
||||
export interface ToolResultRecord {
|
||||
kind: 'tool_result';
|
||||
tool_use_id: string;
|
||||
message_uuid: string;
|
||||
session_id: string;
|
||||
content: string;
|
||||
file_path: string | null;
|
||||
is_error: 0 | 1;
|
||||
}
|
||||
|
||||
export interface SummaryRecord {
|
||||
kind: 'summary';
|
||||
id: string;
|
||||
session_id: string;
|
||||
timestamp: string | null;
|
||||
source: string;
|
||||
content: string;
|
||||
}
|
||||
|
||||
// One codex subagent. Like workflow_agent, a row can be contributed by more than
|
||||
// one point in the parse (the spawn event vs the agent's own thread), so non-key
|
||||
// fields are optional and persist merges them column-wise with COALESCE.
|
||||
export interface SubagentRecord {
|
||||
kind: 'subagent';
|
||||
agent_id: string;
|
||||
session_id: string;
|
||||
parent_tool_use_id?: string | null;
|
||||
agent_type?: string | null;
|
||||
description?: string | null;
|
||||
duration_ms?: number | null;
|
||||
total_tokens?: number | null;
|
||||
}
|
||||
|
||||
// A workflow run. `agent_count` is intentionally absent: it is a derived
|
||||
// aggregate (COUNT of workflow_agents for this run) that persist computes, since
|
||||
// the agents may be indexed on different runs than the workflow metadata.
|
||||
export interface WorkflowRecord {
|
||||
kind: 'workflow';
|
||||
run_id: string;
|
||||
session_id: string;
|
||||
task_id: string | null;
|
||||
script: string | null;
|
||||
result_json: string | null;
|
||||
timestamp: string | null;
|
||||
duration_ms: number | null;
|
||||
total_tokens: number | null;
|
||||
status: string | null;
|
||||
workflow_name: string | null;
|
||||
}
|
||||
|
||||
// One workflow agent. A single row is contributed by TWO independent units, in
|
||||
// any order: the subagent .meta.json unit fills agent_type/description; the
|
||||
// workflow run json unit fills phase/label/model/state/duration_ms/tokens/
|
||||
// tool_calls. So every optional field a unit does not know is omitted, and
|
||||
// persist merges column-wise (ON CONFLICT(agent_id) DO UPDATE SET
|
||||
// col=COALESCE(excluded.col, col)). All contributors MUST use the same unified
|
||||
// agent_id key so the merge lands on the same row.
|
||||
export interface WorkflowAgentRecord {
|
||||
kind: 'workflow_agent';
|
||||
agent_id: string;
|
||||
run_id: string;
|
||||
session_id: string;
|
||||
agent_type?: string | null;
|
||||
description?: string | null;
|
||||
phase?: string | null;
|
||||
label?: string | null;
|
||||
model?: string | null;
|
||||
state?: string | null;
|
||||
duration_ms?: number | null;
|
||||
tokens?: number | null;
|
||||
tool_calls?: number | null;
|
||||
}
|
||||
|
||||
// Update op (not a table): sets messages.turn_duration_ms for a message that was
|
||||
// (or will be) inserted by a separate line, possibly on a different run. Persist
|
||||
// applies it as a targeted UPDATE, so it never clobbers other message columns.
|
||||
export interface MessageTurnDurationRecord {
|
||||
kind: 'message-turn-duration';
|
||||
uuid: string;
|
||||
turn_duration_ms: number | null;
|
||||
}
|
||||
|
||||
// Retraction op (not a table). The adapter emits this when a previously-indexed
|
||||
// session must be removed — e.g. a Codex guardian/auto-review thread. Persist
|
||||
// executes the cascade delete across all tables for that session.
|
||||
export interface DeleteSessionRecord {
|
||||
kind: 'delete-session';
|
||||
sessionId: string;
|
||||
}
|
||||
|
||||
// Session-level aggregate. Emitted once, after the unit's records are produced,
|
||||
// because started_at/ended_at/message_count are computed across the stream.
|
||||
// title/ended_at may be enriched by the adapter from source-specific auxiliary
|
||||
// files (claude history.jsonl, codex session_index.jsonl); persist upserts with
|
||||
// fill-if-null (COALESCE) so those never clobber a value already present.
|
||||
// project_path is NOT set here — the orchestration's global pass derives it from
|
||||
// persisted message cwds (refreshSessionProjectPaths).
|
||||
//
|
||||
// countMode tells persist how to treat message_count, because providers differ:
|
||||
// a line-incremental adapter (claude) yields only new messages ('delta', persist
|
||||
// accumulates onto the existing row); a full-reparse adapter (codex) yields every
|
||||
// message each run ('total', persist replaces). A 'delta' parse from an empty
|
||||
// cursor is equivalent to 'total'.
|
||||
export interface SessionRecord {
|
||||
kind: 'session';
|
||||
id: string;
|
||||
title: string | null;
|
||||
project: string | null;
|
||||
started_at: string | null;
|
||||
ended_at: string | null;
|
||||
git_branch: string | null;
|
||||
version: string | null;
|
||||
message_count: number;
|
||||
countMode: 'total' | 'delta';
|
||||
jsonl_path: string;
|
||||
source: string;
|
||||
}
|
||||
|
||||
// A transcript source. Pure: it never touches the Obelisk database. It owns its
|
||||
// own discovery, change-detection, and resume cursoring, because those are
|
||||
// format-specific (file mtime, DB watermark, …). `parse` is a generator that
|
||||
// yields records for one unit and RETURNS the new cursor to persist.
|
||||
export interface Provider {
|
||||
/** Stable source tag stored on rows, e.g. 'claude' | 'codex'. */
|
||||
readonly name: string;
|
||||
/** Discover units needing (re)indexing, using stored cursors to detect change. */
|
||||
discover(ctx: DiscoverContext): IndexUnit[];
|
||||
/** Stream records for one unit resuming from `cursor`; return the new cursor. */
|
||||
parse(unit: IndexUnit, cursor: Cursor): Generator<IndexRecord, Cursor>;
|
||||
}
|
||||
@@ -0,0 +1,593 @@
|
||||
// Query and attune sandbox helpers for the Core package.
|
||||
import { readLines, fs, path } from './db.mjs';
|
||||
|
||||
function normalizeOpts(optsOrScalar, scalarKey = 'sessionId') {
|
||||
if (optsOrScalar == null) return {};
|
||||
if (typeof optsOrScalar === 'string') return { [scalarKey]: optsOrScalar };
|
||||
if (typeof optsOrScalar === 'number') return { limit: optsOrScalar };
|
||||
return optsOrScalar;
|
||||
}
|
||||
|
||||
function buildWhere(opts, aliases) {
|
||||
const clauses = [];
|
||||
const params = [];
|
||||
if (opts.sessionId) { clauses.push(`${aliases.sessionId} = ?`); params.push(opts.sessionId); }
|
||||
if (opts.sessions?.length) {
|
||||
clauses.push(`${aliases.sessionId} IN (${opts.sessions.map(() => '?').join(',')})`);
|
||||
params.push(...opts.sessions);
|
||||
}
|
||||
if (opts.project) { clauses.push(`${aliases.project} LIKE ?`); params.push(opts.project); }
|
||||
if (opts.after) { clauses.push(`${aliases.timestamp} > ?`); params.push(opts.after); }
|
||||
if (opts.before) { clauses.push(`${aliases.timestamp} < ?`); params.push(opts.before); }
|
||||
if (opts.branch) { clauses.push(`${aliases.branch} = ?`); params.push(opts.branch); }
|
||||
if (opts.source && opts.source !== 'all' && aliases.source) {
|
||||
clauses.push(`COALESCE(${aliases.source}, 'claude') = ?`);
|
||||
params.push(opts.source);
|
||||
}
|
||||
return { where: clauses.length ? clauses.join(' AND ') : '1=1', params };
|
||||
}
|
||||
|
||||
const BASH_EXIT_PAT = 'Exit code %';
|
||||
|
||||
function assertReadOnlySql(sql) {
|
||||
const text = String(sql || '').trim();
|
||||
if (!/^(SELECT|WITH)\b/i.test(text)) {
|
||||
throw new Error('sql() only supports read-only SELECT/WITH queries');
|
||||
}
|
||||
if (/\b(INSERT|UPDATE|DELETE|REPLACE|CREATE|DROP|ALTER|PRAGMA|VACUUM|ATTACH|DETACH)\b/i.test(text)) {
|
||||
throw new Error('sql() only supports read-only SELECT/WITH queries');
|
||||
}
|
||||
}
|
||||
|
||||
const CJK_TEXT_RE = /[\p{Script=Han}\p{Script=Hiragana}\p{Script=Katakana}\p{Script=Hangul}]/u;
|
||||
|
||||
function assertEnglishMemoryText(value, label) {
|
||||
const text = String(value || '');
|
||||
if (!text.trim()) return;
|
||||
if (CJK_TEXT_RE.test(text)) {
|
||||
const requirement = label.includes('query') ? 'must use English terms' : 'must be written in English';
|
||||
throw new Error(`${label} ${requirement}; translate user-language terms before using the memory layer`);
|
||||
}
|
||||
}
|
||||
|
||||
function buildSafeFtsQuery(text) {
|
||||
const tokens = String(text || '').match(/[\p{Letter}\p{Number}]+/gu) || [];
|
||||
return tokens
|
||||
.slice(0, 12)
|
||||
.map(token => `"${token}"`)
|
||||
.join(' ');
|
||||
}
|
||||
|
||||
function createQueryApi(db) {
|
||||
const q = (sql, ...p) => {
|
||||
assertReadOnlySql(sql);
|
||||
return db.prepare(sql).all(...p);
|
||||
};
|
||||
|
||||
const normalizeOverviewOpts = (optsOrScalar) => {
|
||||
if (optsOrScalar == null) return {};
|
||||
if (typeof optsOrScalar === 'string') return { project: optsOrScalar };
|
||||
if (typeof optsOrScalar === 'number') return { limit: optsOrScalar };
|
||||
return optsOrScalar;
|
||||
};
|
||||
|
||||
const search = (text, opts = {}) => {
|
||||
const { limit = 20, sessionId, project, after, before, cwd, source, includeMeta = false } = opts;
|
||||
let where = 'WHERE mf.text MATCH ?';
|
||||
const filterParams = [];
|
||||
if (sessionId) { where += ' AND mf.session_id=?'; filterParams.push(sessionId); }
|
||||
if (project) { where += ' AND s.project LIKE ?'; filterParams.push(project); }
|
||||
if (after) { where += ' AND m.timestamp>?'; filterParams.push(after); }
|
||||
if (before) { where += ' AND m.timestamp<?'; filterParams.push(before); }
|
||||
if (cwd) { where += ' AND m.cwd LIKE ?'; filterParams.push(cwd); }
|
||||
if (source && source !== 'all') { where += " AND COALESCE(m.source, s.source, 'claude')=?"; filterParams.push(source); }
|
||||
if (!includeMeta) where += ' AND COALESCE(m.is_meta,0)=0';
|
||||
const stmt = db.prepare(`
|
||||
SELECT m.uuid,m.session_id,m.text,m.content_type,m.is_meta,m.role,m.timestamp,m.model,m.cwd,m.source as m_source,
|
||||
s.id as s_id,s.title as s_title,s.project as s_project,s.started_at as s_started,
|
||||
s.source as s_source,
|
||||
rank
|
||||
FROM messages_fts mf JOIN messages m ON m.uuid=mf.uuid LEFT JOIN sessions s ON s.id=m.session_id
|
||||
${where} ORDER BY rank LIMIT ?`);
|
||||
const runMatch = (matchText) => stmt.all(matchText, ...filterParams, limit);
|
||||
// Honor raw FTS5 syntax when the query is valid, but never crash on ordinary
|
||||
// input (hyphens, punctuation) that FTS5 would parse as operators: fall back
|
||||
// to safe per-token quoting, the same tokenization memories() uses.
|
||||
let rows;
|
||||
try {
|
||||
rows = runMatch(text);
|
||||
} catch {
|
||||
const safe = buildSafeFtsQuery(text);
|
||||
rows = safe ? runMatch(safe) : [];
|
||||
}
|
||||
return rows.map(r => {
|
||||
const metaClause = includeMeta ? '' : 'AND COALESCE(is_meta,0)=0';
|
||||
const ctx = db.prepare(
|
||||
`SELECT uuid,text,content_type,is_meta,role,timestamp,model,COALESCE(source, 'claude') as source FROM messages WHERE session_id=? AND uuid!=? ${metaClause} ORDER BY ABS(JULIANDAY(timestamp)-JULIANDAY(?)) LIMIT 6`
|
||||
).all(r.session_id, r.uuid, r.timestamp).sort((a,b) => a.timestamp < b.timestamp ? -1 : 1);
|
||||
const sourceValue = r.m_source || r.s_source || 'claude';
|
||||
return {
|
||||
message: { uuid: r.uuid, text: r.text, content_type: r.content_type, is_meta: r.is_meta || 0, role: r.role, timestamp: r.timestamp, model: r.model, cwd: r.cwd, source: sourceValue },
|
||||
session: { id: r.s_id, title: r.s_title, project: r.s_project, started_at: r.s_started, source: r.s_source || sourceValue },
|
||||
rank: r.rank,
|
||||
context: ctx,
|
||||
};
|
||||
});
|
||||
};
|
||||
|
||||
const context = (uuid) => {
|
||||
const msg = db.prepare('SELECT * FROM messages WHERE uuid=?').get(uuid);
|
||||
if (!msg) return null;
|
||||
const session = db.prepare('SELECT * FROM sessions WHERE id=?').get(msg.session_id);
|
||||
const chain = [];
|
||||
let cur = msg;
|
||||
while (cur?.parent_uuid) { cur = db.prepare('SELECT * FROM messages WHERE uuid=?').get(cur.parent_uuid); if (cur) chain.unshift(cur); }
|
||||
let subagent = msg.agent_id ? db.prepare('SELECT * FROM subagents WHERE agent_id=?').get(msg.agent_id) : null;
|
||||
let workflow = null;
|
||||
if (msg.agent_id) {
|
||||
const wa = db.prepare('SELECT * FROM workflow_agents WHERE agent_id=?').get(msg.agent_id);
|
||||
if (wa) workflow = db.prepare('SELECT * FROM workflows WHERE run_id=?').get(wa.run_id);
|
||||
}
|
||||
return { message: msg, parentChain: chain, session, subagent, workflow };
|
||||
};
|
||||
|
||||
const trace = (uuid) => {
|
||||
const chain = [];
|
||||
let cur = db.prepare('SELECT * FROM messages WHERE uuid=?').get(uuid);
|
||||
while (cur) { chain.unshift(cur); cur = cur.parent_uuid ? db.prepare('SELECT * FROM messages WHERE uuid=?').get(cur.parent_uuid) : null; }
|
||||
return chain;
|
||||
};
|
||||
|
||||
const thread = (sid, opts = {}) => {
|
||||
const includeMeta = opts?.includeMeta === true;
|
||||
const metaClause = includeMeta ? '' : 'AND COALESCE(is_meta,0)=0';
|
||||
return db.prepare(`SELECT * FROM messages WHERE session_id=? ${metaClause} ORDER BY timestamp`).all(sid);
|
||||
};
|
||||
|
||||
const subagents = (optsOrSid) => {
|
||||
const opts = normalizeOpts(optsOrSid);
|
||||
const { limit = 100 } = opts;
|
||||
const needsJoin = opts.project || opts.branch || opts.source;
|
||||
const { where, params } = buildWhere(opts, { sessionId: 'sa.session_id', project: 's.project', timestamp: 'sa.session_id', branch: 's.git_branch', source: 's.source' });
|
||||
params.push(limit);
|
||||
const join = needsJoin ? 'LEFT JOIN sessions s ON s.id=sa.session_id' : '';
|
||||
return db.prepare(`SELECT sa.* FROM subagents sa ${join} WHERE ${where} LIMIT ?`).all(...params).map(r => {
|
||||
const c = db.prepare('SELECT COUNT(*) as c FROM messages WHERE agent_id=?').get(r.agent_id);
|
||||
return { ...r, messageCount: c?.c || 0 };
|
||||
});
|
||||
};
|
||||
|
||||
const workflows = (optsOrSid) => {
|
||||
const opts = normalizeOpts(optsOrSid);
|
||||
const { limit = 100 } = opts;
|
||||
const needsJoin = opts.project || opts.branch || opts.source;
|
||||
const { where, params } = buildWhere(opts, { sessionId: 'w.session_id', project: 's.project', timestamp: 'w.timestamp', branch: 's.git_branch', source: 's.source' });
|
||||
params.push(limit);
|
||||
const join = needsJoin ? 'LEFT JOIN sessions s ON s.id=w.session_id' : '';
|
||||
return db.prepare(`SELECT w.* FROM workflows w ${join} WHERE ${where} ORDER BY w.timestamp DESC LIMIT ?`).all(...params);
|
||||
};
|
||||
|
||||
const workflowTree = (runId) => {
|
||||
const wf = db.prepare('SELECT * FROM workflows WHERE run_id=?').get(runId);
|
||||
if (!wf) return null;
|
||||
let result = null;
|
||||
try { result = JSON.parse(wf.result_json); } catch {}
|
||||
const agents = db.prepare('SELECT * FROM workflow_agents WHERE run_id=?').all(runId).map(a => {
|
||||
const mc = db.prepare('SELECT COUNT(*) as c FROM messages WHERE agent_id=?').get(a.agent_id);
|
||||
return { ...a, messageCount: mc?.c || 0 };
|
||||
});
|
||||
return { ...wf, result, agents };
|
||||
};
|
||||
|
||||
const fileHistory = (fp, opts = {}) => {
|
||||
const { limit = 200, after, before, source } = opts;
|
||||
let where = 'tc.file_path=?';
|
||||
const params = [fp];
|
||||
if (after) { where += ' AND m.timestamp > ?'; params.push(after); }
|
||||
if (before) { where += ' AND m.timestamp < ?'; params.push(before); }
|
||||
if (source && source !== 'all') { where += " AND COALESCE(s.source, 'claude') = ?"; params.push(source); }
|
||||
params.push(limit);
|
||||
return db.prepare(
|
||||
`SELECT tc.*,s.title as s_title,s.project as s_project,m.timestamp as ts FROM tool_calls tc LEFT JOIN sessions s ON s.id=tc.session_id LEFT JOIN messages m ON m.uuid=tc.message_uuid WHERE ${where} ORDER BY m.timestamp LIMIT ?`
|
||||
).all(...params).map(r => ({
|
||||
toolCall: { id: r.id, message_uuid: r.message_uuid, name: r.name, input_json: r.input_json },
|
||||
session: { id: r.session_id, title: r.s_title, project: r.s_project },
|
||||
timestamp: r.ts,
|
||||
}));
|
||||
};
|
||||
|
||||
const failures = (optsOrSid) => {
|
||||
const opts = normalizeOpts(optsOrSid);
|
||||
const { limit = 50 } = opts;
|
||||
const needsJoin = opts.project || opts.branch || opts.source;
|
||||
const { where, params: filterParams } = buildWhere(opts, { sessionId: 'tr.session_id', project: 's.project', timestamp: 'rm.timestamp', branch: 's.git_branch', source: 's.source' });
|
||||
const join = needsJoin ? 'LEFT JOIN sessions s ON s.id=tr.session_id' : '';
|
||||
const errorCond = `(tr.is_error = 1 OR tr.content LIKE '${BASH_EXIT_PAT}')`;
|
||||
const allParams = [...filterParams, limit];
|
||||
const rows = db.prepare(`SELECT tr.* FROM tool_results tr ${join} LEFT JOIN messages rm ON rm.uuid=tr.message_uuid WHERE ${errorCond} AND ${where} ORDER BY rm.timestamp DESC LIMIT ?`).all(...allParams);
|
||||
return rows.map(r => {
|
||||
const tc = db.prepare('SELECT * FROM tool_calls WHERE id=?').get(r.tool_use_id);
|
||||
const session = db.prepare('SELECT * FROM sessions WHERE id=?').get(r.session_id);
|
||||
const rm = db.prepare('SELECT * FROM messages WHERE uuid=?').get(r.message_uuid);
|
||||
const next = rm?.timestamp ? db.prepare('SELECT * FROM messages WHERE session_id=? AND timestamp>? ORDER BY timestamp LIMIT 3').all(r.session_id, rm.timestamp) : [];
|
||||
return { toolCall: tc, result: r, session, nextMessages: next };
|
||||
});
|
||||
};
|
||||
|
||||
const sessions = (optsOrN) => {
|
||||
const opts = normalizeOpts(optsOrN, 'sessionId');
|
||||
const { limit = 50 } = opts;
|
||||
const { where, params } = buildWhere(opts, { sessionId: 's.id', project: 's.project', timestamp: 's.started_at', branch: 's.git_branch', source: 's.source' });
|
||||
params.push(limit);
|
||||
return db.prepare(`SELECT * FROM sessions s WHERE ${where} ORDER BY ended_at DESC LIMIT ?`).all(...params);
|
||||
};
|
||||
|
||||
const recent = (n = 10) => sessions({ limit: n });
|
||||
|
||||
const summaries = (optsOrSid) => {
|
||||
const opts = normalizeOpts(optsOrSid);
|
||||
const { limit = 100 } = opts;
|
||||
const { where, params } = buildWhere(opts, { sessionId: 'su.session_id', project: 's.project', timestamp: 'su.timestamp', branch: 's.git_branch', source: 's.source' });
|
||||
params.push(limit);
|
||||
return db.prepare(`SELECT su.*, s.title as session_title, s.project FROM summaries su LEFT JOIN sessions s ON s.id=su.session_id WHERE ${where} ORDER BY su.timestamp DESC LIMIT ?`).all(...params);
|
||||
};
|
||||
|
||||
const overview = (optsOrScalar) => {
|
||||
const opts = normalizeOverviewOpts(optsOrScalar);
|
||||
const cwd = process.cwd();
|
||||
const sessionLimit = opts.limit ?? 8;
|
||||
const projectLimit = opts.projectLimit ?? 20;
|
||||
const memoryLimit = opts.memoryLimit ?? 100;
|
||||
|
||||
const projectDescriptor = (row, source, confidence) => row ? ({
|
||||
project: row.project,
|
||||
project_path: row.project_path || null,
|
||||
source,
|
||||
confidence,
|
||||
}) : null;
|
||||
|
||||
const latestProjectByPattern = (pattern) => {
|
||||
const fromSessions = db.prepare(`
|
||||
SELECT project, project_path
|
||||
FROM sessions
|
||||
WHERE project LIKE ?
|
||||
ORDER BY COALESCE(ended_at, started_at) DESC
|
||||
LIMIT 1
|
||||
`).get(pattern);
|
||||
if (fromSessions) return fromSessions;
|
||||
return db.prepare(`
|
||||
SELECT project, NULL AS project_path
|
||||
FROM memories
|
||||
WHERE project LIKE ?
|
||||
ORDER BY created_at DESC
|
||||
LIMIT 1
|
||||
`).get(pattern);
|
||||
};
|
||||
|
||||
const resolveCurrentProject = () => {
|
||||
if (opts.project) {
|
||||
const row = latestProjectByPattern(opts.project);
|
||||
const confidence = row ? (/[%_]/.test(opts.project) ? 'inferred' : 'exact') : 'unknown';
|
||||
return projectDescriptor(row || { project: opts.project, project_path: null }, 'opts', confidence);
|
||||
}
|
||||
|
||||
const paths = db.prepare(`
|
||||
SELECT project, project_path, MAX(COALESCE(ended_at, started_at)) AS last_seen
|
||||
FROM sessions
|
||||
WHERE project IS NOT NULL AND project_path IS NOT NULL AND project_path != ''
|
||||
GROUP BY project, project_path
|
||||
`).all();
|
||||
const byProjectPath = paths
|
||||
.filter(r => cwd === r.project_path || cwd.startsWith(r.project_path + path.sep))
|
||||
.sort((a, b) => b.project_path.length - a.project_path.length || String(b.last_seen || '').localeCompare(String(a.last_seen || '')))[0];
|
||||
if (byProjectPath) return projectDescriptor(byProjectPath, 'cwd_project_path', 'exact');
|
||||
|
||||
const byMessageCwd = db.prepare(`
|
||||
SELECT s.project, s.project_path, MAX(m.timestamp) AS last_seen
|
||||
FROM messages m
|
||||
LEFT JOIN sessions s ON s.id=m.session_id
|
||||
WHERE m.cwd = ? AND s.project IS NOT NULL
|
||||
GROUP BY s.project, s.project_path
|
||||
ORDER BY last_seen DESC
|
||||
LIMIT 1
|
||||
`).get(cwd);
|
||||
if (byMessageCwd) return projectDescriptor(byMessageCwd, 'cwd_messages', 'inferred');
|
||||
|
||||
return null;
|
||||
};
|
||||
|
||||
const projects = db.prepare(`
|
||||
WITH names AS (
|
||||
SELECT project FROM sessions WHERE project IS NOT NULL GROUP BY project
|
||||
UNION
|
||||
SELECT project FROM memories WHERE project IS NOT NULL AND deleted_at IS NULL GROUP BY project
|
||||
),
|
||||
session_stats AS (
|
||||
SELECT project, COUNT(*) AS session_count, MAX(COALESCE(ended_at, started_at)) AS last_session_at
|
||||
FROM sessions
|
||||
WHERE project IS NOT NULL
|
||||
GROUP BY project
|
||||
),
|
||||
memory_stats AS (
|
||||
SELECT project, COUNT(*) AS memory_count, MAX(created_at) AS last_memory_at
|
||||
FROM memories
|
||||
WHERE project IS NOT NULL AND deleted_at IS NULL
|
||||
GROUP BY project
|
||||
)
|
||||
SELECT
|
||||
n.project,
|
||||
(
|
||||
SELECT s2.project_path
|
||||
FROM sessions s2
|
||||
WHERE s2.project = n.project AND s2.project_path IS NOT NULL
|
||||
ORDER BY COALESCE(s2.ended_at, s2.started_at) DESC
|
||||
LIMIT 1
|
||||
) AS project_path,
|
||||
COALESCE(ss.session_count, 0) AS session_count,
|
||||
COALESCE(ms.memory_count, 0) AS memory_count,
|
||||
ss.last_session_at,
|
||||
ms.last_memory_at
|
||||
FROM names n
|
||||
LEFT JOIN session_stats ss ON ss.project = n.project
|
||||
LEFT JOIN memory_stats ms ON ms.project = n.project
|
||||
ORDER BY COALESCE(ss.last_session_at, ms.last_memory_at) DESC
|
||||
LIMIT ?
|
||||
`).all(projectLimit).map(row => {
|
||||
const branches = db.prepare(`
|
||||
SELECT git_branch
|
||||
FROM sessions
|
||||
WHERE project = ? AND git_branch IS NOT NULL AND git_branch != ''
|
||||
GROUP BY git_branch
|
||||
ORDER BY MAX(COALESCE(ended_at, started_at)) DESC
|
||||
LIMIT 5
|
||||
`).all(row.project).map(r => r.git_branch);
|
||||
return { ...row, recent_branches: branches };
|
||||
});
|
||||
|
||||
const currentProject = resolveCurrentProject();
|
||||
let current_project = null;
|
||||
if (currentProject?.project) {
|
||||
const sessionTotal = db.prepare('SELECT COUNT(*) AS c FROM sessions WHERE project = ?').get(currentProject.project)?.c || 0;
|
||||
const sessionsForProject = db.prepare(`
|
||||
SELECT id, title, project, project_path, started_at, ended_at, git_branch, message_count, COALESCE(source, 'claude') AS source
|
||||
FROM sessions
|
||||
WHERE project = ?
|
||||
ORDER BY COALESCE(ended_at, started_at) DESC
|
||||
LIMIT ?
|
||||
`).all(currentProject.project, sessionLimit);
|
||||
const memoryTotal = db.prepare('SELECT COUNT(*) AS c FROM memories WHERE project = ? AND deleted_at IS NULL').get(currentProject.project)?.c || 0;
|
||||
const memoriesForProject = db.prepare(`
|
||||
SELECT id, path, anchors, summary, session_id, project, created_at
|
||||
FROM memories
|
||||
WHERE project = ? AND deleted_at IS NULL
|
||||
ORDER BY created_at DESC
|
||||
LIMIT ?
|
||||
`).all(currentProject.project, memoryLimit);
|
||||
current_project = {
|
||||
project: currentProject.project,
|
||||
project_path: currentProject.project_path,
|
||||
session_total: sessionTotal,
|
||||
sessions: sessionsForProject,
|
||||
memory_total: memoryTotal,
|
||||
memories: memoriesForProject,
|
||||
};
|
||||
}
|
||||
|
||||
const totalProjects = db.prepare(`
|
||||
SELECT COUNT(*) AS c
|
||||
FROM (
|
||||
SELECT project FROM sessions WHERE project IS NOT NULL GROUP BY project
|
||||
UNION
|
||||
SELECT project FROM memories WHERE project IS NOT NULL AND deleted_at IS NULL GROUP BY project
|
||||
)
|
||||
`).get()?.c || 0;
|
||||
const totalSessions = db.prepare('SELECT COUNT(*) AS c FROM sessions').get()?.c || 0;
|
||||
const totalMemories = db.prepare('SELECT COUNT(*) AS c FROM memories WHERE deleted_at IS NULL').get()?.c || 0;
|
||||
const sources = db.prepare(`
|
||||
SELECT COALESCE(source, 'claude') AS source,
|
||||
COUNT(*) AS session_count,
|
||||
MAX(COALESCE(ended_at, started_at)) AS last_session_at
|
||||
FROM sessions
|
||||
GROUP BY COALESCE(source, 'claude')
|
||||
ORDER BY last_session_at DESC
|
||||
`).all();
|
||||
|
||||
return {
|
||||
current: {
|
||||
cwd,
|
||||
project: currentProject,
|
||||
},
|
||||
current_project,
|
||||
projects,
|
||||
totals: {
|
||||
projects: totalProjects,
|
||||
sessions: totalSessions,
|
||||
memories: totalMemories,
|
||||
sources,
|
||||
},
|
||||
};
|
||||
};
|
||||
|
||||
const resolveJsonlPath = (messageUuid) => {
|
||||
const msg = db.prepare('SELECT session_id, agent_id, source FROM messages WHERE uuid=?').get(messageUuid);
|
||||
if (!msg) return null;
|
||||
if (msg.source === 'codex' || String(messageUuid).startsWith('codex:')) {
|
||||
const match = /^codex:([^:]+):(\d+)$/.exec(String(messageUuid));
|
||||
if (!match) return null;
|
||||
const rawThreadId = match[1];
|
||||
if (!msg.agent_id) {
|
||||
const ses = db.prepare('SELECT jsonl_path FROM sessions WHERE id=?').get(msg.session_id);
|
||||
if (ses?.jsonl_path) return ses.jsonl_path;
|
||||
}
|
||||
return db.prepare(`
|
||||
SELECT jsonl_path FROM index_state
|
||||
WHERE jsonl_path LIKE ? AND jsonl_path LIKE '%.jsonl'
|
||||
ORDER BY length(jsonl_path) ASC
|
||||
LIMIT 1
|
||||
`).get(`%${rawThreadId}.jsonl`)?.jsonl_path || null;
|
||||
}
|
||||
if (msg.agent_id) {
|
||||
const wa = db.prepare('SELECT agent_id, run_id, session_id FROM workflow_agents WHERE agent_id=?').get(msg.agent_id);
|
||||
if (wa) {
|
||||
const ses = db.prepare('SELECT jsonl_path FROM sessions WHERE id=?').get(wa.session_id);
|
||||
if (ses) return path.join(path.dirname(ses.jsonl_path), wa.session_id, 'subagents', 'workflows', wa.run_id, wa.agent_id + '.jsonl');
|
||||
}
|
||||
const sa = db.prepare('SELECT agent_id, session_id FROM subagents WHERE agent_id=?').get(msg.agent_id);
|
||||
if (sa) {
|
||||
const ses = db.prepare('SELECT jsonl_path FROM sessions WHERE id=?').get(sa.session_id);
|
||||
if (ses) return path.join(path.dirname(ses.jsonl_path), sa.session_id, 'subagents', sa.agent_id + '.jsonl');
|
||||
}
|
||||
} else {
|
||||
const ses = db.prepare('SELECT jsonl_path FROM sessions WHERE id=?').get(msg.session_id);
|
||||
if (ses) return ses.jsonl_path;
|
||||
}
|
||||
return null;
|
||||
};
|
||||
|
||||
const findCodexRawLine = (jsonlPath, uuid) => {
|
||||
const match = /^codex:[^:]+:(\d+)$/.exec(String(uuid));
|
||||
if (!match || !jsonlPath || !fs.existsSync(jsonlPath)) return null;
|
||||
const targetLine = Number(match[1]);
|
||||
let lineNum = 0;
|
||||
let found = null;
|
||||
readLines(jsonlPath, (line) => {
|
||||
lineNum++;
|
||||
if (lineNum !== targetLine) return;
|
||||
found = line;
|
||||
return false;
|
||||
});
|
||||
return found;
|
||||
};
|
||||
|
||||
const findRawLine = (jsonlPath, uuid) => {
|
||||
if (!jsonlPath || !fs.existsSync(jsonlPath)) return null;
|
||||
if (String(uuid).startsWith('codex:')) return findCodexRawLine(jsonlPath, uuid);
|
||||
let found = null;
|
||||
readLines(jsonlPath, (line) => {
|
||||
if (!line.includes(uuid)) return;
|
||||
try { const obj = JSON.parse(line); if (obj.uuid === uuid) { found = line; return false; } } catch {}
|
||||
});
|
||||
return found;
|
||||
};
|
||||
|
||||
const raw = (messageUuid, opts = {}) => {
|
||||
const { offset = 0, limit = 10000 } = opts;
|
||||
const jsonlPath = resolveJsonlPath(messageUuid);
|
||||
const line = findRawLine(jsonlPath, messageUuid);
|
||||
if (!line) return null;
|
||||
return {
|
||||
text: line.slice(offset, offset + limit),
|
||||
totalLength: line.length,
|
||||
offset,
|
||||
limit,
|
||||
hasMore: offset + limit < line.length,
|
||||
};
|
||||
};
|
||||
|
||||
const memories = (optsOrSid) => {
|
||||
const opts = normalizeOpts(optsOrSid);
|
||||
const { limit = 50, query } = opts;
|
||||
assertEnglishMemoryText(query, 'memories() query');
|
||||
const needsJoin = opts.branch || opts.source;
|
||||
const { where: baseWhere, params } = buildWhere(opts, {
|
||||
sessionId: 'mem.session_id',
|
||||
project: 'mem.project',
|
||||
timestamp: 'mem.created_at',
|
||||
branch: 's.git_branch',
|
||||
source: 's.source',
|
||||
});
|
||||
let where = baseWhere + ' AND mem.deleted_at IS NULL';
|
||||
const join = needsJoin ? 'LEFT JOIN sessions s ON s.id=mem.session_id' : '';
|
||||
const hasQuery = String(query || '').trim().length > 0;
|
||||
const ftsQuery = buildSafeFtsQuery(query);
|
||||
if (!hasQuery) {
|
||||
params.push(limit);
|
||||
return db.prepare(`SELECT mem.* FROM memories mem ${join} WHERE ${where} ORDER BY mem.created_at DESC LIMIT ?`).all(...params);
|
||||
}
|
||||
if (!ftsQuery) return [];
|
||||
params.unshift(ftsQuery);
|
||||
params.push(limit);
|
||||
return db.prepare(`
|
||||
SELECT mem.*, mf.rank AS rank
|
||||
FROM memories_fts mf
|
||||
JOIN memories mem ON mem.rowid = mf.rowid
|
||||
${join}
|
||||
WHERE memories_fts MATCH ? AND ${where}
|
||||
ORDER BY mf.rank, mem.created_at DESC
|
||||
LIMIT ?
|
||||
`).all(...params);
|
||||
};
|
||||
|
||||
return { sql: q, search, context, trace, thread, subagents, workflows, workflowTree, fileHistory, failures, sessions, recent, summaries, raw, memories, overview };
|
||||
}
|
||||
|
||||
function createAttuneApi(db) {
|
||||
const resolveMemoryPath = (memoryPath, sessionId) => {
|
||||
let base = null;
|
||||
if (sessionId) {
|
||||
base = db.prepare('SELECT project_path FROM sessions WHERE id=?').get(sessionId)?.project_path || null;
|
||||
}
|
||||
const resolved = path.isAbsolute(memoryPath)
|
||||
? path.normalize(memoryPath)
|
||||
: path.resolve(base || process.cwd(), memoryPath);
|
||||
let stat;
|
||||
try {
|
||||
stat = fs.statSync(resolved);
|
||||
} catch {
|
||||
throw new Error(`remember() memory file does not exist: ${resolved}`);
|
||||
}
|
||||
if (!stat.isFile()) throw new Error(`remember() memory path is not a file: ${resolved}`);
|
||||
return resolved;
|
||||
};
|
||||
|
||||
const normalizeAnchors = (anchors) => {
|
||||
if (anchors == null) return null;
|
||||
let parsed = anchors;
|
||||
if (typeof anchors === 'string') {
|
||||
const trimmed = anchors.trim();
|
||||
if (!trimmed) return null;
|
||||
try {
|
||||
parsed = JSON.parse(trimmed);
|
||||
} catch {
|
||||
throw new Error('remember() anchors must be a JSON array');
|
||||
}
|
||||
}
|
||||
if (!Array.isArray(parsed)) throw new Error('remember() anchors must be an array');
|
||||
for (const anchor of parsed) {
|
||||
if (!anchor || typeof anchor !== 'object' || Array.isArray(anchor)) {
|
||||
throw new Error('remember() anchors entries must be objects');
|
||||
}
|
||||
}
|
||||
return parsed.length ? JSON.stringify(parsed) : null;
|
||||
};
|
||||
|
||||
const remember = ({ path: memoryPath, session_id, message_start, message_end, summary, project, anchors }) => {
|
||||
if (!memoryPath || !summary) throw new Error('remember() requires path and summary');
|
||||
assertEnglishMemoryText(summary, 'remember() summary');
|
||||
const normalizedPath = resolveMemoryPath(memoryPath, session_id);
|
||||
const normalizedAnchors = normalizeAnchors(anchors);
|
||||
const id = `mem-${Date.now()}-${Math.random().toString(36).slice(2, 8)}`;
|
||||
const proj = project || db.prepare('SELECT project FROM sessions WHERE id=?').get(session_id)?.project || null;
|
||||
const created_at = new Date().toISOString();
|
||||
db.prepare('INSERT OR REPLACE INTO memories (id, session_id, project, message_start, message_end, path, anchors, summary, created_at) VALUES (?,?,?,?,?,?,?,?,?)').run(
|
||||
id, session_id || null, proj, message_start || null, message_end || null, normalizedPath, normalizedAnchors, summary, created_at);
|
||||
return { id, path: normalizedPath, project: proj, anchors: normalizedAnchors, created_at };
|
||||
};
|
||||
|
||||
const forget = ({ id, reason }) => {
|
||||
const deletionReason = String(reason || '').trim();
|
||||
if (!id || !deletionReason) throw new Error('forget() requires id and reason');
|
||||
const row = db.prepare('SELECT id, deleted_at, deleted_reason FROM memories WHERE id=?').get(id);
|
||||
if (!row) throw new Error(`forget() memory not found: ${id}`);
|
||||
if (row.deleted_at) {
|
||||
return { id, deleted_at: row.deleted_at, deleted_reason: row.deleted_reason, already_deleted: true };
|
||||
}
|
||||
const deleted_at = new Date().toISOString();
|
||||
db.prepare('UPDATE memories SET deleted_at=?, deleted_reason=? WHERE id=?').run(deleted_at, deletionReason, id);
|
||||
return { id, deleted_at, deleted_reason: deletionReason };
|
||||
};
|
||||
|
||||
return { remember, forget };
|
||||
}
|
||||
|
||||
export { createQueryApi, createAttuneApi };
|
||||
@@ -0,0 +1,46 @@
|
||||
#!/usr/bin/env node
|
||||
// Skill transport: a thin CLI shell over the Obelisk Core package.
|
||||
// It only parses args, reads script files, prints JSON, and owns the uniform
|
||||
// { error, stack } + exit-1 error envelope. All logic lives in Core.
|
||||
|
||||
import { createRequire } from 'node:module';
|
||||
const require = createRequire(import.meta.url);
|
||||
const fs = require('node:fs');
|
||||
const path = require('node:path');
|
||||
|
||||
import { DB_PATH, buildIndex, searchText, executeQuery, executeAttune } from './core.ts';
|
||||
|
||||
async function main() {
|
||||
const args = process.argv.slice(2);
|
||||
// Uniform error envelope across all four verbs: a failure is reported as
|
||||
// { error, stack } on stdout with exit code 1, never a raw crash on stderr.
|
||||
const fail = (e) => {
|
||||
process.stdout.write(JSON.stringify({ error: e.message, stack: e.stack }) + '\n');
|
||||
process.exitCode = 1;
|
||||
};
|
||||
const emit = (r) => process.stdout.write(JSON.stringify(r, null, 2) + '\n');
|
||||
|
||||
if (args[0] === '--build') {
|
||||
try {
|
||||
buildIndex({ force: true });
|
||||
process.stdout.write(JSON.stringify({ ok: true, db: DB_PATH }) + '\n');
|
||||
} catch (e) { fail(e); }
|
||||
return;
|
||||
}
|
||||
if (args[0] === '--search' && args[1]) {
|
||||
try { emit(searchText(args.slice(1).join(' '))); } catch (e) { fail(e); }
|
||||
return;
|
||||
}
|
||||
if (args[0] === '--query' && args[1]) {
|
||||
try { emit(await executeQuery(fs.readFileSync(path.resolve(args[1]), 'utf8'))); } catch (e) { fail(e); }
|
||||
return;
|
||||
}
|
||||
if (args[0] === '--attune' && args[1]) {
|
||||
try { emit(await executeAttune(fs.readFileSync(path.resolve(args[1]), 'utf8'))); } catch (e) { fail(e); }
|
||||
return;
|
||||
}
|
||||
process.stderr.write('Usage:\n node runtime.mjs --build\n node runtime.mjs --search "text"\n node runtime.mjs --query <file.js>\n node runtime.mjs --attune <file.js>\n');
|
||||
process.exitCode = 1;
|
||||
}
|
||||
|
||||
main();
|
||||
@@ -0,0 +1,89 @@
|
||||
-- Shared Obelisk Core schema.
|
||||
CREATE TABLE IF NOT EXISTS sessions (
|
||||
id TEXT PRIMARY KEY, title TEXT, project TEXT, project_path TEXT,
|
||||
started_at TEXT, ended_at TEXT, git_branch TEXT, version TEXT,
|
||||
message_count INTEGER DEFAULT 0, jsonl_path TEXT, source TEXT DEFAULT 'claude');
|
||||
CREATE TABLE IF NOT EXISTS messages (
|
||||
uuid TEXT PRIMARY KEY, session_id TEXT, type TEXT, parent_uuid TEXT,
|
||||
timestamp TEXT, role TEXT, text TEXT, content_type TEXT,
|
||||
is_meta INTEGER DEFAULT 0, model TEXT,
|
||||
is_sidechain INTEGER DEFAULT 0, agent_id TEXT,
|
||||
input_tokens INTEGER, output_tokens INTEGER,
|
||||
cwd TEXT, skill TEXT, turn_duration_ms INTEGER,
|
||||
source TEXT DEFAULT 'claude');
|
||||
CREATE TABLE IF NOT EXISTS tool_calls (
|
||||
id TEXT PRIMARY KEY, message_uuid TEXT, session_id TEXT,
|
||||
name TEXT, input_json TEXT, file_path TEXT);
|
||||
CREATE TABLE IF NOT EXISTS tool_results (
|
||||
tool_use_id TEXT PRIMARY KEY, message_uuid TEXT, session_id TEXT,
|
||||
content TEXT, file_path TEXT, is_error INTEGER DEFAULT 0);
|
||||
CREATE TABLE IF NOT EXISTS subagents (
|
||||
agent_id TEXT PRIMARY KEY, session_id TEXT, parent_tool_use_id TEXT,
|
||||
agent_type TEXT, description TEXT, duration_ms INTEGER, total_tokens INTEGER);
|
||||
CREATE TABLE IF NOT EXISTS workflows (
|
||||
run_id TEXT PRIMARY KEY, session_id TEXT, task_id TEXT,
|
||||
script TEXT, result_json TEXT, timestamp TEXT, agent_count INTEGER DEFAULT 0,
|
||||
duration_ms INTEGER, total_tokens INTEGER, status TEXT, workflow_name TEXT);
|
||||
CREATE TABLE IF NOT EXISTS workflow_agents (
|
||||
agent_id TEXT PRIMARY KEY, run_id TEXT, session_id TEXT,
|
||||
agent_type TEXT, description TEXT,
|
||||
phase TEXT, label TEXT, model TEXT, state TEXT,
|
||||
duration_ms INTEGER, tokens INTEGER, tool_calls INTEGER);
|
||||
CREATE TABLE IF NOT EXISTS index_state (
|
||||
jsonl_path TEXT PRIMARY KEY, mtime REAL, lines_processed INTEGER);
|
||||
CREATE TABLE IF NOT EXISTS summaries (
|
||||
id TEXT PRIMARY KEY, session_id TEXT, timestamp TEXT,
|
||||
source TEXT, content TEXT);
|
||||
CREATE VIRTUAL TABLE IF NOT EXISTS messages_fts USING fts5(
|
||||
uuid UNINDEXED, session_id UNINDEXED, text, content=messages, content_rowid=rowid);
|
||||
CREATE TRIGGER IF NOT EXISTS messages_fts_ai AFTER INSERT ON messages BEGIN
|
||||
INSERT INTO messages_fts(rowid, uuid, session_id, text)
|
||||
VALUES (new.rowid, new.uuid, new.session_id, new.text);
|
||||
END;
|
||||
CREATE TRIGGER IF NOT EXISTS messages_fts_ad AFTER DELETE ON messages BEGIN
|
||||
INSERT INTO messages_fts(messages_fts, rowid, uuid, session_id, text)
|
||||
VALUES ('delete', old.rowid, old.uuid, old.session_id, old.text);
|
||||
END;
|
||||
CREATE TRIGGER IF NOT EXISTS messages_fts_au AFTER UPDATE ON messages BEGIN
|
||||
INSERT INTO messages_fts(messages_fts, rowid, uuid, session_id, text)
|
||||
VALUES ('delete', old.rowid, old.uuid, old.session_id, old.text);
|
||||
INSERT INTO messages_fts(rowid, uuid, session_id, text)
|
||||
VALUES (new.rowid, new.uuid, new.session_id, new.text);
|
||||
END;
|
||||
CREATE INDEX IF NOT EXISTS idx_messages_session ON messages(session_id);
|
||||
CREATE INDEX IF NOT EXISTS idx_messages_agent ON messages(agent_id);
|
||||
CREATE INDEX IF NOT EXISTS idx_messages_ts ON messages(session_id, timestamp);
|
||||
CREATE INDEX IF NOT EXISTS idx_sessions_source ON sessions(source);
|
||||
CREATE INDEX IF NOT EXISTS idx_messages_source ON messages(source);
|
||||
CREATE INDEX IF NOT EXISTS idx_tc_session_name ON tool_calls(session_id, name);
|
||||
CREATE INDEX IF NOT EXISTS idx_tc_file ON tool_calls(file_path);
|
||||
CREATE INDEX IF NOT EXISTS idx_sa_session ON subagents(session_id);
|
||||
CREATE INDEX IF NOT EXISTS idx_wf_session ON workflows(session_id);
|
||||
CREATE INDEX IF NOT EXISTS idx_wa_run ON workflow_agents(run_id);
|
||||
CREATE INDEX IF NOT EXISTS idx_summaries_session ON summaries(session_id);
|
||||
CREATE TABLE IF NOT EXISTS memories (
|
||||
id TEXT PRIMARY KEY, session_id TEXT, project TEXT,
|
||||
message_start TEXT, message_end TEXT,
|
||||
path TEXT, anchors TEXT, summary TEXT, created_at TEXT,
|
||||
deleted_at TEXT, deleted_reason TEXT);
|
||||
CREATE VIRTUAL TABLE IF NOT EXISTS memories_fts USING fts5(
|
||||
id UNINDEXED, path, summary,
|
||||
content=memories, content_rowid=rowid,
|
||||
tokenize='unicode61 remove_diacritics 1');
|
||||
CREATE TRIGGER IF NOT EXISTS memories_fts_ai AFTER INSERT ON memories BEGIN
|
||||
INSERT INTO memories_fts(rowid, id, path, summary)
|
||||
VALUES (new.rowid, new.id, new.path, new.summary);
|
||||
END;
|
||||
CREATE TRIGGER IF NOT EXISTS memories_fts_ad AFTER DELETE ON memories BEGIN
|
||||
INSERT INTO memories_fts(memories_fts, rowid, id, path, summary)
|
||||
VALUES ('delete', old.rowid, old.id, old.path, old.summary);
|
||||
END;
|
||||
CREATE TRIGGER IF NOT EXISTS memories_fts_au AFTER UPDATE ON memories BEGIN
|
||||
INSERT INTO memories_fts(memories_fts, rowid, id, path, summary)
|
||||
VALUES ('delete', old.rowid, old.id, old.path, old.summary);
|
||||
INSERT INTO memories_fts(rowid, id, path, summary)
|
||||
VALUES (new.rowid, new.id, new.path, new.summary);
|
||||
END;
|
||||
CREATE INDEX IF NOT EXISTS idx_memories_project ON memories(project);
|
||||
CREATE INDEX IF NOT EXISTS idx_memories_session ON memories(session_id);
|
||||
CREATE INDEX IF NOT EXISTS idx_memories_created ON memories(created_at);
|
||||
@@ -0,0 +1,138 @@
|
||||
// Binding-agnostic SQLite write plumbing shared from the Core package
|
||||
// (docs/adr/0006). The injected db must expose `exec(sql)`; this works for both
|
||||
// node:sqlite (skill/CLI) and better-sqlite3 (app), same injection model as
|
||||
// `persist`.
|
||||
|
||||
export interface WriteTxDb {
|
||||
exec(sql: string): unknown;
|
||||
inTransaction(): boolean;
|
||||
}
|
||||
|
||||
export interface SqliteConnection {
|
||||
exec(sql: string): unknown;
|
||||
}
|
||||
|
||||
type Phase = 'begin' | 'work' | 'commit' | 'rollback';
|
||||
|
||||
export interface WriteTxDiagnostics {
|
||||
phase: Phase;
|
||||
code: string | null;
|
||||
label?: string;
|
||||
rollbackSucceeded: boolean | null;
|
||||
rollbackError: string | null;
|
||||
transactionActive: boolean | null;
|
||||
attempts: number;
|
||||
}
|
||||
|
||||
export interface WriteTxOptions {
|
||||
// Diagnostic label for this transaction (e.g. a file path or 'finalize').
|
||||
label?: string;
|
||||
}
|
||||
|
||||
const BUSY_MESSAGE = /SQLITE_BUSY|database is locked|database is busy/i;
|
||||
|
||||
function busyCode(error: unknown): string | null {
|
||||
const raw = error as { code?: unknown; errcode?: unknown; message?: unknown } | null;
|
||||
const code = (raw?.code ?? raw?.errcode);
|
||||
if (typeof code === 'string' && code.startsWith('SQLITE_BUSY')) return code;
|
||||
if (typeof raw?.message === 'string' && BUSY_MESSAGE.test(raw.message)) return 'SQLITE_BUSY';
|
||||
return null;
|
||||
}
|
||||
|
||||
function errorCode(error: unknown): string | null {
|
||||
const raw = error as { code?: unknown } | null;
|
||||
return typeof raw?.code === 'string' ? raw.code : null;
|
||||
}
|
||||
|
||||
interface BetterSqliteHandle {
|
||||
exec(sql: string): unknown;
|
||||
readonly inTransaction: boolean;
|
||||
}
|
||||
|
||||
interface NodeSqliteHandle {
|
||||
exec(sql: string): unknown;
|
||||
readonly isTransaction: boolean;
|
||||
}
|
||||
|
||||
export function betterSqliteTransactionAdapter(db: BetterSqliteHandle): WriteTxDb {
|
||||
return {
|
||||
exec: sql => db.exec(sql),
|
||||
inTransaction: () => db.inTransaction,
|
||||
};
|
||||
}
|
||||
|
||||
export function nodeSqliteTransactionAdapter(db: NodeSqliteHandle): WriteTxDb {
|
||||
return {
|
||||
exec: sql => db.exec(sql),
|
||||
inTransaction: () => db.isTransaction,
|
||||
};
|
||||
}
|
||||
|
||||
function transactionState(db: WriteTxDb): boolean | null {
|
||||
try {
|
||||
return db.inTransaction();
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
function attachDiagnostics(error: unknown, diagnostics: WriteTxDiagnostics): void {
|
||||
if (!error || typeof error !== 'object') return;
|
||||
try {
|
||||
(error as { obelisk?: WriteTxDiagnostics }).obelisk = diagnostics;
|
||||
} catch {
|
||||
// Frozen/native errors must still be rethrown unchanged.
|
||||
}
|
||||
}
|
||||
|
||||
// Runs `work` exactly once inside a transaction and returns its value. Retry and
|
||||
// scheduling policy belongs to the build coordinator, which knows the operation's
|
||||
// idempotency and total time budget. Cleanup never masks the primary exception.
|
||||
export function runWriteTransaction<T>(db: WriteTxDb, work: () => T, options: WriteTxOptions = {}): T {
|
||||
const { label } = options;
|
||||
let phase: Phase = 'begin';
|
||||
try {
|
||||
db.exec('BEGIN IMMEDIATE');
|
||||
phase = 'work';
|
||||
const value = work();
|
||||
phase = 'commit';
|
||||
db.exec('COMMIT');
|
||||
return value;
|
||||
} catch (error) {
|
||||
let rollbackSucceeded: boolean | null = null;
|
||||
let rollbackError: string | null = null;
|
||||
const activeBeforeRollback = transactionState(db);
|
||||
if (activeBeforeRollback !== false) {
|
||||
try {
|
||||
db.exec('ROLLBACK');
|
||||
rollbackSucceeded = true;
|
||||
} catch (rollbackFailure) {
|
||||
rollbackSucceeded = false;
|
||||
rollbackError = rollbackFailure instanceof Error ? rollbackFailure.message : String(rollbackFailure);
|
||||
}
|
||||
}
|
||||
const busy = busyCode(error);
|
||||
const diagnostics: WriteTxDiagnostics = {
|
||||
phase,
|
||||
code: busy ?? errorCode(error),
|
||||
label,
|
||||
rollbackSucceeded,
|
||||
rollbackError,
|
||||
transactionActive: transactionState(db),
|
||||
attempts: 1,
|
||||
};
|
||||
attachDiagnostics(error, diagnostics);
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
// Applies the connection-level pragmas used by every Obelisk writer/reader. Uses
|
||||
// exec (not better-sqlite3's .pragma) so one implementation covers both bindings.
|
||||
// busy_timeout is a real behavior change for node:sqlite (no default); it is set
|
||||
// explicitly for better-sqlite3 too, whose own default already happens to be
|
||||
// 5000ms. It is NOT the concurrency fix — see docs/adr/0006.
|
||||
export function configureConnection(db: SqliteConnection, { busyTimeoutMs = 5000 } = {}): void {
|
||||
db.exec(`PRAGMA busy_timeout=${busyTimeoutMs}`);
|
||||
db.exec('PRAGMA journal_mode=WAL');
|
||||
db.exec('PRAGMA synchronous=NORMAL');
|
||||
}
|
||||
@@ -0,0 +1,95 @@
|
||||
// Core's bounded retry policy above the transaction primitive. Callers opt in only for
|
||||
// idempotent work; BEGIN contention and an uncertain/live transaction are never
|
||||
// retried here.
|
||||
|
||||
import { runWriteTransaction, type WriteTxDb, type WriteTxOptions } from './tx.ts';
|
||||
|
||||
interface TransactionDiagnostics {
|
||||
phase?: string;
|
||||
code?: string | null;
|
||||
transactionActive?: boolean | null;
|
||||
attempts?: number;
|
||||
}
|
||||
|
||||
export interface WriteRetryOptions {
|
||||
maxAttempts?: number;
|
||||
budgetMs?: number;
|
||||
retryDelayMs?: number;
|
||||
now?: () => number;
|
||||
sleep?: (ms: number) => void;
|
||||
}
|
||||
|
||||
function diagnostics(error: unknown): TransactionDiagnostics | null {
|
||||
if (!error || typeof error !== 'object') return null;
|
||||
return (error as { obelisk?: TransactionDiagnostics }).obelisk ?? null;
|
||||
}
|
||||
|
||||
function isBusyCode(code: unknown): boolean {
|
||||
return typeof code === 'string' && code.startsWith('SQLITE_BUSY');
|
||||
}
|
||||
|
||||
function syncSleep(ms: number): void {
|
||||
if (ms <= 0) return;
|
||||
try {
|
||||
Atomics.wait(new Int32Array(new SharedArrayBuffer(4)), 0, 0, ms);
|
||||
} catch {
|
||||
// Bounded attempts still prevent an infinite retry loop.
|
||||
}
|
||||
}
|
||||
|
||||
export function isBeginBusyFailure(error: unknown): boolean {
|
||||
const info = diagnostics(error);
|
||||
return (
|
||||
info?.phase === 'begin' &&
|
||||
isBusyCode(info.code) &&
|
||||
info.transactionActive === false
|
||||
);
|
||||
}
|
||||
|
||||
export function hasUnusableTransaction(error: unknown): boolean {
|
||||
const info = diagnostics(error);
|
||||
return Boolean(info && info.transactionActive !== false);
|
||||
}
|
||||
|
||||
export function isRetryableWriteFailure(error: unknown): boolean {
|
||||
const info = diagnostics(error);
|
||||
return (
|
||||
(info?.phase === 'work' || info?.phase === 'commit') &&
|
||||
isBusyCode(info.code) &&
|
||||
info.transactionActive === false
|
||||
);
|
||||
}
|
||||
|
||||
export function runWithWriteRetry<T>(operation: () => T, {
|
||||
maxAttempts = 3,
|
||||
budgetMs = 1000,
|
||||
retryDelayMs = 25,
|
||||
now = Date.now,
|
||||
sleep = syncSleep,
|
||||
}: WriteRetryOptions = {}): T {
|
||||
const startedAt = now();
|
||||
for (let attempt = 1; ; attempt += 1) {
|
||||
try {
|
||||
return operation();
|
||||
} catch (error) {
|
||||
const info = diagnostics(error);
|
||||
if (info) info.attempts = attempt;
|
||||
if (!isRetryableWriteFailure(error) || attempt >= maxAttempts) throw error;
|
||||
const remaining = budgetMs - (now() - startedAt);
|
||||
if (remaining <= 0) throw error;
|
||||
sleep(Math.min(retryDelayMs * attempt, remaining));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
export function runRetryableWriteTransaction<T>(
|
||||
db: WriteTxDb,
|
||||
work: () => T,
|
||||
transactionOptions: WriteTxOptions = {},
|
||||
retryOptions: WriteRetryOptions = {},
|
||||
): T {
|
||||
return runWithWriteRetry(
|
||||
() => runWriteTransaction(db, work, transactionOptions),
|
||||
retryOptions,
|
||||
);
|
||||
}
|
||||
@@ -0,0 +1,91 @@
|
||||
// Cross-process single-writer lease shared by every Obelisk mutation. The
|
||||
// lock lives in a dedicated SQLite database so node:sqlite and better-sqlite3
|
||||
// share identical locking semantics on every supported platform.
|
||||
|
||||
import { mkdirSync } from 'node:fs';
|
||||
import { dirname, join } from 'node:path';
|
||||
|
||||
export interface WriterLeaseDb {
|
||||
exec(sql: string): unknown;
|
||||
close(): void;
|
||||
}
|
||||
|
||||
export interface WriterLease {
|
||||
release(): void;
|
||||
}
|
||||
|
||||
export interface AcquireWriterLeaseOptions {
|
||||
lockPath: string;
|
||||
openDb: (path: string) => WriterLeaseDb;
|
||||
waitMs?: number;
|
||||
retryDelayMs?: number;
|
||||
now?: () => number;
|
||||
sleep?: (ms: number) => void;
|
||||
}
|
||||
|
||||
const BUSY_MESSAGE = /SQLITE_BUSY|database is locked|database is busy/i;
|
||||
|
||||
function isBusy(error: unknown): boolean {
|
||||
const raw = error as { code?: unknown; errcode?: unknown; message?: unknown } | null;
|
||||
const code = raw?.code ?? raw?.errcode;
|
||||
return (
|
||||
(typeof code === 'string' && code.startsWith('SQLITE_BUSY')) ||
|
||||
(typeof raw?.message === 'string' && BUSY_MESSAGE.test(raw.message))
|
||||
);
|
||||
}
|
||||
|
||||
function syncSleep(ms: number): void {
|
||||
if (ms <= 0) return;
|
||||
try {
|
||||
Atomics.wait(new Int32Array(new SharedArrayBuffer(4)), 0, 0, ms);
|
||||
} catch {
|
||||
// If synchronous sleeping is unavailable, the bounded attempt count below
|
||||
// still prevents an infinite acquisition loop.
|
||||
}
|
||||
}
|
||||
|
||||
export function writerLockPathFor(dbPath: string): string {
|
||||
return join(dirname(dbPath), 'writer.lock.sqlite');
|
||||
}
|
||||
|
||||
export function acquireWriterLease({
|
||||
lockPath,
|
||||
openDb,
|
||||
waitMs = 0,
|
||||
retryDelayMs = 25,
|
||||
now = Date.now,
|
||||
sleep = syncSleep,
|
||||
}: AcquireWriterLeaseOptions): WriterLease | null {
|
||||
mkdirSync(dirname(lockPath), { recursive: true });
|
||||
const startedAt = now();
|
||||
const maxAttempts = waitMs > 0 ? Math.ceil(waitMs / Math.max(1, retryDelayMs)) + 1 : 1;
|
||||
|
||||
for (let attempt = 0; attempt < maxAttempts; attempt += 1) {
|
||||
const db = openDb(lockPath);
|
||||
try {
|
||||
db.exec('PRAGMA busy_timeout=0');
|
||||
db.exec('BEGIN IMMEDIATE');
|
||||
let released = false;
|
||||
return {
|
||||
release() {
|
||||
if (released) return;
|
||||
released = true;
|
||||
try {
|
||||
db.exec('ROLLBACK');
|
||||
} catch {
|
||||
// Closing the connection releases any remaining SQLite lock.
|
||||
} finally {
|
||||
db.close();
|
||||
}
|
||||
},
|
||||
};
|
||||
} catch (error) {
|
||||
db.close();
|
||||
if (!isBusy(error)) throw error;
|
||||
const remaining = waitMs - (now() - startedAt);
|
||||
if (remaining <= 0 || attempt + 1 >= maxAttempts) return null;
|
||||
sleep(Math.min(retryDelayMs, remaining));
|
||||
}
|
||||
}
|
||||
return null;
|
||||
}
|
||||
@@ -0,0 +1,11 @@
|
||||
{
|
||||
"extends": "./tsconfig.json",
|
||||
"compilerOptions": {
|
||||
"noEmit": false,
|
||||
"outDir": "dist",
|
||||
"rootDir": "src",
|
||||
"declaration": true,
|
||||
"rewriteRelativeImportExtensions": true
|
||||
},
|
||||
"include": ["src/**/*.ts", "src/**/*.mjs"]
|
||||
}
|
||||
@@ -0,0 +1,7 @@
|
||||
{
|
||||
"extends": "../../tsconfig.json",
|
||||
"compilerOptions": {
|
||||
"noEmit": true
|
||||
},
|
||||
"include": ["src/**/*"]
|
||||
}
|
||||
Reference in New Issue
Block a user