feat(core): add first-class Pi session indexing (#23)
Pi cannot be read as another linear JSONL stream. Its history is a tree with a durable leaf, orphan roots, branch summaries, and two compaction forms, so the active context is something the format states rather than something line order implies. The adapter keeps those semantics inside itself and projects the result into the existing canonical tables. Sessions are keyed by (normalized header cwd, header id) rather than by path, because Pi's --session-id lookup is project-local: two projects may reuse an id, while a move or an identical copy is still one session. Discovery covers both layouts Pi writes and fingerprints each file by mtime, ctime, size and inode, so a rewrite that preserves mtime is not read as unchanged. Abandoned branches are preserved rather than dropped. Visibility becomes three-state -- visible, inactive, hidden -- and helpers return only visible rows until includeInactive asks for the superseded path, labeling every row so a caller knows which it holds. Usage counts all three, because an abandoned call still spent tokens; message_count reports only the visible transcript. A committed MIT-licensed oracle transcribed from Pi 0.83.0 pins the context algorithms, and a fixed-seed differential runs 512 generated sessions against it on every test run. Schema changes are additive.
This commit is contained in:
+140
-36
@@ -13,10 +13,14 @@ import { buildEditorUrl, DEFAULT_EDITOR_SCHEME, EDITOR_SCHEMES, resolveFileRefer
|
||||
import { acquireWriterLease, writerLockPathFor } from '../../../packages/core/src/writer-lease.ts';
|
||||
import { migrateCoreSchemaColumns } from '../../../packages/core/src/schema-migrations.ts';
|
||||
import { createBuiltinProviderRegistry } from '../../../packages/core/src/providers/builtins.ts';
|
||||
import {
|
||||
createConfiguredBuiltinProviderRuntime,
|
||||
readPersistedProviderSettings,
|
||||
} from '../../../packages/core/src/provider-settings.ts';
|
||||
import {
|
||||
buildSourceCatalog,
|
||||
resolveProviderRoots,
|
||||
setPersistedSetting,
|
||||
type ProviderSourceIssue,
|
||||
} from './provider-settings.ts';
|
||||
import type {
|
||||
SessionPatchCursor,
|
||||
@@ -66,6 +70,8 @@ const DEFAULT_CODEX_DIR = path.join(os.homedir(), '.codex');
|
||||
let db;
|
||||
let indexerService;
|
||||
let indexerWorker;
|
||||
let latestSourceIssues: ProviderSourceIssue[] = [];
|
||||
let latestSettingsError: string | null = null;
|
||||
|
||||
type WriterLeaseMode = 'acquire' | 'caller-held';
|
||||
|
||||
@@ -78,16 +84,19 @@ function acquireAppWriterLease(dbPath: string, waitMs = 0) {
|
||||
}
|
||||
|
||||
function getRuntimePaths(persisted = loadPersistedSettings()) {
|
||||
const defaultRegistry = createBuiltinProviderRegistry({
|
||||
claude: DEFAULT_CLAUDE_DIR,
|
||||
codex: DEFAULT_CODEX_DIR,
|
||||
const runtime = createConfiguredBuiltinProviderRuntime(persisted, {
|
||||
baseRoots: {
|
||||
claude: DEFAULT_CLAUDE_DIR,
|
||||
codex: DEFAULT_CODEX_DIR,
|
||||
},
|
||||
});
|
||||
const providerRoots = resolveProviderRoots(defaultRegistry, persisted);
|
||||
const providerRegistry = createBuiltinProviderRegistry(providerRoots);
|
||||
const providerRoots = runtime.roots;
|
||||
const providerRegistry = runtime.registry;
|
||||
const claudeDir = providerRoots['claude'] ?? DEFAULT_CLAUDE_DIR;
|
||||
const codexDir = providerRoots['codex'] ?? DEFAULT_CODEX_DIR;
|
||||
return {
|
||||
providerRoots,
|
||||
providerSettings: persisted,
|
||||
providerRegistry,
|
||||
claudeDir,
|
||||
codexDir,
|
||||
@@ -209,11 +218,37 @@ function runAppDbWrite(work: () => void): boolean {
|
||||
}
|
||||
}
|
||||
|
||||
function notifyIndexUpdated(result: { affectedSessionIds?: unknown } = {}) {
|
||||
function notifyIndexUpdated(result: {
|
||||
affectedSessionIds?: unknown;
|
||||
inventoryIssues?: unknown;
|
||||
skippedFiles?: unknown;
|
||||
} = {}) {
|
||||
const affectedSessionIds = Array.isArray(result.affectedSessionIds)
|
||||
? [...new Set(result.affectedSessionIds.filter(Boolean))]
|
||||
: [];
|
||||
const payload = { affectedSessionIds };
|
||||
const issueLists = [result.inventoryIssues, result.skippedFiles]
|
||||
.filter(Array.isArray)
|
||||
.flat();
|
||||
if (Array.isArray(result.inventoryIssues) || Array.isArray(result.skippedFiles)) {
|
||||
const unique = new Map<string, ProviderSourceIssue>();
|
||||
for (const value of issueLists) {
|
||||
const issue = value as Partial<ProviderSourceIssue> | null;
|
||||
if (
|
||||
issue !== null
|
||||
&& typeof issue.provider === 'string'
|
||||
&& typeof issue.path === 'string'
|
||||
&& typeof issue.error === 'string'
|
||||
) {
|
||||
unique.set(`${issue.provider}\0${issue.path}\0${issue.error}`, {
|
||||
provider: issue.provider,
|
||||
path: issue.path,
|
||||
error: issue.error,
|
||||
});
|
||||
}
|
||||
}
|
||||
latestSourceIssues = [...unique.values()];
|
||||
}
|
||||
const payload = { affectedSessionIds, sourceIssues: latestSourceIssues };
|
||||
for (const win of BrowserWindow.getAllWindows()) {
|
||||
win.webContents.send('obelisk:index-updated', payload);
|
||||
for (const sessionId of affectedSessionIds) {
|
||||
@@ -235,6 +270,7 @@ function appendWhere(sql, params, clause) {
|
||||
|
||||
function startIndexerService({ buildOnStart = false } = {}) {
|
||||
const paths = getRuntimePaths();
|
||||
if (latestSettingsError !== null) return null;
|
||||
migrateLegacyDbIfNeeded(paths);
|
||||
indexerService = createIndexerService({
|
||||
projectsDir: paths.projectsDir,
|
||||
@@ -244,13 +280,18 @@ function startIndexerService({ buildOnStart = false } = {}) {
|
||||
reason,
|
||||
changedPaths,
|
||||
providerRoots: paths.providerRoots,
|
||||
providerSettings: paths.providerSettings,
|
||||
claudeDir: paths.claudeDir,
|
||||
codexDir: paths.codexDir,
|
||||
projectsDir: paths.projectsDir,
|
||||
dbPath: paths.dbPath,
|
||||
});
|
||||
if (result?.deferred) {
|
||||
if (Array.isArray(result.affectedSessionIds) && result.affectedSessionIds.length) {
|
||||
if (
|
||||
(Array.isArray(result.affectedSessionIds) && result.affectedSessionIds.length)
|
||||
|| (Array.isArray(result.inventoryIssues) && result.inventoryIssues.length)
|
||||
|| (Array.isArray(result.skippedFiles) && result.skippedFiles.length)
|
||||
) {
|
||||
notifyIndexUpdated(result);
|
||||
}
|
||||
} else {
|
||||
@@ -268,11 +309,11 @@ function startIndexerService({ buildOnStart = false } = {}) {
|
||||
function startBackgroundResources({ runStartupBuild = false } = {}) {
|
||||
if (!indexerWorker) indexerWorker = createWorkerBuildIndex();
|
||||
const paths = getRuntimePaths();
|
||||
migrateLegacyDbIfNeeded(paths);
|
||||
if (latestSettingsError === null) migrateLegacyDbIfNeeded(paths);
|
||||
openDb(paths.dbPath);
|
||||
if (!indexerService) {
|
||||
if (!indexerService && latestSettingsError === null) {
|
||||
const service = startIndexerService({ buildOnStart: false });
|
||||
if (runStartupBuild) service.runBuildNow('startup');
|
||||
if (runStartupBuild) service?.runBuildNow('startup');
|
||||
}
|
||||
if (!obeliskWatcher) startObeliskWatcher();
|
||||
}
|
||||
@@ -440,18 +481,29 @@ function querySessionMessages(sessionId: string): SessionMessageRow[] {
|
||||
SELECT m.uuid, m.session_id, m.type, m.parent_uuid, m.timestamp, m.role, m.text, m.model,
|
||||
m.is_sidechain, m.agent_id, m.input_tokens, m.output_tokens, m.cwd, m.skill, m.turn_duration_ms,
|
||||
m.content_type, m.is_meta, m.visibility, m.source
|
||||
FROM messages m WHERE m.session_id = ? AND m.agent_id IS NULL ORDER BY m.timestamp, m.uuid
|
||||
FROM messages m
|
||||
WHERE m.session_id = ? AND m.agent_id IS NULL
|
||||
AND COALESCE(m.visibility, 'visible') = 'visible'
|
||||
ORDER BY m.timestamp, m.uuid
|
||||
`).all(sessionId) as SessionMessageRow[];
|
||||
}
|
||||
|
||||
function querySessionToolCalls(sessionId: string): SessionToolCallRow[] {
|
||||
if (!db) return [];
|
||||
return db.prepare(`SELECT * FROM tool_calls WHERE session_id = ?`).all(sessionId) as SessionToolCallRow[];
|
||||
return db.prepare(`
|
||||
SELECT tc.* FROM tool_calls tc
|
||||
JOIN messages m ON m.uuid = tc.message_uuid
|
||||
WHERE tc.session_id = ? AND COALESCE(m.visibility, 'visible') = 'visible'
|
||||
`).all(sessionId) as SessionToolCallRow[];
|
||||
}
|
||||
|
||||
function querySessionToolResults(sessionId: string): SessionToolResultRow[] {
|
||||
if (!db) return [];
|
||||
return db.prepare(`SELECT * FROM tool_results WHERE session_id = ?`).all(sessionId) as SessionToolResultRow[];
|
||||
return db.prepare(`
|
||||
SELECT tr.* FROM tool_results tr
|
||||
JOIN messages m ON m.uuid = tr.message_uuid
|
||||
WHERE tr.session_id = ? AND COALESCE(m.visibility, 'visible') = 'visible'
|
||||
`).all(sessionId) as SessionToolResultRow[];
|
||||
}
|
||||
|
||||
function querySessionSubagents(sessionId: string): SessionSubagentRow[] {
|
||||
@@ -470,7 +522,10 @@ function querySessionWorkflows(sessionId: string): SessionWorkflowRow[] {
|
||||
|
||||
function querySessionSummaries(sessionId: string): SessionSummaryRow[] {
|
||||
if (!db) return [];
|
||||
return db.prepare(`SELECT * FROM summaries WHERE session_id = ?`).all(sessionId) as SessionSummaryRow[];
|
||||
return db.prepare(`
|
||||
SELECT * FROM summaries
|
||||
WHERE session_id = ? AND COALESCE(visibility, 'visible') = 'visible'
|
||||
`).all(sessionId) as SessionSummaryRow[];
|
||||
}
|
||||
|
||||
function querySessionSnapshot(sessionId: string): SessionDetailAssemblyInput {
|
||||
@@ -569,7 +624,9 @@ ipcMain.handle('db:getSubagentMessages', (_, agentId) => {
|
||||
SELECT m.uuid, m.session_id, m.type, m.parent_uuid, m.timestamp, m.role, m.text, m.model,
|
||||
m.is_sidechain, m.agent_id, m.input_tokens, m.output_tokens, m.cwd, m.skill, m.turn_duration_ms,
|
||||
m.content_type, m.is_meta, m.visibility, m.source
|
||||
FROM messages m WHERE m.agent_id = ? ORDER BY m.timestamp, m.uuid
|
||||
FROM messages m
|
||||
WHERE m.agent_id = ? AND COALESCE(m.visibility, 'visible') = 'visible'
|
||||
ORDER BY m.timestamp, m.uuid
|
||||
`).all(agentId);
|
||||
});
|
||||
|
||||
@@ -578,7 +635,7 @@ ipcMain.handle('db:getSubagentToolCalls', (_, agentId) => {
|
||||
return db.prepare(`
|
||||
SELECT tc.* FROM tool_calls tc
|
||||
JOIN messages m ON m.uuid = tc.message_uuid
|
||||
WHERE m.agent_id = ?
|
||||
WHERE m.agent_id = ? AND COALESCE(m.visibility, 'visible') = 'visible'
|
||||
`).all(agentId);
|
||||
});
|
||||
|
||||
@@ -587,7 +644,7 @@ ipcMain.handle('db:getSubagentToolResults', (_, agentId) => {
|
||||
return db.prepare(`
|
||||
SELECT tr.* FROM tool_results tr
|
||||
JOIN messages m ON m.uuid = tr.message_uuid
|
||||
WHERE m.agent_id = ?
|
||||
WHERE m.agent_id = ? AND COALESCE(m.visibility, 'visible') = 'visible'
|
||||
`).all(agentId);
|
||||
});
|
||||
|
||||
@@ -606,7 +663,7 @@ ipcMain.handle('db:getMemories', () => {
|
||||
ipcMain.handle('db:getMessageFullText', (_, uuid) => {
|
||||
if (!db) return null;
|
||||
const msg = db.prepare('SELECT * FROM messages WHERE uuid=?').get(uuid);
|
||||
if (!msg) return null;
|
||||
if (!msg || (msg.visibility ?? 'visible') !== 'visible') return null;
|
||||
const session = db.prepare('SELECT * FROM sessions WHERE id=?').get(msg.session_id) ?? null;
|
||||
const subagent = msg.agent_id
|
||||
? db.prepare('SELECT * FROM subagents WHERE agent_id=?').get(msg.agent_id) ?? null
|
||||
@@ -614,12 +671,16 @@ ipcMain.handle('db:getMessageFullText', (_, uuid) => {
|
||||
const workflowAgent = msg.agent_id
|
||||
? db.prepare('SELECT * FROM workflow_agents WHERE agent_id=?').get(msg.agent_id) ?? null
|
||||
: null;
|
||||
const cursorRow = typeof session?.jsonl_path === 'string'
|
||||
? db.prepare('SELECT cursor FROM index_state WHERE jsonl_path=?').get(session.jsonl_path)
|
||||
: undefined;
|
||||
const paths = getRuntimePaths();
|
||||
const raw = paths.providerRegistry.raw({
|
||||
source: msg.source || session?.source || 'claude',
|
||||
messageUuid: String(uuid),
|
||||
session,
|
||||
agentId: msg.agent_id || null,
|
||||
cursor: typeof cursorRow?.cursor === 'string' ? cursorRow.cursor : null,
|
||||
subagent,
|
||||
workflowAgent,
|
||||
});
|
||||
@@ -641,7 +702,9 @@ function querySessionFileRoots(sessionId: unknown): string[] {
|
||||
const roots: string[] = [];
|
||||
try {
|
||||
const rows = db.prepare(
|
||||
`SELECT DISTINCT cwd FROM messages WHERE session_id = ? AND cwd IS NOT NULL AND cwd != ''`
|
||||
`SELECT DISTINCT cwd FROM messages
|
||||
WHERE session_id = ? AND cwd IS NOT NULL AND cwd != ''
|
||||
AND COALESCE(visibility, 'visible') = 'visible'`
|
||||
).all(sessionId);
|
||||
for (const row of rows) roots.push(row.cwd);
|
||||
const session = db.prepare(`SELECT project_path FROM sessions WHERE id = ?`).get(sessionId);
|
||||
@@ -659,7 +722,12 @@ ipcMain.handle('file-ref:open', async (_, ref) => {
|
||||
if (!filePath) return { opened: false };
|
||||
const { editorScheme } = loadPersistedSettings();
|
||||
try {
|
||||
await shell.openExternal(buildEditorUrl({ scheme: editorScheme, filePath, line, column }));
|
||||
await shell.openExternal(buildEditorUrl({
|
||||
scheme: typeof editorScheme === 'string' ? editorScheme : undefined,
|
||||
filePath,
|
||||
line,
|
||||
column,
|
||||
}));
|
||||
return { opened: true, path: filePath };
|
||||
} catch {
|
||||
return { opened: false, path: filePath };
|
||||
@@ -705,11 +773,25 @@ ipcMain.handle('db:getUsageStats', (_, opts = {}) => {
|
||||
if (!db) return { daily: [], totalTokens: 0, peakDay: null, longestTurn: null };
|
||||
const sourceFilter = sourceWhereClause(opts, 'source');
|
||||
const sourceSql = sourceFilter.sql ? `AND ${sourceFilter.sql}` : '';
|
||||
const usageEvents = `
|
||||
WITH usage_events AS (
|
||||
SELECT timestamp, input_tokens, output_tokens, COALESCE(source, 'claude') AS source
|
||||
FROM messages
|
||||
UNION ALL
|
||||
SELECT su.timestamp, su.input_tokens, su.output_tokens,
|
||||
COALESCE(s.source, 'claude') AS source
|
||||
FROM summaries su
|
||||
LEFT JOIN sessions s ON s.id = su.session_id
|
||||
)
|
||||
`;
|
||||
// Visibility controls evidence display, not accounting. Abandoned model calls
|
||||
// still consumed tokens, so aggregate usage intentionally includes them.
|
||||
|
||||
const daily = db.prepare(`
|
||||
${usageEvents}
|
||||
SELECT DATE(timestamp) as day,
|
||||
SUM(COALESCE(input_tokens,0) + COALESCE(output_tokens,0)) as tokens
|
||||
FROM messages
|
||||
FROM usage_events
|
||||
WHERE timestamp IS NOT NULL AND (input_tokens IS NOT NULL OR output_tokens IS NOT NULL)
|
||||
${sourceSql}
|
||||
GROUP BY DATE(timestamp)
|
||||
@@ -717,15 +799,17 @@ ipcMain.handle('db:getUsageStats', (_, opts = {}) => {
|
||||
`).all(...sourceFilter.params);
|
||||
|
||||
const totalTokens = db.prepare(`
|
||||
${usageEvents}
|
||||
SELECT SUM(COALESCE(input_tokens,0) + COALESCE(output_tokens,0)) as total
|
||||
FROM messages
|
||||
FROM usage_events
|
||||
${sourceFilter.sql ? `WHERE ${sourceFilter.sql}` : ''}
|
||||
`).get(...sourceFilter.params)?.total || 0;
|
||||
|
||||
const peakDay = db.prepare(`
|
||||
${usageEvents}
|
||||
SELECT DATE(timestamp) as day,
|
||||
SUM(COALESCE(input_tokens,0) + COALESCE(output_tokens,0)) as tokens
|
||||
FROM messages
|
||||
FROM usage_events
|
||||
WHERE timestamp IS NOT NULL AND (input_tokens IS NOT NULL OR output_tokens IS NOT NULL)
|
||||
${sourceSql}
|
||||
GROUP BY DATE(timestamp)
|
||||
@@ -837,15 +921,25 @@ ipcMain.handle('recap:read', (_, filename) => {
|
||||
const SETTINGS_PATH = path.join(OBELISK_DIR, 'settings.json');
|
||||
|
||||
function loadPersistedSettings() {
|
||||
try {
|
||||
if (fs.existsSync(SETTINGS_PATH)) return JSON.parse(fs.readFileSync(SETTINGS_PATH, 'utf-8'));
|
||||
} catch {}
|
||||
return {};
|
||||
const result = readPersistedProviderSettings(SETTINGS_PATH);
|
||||
latestSettingsError = result.ok ? null : result.error ?? 'Obelisk settings are unavailable';
|
||||
if (latestSettingsError !== null) console.warn(latestSettingsError);
|
||||
return result.settings;
|
||||
}
|
||||
|
||||
function savePersistedSettings(settings) {
|
||||
if (!fs.existsSync(OBELISK_DIR)) fs.mkdirSync(OBELISK_DIR, { recursive: true });
|
||||
fs.writeFileSync(SETTINGS_PATH, JSON.stringify(settings, null, 2));
|
||||
const temporaryPath = `${SETTINGS_PATH}.${process.pid}.tmp`;
|
||||
try {
|
||||
fs.writeFileSync(temporaryPath, JSON.stringify(settings, null, 2));
|
||||
fs.renameSync(temporaryPath, SETTINGS_PATH);
|
||||
latestSettingsError = null;
|
||||
} catch (error) {
|
||||
try {
|
||||
fs.unlinkSync(temporaryPath);
|
||||
} catch {}
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
ipcMain.handle('settings:get', () => {
|
||||
@@ -878,11 +972,13 @@ ipcMain.handle('settings:get', () => {
|
||||
registry: providerRegistry,
|
||||
roots: providerRoots,
|
||||
stats: sourceStats,
|
||||
sourceIssues: latestSourceIssues,
|
||||
pathExists: fs.existsSync,
|
||||
});
|
||||
const sessionCount = sources.reduce((sum, source) => sum + source.sessionCount, 0);
|
||||
const lastIndexed = sources.map((source) => source.lastIndexed).filter(Boolean).sort().at(-1) || '';
|
||||
const connected = sources.some((source) => source.status !== 'error');
|
||||
const connected = latestSettingsError === null
|
||||
&& sources.some((source) => source.status !== 'error');
|
||||
|
||||
return {
|
||||
version: app.getVersion(),
|
||||
@@ -898,7 +994,7 @@ ipcMain.handle('settings:get', () => {
|
||||
sessionCount,
|
||||
lastIndexed,
|
||||
status: connected ? 'ok' : 'error',
|
||||
statusText: connected ? 'Connected' : 'No source folders found',
|
||||
statusText: latestSettingsError ?? (connected ? 'Connected' : 'No source folders found'),
|
||||
};
|
||||
});
|
||||
|
||||
@@ -910,8 +1006,8 @@ ipcMain.handle('settings:set', async (_, key, value) => {
|
||||
if (key === 'autoRefresh') {
|
||||
if (value === false && indexerService) {
|
||||
await stopIndexerServiceAndWait();
|
||||
} else if (value !== false && indexerService) {
|
||||
await stopIndexerServiceAndWait();
|
||||
} else if (value !== false) {
|
||||
if (indexerService) await stopIndexerServiceAndWait();
|
||||
startIndexerService({ buildOnStart: false });
|
||||
}
|
||||
}
|
||||
@@ -928,7 +1024,7 @@ ipcMain.handle('settings:set', async (_, key, value) => {
|
||||
if (persisted.autoRefresh !== false) {
|
||||
startIndexerService({ buildOnStart: true });
|
||||
}
|
||||
notifyIndexUpdated();
|
||||
notifyIndexUpdated({ inventoryIssues: [] });
|
||||
}
|
||||
return true;
|
||||
});
|
||||
@@ -951,6 +1047,7 @@ ipcMain.handle('settings:revealPath', (_, p) => {
|
||||
ipcMain.handle('settings:rebuildIndex', async () => {
|
||||
if (!indexerWorker) return null;
|
||||
const persisted = loadPersistedSettings();
|
||||
if (latestSettingsError !== null) throw new Error(latestSettingsError);
|
||||
const paths = getRuntimePaths(persisted);
|
||||
const tempDbPath = rebuildTempDbPath(paths.dbPath);
|
||||
const shouldRestartWatcher = persisted.autoRefresh !== false;
|
||||
@@ -977,6 +1074,9 @@ ipcMain.handle('settings:rebuildIndex', async () => {
|
||||
skipped: 0,
|
||||
skippedFiles: [],
|
||||
deferred: true,
|
||||
complete: false,
|
||||
incompleteProviders: [],
|
||||
inventoryIssues: [],
|
||||
reason: 'writer_busy',
|
||||
};
|
||||
}
|
||||
@@ -985,6 +1085,7 @@ ipcMain.handle('settings:rebuildIndex', async () => {
|
||||
reason: 'manual-rebuild',
|
||||
force: true,
|
||||
providerRoots: paths.providerRoots,
|
||||
providerSettings: paths.providerSettings,
|
||||
claudeDir: paths.claudeDir,
|
||||
codexDir: paths.codexDir,
|
||||
projectsDir: paths.projectsDir,
|
||||
@@ -993,7 +1094,10 @@ ipcMain.handle('settings:rebuildIndex', async () => {
|
||||
writerLeasePath,
|
||||
writerLeaseMode: 'caller-held',
|
||||
});
|
||||
if (result?.deferred) return result;
|
||||
if (result?.deferred || result?.complete !== true) {
|
||||
notifyIndexUpdated(result);
|
||||
return result;
|
||||
}
|
||||
closeDb();
|
||||
replaceDbWithTemp(tempDbPath, paths.dbPath);
|
||||
openDb(paths.dbPath, { writerLeaseMode: 'caller-held' });
|
||||
|
||||
@@ -21,10 +21,17 @@ interface Timers {
|
||||
|
||||
interface Watcher {
|
||||
close(): unknown;
|
||||
refreshMissingRoots?(): boolean;
|
||||
}
|
||||
|
||||
interface IndexerBuildResult {
|
||||
deferred?: boolean;
|
||||
complete?: boolean;
|
||||
inventoryIssues?: Array<{
|
||||
provider: string;
|
||||
path: string;
|
||||
error: string;
|
||||
}>;
|
||||
}
|
||||
|
||||
type IndexerBuild = (args: {
|
||||
@@ -42,7 +49,7 @@ interface IndexerServiceOptions {
|
||||
deferredRetryMs?: number;
|
||||
buildIndex?: IndexerBuild;
|
||||
writeHeartbeat?: () => unknown;
|
||||
watchProjects?: (onChange: (changedPath: string) => void) => Watcher | null;
|
||||
watchProjects?: (onChange: (changedPath?: string) => void) => Watcher | null;
|
||||
chokidar?: any;
|
||||
timers?: Timers;
|
||||
logger?: { warn?: (msg: string) => void };
|
||||
@@ -71,10 +78,11 @@ function createIndexerService({
|
||||
if (typeof buildIndex !== 'function') throw new Error('createIndexerService() requires buildIndex');
|
||||
const watch = watchProjects || ((onChange) => {
|
||||
const roots = [...new Set((Array.isArray(watchDirs) ? watchDirs : [watchDirs]).filter(Boolean))];
|
||||
const existingRoots = roots.filter(root => fs.existsSync(root));
|
||||
if (!existingRoots.length) return null;
|
||||
if (!roots.length) return null;
|
||||
const watchers: any[] = [];
|
||||
for (const root of existingRoots) {
|
||||
const watchedRoots = new Set<string>();
|
||||
const addRoot = (root: string) => {
|
||||
if (watchedRoots.has(root) || !fs.existsSync(root)) return false;
|
||||
const onFileChange = (filename) => {
|
||||
const name = filename ? String(filename) : '';
|
||||
if (!name || name.endsWith('.jsonl') || name.endsWith('.json')) {
|
||||
@@ -102,11 +110,25 @@ function createIndexerService({
|
||||
logger.warn?.(`Obelisk watcher failed: ${(error as Error).message}`);
|
||||
});
|
||||
watchers.push(watcher);
|
||||
}
|
||||
watchedRoots.add(root);
|
||||
return true;
|
||||
};
|
||||
const refreshMissingRoots = (notify: boolean) => {
|
||||
let added = false;
|
||||
for (const root of roots) {
|
||||
if (addRoot(root)) added = true;
|
||||
}
|
||||
if (added && notify) onChange();
|
||||
return watchedRoots.size === roots.length;
|
||||
};
|
||||
refreshMissingRoots(false);
|
||||
return {
|
||||
close() {
|
||||
return Promise.all(watchers.map(w => Promise.resolve(w.close?.())));
|
||||
},
|
||||
refreshMissingRoots() {
|
||||
return refreshMissingRoots(true);
|
||||
},
|
||||
};
|
||||
});
|
||||
|
||||
@@ -121,18 +143,29 @@ function createIndexerService({
|
||||
let pending = false;
|
||||
let lastReason: string | null = null;
|
||||
let changedPaths = new Set<string>();
|
||||
let fullInventoryPending = false;
|
||||
let idlePromise = Promise.resolve();
|
||||
|
||||
const requestFullInventory = () => {
|
||||
fullInventoryPending = true;
|
||||
changedPaths.clear();
|
||||
};
|
||||
|
||||
const addChangedPath = (changedPath?: string | string[]) => {
|
||||
if (Array.isArray(changedPath)) {
|
||||
for (const p of changedPath) addChangedPath(p);
|
||||
return;
|
||||
}
|
||||
const name = changedPath ? String(changedPath) : '';
|
||||
if (name) changedPaths.add(name);
|
||||
if (name && !fullInventoryPending) changedPaths.add(name);
|
||||
};
|
||||
|
||||
const takeChangedPaths = () => {
|
||||
if (fullInventoryPending) {
|
||||
fullInventoryPending = false;
|
||||
changedPaths.clear();
|
||||
return undefined;
|
||||
}
|
||||
if (!changedPaths.size) return undefined;
|
||||
const paths = [...changedPaths];
|
||||
changedPaths = new Set();
|
||||
@@ -160,8 +193,16 @@ function createIndexerService({
|
||||
const buildChangedPaths = takeChangedPaths();
|
||||
idlePromise = (async () => {
|
||||
const result = await buildIndex({ reason, changedPaths: buildChangedPaths });
|
||||
if (result?.complete === false) {
|
||||
for (const issue of result.inventoryIssues ?? []) {
|
||||
logger.warn?.(
|
||||
`Obelisk indexed a partial ${issue.provider} inventory at ${issue.path}: ${issue.error}`,
|
||||
);
|
||||
}
|
||||
}
|
||||
if (result?.deferred) {
|
||||
addChangedPath(buildChangedPaths);
|
||||
if (buildChangedPaths === undefined) requestFullInventory();
|
||||
else addChangedPath(buildChangedPaths);
|
||||
if (!stopped && !deferredRetryTimer) {
|
||||
deferredRetryTimer = timers.setTimeout(() => {
|
||||
deferredRetryTimer = null;
|
||||
@@ -189,7 +230,8 @@ function createIndexerService({
|
||||
|
||||
const scheduleBuild = (reason = "change", changedPath: string | undefined = undefined) => {
|
||||
if (stopped) return;
|
||||
addChangedPath(changedPath);
|
||||
if (changedPath === undefined) requestFullInventory();
|
||||
else addChangedPath(changedPath);
|
||||
lastReason = reason;
|
||||
if (running) pending = true;
|
||||
if (deferredRetryTimer) timers.clearTimeout(deferredRetryTimer);
|
||||
@@ -209,14 +251,25 @@ function createIndexerService({
|
||||
}, debounceMs);
|
||||
};
|
||||
|
||||
const scheduleWatchRetry = () => {
|
||||
if (stopped || watchRetryTimer) return;
|
||||
watchRetryTimer = timers.setTimeout(() => {
|
||||
watchRetryTimer = null;
|
||||
if (!watcher) {
|
||||
startWatching();
|
||||
return;
|
||||
}
|
||||
if (watcher.refreshMissingRoots?.() === false) scheduleWatchRetry();
|
||||
}, watchRetryMs);
|
||||
};
|
||||
|
||||
const startWatching = () => {
|
||||
if (stopped || watcher) return;
|
||||
watcher = watch((changedPath) => scheduleBuild('watch', changedPath));
|
||||
if (!watcher) {
|
||||
watchRetryTimer = timers.setTimeout(() => {
|
||||
watchRetryTimer = null;
|
||||
startWatching();
|
||||
}, watchRetryMs);
|
||||
scheduleWatchRetry();
|
||||
} else if (watcher.refreshMissingRoots?.() === false) {
|
||||
scheduleWatchRetry();
|
||||
}
|
||||
};
|
||||
|
||||
|
||||
+214
-82
@@ -5,10 +5,18 @@ import { fileURLToPath } from 'node:url';
|
||||
import Database from 'better-sqlite3';
|
||||
import { createBuiltinProviderRegistry } from '../../../packages/core/src/providers/builtins.ts';
|
||||
import type { ProviderRegistry } from '../../../packages/core/src/providers/registry.ts';
|
||||
import {
|
||||
createConfiguredBuiltinProviderRuntime,
|
||||
type PersistedProviderSettings,
|
||||
} from '../../../packages/core/src/provider-settings.ts';
|
||||
import {
|
||||
createProviderIndexPlan,
|
||||
indexProviderPlan,
|
||||
indexProviderPlanStrict,
|
||||
ProviderIndexFailure,
|
||||
writeProviderIndexMarkers,
|
||||
type ProviderInventoryIssue,
|
||||
type ProviderSessionProvenance,
|
||||
} from '../../../packages/core/src/provider-indexing.ts';
|
||||
import { runWriteTransaction, configureConnection, betterSqliteTransactionAdapter } from '../../../packages/core/src/tx.ts';
|
||||
import { migrateCoreSchemaColumns } from '../../../packages/core/src/schema-migrations.ts';
|
||||
@@ -48,40 +56,68 @@ function openIndexDb({ dbPath = DEFAULT_DB_PATH, schemaPath = resolveSchemaPath(
|
||||
return db;
|
||||
}
|
||||
|
||||
function copyMemoriesFromDb(db, sourceDbPath) {
|
||||
if (!sourceDbPath || !fs.existsSync(sourceDbPath)) return false;
|
||||
function copyPreservedDataFromDb(db, sourceDbPath): ProviderSessionProvenance[] {
|
||||
if (!sourceDbPath || !fs.existsSync(sourceDbPath)) {
|
||||
throw new Error(`Preserved Obelisk database is unavailable: ${sourceDbPath}`);
|
||||
}
|
||||
db.prepare('ATTACH DATABASE ? AS previous_obelisk').run(sourceDbPath);
|
||||
try {
|
||||
const hasMemories = db.prepare(`
|
||||
SELECT name FROM previous_obelisk.sqlite_master
|
||||
WHERE type='table' AND name='memories'
|
||||
`).get();
|
||||
if (!hasMemories) return false;
|
||||
if (hasMemories) {
|
||||
const sourceColumns = new Set(
|
||||
db.prepare('PRAGMA previous_obelisk.table_info(memories)').all().map(column => column.name),
|
||||
);
|
||||
const targetColumns = [
|
||||
'id',
|
||||
'session_id',
|
||||
'project',
|
||||
'message_start',
|
||||
'message_end',
|
||||
'path',
|
||||
'anchors',
|
||||
'summary',
|
||||
'created_at',
|
||||
'deleted_at',
|
||||
'deleted_reason',
|
||||
];
|
||||
const selectList = targetColumns
|
||||
.map(column => sourceColumns.has(column) ? column : `NULL AS ${column}`)
|
||||
.join(',');
|
||||
db.exec(`
|
||||
INSERT OR REPLACE INTO memories (${targetColumns.join(',')})
|
||||
SELECT ${selectList} FROM previous_obelisk.memories
|
||||
`);
|
||||
}
|
||||
|
||||
const sourceColumns = new Set(
|
||||
db.prepare('PRAGMA previous_obelisk.table_info(memories)').all().map(column => column.name),
|
||||
const hasSessions = db.prepare(`
|
||||
SELECT name FROM previous_obelisk.sqlite_master
|
||||
WHERE type='table' AND name='sessions'
|
||||
`).get();
|
||||
if (!hasSessions) {
|
||||
throw new Error(`Preserved Obelisk database has no sessions table: ${sourceDbPath}`);
|
||||
}
|
||||
const sessionColumns = new Set(
|
||||
db.prepare('PRAGMA previous_obelisk.table_info(sessions)').all().map(column => column.name),
|
||||
);
|
||||
const targetColumns = [
|
||||
'id',
|
||||
'session_id',
|
||||
'project',
|
||||
'message_start',
|
||||
'message_end',
|
||||
'path',
|
||||
'anchors',
|
||||
'summary',
|
||||
'created_at',
|
||||
'deleted_at',
|
||||
'deleted_reason',
|
||||
];
|
||||
const selectList = targetColumns
|
||||
.map(column => sourceColumns.has(column) ? column : `NULL AS ${column}`)
|
||||
.join(',');
|
||||
db.exec(`
|
||||
INSERT OR REPLACE INTO memories (${targetColumns.join(',')})
|
||||
SELECT ${selectList} FROM previous_obelisk.memories
|
||||
`);
|
||||
return true;
|
||||
if (!sessionColumns.has('id') || !sessionColumns.has('jsonl_path')) {
|
||||
throw new Error(`Preserved Obelisk sessions schema is incomplete: ${sourceDbPath}`);
|
||||
}
|
||||
const sourceExpression = sessionColumns.has('source')
|
||||
? "COALESCE(source, 'claude')"
|
||||
: "'claude'";
|
||||
return db.prepare(`
|
||||
SELECT id, jsonl_path, ${sourceExpression} AS source
|
||||
FROM previous_obelisk.sessions
|
||||
WHERE jsonl_path IS NOT NULL
|
||||
AND jsonl_path != ''
|
||||
`).all().map(row => ({
|
||||
source: String(row.source),
|
||||
sessionId: String(row.id),
|
||||
jsonlPath: String(row.jsonl_path),
|
||||
}));
|
||||
} finally {
|
||||
db.exec('DETACH DATABASE previous_obelisk');
|
||||
}
|
||||
@@ -190,6 +226,7 @@ function writeHeartbeat({
|
||||
|
||||
interface BuildIndexOptions {
|
||||
providerRoots?: Record<string, string>;
|
||||
providerSettings?: PersistedProviderSettings;
|
||||
providerRegistry?: ProviderRegistry;
|
||||
claudeDir?: string;
|
||||
codexDir?: string;
|
||||
@@ -207,6 +244,7 @@ interface BuildIndexOptions {
|
||||
}
|
||||
|
||||
interface SkippedFile {
|
||||
provider: string;
|
||||
path: string;
|
||||
error: string;
|
||||
diagnostics?: unknown;
|
||||
@@ -220,6 +258,9 @@ interface BuildIndexResult {
|
||||
skipped: number;
|
||||
skippedFiles: SkippedFile[];
|
||||
deferred: boolean;
|
||||
complete: boolean;
|
||||
incompleteProviders: string[];
|
||||
inventoryIssues: ProviderInventoryIssue[];
|
||||
reason?: string;
|
||||
}
|
||||
|
||||
@@ -234,6 +275,9 @@ function deferredBuildResult(
|
||||
ftsRebuilt: false,
|
||||
skipped: 0,
|
||||
skippedFiles: [],
|
||||
complete: false,
|
||||
incompleteProviders: [],
|
||||
inventoryIssues: [],
|
||||
...overrides,
|
||||
deferred: true,
|
||||
reason,
|
||||
@@ -242,6 +286,7 @@ function deferredBuildResult(
|
||||
|
||||
function buildIndex({
|
||||
providerRoots = {},
|
||||
providerSettings,
|
||||
providerRegistry,
|
||||
claudeDir = DEFAULT_CLAUDE_DIR,
|
||||
codexDir = path.join(path.dirname(claudeDir), '.codex'),
|
||||
@@ -276,13 +321,15 @@ function buildIndex({
|
||||
const txDb = betterSqliteTransactionAdapter(db);
|
||||
let messageFtsTriggersDropped = false;
|
||||
try {
|
||||
let priorSessions: ProviderSessionProvenance[] | undefined;
|
||||
if (preserveDbPath && path.resolve(preserveDbPath) !== path.resolve(dbPath)) {
|
||||
copyMemoriesFromDb(db, preserveDbPath);
|
||||
priorSessions = copyPreservedDataFromDb(db, preserveDbPath);
|
||||
}
|
||||
const defaultHome = os.homedir();
|
||||
const compatibilityHome = path.dirname(claudeDir);
|
||||
const relocatedDefaults = Object.fromEntries(
|
||||
createBuiltinProviderRegistry().catalog().map((descriptor) => {
|
||||
createBuiltinProviderRegistry().catalog().flatMap((descriptor) => {
|
||||
if (descriptor.requiresExplicitRoot) return [];
|
||||
const relativeDefault = path.relative(defaultHome, descriptor.defaultRoot);
|
||||
const root = compatibilityHome !== defaultHome
|
||||
&& relativeDefault
|
||||
@@ -290,7 +337,7 @@ function buildIndex({
|
||||
&& !path.isAbsolute(relativeDefault)
|
||||
? path.join(compatibilityHome, relativeDefault)
|
||||
: descriptor.defaultRoot;
|
||||
return [descriptor.id, root];
|
||||
return [[descriptor.id, root]];
|
||||
}),
|
||||
);
|
||||
const roots = {
|
||||
@@ -299,8 +346,15 @@ function buildIndex({
|
||||
codex: codexDir,
|
||||
...providerRoots,
|
||||
};
|
||||
const registry = providerRegistry ?? createBuiltinProviderRegistry(roots);
|
||||
const providerPlan = createProviderIndexPlan(db, registry, { force, changedPaths });
|
||||
const registry = providerRegistry
|
||||
?? (providerSettings === undefined
|
||||
? createBuiltinProviderRegistry(roots)
|
||||
: createConfiguredBuiltinProviderRuntime(providerSettings, { baseRoots: roots }).registry);
|
||||
const providerPlan = createProviderIndexPlan(db, registry, {
|
||||
force,
|
||||
changedPaths,
|
||||
priorSessions,
|
||||
});
|
||||
let latestSourceMtime = providerPlan.items.reduce((latest, { unit }) => {
|
||||
const providerCursor = (unit.meta as { currentCursor?: unknown } | undefined)?.currentCursor;
|
||||
if (typeof providerCursor === 'string') {
|
||||
@@ -312,45 +366,32 @@ function buildIndex({
|
||||
return latest;
|
||||
}
|
||||
}, 0);
|
||||
|
||||
try {
|
||||
if (force) {
|
||||
runRetryableWriteTransaction(txDb, () => {
|
||||
dropMessageFtsTriggers(db);
|
||||
db.prepare("DELETE FROM index_state WHERE substr(jsonl_path, 1, 2) != '__'").run();
|
||||
db.prepare("DELETE FROM messages").run();
|
||||
db.prepare("DELETE FROM tool_calls").run();
|
||||
db.prepare("DELETE FROM tool_results").run();
|
||||
db.prepare("DELETE FROM sessions").run();
|
||||
db.prepare("DELETE FROM summaries").run();
|
||||
db.prepare("DELETE FROM subagents").run();
|
||||
db.prepare("DELETE FROM workflows").run();
|
||||
db.prepare("DELETE FROM workflow_agents").run();
|
||||
}, { label: 'force-cleanup' });
|
||||
messageFtsTriggersDropped = true;
|
||||
}
|
||||
} catch (error) {
|
||||
if (isBeginBusyFailure(error)) {
|
||||
return deferredBuildResult('database_busy', {
|
||||
files: providerPlan.items.length,
|
||||
latestSourceMtime,
|
||||
});
|
||||
}
|
||||
throw error;
|
||||
const discoveredSourceMtime = latestSourceMtime;
|
||||
const incompleteProviders = [...providerPlan.incompleteProviders].sort();
|
||||
const inventoryIssues = [...providerPlan.inventoryIssues];
|
||||
if (force && incompleteProviders.length > 0) {
|
||||
return {
|
||||
files: providerPlan.items.length,
|
||||
latestSourceMtime,
|
||||
affectedSessionIds: [],
|
||||
ftsRebuilt: false,
|
||||
skipped: 0,
|
||||
skippedFiles: [],
|
||||
deferred: false,
|
||||
complete: false,
|
||||
incompleteProviders,
|
||||
inventoryIssues,
|
||||
reason: 'incomplete_snapshot',
|
||||
};
|
||||
}
|
||||
|
||||
const affectedSessionIds = new Set<string>();
|
||||
const finalizeAffectedSessionIds = new Set<string>();
|
||||
const changedMetaJsonlPaths = new Set<string>();
|
||||
if (Array.isArray(changedPaths)) {
|
||||
for (const changedPath of changedPaths) {
|
||||
const sessionId = sessionIdFromChangedPath(projectsDir, changedPath);
|
||||
const normalizedChangedPath = normalizeChangedPath(projectsDir, changedPath);
|
||||
const isMetaChange = normalizedChangedPath?.toLowerCase().endsWith('.meta.json');
|
||||
if (isMetaChange && normalizedChangedPath) {
|
||||
changedMetaJsonlPaths.add(
|
||||
normalizedChangedPath.slice(0, -'.meta.json'.length) + '.jsonl',
|
||||
);
|
||||
}
|
||||
// Transcript files report their session only after their own transaction
|
||||
// commits. Workflow changes are applied during finalize, so stage those
|
||||
// IDs until the finalize transaction commits. Meta files map back to their
|
||||
@@ -361,23 +402,118 @@ function buildIndex({
|
||||
}
|
||||
}
|
||||
const skipped: SkippedFile[] = [];
|
||||
let ftsRebuilt = false;
|
||||
const noteCommitted = ({ unit }, nextCursor) => {
|
||||
if (nextCursor) {
|
||||
latestSourceMtime = Math.max(
|
||||
latestSourceMtime,
|
||||
Number(nextCursor.split(':')[0]) || 0,
|
||||
);
|
||||
}
|
||||
if (unit.sessionId) affectedSessionIds.add(unit.sessionId);
|
||||
for (const sessionId of unit.retractSessionIds ?? []) affectedSessionIds.add(sessionId);
|
||||
};
|
||||
const finalize = (providerResult) => {
|
||||
refreshSessionProjectPaths(db);
|
||||
if (messageFtsTriggersDropped) installSchema(db, schemaPath);
|
||||
ftsRebuilt = ensureFtsReady(db, { force });
|
||||
writeIndexMarker(db, '__last_build__');
|
||||
if (providerResult.complete) writeIndexMarker(db, '__app_last_successful_build__');
|
||||
writeIndexMarker(db, '__indexer_owner_app__');
|
||||
writeProviderIndexMarkers(db, providerPlan, providerResult);
|
||||
if (latestSourceMtime) writeIndexMarker(db, '__last_source_mtime__', latestSourceMtime);
|
||||
};
|
||||
|
||||
if (force) {
|
||||
try {
|
||||
runRetryableWriteTransaction(txDb, () => {
|
||||
latestSourceMtime = discoveredSourceMtime;
|
||||
affectedSessionIds.clear();
|
||||
skipped.splice(0);
|
||||
ftsRebuilt = false;
|
||||
dropMessageFtsTriggers(db);
|
||||
messageFtsTriggersDropped = true;
|
||||
// The provider contract reserves no key prefix. A force snapshot
|
||||
// recreates every unit cursor, provider marker, and system marker.
|
||||
db.prepare('DELETE FROM index_state').run();
|
||||
for (const table of ['messages', 'tool_calls', 'tool_results', 'sessions', 'summaries', 'subagents', 'workflows', 'workflow_agents']) {
|
||||
db.prepare(`DELETE FROM ${table}`).run();
|
||||
}
|
||||
const providerResult = indexProviderPlanStrict({
|
||||
db,
|
||||
plan: providerPlan,
|
||||
onCommitted: noteCommitted,
|
||||
});
|
||||
finalize(providerResult);
|
||||
}, { label: 'force-rebuild' });
|
||||
} catch (error) {
|
||||
if (isBeginBusyFailure(error)) {
|
||||
return deferredBuildResult('database_busy', {
|
||||
files: providerPlan.items.length,
|
||||
latestSourceMtime: discoveredSourceMtime,
|
||||
incompleteProviders,
|
||||
inventoryIssues,
|
||||
});
|
||||
}
|
||||
if (error instanceof ProviderIndexFailure) {
|
||||
skipped.push({
|
||||
provider: error.item.provider.name,
|
||||
path: error.item.unit.key,
|
||||
error: error.sourceError instanceof Error
|
||||
? error.sourceError.message
|
||||
: String(error.sourceError),
|
||||
diagnostics: (error as { obelisk?: unknown }).obelisk
|
||||
?? (error.sourceError as { obelisk?: unknown } | null)?.obelisk,
|
||||
});
|
||||
console.warn(
|
||||
`Warning: failed to index ${error.item.provider.name} unit ${error.item.unit.key}: ${error.message}`,
|
||||
);
|
||||
return {
|
||||
files: providerPlan.items.length,
|
||||
latestSourceMtime: discoveredSourceMtime,
|
||||
affectedSessionIds: [],
|
||||
ftsRebuilt: false,
|
||||
skipped: skipped.length,
|
||||
skippedFiles: skipped,
|
||||
deferred: false,
|
||||
complete: false,
|
||||
incompleteProviders,
|
||||
inventoryIssues,
|
||||
reason: 'provider_failure',
|
||||
};
|
||||
}
|
||||
throw error;
|
||||
}
|
||||
for (const sessionId of finalizeAffectedSessionIds) affectedSessionIds.add(sessionId);
|
||||
return {
|
||||
files: providerPlan.items.length,
|
||||
latestSourceMtime,
|
||||
affectedSessionIds: [...affectedSessionIds],
|
||||
ftsRebuilt,
|
||||
skipped: 0,
|
||||
skippedFiles: [],
|
||||
deferred: false,
|
||||
complete: true,
|
||||
incompleteProviders,
|
||||
inventoryIssues,
|
||||
};
|
||||
}
|
||||
|
||||
const providerResult = indexProviderPlan({
|
||||
db,
|
||||
plan: providerPlan,
|
||||
runTransaction: (label, work) => runRetryableWriteTransaction(txDb, work, { label }),
|
||||
onCommitted: ({ unit }, nextCursor) => {
|
||||
if (nextCursor) latestSourceMtime = Math.max(latestSourceMtime, Number(nextCursor.split(':')[0]) || 0);
|
||||
if (unit.sessionId) affectedSessionIds.add(unit.sessionId);
|
||||
},
|
||||
onCommitted: noteCommitted,
|
||||
onError: (error, { provider, unit }) => {
|
||||
if (isBeginBusyFailure(error)) return 'stop';
|
||||
if (hasUnusableTransaction(error)) throw error;
|
||||
skipped.push({
|
||||
provider: provider.name,
|
||||
path: unit.key,
|
||||
error: (error as Error).message,
|
||||
diagnostics: (error as { obelisk?: unknown }).obelisk,
|
||||
error: error instanceof Error ? error.message : String(error),
|
||||
diagnostics: (error as { obelisk?: unknown })?.obelisk,
|
||||
});
|
||||
console.warn(`Warning: failed to index ${provider.name} unit ${unit.key}: ${(error as Error).message}`);
|
||||
console.warn(`Warning: failed to index ${provider.name} unit ${unit.key}: ${error instanceof Error ? error.message : String(error)}`);
|
||||
return 'skip';
|
||||
},
|
||||
});
|
||||
@@ -388,22 +524,13 @@ function buildIndex({
|
||||
affectedSessionIds: [...affectedSessionIds],
|
||||
skipped: skipped.length,
|
||||
skippedFiles: skipped,
|
||||
incompleteProviders,
|
||||
inventoryIssues,
|
||||
});
|
||||
}
|
||||
let ftsRebuilt = false;
|
||||
// Finalize is one transaction; a failure here fails the whole build (the
|
||||
// index would otherwise be left inconsistent).
|
||||
// Finalize is one transaction; a failure here fails the whole build.
|
||||
try {
|
||||
runRetryableWriteTransaction(txDb, () => {
|
||||
refreshSessionProjectPaths(db);
|
||||
if (messageFtsTriggersDropped) installSchema(db, schemaPath);
|
||||
ftsRebuilt = ensureFtsReady(db, { force });
|
||||
writeIndexMarker(db, '__last_build__');
|
||||
writeIndexMarker(db, '__app_last_successful_build__');
|
||||
writeIndexMarker(db, '__indexer_owner_app__');
|
||||
writeProviderIndexMarkers(db, providerPlan, providerResult);
|
||||
if (latestSourceMtime) writeIndexMarker(db, '__last_source_mtime__', latestSourceMtime);
|
||||
}, { label: 'finalize' });
|
||||
runRetryableWriteTransaction(txDb, () => finalize(providerResult), { label: 'finalize' });
|
||||
} catch (error) {
|
||||
if (isBeginBusyFailure(error)) {
|
||||
return deferredBuildResult('database_busy', {
|
||||
@@ -412,6 +539,8 @@ function buildIndex({
|
||||
affectedSessionIds: [...affectedSessionIds],
|
||||
skipped: skipped.length,
|
||||
skippedFiles: skipped,
|
||||
incompleteProviders,
|
||||
inventoryIssues,
|
||||
});
|
||||
}
|
||||
throw error;
|
||||
@@ -425,6 +554,9 @@ function buildIndex({
|
||||
skipped: skipped.length,
|
||||
skippedFiles: skipped,
|
||||
deferred: false,
|
||||
complete: providerResult.complete,
|
||||
incompleteProviders,
|
||||
inventoryIssues,
|
||||
};
|
||||
} finally {
|
||||
if (messageFtsTriggersDropped) {
|
||||
|
||||
@@ -1,4 +1,5 @@
|
||||
import type { ProviderRegistry } from '../../../packages/core/src/providers/registry.ts';
|
||||
export { resolveProviderRoots } from '../../../packages/core/src/provider-settings.ts';
|
||||
|
||||
type PersistedSettings = Record<string, unknown> & {
|
||||
providerRoots?: Record<string, unknown>;
|
||||
@@ -9,30 +10,20 @@ interface SourceStats {
|
||||
lastIndexed: string;
|
||||
}
|
||||
|
||||
export interface ProviderSourceIssue {
|
||||
readonly provider: string;
|
||||
readonly path: string;
|
||||
readonly error: string;
|
||||
}
|
||||
|
||||
interface BuildSourceCatalogOptions {
|
||||
registry: ProviderRegistry;
|
||||
roots: Readonly<Record<string, string>>;
|
||||
stats?: ReadonlyMap<string, SourceStats>;
|
||||
sourceIssues?: readonly ProviderSourceIssue[];
|
||||
pathExists?: (path: string) => boolean;
|
||||
}
|
||||
|
||||
function configuredPath(value: unknown): string | null {
|
||||
return typeof value === 'string' && value.trim().length > 0 ? value : null;
|
||||
}
|
||||
|
||||
export function resolveProviderRoots(
|
||||
registry: ProviderRegistry,
|
||||
persisted: PersistedSettings = {},
|
||||
): Record<string, string> {
|
||||
const configured = persisted.providerRoots ?? {};
|
||||
return Object.fromEntries(registry.catalog().map((descriptor) => {
|
||||
const root = configuredPath(configured[descriptor.id])
|
||||
?? configuredPath(persisted[`${descriptor.id}Dir`])
|
||||
?? descriptor.defaultRoot;
|
||||
return [descriptor.id, root];
|
||||
}));
|
||||
}
|
||||
|
||||
export function setPersistedSetting(
|
||||
persisted: PersistedSettings,
|
||||
key: string,
|
||||
@@ -46,7 +37,9 @@ export function setPersistedSetting(
|
||||
}
|
||||
|
||||
const providerId = providerMatch[1]!;
|
||||
const roots = persisted.providerRoots && typeof persisted.providerRoots === 'object'
|
||||
const roots = persisted.providerRoots
|
||||
&& typeof persisted.providerRoots === 'object'
|
||||
&& !Array.isArray(persisted.providerRoots)
|
||||
? persisted.providerRoots
|
||||
: {};
|
||||
if (value === null) delete roots[providerId];
|
||||
@@ -60,13 +53,23 @@ export function buildSourceCatalog({
|
||||
registry,
|
||||
roots,
|
||||
stats = new Map(),
|
||||
sourceIssues = [],
|
||||
pathExists = () => false,
|
||||
}: BuildSourceCatalogOptions) {
|
||||
return registry.catalog().map((descriptor) => {
|
||||
const path = roots[descriptor.id] ?? descriptor.defaultRoot;
|
||||
const exists = pathExists(path);
|
||||
const needsRoot = descriptor.requiresExplicitRoot === true;
|
||||
const sourceStats = stats.get(descriptor.id) ?? { sessionCount: 0, lastIndexed: '' };
|
||||
const status = !exists ? 'error' : sourceStats.sessionCount > 0 ? 'ok' : 'warn';
|
||||
const issue = sourceIssues.find((candidate) => candidate.provider === descriptor.id);
|
||||
const status = needsRoot || !exists
|
||||
? 'error'
|
||||
: issue !== undefined || sourceStats.sessionCount === 0
|
||||
? 'warn'
|
||||
: 'ok';
|
||||
const partialStatus = issue === undefined
|
||||
? null
|
||||
: `Index issue: ${issue.path} — ${issue.error}`;
|
||||
return {
|
||||
id: descriptor.id,
|
||||
name: descriptor.name,
|
||||
@@ -78,11 +81,11 @@ export function buildSourceCatalog({
|
||||
sessionCount: sourceStats.sessionCount,
|
||||
lastIndexed: sourceStats.lastIndexed,
|
||||
status,
|
||||
statusText: !exists
|
||||
statusText: needsRoot
|
||||
? descriptor.rootResolutionReason ?? 'Select a session folder'
|
||||
: !exists
|
||||
? 'Folder not found'
|
||||
: sourceStats.sessionCount > 0
|
||||
? 'Connected'
|
||||
: 'No sessions found',
|
||||
: partialStatus ?? (sourceStats.sessionCount > 0 ? 'Connected' : 'No sessions found'),
|
||||
};
|
||||
});
|
||||
}
|
||||
|
||||
@@ -22,6 +22,7 @@ import { sourceLabel } from './source-catalog.mjs';
|
||||
const router = useRouter();
|
||||
const route = useRoute();
|
||||
let searchTimer = null;
|
||||
let stopSourceUpdates = null;
|
||||
|
||||
const routeSession = computed(() => {
|
||||
return getSessionSummary(route.params.id);
|
||||
@@ -193,9 +194,14 @@ function handleGlobalKeydown(event) {
|
||||
}
|
||||
}
|
||||
|
||||
onMounted(() => window.addEventListener('keydown', handleGlobalKeydown));
|
||||
onMounted(() => {
|
||||
window.addEventListener('keydown', handleGlobalKeydown);
|
||||
stopSourceUpdates = window.obelisk?.onIndexUpdated?.(() => loadSourceDots()) ?? null;
|
||||
});
|
||||
onUnmounted(() => {
|
||||
window.removeEventListener('keydown', handleGlobalKeydown);
|
||||
stopSourceUpdates?.();
|
||||
stopSourceUpdates = null;
|
||||
clearTimeout(searchTimer);
|
||||
});
|
||||
|
||||
|
||||
@@ -20,14 +20,18 @@ const editorMenuOpen = ref(false);
|
||||
const memoryCount = ref(0);
|
||||
const rebuilding = ref(false);
|
||||
const version = ref('');
|
||||
let stopIndexUpdates = null;
|
||||
|
||||
onMounted(async () => {
|
||||
stopIndexUpdates = window.obelisk?.onIndexUpdated?.(() => loadSettings()) ?? null;
|
||||
document.addEventListener('pointerdown', closeEditorMenuOutside);
|
||||
document.addEventListener('keydown', closeEditorMenuOnEscape);
|
||||
await loadSettings();
|
||||
});
|
||||
|
||||
onBeforeUnmount(() => {
|
||||
stopIndexUpdates?.();
|
||||
stopIndexUpdates = null;
|
||||
document.removeEventListener('pointerdown', closeEditorMenuOutside);
|
||||
document.removeEventListener('keydown', closeEditorMenuOnEscape);
|
||||
});
|
||||
@@ -364,7 +368,7 @@ function fmtRelative(iso) {
|
||||
.source-card-name .vendor { font-size: 11.5px; color: var(--muted); font-weight: 400; }
|
||||
.source-card-status {
|
||||
font-family: var(--font-mono); font-size: 10.5px; color: var(--muted);
|
||||
margin-top: 3px; display: flex; align-items: center; gap: 8px;
|
||||
margin-top: 3px; display: flex; align-items: center; flex-wrap: wrap; gap: 8px;
|
||||
}
|
||||
.source-card-status .stat-dot { width: 6px; height: 6px; border-radius: 50%; position: relative; }
|
||||
.source-card-status .stat-dot.ok { background: #34d399; box-shadow: 0 0 5px rgba(52,211,153,0.5); }
|
||||
@@ -375,7 +379,7 @@ function fmtRelative(iso) {
|
||||
border: 1px solid #34d399; opacity: 0.5; animation: src-pulse 1.6s ease-out infinite;
|
||||
}
|
||||
@keyframes src-pulse { 0% { transform: scale(0.8); opacity: 0.5; } 100% { transform: scale(1.8); opacity: 0; } }
|
||||
.source-card-status .stat-text { color: var(--fg-2); }
|
||||
.source-card-status .stat-text { color: var(--fg-2); min-width: 0; overflow-wrap: anywhere; }
|
||||
.source-card-status .stat-text.ok { color: #34d399; }
|
||||
.source-card-status .stat-text.warn { color: #fbbf24; }
|
||||
.source-card-status .stat-text.error { color: #f87171; }
|
||||
|
||||
Reference in New Issue
Block a user