refactor(app): consume shared indexing core, remove duplicated indexer (Phase 5d-3c-i)
The desktop app now indexes through the shared provider adapters + persist
layer (scripts/providers/{claude,codex}, scripts/persist, scripts/parsing)
instead of maintaining its own parallel indexer. buildIndex shrinks from
~1173 to ~592 lines, eliminating the skill<->app parse duplication that
Phase 5 set out to remove. electron-vite bundles the .ts core from source
with better-sqlite3 injected; the provider->parsing graph stays
node:sqlite-free so nothing drags node:sqlite into the app.
Also fix a misleading log: when a manual rebuild tears down the worker
mid-build, the cancelled background build is a deliberate stop, not a
failure. Guard the service's failure log with the stopped flag so it no
longer prints "Obelisk index build failed: Indexer worker stopped" on
every rebuild.
- CONTEXT.md: provider-adapter + single-persist + node:sqlite-free parsing.
- docs/adr/0005: app builds with electron-vite (TS+ESM), packages with
electron-builder; preload CJS for sandbox; app consumes core from source.
Verified: full suite 121/121; a node:sqlite-adapter dogfood of the rebuild
path over real data (969 files, 285 sessions, FTS rebuilt) runs clean; app
Rebuild confirmed in real Electron/better-sqlite3.
Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
tommy0103
co-authored by
Claude Opus 4.8
Claude Opus 4.8
parent
905c10789a
commit
60d47a852e
@@ -8,3 +8,5 @@ release/
|
||||
.claude/
|
||||
dist/
|
||||
app/out/
|
||||
HANDOFF.md
|
||||
.obelisk/
|
||||
+14
-10
@@ -27,22 +27,26 @@ promoted to an external tool surface.
|
||||
|
||||
## Indexing
|
||||
|
||||
**Parse core**:
|
||||
The pure `jsonl -> records` transform. Given a transcript file and a start line,
|
||||
it yields normalized index records. It does not open, own, or write to a
|
||||
database, and is shared verbatim by every indexing mode. This is the layer that
|
||||
must never be duplicated.
|
||||
_Avoid_: parser, ingest
|
||||
**Provider adapter**:
|
||||
A pure per-source module (claude, codex, later opencode, pi, …) that discovers a
|
||||
source's transcript files and parses one into a stream of records. It never opens
|
||||
or writes a database; adding a source means adding one adapter. The shared pure
|
||||
parse/discover helpers live in `scripts/parsing.mjs`, which imports only
|
||||
node:fs/path/os — deliberately node:sqlite-free so the compiled providers can be
|
||||
consumed by the app (whose Electron runtime has no `node:sqlite`).
|
||||
_Avoid_: parse core, parser, ingest
|
||||
|
||||
**Record**:
|
||||
One normalized row destined for the index (session, message, tool call, tool
|
||||
result, summary, subagent, workflow, …), emitted by the parse core before any
|
||||
result, summary, subagent, workflow, …), emitted by a provider adapter before any
|
||||
persistence happens.
|
||||
|
||||
**Persist layer**:
|
||||
The thin, binding-specific writer that consumes records from the parse core and
|
||||
writes them into SQLite inside a transaction. Two persist layers exist and differ
|
||||
only in binding: `node:sqlite` (skill/CLI) and `better-sqlite3` (app).
|
||||
The single shared, provider- and binding-agnostic writer that consumes records
|
||||
from any adapter and writes them into an injected SQLite handle inside a
|
||||
transaction. The binding is injected — `node:sqlite` (skill/CLI) or
|
||||
`better-sqlite3` (app) — so there is one persist implementation, not one per
|
||||
binding.
|
||||
_Avoid_: writer, sink, DAO
|
||||
|
||||
**Daemon indexing mode**:
|
||||
|
||||
@@ -111,7 +111,9 @@ function createIndexerService({
|
||||
writeHeartbeat();
|
||||
})()
|
||||
.catch((error) => {
|
||||
logger.warn?.(`Obelisk index build failed: ${error.message}`);
|
||||
// A build in flight when the service is stopped (e.g. a manual rebuild
|
||||
// tears down the worker) is a deliberate cancellation, not a failure.
|
||||
if (!stopped) logger.warn?.(`Obelisk index build failed: ${error.message}`);
|
||||
})
|
||||
.finally(() => {
|
||||
running = false;
|
||||
|
||||
+51
-632
@@ -3,10 +3,21 @@ import os from 'node:os';
|
||||
import path from 'node:path';
|
||||
import { fileURLToPath } from 'node:url';
|
||||
import Database from 'better-sqlite3';
|
||||
import { parse as claudeParse } from '../../../scripts/providers/claude.ts';
|
||||
import { parse as codexParse } from '../../../scripts/providers/codex.ts';
|
||||
import { persist } from '../../../scripts/persist.ts';
|
||||
import {
|
||||
inferProjectPath,
|
||||
isDir,
|
||||
readLines,
|
||||
codexDbId,
|
||||
codexRawId,
|
||||
codexParentThreadId,
|
||||
readCodexGuardianThreadInfo,
|
||||
} from '../../../scripts/parsing.mjs';
|
||||
|
||||
const __dirname = path.dirname(fileURLToPath(import.meta.url));
|
||||
|
||||
const TEXT_LIMIT = 10000;
|
||||
const DEFAULT_CLAUDE_DIR = path.join(os.homedir(), '.claude');
|
||||
const DEFAULT_CODEX_DIR = path.join(os.homedir(), '.codex');
|
||||
const DEFAULT_OBELISK_DIR = path.join(os.homedir(), '.obelisk');
|
||||
@@ -93,110 +104,6 @@ function copyMemoriesFromDb(db, sourceDbPath) {
|
||||
}
|
||||
}
|
||||
|
||||
function trunc(s) {
|
||||
return typeof s === 'string' && s.length > TEXT_LIMIT ? s.slice(0, TEXT_LIMIT) : s;
|
||||
}
|
||||
|
||||
function truncJson(obj, limit = TEXT_LIMIT) {
|
||||
if (obj === null || obj === undefined) return null;
|
||||
const walk = (v) => {
|
||||
if (typeof v === 'string') return v.length > limit ? v.slice(0, limit) + '...[truncated]' : v;
|
||||
if (Array.isArray(v)) return v.map(walk);
|
||||
if (typeof v === 'object' && v !== null) {
|
||||
const out = {};
|
||||
for (const [k, val] of Object.entries(v)) out[k] = walk(val);
|
||||
return out;
|
||||
}
|
||||
return v;
|
||||
};
|
||||
return JSON.stringify(walk(obj));
|
||||
}
|
||||
|
||||
function extractText(content) {
|
||||
if (typeof content === 'string') return trunc(content);
|
||||
if (!Array.isArray(content)) return null;
|
||||
const parts = [];
|
||||
for (const b of content) {
|
||||
if (b.type === 'text' && b.text) parts.push(b.text);
|
||||
else if (b.type === 'thinking' && b.thinking) parts.push(b.thinking);
|
||||
}
|
||||
return parts.length ? trunc(parts.join('\n')) : null;
|
||||
}
|
||||
|
||||
function extractContentType(content) {
|
||||
if (typeof content === 'string') return 'text';
|
||||
if (!Array.isArray(content) || !content.length) return 'unknown';
|
||||
const types = new Set();
|
||||
let sawUnknown = false;
|
||||
for (const b of content) {
|
||||
if (!b || typeof b !== 'object') { sawUnknown = true; continue; }
|
||||
if (b.type === 'text') types.add('text');
|
||||
else if (b.type === 'thinking') types.add('thinking');
|
||||
else if (b.type === 'tool_use') types.add('tool_use');
|
||||
else if (b.type === 'tool_result') types.add('tool_result');
|
||||
else sawUnknown = true;
|
||||
}
|
||||
return !sawUnknown && types.size === 1 ? [...types][0] : 'unknown';
|
||||
}
|
||||
|
||||
const COMMAND_ENVELOPE_RE = /^\s*(<command-name>[^<]+<\/command-name>|<(?:task-notification|system-reminder)\b|<local-command(?:\b|-))/;
|
||||
|
||||
function extractMessageIsMeta(record, text = extractText(record?.message?.content)) {
|
||||
const msg = record?.message || {};
|
||||
if (record?.isMeta === true || msg.isMeta === true) return 1;
|
||||
return typeof text === 'string' && COMMAND_ENVELOPE_RE.test(text) ? 1 : 0;
|
||||
}
|
||||
|
||||
function filePath(name, input) {
|
||||
if (!input) return null;
|
||||
return ['Read', 'Edit', 'Write', 'NotebookEdit'].includes(name) ? (input.file_path || null) : null;
|
||||
}
|
||||
|
||||
function isDir(p) {
|
||||
try {
|
||||
return fs.statSync(p).isDirectory();
|
||||
} catch {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
function readLines(filePath, callback) {
|
||||
const data = fs.readFileSync(filePath, 'utf8');
|
||||
const lines = data.split('\n');
|
||||
for (const line of lines) {
|
||||
if (line && callback(line) === false) return;
|
||||
}
|
||||
}
|
||||
|
||||
function legacyProjectPathFromSlug(project) {
|
||||
if (!project) return null;
|
||||
return '/' + project.replace(/-/g, '/').replace(/^\//, '');
|
||||
}
|
||||
|
||||
function normalizeObservedCwd(cwd) {
|
||||
if (typeof cwd !== 'string' || !cwd.trim() || !path.isAbsolute(cwd)) return null;
|
||||
return path.normalize(cwd);
|
||||
}
|
||||
|
||||
function projectSlugFromPath(projectPath) {
|
||||
const normalized = normalizeObservedCwd(projectPath);
|
||||
if (!normalized) return null;
|
||||
return '-' + normalized.replace(/^[\\/]+/, '').replace(/[\\/]+/g, '-');
|
||||
}
|
||||
|
||||
function inferProjectPath(project, observedCwds = []) {
|
||||
const byPath = new Map();
|
||||
for (const cwd of observedCwds) {
|
||||
const normalized = normalizeObservedCwd(cwd);
|
||||
if (!normalized) continue;
|
||||
const current = byPath.get(normalized) || { path: normalized, count: 0, first: byPath.size };
|
||||
current.count++;
|
||||
byPath.set(normalized, current);
|
||||
}
|
||||
const best = [...byPath.values()].sort((a, b) => b.count - a.count || a.first - b.first)[0];
|
||||
return best?.path || legacyProjectPathFromSlug(project);
|
||||
}
|
||||
|
||||
function discoverJsonlFiles({ projectsDir = DEFAULT_PROJECTS_DIR, changedPaths = undefined } = {}) {
|
||||
if (Array.isArray(changedPaths) && changedPaths.length) {
|
||||
const changedFiles = discoverJsonlFilesForChanges({ projectsDir, changedPaths });
|
||||
@@ -383,544 +290,56 @@ function needsReindex(db, fp) {
|
||||
return mt > row.mtime ? { needed: true, skip: row.lines_processed, mtime: mt } : { needed: false, skip: 0, mtime: mt };
|
||||
}
|
||||
|
||||
function indexJsonl(db, fi) {
|
||||
const { needed, skip, mtime } = needsReindex(db, fi.path);
|
||||
if (!needed) return;
|
||||
const ins = {
|
||||
ses: db.prepare('INSERT OR REPLACE INTO sessions (id,title,project,project_path,started_at,ended_at,git_branch,version,message_count,jsonl_path,source) VALUES (?,?,?,?,?,?,?,?,?,?,?)'),
|
||||
msg: db.prepare(`
|
||||
INSERT INTO messages (uuid,session_id,type,parent_uuid,timestamp,role,text,content_type,is_meta,model,is_sidechain,agent_id,input_tokens,output_tokens,cwd,skill,source)
|
||||
VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)
|
||||
ON CONFLICT(uuid) DO UPDATE SET
|
||||
session_id=excluded.session_id,
|
||||
type=excluded.type,
|
||||
parent_uuid=excluded.parent_uuid,
|
||||
timestamp=excluded.timestamp,
|
||||
role=excluded.role,
|
||||
text=excluded.text,
|
||||
content_type=excluded.content_type,
|
||||
is_meta=excluded.is_meta,
|
||||
model=excluded.model,
|
||||
is_sidechain=excluded.is_sidechain,
|
||||
agent_id=excluded.agent_id,
|
||||
input_tokens=excluded.input_tokens,
|
||||
output_tokens=excluded.output_tokens,
|
||||
cwd=excluded.cwd,
|
||||
skill=excluded.skill,
|
||||
source=excluded.source
|
||||
`),
|
||||
tc: db.prepare('INSERT OR REPLACE INTO tool_calls (id,message_uuid,session_id,name,input_json,file_path) VALUES (?,?,?,?,?,?)'),
|
||||
tr: db.prepare('INSERT OR REPLACE INTO tool_results (tool_use_id,message_uuid,session_id,content,file_path,is_error) VALUES (?,?,?,?,?,?)'),
|
||||
sum: db.prepare('INSERT OR REPLACE INTO summaries (id,session_id,timestamp,source,content) VALUES (?,?,?,?,?)'),
|
||||
idx: db.prepare('INSERT OR REPLACE INTO index_state (jsonl_path,mtime,lines_processed) VALUES (?,?,?)'),
|
||||
};
|
||||
const existing = !fi.isSubagent ? db.prepare('SELECT * FROM sessions WHERE id = ?').get(fi.sessionId) : null;
|
||||
const sm = {
|
||||
started_at: existing?.started_at || null,
|
||||
ended_at: existing?.ended_at || null,
|
||||
git_branch: existing?.git_branch || null,
|
||||
version: existing?.version || null,
|
||||
title: existing?.title || null,
|
||||
n: skip > 0 ? (existing?.message_count || 0) : 0,
|
||||
cwds: [],
|
||||
// Index one Claude transcript via the shared provider + persist core.
|
||||
// Returns { sessionId, path } when reindexed, undefined when skipped.
|
||||
function indexClaudeFile(db, file) {
|
||||
const { needed, skip, mtime } = needsReindex(db, file.path);
|
||||
if (!needed) return undefined;
|
||||
const unit = {
|
||||
key: file.path,
|
||||
sessionId: file.sessionId,
|
||||
project: file.project,
|
||||
isSubagent: file.isSubagent,
|
||||
agentId: file.agentId,
|
||||
};
|
||||
const cursor = skip > 0 ? `${mtime}:${skip}` : null;
|
||||
persist(db, unit, claudeParse(unit, cursor));
|
||||
return { sessionId: file.sessionId, path: file.path };
|
||||
}
|
||||
|
||||
let lineNum = 0;
|
||||
readLines(fi.path, (line) => {
|
||||
lineNum++;
|
||||
if (lineNum <= skip) return;
|
||||
function codexSessionMeta(filePath) {
|
||||
let meta = null;
|
||||
readLines(filePath, (line) => {
|
||||
let obj;
|
||||
try { obj = JSON.parse(line); } catch { return; }
|
||||
const sid = fi.sessionId;
|
||||
const ts = obj.timestamp || null;
|
||||
if (obj.type === 'ai-title' && obj.aiTitle) { sm.title = obj.aiTitle; return; }
|
||||
if (obj.type === 'system' && obj.subtype === 'away_summary' && obj.content) {
|
||||
ins.sum.run(obj.uuid || `${sid}-away-${ts}`, sid, ts, 'away_summary', obj.content);
|
||||
return;
|
||||
}
|
||||
if (obj.type === 'system' && obj.subtype === 'turn_duration' && obj.parentUuid && obj.durationMs) {
|
||||
db.prepare('UPDATE messages SET turn_duration_ms=? WHERE uuid=?').run(obj.durationMs, obj.parentUuid);
|
||||
return;
|
||||
}
|
||||
if (obj.type !== 'user' && obj.type !== 'assistant') return;
|
||||
if (ts && (!sm.started_at || ts < sm.started_at)) sm.started_at = ts;
|
||||
if (ts && (!sm.ended_at || ts > sm.ended_at)) sm.ended_at = ts;
|
||||
if (obj.gitBranch) sm.git_branch = obj.gitBranch;
|
||||
if (obj.version) sm.version = obj.version;
|
||||
sm.n++;
|
||||
if (!fi.isSubagent && obj.cwd) sm.cwds.push(obj.cwd);
|
||||
|
||||
const msg = obj.message || {};
|
||||
const text = extractText(msg.content);
|
||||
const contentType = extractContentType(msg.content);
|
||||
const isMeta = extractMessageIsMeta(obj, text);
|
||||
const usage = msg.usage || {};
|
||||
const aid = fi.isSubagent ? fi.agentId : (obj.agentId || null);
|
||||
if (obj.uuid) {
|
||||
ins.msg.run(obj.uuid, sid, obj.type, obj.parentUuid || null, ts,
|
||||
msg.role || obj.type, text, contentType, isMeta, msg.model || null,
|
||||
obj.isSidechain ? 1 : 0, aid, usage.input_tokens || null, usage.output_tokens || null,
|
||||
obj.cwd || null, obj.attributionSkill || null, 'claude');
|
||||
}
|
||||
if (obj.type === 'assistant' && Array.isArray(msg.content)) {
|
||||
for (const b of msg.content) {
|
||||
if (b.type === 'tool_use' && b.id)
|
||||
ins.tc.run(b.id, obj.uuid, sid, b.name, truncJson(b.input || {}), filePath(b.name, b.input));
|
||||
}
|
||||
}
|
||||
if (obj.type === 'user' && Array.isArray(msg.content)) {
|
||||
for (const b of msg.content) {
|
||||
if (b.type !== 'tool_result' || !b.tool_use_id) continue;
|
||||
const rt = typeof b.content === 'string' ? b.content
|
||||
: Array.isArray(b.content) ? b.content.map(c => c.text || '').join('\n') : '';
|
||||
ins.tr.run(b.tool_use_id, obj.uuid, sid, trunc(rt), obj.toolUseResult?.filePath || null, b.is_error ? 1 : 0);
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
if (!fi.isSubagent) {
|
||||
const pp = inferProjectPath(fi.project, sm.cwds);
|
||||
ins.ses.run(fi.sessionId, sm.title, fi.project, pp, sm.started_at, sm.ended_at, sm.git_branch, sm.version, sm.n, fi.path, 'claude');
|
||||
}
|
||||
ins.idx.run(fi.path, mtime, lineNum);
|
||||
return { sessionId: fi.sessionId, path: fi.path };
|
||||
}
|
||||
|
||||
function codexDbId(id) {
|
||||
if (!id) return null;
|
||||
const raw = String(id).replace(/^codex:/, '');
|
||||
return `codex:${raw}`;
|
||||
}
|
||||
|
||||
function codexRawId(id) {
|
||||
return id ? String(id).replace(/^codex:/, '') : null;
|
||||
}
|
||||
|
||||
function codexLineUuid(threadId, lineNum) {
|
||||
return `codex:${codexRawId(threadId)}:${String(lineNum).padStart(6, '0')}`;
|
||||
}
|
||||
|
||||
function codexCallId(callId) {
|
||||
if (!callId) return null;
|
||||
return `codex:${String(callId).replace(/^codex:/, '')}`;
|
||||
}
|
||||
|
||||
function codexParentThreadId(meta) {
|
||||
const subagent = meta?.source?.subagent;
|
||||
return subagent?.thread_spawn?.parent_thread_id
|
||||
|| meta?.forked_from_id
|
||||
|| subagent?.parent_thread_id
|
||||
|| null;
|
||||
}
|
||||
|
||||
function codexIsGuardianThread(meta, records = []) {
|
||||
const subagent = meta?.source?.subagent;
|
||||
if (subagent?.other === 'guardian') return true;
|
||||
if (meta?.thread_source !== 'subagent') return false;
|
||||
return records.some(({ obj }) => obj?.payload?.model === 'codex-auto-review' || obj?.model === 'codex-auto-review');
|
||||
}
|
||||
|
||||
function deleteCodexThreadRows(db, threadRawId) {
|
||||
const threadId = codexDbId(threadRawId);
|
||||
if (!threadId) return;
|
||||
db.prepare(`
|
||||
DELETE FROM tool_results
|
||||
WHERE session_id = ?
|
||||
OR message_uuid IN (SELECT uuid FROM messages WHERE session_id = ? OR agent_id = ?)
|
||||
`).run(threadId, threadId, threadId);
|
||||
db.prepare(`
|
||||
DELETE FROM tool_calls
|
||||
WHERE session_id = ?
|
||||
OR message_uuid IN (SELECT uuid FROM messages WHERE session_id = ? OR agent_id = ?)
|
||||
`).run(threadId, threadId, threadId);
|
||||
db.prepare('DELETE FROM messages WHERE session_id = ? OR agent_id = ?').run(threadId, threadId);
|
||||
db.prepare('DELETE FROM subagents WHERE agent_id = ? OR session_id = ?').run(threadId, threadId);
|
||||
db.prepare('DELETE FROM summaries WHERE session_id = ?').run(threadId);
|
||||
db.prepare('DELETE FROM sessions WHERE id = ?').run(threadId);
|
||||
}
|
||||
|
||||
function readCodexGuardianThreadInfo(filePath) {
|
||||
const records = [];
|
||||
let metaRecord = null;
|
||||
let lineNum = 0;
|
||||
readLines(filePath, (line) => {
|
||||
lineNum++;
|
||||
let obj;
|
||||
try {
|
||||
obj = JSON.parse(line);
|
||||
} catch {
|
||||
return;
|
||||
}
|
||||
records.push({ lineNum, obj });
|
||||
if (obj?.type === 'session_meta' && obj.payload?.id) {
|
||||
metaRecord = { lineNum, obj };
|
||||
if (obj.payload?.source?.subagent?.other === 'guardian') return false;
|
||||
if (obj.payload?.thread_source !== 'subagent') return false;
|
||||
meta = obj.payload;
|
||||
return false;
|
||||
}
|
||||
if (metaRecord && codexIsGuardianThread(metaRecord.obj.payload, records)) return false;
|
||||
});
|
||||
const meta = metaRecord?.obj?.payload;
|
||||
if (!meta || !codexIsGuardianThread(meta, records)) return null;
|
||||
return { threadRawId: codexRawId(meta.id), lineNum };
|
||||
return meta;
|
||||
}
|
||||
|
||||
function codexAgentNickname(meta) {
|
||||
return meta?.agent_nickname
|
||||
|| meta?.source?.subagent?.thread_spawn?.agent_nickname
|
||||
|| null;
|
||||
}
|
||||
|
||||
function codexAgentRole(meta) {
|
||||
return meta?.agent_role
|
||||
|| meta?.source?.subagent?.thread_spawn?.agent_role
|
||||
|| null;
|
||||
}
|
||||
|
||||
function parseCodexJsonInput(value) {
|
||||
if (value === null || value === undefined || value === '') return {};
|
||||
if (typeof value !== 'string') return value;
|
||||
try { return JSON.parse(value); } catch { return value; }
|
||||
}
|
||||
|
||||
function codexUsage(payload) {
|
||||
const usage = payload?.info?.last_token_usage || payload?.info?.total_token_usage || payload?.last_token_usage || null;
|
||||
if (!usage) return {};
|
||||
return {
|
||||
inputTokens: usage.input_tokens ?? null,
|
||||
outputTokens: usage.output_tokens ?? null,
|
||||
};
|
||||
}
|
||||
|
||||
function codexEventText(payload) {
|
||||
if (typeof payload?.message === 'string') return payload.message;
|
||||
if (Array.isArray(payload?.text_elements) && payload.text_elements.length) {
|
||||
const parts = payload.text_elements.map(item => typeof item === 'string' ? item : item?.text).filter(Boolean);
|
||||
if (parts.length) return parts.join('\n');
|
||||
}
|
||||
if (typeof payload?.text === 'string') return payload.text;
|
||||
// Index one Codex rollout via the shared provider + persist core (full reparse).
|
||||
// Returns { sessionId, path } when reindexed, undefined when skipped.
|
||||
function indexCodexFile(db, file) {
|
||||
const { needed } = needsReindex(db, file.path);
|
||||
const guardian = readCodexGuardianThreadInfo(file.path);
|
||||
if (!needed) {
|
||||
if (guardian) {
|
||||
persist(db, { key: file.path, sessionId: '' }, (function* () {
|
||||
yield { kind: 'delete-session', sessionId: codexDbId(guardian.threadRawId) };
|
||||
return null;
|
||||
}
|
||||
|
||||
function codexMessagePayloadText(payload) {
|
||||
if (!Array.isArray(payload?.content)) return null;
|
||||
const parts = [];
|
||||
for (const block of payload.content) {
|
||||
if (typeof block?.text === 'string') parts.push(block.text);
|
||||
})());
|
||||
}
|
||||
return parts.length ? parts.join('\n') : null;
|
||||
}
|
||||
|
||||
function codexVisibleMessageKey(role, text) {
|
||||
return `${role || ''}\u0000${text || ''}`;
|
||||
}
|
||||
|
||||
function codexToolInput(payload) {
|
||||
if (payload?.type === 'custom_tool_call') return parseCodexJsonInput(payload.input);
|
||||
if (payload?.type === 'tool_search_call') return parseCodexJsonInput(payload.arguments);
|
||||
if (payload?.type === 'web_search_call') return { action: payload.action || null };
|
||||
return parseCodexJsonInput(payload?.arguments);
|
||||
}
|
||||
|
||||
function codexToolOutput(payload) {
|
||||
if (typeof payload?.output === 'string') return payload.output;
|
||||
if (payload?.output !== undefined) return JSON.stringify(payload.output);
|
||||
if (payload?.tools !== undefined) return JSON.stringify(payload.tools);
|
||||
if (payload?.execution !== undefined) return JSON.stringify(payload.execution);
|
||||
return null;
|
||||
}
|
||||
|
||||
function upsertCodexSubagent(db, {
|
||||
agentId,
|
||||
sessionId,
|
||||
parentToolUseId = null,
|
||||
agentType = null,
|
||||
description = null,
|
||||
durationMs = null,
|
||||
totalTokens = null,
|
||||
} = {}) {
|
||||
if (!agentId || !sessionId) return;
|
||||
db.prepare(`
|
||||
INSERT INTO subagents (agent_id,session_id,parent_tool_use_id,agent_type,description,duration_ms,total_tokens)
|
||||
VALUES (?,?,?,?,?,?,?)
|
||||
ON CONFLICT(agent_id) DO UPDATE SET
|
||||
session_id=excluded.session_id,
|
||||
parent_tool_use_id=COALESCE(excluded.parent_tool_use_id, subagents.parent_tool_use_id),
|
||||
agent_type=COALESCE(excluded.agent_type, subagents.agent_type),
|
||||
description=COALESCE(excluded.description, subagents.description),
|
||||
duration_ms=COALESCE(excluded.duration_ms, subagents.duration_ms),
|
||||
total_tokens=COALESCE(excluded.total_tokens, subagents.total_tokens)
|
||||
`).run(agentId, sessionId, parentToolUseId, agentType, description, durationMs, totalTokens);
|
||||
}
|
||||
|
||||
function indexCodexJsonl(db, fi) {
|
||||
const state = needsReindex(db, fi.path);
|
||||
if (!state.needed) {
|
||||
const guardian = readCodexGuardianThreadInfo(fi.path);
|
||||
if (guardian) deleteCodexThreadRows(db, guardian.threadRawId);
|
||||
return null;
|
||||
return undefined;
|
||||
}
|
||||
const mtime = state.mtime;
|
||||
const records = [];
|
||||
let lineNum = 0;
|
||||
readLines(fi.path, (line) => {
|
||||
lineNum++;
|
||||
try {
|
||||
records.push({ lineNum, obj: JSON.parse(line) });
|
||||
} catch {}
|
||||
});
|
||||
|
||||
const metaRecord = records.find(r => r.obj?.type === 'session_meta' && r.obj.payload?.id);
|
||||
if (!metaRecord) {
|
||||
db.prepare('INSERT OR REPLACE INTO index_state (jsonl_path,mtime,lines_processed) VALUES (?,?,?)').run(fi.path, mtime, lineNum);
|
||||
return null;
|
||||
}
|
||||
|
||||
const meta = metaRecord.obj.payload;
|
||||
const threadRawId = codexRawId(meta.id);
|
||||
if (codexIsGuardianThread(meta, records)) {
|
||||
deleteCodexThreadRows(db, threadRawId);
|
||||
db.prepare('INSERT OR REPLACE INTO index_state (jsonl_path,mtime,lines_processed) VALUES (?,?,?)').run(fi.path, mtime, lineNum);
|
||||
return null;
|
||||
}
|
||||
const parentRawId = codexParentThreadId(meta);
|
||||
const sessionId = codexDbId(parentRawId || threadRawId);
|
||||
const agentId = parentRawId ? codexDbId(threadRawId) : null;
|
||||
const isSidechain = agentId ? 1 : 0;
|
||||
const projectPath = normalizeObservedCwd(meta.cwd);
|
||||
const project = projectSlugFromPath(projectPath);
|
||||
const sm = {
|
||||
started_at: meta.timestamp || metaRecord.obj.timestamp || null,
|
||||
ended_at: meta.timestamp || metaRecord.obj.timestamp || null,
|
||||
git_branch: meta.git?.branch || null,
|
||||
version: meta.cli_version || null,
|
||||
title: null,
|
||||
n: 0,
|
||||
cwds: projectPath ? [projectPath] : [],
|
||||
lastMessageUuid: null,
|
||||
lastTextAssistantUuid: null,
|
||||
totalInputTokens: 0,
|
||||
totalOutputTokens: 0,
|
||||
};
|
||||
|
||||
const ins = {
|
||||
ses: db.prepare('INSERT OR REPLACE INTO sessions (id,title,project,project_path,started_at,ended_at,git_branch,version,message_count,jsonl_path,source) VALUES (?,?,?,?,?,?,?,?,?,?,?)'),
|
||||
msg: db.prepare(`
|
||||
INSERT INTO messages (uuid,session_id,type,parent_uuid,timestamp,role,text,content_type,is_meta,model,is_sidechain,agent_id,input_tokens,output_tokens,cwd,skill,source)
|
||||
VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)
|
||||
ON CONFLICT(uuid) DO UPDATE SET
|
||||
session_id=excluded.session_id,
|
||||
type=excluded.type,
|
||||
parent_uuid=excluded.parent_uuid,
|
||||
timestamp=excluded.timestamp,
|
||||
role=excluded.role,
|
||||
text=excluded.text,
|
||||
content_type=excluded.content_type,
|
||||
is_meta=excluded.is_meta,
|
||||
model=excluded.model,
|
||||
is_sidechain=excluded.is_sidechain,
|
||||
agent_id=excluded.agent_id,
|
||||
input_tokens=excluded.input_tokens,
|
||||
output_tokens=excluded.output_tokens,
|
||||
cwd=excluded.cwd,
|
||||
skill=excluded.skill,
|
||||
source=excluded.source
|
||||
`),
|
||||
tc: db.prepare('INSERT OR REPLACE INTO tool_calls (id,message_uuid,session_id,name,input_json,file_path) VALUES (?,?,?,?,?,?)'),
|
||||
tr: db.prepare('INSERT OR REPLACE INTO tool_results (tool_use_id,message_uuid,session_id,content,file_path,is_error) VALUES (?,?,?,?,?,?)'),
|
||||
idx: db.prepare('INSERT OR REPLACE INTO index_state (jsonl_path,mtime,lines_processed) VALUES (?,?,?)'),
|
||||
dur: db.prepare('UPDATE messages SET turn_duration_ms=? WHERE uuid=?'),
|
||||
usage: db.prepare('UPDATE messages SET input_tokens=?, output_tokens=? WHERE uuid=?'),
|
||||
};
|
||||
|
||||
let currentCwd = projectPath;
|
||||
let currentModel = null;
|
||||
const eventMessageKeys = new Set();
|
||||
const callMessageUuids = new Map();
|
||||
|
||||
for (const { obj } of records) {
|
||||
if (obj?.type !== 'event_msg') continue;
|
||||
const payload = obj.payload || {};
|
||||
if (payload.type !== 'user_message' && payload.type !== 'agent_message') continue;
|
||||
const text = codexEventText(payload);
|
||||
if (text === null) continue;
|
||||
eventMessageKeys.add(codexVisibleMessageKey(payload.type === 'user_message' ? 'user' : 'assistant', text));
|
||||
}
|
||||
|
||||
const updateBounds = (ts) => {
|
||||
if (!ts) return;
|
||||
if (!sm.started_at || ts < sm.started_at) sm.started_at = ts;
|
||||
if (!sm.ended_at || ts > sm.ended_at) sm.ended_at = ts;
|
||||
};
|
||||
|
||||
const insertMessage = ({ uuid, type, role, text = null, contentType = 'text', timestamp, isMeta = 0 }) => {
|
||||
ins.msg.run(
|
||||
uuid,
|
||||
sessionId,
|
||||
type,
|
||||
sm.lastMessageUuid,
|
||||
timestamp || null,
|
||||
role,
|
||||
trunc(text),
|
||||
contentType,
|
||||
isMeta,
|
||||
currentModel,
|
||||
isSidechain,
|
||||
agentId,
|
||||
null,
|
||||
null,
|
||||
currentCwd,
|
||||
null,
|
||||
'codex',
|
||||
);
|
||||
sm.lastMessageUuid = uuid;
|
||||
if (!agentId) sm.n++;
|
||||
if (type === 'assistant' && contentType === 'text') sm.lastTextAssistantUuid = uuid;
|
||||
updateBounds(timestamp);
|
||||
return uuid;
|
||||
};
|
||||
|
||||
for (const { lineNum: currentLine, obj } of records) {
|
||||
const ts = obj.timestamp || null;
|
||||
if (obj.type === 'session_meta') {
|
||||
if (obj.payload?.cwd) {
|
||||
currentCwd = normalizeObservedCwd(obj.payload.cwd) || currentCwd;
|
||||
if (currentCwd) sm.cwds.push(currentCwd);
|
||||
}
|
||||
if (obj.payload?.git?.branch) sm.git_branch = obj.payload.git.branch;
|
||||
if (obj.payload?.cli_version) sm.version = obj.payload.cli_version;
|
||||
updateBounds(obj.payload?.timestamp || ts);
|
||||
continue;
|
||||
}
|
||||
if (obj.type === 'turn_context') {
|
||||
currentCwd = normalizeObservedCwd(obj.payload?.cwd) || currentCwd;
|
||||
currentModel = obj.payload?.model || currentModel;
|
||||
if (currentCwd) sm.cwds.push(currentCwd);
|
||||
updateBounds(ts);
|
||||
continue;
|
||||
}
|
||||
if (obj.type === 'event_msg') {
|
||||
const payload = obj.payload || {};
|
||||
if (payload.type === 'user_message' || payload.type === 'agent_message' || payload.type === 'agent_reasoning') {
|
||||
const text = codexEventText(payload);
|
||||
if (text === null) continue;
|
||||
const isReasoning = payload.type === 'agent_reasoning';
|
||||
insertMessage({
|
||||
uuid: codexLineUuid(threadRawId, currentLine),
|
||||
type: payload.type === 'user_message' ? 'user' : 'assistant',
|
||||
role: payload.type === 'user_message' ? 'user' : 'assistant',
|
||||
text,
|
||||
contentType: isReasoning ? 'thinking' : 'text',
|
||||
timestamp: ts,
|
||||
});
|
||||
continue;
|
||||
}
|
||||
if (payload.type === 'collab_agent_spawn_end' && payload.call_id && payload.new_thread_id) {
|
||||
const uuid = insertMessage({
|
||||
uuid: codexLineUuid(threadRawId, currentLine),
|
||||
type: 'assistant',
|
||||
role: 'assistant',
|
||||
text: null,
|
||||
contentType: 'tool_use',
|
||||
timestamp: ts,
|
||||
});
|
||||
const toolId = codexCallId(payload.call_id);
|
||||
const description = payload.new_agent_nickname || payload.new_agent_role || 'Agent';
|
||||
const input = {
|
||||
description,
|
||||
subagent_type: payload.new_agent_role || 'Agent',
|
||||
prompt: payload.prompt || '',
|
||||
new_thread_id: payload.new_thread_id,
|
||||
model: payload.model || null,
|
||||
reasoning_effort: payload.reasoning_effort || null,
|
||||
};
|
||||
ins.tc.run(toolId, uuid, sessionId, 'Agent', truncJson(input), null);
|
||||
callMessageUuids.set(toolId, uuid);
|
||||
upsertCodexSubagent(db, {
|
||||
agentId: codexDbId(payload.new_thread_id),
|
||||
sessionId,
|
||||
parentToolUseId: toolId,
|
||||
agentType: payload.new_agent_role || null,
|
||||
description,
|
||||
});
|
||||
continue;
|
||||
}
|
||||
if (payload.type === 'task_complete') {
|
||||
if (sm.lastTextAssistantUuid && payload.duration_ms !== undefined) {
|
||||
ins.dur.run(payload.duration_ms || null, sm.lastTextAssistantUuid);
|
||||
}
|
||||
updateBounds(ts);
|
||||
continue;
|
||||
}
|
||||
if (payload.type === 'token_count') {
|
||||
const usage = codexUsage(payload);
|
||||
if (usage.inputTokens !== null) sm.totalInputTokens = usage.inputTokens;
|
||||
if (usage.outputTokens !== null) sm.totalOutputTokens = usage.outputTokens;
|
||||
if (sm.lastTextAssistantUuid && (usage.inputTokens !== null || usage.outputTokens !== null)) {
|
||||
ins.usage.run(usage.inputTokens, usage.outputTokens, sm.lastTextAssistantUuid);
|
||||
}
|
||||
continue;
|
||||
}
|
||||
if (payload.type === 'thread_name_updated' && payload.thread_name) {
|
||||
sm.title = payload.thread_name;
|
||||
}
|
||||
continue;
|
||||
}
|
||||
if (obj.type !== 'response_item') continue;
|
||||
const payload = obj.payload || {};
|
||||
if (payload.type === 'message' && payload.role !== 'developer') {
|
||||
const text = codexMessagePayloadText(payload);
|
||||
const role = payload.role || 'assistant';
|
||||
if (text !== null && !eventMessageKeys.has(codexVisibleMessageKey(role, text))) {
|
||||
insertMessage({
|
||||
uuid: codexLineUuid(threadRawId, currentLine),
|
||||
type: role === 'user' ? 'user' : 'assistant',
|
||||
role,
|
||||
text,
|
||||
contentType: 'text',
|
||||
timestamp: ts,
|
||||
});
|
||||
}
|
||||
continue;
|
||||
}
|
||||
if (['function_call', 'custom_tool_call', 'tool_search_call', 'web_search_call'].includes(payload.type) && payload.call_id) {
|
||||
const uuid = insertMessage({
|
||||
uuid: codexLineUuid(threadRawId, currentLine),
|
||||
type: 'assistant',
|
||||
role: 'assistant',
|
||||
text: null,
|
||||
contentType: 'tool_use',
|
||||
timestamp: ts,
|
||||
});
|
||||
const name = payload.name || payload.tool || payload.type.replace(/_call$/, '');
|
||||
const toolId = codexCallId(payload.call_id);
|
||||
ins.tc.run(toolId, uuid, sessionId, name, truncJson(codexToolInput(payload)), null);
|
||||
callMessageUuids.set(toolId, uuid);
|
||||
continue;
|
||||
}
|
||||
if (['function_call_output', 'custom_tool_call_output', 'tool_search_output'].includes(payload.type) && payload.call_id) {
|
||||
const toolId = codexCallId(payload.call_id);
|
||||
ins.tr.run(toolId, callMessageUuids.get(toolId) || null, sessionId, trunc(codexToolOutput(payload) || ''), null, payload.is_error ? 1 : 0);
|
||||
}
|
||||
}
|
||||
|
||||
if (agentId) {
|
||||
const tokenTotal = (sm.totalInputTokens || 0) + (sm.totalOutputTokens || 0);
|
||||
const started = sm.started_at ? new Date(sm.started_at).getTime() : null;
|
||||
const ended = sm.ended_at ? new Date(sm.ended_at).getTime() : null;
|
||||
upsertCodexSubagent(db, {
|
||||
agentId,
|
||||
sessionId,
|
||||
agentType: codexAgentRole(meta),
|
||||
description: codexAgentNickname(meta),
|
||||
durationMs: started && ended ? ended - started : null,
|
||||
totalTokens: tokenTotal || null,
|
||||
});
|
||||
} else {
|
||||
const pp = inferProjectPath(project, sm.cwds);
|
||||
ins.ses.run(sessionId, sm.title, project, pp, sm.started_at, sm.ended_at, sm.git_branch, sm.version, sm.n, fi.path, 'codex');
|
||||
}
|
||||
ins.idx.run(fi.path, mtime, lineNum);
|
||||
return { sessionId, path: fi.path };
|
||||
const unit = { key: file.path, sessionId: '' };
|
||||
persist(db, unit, codexParse(unit, null));
|
||||
if (guardian) return undefined;
|
||||
const meta = codexSessionMeta(file.path);
|
||||
const sessionId = meta ? codexDbId(codexParentThreadId(meta) || codexRawId(meta.id)) : undefined;
|
||||
return { sessionId, path: file.path };
|
||||
}
|
||||
|
||||
function indexCodexSessionIndex(db, { codexDir = DEFAULT_CODEX_DIR } = {}) {
|
||||
@@ -1122,7 +541,7 @@ function buildIndex({
|
||||
for (const file of files) {
|
||||
db.exec('BEGIN');
|
||||
try {
|
||||
const indexed = file.source === 'codex' ? indexCodexJsonl(db, file) : indexJsonl(db, file);
|
||||
const indexed = file.source === 'codex' ? indexCodexFile(db, file) : indexClaudeFile(db, file);
|
||||
if (indexed?.sessionId) affectedSessionIds.add(indexed.sessionId);
|
||||
if (file.source !== 'codex') indexSubagentMeta(db, file);
|
||||
db.exec('COMMIT');
|
||||
|
||||
@@ -0,0 +1,39 @@
|
||||
# The app builds with electron-vite (TS + ESM), packages with electron-builder
|
||||
|
||||
**Context.** The desktop app must consume the shared TypeScript/ESM Core
|
||||
(`providers/*` + `persist`) instead of maintaining its own duplicate indexer, and
|
||||
the app itself should be TypeScript + ESM long-term. The app previously ran raw
|
||||
CommonJS on Electron's Node with only the Vue renderer built by Vite; the main
|
||||
process had no build step, and Electron's bundled Node (20 on Electron 33) can
|
||||
neither strip TypeScript nor use `node:sqlite`. Options for the main-process build
|
||||
were a hand-rolled tsc/esbuild step, `vite-plugin-electron`, or `electron-vite`.
|
||||
|
||||
**Decision.** Adopt **electron-vite** to build all three processes (main, preload,
|
||||
renderer) as TypeScript + ESM, and keep **electron-builder** for packaging
|
||||
(dmg/nsis/AppImage). electron-vite is purpose-built for the Electron three-process
|
||||
model and handles the parts a DIY build would force us to hand-maintain forever
|
||||
(per-process module format, native-module externalization, dev reload). Specific
|
||||
decisions within this:
|
||||
|
||||
- **Preload is emitted as CommonJS** even though the app is ESM: the sandboxed
|
||||
renderer (sandbox is on by default since Electron 20, and we keep it on for
|
||||
security) does not support ESM preload. Source stays ESM; only the preload
|
||||
output format is CJS. `main` loads `../preload/index.js`.
|
||||
- **The app consumes the Core from source**: electron-vite/rollup bundles
|
||||
`scripts/providers/*` + `scripts/persist` (and their `scripts/parsing.mjs`
|
||||
dependency) into the app's main/worker build, injecting `better-sqlite3`. This
|
||||
works because the provider→parsing import graph is node:sqlite-free (ADR-0001),
|
||||
so nothing drags `node:sqlite` into the app. The `dist/` from `build:core`
|
||||
(ADR-0003) remains for the skill artifact; the app does not need it.
|
||||
- **better-sqlite3 stays the app's binding**, externalized (not bundled) and
|
||||
unpacked from the asar.
|
||||
|
||||
**Consequences.** The app is restructured into `src/{main,preload,renderer}` with
|
||||
`electron.vite.config.ts`; each main module is a build input so CommonJS-style
|
||||
require resolution and the indexer worker (`{ type: 'module' }`) resolve at
|
||||
runtime. `npm run dev` is `electron-vite dev`. Tests that loaded app modules moved
|
||||
to ESM imports, and `app-main-settings` was rewritten from CJS `Module._load`
|
||||
mocking to `node:test` `mock.module` (needs `--experimental-test-module-mocks`).
|
||||
A future contributor may be tempted to make the preload ESM or disable the
|
||||
sandbox — this ADR records that CJS preload under an on sandbox is the intended,
|
||||
secure default.
|
||||
@@ -73,6 +73,48 @@ test('indexer service runs one pending build after an in-flight build finishes',
|
||||
assert.deepEqual(calls, ['first', 'pending']);
|
||||
});
|
||||
|
||||
test('indexer service does not log a build cancelled by a service stop', async () => {
|
||||
const timers = manualTimers();
|
||||
const warnings = [];
|
||||
let rejectBuild;
|
||||
const service = createIndexerService({
|
||||
buildIndex: () => new Promise((_resolve, reject) => { rejectBuild = reject; }),
|
||||
watchProjects: () => null,
|
||||
writeHeartbeat: () => {},
|
||||
timers,
|
||||
stabilityMs: 0,
|
||||
logger: { warn: (msg) => warnings.push(msg) },
|
||||
});
|
||||
|
||||
const build = service.runBuildNow('startup');
|
||||
service.stop(); // manual rebuild path tears the worker down mid-build
|
||||
rejectBuild(new Error('Indexer worker stopped'));
|
||||
await build;
|
||||
|
||||
assert.deepEqual(warnings, []);
|
||||
});
|
||||
|
||||
test('indexer service logs a build that fails while running', async () => {
|
||||
const timers = manualTimers();
|
||||
const warnings = [];
|
||||
let rejectBuild;
|
||||
const service = createIndexerService({
|
||||
buildIndex: () => new Promise((_resolve, reject) => { rejectBuild = reject; }),
|
||||
watchProjects: () => null,
|
||||
writeHeartbeat: () => {},
|
||||
timers,
|
||||
stabilityMs: 0,
|
||||
logger: { warn: (msg) => warnings.push(msg) },
|
||||
});
|
||||
|
||||
const build = service.runBuildNow('watch');
|
||||
rejectBuild(new Error('disk on fire'));
|
||||
await build;
|
||||
|
||||
assert.equal(warnings.length, 1);
|
||||
assert.match(warnings[0], /Obelisk index build failed: disk on fire/);
|
||||
});
|
||||
|
||||
test('indexer service waits for a stability window before building', async () => {
|
||||
const timers = manualTimers();
|
||||
const calls = [];
|
||||
|
||||
Reference in New Issue
Block a user