From 60d47a852ea269e388c7dc8b616ac3dedff02364 Mon Sep 17 00:00:00 2001 From: tommy0103 Date: Thu, 9 Jul 2026 12:46:02 +0800 Subject: [PATCH] refactor(app): consume shared indexing core, remove duplicated indexer (Phase 5d-3c-i) The desktop app now indexes through the shared provider adapters + persist layer (scripts/providers/{claude,codex}, scripts/persist, scripts/parsing) instead of maintaining its own parallel indexer. buildIndex shrinks from ~1173 to ~592 lines, eliminating the skill<->app parse duplication that Phase 5 set out to remove. electron-vite bundles the .ts core from source with better-sqlite3 injected; the provider->parsing graph stays node:sqlite-free so nothing drags node:sqlite into the app. Also fix a misleading log: when a manual rebuild tears down the worker mid-build, the cancelled background build is a deliberate stop, not a failure. Guard the service's failure log with the stopped flag so it no longer prints "Obelisk index build failed: Indexer worker stopped" on every rebuild. - CONTEXT.md: provider-adapter + single-persist + node:sqlite-free parsing. - docs/adr/0005: app builds with electron-vite (TS+ESM), packages with electron-builder; preload CJS for sandbox; app consumes core from source. Verified: full suite 121/121; a node:sqlite-adapter dogfood of the rebuild path over real data (969 files, 285 sessions, FTS rebuilt) runs clean; app Rebuild confirmed in real Electron/better-sqlite3. Co-Authored-By: Claude Opus 4.8 --- .gitignore | 2 + CONTEXT.md | 24 +- app/src/main/indexer-service.js | 4 +- app/src/main/indexer.js | 685 ++-------------------- docs/adr/0005-app-electron-vite-ts-esm.md | 39 ++ tests/app-indexer-service.test.mjs | 42 ++ 6 files changed, 152 insertions(+), 644 deletions(-) create mode 100644 docs/adr/0005-app-electron-vite-ts-esm.md diff --git a/.gitignore b/.gitignore index 21d2053..fe7675e 100644 --- a/.gitignore +++ b/.gitignore @@ -8,3 +8,5 @@ release/ .claude/ dist/ app/out/ +HANDOFF.md +.obelisk/ \ No newline at end of file diff --git a/CONTEXT.md b/CONTEXT.md index f507b50..c4babc4 100644 --- a/CONTEXT.md +++ b/CONTEXT.md @@ -27,22 +27,26 @@ promoted to an external tool surface. ## Indexing -**Parse core**: -The pure `jsonl -> records` transform. Given a transcript file and a start line, -it yields normalized index records. It does not open, own, or write to a -database, and is shared verbatim by every indexing mode. This is the layer that -must never be duplicated. -_Avoid_: parser, ingest +**Provider adapter**: +A pure per-source module (claude, codex, later opencode, pi, …) that discovers a +source's transcript files and parses one into a stream of records. It never opens +or writes a database; adding a source means adding one adapter. The shared pure +parse/discover helpers live in `scripts/parsing.mjs`, which imports only +node:fs/path/os — deliberately node:sqlite-free so the compiled providers can be +consumed by the app (whose Electron runtime has no `node:sqlite`). +_Avoid_: parse core, parser, ingest **Record**: One normalized row destined for the index (session, message, tool call, tool -result, summary, subagent, workflow, …), emitted by the parse core before any +result, summary, subagent, workflow, …), emitted by a provider adapter before any persistence happens. **Persist layer**: -The thin, binding-specific writer that consumes records from the parse core and -writes them into SQLite inside a transaction. Two persist layers exist and differ -only in binding: `node:sqlite` (skill/CLI) and `better-sqlite3` (app). +The single shared, provider- and binding-agnostic writer that consumes records +from any adapter and writes them into an injected SQLite handle inside a +transaction. The binding is injected — `node:sqlite` (skill/CLI) or +`better-sqlite3` (app) — so there is one persist implementation, not one per +binding. _Avoid_: writer, sink, DAO **Daemon indexing mode**: diff --git a/app/src/main/indexer-service.js b/app/src/main/indexer-service.js index 2532ce7..0b63fe3 100644 --- a/app/src/main/indexer-service.js +++ b/app/src/main/indexer-service.js @@ -111,7 +111,9 @@ function createIndexerService({ writeHeartbeat(); })() .catch((error) => { - logger.warn?.(`Obelisk index build failed: ${error.message}`); + // A build in flight when the service is stopped (e.g. a manual rebuild + // tears down the worker) is a deliberate cancellation, not a failure. + if (!stopped) logger.warn?.(`Obelisk index build failed: ${error.message}`); }) .finally(() => { running = false; diff --git a/app/src/main/indexer.js b/app/src/main/indexer.js index 2eaee65..8c92e77 100644 --- a/app/src/main/indexer.js +++ b/app/src/main/indexer.js @@ -3,10 +3,21 @@ import os from 'node:os'; import path from 'node:path'; import { fileURLToPath } from 'node:url'; import Database from 'better-sqlite3'; +import { parse as claudeParse } from '../../../scripts/providers/claude.ts'; +import { parse as codexParse } from '../../../scripts/providers/codex.ts'; +import { persist } from '../../../scripts/persist.ts'; +import { + inferProjectPath, + isDir, + readLines, + codexDbId, + codexRawId, + codexParentThreadId, + readCodexGuardianThreadInfo, +} from '../../../scripts/parsing.mjs'; const __dirname = path.dirname(fileURLToPath(import.meta.url)); -const TEXT_LIMIT = 10000; const DEFAULT_CLAUDE_DIR = path.join(os.homedir(), '.claude'); const DEFAULT_CODEX_DIR = path.join(os.homedir(), '.codex'); const DEFAULT_OBELISK_DIR = path.join(os.homedir(), '.obelisk'); @@ -93,110 +104,6 @@ function copyMemoriesFromDb(db, sourceDbPath) { } } -function trunc(s) { - return typeof s === 'string' && s.length > TEXT_LIMIT ? s.slice(0, TEXT_LIMIT) : s; -} - -function truncJson(obj, limit = TEXT_LIMIT) { - if (obj === null || obj === undefined) return null; - const walk = (v) => { - if (typeof v === 'string') return v.length > limit ? v.slice(0, limit) + '...[truncated]' : v; - if (Array.isArray(v)) return v.map(walk); - if (typeof v === 'object' && v !== null) { - const out = {}; - for (const [k, val] of Object.entries(v)) out[k] = walk(val); - return out; - } - return v; - }; - return JSON.stringify(walk(obj)); -} - -function extractText(content) { - if (typeof content === 'string') return trunc(content); - if (!Array.isArray(content)) return null; - const parts = []; - for (const b of content) { - if (b.type === 'text' && b.text) parts.push(b.text); - else if (b.type === 'thinking' && b.thinking) parts.push(b.thinking); - } - return parts.length ? trunc(parts.join('\n')) : null; -} - -function extractContentType(content) { - if (typeof content === 'string') return 'text'; - if (!Array.isArray(content) || !content.length) return 'unknown'; - const types = new Set(); - let sawUnknown = false; - for (const b of content) { - if (!b || typeof b !== 'object') { sawUnknown = true; continue; } - if (b.type === 'text') types.add('text'); - else if (b.type === 'thinking') types.add('thinking'); - else if (b.type === 'tool_use') types.add('tool_use'); - else if (b.type === 'tool_result') types.add('tool_result'); - else sawUnknown = true; - } - return !sawUnknown && types.size === 1 ? [...types][0] : 'unknown'; -} - -const COMMAND_ENVELOPE_RE = /^\s*([^<]+<\/command-name>|<(?:task-notification|system-reminder)\b| b.count - a.count || a.first - b.first)[0]; - return best?.path || legacyProjectPathFromSlug(project); -} - function discoverJsonlFiles({ projectsDir = DEFAULT_PROJECTS_DIR, changedPaths = undefined } = {}) { if (Array.isArray(changedPaths) && changedPaths.length) { const changedFiles = discoverJsonlFilesForChanges({ projectsDir, changedPaths }); @@ -383,544 +290,56 @@ function needsReindex(db, fp) { return mt > row.mtime ? { needed: true, skip: row.lines_processed, mtime: mt } : { needed: false, skip: 0, mtime: mt }; } -function indexJsonl(db, fi) { - const { needed, skip, mtime } = needsReindex(db, fi.path); - if (!needed) return; - const ins = { - ses: db.prepare('INSERT OR REPLACE INTO sessions (id,title,project,project_path,started_at,ended_at,git_branch,version,message_count,jsonl_path,source) VALUES (?,?,?,?,?,?,?,?,?,?,?)'), - msg: db.prepare(` - INSERT INTO messages (uuid,session_id,type,parent_uuid,timestamp,role,text,content_type,is_meta,model,is_sidechain,agent_id,input_tokens,output_tokens,cwd,skill,source) - VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?) - ON CONFLICT(uuid) DO UPDATE SET - session_id=excluded.session_id, - type=excluded.type, - parent_uuid=excluded.parent_uuid, - timestamp=excluded.timestamp, - role=excluded.role, - text=excluded.text, - content_type=excluded.content_type, - is_meta=excluded.is_meta, - model=excluded.model, - is_sidechain=excluded.is_sidechain, - agent_id=excluded.agent_id, - input_tokens=excluded.input_tokens, - output_tokens=excluded.output_tokens, - cwd=excluded.cwd, - skill=excluded.skill, - source=excluded.source - `), - tc: db.prepare('INSERT OR REPLACE INTO tool_calls (id,message_uuid,session_id,name,input_json,file_path) VALUES (?,?,?,?,?,?)'), - tr: db.prepare('INSERT OR REPLACE INTO tool_results (tool_use_id,message_uuid,session_id,content,file_path,is_error) VALUES (?,?,?,?,?,?)'), - sum: db.prepare('INSERT OR REPLACE INTO summaries (id,session_id,timestamp,source,content) VALUES (?,?,?,?,?)'), - idx: db.prepare('INSERT OR REPLACE INTO index_state (jsonl_path,mtime,lines_processed) VALUES (?,?,?)'), - }; - const existing = !fi.isSubagent ? db.prepare('SELECT * FROM sessions WHERE id = ?').get(fi.sessionId) : null; - const sm = { - started_at: existing?.started_at || null, - ended_at: existing?.ended_at || null, - git_branch: existing?.git_branch || null, - version: existing?.version || null, - title: existing?.title || null, - n: skip > 0 ? (existing?.message_count || 0) : 0, - cwds: [], +// Index one Claude transcript via the shared provider + persist core. +// Returns { sessionId, path } when reindexed, undefined when skipped. +function indexClaudeFile(db, file) { + const { needed, skip, mtime } = needsReindex(db, file.path); + if (!needed) return undefined; + const unit = { + key: file.path, + sessionId: file.sessionId, + project: file.project, + isSubagent: file.isSubagent, + agentId: file.agentId, }; + const cursor = skip > 0 ? `${mtime}:${skip}` : null; + persist(db, unit, claudeParse(unit, cursor)); + return { sessionId: file.sessionId, path: file.path }; +} - let lineNum = 0; - readLines(fi.path, (line) => { - lineNum++; - if (lineNum <= skip) return; +function codexSessionMeta(filePath) { + let meta = null; + readLines(filePath, (line) => { let obj; try { obj = JSON.parse(line); } catch { return; } - const sid = fi.sessionId; - const ts = obj.timestamp || null; - if (obj.type === 'ai-title' && obj.aiTitle) { sm.title = obj.aiTitle; return; } - if (obj.type === 'system' && obj.subtype === 'away_summary' && obj.content) { - ins.sum.run(obj.uuid || `${sid}-away-${ts}`, sid, ts, 'away_summary', obj.content); - return; - } - if (obj.type === 'system' && obj.subtype === 'turn_duration' && obj.parentUuid && obj.durationMs) { - db.prepare('UPDATE messages SET turn_duration_ms=? WHERE uuid=?').run(obj.durationMs, obj.parentUuid); - return; - } - if (obj.type !== 'user' && obj.type !== 'assistant') return; - if (ts && (!sm.started_at || ts < sm.started_at)) sm.started_at = ts; - if (ts && (!sm.ended_at || ts > sm.ended_at)) sm.ended_at = ts; - if (obj.gitBranch) sm.git_branch = obj.gitBranch; - if (obj.version) sm.version = obj.version; - sm.n++; - if (!fi.isSubagent && obj.cwd) sm.cwds.push(obj.cwd); - - const msg = obj.message || {}; - const text = extractText(msg.content); - const contentType = extractContentType(msg.content); - const isMeta = extractMessageIsMeta(obj, text); - const usage = msg.usage || {}; - const aid = fi.isSubagent ? fi.agentId : (obj.agentId || null); - if (obj.uuid) { - ins.msg.run(obj.uuid, sid, obj.type, obj.parentUuid || null, ts, - msg.role || obj.type, text, contentType, isMeta, msg.model || null, - obj.isSidechain ? 1 : 0, aid, usage.input_tokens || null, usage.output_tokens || null, - obj.cwd || null, obj.attributionSkill || null, 'claude'); - } - if (obj.type === 'assistant' && Array.isArray(msg.content)) { - for (const b of msg.content) { - if (b.type === 'tool_use' && b.id) - ins.tc.run(b.id, obj.uuid, sid, b.name, truncJson(b.input || {}), filePath(b.name, b.input)); - } - } - if (obj.type === 'user' && Array.isArray(msg.content)) { - for (const b of msg.content) { - if (b.type !== 'tool_result' || !b.tool_use_id) continue; - const rt = typeof b.content === 'string' ? b.content - : Array.isArray(b.content) ? b.content.map(c => c.text || '').join('\n') : ''; - ins.tr.run(b.tool_use_id, obj.uuid, sid, trunc(rt), obj.toolUseResult?.filePath || null, b.is_error ? 1 : 0); - } - } - }); - - if (!fi.isSubagent) { - const pp = inferProjectPath(fi.project, sm.cwds); - ins.ses.run(fi.sessionId, sm.title, fi.project, pp, sm.started_at, sm.ended_at, sm.git_branch, sm.version, sm.n, fi.path, 'claude'); - } - ins.idx.run(fi.path, mtime, lineNum); - return { sessionId: fi.sessionId, path: fi.path }; -} - -function codexDbId(id) { - if (!id) return null; - const raw = String(id).replace(/^codex:/, ''); - return `codex:${raw}`; -} - -function codexRawId(id) { - return id ? String(id).replace(/^codex:/, '') : null; -} - -function codexLineUuid(threadId, lineNum) { - return `codex:${codexRawId(threadId)}:${String(lineNum).padStart(6, '0')}`; -} - -function codexCallId(callId) { - if (!callId) return null; - return `codex:${String(callId).replace(/^codex:/, '')}`; -} - -function codexParentThreadId(meta) { - const subagent = meta?.source?.subagent; - return subagent?.thread_spawn?.parent_thread_id - || meta?.forked_from_id - || subagent?.parent_thread_id - || null; -} - -function codexIsGuardianThread(meta, records = []) { - const subagent = meta?.source?.subagent; - if (subagent?.other === 'guardian') return true; - if (meta?.thread_source !== 'subagent') return false; - return records.some(({ obj }) => obj?.payload?.model === 'codex-auto-review' || obj?.model === 'codex-auto-review'); -} - -function deleteCodexThreadRows(db, threadRawId) { - const threadId = codexDbId(threadRawId); - if (!threadId) return; - db.prepare(` - DELETE FROM tool_results - WHERE session_id = ? - OR message_uuid IN (SELECT uuid FROM messages WHERE session_id = ? OR agent_id = ?) - `).run(threadId, threadId, threadId); - db.prepare(` - DELETE FROM tool_calls - WHERE session_id = ? - OR message_uuid IN (SELECT uuid FROM messages WHERE session_id = ? OR agent_id = ?) - `).run(threadId, threadId, threadId); - db.prepare('DELETE FROM messages WHERE session_id = ? OR agent_id = ?').run(threadId, threadId); - db.prepare('DELETE FROM subagents WHERE agent_id = ? OR session_id = ?').run(threadId, threadId); - db.prepare('DELETE FROM summaries WHERE session_id = ?').run(threadId); - db.prepare('DELETE FROM sessions WHERE id = ?').run(threadId); -} - -function readCodexGuardianThreadInfo(filePath) { - const records = []; - let metaRecord = null; - let lineNum = 0; - readLines(filePath, (line) => { - lineNum++; - let obj; - try { - obj = JSON.parse(line); - } catch { - return; - } - records.push({ lineNum, obj }); if (obj?.type === 'session_meta' && obj.payload?.id) { - metaRecord = { lineNum, obj }; - if (obj.payload?.source?.subagent?.other === 'guardian') return false; - if (obj.payload?.thread_source !== 'subagent') return false; + meta = obj.payload; + return false; } - if (metaRecord && codexIsGuardianThread(metaRecord.obj.payload, records)) return false; }); - const meta = metaRecord?.obj?.payload; - if (!meta || !codexIsGuardianThread(meta, records)) return null; - return { threadRawId: codexRawId(meta.id), lineNum }; + return meta; } -function codexAgentNickname(meta) { - return meta?.agent_nickname - || meta?.source?.subagent?.thread_spawn?.agent_nickname - || null; -} - -function codexAgentRole(meta) { - return meta?.agent_role - || meta?.source?.subagent?.thread_spawn?.agent_role - || null; -} - -function parseCodexJsonInput(value) { - if (value === null || value === undefined || value === '') return {}; - if (typeof value !== 'string') return value; - try { return JSON.parse(value); } catch { return value; } -} - -function codexUsage(payload) { - const usage = payload?.info?.last_token_usage || payload?.info?.total_token_usage || payload?.last_token_usage || null; - if (!usage) return {}; - return { - inputTokens: usage.input_tokens ?? null, - outputTokens: usage.output_tokens ?? null, - }; -} - -function codexEventText(payload) { - if (typeof payload?.message === 'string') return payload.message; - if (Array.isArray(payload?.text_elements) && payload.text_elements.length) { - const parts = payload.text_elements.map(item => typeof item === 'string' ? item : item?.text).filter(Boolean); - if (parts.length) return parts.join('\n'); - } - if (typeof payload?.text === 'string') return payload.text; - return null; -} - -function codexMessagePayloadText(payload) { - if (!Array.isArray(payload?.content)) return null; - const parts = []; - for (const block of payload.content) { - if (typeof block?.text === 'string') parts.push(block.text); - } - return parts.length ? parts.join('\n') : null; -} - -function codexVisibleMessageKey(role, text) { - return `${role || ''}\u0000${text || ''}`; -} - -function codexToolInput(payload) { - if (payload?.type === 'custom_tool_call') return parseCodexJsonInput(payload.input); - if (payload?.type === 'tool_search_call') return parseCodexJsonInput(payload.arguments); - if (payload?.type === 'web_search_call') return { action: payload.action || null }; - return parseCodexJsonInput(payload?.arguments); -} - -function codexToolOutput(payload) { - if (typeof payload?.output === 'string') return payload.output; - if (payload?.output !== undefined) return JSON.stringify(payload.output); - if (payload?.tools !== undefined) return JSON.stringify(payload.tools); - if (payload?.execution !== undefined) return JSON.stringify(payload.execution); - return null; -} - -function upsertCodexSubagent(db, { - agentId, - sessionId, - parentToolUseId = null, - agentType = null, - description = null, - durationMs = null, - totalTokens = null, -} = {}) { - if (!agentId || !sessionId) return; - db.prepare(` - INSERT INTO subagents (agent_id,session_id,parent_tool_use_id,agent_type,description,duration_ms,total_tokens) - VALUES (?,?,?,?,?,?,?) - ON CONFLICT(agent_id) DO UPDATE SET - session_id=excluded.session_id, - parent_tool_use_id=COALESCE(excluded.parent_tool_use_id, subagents.parent_tool_use_id), - agent_type=COALESCE(excluded.agent_type, subagents.agent_type), - description=COALESCE(excluded.description, subagents.description), - duration_ms=COALESCE(excluded.duration_ms, subagents.duration_ms), - total_tokens=COALESCE(excluded.total_tokens, subagents.total_tokens) - `).run(agentId, sessionId, parentToolUseId, agentType, description, durationMs, totalTokens); -} - -function indexCodexJsonl(db, fi) { - const state = needsReindex(db, fi.path); - if (!state.needed) { - const guardian = readCodexGuardianThreadInfo(fi.path); - if (guardian) deleteCodexThreadRows(db, guardian.threadRawId); - return null; - } - const mtime = state.mtime; - const records = []; - let lineNum = 0; - readLines(fi.path, (line) => { - lineNum++; - try { - records.push({ lineNum, obj: JSON.parse(line) }); - } catch {} - }); - - const metaRecord = records.find(r => r.obj?.type === 'session_meta' && r.obj.payload?.id); - if (!metaRecord) { - db.prepare('INSERT OR REPLACE INTO index_state (jsonl_path,mtime,lines_processed) VALUES (?,?,?)').run(fi.path, mtime, lineNum); - return null; - } - - const meta = metaRecord.obj.payload; - const threadRawId = codexRawId(meta.id); - if (codexIsGuardianThread(meta, records)) { - deleteCodexThreadRows(db, threadRawId); - db.prepare('INSERT OR REPLACE INTO index_state (jsonl_path,mtime,lines_processed) VALUES (?,?,?)').run(fi.path, mtime, lineNum); - return null; - } - const parentRawId = codexParentThreadId(meta); - const sessionId = codexDbId(parentRawId || threadRawId); - const agentId = parentRawId ? codexDbId(threadRawId) : null; - const isSidechain = agentId ? 1 : 0; - const projectPath = normalizeObservedCwd(meta.cwd); - const project = projectSlugFromPath(projectPath); - const sm = { - started_at: meta.timestamp || metaRecord.obj.timestamp || null, - ended_at: meta.timestamp || metaRecord.obj.timestamp || null, - git_branch: meta.git?.branch || null, - version: meta.cli_version || null, - title: null, - n: 0, - cwds: projectPath ? [projectPath] : [], - lastMessageUuid: null, - lastTextAssistantUuid: null, - totalInputTokens: 0, - totalOutputTokens: 0, - }; - - const ins = { - ses: db.prepare('INSERT OR REPLACE INTO sessions (id,title,project,project_path,started_at,ended_at,git_branch,version,message_count,jsonl_path,source) VALUES (?,?,?,?,?,?,?,?,?,?,?)'), - msg: db.prepare(` - INSERT INTO messages (uuid,session_id,type,parent_uuid,timestamp,role,text,content_type,is_meta,model,is_sidechain,agent_id,input_tokens,output_tokens,cwd,skill,source) - VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?) - ON CONFLICT(uuid) DO UPDATE SET - session_id=excluded.session_id, - type=excluded.type, - parent_uuid=excluded.parent_uuid, - timestamp=excluded.timestamp, - role=excluded.role, - text=excluded.text, - content_type=excluded.content_type, - is_meta=excluded.is_meta, - model=excluded.model, - is_sidechain=excluded.is_sidechain, - agent_id=excluded.agent_id, - input_tokens=excluded.input_tokens, - output_tokens=excluded.output_tokens, - cwd=excluded.cwd, - skill=excluded.skill, - source=excluded.source - `), - tc: db.prepare('INSERT OR REPLACE INTO tool_calls (id,message_uuid,session_id,name,input_json,file_path) VALUES (?,?,?,?,?,?)'), - tr: db.prepare('INSERT OR REPLACE INTO tool_results (tool_use_id,message_uuid,session_id,content,file_path,is_error) VALUES (?,?,?,?,?,?)'), - idx: db.prepare('INSERT OR REPLACE INTO index_state (jsonl_path,mtime,lines_processed) VALUES (?,?,?)'), - dur: db.prepare('UPDATE messages SET turn_duration_ms=? WHERE uuid=?'), - usage: db.prepare('UPDATE messages SET input_tokens=?, output_tokens=? WHERE uuid=?'), - }; - - let currentCwd = projectPath; - let currentModel = null; - const eventMessageKeys = new Set(); - const callMessageUuids = new Map(); - - for (const { obj } of records) { - if (obj?.type !== 'event_msg') continue; - const payload = obj.payload || {}; - if (payload.type !== 'user_message' && payload.type !== 'agent_message') continue; - const text = codexEventText(payload); - if (text === null) continue; - eventMessageKeys.add(codexVisibleMessageKey(payload.type === 'user_message' ? 'user' : 'assistant', text)); - } - - const updateBounds = (ts) => { - if (!ts) return; - if (!sm.started_at || ts < sm.started_at) sm.started_at = ts; - if (!sm.ended_at || ts > sm.ended_at) sm.ended_at = ts; - }; - - const insertMessage = ({ uuid, type, role, text = null, contentType = 'text', timestamp, isMeta = 0 }) => { - ins.msg.run( - uuid, - sessionId, - type, - sm.lastMessageUuid, - timestamp || null, - role, - trunc(text), - contentType, - isMeta, - currentModel, - isSidechain, - agentId, - null, - null, - currentCwd, - null, - 'codex', - ); - sm.lastMessageUuid = uuid; - if (!agentId) sm.n++; - if (type === 'assistant' && contentType === 'text') sm.lastTextAssistantUuid = uuid; - updateBounds(timestamp); - return uuid; - }; - - for (const { lineNum: currentLine, obj } of records) { - const ts = obj.timestamp || null; - if (obj.type === 'session_meta') { - if (obj.payload?.cwd) { - currentCwd = normalizeObservedCwd(obj.payload.cwd) || currentCwd; - if (currentCwd) sm.cwds.push(currentCwd); - } - if (obj.payload?.git?.branch) sm.git_branch = obj.payload.git.branch; - if (obj.payload?.cli_version) sm.version = obj.payload.cli_version; - updateBounds(obj.payload?.timestamp || ts); - continue; - } - if (obj.type === 'turn_context') { - currentCwd = normalizeObservedCwd(obj.payload?.cwd) || currentCwd; - currentModel = obj.payload?.model || currentModel; - if (currentCwd) sm.cwds.push(currentCwd); - updateBounds(ts); - continue; - } - if (obj.type === 'event_msg') { - const payload = obj.payload || {}; - if (payload.type === 'user_message' || payload.type === 'agent_message' || payload.type === 'agent_reasoning') { - const text = codexEventText(payload); - if (text === null) continue; - const isReasoning = payload.type === 'agent_reasoning'; - insertMessage({ - uuid: codexLineUuid(threadRawId, currentLine), - type: payload.type === 'user_message' ? 'user' : 'assistant', - role: payload.type === 'user_message' ? 'user' : 'assistant', - text, - contentType: isReasoning ? 'thinking' : 'text', - timestamp: ts, - }); - continue; - } - if (payload.type === 'collab_agent_spawn_end' && payload.call_id && payload.new_thread_id) { - const uuid = insertMessage({ - uuid: codexLineUuid(threadRawId, currentLine), - type: 'assistant', - role: 'assistant', - text: null, - contentType: 'tool_use', - timestamp: ts, - }); - const toolId = codexCallId(payload.call_id); - const description = payload.new_agent_nickname || payload.new_agent_role || 'Agent'; - const input = { - description, - subagent_type: payload.new_agent_role || 'Agent', - prompt: payload.prompt || '', - new_thread_id: payload.new_thread_id, - model: payload.model || null, - reasoning_effort: payload.reasoning_effort || null, - }; - ins.tc.run(toolId, uuid, sessionId, 'Agent', truncJson(input), null); - callMessageUuids.set(toolId, uuid); - upsertCodexSubagent(db, { - agentId: codexDbId(payload.new_thread_id), - sessionId, - parentToolUseId: toolId, - agentType: payload.new_agent_role || null, - description, - }); - continue; - } - if (payload.type === 'task_complete') { - if (sm.lastTextAssistantUuid && payload.duration_ms !== undefined) { - ins.dur.run(payload.duration_ms || null, sm.lastTextAssistantUuid); - } - updateBounds(ts); - continue; - } - if (payload.type === 'token_count') { - const usage = codexUsage(payload); - if (usage.inputTokens !== null) sm.totalInputTokens = usage.inputTokens; - if (usage.outputTokens !== null) sm.totalOutputTokens = usage.outputTokens; - if (sm.lastTextAssistantUuid && (usage.inputTokens !== null || usage.outputTokens !== null)) { - ins.usage.run(usage.inputTokens, usage.outputTokens, sm.lastTextAssistantUuid); - } - continue; - } - if (payload.type === 'thread_name_updated' && payload.thread_name) { - sm.title = payload.thread_name; - } - continue; - } - if (obj.type !== 'response_item') continue; - const payload = obj.payload || {}; - if (payload.type === 'message' && payload.role !== 'developer') { - const text = codexMessagePayloadText(payload); - const role = payload.role || 'assistant'; - if (text !== null && !eventMessageKeys.has(codexVisibleMessageKey(role, text))) { - insertMessage({ - uuid: codexLineUuid(threadRawId, currentLine), - type: role === 'user' ? 'user' : 'assistant', - role, - text, - contentType: 'text', - timestamp: ts, - }); - } - continue; - } - if (['function_call', 'custom_tool_call', 'tool_search_call', 'web_search_call'].includes(payload.type) && payload.call_id) { - const uuid = insertMessage({ - uuid: codexLineUuid(threadRawId, currentLine), - type: 'assistant', - role: 'assistant', - text: null, - contentType: 'tool_use', - timestamp: ts, - }); - const name = payload.name || payload.tool || payload.type.replace(/_call$/, ''); - const toolId = codexCallId(payload.call_id); - ins.tc.run(toolId, uuid, sessionId, name, truncJson(codexToolInput(payload)), null); - callMessageUuids.set(toolId, uuid); - continue; - } - if (['function_call_output', 'custom_tool_call_output', 'tool_search_output'].includes(payload.type) && payload.call_id) { - const toolId = codexCallId(payload.call_id); - ins.tr.run(toolId, callMessageUuids.get(toolId) || null, sessionId, trunc(codexToolOutput(payload) || ''), null, payload.is_error ? 1 : 0); +// Index one Codex rollout via the shared provider + persist core (full reparse). +// Returns { sessionId, path } when reindexed, undefined when skipped. +function indexCodexFile(db, file) { + const { needed } = needsReindex(db, file.path); + const guardian = readCodexGuardianThreadInfo(file.path); + if (!needed) { + if (guardian) { + persist(db, { key: file.path, sessionId: '' }, (function* () { + yield { kind: 'delete-session', sessionId: codexDbId(guardian.threadRawId) }; + return null; + })()); } + return undefined; } - - if (agentId) { - const tokenTotal = (sm.totalInputTokens || 0) + (sm.totalOutputTokens || 0); - const started = sm.started_at ? new Date(sm.started_at).getTime() : null; - const ended = sm.ended_at ? new Date(sm.ended_at).getTime() : null; - upsertCodexSubagent(db, { - agentId, - sessionId, - agentType: codexAgentRole(meta), - description: codexAgentNickname(meta), - durationMs: started && ended ? ended - started : null, - totalTokens: tokenTotal || null, - }); - } else { - const pp = inferProjectPath(project, sm.cwds); - ins.ses.run(sessionId, sm.title, project, pp, sm.started_at, sm.ended_at, sm.git_branch, sm.version, sm.n, fi.path, 'codex'); - } - ins.idx.run(fi.path, mtime, lineNum); - return { sessionId, path: fi.path }; + const unit = { key: file.path, sessionId: '' }; + persist(db, unit, codexParse(unit, null)); + if (guardian) return undefined; + const meta = codexSessionMeta(file.path); + const sessionId = meta ? codexDbId(codexParentThreadId(meta) || codexRawId(meta.id)) : undefined; + return { sessionId, path: file.path }; } function indexCodexSessionIndex(db, { codexDir = DEFAULT_CODEX_DIR } = {}) { @@ -1122,7 +541,7 @@ function buildIndex({ for (const file of files) { db.exec('BEGIN'); try { - const indexed = file.source === 'codex' ? indexCodexJsonl(db, file) : indexJsonl(db, file); + const indexed = file.source === 'codex' ? indexCodexFile(db, file) : indexClaudeFile(db, file); if (indexed?.sessionId) affectedSessionIds.add(indexed.sessionId); if (file.source !== 'codex') indexSubagentMeta(db, file); db.exec('COMMIT'); diff --git a/docs/adr/0005-app-electron-vite-ts-esm.md b/docs/adr/0005-app-electron-vite-ts-esm.md new file mode 100644 index 0000000..5484d15 --- /dev/null +++ b/docs/adr/0005-app-electron-vite-ts-esm.md @@ -0,0 +1,39 @@ +# The app builds with electron-vite (TS + ESM), packages with electron-builder + +**Context.** The desktop app must consume the shared TypeScript/ESM Core +(`providers/*` + `persist`) instead of maintaining its own duplicate indexer, and +the app itself should be TypeScript + ESM long-term. The app previously ran raw +CommonJS on Electron's Node with only the Vue renderer built by Vite; the main +process had no build step, and Electron's bundled Node (20 on Electron 33) can +neither strip TypeScript nor use `node:sqlite`. Options for the main-process build +were a hand-rolled tsc/esbuild step, `vite-plugin-electron`, or `electron-vite`. + +**Decision.** Adopt **electron-vite** to build all three processes (main, preload, +renderer) as TypeScript + ESM, and keep **electron-builder** for packaging +(dmg/nsis/AppImage). electron-vite is purpose-built for the Electron three-process +model and handles the parts a DIY build would force us to hand-maintain forever +(per-process module format, native-module externalization, dev reload). Specific +decisions within this: + +- **Preload is emitted as CommonJS** even though the app is ESM: the sandboxed + renderer (sandbox is on by default since Electron 20, and we keep it on for + security) does not support ESM preload. Source stays ESM; only the preload + output format is CJS. `main` loads `../preload/index.js`. +- **The app consumes the Core from source**: electron-vite/rollup bundles + `scripts/providers/*` + `scripts/persist` (and their `scripts/parsing.mjs` + dependency) into the app's main/worker build, injecting `better-sqlite3`. This + works because the provider→parsing import graph is node:sqlite-free (ADR-0001), + so nothing drags `node:sqlite` into the app. The `dist/` from `build:core` + (ADR-0003) remains for the skill artifact; the app does not need it. +- **better-sqlite3 stays the app's binding**, externalized (not bundled) and + unpacked from the asar. + +**Consequences.** The app is restructured into `src/{main,preload,renderer}` with +`electron.vite.config.ts`; each main module is a build input so CommonJS-style +require resolution and the indexer worker (`{ type: 'module' }`) resolve at +runtime. `npm run dev` is `electron-vite dev`. Tests that loaded app modules moved +to ESM imports, and `app-main-settings` was rewritten from CJS `Module._load` +mocking to `node:test` `mock.module` (needs `--experimental-test-module-mocks`). +A future contributor may be tempted to make the preload ESM or disable the +sandbox — this ADR records that CJS preload under an on sandbox is the intended, +secure default. diff --git a/tests/app-indexer-service.test.mjs b/tests/app-indexer-service.test.mjs index 113adbb..7e23903 100644 --- a/tests/app-indexer-service.test.mjs +++ b/tests/app-indexer-service.test.mjs @@ -73,6 +73,48 @@ test('indexer service runs one pending build after an in-flight build finishes', assert.deepEqual(calls, ['first', 'pending']); }); +test('indexer service does not log a build cancelled by a service stop', async () => { + const timers = manualTimers(); + const warnings = []; + let rejectBuild; + const service = createIndexerService({ + buildIndex: () => new Promise((_resolve, reject) => { rejectBuild = reject; }), + watchProjects: () => null, + writeHeartbeat: () => {}, + timers, + stabilityMs: 0, + logger: { warn: (msg) => warnings.push(msg) }, + }); + + const build = service.runBuildNow('startup'); + service.stop(); // manual rebuild path tears the worker down mid-build + rejectBuild(new Error('Indexer worker stopped')); + await build; + + assert.deepEqual(warnings, []); +}); + +test('indexer service logs a build that fails while running', async () => { + const timers = manualTimers(); + const warnings = []; + let rejectBuild; + const service = createIndexerService({ + buildIndex: () => new Promise((_resolve, reject) => { rejectBuild = reject; }), + watchProjects: () => null, + writeHeartbeat: () => {}, + timers, + stabilityMs: 0, + logger: { warn: (msg) => warnings.push(msg) }, + }); + + const build = service.runBuildNow('watch'); + rejectBuild(new Error('disk on fire')); + await build; + + assert.equal(warnings.length, 1); + assert.match(warnings[0], /Obelisk index build failed: disk on fire/); +}); + test('indexer service waits for a stability window before building', async () => { const timers = manualTimers(); const calls = [];