From 4eec6b38c92eacbebef75466e08c2f683056b6b7 Mon Sep 17 00:00:00 2001 From: tommy0103 Date: Sat, 13 Jun 2026 03:42:01 +0800 Subject: [PATCH] feat(app): embed indexer in Electron with file-watching service and UI refinements Extract schema DDL into scripts/schema.sql shared between CLI and app. Add an in-process chokidar-based indexer-service that watches ~/.claude/projects for JSONL changes, debounces, and triggers background rebuilds via a worker thread. Rename Usage view to Activity, flesh out MemoryDetail and SubagentDetail views, and refine App.vue layout/routing. The main process now starts/stops the indexer lifecycle and notifies renderer windows on index updates. --- README.md | 12 +- SKILL.md | 4 +- app/indexer-service.js | 163 +++++++ app/indexer-worker-client.js | 59 +++ app/indexer-worker.js | 17 + app/indexer.js | 439 ++++++++++++++++++ app/main.js | 35 +- app/package-lock.json | 31 +- app/package.json | 17 +- app/preload.js | 5 + app/renderer/src/App.vue | 368 +++++++-------- app/renderer/src/components/Sidebar.vue | 12 +- app/renderer/src/components/Toolbar.vue | 4 +- app/renderer/src/data.js | 17 +- app/renderer/src/main.js | 13 +- app/renderer/src/router.js | 8 +- app/renderer/src/utils.js | 10 +- .../src/views/{Usage.vue => Activity.vue} | 2 +- app/renderer/src/views/MemoryDetail.vue | 112 ++++- app/renderer/src/views/SessionDetail.vue | 16 +- app/renderer/src/views/SessionList.vue | 40 +- app/renderer/src/views/SubagentDetail.vue | 138 +++++- app/renderer/styles/list.css | 6 + app/renderer/styles/sidebar.css | 2 +- app/renderer/styles/toolbar.css | 2 +- references/schema.md | 11 + scripts/db.mjs | 61 +-- scripts/indexer.mjs | 23 +- scripts/schema.sql | 57 +++ 29 files changed, 1370 insertions(+), 314 deletions(-) create mode 100644 app/indexer-service.js create mode 100644 app/indexer-worker-client.js create mode 100644 app/indexer-worker.js create mode 100644 app/indexer.js rename app/renderer/src/views/{Usage.vue => Activity.vue} (99%) create mode 100644 scripts/schema.sql diff --git a/README.md b/README.md index 750e912..48188f8 100644 --- a/README.md +++ b/README.md @@ -123,8 +123,12 @@ same SQLite data. - `references/schema.md` — full SQLite schema and API reference - `references/query-patterns.md` — copyable CodeAct recipes for common retrieval tasks +- `references/retrieval-semantics.md` — query design frame for scoped and synthesis retrieval - `references/pitfalls.md` — scope, FTS, ordering, compact/raw, and field-name traps +The executable SQLite schema lives in `scripts/schema.sql`; `references/schema.md` +is the human/agent explanation of that contract. + The design is progressive disclosure with guardrails: the main skill keeps the core contract and high-risk pitfalls visible, while longer recipes and the full schema stay out of the first prompt until the agent needs them. @@ -149,16 +153,22 @@ Full-text search via FTS5 covers message text across every session layer and ran .claude/skills/obelisk/ ├── SKILL.md # Skill definition + simple API + examples ├── scripts/ -│ └── runtime.mjs # Indexer + query runtime (400 lines, zero deps) +│ ├── schema.sql # Executable SQLite schema +│ └── runtime.mjs # Indexer + query runtime (zero deps) └── references/ ├── schema.md # Full table schema + advanced API reference ├── query-patterns.md # Copyable retrieval recipes + ├── retrieval-semantics.md # Query design frame for retrieval semantics └── pitfalls.md # Scope, FTS, ordering, and compactness traps ``` ## Implementation Notes The index rebuilds incrementally — only new or modified JSONL files are re-parsed. +When the optional app is running, it is the active indexer: it watches Claude +project files, builds in a worker thread, writes `__app_heartbeat__` plus +`__app_last_successful_build__` into `index_state`, and the skill-side lazy +build skips work only while both markers are fresh. Zero npm dependencies. Uses Node 22's built-in node:sqlite with FTS5. The entire runtime is ~400 lines. diff --git a/SKILL.md b/SKILL.md index 6dadc8b..132f117 100644 --- a/SKILL.md +++ b/SKILL.md @@ -144,7 +144,9 @@ Read-only SQL SELECT/WITH with `?` placeholders. Returns array rows. SQL is an escape hatch for exact structured joins and aggregations after the helper-first surface is insufficient; it is not the default retrieval entry point. -Before writing non-trivial SQL, read `references/schema.md`. Common safe joins: +Before writing non-trivial SQL, read `references/schema.md`. The executable DDL +lives in `scripts/schema.sql`; use the reference for query semantics and the SQL +file for schema-source alignment. Common safe joins: - `tool_calls` does not have timestamps. Join `messages m ON m.uuid = tc.message_uuid`. - `tool_results` does not have timestamps. Join `messages m ON m.uuid = tr.message_uuid`. diff --git a/app/indexer-service.js b/app/indexer-service.js new file mode 100644 index 0000000..4d9de59 --- /dev/null +++ b/app/indexer-service.js @@ -0,0 +1,163 @@ +const fs = require('fs'); +const os = require('os'); +const path = require('path'); + +const DEFAULT_PROJECTS_DIR = path.join(os.homedir(), '.claude', 'projects'); +const DEFAULT_DEBOUNCE_MS = 2000; +const DEFAULT_STABILITY_MS = 500; +const DEFAULT_HEARTBEAT_MS = 30000; +const DEFAULT_WATCH_RETRY_MS = 5000; + +function createIndexerService({ + projectsDir = DEFAULT_PROJECTS_DIR, + debounceMs = DEFAULT_DEBOUNCE_MS, + stabilityMs = DEFAULT_STABILITY_MS, + heartbeatMs = DEFAULT_HEARTBEAT_MS, + watchRetryMs = DEFAULT_WATCH_RETRY_MS, + buildIndex, + writeHeartbeat = () => {}, + watchProjects, + chokidar, + timers = { + setTimeout, + clearTimeout, + setInterval, + clearInterval, + }, + logger = console, +} = {}) { + if (typeof buildIndex !== 'function') throw new Error('createIndexerService() requires buildIndex'); + const watch = watchProjects || ((onChange) => { + if (!fs.existsSync(projectsDir)) return null; + const watcher = (chokidar || require('chokidar')).watch(projectsDir, { + cwd: projectsDir, + ignoreInitial: true, + awaitWriteFinish: { + stabilityThreshold: Math.max(stabilityMs, 500), + pollInterval: 100, + }, + ignored: (targetPath, stats) => { + if (stats?.isDirectory()) return false; + if (!stats) return false; + return !String(targetPath).endsWith('.jsonl') && !String(targetPath).endsWith('.json'); + }, + }); + const onFileChange = (filename) => { + const name = filename ? String(filename) : ''; + if (!name || name.endsWith('.jsonl') || name.endsWith('.json')) onChange(name); + }; + return watcher + .on('add', onFileChange) + .on('change', onFileChange) + .on('unlink', onFileChange) + .on('error', (error) => { + logger.warn?.(`Obelisk watcher failed: ${error.message}`); + }); + }); + + let buildTimer = null; + let stabilityTimer = null; + let heartbeatTimer = null; + let watchRetryTimer = null; + let watcher = null; + let stopped = false; + let running = false; + let pending = false; + let lastReason = null; + let idlePromise = Promise.resolve(); + + const runBuildNow = (reason = 'manual') => { + if (stopped) return idlePromise; + if (running) { + pending = true; + return idlePromise; + } + running = true; + pending = false; + idlePromise = (async () => { + await buildIndex({ reason }); + writeHeartbeat(); + })() + .catch((error) => { + logger.warn?.(`Obelisk index build failed: ${error.message}`); + }) + .finally(() => { + running = false; + if (pending && !stopped) { + pending = false; + runBuildNow('pending'); + } + }); + return idlePromise; + }; + + const scheduleBuild = (reason = 'change') => { + if (stopped) return; + lastReason = reason; + if (running) pending = true; + if (buildTimer) timers.clearTimeout(buildTimer); + if (stabilityTimer) timers.clearTimeout(stabilityTimer); + buildTimer = timers.setTimeout(() => { + buildTimer = null; + if (stabilityMs <= 0) { + runBuildNow(lastReason || reason); + return; + } + stabilityTimer = timers.setTimeout(() => { + stabilityTimer = null; + runBuildNow(lastReason || reason); + }, stabilityMs); + }, debounceMs); + }; + + const startWatching = () => { + if (stopped || watcher) return; + watcher = watch(() => scheduleBuild('watch')); + if (!watcher) { + watchRetryTimer = timers.setTimeout(() => { + watchRetryTimer = null; + startWatching(); + }, watchRetryMs); + } + }; + + const start = ({ buildOnStart = true } = {}) => { + stopped = false; + if (buildOnStart) scheduleBuild('startup'); + startWatching(); + if (typeof timers.setInterval === 'function') { + heartbeatTimer = timers.setInterval(() => { + try { + writeHeartbeat(); + } catch (error) { + logger.warn?.(`Obelisk heartbeat failed: ${error.message}`); + } + }, heartbeatMs); + } + }; + + const stop = () => { + stopped = true; + pending = false; + if (buildTimer) timers.clearTimeout(buildTimer); + buildTimer = null; + if (stabilityTimer) timers.clearTimeout(stabilityTimer); + stabilityTimer = null; + if (watchRetryTimer) timers.clearTimeout(watchRetryTimer); + watchRetryTimer = null; + if (heartbeatTimer && typeof timers.clearInterval === 'function') timers.clearInterval(heartbeatTimer); + heartbeatTimer = null; + if (watcher?.close) watcher.close(); + watcher = null; + }; + + return { + start, + stop, + scheduleBuild, + runBuildNow, + idle: () => idlePromise, + }; +} + +module.exports = { createIndexerService }; diff --git a/app/indexer-worker-client.js b/app/indexer-worker-client.js new file mode 100644 index 0000000..7286b30 --- /dev/null +++ b/app/indexer-worker-client.js @@ -0,0 +1,59 @@ +const path = require('path'); +const { Worker } = require('worker_threads'); + +function createWorkerBuildIndex({ + workerPath = path.join(__dirname, 'indexer-worker.js'), + WorkerImpl = Worker, +} = {}) { + let worker = null; + let nextId = 1; + const pending = new Map(); + + const rejectPending = (error) => { + for (const { reject } of pending.values()) reject(error); + pending.clear(); + }; + + const ensureWorker = () => { + if (worker) return worker; + worker = new WorkerImpl(workerPath); + worker.on('message', (message) => { + const current = pending.get(message.id); + if (!current) return; + pending.delete(message.id); + if (message.error) { + const error = new Error(message.error.message); + error.stack = message.error.stack; + current.reject(error); + } else { + current.resolve(message.result); + } + }); + worker.on('error', (error) => { + rejectPending(error); + worker = null; + }); + worker.on('exit', (code) => { + if (pending.size) rejectPending(new Error(`Indexer worker exited with code ${code}`)); + worker = null; + }); + return worker; + }; + + const buildIndex = (args = {}) => new Promise((resolve, reject) => { + const id = nextId++; + pending.set(id, { resolve, reject }); + ensureWorker().postMessage({ id, args }); + }); + + const stop = () => { + const current = worker; + worker = null; + if (current?.terminate) current.terminate(); + rejectPending(new Error('Indexer worker stopped')); + }; + + return { buildIndex, stop }; +} + +module.exports = { createWorkerBuildIndex }; diff --git a/app/indexer-worker.js b/app/indexer-worker.js new file mode 100644 index 0000000..a1241d6 --- /dev/null +++ b/app/indexer-worker.js @@ -0,0 +1,17 @@ +const { parentPort } = require('worker_threads'); +const { buildIndex } = require('./indexer'); + +parentPort.on('message', ({ id, args }) => { + try { + const result = buildIndex(args || {}); + parentPort.postMessage({ id, result }); + } catch (error) { + parentPort.postMessage({ + id, + error: { + message: error.message, + stack: error.stack, + }, + }); + } +}); diff --git a/app/indexer.js b/app/indexer.js new file mode 100644 index 0000000..f29304e --- /dev/null +++ b/app/indexer.js @@ -0,0 +1,439 @@ +const fs = require('fs'); +const os = require('os'); +const path = require('path'); +const Database = require('better-sqlite3'); + +const TEXT_LIMIT = 10000; +const DEFAULT_CLAUDE_DIR = path.join(os.homedir(), '.claude'); +const DEFAULT_DB_PATH = path.join(DEFAULT_CLAUDE_DIR, 'obelisk.sqlite'); +const DEFAULT_PROJECTS_DIR = path.join(DEFAULT_CLAUDE_DIR, 'projects'); +const DEFAULT_HISTORY_PATH = path.join(DEFAULT_CLAUDE_DIR, 'history.jsonl'); + +function resolveSchemaPath() { + const candidates = [ + path.join(__dirname, 'schema.sql'), + path.join(__dirname, '..', 'scripts', 'schema.sql'), + process.resourcesPath ? path.join(process.resourcesPath, 'scripts', 'schema.sql') : null, + ].filter(Boolean); + const found = candidates.find(p => fs.existsSync(p)); + if (!found) throw new Error('Obelisk schema.sql not found'); + return found; +} + +function openIndexDb({ dbPath = DEFAULT_DB_PATH, schemaPath = resolveSchemaPath(), DatabaseImpl = Database } = {}) { + fs.mkdirSync(path.dirname(dbPath), { recursive: true }); + const db = new DatabaseImpl(dbPath); + db.pragma('journal_mode = WAL'); + db.pragma('synchronous = NORMAL'); + db.exec(fs.readFileSync(schemaPath, 'utf8')); + migrateDb(db); + return db; +} + +function ensureColumn(db, table, column, definition) { + const columns = db.prepare(`PRAGMA table_info(${table})`).all().map(c => c.name); + if (!columns.includes(column)) db.exec(`ALTER TABLE ${table} ADD COLUMN ${column} ${definition}`); +} + +function migrateDb(db) { + ensureColumn(db, 'messages', 'content_type', 'TEXT'); + ensureColumn(db, 'messages', 'is_meta', 'INTEGER DEFAULT 0'); + ensureColumn(db, 'memories', 'anchors', 'TEXT'); + ensureColumn(db, 'memories', 'deleted_at', 'TEXT'); + ensureColumn(db, 'memories', 'deleted_reason', 'TEXT'); +} + +function trunc(s) { + return typeof s === 'string' && s.length > TEXT_LIMIT ? s.slice(0, TEXT_LIMIT) : s; +} + +function truncJson(obj, limit = TEXT_LIMIT) { + if (obj === null || obj === undefined) return null; + const walk = (v) => { + if (typeof v === 'string') return v.length > limit ? v.slice(0, limit) + '...[truncated]' : v; + if (Array.isArray(v)) return v.map(walk); + if (typeof v === 'object' && v !== null) { + const out = {}; + for (const [k, val] of Object.entries(v)) out[k] = walk(val); + return out; + } + return v; + }; + return JSON.stringify(walk(obj)); +} + +function extractText(content) { + if (typeof content === 'string') return trunc(content); + if (!Array.isArray(content)) return null; + const parts = []; + for (const b of content) { + if (b.type === 'text' && b.text) parts.push(b.text); + else if (b.type === 'thinking' && b.thinking) parts.push(b.thinking); + } + return parts.length ? trunc(parts.join('\n')) : null; +} + +function extractContentType(content) { + if (typeof content === 'string') return 'text'; + if (!Array.isArray(content) || !content.length) return 'unknown'; + const types = new Set(); + let sawUnknown = false; + for (const b of content) { + if (!b || typeof b !== 'object') { sawUnknown = true; continue; } + if (b.type === 'text') types.add('text'); + else if (b.type === 'thinking') types.add('thinking'); + else if (b.type === 'tool_use') types.add('tool_use'); + else if (b.type === 'tool_result') types.add('tool_result'); + else sawUnknown = true; + } + return !sawUnknown && types.size === 1 ? [...types][0] : 'unknown'; +} + +const COMMAND_ENVELOPE_RE = /^\s*([^<]+<\/command-name>|||)/; + +function extractMessageIsMeta(record, text = extractText(record?.message?.content)) { + const msg = record?.message || {}; + if (record?.isMeta === true || msg.isMeta === true) return 1; + return typeof text === 'string' && COMMAND_ENVELOPE_RE.test(text) ? 1 : 0; +} + +function filePath(name, input) { + if (!input) return null; + return ['Read', 'Edit', 'Write', 'NotebookEdit'].includes(name) ? (input.file_path || null) : null; +} + +function isDir(p) { + try { + return fs.statSync(p).isDirectory(); + } catch { + return false; + } +} + +function readLines(filePath, callback) { + const data = fs.readFileSync(filePath, 'utf8'); + const lines = data.split('\n'); + for (const line of lines) { + if (line && callback(line) === false) return; + } +} + +function legacyProjectPathFromSlug(project) { + if (!project) return null; + return '/' + project.replace(/-/g, '/').replace(/^\//, ''); +} + +function normalizeObservedCwd(cwd) { + if (typeof cwd !== 'string' || !cwd.trim() || !path.isAbsolute(cwd)) return null; + return path.normalize(cwd); +} + +function inferProjectPath(project, observedCwds = []) { + const byPath = new Map(); + for (const cwd of observedCwds) { + const normalized = normalizeObservedCwd(cwd); + if (!normalized) continue; + const current = byPath.get(normalized) || { path: normalized, count: 0, first: byPath.size }; + current.count++; + byPath.set(normalized, current); + } + const best = [...byPath.values()].sort((a, b) => b.count - a.count || a.first - b.first)[0]; + return best?.path || legacyProjectPathFromSlug(project); +} + +function discoverJsonlFiles({ projectsDir = DEFAULT_PROJECTS_DIR } = {}) { + const files = []; + if (!fs.existsSync(projectsDir)) return files; + let projects; + try { projects = fs.readdirSync(projectsDir); } catch { return files; } + for (const proj of projects) { + const projPath = path.join(projectsDir, proj); + if (!isDir(projPath)) continue; + let entries; + try { entries = fs.readdirSync(projPath); } catch { continue; } + for (const f of entries) { + if (f.endsWith('.jsonl')) + files.push({ path: path.join(projPath, f), sessionId: f.slice(0, -6), project: proj, isSubagent: false }); + } + for (const sd of entries) { + const saDir = path.join(projPath, sd, 'subagents'); + if (!isDir(saDir)) continue; + let saEntries; + try { saEntries = fs.readdirSync(saDir); } catch { continue; } + for (const sf of saEntries) { + if (sf.endsWith('.jsonl')) + files.push({ path: path.join(saDir, sf), sessionId: sd, project: proj, isSubagent: true, agentId: sf.slice(0, -6) }); + } + const wfRoot = path.join(saDir, 'workflows'); + if (!isDir(wfRoot)) continue; + let wfDirs; + try { wfDirs = fs.readdirSync(wfRoot); } catch { continue; } + for (const wfDir of wfDirs) { + const wfPath = path.join(wfRoot, wfDir); + if (!isDir(wfPath)) continue; + let wfEntries; + try { wfEntries = fs.readdirSync(wfPath); } catch { continue; } + for (const wf of wfEntries) { + if (wf.endsWith('.jsonl')) + files.push({ path: path.join(wfPath, wf), sessionId: sd, project: proj, isSubagent: true, agentId: wf.slice(0, -6), workflowRunId: wfDir }); + } + } + } + } + return files; +} + +function needsReindex(db, fp) { + const mt = fs.statSync(fp).mtimeMs; + const row = db.prepare('SELECT mtime, lines_processed FROM index_state WHERE jsonl_path = ?').get(fp); + if (!row) return { needed: true, skip: 0, mtime: mt }; + return mt > row.mtime ? { needed: true, skip: row.lines_processed, mtime: mt } : { needed: false, skip: 0, mtime: mt }; +} + +function indexJsonl(db, fi) { + const { needed, skip, mtime } = needsReindex(db, fi.path); + if (!needed) return; + const ins = { + ses: db.prepare('INSERT OR REPLACE INTO sessions (id,title,project,project_path,started_at,ended_at,git_branch,version,message_count,jsonl_path) VALUES (?,?,?,?,?,?,?,?,?,?)'), + msg: db.prepare('INSERT OR REPLACE INTO messages (uuid,session_id,type,parent_uuid,timestamp,role,text,content_type,is_meta,model,is_sidechain,agent_id,input_tokens,output_tokens,cwd,skill) VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)'), + tc: db.prepare('INSERT OR REPLACE INTO tool_calls (id,message_uuid,session_id,name,input_json,file_path) VALUES (?,?,?,?,?,?)'), + tr: db.prepare('INSERT OR REPLACE INTO tool_results (tool_use_id,message_uuid,session_id,content,file_path,is_error) VALUES (?,?,?,?,?,?)'), + sum: db.prepare('INSERT OR REPLACE INTO summaries (id,session_id,timestamp,source,content) VALUES (?,?,?,?,?)'), + idx: db.prepare('INSERT OR REPLACE INTO index_state (jsonl_path,mtime,lines_processed) VALUES (?,?,?)'), + }; + const existing = !fi.isSubagent ? db.prepare('SELECT * FROM sessions WHERE id = ?').get(fi.sessionId) : null; + const sm = { + started_at: existing?.started_at || null, + ended_at: existing?.ended_at || null, + git_branch: existing?.git_branch || null, + version: existing?.version || null, + title: existing?.title || null, + n: existing?.message_count || 0, + cwds: [], + }; + + let lineNum = 0; + readLines(fi.path, (line) => { + lineNum++; + if (lineNum <= skip) return; + let obj; + try { obj = JSON.parse(line); } catch { return; } + const sid = fi.sessionId; + const ts = obj.timestamp || null; + if (obj.type === 'ai-title' && obj.aiTitle) { sm.title = obj.aiTitle; return; } + if (obj.type === 'system' && obj.subtype === 'away_summary' && obj.content) { + ins.sum.run(obj.uuid || `${sid}-away-${ts}`, sid, ts, 'away_summary', obj.content); + return; + } + if (obj.type === 'system' && obj.subtype === 'turn_duration' && obj.parentUuid && obj.durationMs) { + db.prepare('UPDATE messages SET turn_duration_ms=? WHERE uuid=?').run(obj.durationMs, obj.parentUuid); + return; + } + if (obj.type !== 'user' && obj.type !== 'assistant') return; + if (ts && (!sm.started_at || ts < sm.started_at)) sm.started_at = ts; + if (ts && (!sm.ended_at || ts > sm.ended_at)) sm.ended_at = ts; + if (obj.gitBranch) sm.git_branch = obj.gitBranch; + if (obj.version) sm.version = obj.version; + sm.n++; + if (!fi.isSubagent && obj.cwd) sm.cwds.push(obj.cwd); + + const msg = obj.message || {}; + const text = extractText(msg.content); + const contentType = extractContentType(msg.content); + const isMeta = extractMessageIsMeta(obj, text); + const usage = msg.usage || {}; + const aid = fi.isSubagent ? fi.agentId : (obj.agentId || null); + if (obj.uuid) { + ins.msg.run(obj.uuid, sid, obj.type, obj.parentUuid || null, ts, + msg.role || obj.type, text, contentType, isMeta, msg.model || null, + obj.isSidechain ? 1 : 0, aid, usage.input_tokens || null, usage.output_tokens || null, + obj.cwd || null, obj.attributionSkill || null); + } + if (obj.type === 'assistant' && Array.isArray(msg.content)) { + for (const b of msg.content) { + if (b.type === 'tool_use' && b.id) + ins.tc.run(b.id, obj.uuid, sid, b.name, truncJson(b.input || {}), filePath(b.name, b.input)); + } + } + if (obj.type === 'user' && Array.isArray(msg.content)) { + for (const b of msg.content) { + if (b.type !== 'tool_result' || !b.tool_use_id) continue; + const rt = typeof b.content === 'string' ? b.content + : Array.isArray(b.content) ? b.content.map(c => c.text || '').join('\n') : ''; + ins.tr.run(b.tool_use_id, obj.uuid, sid, trunc(rt), obj.toolUseResult?.filePath || null, b.is_error ? 1 : 0); + } + } + }); + + if (!fi.isSubagent) { + const pp = inferProjectPath(fi.project, sm.cwds); + ins.ses.run(fi.sessionId, sm.title, fi.project, pp, sm.started_at, sm.ended_at, sm.git_branch, sm.version, sm.n, fi.path); + } + ins.idx.run(fi.path, mtime, lineNum); +} + +function refreshSessionProjectPaths(db) { + const sessions = db.prepare('SELECT id, project FROM sessions').all(); + const cwdStmt = db.prepare(` + SELECT cwd FROM messages + WHERE session_id = ? AND cwd IS NOT NULL AND cwd != '' + ORDER BY timestamp IS NULL, timestamp + `); + const update = db.prepare('UPDATE sessions SET project_path = ? WHERE id = ?'); + for (const session of sessions) { + const cwds = cwdStmt.all(session.id).map(row => row.cwd); + const projectPath = inferProjectPath(session.project, cwds); + if (projectPath) update.run(projectPath, session.id); + } +} + +function indexSubagentMeta(db, fi) { + if (!fi.isSubagent) return; + const mp = fi.path.replace('.jsonl', '.meta.json'); + if (!fs.existsSync(mp)) return; + try { + const meta = JSON.parse(fs.readFileSync(mp, 'utf8')); + const tok = db.prepare('SELECT COALESCE(SUM(input_tokens),0)+COALESCE(SUM(output_tokens),0) as t FROM messages WHERE agent_id=?').get(fi.agentId); + const ts = db.prepare('SELECT MIN(timestamp) as t0, MAX(timestamp) as t1 FROM messages WHERE agent_id=?').get(fi.agentId); + const dur = ts?.t0 && ts?.t1 ? new Date(ts.t1).getTime() - new Date(ts.t0).getTime() : null; + if (fi.workflowRunId) { + db.prepare('INSERT OR REPLACE INTO workflow_agents (agent_id,run_id,session_id,agent_type,description) VALUES(?,?,?,?,?)').run(fi.agentId, fi.workflowRunId, fi.sessionId, meta.agentType||null, meta.description||null); + } else { + db.prepare('INSERT OR REPLACE INTO subagents VALUES(?,?,?,?,?,?,?)').run(fi.agentId, fi.sessionId, meta.toolUseId||null, meta.agentType||null, meta.description||null, dur, tok?.t||0); + } + } catch (error) { + console.warn(`Warning: failed to read subagent meta ${mp}: ${error.message}`); + } +} + +function indexWorkflows(db, { projectsDir = DEFAULT_PROJECTS_DIR } = {}) { + if (!fs.existsSync(projectsDir)) return; + let projects; + try { projects = fs.readdirSync(projectsDir); } catch { return; } + for (const proj of projects) { + const pp = path.join(projectsDir, proj); + if (!isDir(pp)) continue; + let entries; + try { entries = fs.readdirSync(pp); } catch { continue; } + for (const sd of entries) { + const wd = path.join(pp, sd, 'workflows'); + if (!isDir(wd)) continue; + let wfFiles; + try { wfFiles = fs.readdirSync(wd); } catch { continue; } + for (const f of wfFiles) { + if (!f.endsWith('.json')) continue; + try { + const wf = JSON.parse(fs.readFileSync(path.join(wd, f), 'utf8')); + if (!wf.runId) continue; + const ac = db.prepare('SELECT COUNT(*) as c FROM workflow_agents WHERE run_id=?').get(wf.runId); + db.prepare('INSERT OR REPLACE INTO workflows (run_id,session_id,task_id,script,result_json,timestamp,agent_count,duration_ms,total_tokens,status,workflow_name) VALUES(?,?,?,?,?,?,?,?,?,?,?)').run( + wf.runId, sd, wf.taskId||null, wf.script||null, + wf.result ? JSON.stringify(wf.result) : null, wf.timestamp||null, ac?.c||0, + wf.durationMs||null, wf.totalTokens||null, wf.status||null, wf.workflowName||null); + const progress = wf.workflowProgress || []; + for (const item of progress) { + if (item.type !== 'workflow_agent' || !item.agentId) continue; + db.prepare('UPDATE workflow_agents SET phase=?, label=?, model=?, state=?, duration_ms=?, tokens=?, tool_calls=? WHERE agent_id=?').run( + item.phaseTitle||null, item.label||null, item.model||null, item.state||null, + item.durationMs||null, item.tokens||null, item.toolCalls||null, 'agent-' + item.agentId); + } + } catch (error) { + console.warn(`Warning: failed to index workflow ${f}: ${error.message}`); + } + } + } + } +} + +function indexHistory(db, { historyPath = DEFAULT_HISTORY_PATH } = {}) { + if (!fs.existsSync(historyPath)) return; + readLines(historyPath, (line) => { + try { + const o = JSON.parse(line); + if (o.sessionId && o.title) db.prepare('UPDATE sessions SET title=? WHERE id=? AND title IS NULL').run(o.title, o.sessionId); + } catch (error) { + console.warn(`Warning: malformed history line: ${error.message}`); + } + }); +} + +function rebuildFts(db) { + db.exec("INSERT INTO messages_fts(messages_fts) VALUES('rebuild')"); + db.exec("INSERT INTO memories_fts(memories_fts) VALUES('rebuild')"); +} + +function writeIndexMarker(db, key, value = Date.now()) { + db.prepare('INSERT OR REPLACE INTO index_state (jsonl_path, mtime, lines_processed) VALUES (?, ?, 0)').run(key, value); +} + +function writeHeartbeat({ dbPath = DEFAULT_DB_PATH, DatabaseImpl = Database } = {}) { + if (!fs.existsSync(dbPath)) return; + const db = new DatabaseImpl(dbPath); + try { + writeIndexMarker(db, '__app_heartbeat__'); + } finally { + db.close(); + } +} + +function buildIndex({ + claudeDir = DEFAULT_CLAUDE_DIR, + projectsDir = path.join(claudeDir, 'projects'), + historyPath = path.join(claudeDir, 'history.jsonl'), + dbPath = path.join(claudeDir, 'obelisk.sqlite'), + schemaPath = resolveSchemaPath(), + DatabaseImpl = Database, + force = false, +} = {}) { + const db = openIndexDb({ dbPath, schemaPath, DatabaseImpl }); + const files = discoverJsonlFiles({ projectsDir }); + const latestSourceMtime = files.reduce((latest, file) => { + try { + return Math.max(latest, fs.statSync(file.path).mtimeMs); + } catch { + return latest; + } + }, 0); + + try { + if (force) db.prepare("DELETE FROM index_state WHERE jsonl_path NOT LIKE '__%'").run(); + for (const file of files) { + db.exec('BEGIN'); + try { + indexJsonl(db, file); + indexSubagentMeta(db, file); + db.exec('COMMIT'); + } catch (error) { + db.exec('ROLLBACK'); + console.warn(`Warning: failed to index ${file.path}: ${error.message}`); + } + } + db.exec('BEGIN'); + try { + indexWorkflows(db, { projectsDir }); + refreshSessionProjectPaths(db); + indexHistory(db, { historyPath }); + rebuildFts(db); + writeIndexMarker(db, '__last_build__'); + writeIndexMarker(db, '__app_heartbeat__'); + writeIndexMarker(db, '__app_last_successful_build__'); + writeIndexMarker(db, '__indexer_owner_app__'); + if (latestSourceMtime) writeIndexMarker(db, '__last_source_mtime__', latestSourceMtime); + db.exec('COMMIT'); + } catch (error) { + db.exec('ROLLBACK'); + throw error; + } + return { files: files.length, latestSourceMtime }; + } finally { + db.close(); + } +} + +module.exports = { + buildIndex, + writeHeartbeat, + openIndexDb, + discoverJsonlFiles, + inferProjectPath, +}; diff --git a/app/main.js b/app/main.js index a5a286c..d2f1a32 100644 --- a/app/main.js +++ b/app/main.js @@ -3,18 +3,44 @@ const path = require('path'); const os = require('os'); const fs = require('fs'); const Database = require('better-sqlite3'); +const { writeHeartbeat } = require('./indexer'); +const { createIndexerService } = require('./indexer-service'); +const { createWorkerBuildIndex } = require('./indexer-worker-client'); const DB_PATH = path.join(os.homedir(), '.claude', 'obelisk.sqlite'); let db; +let indexerService; +let indexerWorker; function openDb() { if (!fs.existsSync(DB_PATH)) return null; + if (db) db.close(); db = new Database(DB_PATH, { readonly: false }); db.pragma('journal_mode = WAL'); return db; } +function notifyIndexUpdated() { + for (const win of BrowserWindow.getAllWindows()) { + win.webContents.send('obelisk:index-updated'); + } +} + +function startIndexerService() { + indexerService = createIndexerService({ + buildIndex: async ({ reason }) => { + const result = await indexerWorker.buildIndex({ reason }); + openDb(); + notifyIndexUpdated(); + return result; + }, + writeHeartbeat, + }); + indexerService.start({ buildOnStart: false }); + return indexerService; +} + function createWindow() { const win = new BrowserWindow({ width: 1200, @@ -41,14 +67,21 @@ function createWindow() { } app.whenReady().then(() => { + indexerWorker = createWorkerBuildIndex(); openDb(); createWindow(); + startIndexerService().runBuildNow('startup'); app.on('activate', () => { if (BrowserWindow.getAllWindows().length === 0) createWindow(); }); }); +app.on('before-quit', () => { + if (indexerService) indexerService.stop(); + if (indexerWorker) indexerWorker.stop(); +}); + app.on('window-all-closed', () => { if (db) db.close(); if (process.platform !== 'darwin') app.quit(); @@ -137,7 +170,7 @@ ipcMain.handle('db:getSessionSummaries', (_, sessionId) => { ipcMain.handle('db:getMemories', () => { if (!db) return []; return db.prepare(` - SELECT id, session_id, project, message_start, message_end, path, summary, created_at, deleted_at, deleted_reason + SELECT id, session_id, project, message_start, message_end, path, anchors, summary, created_at, deleted_at, deleted_reason FROM memories ORDER BY created_at DESC `).all(); }); diff --git a/app/package-lock.json b/app/package-lock.json index 1791ff0..0a5fdb0 100644 --- a/app/package-lock.json +++ b/app/package-lock.json @@ -8,7 +8,8 @@ "name": "obelisk", "version": "0.1.0", "dependencies": { - "better-sqlite3": "^11.0.0" + "better-sqlite3": "^11.0.0", + "chokidar": "^4.0.3" }, "devDependencies": { "@vitejs/plugin-vue": "^5.0.0", @@ -2578,6 +2579,21 @@ "url": "https://github.com/chalk/chalk?sponsor=1" } }, + "node_modules/chokidar": { + "version": "4.0.3", + "resolved": "https://registry.npmjs.org/chokidar/-/chokidar-4.0.3.tgz", + "integrity": "sha512-Qgzu8kfBvo+cA4962jnP1KkS6Dop5NS6g7R5LFYJr4b8Ub94PPQXUksCw9PvXoeXPRRddRNC5C1JQUR2SMGtnA==", + "license": "MIT", + "dependencies": { + "readdirp": "^4.0.1" + }, + "engines": { + "node": ">= 14.16.0" + }, + "funding": { + "url": "https://paulmillr.com/funding/" + } + }, "node_modules/chownr": { "version": "1.1.4", "resolved": "https://registry.npmjs.org/chownr/-/chownr-1.1.4.tgz", @@ -5681,6 +5697,19 @@ "node": ">=10" } }, + "node_modules/readdirp": { + "version": "4.1.2", + "resolved": "https://registry.npmjs.org/readdirp/-/readdirp-4.1.2.tgz", + "integrity": "sha512-GDhwkLfywWL2s6vEjyhri+eXmfH6j1L7JE27WhqLeYzoh/A3DBaYGEj2H/HFZCn/kMfim73FXxEJTw06WtxQwg==", + "license": "MIT", + "engines": { + "node": ">= 14.18.0" + }, + "funding": { + "type": "individual", + "url": "https://paulmillr.com/funding/" + } + }, "node_modules/require-directory": { "version": "2.1.1", "resolved": "https://registry.npmjs.org/require-directory/-/require-directory-2.1.1.tgz", diff --git a/app/package.json b/app/package.json index b38725e..6b41ef8 100644 --- a/app/package.json +++ b/app/package.json @@ -20,15 +20,28 @@ "files": [ "main.js", "preload.js", + "indexer.js", + "indexer-service.js", + "indexer-worker.js", + "indexer-worker-client.js", "dist-renderer/**/*", - "node_modules/better-sqlite3/**/*" + "node_modules/better-sqlite3/**/*", + "node_modules/chokidar/**/*", + "node_modules/readdirp/**/*" + ], + "extraResources": [ + { + "from": "../scripts/schema.sql", + "to": "scripts/schema.sql" + } ], "asarUnpack": [ "node_modules/better-sqlite3/**/*" ] }, "dependencies": { - "better-sqlite3": "^11.0.0" + "better-sqlite3": "^11.0.0", + "chokidar": "^4.0.3" }, "devDependencies": { "@vitejs/plugin-vue": "^5.0.0", diff --git a/app/preload.js b/app/preload.js index b6e1e7d..2450c44 100644 --- a/app/preload.js +++ b/app/preload.js @@ -19,4 +19,9 @@ contextBridge.exposeInMainWorld('obelisk', { getProjects: () => ipcRenderer.invoke('db:getProjects'), getStats: () => ipcRenderer.invoke('db:getStats'), getUsageStats: () => ipcRenderer.invoke('db:getUsageStats'), + onIndexUpdated: (callback) => { + const listener = () => callback(); + ipcRenderer.on('obelisk:index-updated', listener); + return () => ipcRenderer.removeListener('obelisk:index-updated', listener); + }, }); diff --git a/app/renderer/src/App.vue b/app/renderer/src/App.vue index 9ffe3a9..af73463 100644 --- a/app/renderer/src/App.vue +++ b/app/renderer/src/App.vue @@ -25,16 +25,23 @@ const archivedCount = computed(() => state.memories.filter(m => m.archived).leng const totalMemoryCount = computed(() => state.memories.length); const sessionCount = computed(() => state.sessions.length); +const currentRouteType = computed(() => { + const name = route.name; + if (name === 'SessionList' || name === 'SessionDetail' || name === 'SubagentDetail') return 'sessions'; + if (name === 'Activity') return 'activity'; + return 'memory'; +}); + const sidebarProjects = computed(() => { - const items = state.route === 'sessions' ? state.sessions : state.memories; + const items = currentRouteType.value === 'sessions' ? state.sessions : state.memories; const filtered = items.filter(item => { - if (state.route === 'sessions') return true; + if (currentRouteType.value === 'sessions') return true; return state.view === 'archived' ? item.archived : !item.archived; }); let projects = [...new Set(filtered.map(item => item.project).filter(Boolean))]; if (state.projectSearch) { const q = state.projectSearch.toLowerCase(); - projects = projects.filter(p => p.toLowerCase().includes(q)); + projects = projects.filter(p => formatProjectLabel(p).toLowerCase().includes(q)); } projects.sort((a, b) => formatProjectLabel(a).localeCompare(formatProjectLabel(b))); @@ -51,6 +58,11 @@ const sidebarProjects = computed(() => { })); }); +const totalProjectCount = computed(() => { + const items = currentRouteType.value === 'sessions' ? state.sessions : state.memories; + return new Set(items.map(i => i.project).filter(Boolean)).size; +}); + // --- Toolbar visibility --- const showToolbar = computed(() => { @@ -67,8 +79,8 @@ const showSearchMsgsToggle = computed(() => { const windowTitle = computed(() => { const appName = 'Obelisk'; let scopeText = ''; - if (route.name === 'Usage') { - scopeText = 'Usage'; + if (route.name === 'Activity') { + scopeText = 'Activity'; } else if (route.name?.startsWith('Session')) { if (route.name === 'SessionDetail' || route.name === 'SubagentDetail') { const s = state.sessions.find(x => x.id === route.params.id); @@ -100,8 +112,8 @@ function handleSidebarRoute(routeName) { setRoute(routeName); if (routeName === 'sessions') { router.push('/sessions'); - } else if (routeName === 'usage') { - router.push('/usage'); + } else if (routeName === 'activity') { + router.push('/activity'); } else { router.push('/memory'); } @@ -112,6 +124,10 @@ function handleSidebarView(view) { router.push('/memory'); } +function handleClearProject() { + setProject('all'); +} + function handleSidebarProject(slug) { setProject(slug); // Stay on current list route @@ -147,94 +163,125 @@ const keepAliveIncludes = ['SessionDetail'];