Files
obelisk/scripts/query.mjs
T
tommy0103 b524339d85 feat(app): add Electron desktop UI and evolve memory/retrieval layer
Introduce an Electron app with session browser, memory list, and usage
  views (vanilla JS + Vue scaffolding). On the data layer: add content_type
  and is_meta to messages for transcript control-plane filtering, introduce
  FTS5-backed memory recall with safe tokenization, support memory archival
  via forget() through the renamed --attune runtime, and expose anchors on
  memory records.
2026-06-12 22:20:29 +08:00

538 lines
23 KiB
JavaScript

import { readLines, fs, path } from './db.mjs';
function normalizeOpts(optsOrScalar, scalarKey = 'sessionId') {
if (optsOrScalar == null) return {};
if (typeof optsOrScalar === 'string') return { [scalarKey]: optsOrScalar };
if (typeof optsOrScalar === 'number') return { limit: optsOrScalar };
return optsOrScalar;
}
function buildWhere(opts, aliases) {
const clauses = [];
const params = [];
if (opts.sessionId) { clauses.push(`${aliases.sessionId} = ?`); params.push(opts.sessionId); }
if (opts.sessions?.length) {
clauses.push(`${aliases.sessionId} IN (${opts.sessions.map(() => '?').join(',')})`);
params.push(...opts.sessions);
}
if (opts.project) { clauses.push(`${aliases.project} LIKE ?`); params.push(opts.project); }
if (opts.after) { clauses.push(`${aliases.timestamp} > ?`); params.push(opts.after); }
if (opts.before) { clauses.push(`${aliases.timestamp} < ?`); params.push(opts.before); }
if (opts.branch) { clauses.push(`${aliases.branch} = ?`); params.push(opts.branch); }
return { where: clauses.length ? clauses.join(' AND ') : '1=1', params };
}
const BASH_EXIT_PAT = 'Exit code %';
function assertReadOnlySql(sql) {
const text = String(sql || '').trim();
if (!/^(SELECT|WITH)\b/i.test(text)) {
throw new Error('sql() only supports read-only SELECT/WITH queries');
}
if (/\b(INSERT|UPDATE|DELETE|REPLACE|CREATE|DROP|ALTER|PRAGMA|VACUUM|ATTACH|DETACH)\b/i.test(text)) {
throw new Error('sql() only supports read-only SELECT/WITH queries');
}
}
const CJK_TEXT_RE = /[\p{Script=Han}\p{Script=Hiragana}\p{Script=Katakana}\p{Script=Hangul}]/u;
function assertEnglishMemoryText(value, label) {
const text = String(value || '');
if (!text.trim()) return;
if (CJK_TEXT_RE.test(text)) {
const requirement = label.includes('query') ? 'must use English terms' : 'must be written in English';
throw new Error(`${label} ${requirement}; translate user-language terms before using the memory layer`);
}
}
function buildSafeFtsQuery(text) {
const tokens = String(text || '').match(/[\p{Letter}\p{Number}]+/gu) || [];
return tokens
.slice(0, 12)
.map(token => `"${token}"`)
.join(' ');
}
function createQueryApi(db) {
const q = (sql, ...p) => {
assertReadOnlySql(sql);
return db.prepare(sql).all(...p);
};
const normalizeOverviewOpts = (optsOrScalar) => {
if (optsOrScalar == null) return {};
if (typeof optsOrScalar === 'string') return { project: optsOrScalar };
if (typeof optsOrScalar === 'number') return { limit: optsOrScalar };
return optsOrScalar;
};
const search = (text, opts = {}) => {
const { limit = 20, sessionId, project, after, before, cwd, includeMeta = false } = opts;
let where = 'WHERE mf.text MATCH ?';
const p = [text];
if (sessionId) { where += ' AND mf.session_id=?'; p.push(sessionId); }
if (project) { where += ' AND s.project LIKE ?'; p.push(project); }
if (after) { where += ' AND m.timestamp>?'; p.push(after); }
if (before) { where += ' AND m.timestamp<?'; p.push(before); }
if (cwd) { where += ' AND m.cwd LIKE ?'; p.push(cwd); }
if (!includeMeta) where += ' AND COALESCE(m.is_meta,0)=0';
p.push(limit);
const rows = db.prepare(`
SELECT m.uuid,m.session_id,m.text,m.content_type,m.is_meta,m.role,m.timestamp,m.model,m.cwd,
s.id as s_id,s.title as s_title,s.project as s_project,s.started_at as s_started,
rank
FROM messages_fts mf JOIN messages m ON m.uuid=mf.uuid LEFT JOIN sessions s ON s.id=m.session_id
${where} ORDER BY rank LIMIT ?`).all(...p);
return rows.map(r => {
const metaClause = includeMeta ? '' : 'AND COALESCE(is_meta,0)=0';
const ctx = db.prepare(
`SELECT uuid,text,content_type,is_meta,role,timestamp,model FROM messages WHERE session_id=? AND uuid!=? ${metaClause} ORDER BY ABS(JULIANDAY(timestamp)-JULIANDAY(?)) LIMIT 6`
).all(r.session_id, r.uuid, r.timestamp).sort((a,b) => a.timestamp < b.timestamp ? -1 : 1);
return {
message: { uuid: r.uuid, text: r.text, content_type: r.content_type, is_meta: r.is_meta || 0, role: r.role, timestamp: r.timestamp, model: r.model, cwd: r.cwd },
session: { id: r.s_id, title: r.s_title, project: r.s_project, started_at: r.s_started },
rank: r.rank,
context: ctx,
};
});
};
const context = (uuid) => {
const msg = db.prepare('SELECT * FROM messages WHERE uuid=?').get(uuid);
if (!msg) return null;
const session = db.prepare('SELECT * FROM sessions WHERE id=?').get(msg.session_id);
const chain = [];
let cur = msg;
while (cur?.parent_uuid) { cur = db.prepare('SELECT * FROM messages WHERE uuid=?').get(cur.parent_uuid); if (cur) chain.unshift(cur); }
let subagent = msg.agent_id ? db.prepare('SELECT * FROM subagents WHERE agent_id=?').get(msg.agent_id) : null;
let workflow = null;
if (msg.agent_id) {
const wa = db.prepare('SELECT * FROM workflow_agents WHERE agent_id=?').get(msg.agent_id);
if (wa) workflow = db.prepare('SELECT * FROM workflows WHERE run_id=?').get(wa.run_id);
}
return { message: msg, parentChain: chain, session, subagent, workflow };
};
const trace = (uuid) => {
const chain = [];
let cur = db.prepare('SELECT * FROM messages WHERE uuid=?').get(uuid);
while (cur) { chain.unshift(cur); cur = cur.parent_uuid ? db.prepare('SELECT * FROM messages WHERE uuid=?').get(cur.parent_uuid) : null; }
return chain;
};
const thread = (sid, opts = {}) => {
const includeMeta = opts?.includeMeta === true;
const metaClause = includeMeta ? '' : 'AND COALESCE(is_meta,0)=0';
return db.prepare(`SELECT * FROM messages WHERE session_id=? ${metaClause} ORDER BY timestamp`).all(sid);
};
const subagents = (optsOrSid) => {
const opts = normalizeOpts(optsOrSid);
const { limit = 100 } = opts;
const needsJoin = opts.project || opts.branch;
const { where, params } = buildWhere(opts, { sessionId: 'sa.session_id', project: 's.project', timestamp: 'sa.session_id', branch: 's.git_branch' });
params.push(limit);
const join = needsJoin ? 'LEFT JOIN sessions s ON s.id=sa.session_id' : '';
return db.prepare(`SELECT sa.* FROM subagents sa ${join} WHERE ${where} LIMIT ?`).all(...params).map(r => {
const c = db.prepare('SELECT COUNT(*) as c FROM messages WHERE agent_id=?').get(r.agent_id);
return { ...r, messageCount: c?.c || 0 };
});
};
const workflows = (optsOrSid) => {
const opts = normalizeOpts(optsOrSid);
const { limit = 100 } = opts;
const needsJoin = opts.project || opts.branch;
const { where, params } = buildWhere(opts, { sessionId: 'w.session_id', project: 's.project', timestamp: 'w.timestamp', branch: 's.git_branch' });
params.push(limit);
const join = needsJoin ? 'LEFT JOIN sessions s ON s.id=w.session_id' : '';
return db.prepare(`SELECT w.* FROM workflows w ${join} WHERE ${where} ORDER BY w.timestamp DESC LIMIT ?`).all(...params);
};
const workflowTree = (runId) => {
const wf = db.prepare('SELECT * FROM workflows WHERE run_id=?').get(runId);
if (!wf) return null;
let result = null;
try { result = JSON.parse(wf.result_json); } catch {}
const agents = db.prepare('SELECT * FROM workflow_agents WHERE run_id=?').all(runId).map(a => {
const mc = db.prepare('SELECT COUNT(*) as c FROM messages WHERE agent_id=?').get(a.agent_id);
return { ...a, messageCount: mc?.c || 0 };
});
return { ...wf, result, agents };
};
const fileHistory = (fp, opts = {}) => {
const { limit = 200, after, before } = opts;
let where = 'tc.file_path=?';
const params = [fp];
if (after) { where += ' AND m.timestamp > ?'; params.push(after); }
if (before) { where += ' AND m.timestamp < ?'; params.push(before); }
params.push(limit);
return db.prepare(
`SELECT tc.*,s.title as s_title,s.project as s_project,m.timestamp as ts FROM tool_calls tc LEFT JOIN sessions s ON s.id=tc.session_id LEFT JOIN messages m ON m.uuid=tc.message_uuid WHERE ${where} ORDER BY m.timestamp LIMIT ?`
).all(...params).map(r => ({
toolCall: { id: r.id, message_uuid: r.message_uuid, name: r.name, input_json: r.input_json },
session: { id: r.session_id, title: r.s_title, project: r.s_project },
timestamp: r.ts,
}));
};
const failures = (optsOrSid) => {
const opts = normalizeOpts(optsOrSid);
const { limit = 50 } = opts;
const needsJoin = opts.project || opts.branch;
const { where, params: filterParams } = buildWhere(opts, { sessionId: 'tr.session_id', project: 's.project', timestamp: 'rm.timestamp', branch: 's.git_branch' });
const join = needsJoin ? 'LEFT JOIN sessions s ON s.id=tr.session_id' : '';
const errorCond = `(tr.is_error = 1 OR tr.content LIKE '${BASH_EXIT_PAT}')`;
const allParams = [...filterParams, limit];
const rows = db.prepare(`SELECT tr.* FROM tool_results tr ${join} LEFT JOIN messages rm ON rm.uuid=tr.message_uuid WHERE ${errorCond} AND ${where} ORDER BY rm.timestamp DESC LIMIT ?`).all(...allParams);
return rows.map(r => {
const tc = db.prepare('SELECT * FROM tool_calls WHERE id=?').get(r.tool_use_id);
const session = db.prepare('SELECT * FROM sessions WHERE id=?').get(r.session_id);
const rm = db.prepare('SELECT * FROM messages WHERE uuid=?').get(r.message_uuid);
const next = rm?.timestamp ? db.prepare('SELECT * FROM messages WHERE session_id=? AND timestamp>? ORDER BY timestamp LIMIT 3').all(r.session_id, rm.timestamp) : [];
return { toolCall: tc, result: r, session, nextMessages: next };
});
};
const sessions = (optsOrN) => {
const opts = normalizeOpts(optsOrN, 'sessionId');
const { limit = 50 } = opts;
const { where, params } = buildWhere(opts, { sessionId: 's.id', project: 's.project', timestamp: 's.started_at', branch: 's.git_branch' });
params.push(limit);
return db.prepare(`SELECT * FROM sessions s WHERE ${where} ORDER BY ended_at DESC LIMIT ?`).all(...params);
};
const recent = (n = 10) => sessions({ limit: n });
const summaries = (optsOrSid) => {
const opts = normalizeOpts(optsOrSid);
const { limit = 100 } = opts;
const { where, params } = buildWhere(opts, { sessionId: 'su.session_id', project: 's.project', timestamp: 'su.timestamp', branch: 's.git_branch' });
params.push(limit);
return db.prepare(`SELECT su.*, s.title as session_title, s.project FROM summaries su LEFT JOIN sessions s ON s.id=su.session_id WHERE ${where} ORDER BY su.timestamp DESC LIMIT ?`).all(...params);
};
const overview = (optsOrScalar) => {
const opts = normalizeOverviewOpts(optsOrScalar);
const cwd = process.cwd();
const sessionLimit = opts.limit ?? 8;
const projectLimit = opts.projectLimit ?? 20;
const memoryLimit = opts.memoryLimit ?? 100;
const projectDescriptor = (row, source, confidence) => row ? ({
project: row.project,
project_path: row.project_path || null,
source,
confidence,
}) : null;
const latestProjectByPattern = (pattern) => {
const fromSessions = db.prepare(`
SELECT project, project_path
FROM sessions
WHERE project LIKE ?
ORDER BY COALESCE(ended_at, started_at) DESC
LIMIT 1
`).get(pattern);
if (fromSessions) return fromSessions;
return db.prepare(`
SELECT project, NULL AS project_path
FROM memories
WHERE project LIKE ?
ORDER BY created_at DESC
LIMIT 1
`).get(pattern);
};
const resolveCurrentProject = () => {
if (opts.project) {
const row = latestProjectByPattern(opts.project);
const confidence = row ? (/[%_]/.test(opts.project) ? 'inferred' : 'exact') : 'unknown';
return projectDescriptor(row || { project: opts.project, project_path: null }, 'opts', confidence);
}
const paths = db.prepare(`
SELECT project, project_path, MAX(COALESCE(ended_at, started_at)) AS last_seen
FROM sessions
WHERE project IS NOT NULL AND project_path IS NOT NULL AND project_path != ''
GROUP BY project, project_path
`).all();
const byProjectPath = paths
.filter(r => cwd === r.project_path || cwd.startsWith(r.project_path + path.sep))
.sort((a, b) => b.project_path.length - a.project_path.length || String(b.last_seen || '').localeCompare(String(a.last_seen || '')))[0];
if (byProjectPath) return projectDescriptor(byProjectPath, 'cwd_project_path', 'exact');
const byMessageCwd = db.prepare(`
SELECT s.project, s.project_path, MAX(m.timestamp) AS last_seen
FROM messages m
LEFT JOIN sessions s ON s.id=m.session_id
WHERE m.cwd = ? AND s.project IS NOT NULL
GROUP BY s.project, s.project_path
ORDER BY last_seen DESC
LIMIT 1
`).get(cwd);
if (byMessageCwd) return projectDescriptor(byMessageCwd, 'cwd_messages', 'inferred');
return null;
};
const projects = db.prepare(`
WITH names AS (
SELECT project FROM sessions WHERE project IS NOT NULL GROUP BY project
UNION
SELECT project FROM memories WHERE project IS NOT NULL AND deleted_at IS NULL GROUP BY project
),
session_stats AS (
SELECT project, COUNT(*) AS session_count, MAX(COALESCE(ended_at, started_at)) AS last_session_at
FROM sessions
WHERE project IS NOT NULL
GROUP BY project
),
memory_stats AS (
SELECT project, COUNT(*) AS memory_count, MAX(created_at) AS last_memory_at
FROM memories
WHERE project IS NOT NULL AND deleted_at IS NULL
GROUP BY project
)
SELECT
n.project,
(
SELECT s2.project_path
FROM sessions s2
WHERE s2.project = n.project AND s2.project_path IS NOT NULL
ORDER BY COALESCE(s2.ended_at, s2.started_at) DESC
LIMIT 1
) AS project_path,
COALESCE(ss.session_count, 0) AS session_count,
COALESCE(ms.memory_count, 0) AS memory_count,
ss.last_session_at,
ms.last_memory_at
FROM names n
LEFT JOIN session_stats ss ON ss.project = n.project
LEFT JOIN memory_stats ms ON ms.project = n.project
ORDER BY COALESCE(ss.last_session_at, ms.last_memory_at) DESC
LIMIT ?
`).all(projectLimit).map(row => {
const branches = db.prepare(`
SELECT git_branch
FROM sessions
WHERE project = ? AND git_branch IS NOT NULL AND git_branch != ''
GROUP BY git_branch
ORDER BY MAX(COALESCE(ended_at, started_at)) DESC
LIMIT 5
`).all(row.project).map(r => r.git_branch);
return { ...row, recent_branches: branches };
});
const currentProject = resolveCurrentProject();
let current_project = null;
if (currentProject?.project) {
const sessionTotal = db.prepare('SELECT COUNT(*) AS c FROM sessions WHERE project = ?').get(currentProject.project)?.c || 0;
const sessionsForProject = db.prepare(`
SELECT id, title, project, project_path, started_at, ended_at, git_branch, message_count
FROM sessions
WHERE project = ?
ORDER BY COALESCE(ended_at, started_at) DESC
LIMIT ?
`).all(currentProject.project, sessionLimit);
const memoryTotal = db.prepare('SELECT COUNT(*) AS c FROM memories WHERE project = ? AND deleted_at IS NULL').get(currentProject.project)?.c || 0;
const memoriesForProject = db.prepare(`
SELECT id, path, anchors, summary, session_id, project, created_at
FROM memories
WHERE project = ? AND deleted_at IS NULL
ORDER BY created_at DESC
LIMIT ?
`).all(currentProject.project, memoryLimit);
current_project = {
project: currentProject.project,
project_path: currentProject.project_path,
session_total: sessionTotal,
sessions: sessionsForProject,
memory_total: memoryTotal,
memories: memoriesForProject,
};
}
const totalProjects = db.prepare(`
SELECT COUNT(*) AS c
FROM (
SELECT project FROM sessions WHERE project IS NOT NULL GROUP BY project
UNION
SELECT project FROM memories WHERE project IS NOT NULL AND deleted_at IS NULL GROUP BY project
)
`).get()?.c || 0;
const totalSessions = db.prepare('SELECT COUNT(*) AS c FROM sessions').get()?.c || 0;
const totalMemories = db.prepare('SELECT COUNT(*) AS c FROM memories WHERE deleted_at IS NULL').get()?.c || 0;
return {
current: {
cwd,
project: currentProject,
},
current_project,
projects,
totals: {
projects: totalProjects,
sessions: totalSessions,
memories: totalMemories,
},
};
};
const resolveJsonlPath = (messageUuid) => {
const msg = db.prepare('SELECT session_id, agent_id FROM messages WHERE uuid=?').get(messageUuid);
if (!msg) return null;
if (msg.agent_id) {
const wa = db.prepare('SELECT agent_id, run_id, session_id FROM workflow_agents WHERE agent_id=?').get(msg.agent_id);
if (wa) {
const ses = db.prepare('SELECT jsonl_path FROM sessions WHERE id=?').get(wa.session_id);
if (ses) return path.join(path.dirname(ses.jsonl_path), wa.session_id, 'subagents', 'workflows', wa.run_id, wa.agent_id + '.jsonl');
}
const sa = db.prepare('SELECT agent_id, session_id FROM subagents WHERE agent_id=?').get(msg.agent_id);
if (sa) {
const ses = db.prepare('SELECT jsonl_path FROM sessions WHERE id=?').get(sa.session_id);
if (ses) return path.join(path.dirname(ses.jsonl_path), sa.session_id, 'subagents', sa.agent_id + '.jsonl');
}
} else {
const ses = db.prepare('SELECT jsonl_path FROM sessions WHERE id=?').get(msg.session_id);
if (ses) return ses.jsonl_path;
}
return null;
};
const findRawLine = (jsonlPath, uuid) => {
if (!jsonlPath || !fs.existsSync(jsonlPath)) return null;
let found = null;
readLines(jsonlPath, (line) => {
if (!line.includes(uuid)) return;
try { const obj = JSON.parse(line); if (obj.uuid === uuid) { found = line; return false; } } catch {}
});
return found;
};
const raw = (messageUuid, opts = {}) => {
const { offset = 0, limit = 10000 } = opts;
const jsonlPath = resolveJsonlPath(messageUuid);
const line = findRawLine(jsonlPath, messageUuid);
if (!line) return null;
return {
text: line.slice(offset, offset + limit),
totalLength: line.length,
offset,
limit,
hasMore: offset + limit < line.length,
};
};
const memories = (optsOrSid) => {
const opts = normalizeOpts(optsOrSid);
const { limit = 50, query } = opts;
assertEnglishMemoryText(query, 'memories() query');
const needsJoin = opts.branch;
const { where: baseWhere, params } = buildWhere(opts, {
sessionId: 'mem.session_id',
project: 'mem.project',
timestamp: 'mem.created_at',
branch: 's.git_branch',
});
let where = baseWhere + ' AND mem.deleted_at IS NULL';
const join = needsJoin ? 'LEFT JOIN sessions s ON s.id=mem.session_id' : '';
const hasQuery = String(query || '').trim().length > 0;
const ftsQuery = buildSafeFtsQuery(query);
if (!hasQuery) {
params.push(limit);
return db.prepare(`SELECT mem.* FROM memories mem ${join} WHERE ${where} ORDER BY mem.created_at DESC LIMIT ?`).all(...params);
}
if (!ftsQuery) return [];
params.unshift(ftsQuery);
params.push(limit);
return db.prepare(`
SELECT mem.*, mf.rank AS rank
FROM memories_fts mf
JOIN memories mem ON mem.rowid = mf.rowid
${join}
WHERE memories_fts MATCH ? AND ${where}
ORDER BY mf.rank, mem.created_at DESC
LIMIT ?
`).all(...params);
};
return { sql: q, search, context, trace, thread, subagents, workflows, workflowTree, fileHistory, failures, sessions, recent, summaries, raw, memories, overview };
}
function createAttuneApi(db) {
const resolveMemoryPath = (memoryPath, sessionId) => {
let base = null;
if (sessionId) {
base = db.prepare('SELECT project_path FROM sessions WHERE id=?').get(sessionId)?.project_path || null;
}
const resolved = path.isAbsolute(memoryPath)
? path.normalize(memoryPath)
: path.resolve(base || process.cwd(), memoryPath);
let stat;
try {
stat = fs.statSync(resolved);
} catch {
throw new Error(`remember() memory file does not exist: ${resolved}`);
}
if (!stat.isFile()) throw new Error(`remember() memory path is not a file: ${resolved}`);
return resolved;
};
const normalizeAnchors = (anchors) => {
if (anchors == null) return null;
let parsed = anchors;
if (typeof anchors === 'string') {
const trimmed = anchors.trim();
if (!trimmed) return null;
try {
parsed = JSON.parse(trimmed);
} catch {
throw new Error('remember() anchors must be a JSON array');
}
}
if (!Array.isArray(parsed)) throw new Error('remember() anchors must be an array');
for (const anchor of parsed) {
if (!anchor || typeof anchor !== 'object' || Array.isArray(anchor)) {
throw new Error('remember() anchors entries must be objects');
}
}
return parsed.length ? JSON.stringify(parsed) : null;
};
const remember = ({ path: memoryPath, session_id, message_start, message_end, summary, project, anchors }) => {
if (!memoryPath || !summary) throw new Error('remember() requires path and summary');
assertEnglishMemoryText(summary, 'remember() summary');
const normalizedPath = resolveMemoryPath(memoryPath, session_id);
const normalizedAnchors = normalizeAnchors(anchors);
const id = `mem-${Date.now()}-${Math.random().toString(36).slice(2, 8)}`;
const proj = project || db.prepare('SELECT project FROM sessions WHERE id=?').get(session_id)?.project || null;
const created_at = new Date().toISOString();
db.prepare('INSERT OR REPLACE INTO memories (id, session_id, project, message_start, message_end, path, anchors, summary, created_at) VALUES (?,?,?,?,?,?,?,?,?)').run(
id, session_id || null, proj, message_start || null, message_end || null, normalizedPath, normalizedAnchors, summary, created_at);
db.prepare(`
INSERT INTO memories_fts(rowid, id, path, summary)
SELECT rowid, id, path, summary FROM memories WHERE id = ?
`).run(id);
return { id, path: normalizedPath, project: proj, anchors: normalizedAnchors, created_at };
};
const forget = ({ id, reason }) => {
const deletionReason = String(reason || '').trim();
if (!id || !deletionReason) throw new Error('forget() requires id and reason');
const row = db.prepare('SELECT id, deleted_at, deleted_reason FROM memories WHERE id=?').get(id);
if (!row) throw new Error(`forget() memory not found: ${id}`);
if (row.deleted_at) {
return { id, deleted_at: row.deleted_at, deleted_reason: row.deleted_reason, already_deleted: true };
}
const deleted_at = new Date().toISOString();
db.prepare('UPDATE memories SET deleted_at=?, deleted_reason=? WHERE id=?').run(deleted_at, deletionReason, id);
return { id, deleted_at, deleted_reason: deletionReason };
};
return { remember, forget };
}
export { createQueryApi, createAttuneApi };