feat(index): capture cwd, skill, turn_duration, is_error from JSONL;
expose FTS5 rank and cwd filter in search()
This commit is contained in:
tommy0103
@@ -1,2 +1,3 @@
|
||||
.DS_Store
|
||||
plans/
|
||||
.skillopt-backups
|
||||
@@ -40,9 +40,11 @@ The query file body is executed inside `(async () => { ... })()` with the API be
|
||||
|
||||
Full-text search across all messages (user, assistant, subagent, workflow agent).
|
||||
|
||||
Returns: `[{ message: {uuid, text, role, timestamp, model}, session: {id, title, project, started_at}, context: [...surrounding messages] }]`
|
||||
Returns: `[{ message: {uuid, text, role, timestamp, model, cwd}, session: {id, title, project, started_at}, rank, context: [...surrounding messages] }]`
|
||||
|
||||
opts: `{ limit, sessionId, project, after, before }`
|
||||
opts: `{ limit, sessionId, project, after, before, cwd }`
|
||||
|
||||
`rank` is the FTS5 relevance score (negative; closer to 0 = more relevant). Use it to judge result quality and stop early when results become irrelevant.
|
||||
|
||||
### sessions(opts?)
|
||||
|
||||
@@ -71,6 +73,8 @@ Raw SQL. Use `?` placeholders. Returns array of row objects.
|
||||
|
||||
**Before writing your first SQL query, read `references/schema.md` for the full table schema, column names, and relationships.** Don't guess column names — the schema is your source of truth.
|
||||
|
||||
**Schema-safe SQL pattern:** when aggregating event tables, join to the table that actually owns the metadata instead of inventing columns. For example, `tool_calls` does **not** own timestamps; join `messages m ON m.uuid = tc.message_uuid` for `m.timestamp`, and join `sessions s ON s.id = tc.session_id` for project/session filters. Prefer SQL-side `GROUP BY`/`COUNT`/`MAX` with `LIMIT`, and return compact evidence rows with stable IDs rather than raw event records.
|
||||
|
||||
Tables: `sessions`, `messages`, `tool_calls`, `tool_results`, `subagents`, `workflows`, `workflow_agents`, `messages_fts`
|
||||
|
||||
### Other APIs
|
||||
@@ -89,6 +93,10 @@ All list-returning functions accept a common filter opts object: `{ project, aft
|
||||
|
||||
### Retrieval strategy
|
||||
|
||||
**For helper field/schema confirmation questions:** use the relevant helper under the user's explicit scope with a small `limit`, then return only `Object.keys(row)` or a projection of the fields being verified plus short snippets. Do not invent alias fields when helper docs/rows use different names (for summaries, use `source`, `content`, `session_id`, `project`; not `text` or `summary_type`). Include stable evidence IDs such as `id`, `session_id`, or `uuid` in the final answer.
|
||||
|
||||
**Respect explicit scopes and empty results.** If the user asks for a specific project/session/file/time range, keep every query inside that scope. When a scoped helper call such as `summaries({ project, limit })` returns `[]`, report no results; do not broaden to all projects or all summaries unless the user asks for fallback.
|
||||
|
||||
**Never pull an entire session.** Navigate incrementally:
|
||||
|
||||
1. `sessions({ project: '...' })` or `recent()` — find relevant sessions
|
||||
@@ -103,6 +111,8 @@ All list-returning functions accept a common filter opts object: `{ project, aft
|
||||
4. `raw(uuid, opts?)` — recover truncated content from a specific message
|
||||
5. `thread(sessionId)` — full session dump, **last resort only**
|
||||
|
||||
**File history queries:** `fileHistory()` can include many `Read` rows and large tool inputs. For questions about how a file changed, filter to `Edit`/`Write` before returning, cap the filtered list to the requested evidence count, and return compact evidence records only.
|
||||
|
||||
### raw(uuid, opts?)
|
||||
|
||||
Some indexed fields (tool call inputs, tool results) are truncated to 10k chars. `raw()` reads the original JSONL line to recover the full content.
|
||||
|
||||
@@ -45,7 +45,10 @@ CREATE TABLE messages (
|
||||
is_sidechain INTEGER DEFAULT 0, -- 1 if this message is on a sidechain (retry/branch)
|
||||
agent_id TEXT, -- subagent or workflow agent UUID (NULL for main conversation)
|
||||
input_tokens INTEGER, -- token usage (assistant messages only)
|
||||
output_tokens INTEGER -- token usage (assistant messages only)
|
||||
output_tokens INTEGER, -- token usage (assistant messages only)
|
||||
cwd TEXT, -- working directory at message time (may differ from session project_path)
|
||||
skill TEXT, -- skill that generated this response (e.g. "obelisk"), NULL if none
|
||||
turn_duration_ms INTEGER -- wall-clock duration of the turn ending at this message (from system turn_duration event)
|
||||
);
|
||||
```
|
||||
|
||||
@@ -94,7 +97,8 @@ CREATE TABLE tool_results (
|
||||
message_uuid TEXT, -- FK -> messages.uuid (the user message carrying this result)
|
||||
session_id TEXT, -- FK -> sessions.id (denormalized)
|
||||
content TEXT, -- result text (truncated to 10k chars)
|
||||
file_path TEXT -- file path from toolUseResult metadata (if any)
|
||||
file_path TEXT, -- file path from toolUseResult metadata (if any)
|
||||
is_error INTEGER DEFAULT 0 -- 1 if the tool call returned an error (from API is_error field)
|
||||
);
|
||||
```
|
||||
|
||||
@@ -199,8 +203,9 @@ Full-text search across all message text using FTS5.
|
||||
| `opts.project` | `string` | Restrict to a project slug |
|
||||
| `opts.after` | `string` | ISO 8601 lower bound on timestamp |
|
||||
| `opts.before` | `string` | ISO 8601 upper bound on timestamp |
|
||||
| `opts.cwd` | `string` | Filter by working directory (supports LIKE) |
|
||||
|
||||
**Returns:** `Array<{ message, session, context }>` where `context` is the 6 nearest messages by timestamp.
|
||||
**Returns:** `Array<{ message, session, rank, context }>` where `context` is the 6 nearest messages by timestamp. `rank` is the FTS5 relevance score (negative; closer to 0 = more relevant).
|
||||
|
||||
```js
|
||||
const hits = search('MCTS exploration');
|
||||
|
||||
+2
-1
@@ -18,7 +18,8 @@ CREATE TABLE IF NOT EXISTS messages (
|
||||
uuid TEXT PRIMARY KEY, session_id TEXT, type TEXT, parent_uuid TEXT,
|
||||
timestamp TEXT, role TEXT, text TEXT, model TEXT,
|
||||
is_sidechain INTEGER DEFAULT 0, agent_id TEXT,
|
||||
input_tokens INTEGER, output_tokens INTEGER);
|
||||
input_tokens INTEGER, output_tokens INTEGER,
|
||||
cwd TEXT, skill TEXT, turn_duration_ms INTEGER);
|
||||
CREATE TABLE IF NOT EXISTS tool_calls (
|
||||
id TEXT PRIMARY KEY, message_uuid TEXT, session_id TEXT,
|
||||
name TEXT, input_json TEXT, file_path TEXT);
|
||||
|
||||
+7
-2
@@ -59,7 +59,7 @@ function indexJsonl(db, fi) {
|
||||
|
||||
const ins = {
|
||||
ses: db.prepare('INSERT OR REPLACE INTO sessions (id,title,project,project_path,started_at,ended_at,git_branch,version,message_count,jsonl_path) VALUES (?,?,?,?,?,?,?,?,?,?)'),
|
||||
msg: db.prepare('INSERT OR REPLACE INTO messages (uuid,session_id,type,parent_uuid,timestamp,role,text,model,is_sidechain,agent_id,input_tokens,output_tokens) VALUES (?,?,?,?,?,?,?,?,?,?,?,?)'),
|
||||
msg: db.prepare('INSERT OR REPLACE INTO messages (uuid,session_id,type,parent_uuid,timestamp,role,text,model,is_sidechain,agent_id,input_tokens,output_tokens,cwd,skill) VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?)'),
|
||||
tc: db.prepare('INSERT OR REPLACE INTO tool_calls (id,message_uuid,session_id,name,input_json,file_path) VALUES (?,?,?,?,?,?)'),
|
||||
tr: db.prepare('INSERT OR REPLACE INTO tool_results (tool_use_id,message_uuid,session_id,content,file_path,is_error) VALUES (?,?,?,?,?,?)'),
|
||||
sum: db.prepare('INSERT OR REPLACE INTO summaries (id,session_id,timestamp,source,content) VALUES (?,?,?,?,?)'),
|
||||
@@ -90,6 +90,10 @@ function indexJsonl(db, fi) {
|
||||
ins.sum.run(obj.uuid || `${sid}-away-${ts}`, sid, ts, 'away_summary', obj.content);
|
||||
return;
|
||||
}
|
||||
if (obj.type === 'system' && obj.subtype === 'turn_duration' && obj.parentUuid && obj.durationMs) {
|
||||
db.prepare('UPDATE messages SET turn_duration_ms=? WHERE uuid=?').run(obj.durationMs, obj.parentUuid);
|
||||
return;
|
||||
}
|
||||
if (obj.type !== 'user' && obj.type !== 'assistant') return;
|
||||
|
||||
if (ts && (!sm.started_at || ts < sm.started_at)) sm.started_at = ts;
|
||||
@@ -106,7 +110,8 @@ function indexJsonl(db, fi) {
|
||||
if (obj.uuid) {
|
||||
ins.msg.run(obj.uuid, sid, obj.type, obj.parentUuid || null, ts,
|
||||
msg.role || obj.type, text, msg.model || null,
|
||||
obj.isSidechain ? 1 : 0, aid, usage.input_tokens || null, usage.output_tokens || null);
|
||||
obj.isSidechain ? 1 : 0, aid, usage.input_tokens || null, usage.output_tokens || null,
|
||||
obj.cwd || null, obj.attributionSkill || null);
|
||||
}
|
||||
|
||||
if (obj.type === 'assistant' && Array.isArray(msg.content)) {
|
||||
|
||||
+8
-5
@@ -28,17 +28,19 @@ function createQueryApi(db) {
|
||||
const q = (sql, ...p) => db.prepare(sql).all(...p);
|
||||
|
||||
const search = (text, opts = {}) => {
|
||||
const { limit = 20, sessionId, project, after, before } = opts;
|
||||
const { limit = 20, sessionId, project, after, before, cwd } = opts;
|
||||
let where = 'WHERE mf.text MATCH ?';
|
||||
const p = [text];
|
||||
if (sessionId) { where += ' AND mf.session_id=?'; p.push(sessionId); }
|
||||
if (project) { where += ' AND s.project=?'; p.push(project); }
|
||||
if (project) { where += ' AND s.project LIKE ?'; p.push(project); }
|
||||
if (after) { where += ' AND m.timestamp>?'; p.push(after); }
|
||||
if (before) { where += ' AND m.timestamp<?'; p.push(before); }
|
||||
if (cwd) { where += ' AND m.cwd LIKE ?'; p.push(cwd); }
|
||||
p.push(limit);
|
||||
const rows = db.prepare(`
|
||||
SELECT m.uuid,m.session_id,m.text,m.role,m.timestamp,m.model,
|
||||
s.id as s_id,s.title as s_title,s.project as s_project,s.started_at as s_started
|
||||
SELECT m.uuid,m.session_id,m.text,m.role,m.timestamp,m.model,m.cwd,
|
||||
s.id as s_id,s.title as s_title,s.project as s_project,s.started_at as s_started,
|
||||
rank
|
||||
FROM messages_fts mf JOIN messages m ON m.uuid=mf.uuid LEFT JOIN sessions s ON s.id=m.session_id
|
||||
${where} ORDER BY rank LIMIT ?`).all(...p);
|
||||
return rows.map(r => {
|
||||
@@ -46,8 +48,9 @@ function createQueryApi(db) {
|
||||
'SELECT uuid,text,role,timestamp,model FROM messages WHERE session_id=? AND uuid!=? ORDER BY ABS(JULIANDAY(timestamp)-JULIANDAY(?)) LIMIT 6'
|
||||
).all(r.session_id, r.uuid, r.timestamp).sort((a,b) => a.timestamp < b.timestamp ? -1 : 1);
|
||||
return {
|
||||
message: { uuid: r.uuid, text: r.text, role: r.role, timestamp: r.timestamp, model: r.model },
|
||||
message: { uuid: r.uuid, text: r.text, role: r.role, timestamp: r.timestamp, model: r.model, cwd: r.cwd },
|
||||
session: { id: r.s_id, title: r.s_title, project: r.s_project, started_at: r.s_started },
|
||||
rank: r.rank,
|
||||
context: ctx,
|
||||
};
|
||||
});
|
||||
|
||||
Reference in New Issue
Block a user