From 2b30d9596d4f63ce3e13c5e4260cce24a924a3bc Mon Sep 17 00:00:00 2001 From: tommy0103 Date: Wed, 8 Jul 2026 18:27:28 +0800 Subject: [PATCH] feat(providers): add pure claude adapter with record-stream golden tests (5b-1) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit claude.parse mirrors indexJsonl line-for-line but yields IndexRecords (no db). Adds MessageTurnDurationRecord op. Purely additive — buildIndex still uses the old path. 111/111 green. --- eslint.config.js | 3 + scripts/indexer.mjs | 2 +- scripts/providers/claude.ts | 131 ++++++++++++++++++++++++++++++++++++ scripts/providers/types.ts | 10 +++ tests/claude-parse.test.mjs | 86 +++++++++++++++++++++++ 5 files changed, 231 insertions(+), 1 deletion(-) create mode 100644 scripts/providers/claude.ts create mode 100644 tests/claude-parse.test.mjs diff --git a/eslint.config.js b/eslint.config.js index 1c7f2eb..d158081 100644 --- a/eslint.config.js +++ b/eslint.config.js @@ -40,6 +40,9 @@ export default tseslint.config( }, rules: { '@typescript-eslint/no-unused-vars': ['warn', { argsIgnorePattern: '^_', varsIgnorePattern: '^_' }], + // Provider adapters parse untyped external transcript JSON; `any` at those + // boundaries is deliberate, not a smell. + '@typescript-eslint/no-explicit-any': 'off', }, }, ); diff --git a/scripts/indexer.mjs b/scripts/indexer.mjs index e959002..b3c28c8 100644 --- a/scripts/indexer.mjs +++ b/scripts/indexer.mjs @@ -808,4 +808,4 @@ function buildIndex({ force = false } = {}) { db.close(); } -export { buildIndex, indexJsonl, inferProjectPath, refreshSessionProjectPaths, shouldSkipBuild }; +export { buildIndex, indexJsonl, discoverJsonlFiles, inferProjectPath, refreshSessionProjectPaths, shouldSkipBuild }; diff --git a/scripts/providers/claude.ts b/scripts/providers/claude.ts new file mode 100644 index 0000000..2edc698 --- /dev/null +++ b/scripts/providers/claude.ts @@ -0,0 +1,131 @@ +// Claude Code provider adapter (see docs/adr/0001). +// +// Pure: discovers Claude transcript files and parses one into a record stream. +// It never touches the Obelisk database. The per-line logic mirrors the original +// indexJsonl exactly, but yields IndexRecords instead of writing rows; the shared +// persist layer consumes them. Session aggregates here reflect only THIS chunk +// (started_at/ended_at/message_count); persist merges them with any existing row. + +import { createRequire } from 'node:module'; +const require = createRequire(import.meta.url); +const fs = require('node:fs'); + +import { + extractText, extractContentType, extractMessageIsMeta, + filePath, trunc, truncJson, readLines, +} from '../db.mjs'; +import { discoverJsonlFiles } from '../indexer.mjs'; + +import type { Cursor, DiscoverContext, IndexRecord, IndexUnit, Provider } from './types.ts'; + +// Claude cursor encodes the file mtime and the number of lines already indexed: +// ":". mtime lets discovery detect change; lines lets +// parse resume without reprocessing. +function cursorToSkip(cursor: Cursor): number { + if (!cursor) return 0; + const n = Number(cursor.split(':')[1]); + return Number.isFinite(n) ? n : 0; +} + +export const name = 'claude'; + +export function discover(_ctx: DiscoverContext): IndexUnit[] { + return discoverJsonlFiles().map((f: any) => ({ + key: f.path, + sessionId: f.sessionId, + project: f.project, + isSubagent: f.isSubagent, + agentId: f.agentId, + meta: f.workflowRunId ? { workflowRunId: f.workflowRunId } : undefined, + })); +} + +export function* parse(unit: IndexUnit, cursor: Cursor): Generator { + const skip = cursorToSkip(cursor); + const mtime = fs.statSync(unit.key).mtimeMs; + const isSubagent = unit.isSubagent === true; + const records: IndexRecord[] = []; + const sm = { + started_at: null as string | null, + ended_at: null as string | null, + git_branch: null as string | null, + version: null as string | null, + title: null as string | null, + n: 0, + }; + + let lineNum = 0; + readLines(unit.key, (line: string) => { + lineNum++; + if (lineNum <= skip) return; + let obj: any; + try { obj = JSON.parse(line); } catch { return; } + const sid = unit.sessionId; + const ts = obj.timestamp || null; + + if (obj.type === 'ai-title' && obj.aiTitle) { sm.title = obj.aiTitle; return; } + if (obj.type === 'system' && obj.subtype === 'away_summary' && obj.content) { + records.push({ kind: 'summary', id: obj.uuid || `${sid}-away-${ts}`, session_id: sid, timestamp: ts, source: 'away_summary', content: obj.content }); + return; + } + if (obj.type === 'system' && obj.subtype === 'turn_duration' && obj.parentUuid && obj.durationMs) { + records.push({ kind: 'message-turn-duration', uuid: obj.parentUuid, turn_duration_ms: obj.durationMs }); + return; + } + if (obj.type !== 'user' && obj.type !== 'assistant') return; + + if (ts && (!sm.started_at || ts < sm.started_at)) sm.started_at = ts; + if (ts && (!sm.ended_at || ts > sm.ended_at)) sm.ended_at = ts; + if (obj.gitBranch) sm.git_branch = obj.gitBranch; + if (obj.version) sm.version = obj.version; + sm.n++; + + const msg = obj.message || {}; + const text = extractText(msg.content); + const contentType = extractContentType(msg.content); + const isMeta = extractMessageIsMeta(obj, text); + const usage = msg.usage || {}; + const aid = isSubagent ? (unit.agentId ?? null) : (obj.agentId || null); + + if (obj.uuid) { + records.push({ + kind: 'message', uuid: obj.uuid, session_id: sid, type: obj.type, + parent_uuid: obj.parentUuid || null, timestamp: ts, role: msg.role || obj.type, + text, content_type: contentType, is_meta: (isMeta ? 1 : 0), model: msg.model || null, + is_sidechain: obj.isSidechain ? 1 : 0, agent_id: aid, + input_tokens: usage.input_tokens || null, output_tokens: usage.output_tokens || null, + cwd: obj.cwd || null, skill: obj.attributionSkill || null, source: 'claude', + }); + } + + if (obj.type === 'assistant' && Array.isArray(msg.content)) { + for (const b of msg.content) { + if (b.type === 'tool_use' && b.id) + records.push({ kind: 'tool_call', id: b.id, message_uuid: obj.uuid, session_id: sid, name: b.name, input_json: truncJson(b.input || {}) as string, file_path: filePath(b.name, b.input) }); + } + } + + if (obj.type === 'user' && Array.isArray(msg.content)) { + for (const b of msg.content) { + if (b.type !== 'tool_result' || !b.tool_use_id) continue; + const rt = typeof b.content === 'string' ? b.content + : Array.isArray(b.content) ? b.content.map((c: any) => c.text || '').join('\n') : ''; + records.push({ kind: 'tool_result', tool_use_id: b.tool_use_id, message_uuid: obj.uuid, session_id: sid, content: trunc(rt), file_path: obj.toolUseResult?.filePath || null, is_error: b.is_error ? 1 : 0 }); + } + } + }); + + // Subagent transcripts do not own a session row (matches indexJsonl). + if (!isSubagent) { + records.push({ + kind: 'session', id: unit.sessionId, title: sm.title, project: unit.project || null, + started_at: sm.started_at, ended_at: sm.ended_at, git_branch: sm.git_branch, + version: sm.version, message_count: sm.n, jsonl_path: unit.key, source: 'claude', + }); + } + + yield* records; + return `${mtime}:${lineNum}`; +} + +export const claudeProvider: Provider = { name, discover, parse }; diff --git a/scripts/providers/types.ts b/scripts/providers/types.ts index 655c9e6..713bd6d 100644 --- a/scripts/providers/types.ts +++ b/scripts/providers/types.ts @@ -60,6 +60,7 @@ export type IndexRecord = | SubagentRecord | WorkflowRecord | WorkflowAgentRecord + | MessageTurnDurationRecord | DeleteSessionRecord; export interface MessageRecord { @@ -163,6 +164,15 @@ export interface WorkflowAgentRecord { tool_calls?: number | null; } +// Update op (not a table): sets messages.turn_duration_ms for a message that was +// (or will be) inserted by a separate line, possibly on a different run. Persist +// applies it as a targeted UPDATE, so it never clobbers other message columns. +export interface MessageTurnDurationRecord { + kind: 'message-turn-duration'; + uuid: string; + turn_duration_ms: number; +} + // Retraction op (not a table). The adapter emits this when a previously-indexed // session must be removed — e.g. a Codex guardian/auto-review thread. Persist // executes the cascade delete across all tables for that session. diff --git a/tests/claude-parse.test.mjs b/tests/claude-parse.test.mjs new file mode 100644 index 0000000..ffae8dc --- /dev/null +++ b/tests/claude-parse.test.mjs @@ -0,0 +1,86 @@ +// Phase 5b golden test: pins the claude adapter's parse() record stream. +// This is the binding-independent contract — no database is involved. If the +// per-line parse behavior drifts, this fails before persist ever runs. + +import { test } from 'node:test'; +import assert from 'node:assert/strict'; +import { mkdtempSync, writeFileSync, statSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; + +import { parse } from '../scripts/providers/claude.ts'; + +function writeFixture() { + const dir = mkdtempSync(join(tmpdir(), 'obelisk-claude-parse-')); + const path = join(dir, 'sid-x.jsonl'); + const lines = [ + { type: 'ai-title', aiTitle: 'My Session' }, + { uuid: 'u1', type: 'user', timestamp: '2026-06-10T10:00:00Z', cwd: '/proj', gitBranch: 'main', message: { role: 'user', content: 'hi' } }, + { uuid: 'a1', type: 'assistant', timestamp: '2026-06-10T10:00:05Z', message: { role: 'assistant', model: 'claude-opus', content: [{ type: 'text', text: 'ok' }, { type: 'tool_use', id: 'tc1', name: 'Read', input: { file_path: '/f' } }] } }, + { type: 'system', subtype: 'turn_duration', parentUuid: 'a1', durationMs: 1234 }, + { uuid: 'u2', type: 'user', timestamp: '2026-06-10T10:00:10Z', message: { role: 'user', content: [{ type: 'tool_result', tool_use_id: 'tc1', content: 'file body', is_error: false }] } }, + { type: 'system', subtype: 'away_summary', uuid: 's1', timestamp: '2026-06-10T10:00:11Z', content: 'a summary' }, + ]; + writeFileSync(path, lines.map(l => JSON.stringify(l)).join('\n') + '\n'); + return path; +} + +// Drain a generator, returning both the yielded values and its return value. +function drain(gen) { + const values = []; + let step = gen.next(); + while (!step.done) { values.push(step.value); step = gen.next(); } + return { values, ret: step.value }; +} + +test('claude parse() yields the expected record stream for a main session', () => { + const path = writeFixture(); + const { values, ret } = drain(parse({ key: path, sessionId: 'sid-x', project: 'quiet-zero' }, null)); + + const byKind = k => values.filter(r => r.kind === k); + + // Three user/assistant messages, correct order and fields. + assert.deepEqual(byKind('message').map(m => m.uuid), ['u1', 'a1', 'u2']); + assert.equal(byKind('message').find(m => m.uuid === 'a1').model, 'claude-opus'); + assert.equal(byKind('message').every(m => m.source === 'claude'), true); + + // Tool call + tool result extracted. + assert.deepEqual(byKind('tool_call').map(t => ({ id: t.id, name: t.name })), [{ id: 'tc1', name: 'Read' }]); + assert.deepEqual(byKind('tool_result').map(t => ({ id: t.tool_use_id, err: t.is_error })), [{ id: 'tc1', err: 0 }]); + + // turn_duration is an update op keyed on the assistant message. + assert.deepEqual(byKind('message-turn-duration'), [{ kind: 'message-turn-duration', uuid: 'a1', turn_duration_ms: 1234 }]); + + // Away summary. + assert.deepEqual(byKind('summary').map(s => s.id), ['s1']); + + // Exactly one session aggregate, reflecting THIS chunk. + const sessions = byKind('session'); + assert.equal(sessions.length, 1); + assert.equal(sessions[0].title, 'My Session'); + assert.equal(sessions[0].message_count, 3); + assert.equal(sessions[0].started_at, '2026-06-10T10:00:00Z'); + assert.equal(sessions[0].ended_at, '2026-06-10T10:00:10Z'); + assert.equal(sessions[0].git_branch, 'main'); + + // Cursor encodes mtime:lines (6 lines consumed). + assert.equal(ret, `${statSync(path).mtimeMs}:6`); +}); + +test('claude parse() emits no session record for a subagent transcript', () => { + const path = writeFixture(); + const { values } = drain(parse({ key: path, sessionId: 'sid-x', isSubagent: true, agentId: 'agent-7' }, null)); + + assert.equal(values.filter(r => r.kind === 'session').length, 0); + // Subagent messages carry the unit's agent id. + assert.equal(values.filter(r => r.kind === 'message').every(m => m.agent_id === 'agent-7'), true); +}); + +test('claude parse() resumes from a cursor, skipping already-indexed lines', () => { + const path = writeFixture(); + // Cursor with 6 lines already processed → nothing new to parse. + const { values } = drain(parse({ key: path, sessionId: 'sid-x', project: 'quiet-zero' }, '0:6')); + // Only the (empty-chunk) session record, with message_count 0. + assert.deepEqual(values.filter(r => r.kind !== 'session'), []); + assert.equal(values.find(r => r.kind === 'session').message_count, 0); +});