fix: normalize missing Codex token usage

Treat token_count events without usage as null canonical token values so node:sqlite never receives undefined. Covers issue #1 with a parse-to-persist regression test.
This commit is contained in:
tommy0103
2026-07-22 01:02:25 +08:00
parent 28fa99ac94
commit 476d4a94fa
3 changed files with 52 additions and 7 deletions
+13 -4
View File
@@ -296,12 +296,21 @@ function parseCodexJsonInput(value: JsonValue): JsonValue {
try { return JSON.parse(value); } catch { return value; } try { return JSON.parse(value); } catch { return value; }
} }
function codexUsage(payload: JsonRecord) { interface CodexUsage {
inputTokens: number | null;
outputTokens: number | null;
}
function codexUsage(payload: JsonRecord): CodexUsage {
const usage = payload?.info?.last_token_usage || payload?.info?.total_token_usage || payload?.last_token_usage || null; const usage = payload?.info?.last_token_usage || payload?.info?.total_token_usage || payload?.last_token_usage || null;
if (!usage) return {}; if (!usage) return { inputTokens: null, outputTokens: null };
return { return {
inputTokens: usage.input_tokens ?? null, inputTokens: typeof usage.input_tokens === 'number' && Number.isFinite(usage.input_tokens)
outputTokens: usage.output_tokens ?? null, ? usage.input_tokens
: null,
outputTokens: typeof usage.output_tokens === 'number' && Number.isFinite(usage.output_tokens)
? usage.output_tokens
: null,
}; };
} }
+3 -3
View File
@@ -256,9 +256,9 @@ export function* parse(unit: IndexUnit, _cursor: Cursor): Generator<TranscriptRe
} }
if (payload.type === 'token_count') { if (payload.type === 'token_count') {
const usage = codexUsage(payload); const usage = codexUsage(payload);
if (usage.inputTokens !== null) sm.totalInputTokens = usage.inputTokens; if (usage.inputTokens != null) sm.totalInputTokens = usage.inputTokens;
if (usage.outputTokens !== null) sm.totalOutputTokens = usage.outputTokens; if (usage.outputTokens != null) sm.totalOutputTokens = usage.outputTokens;
if (sm.lastTextAssistantUuid && (usage.inputTokens !== null || usage.outputTokens !== null)) { if (sm.lastTextAssistantUuid && (usage.inputTokens != null || usage.outputTokens != null)) {
const rec = msgByUuid.get(sm.lastTextAssistantUuid); const rec = msgByUuid.get(sm.lastTextAssistantUuid);
if (rec) { rec.input_tokens = usage.inputTokens; rec.output_tokens = usage.outputTokens; } if (rec) { rec.input_tokens = usage.inputTokens; rec.output_tokens = usage.outputTokens; }
} }
+36
View File
@@ -71,6 +71,42 @@ test('a provider record stream assembles directly into session detail', () => {
db.close(); db.close();
}); });
test('Codex token_count without usage remains persistable', () => {
const threadId = '019e8951-3e7d-7343-a3e3-05bff48a3180';
const path = writeCodexFixture([
{
type: 'session_meta',
timestamp: '2026-07-14T12:21:21.000Z',
payload: { id: threadId, cwd: '/tmp/demo', timestamp: '2026-07-14T12:21:21.000Z' },
},
{
type: 'event_msg',
timestamp: '2026-07-14T12:21:30.000Z',
payload: { type: 'agent_message', message: 'hello' },
},
{
type: 'event_msg',
timestamp: '2026-07-14T12:21:31.000Z',
payload: { type: 'token_count', info: null },
},
]);
const db = new DatabaseSync(':memory:');
db.exec(SCHEMA);
assert.doesNotThrow(() => {
persist(db, { key: path, sessionId: `codex:${threadId}` }, parseCodex({
key: path,
sessionId: `codex:${threadId}`,
}, null));
});
const usage = db.prepare(
'SELECT input_tokens, output_tokens FROM messages WHERE text = ?',
).get('hello');
assert.equal(usage.input_tokens, null);
assert.equal(usage.output_tokens, null);
db.close();
});
test('provider-classified hidden context never reaches session detail', () => { test('provider-classified hidden context never reaches session detail', () => {
const threadId = '019e8951-3e7d-7343-a3e3-05bff48a317e'; const threadId = '019e8951-3e7d-7343-a3e3-05bff48a317e';
const path = writeCodexFixture([ const path = writeCodexFixture([