fix: normalize missing Codex token usage
Treat token_count events without usage as null canonical token values so node:sqlite never receives undefined. Covers issue #1 with a parse-to-persist regression test.
This commit is contained in:
@@ -296,12 +296,21 @@ function parseCodexJsonInput(value: JsonValue): JsonValue {
|
|||||||
try { return JSON.parse(value); } catch { return value; }
|
try { return JSON.parse(value); } catch { return value; }
|
||||||
}
|
}
|
||||||
|
|
||||||
function codexUsage(payload: JsonRecord) {
|
interface CodexUsage {
|
||||||
|
inputTokens: number | null;
|
||||||
|
outputTokens: number | null;
|
||||||
|
}
|
||||||
|
|
||||||
|
function codexUsage(payload: JsonRecord): CodexUsage {
|
||||||
const usage = payload?.info?.last_token_usage || payload?.info?.total_token_usage || payload?.last_token_usage || null;
|
const usage = payload?.info?.last_token_usage || payload?.info?.total_token_usage || payload?.last_token_usage || null;
|
||||||
if (!usage) return {};
|
if (!usage) return { inputTokens: null, outputTokens: null };
|
||||||
return {
|
return {
|
||||||
inputTokens: usage.input_tokens ?? null,
|
inputTokens: typeof usage.input_tokens === 'number' && Number.isFinite(usage.input_tokens)
|
||||||
outputTokens: usage.output_tokens ?? null,
|
? usage.input_tokens
|
||||||
|
: null,
|
||||||
|
outputTokens: typeof usage.output_tokens === 'number' && Number.isFinite(usage.output_tokens)
|
||||||
|
? usage.output_tokens
|
||||||
|
: null,
|
||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -256,9 +256,9 @@ export function* parse(unit: IndexUnit, _cursor: Cursor): Generator<TranscriptRe
|
|||||||
}
|
}
|
||||||
if (payload.type === 'token_count') {
|
if (payload.type === 'token_count') {
|
||||||
const usage = codexUsage(payload);
|
const usage = codexUsage(payload);
|
||||||
if (usage.inputTokens !== null) sm.totalInputTokens = usage.inputTokens;
|
if (usage.inputTokens != null) sm.totalInputTokens = usage.inputTokens;
|
||||||
if (usage.outputTokens !== null) sm.totalOutputTokens = usage.outputTokens;
|
if (usage.outputTokens != null) sm.totalOutputTokens = usage.outputTokens;
|
||||||
if (sm.lastTextAssistantUuid && (usage.inputTokens !== null || usage.outputTokens !== null)) {
|
if (sm.lastTextAssistantUuid && (usage.inputTokens != null || usage.outputTokens != null)) {
|
||||||
const rec = msgByUuid.get(sm.lastTextAssistantUuid);
|
const rec = msgByUuid.get(sm.lastTextAssistantUuid);
|
||||||
if (rec) { rec.input_tokens = usage.inputTokens; rec.output_tokens = usage.outputTokens; }
|
if (rec) { rec.input_tokens = usage.inputTokens; rec.output_tokens = usage.outputTokens; }
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -71,6 +71,42 @@ test('a provider record stream assembles directly into session detail', () => {
|
|||||||
db.close();
|
db.close();
|
||||||
});
|
});
|
||||||
|
|
||||||
|
test('Codex token_count without usage remains persistable', () => {
|
||||||
|
const threadId = '019e8951-3e7d-7343-a3e3-05bff48a3180';
|
||||||
|
const path = writeCodexFixture([
|
||||||
|
{
|
||||||
|
type: 'session_meta',
|
||||||
|
timestamp: '2026-07-14T12:21:21.000Z',
|
||||||
|
payload: { id: threadId, cwd: '/tmp/demo', timestamp: '2026-07-14T12:21:21.000Z' },
|
||||||
|
},
|
||||||
|
{
|
||||||
|
type: 'event_msg',
|
||||||
|
timestamp: '2026-07-14T12:21:30.000Z',
|
||||||
|
payload: { type: 'agent_message', message: 'hello' },
|
||||||
|
},
|
||||||
|
{
|
||||||
|
type: 'event_msg',
|
||||||
|
timestamp: '2026-07-14T12:21:31.000Z',
|
||||||
|
payload: { type: 'token_count', info: null },
|
||||||
|
},
|
||||||
|
]);
|
||||||
|
const db = new DatabaseSync(':memory:');
|
||||||
|
db.exec(SCHEMA);
|
||||||
|
|
||||||
|
assert.doesNotThrow(() => {
|
||||||
|
persist(db, { key: path, sessionId: `codex:${threadId}` }, parseCodex({
|
||||||
|
key: path,
|
||||||
|
sessionId: `codex:${threadId}`,
|
||||||
|
}, null));
|
||||||
|
});
|
||||||
|
const usage = db.prepare(
|
||||||
|
'SELECT input_tokens, output_tokens FROM messages WHERE text = ?',
|
||||||
|
).get('hello');
|
||||||
|
assert.equal(usage.input_tokens, null);
|
||||||
|
assert.equal(usage.output_tokens, null);
|
||||||
|
db.close();
|
||||||
|
});
|
||||||
|
|
||||||
test('provider-classified hidden context never reaches session detail', () => {
|
test('provider-classified hidden context never reaches session detail', () => {
|
||||||
const threadId = '019e8951-3e7d-7343-a3e3-05bff48a317e';
|
const threadId = '019e8951-3e7d-7343-a3e3-05bff48a317e';
|
||||||
const path = writeCodexFixture([
|
const path = writeCodexFixture([
|
||||||
|
|||||||
Reference in New Issue
Block a user