From 4d0bedd85a4eb0cfbf01a22cced9c36a31ee2e3a Mon Sep 17 00:00:00 2001 From: Zack Zatkin-Gold Date: Sat, 19 Sep 2026 20:31:53 -0700 Subject: [PATCH] Read insight attributes as aliases, not as one emitter's vocabulary Fathom ingests OTLP from anything, and ingest already knew emitters disagree about names: classify() keys on gen_ai.operation.name as well as Claude Code's span.type, and the correlator's join keys list gen_ai.tool.call.id alongside tool_use_id. The insights analyzer did not. It read cost_usd, input_tokens, output_tokens, cache_read_tokens, model, stop_reason, tool_name and user_prompt -- Claude Code's private vocabulary, exclusively. A trace from any other emitter therefore rendered correctly in the trace view and reported no cost, no tokens, and every tool named "tool". That is a silent wrong answer, and the kind that presents as "you have no findings" rather than as a bug. Each fact now resolves through an ordered list of names with Claude Code's own first, so a trace carrying both vocabularies resolves to the name the analyzer was written against and no existing answer changes. Resolution is first-match-wins rather than sum-across-matches, so an emitter stamping both names on one span does not get its tokens counted twice -- there is a test pinning that. Cost gets one extra wrinkle: Claude Code reports it on the log records, a GenAI emitter reports it on the span. costOf() checks the span first and short-circuits, so a trace carrying both does not pay twice. This is a read-time alias, not an ingest-time rewrite. What an emitter sent stays on disk verbatim, the same reason classify() keeps the original span name whatever kind it decides on. Verified against a real store, not only in unit tests: a pi run emitted by costslayer's new OTLP layer previously extracted to $0.00000 and zero tokens, and now extracts to $0.04218, 12,560 tokens, 11,000 cached, both tool names, and the correct model and stop reason. Co-Authored-By: Claude Opus 5 --- README.md | 13 +++ internal/insights/analyzer.go | 82 +++++----------- internal/insights/attrs.go | 102 ++++++++++++++++++++ internal/insights/attrs_test.go | 162 ++++++++++++++++++++++++++++++++ 4 files changed, 301 insertions(+), 58 deletions(-) create mode 100644 internal/insights/attrs.go create mode 100644 internal/insights/attrs_test.go diff --git a/README.md b/README.md index dbdccc7..40a80d9 100644 --- a/README.md +++ b/README.md @@ -156,6 +156,19 @@ substance — the user's prompt, what a tool was passed, what it returned, what the model said back — arrives as OTLP log records. Spans alone give you a bar chart; the logs are what make it a transcript. +**Attribute names are read as aliases, not as one vocabulary.** Emitters +disagree about what to call the same fact: Claude Code reports `cost_usd`, +`input_tokens`, and `tool_name`, while an emitter following OpenTelemetry's +GenAI conventions reports `gen_ai.usage.cost`, `gen_ai.usage.input_tokens`, +and `gen_ai.tool.name`. Ingest and correlation already knew this; the insights +analyzer originally did not, so a trace from any other emitter rendered +correctly in the trace view and reported no cost, no tokens, and every tool +named "tool" — a wrong answer that presents as "no findings" rather than as a +bug. The analyzer now resolves each fact through an ordered list of names, +Claude Code's first. It is a read-time alias: what an emitter sent stays on +disk verbatim, the same way `classify()` keeps the original span name whatever +kind it decides on. + Correlating the two is not a `span_id` join, and assuming it is produces a broken UI. Emitters routinely stamp every log record in a run with the *root* span's id, so joining on `span_id` collapses the entire transcript onto the root diff --git a/internal/insights/analyzer.go b/internal/insights/analyzer.go index 0e1c4f9..c3732b2 100644 --- a/internal/insights/analyzer.go +++ b/internal/insights/analyzer.go @@ -440,17 +440,10 @@ func buildFacts(corr *tracetree.Correlator, id string, spans []model.Span, logs sp := n.Span if f.Prompt == "" { - if v, ok := sp.Attrs["user_prompt"]; ok { - f.Prompt = v.Str - } + f.Prompt = firstStr(sp.Attrs, promptKeys) } if f.Model == "" { - for _, k := range []string{"gen_ai.request.model", "model"} { - if v, ok := sp.Attrs[k]; ok && v.Str != "" { - f.Model = v.Str - break - } - } + f.Model = firstStr(sp.Attrs, modelKeys) } switch sp.Kind { @@ -467,33 +460,24 @@ func buildFacts(corr *tracetree.Correlator, id string, spans []model.Span, logs f.Calls = append(f.Calls, toolCallFrom(sp, n.Logs)) } + f.CostUSD += costOf(sp, n.Logs) + for _, l := range n.Logs { - if v, ok := l.Attrs["cost_usd"]; ok { - if c, ok := v.Float(); ok { - f.CostUSD += c - } + if f.Prompt == "" && isPromptEvent(l.EventName) { + f.Prompt = firstStr(l.Attrs, promptKeys) } - if f.Prompt == "" && l.EventName == "user_prompt" { - if v, ok := l.Attrs["prompt"]; ok { - f.Prompt = v.Str + } + if f.Prompt == "" { + for _, l := range n.Logs { + if isPromptEvent(l.EventName) && l.Body != "" { + f.Prompt = l.Body + break } } } - for _, k := range []string{"input_tokens", "output_tokens"} { - if v, ok := sp.Attrs[k]; ok { - if n, ok := v.Float(); ok { - f.Tokens += int64(n) - } - } - } - for _, k := range []string{"cache_read_tokens", "cache_creation_tokens"} { - if v, ok := sp.Attrs[k]; ok { - if n, ok := v.Float(); ok { - f.Cached += int64(n) - } - } - } + f.Tokens += firstNum(sp.Attrs, inputTokenKeys) + firstNum(sp.Attrs, outputTokenKeys) + f.Cached += firstNum(sp.Attrs, cacheReadKeys) + firstNum(sp.Attrs, cacheWriteKeys) } sort.SliceStable(f.LLMCalls, func(i, j int) bool { return i < j }) @@ -515,8 +499,8 @@ func toolCallFrom(sp model.Span, logs []model.LogRecord) ToolCall { Succeeded: sp.Status != model.StatusError, DurMicros: sp.DurationMicros(), } - if v, ok := sp.Attrs["tool_name"]; ok && v.Str != "" { - c.Name = v.Str + if n := firstStr(sp.Attrs, toolNameKeys); n != "" { + c.Name = n } // The arguments live on the correlated tool_result record, not the span — @@ -525,8 +509,8 @@ func toolCallFrom(sp model.Span, logs []model.LogRecord) ToolCall { if l.EventName != "tool_result" && l.EventName != "tool_decision" { continue } - if v, ok := l.Attrs["tool_name"]; ok && v.Str != "" { - c.Name = v.Str + if n := firstStr(l.Attrs, toolNameKeys); n != "" { + c.Name = n } for _, k := range []string{"tool_input", "tool_parameters"} { if v, ok := l.Attrs[k]; ok && v.Str != "" && c.Input == "" { @@ -556,30 +540,12 @@ func toolCallFrom(sp model.Span, logs []model.LogRecord) ToolCall { func llmCallFrom(sp model.Span, logs []model.LogRecord) LLMCall { c := LLMCall{DurMicros: sp.DurationMicros()} - num := func(attrs model.Attrs, key string) int64 { - if v, ok := attrs[key]; ok { - if f, ok := v.Float(); ok { - return int64(f) - } - } - return 0 - } - c.InputTokens = num(sp.Attrs, "input_tokens") - c.OutputToken = num(sp.Attrs, "output_tokens") - c.CacheRead = num(sp.Attrs, "cache_read_tokens") - if v, ok := sp.Attrs["model"]; ok { - c.Model = v.Str - } - if v, ok := sp.Attrs["stop_reason"]; ok { - c.StopReason = v.Str - } - for _, l := range logs { - if v, ok := l.Attrs["cost_usd"]; ok { - if f, ok := v.Float(); ok { - c.CostUSD += f - } - } - } + c.InputTokens = firstNum(sp.Attrs, inputTokenKeys) + c.OutputToken = firstNum(sp.Attrs, outputTokenKeys) + c.CacheRead = firstNum(sp.Attrs, cacheReadKeys) + c.Model = firstStr(sp.Attrs, modelKeys) + c.StopReason = firstStr(sp.Attrs, stopReasonKeys) + c.CostUSD = costOf(sp, logs) return c } diff --git a/internal/insights/attrs.go b/internal/insights/attrs.go new file mode 100644 index 0000000..a4d7d09 --- /dev/null +++ b/internal/insights/attrs.go @@ -0,0 +1,102 @@ +package insights + +import "git.z.gd/zzg/fathom/internal/model" + +// Emitters disagree about what to call the same fact, and the analyzer has to +// read facts rather than names. +// +// Ingest and correlation already knew this: classify() keys on +// gen_ai.operation.name as well as Claude Code's span.type, and the +// correlator's join keys list gen_ai.tool.call.id alongside tool_use_id. The +// analyzer did not, and read Claude Code's private vocabulary exclusively. A +// trace from any other emitter therefore rendered correctly in the trace view +// and reported no cost, no tokens, and every tool named "tool" — a silent +// wrong answer that presents as "no findings" rather than as a bug. +// +// The lists below are ordered, and Claude Code's own names come first in +// every one. A trace carrying both vocabularies resolves to the name the +// analyzer was originally written against, so adding these cannot change any +// existing answer. Resolution is first-match-wins rather than +// sum-across-matches, for the same reason: an emitter that stamps both names +// on one span must not have its tokens counted twice. +// +// This is a read-time alias, not an ingest-time rewrite. What an emitter sent +// stays on disk verbatim — the same reason classify() keeps the original span +// name whatever kind it decides on. +var ( + costKeys = []string{"cost_usd", "gen_ai.usage.cost"} + inputTokenKeys = []string{"input_tokens", "gen_ai.usage.input_tokens"} + outputTokenKeys = []string{"output_tokens", "gen_ai.usage.output_tokens"} + // Anthropic's cache accounting is not in the GenAI conventions proper; + // these are the names the conventions' Anthropic mapping uses. + cacheReadKeys = []string{"cache_read_tokens", "gen_ai.usage.cache_read_input_tokens"} + cacheWriteKeys = []string{"cache_creation_tokens", "gen_ai.usage.cache_creation_input_tokens"} + // Request before response: the analyzer groups by what was asked for, and + // a response model can name a dated build of the same requested alias. + modelKeys = []string{"model", "gen_ai.request.model", "gen_ai.response.model"} + toolNameKeys = []string{"tool_name", "gen_ai.tool.name"} + stopReasonKeys = []string{"stop_reason", "gen_ai.response.finish_reasons"} + promptKeys = []string{"user_prompt", "prompt", "gen_ai.prompt"} +) + +// promptEventNames are the log-record event names that carry a user's prompt. +var promptEventNames = []string{"user_prompt", "gen_ai.user.message"} + +func isPromptEvent(name string) bool { + for _, n := range promptEventNames { + if name == n { + return true + } + } + return false +} + +// firstStr returns the first key present with a non-empty string value. +func firstStr(a model.Attrs, keys []string) string { + for _, k := range keys { + if v, ok := a[k]; ok && v.Str != "" { + return v.Str + } + } + return "" +} + +// firstFloat returns the first key present with a numeric value, and whether +// any was found. The two are distinct: a genuine zero cost and a missing cost +// are different facts, and collapsing them is how "this run produced nothing" +// becomes unanswerable. +func firstFloat(a model.Attrs, keys []string) (float64, bool) { + for _, k := range keys { + if v, ok := a[k]; ok { + if f, ok := v.Float(); ok { + return f, true + } + } + } + return 0, false +} + +// firstNum is firstFloat truncated to an integer, for token counts. +func firstNum(a model.Attrs, keys []string) int64 { + if f, ok := firstFloat(a, keys); ok { + return int64(f) + } + return 0 +} + +// costOf resolves a cost for one span and its correlated records. Claude Code +// reports cost on the log records; a GenAI emitter reports it on the span +// itself. The span is checked first and short-circuits, so a trace that +// carries both does not pay twice. +func costOf(sp model.Span, logs []model.LogRecord) float64 { + if c, ok := firstFloat(sp.Attrs, costKeys); ok { + return c + } + var total float64 + for _, l := range logs { + if c, ok := firstFloat(l.Attrs, costKeys); ok { + total += c + } + } + return total +} diff --git a/internal/insights/attrs_test.go b/internal/insights/attrs_test.go new file mode 100644 index 0000000..6505b77 --- /dev/null +++ b/internal/insights/attrs_test.go @@ -0,0 +1,162 @@ +package insights + +import ( + "testing" + + "git.z.gd/zzg/fathom/internal/model" + "git.z.gd/zzg/fathom/internal/tracetree" +) + +// Fathom ingests OTLP from anything, and classify() and the correlator both +// already understand OpenTelemetry's GenAI conventions. The insights analyzer +// did not: it read Claude Code's private attribute names and nothing else, so +// a trace from any other emitter rendered perfectly in the UI and reported no +// cost, no tokens and no tool names at all. That is a silent wrong answer, and +// the kind that looks like "you have no findings" rather than like a bug. +// +// These tests use a pi/costslayer-shaped trace -- plain GenAI conventions, +// nothing Fathom-specific -- because that is the case that was broken. + +func genAITrace() ([]model.Span, []model.LogRecord) { + const traceID = "aaaabbbbccccddddeeeeffff00001111" + spans := []model.Span{ + { + TraceID: traceID, SpanID: "1111111111111111", Name: "pi agent run", + Kind: model.KindAgentStep, StartTS: 1_000_000, EndTS: 9_000_000, + }, + { + TraceID: traceID, SpanID: "2222222222222222", ParentSpanID: "1111111111111111", + Name: "chat claude-sonnet-5", Kind: model.KindLLMCall, + StartTS: 2_000_000, EndTS: 5_000_000, + Attrs: model.Attrs{ + "gen_ai.request.model": model.StringValue("claude-sonnet-5"), + "gen_ai.usage.input_tokens": model.IntValue(12043), + "gen_ai.usage.output_tokens": model.IntValue(517), + "gen_ai.usage.cache_read_input_tokens": model.IntValue(9800), + "gen_ai.usage.cache_creation_input_tokens": model.IntValue(1200), + "gen_ai.usage.cost": model.FloatValue(0.04218), + "gen_ai.response.finish_reasons": model.StringValue("tool_use"), + }, + }, + { + TraceID: traceID, SpanID: "3333333333333333", ParentSpanID: "1111111111111111", + Name: "execute_tool read", Kind: model.KindToolCall, + StartTS: 6_000_000, EndTS: 7_000_000, + Attrs: model.Attrs{ + "gen_ai.operation.name": model.StringValue("execute_tool"), + "gen_ai.tool.name": model.StringValue("read"), + "gen_ai.tool.call.id": model.StringValue("toolu_1"), + }, + }, + } + logs := []model.LogRecord{ + { + TraceID: traceID, SpanID: "1111111111111111", EventName: "gen_ai.user.message", + TS: 1_000_000, Body: "refactor the auth module", + Attrs: model.Attrs{"gen_ai.prompt": model.StringValue("refactor the auth module")}, + }, + } + return spans, logs +} + +func factsFromGenAI(t *testing.T) TraceFacts { + t.Helper() + spans, logs := genAITrace() + return buildFacts(tracetree.NewCorrelator(), spans[0].TraceID, spans, logs) +} + +// The headline failure. Every cost number in every finding is a sum of this. +func TestGenAICostIsCounted(t *testing.T) { + f := factsFromGenAI(t) + if got, want := f.CostUSD, 0.04218; got != want { + t.Errorf("CostUSD = %v, want %v — a gen_ai emitter's spend must not read as zero", got, want) + } +} + +func TestGenAITokensAreCounted(t *testing.T) { + f := factsFromGenAI(t) + if got, want := f.Tokens, int64(12043+517); got != want { + t.Errorf("Tokens = %d, want %d", got, want) + } + if got, want := f.Cached, int64(9800+1200); got != want { + t.Errorf("Cached = %d, want %d", got, want) + } +} + +// Without this the context-waste detector cannot tell repeated context from +// fresh context, because everything looks like zero cached tokens. +func TestGenAILLMCallCarriesItsNumbers(t *testing.T) { + f := factsFromGenAI(t) + if len(f.LLMCalls) != 1 { + t.Fatalf("got %d LLM calls, want 1", len(f.LLMCalls)) + } + c := f.LLMCalls[0] + if c.InputTokens != 12043 { + t.Errorf("InputTokens = %d, want 12043", c.InputTokens) + } + if c.OutputToken != 517 { + t.Errorf("OutputToken = %d, want 517", c.OutputToken) + } + if c.CacheRead != 9800 { + t.Errorf("CacheRead = %d, want 9800", c.CacheRead) + } + if c.Model != "claude-sonnet-5" { + t.Errorf("Model = %q, want claude-sonnet-5", c.Model) + } + if c.StopReason != "tool_use" { + t.Errorf("StopReason = %q, want tool_use", c.StopReason) + } + if c.CostUSD != 0.04218 { + t.Errorf("CostUSD = %v, want 0.04218", c.CostUSD) + } +} + +// A tool call whose name is "tool" collapses every distinct action in a run +// into one signature, which is precisely what the repeated-sequence detector +// is looking for. Unnamed tools produce confident nonsense. +func TestGenAIToolNameIsRead(t *testing.T) { + f := factsFromGenAI(t) + if len(f.Calls) != 1 { + t.Fatalf("got %d tool calls, want 1", len(f.Calls)) + } + if got := f.Calls[0].Name; got != "read" { + t.Errorf("tool name = %q, want %q", got, "read") + } +} + +func TestGenAIPromptIsRead(t *testing.T) { + f := factsFromGenAI(t) + if got, want := f.Prompt, "refactor the auth module"; got != want { + t.Errorf("Prompt = %q, want %q", got, want) + } +} + +// The aliases must not cost Claude Code anything. Its names stay first in +// every list, so a trace carrying both vocabularies resolves to the one the +// analyzer was written against. +func TestClaudeCodeAttributesStillWin(t *testing.T) { + const traceID = "11112222333344445555666677778888" + spans := []model.Span{{ + TraceID: traceID, SpanID: "1111111111111111", Name: "llm_request", + Kind: model.KindLLMCall, StartTS: 1_000_000, EndTS: 2_000_000, + Attrs: model.Attrs{ + "input_tokens": model.IntValue(100), + "gen_ai.usage.input_tokens": model.IntValue(999), + "model": model.StringValue("claude-opus-5"), + "gen_ai.request.model": model.StringValue("wrong"), + }, + }} + f := buildFacts(tracetree.NewCorrelator(), traceID, spans, nil) + if len(f.LLMCalls) != 1 { + t.Fatalf("got %d LLM calls, want 1", len(f.LLMCalls)) + } + if got := f.LLMCalls[0].InputTokens; got != 100 { + t.Errorf("InputTokens = %d, want 100 — Claude Code's own name must win", got) + } + if got := f.LLMCalls[0].Model; got != "claude-opus-5" { + t.Errorf("Model = %q, want claude-opus-5", got) + } + if got := f.Tokens; got != 100 { + t.Errorf("Tokens = %d, want 100 — an aliased key must not be summed twice", got) + } +} -- 2.56.0