fathom/internal/insights/attrs_test.go

package insights

import (
	"testing"

	"git.z.gd/zzg/fathom/internal/model"
	"git.z.gd/zzg/fathom/internal/tracetree"
)

// Fathom ingests OTLP from anything, and classify() and the correlator both
// already understand OpenTelemetry's GenAI conventions. The insights analyzer
// did not: it read Claude Code's private attribute names and nothing else, so
// a trace from any other emitter rendered perfectly in the UI and reported no
// cost, no tokens and no tool names at all. That is a silent wrong answer, and
// the kind that looks like "you have no findings" rather than like a bug.
//
// These tests use a pi/costslayer-shaped trace -- plain GenAI conventions,
// nothing Fathom-specific -- because that is the case that was broken.

func genAITrace() ([]model.Span, []model.LogRecord) {
	const traceID = "aaaabbbbccccddddeeeeffff00001111"
	spans := []model.Span{
		{
			TraceID: traceID, SpanID: "1111111111111111", Name: "pi agent run",
			Kind: model.KindAgentStep, StartTS: 1_000_000, EndTS: 9_000_000,
		},
		{
			TraceID: traceID, SpanID: "2222222222222222", ParentSpanID: "1111111111111111",
			Name: "chat claude-sonnet-5", Kind: model.KindLLMCall,
			StartTS: 2_000_000, EndTS: 5_000_000,
			Attrs: model.Attrs{
				"gen_ai.request.model":                     model.StringValue("claude-sonnet-5"),
				"gen_ai.usage.input_tokens":                model.IntValue(12043),
				"gen_ai.usage.output_tokens":               model.IntValue(517),
				"gen_ai.usage.cache_read_input_tokens":     model.IntValue(9800),
				"gen_ai.usage.cache_creation_input_tokens": model.IntValue(1200),
				"gen_ai.usage.cost":                        model.FloatValue(0.04218),
				"gen_ai.response.finish_reasons":           model.StringValue("tool_use"),
			},
		},
		{
			TraceID: traceID, SpanID: "3333333333333333", ParentSpanID: "1111111111111111",
			Name: "execute_tool read", Kind: model.KindToolCall,
			StartTS: 6_000_000, EndTS: 7_000_000,
			Attrs: model.Attrs{
				"gen_ai.operation.name": model.StringValue("execute_tool"),
				"gen_ai.tool.name":      model.StringValue("read"),
				"gen_ai.tool.call.id":   model.StringValue("toolu_1"),
			},
		},
	}
	logs := []model.LogRecord{
		{
			TraceID: traceID, SpanID: "1111111111111111", EventName: "gen_ai.user.message",
			TS: 1_000_000, Body: "refactor the auth module",
			Attrs: model.Attrs{"gen_ai.prompt": model.StringValue("refactor the auth module")},
		},
	}
	return spans, logs
}

func factsFromGenAI(t *testing.T) TraceFacts {
	t.Helper()
	spans, logs := genAITrace()
	return buildFacts(tracetree.NewCorrelator(), spans[0].TraceID, spans, logs)
}

// The headline failure. Every cost number in every finding is a sum of this.
func TestGenAICostIsCounted(t *testing.T) {
	f := factsFromGenAI(t)
	if got, want := f.CostUSD, 0.04218; got != want {
		t.Errorf("CostUSD = %v, want %v — a gen_ai emitter's spend must not read as zero", got, want)
	}
}

func TestGenAITokensAreCounted(t *testing.T) {
	f := factsFromGenAI(t)
	if got, want := f.Tokens, int64(12043+517); got != want {
		t.Errorf("Tokens = %d, want %d", got, want)
	}
	if got, want := f.Cached, int64(9800+1200); got != want {
		t.Errorf("Cached = %d, want %d", got, want)
	}
}

// Without this the context-waste detector cannot tell repeated context from
// fresh context, because everything looks like zero cached tokens.
func TestGenAILLMCallCarriesItsNumbers(t *testing.T) {
	f := factsFromGenAI(t)
	if len(f.LLMCalls) != 1 {
		t.Fatalf("got %d LLM calls, want 1", len(f.LLMCalls))
	}
	c := f.LLMCalls[0]
	if c.InputTokens != 12043 {
		t.Errorf("InputTokens = %d, want 12043", c.InputTokens)
	}
	if c.OutputToken != 517 {
		t.Errorf("OutputToken = %d, want 517", c.OutputToken)
	}
	if c.CacheRead != 9800 {
		t.Errorf("CacheRead = %d, want 9800", c.CacheRead)
	}
	if c.Model != "claude-sonnet-5" {
		t.Errorf("Model = %q, want claude-sonnet-5", c.Model)
	}
	if c.StopReason != "tool_use" {
		t.Errorf("StopReason = %q, want tool_use", c.StopReason)
	}
	if c.CostUSD != 0.04218 {
		t.Errorf("CostUSD = %v, want 0.04218", c.CostUSD)
	}
}

// A tool call whose name is "tool" collapses every distinct action in a run
// into one signature, which is precisely what the repeated-sequence detector
// is looking for. Unnamed tools produce confident nonsense.
func TestGenAIToolNameIsRead(t *testing.T) {
	f := factsFromGenAI(t)
	if len(f.Calls) != 1 {
		t.Fatalf("got %d tool calls, want 1", len(f.Calls))
	}
	if got := f.Calls[0].Name; got != "read" {
		t.Errorf("tool name = %q, want %q", got, "read")
	}
}

func TestGenAIPromptIsRead(t *testing.T) {
	f := factsFromGenAI(t)
	if got, want := f.Prompt, "refactor the auth module"; got != want {
		t.Errorf("Prompt = %q, want %q", got, want)
	}
}

// The aliases must not cost Claude Code anything. Its names stay first in
// every list, so a trace carrying both vocabularies resolves to the one the
// analyzer was written against.
func TestClaudeCodeAttributesStillWin(t *testing.T) {
	const traceID = "11112222333344445555666677778888"
	spans := []model.Span{{
		TraceID: traceID, SpanID: "1111111111111111", Name: "llm_request",
		Kind: model.KindLLMCall, StartTS: 1_000_000, EndTS: 2_000_000,
		Attrs: model.Attrs{
			"input_tokens":              model.IntValue(100),
			"gen_ai.usage.input_tokens": model.IntValue(999),
			"model":                     model.StringValue("claude-opus-5"),
			"gen_ai.request.model":      model.StringValue("wrong"),
		},
	}}
	f := buildFacts(tracetree.NewCorrelator(), traceID, spans, nil)
	if len(f.LLMCalls) != 1 {
		t.Fatalf("got %d LLM calls, want 1", len(f.LLMCalls))
	}
	if got := f.LLMCalls[0].InputTokens; got != 100 {
		t.Errorf("InputTokens = %d, want 100 — Claude Code's own name must win", got)
	}
	if got := f.LLMCalls[0].Model; got != "claude-opus-5" {
		t.Errorf("Model = %q, want claude-opus-5", got)
	}
	if got := f.Tokens; got != 100 {
		t.Errorf("Tokens = %d, want 100 — an aliased key must not be summed twice", got)
	}
}