fathom/internal/insights/insights_test.go

package insights

import (
	"encoding/json"
	"fmt"
	"strings"
	"testing"
)

// --- normalisation ---

// TestCommandShape pins the claim that structure, not semantics, decides when
// two commands are the same action. Every pair here differs in wording and
// matches on shape.
func TestCommandShape(t *testing.T) {
	cases := map[string]string{
		"go test ./internal/payments":               "go test",
		"go test ./internal/orders -run TestFoo -v": "go test",
		"cd /srv/app && go test ./...":              "go test",
		"git commit -m 'fix the thing'":             "git commit",
		"git status":                                "git status",
		"sudo systemctl restart nginx":              "systemctl restart",
		"npx prettier --write src/":                 "prettier",
		"ls *.txt 2>/dev/null":                      "ls",
		"cat foo.json | jq '.items'":                "jq",
		"FOO=1 BAR=2 python3 script.py":             "python3",
		"make build":                                "make build",
		"":                                          "?",
	}
	for in, want := range cases {
		if got := commandShape(in); got != want {
			t.Errorf("commandShape(%q) = %q, want %q", in, got, want)
		}
	}
}

func TestPathShape(t *testing.T) {
	cases := map[string]string{
		// Absolute and relative forms of the same file agree, and no part of
		// the home path survives into the shape.
		"/Users/zzg/work/fathom/internal/store/chunk.go": "store/*.go",
		"internal/store/shard.go":                        "store/*.go",
		"internal/query/exec.go":                         "query/*.go",
		"README.md":                                      "*.md",
		"/tmp/scratch/a.txt":                             "scratch/*.txt",
		"":                                               "?",
	}
	for in, want := range cases {
		if got := pathShape(in); got != want {
			t.Errorf("pathShape(%q) = %q, want %q", in, got, want)
		}
	}
}

// TestNormalizeCallGroupsVaryingArguments is the central claim of the whole
// approach: the same workflow with different arguments produces one signature.
func TestNormalizeCallGroupsVaryingArguments(t *testing.T) {
	a, _ := normalizeCall("Bash", `{"command":"go test ./internal/payments -run TestRetry"}`)
	b, _ := normalizeCall("Bash", `{"command":"go test ./internal/orders"}`)
	if a != b {
		t.Errorf("two runs of go test produced different signatures: %q vs %q", a, b)
	}

	c, _ := normalizeCall("Bash", `{"command":"git commit -m x"}`)
	if a == c {
		t.Errorf("go test and git commit collapsed into one signature (%q)", a)
	}

	// A malformed input must not crash or produce a wildcard that matches
	// everything.
	if sig, _ := normalizeCall("Bash", "not json"); sig != "Bash:?" {
		t.Errorf("unparseable input gave %q", sig)
	}
}

// --- MinHash ---

func TestMinHashSimilarity(t *testing.T) {
	sig := func(s string) signature { return minhash(shingle(canonPrompt(s), shingleSize)) }

	a := sig("the flaky test in payments_test.go fails about one run in five")
	b := sig("the flaky test in orders_test.go fails about one run in five")
	c := sig("add pagination to the orders endpoint keeping the response shape stable")

	if s := a.similarity(b); s < 0.5 {
		t.Errorf("near-identical prompts scored %.2f, want >= 0.5", s)
	}
	if s := a.similarity(c); s > 0.3 {
		t.Errorf("unrelated prompts scored %.2f, want <= 0.3", s)
	}
	if s := a.similarity(a); s != 1 {
		t.Errorf("a prompt is %.2f similar to itself, want 1", s)
	}
}

// TestEmptySignatureNeverClusters guards the case that would otherwise group
// every trace with no prompt into one meaningless finding.
func TestEmptySignatureNeverClusters(t *testing.T) {
	var sigs []signature
	for i := 0; i < 5; i++ {
		sigs = append(sigs, minhash(nil))
	}
	if got := clusterBySimilarity(sigs, 0.5); len(got) != 0 {
		t.Errorf("empty signatures formed %d clusters, want 0", len(got))
	}
}

func TestClusterBySimilarity(t *testing.T) {
	sig := func(s string) signature { return minhash(shingle(canonPrompt(s), shingleSize)) }
	sigs := []signature{
		sig("fix the flaky test in payments"),
		sig("add pagination to the orders endpoint"),
		sig("fix the flaky test in billing"),
		sig("fix the flaky test in shipping"),
	}
	got := clusterBySimilarity(sigs, 0.5)
	if len(got) != 1 {
		t.Fatalf("got %d clusters, want 1; clusters=%v", len(got), got)
	}
	if len(got[0]) != 3 {
		t.Errorf("cluster has %d members, want the 3 flaky-test prompts: %v", len(got[0]), got[0])
	}
}

// --- detectors ---

func call(name, input string, ok bool) ToolCall {
	c := ToolCall{Name: name, Input: input, Succeeded: ok, DurMicros: 100_000}
	c.Signature, c.Target = normalizeCall(name, input)
	return c
}

func trace(id string, ts int64, prompt string, cost float64, calls ...ToolCall) TraceFacts {
	return TraceFacts{
		TraceID: id, StartTS: ts, EndTS: ts + 10_000_000,
		Prompt: prompt, CostUSD: cost, Calls: calls,
	}
}

// grepReadEditTest is the workflow under test: find it, read it, change it,
// check it. The arguments differ every time.
func grepReadEditTest(n int) []ToolCall {
	return []ToolCall{
		call("Grep", fmt.Sprintf(`{"pattern":"handler%d","type":"go"}`, n), true),
		call("Read", fmt.Sprintf(`{"file_path":"internal/api/handler%d.go"}`, n), true),
		call("Edit", fmt.Sprintf(`{"file_path":"internal/api/handler%d.go"}`, n), true),
		call("Bash", fmt.Sprintf(`{"command":"go test ./internal/api -run Test%d"}`, n), true),
	}
}

func TestCrystallizeFindsRepeatedSequence(t *testing.T) {
	var traces []TraceFacts
	for i := 0; i < 4; i++ {
		traces = append(traces, trace(
			fmt.Sprintf("t%d", i), int64(1_700_000_000_000_000+i*1_000_000),
			fmt.Sprintf("update handler %d", i), 0.50, grepReadEditTest(i)...))
	}

	got := NewCrystallize().Detect(traces)

	var seq *Insight
	for i := range got {
		if strings.HasPrefix(got[i].ID, "crystallize/seq/") {
			seq = &got[i]
			break
		}
	}
	if seq == nil {
		t.Fatalf("no sequence insight found; got %d insights: %+v", len(got), ids(got))
	}
	if seq.Occurrences != 4 {
		t.Errorf("occurrences = %d, want 4", seq.Occurrences)
	}
	if len(seq.Steps) != 4 {
		t.Errorf("steps = %v, want the 4-step sequence", seq.Steps)
	}
	if len(seq.Evidence) != 4 {
		t.Errorf("evidence covers %d traces, want 4", len(seq.Evidence))
	}
	// Evidence must name the actual files, or the finding is unverifiable.
	if !strings.Contains(seq.Evidence[0].Detail, "handler") {
		t.Errorf("evidence detail %q does not show what the pattern did", seq.Evidence[0].Detail)
	}
	if seq.CostUSD <= 0 {
		t.Error("insight carries no cost, so there is nothing to weigh it by")
	}
}

// TestCrystallizeIgnoresOneOffs is the property that keeps the tab quiet. Two
// occurrences is a coincidence, not a habit.
func TestCrystallizeIgnoresOneOffs(t *testing.T) {
	traces := []TraceFacts{
		trace("t0", 1_700_000_000_000_000, "do a thing", 0.1, grepReadEditTest(0)...),
		trace("t1", 1_700_000_001_000_000, "do another thing", 0.1, grepReadEditTest(1)...),
	}
	for _, in := range NewCrystallize().Detect(traces) {
		if strings.HasPrefix(in.ID, "crystallize/seq/") {
			t.Errorf("reported a sequence seen only twice: %s", in.Title)
		}
	}
}

// TestCrystallizeIgnoresRepetitionWithinOneTrace stops a single pathological
// run from manufacturing a pattern out of its own loop.
func TestCrystallizeIgnoresRepetitionWithinOneTrace(t *testing.T) {
	var calls []ToolCall
	for i := 0; i < 10; i++ {
		calls = append(calls, grepReadEditTest(i)...)
	}
	traces := []TraceFacts{trace("only", 1_700_000_000_000_000, "big loop", 1.0, calls...)}

	for _, in := range NewCrystallize().Detect(traces) {
		if strings.HasPrefix(in.ID, "crystallize/seq/") {
			t.Errorf("one trace looping produced a cross-trace pattern: %s", in.Title)
		}
	}
}

// TestCrystallizeRejectsSingleToolRuns: reading four files in a row is reading,
// not a procedure worth scripting.
func TestCrystallizeRejectsSingleToolRuns(t *testing.T) {
	var traces []TraceFacts
	for i := 0; i < 5; i++ {
		traces = append(traces, trace(fmt.Sprintf("t%d", i), int64(1_700_000_000_000_000+i), "read things", 0.1,
			call("Read", `{"file_path":"internal/a/x.go"}`, true),
			call("Read", `{"file_path":"internal/a/y.go"}`, true),
			call("Read", `{"file_path":"internal/a/z.go"}`, true),
		))
	}
	for _, in := range NewCrystallize().Detect(traces) {
		if strings.HasPrefix(in.ID, "crystallize/seq/") {
			t.Errorf("a run of one repeated tool was reported as a workflow: %s", in.Title)
		}
	}
}

func TestCrystallizePrefersLongestPattern(t *testing.T) {
	var traces []TraceFacts
	for i := 0; i < 4; i++ {
		traces = append(traces, trace(fmt.Sprintf("t%d", i), int64(1_700_000_000_000_000+i), "x", 0.2, grepReadEditTest(i)...))
	}
	var seqs []Insight
	for _, in := range NewCrystallize().Detect(traces) {
		if strings.HasPrefix(in.ID, "crystallize/seq/") {
			seqs = append(seqs, in)
		}
	}
	if len(seqs) != 1 {
		t.Fatalf("got %d sequence insights, want 1 (the 3-step prefixes are implied): %v", len(seqs), ids(seqs))
	}
	if len(seqs[0].Steps) != 4 {
		t.Errorf("kept the %d-step pattern, want the full 4-step one", len(seqs[0].Steps))
	}
}

func TestCrystallizeFindsRepeatedAsks(t *testing.T) {
	traces := []TraceFacts{
		trace("a", 1_700_000_000_000_000, "why is the nightly ETL job timing out check the logs", 0.3),
		trace("b", 1_700_000_100_000_000, "why is the nightly ETL job timing out again check the logs", 0.4),
		trace("c", 1_700_000_200_000_000, "why is the nightly ETL job timing out please check the logs", 0.5),
	}
	var ask *Insight
	for _, in := range NewCrystallize().Detect(traces) {
		if strings.HasPrefix(in.ID, "crystallize/ask/") {
			c := in
			ask = &c
		}
	}
	if ask == nil {
		t.Fatal("three near-identical prompts produced no repeated-ask insight")
	}
	if ask.Occurrences != 3 {
		t.Errorf("occurrences = %d, want 3", ask.Occurrences)
	}
	if got := ask.CostUSD; got < 1.19 || got > 1.21 {
		t.Errorf("cost = %.2f, want the sum 1.20", got)
	}
}

func TestContextWeightFindsRereads(t *testing.T) {
	mk := func(id string, ts int64) TraceFacts {
		return trace(id, ts, "work on the parser", 0.4,
			call("Read", `{"file_path":"internal/parse/parse.go"}`, true),
			call("Edit", `{"file_path":"internal/parse/parse.go"}`, true),
			call("Read", `{"file_path":"internal/parse/parse.go"}`, true),
			call("Bash", `{"command":"go build ./..."}`, true),
			call("Read", `{"file_path":"internal/parse/parse.go"}`, true),
		)
	}
	traces := []TraceFacts{mk("a", 1_700_000_000_000_000), mk("b", 1_700_000_100_000_000)}

	var found *Insight
	for _, in := range NewContextWeight().Detect(traces) {
		if in.ID == "context/rereads" {
			c := in
			found = &c
		}
	}
	if found == nil {
		t.Fatal("a file read three times per run was not reported")
	}
	if !strings.Contains(found.Evidence[0].Detail, "3 times") {
		t.Errorf("evidence %q should say how many times", found.Evidence[0].Detail)
	}
}

func TestContextWeightIgnoresOrdinaryRereads(t *testing.T) {
	// Read, edit, read back is normal and must not be flagged.
	mk := func(id string, ts int64) TraceFacts {
		return trace(id, ts, "small edit", 0.1,
			call("Read", `{"file_path":"internal/a/x.go"}`, true),
			call("Edit", `{"file_path":"internal/a/x.go"}`, true),
			call("Read", `{"file_path":"internal/a/x.go"}`, true),
		)
	}
	for _, in := range NewContextWeight().Detect([]TraceFacts{mk("a", 1), mk("b", 2)}) {
		if in.ID == "context/rereads" {
			t.Error("reading a file twice was reported as waste")
		}
	}
}

func TestWasteFindsRepeatedFailures(t *testing.T) {
	var traces []TraceFacts
	for i := 0; i < 3; i++ {
		traces = append(traces, trace(fmt.Sprintf("t%d", i), int64(1_700_000_000_000_000+i), "run the suite", 0.25,
			call("Bash", `{"command":"go test ./..."}`, false),
		))
	}
	got := NewWaste().Detect(traces)
	if len(got) != 1 {
		t.Fatalf("got %d waste insights, want 1: %v", len(got), ids(got))
	}
	if got[0].Occurrences != 3 {
		t.Errorf("occurrences = %d, want 3", got[0].Occurrences)
	}
	if !strings.Contains(got[0].Title, "go test") {
		t.Errorf("title %q should name the failing command", got[0].Title)
	}
}

func TestWasteIgnoresIsolatedFailure(t *testing.T) {
	traces := []TraceFacts{
		trace("a", 1, "x", 0.1, call("Bash", `{"command":"go test ./..."}`, false)),
	}
	if got := NewWaste().Detect(traces); len(got) != 0 {
		t.Errorf("a single failure was reported: %v", ids(got))
	}
}

// TestNoTracesProducesNoInsights: an empty store must produce an empty tab, not
// a crash and not invented findings.
func TestNoTracesProducesNoInsights(t *testing.T) {
	for _, d := range []Detector{NewCrystallize(), NewContextWeight(), NewWaste()} {
		if got := d.Detect(nil); len(got) != 0 {
			t.Errorf("%s invented %d insights from no data", d.Name(), len(got))
		}
	}
}

// TestAddressableCostIsNotDoubleCounted: a trace flagged by several detectors
// must be counted once, or the page claims more savings than was ever spent.
func TestAddressableCostIsNotDoubleCounted(t *testing.T) {
	traces := []TraceFacts{{TraceID: "a", CostUSD: 1.0}}
	all := []Insight{
		{Theme: ThemeCrystallize, Evidence: []Evidence{{TraceID: "a", CostUSD: 1.0}}},
		{Theme: ThemeWaste, Evidence: []Evidence{{TraceID: "a", CostUSD: 1.0}}},
	}
	res := group(all, Window{}, traces, 0)
	if res.AddressableCost != 1.0 {
		t.Errorf("addressable cost = %.2f, want 1.00 (one trace counted once)", res.AddressableCost)
	}
	if res.TotalCost != 1.0 {
		t.Errorf("total cost = %.2f, want 1.00", res.TotalCost)
	}
}

func ids(in []Insight) []string {
	out := make([]string, len(in))
	for i, x := range in {
		out[i] = x.ID
	}
	return out
}

// TestPathShapeLeaksNoHomePath: insight titles are shown in a UI and derived
// from paths on this machine, so the username must not survive normalisation.
func TestPathShapeLeaksNoHomePath(t *testing.T) {
	for _, p := range []string{
		"/Users/zzg/notes.md",
		"/home/zzg/src/thing/main.go",
		"/Users/zzg/work/fathom/README.md",
	} {
		if got := pathShape(p); strings.Contains(got, "zzg") {
			t.Errorf("pathShape(%q) = %q, which carries the username", p, got)
		}
	}
}

// TestThemesAreCapped keeps the page readable. Findings are ranked by cost, so
// the cut falls where the remainder stops mattering — but the count of what was
// dropped is reported, because silently hiding findings would misrepresent how
// much was found.
func TestThemesAreCapped(t *testing.T) {
	var all []Insight
	for i := 0; i < maxPerTheme+5; i++ {
		all = append(all, Insight{
			ID: fmt.Sprintf("x%d", i), Theme: ThemeWaste,
			CostUSD: float64(i), Occurrences: 3,
			Evidence: []Evidence{{TraceID: fmt.Sprintf("t%d", i), CostUSD: float64(i)}},
		})
	}
	res := group(all, Window{}, nil, 0)
	if len(res.Themes) != 1 {
		t.Fatalf("got %d themes, want 1", len(res.Themes))
	}
	g := res.Themes[0]
	if len(g.Insights) != maxPerTheme {
		t.Errorf("showed %d insights, want the %d cap", len(g.Insights), maxPerTheme)
	}
	if g.Truncated != 5 {
		t.Errorf("truncated = %d, want 5", g.Truncated)
	}
	// The most expensive finding must survive the cut.
	if g.Insights[0].CostUSD != float64(maxPerTheme+4) {
		t.Errorf("top insight cost %.0f; the cap dropped the wrong end", g.Insights[0].CostUSD)
	}
}

// TestCapTakesTurnsBetweenFamilies is the fix for a real ranking failure: the
// costs reported by different detectors are not comparable, so a pure cost sort
// buried every sequence finding beneath the prompt clusters.
func TestCapTakesTurnsBetweenFamilies(t *testing.T) {
	var all []Insight
	// Prompt clusters claim whole-trace cost, so they are always larger.
	for i := 0; i < 10; i++ {
		all = append(all, Insight{
			ID: fmt.Sprintf("crystallize/ask/%d", i), Theme: ThemeCrystallize,
			CostUSD: 10 + float64(i),
		})
	}
	// Sequence findings claim only their slice, so they are always smaller.
	for i := 0; i < 10; i++ {
		all = append(all, Insight{
			ID: fmt.Sprintf("crystallize/seq/%d", i), Theme: ThemeCrystallize,
			CostUSD: 1 + float64(i),
		})
	}

	res := group(all, Window{}, nil, 0)
	if len(res.Themes) != 1 {
		t.Fatalf("got %d themes, want 1", len(res.Themes))
	}

	seq, ask := 0, 0
	for _, in := range res.Themes[0].Insights {
		if strings.HasPrefix(in.ID, "crystallize/seq/") {
			seq++
		} else {
			ask++
		}
	}
	if seq == 0 {
		t.Error("no sequence findings shown; the cheaper family was buried entirely")
	}
	if ask == 0 {
		t.Error("no prompt-cluster findings shown")
	}
	if got := len(res.Themes[0].Insights); got != maxPerTheme {
		t.Errorf("showed %d, want %d", got, maxPerTheme)
	}
}

func TestFamily(t *testing.T) {
	cases := map[string]string{
		"crystallize/seq/grep-read-edit": "crystallize/seq",
		"crystallize/ask/why-is-the":     "crystallize/ask",
		"context/rereads":                "context/rereads",
		"waste/failing/go-test":          "waste/failing",
	}
	for in, want := range cases {
		if got := family(in); got != want {
			t.Errorf("family(%q) = %q, want %q", in, got, want)
		}
	}
}

func TestSharedRun(t *testing.T) {
	cases := []struct {
		a, b []string
		want int
	}{
		{[]string{"a", "b", "c"}, []string{"a", "b", "c"}, 3},
		{[]string{"x", "a", "b", "c"}, []string{"y", "a", "b", "c"}, 3},
		{[]string{"a", "b"}, []string{"c", "d"}, 0},
		{[]string{"a", "b", "c"}, []string{"b", "c", "d"}, 2},
		{nil, []string{"a"}, 0},
	}
	for _, tc := range cases {
		if got := sharedRun(tc.a, tc.b); got != tc.want {
			t.Errorf("sharedRun(%v, %v) = %d, want %d", tc.a, tc.b, got, tc.want)
		}
	}
}

// TestVariantsOfOneHabitCollapse: the same routine with a different lead-in is
// one finding, not three. Substring containment alone does not catch this,
// because the variants contain neither one another.
func TestVariantsOfOneHabitCollapse(t *testing.T) {
	core := []ToolCall{
		call("Grep", `{"pattern":"x","type":"go"}`, true),
		call("Read", `{"file_path":"internal/api/a.go"}`, true),
		call("Edit", `{"file_path":"internal/api/a.go"}`, true),
		call("Bash", `{"command":"go test ./internal/api"}`, true),
	}
	leadIns := []ToolCall{
		call("WebFetch", `{"url":"https://docs.example.com/x"}`, true),
		call("Bash", `{"command":"go build ./..."}`, true),
		call("Glob", `{"pattern":"**/*.go"}`, true),
	}

	var traces []TraceFacts
	for i := 0; i < 9; i++ {
		calls := append([]ToolCall{leadIns[i%len(leadIns)]}, core...)
		traces = append(traces, trace(fmt.Sprintf("t%d", i), int64(1_700_000_000_000_000+i), "work", 0.3, calls...))
	}

	var seqs []Insight
	for _, in := range NewCrystallize().Detect(traces) {
		if strings.HasPrefix(in.ID, "crystallize/seq/") {
			seqs = append(seqs, in)
		}
	}
	if len(seqs) != 1 {
		t.Fatalf("got %d sequence findings, want 1; the same habit was reported once per lead-in: %v",
			len(seqs), titles(seqs))
	}
}

// TestSequenceTitleIsReadable: the title is a sentence, not a dump of internal
// signatures. The signatures stay available as steps.
func TestSequenceTitleIsReadable(t *testing.T) {
	var traces []TraceFacts
	for i := 0; i < 4; i++ {
		traces = append(traces, trace(fmt.Sprintf("t%d", i), int64(1_700_000_000_000_000+i), "x", 0.2, grepReadEditTest(i)...))
	}
	for _, in := range NewCrystallize().Detect(traces) {
		if !strings.HasPrefix(in.ID, "crystallize/seq/") {
			continue
		}
		if strings.Contains(in.Title, ":") {
			t.Errorf("title carries raw signatures: %q", in.Title)
		}
		if !strings.Contains(in.Title, "Grep → Read → Edit → Bash") {
			t.Errorf("title = %q, want the tool names", in.Title)
		}
		if len(in.Steps) != 4 || !strings.Contains(in.Steps[0], ":") {
			t.Errorf("steps should keep the full signatures, got %v", in.Steps)
		}
	}
}

func titles(in []Insight) []string {
	out := make([]string, len(in))
	for i, x := range in {
		out[i] = x.Title
	}
	return out
}

// TestThemesIsNeverNull guards a crash rather than a cosmetic wart. A nil slice
// marshals to JSON null, and the client iterating that rendered a raw
// JavaScript error where the empty state should have been — on exactly the
// store a new user starts with.
func TestThemesIsNeverNull(t *testing.T) {
	res := group(nil, Window{}, nil, 0)
	if res.Themes == nil {
		t.Fatal("Themes is nil; it will marshal to null and break the client")
	}
	b, err := json.Marshal(res)
	if err != nil {
		t.Fatal(err)
	}
	if !strings.Contains(string(b), `"themes":[]`) {
		t.Errorf("themes did not marshal as an empty list: %s", b)
	}
}

// TestCoveredCostCountsEachTraceOnce guards a figure shown in a notification.
//
// Several findings routinely name the same traces — one run can be both a
// repeated sequence and a repeated ask — so summing their costs can claim more
// money than the window ever spent. The first version of the nudge did exactly
// that: "$12.52 covered" against a window costing $7.94.
func TestCoveredCostCountsEachTraceOnce(t *testing.T) {
	ins := []Insight{
		{CostUSD: 5, Evidence: []Evidence{{TraceID: "a", CostUSD: 3}, {TraceID: "b", CostUSD: 2}}},
		{CostUSD: 4, Evidence: []Evidence{{TraceID: "a", CostUSD: 3}, {TraceID: "c", CostUSD: 1}}},
	}
	// a=3, b=2, c=1 -> 6. Summing the findings would say 9.
	if got := coveredCost(ins); got != 6 {
		t.Errorf("coveredCost = %v, want 6 (each trace once)", got)
	}

	// With no evidence there is nothing to attribute, so the findings' own
	// figures are the only thing left.
	bare := []Insight{{CostUSD: 2}, {CostUSD: 3}}
	if got := coveredCost(bare); got != 5 {
		t.Errorf("coveredCost with no evidence = %v, want 5", got)
	}
}