fathom/internal/insights/local_test.go

package insights

import (
	"context"
	"errors"
	"fmt"
	"strings"
	"testing"
)

// fakeEmbedder produces deterministic vectors from a topic keyword, so the
// embedding path can be tested without a model. Prompts sharing a topic get
// near-identical vectors even when they share no other vocabulary — which is
// exactly the case MinHash cannot handle and the reason this path exists.
type fakeEmbedder struct {
	topics map[string]int // substring -> topic id
	dims   int
	err    error
	calls  int
}

func (f *fakeEmbedder) Embed(ctx context.Context, texts []string) ([][]float32, error) {
	f.calls++
	if f.err != nil {
		return nil, f.err
	}
	out := make([][]float32, len(texts))
	for i, t := range texts {
		v := make([]float32, f.dims)
		topic := 0
		for sub, id := range f.topics {
			if strings.Contains(strings.ToLower(t), sub) {
				topic = id
				break
			}
		}
		// One dominant axis per topic, with a little spread so vectors within a
		// topic are close but not identical.
		v[topic%f.dims] = 1
		v[(topic*7+i)%f.dims] += 0.05
		out[i] = v
	}
	return out, nil
}

// TestEmbeddingClusteringCatchesWhatMinHashMisses is the whole argument for
// using a local model in detection. These two prompts mean the same thing and
// share almost no vocabulary, so lexical clustering provably cannot group them.
func TestEmbeddingClusteringCatchesWhatMinHashMisses(t *testing.T) {
	prompts := []string{
		"the flaky test in payments keeps failing intermittently",
		"diagnose the intermittent failure in the billing suite",
		"work out why that unreliable spec breaks now and then",
		"add pagination to the orders endpoint keeping the shape stable",
	}

	lexical, method := clusterPrompts(context.Background(), nil, prompts, 0.55)
	if method != MethodMinHash {
		t.Fatalf("with no embedder the method should be %q, got %q", MethodMinHash, method)
	}
	if len(lexical) != 0 {
		t.Errorf("MinHash unexpectedly grouped these; the test no longer demonstrates the gap: %v", lexical)
	}

	emb := &fakeEmbedder{
		topics: map[string]int{
			"flaky": 1, "intermittent": 1, "unreliable": 1,
			"pagination": 2,
		},
		dims: 16,
	}
	semantic, method := clusterPrompts(context.Background(), emb, prompts, 0.55)
	if method != MethodEmbedding {
		t.Fatalf("method = %q, want %q", method, MethodEmbedding)
	}
	if len(semantic) != 1 {
		t.Fatalf("got %d clusters, want 1: %v", len(semantic), semantic)
	}
	if len(semantic[0]) != 3 {
		t.Errorf("cluster has %d members, want the 3 flaky-test prompts: %v", len(semantic[0]), semantic[0])
	}
	// The unrelated prompt must stay out.
	for _, m := range semantic[0] {
		if m == 3 {
			t.Error("the pagination prompt was grouped with the flaky-test ones")
		}
	}
}

// TestEmbedderFailureFallsBackSilently is the property that keeps a local model
// optional: anything going wrong costs clustering quality, never the analysis.
func TestEmbedderFailureFallsBackSilently(t *testing.T) {
	prompts := []string{
		"fix the flaky test in payments it fails about one run in five",
		"fix the flaky test in billing it fails about one run in five",
		"fix the flaky test in orders it fails about one run in five",
		"add pagination to the reporting endpoint",
	}
	const threshold = 0.55

	// The fallback must reproduce the answer MinHash gives on its own, so the
	// assertion is against that rather than a hardcoded count — the point is
	// that a failing model changes nothing, not that it yields some specific
	// number.
	want, wantMethod := clusterPrompts(context.Background(), nil, prompts, threshold)
	if wantMethod != MethodMinHash {
		t.Fatalf("baseline method = %q", wantMethod)
	}
	if len(want) == 0 {
		t.Fatal("the baseline found no clusters, so this test would pass vacuously")
	}

	for name, emb := range map[string]Embedder{
		"error":        &fakeEmbedder{err: errors.New("connection refused"), dims: 8},
		"wrong length": shortEmbedder{},
		"empty vector": zeroDimEmbedder{},
	} {
		got, method := clusterPrompts(context.Background(), emb, prompts, threshold)
		if method != MethodMinHash {
			t.Errorf("%s: method = %q, want a silent fallback to %q", name, method, MethodMinHash)
		}
		if fmt.Sprint(got) != fmt.Sprint(want) {
			t.Errorf("%s: fallback gave %v, want the MinHash answer %v", name, got, want)
		}
	}
}

type shortEmbedder struct{}

func (shortEmbedder) Embed(ctx context.Context, texts []string) ([][]float32, error) {
	return [][]float32{{1, 0}}, nil // fewer vectors than inputs
}

type zeroDimEmbedder struct{}

func (zeroDimEmbedder) Embed(ctx context.Context, texts []string) ([][]float32, error) {
	out := make([][]float32, len(texts))
	for i := range out {
		out[i] = nil
	}
	return out, nil
}

// TestBlankPromptsNeverCluster: traces with no prompt would otherwise all embed
// to the same vector and form one meaningless finding.
func TestBlankPromptsNeverCluster(t *testing.T) {
	prompts := []string{"", "", "", "  "}
	emb := &fakeEmbedder{dims: 8}
	got, _ := clusterPrompts(context.Background(), emb, prompts, 0.55)
	if len(got) != 0 {
		t.Errorf("blank prompts formed %d clusters, want 0", len(got))
	}
	if emb.calls != 0 {
		t.Error("blank prompts were sent to the model")
	}
}

func TestNormalizeAndDot(t *testing.T) {
	if v := normalize([]float32{0, 0, 0}); v != nil {
		t.Error("a zero vector must normalize to nil, or it is similar to everything")
	}
	a := normalize([]float32{3, 4})
	if got := dot(a, a); got < 0.999 || got > 1.001 {
		t.Errorf("a unit vector dotted with itself = %v, want 1", got)
	}
	if got := dot([]float32{1, 0}, []float32{1, 0, 0}); got != 0 {
		t.Errorf("mismatched lengths gave %v, want 0", got)
	}
}

// fakeLabeler stands in for a chat model.
type fakeLabeler struct {
	reply string
	err   error
	calls int
}

func (f *fakeLabeler) Label(ctx context.Context, prompts []string) (string, error) {
	f.calls++
	return f.reply, f.err
}

// TestLabelsNeverInventNumbers is the line the model is not allowed to cross.
// It may write a phrase; the occurrence count and cost stay as computed.
func TestLabelsNeverInventNumbers(t *testing.T) {
	res := Result{Themes: []ThemeGroup{{
		ThemeInfo: ThemeInfo{Theme: ThemeCrystallize},
		Insights: []Insight{{
			ID: "crystallize/ask/x", Occurrences: 7, CostUSD: 1.25,
			Title: "You asked for much the same thing 7 times",
			Evidence: []Evidence{
				{TraceID: "a", Prompt: "fix the flaky test"},
				{TraceID: "b", Prompt: "fix the other flaky test"},
			},
		}},
	}}}

	a := &Analyzer{
		local:       &fakeLabeler{reply: "Flaky test triage in payments"},
		crystallize: NewCrystallize(),
		status:      LocalStatus{Available: true, Enabled: true},
	}
	if n := a.labelFindings(&res); n != 1 {
		t.Fatalf("applied %d labels, want 1", n)
	}

	got := res.Themes[0].Insights[0]
	if got.Label != "Flaky test triage in payments" {
		t.Errorf("label = %q", got.Label)
	}
	if got.Occurrences != 7 || got.CostUSD != 1.25 {
		t.Errorf("the model changed the numbers: occurrences=%d cost=%.2f", got.Occurrences, got.CostUSD)
	}
	if !strings.Contains(got.Title, "7 times") {
		t.Errorf("title lost the computed count: %q", got.Title)
	}
}

func TestLabelFailureKeepsHeuristicTitle(t *testing.T) {
	original := "You asked for much the same thing 4 times"
	for name, lab := range map[string]Labeler{
		"error": &fakeLabeler{err: errors.New("model is loading")},
		"empty": &fakeLabeler{reply: ""},
	} {
		res := Result{Themes: []ThemeGroup{{
			Insights: []Insight{{
				ID: "crystallize/ask/x", Occurrences: 4, Title: original,
				Evidence: []Evidence{{Prompt: "a"}, {Prompt: "b"}},
			}},
		}}}
		a := &Analyzer{local: lab, crystallize: NewCrystallize(),
			status: LocalStatus{Available: true, Enabled: true}}

		if n := a.labelFindings(&res); n != 0 {
			t.Errorf("%s: applied %d labels, want 0", name, n)
		}
		if got := res.Themes[0].Insights[0].Title; got != original {
			t.Errorf("%s: title changed to %q on failure", name, got)
		}
	}
}

// TestOnlyClusterFindingsAreLabelled: a sequence finding is already precisely
// named by its steps, and asking a model to rename it would only add noise.
func TestOnlyClusterFindingsAreLabelled(t *testing.T) {
	res := Result{Themes: []ThemeGroup{{
		Insights: []Insight{
			{ID: "crystallize/seq/a", Occurrences: 5, Title: "Read → Edit → Bash",
				Evidence: []Evidence{{Prompt: "a"}, {Prompt: "b"}}},
			{ID: "waste/failing/go-test", Occurrences: 3, Title: "`go test` failed",
				Evidence: []Evidence{{Prompt: "a"}, {Prompt: "b"}}},
		},
	}}}
	lab := &fakeLabeler{reply: "Something"}
	a := &Analyzer{local: lab, crystallize: NewCrystallize(),
		status: LocalStatus{Available: true, Enabled: true}}

	if n := a.labelFindings(&res); n != 0 {
		t.Errorf("labelled %d non-cluster findings, want 0", n)
	}
	if lab.calls != 0 {
		t.Errorf("the model was called %d times for findings that do not need naming", lab.calls)
	}
}

// TestDisabledLocalUsesMinHash: turning the toggle off must actually change the
// detection path, not just the label.
func TestDisabledLocalUsesMinHash(t *testing.T) {
	c := NewCrystallize()
	c.Embedder = &fakeEmbedder{dims: 8}

	a := &Analyzer{crystallize: c, embedder: c.Embedder,
		status: LocalStatus{Available: true, Enabled: false}}
	a.applyLocal()
	if a.crystallize.Embedder != nil {
		t.Error("the embedder is still wired in with local use disabled")
	}

	a.status.Enabled = true
	a.applyLocal()
	if a.crystallize.Embedder == nil {
		t.Error("the embedder was not restored when enabled")
	}
}

func TestClusterMethodIsReported(t *testing.T) {
	c := NewCrystallize()
	if got := c.LastMethod(); got != MethodMinHash {
		t.Errorf("before any run the method should default to %q, got %q", MethodMinHash, got)
	}

	var traces []TraceFacts
	for i := 0; i < 4; i++ {
		traces = append(traces, trace(fmt.Sprintf("t%d", i), int64(1_700_000_000_000_000+i),
			"fix the flaky test in payments", 0.2))
	}
	c.Detect(traces)
	if got := c.LastMethod(); got != MethodMinHash {
		t.Errorf("method = %q, want %q with no embedder", got, MethodMinHash)
	}

	c.Embedder = &fakeEmbedder{dims: 8}
	c.Detect(traces)
	if got := c.LastMethod(); got != MethodEmbedding {
		t.Errorf("method = %q, want %q with an embedder", got, MethodEmbedding)
	}
}