fathom/internal/insights/insights.go

// Package insights finds cost patterns in stored traces.
//
// The whole subsystem is deliberately heuristic. No model is called, no
// embedding is computed, and nothing leaves the machine — not only to preserve
// Fathom's single-binary property, but because the questions being asked do not
// need semantics. "Which mechanical sequences repeat" is a categorical question
// over a tiny alphabet of tool names, and sequence mining answers it exactly.
// "Are these two commands the same" is a structural question about argv shape.
// "Have I asked this before" is near-duplicate detection, which MinHash has
// solved for twenty years.
//
// The one place semantics would genuinely help is naming a detected cluster in
// prose. That is also the one thing this deliberately does not do: an insight
// presents the traces it found and lets the reader draw the conclusion, because
// the evidence is more convincing than any summary of it would be.
package insights

import (
	"fmt"
	"sort"
	"strings"
)

// Theme groups insights by the kind of change they suggest.
type Theme string

const (
	// ThemeCrystallize covers work that has become routine enough to be worth
	// capturing as a tool or a throwaway skill.
	ThemeCrystallize Theme = "crystallize"

	// ThemeContext covers context that is being paid for repeatedly.
	ThemeContext Theme = "context"

	// ThemeWaste covers work that was paid for and produced nothing.
	ThemeWaste Theme = "waste"
)

// ThemeInfo describes a theme for the UI.
type ThemeInfo struct {
	Theme    Theme  `json:"theme"`
	Title    string `json:"title"`
	Subtitle string `json:"subtitle"`
}

func Themes() []ThemeInfo {
	return []ThemeInfo{
		{
			Theme: ThemeCrystallize,
			Title: "Worth crystallizing",
			Subtitle: "Sequences you have repeated enough times that a small script " +
				"or a temporary skill would pay for itself. These are usually tied to " +
				"whatever you are working on right now, so they are worth writing " +
				"cheaply and throwing away later.",
		},
		{
			Theme:    ThemeContext,
			Title:    "Context you are paying for twice",
			Subtitle: "Material that was sent to the model more than once, or that grew without being used.",
		},
		{
			Theme:    ThemeWaste,
			Title:    "Spend with nothing to show",
			Subtitle: "Calls that failed, retried, or produced no result you kept.",
		},
	}
}

// Severity orders insights within a theme. It is about how much money is on the
// table, not about how alarming something looks.
type Severity string

const (
	SevHigh   Severity = "high"
	SevMedium Severity = "medium"
	SevLow    Severity = "low"
)

// Evidence points at the traces that produced an insight. An insight without
// evidence is an opinion, so every detector is required to supply it.
type Evidence struct {
	TraceID string  `json:"trace_id"`
	When    int64   `json:"when"`             // unix micros
	Prompt  string  `json:"prompt,omitempty"` // what the run was asked to do
	Detail  string  `json:"detail,omitempty"` // what this trace contributed
	CostUSD float64 `json:"cost_usd"`
}

// Insight is one finding.
type Insight struct {
	ID       string   `json:"id"`
	Theme    Theme    `json:"theme"`
	Severity Severity `json:"severity"`

	// Title states the observation. Body explains why it costs money.
	Title string `json:"title"`
	Body  string `json:"body"`

	// Suggestion is what to do about it, phrased as an option rather than an
	// instruction — the reader knows things about their work that Fathom does
	// not.
	Suggestion string `json:"suggestion,omitempty"`

	// The numbers that justify showing this at all.
	Occurrences int     `json:"occurrences"`
	CostUSD     float64 `json:"cost_usd"`
	Tokens      int64   `json:"tokens,omitempty"`
	WallMicros  int64   `json:"wall_micros,omitempty"`

	// Steps is the detected sequence, for crystallize-style findings.
	Steps []string `json:"steps,omitempty"`

	// Method records how a clustered finding was grouped, where that could
	// have gone more than one way.
	Method ClusterMethod `json:"method,omitempty"`

	// Label is an optional name for the finding, written by a local model over
	// facts already computed. It never replaces a number and is absent when no
	// model is available.
	Label string `json:"label,omitempty"`

	Evidence []Evidence `json:"evidence,omitempty"`

	// FirstSeen and LastSeen bound the window the pattern was observed in.
	FirstSeen int64 `json:"first_seen"`
	LastSeen  int64 `json:"last_seen"`
}

// Detector finds one kind of pattern.
//
// Detectors receive facts already extracted from the store rather than the
// store itself, so one scan feeds all of them and each is a pure function of
// its input — which is what makes them testable without any I/O.
type Detector interface {
	Name() string
	Detect(traces []TraceFacts) []Insight
}

// TraceFacts is the analysed form of one trace: everything the detectors need
// and nothing else. Building this once per scan keeps detectors from each
// walking the span tree again.
type TraceFacts struct {
	TraceID   string
	StartTS   int64
	EndTS     int64
	Prompt    string
	Model     string
	CostUSD   float64
	Tokens    int64
	Cached    int64
	SpanCount int

	// Calls is the tool calls in the order they happened.
	Calls []ToolCall

	// LLMCalls is per-model-request accounting, used to spot context growth.
	LLMCalls []LLMCall
}

// ToolCall is one tool invocation.
type ToolCall struct {
	Name       string // Bash, Read, Edit, ...
	Signature  string // normalised shape, e.g. "Bash:go test"
	Input      string // raw input JSON, for evidence
	Target     string // the file or URL acted on, where there is one
	Succeeded  bool
	DurMicros  int64
	OutputSize int64
}

// LLMCall is one model request.
type LLMCall struct {
	Model       string
	InputTokens int64
	OutputToken int64
	CacheRead   int64
	CostUSD     float64
	DurMicros   int64
	StopReason  string
}

// DurMicros is the trace's wall time.
func (t TraceFacts) DurMicros() int64 {
	if t.EndTS < t.StartTS {
		return 0
	}
	return t.EndTS - t.StartTS
}

// Result is the full analysis, grouped for display.
type Result struct {
	Themes     []ThemeGroup `json:"themes"`
	Window     Window       `json:"window"`
	TraceCount int          `json:"trace_count"`
	TotalCost  float64      `json:"total_cost_usd"`
	// Addressable is the share of spend the insights actually speak to. Showing
	// it keeps the page honest: if it is small, the findings are marginal and
	// should be read that way.
	AddressableCost float64 `json:"addressable_cost_usd"`
	GeneratedAt     int64   `json:"generated_at"`

	// Local describes the model server used, if any.
	Local LocalStatus `json:"local"`

	// MinOccurrences is how many times a pattern must recur to be reported.
	// Sent to the client so the empty-state copy cannot drift away from the
	// thresholds the detectors actually use.
	MinOccurrences int `json:"min_occurrences"`

	// Dismissed is how many findings are currently hidden by the reader. An
	// empty page because everything was dismissed is a different situation
	// from an empty page because nothing was found.
	Dismissed int `json:"dismissed"`
}

// ThemeGroup is one theme and its findings.
type ThemeGroup struct {
	ThemeInfo
	Insights []Insight `json:"insights"`
	// Truncated is how many further findings were suppressed. A page listing
	// everything it noticed is a page nobody reads, but silently dropping
	// findings would misrepresent how much was found.
	Truncated int `json:"truncated,omitempty"`
}

// maxPerTheme caps what is shown, so the page stays readable.
const maxPerTheme = 6

// capWithDiversity fills the cap by taking turns between kinds of finding.
//
// A straight cost ranking does not work here, because the costs are not
// comparable across detectors. A repeated-ask finding claims the whole cost of
// every trace it names — you re-ran the entire thing — while a repeated-sequence
// finding claims only its slice of each trace. The ask numbers are therefore
// always larger, and a pure sort buries every sequence finding beneath them even
// when the sequences are the more actionable result.
//
// Taking turns by family, in cost order within each, shows the best of each kind
// instead of every instance of the loudest kind.
func capWithDiversity(ins []Insight, limit int) []Insight {
	if len(ins) <= limit {
		return ins
	}

	order := []string{}
	byFamily := map[string][]Insight{}
	for _, in := range ins {
		f := family(in.ID)
		if _, seen := byFamily[f]; !seen {
			order = append(order, f)
		}
		byFamily[f] = append(byFamily[f], in)
	}

	out := make([]Insight, 0, limit)
	for len(out) < limit {
		progressed := false
		for _, f := range order {
			if len(out) >= limit {
				break
			}
			q := byFamily[f]
			if len(q) == 0 {
				continue
			}
			out = append(out, q[0])
			byFamily[f] = q[1:]
			progressed = true
		}
		if !progressed {
			break
		}
	}
	// Restore cost order for display; the round-robin was only for selection.
	sort.SliceStable(out, func(i, j int) bool { return out[i].CostUSD > out[j].CostUSD })
	return out
}

// family is the detector-specific prefix of an insight id: "crystallize/seq"
// and "crystallize/ask" are different kinds of finding within one theme.
func family(id string) string {
	first := strings.Index(id, "/")
	if first < 0 {
		return id
	}
	second := strings.Index(id[first+1:], "/")
	if second < 0 {
		return id
	}
	return id[:first+1+second]
}

// Window is the time range analysed.
type Window struct {
	From int64 `json:"from"`
	To   int64 `json:"to"`
}

// group assembles detector output into display order: themes in their declared
// order, insights within a theme by money at stake.
func group(all []Insight, window Window, traces []TraceFacts, generatedAt int64) Result {
	byTheme := make(map[Theme][]Insight)
	for _, in := range all {
		byTheme[in.Theme] = append(byTheme[in.Theme], in)
	}

	// Themes is always a list, never null. A nil slice marshals to JSON null,
	// which the client then tries to iterate — that crashed the insights page
	// on an empty store and rendered a raw JavaScript error.
	res := Result{
		Themes:      []ThemeGroup{},
		Window:      window,
		TraceCount:  len(traces),
		GeneratedAt: generatedAt,
	}
	for _, t := range traces {
		res.TotalCost += t.CostUSD
	}

	// Addressable spend is counted per trace, not per insight, so a trace
	// flagged by three detectors is not counted three times. Summing insight
	// costs would routinely claim more savings than was ever spent.
	addressed := make(map[string]float64)
	for _, in := range all {
		for _, e := range in.Evidence {
			if e.CostUSD > addressed[e.TraceID] {
				addressed[e.TraceID] = e.CostUSD
			}
		}
	}
	for _, c := range addressed {
		res.AddressableCost += c
	}

	for _, info := range Themes() {
		ins := byTheme[info.Theme]
		if len(ins) == 0 {
			continue
		}
		sort.SliceStable(ins, func(i, j int) bool {
			if ins[i].CostUSD != ins[j].CostUSD {
				return ins[i].CostUSD > ins[j].CostUSD
			}
			return ins[i].Occurrences > ins[j].Occurrences
		})
		g := ThemeGroup{ThemeInfo: info}
		shown := capWithDiversity(ins, maxPerTheme)
		g.Truncated = len(ins) - len(shown)
		g.Insights = shown
		res.Themes = append(res.Themes, g)
	}
	return res
}

// severityFor maps money at stake onto a label.
//
// The thresholds are absolute dollars rather than a share of spend, because a
// share is misleading at both ends: on a quiet week every finding looks
// enormous, and on a busy one a genuinely expensive habit looks negligible.
func severityFor(costUSD float64, occurrences int) Severity {
	switch {
	case costUSD >= 5 || occurrences >= 20:
		return SevHigh
	case costUSD >= 1 || occurrences >= 8:
		return SevMedium
	default:
		return SevLow
	}
}

func plural(n int, one, many string) string {
	if n == 1 {
		return one
	}
	return many
}

func money(v float64) string {
	if v < 0.01 {
		return fmt.Sprintf("$%.4f", v)
	}
	return fmt.Sprintf("$%.2f", v)
}