fathom/internal/insights/context_waste.go

package insights

import (
	"fmt"
	"sort"
	"strings"
)

// ContextWeight finds material that was paid for more than once.
//
// This is usually the largest real line item and the least visible one, because
// nothing in a normal trace view shows that the same file went into the context
// four times.
type ContextWeight struct {
	// RereadThreshold is how many times one file must be read within a single
	// trace before it counts. Twice is ordinary — look, edit, look again.
	RereadThreshold int

	// LargeOutputBytes flags a single tool result big enough to matter on its
	// own once it enters the context.
	LargeOutputBytes int64

	// GrowthFactor is how much the last model call's input must exceed the
	// first before the growth is worth remarking on.
	GrowthFactor float64
}

func NewContextWeight() *ContextWeight {
	return &ContextWeight{
		RereadThreshold:  3,
		LargeOutputBytes: 50_000,
		GrowthFactor:     4,
	}
}

func (c *ContextWeight) Name() string { return "context" }

func (c *ContextWeight) Detect(traces []TraceFacts) []Insight {
	var out []Insight
	if in, ok := c.rereads(traces); ok {
		out = append(out, in)
	}
	if in, ok := c.largeOutputs(traces); ok {
		out = append(out, in)
	}
	if in, ok := c.growth(traces); ok {
		out = append(out, in)
	}
	return out
}

// rereads finds files read repeatedly within a single run.
func (c *ContextWeight) rereads(traces []TraceFacts) (Insight, bool) {
	var ev []Evidence
	var cost float64
	total := 0
	first, last := int64(0), int64(0)

	for _, t := range traces {
		counts := map[string]int{}
		for _, call := range t.Calls {
			if !isRead(call.Name) || call.Target == "" {
				continue
			}
			counts[call.Target]++
		}
		var worst string
		worstN := 0
		repeats := 0
		for target, n := range counts {
			if n >= c.RereadThreshold {
				repeats += n - 1
				if n > worstN {
					worst, worstN = target, n
				}
			}
		}
		if worstN == 0 {
			continue
		}
		total += repeats
		cost += t.CostUSD
		if first == 0 || t.StartTS < first {
			first = t.StartTS
		}
		if t.StartTS > last {
			last = t.StartTS
		}
		ev = append(ev, Evidence{
			TraceID: t.TraceID, When: t.StartTS, Prompt: t.Prompt, CostUSD: t.CostUSD,
			Detail: fmt.Sprintf("read %s %d times", trimMiddle(worst, 48), worstN),
		})
	}
	if len(ev) < 2 {
		return Insight{}, false
	}
	sort.SliceStable(ev, func(i, j int) bool { return ev[i].When > ev[j].When })
	if len(ev) > 8 {
		ev = ev[:8]
	}

	in := Insight{
		ID:          "context/rereads",
		Theme:       ThemeContext,
		Severity:    severityFor(cost, total),
		Title:       fmt.Sprintf("The same files were re-read %d times within single runs", total),
		Occurrences: total,
		CostUSD:     cost,
		FirstSeen:   first,
		LastSeen:    last,
		Evidence:    ev,
	}
	in.Body = "A file read three or more times inside one run is usually the agent losing track of something it " +
		"already had. Each read goes back into the context and is paid for on every subsequent turn of that run, " +
		"so the cost compounds rather than repeating."
	in.Suggestion = "Where a file is central to a task, putting the relevant part somewhere durable — a note in the " +
		"repo, or a project instruction file — costs one read instead of several."
	return in, true
}

// largeOutputs finds single tool results big enough to be worth truncating.
func (c *ContextWeight) largeOutputs(traces []TraceFacts) (Insight, bool) {
	type hit struct {
		t    TraceFacts
		call ToolCall
	}
	var hits []hit
	var bytes int64

	for _, t := range traces {
		for _, call := range t.Calls {
			if call.OutputSize >= c.LargeOutputBytes {
				hits = append(hits, hit{t, call})
				bytes += call.OutputSize
			}
		}
	}
	if len(hits) < 2 {
		return Insight{}, false
	}
	sort.SliceStable(hits, func(i, j int) bool { return hits[i].call.OutputSize > hits[j].call.OutputSize })

	var cost float64
	seen := map[string]bool{}
	var ev []Evidence
	first, last := int64(0), int64(0)
	for _, h := range hits {
		if !seen[h.t.TraceID] {
			seen[h.t.TraceID] = true
			cost += h.t.CostUSD
		}
		if first == 0 || h.t.StartTS < first {
			first = h.t.StartTS
		}
		if h.t.StartTS > last {
			last = h.t.StartTS
		}
		if len(ev) < 8 {
			ev = append(ev, Evidence{
				TraceID: h.t.TraceID, When: h.t.StartTS, Prompt: h.t.Prompt, CostUSD: h.t.CostUSD,
				Detail: fmt.Sprintf("%s returned %s", h.call.Name, humanBytes(h.call.OutputSize)),
			})
		}
	}

	in := Insight{
		ID:          "context/large-outputs",
		Theme:       ThemeContext,
		Severity:    severityFor(cost, len(hits)),
		Title:       fmt.Sprintf("%d tool results were large enough to dominate the context", len(hits)),
		Occurrences: len(hits),
		CostUSD:     cost,
		FirstSeen:   first,
		LastSeen:    last,
		Evidence:    ev,
	}
	in.Body = fmt.Sprintf(
		"These results total %s. A large tool output is not paid for once — it stays in the context and is re-sent "+
			"with every following turn in that run.", humanBytes(bytes))
	in.Suggestion = "Narrowing these calls so they return the part that matters — a filter, a head, a smaller glob — " +
		"usually costs nothing in usefulness."
	return in, true
}

// growth finds runs whose context ballooned without obvious cause.
func (c *ContextWeight) growth(traces []TraceFacts) (Insight, bool) {
	var ev []Evidence
	var cost float64
	first, last := int64(0), int64(0)

	for _, t := range traces {
		if len(t.LLMCalls) < 3 {
			continue
		}
		start := t.LLMCalls[0].InputTokens + t.LLMCalls[0].CacheRead
		end := t.LLMCalls[len(t.LLMCalls)-1].InputTokens + t.LLMCalls[len(t.LLMCalls)-1].CacheRead
		if start == 0 || float64(end) < float64(start)*c.GrowthFactor {
			continue
		}
		cost += t.CostUSD
		if first == 0 || t.StartTS < first {
			first = t.StartTS
		}
		if t.StartTS > last {
			last = t.StartTS
		}
		ev = append(ev, Evidence{
			TraceID: t.TraceID, When: t.StartTS, Prompt: t.Prompt, CostUSD: t.CostUSD,
			Detail: fmt.Sprintf("context grew %s → %s over %d model calls",
				humanCount(start), humanCount(end), len(t.LLMCalls)),
		})
	}
	if len(ev) < 2 {
		return Insight{}, false
	}
	sort.SliceStable(ev, func(i, j int) bool { return ev[i].CostUSD > ev[j].CostUSD })
	if len(ev) > 8 {
		ev = ev[:8]
	}

	in := Insight{
		ID:          "context/growth",
		Theme:       ThemeContext,
		Severity:    severityFor(cost, len(ev)),
		Title:       fmt.Sprintf("%d runs ended with several times the context they started with", len(ev)),
		Occurrences: len(ev),
		CostUSD:     cost,
		FirstSeen:   first,
		LastSeen:    last,
		Evidence:    ev,
	}
	in.Body = "Context grows as a run goes on, which is normal. Growing several times over usually means the run " +
		"accumulated material it never needed again — early exploration that stayed in the transcript."
	in.Suggestion = "Long exploratory runs are cheaper split in two: find the answer in one, then start a fresh run " +
		"that begins from it."
	return in, true
}

// Waste finds spend that produced nothing.
type Waste struct {
	// MinFailures is how many failures of one command shape are needed before
	// it reads as a pattern rather than an accident.
	MinFailures int
}

func NewWaste() *Waste { return &Waste{MinFailures: 3} }

func (w *Waste) Name() string { return "waste" }

func (w *Waste) Detect(traces []TraceFacts) []Insight {
	type fail struct {
		count  int
		cost   float64
		traces map[string]Evidence
	}
	byShape := map[string]*fail{}

	for _, t := range traces {
		for _, call := range t.Calls {
			if call.Succeeded {
				continue
			}
			f := byShape[call.Signature]
			if f == nil {
				f = &fail{traces: map[string]Evidence{}}
				byShape[call.Signature] = f
			}
			f.count++
			if _, ok := f.traces[t.TraceID]; !ok {
				f.cost += t.CostUSD
				f.traces[t.TraceID] = Evidence{
					TraceID: t.TraceID, When: t.StartTS, Prompt: t.Prompt, CostUSD: t.CostUSD,
					Detail: fmt.Sprintf("%s failed", call.Name),
				}
			}
		}
	}

	var out []Insight
	for shape, f := range byShape {
		if f.count < w.MinFailures {
			continue
		}
		var ev []Evidence
		first, last := int64(0), int64(0)
		for _, e := range f.traces {
			if first == 0 || e.When < first {
				first = e.When
			}
			if e.When > last {
				last = e.When
			}
			ev = append(ev, e)
		}
		sort.SliceStable(ev, func(i, j int) bool { return ev[i].When > ev[j].When })
		if len(ev) > 8 {
			ev = ev[:8]
		}

		in := Insight{
			ID:          "waste/failing/" + slug(shape),
			Theme:       ThemeWaste,
			Severity:    severityFor(f.cost, f.count),
			Title:       fmt.Sprintf("`%s` failed %d times", displayShape(shape), f.count),
			Occurrences: f.count,
			CostUSD:     f.cost,
			FirstSeen:   first,
			LastSeen:    last,
			Evidence:    ev,
			Steps:       []string{shape},
		}
		in.Body = fmt.Sprintf(
			"This call shape failed %d times across %d %s. Every failure is paid for twice: once for the call, "+
				"and again for the model turn that reads the error and decides what to do about it.",
			f.count, len(f.traces), plural(len(f.traces), "run", "runs"))
		in.Suggestion = "A command that fails this reliably is usually wrong in a fixable way — a missing tool, a " +
			"wrong path, a flag that does not exist on this machine. Fixing it once removes the whole series."
		out = append(out, in)
	}
	return out
}

func isRead(name string) bool {
	switch name {
	case "Read", "read_file", "View", "cat":
		return true
	}
	return false
}

func displayShape(s string) string {
	if i := strings.Index(s, ":"); i >= 0 && i+1 < len(s) {
		return s[i+1:]
	}
	return s
}

func humanBytes(n int64) string {
	switch {
	case n >= 1<<20:
		return fmt.Sprintf("%.1fMB", float64(n)/(1<<20))
	case n >= 1<<10:
		return fmt.Sprintf("%.0fKB", float64(n)/(1<<10))
	}
	return fmt.Sprintf("%dB", n)
}

func humanCount(n int64) string {
	switch {
	case n >= 1_000_000:
		return fmt.Sprintf("%.1fM", float64(n)/1e6)
	case n >= 1000:
		return fmt.Sprintf("%.0fk", float64(n)/1000)
	}
	return fmt.Sprintf("%d", n)
}