fathom/internal/insights/normalize.go

package insights

import (
	"encoding/json"
	"path/filepath"
	"regexp"
	"strings"
)

// Normalisation turns a concrete tool call into the shape of the tool call.
//
// This is the piece that would have been reached for an embedding. It is not
// needed: `go test ./internal/payments` and `go test ./internal/orders` are the
// same action not because they mean the same thing but because they have the
// same argv shape with a path substituted. Structure answers it exactly, and
// unlike a similarity score it is inspectable when it gets something wrong.

var (
	// Wrappers contribute nothing to what a command *does*, so they are peeled
	// off before the verb is read. Otherwise every command through `sudo` or
	// `npx` collapses into one bucket.
	cmdWrappers = map[string]bool{
		"sudo": true, "env": true, "time": true, "nice": true,
		"npx": true, "bunx": true, "pnpm": true, "yarn": true, "npm": true,
		"uv": true, "uvx": true, "poetry": true, "pipx": true,
	}

	// Tools whose first argument is a meaningful subcommand rather than a
	// target. `git commit` and `git status` are different actions; `cat a.txt`
	// and `cat b.txt` are the same one.
	subcommandTools = map[string]bool{
		"git": true, "go": true, "cargo": true, "docker": true, "kubectl": true,
		"npm": true, "pnpm": true, "yarn": true, "make": true, "terraform": true,
		"gh": true, "brew": true, "systemctl": true, "tea": true, "pip": true,
	}

	flagRe   = regexp.MustCompile(`^-{1,2}[A-Za-z0-9]`)
	numberRe = regexp.MustCompile(`\d+`)
	hexRe    = regexp.MustCompile(`\b[0-9a-f]{8,}\b`)
)

// normalizeCall derives the stable signature and human target for a tool call.
func normalizeCall(toolName, inputJSON string) (signature, target string) {
	args := parseArgs(inputJSON)

	switch toolName {
	case "Bash", "bash", "shell", "run_command":
		cmd := firstNonEmpty(args, "command", "full_command", "cmd", "script")
		verb := commandShape(cmd)
		return "Bash:" + verb, verb

	case "Read", "read_file", "View":
		p := firstNonEmpty(args, "file_path", "path", "filename")
		return "Read:" + pathShape(p), p

	case "Edit", "edit_file", "Write", "write_file", "NotebookEdit":
		p := firstNonEmpty(args, "file_path", "path", "filename")
		return toolName + ":" + pathShape(p), p

	case "Grep", "search", "ripgrep":
		// The pattern is the variable part; what repeats is the act of
		// searching, optionally scoped to a file type.
		t := firstNonEmpty(args, "type", "glob")
		if t != "" {
			return "Grep:" + t, t
		}
		return "Grep", firstNonEmpty(args, "pattern", "query")

	case "Glob", "list_files", "LS":
		return "Glob", firstNonEmpty(args, "pattern", "path")

	case "WebFetch", "fetch", "WebSearch":
		u := firstNonEmpty(args, "url", "query")
		return toolName + ":" + hostShape(u), u

	case "Task", "Agent":
		return "Task:" + firstNonEmpty(args, "subagent_type", "description"), ""

	case "Skill":
		s := firstNonEmpty(args, "skill", "name")
		return "Skill:" + s, s
	}

	// Unknown tools, including MCP ones, keep their name. That is enough to
	// detect repetition; it just cannot say anything about the arguments.
	return toolName, firstNonEmpty(args, "path", "file_path", "url", "query")
}

// commandShape reduces a shell command to its verb.
//
// `cd /srv && go test ./internal/... -run TestFoo -v` becomes `go test`. The
// paths, the test name, and the flags are exactly the parts that differ between
// two runs of the same habit.
func commandShape(cmd string) string {
	cmd = strings.TrimSpace(cmd)
	if cmd == "" {
		return "?"
	}

	// Take the last stage of a pipeline or chain: `cd x && go test` is a `go
	// test`, and `cat f | jq .` is a `jq`.
	if seg := lastSegment(cmd); seg != "" {
		cmd = seg
	}

	fields := splitFields(cmd)
	i := 0
	for i < len(fields) && (cmdWrappers[fields[i]] || strings.Contains(fields[i], "=")) {
		i++
	}
	if i >= len(fields) {
		return "?"
	}

	base := filepath.Base(fields[i])
	if !subcommandTools[base] {
		return base
	}
	// Take the first following word that is not a flag as the subcommand.
	for _, f := range fields[i+1:] {
		if flagRe.MatchString(f) {
			continue
		}
		return base + " " + f
	}
	return base
}

// lastSegment returns the final stage of a pipeline or && / ; chain. Quoting is
// ignored deliberately: a mis-split yields a slightly odd bucket label, never a
// wrong answer about which calls happened.
func lastSegment(cmd string) string {
	best := cmd
	for _, sep := range []string{"&&", "||", ";", "|"} {
		if i := strings.LastIndex(best, sep); i >= 0 {
			if tail := strings.TrimSpace(best[i+len(sep):]); tail != "" {
				best = tail
			}
		}
	}
	return strings.TrimSpace(best)
}

func splitFields(s string) []string {
	return strings.Fields(strings.ReplaceAll(s, "\n", " "))
}

// pathShape keeps the immediate parent directory and the extension, and drops
// the filename.
//
// The parent directory is the right grain: two edits inside internal/api are
// the same habit, while an edit in internal/api and one in internal/store are
// not. Walking down from the filesystem root instead would make the shape
// depend on where the repository happens to be checked out, and would put the
// operator's username in an insight title.
func pathShape(p string) string {
	if p == "" {
		return "?"
	}
	ext := strings.TrimPrefix(filepath.Ext(p), ".")
	if ext == "" {
		ext = "file"
	}
	dir := filepath.Dir(p)
	parent := filepath.Base(dir)
	switch parent {
	case "", ".", "/":
		return "*." + ext
	}
	// A file sitting directly in a home directory has the account name as its
	// parent. That is not a useful grouping and it would put the username into
	// an insight title, so it is dropped.
	if grand := filepath.Base(filepath.Dir(dir)); isHomeRoot(grand) {
		return "*." + ext
	}
	return parent + "/*." + ext
}

func isHomeRoot(s string) bool {
	switch s {
	case "Users", "users", "home":
		return true
	}
	return false
}

func hostShape(u string) string {
	u = strings.TrimPrefix(strings.TrimPrefix(u, "https://"), "http://")
	if i := strings.IndexAny(u, "/?"); i > 0 {
		u = u[:i]
	}
	if u == "" {
		return "?"
	}
	return u
}

// parseArgs reads a tool input, which emitters send as a JSON object but not
// always. A non-object input is not an error worth reporting; the call simply
// contributes no argument detail.
func parseArgs(inputJSON string) map[string]string {
	out := map[string]string{}
	if inputJSON == "" {
		return out
	}
	var raw map[string]any
	if err := json.Unmarshal([]byte(inputJSON), &raw); err != nil {
		return out
	}
	for k, v := range raw {
		switch t := v.(type) {
		case string:
			out[k] = t
		case float64, bool:
			out[k] = strings.TrimSpace(strings.Trim(jsonString(t), `"`))
		}
	}
	return out
}

func jsonString(v any) string {
	b, err := json.Marshal(v)
	if err != nil {
		return ""
	}
	return string(b)
}

func firstNonEmpty(args map[string]string, keys ...string) string {
	for _, k := range keys {
		if v, ok := args[k]; ok && strings.TrimSpace(v) != "" {
			return strings.TrimSpace(v)
		}
	}
	return ""
}

// canonPrompt reduces a prompt to comparable tokens: lowercased words with
// digits and hex ids flattened, so "fix the flaky test in payments_test.go" and
// "fix the flaky test in orders_test.go" share most of their shingles.
func canonPrompt(s string) []string {
	s = strings.ToLower(s)
	s = hexRe.ReplaceAllString(s, "#")
	s = numberRe.ReplaceAllString(s, "#")
	fields := strings.FieldsFunc(s, func(r rune) bool {
		return !(r == '_' || r == '#' || r == '.' ||
			(r >= 'a' && r <= 'z') || (r >= '0' && r <= '9'))
	})
	out := fields[:0]
	for _, f := range fields {
		if len(f) < 2 || stopWords[f] {
			continue
		}
		out = append(out, f)
	}
	return out
}

var stopWords = map[string]bool{
	"the": true, "a": true, "an": true, "and": true, "or": true, "to": true,
	"of": true, "in": true, "for": true, "on": true, "is": true, "it": true,
	"be": true, "as": true, "at": true, "by": true, "with": true, "that": true,
	"this": true, "then": true, "so": true, "if": true, "me": true, "my": true,
	"you": true, "your": true, "please": true, "can": true, "could": true,
	"should": true, "would": true, "do": true, "does": true, "did": true,
	"use": true, "using": true, "make": true, "sure": true, "just": true,
}