fathom/internal/insights/local/local.go

// Package local talks to a model server already running on this machine.
//
// This is the one place Fathom makes an outbound request, and the constraints
// it was built under still hold: no credential, no per-call cost, no data
// leaving the machine, and no dependency beyond the standard library. If
// nothing is listening, every entry point here returns "unavailable" and the
// caller carries on with its own deterministic path.
//
// Two wire formats cover the field. Ollama has its own, and LM Studio,
// llama.cpp's server, and most everything else expose an OpenAI-compatible
// /v1. Supporting both is a few dozen lines and removes the need to ask anyone
// which one they run.
package local

import (
	"bytes"
	"context"
	"encoding/json"
	"fmt"
	"net/http"
	"os"
	"strings"
	"time"
)

// Kind is the wire format a provider speaks.
type Kind string

const (
	KindOllama Kind = "ollama"
	KindOpenAI Kind = "openai"
)

// Provider is a reachable local model server.
type Provider struct {
	BaseURL string `json:"base_url"`
	Kind    Kind   `json:"kind"`

	// EmbedModel and ChatModel are empty when the server has nothing suitable.
	// The two capabilities are independent: a machine may have an embedding
	// model and no chat model, or the reverse, and each feature degrades on its
	// own.
	EmbedModel string `json:"embed_model,omitempty"`
	ChatModel  string `json:"chat_model,omitempty"`

	client *http.Client
}

// CanEmbed and CanLabel report the capabilities actually present.
func (p *Provider) CanEmbed() bool { return p != nil && p.EmbedModel != "" }
func (p *Provider) CanLabel() bool { return p != nil && p.ChatModel != "" }

// Candidates are probed in order. OLLAMA_HOST first so an explicit setting
// wins, then the default ports for Ollama, LM Studio, and llama.cpp.
func candidates() []string {
	var out []string
	if h := strings.TrimSpace(os.Getenv("OLLAMA_HOST")); h != "" {
		if !strings.HasPrefix(h, "http") {
			h = "http://" + h
		}
		out = append(out, strings.TrimRight(h, "/"))
	}
	if u := strings.TrimSpace(os.Getenv("FATHOM_LOCAL_MODEL_URL")); u != "" {
		out = append(out, strings.TrimRight(u, "/"))
	}
	return append(out,
		"http://localhost:11434", // Ollama
		"http://localhost:1234",  // LM Studio
		"http://localhost:8080",  // llama.cpp server
	)
}

// probeTimeout is short on purpose. Discovery runs at startup on the path to
// serving requests, and a machine with nothing listening must not pay for it.
const probeTimeout = 900 * time.Millisecond

// Discover finds a usable local model server, or returns nil.
//
// A nil provider is the normal result on a machine with nothing installed and
// is not an error: it means the insights tab behaves exactly as it does without
// this package.
func Discover(ctx context.Context) *Provider {
	client := &http.Client{Timeout: probeTimeout}
	for _, base := range candidates() {
		if p := probe(ctx, client, base); p != nil {
			p.client = &http.Client{Timeout: 30 * time.Second}
			return p
		}
	}
	return nil
}

func probe(ctx context.Context, client *http.Client, base string) *Provider {
	if models := listOllama(ctx, client, base); len(models) > 0 {
		p := &Provider{BaseURL: base, Kind: KindOllama}
		p.EmbedModel, p.ChatModel = pickModels(models)
		return p
	}
	if models := listOpenAI(ctx, client, base); len(models) > 0 {
		p := &Provider{BaseURL: base, Kind: KindOpenAI}
		p.EmbedModel, p.ChatModel = pickModels(models)
		return p
	}
	return nil
}

func listOllama(ctx context.Context, client *http.Client, base string) []string {
	var body struct {
		Models []struct {
			Name  string `json:"name"`
			Model string `json:"model"`
		} `json:"models"`
	}
	if err := getJSON(ctx, client, base+"/api/tags", &body); err != nil {
		return nil
	}
	var out []string
	for _, m := range body.Models {
		if m.Name != "" {
			out = append(out, m.Name)
			continue
		}
		if m.Model != "" {
			out = append(out, m.Model)
		}
	}
	return out
}

func listOpenAI(ctx context.Context, client *http.Client, base string) []string {
	var body struct {
		Data []struct {
			ID string `json:"id"`
		} `json:"data"`
	}
	if err := getJSON(ctx, client, base+"/v1/models", &body); err != nil {
		return nil
	}
	var out []string
	for _, m := range body.Data {
		if m.ID != "" {
			out = append(out, m.ID)
		}
	}
	return out
}

// embedHints and chatHints pick a model by name.
//
// Naming is the only signal available without downloading anything, and it is a
// reliable one in practice: embedding models say so. A wrong guess degrades to
// the heuristic path rather than producing bad output, because a model that
// cannot embed returns an error and the caller falls back.
var embedHints = []string{"embed", "minilm", "bge", "gte", "e5-"}

// Small chat models are preferred: the job is writing a six-word label, and a
// 70B model would spend a minute on it for no gain.
var chatHints = []string{
	"qwen2.5:1.5b", "qwen2.5:3b", "llama3.2:1b", "llama3.2:3b",
	"phi3", "phi4-mini", "gemma2:2b", "gemma3:1b", "smollm",
	"qwen", "llama3", "gemma", "mistral", "phi",
}

func pickModels(models []string) (embed, chat string) {
	for _, m := range models {
		l := strings.ToLower(m)
		for _, h := range embedHints {
			if strings.Contains(l, h) {
				embed = m
				break
			}
		}
		if embed != "" {
			break
		}
	}

	for _, hint := range chatHints {
		for _, m := range models {
			l := strings.ToLower(m)
			if isEmbedName(l) {
				continue
			}
			if strings.Contains(l, hint) {
				return embed, m
			}
		}
	}
	// No recognised name: take the first non-embedding model.
	for _, m := range models {
		if !isEmbedName(strings.ToLower(m)) {
			return embed, m
		}
	}
	return embed, ""
}

func isEmbedName(l string) bool {
	for _, h := range embedHints {
		if strings.Contains(l, h) {
			return true
		}
	}
	return false
}

// Embed returns one vector per input.
//
// A provider that cannot embed, or that fails, returns an error rather than
// partial results — the caller clusters with MinHash instead, and half a set of
// vectors would produce a worse answer than either method alone.
func (p *Provider) Embed(ctx context.Context, texts []string) ([][]float32, error) {
	if !p.CanEmbed() {
		return nil, fmt.Errorf("no embedding model available")
	}
	if len(texts) == 0 {
		return nil, nil
	}
	if p.Kind == KindOllama {
		return p.embedOllama(ctx, texts)
	}
	return p.embedOpenAI(ctx, texts)
}

func (p *Provider) embedOllama(ctx context.Context, texts []string) ([][]float32, error) {
	// Newer Ollama takes a batch on /api/embed; older builds take one prompt at
	// a time on /api/embeddings. Try the batch endpoint and fall back.
	var batch struct {
		Embeddings [][]float32 `json:"embeddings"`
	}
	err := postJSON(ctx, p.client, p.BaseURL+"/api/embed", map[string]any{
		"model": p.EmbedModel,
		"input": texts,
	}, &batch)
	if err == nil && len(batch.Embeddings) == len(texts) {
		return batch.Embeddings, nil
	}

	out := make([][]float32, 0, len(texts))
	for _, t := range texts {
		var one struct {
			Embedding []float32 `json:"embedding"`
		}
		if err := postJSON(ctx, p.client, p.BaseURL+"/api/embeddings", map[string]any{
			"model":  p.EmbedModel,
			"prompt": t,
		}, &one); err != nil {
			return nil, err
		}
		if len(one.Embedding) == 0 {
			return nil, fmt.Errorf("embedding model returned an empty vector")
		}
		out = append(out, one.Embedding)
	}
	return out, nil
}

func (p *Provider) embedOpenAI(ctx context.Context, texts []string) ([][]float32, error) {
	var body struct {
		Data []struct {
			Embedding []float32 `json:"embedding"`
		} `json:"data"`
	}
	if err := postJSON(ctx, p.client, p.BaseURL+"/v1/embeddings", map[string]any{
		"model": p.EmbedModel,
		"input": texts,
	}, &body); err != nil {
		return nil, err
	}
	if len(body.Data) != len(texts) {
		return nil, fmt.Errorf("asked for %d embeddings, got %d", len(texts), len(body.Data))
	}
	out := make([][]float32, len(body.Data))
	for i, d := range body.Data {
		out[i] = d.Embedding
	}
	return out, nil
}

// labelPrompt asks for a short noun phrase and nothing else.
//
// The instruction is blunt about format because small models narrate by
// default, and a label that arrives as "Sure! Here's a summary:..." is worse
// than the heuristic title it would replace. The caller validates the result
// anyway and discards anything that does not look like a label.
const labelPrompt = `Below are prompts a developer gave an AI coding agent. They are all variations of the same request.

Reply with a short noun phrase, at most eight words, naming what they have in common. No preamble, no quotes, no trailing period. Reply with the phrase only.

Prompts:
%s`

// Label names a cluster of related prompts.
func (p *Provider) Label(ctx context.Context, prompts []string) (string, error) {
	if !p.CanLabel() {
		return "", fmt.Errorf("no chat model available")
	}
	if len(prompts) == 0 {
		return "", fmt.Errorf("nothing to label")
	}
	if len(prompts) > 6 {
		prompts = prompts[:6]
	}
	var b strings.Builder
	for _, s := range prompts {
		s = strings.TrimSpace(strings.ReplaceAll(s, "\n", " "))
		if len(s) > 200 {
			s = s[:200]
		}
		fmt.Fprintf(&b, "- %s\n", s)
	}
	prompt := fmt.Sprintf(labelPrompt, b.String())

	if p.Kind == KindOllama {
		var body struct {
			Response string `json:"response"`
		}
		if err := postJSON(ctx, p.client, p.BaseURL+"/api/generate", map[string]any{
			"model":   p.ChatModel,
			"prompt":  prompt,
			"stream":  false,
			"options": map[string]any{"temperature": 0.1, "num_predict": 32},
		}, &body); err != nil {
			return "", err
		}
		return cleanLabel(body.Response), nil
	}

	var body struct {
		Choices []struct {
			Message struct {
				Content string `json:"content"`
			} `json:"message"`
		} `json:"choices"`
	}
	if err := postJSON(ctx, p.client, p.BaseURL+"/v1/chat/completions", map[string]any{
		"model":       p.ChatModel,
		"messages":    []map[string]string{{"role": "user", "content": prompt}},
		"temperature": 0.1,
		"max_tokens":  32,
	}, &body); err != nil {
		return "", err
	}
	if len(body.Choices) == 0 {
		return "", fmt.Errorf("model returned no choices")
	}
	return cleanLabel(body.Choices[0].Message.Content), nil
}

// cleanLabel strips the scaffolding small models add, and rejects anything that
// does not look like a label.
//
// Returning an empty string is a valid outcome and means the caller keeps its
// own title. That is the safe direction: a missing label costs a little
// polish, whereas a rambling one would be shown to the reader as fact.
func cleanLabel(s string) string {
	s = strings.TrimSpace(s)

	// Some models emit a reasoning block before the answer.
	if i := strings.LastIndex(s, "</think>"); i >= 0 {
		s = strings.TrimSpace(s[i+len("</think>"):])
	}
	// Take the last non-empty line, which is where the answer lands when a
	// model preambles.
	lines := strings.Split(s, "\n")
	for i := len(lines) - 1; i >= 0; i-- {
		if t := strings.TrimSpace(lines[i]); t != "" {
			s = t
			break
		}
	}

	s = strings.Trim(s, ` "'*.`)
	s = strings.TrimPrefix(s, "- ")
	// A colon usually separates preamble from answer: "Common theme: flaky tests".
	if i := strings.LastIndex(s, ":"); i >= 0 && i < len(s)-1 {
		s = strings.TrimSpace(s[i+1:])
	}
	s = strings.Trim(s, ` "'*.`)

	if s == "" {
		return ""
	}
	// Reject anything sentence-shaped. A label is a phrase; prose means the
	// model ignored the instruction and the fallback title is better.
	if words := strings.Fields(s); len(words) > 10 {
		return ""
	}
	if strings.ContainsAny(s, "\n") {
		return ""
	}
	return s
}

// --- HTTP plumbing ---

func getJSON(ctx context.Context, client *http.Client, url string, out any) error {
	req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil)
	if err != nil {
		return err
	}
	res, err := client.Do(req)
	if err != nil {
		return err
	}
	defer res.Body.Close()
	if res.StatusCode != http.StatusOK {
		return fmt.Errorf("%s: %s", url, res.Status)
	}
	return json.NewDecoder(res.Body).Decode(out)
}

func postJSON(ctx context.Context, client *http.Client, url string, in, out any) error {
	b, err := json.Marshal(in)
	if err != nil {
		return err
	}
	req, err := http.NewRequestWithContext(ctx, http.MethodPost, url, bytes.NewReader(b))
	if err != nil {
		return err
	}
	req.Header.Set("Content-Type", "application/json")
	res, err := client.Do(req)
	if err != nil {
		return err
	}
	defer res.Body.Close()
	if res.StatusCode != http.StatusOK {
		return fmt.Errorf("%s: %s", url, res.Status)
	}
	return json.NewDecoder(res.Body).Decode(out)
}

// worthPrompt asks whether a finding justifies interrupting someone.
//
// The model is given the finding's own words and numbers and nothing else: no
// traces, no prompts, no file contents. It is answering a question about
// salience, not analysing anyone's work.
//
// The framing is deliberately sceptical. A model asked "is this useful?" says
// yes to almost anything, and the failure mode being guarded against is a
// notification nobody wanted.
const worthPrompt = `A tool watches a developer's AI coding sessions and has found this pattern:

%s
%s
(seen %d times, about $%.2f)

Should the developer be interrupted with a one-line notification about this at the end of their current task? Interrupt only if acting on it would plausibly change how they work. Do not interrupt for something trivial, obvious, or too small to be worth the distraction.

Answer with exactly one word, YES or NO, then a dash and at most ten words of reason.`

// Worth implements the insights.Judge interface.
func (p *Provider) Worth(ctx context.Context, title, body string, occurrences int, costUSD float64) (bool, string, error) {
	if !p.CanLabel() {
		return false, "", fmt.Errorf("no chat model available")
	}
	prompt := fmt.Sprintf(worthPrompt, title, body, occurrences, costUSD)

	raw, err := p.complete(ctx, prompt, 48)
	if err != nil {
		return false, "", err
	}
	return parseVerdict(raw)
}

// parseVerdict reads the model's answer.
//
// An unparseable answer is an error rather than a NO, so that the caller's
// failure path — keep the finding — applies. A model that rambles should not be
// able to silence something that cleared every numeric bar.
func parseVerdict(raw string) (bool, string, error) {
	s := strings.TrimSpace(raw)
	if i := strings.LastIndex(s, "</think>"); i >= 0 {
		s = strings.TrimSpace(s[i+len("</think>"):])
	}
	upper := strings.ToUpper(s)

	yes := strings.HasPrefix(upper, "YES")
	no := strings.HasPrefix(upper, "NO")
	if !yes && !no {
		// Fall back to whichever word appears first anywhere in the reply.
		iy, in := strings.Index(upper, "YES"), strings.Index(upper, "NO")
		switch {
		case iy >= 0 && (in < 0 || iy < in):
			yes = true
		case in >= 0:
			no = true
		default:
			return false, "", fmt.Errorf("could not read a verdict from %q", raw)
		}
	}

	reason := s
	if i := strings.IndexAny(s, "-–—:"); i >= 0 && i+1 < len(s) {
		reason = strings.TrimSpace(s[i+1:])
	}
	if len(reason) > 120 {
		reason = reason[:120]
	}
	return yes, reason, nil
}

// complete is the shared one-shot completion path for both wire formats.
func (p *Provider) complete(ctx context.Context, prompt string, maxTokens int) (string, error) {
	if p.Kind == KindOllama {
		var body struct {
			Response string `json:"response"`
		}
		if err := postJSON(ctx, p.client, p.BaseURL+"/api/generate", map[string]any{
			"model":   p.ChatModel,
			"prompt":  prompt,
			"stream":  false,
			"options": map[string]any{"temperature": 0, "num_predict": maxTokens},
		}, &body); err != nil {
			return "", err
		}
		return body.Response, nil
	}

	var body struct {
		Choices []struct {
			Message struct {
				Content string `json:"content"`
			} `json:"message"`
		} `json:"choices"`
	}
	if err := postJSON(ctx, p.client, p.BaseURL+"/v1/chat/completions", map[string]any{
		"model":       p.ChatModel,
		"messages":    []map[string]string{{"role": "user", "content": prompt}},
		"temperature": 0,
		"max_tokens":  maxTokens,
	}, &body); err != nil {
		return "", err
	}
	if len(body.Choices) == 0 {
		return "", fmt.Errorf("model returned no choices")
	}
	return body.Choices[0].Message.Content, nil
}