Files
archie/internal/tools/webfetch/webfetch.go
T
2026-08-17 00:02:31 +10:00

354 lines
11 KiB
Go

// Package webfetch retrieves a URL on the model's behalf and returns readable
// text.
//
// The interesting part is not the fetch, it is the refusals. archied runs on a
// host where the Docker API, NATS and its own dashboard answer on private
// addresses, and the URL it is asked to fetch arrives in chat -- an untrusted
// path, since a forge issue body or an inbound email can end up quoted into a
// turn. So the address is vetted at dial time rather than by inspecting the
// URL: a name that looks public can resolve to a private address, and only the
// resolved address tells the truth.
package webfetch
import (
"context"
"errors"
"fmt"
"io"
"net"
"net/http"
"net/url"
"strings"
"time"
"golang.org/x/net/html"
)
const (
// maxRedirects bounds a redirect chain. Each hop is re-vetted by the
// dialer, so this only stops a loop from spending the timeout.
maxRedirects = 5
// statusExcerpt is how much of a failed response body is quoted back.
// Enough to tell an API error message from an HTML error page.
statusExcerpt = 200
)
// Config controls the fetcher. The zero value is disabled.
type Config struct {
// Enabled advertises the tool. When false, Tool returns nil so the
// model is never shown a capability the operator turned off.
Enabled bool
// Timeout bounds one fetch, redirects included.
Timeout time.Duration
// MaxBytes bounds how much of a response body is read.
MaxBytes int64
// AllowPrivateNetworks permits loopback, private, link-local and
// carrier-grade-NAT addresses. Off by default: leaving it off is what
// keeps the daemon's own control surfaces out of reach.
AllowPrivateNetworks bool
}
// Client fetches URLs under a Config.
type Client struct {
cfg Config
http *http.Client
}
// New builds a Client whose transport refuses disallowed addresses at dial
// time.
func New(cfg Config) *Client {
c := &Client{cfg: cfg}
transport := &http.Transport{
DialContext: c.dialContext,
TLSHandshakeTimeout: cfg.Timeout,
ResponseHeaderTimeout: cfg.Timeout,
DisableKeepAlives: true,
}
c.http = &http.Client{
Transport: transport,
Timeout: cfg.Timeout,
CheckRedirect: func(req *http.Request, via []*http.Request) error {
if len(via) >= maxRedirects {
return fmt.Errorf("stopped after %d redirects", maxRedirects)
}
return checkScheme(req.URL)
},
}
return c
}
// Fetch retrieves rawURL and returns its readable content. When raw is true
// the body is returned as received rather than reduced to text.
func (c *Client) Fetch(ctx context.Context, rawURL string, raw bool) (string, error) {
parsed, err := url.Parse(strings.TrimSpace(rawURL))
if err != nil {
return "", fmt.Errorf("invalid url %q: %w", rawURL, err)
}
if err := checkScheme(parsed); err != nil {
return "", err
}
req, err := http.NewRequestWithContext(ctx, http.MethodGet, parsed.String(), nil)
if err != nil {
return "", fmt.Errorf("build request: %w", err)
}
// Some sites serve a different, poorer page to an unidentified client,
// and a few refuse one outright.
req.Header.Set("User-Agent", "archie-webfetch/1.0 (+https://github.com/samcharles93/archie-core)")
req.Header.Set("Accept", "text/html,application/xhtml+xml,text/plain,application/json;q=0.9,*/*;q=0.1")
resp, err := c.http.Do(req)
if err != nil {
return "", unwrapFetchError(err)
}
defer resp.Body.Close()
body, truncated, err := c.readBody(resp)
if err != nil {
return "", err
}
if resp.StatusCode < 200 || resp.StatusCode >= 300 {
return "", fmt.Errorf("%s returned %d %s: %s",
parsed.Redacted(), resp.StatusCode, http.StatusText(resp.StatusCode), excerpt(body))
}
mediaType := contentType(resp)
rendered, err := render(mediaType, body, raw, len(body))
if err != nil {
return "", err
}
if truncated {
rendered += fmt.Sprintf("\n\n[webfetch: response truncated at %d bytes]", c.cfg.MaxBytes)
}
return rendered, nil
}
// readBody reads at most MaxBytes, reporting whether more was available.
func (c *Client) readBody(resp *http.Response) (string, bool, error) {
limit := c.cfg.MaxBytes
if limit <= 0 {
limit = 1 << 20
}
// One byte past the limit distinguishes "exactly at the limit" from
// "there was more".
data, err := io.ReadAll(io.LimitReader(resp.Body, limit+1))
if err != nil {
return "", false, fmt.Errorf("read response: %w", err)
}
if int64(len(data)) > limit {
return string(data[:limit]), true, nil
}
return string(data), false, nil
}
// render turns a body into what the model sees, refusing anything that is not
// text. Returning binary would spend the context window on bytes the model
// cannot read.
func render(mediaType, body string, raw bool, size int) (string, error) {
switch {
case raw:
return body, nil
case mediaType == "text/html", mediaType == "application/xhtml+xml":
return htmlToText(body), nil
case strings.HasPrefix(mediaType, "text/"),
mediaType == "application/json",
mediaType == "application/xml",
strings.HasSuffix(mediaType, "+json"),
strings.HasSuffix(mediaType, "+xml"):
return body, nil
default:
return "", fmt.Errorf("refusing %s (%d bytes): only text, html, json and xml are readable", mediaType, size)
}
}
// contentType returns the media type without parameters, defaulting to plain
// text when the server sends nothing.
func contentType(resp *http.Response) string {
raw := resp.Header.Get("Content-Type")
if raw == "" {
return "text/plain"
}
if i := strings.IndexByte(raw, ';'); i >= 0 {
raw = raw[:i]
}
return strings.ToLower(strings.TrimSpace(raw))
}
// excerpt quotes the start of a failed response so the model can tell an API
// error message from a generic error page.
func excerpt(body string) string {
body = strings.TrimSpace(body)
if len(body) <= statusExcerpt {
return body
}
return body[:statusExcerpt] + "..."
}
// checkScheme allows only http and https. Anything else -- file:, data:,
// gopher: -- reaches something that is not a web server.
func checkScheme(u *url.URL) error {
switch u.Scheme {
case "http", "https":
return nil
case "":
return fmt.Errorf("url %q has no scheme: give an absolute http:// or https:// url", u.String())
default:
return fmt.Errorf("refusing scheme %q: only http and https are fetched", u.Scheme)
}
}
// unwrapFetchError strips the url.Error wrapper so the refusal the dialer or
// the redirect check produced is what the model reads, rather than being
// buried behind Go's transport prose.
func unwrapFetchError(err error) error {
var urlErr *url.Error
if errors.As(err, &urlErr) && urlErr.Err != nil {
return urlErr.Err
}
return err
}
// dialContext resolves the host and refuses addresses the daemon must not be
// talked into reaching, then dials a vetted address directly.
//
// Vetting here rather than on the URL is the whole point: between a pre-flight
// check and the connection, a name can resolve to something else. Dialing the
// address that was actually approved closes that window.
func (c *Client) dialContext(ctx context.Context, network, addr string) (net.Conn, error) {
host, port, err := net.SplitHostPort(addr)
if err != nil {
return nil, fmt.Errorf("invalid address %q: %w", addr, err)
}
ips, err := net.DefaultResolver.LookupIPAddr(ctx, host)
if err != nil {
return nil, err
}
dialer := &net.Dialer{Timeout: c.cfg.Timeout}
var refused []string
for _, ip := range ips {
if !c.addressAllowed(ip.IP) {
refused = append(refused, ip.IP.String())
continue
}
conn, err := dialer.DialContext(ctx, network, net.JoinHostPort(ip.IP.String(), port))
if err == nil {
return conn, nil
}
}
if len(refused) > 0 {
return nil, fmt.Errorf(
"refusing to connect to %s: it resolves to the private or loopback address %s, which is where this host's own services live; set tools.web_fetch.allow_private_networks to permit it",
host, strings.Join(refused, ", "),
)
}
return nil, fmt.Errorf("could not connect to %s", host)
}
// addressAllowed reports whether an address may be reached.
func (c *Client) addressAllowed(ip net.IP) bool {
if c.cfg.AllowPrivateNetworks {
return true
}
switch {
case ip.IsLoopback(), ip.IsPrivate(), ip.IsUnspecified(),
ip.IsLinkLocalUnicast(), ip.IsLinkLocalMulticast(),
ip.IsInterfaceLocalMulticast(), ip.IsMulticast():
return false
case isCarrierGradeNAT(ip):
return false
default:
return true
}
}
// isCarrierGradeNAT reports whether ip is in 100.64.0.0/10, which net.IP has
// no predicate for and which routes to infrastructure rather than the public
// internet.
func isCarrierGradeNAT(ip net.IP) bool {
v4 := ip.To4()
return v4 != nil && v4[0] == 100 && v4[1] >= 64 && v4[1] <= 127
}
// blockedElements hold text that is markup machinery rather than content.
var blockedElements = map[string]bool{
"script": true, "style": true, "noscript": true,
"head": true, "template": true, "svg": true,
}
// htmlToText reduces a document to the text a reader would see. It is
// deliberately simple: the goal is to spend the context window on prose
// instead of markup, not to reproduce a browser.
func htmlToText(body string) string {
doc, err := html.Parse(strings.NewReader(body))
if err != nil {
// Unparseable markup still has readable text in it; handing back
// the raw document beats returning nothing.
return body
}
var b strings.Builder
var walk func(*html.Node)
walk = func(n *html.Node) {
if n.Type == html.ElementNode && blockedElements[n.Data] {
return
}
if n.Type == html.TextNode {
if text := strings.TrimSpace(n.Data); text != "" {
b.WriteString(text)
b.WriteByte(' ')
}
}
for child := n.FirstChild; child != nil; child = child.NextSibling {
walk(child)
}
// Block-level elements end a line so headings and paragraphs do not
// run together into one wall of text.
if n.Type == html.ElementNode && breaksLine(n.Data) {
b.WriteByte('\n')
}
}
walk(doc)
return collapseBlankLines(b.String())
}
// breaksLine reports whether an element should end the current line.
func breaksLine(tag string) bool {
switch tag {
case "p", "div", "br", "li", "tr", "section", "article", "header", "footer",
"h1", "h2", "h3", "h4", "h5", "h6", "blockquote", "pre":
return true
default:
return false
}
}
// collapseBlankLines trims each line and squeezes runs of blank lines, which
// nested markup produces in quantity.
func collapseBlankLines(s string) string {
lines := strings.Split(s, "\n")
out := make([]string, 0, len(lines))
blank := false
for _, line := range lines {
line = strings.Join(strings.Fields(line), " ")
if line == "" {
if blank {
continue
}
blank = true
} else {
blank = false
}
out = append(out, line)
}
return strings.TrimSpace(strings.Join(out, "\n"))
}