wanderer/db/util/safe_fetch.go

package util

import (
	"context"
	"crypto/tls"
	"crypto/x509"
	"fmt"
	"io"
	"net"
	"net/http"
	"net/netip"
	"net/url"
	"time"

	"github.com/doyensec/safeurl"
)

const (
	DefaultPluginMediaMaxBytes       int64 = 50 << 20
	DefaultPluginMaxImportMediaItems       = 20
	DefaultPluginMaxImportMediaBytes int64 = 200 << 20
)

type SafeFetchResult struct {
	Body        []byte
	ContentType string
	FinalURL    string
}

type ConnectorHTTPPolicy struct {
	BaseURL      string
	AllowPrivate bool
	TLSMode      string
	TLSCABundle  []byte
}

func FetchPublicURL(ctx context.Context, rawURL string, maxBytes int64) (*SafeFetchResult, error) {
	if maxBytes <= 0 {
		maxBytes = DefaultPluginMediaMaxBytes
	}
	parsed, err := url.Parse(rawURL)
	if err != nil || parsed.Scheme == "" || parsed.Host == "" {
		return nil, fmt.Errorf("invalid public URL")
	}
	if parsed.User != nil {
		return nil, fmt.Errorf("public URL must not include credentials")
	}
	config := safeurl.GetConfigBuilder().
		SetTimeout(60*time.Second).
		SetAllowedSchemes("http", "https").
		SetAllowedPorts(80, 443).
		EnableIPv6(true).
		AllowSendingCredentials(false).
		SetCheckRedirect(publicMediaRedirectPolicy).
		Build()
	client := safeurl.Client(config)

	req, err := http.NewRequestWithContext(ctx, http.MethodGet, rawURL, nil)
	if err != nil {
		return nil, err
	}
	resp, err := client.Do(req)
	if err != nil {
		return nil, err
	}
	defer resp.Body.Close()

	body, err := ReadBoundedForPlugin(resp.Body, maxBytes)
	if err != nil {
		return nil, err
	}
	return &SafeFetchResult{
		Body:        body,
		ContentType: resp.Header.Get("Content-Type"),
		FinalURL:    resp.Request.URL.String(),
	}, nil
}

func publicMediaRedirectPolicy(req *http.Request, via []*http.Request) error {
	if len(via) >= 10 {
		return fmt.Errorf("too many redirects")
	}
	if req.URL.User != nil {
		return fmt.Errorf("redirect URL must not include credentials")
	}
	if req.URL.Scheme != "http" && req.URL.Scheme != "https" {
		return fmt.Errorf("redirect scheme must be http or https")
	}
	if len(via) > 0 && via[len(via)-1].URL.Scheme == "https" && req.URL.Scheme == "http" {
		return fmt.Errorf("redirect downgrades https to http")
	}
	return nil
}

func ConnectorHTTPClient(policy ConnectorHTTPPolicy, checkRedirect func(req *http.Request, via []*http.Request) error) (*http.Client, error) {
	base, err := url.Parse(policy.BaseURL)
	if err != nil || base.Scheme == "" || base.Host == "" {
		return nil, fmt.Errorf("invalid connector baseURL")
	}
	tlsConfig, err := connectorTLSConfig(policy.TLSMode, policy.TLSCABundle)
	if err != nil {
		return nil, err
	}
	dialer := &net.Dialer{Timeout: 30 * time.Second}
	transport := &http.Transport{
		TLSClientConfig: tlsConfig,
		DialContext: func(ctx context.Context, network string, addr string) (net.Conn, error) {
			host, port, err := net.SplitHostPort(addr)
			if err != nil {
				return nil, err
			}
			ips, err := net.DefaultResolver.LookupIP(ctx, "ip", host)
			if err != nil || len(ips) == 0 {
				return nil, fmt.Errorf("failed to resolve connector host: %w", err)
			}
			var selected net.IP
			for _, ip := range ips {
				if connectorIPAllowed(ip, policy.AllowPrivate) {
					selected = ip
					break
				}
			}
			if selected == nil {
				return nil, fmt.Errorf("connector host resolved outside allowed IP policy")
			}
			return dialer.DialContext(ctx, network, net.JoinHostPort(selected.String(), port))
		},
	}
	return &http.Client{
		Timeout:       60 * time.Second,
		Transport:     transport,
		CheckRedirect: checkRedirect,
	}, nil
}

func connectorTLSConfig(mode string, caBundle []byte) (*tls.Config, error) {
	switch mode {
	case "", "system":
		return nil, nil
	case "customCA":
		roots, err := x509.SystemCertPool()
		if err != nil || roots == nil {
			roots = x509.NewCertPool()
		}
		if len(caBundle) == 0 || !roots.AppendCertsFromPEM(caBundle) {
			return nil, fmt.Errorf("connector customCA bundle is invalid")
		}
		return &tls.Config{RootCAs: roots}, nil
	default:
		return nil, fmt.Errorf("unsupported connector TLS mode %q", mode)
	}
}

func connectorIPAllowed(ip net.IP, allowPrivate bool) bool {
	addr, ok := netip.AddrFromSlice(ip)
	if !ok {
		return false
	}
	if addr.Is4In6() {
		addr = addr.Unmap()
	}
	if addr.IsLoopback() || addr.IsLinkLocalUnicast() || addr.IsLinkLocalMulticast() ||
		addr.IsMulticast() || addr.IsUnspecified() {
		return false
	}
	if isSpecialPurposeIP(addr) {
		return false
	}
	if addr.IsPrivate() {
		return allowPrivate
	}
	return true
}

func isSpecialPurposeIP(addr netip.Addr) bool {
	for _, prefix := range specialPurposePrefixes {
		if prefix.Contains(addr) {
			return true
		}
	}
	return false
}

var specialPurposePrefixes = mustPrefixes(
	"0.0.0.0/8",
	"100.64.0.0/10",
	"127.0.0.0/8",
	"169.254.0.0/16",
	"192.0.0.0/24",
	"192.0.2.0/24",
	"198.18.0.0/15",
	"198.51.100.0/24",
	"203.0.113.0/24",
	"224.0.0.0/4",
	"240.0.0.0/4",
	"::/128",
	"::1/128",
	"64:ff9b::/96",
	"100::/64",
	"2001:db8::/32",
	"fe80::/10",
	"ff00::/8",
)

func mustPrefixes(values ...string) []netip.Prefix {
	prefixes := make([]netip.Prefix, 0, len(values))
	for _, value := range values {
		prefix, err := netip.ParsePrefix(value)
		if err != nil {
			panic(err)
		}
		prefixes = append(prefixes, prefix)
	}
	return prefixes
}

func ReadBoundedForPlugin(reader io.Reader, maxBytes int64) ([]byte, error) {
	body, err := io.ReadAll(io.LimitReader(reader, maxBytes+1))
	if err != nil {
		return nil, err
	}
	if int64(len(body)) > maxBytes {
		return nil, fmt.Errorf("response exceeds maximum size")
	}
	return body, nil
}