Refuse feed connections to non-public addresses

docs/feed-backed-news.md

@@ -1,6 +1,6 @@
 # Feed-backed threaded news
 
-Mobius can import a public RSS or Atom feed into an existing Hotline news
+Mobius can import an RSS or Atom feed into an existing Hotline news
 category. This is useful for project announcements, Sparkle appcasts, and
 GitHub release feeds.
 
@@ -28,8 +28,18 @@ NewsFeeds:
 ```
 
 `CategoryPath` is the complete path from the root of threaded news to the
-existing category. `URL` must be an absolute public `http` or `https` URL and
+existing category. `URL` must be an absolute `http` or `https` URL and
 cannot contain embedded credentials. Only one feed may map to a category.
+A feed on the local network is fine: when the configured URL names a loopback,
+private, link-local or carrier-grade NAT address, or a host that resolves only
+to such addresses, Mobius fetches it as configured, connecting to the
+addresses that lookup gave. A public feed is held to
+public addresses on every connection, so a redirect it serves cannot reach the
+server's local network. No redirect from `https` to `http` is followed. A proxy
+set with `HTTP_PROXY` or `HTTPS_PROXY` may be on a private address. Through a
+proxy, Mobius cannot see what a host name resolves to, so for a public feed it
+refuses only a URL that names a non-public address directly; the proxy's own
+rules decide the rest.
 
 Mobius detects RSS and Atom automatically. Sparkle appcasts are RSS; GitHub's
 `releases.atom` endpoints are Atom. JSON Feed and provider-specific APIs are not
@@ -120,7 +130,7 @@ rewrite history.
   news category rather than a bundle. Also check for duplicate paths and
   invalid or credential-bearing URLs.
 - If new entries do not appear, open the category and inspect the server log.
-  Confirm that the URL is public RSS or Atom and that entries have stable IDs,
+  Confirm that the URL is RSS or Atom and that entries have stable IDs,
   links, or enclosures.
 - If old releases are missing on the first import, inspect the source feed. Its
   published window is the initial cutoff; Mobius does not use pagination or a

internal/mobius/feed_news_fetch.go

@@ -7,11 +7,15 @@ import (
 	"fmt"
 	stdhtml "html"
 	"io"
+	"net"
 	"net/http"
+	"net/netip"
 	"net/url"
 	"slices"
 	"sort"
 	"strings"
+	"sync"
+	"syscall"
 	"time"
 	"unicode/utf8"
 
@@ -27,7 +31,71 @@ const (
 	maxNewsBodyBytes   = 65535
 )
 
-var errFeedResponseTooLarge = errors.New("feed response exceeds 2 MiB")
+var (
+	errFeedResponseTooLarge = errors.New("feed response exceeds 2 MiB")
+	errFeedAddressNotPublic = errors.New("feed address is not public")
+)
+
+// sharedAddressSpace is carrier-grade NAT space (RFC 6598), which netip does not count as private.
+var sharedAddressSpace = netip.MustParsePrefix("100.64.0.0/10")
+
+func isPublicAddr(ip netip.Addr) bool {
+	ip = ip.Unmap()
+	return ip.IsGlobalUnicast() && !ip.IsPrivate() && !sharedAddressSpace.Contains(ip)
+}
+
+// privateFeed marks a fetch whose configured URL is itself on a non-public address. The operator
+// chose that, so nothing it connects to is refused. For a host name, addrs are the answers that
+// decided it, and connections to that host go to them: a second lookup could answer otherwise, and
+// a name server answering loopback to the check and a public address to the connection would have
+// its redirects followed unchecked.
+type privateFeed struct {
+	host  string
+	addrs []netip.Addr
+}
+
+type privateFeedKey struct{}
+
+// privateFeedFor reports whether the configured URL names a non-public address, or a host whose
+// every address is one. Behind a proxy, which resolves names where this cannot see, only an address
+// makes a feed private, and a host that does not resolve here counts as public.
+func (f *feedFetcher) privateFeedFor(ctx context.Context, u *url.URL, proxied bool) *privateFeed {
+	if ip, err := netip.ParseAddr(u.Hostname()); err == nil {
+		if isPublicAddr(ip) {
+			return nil
+		}
+		return &privateFeed{}
+	}
+	if proxied {
+		return nil
+	}
+	addrs, err := f.lookup(ctx, u.Hostname())
+	if err != nil || len(addrs) == 0 || slices.ContainsFunc(addrs, isPublicAddr) {
+		return nil
+	}
+	return &privateFeed{host: u.Hostname(), addrs: addrs}
+}
+
+// refuseNonPublicDial is checked against the resolved address of every direct connection a public
+// feed's fetch makes, so a redirect it serves cannot point the server at loopback, the local
+// network, or a cloud metadata endpoint.
+func refuseNonPublicDial(ctx context.Context, _, address string, _ syscall.RawConn) error {
+	if ctx.Value(privateFeedKey{}) != nil {
+		return nil
+	}
+	host, _, err := net.SplitHostPort(address)
+	if err != nil {
+		return err
+	}
+	ip, err := netip.ParseAddr(host)
+	if err != nil {
+		return err
+	}
+	if !isPublicAddr(ip) {
+		return fmt.Errorf("%w: %s", errFeedAddressNotPublic, ip)
+	}
+	return nil
+}
 
 type normalizedFeedItem struct {
 	identity    string
@@ -54,21 +122,62 @@ type feedFetcher struct {
 	parser    *gofeed.Parser
 	now       func() time.Time
 	userAgent string
+	proxy     func(*http.Request) (*url.URL, error)
+	lookup    func(ctx context.Context, host string) ([]netip.Addr, error)
 }
 
 func newFeedFetcher(userAgent string) *feedFetcher {
+	return newFeedFetcherVia(userAgent, http.ProxyFromEnvironment)
+}
+
+// newFeedFetcherVia fetches through the proxies proxy chooses. A proxy is the operator's choice,
+// and is commonly on the local network, so connections to it are not checked; through it the
+// server cannot see what a host name resolves to, so for a public feed only an address written
+// into a URL is.
+func newFeedFetcherVia(userAgent string, proxy func(*http.Request) (*url.URL, error)) *feedFetcher {
 	if userAgent == "" {
 		userAgent = "mobius-hotline-server"
 	}
+	var proxies sync.Map
+	direct := &net.Dialer{Timeout: 30 * time.Second, KeepAlive: 30 * time.Second}
+	guarded := &net.Dialer{Timeout: 30 * time.Second, KeepAlive: 30 * time.Second, ControlContext: refuseNonPublicDial}
+	transport := http.DefaultTransport.(*http.Transport).Clone()
+	transport.Proxy = func(req *http.Request) (*url.URL, error) {
+		proxyURL, err := proxy(req)
+		if err != nil || proxyURL == nil {
+			return proxyURL, err
+		}
+		ip, err := netip.ParseAddr(req.URL.Hostname())
+		if err == nil && !isPublicAddr(ip) && req.Context().Value(privateFeedKey{}) == nil {
+			return nil, fmt.Errorf("%w: %s", errFeedAddressNotPublic, ip)
+		}
+		proxies.Store(proxyAddr(proxyURL), struct{}{})
+		return proxyURL, nil
+	}
+	transport.DialContext = func(ctx context.Context, network, address string) (net.Conn, error) {
+		if _, ok := proxies.Load(address); ok {
+			return direct.DialContext(ctx, network, address)
+		}
+		if pin, _ := ctx.Value(privateFeedKey{}).(*privateFeed); pin != nil && pin.addrs != nil {
+			if host, port, err := net.SplitHostPort(address); err == nil && host == pin.host {
+				return dialFirst(ctx, direct, network, pin.addrs, port)
+			}
+		}
+		return guarded.DialContext(ctx, network, address)
+	}
 	return &feedFetcher{
 		client: &http.Client{
+			Transport: transport,
 			CheckRedirect: func(req *http.Request, via []*http.Request) error {
 				if len(via) >= 10 {
 					return errors.New("stopped after 10 redirects")
 				}
 				if req.URL.Scheme != "http" && req.URL.Scheme != "https" {
 					return fmt.Errorf("redirected to unsupported URL scheme %q", req.URL.Scheme)
 				}
+				if via[len(via)-1].URL.Scheme == "https" && req.URL.Scheme == "http" {
+					return errors.New("redirected from https to http")
+				}
 				if req.URL.User != nil {
 					return errors.New("redirected to a URL containing credentials")
 				}
@@ -78,7 +187,32 @@ func newFeedFetcher(userAgent string) *feedFetcher {
 		parser:    gofeed.NewParser(),
 		now:       time.Now,
 		userAgent: userAgent,
+		proxy:     proxy,
+		lookup: func(ctx context.Context, host string) ([]netip.Addr, error) {
+			return net.DefaultResolver.LookupNetIP(ctx, "ip", host)
+		},
+	}
+}
+
+// dialFirst connects to the first of addrs that answers.
+func dialFirst(ctx context.Context, d *net.Dialer, network string, addrs []netip.Addr, port string) (net.Conn, error) {
+	var err error
+	for _, addr := range addrs {
+		var conn net.Conn
+		if conn, err = d.DialContext(ctx, network, net.JoinHostPort(addr.Unmap().String(), port)); err == nil {
+			return conn, nil
+		}
 	}
+	return nil, err
+}
+
+// proxyAddr is the address the transport dials for proxyURL.
+func proxyAddr(proxyURL *url.URL) string {
+	if port := proxyURL.Port(); port != "" {
+		return net.JoinHostPort(proxyURL.Hostname(), port)
+	}
+	port := map[string]string{"http": "80", "https": "443", "socks5": "1080", "socks5h": "1080"}[proxyURL.Scheme]
+	return net.JoinHostPort(proxyURL.Hostname(), port)
 }
 
 func (f *feedFetcher) fetch(ctx context.Context, sourceURL string, validators feedHTTPValidators) (feedFetchResult, error) {
@@ -89,6 +223,10 @@ func (f *feedFetcher) fetch(ctx context.Context, sourceURL string, validators fe
 	if err != nil {
 		return feedFetchResult{}, errors.New("create request from configured feed URL")
 	}
+	proxyURL, _ := f.proxy(req)
+	if pin := f.privateFeedFor(ctx, req.URL, proxyURL != nil); pin != nil {
+		req = req.WithContext(context.WithValue(ctx, privateFeedKey{}, pin))
+	}
 	req.Header.Set("Accept", "application/rss+xml, application/atom+xml, application/xml;q=0.9, text/xml;q=0.8")
 	req.Header.Set("User-Agent", f.userAgent)
 	if validators.etag != "" {

internal/mobius/feed_news_test.go

@@ -2,11 +2,16 @@ package mobius
 
 import (
 	"bytes"
+	"context"
 	"errors"
 	"fmt"
 	"io"
 	"log/slog"
+	"net"
 	"net/http"
+	"net/http/httptest"
+	"net/netip"
+	"net/url"
 	"os"
 	"path/filepath"
 	"strings"
@@ -425,6 +430,129 @@ func TestFeedFetcherLimitsAndConditional304(t *testing.T) {
 	})
 }
 
+func TestIsPublicAddr(t *testing.T) {
+	tests := []struct {
+		addr string
+		want bool
+	}{
+		{"93.184.215.14", true},
+		{"2606:4700::6810:84e5", true},
+		{"127.0.0.1", false},
+		{"::1", false},
+		{"10.1.2.3", false},
+		{"172.16.0.1", false},
+		{"192.168.1.1", false},
+		{"169.254.169.254", false},
+		{"100.64.0.1", false},
+		{"0.0.0.0", false},
+		{"::", false},
+		{"fd00::1", false},
+		{"fe80::1", false},
+		{"224.0.0.1", false},
+		{"::ffff:127.0.0.1", false},
+		{"::ffff:192.168.1.1", false},
+	}
+	for _, tt := range tests {
+		assert.Equal(t, tt.want, isPublicAddr(netip.MustParseAddr(tt.addr)), tt.addr)
+	}
+}
+
+// publicFeedURL is a literal public address, so whether the feed is public never depends on DNS.
+const publicFeedURL = "http://93.184.215.14/feed.xml"
+
+func TestFeedFetcherKeepsAPublicFeedOffTheLocalNetwork(t *testing.T) {
+	var hits atomic.Int32
+	server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, request *http.Request) {
+		if request.URL.Path == "/moved" {
+			http.Redirect(w, request, "/feed.xml", http.StatusFound)
+			return
+		}
+		hits.Add(1)
+		_, _ = io.WriteString(w, rssFixture("Release", "release-1", "Mon, 02 Jan 2006 15:04:05 GMT"))
+	}))
+	defer server.Close()
+
+	t.Run("a feed on the local network is fetched as configured", func(t *testing.T) {
+		result, err := newFeedFetcher("test").fetch(t.Context(), server.URL+"/moved", feedHTTPValidators{})
+		require.NoError(t, err)
+		assert.Len(t, result.items, 1)
+	})
+
+	t.Run("a public feed cannot redirect into it", func(t *testing.T) {
+		before := hits.Load()
+		fetcher := newFeedFetcher("test")
+		dialing := fetcher.client.Transport
+		fetcher.client.Transport = roundTripFunc(func(request *http.Request) (*http.Response, error) {
+			if request.URL.String() == publicFeedURL {
+				return testFeedResponse(http.StatusFound, "", map[string]string{"Location": server.URL + "/feed.xml"}), nil
+			}
+			return dialing.RoundTrip(request)
+		})
+		_, err := fetcher.fetch(t.Context(), publicFeedURL, feedHTTPValidators{})
+		assert.ErrorIs(t, err, errFeedAddressNotPublic)
+		assert.Equal(t, before, hits.Load())
+	})
+}
+
+// A name server can answer loopback to the lookup that judges the feed and its real address to
+// the connection, then redirect into the network. The judging lookup's answers are where the
+// feed connects, so the server behind the second answer is never reached.
+func TestFeedFetcherConnectsWhereTheJudgingLookupSaid(t *testing.T) {
+	var hits atomic.Int32
+	server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, request *http.Request) {
+		hits.Add(1)
+		http.Redirect(w, request, "http://169.254.169.254/latest/meta-data/", http.StatusFound)
+	}))
+	defer server.Close()
+	_, port, err := net.SplitHostPort(strings.TrimPrefix(server.URL, "http://"))
+	require.NoError(t, err)
+
+	fetcher := newFeedFetcher("test")
+	fetcher.lookup = func(context.Context, string) ([]netip.Addr, error) {
+		return []netip.Addr{netip.MustParseAddr("127.0.0.2")}, nil
+	}
+	_, err = fetcher.fetch(t.Context(), "http://localhost:"+port+"/feed.xml", feedHTTPValidators{})
+	require.Error(t, err)
+	assert.Zero(t, hits.Load(), "connected to the second answer, not the first")
+}
+
+func TestFeedFetcherConnectsToAPrivateProxy(t *testing.T) {
+	var proxied []string
+	proxy := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, request *http.Request) {
+		proxied = append(proxied, request.URL.String())
+		if request.URL.String() == publicFeedURL {
+			http.Redirect(w, request, "http://169.254.169.254/feed.xml", http.StatusFound)
+			return
+		}
+		_, _ = io.WriteString(w, rssFixture("Release", "release-1", "Mon, 02 Jan 2006 15:04:05 GMT"))
+	}))
+	defer proxy.Close()
+	proxyURL, err := url.Parse(proxy.URL)
+	require.NoError(t, err)
+	fetcher := newFeedFetcherVia("test", http.ProxyURL(proxyURL))
+
+	result, err := fetcher.fetch(t.Context(), "http://10.1.2.3/feed.xml", feedHTTPValidators{})
+	require.NoError(t, err, "a feed on the local network, through the proxy")
+	assert.Len(t, result.items, 1)
+
+	_, err = fetcher.fetch(t.Context(), publicFeedURL, feedHTTPValidators{})
+	assert.ErrorIs(t, err, errFeedAddressNotPublic)
+	assert.Equal(t, []string{"http://10.1.2.3/feed.xml", publicFeedURL}, proxied,
+		"the redirect to a metadata address never reaches the proxy")
+}
+
+func TestFeedFetcherRefusesHTTPSDowngrade(t *testing.T) {
+	fetcher := newFeedFetcher("test")
+	var requested []string
+	fetcher.client.Transport = roundTripFunc(func(request *http.Request) (*http.Response, error) {
+		requested = append(requested, request.URL.String())
+		return testFeedResponse(http.StatusFound, "", map[string]string{"Location": "http://feeds.example.com/feed.xml"}), nil
+	})
+	_, err := fetcher.fetch(t.Context(), "https://feeds.example.com/feed.xml", feedHTTPValidators{})
+	require.ErrorContains(t, err, "redirected from https to http")
+	assert.Equal(t, []string{"https://feeds.example.com/feed.xml"}, requested)
+}
+
 func feedFixture(t *testing.T, name string) string {
 	t.Helper()
 	data, err := os.ReadFile(filepath.Join("testdata", name))