From 26177023e075d6c5dc9b01030b1c05711e81293f Mon Sep 17 00:00:00 2001 From: Misha Nasledov Date: Fri, 9 Oct 2026 16:39:52 -0700 Subject: [PATCH 1/3] Refuse feed connections to non-public addresses The feed URL is documented as public, but nothing enforced it, and redirects were followed anywhere http(s) goes: a feed host could send the server's fetch to loopback, the local network or 169.254.169.254, and from https down to http. Check every direct connection's resolved address at dial time, so neither the URL's host name nor a redirect reaches a loopback, private, link-local or carrier-grade NAT address, and refuse a redirect from https to http. A proxy from HTTP_PROXY or HTTPS_PROXY is the operator's choice and is often on the local network, so connections to it are not checked. Through it the server cannot see what a host name resolves to, so only an address written into the URL is refused. --- docs/feed-backed-news.md | 6 ++ internal/mobius/feed_news_fetch.go | 76 ++++++++++++++++++++++++- internal/mobius/feed_news_test.go | 90 ++++++++++++++++++++++++++++++ 3 files changed, 171 insertions(+), 1 deletion(-) diff --git a/docs/feed-backed-news.md b/docs/feed-backed-news.md index 8edcf79..bb38778 100644 --- a/docs/feed-backed-news.md +++ b/docs/feed-backed-news.md @@ -30,6 +30,12 @@ NewsFeeds: `CategoryPath` is the complete path from the root of threaded news to the existing category. `URL` must be an absolute public `http` or `https` URL and cannot contain embedded credentials. Only one feed may map to a category. +Mobius refuses to connect to loopback, private, link-local or carrier-grade NAT +addresses, whether the URL names one or a redirect leads to one, and does not +follow a redirect from `https` to `http`. A proxy set with `HTTP_PROXY` or +`HTTPS_PROXY` may be on a private address. Through a proxy, Mobius cannot see +what a host name resolves to, so it refuses only a URL that names a non-public +address directly; the proxy's own rules decide the rest. Mobius detects RSS and Atom automatically. Sparkle appcasts are RSS; GitHub's `releases.atom` endpoints are Atom. JSON Feed and provider-specific APIs are not diff --git a/internal/mobius/feed_news_fetch.go b/internal/mobius/feed_news_fetch.go index 18d25ee..aa22095 100644 --- a/internal/mobius/feed_news_fetch.go +++ b/internal/mobius/feed_news_fetch.go @@ -7,11 +7,15 @@ import ( "fmt" stdhtml "html" "io" + "net" "net/http" + "net/netip" "net/url" "slices" "sort" "strings" + "sync" + "syscall" "time" "unicode/utf8" @@ -27,7 +31,36 @@ const ( maxNewsBodyBytes = 65535 ) -var errFeedResponseTooLarge = errors.New("feed response exceeds 2 MiB") +var ( + errFeedResponseTooLarge = errors.New("feed response exceeds 2 MiB") + errFeedAddressNotPublic = errors.New("feed address is not public") +) + +// sharedAddressSpace is carrier-grade NAT space (RFC 6598), which netip does not count as private. +var sharedAddressSpace = netip.MustParsePrefix("100.64.0.0/10") + +func isPublicAddr(ip netip.Addr) bool { + ip = ip.Unmap() + return ip.IsGlobalUnicast() && !ip.IsPrivate() && !sharedAddressSpace.Contains(ip) +} + +// refuseNonPublicDial is checked against the resolved address of every direct connection, so +// neither a feed's host name nor a redirect it serves can point the server at loopback, the local +// network, or a cloud metadata endpoint. +func refuseNonPublicDial(_ context.Context, _, address string, _ syscall.RawConn) error { + host, _, err := net.SplitHostPort(address) + if err != nil { + return err + } + ip, err := netip.ParseAddr(host) + if err != nil { + return err + } + if !isPublicAddr(ip) { + return fmt.Errorf("%w: %s", errFeedAddressNotPublic, ip) + } + return nil +} type normalizedFeedItem struct { identity string @@ -57,11 +90,40 @@ type feedFetcher struct { } func newFeedFetcher(userAgent string) *feedFetcher { + return newFeedFetcherVia(userAgent, http.ProxyFromEnvironment) +} + +// newFeedFetcherVia fetches through the proxies proxy chooses. A proxy is the operator's choice, +// and is commonly on the local network, so connections to it are not checked; through it the +// server cannot see what a host name resolves to, so only an address written into the URL is. +func newFeedFetcherVia(userAgent string, proxy func(*http.Request) (*url.URL, error)) *feedFetcher { if userAgent == "" { userAgent = "mobius-hotline-server" } + var proxies sync.Map + direct := &net.Dialer{Timeout: 30 * time.Second, KeepAlive: 30 * time.Second} + guarded := &net.Dialer{Timeout: 30 * time.Second, KeepAlive: 30 * time.Second, ControlContext: refuseNonPublicDial} + transport := http.DefaultTransport.(*http.Transport).Clone() + transport.Proxy = func(req *http.Request) (*url.URL, error) { + proxyURL, err := proxy(req) + if err != nil || proxyURL == nil { + return proxyURL, err + } + if ip, err := netip.ParseAddr(req.URL.Hostname()); err == nil && !isPublicAddr(ip) { + return nil, fmt.Errorf("%w: %s", errFeedAddressNotPublic, ip) + } + proxies.Store(proxyAddr(proxyURL), struct{}{}) + return proxyURL, nil + } + transport.DialContext = func(ctx context.Context, network, address string) (net.Conn, error) { + if _, ok := proxies.Load(address); ok { + return direct.DialContext(ctx, network, address) + } + return guarded.DialContext(ctx, network, address) + } return &feedFetcher{ client: &http.Client{ + Transport: transport, CheckRedirect: func(req *http.Request, via []*http.Request) error { if len(via) >= 10 { return errors.New("stopped after 10 redirects") @@ -69,6 +131,9 @@ func newFeedFetcher(userAgent string) *feedFetcher { if req.URL.Scheme != "http" && req.URL.Scheme != "https" { return fmt.Errorf("redirected to unsupported URL scheme %q", req.URL.Scheme) } + if via[len(via)-1].URL.Scheme == "https" && req.URL.Scheme == "http" { + return errors.New("redirected from https to http") + } if req.URL.User != nil { return errors.New("redirected to a URL containing credentials") } @@ -81,6 +146,15 @@ func newFeedFetcher(userAgent string) *feedFetcher { } } +// proxyAddr is the address the transport dials for proxyURL. +func proxyAddr(proxyURL *url.URL) string { + if port := proxyURL.Port(); port != "" { + return net.JoinHostPort(proxyURL.Hostname(), port) + } + port := map[string]string{"http": "80", "https": "443", "socks5": "1080", "socks5h": "1080"}[proxyURL.Scheme] + return net.JoinHostPort(proxyURL.Hostname(), port) +} + func (f *feedFetcher) fetch(ctx context.Context, sourceURL string, validators feedHTTPValidators) (feedFetchResult, error) { ctx, cancel := context.WithTimeout(ctx, feedRequestTimeout) defer cancel() diff --git a/internal/mobius/feed_news_test.go b/internal/mobius/feed_news_test.go index 037d1a0..0b06c28 100644 --- a/internal/mobius/feed_news_test.go +++ b/internal/mobius/feed_news_test.go @@ -7,6 +7,9 @@ import ( "io" "log/slog" "net/http" + "net/http/httptest" + "net/netip" + "net/url" "os" "path/filepath" "strings" @@ -425,6 +428,93 @@ func TestFeedFetcherLimitsAndConditional304(t *testing.T) { }) } +func TestIsPublicAddr(t *testing.T) { + tests := []struct { + addr string + want bool + }{ + {"93.184.215.14", true}, + {"2606:4700::6810:84e5", true}, + {"127.0.0.1", false}, + {"::1", false}, + {"10.1.2.3", false}, + {"172.16.0.1", false}, + {"192.168.1.1", false}, + {"169.254.169.254", false}, + {"100.64.0.1", false}, + {"0.0.0.0", false}, + {"::", false}, + {"fd00::1", false}, + {"fe80::1", false}, + {"224.0.0.1", false}, + {"::ffff:127.0.0.1", false}, + {"::ffff:192.168.1.1", false}, + } + for _, tt := range tests { + assert.Equal(t, tt.want, isPublicAddr(netip.MustParseAddr(tt.addr)), tt.addr) + } +} + +func TestFeedFetcherRefusesNonPublicDestinations(t *testing.T) { + var hits atomic.Int32 + server := httptest.NewServer(http.HandlerFunc(func(http.ResponseWriter, *http.Request) { + hits.Add(1) + })) + defer server.Close() + + t.Run("configured URL", func(t *testing.T) { + _, err := newFeedFetcher("test").fetch(t.Context(), server.URL+"/feed.xml", feedHTTPValidators{}) + assert.ErrorIs(t, err, errFeedAddressNotPublic) + }) + + t.Run("redirect", func(t *testing.T) { + fetcher := newFeedFetcher("test") + dialing := fetcher.client.Transport + fetcher.client.Transport = roundTripFunc(func(request *http.Request) (*http.Response, error) { + if request.URL.Host == "feeds.example.com" { + return testFeedResponse(http.StatusFound, "", map[string]string{"Location": server.URL + "/feed.xml"}), nil + } + return dialing.RoundTrip(request) + }) + _, err := fetcher.fetch(t.Context(), "http://feeds.example.com/feed.xml", feedHTTPValidators{}) + assert.ErrorIs(t, err, errFeedAddressNotPublic) + }) + + assert.Zero(t, hits.Load()) +} + +func TestFeedFetcherConnectsToAPrivateProxy(t *testing.T) { + var proxied []string + proxy := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, request *http.Request) { + proxied = append(proxied, request.URL.String()) + _, _ = io.WriteString(w, rssFixture("Release", "release-1", "Mon, 02 Jan 2006 15:04:05 GMT")) + })) + defer proxy.Close() + proxyURL, err := url.Parse(proxy.URL) + require.NoError(t, err) + fetcher := newFeedFetcherVia("test", http.ProxyURL(proxyURL)) + + result, err := fetcher.fetch(t.Context(), "http://feeds.example.com/feed.xml", feedHTTPValidators{}) + require.NoError(t, err) + assert.Len(t, result.items, 1) + + _, err = fetcher.fetch(t.Context(), "http://169.254.169.254/feed.xml", feedHTTPValidators{}) + assert.ErrorIs(t, err, errFeedAddressNotPublic) + assert.Equal(t, []string{"http://feeds.example.com/feed.xml"}, proxied) +} + +func TestFeedFetcherRefusesHTTPSDowngrade(t *testing.T) { + fetcher := newFeedFetcher("test") + var requested []string + fetcher.client.Transport = roundTripFunc(func(request *http.Request) (*http.Response, error) { + requested = append(requested, request.URL.String()) + return testFeedResponse(http.StatusFound, "", map[string]string{"Location": "http://feeds.example.com/feed.xml"}), nil + }) + _, err := fetcher.fetch(t.Context(), "https://feeds.example.com/feed.xml", feedHTTPValidators{}) + require.ErrorContains(t, err, "redirected from https to http") + assert.Equal(t, []string{"https://feeds.example.com/feed.xml"}, requested) +} + func feedFixture(t *testing.T, name string) string { t.Helper() data, err := os.ReadFile(filepath.Join("testdata", name)) From 4960947fbd12644fdd73280e45dd8a759fc83258 Mon Sep 17 00:00:00 2001 From: Misha Nasledov Date: Fri, 9 Oct 2026 17:16:46 -0700 Subject: [PATCH 2/3] Fetch a feed on the local network as configured A feed whose configured URL is on a private address is the operator's choice and exposes nothing, so it is fetched as configured, redirects included. What stays refused is a public feed reaching the local network: every connection its fetch makes, redirects and all, must be to a public address, and through a proxy a redirect to a literal private address is refused before the proxy sees it. A host counts as private when it is a literal non-public address or resolves only to them; one that does not resolve, as behind a proxy, counts as public. --- docs/feed-backed-news.md | 21 ++++++++------- internal/mobius/feed_news_fetch.go | 35 +++++++++++++++++++++---- internal/mobius/feed_news_test.go | 42 ++++++++++++++++++++---------- 3 files changed, 70 insertions(+), 28 deletions(-) diff --git a/docs/feed-backed-news.md b/docs/feed-backed-news.md index bb38778..cc1b0e1 100644 --- a/docs/feed-backed-news.md +++ b/docs/feed-backed-news.md @@ -1,6 +1,6 @@ # Feed-backed threaded news -Mobius can import a public RSS or Atom feed into an existing Hotline news +Mobius can import an RSS or Atom feed into an existing Hotline news category. This is useful for project announcements, Sparkle appcasts, and GitHub release feeds. @@ -28,14 +28,17 @@ NewsFeeds: ``` `CategoryPath` is the complete path from the root of threaded news to the -existing category. `URL` must be an absolute public `http` or `https` URL and +existing category. `URL` must be an absolute `http` or `https` URL and cannot contain embedded credentials. Only one feed may map to a category. -Mobius refuses to connect to loopback, private, link-local or carrier-grade NAT -addresses, whether the URL names one or a redirect leads to one, and does not -follow a redirect from `https` to `http`. A proxy set with `HTTP_PROXY` or -`HTTPS_PROXY` may be on a private address. Through a proxy, Mobius cannot see -what a host name resolves to, so it refuses only a URL that names a non-public -address directly; the proxy's own rules decide the rest. +A feed on the local network is fine: when the configured URL names a loopback, +private, link-local or carrier-grade NAT address, or a host that resolves only +to such addresses, Mobius fetches it as configured. A public feed is held to +public addresses on every connection, so a redirect it serves cannot reach the +server's local network. No redirect from `https` to `http` is followed. A proxy +set with `HTTP_PROXY` or `HTTPS_PROXY` may be on a private address. Through a +proxy, Mobius cannot see what a host name resolves to, so for a public feed it +refuses only a URL that names a non-public address directly; the proxy's own +rules decide the rest. Mobius detects RSS and Atom automatically. Sparkle appcasts are RSS; GitHub's `releases.atom` endpoints are Atom. JSON Feed and provider-specific APIs are not @@ -126,7 +129,7 @@ rewrite history. news category rather than a bundle. Also check for duplicate paths and invalid or credential-bearing URLs. - If new entries do not appear, open the category and inspect the server log. - Confirm that the URL is public RSS or Atom and that entries have stable IDs, + Confirm that the URL is RSS or Atom and that entries have stable IDs, links, or enclosures. - If old releases are missing on the first import, inspect the source feed. Its published window is the initial cutoff; Mobius does not use pagination or a diff --git a/internal/mobius/feed_news_fetch.go b/internal/mobius/feed_news_fetch.go index aa22095..d674f3a 100644 --- a/internal/mobius/feed_news_fetch.go +++ b/internal/mobius/feed_news_fetch.go @@ -44,10 +44,30 @@ func isPublicAddr(ip netip.Addr) bool { return ip.IsGlobalUnicast() && !ip.IsPrivate() && !sharedAddressSpace.Contains(ip) } -// refuseNonPublicDial is checked against the resolved address of every direct connection, so -// neither a feed's host name nor a redirect it serves can point the server at loopback, the local +// privateFeed marks a fetch whose configured URL is itself on a non-public address. The operator +// chose that, so nothing it connects to is refused. +type privateFeed struct{} + +// isPrivateFeed reports whether the configured URL names a non-public address, or a host whose +// every address is one. A host that does not resolve here, as behind a proxy, counts as public. +func isPrivateFeed(ctx context.Context, u *url.URL) bool { + if ip, err := netip.ParseAddr(u.Hostname()); err == nil { + return !isPublicAddr(ip) + } + addrs, err := net.DefaultResolver.LookupNetIP(ctx, "ip", u.Hostname()) + if err != nil || len(addrs) == 0 { + return false + } + return !slices.ContainsFunc(addrs, isPublicAddr) +} + +// refuseNonPublicDial is checked against the resolved address of every direct connection a public +// feed's fetch makes, so a redirect it serves cannot point the server at loopback, the local // network, or a cloud metadata endpoint. -func refuseNonPublicDial(_ context.Context, _, address string, _ syscall.RawConn) error { +func refuseNonPublicDial(ctx context.Context, _, address string, _ syscall.RawConn) error { + if ctx.Value(privateFeed{}) != nil { + return nil + } host, _, err := net.SplitHostPort(address) if err != nil { return err @@ -95,7 +115,8 @@ func newFeedFetcher(userAgent string) *feedFetcher { // newFeedFetcherVia fetches through the proxies proxy chooses. A proxy is the operator's choice, // and is commonly on the local network, so connections to it are not checked; through it the -// server cannot see what a host name resolves to, so only an address written into the URL is. +// server cannot see what a host name resolves to, so for a public feed only an address written +// into a URL is. func newFeedFetcherVia(userAgent string, proxy func(*http.Request) (*url.URL, error)) *feedFetcher { if userAgent == "" { userAgent = "mobius-hotline-server" @@ -109,7 +130,8 @@ func newFeedFetcherVia(userAgent string, proxy func(*http.Request) (*url.URL, er if err != nil || proxyURL == nil { return proxyURL, err } - if ip, err := netip.ParseAddr(req.URL.Hostname()); err == nil && !isPublicAddr(ip) { + ip, err := netip.ParseAddr(req.URL.Hostname()) + if err == nil && !isPublicAddr(ip) && req.Context().Value(privateFeed{}) == nil { return nil, fmt.Errorf("%w: %s", errFeedAddressNotPublic, ip) } proxies.Store(proxyAddr(proxyURL), struct{}{}) @@ -163,6 +185,9 @@ func (f *feedFetcher) fetch(ctx context.Context, sourceURL string, validators fe if err != nil { return feedFetchResult{}, errors.New("create request from configured feed URL") } + if isPrivateFeed(ctx, req.URL) { + req = req.WithContext(context.WithValue(ctx, privateFeed{}, true)) + } req.Header.Set("Accept", "application/rss+xml, application/atom+xml, application/xml;q=0.9, text/xml;q=0.8") req.Header.Set("User-Agent", f.userAgent) if validators.etag != "" { diff --git a/internal/mobius/feed_news_test.go b/internal/mobius/feed_news_test.go index 0b06c28..4f85ec1 100644 --- a/internal/mobius/feed_news_test.go +++ b/internal/mobius/feed_news_test.go @@ -455,38 +455,51 @@ func TestIsPublicAddr(t *testing.T) { } } -func TestFeedFetcherRefusesNonPublicDestinations(t *testing.T) { +// publicFeedURL is a literal public address, so whether the feed is public never depends on DNS. +const publicFeedURL = "http://93.184.215.14/feed.xml" + +func TestFeedFetcherKeepsAPublicFeedOffTheLocalNetwork(t *testing.T) { var hits atomic.Int32 - server := httptest.NewServer(http.HandlerFunc(func(http.ResponseWriter, *http.Request) { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, request *http.Request) { + if request.URL.Path == "/moved" { + http.Redirect(w, request, "/feed.xml", http.StatusFound) + return + } hits.Add(1) + _, _ = io.WriteString(w, rssFixture("Release", "release-1", "Mon, 02 Jan 2006 15:04:05 GMT")) })) defer server.Close() - t.Run("configured URL", func(t *testing.T) { - _, err := newFeedFetcher("test").fetch(t.Context(), server.URL+"/feed.xml", feedHTTPValidators{}) - assert.ErrorIs(t, err, errFeedAddressNotPublic) + t.Run("a feed on the local network is fetched as configured", func(t *testing.T) { + result, err := newFeedFetcher("test").fetch(t.Context(), server.URL+"/moved", feedHTTPValidators{}) + require.NoError(t, err) + assert.Len(t, result.items, 1) }) - t.Run("redirect", func(t *testing.T) { + t.Run("a public feed cannot redirect into it", func(t *testing.T) { + before := hits.Load() fetcher := newFeedFetcher("test") dialing := fetcher.client.Transport fetcher.client.Transport = roundTripFunc(func(request *http.Request) (*http.Response, error) { - if request.URL.Host == "feeds.example.com" { + if request.URL.String() == publicFeedURL { return testFeedResponse(http.StatusFound, "", map[string]string{"Location": server.URL + "/feed.xml"}), nil } return dialing.RoundTrip(request) }) - _, err := fetcher.fetch(t.Context(), "http://feeds.example.com/feed.xml", feedHTTPValidators{}) + _, err := fetcher.fetch(t.Context(), publicFeedURL, feedHTTPValidators{}) assert.ErrorIs(t, err, errFeedAddressNotPublic) + assert.Equal(t, before, hits.Load()) }) - - assert.Zero(t, hits.Load()) } func TestFeedFetcherConnectsToAPrivateProxy(t *testing.T) { var proxied []string proxy := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, request *http.Request) { proxied = append(proxied, request.URL.String()) + if request.URL.String() == publicFeedURL { + http.Redirect(w, request, "http://169.254.169.254/feed.xml", http.StatusFound) + return + } _, _ = io.WriteString(w, rssFixture("Release", "release-1", "Mon, 02 Jan 2006 15:04:05 GMT")) })) defer proxy.Close() @@ -494,13 +507,14 @@ func TestFeedFetcherConnectsToAPrivateProxy(t *testing.T) { require.NoError(t, err) fetcher := newFeedFetcherVia("test", http.ProxyURL(proxyURL)) - result, err := fetcher.fetch(t.Context(), "http://feeds.example.com/feed.xml", feedHTTPValidators{}) - require.NoError(t, err) + result, err := fetcher.fetch(t.Context(), "http://10.1.2.3/feed.xml", feedHTTPValidators{}) + require.NoError(t, err, "a feed on the local network, through the proxy") assert.Len(t, result.items, 1) - _, err = fetcher.fetch(t.Context(), "http://169.254.169.254/feed.xml", feedHTTPValidators{}) + _, err = fetcher.fetch(t.Context(), publicFeedURL, feedHTTPValidators{}) assert.ErrorIs(t, err, errFeedAddressNotPublic) - assert.Equal(t, []string{"http://feeds.example.com/feed.xml"}, proxied) + assert.Equal(t, []string{"http://10.1.2.3/feed.xml", publicFeedURL}, proxied, + "the redirect to a metadata address never reaches the proxy") } func TestFeedFetcherRefusesHTTPSDowngrade(t *testing.T) { From 5eaa7cdf1c3d70d22a00dc4a2aaa09b6edbe3734 Mon Sep 17 00:00:00 2001 From: Misha Nasledov Date: Fri, 9 Oct 2026 17:49:19 -0700 Subject: [PATCH 3/3] Connect a private feed to the addresses that judged it private A feed's host name was looked up once to judge the feed private and again to connect to it. A name server could answer loopback to the first and its public address to the second, and the redirect that server then sent toward 169.254.169.254 was followed unchecked. The judging lookup's answers are now where connections to that host go. Behind a proxy, which resolves names out of sight, only an address written in the URL makes a feed private. --- docs/feed-backed-news.md | 3 +- internal/mobius/feed_news_fetch.go | 67 +++++++++++++++++++++++------- internal/mobius/feed_news_test.go | 24 +++++++++++ 3 files changed, 79 insertions(+), 15 deletions(-) diff --git a/docs/feed-backed-news.md b/docs/feed-backed-news.md index cc1b0e1..dd0dae8 100644 --- a/docs/feed-backed-news.md +++ b/docs/feed-backed-news.md @@ -32,7 +32,8 @@ existing category. `URL` must be an absolute `http` or `https` URL and cannot contain embedded credentials. Only one feed may map to a category. A feed on the local network is fine: when the configured URL names a loopback, private, link-local or carrier-grade NAT address, or a host that resolves only -to such addresses, Mobius fetches it as configured. A public feed is held to +to such addresses, Mobius fetches it as configured, connecting to the +addresses that lookup gave. A public feed is held to public addresses on every connection, so a redirect it serves cannot reach the server's local network. No redirect from `https` to `http` is followed. A proxy set with `HTTP_PROXY` or `HTTPS_PROXY` may be on a private address. Through a diff --git a/internal/mobius/feed_news_fetch.go b/internal/mobius/feed_news_fetch.go index d674f3a..8a2e6ef 100644 --- a/internal/mobius/feed_news_fetch.go +++ b/internal/mobius/feed_news_fetch.go @@ -45,27 +45,42 @@ func isPublicAddr(ip netip.Addr) bool { } // privateFeed marks a fetch whose configured URL is itself on a non-public address. The operator -// chose that, so nothing it connects to is refused. -type privateFeed struct{} +// chose that, so nothing it connects to is refused. For a host name, addrs are the answers that +// decided it, and connections to that host go to them: a second lookup could answer otherwise, and +// a name server answering loopback to the check and a public address to the connection would have +// its redirects followed unchecked. +type privateFeed struct { + host string + addrs []netip.Addr +} + +type privateFeedKey struct{} -// isPrivateFeed reports whether the configured URL names a non-public address, or a host whose -// every address is one. A host that does not resolve here, as behind a proxy, counts as public. -func isPrivateFeed(ctx context.Context, u *url.URL) bool { +// privateFeedFor reports whether the configured URL names a non-public address, or a host whose +// every address is one. Behind a proxy, which resolves names where this cannot see, only an address +// makes a feed private, and a host that does not resolve here counts as public. +func (f *feedFetcher) privateFeedFor(ctx context.Context, u *url.URL, proxied bool) *privateFeed { if ip, err := netip.ParseAddr(u.Hostname()); err == nil { - return !isPublicAddr(ip) + if isPublicAddr(ip) { + return nil + } + return &privateFeed{} } - addrs, err := net.DefaultResolver.LookupNetIP(ctx, "ip", u.Hostname()) - if err != nil || len(addrs) == 0 { - return false + if proxied { + return nil + } + addrs, err := f.lookup(ctx, u.Hostname()) + if err != nil || len(addrs) == 0 || slices.ContainsFunc(addrs, isPublicAddr) { + return nil } - return !slices.ContainsFunc(addrs, isPublicAddr) + return &privateFeed{host: u.Hostname(), addrs: addrs} } // refuseNonPublicDial is checked against the resolved address of every direct connection a public // feed's fetch makes, so a redirect it serves cannot point the server at loopback, the local // network, or a cloud metadata endpoint. func refuseNonPublicDial(ctx context.Context, _, address string, _ syscall.RawConn) error { - if ctx.Value(privateFeed{}) != nil { + if ctx.Value(privateFeedKey{}) != nil { return nil } host, _, err := net.SplitHostPort(address) @@ -107,6 +122,8 @@ type feedFetcher struct { parser *gofeed.Parser now func() time.Time userAgent string + proxy func(*http.Request) (*url.URL, error) + lookup func(ctx context.Context, host string) ([]netip.Addr, error) } func newFeedFetcher(userAgent string) *feedFetcher { @@ -131,7 +148,7 @@ func newFeedFetcherVia(userAgent string, proxy func(*http.Request) (*url.URL, er return proxyURL, err } ip, err := netip.ParseAddr(req.URL.Hostname()) - if err == nil && !isPublicAddr(ip) && req.Context().Value(privateFeed{}) == nil { + if err == nil && !isPublicAddr(ip) && req.Context().Value(privateFeedKey{}) == nil { return nil, fmt.Errorf("%w: %s", errFeedAddressNotPublic, ip) } proxies.Store(proxyAddr(proxyURL), struct{}{}) @@ -141,6 +158,11 @@ func newFeedFetcherVia(userAgent string, proxy func(*http.Request) (*url.URL, er if _, ok := proxies.Load(address); ok { return direct.DialContext(ctx, network, address) } + if pin, _ := ctx.Value(privateFeedKey{}).(*privateFeed); pin != nil && pin.addrs != nil { + if host, port, err := net.SplitHostPort(address); err == nil && host == pin.host { + return dialFirst(ctx, direct, network, pin.addrs, port) + } + } return guarded.DialContext(ctx, network, address) } return &feedFetcher{ @@ -165,7 +187,23 @@ func newFeedFetcherVia(userAgent string, proxy func(*http.Request) (*url.URL, er parser: gofeed.NewParser(), now: time.Now, userAgent: userAgent, + proxy: proxy, + lookup: func(ctx context.Context, host string) ([]netip.Addr, error) { + return net.DefaultResolver.LookupNetIP(ctx, "ip", host) + }, + } +} + +// dialFirst connects to the first of addrs that answers. +func dialFirst(ctx context.Context, d *net.Dialer, network string, addrs []netip.Addr, port string) (net.Conn, error) { + var err error + for _, addr := range addrs { + var conn net.Conn + if conn, err = d.DialContext(ctx, network, net.JoinHostPort(addr.Unmap().String(), port)); err == nil { + return conn, nil + } } + return nil, err } // proxyAddr is the address the transport dials for proxyURL. @@ -185,8 +223,9 @@ func (f *feedFetcher) fetch(ctx context.Context, sourceURL string, validators fe if err != nil { return feedFetchResult{}, errors.New("create request from configured feed URL") } - if isPrivateFeed(ctx, req.URL) { - req = req.WithContext(context.WithValue(ctx, privateFeed{}, true)) + proxyURL, _ := f.proxy(req) + if pin := f.privateFeedFor(ctx, req.URL, proxyURL != nil); pin != nil { + req = req.WithContext(context.WithValue(ctx, privateFeedKey{}, pin)) } req.Header.Set("Accept", "application/rss+xml, application/atom+xml, application/xml;q=0.9, text/xml;q=0.8") req.Header.Set("User-Agent", f.userAgent) diff --git a/internal/mobius/feed_news_test.go b/internal/mobius/feed_news_test.go index 4f85ec1..d50cad9 100644 --- a/internal/mobius/feed_news_test.go +++ b/internal/mobius/feed_news_test.go @@ -2,10 +2,12 @@ package mobius import ( "bytes" + "context" "errors" "fmt" "io" "log/slog" + "net" "net/http" "net/http/httptest" "net/netip" @@ -492,6 +494,28 @@ func TestFeedFetcherKeepsAPublicFeedOffTheLocalNetwork(t *testing.T) { }) } +// A name server can answer loopback to the lookup that judges the feed and its real address to +// the connection, then redirect into the network. The judging lookup's answers are where the +// feed connects, so the server behind the second answer is never reached. +func TestFeedFetcherConnectsWhereTheJudgingLookupSaid(t *testing.T) { + var hits atomic.Int32 + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, request *http.Request) { + hits.Add(1) + http.Redirect(w, request, "http://169.254.169.254/latest/meta-data/", http.StatusFound) + })) + defer server.Close() + _, port, err := net.SplitHostPort(strings.TrimPrefix(server.URL, "http://")) + require.NoError(t, err) + + fetcher := newFeedFetcher("test") + fetcher.lookup = func(context.Context, string) ([]netip.Addr, error) { + return []netip.Addr{netip.MustParseAddr("127.0.0.2")}, nil + } + _, err = fetcher.fetch(t.Context(), "http://localhost:"+port+"/feed.xml", feedHTTPValidators{}) + require.Error(t, err) + assert.Zero(t, hits.Load(), "connected to the second answer, not the first") +} + func TestFeedFetcherConnectsToAPrivateProxy(t *testing.T) { var proxied []string proxy := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, request *http.Request) {