diff --git a/docs/feed-backed-news.md b/docs/feed-backed-news.md index 8edcf79..dd0dae8 100644 --- a/docs/feed-backed-news.md +++ b/docs/feed-backed-news.md @@ -1,6 +1,6 @@ # Feed-backed threaded news -Mobius can import a public RSS or Atom feed into an existing Hotline news +Mobius can import an RSS or Atom feed into an existing Hotline news category. This is useful for project announcements, Sparkle appcasts, and GitHub release feeds. @@ -28,8 +28,18 @@ NewsFeeds: ``` `CategoryPath` is the complete path from the root of threaded news to the -existing category. `URL` must be an absolute public `http` or `https` URL and +existing category. `URL` must be an absolute `http` or `https` URL and cannot contain embedded credentials. Only one feed may map to a category. +A feed on the local network is fine: when the configured URL names a loopback, +private, link-local or carrier-grade NAT address, or a host that resolves only +to such addresses, Mobius fetches it as configured, connecting to the +addresses that lookup gave. A public feed is held to +public addresses on every connection, so a redirect it serves cannot reach the +server's local network. No redirect from `https` to `http` is followed. A proxy +set with `HTTP_PROXY` or `HTTPS_PROXY` may be on a private address. Through a +proxy, Mobius cannot see what a host name resolves to, so for a public feed it +refuses only a URL that names a non-public address directly; the proxy's own +rules decide the rest. Mobius detects RSS and Atom automatically. Sparkle appcasts are RSS; GitHub's `releases.atom` endpoints are Atom. JSON Feed and provider-specific APIs are not @@ -120,7 +130,7 @@ rewrite history. news category rather than a bundle. Also check for duplicate paths and invalid or credential-bearing URLs. - If new entries do not appear, open the category and inspect the server log. - Confirm that the URL is public RSS or Atom and that entries have stable IDs, + Confirm that the URL is RSS or Atom and that entries have stable IDs, links, or enclosures. - If old releases are missing on the first import, inspect the source feed. Its published window is the initial cutoff; Mobius does not use pagination or a diff --git a/internal/mobius/feed_news_fetch.go b/internal/mobius/feed_news_fetch.go index 18d25ee..8a2e6ef 100644 --- a/internal/mobius/feed_news_fetch.go +++ b/internal/mobius/feed_news_fetch.go @@ -7,11 +7,15 @@ import ( "fmt" stdhtml "html" "io" + "net" "net/http" + "net/netip" "net/url" "slices" "sort" "strings" + "sync" + "syscall" "time" "unicode/utf8" @@ -27,7 +31,71 @@ const ( maxNewsBodyBytes = 65535 ) -var errFeedResponseTooLarge = errors.New("feed response exceeds 2 MiB") +var ( + errFeedResponseTooLarge = errors.New("feed response exceeds 2 MiB") + errFeedAddressNotPublic = errors.New("feed address is not public") +) + +// sharedAddressSpace is carrier-grade NAT space (RFC 6598), which netip does not count as private. +var sharedAddressSpace = netip.MustParsePrefix("100.64.0.0/10") + +func isPublicAddr(ip netip.Addr) bool { + ip = ip.Unmap() + return ip.IsGlobalUnicast() && !ip.IsPrivate() && !sharedAddressSpace.Contains(ip) +} + +// privateFeed marks a fetch whose configured URL is itself on a non-public address. The operator +// chose that, so nothing it connects to is refused. For a host name, addrs are the answers that +// decided it, and connections to that host go to them: a second lookup could answer otherwise, and +// a name server answering loopback to the check and a public address to the connection would have +// its redirects followed unchecked. +type privateFeed struct { + host string + addrs []netip.Addr +} + +type privateFeedKey struct{} + +// privateFeedFor reports whether the configured URL names a non-public address, or a host whose +// every address is one. Behind a proxy, which resolves names where this cannot see, only an address +// makes a feed private, and a host that does not resolve here counts as public. +func (f *feedFetcher) privateFeedFor(ctx context.Context, u *url.URL, proxied bool) *privateFeed { + if ip, err := netip.ParseAddr(u.Hostname()); err == nil { + if isPublicAddr(ip) { + return nil + } + return &privateFeed{} + } + if proxied { + return nil + } + addrs, err := f.lookup(ctx, u.Hostname()) + if err != nil || len(addrs) == 0 || slices.ContainsFunc(addrs, isPublicAddr) { + return nil + } + return &privateFeed{host: u.Hostname(), addrs: addrs} +} + +// refuseNonPublicDial is checked against the resolved address of every direct connection a public +// feed's fetch makes, so a redirect it serves cannot point the server at loopback, the local +// network, or a cloud metadata endpoint. +func refuseNonPublicDial(ctx context.Context, _, address string, _ syscall.RawConn) error { + if ctx.Value(privateFeedKey{}) != nil { + return nil + } + host, _, err := net.SplitHostPort(address) + if err != nil { + return err + } + ip, err := netip.ParseAddr(host) + if err != nil { + return err + } + if !isPublicAddr(ip) { + return fmt.Errorf("%w: %s", errFeedAddressNotPublic, ip) + } + return nil +} type normalizedFeedItem struct { identity string @@ -54,14 +122,52 @@ type feedFetcher struct { parser *gofeed.Parser now func() time.Time userAgent string + proxy func(*http.Request) (*url.URL, error) + lookup func(ctx context.Context, host string) ([]netip.Addr, error) } func newFeedFetcher(userAgent string) *feedFetcher { + return newFeedFetcherVia(userAgent, http.ProxyFromEnvironment) +} + +// newFeedFetcherVia fetches through the proxies proxy chooses. A proxy is the operator's choice, +// and is commonly on the local network, so connections to it are not checked; through it the +// server cannot see what a host name resolves to, so for a public feed only an address written +// into a URL is. +func newFeedFetcherVia(userAgent string, proxy func(*http.Request) (*url.URL, error)) *feedFetcher { if userAgent == "" { userAgent = "mobius-hotline-server" } + var proxies sync.Map + direct := &net.Dialer{Timeout: 30 * time.Second, KeepAlive: 30 * time.Second} + guarded := &net.Dialer{Timeout: 30 * time.Second, KeepAlive: 30 * time.Second, ControlContext: refuseNonPublicDial} + transport := http.DefaultTransport.(*http.Transport).Clone() + transport.Proxy = func(req *http.Request) (*url.URL, error) { + proxyURL, err := proxy(req) + if err != nil || proxyURL == nil { + return proxyURL, err + } + ip, err := netip.ParseAddr(req.URL.Hostname()) + if err == nil && !isPublicAddr(ip) && req.Context().Value(privateFeedKey{}) == nil { + return nil, fmt.Errorf("%w: %s", errFeedAddressNotPublic, ip) + } + proxies.Store(proxyAddr(proxyURL), struct{}{}) + return proxyURL, nil + } + transport.DialContext = func(ctx context.Context, network, address string) (net.Conn, error) { + if _, ok := proxies.Load(address); ok { + return direct.DialContext(ctx, network, address) + } + if pin, _ := ctx.Value(privateFeedKey{}).(*privateFeed); pin != nil && pin.addrs != nil { + if host, port, err := net.SplitHostPort(address); err == nil && host == pin.host { + return dialFirst(ctx, direct, network, pin.addrs, port) + } + } + return guarded.DialContext(ctx, network, address) + } return &feedFetcher{ client: &http.Client{ + Transport: transport, CheckRedirect: func(req *http.Request, via []*http.Request) error { if len(via) >= 10 { return errors.New("stopped after 10 redirects") @@ -69,6 +175,9 @@ func newFeedFetcher(userAgent string) *feedFetcher { if req.URL.Scheme != "http" && req.URL.Scheme != "https" { return fmt.Errorf("redirected to unsupported URL scheme %q", req.URL.Scheme) } + if via[len(via)-1].URL.Scheme == "https" && req.URL.Scheme == "http" { + return errors.New("redirected from https to http") + } if req.URL.User != nil { return errors.New("redirected to a URL containing credentials") } @@ -78,7 +187,32 @@ func newFeedFetcher(userAgent string) *feedFetcher { parser: gofeed.NewParser(), now: time.Now, userAgent: userAgent, + proxy: proxy, + lookup: func(ctx context.Context, host string) ([]netip.Addr, error) { + return net.DefaultResolver.LookupNetIP(ctx, "ip", host) + }, + } +} + +// dialFirst connects to the first of addrs that answers. +func dialFirst(ctx context.Context, d *net.Dialer, network string, addrs []netip.Addr, port string) (net.Conn, error) { + var err error + for _, addr := range addrs { + var conn net.Conn + if conn, err = d.DialContext(ctx, network, net.JoinHostPort(addr.Unmap().String(), port)); err == nil { + return conn, nil + } } + return nil, err +} + +// proxyAddr is the address the transport dials for proxyURL. +func proxyAddr(proxyURL *url.URL) string { + if port := proxyURL.Port(); port != "" { + return net.JoinHostPort(proxyURL.Hostname(), port) + } + port := map[string]string{"http": "80", "https": "443", "socks5": "1080", "socks5h": "1080"}[proxyURL.Scheme] + return net.JoinHostPort(proxyURL.Hostname(), port) } func (f *feedFetcher) fetch(ctx context.Context, sourceURL string, validators feedHTTPValidators) (feedFetchResult, error) { @@ -89,6 +223,10 @@ func (f *feedFetcher) fetch(ctx context.Context, sourceURL string, validators fe if err != nil { return feedFetchResult{}, errors.New("create request from configured feed URL") } + proxyURL, _ := f.proxy(req) + if pin := f.privateFeedFor(ctx, req.URL, proxyURL != nil); pin != nil { + req = req.WithContext(context.WithValue(ctx, privateFeedKey{}, pin)) + } req.Header.Set("Accept", "application/rss+xml, application/atom+xml, application/xml;q=0.9, text/xml;q=0.8") req.Header.Set("User-Agent", f.userAgent) if validators.etag != "" { diff --git a/internal/mobius/feed_news_test.go b/internal/mobius/feed_news_test.go index 037d1a0..d50cad9 100644 --- a/internal/mobius/feed_news_test.go +++ b/internal/mobius/feed_news_test.go @@ -2,11 +2,16 @@ package mobius import ( "bytes" + "context" "errors" "fmt" "io" "log/slog" + "net" "net/http" + "net/http/httptest" + "net/netip" + "net/url" "os" "path/filepath" "strings" @@ -425,6 +430,129 @@ func TestFeedFetcherLimitsAndConditional304(t *testing.T) { }) } +func TestIsPublicAddr(t *testing.T) { + tests := []struct { + addr string + want bool + }{ + {"93.184.215.14", true}, + {"2606:4700::6810:84e5", true}, + {"127.0.0.1", false}, + {"::1", false}, + {"10.1.2.3", false}, + {"172.16.0.1", false}, + {"192.168.1.1", false}, + {"169.254.169.254", false}, + {"100.64.0.1", false}, + {"0.0.0.0", false}, + {"::", false}, + {"fd00::1", false}, + {"fe80::1", false}, + {"224.0.0.1", false}, + {"::ffff:127.0.0.1", false}, + {"::ffff:192.168.1.1", false}, + } + for _, tt := range tests { + assert.Equal(t, tt.want, isPublicAddr(netip.MustParseAddr(tt.addr)), tt.addr) + } +} + +// publicFeedURL is a literal public address, so whether the feed is public never depends on DNS. +const publicFeedURL = "http://93.184.215.14/feed.xml" + +func TestFeedFetcherKeepsAPublicFeedOffTheLocalNetwork(t *testing.T) { + var hits atomic.Int32 + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, request *http.Request) { + if request.URL.Path == "/moved" { + http.Redirect(w, request, "/feed.xml", http.StatusFound) + return + } + hits.Add(1) + _, _ = io.WriteString(w, rssFixture("Release", "release-1", "Mon, 02 Jan 2006 15:04:05 GMT")) + })) + defer server.Close() + + t.Run("a feed on the local network is fetched as configured", func(t *testing.T) { + result, err := newFeedFetcher("test").fetch(t.Context(), server.URL+"/moved", feedHTTPValidators{}) + require.NoError(t, err) + assert.Len(t, result.items, 1) + }) + + t.Run("a public feed cannot redirect into it", func(t *testing.T) { + before := hits.Load() + fetcher := newFeedFetcher("test") + dialing := fetcher.client.Transport + fetcher.client.Transport = roundTripFunc(func(request *http.Request) (*http.Response, error) { + if request.URL.String() == publicFeedURL { + return testFeedResponse(http.StatusFound, "", map[string]string{"Location": server.URL + "/feed.xml"}), nil + } + return dialing.RoundTrip(request) + }) + _, err := fetcher.fetch(t.Context(), publicFeedURL, feedHTTPValidators{}) + assert.ErrorIs(t, err, errFeedAddressNotPublic) + assert.Equal(t, before, hits.Load()) + }) +} + +// A name server can answer loopback to the lookup that judges the feed and its real address to +// the connection, then redirect into the network. The judging lookup's answers are where the +// feed connects, so the server behind the second answer is never reached. +func TestFeedFetcherConnectsWhereTheJudgingLookupSaid(t *testing.T) { + var hits atomic.Int32 + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, request *http.Request) { + hits.Add(1) + http.Redirect(w, request, "http://169.254.169.254/latest/meta-data/", http.StatusFound) + })) + defer server.Close() + _, port, err := net.SplitHostPort(strings.TrimPrefix(server.URL, "http://")) + require.NoError(t, err) + + fetcher := newFeedFetcher("test") + fetcher.lookup = func(context.Context, string) ([]netip.Addr, error) { + return []netip.Addr{netip.MustParseAddr("127.0.0.2")}, nil + } + _, err = fetcher.fetch(t.Context(), "http://localhost:"+port+"/feed.xml", feedHTTPValidators{}) + require.Error(t, err) + assert.Zero(t, hits.Load(), "connected to the second answer, not the first") +} + +func TestFeedFetcherConnectsToAPrivateProxy(t *testing.T) { + var proxied []string + proxy := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, request *http.Request) { + proxied = append(proxied, request.URL.String()) + if request.URL.String() == publicFeedURL { + http.Redirect(w, request, "http://169.254.169.254/feed.xml", http.StatusFound) + return + } + _, _ = io.WriteString(w, rssFixture("Release", "release-1", "Mon, 02 Jan 2006 15:04:05 GMT")) + })) + defer proxy.Close() + proxyURL, err := url.Parse(proxy.URL) + require.NoError(t, err) + fetcher := newFeedFetcherVia("test", http.ProxyURL(proxyURL)) + + result, err := fetcher.fetch(t.Context(), "http://10.1.2.3/feed.xml", feedHTTPValidators{}) + require.NoError(t, err, "a feed on the local network, through the proxy") + assert.Len(t, result.items, 1) + + _, err = fetcher.fetch(t.Context(), publicFeedURL, feedHTTPValidators{}) + assert.ErrorIs(t, err, errFeedAddressNotPublic) + assert.Equal(t, []string{"http://10.1.2.3/feed.xml", publicFeedURL}, proxied, + "the redirect to a metadata address never reaches the proxy") +} + +func TestFeedFetcherRefusesHTTPSDowngrade(t *testing.T) { + fetcher := newFeedFetcher("test") + var requested []string + fetcher.client.Transport = roundTripFunc(func(request *http.Request) (*http.Response, error) { + requested = append(requested, request.URL.String()) + return testFeedResponse(http.StatusFound, "", map[string]string{"Location": "http://feeds.example.com/feed.xml"}), nil + }) + _, err := fetcher.fetch(t.Context(), "https://feeds.example.com/feed.xml", feedHTTPValidators{}) + require.ErrorContains(t, err, "redirected from https to http") + assert.Equal(t, []string{"https://feeds.example.com/feed.xml"}, requested) +} + func feedFixture(t *testing.T, name string) string { t.Helper() data, err := os.ReadFile(filepath.Join("testdata", name))