From fa5caac1e79b95a5d1479596eff021f4d0f822e7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ste=CC=81phane=20Duchesneau?= Date: Thu, 12 Nov 2026 17:56:21 -0500 Subject: [PATCH] Start Azure walks at the starting point WalkFrom listed the whole prefix and skipped the files before the starting point, which times out on stores with millions of files. --- CHANGELOG.md | 2 ++ azure.go | 32 ++++++++++++++++++++++++++++++-- azure_test.go | 48 ++++++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 80 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 5acbf8b..cead260 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -26,6 +26,8 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), ### Changed +* Azure store: `WalkFrom` and `WalkFromTo` have Azure start the listing at the starting point, with the `startFrom` parameter of List Blobs, instead of listing every file under the prefix and skipping those before it. A walk from near the end of a store holding millions of files now takes a few calls instead of one per 5000 files before the starting point. The names are still checked against both bounds, so an endpoint that ignores `startFrom` returns the same files, at the old cost. + * zstd stores read with decoders kept in a process-wide `default` pool unless `compression_config` names another pool, and those decoders work on the reading goroutine instead of 4 background ones. Reading 100 MiB `best/32` objects 10 at a time, that used 6% less CPU and 53% less peak heap; on 2 MiB objects, 42% less CPU. `compression_config=pool=none` brings back a decoder per object with 4 background goroutines. Reading from a zstd reader after closing it now returns an error. * Bumped `github.com/klauspost/compress` from v1.10.2 to v1.20.1, which requires Go 1.25. In v1.10.2 the `better` and `best` levels were aliases of `default`. diff --git a/azure.go b/azure.go index 2bd25cf..d81bffd 100644 --- a/azure.go +++ b/azure.go @@ -314,15 +314,37 @@ func (s *AzureStore) PushLocalFile(ctx context.Context, localFile, toBaseName st return remove() } +// WalkFrom has Azure start the listing at startingPoint (the startFrom parameter of List +// Blobs), so it does not page through the files before it. The names listed are still +// checked against startingPoint, for an endpoint that ignores startFrom. func (s *AzureStore) WalkFrom(ctx context.Context, prefix, startingPoint string, f func(filename string) (err error)) error { - return commonWalkFrom(s, ctx, prefix, startingPoint, f) + return commonWalkFrom(azureWalkStartingAt{s, startingPoint}, ctx, prefix, startingPoint, f) } +// WalkFromTo starts the listing at startingPoint like WalkFrom does. Azure only takes an +// end bound with the Arrow response format, so the walk stops at exclusiveEndPoint on +// this side. func (s *AzureStore) WalkFromTo(ctx context.Context, prefix, startingPoint, exclusiveEndPoint string, f func(filename string) (err error)) error { - return commonWalkFromTo(s, ctx, prefix, startingPoint, exclusiveEndPoint, f) + return commonWalkFromTo(azureWalkStartingAt{s, startingPoint}, ctx, prefix, startingPoint, exclusiveEndPoint, f) +} + +// azureWalkStartingAt is the store whose Walk lists from startingPoint, for the common +// WalkFrom and WalkFromTo to check the bounds and filter the names on top of it. +type azureWalkStartingAt struct { + *AzureStore + startingPoint string +} + +func (w azureWalkStartingAt) Walk(ctx context.Context, prefix string, f func(filename string) (err error)) error { + return w.walk(ctx, prefix, w.startingPoint, f) } func (s *AzureStore) Walk(ctx context.Context, prefix string, f func(filename string) (err error)) error { + return s.walk(ctx, prefix, "", f) +} + +// walk lists the files under prefix, from startingPoint when it is not empty. +func (s *AzureStore) walk(ctx context.Context, prefix, startingPoint string, f func(filename string) (err error)) error { p := strings.TrimLeft(s.baseURL.Path, "/") + "/" if prefix != "" { p = filepath.Join(p, prefix) @@ -336,6 +358,12 @@ func (s *AzureStore) Walk(ctx context.Context, prefix string, f func(filename st listOptions := &azblob.ListBlobsFlatOptions{ Prefix: &p, } + if startingPoint != "" { + // startFrom is a blob name, inclusive, and the extension is left out so a name + // that extends startingPoint, like "a-b" for "a", is not skipped + startFrom := path.Join(strings.TrimLeft(s.baseURL.Path, "/"), startingPoint) + listOptions.StartFrom = &startFrom + } pager := s.client.NewListBlobsFlatPager(s.containerName, listOptions) diff --git a/azure_test.go b/azure_test.go index 8741333..51a90a9 100644 --- a/azure_test.go +++ b/azure_test.go @@ -4,6 +4,8 @@ import ( "bytes" "context" "fmt" + "net/http" + "net/http/httptest" "net/url" "os" "testing" @@ -119,3 +121,49 @@ func TestAzureStoreWriteObject(t *testing.T) { require.NoError(t, err) assert.ElementsMatch(t, []string{}, readFiles) } + +func TestAzureStoreWalkFromSendsStartFrom(t *testing.T) { + var startFroms []string + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + startFroms = append(startFroms, r.URL.Query().Get("startFrom")) + + // "base/00000001" is below the starting point, as an endpoint ignoring startFrom + // would return it + w.Header().Set("Content-Type", "application/xml") + fmt.Fprint(w, ` + +base/00000001.dbin +base/00000002.dbin +base/00000003.dbin +`) + })) + defer server.Close() + + t.Setenv("AZURE_STORAGE_ENDPOINT", server.URL) + t.Setenv("AZURE_STORAGE_KEY", "a2V5") + + base, err := url.Parse("az://account.cont/base") + require.NoError(t, err) + s, err := NewAzureStore(base, "dbin", "", false) + require.NoError(t, err) + + var seen []string + err = s.WalkFrom(context.Background(), "", "00000002", func(filename string) error { + seen = append(seen, filename) + return nil + }) + require.NoError(t, err) + assert.Equal(t, []string{"00000002", "00000003"}, seen) + + seen = nil + err = s.WalkFromTo(context.Background(), "", "00000002", "00000003", func(filename string) error { + seen = append(seen, filename) + return nil + }) + require.NoError(t, err) + assert.Equal(t, []string{"00000002"}, seen) + + require.NoError(t, s.Walk(context.Background(), "", func(string) error { return nil })) + + assert.Equal(t, []string{"base/00000002", "base/00000002", ""}, startFroms) +}