Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions CHANGELOG.md
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,8 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),

### Changed

* Azure store: `WalkFrom` and `WalkFromTo` have Azure start the listing at the starting point, with the `startFrom` parameter of List Blobs, instead of listing every file under the prefix and skipping those before it. A walk from near the end of a store holding millions of files now takes a few calls instead of one per 5000 files before the starting point. The names are still checked against both bounds, so an endpoint that ignores `startFrom` returns the same files, at the old cost.

* zstd stores read with decoders kept in a process-wide `default` pool unless `compression_config` names another pool, and those decoders work on the reading goroutine instead of 4 background ones. Reading 100 MiB `best/32` objects 10 at a time, that used 6% less CPU and 53% less peak heap; on 2 MiB objects, 42% less CPU. `compression_config=pool=none` brings back a decoder per object with 4 background goroutines. Reading from a zstd reader after closing it now returns an error.

* Bumped `github.com/klauspost/compress` from v1.10.2 to v1.20.1, which requires Go 1.25. In v1.10.2 the `better` and `best` levels were aliases of `default`.
Expand Down
32 changes: 30 additions & 2 deletions azure.go
Original file line number Diff line number Diff line change
Expand Up @@ -314,15 +314,37 @@ func (s *AzureStore) PushLocalFile(ctx context.Context, localFile, toBaseName st
return remove()
}

// WalkFrom has Azure start the listing at startingPoint (the startFrom parameter of List
// Blobs), so it does not page through the files before it. The names listed are still
// checked against startingPoint, for an endpoint that ignores startFrom.
func (s *AzureStore) WalkFrom(ctx context.Context, prefix, startingPoint string, f func(filename string) (err error)) error {
return commonWalkFrom(s, ctx, prefix, startingPoint, f)
return commonWalkFrom(azureWalkStartingAt{s, startingPoint}, ctx, prefix, startingPoint, f)
}

// WalkFromTo starts the listing at startingPoint like WalkFrom does. Azure only takes an
// end bound with the Arrow response format, so the walk stops at exclusiveEndPoint on
// this side.
func (s *AzureStore) WalkFromTo(ctx context.Context, prefix, startingPoint, exclusiveEndPoint string, f func(filename string) (err error)) error {
return commonWalkFromTo(s, ctx, prefix, startingPoint, exclusiveEndPoint, f)
return commonWalkFromTo(azureWalkStartingAt{s, startingPoint}, ctx, prefix, startingPoint, exclusiveEndPoint, f)
}

// azureWalkStartingAt is the store whose Walk lists from startingPoint, for the common
// WalkFrom and WalkFromTo to check the bounds and filter the names on top of it.
type azureWalkStartingAt struct {
*AzureStore
startingPoint string
}

func (w azureWalkStartingAt) Walk(ctx context.Context, prefix string, f func(filename string) (err error)) error {
return w.walk(ctx, prefix, w.startingPoint, f)
}

func (s *AzureStore) Walk(ctx context.Context, prefix string, f func(filename string) (err error)) error {
return s.walk(ctx, prefix, "", f)
}

// walk lists the files under prefix, from startingPoint when it is not empty.
func (s *AzureStore) walk(ctx context.Context, prefix, startingPoint string, f func(filename string) (err error)) error {
p := strings.TrimLeft(s.baseURL.Path, "/") + "/"
if prefix != "" {
p = filepath.Join(p, prefix)
Expand All @@ -336,6 +358,12 @@ func (s *AzureStore) Walk(ctx context.Context, prefix string, f func(filename st
listOptions := &azblob.ListBlobsFlatOptions{
Prefix: &p,
}
if startingPoint != "" {
// startFrom is a blob name, inclusive, and the extension is left out so a name
// that extends startingPoint, like "a-b" for "a", is not skipped
startFrom := path.Join(strings.TrimLeft(s.baseURL.Path, "/"), startingPoint)
listOptions.StartFrom = &startFrom
}

pager := s.client.NewListBlobsFlatPager(s.containerName, listOptions)

Expand Down
48 changes: 48 additions & 0 deletions azure_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,8 @@ import (
"bytes"
"context"
"fmt"
"net/http"
"net/http/httptest"
"net/url"
"os"
"testing"
Expand Down Expand Up @@ -119,3 +121,49 @@ func TestAzureStoreWriteObject(t *testing.T) {
require.NoError(t, err)
assert.ElementsMatch(t, []string{}, readFiles)
}

func TestAzureStoreWalkFromSendsStartFrom(t *testing.T) {
var startFroms []string
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
startFroms = append(startFroms, r.URL.Query().Get("startFrom"))

// "base/00000001" is below the starting point, as an endpoint ignoring startFrom
// would return it
w.Header().Set("Content-Type", "application/xml")
fmt.Fprint(w, `<?xml version="1.0" encoding="utf-8"?>
<EnumerationResults ContainerName="cont"><Blobs>
<Blob><Name>base/00000001.dbin</Name><Properties/></Blob>
<Blob><Name>base/00000002.dbin</Name><Properties/></Blob>
<Blob><Name>base/00000003.dbin</Name><Properties/></Blob>
</Blobs><NextMarker/></EnumerationResults>`)
}))
defer server.Close()

t.Setenv("AZURE_STORAGE_ENDPOINT", server.URL)
t.Setenv("AZURE_STORAGE_KEY", "a2V5")

base, err := url.Parse("az://account.cont/base")
require.NoError(t, err)
s, err := NewAzureStore(base, "dbin", "", false)
require.NoError(t, err)

var seen []string
err = s.WalkFrom(context.Background(), "", "00000002", func(filename string) error {
seen = append(seen, filename)
return nil
})
require.NoError(t, err)
assert.Equal(t, []string{"00000002", "00000003"}, seen)

seen = nil
err = s.WalkFromTo(context.Background(), "", "00000002", "00000003", func(filename string) error {
seen = append(seen, filename)
return nil
})
require.NoError(t, err)
assert.Equal(t, []string{"00000002"}, seen)

require.NoError(t, s.Walk(context.Background(), "", func(string) error { return nil }))

assert.Equal(t, []string{"base/00000002", "base/00000002", ""}, startFroms)
}
Loading