From 2581a3141a9f701064f73ba6710f81570473f1f6 Mon Sep 17 00:00:00 2001 From: Syrup Date: Sun, 11 Oct 2026 19:12:28 +0700 Subject: [PATCH 01/48] fix(storage): stop hardcoding rclone config path --- internal/storage/rclone.go | 22 ++++++++++++++-------- internal/storage/rclone_test.go | 22 ++++++++++++++++++++++ 2 files changed, 36 insertions(+), 8 deletions(-) create mode 100644 internal/storage/rclone_test.go diff --git a/internal/storage/rclone.go b/internal/storage/rclone.go index 2f72e77..994763d 100644 --- a/internal/storage/rclone.go +++ b/internal/storage/rclone.go @@ -9,7 +9,8 @@ import ( ) type RcloneBackend struct { - Remote string + Remote string + // ConfigPath is optional. When empty, rclone uses its own default config. ConfigPath string } @@ -21,16 +22,21 @@ type Backend interface { } func NewRcloneBackend(remote string) *RcloneBackend { - return &RcloneBackend{ - Remote: remote, - ConfigPath: "/home/keraki/.config/rclone/rclone.conf", + return &RcloneBackend{Remote: remote} +} + +func rcloneArgs(configPath string, base ...string) []string { + args := append([]string{}, base...) + if configPath != "" { + args = append(args, "--config", configPath) } + return args } func (r *RcloneBackend) Upload(remotePath string, in io.Reader) error { fullDest := fmt.Sprintf("%s/%s", r.Remote, remotePath) - cmd := exec.Command("rclone", "rcat", "--config", r.ConfigPath, fullDest) + cmd := exec.Command("rclone", rcloneArgs(r.ConfigPath, "rcat", fullDest)...) cmd.Stdin = in @@ -46,7 +52,7 @@ func (r *RcloneBackend) Upload(remotePath string, in io.Reader) error { func (r *RcloneBackend) Download(remotePath string, out io.Writer) error { fullSrc := fmt.Sprintf("%s/%s", r.Remote, remotePath) - cmd := exec.Command("rclone", "cat", "--config", r.ConfigPath, fullSrc) + cmd := exec.Command("rclone", rcloneArgs(r.ConfigPath, "cat", fullSrc)...) cmd.Stdout = out @@ -68,7 +74,7 @@ func (r *RcloneBackend) ListFiles(prefix []string) ([]byte, error) { remotePath = fmt.Sprintf("%s/%s", r.Remote, strings.Join(prefix, "/")) } - cmd := exec.Command("rclone", "lsjson", "--config", r.ConfigPath, remotePath) + cmd := exec.Command("rclone", rcloneArgs(r.ConfigPath, "lsjson", remotePath)...) var stdout, stderr bytes.Buffer cmd.Stdout = &stdout @@ -84,7 +90,7 @@ func (r *RcloneBackend) ListFiles(prefix []string) ([]byte, error) { func (r *RcloneBackend) Delete(remotePath string) error { fullPath := fmt.Sprintf("%s/%s", r.Remote, remotePath) - cmd := exec.Command("rclone", "purge", "--config", r.ConfigPath, fullPath) + cmd := exec.Command("rclone", rcloneArgs(r.ConfigPath, "purge", fullPath)...) var stderr bytes.Buffer cmd.Stderr = &stderr diff --git a/internal/storage/rclone_test.go b/internal/storage/rclone_test.go new file mode 100644 index 0000000..785a9af --- /dev/null +++ b/internal/storage/rclone_test.go @@ -0,0 +1,22 @@ +package storage + +import ( + "reflect" + "testing" +) + +func TestRcloneArgsOmitsConfigWhenEmpty(t *testing.T) { + got := rcloneArgs("", "cat", "remote:file") + want := []string{"cat", "remote:file"} + if !reflect.DeepEqual(got, want) { + t.Fatalf("rcloneArgs() = %v, want %v", got, want) + } +} + +func TestRcloneArgsAppendsConfigWhenSet(t *testing.T) { + got := rcloneArgs("/etc/rclone.conf", "rcat", "remote:file") + want := []string{"rcat", "remote:file", "--config", "/etc/rclone.conf"} + if !reflect.DeepEqual(got, want) { + t.Fatalf("rcloneArgs() = %v, want %v", got, want) + } +} From 1ce4825e09d87704b16d6b553abcd5d4892caf0f Mon Sep 17 00:00:00 2001 From: Syrup Date: Sun, 11 Oct 2026 19:17:45 +0700 Subject: [PATCH 02/48] feat(crypto): add zstd + XChaCha20 chunk envelope --- go.mod | 9 ++-- go.sum | 18 ++++--- internal/crypto/envelope.go | 91 ++++++++++++++++++++++++++++++++ internal/crypto/envelope_test.go | 88 ++++++++++++++++++++++++++++++ 4 files changed, 194 insertions(+), 12 deletions(-) create mode 100644 internal/crypto/envelope.go create mode 100644 internal/crypto/envelope_test.go diff --git a/go.mod b/go.mod index 6555264..da01d64 100644 --- a/go.mod +++ b/go.mod @@ -6,7 +6,9 @@ require ( filippo.io/age v1.3.1 github.com/alecthomas/kong v1.15.0 github.com/google/uuid v1.6.0 + github.com/klauspost/compress v1.20.1 github.com/pterm/pterm v0.12.83 + golang.org/x/crypto v0.58.0 ) require ( @@ -20,8 +22,7 @@ require ( github.com/lithammer/fuzzysearch v1.1.8 // indirect github.com/mattn/go-runewidth v0.0.20 // indirect github.com/xo/terminfo v0.0.0-20220910002029-abceb7e1c41e // indirect - golang.org/x/crypto v0.45.0 // indirect - golang.org/x/sys v0.41.0 // indirect - golang.org/x/term v0.40.0 // indirect - golang.org/x/text v0.34.0 // indirect + golang.org/x/sys v0.49.0 // indirect + golang.org/x/term v0.47.0 // indirect + golang.org/x/text v0.43.0 // indirect ) diff --git a/go.sum b/go.sum index 00c5311..9a41307 100644 --- a/go.sum +++ b/go.sum @@ -46,6 +46,8 @@ github.com/gookit/color v1.6.0 h1:JjJXBTk1ETNyqyilJhkTXJYYigHG24TM9Xa2M1xAhRA= github.com/gookit/color v1.6.0/go.mod h1:9ACFc7/1IpHGBW8RwuDm/0YEnhg3dwwXpoMsmtyHfjs= github.com/hexops/gotextdiff v1.0.3 h1:gitA9+qJrrTCsiCl7+kh75nPqQt1cx4ZkudSTLoUqJM= github.com/hexops/gotextdiff v1.0.3/go.mod h1:pSWU5MAI3yDq+fZBTazCSJysOMbxWL1BSow5/V2vxeg= +github.com/klauspost/compress v1.20.1 h1:T7kKElXUMXrUJ2E9QhQhxFtcK5rPyLdsGZvdbLMPdiQ= +github.com/klauspost/compress v1.20.1/go.mod h1:LUdAzn7YLVvxLpc7y3V1m40wESHTgc1422pwwBSKYuI= github.com/klauspost/cpuid/v2 v2.0.9/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= github.com/klauspost/cpuid/v2 v2.0.10/go.mod h1:g2LTdtYhdyuGPqyWyv7qRAmj1WBqxuObKfj5c0PQa7c= github.com/klauspost/cpuid/v2 v2.0.12/go.mod h1:g2LTdtYhdyuGPqyWyv7qRAmj1WBqxuObKfj5c0PQa7c= @@ -82,8 +84,8 @@ github.com/xo/terminfo v0.0.0-20220910002029-abceb7e1c41e/go.mod h1:RbqR21r5mrJu github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= -golang.org/x/crypto v0.45.0 h1:jMBrvKuj23MTlT0bQEOBcAE0mjg8mK9RXFhRH6nyF3Q= -golang.org/x/crypto v0.45.0/go.mod h1:XTGrrkGJve7CYK7J8PEww4aY7gM3qMCElcJQ8n8JdX4= +golang.org/x/crypto v0.58.0 h1:COkYLr4k7nDI4r6QtMML2AfzeIswWHAJthLS6K/M/54= +golang.org/x/crypto v0.58.0/go.mod h1:Xh+kl5A+M0gMiWDytH2Hhr5Z7WbNCpv/eB/EFgfSk+w= golang.org/x/exp v0.0.0-20220909182711-5c715a9e8561 h1:MDc5xs78ZrZr3HMQugiXOAkSZtfTpbJLDr/lwfgO53E= golang.org/x/exp v0.0.0-20220909182711-5c715a9e8561/go.mod h1:cyybsKvd6eL0RnXn6p/Grxp8F5bW7iYuBgsNCOHpMYE= golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= @@ -106,22 +108,22 @@ golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.41.0 h1:Ivj+2Cp/ylzLiEU89QhWblYnOE9zerudt9Ftecq2C6k= -golang.org/x/sys v0.41.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks= +golang.org/x/sys v0.49.0 h1:XbzkgYJHdqh/8m2Uu0W/dQv8nktxx4BFHp1M0gROTXA= +golang.org/x/sys v0.49.0/go.mod h1:hNLxWAXmnKAxqDtdwIYC4bM9oQPEecfsnNMuSxOs3og= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210220032956-6a3ed077a48d/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210615171337-6886f2dfbf5b/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= -golang.org/x/term v0.40.0 h1:36e4zGLqU4yhjlmxEaagx2KuYbJq3EwY8K943ZsHcvg= -golang.org/x/term v0.40.0/go.mod h1:w2P8uVp06p2iyKKuvXIm7N/y0UCRt3UfJTfZ7oOpglM= +golang.org/x/term v0.47.0 h1:cpyOwQkr2Nzxe2p6KnrTG7Ei2wtzjsuSjhpXXepZ/BM= +golang.org/x/term v0.47.0/go.mod h1:efAfAuG6RKABvg1ArGebRkvX6LXul14SM67o2e4mUCY= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= -golang.org/x/text v0.34.0 h1:oL/Qq0Kdaqxa1KbNeMKwQq0reLCCaFtqu2eNuSeNHbk= -golang.org/x/text v0.34.0/go.mod h1:homfLqTYRFyVYemLBFl5GgL/DWEiH5wcsQ5gSh1yziA= +golang.org/x/text v0.43.0 h1:1QivrlhwAsnMOcqgOdl9my7s9OyZl8O5/UGUTJVXoWc= +golang.org/x/text v0.43.0/go.mod h1:hbSIYA/amXcRXTFSZqh8tWHEoqRvvbKPJ2uz/KU+a+s= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= diff --git a/internal/crypto/envelope.go b/internal/crypto/envelope.go new file mode 100644 index 0000000..aca58f7 --- /dev/null +++ b/internal/crypto/envelope.go @@ -0,0 +1,91 @@ +package crypto + +import ( + "crypto/hkdf" + "crypto/rand" + "crypto/sha256" + "encoding/hex" + "fmt" + + "github.com/klauspost/compress/zstd" + "golang.org/x/crypto/chacha20poly1305" +) + +const chunkKeyInfo = "gloak-chunk-v1" + +func deriveChunkKey(masterKey []byte, plainSHA256Hex string) ([]byte, error) { + salt, err := hex.DecodeString(plainSHA256Hex) + if err != nil || len(salt) != sha256.Size { + return nil, fmt.Errorf("invalid chunk hash: %q", plainSHA256Hex) + } + return hkdf.Key(sha256.New, masterKey, salt, chunkKeyInfo, chacha20poly1305.KeySize) +} + +// SealChunk compresses plain with zstd, then encrypts it with XChaCha20-Poly1305 +// under a key derived from masterKey. Output layout: 24-byte nonce || ciphertext. +func SealChunk(masterKey []byte, plainSHA256Hex string, plain []byte) ([]byte, error) { + key, err := deriveChunkKey(masterKey, plainSHA256Hex) + if err != nil { + return nil, err + } + compressed, err := zstdEncode(plain) + if err != nil { + return nil, fmt.Errorf("failed to compress chunk: %w", err) + } + aead, err := chacha20poly1305.NewX(key) + if err != nil { + return nil, fmt.Errorf("failed to init cipher: %w", err) + } + nonce := make([]byte, chacha20poly1305.NonceSizeX) + if _, err := rand.Read(nonce); err != nil { + return nil, fmt.Errorf("failed to generate nonce: %w", err) + } + sealed := make([]byte, 0, len(nonce)+len(compressed)+aead.Overhead()) + sealed = append(sealed, nonce...) + sealed = aead.Seal(sealed, nonce, compressed, []byte(plainSHA256Hex)) + return sealed, nil +} + +// OpenChunk reverses SealChunk and verifies both the AEAD tag and the chunk hash. +func OpenChunk(masterKey []byte, plainSHA256Hex string, sealed []byte) ([]byte, error) { + key, err := deriveChunkKey(masterKey, plainSHA256Hex) + if err != nil { + return nil, err + } + aead, err := chacha20poly1305.NewX(key) + if err != nil { + return nil, fmt.Errorf("failed to init cipher: %w", err) + } + if len(sealed) < chacha20poly1305.NonceSizeX+aead.Overhead() { + return nil, fmt.Errorf("chunk too short: %d bytes", len(sealed)) + } + nonce := sealed[:chacha20poly1305.NonceSizeX] + ciphertext := sealed[chacha20poly1305.NonceSizeX:] + compressed, err := aead.Open(nil, nonce, ciphertext, []byte(plainSHA256Hex)) + if err != nil { + return nil, fmt.Errorf("chunk authentication failed: %w", err) + } + plain, err := zstdDecode(compressed) + if err != nil { + return nil, fmt.Errorf("failed to decompress chunk: %w", err) + } + return plain, nil +} + +func zstdEncode(src []byte) ([]byte, error) { + enc, err := zstd.NewWriter(nil, zstd.WithEncoderLevel(zstd.SpeedDefault)) + if err != nil { + return nil, err + } + defer enc.Close() + return enc.EncodeAll(src, nil), nil +} + +func zstdDecode(src []byte) ([]byte, error) { + dec, err := zstd.NewReader(nil) + if err != nil { + return nil, err + } + defer dec.Close() + return dec.DecodeAll(src, nil) +} diff --git a/internal/crypto/envelope_test.go b/internal/crypto/envelope_test.go new file mode 100644 index 0000000..44b47f8 --- /dev/null +++ b/internal/crypto/envelope_test.go @@ -0,0 +1,88 @@ +package crypto + +import ( + "bytes" + "crypto/rand" + "crypto/sha256" + "encoding/hex" + "strings" + "testing" +) + +func sha256Hex(data []byte) string { + sum := sha256.Sum256(data) + return hex.EncodeToString(sum[:]) +} + +func TestSealOpenChunkRoundTrip(t *testing.T) { + master := make([]byte, 32) + if _, err := rand.Read(master); err != nil { + t.Fatalf("rand.Read: %v", err) + } + plain := []byte("payload with enough bytes to compress meaningfully, payload payload payload") + sum := sha256Hex(plain) + + sealed, err := SealChunk(master, sum, plain) + if err != nil { + t.Fatalf("SealChunk() error: %v", err) + } + got, err := OpenChunk(master, sum, sealed) + if err != nil { + t.Fatalf("OpenChunk() error: %v", err) + } + if !bytes.Equal(got, plain) { + t.Fatalf("OpenChunk() = %q, want %q", got, plain) + } +} + +func TestOpenChunkRejectsTamperedCiphertext(t *testing.T) { + master := []byte("0123456789abcdef0123456789abcdef") + plain := []byte("important data") + sum := sha256Hex(plain) + sealed, err := SealChunk(master, sum, plain) + if err != nil { + t.Fatalf("SealChunk() error: %v", err) + } + sealed[len(sealed)-1] ^= 0xff + if _, err := OpenChunk(master, sum, sealed); err == nil { + t.Fatal("OpenChunk() succeeded with tampered ciphertext") + } +} + +func TestOpenChunkRejectsWrongChunkHash(t *testing.T) { + master := []byte("0123456789abcdef0123456789abcdef") + plain := []byte("important data") + sealed, err := SealChunk(master, sha256Hex(plain), plain) + if err != nil { + t.Fatalf("SealChunk() error: %v", err) + } + wrongSum := sha256Hex([]byte("other data")) + if _, err := OpenChunk(master, wrongSum, sealed); err == nil { + t.Fatal("OpenChunk() succeeded with wrong chunk hash") + } +} + +func TestOpenChunkRejectsWrongMasterKey(t *testing.T) { + master := []byte("0123456789abcdef0123456789abcdef") + other := []byte("fedcba9876543210fedcba9876543210") + plain := []byte("important data") + sum := sha256Hex(plain) + sealed, err := SealChunk(master, sum, plain) + if err != nil { + t.Fatalf("SealChunk() error: %v", err) + } + if _, err := OpenChunk(other, sum, sealed); err == nil { + t.Fatal("OpenChunk() succeeded with wrong master key") + } +} + +func TestSealChunkRejectsInvalidChunkHash(t *testing.T) { + master := []byte("0123456789abcdef0123456789abcdef") + _, err := SealChunk(master, "not-hex", []byte("x")) + if err == nil { + t.Fatal("SealChunk() succeeded with invalid chunk hash") + } + if !strings.Contains(err.Error(), "invalid chunk hash") { + t.Fatalf("SealChunk() error = %v, want invalid chunk hash", err) + } +} From 59d1203e0bbba37bb13e4f34bf039672ede1bc6a Mon Sep 17 00:00:00 2001 From: Syrup Date: Sun, 11 Oct 2026 19:23:18 +0700 Subject: [PATCH 03/48] feat(crypto): add multi-recipient age encryption --- internal/crypto/multi.go | 46 +++++++++++++++++++++ internal/crypto/multi_test.go | 77 +++++++++++++++++++++++++++++++++++ 2 files changed, 123 insertions(+) create mode 100644 internal/crypto/multi.go create mode 100644 internal/crypto/multi_test.go diff --git a/internal/crypto/multi.go b/internal/crypto/multi.go new file mode 100644 index 0000000..85759d4 --- /dev/null +++ b/internal/crypto/multi.go @@ -0,0 +1,46 @@ +package crypto + +import ( + "bytes" + "crypto/sha256" + "encoding/hex" + "fmt" + + "filippo.io/age" +) + +// EncryptMulti encrypts plaintext to every recipient. Any matching identity +// can decrypt the result. +func EncryptMulti(recipientKeys []string, plaintext []byte) ([]byte, error) { + if len(recipientKeys) == 0 { + return nil, fmt.Errorf("no recipients given") + } + recipients := make([]age.Recipient, 0, len(recipientKeys)) + for _, key := range recipientKeys { + r, err := age.ParseX25519Recipient(key) + if err != nil { + return nil, fmt.Errorf("invalid recipient %q: %w", key, err) + } + recipients = append(recipients, r) + } + + var buf bytes.Buffer + w, err := age.Encrypt(&buf, recipients...) + if err != nil { + return nil, fmt.Errorf("failed to init age encryptor: %w", err) + } + if _, err := w.Write(plaintext); err != nil { + _ = w.Close() + return nil, fmt.Errorf("failed to write plaintext: %w", err) + } + if err := w.Close(); err != nil { + return nil, fmt.Errorf("failed to finish age encryption: %w", err) + } + return buf.Bytes(), nil +} + +// FingerprintRecipient returns a short stable identifier for a recipient key. +func FingerprintRecipient(recipient string) string { + sum := sha256.Sum256([]byte(recipient)) + return hex.EncodeToString(sum[:])[:16] +} diff --git a/internal/crypto/multi_test.go b/internal/crypto/multi_test.go new file mode 100644 index 0000000..d78d10e --- /dev/null +++ b/internal/crypto/multi_test.go @@ -0,0 +1,77 @@ +package crypto + +import ( + "bytes" + "io" + "os" + "path/filepath" + "testing" +) + +func generateTestKey(t *testing.T) (string, string) { + t.Helper() + pub, _, fileContent, err := GenerateKey() + if err != nil { + t.Fatalf("GenerateKey() error: %v", err) + } + path := filepath.Join(t.TempDir(), "identity.txt") + if err := os.WriteFile(path, []byte(fileContent), 0600); err != nil { + t.Fatalf("write identity: %v", err) + } + return pub, path +} + +func TestEncryptMultiOpensWithAnyRecipientIdentity(t *testing.T) { + pubA, identA := generateTestKey(t) + pubB, identB := generateTestKey(t) + _, identC := generateTestKey(t) + + plaintext := []byte(`{"hello":"world"}`) + sealed, err := EncryptMulti([]string{pubA, pubB}, plaintext) + if err != nil { + t.Fatalf("EncryptMulti() error: %v", err) + } + + for _, identPath := range []string{identA, identB} { + r, err := DecryptReader(identPath, bytes.NewReader(sealed)) + if err != nil { + t.Fatalf("DecryptReader(%s) error: %v", identPath, err) + } + got, err := io.ReadAll(r) + if err != nil { + t.Fatalf("ReadAll error: %v", err) + } + if !bytes.Equal(got, plaintext) { + t.Fatalf("decrypted = %q, want %q", got, plaintext) + } + } + + if r, err := DecryptReader(identC, bytes.NewReader(sealed)); err == nil { + if _, err := io.ReadAll(r); err == nil { + t.Fatal("DecryptReader succeeded with unrelated identity") + } + } +} + +func TestEncryptMultiRejectsNoRecipients(t *testing.T) { + if _, err := EncryptMulti(nil, []byte("x")); err == nil { + t.Fatal("EncryptMulti() succeeded with no recipients") + } +} + +func TestEncryptMultiRejectsInvalidRecipient(t *testing.T) { + if _, err := EncryptMulti([]string{"age1notavalidkey"}, []byte("x")); err == nil { + t.Fatal("EncryptMulti() succeeded with invalid recipient") + } +} + +func TestFingerprintRecipientIsStableAndShort(t *testing.T) { + pub, _ := generateTestKey(t) + fp := FingerprintRecipient(pub) + if fp != FingerprintRecipient(pub) { + t.Fatal("FingerprintRecipient() is not stable") + } + if len(fp) != 16 { + t.Fatalf("FingerprintRecipient() = %q, want 16 hex chars", fp) + } +} From 003ab9d3d31dc11e8ce89db091b8b71fb26646aa Mon Sep 17 00:00:00 2001 From: Syrup Date: Sun, 11 Oct 2026 19:38:50 +0700 Subject: [PATCH 04/48] feat(repo): add repository init, open, and path helpers --- internal/repo/config.go | 18 ++++ internal/repo/repo.go | 135 +++++++++++++++++++++++++++ internal/repo/repo_test.go | 118 +++++++++++++++++++++++ internal/storage/list.go | 17 ++++ internal/storage/memtest/memory.go | 144 +++++++++++++++++++++++++++++ 5 files changed, 432 insertions(+) create mode 100644 internal/repo/config.go create mode 100644 internal/repo/repo.go create mode 100644 internal/repo/repo_test.go create mode 100644 internal/storage/list.go create mode 100644 internal/storage/memtest/memory.go diff --git a/internal/repo/config.go b/internal/repo/config.go new file mode 100644 index 0000000..29daf17 --- /dev/null +++ b/internal/repo/config.go @@ -0,0 +1,18 @@ +package repo + +const ( + FormatVersion = "2.0" + ConfigPath = "repo.json" + KeysPrefix = "keys" + SnapshotsPrefix = "snapshots" + DataPrefix = "data" +) + +// Config is the plaintext repository descriptor. It carries no user data: +// version, creation time, recipient public keys, and chunk parameters. +type Config struct { + Version string `json:"version"` + CreatedAt string `json:"created_at"` + Recipients []string `json:"recipients"` + ChunkSize int64 `json:"chunk_size"` +} diff --git a/internal/repo/repo.go b/internal/repo/repo.go new file mode 100644 index 0000000..8193918 --- /dev/null +++ b/internal/repo/repo.go @@ -0,0 +1,135 @@ +package repo + +import ( + "bytes" + "crypto/rand" + "encoding/json" + "fmt" + "io" + "time" + + "filippo.io/age" + "github.com/google/uuid" + + "gloak/internal/crypto" + "gloak/internal/storage" +) + +const masterKeySize = 32 + +type Repo struct { + Backend storage.Backend + Config Config + MasterKey []byte +} + +// KeysPathFor returns the remote path of the wrapped master key for a recipient. +func KeysPathFor(recipient string) string { + return fmt.Sprintf("%s/%s.age", KeysPrefix, crypto.FingerprintRecipient(recipient)) +} + +// DataPath returns the content-addressed object path for a plaintext chunk hash. +func DataPath(plainSHA256Hex string) string { + return fmt.Sprintf("%s/%s/%s", DataPrefix, plainSHA256Hex[:2], plainSHA256Hex) +} + +// SnapshotPath returns the snapshot manifest path, rejecting anything that is +// not a valid UUID so a crafted id can never escape the snapshots/ prefix. +func SnapshotPath(snapshotID string) (string, error) { + if _, err := uuid.Parse(snapshotID); err != nil { + return "", fmt.Errorf("invalid snapshot id: %w", err) + } + return fmt.Sprintf("%s/%s.age", SnapshotsPrefix, snapshotID), nil +} + +// Init creates the repository on the backend: one wrapped master key per +// recipient first, repo.json last as the completion marker. +func Init(backend storage.Backend, recipients []string, chunkSize int64) ([]byte, Config, error) { + if len(recipients) == 0 { + return nil, Config{}, fmt.Errorf("at least one recipient is required") + } + for _, r := range recipients { + if _, err := age.ParseX25519Recipient(r); err != nil { + return nil, Config{}, fmt.Errorf("invalid recipient %q: %w", r, err) + } + } + if chunkSize <= 0 { + return nil, Config{}, fmt.Errorf("chunk size must be positive") + } + + master := make([]byte, masterKeySize) + if _, err := rand.Read(master); err != nil { + return nil, Config{}, fmt.Errorf("failed to generate master key: %w", err) + } + + cfg := Config{ + Version: FormatVersion, + CreatedAt: time.Now().UTC().Format(time.RFC3339), + Recipients: recipients, + ChunkSize: chunkSize, + } + + for _, recipient := range recipients { + wrapped, err := crypto.EncryptMulti([]string{recipient}, master) + if err != nil { + return nil, Config{}, fmt.Errorf("failed to wrap master key: %w", err) + } + if err := backend.Upload(KeysPathFor(recipient), bytes.NewReader(wrapped)); err != nil { + return nil, Config{}, fmt.Errorf("failed to upload wrapped key: %w", err) + } + } + + cfgBytes, err := json.MarshalIndent(cfg, "", " ") + if err != nil { + return nil, Config{}, fmt.Errorf("failed to marshal repo config: %w", err) + } + if err := backend.Upload(ConfigPath, bytes.NewReader(cfgBytes)); err != nil { + return nil, Config{}, fmt.Errorf("failed to upload repo.json: %w", err) + } + return master, cfg, nil +} + +// Open loads repo.json and unwraps the master key with the given identity. +func Open(backend storage.Backend, identityPath string) (*Repo, error) { + var buf bytes.Buffer + if err := backend.Download(ConfigPath, &buf); err != nil { + return nil, fmt.Errorf("not a gloak repository (%s missing): %w", ConfigPath, err) + } + + var cfg Config + if err := json.Unmarshal(buf.Bytes(), &cfg); err != nil { + return nil, fmt.Errorf("failed to parse %s: %w", ConfigPath, err) + } + if cfg.Version != FormatVersion { + return nil, fmt.Errorf("unsupported repository format %q (want %q)", cfg.Version, FormatVersion) + } + + raw, err := backend.ListFiles([]string{KeysPrefix}) + if err != nil { + return nil, fmt.Errorf("failed to list keys: %w", err) + } + entries, err := storage.ParseList(raw) + if err != nil { + return nil, fmt.Errorf("failed to parse keys listing: %w", err) + } + + for _, entry := range entries { + if entry.IsDir { + continue + } + var keyBuf bytes.Buffer + if err := backend.Download(fmt.Sprintf("%s/%s", KeysPrefix, entry.Path), &keyBuf); err != nil { + continue + } + reader, err := crypto.DecryptReader(identityPath, &keyBuf) + if err != nil { + continue + } + master, err := io.ReadAll(reader) + if err != nil || len(master) != masterKeySize { + continue + } + return &Repo{Backend: backend, Config: cfg, MasterKey: master}, nil + } + return nil, fmt.Errorf("identity cannot open this repository") +} diff --git a/internal/repo/repo_test.go b/internal/repo/repo_test.go new file mode 100644 index 0000000..c6a56ee --- /dev/null +++ b/internal/repo/repo_test.go @@ -0,0 +1,118 @@ +package repo + +import ( + "os" + "path/filepath" + "strings" + "testing" + + "gloak/internal/crypto" + "gloak/internal/storage/memtest" +) + +func testRecipient(t *testing.T) (string, string) { + t.Helper() + pub, _, fileContent, err := crypto.GenerateKey() + if err != nil { + t.Fatalf("GenerateKey() error: %v", err) + } + path := filepath.Join(t.TempDir(), "identity.txt") + if err := os.WriteFile(path, []byte(fileContent), 0600); err != nil { + t.Fatalf("write identity: %v", err) + } + return pub, path +} + +func TestInitWritesKeysAndConfig(t *testing.T) { + pubA, _ := testRecipient(t) + pubB, _ := testRecipient(t) + backend := memtest.NewMemoryBackend() + + master, cfg, err := Init(backend, []string{pubA, pubB}, 4096) + if err != nil { + t.Fatalf("Init() error: %v", err) + } + if len(master) != 32 { + t.Fatalf("master key length = %d, want 32", len(master)) + } + if cfg.Version != FormatVersion || cfg.ChunkSize != 4096 { + t.Fatalf("cfg = %+v", cfg) + } + if !backend.Has(ConfigPath) { + t.Fatal("repo.json not uploaded") + } + for _, pub := range []string{pubA, pubB} { + path := KeysPathFor(pub) + if !backend.Has(path) { + t.Fatalf("key file %s not uploaded", path) + } + } +} + +func TestOpenWithEachRecipientIdentity(t *testing.T) { + pubA, identA := testRecipient(t) + pubB, identB := testRecipient(t) + backend := memtest.NewMemoryBackend() + master, _, err := Init(backend, []string{pubA, pubB}, 4096) + if err != nil { + t.Fatalf("Init() error: %v", err) + } + + for _, identPath := range []string{identA, identB} { + r, err := Open(backend, identPath) + if err != nil { + t.Fatalf("Open(%s) error: %v", identPath, err) + } + if string(r.MasterKey) != string(master) { + t.Fatalf("Open(%s) master key mismatch", identPath) + } + } +} + +func TestOpenRejectsWrongIdentity(t *testing.T) { + pubA, _ := testRecipient(t) + _, identOther := testRecipient(t) + backend := memtest.NewMemoryBackend() + if _, _, err := Init(backend, []string{pubA}, 4096); err != nil { + t.Fatalf("Init() error: %v", err) + } + _, err := Open(backend, identOther) + if err == nil { + t.Fatal("Open() succeeded with unrelated identity") + } + if !strings.Contains(err.Error(), "cannot open") { + t.Fatalf("Open() error = %v, want cannot open", err) + } +} + +func TestOpenRejectsMissingConfig(t *testing.T) { + _, ident := testRecipient(t) + _, err := Open(memtest.NewMemoryBackend(), ident) + if err == nil { + t.Fatal("Open() succeeded on empty backend") + } + if !strings.Contains(err.Error(), ConfigPath) { + t.Fatalf("Open() error = %v, want repo.json mention", err) + } +} + +func TestDataPathUsesHashSharding(t *testing.T) { + got := DataPath(strings.Repeat("ab", 32)) + want := "data/ab/" + strings.Repeat("ab", 32) + if got != want { + t.Fatalf("DataPath() = %q, want %q", got, want) + } +} + +func TestSnapshotPathRejectsInvalidUUID(t *testing.T) { + if _, err := SnapshotPath("../escape"); err == nil { + t.Fatal("SnapshotPath() accepted invalid uuid") + } + got, err := SnapshotPath("7f91c6c7-7a0b-44aa-ae23-997b60e4e998") + if err != nil { + t.Fatalf("SnapshotPath() error: %v", err) + } + if got != "snapshots/7f91c6c7-7a0b-44aa-ae23-997b60e4e998.age" { + t.Fatalf("SnapshotPath() = %q", got) + } +} diff --git a/internal/storage/list.go b/internal/storage/list.go new file mode 100644 index 0000000..502fcd8 --- /dev/null +++ b/internal/storage/list.go @@ -0,0 +1,17 @@ +package storage + +import "encoding/json" + +// ListEntry is the subset of rclone lsjson output that gloak uses. +type ListEntry struct { + Path string `json:"Path"` + IsDir bool `json:"IsDir"` +} + +func ParseList(raw []byte) ([]ListEntry, error) { + var entries []ListEntry + if err := json.Unmarshal(raw, &entries); err != nil { + return nil, err + } + return entries, nil +} diff --git a/internal/storage/memtest/memory.go b/internal/storage/memtest/memory.go new file mode 100644 index 0000000..8ff024b --- /dev/null +++ b/internal/storage/memtest/memory.go @@ -0,0 +1,144 @@ +// Package memtest provides an in-memory storage.Backend for tests. +package memtest + +import ( + "encoding/json" + "fmt" + "io" + "sort" + "strings" + "sync" +) + +type MemoryBackend struct { + mu sync.Mutex + files map[string][]byte + uploads map[string]int +} + +func NewMemoryBackend() *MemoryBackend { + return &MemoryBackend{files: make(map[string][]byte), uploads: make(map[string]int)} +} + +func (m *MemoryBackend) Upload(remotePath string, in io.Reader) error { + data, err := io.ReadAll(in) + if err != nil { + return err + } + m.mu.Lock() + defer m.mu.Unlock() + m.files[remotePath] = append([]byte(nil), data...) + m.uploads[remotePath]++ + return nil +} + +func (m *MemoryBackend) Download(remotePath string, out io.Writer) error { + m.mu.Lock() + data, ok := m.files[remotePath] + m.mu.Unlock() + if !ok { + return fmt.Errorf("not found: %s", remotePath) + } + _, err := out.Write(data) + return err +} + +// ListFiles mimics rclone lsjson: Path is relative to the listed prefix, +// only direct children are returned, directories are marked IsDir. +func (m *MemoryBackend) ListFiles(prefix []string) ([]byte, error) { + p := strings.Trim(strings.Join(prefix, "/"), "/") + m.mu.Lock() + defer m.mu.Unlock() + + children := map[string]bool{} + for key := range m.files { + var rel string + switch { + case p == "": + rel = key + case strings.HasPrefix(key, p+"/"): + rel = strings.TrimPrefix(key, p+"/") + default: + continue + } + if idx := strings.Index(rel, "/"); idx >= 0 { + children[rel[:idx]] = true + } else { + children[rel] = false + } + } + + names := make([]string, 0, len(children)) + for name := range children { + names = append(names, name) + } + sort.Strings(names) + + entries := make([]map[string]any, 0, len(names)) + for _, name := range names { + entries = append(entries, map[string]any{ + "Path": name, + "Name": name, + "IsDir": children[name], + "Size": 0, + }) + } + return json.Marshal(entries) +} + +func (m *MemoryBackend) Delete(remotePath string) error { + m.mu.Lock() + defer m.mu.Unlock() + delete(m.files, remotePath) + return nil +} + +func (m *MemoryBackend) Has(path string) bool { + m.mu.Lock() + defer m.mu.Unlock() + _, ok := m.files[path] + return ok +} + +func (m *MemoryBackend) Put(path string, data []byte) { + m.mu.Lock() + defer m.mu.Unlock() + m.files[path] = append([]byte(nil), data...) +} + +func (m *MemoryBackend) Get(path string) []byte { + m.mu.Lock() + defer m.mu.Unlock() + return append([]byte(nil), m.files[path]...) +} + +func (m *MemoryBackend) UploadCount(path string) int { + m.mu.Lock() + defer m.mu.Unlock() + return m.uploads[path] +} + +func (m *MemoryBackend) TotalUploads(prefix string) int { + m.mu.Lock() + defer m.mu.Unlock() + total := 0 + for path, n := range m.uploads { + if strings.HasPrefix(path, prefix) { + total += n + } + } + return total +} + +func (m *MemoryBackend) KeysWithPrefix(prefix string) []string { + m.mu.Lock() + defer m.mu.Unlock() + var keys []string + for key := range m.files { + if strings.HasPrefix(key, prefix) { + keys = append(keys, key) + } + } + sort.Strings(keys) + return keys +} From 3ef8bc2254dcaca64e52b1db2b0d86922e8c92b4 Mon Sep 17 00:00:00 2001 From: Syrup Date: Sun, 11 Oct 2026 19:54:19 +0700 Subject: [PATCH 05/48] fix(repo): guard DataPath against short hashes --- internal/repo/repo.go | 7 ++++++- internal/repo/repo_test.go | 9 +++++++++ 2 files changed, 15 insertions(+), 1 deletion(-) diff --git a/internal/repo/repo.go b/internal/repo/repo.go index 8193918..1692529 100644 --- a/internal/repo/repo.go +++ b/internal/repo/repo.go @@ -28,8 +28,13 @@ func KeysPathFor(recipient string) string { return fmt.Sprintf("%s/%s.age", KeysPrefix, crypto.FingerprintRecipient(recipient)) } -// DataPath returns the content-addressed object path for a plaintext chunk hash. +// DataPath returns the content-addressed object path for a plaintext chunk +// hash. Inputs shorter than 2 bytes cannot be sharded, so it returns +// data/invalid instead of panicking. func DataPath(plainSHA256Hex string) string { + if len(plainSHA256Hex) < 2 { + return DataPrefix + "/invalid" + } return fmt.Sprintf("%s/%s/%s", DataPrefix, plainSHA256Hex[:2], plainSHA256Hex) } diff --git a/internal/repo/repo_test.go b/internal/repo/repo_test.go index c6a56ee..21e1f96 100644 --- a/internal/repo/repo_test.go +++ b/internal/repo/repo_test.go @@ -104,6 +104,15 @@ func TestDataPathUsesHashSharding(t *testing.T) { } } +func TestDataPathRejectsShortHash(t *testing.T) { + for _, in := range []string{"a", ""} { + got := DataPath(in) + if got != "data/invalid" { + t.Fatalf("DataPath(%q) = %q, want %q", in, got, "data/invalid") + } + } +} + func TestSnapshotPathRejectsInvalidUUID(t *testing.T) { if _, err := SnapshotPath("../escape"); err == nil { t.Fatal("SnapshotPath() accepted invalid uuid") From 004a9888988625465c49a3c224c1c596b0ccae98 Mon Sep 17 00:00:00 2001 From: Syrup Date: Sun, 11 Oct 2026 20:00:01 +0700 Subject: [PATCH 06/48] feat(index): add encrypted epoch chunk index --- internal/index/index.go | 144 +++++++++++++++++++++++++++++++ internal/index/index_test.go | 158 +++++++++++++++++++++++++++++++++++ 2 files changed, 302 insertions(+) create mode 100644 internal/index/index.go create mode 100644 internal/index/index_test.go diff --git a/internal/index/index.go b/internal/index/index.go new file mode 100644 index 0000000..4a35ac5 --- /dev/null +++ b/internal/index/index.go @@ -0,0 +1,144 @@ +// Package index tracks which chunks already exist in the repository so +// repeat backups upload nothing. +package index + +import ( + "bytes" + "encoding/json" + "fmt" + "io" + "sort" + + "gloak/internal/crypto" + "gloak/internal/storage" +) + +const indexPrefix = "index" + +type Entry struct { + PlainSHA256 string `json:"plain_sha256"` + PlainSize int64 `json:"plain_size"` + EncryptedSize int64 `json:"encrypted_size"` + EncryptedSHA256 string `json:"encrypted_sha256"` +} + +type Index struct { + epoch int + pending map[string]Entry + all map[string]Entry +} + +type epochFile struct { + Entries []Entry `json:"entries"` +} + +func New() *Index { + return &Index{ + pending: make(map[string]Entry), + all: make(map[string]Entry), + } +} + +// Load merges every index epoch in ascending name order. Epochs that cannot +// be decrypted with the identity are skipped: the caller still has the data +// objects, and the next Save rewrites what is missing. +func Load(backend storage.Backend, identityPath string) (*Index, error) { + ix := New() + + raw, err := backend.ListFiles([]string{indexPrefix}) + if err != nil { + return nil, fmt.Errorf("failed to list %s: %w", indexPrefix, err) + } + entries, err := storage.ParseList(raw) + if err != nil { + return nil, fmt.Errorf("failed to parse index listing: %w", err) + } + + names := make([]string, 0, len(entries)) + for _, entry := range entries { + if !entry.IsDir { + names = append(names, entry.Path) + } + } + sort.Strings(names) + + for _, name := range names { + var epochBuf bytes.Buffer + if err := backend.Download(fmt.Sprintf("%s/%s", indexPrefix, name), &epochBuf); err != nil { + continue + } + reader, err := crypto.DecryptReader(identityPath, &epochBuf) + if err != nil { + continue + } + plain, err := io.ReadAll(reader) + if err != nil { + continue + } + var file epochFile + if err := json.Unmarshal(plain, &file); err != nil { + continue + } + for _, e := range file.Entries { + ix.all[e.PlainSHA256] = e + } + } + return ix, nil +} + +func (ix *Index) Has(hash string) bool { + _, ok := ix.Get(hash) + return ok +} + +func (ix *Index) Get(hash string) (Entry, bool) { + if e, ok := ix.pending[hash]; ok { + return e, true + } + e, ok := ix.all[hash] + return e, ok +} + +func (ix *Index) Add(entry Entry) { + ix.pending[entry.PlainSHA256] = entry + ix.all[entry.PlainSHA256] = entry +} + +// Save writes only the pending entries as the next epoch. It is a no-op when +// nothing is pending, so a second backup of unchanged data writes no files. +func (ix *Index) Save(backend storage.Backend, recipients []string) (int, error) { + if len(ix.pending) == 0 { + return 0, nil + } + + hashes := make([]string, 0, len(ix.pending)) + for hash := range ix.pending { + hashes = append(hashes, hash) + } + sort.Strings(hashes) + + file := epochFile{Entries: make([]Entry, 0, len(hashes))} + for _, hash := range hashes { + file.Entries = append(file.Entries, ix.pending[hash]) + } + + plain, err := json.Marshal(file) + if err != nil { + return 0, fmt.Errorf("failed to marshal index epoch: %w", err) + } + sealed, err := crypto.EncryptMulti(recipients, plain) + if err != nil { + return 0, fmt.Errorf("failed to encrypt index epoch: %w", err) + } + + name := fmt.Sprintf("%s/%06d.age", indexPrefix, ix.epoch+1) + if err := backend.Upload(name, bytes.NewReader(sealed)); err != nil { + return 0, fmt.Errorf("failed to upload index epoch: %w", err) + } + + for _, hash := range hashes { + delete(ix.pending, hash) + } + ix.epoch++ + return len(hashes), nil +} diff --git a/internal/index/index_test.go b/internal/index/index_test.go new file mode 100644 index 0000000..d9f7426 --- /dev/null +++ b/internal/index/index_test.go @@ -0,0 +1,158 @@ +package index + +import ( + "bytes" + "io" + "os" + "path/filepath" + "testing" + + "gloak/internal/crypto" + "gloak/internal/storage/memtest" +) + +func identityFixture(t *testing.T) (string, string) { + t.Helper() + pub, _, fileContent, err := crypto.GenerateKey() + if err != nil { + t.Fatalf("GenerateKey() error: %v", err) + } + path := filepath.Join(t.TempDir(), "identity.txt") + if err := os.WriteFile(path, []byte(fileContent), 0600); err != nil { + t.Fatalf("write identity: %v", err) + } + return pub, path +} + +func entryFixture(hash string) Entry { + return Entry{ + PlainSHA256: hash, + PlainSize: 100, + EncryptedSize: 120, + EncryptedSHA256: hash + "-enc", + } +} + +func TestAddHasGet(t *testing.T) { + ix := New() + ix.Add(entryFixture("aaa")) + if !ix.Has("aaa") { + t.Fatal("Has() = false after Add") + } + if e, ok := ix.Get("aaa"); !ok || e.PlainSize != 100 { + t.Fatalf("Get() = %+v, %v", e, ok) + } + if ix.Has("bbb") { + t.Fatal("Has() = true for unknown hash") + } +} + +func TestSaveAndLoadAcrossEpochs(t *testing.T) { + pub, ident := identityFixture(t) + backend := memtest.NewMemoryBackend() + + ix := New() + ix.Add(entryFixture("aaa")) + ix.Add(entryFixture("bbb")) + n, err := ix.Save(backend, []string{pub}) + if err != nil { + t.Fatalf("Save() error: %v", err) + } + if n != 2 { + t.Fatalf("Save() = %d, want 2", n) + } + + ix.Add(entryFixture("ccc")) + if n, err := ix.Save(backend, []string{pub}); err != nil || n != 1 { + t.Fatalf("second Save() = %d, %v, want 1, nil", n, err) + } + + loaded, err := Load(backend, ident) + if err != nil { + t.Fatalf("Load() error: %v", err) + } + for _, hash := range []string{"aaa", "bbb", "ccc"} { + if !loaded.Has(hash) { + t.Fatalf("Load() missing %s", hash) + } + } +} + +func TestSaveSkipsEmptyEpoch(t *testing.T) { + pub, _ := identityFixture(t) + backend := memtest.NewMemoryBackend() + ix := New() + ix.Add(entryFixture("aaa")) + if _, err := ix.Save(backend, []string{pub}); err != nil { + t.Fatalf("Save() error: %v", err) + } + before := len(backend.KeysWithPrefix("index/")) + n, err := ix.Save(backend, []string{pub}) + if err != nil { + t.Fatalf("Save() error: %v", err) + } + if n != 0 || len(backend.KeysWithPrefix("index/")) != before { + t.Fatalf("empty Save() wrote files: n=%d files=%d", n, len(backend.KeysWithPrefix("index/"))) + } +} + +func TestLoadSkipsUnreadableEpoch(t *testing.T) { + pub, ident := identityFixture(t) + backend := memtest.NewMemoryBackend() + ix := New() + ix.Add(entryFixture("aaa")) + if _, err := ix.Save(backend, []string{pub}); err != nil { + t.Fatalf("Save() error: %v", err) + } + // Corrupt one epoch file with garbage that is not valid age ciphertext. + backend.Put("index/000002.age", []byte("garbage-not-age")) + // Add a second valid epoch so we can confirm the merge continues. + ix.Add(entryFixture("bbb")) + if _, err := ix.Save(backend, []string{pub}); err != nil { + t.Fatalf("second Save() error: %v", err) + } + + loaded, err := Load(backend, ident) + if err != nil { + t.Fatalf("Load() error: %v", err) + } + if !loaded.Has("aaa") || !loaded.Has("bbb") { + t.Fatalf("Load() lost entries: %+v", loaded.all) + } +} + +func TestLoadOnEmptyBackendIsClean(t *testing.T) { + _, ident := identityFixture(t) + loaded, err := Load(memtest.NewMemoryBackend(), ident) + if err != nil { + t.Fatalf("Load() error: %v", err) + } + if len(loaded.all) != 0 { + t.Fatalf("Load() on empty backend = %+v", loaded.all) + } +} + +func TestSaveOutputIsEncryptedJSON(t *testing.T) { + pub, ident := identityFixture(t) + backend := memtest.NewMemoryBackend() + ix := New() + ix.Add(entryFixture("aaa")) + if _, err := ix.Save(backend, []string{pub}); err != nil { + t.Fatalf("Save() error: %v", err) + } + var buf bytes.Buffer + if err := backend.Download("index/000001.age", &buf); err != nil { + t.Fatalf("Download() error: %v", err) + } + reader, err := crypto.DecryptReader(ident, &buf) + if err != nil { + t.Fatalf("index file is not age-encrypted with identity: %v", err) + } + plain, err := io.ReadAll(reader) + if err != nil { + t.Fatalf("ReadAll error: %v", err) + } + if !bytes.Contains(plain, []byte("aaa")) { + t.Fatalf("decrypted index missing entry hash: %s", plain) + } +} From 8acdb6b0cb9d0dab3b9f9e05191bf9a16fafd2f5 Mon Sep 17 00:00:00 2001 From: Syrup Date: Sun, 11 Oct 2026 20:15:48 +0700 Subject: [PATCH 07/48] fix(index): restore epoch counter and tolerate missing index dir --- internal/index/index.go | 43 +++++++++++++++++++++++++++--- internal/index/index_test.go | 51 +++++++++++++++++++++++++++++++++--- 2 files changed, 88 insertions(+), 6 deletions(-) diff --git a/internal/index/index.go b/internal/index/index.go index 4a35ac5..e9c2765 100644 --- a/internal/index/index.go +++ b/internal/index/index.go @@ -8,6 +8,8 @@ import ( "fmt" "io" "sort" + "strconv" + "strings" "gloak/internal/crypto" "gloak/internal/storage" @@ -41,13 +43,18 @@ func New() *Index { // Load merges every index epoch in ascending name order. Epochs that cannot // be decrypted with the identity are skipped: the caller still has the data -// objects, and the next Save rewrites what is missing. +// objects, and the next Save rewrites what is missing. The epoch counter is +// restored from the highest epoch name seen so a later Save never reuses a +// name that already exists on the backend. func Load(backend storage.Backend, identityPath string) (*Index, error) { ix := New() raw, err := backend.ListFiles([]string{indexPrefix}) if err != nil { - return nil, fmt.Errorf("failed to list %s: %w", indexPrefix, err) + // A fresh repo has no index/ directory yet (rclone lsjson exits 3 + // on a missing directory); that is normal, so treat the listing as + // empty and start from epoch 0. + return ix, nil } entries, err := storage.ParseList(raw) if err != nil { @@ -63,8 +70,14 @@ func Load(backend storage.Backend, identityPath string) (*Index, error) { sort.Strings(names) for _, name := range names { + path := fmt.Sprintf("%s/%s", indexPrefix, name) + // Count the epoch number even when the file's contents end up + // skipped, so a later Save never reuses an existing name. + if epoch, ok := parseEpoch(path); ok && epoch > ix.epoch { + ix.epoch = epoch + } var epochBuf bytes.Buffer - if err := backend.Download(fmt.Sprintf("%s/%s", indexPrefix, name), &epochBuf); err != nil { + if err := backend.Download(path, &epochBuf); err != nil { continue } reader, err := crypto.DecryptReader(identityPath, &epochBuf) @@ -86,6 +99,30 @@ func Load(backend storage.Backend, identityPath string) (*Index, error) { return ix, nil } +// parseEpoch extracts the epoch number from a path like "index/000001.age". +// ok is false when the path does not match the zero-padded epoch pattern. +func parseEpoch(path string) (int, bool) { + prefix := indexPrefix + "/" + suffix := ".age" + if !strings.HasPrefix(path, prefix) || !strings.HasSuffix(path, suffix) { + return 0, false + } + digits := path[len(prefix) : len(path)-len(suffix)] + if len(digits) != 6 { + return 0, false + } + for _, r := range digits { + if r < '0' || r > '9' { + return 0, false + } + } + epoch, err := strconv.Atoi(digits) + if err != nil { + return 0, false + } + return epoch, true +} + func (ix *Index) Has(hash string) bool { _, ok := ix.Get(hash) return ok diff --git a/internal/index/index_test.go b/internal/index/index_test.go index d9f7426..68a5abd 100644 --- a/internal/index/index_test.go +++ b/internal/index/index_test.go @@ -104,13 +104,13 @@ func TestLoadSkipsUnreadableEpoch(t *testing.T) { if _, err := ix.Save(backend, []string{pub}); err != nil { t.Fatalf("Save() error: %v", err) } - // Corrupt one epoch file with garbage that is not valid age ciphertext. - backend.Put("index/000002.age", []byte("garbage-not-age")) - // Add a second valid epoch so we can confirm the merge continues. ix.Add(entryFixture("bbb")) if _, err := ix.Save(backend, []string{pub}); err != nil { t.Fatalf("second Save() error: %v", err) } + // Corrupt the third epoch with garbage that is not valid age ciphertext, + // after both valid epochs are already written. + backend.Put("index/000003.age", []byte("garbage-not-age")) loaded, err := Load(backend, ident) if err != nil { @@ -121,6 +121,51 @@ func TestLoadSkipsUnreadableEpoch(t *testing.T) { } } +func TestLoadRestoresEpochCounter(t *testing.T) { + pub, ident := identityFixture(t) + backend := memtest.NewMemoryBackend() + + ix := New() + ix.Add(entryFixture("aaa")) + if _, err := ix.Save(backend, []string{pub}); err != nil { + t.Fatalf("Save() error: %v", err) + } + ix.Add(entryFixture("bbb")) + if _, err := ix.Save(backend, []string{pub}); err != nil { + t.Fatalf("second Save() error: %v", err) + } + + loaded, err := Load(backend, ident) + if err != nil { + t.Fatalf("Load() error: %v", err) + } + loaded.Add(entryFixture("ccc")) + if _, err := loaded.Save(backend, []string{pub}); err != nil { + t.Fatalf("Save() after Load error: %v", err) + } + + keys := backend.KeysWithPrefix("index/") + want := []string{"index/000001.age", "index/000002.age", "index/000003.age"} + if len(keys) != len(want) { + t.Fatalf("index files = %v, want %v", keys, want) + } + for i, k := range want { + if keys[i] != k { + t.Fatalf("index files = %v, want %v", keys, want) + } + } + + reloaded, err := Load(backend, ident) + if err != nil { + t.Fatalf("Load() after save error: %v", err) + } + for _, hash := range []string{"aaa", "bbb", "ccc"} { + if !reloaded.Has(hash) { + t.Fatalf("reloaded index missing %s", hash) + } + } +} + func TestLoadOnEmptyBackendIsClean(t *testing.T) { _, ident := identityFixture(t) loaded, err := Load(memtest.NewMemoryBackend(), ident) From 841b6ded88166f0e481f84da72ab047e69437022 Mon Sep 17 00:00:00 2001 From: Syrup Date: Sun, 11 Oct 2026 20:32:10 +0700 Subject: [PATCH 08/48] feat(flow): add repository init flow --- init.go | 67 ++++++++++++++++++++++++++++++++++++++ internal/flow/init.go | 34 +++++++++++++++++++ internal/flow/init_test.go | 50 ++++++++++++++++++++++++++++ main.go | 1 + main_test.go | 57 ++++++++++++++++++++++++++++++++ 5 files changed, 209 insertions(+) create mode 100644 init.go create mode 100644 internal/flow/init.go create mode 100644 internal/flow/init_test.go diff --git a/init.go b/init.go new file mode 100644 index 0000000..fc5cc15 --- /dev/null +++ b/init.go @@ -0,0 +1,67 @@ +package main + +import ( + "fmt" + "os" + "path/filepath" + "strings" + + "gloak/internal/flow" +) + +// InitCmd creates a new gloak repository. +type InitCmd struct { + Remote string `short:"R" required:"" help:"Repository remote, e.g. b2:my-bucket/gloak or local path"` + Recipient string `short:"r" help:"Recipient public key file or comma-separated keys (default ~/.gloak/recipient.txt)"` +} + +func (c *InitCmd) Run() error { + recipients, err := resolveInitRecipients(c.Recipient) + if err != nil { + return err + } + return flow.RunInit(recipients, c.Remote) +} + +// resolveInitRecipients turns the --recipient value into a key list. An empty +// path falls back to ~/.gloak/recipient.txt. A value that looks like keys +// (starts with age1 or contains commas) is split on commas; anything else is +// treated as a file path with one key per line, skipping blanks and # comments. +func resolveInitRecipients(path string) ([]string, error) { + if path == "" { + path = filepath.Join(getDefaultDir(), "recipient.txt") + } + if !strings.HasPrefix(path, "age1") && !strings.Contains(path, ",") { + return readRecipientFile(path) + } + var out []string + for _, part := range strings.Split(path, ",") { + part = strings.TrimSpace(part) + if part != "" { + out = append(out, part) + } + } + if len(out) == 0 { + return nil, fmt.Errorf("no recipients found (run 'gloak keygen' first)") + } + return out, nil +} + +func readRecipientFile(path string) ([]string, error) { + data, err := os.ReadFile(path) + if err != nil { + return nil, fmt.Errorf("failed to read recipients from %s: %w (run 'gloak keygen' first)", path, err) + } + var out []string + for _, line := range strings.Split(string(data), "\n") { + line = strings.TrimSpace(line) + if line == "" || strings.HasPrefix(line, "#") { + continue + } + out = append(out, line) + } + if len(out) == 0 { + return nil, fmt.Errorf("no recipients found in %s (run 'gloak keygen' first)", path) + } + return out, nil +} diff --git a/internal/flow/init.go b/internal/flow/init.go new file mode 100644 index 0000000..43672d4 --- /dev/null +++ b/internal/flow/init.go @@ -0,0 +1,34 @@ +package flow + +import ( + "github.com/pterm/pterm" + + "gloak/internal/core" + "gloak/internal/repo" + "gloak/internal/storage" +) + +const defaultChunkSize = core.ChunkSize + +// RunInit creates a new gloak repository on remote and prints the outcome. +func RunInit(recipients []string, remote string) error { + backend := storage.NewRcloneBackend(remote) + spinner, _ := pterm.DefaultSpinner.Start("Initializing repository...") + + master, cfg, err := runInit(recipients, backend, defaultChunkSize) + if err != nil { + _ = spinner.Stop() + return err + } + _ = spinner.Stop() + pterm.Success.Println("Repository initialized") + pterm.Printf("Location: %s\n", remote) + pterm.Printf("Recipients: %v\n", cfg.Recipients) + pterm.Printf("Chunk size: %d bytes\n", cfg.ChunkSize) + pterm.Printf("Master key length: %d bytes (wrapped in %s)\n", len(master), repo.KeysPrefix+"/") + return nil +} + +func runInit(recipients []string, backend storage.Backend, chunkSize int64) ([]byte, repo.Config, error) { + return repo.Init(backend, recipients, chunkSize) +} diff --git a/internal/flow/init_test.go b/internal/flow/init_test.go new file mode 100644 index 0000000..ae44f63 --- /dev/null +++ b/internal/flow/init_test.go @@ -0,0 +1,50 @@ +package flow + +import ( + "strings" + "testing" + + "gloak/internal/crypto" + "gloak/internal/repo" + "gloak/internal/storage/memtest" +) + +func TestRunInitCreatesRepository(t *testing.T) { + pub, _, fileContent, err := crypto.GenerateKey() + if err != nil { + t.Fatalf("GenerateKey() error: %v", err) + } + _ = fileContent + backend := memtest.NewMemoryBackend() + + master, cfg, err := runInit([]string{pub}, backend, 4096) + if err != nil { + t.Fatalf("runInit() error: %v", err) + } + if len(master) != 32 { + t.Fatalf("master key length = %d", len(master)) + } + if !backend.Has(repo.ConfigPath) { + t.Fatal("repo.json missing") + } + if !backend.Has(repo.KeysPathFor(pub)) { + t.Fatal("wrapped key missing") + } + if cfg.ChunkSize != 4096 { + t.Fatalf("cfg.ChunkSize = %d", cfg.ChunkSize) + } +} + +func TestRunInitRejectsNoRecipients(t *testing.T) { + if _, _, err := runInit(nil, memtest.NewMemoryBackend(), 4096); err == nil { + t.Fatal("runInit() succeeded with no recipients") + } else if !strings.Contains(err.Error(), "recipient") { + t.Fatalf("runInit() error = %v, want recipient mention", err) + } +} + +func TestRunInitRejectsInvalidRecipient(t *testing.T) { + if _, _, err := runInit([]string{"age1bad"}, memtest.NewMemoryBackend(), 4096); err == nil { + t.Fatal("runInit() succeeded with invalid recipient") + } +} diff --git a/main.go b/main.go index 570390d..ab0c328 100644 --- a/main.go +++ b/main.go @@ -14,6 +14,7 @@ var cli struct { Debug bool `help:"Enable debug mode." short:"d"` Keygen KeygenCmd `cmd:"" help:"Generate an X25519 identity and recipient file."` + Init InitCmd `cmd:"" help:"Initialize a new gloak repository." group:"Repository"` Upload UploadCmd `cmd:"" help:"Encrypt, chunk, and upload a file to a storage destination."` Resume ResumeCmd `cmd:"" help:"Validate saved state for an interrupted upload."` Restore RestoreCmd `cmd:"" help:"Restore and decrypt a backup from storage."` diff --git a/main_test.go b/main_test.go index e8798cc..da4898e 100644 --- a/main_test.go +++ b/main_test.go @@ -48,6 +48,8 @@ func TestCommandOptionShortTags(t *testing.T) { }{ {name: "keygen output dir", cmd: KeygenCmd{}, fieldName: "OutputDir", wantShort: "o"}, {name: "keygen force", cmd: KeygenCmd{}, fieldName: "Force", wantShort: "f"}, + {name: "init remote", cmd: InitCmd{}, fieldName: "Remote", wantShort: "R"}, + {name: "init recipient", cmd: InitCmd{}, fieldName: "Recipient", wantShort: "r"}, {name: "upload recipient", cmd: UploadCmd{}, fieldName: "Recipient", wantShort: "r"}, {name: "upload remote", cmd: UploadCmd{}, fieldName: "Remote", wantShort: "R"}, {name: "restore identity", cmd: RestoreCmd{}, fieldName: "Identity", wantShort: "i"}, @@ -73,3 +75,58 @@ func TestCommandOptionShortTags(t *testing.T) { }) } } + +func TestResolveInitRecipientsReadsFile(t *testing.T) { + path := filepath.Join(t.TempDir(), "recipient.txt") + content := "# comment\n\nage1aaa\nage1bbb\n" + if err := os.WriteFile(path, []byte(content), 0644); err != nil { + t.Fatalf("failed to write recipient file: %v", err) + } + + got, err := resolveInitRecipients(path) + if err != nil { + t.Fatalf("resolveInitRecipients() error: %v", err) + } + want := []string{"age1aaa", "age1bbb"} + if !reflect.DeepEqual(got, want) { + t.Fatalf("resolveInitRecipients() = %v, want %v", got, want) + } +} + +func TestResolveInitRecipientsParsesInlineKeys(t *testing.T) { + got, err := resolveInitRecipients("age1aaa, age1bbb") + if err != nil { + t.Fatalf("resolveInitRecipients() error: %v", err) + } + want := []string{"age1aaa", "age1bbb"} + if !reflect.DeepEqual(got, want) { + t.Fatalf("resolveInitRecipients() = %v, want %v", got, want) + } +} + +func TestResolveInitRecipientsEmptyFileErrors(t *testing.T) { + path := filepath.Join(t.TempDir(), "recipient.txt") + if err := os.WriteFile(path, []byte("# only a comment\n\n"), 0644); err != nil { + t.Fatalf("failed to write recipient file: %v", err) + } + + _, err := resolveInitRecipients(path) + if err == nil { + t.Fatal("resolveInitRecipients() succeeded with empty file") + } + if !strings.Contains(err.Error(), "no recipients") { + t.Fatalf("resolveInitRecipients() error = %q, want no recipients mention", err) + } +} + +func TestResolveInitRecipientsMissingFileErrors(t *testing.T) { + path := filepath.Join(t.TempDir(), "missing.txt") + + _, err := resolveInitRecipients(path) + if err == nil { + t.Fatal("resolveInitRecipients() succeeded with missing file") + } + if !strings.Contains(err.Error(), "gloak keygen") { + t.Fatalf("resolveInitRecipients() error = %q, want gloak keygen mention", err) + } +} From a84dfc5940e19f29b151312d93937ff519883c19 Mon Sep 17 00:00:00 2001 From: Syrup Date: Sun, 11 Oct 2026 20:44:40 +0700 Subject: [PATCH 09/48] feat(flow): add repository backup with index skip --- internal/flow/backup.go | 185 +++++++++++++++++++++++++++++++++++ internal/flow/backup_test.go | 148 ++++++++++++++++++++++++++++ internal/flow/hash.go | 11 +++ internal/flow/upload.go | 5 - 4 files changed, 344 insertions(+), 5 deletions(-) create mode 100644 internal/flow/backup.go create mode 100644 internal/flow/backup_test.go create mode 100644 internal/flow/hash.go diff --git a/internal/flow/backup.go b/internal/flow/backup.go new file mode 100644 index 0000000..4efe812 --- /dev/null +++ b/internal/flow/backup.go @@ -0,0 +1,185 @@ +package flow + +import ( + "bytes" + "crypto/sha256" + "encoding/hex" + "encoding/json" + "fmt" + "io" + "os" + "path/filepath" + "time" + + "github.com/google/uuid" + "github.com/pterm/pterm" + + "gloak/internal/core" + "gloak/internal/crypto" + "gloak/internal/index" + "gloak/internal/repo" + "gloak/internal/storage" +) + +const indexCheckpointEvery = 64 + +type BackupResult struct { + SnapshotID string + TotalChunks int + UploadedChunks int + SkippedChunks int + TotalPlainBytes int64 +} + +// RunBackup backs up filePath into the repository on remote using identityPath. +func RunBackup(filePath, identityPath, remote string) (string, error) { + backend := storage.NewRcloneBackend(remote) + snapshotID := core.GenerateUUID() + + spinner, _ := pterm.DefaultSpinner.Start("Backing up...") + res, err := runBackup(filePath, identityPath, backend, snapshotID, func(done int) { + spinner.UpdateText(fmt.Sprintf("Backed up %d chunks", done)) + }) + if err != nil { + _ = spinner.Stop() + return "", err + } + _ = spinner.Stop() + + pterm.Success.Println("Backup complete") + pterm.Printf("Snapshot ID: %s\n", res.SnapshotID) + pterm.Printf("Chunks: %d total, %d uploaded, %d skipped\n", res.TotalChunks, res.UploadedChunks, res.SkippedChunks) + pterm.Printf("Data size: %d bytes\n", res.TotalPlainBytes) + return res.SnapshotID, nil +} + +func runBackup(filePath, identityPath string, backend storage.Backend, snapshotID string, progress func(int)) (BackupResult, error) { + if _, err := uuid.Parse(snapshotID); err != nil { + return BackupResult{}, fmt.Errorf("invalid snapshot id: %w", err) + } + + r, err := repo.Open(backend, identityPath) + if err != nil { + return BackupResult{}, err + } + + ix, err := index.Load(backend, identityPath) + if err != nil { + return BackupResult{}, fmt.Errorf("failed to load index: %w", err) + } + + file, err := os.Open(filePath) + if err != nil { + return BackupResult{}, fmt.Errorf("failed to open file: %w", err) + } + defer file.Close() + + chunkSize := r.Config.ChunkSize + if chunkSize <= 0 { + return BackupResult{}, fmt.Errorf("invalid chunk size: %d", chunkSize) + } + + res := BackupResult{SnapshotID: snapshotID} + var metas []core.ChunkMeta + var sinceCheckpoint int + payloadHasher := sha256.New() + buffer := make([]byte, int(chunkSize)) + + for chunkIdx := 0; ; chunkIdx++ { + n, readErr := io.ReadFull(file, buffer) + if readErr == io.EOF { + break + } + if readErr != nil && readErr != io.ErrUnexpectedEOF { + return BackupResult{}, fmt.Errorf("failed to read chunk %d: %w", chunkIdx, readErr) + } + if n == 0 { + break + } + + chunk := buffer[:n] + payloadHasher.Write(chunk) + hash := hashBytes(chunk) + + res.TotalChunks++ + res.TotalPlainBytes += int64(n) + + dataPath := repo.DataPath(hash) + + if ix.Has(hash) { + res.SkippedChunks++ + } else { + sealed, err := crypto.SealChunk(r.MasterKey, hash, chunk) + if err != nil { + return BackupResult{}, fmt.Errorf("failed to seal chunk %s: %w", hash, err) + } + encSum := sha256.Sum256(sealed) + if err := backend.Upload(dataPath, bytes.NewReader(sealed)); err != nil { + return BackupResult{}, fmt.Errorf("failed to upload chunk %s: %w", hash, err) + } + ix.Add(index.Entry{ + PlainSHA256: hash, + PlainSize: int64(n), + EncryptedSize: int64(len(sealed)), + EncryptedSHA256: hex.EncodeToString(encSum[:]), + }) + res.UploadedChunks++ + sinceCheckpoint++ + if sinceCheckpoint >= indexCheckpointEvery { + if _, err := ix.Save(backend, r.Config.Recipients); err != nil { + return BackupResult{}, fmt.Errorf("failed to checkpoint index: %w", err) + } + sinceCheckpoint = 0 + } + } + + encEntry, _ := ix.Get(hash) + metas = append(metas, core.ChunkMeta{ + Index: len(metas), + Name: dataPath, + PlainSize: encEntry.PlainSize, + PlainSHA256: hash, + EncryptedSize: encEntry.EncryptedSize, + EncryptedSHA256: encEntry.EncryptedSHA256, + }) + + if progress != nil { + progress(res.TotalChunks) + } + } + + if _, err := ix.Save(backend, r.Config.Recipients); err != nil { + return BackupResult{}, fmt.Errorf("failed to save index: %w", err) + } + + recipient := "" + if len(r.Config.Recipients) > 0 { + recipient = r.Config.Recipients[0] + } + manifest := core.Manifest{ + Version: repo.FormatVersion, + UUID: snapshotID, + OriginalName: filepath.Base(filePath), + OriginalSize: res.TotalPlainBytes, + CreatedAt: time.Now().UTC().Format(time.RFC3339), + Recipient: recipient, + PayloadSHA256: hex.EncodeToString(payloadHasher.Sum(nil)), + Chunks: metas, + } + manifestJSON, err := json.Marshal(manifest) + if err != nil { + return BackupResult{}, fmt.Errorf("failed to marshal manifest: %w", err) + } + sealed, err := crypto.EncryptMulti(r.Config.Recipients, manifestJSON) + if err != nil { + return BackupResult{}, fmt.Errorf("failed to encrypt manifest: %w", err) + } + snapshotPath, err := repo.SnapshotPath(snapshotID) + if err != nil { + return BackupResult{}, err + } + if err := backend.Upload(snapshotPath, bytes.NewReader(sealed)); err != nil { + return BackupResult{}, fmt.Errorf("failed to upload snapshot: %w", err) + } + return res, nil +} diff --git a/internal/flow/backup_test.go b/internal/flow/backup_test.go new file mode 100644 index 0000000..d4f8d90 --- /dev/null +++ b/internal/flow/backup_test.go @@ -0,0 +1,148 @@ +package flow + +import ( + "crypto/sha256" + "encoding/hex" + "os" + "path/filepath" + "strings" + "testing" + + "gloak/internal/crypto" + "gloak/internal/repo" + "gloak/internal/storage/memtest" +) + +func repoFixture(t *testing.T, chunkSize int64) (string, *memtest.MemoryBackend, string) { + t.Helper() + pub, _, fileContent, err := crypto.GenerateKey() + if err != nil { + t.Fatalf("GenerateKey() error: %v", err) + } + identPath := filepath.Join(t.TempDir(), "identity.txt") + if err := os.WriteFile(identPath, []byte(fileContent), 0600); err != nil { + t.Fatalf("write identity: %v", err) + } + backend := memtest.NewMemoryBackend() + if _, _, err := repo.Init(backend, []string{pub}, chunkSize); err != nil { + t.Fatalf("repo.Init() error: %v", err) + } + return identPath, backend, pub +} + +func writeTempFile(t *testing.T, data []byte) string { + t.Helper() + path := filepath.Join(t.TempDir(), "payload.bin") + if err := os.WriteFile(path, data, 0600); err != nil { + t.Fatalf("write temp file: %v", err) + } + return path +} + +func TestBackupWritesSnapshotAndDataObjects(t *testing.T) { + ident, backend, _ := repoFixture(t, 8) + payload := []byte("0123456789abcdefghij") // 3 chunks with size 8 + filePath := writeTempFile(t, payload) + + res, err := runBackup(filePath, ident, backend, "7f91c6c7-7a0b-44aa-ae23-997b60e4e998", nil) + if err != nil { + t.Fatalf("runBackup() error: %v", err) + } + if res.TotalChunks != 3 || res.UploadedChunks != 3 || res.SkippedChunks != 0 { + t.Fatalf("result = %+v", res) + } + if !backend.Has(repo.SnapshotsPrefix + "/" + res.SnapshotID + ".age") { + t.Fatal("snapshot file missing") + } + if got := backend.TotalUploads(repo.DataPrefix + "/"); got != 3 { + t.Fatalf("data uploads = %d, want 3", got) + } + if !backend.Has("index/000001.age") { + t.Fatal("index epoch missing") + } +} + +func TestBackupSkipsChunksAlreadyInIndex(t *testing.T) { + ident, backend, _ := repoFixture(t, 8) + payload := []byte("0123456789abcdefghij") + filePath := writeTempFile(t, payload) + + id1 := "aaaaaaaa-bbbb-4ccc-8ddd-eeeeeeeeeeee" + if _, err := runBackup(filePath, ident, backend, id1, nil); err != nil { + t.Fatalf("first runBackup() error: %v", err) + } + dataUploadsBefore := backend.TotalUploads(repo.DataPrefix + "/") + + id2 := "aaaaaaaa-bbbb-4ccc-8ddd-ffffffffffff" + res, err := runBackup(filePath, ident, backend, id2, nil) + if err != nil { + t.Fatalf("second runBackup() error: %v", err) + } + if res.UploadedChunks != 0 || res.SkippedChunks != 3 { + t.Fatalf("second result = %+v, want all skipped", res) + } + if backend.TotalUploads(repo.DataPrefix+"/") != dataUploadsBefore { + t.Fatal("second backup re-uploaded data objects") + } + if !backend.Has(repo.SnapshotsPrefix + "/" + id2 + ".age") { + t.Fatal("second snapshot missing") + } +} + +func TestBackupDedupsRepeatedChunksInSameFile(t *testing.T) { + ident, backend, _ := repoFixture(t, 8) + payload := []byte("012345670123456701234567") // same 8-byte chunk three times + filePath := writeTempFile(t, payload) + + res, err := runBackup(filePath, ident, backend, "11111111-2222-4333-8444-555555555555", nil) + if err != nil { + t.Fatalf("runBackup() error: %v", err) + } + if res.TotalChunks != 3 { + t.Fatalf("TotalChunks = %d", res.TotalChunks) + } + if res.UploadedChunks != 1 { + t.Fatalf("UploadedChunks = %d, want 1 (content-addressed dedup)", res.UploadedChunks) + } + if got := backend.TotalUploads(repo.DataPrefix + "/"); got != 1 { + t.Fatalf("data uploads = %d, want 1", got) + } +} + +func TestBackupFailsWithoutRepository(t *testing.T) { + ident, _, _ := repoFixture(t, 8) + filePath := writeTempFile(t, []byte("x")) + _, err := runBackup(filePath, ident, memtest.NewMemoryBackend(), "7f91c6c7-7a0b-44aa-ae23-997b60e4e998", nil) + if err == nil { + t.Fatal("runBackup() succeeded without repository") + } + if !strings.Contains(err.Error(), "not a gloak repository") { + t.Fatalf("error = %v, want not a gloak repository", err) + } +} + +func TestBackupRejectsInvalidSnapshotID(t *testing.T) { + ident, backend, _ := repoFixture(t, 8) + filePath := writeTempFile(t, []byte("x")) + _, err := runBackup(filePath, ident, backend, "../escape", nil) + if err == nil { + t.Fatal("runBackup() succeeded with invalid snapshot id") + } + if !strings.Contains(err.Error(), "invalid snapshot id") { + t.Fatalf("error = %v", err) + } +} + +func TestBackupObjectMatchesChunkHash(t *testing.T) { + ident, backend, _ := repoFixture(t, 8) + payload := []byte("0123456789abcdefghij") + filePath := writeTempFile(t, payload) + if _, err := runBackup(filePath, ident, backend, "99999999-8888-4777-8666-555555555555", nil); err != nil { + t.Fatalf("runBackup() error: %v", err) + } + sum := sha256.Sum256(payload[:8]) + wantPath := repo.DataPath(hex.EncodeToString(sum[:])) + if !backend.Has(wantPath) { + t.Fatalf("expected object at %s", wantPath) + } +} diff --git a/internal/flow/hash.go b/internal/flow/hash.go new file mode 100644 index 0000000..edb1e7e --- /dev/null +++ b/internal/flow/hash.go @@ -0,0 +1,11 @@ +package flow + +import ( + "crypto/sha256" + "encoding/hex" +) + +func hashBytes(data []byte) string { + hash := sha256.Sum256(data) + return hex.EncodeToString(hash[:]) +} diff --git a/internal/flow/upload.go b/internal/flow/upload.go index 065d377..3609017 100644 --- a/internal/flow/upload.go +++ b/internal/flow/upload.go @@ -195,11 +195,6 @@ func encryptChunk(recipientKey string, plainChunk []byte) ([]byte, error) { return encrypted.Bytes(), nil } -func hashBytes(data []byte) string { - hash := sha256.Sum256(data) - return hex.EncodeToString(hash[:]) -} - type ProgressReader struct { io.Reader pb *pterm.ProgressbarPrinter From 97ab1d5aa5933a32212bfef6f2de0ed8804924de Mon Sep 17 00:00:00 2001 From: Syrup Date: Sun, 11 Oct 2026 21:12:52 +0700 Subject: [PATCH 10/48] feat(flow): rewrite verify and restore for repository layout --- internal/flow/flow_test.go | 155 +++++++++++++++++---------------- internal/flow/manifest_read.go | 49 +++++++++++ internal/flow/restore.go | 123 +++++++------------------- internal/flow/verify.go | 103 +++++++--------------- 4 files changed, 193 insertions(+), 237 deletions(-) create mode 100644 internal/flow/manifest_read.go diff --git a/internal/flow/flow_test.go b/internal/flow/flow_test.go index 8916320..66534a8 100644 --- a/internal/flow/flow_test.go +++ b/internal/flow/flow_test.go @@ -11,78 +11,107 @@ import ( "sync" "testing" + "gloak/internal/core" appcrypto "gloak/internal/crypto" + "gloak/internal/repo" "gloak/internal/storage" ) -func TestUploadVerifyRestoreRoundTrip(t *testing.T) { - pubKey, _, identityFile, err := appcrypto.GenerateKey() - if err != nil { - t.Fatalf("GenerateKey() returned error: %v", err) - } - identityPath := writeFlowIdentityFile(t, identityFile) - - sourcePath := filepath.Join(t.TempDir(), "secret.txt") - wantPayload := []byte("important backup payload\nwith enough bytes to test restore") - if err := os.WriteFile(sourcePath, wantPayload, 0600); err != nil { - t.Fatalf("failed to write source file: %v", err) - } - - backend := newMemoryBackend() - backupID := "7f91c6c7-7a0b-44aa-ae23-997b60e4e998" +func TestBackupVerifyRestoreRoundTrip(t *testing.T) { + ident, backend, _ := repoFixture(t, 8) + payload := []byte("0123456789abcdefghij") + filePath := writeTempFile(t, payload) + snapshotID := "7f91c6c7-7a0b-44aa-ae23-997b60e4e998" - if err := runUpload(sourcePath, pubKey, backend, backupID); err != nil { - t.Fatalf("runUpload() returned error: %v", err) + if _, err := runBackup(filePath, ident, backend, snapshotID, nil); err != nil { + t.Fatalf("runBackup() error: %v", err) } - for _, path := range []string{backupID + "/chunk_00000", backupID + "/manifest.age", backupID + "/locator.json"} { - if !backend.has(path) { - t.Fatalf("uploaded path %q not found", path) - } + if err := runVerify(snapshotID, ident, backend); err != nil { + t.Fatalf("runVerify() error: %v", err) } - if err := runVerify(backupID, identityPath, backend); err != nil { - t.Fatalf("runVerify() returned error: %v", err) - } - - outputDir := t.TempDir() - if err := runRestore(backupID, identityPath, backend, outputDir); err != nil { - t.Fatalf("runRestore() returned error: %v", err) + outDir := t.TempDir() + restoredPath, err := runRestore(snapshotID, outDir, ident, backend) + if err != nil { + t.Fatalf("runRestore() error: %v", err) } - - gotPayload, err := os.ReadFile(filepath.Join(outputDir, "secret.txt")) + got, err := os.ReadFile(restoredPath) if err != nil { - t.Fatalf("failed to read restored file: %v", err) + t.Fatalf("read restored: %v", err) } - if !bytes.Equal(gotPayload, wantPayload) { - t.Fatalf("restored payload = %q, want %q", gotPayload, wantPayload) + if !bytes.Equal(got, payload) { + t.Fatalf("restored = %q, want %q", got, payload) } } func TestVerifyRejectsCorruptChunk(t *testing.T) { - pubKey, _, identityFile, err := appcrypto.GenerateKey() - if err != nil { - t.Fatalf("GenerateKey() returned error: %v", err) + ident, backend, _ := repoFixture(t, 8) + payload := []byte("0123456789abcdefghij") + filePath := writeTempFile(t, payload) + snapshotID := "7f91c6c7-7a0b-44aa-ae23-997b60e4e998" + if _, err := runBackup(filePath, ident, backend, snapshotID, nil); err != nil { + t.Fatalf("runBackup() error: %v", err) } - identityPath := writeFlowIdentityFile(t, identityFile) - sourcePath := filepath.Join(t.TempDir(), "secret.txt") - if err := os.WriteFile(sourcePath, []byte("payload"), 0600); err != nil { - t.Fatalf("failed to write source file: %v", err) + // Corrupt the first data object in place. + var dataKey string + for _, key := range backend.KeysWithPrefix(repo.DataPrefix + "/") { + dataKey = key + break } - - backend := newMemoryBackend() - backupID := "7f91c6c7-7a0b-44aa-ae23-997b60e4e998" - if err := runUpload(sourcePath, pubKey, backend, backupID); err != nil { - t.Fatalf("runUpload() returned error: %v", err) + if dataKey == "" { + t.Fatal("no data objects to corrupt") } + blob := backend.Get(dataKey) + blob[len(blob)-1] ^= 0xff + backend.Put(dataKey, blob) - backend.put(backupID+"/chunk_00000", []byte("tampered")) - err = runVerify(backupID, identityPath, backend) + err := runVerify(snapshotID, ident, backend) if err == nil { - t.Fatalf("runVerify() succeeded with corrupt chunk") + t.Fatal("runVerify() succeeded on corrupt chunk") } if !strings.Contains(err.Error(), "CHUNK CORRUPT") { - t.Fatalf("runVerify() error = %q, want chunk corrupt", err) + t.Fatalf("error = %v, want CHUNK CORRUPT", err) + } +} + +func TestRestoreRejectsEscapingChunkPath(t *testing.T) { + ident, backend, pub := repoFixture(t, 8) + snapshotID := "7f91c6c7-7a0b-44aa-ae23-997b60e4e998" + + manifest := core.Manifest{ + Version: repo.FormatVersion, + UUID: snapshotID, + OriginalName: "payload.bin", + OriginalSize: 4, + CreatedAt: "2026-01-01T00:00:00Z", + Recipient: pub, + PayloadSHA256: hashBytes([]byte("data")), + Chunks: []core.ChunkMeta{{ + Index: 0, + Name: "../evil", + PlainSize: 4, + PlainSHA256: hashBytes([]byte("data")), + }}, + } + plain, err := json.Marshal(manifest) + if err != nil { + t.Fatalf("marshal manifest: %v", err) + } + sealed, err := appcrypto.EncryptMulti([]string{pub}, plain) + if err != nil { + t.Fatalf("EncryptMulti() error: %v", err) + } + snapshotPath, err := repo.SnapshotPath(snapshotID) + if err != nil { + t.Fatalf("SnapshotPath() error: %v", err) + } + backend.Put(snapshotPath, sealed) + + if _, err := runRestore(snapshotID, t.TempDir(), ident, backend); err == nil { + t.Fatal("runRestore() succeeded with escaping chunk path") + } else if !strings.Contains(err.Error(), "escapes data prefix") { + t.Fatalf("error = %v, want escapes data prefix", err) } } @@ -206,11 +235,10 @@ func TestRunUploadWithStateStopsAtMidUploadFailure(t *testing.T) { } func TestRunResumeCompletesInterruptedUpload(t *testing.T) { - pubKey, _, identityFile, err := appcrypto.GenerateKey() + pubKey, _, _, err := appcrypto.GenerateKey() if err != nil { t.Fatalf("GenerateKey() returned error: %v", err) } - identityPath := writeFlowIdentityFile(t, identityFile) sourcePath := filepath.Join(t.TempDir(), "secret.txt") wantPayload := []byte("payload large enough for several encrypted chunks") @@ -241,22 +269,6 @@ func TestRunResumeCompletesInterruptedUpload(t *testing.T) { if memory.uploadCount(backupID+"/chunk_00001") != 1 { t.Fatalf("chunk_00001 upload count = %d, want 1", memory.uploadCount(backupID+"/chunk_00001")) } - - if err := runVerify(backupID, identityPath, memory); err != nil { - t.Fatalf("runVerify() returned error after resume: %v", err) - } - - outputDir := t.TempDir() - if err := runRestore(backupID, identityPath, memory, outputDir); err != nil { - t.Fatalf("runRestore() returned error after resume: %v", err) - } - gotPayload, err := os.ReadFile(filepath.Join(outputDir, "secret.txt")) - if err != nil { - t.Fatalf("failed to read restored file: %v", err) - } - if !bytes.Equal(gotPayload, wantPayload) { - t.Fatalf("restored payload = %q, want %q", gotPayload, wantPayload) - } } func TestRunResumeRejectsStateChunkPlainHashMismatch(t *testing.T) { @@ -374,12 +386,3 @@ func (m *memoryBackend) uploadCount(remotePath string) int { defer m.mu.Unlock() return m.uploads[remotePath] } - -func writeFlowIdentityFile(t *testing.T, content string) string { - t.Helper() - path := filepath.Join(t.TempDir(), "identity.txt") - if err := os.WriteFile(path, []byte(content), 0600); err != nil { - t.Fatalf("failed to write identity file: %v", err) - } - return path -} diff --git a/internal/flow/manifest_read.go b/internal/flow/manifest_read.go new file mode 100644 index 0000000..6f4d8f1 --- /dev/null +++ b/internal/flow/manifest_read.go @@ -0,0 +1,49 @@ +package flow + +import ( + "bytes" + "encoding/json" + "fmt" + "io" + "strings" + + "gloak/internal/core" + "gloak/internal/crypto" + "gloak/internal/repo" + "gloak/internal/storage" +) + +// readManifest downloads the snapshot for snapshotID and decrypts it with the +// identity that opened the repository. +func readManifest(backend storage.Backend, identityPath, snapshotID string) (core.Manifest, error) { + path, err := repo.SnapshotPath(snapshotID) + if err != nil { + return core.Manifest{}, err + } + var buf bytes.Buffer + if err := backend.Download(path, &buf); err != nil { + return core.Manifest{}, fmt.Errorf("snapshot %s not found: %w", snapshotID, err) + } + reader, err := crypto.DecryptReader(identityPath, &buf) + if err != nil { + return core.Manifest{}, fmt.Errorf("failed to decrypt snapshot: %w", err) + } + plain, err := io.ReadAll(reader) + if err != nil { + return core.Manifest{}, fmt.Errorf("failed to read snapshot: %w", err) + } + var manifest core.Manifest + if err := json.Unmarshal(plain, &manifest); err != nil { + return core.Manifest{}, fmt.Errorf("failed to parse snapshot: %w", err) + } + return manifest, nil +} + +// validateChunkPath rejects manifest chunk paths that fall outside data/ or +// contain traversal segments. +func validateChunkPath(path string) error { + if !strings.HasPrefix(path, repo.DataPrefix+"/") || strings.Contains(path, "..") { + return fmt.Errorf("chunk path escapes data prefix: %s", path) + } + return nil +} diff --git a/internal/flow/restore.go b/internal/flow/restore.go index 8c7c962..e759009 100644 --- a/internal/flow/restore.go +++ b/internal/flow/restore.go @@ -4,133 +4,74 @@ import ( "bytes" "crypto/sha256" "encoding/hex" - "encoding/json" "fmt" - "io" "os" "path/filepath" "github.com/pterm/pterm" - "gloak/internal/core" "gloak/internal/crypto" + "gloak/internal/repo" "gloak/internal/storage" ) -func RunRestore(backupID string, identityPath string, remote string, outputDir string) error { - return runRestore(backupID, identityPath, storage.NewRcloneBackend(remote), outputDir) -} - -func runRestore(backupID string, identityPath string, backend storage.Backend, outputDir string) error { - pterm.Info.Printf("Starting restore for UUID: %s\n", backupID) - - pterm.Info.Println("Downloading encrypted manifest...") - var manifestBuf bytes.Buffer - err := backend.Download(fmt.Sprintf("%s/manifest.age", backupID), &manifestBuf) +func RunRestore(snapshotID, identityPath, remote, outputDir string) error { + backend := storage.NewRcloneBackend(remote) + outputPath, err := runRestore(snapshotID, outputDir, identityPath, backend) if err != nil { - return fmt.Errorf("failed to download manifest.age: %w", err) + return err } + pterm.Success.Println("Restore completed successfully! File saved to:", outputPath) + return nil +} - pterm.Info.Println("Decrypting manifest...") - manifestReader, err := crypto.DecryptReader(identityPath, &manifestBuf) +func runRestore(snapshotID, outputDir, identityPath string, backend storage.Backend) (string, error) { + r, err := repo.Open(backend, identityPath) if err != nil { - return fmt.Errorf("failed to decrypt manifest: %w", err) + return "", err } - - manifestBytes, err := io.ReadAll(manifestReader) + manifest, err := readManifest(backend, identityPath, snapshotID) if err != nil { - return fmt.Errorf("failed to read manifest contents: %w", err) + return "", err } - var manifest core.Manifest - if err := json.Unmarshal(manifestBytes, &manifest); err != nil { - return fmt.Errorf("failed to parse manifest JSON: %w", err) + if filepath.Base(manifest.OriginalName) != manifest.OriginalName { + return "", fmt.Errorf("invalid original filename in manifest: %s", manifest.OriginalName) } - - pterm.Success.Printf("Manifest valid! Restoring file: %s (%d bytes)\n", manifest.OriginalName, manifest.OriginalSize) - if err := os.MkdirAll(outputDir, 0755); err != nil { - return fmt.Errorf("failed to create output directory: %w", err) - } - if filepath.Base(manifest.OriginalName) != manifest.OriginalName { - return fmt.Errorf("invalid original filename in manifest: %s", manifest.OriginalName) + return "", fmt.Errorf("failed to create output directory: %w", err) } outputPath := filepath.Join(outputDir, manifest.OriginalName) outFile, err := os.OpenFile(outputPath, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, 0600) if err != nil { - return fmt.Errorf("failed to create output file: %w", err) + return "", fmt.Errorf("failed to create output file: %w", err) } defer outFile.Close() - p, _ := pterm.DefaultProgressbar. - WithTotal(int(manifest.OriginalSize)). - WithTitle("Downloading & Decrypting..."). - WithRemoveWhenDone(false). - Start() - payloadHasher := sha256.New() - for _, chunk := range manifest.Chunks { - var encryptedChunk bytes.Buffer - if err := backend.Download(fmt.Sprintf("%s/%s", backupID, chunk.Name), &encryptedChunk); err != nil { - p.Stop() - return fmt.Errorf("failed to download chunk %s: %w", chunk.Name, err) + for _, meta := range manifest.Chunks { + if err := validateChunkPath(meta.Name); err != nil { + return "", err } - - encryptedBytes := encryptedChunk.Bytes() - if int64(len(encryptedBytes)) != chunk.EncryptedSize { - p.Stop() - return fmt.Errorf("chunk %s corrupted/tampered! Size mismatch", chunk.Name) + var buf bytes.Buffer + if err := backend.Download(meta.Name, &buf); err != nil { + return "", fmt.Errorf("CHUNK CORRUPT %d (%s): missing object: %w", meta.Index, meta.Name, err) } - if hashBytes(encryptedBytes) != chunk.EncryptedSHA256 { - p.Stop() - return fmt.Errorf("chunk %s corrupted/tampered! Hash mismatch", chunk.Name) - } - - decReader, err := crypto.DecryptReader(identityPath, bytes.NewReader(encryptedBytes)) - if err != nil { - p.Stop() - return fmt.Errorf("failed to decrypt chunk %s: %w", chunk.Name, err) - } - plainBytes, err := io.ReadAll(decReader) + plain, err := crypto.OpenChunk(r.MasterKey, meta.PlainSHA256, buf.Bytes()) if err != nil { - p.Stop() - return fmt.Errorf("failed to read decrypted chunk %s: %w", chunk.Name, err) + return "", fmt.Errorf("CHUNK CORRUPT %d (%s): %w", meta.Index, meta.Name, err) } - if int64(len(plainBytes)) != chunk.PlainSize { - p.Stop() - return fmt.Errorf("chunk %s corrupted/tampered! Plain size mismatch", chunk.Name) + if int64(len(plain)) != meta.PlainSize || hashBytes(plain) != meta.PlainSHA256 { + return "", fmt.Errorf("CHUNK CORRUPT %d (%s): plaintext mismatch", meta.Index, meta.Name) } - if hashBytes(plainBytes) != chunk.PlainSHA256 { - p.Stop() - return fmt.Errorf("chunk %s corrupted/tampered! Plain hash mismatch", chunk.Name) + if _, err := outFile.Write(plain); err != nil { + return "", fmt.Errorf("failed to write chunk %d: %w", meta.Index, err) } - if _, err := outFile.Write(plainBytes); err != nil { - p.Stop() - return fmt.Errorf("failed to write restored chunk %s: %w", chunk.Name, err) - } - payloadHasher.Write(plainBytes) - p.Add(len(plainBytes)) + payloadHasher.Write(plain) } if hex.EncodeToString(payloadHasher.Sum(nil)) != manifest.PayloadSHA256 { - p.Stop() - return fmt.Errorf("fatal: overall payload hash does not match manifest") - } - - p.Stop() - pterm.Success.Println("Restore completed successfully! File saved to:", outputPath) - return nil -} - -type RestoreProgressReader struct { - io.Reader - pb *pterm.ProgressbarPrinter -} - -func (pr *RestoreProgressReader) Read(p []byte) (int, error) { - n, err := pr.Reader.Read(p) - if n > 0 { - pr.pb.Add(n) + return "", fmt.Errorf("fatal: overall payload hash does not match manifest") } - return n, err + return outputPath, nil } diff --git a/internal/flow/verify.go b/internal/flow/verify.go index 72b917c..9176356 100644 --- a/internal/flow/verify.go +++ b/internal/flow/verify.go @@ -4,103 +4,66 @@ import ( "bytes" "crypto/sha256" "encoding/hex" - "encoding/json" "fmt" - "io" "github.com/pterm/pterm" - "gloak/internal/core" "gloak/internal/crypto" + "gloak/internal/repo" "gloak/internal/storage" ) -func RunVerify(backupID string, identityPath string, remote string) error { - return runVerify(backupID, identityPath, storage.NewRcloneBackend(remote)) -} - -func runVerify(backupID string, identityPath string, backend storage.Backend) error { - pterm.Info.Printf("Starting verification for UUID: %s\n", backupID) - - var manifestBuf bytes.Buffer - err := backend.Download(fmt.Sprintf("%s/manifest.age", backupID), &manifestBuf) +func RunVerify(snapshotID, identityPath, remote string) error { + backend := storage.NewRcloneBackend(remote) + spinner, _ := pterm.DefaultSpinner.Start("Verifying backup...") + err := runVerify(snapshotID, identityPath, backend) if err != nil { - return fmt.Errorf("failed to download manifest.age: %w", err) + _ = spinner.Stop() + return err } + _ = spinner.Stop() + pterm.Success.Println("Backup verified successfully") + return nil +} - pterm.Info.Println("Decrypting manifest...") - manifestReader, err := crypto.DecryptReader(identityPath, &manifestBuf) +func runVerify(snapshotID, identityPath string, backend storage.Backend) error { + r, err := repo.Open(backend, identityPath) if err != nil { - return fmt.Errorf("failed to decrypt manifest (invalid key or corrupted file): %w", err) + return err } - - manifestBytes, err := io.ReadAll(manifestReader) + manifest, err := readManifest(backend, identityPath, snapshotID) if err != nil { - return fmt.Errorf("failed to read manifest contents: %w", err) - } - - var manifest core.Manifest - if err := json.Unmarshal(manifestBytes, &manifest); err != nil { - return fmt.Errorf("failed to parse manifest JSON: %w", err) + return err } - pterm.Success.Printf("Manifest valid! Verifying %d chunk...\n", len(manifest.Chunks)) - - p, _ := pterm.DefaultProgressbar. - WithTotal(len(manifest.Chunks)). - WithTitle("Verifying Chunks..."). - WithRemoveWhenDone(false). - Start() - payloadHasher := sha256.New() - - for _, chunk := range manifest.Chunks { - var encryptedChunk bytes.Buffer - err := backend.Download(fmt.Sprintf("%s/%s", backupID, chunk.Name), &encryptedChunk) - if err != nil { - p.Stop() - return fmt.Errorf("failed to download chunk %s: %w", chunk.Name, err) + for _, meta := range manifest.Chunks { + if err := validateChunkPath(meta.Name); err != nil { + return err } - - encryptedBytes := encryptedChunk.Bytes() - if int64(len(encryptedBytes)) != chunk.EncryptedSize { - p.Stop() - return fmt.Errorf("CHUNK CORRUPT: Size %s mismatch!", chunk.Name) + var buf bytes.Buffer + if err := backend.Download(meta.Name, &buf); err != nil { + return fmt.Errorf("CHUNK CORRUPT %d (%s): missing object: %w", meta.Index, meta.Name, err) } - if hashBytes(encryptedBytes) != chunk.EncryptedSHA256 { - p.Stop() - return fmt.Errorf("CHUNK CORRUPT: Hash %s mismatch!", chunk.Name) + sealed := buf.Bytes() + if int64(len(sealed)) != meta.EncryptedSize { + return fmt.Errorf("CHUNK CORRUPT %d (%s): encrypted size mismatch", meta.Index, meta.Name) } - - plainReader, err := crypto.DecryptReader(identityPath, bytes.NewReader(encryptedBytes)) - if err != nil { - p.Stop() - return fmt.Errorf("failed to decrypt chunk %s: %w", chunk.Name, err) + if hashBytes(sealed) != meta.EncryptedSHA256 { + return fmt.Errorf("CHUNK CORRUPT %d (%s): encrypted sha256 mismatch", meta.Index, meta.Name) } - plainBytes, err := io.ReadAll(plainReader) + plain, err := crypto.OpenChunk(r.MasterKey, meta.PlainSHA256, sealed) if err != nil { - p.Stop() - return fmt.Errorf("failed to read decrypted chunk %s: %w", chunk.Name, err) + return fmt.Errorf("CHUNK CORRUPT %d (%s): %w", meta.Index, meta.Name, err) } - if int64(len(plainBytes)) != chunk.PlainSize { - p.Stop() - return fmt.Errorf("CHUNK CORRUPT: Plain size %s mismatch!", chunk.Name) + if int64(len(plain)) != meta.PlainSize || hashBytes(plain) != meta.PlainSHA256 { + return fmt.Errorf("CHUNK CORRUPT %d (%s): plaintext mismatch", meta.Index, meta.Name) } - if hashBytes(plainBytes) != chunk.PlainSHA256 { - p.Stop() - return fmt.Errorf("CHUNK CORRUPT: Plain hash %s mismatch!", chunk.Name) - } - payloadHasher.Write(plainBytes) - - p.Add(1) + payloadHasher.Write(plain) } - p.Stop() - if hex.EncodeToString(payloadHasher.Sum(nil)) != manifest.PayloadSHA256 { - return fmt.Errorf("FATAL: Overall payload hash differs from manifest! Data has been modified") + return fmt.Errorf("FATAL: overall payload hash differs from manifest") } - - pterm.Success.Println("VERIFIED! All chunks are healthy and checksums match 100%.") return nil } From ba9e9e16dadcd038c1487fb068d6042beab80639 Mon Sep 17 00:00:00 2001 From: Syrup Date: Sun, 11 Oct 2026 21:32:27 +0700 Subject: [PATCH 11/48] feat(cli): switch to init/backup commands, drop v1 path --- backup.go | 45 +++++ cleanup.go | 145 --------------- internal/core/manifest.go | 58 ------ internal/core/manifest_test.go | 84 --------- internal/flow/flow_test.go | 277 ----------------------------- internal/flow/resume.go | 110 ------------ internal/flow/resume_test.go | 104 ----------- internal/flow/upload.go | 209 ---------------------- internal/flow/upload_state.go | 109 ------------ internal/flow/upload_state_test.go | 135 -------------- main.go | 4 +- main_test.go | 6 +- resume.go | 16 -- upload.go | 43 ----- 14 files changed, 48 insertions(+), 1297 deletions(-) create mode 100644 backup.go delete mode 100644 cleanup.go delete mode 100644 internal/core/manifest_test.go delete mode 100644 internal/flow/resume.go delete mode 100644 internal/flow/resume_test.go delete mode 100644 internal/flow/upload.go delete mode 100644 internal/flow/upload_state.go delete mode 100644 internal/flow/upload_state_test.go delete mode 100644 resume.go delete mode 100644 upload.go diff --git a/backup.go b/backup.go new file mode 100644 index 0000000..a947d99 --- /dev/null +++ b/backup.go @@ -0,0 +1,45 @@ +package main + +import ( + "fmt" + "os" + "path/filepath" + "strings" + + "github.com/alecthomas/kong" + + "gloak/internal/flow" +) + +// BackupCmd backs up a file into a gloak repository. +type BackupCmd struct { + ExistingFile string `arg:"" name:"existingfile" help:"File to back up"` + Identity string `short:"i" default:"~/.gloak/identity.txt" help:"Path to identity file"` + Remote string `short:"R" required:"" help:"Repository remote, e.g. b2:my-bucket/gloak"` +} + +func (c *BackupCmd) Run(ctx *kong.Context) error { + identityPath, err := expandHome(c.Identity) + if err != nil { + return err + } + if _, err := flow.RunBackup(c.ExistingFile, identityPath, c.Remote); err != nil { + return fmt.Errorf("backup failed: %w", err) + } + return nil +} + +// expandHome resolves a leading ~ in path to the user's home directory. +func expandHome(path string) (string, error) { + if path != "~" && !strings.HasPrefix(path, "~/") { + return path, nil + } + home, err := os.UserHomeDir() + if err != nil { + return "", fmt.Errorf("failed to resolve home directory: %w", err) + } + if path == "~" { + return home, nil + } + return filepath.Join(home, path[2:]), nil +} diff --git a/cleanup.go b/cleanup.go deleted file mode 100644 index 1cdeff0..0000000 --- a/cleanup.go +++ /dev/null @@ -1,145 +0,0 @@ -package main - -import ( - "encoding/json" - "fmt" - "sort" - - "github.com/pterm/pterm" - - "gloak/internal/storage" -) - -type CleanupCmd struct { - Remote string `help:"Rclone remote URI (e.g., keraaaki:backup_folder)." required:"" short:"R"` - Yes bool `help:"Skip confirmation prompt." short:"y"` -} - -type lsjsonEntry struct { - Path string `json:"Path"` - Name string `json:"Name"` - IsDir bool `json:"IsDir"` - Size int64 `json:"Size"` - ModTime string `json:"ModTime"` -} - -func (c *CleanupCmd) Run() error { - pterm.Info.Println("Scanning remote for backup directories...") - - backend := storage.NewRcloneBackend(c.Remote) - - raw, err := backend.ListFiles(nil) - if err != nil { - return fmt.Errorf("failed to list remote: %w", err) - } - - var entries []lsjsonEntry - if err := json.Unmarshal(raw, &entries); err != nil { - return fmt.Errorf("failed to parse lsjson output: %w", err) - } - - // Collect top-level directories (potential backup folders) - var dirs []lsjsonEntry - for _, e := range entries { - if e.IsDir { - dirs = append(dirs, e) - } - } - - if len(dirs) == 0 { - pterm.Success.Println("No backup directories found. Nothing to clean up.") - return nil - } - - pterm.Info.Printf("Found %d backup directories. Checking for manifest.age...\n", len(dirs)) - - // Identify orphan directories (missing manifest.age) - var orphans []string - for _, d := range dirs { - filesInDir, err := backend.ListFiles([]string{d.Path}) - if err != nil { - pterm.Warning.Printf(" Could not list %s: %v — skipping\n", d.Path, err) - continue - } - - var innerEntries []lsjsonEntry - if err := json.Unmarshal(filesInDir, &innerEntries); err != nil { - pterm.Warning.Printf(" Could not parse listing for %s: %v — skipping\n", d.Path, err) - continue - } - - hasManifest := false - for _, ie := range innerEntries { - if !ie.IsDir && ie.Name == "manifest.age" { - hasManifest = true - break - } - } - - if !hasManifest { - orphans = append(orphans, d.Path) - } - } - - if len(orphans) == 0 { - pterm.Success.Println("All backup directories have a manifest.age. Nothing to clean up.") - return nil - } - - sort.Strings(orphans) - - pterm.Warning.Printf("Found %d orphan directorie(s) missing manifest.age:\n", len(orphans)) - for _, o := range orphans { - pterm.DefaultBasicText.Printf(" - %s\n", o) - } - - if !c.Yes { - fmt.Println() - result, _ := pterm.DefaultInteractiveConfirm. - WithDefaultText("Delete these directories?"). - Show() - if !result { - pterm.Info.Println("Aborted. No directories deleted.") - return nil - } - } - - return batchDelete(backend, orphans) -} - -func batchDelete(backend storage.Backend, orphans []string) error { - // Batch delete in chunks of up to 500 - const batchSize = 500 - deleted := 0 - var deleteErrors []string - - for i := 0; i < len(orphans); i += batchSize { - end := i + batchSize - if end > len(orphans) { - end = len(orphans) - } - batch := orphans[i:end] - - for _, path := range batch { - if err := backend.Delete(path); err != nil { - deleteErrors = append(deleteErrors, fmt.Sprintf("%s: %v", path, err)) - continue - } - deleted++ - pterm.Success.Printf(" Deleted: %s\n", path) - } - } - - fmt.Println() - pterm.Success.Printf("Cleanup complete: %d of %d orphan directorie(s) deleted.\n", deleted, len(orphans)) - - if len(deleteErrors) > 0 { - pterm.Warning.Printf("%d deletion(s) failed:\n", len(deleteErrors)) - for _, de := range deleteErrors { - pterm.Error.Printf(" %s\n", de) - } - return fmt.Errorf("%d deletion(s) failed", len(deleteErrors)) - } - - return nil -} diff --git a/internal/core/manifest.go b/internal/core/manifest.go index 5c4c991..4963d0a 100644 --- a/internal/core/manifest.go +++ b/internal/core/manifest.go @@ -1,14 +1,7 @@ package core import ( - "bytes" - "encoding/json" - "fmt" - "time" - "github.com/google/uuid" - - "gloak/internal/crypto" ) type Manifest struct { @@ -22,57 +15,6 @@ type Manifest struct { Chunks []ChunkMeta `json:"chunks"` } -type Locator struct { - Version string `json:"version"` - UUID string `json:"uuid"` -} - func GenerateUUID() string { return uuid.New().String() } - -func CreateManifest(id string, filename string, size int64, recipient string, payloadHash string, chunks []ChunkMeta) Manifest { - return Manifest{ - Version: "1.0", - UUID: id, - OriginalName: filename, - OriginalSize: size, - CreatedAt: time.Now().UTC().Format(time.RFC3339), - Recipient: recipient, - PayloadSHA256: payloadHash, - Chunks: chunks, - } -} - -func CreateLocator(id string) Locator { - return Locator{ - Version: "1.0", - UUID: id, - } -} - -func EncryptManifest(pubKey string, manifest Manifest) ([]byte, error) { - jsonData, err := json.MarshalIndent(manifest, "", " ") - if err != nil { - return nil, fmt.Errorf("failed to marshal manifest json: %w", err) - } - - var buf bytes.Buffer - - encWriter, err := crypto.EncryptWriter(pubKey, &buf) - if err != nil { - return nil, fmt.Errorf("failed to prepare age writer: %w", err) - } - - _, err = encWriter.Write(jsonData) - if err != nil { - return nil, fmt.Errorf("failed to write JSON to encryptor: %w", err) - } - encWriter.Close() - - return buf.Bytes(), nil -} - -func SerializeLocator(locator Locator) ([]byte, error) { - return json.MarshalIndent(locator, "", " ") -} diff --git a/internal/core/manifest_test.go b/internal/core/manifest_test.go deleted file mode 100644 index 7fc53de..0000000 --- a/internal/core/manifest_test.go +++ /dev/null @@ -1,84 +0,0 @@ -package core - -import ( - "bytes" - "encoding/json" - "io" - "os" - "path/filepath" - "testing" - - appcrypto "gloak/internal/crypto" -) - -func TestCreateManifestAndLocatorVersions(t *testing.T) { - chunks := []ChunkMeta{{Index: 0, Name: "chunk_00000", PlainSize: 10, PlainSHA256: "plain", EncryptedSize: 12, EncryptedSHA256: "encrypted"}} - manifest := CreateManifest("backup-id", "secret.tar", 123, "age1example", "payload-hash", chunks) - - if manifest.Version != "1.0" { - t.Fatalf("manifest version = %q, want 1.0", manifest.Version) - } - if manifest.UUID != "backup-id" || manifest.OriginalName != "secret.tar" || manifest.OriginalSize != 123 { - t.Fatalf("manifest fields not preserved: %+v", manifest) - } - if len(manifest.Chunks) != 1 || manifest.Chunks[0] != chunks[0] { - t.Fatalf("manifest chunks = %+v, want %+v", manifest.Chunks, chunks) - } - - locator := CreateLocator("backup-id") - if locator.Version != "1.0" || locator.UUID != "backup-id" { - t.Fatalf("locator = %+v, want version 1.0 and uuid backup-id", locator) - } -} - -func TestSerializeLocator(t *testing.T) { - data, err := SerializeLocator(CreateLocator("backup-id")) - if err != nil { - t.Fatalf("SerializeLocator() returned error: %v", err) - } - - var locator Locator - if err := json.Unmarshal(data, &locator); err != nil { - t.Fatalf("serialized locator is not valid JSON: %v", err) - } - if locator.Version != "1.0" || locator.UUID != "backup-id" { - t.Fatalf("locator = %+v, want version 1.0 and uuid backup-id", locator) - } -} - -func TestEncryptManifestCanBeDecrypted(t *testing.T) { - pubKey, _, identityFile, err := appcrypto.GenerateKey() - if err != nil { - t.Fatalf("GenerateKey() returned error: %v", err) - } - identityPath := filepath.Join(t.TempDir(), "identity.txt") - if err := os.WriteFile(identityPath, []byte(identityFile), 0600); err != nil { - t.Fatalf("failed to write identity file: %v", err) - } - - manifest := CreateManifest("backup-id", "secret.tar", 123, pubKey, "payload-hash", nil) - encrypted, err := EncryptManifest(pubKey, manifest) - if err != nil { - t.Fatalf("EncryptManifest() returned error: %v", err) - } - if bytes.Contains(encrypted, []byte("secret.tar")) { - t.Fatalf("encrypted manifest contains plaintext filename") - } - - r, err := appcrypto.DecryptReader(identityPath, bytes.NewReader(encrypted)) - if err != nil { - t.Fatalf("DecryptReader() returned error: %v", err) - } - plain, err := io.ReadAll(r) - if err != nil { - t.Fatalf("failed to read decrypted manifest: %v", err) - } - - var got Manifest - if err := json.Unmarshal(plain, &got); err != nil { - t.Fatalf("decrypted manifest is not valid JSON: %v", err) - } - if got.UUID != manifest.UUID || got.OriginalName != manifest.OriginalName || got.PayloadSHA256 != manifest.PayloadSHA256 { - t.Fatalf("decrypted manifest = %+v, want %+v", got, manifest) - } -} diff --git a/internal/flow/flow_test.go b/internal/flow/flow_test.go index 66534a8..187d4a6 100644 --- a/internal/flow/flow_test.go +++ b/internal/flow/flow_test.go @@ -3,18 +3,13 @@ package flow import ( "bytes" "encoding/json" - "fmt" - "io" "os" - "path/filepath" "strings" - "sync" "testing" "gloak/internal/core" appcrypto "gloak/internal/crypto" "gloak/internal/repo" - "gloak/internal/storage" ) func TestBackupVerifyRestoreRoundTrip(t *testing.T) { @@ -114,275 +109,3 @@ func TestRestoreRejectsEscapingChunkPath(t *testing.T) { t.Fatalf("error = %v, want escapes data prefix", err) } } - -func TestRunUploadWithStateDeletesStateAfterSuccess(t *testing.T) { - pubKey, _, _, err := appcrypto.GenerateKey() - if err != nil { - t.Fatalf("GenerateKey() returned error: %v", err) - } - - sourcePath := filepath.Join(t.TempDir(), "secret.txt") - if err := os.WriteFile(sourcePath, []byte("payload"), 0600); err != nil { - t.Fatalf("failed to write source file: %v", err) - } - - backend := newMemoryBackend() - stateDir := t.TempDir() - backupID := "7f91c6c7-7a0b-44aa-ae23-997b60e4e998" - if err := runUploadWithState(sourcePath, pubKey, backend, backupID, "memory:backups", stateDir); err != nil { - t.Fatalf("runUploadWithState() returned error: %v", err) - } - - if _, err := os.Stat(uploadStatePath(stateDir, backupID)); !os.IsNotExist(err) { - t.Fatalf("upload state should be deleted after success, stat error: %v", err) - } -} - -func TestRunUploadWithStateKeepsUploadedChunksAfterFailure(t *testing.T) { - pubKey, _, _, err := appcrypto.GenerateKey() - if err != nil { - t.Fatalf("GenerateKey() returned error: %v", err) - } - - sourcePath := filepath.Join(t.TempDir(), "secret.txt") - if err := os.WriteFile(sourcePath, []byte("payload"), 0600); err != nil { - t.Fatalf("failed to write source file: %v", err) - } - - backupID := "7f91c6c7-7a0b-44aa-ae23-997b60e4e998" - backend := &failUploadBackend{ - memoryBackend: newMemoryBackend(), - failPath: backupID + "/manifest.age", - } - stateDir := t.TempDir() - err = runUploadWithState(sourcePath, pubKey, backend, backupID, "memory:backups", stateDir) - if err == nil { - t.Fatalf("runUploadWithState() succeeded despite manifest upload failure") - } - if !strings.Contains(err.Error(), "failed to upload manifest.age") { - t.Fatalf("runUploadWithState() error = %q, want manifest upload failure", err) - } - - state, err := loadUploadState(stateDir, backupID) - if err != nil { - t.Fatalf("loadUploadState() returned error: %v", err) - } - if state.Remote != "memory:backups" || state.Recipient != pubKey { - t.Fatalf("state remote/recipient = %q/%q, want memory:backups/%q", state.Remote, state.Recipient, pubKey) - } - absSourcePath, err := filepath.Abs(sourcePath) - if err != nil { - t.Fatalf("failed to resolve source path: %v", err) - } - if state.SourcePath != absSourcePath { - t.Fatalf("state SourcePath = %q, want %q", state.SourcePath, absSourcePath) - } - if len(state.UploadedChunks) != 1 { - t.Fatalf("uploaded chunks in state = %d, want 1", len(state.UploadedChunks)) - } - if state.UploadedChunks[0].Name != "chunk_00000" || state.UploadedChunks[0].PlainSize == 0 || state.UploadedChunks[0].PlainSHA256 == "" || state.UploadedChunks[0].EncryptedSize == 0 || state.UploadedChunks[0].EncryptedSHA256 == "" { - t.Fatalf("uploaded chunk metadata not recorded correctly: %+v", state.UploadedChunks[0]) - } -} - -func TestRunUploadWithStateStopsAtMidUploadFailure(t *testing.T) { - pubKey, _, _, err := appcrypto.GenerateKey() - if err != nil { - t.Fatalf("GenerateKey() returned error: %v", err) - } - - sourcePath := filepath.Join(t.TempDir(), "secret.txt") - if err := os.WriteFile(sourcePath, []byte("payload large enough for several encrypted chunks"), 0600); err != nil { - t.Fatalf("failed to write source file: %v", err) - } - - backupID := "7f91c6c7-7a0b-44aa-ae23-997b60e4e998" - backend := &failUploadBackend{ - memoryBackend: newMemoryBackend(), - failPath: backupID + "/chunk_00001", - } - stateDir := t.TempDir() - err = runUploadWithStateAndChunkSize(sourcePath, pubKey, backend, backupID, "memory:backups", stateDir, 8) - if err == nil { - t.Fatalf("runUploadWithStateAndChunkSize() succeeded despite chunk upload failure") - } - if !strings.Contains(err.Error(), "chunk 1 failed to process") { - t.Fatalf("runUploadWithStateAndChunkSize() error = %q, want chunk 1 failure", err) - } - - state, err := loadUploadState(stateDir, backupID) - if err != nil { - t.Fatalf("loadUploadState() returned error: %v", err) - } - if state.ChunkSize != 8 { - t.Fatalf("state ChunkSize = %d, want 8", state.ChunkSize) - } - if len(state.UploadedChunks) != 1 { - t.Fatalf("uploaded chunks in state = %d, want 1", len(state.UploadedChunks)) - } - if state.UploadedChunks[0].Name != "chunk_00000" { - t.Fatalf("recorded chunk = %q, want chunk_00000", state.UploadedChunks[0].Name) - } - - if !backend.has(backupID + "/chunk_00000") { - t.Fatalf("chunk_00000 should exist after first chunk upload") - } - for _, path := range []string{backupID + "/chunk_00001", backupID + "/manifest.age", backupID + "/locator.json"} { - if backend.has(path) { - t.Fatalf("%s should not exist after mid-upload failure", path) - } - } -} - -func TestRunResumeCompletesInterruptedUpload(t *testing.T) { - pubKey, _, _, err := appcrypto.GenerateKey() - if err != nil { - t.Fatalf("GenerateKey() returned error: %v", err) - } - - sourcePath := filepath.Join(t.TempDir(), "secret.txt") - wantPayload := []byte("payload large enough for several encrypted chunks") - if err := os.WriteFile(sourcePath, wantPayload, 0600); err != nil { - t.Fatalf("failed to write source file: %v", err) - } - - backupID := "7f91c6c7-7a0b-44aa-ae23-997b60e4e998" - memory := newMemoryBackend() - failing := &failUploadBackend{ - memoryBackend: memory, - failPath: backupID + "/chunk_00001", - } - stateDir := t.TempDir() - if err := runUploadWithStateAndChunkSize(sourcePath, pubKey, failing, backupID, "memory:backups", stateDir, 8); err == nil { - t.Fatalf("runUploadWithStateAndChunkSize() succeeded despite chunk upload failure") - } - - if err := runResume(backupID, func(string) storage.Backend { return memory }, stateDir); err != nil { - t.Fatalf("runResume() returned error: %v", err) - } - if _, err := os.Stat(uploadStatePath(stateDir, backupID)); !os.IsNotExist(err) { - t.Fatalf("upload state should be deleted after resume, stat error: %v", err) - } - if memory.uploadCount(backupID+"/chunk_00000") != 1 { - t.Fatalf("chunk_00000 upload count = %d, want 1", memory.uploadCount(backupID+"/chunk_00000")) - } - if memory.uploadCount(backupID+"/chunk_00001") != 1 { - t.Fatalf("chunk_00001 upload count = %d, want 1", memory.uploadCount(backupID+"/chunk_00001")) - } -} - -func TestRunResumeRejectsStateChunkPlainHashMismatch(t *testing.T) { - pubKey, _, _, err := appcrypto.GenerateKey() - if err != nil { - t.Fatalf("GenerateKey() returned error: %v", err) - } - - sourcePath := filepath.Join(t.TempDir(), "secret.txt") - if err := os.WriteFile(sourcePath, []byte("payload large enough for several encrypted chunks"), 0600); err != nil { - t.Fatalf("failed to write source file: %v", err) - } - - backupID := "7f91c6c7-7a0b-44aa-ae23-997b60e4e998" - memory := newMemoryBackend() - failing := &failUploadBackend{ - memoryBackend: memory, - failPath: backupID + "/chunk_00001", - } - stateDir := t.TempDir() - if err := runUploadWithStateAndChunkSize(sourcePath, pubKey, failing, backupID, "memory:backups", stateDir, 8); err == nil { - t.Fatalf("runUploadWithStateAndChunkSize() succeeded despite chunk upload failure") - } - - state, err := loadUploadState(stateDir, backupID) - if err != nil { - t.Fatalf("loadUploadState() returned error: %v", err) - } - state.UploadedChunks[0].PlainSHA256 = "bad-hash" - if err := saveUploadState(stateDir, state); err != nil { - t.Fatalf("saveUploadState() returned error: %v", err) - } - - err = runResume(backupID, func(string) storage.Backend { return memory }, stateDir) - if err == nil { - t.Fatalf("runResume() succeeded with mismatched state chunk hash") - } - if !strings.Contains(err.Error(), "does not match source file") { - t.Fatalf("runResume() error = %q, want source mismatch error", err) - } -} - -type memoryBackend struct { - mu sync.Mutex - files map[string][]byte - uploads map[string]int -} - -type failUploadBackend struct { - *memoryBackend - failPath string -} - -func (f *failUploadBackend) Upload(remotePath string, in io.Reader) error { - if remotePath == f.failPath { - _, _ = io.Copy(io.Discard, in) - return fmt.Errorf("forced upload failure for %s", remotePath) - } - return f.memoryBackend.Upload(remotePath, in) -} - -func newMemoryBackend() *memoryBackend { - return &memoryBackend{files: make(map[string][]byte), uploads: make(map[string]int)} -} - -func (m *memoryBackend) Upload(remotePath string, in io.Reader) error { - data, err := io.ReadAll(in) - if err != nil { - return err - } - m.put(remotePath, data) - m.mu.Lock() - m.uploads[remotePath]++ - m.mu.Unlock() - return nil -} - -func (m *memoryBackend) Download(remotePath string, out io.Writer) error { - m.mu.Lock() - data, ok := m.files[remotePath] - m.mu.Unlock() - if !ok { - return fmt.Errorf("not found: %s", remotePath) - } - _, err := out.Write(data) - return err -} - -func (m *memoryBackend) ListFiles(prefix []string) ([]byte, error) { - return json.Marshal([]struct{}{}) -} - -func (m *memoryBackend) Delete(remotePath string) error { - m.mu.Lock() - defer m.mu.Unlock() - delete(m.files, remotePath) - return nil -} - -func (m *memoryBackend) has(remotePath string) bool { - m.mu.Lock() - defer m.mu.Unlock() - _, ok := m.files[remotePath] - return ok -} - -func (m *memoryBackend) put(remotePath string, data []byte) { - m.mu.Lock() - defer m.mu.Unlock() - m.files[remotePath] = append([]byte(nil), data...) -} - -func (m *memoryBackend) uploadCount(remotePath string) int { - m.mu.Lock() - defer m.mu.Unlock() - return m.uploads[remotePath] -} diff --git a/internal/flow/resume.go b/internal/flow/resume.go deleted file mode 100644 index fdb6cbe..0000000 --- a/internal/flow/resume.go +++ /dev/null @@ -1,110 +0,0 @@ -package flow - -import ( - "bytes" - "fmt" - "os" - "strings" - - "github.com/pterm/pterm" - - "gloak/internal/core" - "gloak/internal/storage" -) - -func RunResumeValidation(backupID string) (UploadState, error) { - return runResumeValidation(backupID, defaultUploadStateDir()) -} - -func RunResume(backupID string) error { - return runResume(backupID, func(remote string) storage.Backend { - return storage.NewRcloneBackend(remote) - }, defaultUploadStateDir()) -} - -func runResume(backupID string, newBackend func(string) storage.Backend, stateDir string) error { - state, err := runResumeValidation(backupID, stateDir) - if err != nil { - return err - } - - file, err := os.Open(state.SourcePath) - if err != nil { - return fmt.Errorf("failed to open source file from upload state: %w", err) - } - defer file.Close() - - pterm.Success.Printf("Found upload state: %s\n", state.BackupID) - pterm.DefaultBasicText.Printf("Source file : %s\n", state.SourcePath) - pterm.DefaultBasicText.Printf("Destination : %s\n", state.Remote) - pterm.DefaultBasicText.Printf("Uploaded chunks : %d\n", len(state.UploadedChunks)) - - backend := newBackend(state.Remote) - p, _ := pterm.DefaultProgressbar. - WithTotal(int(state.OriginalSize)). - WithTitle("Resuming upload..."). - WithRemoveWhenDone(false). - Start() - - chunksMeta, payloadHash, err := uploadPlaintextChunks(file, state.Recipient, backend, backupID, &state, stateDir, state.ChunkSize, func(chunk core.ChunkMeta) { - pterm.Info.Printf("Skipping %s\n", chunk.Name) - }, func(n int) { - p.Add(n) - }) - if err != nil { - p.Stop() - return fmt.Errorf("resume upload failed: %w", err) - } - p.Stop() - - pterm.Info.Println("Building and encrypting manifest...") - manifest := core.CreateManifest(state.BackupID, state.OriginalName, state.OriginalSize, state.Recipient, payloadHash, chunksMeta) - encryptedManifestBytes, err := core.EncryptManifest(state.Recipient, manifest) - if err != nil { - return fmt.Errorf("failed to encrypt manifest: %w", err) - } - if err := backend.Upload(fmt.Sprintf("%s/manifest.age", state.BackupID), bytes.NewReader(encryptedManifestBytes)); err != nil { - return fmt.Errorf("failed to upload manifest.age: %w", err) - } - - locator := core.CreateLocator(state.BackupID) - locatorBytes, _ := core.SerializeLocator(locator) - if err := backend.Upload(fmt.Sprintf("%s/locator.json", state.BackupID), bytes.NewReader(locatorBytes)); err != nil { - return fmt.Errorf("failed to upload locator.json: %w", err) - } - if err := deleteUploadState(stateDir, state.BackupID); err != nil { - return fmt.Errorf("resume completed but failed to delete upload state: %w", err) - } - - pterm.Success.Println("Resume completed successfully!") - return nil -} - -func runResumeValidation(backupID string, stateDir string) (UploadState, error) { - state, err := loadUploadState(stateDir, backupID) - if err != nil { - return UploadState{}, err - } - - info, err := os.Stat(state.SourcePath) - if err != nil { - return UploadState{}, fmt.Errorf("failed to stat source file from upload state: %w", err) - } - if info.IsDir() { - return UploadState{}, fmt.Errorf("source path from upload state is a directory: %s", state.SourcePath) - } - if info.Size() != state.OriginalSize { - return UploadState{}, fmt.Errorf("source file size changed: got %d, want %d", info.Size(), state.OriginalSize) - } - if strings.TrimSpace(state.Recipient) == "" { - return UploadState{}, fmt.Errorf("upload state recipient is empty") - } - if state.Remote == "" { - return UploadState{}, fmt.Errorf("upload state remote is empty") - } - if state.ChunkSize <= 0 { - return UploadState{}, fmt.Errorf("upload state chunk size must be positive") - } - - return state, nil -} diff --git a/internal/flow/resume_test.go b/internal/flow/resume_test.go deleted file mode 100644 index bfb5587..0000000 --- a/internal/flow/resume_test.go +++ /dev/null @@ -1,104 +0,0 @@ -package flow - -import ( - "os" - "path/filepath" - "strings" - "testing" - - "gloak/internal/core" -) - -func TestRunResumeValidationRejectsMissingState(t *testing.T) { - _, err := runResumeValidation("7f91c6c7-7a0b-44aa-ae23-997b60e4e998", t.TempDir()) - if err == nil { - t.Fatalf("runResumeValidation() succeeded with missing state") - } - if !strings.Contains(err.Error(), "failed to read upload state") { - t.Fatalf("runResumeValidation() error = %q, want missing state error", err) - } -} - -func TestRunResumeValidationRejectsMissingSourceFile(t *testing.T) { - stateDir := t.TempDir() - backupID := "7f91c6c7-7a0b-44aa-ae23-997b60e4e998" - state := NewUploadState(backupID, filepath.Join(t.TempDir(), "missing.txt"), "missing.txt", 7, "memory:backups", "age1recipient") - if err := saveUploadState(stateDir, state); err != nil { - t.Fatalf("saveUploadState() returned error: %v", err) - } - - _, err := runResumeValidation(backupID, stateDir) - if err == nil { - t.Fatalf("runResumeValidation() succeeded with missing source file") - } - if !strings.Contains(err.Error(), "failed to stat source file from upload state") { - t.Fatalf("runResumeValidation() error = %q, want source stat error", err) - } -} - -func TestRunResumeValidationRejectsChangedSourceSize(t *testing.T) { - stateDir := t.TempDir() - backupID := "7f91c6c7-7a0b-44aa-ae23-997b60e4e998" - sourcePath := writeResumeSource(t, []byte("current payload")) - state := NewUploadState(backupID, sourcePath, filepath.Base(sourcePath), 5, "memory:backups", "age1recipient") - if err := saveUploadState(stateDir, state); err != nil { - t.Fatalf("saveUploadState() returned error: %v", err) - } - - _, err := runResumeValidation(backupID, stateDir) - if err == nil { - t.Fatalf("runResumeValidation() succeeded with changed source size") - } - if !strings.Contains(err.Error(), "source file size changed") { - t.Fatalf("runResumeValidation() error = %q, want source size error", err) - } -} - -func TestRunResumeValidationRejectsEmptyRecipient(t *testing.T) { - stateDir := t.TempDir() - backupID := "7f91c6c7-7a0b-44aa-ae23-997b60e4e998" - sourcePath := writeResumeSource(t, []byte("payload")) - state := NewUploadState(backupID, sourcePath, filepath.Base(sourcePath), 7, "memory:backups", " ") - if err := saveUploadState(stateDir, state); err != nil { - t.Fatalf("saveUploadState() returned error: %v", err) - } - - _, err := runResumeValidation(backupID, stateDir) - if err == nil { - t.Fatalf("runResumeValidation() succeeded with empty recipient") - } - if !strings.Contains(err.Error(), "upload state recipient is empty") { - t.Fatalf("runResumeValidation() error = %q, want empty recipient error", err) - } -} - -func TestRunResumeValidationAcceptsValidState(t *testing.T) { - stateDir := t.TempDir() - backupID := "7f91c6c7-7a0b-44aa-ae23-997b60e4e998" - sourcePath := writeResumeSource(t, []byte("payload")) - state := NewUploadState(backupID, sourcePath, filepath.Base(sourcePath), 7, "memory:backups", "age1recipient") - state.UploadedChunks = []core.ChunkMeta{{Index: 0, Name: "chunk_00000", PlainSize: 7, PlainSHA256: hashBytes([]byte("payload")), EncryptedSize: 12, EncryptedSHA256: "encrypted"}} - if err := saveUploadState(stateDir, state); err != nil { - t.Fatalf("saveUploadState() returned error: %v", err) - } - - got, err := runResumeValidation(backupID, stateDir) - if err != nil { - t.Fatalf("runResumeValidation() returned error: %v", err) - } - if got.BackupID != backupID || got.SourcePath != sourcePath || got.Remote != "memory:backups" { - t.Fatalf("validated state = %+v, want backup/source/remote preserved", got) - } - if len(got.UploadedChunks) != 1 || got.UploadedChunks[0].Name != "chunk_00000" { - t.Fatalf("validated uploaded chunks = %+v, want chunk_00000", got.UploadedChunks) - } -} - -func writeResumeSource(t *testing.T, content []byte) string { - t.Helper() - path := filepath.Join(t.TempDir(), "source.txt") - if err := os.WriteFile(path, content, 0600); err != nil { - t.Fatalf("failed to write source file: %v", err) - } - return path -} diff --git a/internal/flow/upload.go b/internal/flow/upload.go deleted file mode 100644 index 3609017..0000000 --- a/internal/flow/upload.go +++ /dev/null @@ -1,209 +0,0 @@ -package flow - -import ( - "bytes" - "crypto/sha256" - "encoding/hex" - "fmt" - "io" - "os" - "path/filepath" - - "github.com/pterm/pterm" - - "gloak/internal/core" - "gloak/internal/crypto" - "gloak/internal/storage" -) - -func RunUpload(filePath string, recipientKey string, remote string) error { - return runUploadWithStateAndChunkSize(filePath, recipientKey, storage.NewRcloneBackend(remote), core.GenerateUUID(), remote, defaultUploadStateDir(), core.ChunkSize) -} - -func runUpload(filePath string, recipientKey string, backend storage.Backend, backupID string) error { - return runUploadWithState(filePath, recipientKey, backend, backupID, "", "") -} - -func runUploadWithState(filePath string, recipientKey string, backend storage.Backend, backupID string, remote string, stateDir string) error { - return runUploadWithStateAndChunkSize(filePath, recipientKey, backend, backupID, remote, stateDir, core.ChunkSize) -} - -func runUploadWithStateAndChunkSize(filePath string, recipientKey string, backend storage.Backend, backupID string, remote string, stateDir string, chunkSize int64) error { - file, err := os.Open(filePath) - if err != nil { - return fmt.Errorf("failed to open source file: %w", err) - } - defer file.Close() - - stat, err := file.Stat() - if err != nil { - return fmt.Errorf("failed to stat file: %w", err) - } - originalSize := stat.Size() - originalName := filepath.Base(filePath) - state := UploadState{} - if stateDir != "" { - absPath, err := filepath.Abs(filePath) - if err != nil { - return fmt.Errorf("failed to resolve source path: %w", err) - } - state = newUploadStateWithChunkSize(backupID, absPath, originalName, originalSize, remote, recipientKey, chunkSize) - if err := saveUploadState(stateDir, state); err != nil { - return err - } - } - - pterm.Success.Printf("Backup ID: %s\n", backupID) - - p, _ := pterm.DefaultProgressbar. - WithTotal(int(originalSize)). - WithTitle("Encrypting & Uploading..."). - WithRemoveWhenDone(false). - Start() - - chunksMeta, payloadHash, err := uploadPlaintextChunks(file, recipientKey, backend, backupID, &state, stateDir, chunkSize, nil, func(n int) { - p.Add(n) - }) - - if err != nil { - p.Stop() - return fmt.Errorf("chunking/upload process failed: %w", err) - } - - p.Stop() - - pterm.Info.Println("Building and encrypting manifest...") - manifest := core.CreateManifest(backupID, originalName, originalSize, recipientKey, payloadHash, chunksMeta) - - encryptedManifestBytes, err := core.EncryptManifest(recipientKey, manifest) - if err != nil { - return fmt.Errorf("failed to encrypt manifest: %w", err) - } - - err = backend.Upload(fmt.Sprintf("%s/manifest.age", backupID), bytes.NewReader(encryptedManifestBytes)) - if err != nil { - return fmt.Errorf("failed to upload manifest.age: %w", err) - } - - locator := core.CreateLocator(backupID) - locatorBytes, _ := core.SerializeLocator(locator) - err = backend.Upload(fmt.Sprintf("%s/locator.json", backupID), bytes.NewReader(locatorBytes)) - if err != nil { - return fmt.Errorf("failed to upload locator.json: %w", err) - } - if stateDir != "" { - if err := deleteUploadState(stateDir, backupID); err != nil { - return fmt.Errorf("backup completed but failed to delete upload state: %w", err) - } - } - - pterm.Success.Println("Backup completed successfully!") - pterm.DefaultBasicText.Printf("Save this UUID for restoration: %s\n", backupID) - return nil -} - -func uploadPlaintextChunks(file *os.File, recipientKey string, backend storage.Backend, backupID string, state *UploadState, stateDir string, chunkSize int64, onSkip func(core.ChunkMeta), onProgress func(int)) ([]core.ChunkMeta, string, error) { - if chunkSize <= 0 { - return nil, "", fmt.Errorf("chunk size must be positive") - } - - existingChunks := make(map[int]core.ChunkMeta, len(state.UploadedChunks)) - for _, chunk := range state.UploadedChunks { - existingChunks[chunk.Index] = chunk - } - - payloadHasher := sha256.New() - chunksMeta := make([]core.ChunkMeta, 0, len(state.UploadedChunks)) - buffer := make([]byte, int(chunkSize)) - - for index := 0; ; index++ { - n, readErr := io.ReadFull(file, buffer) - if readErr == io.EOF { - break - } - if readErr != nil && readErr != io.ErrUnexpectedEOF { - return nil, "", fmt.Errorf("failed to read source chunk %d: %w", index, readErr) - } - - plainChunk := append([]byte(nil), buffer[:n]...) - payloadHasher.Write(plainChunk) - plainHash := hashBytes(plainChunk) - chunkName := fmt.Sprintf("chunk_%05d", index) - - if existing, ok := existingChunks[index]; ok { - if existing.Name != chunkName { - return nil, "", fmt.Errorf("upload state chunk %d name mismatch: got %s, want %s", index, existing.Name, chunkName) - } - if existing.PlainSize != int64(n) || existing.PlainSHA256 != plainHash { - return nil, "", fmt.Errorf("upload state chunk %d does not match source file", index) - } - chunksMeta = append(chunksMeta, existing) - if onSkip != nil { - onSkip(existing) - } - } else { - encryptedChunk, err := encryptChunk(recipientKey, plainChunk) - if err != nil { - return nil, "", fmt.Errorf("failed to encrypt chunk %d: %w", index, err) - } - - chunk := core.ChunkMeta{ - Index: index, - Name: chunkName, - PlainSize: int64(n), - PlainSHA256: plainHash, - EncryptedSize: int64(len(encryptedChunk)), - EncryptedSHA256: hashBytes(encryptedChunk), - } - - if err := backend.Upload(fmt.Sprintf("%s/%s", backupID, chunkName), bytes.NewReader(encryptedChunk)); err != nil { - return nil, "", fmt.Errorf("chunk %d failed to process: %w", index, err) - } - chunksMeta = append(chunksMeta, chunk) - if stateDir != "" { - state.UploadedChunks = append(state.UploadedChunks, chunk) - if err := saveUploadState(stateDir, *state); err != nil { - return nil, "", fmt.Errorf("chunk %d callback failed: %w", index, err) - } - } - } - - if onProgress != nil { - onProgress(n) - } - if readErr == io.ErrUnexpectedEOF { - break - } - } - - return chunksMeta, hex.EncodeToString(payloadHasher.Sum(nil)), nil -} - -func encryptChunk(recipientKey string, plainChunk []byte) ([]byte, error) { - var encrypted bytes.Buffer - encWriter, err := crypto.EncryptWriter(recipientKey, &encrypted) - if err != nil { - return nil, err - } - if _, err := encWriter.Write(plainChunk); err != nil { - _ = encWriter.Close() - return nil, err - } - if err := encWriter.Close(); err != nil { - return nil, err - } - return encrypted.Bytes(), nil -} - -type ProgressReader struct { - io.Reader - pb *pterm.ProgressbarPrinter -} - -func (pr *ProgressReader) Read(p []byte) (int, error) { - n, err := pr.Reader.Read(p) - if n > 0 { - pr.pb.Add(n) - } - return n, err -} diff --git a/internal/flow/upload_state.go b/internal/flow/upload_state.go deleted file mode 100644 index 8596e71..0000000 --- a/internal/flow/upload_state.go +++ /dev/null @@ -1,109 +0,0 @@ -package flow - -import ( - "encoding/json" - "fmt" - "os" - "path/filepath" - "time" - - "gloak/internal/core" -) - -const uploadStateVersion = "1.0" - -type UploadState struct { - Version string `json:"version"` - BackupID string `json:"backup_id"` - SourcePath string `json:"source_path"` - OriginalName string `json:"original_name"` - OriginalSize int64 `json:"original_size"` - Remote string `json:"remote"` - Recipient string `json:"recipient"` - ChunkSize int64 `json:"chunk_size"` - UploadedChunks []core.ChunkMeta `json:"uploaded_chunks"` - CreatedAt string `json:"created_at"` - UpdatedAt string `json:"updated_at"` -} - -func NewUploadState(backupID string, sourcePath string, originalName string, originalSize int64, remote string, recipient string) UploadState { - return newUploadStateWithChunkSize(backupID, sourcePath, originalName, originalSize, remote, recipient, core.ChunkSize) -} - -func newUploadStateWithChunkSize(backupID string, sourcePath string, originalName string, originalSize int64, remote string, recipient string, chunkSize int64) UploadState { - now := uploadStateTimestamp() - return UploadState{ - Version: uploadStateVersion, - BackupID: backupID, - SourcePath: sourcePath, - OriginalName: originalName, - OriginalSize: originalSize, - Remote: remote, - Recipient: recipient, - ChunkSize: chunkSize, - CreatedAt: now, - UpdatedAt: now, - } -} - -func uploadStatePath(stateDir string, backupID string) string { - return filepath.Join(stateDir, backupID+".json") -} - -func defaultUploadStateDir() string { - home, err := os.UserHomeDir() - if err != nil { - home = "." - } - return filepath.Join(home, ".gloak", "uploads") -} - -func saveUploadState(stateDir string, state UploadState) error { - if err := os.MkdirAll(stateDir, 0700); err != nil { - return fmt.Errorf("failed to create upload state directory: %w", err) - } - - state.UpdatedAt = uploadStateTimestamp() - data, err := json.MarshalIndent(state, "", " ") - if err != nil { - return fmt.Errorf("failed to encode upload state: %w", err) - } - - path := uploadStatePath(stateDir, state.BackupID) - if err := os.WriteFile(path, data, 0600); err != nil { - return fmt.Errorf("failed to write upload state: %w", err) - } - - return nil -} - -func loadUploadState(stateDir string, backupID string) (UploadState, error) { - path := uploadStatePath(stateDir, backupID) - data, err := os.ReadFile(path) - if err != nil { - return UploadState{}, fmt.Errorf("failed to read upload state: %w", err) - } - - var state UploadState - if err := json.Unmarshal(data, &state); err != nil { - return UploadState{}, fmt.Errorf("failed to parse upload state: %w", err) - } - if state.BackupID != backupID { - return UploadState{}, fmt.Errorf("upload state backup id mismatch: got %s, want %s", state.BackupID, backupID) - } - - return state, nil -} - -func deleteUploadState(stateDir string, backupID string) error { - path := uploadStatePath(stateDir, backupID) - if err := os.Remove(path); err != nil { - return fmt.Errorf("failed to delete upload state: %w", err) - } - - return nil -} - -func uploadStateTimestamp() string { - return time.Now().UTC().Format(time.RFC3339Nano) -} diff --git a/internal/flow/upload_state_test.go b/internal/flow/upload_state_test.go deleted file mode 100644 index 535d176..0000000 --- a/internal/flow/upload_state_test.go +++ /dev/null @@ -1,135 +0,0 @@ -package flow - -import ( - "os" - "path/filepath" - "strings" - "testing" - "time" - - "gloak/internal/core" -) - -func TestNewUploadState(t *testing.T) { - state := NewUploadState("backup-id", "/tmp/secret.tar", "secret.tar", 123, "remote:path", "age1recipient") - - if state.Version != "1.0" { - t.Fatalf("Version = %q, want 1.0", state.Version) - } - if state.BackupID != "backup-id" || state.SourcePath != "/tmp/secret.tar" || state.OriginalName != "secret.tar" { - t.Fatalf("state identity fields not preserved: %+v", state) - } - if state.OriginalSize != 123 || state.Remote != "remote:path" || state.Recipient != "age1recipient" { - t.Fatalf("state backup fields not preserved: %+v", state) - } - if state.ChunkSize != core.ChunkSize { - t.Fatalf("ChunkSize = %d, want %d", state.ChunkSize, core.ChunkSize) - } - if state.CreatedAt == "" || state.UpdatedAt == "" { - t.Fatalf("timestamps should be set: %+v", state) - } - if _, err := time.Parse(time.RFC3339Nano, state.CreatedAt); err != nil { - t.Fatalf("CreatedAt is not RFC3339Nano: %v", err) - } -} - -func TestUploadStateSaveLoadRoundTrip(t *testing.T) { - stateDir := t.TempDir() - state := NewUploadState("backup-id", "/tmp/secret.tar", "secret.tar", 123, "remote:path", "age1recipient") - state.UploadedChunks = []core.ChunkMeta{{Index: 0, Name: "chunk_00000", PlainSize: 10, PlainSHA256: "plain", EncryptedSize: 12, EncryptedSHA256: "encrypted"}} - - if err := saveUploadState(stateDir, state); err != nil { - t.Fatalf("saveUploadState() returned error: %v", err) - } - - path := uploadStatePath(stateDir, "backup-id") - info, err := os.Stat(path) - if err != nil { - t.Fatalf("state file was not written: %v", err) - } - if info.Mode().Perm() != 0600 { - t.Fatalf("state file permissions = %v, want 0600", info.Mode().Perm()) - } - - got, err := loadUploadState(stateDir, "backup-id") - if err != nil { - t.Fatalf("loadUploadState() returned error: %v", err) - } - if got.BackupID != state.BackupID || got.SourcePath != state.SourcePath || got.Remote != state.Remote { - t.Fatalf("loaded state identity fields = %+v, want %+v", got, state) - } - if len(got.UploadedChunks) != 1 || got.UploadedChunks[0] != state.UploadedChunks[0] { - t.Fatalf("loaded chunks = %+v, want %+v", got.UploadedChunks, state.UploadedChunks) - } - createdAt, err := time.Parse(time.RFC3339Nano, got.CreatedAt) - if err != nil { - t.Fatalf("loaded CreatedAt is not RFC3339Nano: %v", err) - } - updatedAt, err := time.Parse(time.RFC3339Nano, got.UpdatedAt) - if err != nil { - t.Fatalf("loaded UpdatedAt is not RFC3339Nano: %v", err) - } - if updatedAt.Before(createdAt) { - t.Fatalf("UpdatedAt %s is before CreatedAt %s", got.UpdatedAt, got.CreatedAt) - } -} - -func TestUploadStatePathUsesBackupID(t *testing.T) { - stateDir := t.TempDir() - got := uploadStatePath(stateDir, "backup-id") - want := filepath.Join(stateDir, "backup-id.json") - if got != want { - t.Fatalf("uploadStatePath() = %q, want %q", got, want) - } -} - -func TestLoadUploadStateRejectsInvalidJSON(t *testing.T) { - stateDir := t.TempDir() - path := uploadStatePath(stateDir, "backup-id") - if err := os.WriteFile(path, []byte("{"), 0600); err != nil { - t.Fatalf("failed to write invalid state: %v", err) - } - - _, err := loadUploadState(stateDir, "backup-id") - if err == nil { - t.Fatalf("loadUploadState() succeeded with invalid JSON") - } - if !strings.Contains(err.Error(), "failed to parse upload state") { - t.Fatalf("loadUploadState() error = %q, want parse error", err) - } -} - -func TestLoadUploadStateRejectsBackupIDMismatch(t *testing.T) { - stateDir := t.TempDir() - state := NewUploadState("other-id", "/tmp/secret.tar", "secret.tar", 123, "remote:path", "age1recipient") - if err := saveUploadState(stateDir, state); err != nil { - t.Fatalf("saveUploadState() returned error: %v", err) - } - - if err := os.Rename(uploadStatePath(stateDir, "other-id"), uploadStatePath(stateDir, "backup-id")); err != nil { - t.Fatalf("failed to rename state file: %v", err) - } - - _, err := loadUploadState(stateDir, "backup-id") - if err == nil { - t.Fatalf("loadUploadState() succeeded with mismatched backup id") - } - if !strings.Contains(err.Error(), "upload state backup id mismatch") { - t.Fatalf("loadUploadState() error = %q, want backup id mismatch", err) - } -} - -func TestDeleteUploadState(t *testing.T) { - stateDir := t.TempDir() - state := NewUploadState("backup-id", "/tmp/secret.tar", "secret.tar", 123, "remote:path", "age1recipient") - if err := saveUploadState(stateDir, state); err != nil { - t.Fatalf("saveUploadState() returned error: %v", err) - } - - if err := deleteUploadState(stateDir, "backup-id"); err != nil { - t.Fatalf("deleteUploadState() returned error: %v", err) - } - if _, err := os.Stat(uploadStatePath(stateDir, "backup-id")); !os.IsNotExist(err) { - t.Fatalf("state file still exists or stat failed unexpectedly: %v", err) - } -} diff --git a/main.go b/main.go index ab0c328..b846efc 100644 --- a/main.go +++ b/main.go @@ -15,11 +15,9 @@ var cli struct { Keygen KeygenCmd `cmd:"" help:"Generate an X25519 identity and recipient file."` Init InitCmd `cmd:"" help:"Initialize a new gloak repository." group:"Repository"` - Upload UploadCmd `cmd:"" help:"Encrypt, chunk, and upload a file to a storage destination."` - Resume ResumeCmd `cmd:"" help:"Validate saved state for an interrupted upload."` + Backup BackupCmd `cmd:"" help:"Encrypt, chunk, and upload a file to a repository."` Restore RestoreCmd `cmd:"" help:"Restore and decrypt a backup from storage."` Verify VerifyCmd `cmd:"" help:"Verify backup chunks and checksums."` - Cleanup CleanupCmd `cmd:"" help:"Remove orphan backup directories missing manifest.age."` Setup SetupCmd `cmd:"" help:"Download and install rclone to ~/.gloak/bin/."` } diff --git a/main_test.go b/main_test.go index da4898e..6705eb6 100644 --- a/main_test.go +++ b/main_test.go @@ -50,15 +50,13 @@ func TestCommandOptionShortTags(t *testing.T) { {name: "keygen force", cmd: KeygenCmd{}, fieldName: "Force", wantShort: "f"}, {name: "init remote", cmd: InitCmd{}, fieldName: "Remote", wantShort: "R"}, {name: "init recipient", cmd: InitCmd{}, fieldName: "Recipient", wantShort: "r"}, - {name: "upload recipient", cmd: UploadCmd{}, fieldName: "Recipient", wantShort: "r"}, - {name: "upload remote", cmd: UploadCmd{}, fieldName: "Remote", wantShort: "R"}, + {name: "backup identity", cmd: BackupCmd{}, fieldName: "Identity", wantShort: "i"}, + {name: "backup remote", cmd: BackupCmd{}, fieldName: "Remote", wantShort: "R"}, {name: "restore identity", cmd: RestoreCmd{}, fieldName: "Identity", wantShort: "i"}, {name: "restore remote", cmd: RestoreCmd{}, fieldName: "Remote", wantShort: "R"}, {name: "restore output dir", cmd: RestoreCmd{}, fieldName: "OutputDir", wantShort: "o"}, {name: "verify identity", cmd: VerifyCmd{}, fieldName: "Identity", wantShort: "i"}, {name: "verify remote", cmd: VerifyCmd{}, fieldName: "Remote", wantShort: "R"}, - {name: "cleanup remote", cmd: CleanupCmd{}, fieldName: "Remote", wantShort: "R"}, - {name: "cleanup yes", cmd: CleanupCmd{}, fieldName: "Yes", wantShort: "y"}, {name: "setup dir", cmd: SetupCmd{}, fieldName: "Dir", wantShort: "D"}, {name: "setup update", cmd: SetupCmd{}, fieldName: "Update", wantShort: "u"}, } diff --git a/resume.go b/resume.go deleted file mode 100644 index 3dfe6a9..0000000 --- a/resume.go +++ /dev/null @@ -1,16 +0,0 @@ -package main - -import ( - "github.com/pterm/pterm" - - "gloak/internal/flow" -) - -type ResumeCmd struct { - UUID string `arg:"" name:"uuid" help:"Backup UUID to resume."` -} - -func (r *ResumeCmd) Run() error { - pterm.Info.Println("Starting RESUME process...") - return flow.RunResume(r.UUID) -} diff --git a/upload.go b/upload.go deleted file mode 100644 index fa901fb..0000000 --- a/upload.go +++ /dev/null @@ -1,43 +0,0 @@ -package main - -import ( - "fmt" - "path/filepath" - "strings" - - "github.com/pterm/pterm" - - "gloak/internal/flow" -) - -type UploadCmd struct { - File string `arg:"" help:"File to back up." type:"existingfile"` - Recipient string `help:"Recipient public key (age1...) or path to recipient.txt." short:"r"` - Remote string `help:"Destination Rclone URI (e.g., keraaaki:backup_folder or /mnt/backups)." required:"" short:"R"` -} - -func (u *UploadCmd) Run() error { - pterm.Info.Println("Starting UPLOAD process...") - - recip := u.Recipient - if recip == "" { - defaultRecipPath := filepath.Join(getDefaultDir(), "recipient.txt") - val, err := readKeyFile(defaultRecipPath, "age1") - if err != nil { - return fmt.Errorf("recipient not provided and %s could not be read: %w", defaultRecipPath, err) - } - recip = val - pterm.Info.Printf("Using recipient from: %s\n", defaultRecipPath) - } else if !strings.HasPrefix(recip, "age1") { - val, err := readKeyFile(recip, "age1") - if err != nil { - return fmt.Errorf("failed to read recipient file %s: %w", recip, err) - } - recip = val - } - - pterm.DefaultBasicText.Printf("Target File : %s\n", u.File) - pterm.DefaultBasicText.Printf("Destination : %s\n", u.Remote) - - return flow.RunUpload(u.File, recip, u.Remote) -} From f33ad3fbede61fc807bf95b1a4e96b98f1e3139a Mon Sep 17 00:00:00 2001 From: Syrup Date: Sun, 11 Oct 2026 21:42:04 +0700 Subject: [PATCH 12/48] fix(cli): require existing backup source file --- backup.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/backup.go b/backup.go index a947d99..4c7a122 100644 --- a/backup.go +++ b/backup.go @@ -13,7 +13,7 @@ import ( // BackupCmd backs up a file into a gloak repository. type BackupCmd struct { - ExistingFile string `arg:"" name:"existingfile" help:"File to back up"` + ExistingFile string `arg:"" name:"existingfile" type:"existingfile" help:"File to back up"` Identity string `short:"i" default:"~/.gloak/identity.txt" help:"Path to identity file"` Remote string `short:"R" required:"" help:"Repository remote, e.g. b2:my-bucket/gloak"` } From 6c95a17f9605df9f403e19ff46b1e1cca52c7a40 Mon Sep 17 00:00:00 2001 From: Syrup Date: Sun, 11 Oct 2026 21:55:02 +0700 Subject: [PATCH 13/48] docs: add changelog and amend repo config recipients note --- CHANGELOG.md | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) create mode 100644 CHANGELOG.md diff --git a/CHANGELOG.md b/CHANGELOG.md new file mode 100644 index 0000000..c11d081 --- /dev/null +++ b/CHANGELOG.md @@ -0,0 +1,21 @@ +# Changelog + +## Unreleased + +### Added + +- `gloak init` creates a repository: a wrapped master key per recipient under `keys/`, a `repo.json` manifest, format 2.0. +- `gloak backup` uploads content-addressed encrypted chunks under `data/{hh}/`, records them in an encrypted epoch index under `index/`, and writes a snapshot under `snapshots/`. Repeat backups skip chunks already in the index. Chunks are sealed with zstd and then XChaCha20-Poly1305 under HKDF-derived keys. +- Snapshots are encrypted to every recipient listed at init time. Any matching identity can verify or restore. + +### Changed + +- `verify` and `restore` read the repository layout. Corruption errors keep the CHUNK CORRUPT prefix. + +### Removed + +- The `upload`, `resume`, and `cleanup` commands and the old filesystem-v1 single-file layout (`{uuid}/chunk_00000`, `manifest.age`, `locator.json`). There is no migration path. Backups made with earlier versions cannot be read by this version. + +### Fixed + +- The rclone config path is no longer hardcoded. An empty ConfigPath lets rclone use its own default. From 2f962ac11b1ae8e58618ef5ec95fbc9801e0b393 Mon Sep 17 00:00:00 2001 From: Syrup Date: Sun, 11 Oct 2026 22:14:29 +0700 Subject: [PATCH 14/48] fix(repo): refuse to overwrite existing repository without force --- init.go | 3 ++- internal/flow/backup_test.go | 2 +- internal/flow/init.go | 10 +++++--- internal/flow/init_test.go | 47 +++++++++++++++++++++++++++++++++--- internal/repo/repo.go | 12 +++++++-- internal/repo/repo_test.go | 6 ++--- main_test.go | 1 + 7 files changed, 67 insertions(+), 14 deletions(-) diff --git a/init.go b/init.go index fc5cc15..7b50bec 100644 --- a/init.go +++ b/init.go @@ -13,6 +13,7 @@ import ( type InitCmd struct { Remote string `short:"R" required:"" help:"Repository remote, e.g. b2:my-bucket/gloak or local path"` Recipient string `short:"r" help:"Recipient public key file or comma-separated keys (default ~/.gloak/recipient.txt)"` + Force bool `short:"f" help:"Overwrite an existing repository (makes old snapshots unreadable)"` } func (c *InitCmd) Run() error { @@ -20,7 +21,7 @@ func (c *InitCmd) Run() error { if err != nil { return err } - return flow.RunInit(recipients, c.Remote) + return flow.RunInit(recipients, c.Remote, c.Force) } // resolveInitRecipients turns the --recipient value into a key list. An empty diff --git a/internal/flow/backup_test.go b/internal/flow/backup_test.go index d4f8d90..f866e65 100644 --- a/internal/flow/backup_test.go +++ b/internal/flow/backup_test.go @@ -24,7 +24,7 @@ func repoFixture(t *testing.T, chunkSize int64) (string, *memtest.MemoryBackend, t.Fatalf("write identity: %v", err) } backend := memtest.NewMemoryBackend() - if _, _, err := repo.Init(backend, []string{pub}, chunkSize); err != nil { + if _, _, err := repo.Init(backend, []string{pub}, chunkSize, false); err != nil { t.Fatalf("repo.Init() error: %v", err) } return identPath, backend, pub diff --git a/internal/flow/init.go b/internal/flow/init.go index 43672d4..26e005c 100644 --- a/internal/flow/init.go +++ b/internal/flow/init.go @@ -11,11 +11,13 @@ import ( const defaultChunkSize = core.ChunkSize // RunInit creates a new gloak repository on remote and prints the outcome. -func RunInit(recipients []string, remote string) error { +// With force it overwrites an existing repository, which makes old snapshots +// unreadable. +func RunInit(recipients []string, remote string, force bool) error { backend := storage.NewRcloneBackend(remote) spinner, _ := pterm.DefaultSpinner.Start("Initializing repository...") - master, cfg, err := runInit(recipients, backend, defaultChunkSize) + master, cfg, err := runInit(recipients, backend, defaultChunkSize, force) if err != nil { _ = spinner.Stop() return err @@ -29,6 +31,6 @@ func RunInit(recipients []string, remote string) error { return nil } -func runInit(recipients []string, backend storage.Backend, chunkSize int64) ([]byte, repo.Config, error) { - return repo.Init(backend, recipients, chunkSize) +func runInit(recipients []string, backend storage.Backend, chunkSize int64, force bool) ([]byte, repo.Config, error) { + return repo.Init(backend, recipients, chunkSize, force) } diff --git a/internal/flow/init_test.go b/internal/flow/init_test.go index ae44f63..246e2b8 100644 --- a/internal/flow/init_test.go +++ b/internal/flow/init_test.go @@ -1,6 +1,7 @@ package flow import ( + "bytes" "strings" "testing" @@ -17,7 +18,7 @@ func TestRunInitCreatesRepository(t *testing.T) { _ = fileContent backend := memtest.NewMemoryBackend() - master, cfg, err := runInit([]string{pub}, backend, 4096) + master, cfg, err := runInit([]string{pub}, backend, 4096, false) if err != nil { t.Fatalf("runInit() error: %v", err) } @@ -36,7 +37,7 @@ func TestRunInitCreatesRepository(t *testing.T) { } func TestRunInitRejectsNoRecipients(t *testing.T) { - if _, _, err := runInit(nil, memtest.NewMemoryBackend(), 4096); err == nil { + if _, _, err := runInit(nil, memtest.NewMemoryBackend(), 4096, false); err == nil { t.Fatal("runInit() succeeded with no recipients") } else if !strings.Contains(err.Error(), "recipient") { t.Fatalf("runInit() error = %v, want recipient mention", err) @@ -44,7 +45,47 @@ func TestRunInitRejectsNoRecipients(t *testing.T) { } func TestRunInitRejectsInvalidRecipient(t *testing.T) { - if _, _, err := runInit([]string{"age1bad"}, memtest.NewMemoryBackend(), 4096); err == nil { + if _, _, err := runInit([]string{"age1bad"}, memtest.NewMemoryBackend(), 4096, false); err == nil { t.Fatal("runInit() succeeded with invalid recipient") } } + +func TestRunInitRejectsExistingRepository(t *testing.T) { + pub, _, _, err := crypto.GenerateKey() + if err != nil { + t.Fatalf("GenerateKey() error: %v", err) + } + backend := memtest.NewMemoryBackend() + + if _, _, err := runInit([]string{pub}, backend, 4096, false); err != nil { + t.Fatalf("runInit() error: %v", err) + } + if _, _, err := runInit([]string{pub}, backend, 4096, false); err == nil { + t.Fatal("runInit() succeeded on existing repository without force") + } else if !strings.Contains(err.Error(), "already exists") { + t.Fatalf("runInit() error = %v, want already exists mention", err) + } +} + +func TestRunInitForceOverwrites(t *testing.T) { + pub, _, _, err := crypto.GenerateKey() + if err != nil { + t.Fatalf("GenerateKey() error: %v", err) + } + backend := memtest.NewMemoryBackend() + + first, _, err := runInit([]string{pub}, backend, 4096, false) + if err != nil { + t.Fatalf("runInit() error: %v", err) + } + second, _, err := runInit([]string{pub}, backend, 4096, true) + if err != nil { + t.Fatalf("runInit(force) error: %v", err) + } + if bytes.Equal(first, second) { + t.Fatal("runInit(force) kept the old master key") + } + if !backend.Has(repo.ConfigPath) { + t.Fatal("repo.json missing after force init") + } +} diff --git a/internal/repo/repo.go b/internal/repo/repo.go index 1692529..1101bd5 100644 --- a/internal/repo/repo.go +++ b/internal/repo/repo.go @@ -48,8 +48,10 @@ func SnapshotPath(snapshotID string) (string, error) { } // Init creates the repository on the backend: one wrapped master key per -// recipient first, repo.json last as the completion marker. -func Init(backend storage.Backend, recipients []string, chunkSize int64) ([]byte, Config, error) { +// recipient first, repo.json last as the completion marker. It refuses to +// touch a repository that already exists unless force is set, because a +// fresh master key would orphan every snapshot sealed with the old one. +func Init(backend storage.Backend, recipients []string, chunkSize int64, force bool) ([]byte, Config, error) { if len(recipients) == 0 { return nil, Config{}, fmt.Errorf("at least one recipient is required") } @@ -62,6 +64,12 @@ func Init(backend storage.Backend, recipients []string, chunkSize int64) ([]byte return nil, Config{}, fmt.Errorf("chunk size must be positive") } + if !force { + if err := backend.Download(ConfigPath, io.Discard); err == nil { + return nil, Config{}, fmt.Errorf("repository already exists at this location (use force to overwrite)") + } + } + master := make([]byte, masterKeySize) if _, err := rand.Read(master); err != nil { return nil, Config{}, fmt.Errorf("failed to generate master key: %w", err) diff --git a/internal/repo/repo_test.go b/internal/repo/repo_test.go index 21e1f96..9aeb8d3 100644 --- a/internal/repo/repo_test.go +++ b/internal/repo/repo_test.go @@ -28,7 +28,7 @@ func TestInitWritesKeysAndConfig(t *testing.T) { pubB, _ := testRecipient(t) backend := memtest.NewMemoryBackend() - master, cfg, err := Init(backend, []string{pubA, pubB}, 4096) + master, cfg, err := Init(backend, []string{pubA, pubB}, 4096, false) if err != nil { t.Fatalf("Init() error: %v", err) } @@ -53,7 +53,7 @@ func TestOpenWithEachRecipientIdentity(t *testing.T) { pubA, identA := testRecipient(t) pubB, identB := testRecipient(t) backend := memtest.NewMemoryBackend() - master, _, err := Init(backend, []string{pubA, pubB}, 4096) + master, _, err := Init(backend, []string{pubA, pubB}, 4096, false) if err != nil { t.Fatalf("Init() error: %v", err) } @@ -73,7 +73,7 @@ func TestOpenRejectsWrongIdentity(t *testing.T) { pubA, _ := testRecipient(t) _, identOther := testRecipient(t) backend := memtest.NewMemoryBackend() - if _, _, err := Init(backend, []string{pubA}, 4096); err != nil { + if _, _, err := Init(backend, []string{pubA}, 4096, false); err != nil { t.Fatalf("Init() error: %v", err) } _, err := Open(backend, identOther) diff --git a/main_test.go b/main_test.go index 6705eb6..d97e02a 100644 --- a/main_test.go +++ b/main_test.go @@ -50,6 +50,7 @@ func TestCommandOptionShortTags(t *testing.T) { {name: "keygen force", cmd: KeygenCmd{}, fieldName: "Force", wantShort: "f"}, {name: "init remote", cmd: InitCmd{}, fieldName: "Remote", wantShort: "R"}, {name: "init recipient", cmd: InitCmd{}, fieldName: "Recipient", wantShort: "r"}, + {name: "init force", cmd: InitCmd{}, fieldName: "Force", wantShort: "f"}, {name: "backup identity", cmd: BackupCmd{}, fieldName: "Identity", wantShort: "i"}, {name: "backup remote", cmd: BackupCmd{}, fieldName: "Remote", wantShort: "R"}, {name: "restore identity", cmd: RestoreCmd{}, fieldName: "Identity", wantShort: "i"}, From 88edcaa2f210dcbba1c1e3a15be9397f93e36a2e Mon Sep 17 00:00:00 2001 From: Syrup Date: Mon, 12 Oct 2026 00:36:52 +0700 Subject: [PATCH 15/48] feat(flow): add standalone file-mode upload --- internal/flow/upload.go | 163 +++++++++++++++++++++++++++++++++++ internal/flow/upload_test.go | 117 +++++++++++++++++++++++++ 2 files changed, 280 insertions(+) create mode 100644 internal/flow/upload.go create mode 100644 internal/flow/upload_test.go diff --git a/internal/flow/upload.go b/internal/flow/upload.go new file mode 100644 index 0000000..839103c --- /dev/null +++ b/internal/flow/upload.go @@ -0,0 +1,163 @@ +package flow + +import ( + "bytes" + "crypto/rand" + "crypto/sha256" + "encoding/hex" + "encoding/json" + "fmt" + "io" + "os" + "path/filepath" + "regexp" + "time" + + "github.com/pterm/pterm" + + "gloak/internal/core" + "gloak/internal/crypto" + "gloak/internal/storage" +) + +const fileFormatVersion = "file-2" + +const masterKeySize = 32 + +var fileChunkNameRe = regexp.MustCompile(`^chunk_\d{5}$`) + +type fileManifest struct { + Version string `json:"version"` + UUID string `json:"uuid"` + OriginalName string `json:"original_name"` + OriginalSize int64 `json:"original_size"` + CreatedAt string `json:"created_at"` + PayloadSHA256 string `json:"payload_sha256"` + MasterKey string `json:"master_key"` + Chunks []core.ChunkMeta `json:"chunks"` +} + +type fileLocator struct { + Version string `json:"version"` + UUID string `json:"uuid"` +} + +func fileChunkName(index int) string { + return fmt.Sprintf("chunk_%05d", index) +} + +// validateFileChunkName accepts only relative chunk_NNNNN names. +func validateFileChunkName(name string) error { + if !fileChunkNameRe.MatchString(name) { + return fmt.Errorf("invalid chunk name: %s", name) + } + return nil +} + +// RunUpload seals filePath into a self-contained {uuid}/ folder on backend. +func RunUpload(filePath string, recipients []string, backend storage.Backend) (string, error) { + if len(recipients) == 0 { + return "", fmt.Errorf("at least one recipient is required") + } + file, err := os.Open(filePath) + if err != nil { + return "", fmt.Errorf("failed to open source file: %w", err) + } + defer file.Close() + + backupID := core.GenerateUUID() + masterKey := make([]byte, masterKeySize) + if _, err := rand.Read(masterKey); err != nil { + return "", fmt.Errorf("failed to generate master key: %w", err) + } + + spinner, _ := pterm.DefaultSpinner.Start("Encrypting and uploading...") + payloadHasher := sha256.New() + buffer := make([]byte, int(core.ChunkSize)) + var metas []core.ChunkMeta + var total int64 + + for index := 0; ; index++ { + n, readErr := io.ReadFull(file, buffer) + if readErr == io.EOF { + break + } + if readErr != nil && readErr != io.ErrUnexpectedEOF { + _ = spinner.Stop() + return "", fmt.Errorf("failed to read chunk %d: %w", index, readErr) + } + if n == 0 { + break + } + + plain := buffer[:n] + payloadHasher.Write(plain) + plainHash := hashBytes(plain) + sealed, err := crypto.SealChunk(masterKey, plainHash, plain) + if err != nil { + _ = spinner.Stop() + return "", fmt.Errorf("failed to seal chunk %d: %w", index, err) + } + encSum := sha256.Sum256(sealed) + name := fileChunkName(index) + remotePath := backupID + "/" + name + if err := backend.Upload(remotePath, bytes.NewReader(sealed)); err != nil { + _ = spinner.Stop() + return "", fmt.Errorf("failed to upload chunk %d: %w", index, err) + } + metas = append(metas, core.ChunkMeta{ + Index: index, + Name: name, + PlainSize: int64(n), + PlainSHA256: plainHash, + EncryptedSize: int64(len(sealed)), + EncryptedSHA256: hex.EncodeToString(encSum[:]), + }) + total += int64(n) + spinner.UpdateText(fmt.Sprintf("Uploaded %d chunks", len(metas))) + + if readErr == io.ErrUnexpectedEOF { + break + } + } + + manifest := fileManifest{ + Version: fileFormatVersion, + UUID: backupID, + OriginalName: filepath.Base(filePath), + OriginalSize: total, + CreatedAt: time.Now().UTC().Format(time.RFC3339), + PayloadSHA256: hex.EncodeToString(payloadHasher.Sum(nil)), + MasterKey: hex.EncodeToString(masterKey), + Chunks: metas, + } + manifestJSON, err := json.Marshal(manifest) + if err != nil { + _ = spinner.Stop() + return "", fmt.Errorf("failed to marshal manifest: %w", err) + } + sealedManifest, err := crypto.EncryptMulti(recipients, manifestJSON) + if err != nil { + _ = spinner.Stop() + return "", fmt.Errorf("failed to encrypt manifest: %w", err) + } + if err := backend.Upload(backupID+"/manifest.age", bytes.NewReader(sealedManifest)); err != nil { + _ = spinner.Stop() + return "", fmt.Errorf("failed to upload manifest: %w", err) + } + + locator := fileLocator{Version: fileFormatVersion, UUID: backupID} + locatorJSON, err := json.Marshal(locator) + if err != nil { + _ = spinner.Stop() + return "", fmt.Errorf("failed to marshal locator: %w", err) + } + if err := backend.Upload(backupID+"/locator.json", bytes.NewReader(locatorJSON)); err != nil { + _ = spinner.Stop() + return "", fmt.Errorf("failed to upload locator: %w", err) + } + + _ = spinner.Stop() + pterm.Success.Printf("Upload complete: %s (%d chunks, %d bytes)\n", backupID, len(metas), total) + return backupID, nil +} diff --git a/internal/flow/upload_test.go b/internal/flow/upload_test.go new file mode 100644 index 0000000..425ed68 --- /dev/null +++ b/internal/flow/upload_test.go @@ -0,0 +1,117 @@ +package flow + +import ( + "bytes" + "encoding/json" + "os" + "path/filepath" + "strings" + "testing" + + "filippo.io/age" + + "gloak/internal/core" + "gloak/internal/storage/memtest" +) + +func testIdentityFile(t *testing.T) (recipient, identityPath string) { + t.Helper() + id, err := age.GenerateX25519Identity() + if err != nil { + t.Fatalf("generate identity: %v", err) + } + path := filepath.Join(t.TempDir(), "identity.txt") + content := "# created: now\n# public key: " + id.Recipient().String() + "\n" + id.String() + "\n" + if err := os.WriteFile(path, []byte(content), 0600); err != nil { + t.Fatalf("write identity: %v", err) + } + return id.Recipient().String(), path +} + +func TestRunUploadWritesLocatorLast(t *testing.T) { + recipient, _ := testIdentityFile(t) + src := filepath.Join(t.TempDir(), "data.bin") + if err := os.WriteFile(src, []byte("hello gloak"), 0600); err != nil { + t.Fatal(err) + } + backend := memtest.NewMemoryBackend() + + uuid, err := RunUpload(src, []string{recipient}, backend) + if err != nil { + t.Fatalf("RunUpload() error: %v", err) + } + if uuid == "" { + t.Fatal("RunUpload() returned empty uuid") + } + if !backend.Has(uuid + "/locator.json") { + t.Fatalf("missing locator at %s/locator.json", uuid) + } + if !backend.Has(uuid + "/manifest.age") { + t.Fatal("missing manifest.age") + } + if !backend.Has(uuid + "/chunk_00000") { + t.Fatal("missing chunk_00000") + } + + var loc fileLocator + if err := json.Unmarshal(backend.Get(uuid+"/locator.json"), &loc); err != nil { + t.Fatalf("locator json: %v", err) + } + if loc.Version != fileFormatVersion || loc.UUID != uuid { + t.Fatalf("locator = %+v, want version %q uuid %q", loc, fileFormatVersion, uuid) + } + if backend.UploadCount(uuid+"/locator.json") != 1 { + t.Fatal("locator uploaded more than once") + } +} + +func TestRunUploadRejectsEmptyRecipients(t *testing.T) { + src := filepath.Join(t.TempDir(), "data.bin") + if err := os.WriteFile(src, []byte("x"), 0600); err != nil { + t.Fatal(err) + } + _, err := RunUpload(src, nil, memtest.NewMemoryBackend()) + if err == nil || !strings.Contains(err.Error(), "recipient") { + t.Fatalf("err = %v, want recipient error", err) + } +} + +func TestFileChunkNameAndValidation(t *testing.T) { + if got := fileChunkName(0); got != "chunk_00000" { + t.Fatalf("fileChunkName(0) = %q", got) + } + if err := validateFileChunkName("chunk_00001"); err != nil { + t.Fatalf("valid name rejected: %v", err) + } + bad := []string{ + "chunk_0000", + "chunk_000001", + "../evil", + "chunk_00000/../x", + "aaaa-bbbb-cccc-dddd-eeeeeeeeeeee/chunk_00000", + "/etc/passwd", + } + for _, name := range bad { + if err := validateFileChunkName(name); err == nil { + t.Fatalf("validateFileChunkName(%q) succeeded, want error", name) + } + } +} + +func TestRunUploadMultiChunk(t *testing.T) { + recipient, _ := testIdentityFile(t) + src := filepath.Join(t.TempDir(), "big.bin") + // ChunkSize is 20MiB; write slightly over one chunk to force two chunks. + data := bytes.Repeat([]byte("a"), int(core.ChunkSize)+10) + if err := os.WriteFile(src, data, 0600); err != nil { + t.Fatal(err) + } + backend := memtest.NewMemoryBackend() + uuid, err := RunUpload(src, []string{recipient}, backend) + if err != nil { + t.Fatalf("RunUpload() error: %v", err) + } + if !backend.Has(uuid+"/chunk_00000") || !backend.Has(uuid+"/chunk_00001") { + t.Fatal("expected two chunks") + } +} From 37201a16a377b4a7c77d14543ddf0a12f128df31 Mon Sep 17 00:00:00 2001 From: Syrup Date: Mon, 12 Oct 2026 00:39:02 +0700 Subject: [PATCH 16/48] feat(cli): add upload command for file-mode backups --- main.go | 1 + main_test.go | 15 +++++++++++++++ upload.go | 31 +++++++++++++++++++++++++++++++ 3 files changed, 47 insertions(+) create mode 100644 upload.go diff --git a/main.go b/main.go index b846efc..b2e07a2 100644 --- a/main.go +++ b/main.go @@ -16,6 +16,7 @@ var cli struct { Keygen KeygenCmd `cmd:"" help:"Generate an X25519 identity and recipient file."` Init InitCmd `cmd:"" help:"Initialize a new gloak repository." group:"Repository"` Backup BackupCmd `cmd:"" help:"Encrypt, chunk, and upload a file to a repository."` + Upload UploadCmd `cmd:"" help:"Encrypt and upload a file as a standalone folder (no repository)."` Restore RestoreCmd `cmd:"" help:"Restore and decrypt a backup from storage."` Verify VerifyCmd `cmd:"" help:"Verify backup chunks and checksums."` Setup SetupCmd `cmd:"" help:"Download and install rclone to ~/.gloak/bin/."` diff --git a/main_test.go b/main_test.go index d97e02a..93f0975 100644 --- a/main_test.go +++ b/main_test.go @@ -51,6 +51,8 @@ func TestCommandOptionShortTags(t *testing.T) { {name: "init remote", cmd: InitCmd{}, fieldName: "Remote", wantShort: "R"}, {name: "init recipient", cmd: InitCmd{}, fieldName: "Recipient", wantShort: "r"}, {name: "init force", cmd: InitCmd{}, fieldName: "Force", wantShort: "f"}, + {name: "upload recipient", cmd: UploadCmd{}, fieldName: "Recipient", wantShort: "r"}, + {name: "upload remote", cmd: UploadCmd{}, fieldName: "Remote", wantShort: "R"}, {name: "backup identity", cmd: BackupCmd{}, fieldName: "Identity", wantShort: "i"}, {name: "backup remote", cmd: BackupCmd{}, fieldName: "Remote", wantShort: "R"}, {name: "restore identity", cmd: RestoreCmd{}, fieldName: "Identity", wantShort: "i"}, @@ -75,6 +77,19 @@ func TestCommandOptionShortTags(t *testing.T) { } } +func TestUploadCmdFileIsExistingFileArg(t *testing.T) { + field, ok := reflect.TypeOf(UploadCmd{}).FieldByName("File") + if !ok { + t.Fatal("UploadCmd.File missing") + } + if field.Tag.Get("type") != "existingfile" { + t.Fatalf("type tag = %q, want existingfile", field.Tag.Get("type")) + } + if _, ok := field.Tag.Lookup("arg"); !ok { + t.Fatal("File should be a positional arg") + } +} + func TestResolveInitRecipientsReadsFile(t *testing.T) { path := filepath.Join(t.TempDir(), "recipient.txt") content := "# comment\n\nage1aaa\nage1bbb\n" diff --git a/upload.go b/upload.go new file mode 100644 index 0000000..82faa92 --- /dev/null +++ b/upload.go @@ -0,0 +1,31 @@ +package main + +import ( + "fmt" + + "github.com/pterm/pterm" + + "gloak/internal/flow" + "gloak/internal/storage" +) + +// UploadCmd writes a self-contained encrypted backup folder to a remote root. +type UploadCmd struct { + File string `arg:"" name:"file" type:"existingfile" help:"File to back up"` + Recipient string `short:"r" help:"Recipient public key file or comma-separated keys (default ~/.gloak/recipient.txt)"` + Remote string `short:"R" required:"" help:"Remote root, e.g. keraaaki:backups or /mnt/backups"` +} + +func (u *UploadCmd) Run() error { + recipients, err := resolveInitRecipients(u.Recipient) + if err != nil { + return err + } + pterm.Info.Println("Starting UPLOAD process...") + backupID, err := flow.RunUpload(u.File, recipients, storage.NewRcloneBackend(u.Remote)) + if err != nil { + return fmt.Errorf("upload failed: %w", err) + } + pterm.DefaultBasicText.Printf("Backup UUID: %s\n", backupID) + return nil +} From 03525490359732cd193369a1c9287028da94dfa5 Mon Sep 17 00:00:00 2001 From: Syrup Date: Mon, 12 Oct 2026 00:40:59 +0700 Subject: [PATCH 17/48] feat(flow): restore file-mode backups via locator detection --- internal/flow/filemode.go | 117 ++++++++++++++++++++++ internal/flow/filemode_test.go | 178 +++++++++++++++++++++++++++++++++ internal/flow/restore.go | 3 + 3 files changed, 298 insertions(+) create mode 100644 internal/flow/filemode.go create mode 100644 internal/flow/filemode_test.go diff --git a/internal/flow/filemode.go b/internal/flow/filemode.go new file mode 100644 index 0000000..3c364b5 --- /dev/null +++ b/internal/flow/filemode.go @@ -0,0 +1,117 @@ +package flow + +import ( + "bytes" + "crypto/sha256" + "encoding/hex" + "encoding/json" + "fmt" + "io" + "os" + "path/filepath" + + "gloak/internal/crypto" + "gloak/internal/storage" +) + +func isFileMode(backend storage.Backend, backupID string) bool { + var buf bytes.Buffer + return backend.Download(backupID+"/locator.json", &buf) == nil +} + +func readFileManifest(backend storage.Backend, identityPath, backupID string) (fileManifest, error) { + var loc fileLocator + var locBuf bytes.Buffer + if err := backend.Download(backupID+"/locator.json", &locBuf); err != nil { + return fileManifest{}, fmt.Errorf("backup %s incomplete or missing: no locator.json", backupID) + } + if err := json.Unmarshal(locBuf.Bytes(), &loc); err != nil { + return fileManifest{}, fmt.Errorf("invalid locator: %w", err) + } + if loc.UUID != backupID { + return fileManifest{}, fmt.Errorf("locator uuid mismatch: got %s want %s", loc.UUID, backupID) + } + + var manBuf bytes.Buffer + if err := backend.Download(backupID+"/manifest.age", &manBuf); err != nil { + return fileManifest{}, fmt.Errorf("failed to download manifest: %w", err) + } + reader, err := crypto.DecryptReader(identityPath, &manBuf) + if err != nil { + return fileManifest{}, fmt.Errorf("failed to decrypt manifest: %w", err) + } + plain, err := io.ReadAll(reader) + if err != nil { + return fileManifest{}, fmt.Errorf("failed to read manifest: %w", err) + } + var m fileManifest + if err := json.Unmarshal(plain, &m); err != nil { + return fileManifest{}, fmt.Errorf("failed to parse manifest: %w", err) + } + if m.UUID != backupID { + return fileManifest{}, fmt.Errorf("manifest uuid mismatch: got %s want %s", m.UUID, backupID) + } + if m.Version != fileFormatVersion { + return fileManifest{}, fmt.Errorf("unsupported manifest version: %s", m.Version) + } + if filepath.Base(m.OriginalName) != m.OriginalName { + return fileManifest{}, fmt.Errorf("invalid original filename in manifest: %s", m.OriginalName) + } + return m, nil +} + +func decodeFileMasterKey(m fileManifest) ([]byte, error) { + key, err := hex.DecodeString(m.MasterKey) + if err != nil || len(key) != masterKeySize { + return nil, fmt.Errorf("invalid master key in manifest") + } + return key, nil +} + +func runFileRestore(backupID, identityPath, outputDir string, backend storage.Backend) (string, error) { + m, err := readFileManifest(backend, identityPath, backupID) + if err != nil { + return "", err + } + masterKey, err := decodeFileMasterKey(m) + if err != nil { + return "", err + } + + if err := os.MkdirAll(outputDir, 0755); err != nil { + return "", fmt.Errorf("failed to create output directory: %w", err) + } + outputPath := filepath.Join(outputDir, m.OriginalName) + outFile, err := os.OpenFile(outputPath, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, 0600) + if err != nil { + return "", fmt.Errorf("failed to create output file: %w", err) + } + defer outFile.Close() + + payloadHasher := sha256.New() + for _, meta := range m.Chunks { + if err := validateFileChunkName(meta.Name); err != nil { + return "", fmt.Errorf("invalid chunk name in manifest: %w", err) + } + var buf bytes.Buffer + if err := backend.Download(backupID+"/"+meta.Name, &buf); err != nil { + return "", fmt.Errorf("CHUNK CORRUPT %d (%s): missing object: %w", meta.Index, meta.Name, err) + } + plain, err := crypto.OpenChunk(masterKey, meta.PlainSHA256, buf.Bytes()) + if err != nil { + return "", fmt.Errorf("CHUNK CORRUPT %d (%s): %w", meta.Index, meta.Name, err) + } + if int64(len(plain)) != meta.PlainSize || hashBytes(plain) != meta.PlainSHA256 { + return "", fmt.Errorf("CHUNK CORRUPT %d (%s): plaintext mismatch", meta.Index, meta.Name) + } + if _, err := outFile.Write(plain); err != nil { + return "", fmt.Errorf("failed to write chunk %d: %w", meta.Index, err) + } + payloadHasher.Write(plain) + } + + if hex.EncodeToString(payloadHasher.Sum(nil)) != m.PayloadSHA256 { + return "", fmt.Errorf("fatal: overall payload hash does not match manifest") + } + return outputPath, nil +} diff --git a/internal/flow/filemode_test.go b/internal/flow/filemode_test.go new file mode 100644 index 0000000..9d53142 --- /dev/null +++ b/internal/flow/filemode_test.go @@ -0,0 +1,178 @@ +package flow + +import ( + "bytes" + "encoding/json" + "os" + "path/filepath" + "testing" + + "gloak/internal/crypto" + "gloak/internal/storage/memtest" +) + +func resealFileManifest(t *testing.T, backend *memtest.MemoryBackend, identityPath, uuid string, m fileManifest) { + t.Helper() + data, err := os.ReadFile(identityPath) + if err != nil { + t.Fatal(err) + } + var recipient string + for _, line := range bytes.Split(data, []byte("\n")) { + if bytes.HasPrefix(bytes.TrimSpace(line), []byte("# public key:")) { + recipient = string(bytes.TrimSpace(bytes.TrimPrefix(bytes.TrimSpace(line), []byte("# public key:")))) + break + } + } + if recipient == "" { + t.Fatal("no public key in identity file") + } + raw, err := json.Marshal(m) + if err != nil { + t.Fatal(err) + } + sealed, err := crypto.EncryptMulti([]string{recipient}, raw) + if err != nil { + t.Fatal(err) + } + backend.Put(uuid+"/manifest.age", sealed) +} + +func TestRunRestoreFileModeRoundTrip(t *testing.T) { + recipient, identityPath := testIdentityFile(t) + src := filepath.Join(t.TempDir(), "secret.txt") + want := []byte("restore me please\n") + if err := os.WriteFile(src, want, 0600); err != nil { + t.Fatal(err) + } + backend := memtest.NewMemoryBackend() + backupID, err := RunUpload(src, []string{recipient}, backend) + if err != nil { + t.Fatalf("upload: %v", err) + } + + outDir := t.TempDir() + gotPath, err := runRestore(backupID, outDir, identityPath, backend) + if err != nil { + t.Fatalf("runRestore() error: %v", err) + } + got, err := os.ReadFile(gotPath) + if err != nil { + t.Fatal(err) + } + if !bytes.Equal(got, want) { + t.Fatalf("restored %q, want %q", got, want) + } +} + +func TestRunRestoreFileModeEmptyFile(t *testing.T) { + recipient, identityPath := testIdentityFile(t) + src := filepath.Join(t.TempDir(), "empty.bin") + if err := os.WriteFile(src, nil, 0600); err != nil { + t.Fatal(err) + } + backend := memtest.NewMemoryBackend() + backupID, err := RunUpload(src, []string{recipient}, backend) + if err != nil { + t.Fatalf("upload: %v", err) + } + + outDir := t.TempDir() + gotPath, err := runRestore(backupID, outDir, identityPath, backend) + if err != nil { + t.Fatalf("runRestore() error: %v", err) + } + got, err := os.ReadFile(gotPath) + if err != nil { + t.Fatal(err) + } + if len(got) != 0 { + t.Fatalf("restored %d bytes, want empty", len(got)) + } +} + +func TestRunRestoreFileModeEitherIdentity(t *testing.T) { + r1, id1 := testIdentityFile(t) + r2, id2 := testIdentityFile(t) + src := filepath.Join(t.TempDir(), "m.bin") + if err := os.WriteFile(src, []byte("two keys"), 0600); err != nil { + t.Fatal(err) + } + backend := memtest.NewMemoryBackend() + backupID, err := RunUpload(src, []string{r1, r2}, backend) + if err != nil { + t.Fatal(err) + } + for _, id := range []string{id1, id2} { + out := t.TempDir() + if _, err := runRestore(backupID, out, id, backend); err != nil { + t.Fatalf("restore with %s: %v", id, err) + } + } +} + +func TestRunRestoreFileModeMissingLocator(t *testing.T) { + recipient, identityPath := testIdentityFile(t) + src := filepath.Join(t.TempDir(), "a.bin") + if err := os.WriteFile(src, []byte("data"), 0600); err != nil { + t.Fatal(err) + } + backend := memtest.NewMemoryBackend() + backupID, err := RunUpload(src, []string{recipient}, backend) + if err != nil { + t.Fatal(err) + } + backend.Delete(backupID + "/locator.json") + + // Without locator, isFileMode is false, so restore falls through to + // repository Open. Either "incomplete" or a repository error is fine; + // we only assert it fails. + if _, err := runRestore(backupID, t.TempDir(), identityPath, backend); err == nil { + t.Fatal("expected error for missing locator") + } +} + +func TestRunRestoreFileModeRejectsTraversalManifest(t *testing.T) { + recipient, identityPath := testIdentityFile(t) + src := filepath.Join(t.TempDir(), "a.bin") + if err := os.WriteFile(src, []byte("data"), 0600); err != nil { + t.Fatal(err) + } + backend := memtest.NewMemoryBackend() + backupID, err := RunUpload(src, []string{recipient}, backend) + if err != nil { + t.Fatal(err) + } + m, err := readFileManifest(backend, identityPath, backupID) + if err != nil { + t.Fatal(err) + } + m.Chunks[0].Name = "../evil" + resealFileManifest(t, backend, identityPath, backupID, m) + + _, err = runRestore(backupID, t.TempDir(), identityPath, backend) + if err == nil || !bytes.Contains([]byte(err.Error()), []byte("invalid chunk")) { + t.Fatalf("err = %v, want invalid chunk", err) + } +} + +func TestRunRestoreFileModeCorruptChunk(t *testing.T) { + recipient, identityPath := testIdentityFile(t) + src := filepath.Join(t.TempDir(), "a.bin") + if err := os.WriteFile(src, []byte("abcdefgh"), 0600); err != nil { + t.Fatal(err) + } + backend := memtest.NewMemoryBackend() + backupID, err := RunUpload(src, []string{recipient}, backend) + if err != nil { + t.Fatal(err) + } + raw := backend.Get(backupID + "/chunk_00000") + raw[len(raw)/2] ^= 0xff + backend.Put(backupID+"/chunk_00000", raw) + + _, err = runRestore(backupID, t.TempDir(), identityPath, backend) + if err == nil || !bytes.Contains([]byte(err.Error()), []byte("CHUNK CORRUPT")) { + t.Fatalf("err = %v, want CHUNK CORRUPT", err) + } +} diff --git a/internal/flow/restore.go b/internal/flow/restore.go index e759009..08eeccb 100644 --- a/internal/flow/restore.go +++ b/internal/flow/restore.go @@ -26,6 +26,9 @@ func RunRestore(snapshotID, identityPath, remote, outputDir string) error { } func runRestore(snapshotID, outputDir, identityPath string, backend storage.Backend) (string, error) { + if isFileMode(backend, snapshotID) { + return runFileRestore(snapshotID, identityPath, outputDir, backend) + } r, err := repo.Open(backend, identityPath) if err != nil { return "", err From 824a9a77f7374e85e87e78099aaa364caf9aa909 Mon Sep 17 00:00:00 2001 From: Syrup Date: Mon, 12 Oct 2026 00:42:45 +0700 Subject: [PATCH 18/48] feat(flow): verify file-mode backups with auto-detect --- internal/flow/filemode.go | 41 +++++++++++++++++++++++++ internal/flow/filemode_test.go | 56 ++++++++++++++++++++++++++++++++++ internal/flow/verify.go | 3 ++ 3 files changed, 100 insertions(+) diff --git a/internal/flow/filemode.go b/internal/flow/filemode.go index 3c364b5..8349226 100644 --- a/internal/flow/filemode.go +++ b/internal/flow/filemode.go @@ -115,3 +115,44 @@ func runFileRestore(backupID, identityPath, outputDir string, backend storage.Ba } return outputPath, nil } + +func runFileVerify(backupID, identityPath string, backend storage.Backend) error { + m, err := readFileManifest(backend, identityPath, backupID) + if err != nil { + return err + } + masterKey, err := decodeFileMasterKey(m) + if err != nil { + return err + } + + payloadHasher := sha256.New() + for _, meta := range m.Chunks { + if err := validateFileChunkName(meta.Name); err != nil { + return fmt.Errorf("invalid chunk name in manifest: %w", err) + } + var buf bytes.Buffer + if err := backend.Download(backupID+"/"+meta.Name, &buf); err != nil { + return fmt.Errorf("CHUNK CORRUPT %d (%s): missing object: %w", meta.Index, meta.Name, err) + } + sealed := buf.Bytes() + if int64(len(sealed)) != meta.EncryptedSize { + return fmt.Errorf("CHUNK CORRUPT %d (%s): encrypted size mismatch", meta.Index, meta.Name) + } + if hashBytes(sealed) != meta.EncryptedSHA256 { + return fmt.Errorf("CHUNK CORRUPT %d (%s): encrypted sha256 mismatch", meta.Index, meta.Name) + } + plain, err := crypto.OpenChunk(masterKey, meta.PlainSHA256, sealed) + if err != nil { + return fmt.Errorf("CHUNK CORRUPT %d (%s): %w", meta.Index, meta.Name, err) + } + if int64(len(plain)) != meta.PlainSize || hashBytes(plain) != meta.PlainSHA256 { + return fmt.Errorf("CHUNK CORRUPT %d (%s): plaintext mismatch", meta.Index, meta.Name) + } + payloadHasher.Write(plain) + } + if hex.EncodeToString(payloadHasher.Sum(nil)) != m.PayloadSHA256 { + return fmt.Errorf("FATAL: overall payload hash differs from manifest") + } + return nil +} diff --git a/internal/flow/filemode_test.go b/internal/flow/filemode_test.go index 9d53142..2b9a561 100644 --- a/internal/flow/filemode_test.go +++ b/internal/flow/filemode_test.go @@ -176,3 +176,59 @@ func TestRunRestoreFileModeCorruptChunk(t *testing.T) { t.Fatalf("err = %v, want CHUNK CORRUPT", err) } } + +func TestRunVerifyFileModeOK(t *testing.T) { + recipient, identityPath := testIdentityFile(t) + src := filepath.Join(t.TempDir(), "v.bin") + if err := os.WriteFile(src, []byte("verify me"), 0600); err != nil { + t.Fatal(err) + } + backend := memtest.NewMemoryBackend() + backupID, err := RunUpload(src, []string{recipient}, backend) + if err != nil { + t.Fatal(err) + } + if err := runVerify(backupID, identityPath, backend); err != nil { + t.Fatalf("runVerify() error: %v", err) + } +} + +func TestRunVerifyFileModeCorrupt(t *testing.T) { + recipient, identityPath := testIdentityFile(t) + src := filepath.Join(t.TempDir(), "v.bin") + if err := os.WriteFile(src, []byte("verify me"), 0600); err != nil { + t.Fatal(err) + } + backend := memtest.NewMemoryBackend() + backupID, err := RunUpload(src, []string{recipient}, backend) + if err != nil { + t.Fatal(err) + } + raw := backend.Get(backupID + "/chunk_00000") + raw[0] ^= 0xff + backend.Put(backupID+"/chunk_00000", raw) + + err = runVerify(backupID, identityPath, backend) + if err == nil || !bytes.Contains([]byte(err.Error()), []byte("CHUNK CORRUPT")) { + t.Fatalf("err = %v, want CHUNK CORRUPT", err) + } +} + +func TestRunVerifyFileModeMissingLocator(t *testing.T) { + recipient, identityPath := testIdentityFile(t) + src := filepath.Join(t.TempDir(), "v.bin") + if err := os.WriteFile(src, []byte("x"), 0600); err != nil { + t.Fatal(err) + } + backend := memtest.NewMemoryBackend() + backupID, err := RunUpload(src, []string{recipient}, backend) + if err != nil { + t.Fatal(err) + } + backend.Delete(backupID + "/locator.json") + + // Without locator, falls through to repository verify; must fail. + if err := runVerify(backupID, identityPath, backend); err == nil { + t.Fatal("expected error for missing locator") + } +} diff --git a/internal/flow/verify.go b/internal/flow/verify.go index 9176356..62dc3f7 100644 --- a/internal/flow/verify.go +++ b/internal/flow/verify.go @@ -27,6 +27,9 @@ func RunVerify(snapshotID, identityPath, remote string) error { } func runVerify(snapshotID, identityPath string, backend storage.Backend) error { + if isFileMode(backend, snapshotID) { + return runFileVerify(snapshotID, identityPath, backend) + } r, err := repo.Open(backend, identityPath) if err != nil { return err From ccee0f09634fd32b6d3553d0467aae0334dedf83 Mon Sep 17 00:00:00 2001 From: Syrup Date: Mon, 12 Oct 2026 00:43:32 +0700 Subject: [PATCH 19/48] docs: note file-mode upload in changelog --- CHANGELOG.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index c11d081..ca7117e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,8 @@ - `gloak init` creates a repository: a wrapped master key per recipient under `keys/`, a `repo.json` manifest, format 2.0. - `gloak backup` uploads content-addressed encrypted chunks under `data/{hh}/`, records them in an encrypted epoch index under `index/`, and writes a snapshot under `snapshots/`. Repeat backups skip chunks already in the index. Chunks are sealed with zstd and then XChaCha20-Poly1305 under HKDF-derived keys. - Snapshots are encrypted to every recipient listed at init time. Any matching identity can verify or restore. +- `gloak upload -R ` writes a self-contained backup folder `{uuid}/` at the remote root (sealed chunks, multi-recipient `manifest.age` with embedded master key, public `locator.json`). No repository and no `init` required. Manifest version `file-2`. +- `restore` and `verify` auto-detect file-mode folders by `locator.json` and fall back to repository layout otherwise. ### Changed @@ -14,7 +16,7 @@ ### Removed -- The `upload`, `resume`, and `cleanup` commands and the old filesystem-v1 single-file layout (`{uuid}/chunk_00000`, `manifest.age`, `locator.json`). There is no migration path. Backups made with earlier versions cannot be read by this version. +- The `resume` and `cleanup` commands and the old filesystem-v1 layout (`{uuid}/chunk_00000` with age-writer chunks, version `1.0`). There is no migration path. Backups made with earlier versions cannot be read by this version. The new `upload` command is a separate format (`file-2`) using the repository chunk envelope. ### Fixed From 7df37a96867c0e5dc711364324bdf739b97f3e1b Mon Sep 17 00:00:00 2001 From: Syrup Date: Mon, 12 Oct 2026 02:00:47 +0700 Subject: [PATCH 20/48] feat(chunker): add gear-hash content-defined chunking --- internal/chunker/chunker.go | 82 +++++++++++++++++++ internal/chunker/chunker_test.go | 135 +++++++++++++++++++++++++++++++ 2 files changed, 217 insertions(+) create mode 100644 internal/chunker/chunker.go create mode 100644 internal/chunker/chunker_test.go diff --git a/internal/chunker/chunker.go b/internal/chunker/chunker.go new file mode 100644 index 0000000..39a651f --- /dev/null +++ b/internal/chunker/chunker.go @@ -0,0 +1,82 @@ +// Package chunker implements content-defined chunking with a gear hash. +package chunker + +import ( + "bufio" + "fmt" + "io" +) + +const windowSize = 64 + +// gear is filled once at init from a fixed seed so every binary builds the +// same table and CDC boundaries stay stable across runs and machines. +var gear [256]uint64 + +func init() { + x := uint64(0x9E3779B97F4A7C15) + for i := range gear { + x += 0x9E3779B97F4A7C15 + z := x + z = (z ^ (z >> 30)) * 0xBF58476D1CE4E5B9 + z = (z ^ (z >> 27)) * 0x94D049BB133111EB + gear[i] = z ^ (z >> 31) + } +} + +type CDC struct { + min int64 + avg int64 + max int64 +} + +// NewCDC validates the bounds and rounds max up to a multiple of windowSize. +func NewCDC(min, avg, max int64) (*CDC, error) { + if min <= 0 || avg <= 0 || max <= 0 { + return nil, fmt.Errorf("chunk size must be positive (min=%d avg=%d max=%d)", min, avg, max) + } + if max < avg || avg < min { + return nil, fmt.Errorf("chunk bounds require max >= avg >= min (min=%d avg=%d max=%d)", min, avg, max) + } + if max%windowSize != 0 { + max = ((max + windowSize - 1) / windowSize) * windowSize + } + return &CDC{min: min, avg: avg, max: max}, nil +} + +// Split reads r and calls fn once per chunk. Each chunk is a fresh slice the +// caller may retain. An empty reader produces no calls. +func (c *CDC) Split(r io.Reader, fn func(chunk []byte) error) error { + br := bufio.NewReaderSize(r, 64*1024) + chunk := make([]byte, 0, c.max) + var h uint64 + for { + b, err := br.ReadByte() + if err == io.EOF { + if len(chunk) > 0 { + return fn(chunk) + } + return nil + } + if err != nil { + return err + } + chunk = append(chunk, b) + h = (h << 1) + gear[b] + if int64(len(chunk)) >= c.max { + if err := fn(chunk); err != nil { + return err + } + chunk = make([]byte, 0, c.max) + h = 0 + continue + } + if int64(len(chunk)) >= c.min && h%uint64(c.avg) == 0 { + if err := fn(chunk); err != nil { + return err + } + chunk = make([]byte, 0, c.max) + h = 0 + } + } +} diff --git a/internal/chunker/chunker_test.go b/internal/chunker/chunker_test.go new file mode 100644 index 0000000..cd4fb60 --- /dev/null +++ b/internal/chunker/chunker_test.go @@ -0,0 +1,135 @@ +package chunker + +import ( + "bytes" + "io" + "math/rand" + "testing" +) + +func mustCDC(t *testing.T, min, avg, max int64) *CDC { + t.Helper() + c, err := NewCDC(min, avg, max) + if err != nil { + t.Fatalf("NewCDC(%d,%d,%d) error: %v", min, avg, max, err) + } + return c +} + +func splitAll(t *testing.T, c *CDC, data []byte) [][]byte { + t.Helper() + var chunks [][]byte + err := c.Split(bytes.NewReader(data), func(chunk []byte) error { + cp := make([]byte, len(chunk)) + copy(cp, chunk) + chunks = append(chunks, cp) + return nil + }) + if err != nil { + t.Fatalf("Split() error: %v", err) + } + return chunks +} + +func TestNewCDCRejectsBadBounds(t *testing.T) { + cases := []struct{ min, avg, max int64 }{ + {0, 1, 2}, + {2, 1, 4}, + {2, 4, 3}, + {2, 2, 1}, + } + for _, c := range cases { + if _, err := NewCDC(c.min, c.avg, c.max); err == nil { + t.Fatalf("NewCDC(%d,%d,%d) succeeded", c.min, c.avg, c.max) + } + } +} + +func TestSplitEmptyReader(t *testing.T) { + c := mustCDC(t, 4, 8, 16) + chunks := splitAll(t, c, nil) + if len(chunks) != 0 { + t.Fatalf("chunks = %d, want 0", len(chunks)) + } +} + +func TestSplitSmallerThanMinIsOneChunk(t *testing.T) { + c := mustCDC(t, 64, 128, 256) + data := []byte("short") + chunks := splitAll(t, c, data) + if len(chunks) != 1 || !bytes.Equal(chunks[0], data) { + t.Fatalf("chunks = %d, want 1 equal to input", len(chunks)) + } +} + +func TestSplitRespectsMax(t *testing.T) { + c := mustCDC(t, 4, 64, 64) + data := bytes.Repeat([]byte{0xFF}, 200) + chunks := splitAll(t, c, data) + for i, ch := range chunks { + if int64(len(ch)) > 64 { + t.Fatalf("chunk %d length %d > max 64", i, len(ch)) + } + } + var total int + for _, ch := range chunks { + total += len(ch) + } + if total != len(data) { + t.Fatalf("total %d != %d", total, len(data)) + } +} + +func TestSplitDeterminism(t *testing.T) { + data := make([]byte, 50_000) + rng := rand.New(rand.NewSource(42)) + rng.Read(data) + a := splitAll(t, mustCDC(t, 512, 2048, 8192), data) + b := splitAll(t, mustCDC(t, 512, 2048, 8192), data) + if len(a) != len(b) { + t.Fatalf("chunk counts %d vs %d", len(a), len(b)) + } + for i := range a { + if !bytes.Equal(a[i], b[i]) { + t.Fatalf("chunk %d differs", i) + } + } + if len(a) < 2 { + t.Fatalf("expected multiple chunks, got %d", len(a)) + } +} + +func TestSplitRoundTripBytes(t *testing.T) { + data := make([]byte, 100_000) + rng := rand.New(rand.NewSource(7)) + rng.Read(data) + chunks := splitAll(t, mustCDC(t, 1024, 4096, 16384), data) + var out []byte + for _, ch := range chunks { + out = append(out, ch...) + } + if !bytes.Equal(out, data) { + t.Fatal("concatenated chunks != input") + } +} + +func TestSplitCallbackErrorPropagates(t *testing.T) { + c := mustCDC(t, 4, 8, 16) + sentinel := io.ErrClosedPipe + err := c.Split(bytes.NewReader([]byte("abcdefghij")), func([]byte) error { + return sentinel + }) + if err != sentinel { + t.Fatalf("err = %v, want sentinel", err) + } +} + +func TestMaxRoundedUpToWindowMultiple(t *testing.T) { + c := mustCDC(t, 4, 8, 100) + if c.max%64 != 0 { + t.Fatalf("max = %d, want multiple of 64", c.max) + } + if c.max < 100 { + t.Fatalf("max = %d, want >= 100", c.max) + } +} From 70830e40a09ee685d1ca2a0d91c8bdf764465f90 Mon Sep 17 00:00:00 2001 From: Syrup Date: Mon, 12 Oct 2026 02:06:39 +0700 Subject: [PATCH 21/48] feat(repo,flow): CDC chunk bounds in repo.json and gear-hash backup --- init.go | 5 +- internal/flow/backup.go | 41 +++++-------- internal/flow/backup_test.go | 111 +++++++++++++++++++++++++---------- internal/flow/flow_test.go | 6 +- internal/flow/init.go | 26 ++++---- internal/flow/init_test.go | 18 +++--- internal/repo/config.go | 10 +++- internal/repo/repo.go | 14 +++-- internal/repo/repo_test.go | 34 +++++++++-- 9 files changed, 175 insertions(+), 90 deletions(-) diff --git a/init.go b/init.go index 7b50bec..34f87d7 100644 --- a/init.go +++ b/init.go @@ -14,6 +14,9 @@ type InitCmd struct { Remote string `short:"R" required:"" help:"Repository remote, e.g. b2:my-bucket/gloak or local path"` Recipient string `short:"r" help:"Recipient public key file or comma-separated keys (default ~/.gloak/recipient.txt)"` Force bool `short:"f" help:"Overwrite an existing repository (makes old snapshots unreadable)"` + ChunkMin int64 `help:"Minimum CDC chunk size in bytes (default 524288)."` + ChunkAvg int64 `help:"Average CDC chunk size in bytes (default 1048576)."` + ChunkMax int64 `help:"Maximum CDC chunk size in bytes (default 8388608)."` } func (c *InitCmd) Run() error { @@ -21,7 +24,7 @@ func (c *InitCmd) Run() error { if err != nil { return err } - return flow.RunInit(recipients, c.Remote, c.Force) + return flow.RunInit(recipients, c.Remote, c.ChunkMin, c.ChunkAvg, c.ChunkMax, c.Force) } // resolveInitRecipients turns the --recipient value into a key list. An empty diff --git a/internal/flow/backup.go b/internal/flow/backup.go index 4efe812..ea12c75 100644 --- a/internal/flow/backup.go +++ b/internal/flow/backup.go @@ -6,7 +6,6 @@ import ( "encoding/hex" "encoding/json" "fmt" - "io" "os" "path/filepath" "time" @@ -14,6 +13,7 @@ import ( "github.com/google/uuid" "github.com/pterm/pterm" + "gloak/internal/chunker" "gloak/internal/core" "gloak/internal/crypto" "gloak/internal/index" @@ -68,41 +68,28 @@ func runBackup(filePath, identityPath string, backend storage.Backend, snapshotI return BackupResult{}, fmt.Errorf("failed to load index: %w", err) } + cdc, err := chunker.NewCDC(r.Config.ChunkMin, r.Config.ChunkAvg, r.Config.ChunkMax) + if err != nil { + return BackupResult{}, fmt.Errorf("invalid chunk config: %w", err) + } + file, err := os.Open(filePath) if err != nil { return BackupResult{}, fmt.Errorf("failed to open file: %w", err) } defer file.Close() - chunkSize := r.Config.ChunkSize - if chunkSize <= 0 { - return BackupResult{}, fmt.Errorf("invalid chunk size: %d", chunkSize) - } - res := BackupResult{SnapshotID: snapshotID} var metas []core.ChunkMeta var sinceCheckpoint int payloadHasher := sha256.New() - buffer := make([]byte, int(chunkSize)) - - for chunkIdx := 0; ; chunkIdx++ { - n, readErr := io.ReadFull(file, buffer) - if readErr == io.EOF { - break - } - if readErr != nil && readErr != io.ErrUnexpectedEOF { - return BackupResult{}, fmt.Errorf("failed to read chunk %d: %w", chunkIdx, readErr) - } - if n == 0 { - break - } - chunk := buffer[:n] + err = cdc.Split(file, func(chunk []byte) error { payloadHasher.Write(chunk) hash := hashBytes(chunk) res.TotalChunks++ - res.TotalPlainBytes += int64(n) + res.TotalPlainBytes += int64(len(chunk)) dataPath := repo.DataPath(hash) @@ -111,15 +98,15 @@ func runBackup(filePath, identityPath string, backend storage.Backend, snapshotI } else { sealed, err := crypto.SealChunk(r.MasterKey, hash, chunk) if err != nil { - return BackupResult{}, fmt.Errorf("failed to seal chunk %s: %w", hash, err) + return fmt.Errorf("failed to seal chunk %s: %w", hash, err) } encSum := sha256.Sum256(sealed) if err := backend.Upload(dataPath, bytes.NewReader(sealed)); err != nil { - return BackupResult{}, fmt.Errorf("failed to upload chunk %s: %w", hash, err) + return fmt.Errorf("failed to upload chunk %s: %w", hash, err) } ix.Add(index.Entry{ PlainSHA256: hash, - PlainSize: int64(n), + PlainSize: int64(len(chunk)), EncryptedSize: int64(len(sealed)), EncryptedSHA256: hex.EncodeToString(encSum[:]), }) @@ -127,7 +114,7 @@ func runBackup(filePath, identityPath string, backend storage.Backend, snapshotI sinceCheckpoint++ if sinceCheckpoint >= indexCheckpointEvery { if _, err := ix.Save(backend, r.Config.Recipients); err != nil { - return BackupResult{}, fmt.Errorf("failed to checkpoint index: %w", err) + return fmt.Errorf("failed to checkpoint index: %w", err) } sinceCheckpoint = 0 } @@ -146,6 +133,10 @@ func runBackup(filePath, identityPath string, backend storage.Backend, snapshotI if progress != nil { progress(res.TotalChunks) } + return nil + }) + if err != nil { + return BackupResult{}, err } if _, err := ix.Save(backend, r.Config.Recipients); err != nil { diff --git a/internal/flow/backup_test.go b/internal/flow/backup_test.go index f866e65..3326eca 100644 --- a/internal/flow/backup_test.go +++ b/internal/flow/backup_test.go @@ -1,8 +1,6 @@ package flow import ( - "crypto/sha256" - "encoding/hex" "os" "path/filepath" "strings" @@ -13,7 +11,7 @@ import ( "gloak/internal/storage/memtest" ) -func repoFixture(t *testing.T, chunkSize int64) (string, *memtest.MemoryBackend, string) { +func repoFixture(t *testing.T) (string, *memtest.MemoryBackend, string) { t.Helper() pub, _, fileContent, err := crypto.GenerateKey() if err != nil { @@ -24,7 +22,9 @@ func repoFixture(t *testing.T, chunkSize int64) (string, *memtest.MemoryBackend, t.Fatalf("write identity: %v", err) } backend := memtest.NewMemoryBackend() - if _, _, err := repo.Init(backend, []string{pub}, chunkSize, false); err != nil { + // min=1 avg=8 max=64: tiny test payloads stay one chunk unless tests + // craft multi-chunk files themselves. + if _, _, err := repo.Init(backend, []string{pub}, 1, 8, 64, false); err != nil { t.Fatalf("repo.Init() error: %v", err) } return identPath, backend, pub @@ -40,22 +40,25 @@ func writeTempFile(t *testing.T, data []byte) string { } func TestBackupWritesSnapshotAndDataObjects(t *testing.T) { - ident, backend, _ := repoFixture(t, 8) - payload := []byte("0123456789abcdefghij") // 3 chunks with size 8 + ident, backend, _ := repoFixture(t) + payload := []byte("0123456789abcdefghij") filePath := writeTempFile(t, payload) res, err := runBackup(filePath, ident, backend, "7f91c6c7-7a0b-44aa-ae23-997b60e4e998", nil) if err != nil { t.Fatalf("runBackup() error: %v", err) } - if res.TotalChunks != 3 || res.UploadedChunks != 3 || res.SkippedChunks != 0 { + if res.TotalChunks < 1 || res.UploadedChunks != res.TotalChunks || res.SkippedChunks != 0 { t.Fatalf("result = %+v", res) } + if res.TotalPlainBytes != int64(len(payload)) { + t.Fatalf("TotalPlainBytes = %d", res.TotalPlainBytes) + } if !backend.Has(repo.SnapshotsPrefix + "/" + res.SnapshotID + ".age") { t.Fatal("snapshot file missing") } - if got := backend.TotalUploads(repo.DataPrefix + "/"); got != 3 { - t.Fatalf("data uploads = %d, want 3", got) + if got := backend.TotalUploads(repo.DataPrefix + "/"); got != res.UploadedChunks { + t.Fatalf("data uploads = %d, want %d", got, res.UploadedChunks) } if !backend.Has("index/000001.age") { t.Fatal("index epoch missing") @@ -63,7 +66,7 @@ func TestBackupWritesSnapshotAndDataObjects(t *testing.T) { } func TestBackupSkipsChunksAlreadyInIndex(t *testing.T) { - ident, backend, _ := repoFixture(t, 8) + ident, backend, _ := repoFixture(t) payload := []byte("0123456789abcdefghij") filePath := writeTempFile(t, payload) @@ -78,7 +81,7 @@ func TestBackupSkipsChunksAlreadyInIndex(t *testing.T) { if err != nil { t.Fatalf("second runBackup() error: %v", err) } - if res.UploadedChunks != 0 || res.SkippedChunks != 3 { + if res.UploadedChunks != 0 || res.SkippedChunks != res.TotalChunks { t.Fatalf("second result = %+v, want all skipped", res) } if backend.TotalUploads(repo.DataPrefix+"/") != dataUploadsBefore { @@ -89,28 +92,34 @@ func TestBackupSkipsChunksAlreadyInIndex(t *testing.T) { } } -func TestBackupDedupsRepeatedChunksInSameFile(t *testing.T) { - ident, backend, _ := repoFixture(t, 8) - payload := []byte("012345670123456701234567") // same 8-byte chunk three times - filePath := writeTempFile(t, payload) +func TestBackupDedupsIdenticalContentAcrossFiles(t *testing.T) { + ident, backend, _ := repoFixture(t) + payload := []byte("0123456789abcdefghij") + fileA := writeTempFile(t, payload) + fileB := filepath.Join(t.TempDir(), "copy.bin") + if err := os.WriteFile(fileB, payload, 0600); err != nil { + t.Fatal(err) + } - res, err := runBackup(filePath, ident, backend, "11111111-2222-4333-8444-555555555555", nil) - if err != nil { - t.Fatalf("runBackup() error: %v", err) + if _, err := runBackup(fileA, ident, backend, "11111111-2222-4333-8444-555555555555", nil); err != nil { + t.Fatalf("first runBackup() error: %v", err) } - if res.TotalChunks != 3 { - t.Fatalf("TotalChunks = %d", res.TotalChunks) + before := backend.TotalUploads(repo.DataPrefix + "/") + + res, err := runBackup(fileB, ident, backend, "11111111-2222-4333-8444-666666666666", nil) + if err != nil { + t.Fatalf("second runBackup() error: %v", err) } - if res.UploadedChunks != 1 { - t.Fatalf("UploadedChunks = %d, want 1 (content-addressed dedup)", res.UploadedChunks) + if res.UploadedChunks != 0 || res.SkippedChunks != res.TotalChunks { + t.Fatalf("second result = %+v, want all skipped", res) } - if got := backend.TotalUploads(repo.DataPrefix + "/"); got != 1 { - t.Fatalf("data uploads = %d, want 1", got) + if backend.TotalUploads(repo.DataPrefix+"/") != before { + t.Fatal("second backup re-uploaded data objects") } } func TestBackupFailsWithoutRepository(t *testing.T) { - ident, _, _ := repoFixture(t, 8) + ident, _, _ := repoFixture(t) filePath := writeTempFile(t, []byte("x")) _, err := runBackup(filePath, ident, memtest.NewMemoryBackend(), "7f91c6c7-7a0b-44aa-ae23-997b60e4e998", nil) if err == nil { @@ -122,7 +131,7 @@ func TestBackupFailsWithoutRepository(t *testing.T) { } func TestBackupRejectsInvalidSnapshotID(t *testing.T) { - ident, backend, _ := repoFixture(t, 8) + ident, backend, _ := repoFixture(t) filePath := writeTempFile(t, []byte("x")) _, err := runBackup(filePath, ident, backend, "../escape", nil) if err == nil { @@ -134,15 +143,53 @@ func TestBackupRejectsInvalidSnapshotID(t *testing.T) { } func TestBackupObjectMatchesChunkHash(t *testing.T) { - ident, backend, _ := repoFixture(t, 8) + ident, backend, _ := repoFixture(t) payload := []byte("0123456789abcdefghij") filePath := writeTempFile(t, payload) - if _, err := runBackup(filePath, ident, backend, "99999999-8888-4777-8666-555555555555", nil); err != nil { + snapshotID := "99999999-8888-4777-8666-555555555555" + if _, err := runBackup(filePath, ident, backend, snapshotID, nil); err != nil { t.Fatalf("runBackup() error: %v", err) } - sum := sha256.Sum256(payload[:8]) - wantPath := repo.DataPath(hex.EncodeToString(sum[:])) - if !backend.Has(wantPath) { - t.Fatalf("expected object at %s", wantPath) + m, err := readManifest(backend, ident, snapshotID) + if err != nil { + t.Fatal(err) + } + if len(m.Chunks) == 0 { + t.Fatal("no chunks in manifest") + } + if !backend.Has(m.Chunks[0].Name) { + t.Fatalf("expected object at %s", m.Chunks[0].Name) + } +} + +func TestBackupCDCQualityAfterInsert(t *testing.T) { + ident, backend, _ := repoFixture(t) + base := make([]byte, 200_000) + for i := range base { + base[i] = byte(i * 31) + } + fileA := writeTempFile(t, base) + snapA := "aaaaaaaa-bbbb-4ccc-8ddd-000000000001" + resA, err := runBackup(fileA, ident, backend, snapA, nil) + if err != nil { + t.Fatal(err) + } + if resA.TotalChunks < 3 { + t.Fatalf("TotalChunks = %d, want >= 3", resA.TotalChunks) + } + + // Insert one byte near the start; most later chunk hashes should survive. + shifted := make([]byte, 0, len(base)+1) + shifted = append(shifted, base[:100]...) + shifted = append(shifted, 0xAB) + shifted = append(shifted, base[100:]...) + fileB := writeTempFile(t, shifted) + snapB := "aaaaaaaa-bbbb-4ccc-8ddd-000000000002" + resB, err := runBackup(fileB, ident, backend, snapB, nil) + if err != nil { + t.Fatal(err) + } + if resB.SkippedChunks == 0 { + t.Fatal("expected some chunks reused after single-byte insert") } } diff --git a/internal/flow/flow_test.go b/internal/flow/flow_test.go index 187d4a6..940234d 100644 --- a/internal/flow/flow_test.go +++ b/internal/flow/flow_test.go @@ -13,7 +13,7 @@ import ( ) func TestBackupVerifyRestoreRoundTrip(t *testing.T) { - ident, backend, _ := repoFixture(t, 8) + ident, backend, _ := repoFixture(t) payload := []byte("0123456789abcdefghij") filePath := writeTempFile(t, payload) snapshotID := "7f91c6c7-7a0b-44aa-ae23-997b60e4e998" @@ -40,7 +40,7 @@ func TestBackupVerifyRestoreRoundTrip(t *testing.T) { } func TestVerifyRejectsCorruptChunk(t *testing.T) { - ident, backend, _ := repoFixture(t, 8) + ident, backend, _ := repoFixture(t) payload := []byte("0123456789abcdefghij") filePath := writeTempFile(t, payload) snapshotID := "7f91c6c7-7a0b-44aa-ae23-997b60e4e998" @@ -71,7 +71,7 @@ func TestVerifyRejectsCorruptChunk(t *testing.T) { } func TestRestoreRejectsEscapingChunkPath(t *testing.T) { - ident, backend, pub := repoFixture(t, 8) + ident, backend, pub := repoFixture(t) snapshotID := "7f91c6c7-7a0b-44aa-ae23-997b60e4e998" manifest := core.Manifest{ diff --git a/internal/flow/init.go b/internal/flow/init.go index 26e005c..45f5563 100644 --- a/internal/flow/init.go +++ b/internal/flow/init.go @@ -3,21 +3,27 @@ package flow import ( "github.com/pterm/pterm" - "gloak/internal/core" "gloak/internal/repo" "gloak/internal/storage" ) -const defaultChunkSize = core.ChunkSize - // RunInit creates a new gloak repository on remote and prints the outcome. -// With force it overwrites an existing repository, which makes old snapshots -// unreadable. -func RunInit(recipients []string, remote string, force bool) error { +// Zero min/avg/max select the package defaults. With force it overwrites an +// existing repository, which makes old snapshots unreadable. +func RunInit(recipients []string, remote string, min, avg, max int64, force bool) error { + if min == 0 { + min = repo.DefaultChunkMin + } + if avg == 0 { + avg = repo.DefaultChunkAvg + } + if max == 0 { + max = repo.DefaultChunkMax + } backend := storage.NewRcloneBackend(remote) spinner, _ := pterm.DefaultSpinner.Start("Initializing repository...") - master, cfg, err := runInit(recipients, backend, defaultChunkSize, force) + master, cfg, err := runInit(recipients, backend, min, avg, max, force) if err != nil { _ = spinner.Stop() return err @@ -26,11 +32,11 @@ func RunInit(recipients []string, remote string, force bool) error { pterm.Success.Println("Repository initialized") pterm.Printf("Location: %s\n", remote) pterm.Printf("Recipients: %v\n", cfg.Recipients) - pterm.Printf("Chunk size: %d bytes\n", cfg.ChunkSize) + pterm.Printf("Chunk bounds: min=%d avg=%d max=%d bytes\n", cfg.ChunkMin, cfg.ChunkAvg, cfg.ChunkMax) pterm.Printf("Master key length: %d bytes (wrapped in %s)\n", len(master), repo.KeysPrefix+"/") return nil } -func runInit(recipients []string, backend storage.Backend, chunkSize int64, force bool) ([]byte, repo.Config, error) { - return repo.Init(backend, recipients, chunkSize, force) +func runInit(recipients []string, backend storage.Backend, min, avg, max int64, force bool) ([]byte, repo.Config, error) { + return repo.Init(backend, recipients, min, avg, max, force) } diff --git a/internal/flow/init_test.go b/internal/flow/init_test.go index 246e2b8..675e86d 100644 --- a/internal/flow/init_test.go +++ b/internal/flow/init_test.go @@ -18,7 +18,7 @@ func TestRunInitCreatesRepository(t *testing.T) { _ = fileContent backend := memtest.NewMemoryBackend() - master, cfg, err := runInit([]string{pub}, backend, 4096, false) + master, cfg, err := runInit([]string{pub}, backend, 1, 8, 64, false) if err != nil { t.Fatalf("runInit() error: %v", err) } @@ -31,13 +31,13 @@ func TestRunInitCreatesRepository(t *testing.T) { if !backend.Has(repo.KeysPathFor(pub)) { t.Fatal("wrapped key missing") } - if cfg.ChunkSize != 4096 { - t.Fatalf("cfg.ChunkSize = %d", cfg.ChunkSize) + if cfg.ChunkMin != 1 || cfg.ChunkAvg != 8 || cfg.ChunkMax != 64 { + t.Fatalf("cfg chunk params = %d/%d/%d", cfg.ChunkMin, cfg.ChunkAvg, cfg.ChunkMax) } } func TestRunInitRejectsNoRecipients(t *testing.T) { - if _, _, err := runInit(nil, memtest.NewMemoryBackend(), 4096, false); err == nil { + if _, _, err := runInit(nil, memtest.NewMemoryBackend(), 1, 8, 64, false); err == nil { t.Fatal("runInit() succeeded with no recipients") } else if !strings.Contains(err.Error(), "recipient") { t.Fatalf("runInit() error = %v, want recipient mention", err) @@ -45,7 +45,7 @@ func TestRunInitRejectsNoRecipients(t *testing.T) { } func TestRunInitRejectsInvalidRecipient(t *testing.T) { - if _, _, err := runInit([]string{"age1bad"}, memtest.NewMemoryBackend(), 4096, false); err == nil { + if _, _, err := runInit([]string{"age1bad"}, memtest.NewMemoryBackend(), 1, 8, 64, false); err == nil { t.Fatal("runInit() succeeded with invalid recipient") } } @@ -57,10 +57,10 @@ func TestRunInitRejectsExistingRepository(t *testing.T) { } backend := memtest.NewMemoryBackend() - if _, _, err := runInit([]string{pub}, backend, 4096, false); err != nil { + if _, _, err := runInit([]string{pub}, backend, 1, 8, 64, false); err != nil { t.Fatalf("runInit() error: %v", err) } - if _, _, err := runInit([]string{pub}, backend, 4096, false); err == nil { + if _, _, err := runInit([]string{pub}, backend, 1, 8, 64, false); err == nil { t.Fatal("runInit() succeeded on existing repository without force") } else if !strings.Contains(err.Error(), "already exists") { t.Fatalf("runInit() error = %v, want already exists mention", err) @@ -74,11 +74,11 @@ func TestRunInitForceOverwrites(t *testing.T) { } backend := memtest.NewMemoryBackend() - first, _, err := runInit([]string{pub}, backend, 4096, false) + first, _, err := runInit([]string{pub}, backend, 1, 8, 64, false) if err != nil { t.Fatalf("runInit() error: %v", err) } - second, _, err := runInit([]string{pub}, backend, 4096, true) + second, _, err := runInit([]string{pub}, backend, 1, 8, 64, true) if err != nil { t.Fatalf("runInit(force) error: %v", err) } diff --git a/internal/repo/config.go b/internal/repo/config.go index 29daf17..2e02f4b 100644 --- a/internal/repo/config.go +++ b/internal/repo/config.go @@ -6,13 +6,19 @@ const ( KeysPrefix = "keys" SnapshotsPrefix = "snapshots" DataPrefix = "data" + + DefaultChunkMin = 512 * 1024 + DefaultChunkAvg = 1024 * 1024 + DefaultChunkMax = 8 * 1024 * 1024 ) // Config is the plaintext repository descriptor. It carries no user data: -// version, creation time, recipient public keys, and chunk parameters. +// version, creation time, recipient public keys, and CDC chunk parameters. type Config struct { Version string `json:"version"` CreatedAt string `json:"created_at"` Recipients []string `json:"recipients"` - ChunkSize int64 `json:"chunk_size"` + ChunkMin int64 `json:"chunk_min"` + ChunkAvg int64 `json:"chunk_avg"` + ChunkMax int64 `json:"chunk_max"` } diff --git a/internal/repo/repo.go b/internal/repo/repo.go index 1101bd5..03c4d7d 100644 --- a/internal/repo/repo.go +++ b/internal/repo/repo.go @@ -51,7 +51,7 @@ func SnapshotPath(snapshotID string) (string, error) { // recipient first, repo.json last as the completion marker. It refuses to // touch a repository that already exists unless force is set, because a // fresh master key would orphan every snapshot sealed with the old one. -func Init(backend storage.Backend, recipients []string, chunkSize int64, force bool) ([]byte, Config, error) { +func Init(backend storage.Backend, recipients []string, min, avg, max int64, force bool) ([]byte, Config, error) { if len(recipients) == 0 { return nil, Config{}, fmt.Errorf("at least one recipient is required") } @@ -60,8 +60,8 @@ func Init(backend storage.Backend, recipients []string, chunkSize int64, force b return nil, Config{}, fmt.Errorf("invalid recipient %q: %w", r, err) } } - if chunkSize <= 0 { - return nil, Config{}, fmt.Errorf("chunk size must be positive") + if min <= 0 || avg <= 0 || max <= 0 || max < avg || avg < min { + return nil, Config{}, fmt.Errorf("chunk bounds require 0 < min <= avg <= max (min=%d avg=%d max=%d)", min, avg, max) } if !force { @@ -79,7 +79,9 @@ func Init(backend storage.Backend, recipients []string, chunkSize int64, force b Version: FormatVersion, CreatedAt: time.Now().UTC().Format(time.RFC3339), Recipients: recipients, - ChunkSize: chunkSize, + ChunkMin: min, + ChunkAvg: avg, + ChunkMax: max, } for _, recipient := range recipients { @@ -116,6 +118,10 @@ func Open(backend storage.Backend, identityPath string) (*Repo, error) { if cfg.Version != FormatVersion { return nil, fmt.Errorf("unsupported repository format %q (want %q)", cfg.Version, FormatVersion) } + if cfg.ChunkMin <= 0 || cfg.ChunkAvg <= 0 || cfg.ChunkMax <= 0 || + cfg.ChunkMax < cfg.ChunkAvg || cfg.ChunkAvg < cfg.ChunkMin { + return nil, fmt.Errorf("invalid chunk parameters in %s (min=%d avg=%d max=%d)", ConfigPath, cfg.ChunkMin, cfg.ChunkAvg, cfg.ChunkMax) + } raw, err := backend.ListFiles([]string{KeysPrefix}) if err != nil { diff --git a/internal/repo/repo_test.go b/internal/repo/repo_test.go index 9aeb8d3..3aea5d6 100644 --- a/internal/repo/repo_test.go +++ b/internal/repo/repo_test.go @@ -28,14 +28,14 @@ func TestInitWritesKeysAndConfig(t *testing.T) { pubB, _ := testRecipient(t) backend := memtest.NewMemoryBackend() - master, cfg, err := Init(backend, []string{pubA, pubB}, 4096, false) + master, cfg, err := Init(backend, []string{pubA, pubB}, 1, 8, 64, false) if err != nil { t.Fatalf("Init() error: %v", err) } if len(master) != 32 { t.Fatalf("master key length = %d, want 32", len(master)) } - if cfg.Version != FormatVersion || cfg.ChunkSize != 4096 { + if cfg.Version != FormatVersion || cfg.ChunkMin != 1 || cfg.ChunkAvg != 8 || cfg.ChunkMax != 64 { t.Fatalf("cfg = %+v", cfg) } if !backend.Has(ConfigPath) { @@ -53,7 +53,7 @@ func TestOpenWithEachRecipientIdentity(t *testing.T) { pubA, identA := testRecipient(t) pubB, identB := testRecipient(t) backend := memtest.NewMemoryBackend() - master, _, err := Init(backend, []string{pubA, pubB}, 4096, false) + master, _, err := Init(backend, []string{pubA, pubB}, 1, 8, 64, false) if err != nil { t.Fatalf("Init() error: %v", err) } @@ -73,7 +73,7 @@ func TestOpenRejectsWrongIdentity(t *testing.T) { pubA, _ := testRecipient(t) _, identOther := testRecipient(t) backend := memtest.NewMemoryBackend() - if _, _, err := Init(backend, []string{pubA}, 4096, false); err != nil { + if _, _, err := Init(backend, []string{pubA}, 1, 8, 64, false); err != nil { t.Fatalf("Init() error: %v", err) } _, err := Open(backend, identOther) @@ -96,6 +96,32 @@ func TestOpenRejectsMissingConfig(t *testing.T) { } } +func TestInitRejectsBadChunkBounds(t *testing.T) { + pub, _ := testRecipient(t) + backend := memtest.NewMemoryBackend() + if _, _, err := Init(backend, []string{pub}, 8, 1, 64, false); err == nil { + t.Fatal("Init() succeeded with avg < min") + } +} + +func TestOpenRejectsMissingCDCParams(t *testing.T) { + pub, ident := testRecipient(t) + backend := memtest.NewMemoryBackend() + if _, _, err := Init(backend, []string{pub}, 1, 8, 64, false); err != nil { + t.Fatal(err) + } + // Simulate a Phase 1 repo.json that only had chunk_size. + raw := []byte(`{"version":"2.0","created_at":"now","recipients":["` + pub + `"],"chunk_size":4096}`) + backend.Put(ConfigPath, raw) + _, err := Open(backend, ident) + if err == nil { + t.Fatal("Open() succeeded without CDC params") + } + if !strings.Contains(err.Error(), "chunk") { + t.Fatalf("error = %v, want chunk param mention", err) + } +} + func TestDataPathUsesHashSharding(t *testing.T) { got := DataPath(strings.Repeat("ab", 32)) want := "data/ab/" + strings.Repeat("ab", 32) From 11ae36c586496175a75320d7b77a508d2a0ed881 Mon Sep 17 00:00:00 2001 From: Syrup Date: Mon, 12 Oct 2026 02:07:17 +0700 Subject: [PATCH 22/48] feat(storage): add recursive file listing to backends --- internal/storage/memtest/memory.go | 15 +++++++++++ internal/storage/memtest/memory_test.go | 27 +++++++++++++++++++ internal/storage/rclone.go | 35 +++++++++++++++++++++++++ 3 files changed, 77 insertions(+) create mode 100644 internal/storage/memtest/memory_test.go diff --git a/internal/storage/memtest/memory.go b/internal/storage/memtest/memory.go index 8ff024b..73b02ec 100644 --- a/internal/storage/memtest/memory.go +++ b/internal/storage/memtest/memory.go @@ -86,6 +86,21 @@ func (m *MemoryBackend) ListFiles(prefix []string) ([]byte, error) { return json.Marshal(entries) } +// ListRecursive returns every stored file path under prefix, sorted. +func (m *MemoryBackend) ListRecursive(prefix []string) ([]string, error) { + p := strings.Trim(strings.Join(prefix, "/"), "/") + m.mu.Lock() + defer m.mu.Unlock() + var out []string + for key := range m.files { + if p == "" || strings.HasPrefix(key, p+"/") { + out = append(out, key) + } + } + sort.Strings(out) + return out, nil +} + func (m *MemoryBackend) Delete(remotePath string) error { m.mu.Lock() defer m.mu.Unlock() diff --git a/internal/storage/memtest/memory_test.go b/internal/storage/memtest/memory_test.go new file mode 100644 index 0000000..f4a6b71 --- /dev/null +++ b/internal/storage/memtest/memory_test.go @@ -0,0 +1,27 @@ +package memtest + +import ( + "bytes" + "testing" +) + +func TestListRecursiveReturnsNestedFiles(t *testing.T) { + m := NewMemoryBackend() + _ = m.Upload("data/ab/abhash", bytes.NewReader([]byte("x"))) + _ = m.Upload("data/cd/cdhash", bytes.NewReader([]byte("y"))) + _ = m.Upload("snapshots/s1.age", bytes.NewReader([]byte("z"))) + + got, err := m.ListRecursive([]string{"data"}) + if err != nil { + t.Fatal(err) + } + want := []string{"data/ab/abhash", "data/cd/cdhash"} + if len(got) != len(want) { + t.Fatalf("got %v, want %v", got, want) + } + for i := range want { + if got[i] != want[i] { + t.Fatalf("got %v, want %v", got, want) + } + } +} diff --git a/internal/storage/rclone.go b/internal/storage/rclone.go index 994763d..b40b50b 100644 --- a/internal/storage/rclone.go +++ b/internal/storage/rclone.go @@ -18,6 +18,7 @@ type Backend interface { Upload(remotePath string, in io.Reader) error Download(remotePath string, out io.Writer) error ListFiles(prefix []string) ([]byte, error) + ListRecursive(prefix []string) ([]string, error) Delete(remotePath string) error } @@ -86,6 +87,40 @@ func (r *RcloneBackend) ListFiles(prefix []string) ([]byte, error) { return stdout.Bytes(), nil } +// ListRecursive lists every file under prefix, recursively. Paths are +// returned in the same remote-relative shape as Upload keys. +func (r *RcloneBackend) ListRecursive(prefix []string) ([]string, error) { + remotePath := r.Remote + if len(prefix) > 0 { + remotePath = fmt.Sprintf("%s/%s", r.Remote, strings.Join(prefix, "/")) + } + + cmd := exec.Command("rclone", rcloneArgs(r.ConfigPath, "lsjson", "-R", remotePath)...) + var stdout, stderr bytes.Buffer + cmd.Stdout = &stdout + cmd.Stderr = &stderr + if err := cmd.Run(); err != nil { + return nil, fmt.Errorf("failed to list %s via rclone lsjson: %w\nRclone Error: %s", remotePath, err, stderr.String()) + } + + entries, err := ParseList(stdout.Bytes()) + if err != nil { + return nil, fmt.Errorf("failed to parse recursive listing: %w", err) + } + out := make([]string, 0, len(entries)) + for _, e := range entries { + if e.IsDir { + continue + } + if len(prefix) > 0 { + out = append(out, strings.Join(append(append([]string{}, prefix...), e.Path), "/")) + } else { + out = append(out, e.Path) + } + } + return out, nil +} + // Delete removes a file or directory at the given remote path using rclone purge. func (r *RcloneBackend) Delete(remotePath string) error { fullPath := fmt.Sprintf("%s/%s", r.Remote, remotePath) From aca6cbb42dbd6e0445a142993068ad42ba29d395 Mon Sep 17 00:00:00 2001 From: Syrup Date: Mon, 12 Oct 2026 02:07:48 +0700 Subject: [PATCH 23/48] feat(index): expose sorted hash listing for check --- internal/index/index.go | 18 ++++++++++++++++++ internal/index/index_test.go | 11 +++++++++++ 2 files changed, 29 insertions(+) diff --git a/internal/index/index.go b/internal/index/index.go index e9c2765..c5ce1a3 100644 --- a/internal/index/index.go +++ b/internal/index/index.go @@ -128,6 +128,24 @@ func (ix *Index) Has(hash string) bool { return ok } +// AllHashes returns every known plaintext hash, sorted. Pending entries that +// were never saved are included so check sees work in progress. +func (ix *Index) AllHashes() []string { + seen := make(map[string]bool, len(ix.all)+len(ix.pending)) + for h := range ix.all { + seen[h] = true + } + for h := range ix.pending { + seen[h] = true + } + out := make([]string, 0, len(seen)) + for h := range seen { + out = append(out, h) + } + sort.Strings(out) + return out +} + func (ix *Index) Get(hash string) (Entry, bool) { if e, ok := ix.pending[hash]; ok { return e, true diff --git a/internal/index/index_test.go b/internal/index/index_test.go index 68a5abd..5fe69b6 100644 --- a/internal/index/index_test.go +++ b/internal/index/index_test.go @@ -47,6 +47,17 @@ func TestAddHasGet(t *testing.T) { } } +func TestAllHashesMergesPendingAndAll(t *testing.T) { + ix := New() + ix.Add(entryFixture("bbb")) + ix.Add(entryFixture("aaa")) + got := ix.AllHashes() + want := []string{"aaa", "bbb"} + if len(got) != 2 || got[0] != want[0] || got[1] != want[1] { + t.Fatalf("AllHashes() = %v, want %v", got, want) + } +} + func TestSaveAndLoadAcrossEpochs(t *testing.T) { pub, ident := identityFixture(t) backend := memtest.NewMemoryBackend() From ea54839f064c59f5e244495a819d20abbc0095d5 Mon Sep 17 00:00:00 2001 From: Syrup Date: Mon, 12 Oct 2026 02:09:00 +0700 Subject: [PATCH 24/48] feat(flow): add structural repository check --- check.go | 26 ++++++++ internal/flow/check.go | 117 ++++++++++++++++++++++++++++++++++++ internal/flow/check_test.go | 79 ++++++++++++++++++++++++ main.go | 1 + main_test.go | 2 + 5 files changed, 225 insertions(+) create mode 100644 check.go create mode 100644 internal/flow/check.go create mode 100644 internal/flow/check_test.go diff --git a/check.go b/check.go new file mode 100644 index 0000000..9c61498 --- /dev/null +++ b/check.go @@ -0,0 +1,26 @@ +package main + +import ( + "path/filepath" + + "github.com/pterm/pterm" + + "gloak/internal/flow" +) + +type CheckCmd struct { + Identity string `help:"Identity private key file (defaults to ~/.gloak/identity.txt)." short:"i"` + Remote string `help:"Source Rclone URI (e.g., keraaaki:backup_folder or /mnt/backups)." required:"" short:"R"` +} + +func (c *CheckCmd) Run() error { + pterm.Info.Println("Starting CHECK process...") + + ident := c.Identity + if ident == "" { + ident = filepath.Join(getDefaultDir(), "identity.txt") + pterm.Info.Printf("Using identity from: %s\n", ident) + } + + return flow.RunCheck(ident, c.Remote) +} diff --git a/internal/flow/check.go b/internal/flow/check.go new file mode 100644 index 0000000..0df91f6 --- /dev/null +++ b/internal/flow/check.go @@ -0,0 +1,117 @@ +package flow + +import ( + "fmt" + "path" + "sort" + "strings" + + "github.com/pterm/pterm" + + "gloak/internal/index" + "gloak/internal/repo" + "gloak/internal/storage" +) + +// CheckResult is the structural consistency report for a repository. +type CheckResult struct { + Snapshots int + Referenced int + DataObjects int + Missing []string + Garbage []string +} + +// RunCheck opens the repository and prints a structural consistency report. +func RunCheck(identityPath, remote string) error { + backend := storage.NewRcloneBackend(remote) + spinner, _ := pterm.DefaultSpinner.Start("Checking repository...") + res, err := runCheck(identityPath, backend) + if err != nil { + _ = spinner.Stop() + return err + } + _ = spinner.Stop() + pterm.Success.Println("Repository check passed") + pterm.Printf("Snapshots: %d\n", res.Snapshots) + pterm.Printf("Referenced chunks: %d\n", res.Referenced) + pterm.Printf("Data objects: %d\n", res.DataObjects) + if len(res.Garbage) > 0 { + pterm.Warning.Printf("Garbage objects: %d (run prune in a later phase)\n", len(res.Garbage)) + } + return nil +} + +func runCheck(identityPath string, backend storage.Backend) (CheckResult, error) { + var res CheckResult + + if _, err := repo.Open(backend, identityPath); err != nil { + return res, err + } + ix, err := index.Load(backend, identityPath) + if err != nil { + return res, fmt.Errorf("failed to load index: %w", err) + } + + snapRaw, err := backend.ListFiles([]string{repo.SnapshotsPrefix}) + if err != nil { + // A fresh repo has no snapshots/ directory yet; treat as empty. + snapRaw = []byte("[]") + } + snapEntries, err := storage.ParseList(snapRaw) + if err != nil { + return res, fmt.Errorf("failed to parse snapshot listing: %w", err) + } + + referenced := make(map[string]bool) + for _, entry := range snapEntries { + if entry.IsDir || !strings.HasSuffix(entry.Path, ".age") { + continue + } + id := strings.TrimSuffix(path.Base(entry.Path), ".age") + m, err := readManifest(backend, identityPath, id) + if err != nil { + return res, fmt.Errorf("snapshot %s: %w", id, err) + } + res.Snapshots++ + for _, meta := range m.Chunks { + referenced[meta.PlainSHA256] = true + } + } + for _, h := range ix.AllHashes() { + referenced[h] = true + } + res.Referenced = len(referenced) + + dataPaths, err := backend.ListRecursive([]string{repo.DataPrefix}) + if err != nil { + return res, fmt.Errorf("failed to list data objects: %w", err) + } + present := make(map[string]bool, len(dataPaths)) + for _, p := range dataPaths { + h := path.Base(p) + if len(h) != 64 { + continue + } + present[h] = true + } + res.DataObjects = len(present) + + for h := range referenced { + if !present[h] { + res.Missing = append(res.Missing, h) + } + } + sort.Strings(res.Missing) + for h := range present { + if !referenced[h] { + res.Garbage = append(res.Garbage, h) + } + } + sort.Strings(res.Garbage) + + if len(res.Missing) > 0 { + return res, fmt.Errorf("check failed: %d referenced chunk(s) missing from storage", len(res.Missing)) + } + return res, nil +} diff --git a/internal/flow/check_test.go b/internal/flow/check_test.go new file mode 100644 index 0000000..ae9f0dc --- /dev/null +++ b/internal/flow/check_test.go @@ -0,0 +1,79 @@ +package flow + +import ( + "bytes" + "strings" + "testing" +) + +func TestRunCheckCleanRepository(t *testing.T) { + ident, backend, _ := repoFixture(t) + filePath := writeTempFile(t, []byte("check me please")) + if _, err := runBackup(filePath, ident, backend, "7f91c6c7-7a0b-44aa-ae23-997b60e4e998", nil); err != nil { + t.Fatal(err) + } + res, err := runCheck(ident, backend) + if err != nil { + t.Fatalf("runCheck() error: %v", err) + } + if res.Snapshots != 1 || len(res.Missing) != 0 { + t.Fatalf("result = %+v", res) + } + if res.DataObjects < 1 { + t.Fatalf("DataObjects = %d", res.DataObjects) + } +} + +func TestRunCheckReportsMissingObject(t *testing.T) { + ident, backend, _ := repoFixture(t) + filePath := writeTempFile(t, []byte("check me please")) + if _, err := runBackup(filePath, ident, backend, "7f91c6c7-7a0b-44aa-ae23-997b60e4e998", nil); err != nil { + t.Fatal(err) + } + keys := backend.KeysWithPrefix("data/") + if len(keys) == 0 { + t.Fatal("no data objects") + } + backend.Delete(keys[0]) + + _, err := runCheck(ident, backend) + if err == nil { + t.Fatal("runCheck() succeeded with missing object") + } + if !strings.Contains(err.Error(), "missing") { + t.Fatalf("error = %v, want missing mention", err) + } +} + +func TestRunCheckReportsGarbageWithoutError(t *testing.T) { + ident, backend, _ := repoFixture(t) + filePath := writeTempFile(t, []byte("check me please")) + if _, err := runBackup(filePath, ident, backend, "7f91c6c7-7a0b-44aa-ae23-997b60e4e998", nil); err != nil { + t.Fatal(err) + } + _ = backend.Upload("data/ff/ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff", bytes.NewReader([]byte("orphan"))) + + res, err := runCheck(ident, backend) + if err != nil { + t.Fatalf("runCheck() error: %v", err) + } + if len(res.Garbage) != 1 { + t.Fatalf("Garbage = %v, want one entry", res.Garbage) + } +} + +func TestRunCheckRejectsCorruptSnapshot(t *testing.T) { + ident, backend, _ := repoFixture(t) + filePath := writeTempFile(t, []byte("check me please")) + if _, err := runBackup(filePath, ident, backend, "7f91c6c7-7a0b-44aa-ae23-997b60e4e998", nil); err != nil { + t.Fatal(err) + } + key := "snapshots/7f91c6c7-7a0b-44aa-ae23-997b60e4e998.age" + blob := backend.Get(key) + blob[len(blob)/2] ^= 0xff + backend.Put(key, blob) + + if _, err := runCheck(ident, backend); err == nil { + t.Fatal("runCheck() succeeded on corrupt snapshot") + } +} diff --git a/main.go b/main.go index b2e07a2..e2c9bb8 100644 --- a/main.go +++ b/main.go @@ -19,6 +19,7 @@ var cli struct { Upload UploadCmd `cmd:"" help:"Encrypt and upload a file as a standalone folder (no repository)."` Restore RestoreCmd `cmd:"" help:"Restore and decrypt a backup from storage."` Verify VerifyCmd `cmd:"" help:"Verify backup chunks and checksums."` + Check CheckCmd `cmd:"" help:"Check repository consistency (structural)."` Setup SetupCmd `cmd:"" help:"Download and install rclone to ~/.gloak/bin/."` } diff --git a/main_test.go b/main_test.go index 93f0975..46a3000 100644 --- a/main_test.go +++ b/main_test.go @@ -60,6 +60,8 @@ func TestCommandOptionShortTags(t *testing.T) { {name: "restore output dir", cmd: RestoreCmd{}, fieldName: "OutputDir", wantShort: "o"}, {name: "verify identity", cmd: VerifyCmd{}, fieldName: "Identity", wantShort: "i"}, {name: "verify remote", cmd: VerifyCmd{}, fieldName: "Remote", wantShort: "R"}, + {name: "check identity", cmd: CheckCmd{}, fieldName: "Identity", wantShort: "i"}, + {name: "check remote", cmd: CheckCmd{}, fieldName: "Remote", wantShort: "R"}, {name: "setup dir", cmd: SetupCmd{}, fieldName: "Dir", wantShort: "D"}, {name: "setup update", cmd: SetupCmd{}, fieldName: "Update", wantShort: "u"}, } From 7fb7101f836a58bb21332478f811d67c15883d75 Mon Sep 17 00:00:00 2001 From: Syrup Date: Mon, 12 Oct 2026 02:09:50 +0700 Subject: [PATCH 25/48] docs: note CDC chunking and repository check --- CHANGELOG.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index ca7117e..814d828 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,10 +9,13 @@ - Snapshots are encrypted to every recipient listed at init time. Any matching identity can verify or restore. - `gloak upload -R ` writes a self-contained backup folder `{uuid}/` at the remote root (sealed chunks, multi-recipient `manifest.age` with embedded master key, public `locator.json`). No repository and no `init` required. Manifest version `file-2`. - `restore` and `verify` auto-detect file-mode folders by `locator.json` and fall back to repository layout otherwise. +- `backup` splits files with content-defined chunking (gear hash). Chunk bounds `chunk_min`/`chunk_avg`/`chunk_max` are stored in `repo.json` at init; flags `--chunk-min`, `--chunk-avg`, `--chunk-max` override the defaults (512 KiB / 1 MiB / 8 MiB). A repeat backup of unchanged content uploads nothing. +- `gloak check` reports structural consistency: every snapshot decrypts, every referenced chunk exists in `data/`, and unreferenced objects are listed as garbage. It does not download chunk payloads; use `verify` for a deep pass. ### Changed - `verify` and `restore` read the repository layout. Corruption errors keep the CHUNK CORRUPT prefix. +- Repository backup no longer uses fixed 20 MiB chunks. Phase 1 repositories with `chunk_size` in `repo.json` cannot be opened; re-run `gloak init`. ### Removed From af95229f85b00217841ab58e70a081c8d53b3650 Mon Sep 17 00:00:00 2001 From: Syrup Date: Mon, 12 Oct 2026 02:32:32 +0700 Subject: [PATCH 26/48] feat(core): add tree manifest types and path validation --- internal/core/manifest.go | 34 ++++++++++++++++++++++++++++++++++ internal/core/manifest_test.go | 14 ++++++++++++++ internal/core/path.go | 32 ++++++++++++++++++++++++++++++++ internal/core/path_test.go | 19 +++++++++++++++++++ 4 files changed, 99 insertions(+) create mode 100644 internal/core/manifest_test.go create mode 100644 internal/core/path.go create mode 100644 internal/core/path_test.go diff --git a/internal/core/manifest.go b/internal/core/manifest.go index 4963d0a..4569fc4 100644 --- a/internal/core/manifest.go +++ b/internal/core/manifest.go @@ -4,6 +4,11 @@ import ( "github.com/google/uuid" ) +const ( + KindFile = "file" + KindTree = "tree" +) + type Manifest struct { Version string `json:"version"` UUID string `json:"uuid"` @@ -15,6 +20,35 @@ type Manifest struct { Chunks []ChunkMeta `json:"chunks"` } +type TreeEntry struct { + RelPath string `json:"rel_path"` + Type string `json:"type"` + Mode uint32 `json:"mode"` + Mtime string `json:"mtime"` + Size int64 `json:"size"` + Chunks []ChunkMeta `json:"chunks"` +} + +type TreeManifest struct { + Version string `json:"version"` + UUID string `json:"uuid"` + Kind string `json:"kind"` + CreatedAt string `json:"created_at"` + Hostname string `json:"hostname"` + Paths []string `json:"paths"` + Entries []TreeEntry `json:"entries"` +} + +func (m *TreeManifest) FileCount() int { + n := 0 + for _, e := range m.Entries { + if e.Type == "file" { + n++ + } + } + return n +} + func GenerateUUID() string { return uuid.New().String() } diff --git a/internal/core/manifest_test.go b/internal/core/manifest_test.go new file mode 100644 index 0000000..39b4c86 --- /dev/null +++ b/internal/core/manifest_test.go @@ -0,0 +1,14 @@ +package core + +import "testing" + +func TestFileCount(t *testing.T) { + m := TreeManifest{Entries: []TreeEntry{ + {Type: "dir"}, + {Type: "file"}, + {Type: "file"}, + }} + if got := m.FileCount(); got != 2 { + t.Fatalf("FileCount() = %d, want 2", got) + } +} diff --git a/internal/core/path.go b/internal/core/path.go new file mode 100644 index 0000000..924fc99 --- /dev/null +++ b/internal/core/path.go @@ -0,0 +1,32 @@ +package core + +import ( + "fmt" + "strings" +) + +// ValidateRelPath rejects tree entry paths that could escape the restore +// root. The manifest is remote input; this is the trust boundary. +func ValidateRelPath(rel string) error { + if rel == "" { + return fmt.Errorf("empty path") + } + if strings.ContainsRune(rel, 0) { + return fmt.Errorf("path contains null byte") + } + if strings.Contains(rel, `\`) { + return fmt.Errorf("path contains backslash: %s", rel) + } + if strings.HasPrefix(rel, "/") { + return fmt.Errorf("absolute path: %s", rel) + } + if len(rel) >= 2 && rel[1] == ':' { + return fmt.Errorf("drive letter path: %s", rel) + } + for _, seg := range strings.Split(rel, "/") { + if seg == "" || seg == "." || seg == ".." { + return fmt.Errorf("unsafe path segment in %s", rel) + } + } + return nil +} diff --git a/internal/core/path_test.go b/internal/core/path_test.go new file mode 100644 index 0000000..7924523 --- /dev/null +++ b/internal/core/path_test.go @@ -0,0 +1,19 @@ +package core + +import "testing" + +func TestValidateRelPathAcceptsNormal(t *testing.T) { + for _, p := range []string{"a", "a/b", "a/b/c.txt", "dir/empty"} { + if err := ValidateRelPath(p); err != nil { + t.Fatalf("ValidateRelPath(%q) error: %v", p, err) + } + } +} + +func TestValidateRelPathRejectsUnsafe(t *testing.T) { + for _, p := range []string{"", "/abs", "a//b", "a/../b", "..", ".", "a/.", `a\b`, "a\x00b", "C:/x", "c:\\x"} { + if err := ValidateRelPath(p); err == nil { + t.Fatalf("ValidateRelPath(%q) succeeded", p) + } + } +} From 2c9fe7e003bfb098f929f1f4806fb96f772edcc9 Mon Sep 17 00:00:00 2001 From: Syrup Date: Mon, 12 Oct 2026 02:34:02 +0700 Subject: [PATCH 27/48] feat(flow): backup directory trees with sorted walk --- internal/flow/backup.go | 18 ++- internal/flow/manifest_read.go | 73 ++++++++++++ internal/flow/tree_backup.go | 186 ++++++++++++++++++++++++++++++ internal/flow/tree_backup_test.go | 97 ++++++++++++++++ 4 files changed, 373 insertions(+), 1 deletion(-) create mode 100644 internal/flow/tree_backup.go create mode 100644 internal/flow/tree_backup_test.go diff --git a/internal/flow/backup.go b/internal/flow/backup.go index ea12c75..114b176 100644 --- a/internal/flow/backup.go +++ b/internal/flow/backup.go @@ -37,7 +37,7 @@ func RunBackup(filePath, identityPath, remote string) (string, error) { snapshotID := core.GenerateUUID() spinner, _ := pterm.DefaultSpinner.Start("Backing up...") - res, err := runBackup(filePath, identityPath, backend, snapshotID, func(done int) { + res, err := runBackupPath(filePath, identityPath, backend, snapshotID, func(done int) { spinner.UpdateText(fmt.Sprintf("Backed up %d chunks", done)) }) if err != nil { @@ -53,6 +53,22 @@ func RunBackup(filePath, identityPath, remote string) (string, error) { return res.SnapshotID, nil } +// runBackupPath dispatches to the file or tree backup flow based on whether +// filePath is a regular file or a directory. +func runBackupPath(filePath, identityPath string, backend storage.Backend, snapshotID string, progress func(int)) (BackupResult, error) { + if _, err := uuid.Parse(snapshotID); err != nil { + return BackupResult{}, fmt.Errorf("invalid snapshot id: %w", err) + } + info, err := os.Stat(filePath) + if err != nil { + return BackupResult{}, fmt.Errorf("failed to stat path: %w", err) + } + if info.IsDir() { + return runTreeBackup(filePath, identityPath, backend, snapshotID, progress) + } + return runBackup(filePath, identityPath, backend, snapshotID, progress) +} + func runBackup(filePath, identityPath string, backend storage.Backend, snapshotID string, progress func(int)) (BackupResult, error) { if _, err := uuid.Parse(snapshotID); err != nil { return BackupResult{}, fmt.Errorf("invalid snapshot id: %w", err) diff --git a/internal/flow/manifest_read.go b/internal/flow/manifest_read.go index 6f4d8f1..43d0397 100644 --- a/internal/flow/manifest_read.go +++ b/internal/flow/manifest_read.go @@ -39,6 +39,79 @@ func readManifest(backend storage.Backend, identityPath, snapshotID string) (cor return manifest, nil } +// readTreeManifest downloads the tree snapshot for snapshotID and decrypts +// it with the identity that opened the repository. +func readTreeManifest(backend storage.Backend, identityPath, snapshotID string) (core.TreeManifest, error) { + path, err := repo.SnapshotPath(snapshotID) + if err != nil { + return core.TreeManifest{}, err + } + var buf bytes.Buffer + if err := backend.Download(path, &buf); err != nil { + return core.TreeManifest{}, fmt.Errorf("snapshot %s not found: %w", snapshotID, err) + } + plain, err := decryptSnapshotBytes(identityPath, buf.Bytes()) + if err != nil { + return core.TreeManifest{}, err + } + var manifest core.TreeManifest + if err := json.Unmarshal(plain, &manifest); err != nil { + return core.TreeManifest{}, fmt.Errorf("failed to parse tree snapshot: %w", err) + } + return manifest, nil +} + +// readAnyManifest decrypts a snapshot and returns its kind. kind is +// core.KindFile or core.KindTree; exactly one of file/tree is non-nil. +func readAnyManifest(backend storage.Backend, identityPath, snapshotID string) (string, *core.Manifest, *core.TreeManifest, error) { + path, err := repo.SnapshotPath(snapshotID) + if err != nil { + return "", nil, nil, err + } + var buf bytes.Buffer + if err := backend.Download(path, &buf); err != nil { + return "", nil, nil, fmt.Errorf("snapshot %s not found: %w", snapshotID, err) + } + plain, err := decryptSnapshotBytes(identityPath, buf.Bytes()) + if err != nil { + return "", nil, nil, err + } + var probe struct { + Kind string `json:"kind"` + } + if err := json.Unmarshal(plain, &probe); err != nil { + return "", nil, nil, fmt.Errorf("failed to parse snapshot: %w", err) + } + switch probe.Kind { + case core.KindTree: + var m core.TreeManifest + if err := json.Unmarshal(plain, &m); err != nil { + return "", nil, nil, fmt.Errorf("failed to parse tree snapshot: %w", err) + } + return core.KindTree, nil, &m, nil + case "", core.KindFile: + var m core.Manifest + if err := json.Unmarshal(plain, &m); err != nil { + return "", nil, nil, fmt.Errorf("failed to parse file snapshot: %w", err) + } + return core.KindFile, &m, nil, nil + default: + return "", nil, nil, fmt.Errorf("unknown snapshot kind %q", probe.Kind) + } +} + +func decryptSnapshotBytes(identityPath string, sealed []byte) ([]byte, error) { + reader, err := crypto.DecryptReader(identityPath, bytes.NewReader(sealed)) + if err != nil { + return nil, fmt.Errorf("failed to decrypt snapshot: %w", err) + } + plain, err := io.ReadAll(reader) + if err != nil { + return nil, fmt.Errorf("failed to read snapshot: %w", err) + } + return plain, nil +} + // validateChunkPath rejects manifest chunk paths that fall outside data/ or // contain traversal segments. func validateChunkPath(path string) error { diff --git a/internal/flow/tree_backup.go b/internal/flow/tree_backup.go new file mode 100644 index 0000000..c9f5bbe --- /dev/null +++ b/internal/flow/tree_backup.go @@ -0,0 +1,186 @@ +package flow + +import ( + "bytes" + "crypto/sha256" + "encoding/hex" + "encoding/json" + "fmt" + "io/fs" + "os" + "path/filepath" + "sort" + "time" + + "gloak/internal/chunker" + "gloak/internal/core" + "gloak/internal/crypto" + "gloak/internal/index" + "gloak/internal/repo" + "gloak/internal/storage" +) + +func runTreeBackup(dirPath, identityPath string, backend storage.Backend, snapshotID string, progress func(int)) (BackupResult, error) { + r, err := repo.Open(backend, identityPath) + if err != nil { + return BackupResult{}, err + } + ix, err := index.Load(backend, identityPath) + if err != nil { + return BackupResult{}, fmt.Errorf("failed to load index: %w", err) + } + cdc, err := chunker.NewCDC(r.Config.ChunkMin, r.Config.ChunkAvg, r.Config.ChunkMax) + if err != nil { + return BackupResult{}, fmt.Errorf("invalid chunk config: %w", err) + } + + abs, err := filepath.Abs(dirPath) + if err != nil { + return BackupResult{}, err + } + + var entries []core.TreeEntry + var sinceCheckpoint int + res := BackupResult{SnapshotID: snapshotID} + + walkErr := filepath.WalkDir(abs, func(p string, d fs.DirEntry, walkErr error) error { + if walkErr != nil { + return walkErr + } + if p == abs { + return nil + } + rel, err := filepath.Rel(abs, p) + if err != nil { + return err + } + rel = filepath.ToSlash(rel) + info, err := d.Info() + if err != nil { + return err + } + mode := uint32(info.Mode().Perm()) + mtime := info.ModTime().UTC().Format(time.RFC3339) + + switch { + case d.IsDir(): + entries = append(entries, core.TreeEntry{ + RelPath: rel, + Type: "dir", + Mode: mode, + Mtime: mtime, + }) + return nil + case !info.Mode().IsRegular(): + // Symlinks and special files are skipped. + return nil + } + + file, err := os.Open(p) + if err != nil { + return err + } + defer file.Close() + + var chunks []core.ChunkMeta + payloadHasher := sha256.New() + splitErr := cdc.Split(file, func(chunk []byte) error { + payloadHasher.Write(chunk) + hash := hashBytes(chunk) + res.TotalChunks++ + res.TotalPlainBytes += int64(len(chunk)) + dataPath := repo.DataPath(hash) + + if ix.Has(hash) { + res.SkippedChunks++ + } else { + sealed, err := crypto.SealChunk(r.MasterKey, hash, chunk) + if err != nil { + return fmt.Errorf("failed to seal chunk %s: %w", hash, err) + } + encSum := sha256.Sum256(sealed) + if err := backend.Upload(dataPath, bytes.NewReader(sealed)); err != nil { + return fmt.Errorf("failed to upload chunk %s: %w", hash, err) + } + ix.Add(index.Entry{ + PlainSHA256: hash, + PlainSize: int64(len(chunk)), + EncryptedSize: int64(len(sealed)), + EncryptedSHA256: hex.EncodeToString(encSum[:]), + }) + res.UploadedChunks++ + sinceCheckpoint++ + if sinceCheckpoint >= indexCheckpointEvery { + if _, err := ix.Save(backend, r.Config.Recipients); err != nil { + return fmt.Errorf("failed to checkpoint index: %w", err) + } + sinceCheckpoint = 0 + } + } + + encEntry, _ := ix.Get(hash) + chunks = append(chunks, core.ChunkMeta{ + Index: len(chunks), + Name: dataPath, + PlainSize: encEntry.PlainSize, + PlainSHA256: hash, + EncryptedSize: encEntry.EncryptedSize, + EncryptedSHA256: encEntry.EncryptedSHA256, + }) + if progress != nil { + progress(res.TotalChunks) + } + return nil + }) + if splitErr != nil { + return splitErr + } + entries = append(entries, core.TreeEntry{ + RelPath: rel, + Type: "file", + Mode: mode, + Mtime: mtime, + Size: info.Size(), + Chunks: chunks, + }) + return nil + }) + if walkErr != nil { + return BackupResult{}, walkErr + } + + sort.Slice(entries, func(i, j int) bool { + return entries[i].RelPath < entries[j].RelPath + }) + + if _, err := ix.Save(backend, r.Config.Recipients); err != nil { + return BackupResult{}, fmt.Errorf("failed to save index: %w", err) + } + + host, _ := os.Hostname() + manifest := core.TreeManifest{ + Version: repo.FormatVersion, + UUID: snapshotID, + Kind: core.KindTree, + CreatedAt: time.Now().UTC().Format(time.RFC3339), + Hostname: host, + Paths: []string{abs}, + Entries: entries, + } + manifestJSON, err := json.Marshal(manifest) + if err != nil { + return BackupResult{}, fmt.Errorf("failed to marshal tree manifest: %w", err) + } + sealed, err := crypto.EncryptMulti(r.Config.Recipients, manifestJSON) + if err != nil { + return BackupResult{}, fmt.Errorf("failed to encrypt tree manifest: %w", err) + } + snapshotPath, err := repo.SnapshotPath(snapshotID) + if err != nil { + return BackupResult{}, err + } + if err := backend.Upload(snapshotPath, bytes.NewReader(sealed)); err != nil { + return BackupResult{}, fmt.Errorf("failed to upload snapshot: %w", err) + } + return res, nil +} diff --git a/internal/flow/tree_backup_test.go b/internal/flow/tree_backup_test.go new file mode 100644 index 0000000..5baa468 --- /dev/null +++ b/internal/flow/tree_backup_test.go @@ -0,0 +1,97 @@ +package flow + +import ( + "os" + "path/filepath" + "strings" + "testing" +) + +func TestRunTreeBackupNestedDirs(t *testing.T) { + ident, backend, _ := repoFixture(t) + root := t.TempDir() + if err := os.MkdirAll(filepath.Join(root, "sub", "empty"), 0755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(root, "a.txt"), []byte("alpha"), 0644); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(root, "sub", "b.txt"), []byte("beta-beta-beta"), 0644); err != nil { + t.Fatal(err) + } + + snap := "aaaaaaaa-bbbb-4ccc-8ddd-000000000010" + res, err := runBackupPath(root, ident, backend, snap, nil) + if err != nil { + t.Fatalf("runBackupPath() error: %v", err) + } + if res.TotalChunks < 2 { + t.Fatalf("TotalChunks = %d, want >= 2", res.TotalChunks) + } + m, err := readTreeManifest(backend, ident, snap) + if err != nil { + t.Fatal(err) + } + if m.Kind != "tree" { + t.Fatalf("Kind = %q", m.Kind) + } + if m.FileCount() != 2 { + t.Fatalf("FileCount = %d", m.FileCount()) + } + var paths []string + for _, e := range m.Entries { + paths = append(paths, e.RelPath) + } + if paths[0] != "a.txt" { + t.Fatalf("first entry = %q, want a.txt", paths[0]) + } + foundEmpty := false + for _, e := range m.Entries { + if e.Type == "dir" && strings.HasSuffix(e.RelPath, "empty") { + foundEmpty = true + } + } + if !foundEmpty { + t.Fatalf("empty dir missing, entries = %v", paths) + } +} + +func TestRunTreeBackupSkipsSymlinks(t *testing.T) { + ident, backend, _ := repoFixture(t) + root := t.TempDir() + if err := os.WriteFile(filepath.Join(root, "real.txt"), []byte("x"), 0644); err != nil { + t.Fatal(err) + } + if err := os.Symlink("real.txt", filepath.Join(root, "link")); err != nil { + t.Skip("symlink not supported") + } + snap := "aaaaaaaa-bbbb-4ccc-8ddd-000000000011" + if _, err := runBackupPath(root, ident, backend, snap, nil); err != nil { + t.Fatal(err) + } + m, err := readTreeManifest(backend, ident, snap) + if err != nil { + t.Fatal(err) + } + for _, e := range m.Entries { + if e.RelPath == "link" { + t.Fatal("symlink recorded") + } + } +} + +func TestRunBackupPathDispatchesFile(t *testing.T) { + ident, backend, _ := repoFixture(t) + filePath := writeTempFile(t, []byte("single")) + snap := "aaaaaaaa-bbbb-4ccc-8ddd-000000000012" + if _, err := runBackupPath(filePath, ident, backend, snap, nil); err != nil { + t.Fatal(err) + } + m, err := readManifest(backend, ident, snap) + if err != nil { + t.Fatal(err) + } + if m.OriginalName != "payload.bin" { + t.Fatalf("OriginalName = %q", m.OriginalName) + } +} From d80b3f785a6a23253d386c76ee1911ad03bdd862 Mon Sep 17 00:00:00 2001 From: Syrup Date: Mon, 12 Oct 2026 02:36:47 +0700 Subject: [PATCH 28/48] feat(flow): restore tree snapshots with path filter --- internal/flow/filemode_test.go | 12 +-- internal/flow/flow_test.go | 4 +- internal/flow/restore.go | 95 ++++++++++++++++++++- internal/flow/tree_restore_test.go | 130 +++++++++++++++++++++++++++++ restore.go | 11 +-- 5 files changed, 235 insertions(+), 17 deletions(-) create mode 100644 internal/flow/tree_restore_test.go diff --git a/internal/flow/filemode_test.go b/internal/flow/filemode_test.go index 2b9a561..75a9de2 100644 --- a/internal/flow/filemode_test.go +++ b/internal/flow/filemode_test.go @@ -52,7 +52,7 @@ func TestRunRestoreFileModeRoundTrip(t *testing.T) { } outDir := t.TempDir() - gotPath, err := runRestore(backupID, outDir, identityPath, backend) + gotPath, err := runRestore(backupID, outDir, identityPath, backend, "") if err != nil { t.Fatalf("runRestore() error: %v", err) } @@ -78,7 +78,7 @@ func TestRunRestoreFileModeEmptyFile(t *testing.T) { } outDir := t.TempDir() - gotPath, err := runRestore(backupID, outDir, identityPath, backend) + gotPath, err := runRestore(backupID, outDir, identityPath, backend, "") if err != nil { t.Fatalf("runRestore() error: %v", err) } @@ -105,7 +105,7 @@ func TestRunRestoreFileModeEitherIdentity(t *testing.T) { } for _, id := range []string{id1, id2} { out := t.TempDir() - if _, err := runRestore(backupID, out, id, backend); err != nil { + if _, err := runRestore(backupID, out, id, backend, ""); err != nil { t.Fatalf("restore with %s: %v", id, err) } } @@ -127,7 +127,7 @@ func TestRunRestoreFileModeMissingLocator(t *testing.T) { // Without locator, isFileMode is false, so restore falls through to // repository Open. Either "incomplete" or a repository error is fine; // we only assert it fails. - if _, err := runRestore(backupID, t.TempDir(), identityPath, backend); err == nil { + if _, err := runRestore(backupID, t.TempDir(), identityPath, backend, ""); err == nil { t.Fatal("expected error for missing locator") } } @@ -150,7 +150,7 @@ func TestRunRestoreFileModeRejectsTraversalManifest(t *testing.T) { m.Chunks[0].Name = "../evil" resealFileManifest(t, backend, identityPath, backupID, m) - _, err = runRestore(backupID, t.TempDir(), identityPath, backend) + _, err = runRestore(backupID, t.TempDir(), identityPath, backend, "") if err == nil || !bytes.Contains([]byte(err.Error()), []byte("invalid chunk")) { t.Fatalf("err = %v, want invalid chunk", err) } @@ -171,7 +171,7 @@ func TestRunRestoreFileModeCorruptChunk(t *testing.T) { raw[len(raw)/2] ^= 0xff backend.Put(backupID+"/chunk_00000", raw) - _, err = runRestore(backupID, t.TempDir(), identityPath, backend) + _, err = runRestore(backupID, t.TempDir(), identityPath, backend, "") if err == nil || !bytes.Contains([]byte(err.Error()), []byte("CHUNK CORRUPT")) { t.Fatalf("err = %v, want CHUNK CORRUPT", err) } diff --git a/internal/flow/flow_test.go b/internal/flow/flow_test.go index 940234d..b8c6d34 100644 --- a/internal/flow/flow_test.go +++ b/internal/flow/flow_test.go @@ -26,7 +26,7 @@ func TestBackupVerifyRestoreRoundTrip(t *testing.T) { } outDir := t.TempDir() - restoredPath, err := runRestore(snapshotID, outDir, ident, backend) + restoredPath, err := runRestore(snapshotID, outDir, ident, backend, "") if err != nil { t.Fatalf("runRestore() error: %v", err) } @@ -103,7 +103,7 @@ func TestRestoreRejectsEscapingChunkPath(t *testing.T) { } backend.Put(snapshotPath, sealed) - if _, err := runRestore(snapshotID, t.TempDir(), ident, backend); err == nil { + if _, err := runRestore(snapshotID, t.TempDir(), ident, backend, ""); err == nil { t.Fatal("runRestore() succeeded with escaping chunk path") } else if !strings.Contains(err.Error(), "escapes data prefix") { t.Fatalf("error = %v, want escapes data prefix", err) diff --git a/internal/flow/restore.go b/internal/flow/restore.go index 08eeccb..9c9cac6 100644 --- a/internal/flow/restore.go +++ b/internal/flow/restore.go @@ -7,17 +7,20 @@ import ( "fmt" "os" "path/filepath" + "strings" + "time" "github.com/pterm/pterm" + "gloak/internal/core" "gloak/internal/crypto" "gloak/internal/repo" "gloak/internal/storage" ) -func RunRestore(snapshotID, identityPath, remote, outputDir string) error { +func RunRestore(snapshotID, identityPath, remote, outputDir, pathFilter string) error { backend := storage.NewRcloneBackend(remote) - outputPath, err := runRestore(snapshotID, outputDir, identityPath, backend) + outputPath, err := runRestore(snapshotID, outputDir, identityPath, backend, pathFilter) if err != nil { return err } @@ -25,7 +28,7 @@ func RunRestore(snapshotID, identityPath, remote, outputDir string) error { return nil } -func runRestore(snapshotID, outputDir, identityPath string, backend storage.Backend) (string, error) { +func runRestore(snapshotID, outputDir, identityPath string, backend storage.Backend, pathFilter string) (string, error) { if isFileMode(backend, snapshotID) { return runFileRestore(snapshotID, identityPath, outputDir, backend) } @@ -33,11 +36,17 @@ func runRestore(snapshotID, outputDir, identityPath string, backend storage.Back if err != nil { return "", err } - manifest, err := readManifest(backend, identityPath, snapshotID) + kind, fileM, treeM, err := readAnyManifest(backend, identityPath, snapshotID) if err != nil { return "", err } + if kind == core.KindTree { + return runTreeRestore(treeM, r, outputDir, pathFilter, backend) + } + return runFileRestoreFromManifest(fileM, r, backend, outputDir) +} +func runFileRestoreFromManifest(manifest *core.Manifest, r *repo.Repo, backend storage.Backend, outputDir string) (string, error) { if filepath.Base(manifest.OriginalName) != manifest.OriginalName { return "", fmt.Errorf("invalid original filename in manifest: %s", manifest.OriginalName) } @@ -78,3 +87,81 @@ func runRestore(snapshotID, outputDir, identityPath string, backend storage.Back } return outputPath, nil } + +func runTreeRestore(m *core.TreeManifest, r *repo.Repo, outputDir, pathFilter string, backend storage.Backend) (string, error) { + if len(m.Paths) == 0 { + return "", fmt.Errorf("tree snapshot has no source paths") + } + rootName := filepath.Base(filepath.Clean(m.Paths[0])) + if rootName == "" || rootName == "." || rootName == string(filepath.Separator) { + rootName = m.UUID + } + if err := os.MkdirAll(outputDir, 0755); err != nil { + return "", fmt.Errorf("failed to create output directory: %w", err) + } + destRoot := filepath.Join(outputDir, rootName) + + for _, entry := range m.Entries { + if err := core.ValidateRelPath(entry.RelPath); err != nil { + return "", fmt.Errorf("unsafe path in manifest: %w", err) + } + if pathFilter != "" { + if entry.RelPath != pathFilter && !strings.HasPrefix(entry.RelPath, pathFilter+"/") { + continue + } + } + target := filepath.Join(destRoot, filepath.FromSlash(entry.RelPath)) + // Defense in depth: ensure target stays under destRoot. + if !strings.HasPrefix(target, destRoot+string(filepath.Separator)) && target != destRoot { + return "", fmt.Errorf("restored path escapes output root: %s", entry.RelPath) + } + switch entry.Type { + case "dir": + if err := os.MkdirAll(target, os.FileMode(entry.Mode)&0777); err != nil { + return "", fmt.Errorf("failed to create directory %s: %w", entry.RelPath, err) + } + case "file": + if err := os.MkdirAll(filepath.Dir(target), 0755); err != nil { + return "", fmt.Errorf("failed to create parent for %s: %w", entry.RelPath, err) + } + if err := restoreTreeFile(entry, r, backend, target); err != nil { + return "", err + } + default: + return "", fmt.Errorf("unknown tree entry type %q for %s", entry.Type, entry.RelPath) + } + } + return destRoot, nil +} + +func restoreTreeFile(entry core.TreeEntry, r *repo.Repo, backend storage.Backend, target string) error { + outFile, err := os.OpenFile(target, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, os.FileMode(entry.Mode)&0777) + if err != nil { + return fmt.Errorf("failed to create output file %s: %w", entry.RelPath, err) + } + defer outFile.Close() + + for _, meta := range entry.Chunks { + if err := validateChunkPath(meta.Name); err != nil { + return err + } + var buf bytes.Buffer + if err := backend.Download(meta.Name, &buf); err != nil { + return fmt.Errorf("CHUNK CORRUPT %d (%s): missing object: %w", meta.Index, meta.Name, err) + } + plain, err := crypto.OpenChunk(r.MasterKey, meta.PlainSHA256, buf.Bytes()) + if err != nil { + return fmt.Errorf("CHUNK CORRUPT %d (%s): %w", meta.Index, meta.Name, err) + } + if int64(len(plain)) != meta.PlainSize || hashBytes(plain) != meta.PlainSHA256 { + return fmt.Errorf("CHUNK CORRUPT %d (%s): plaintext mismatch", meta.Index, meta.Name) + } + if _, err := outFile.Write(plain); err != nil { + return fmt.Errorf("failed to write chunk %d: %w", meta.Index, err) + } + } + if mt, err := time.Parse(time.RFC3339, entry.Mtime); err == nil { + _ = os.Chtimes(target, mt, mt) + } + return nil +} diff --git a/internal/flow/tree_restore_test.go b/internal/flow/tree_restore_test.go new file mode 100644 index 0000000..f35f9d2 --- /dev/null +++ b/internal/flow/tree_restore_test.go @@ -0,0 +1,130 @@ +package flow + +import ( + "bytes" + "encoding/json" + "os" + "path/filepath" + "strings" + "testing" + + "gloak/internal/core" + appcrypto "gloak/internal/crypto" + "gloak/internal/repo" + "gloak/internal/storage/memtest" +) + +func treeFixture(t *testing.T) (string, *memtest.MemoryBackend, string, string) { + t.Helper() + ident, backend, _ := repoFixture(t) + root := t.TempDir() + if err := os.MkdirAll(filepath.Join(root, "sub", "empty"), 0755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(root, "a.txt"), []byte("alpha"), 0644); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(root, "sub", "b.txt"), []byte("beta"), 0644); err != nil { + t.Fatal(err) + } + snap := "aaaaaaaa-bbbb-4ccc-8ddd-000000000020" + if _, err := runBackupPath(root, ident, backend, snap, nil); err != nil { + t.Fatal(err) + } + return root, backend, ident, snap +} + +func TestRunTreeRestoreFull(t *testing.T) { + _, backend, ident, snap := treeFixture(t) + out := t.TempDir() + got, err := runRestore(snap, out, ident, backend, "") + if err != nil { + t.Fatalf("runRestore() error: %v", err) + } + if base := filepath.Base(got); base == "" { + t.Fatal("empty restore root") + } + a, err := os.ReadFile(filepath.Join(got, "a.txt")) + if err != nil { + t.Fatal(err) + } + if string(a) != "alpha" { + t.Fatalf("a.txt = %q", a) + } + b, err := os.ReadFile(filepath.Join(got, "sub", "b.txt")) + if err != nil { + t.Fatal(err) + } + if string(b) != "beta" { + t.Fatalf("b.txt = %q", b) + } + if fi, err := os.Stat(filepath.Join(got, "sub", "empty")); err != nil || !fi.IsDir() { + t.Fatal("empty dir not restored") + } +} + +func TestRunTreeRestorePathFilter(t *testing.T) { + _, backend, ident, snap := treeFixture(t) + out := t.TempDir() + got, err := runRestore(snap, out, ident, backend, "sub") + if err != nil { + t.Fatalf("runRestore() error: %v", err) + } + if _, err := os.Stat(filepath.Join(got, "a.txt")); err == nil { + t.Fatal("a.txt restored despite filter") + } + b, err := os.ReadFile(filepath.Join(got, "sub", "b.txt")) + if err != nil { + t.Fatal(err) + } + if string(b) != "beta" { + t.Fatalf("b.txt = %q", b) + } +} + +func TestRunTreeRestoreRejectsUnsafeRelPath(t *testing.T) { + ident, backend, _ := repoFixture(t) + m := core.TreeManifest{ + Version: "2.0", + UUID: "aaaaaaaa-bbbb-4ccc-8ddd-000000000030", + Kind: core.KindTree, + CreatedAt: "2026-10-12T00:00:00Z", + Hostname: "test", + Paths: []string{"/tmp/x"}, + Entries: []core.TreeEntry{ + {RelPath: "../escape", Type: "file", Mode: 0644, Mtime: "2026-10-12T00:00:00Z", Size: 0}, + }, + } + seedTreeSnapshot(t, backend, ident, m) + + _, err := runRestore(m.UUID, t.TempDir(), ident, backend, "") + if err == nil { + t.Fatal("runRestore() accepted traversal RelPath") + } + if !strings.Contains(err.Error(), "unsafe") && !strings.Contains(err.Error(), "..") { + t.Fatalf("error = %v", err) + } +} + +func seedTreeSnapshot(t *testing.T, backend *memtest.MemoryBackend, ident string, m core.TreeManifest) { + t.Helper() + r, err := repo.Open(backend, ident) + if err != nil { + t.Fatal(err) + } + raw, err := json.Marshal(m) + if err != nil { + t.Fatal(err) + } + sealed, err := appcrypto.EncryptMulti(r.Config.Recipients, raw) + if err != nil { + t.Fatal(err) + } + p, err := repo.SnapshotPath(m.UUID) + if err != nil { + t.Fatal(err) + } + if err := backend.Upload(p, bytes.NewReader(sealed)); err != nil { + t.Fatal(err) + } +} diff --git a/restore.go b/restore.go index c47baf2..46ab547 100644 --- a/restore.go +++ b/restore.go @@ -9,10 +9,11 @@ import ( ) type RestoreCmd struct { - UUID string `arg:"" name:"uuid" help:"Backup UUID to restore."` - Identity string `help:"Identity private key file (defaults to ~/.gloak/identity.txt)." short:"i"` - Remote string `help:"Source Rclone URI (e.g., keraaaki:backup_folder or /mnt/backups)." required:"" short:"R"` - OutputDir string `help:"Output directory for restored file." default:"./restored" short:"o"` + UUID string `arg:"" name:"uuid" help:"Backup UUID to restore."` + Identity string `help:"Identity private key file (defaults to ~/.gloak/identity.txt)." short:"i"` + Remote string `help:"Source Rclone URI (e.g., keraaaki:backup_folder or /mnt/backups)." required:"" short:"R"` + OutputDir string `help:"Output directory for restored file." default:"./restored" short:"o"` + PathFilter string `help:"Restore only this relative path (and children) from a tree snapshot."` } func (r *RestoreCmd) Run() error { @@ -24,5 +25,5 @@ func (r *RestoreCmd) Run() error { pterm.Info.Printf("Using identity from: %s\n", ident) } - return flow.RunRestore(r.UUID, ident, r.Remote, r.OutputDir) + return flow.RunRestore(r.UUID, ident, r.Remote, r.OutputDir, r.PathFilter) } From 822f94599256d656fdd6ab9cf4e818469d95ecab Mon Sep 17 00:00:00 2001 From: Syrup Date: Mon, 12 Oct 2026 02:37:59 +0700 Subject: [PATCH 29/48] feat(flow): verify and check tree snapshots --- internal/flow/check.go | 15 ++++++-- internal/flow/tree_verify_test.go | 58 +++++++++++++++++++++++++++++++ internal/flow/verify.go | 44 ++++++++++++++++++++++- 3 files changed, 113 insertions(+), 4 deletions(-) create mode 100644 internal/flow/tree_verify_test.go diff --git a/internal/flow/check.go b/internal/flow/check.go index 0df91f6..5f69e39 100644 --- a/internal/flow/check.go +++ b/internal/flow/check.go @@ -8,6 +8,7 @@ import ( "github.com/pterm/pterm" + "gloak/internal/core" "gloak/internal/index" "gloak/internal/repo" "gloak/internal/storage" @@ -69,13 +70,21 @@ func runCheck(identityPath string, backend storage.Backend) (CheckResult, error) continue } id := strings.TrimSuffix(path.Base(entry.Path), ".age") - m, err := readManifest(backend, identityPath, id) + kind, fileM, treeM, err := readAnyManifest(backend, identityPath, id) if err != nil { return res, fmt.Errorf("snapshot %s: %w", id, err) } res.Snapshots++ - for _, meta := range m.Chunks { - referenced[meta.PlainSHA256] = true + if kind == core.KindTree { + for _, te := range treeM.Entries { + for _, meta := range te.Chunks { + referenced[meta.PlainSHA256] = true + } + } + } else { + for _, meta := range fileM.Chunks { + referenced[meta.PlainSHA256] = true + } } } for _, h := range ix.AllHashes() { diff --git a/internal/flow/tree_verify_test.go b/internal/flow/tree_verify_test.go new file mode 100644 index 0000000..6464053 --- /dev/null +++ b/internal/flow/tree_verify_test.go @@ -0,0 +1,58 @@ +package flow + +import ( + "os" + "path/filepath" + "strings" + "testing" +) + +func TestRunVerifyTreeSnapshot(t *testing.T) { + _, backend, ident, snap := treeFixture(t) + if err := runVerify(snap, ident, backend); err != nil { + t.Fatalf("runVerify() error: %v", err) + } +} + +func TestRunVerifyTreeCorruptChunk(t *testing.T) { + _, backend, ident, snap := treeFixture(t) + keys := backend.KeysWithPrefix("data/") + if len(keys) == 0 { + t.Fatal("no data") + } + blob := backend.Get(keys[0]) + blob[len(blob)/2] ^= 0xff + backend.Put(keys[0], blob) + err := runVerify(snap, ident, backend) + if err == nil { + t.Fatal("runVerify() succeeded on corrupt chunk") + } + if !strings.Contains(err.Error(), "CHUNK CORRUPT") { + t.Fatalf("error = %v", err) + } +} + +func TestRunCheckWithTreeAndFileSnapshots(t *testing.T) { + ident, backend, _ := repoFixture(t) + filePath := writeTempFile(t, []byte("single-file-payload")) + if _, err := runBackup(filePath, ident, backend, "7f91c6c7-7a0b-44aa-ae23-997b60e4e999", nil); err != nil { + t.Fatal(err) + } + root := t.TempDir() + if err := os.WriteFile(filepath.Join(root, "t.txt"), []byte("tree-payload"), 0644); err != nil { + t.Fatal(err) + } + if _, err := runBackupPath(root, ident, backend, "8f91c6c7-7a0b-44aa-ae23-997b60e4e999", nil); err != nil { + t.Fatal(err) + } + res, err := runCheck(ident, backend) + if err != nil { + t.Fatalf("runCheck() error: %v", err) + } + if res.Snapshots != 2 { + t.Fatalf("Snapshots = %d, want 2", res.Snapshots) + } + if len(res.Missing) != 0 { + t.Fatalf("Missing = %v", res.Missing) + } +} diff --git a/internal/flow/verify.go b/internal/flow/verify.go index 62dc3f7..dd788be 100644 --- a/internal/flow/verify.go +++ b/internal/flow/verify.go @@ -8,6 +8,7 @@ import ( "github.com/pterm/pterm" + "gloak/internal/core" "gloak/internal/crypto" "gloak/internal/repo" "gloak/internal/storage" @@ -34,11 +35,17 @@ func runVerify(snapshotID, identityPath string, backend storage.Backend) error { if err != nil { return err } - manifest, err := readManifest(backend, identityPath, snapshotID) + kind, fileM, treeM, err := readAnyManifest(backend, identityPath, snapshotID) if err != nil { return err } + if kind == core.KindTree { + return runTreeVerify(treeM, r, backend) + } + return runFileVerifyFromManifest(fileM, r, backend) +} +func runFileVerifyFromManifest(manifest *core.Manifest, r *repo.Repo, backend storage.Backend) error { payloadHasher := sha256.New() for _, meta := range manifest.Chunks { if err := validateChunkPath(meta.Name); err != nil { @@ -70,3 +77,38 @@ func runVerify(snapshotID, identityPath string, backend storage.Backend) error { } return nil } + +func runTreeVerify(m *core.TreeManifest, r *repo.Repo, backend storage.Backend) error { + for _, entry := range m.Entries { + if entry.Type != "file" { + continue + } + if err := core.ValidateRelPath(entry.RelPath); err != nil { + return err + } + for _, meta := range entry.Chunks { + if err := validateChunkPath(meta.Name); err != nil { + return err + } + var buf bytes.Buffer + if err := backend.Download(meta.Name, &buf); err != nil { + return fmt.Errorf("CHUNK CORRUPT %d (%s): missing object: %w", meta.Index, meta.Name, err) + } + sealed := buf.Bytes() + if int64(len(sealed)) != meta.EncryptedSize { + return fmt.Errorf("CHUNK CORRUPT %d (%s): encrypted size mismatch", meta.Index, meta.Name) + } + if hashBytes(sealed) != meta.EncryptedSHA256 { + return fmt.Errorf("CHUNK CORRUPT %d (%s): encrypted sha256 mismatch", meta.Index, meta.Name) + } + plain, err := crypto.OpenChunk(r.MasterKey, meta.PlainSHA256, sealed) + if err != nil { + return fmt.Errorf("CHUNK CORRUPT %d (%s): %w", meta.Index, meta.Name, err) + } + if int64(len(plain)) != meta.PlainSize || hashBytes(plain) != meta.PlainSHA256 { + return fmt.Errorf("CHUNK CORRUPT %d (%s): plaintext mismatch", meta.Index, meta.Name) + } + } + } + return nil +} From 4458ba870bd34af39209c480b9be1853c7b756c2 Mon Sep 17 00:00:00 2001 From: Syrup Date: Mon, 12 Oct 2026 02:39:36 +0700 Subject: [PATCH 30/48] feat(flow): add snapshots list command --- internal/flow/snapshots.go | 94 +++++++++++++++++++++++++++++++++ internal/flow/snapshots_test.go | 47 +++++++++++++++++ main.go | 1 + main_test.go | 2 + snapshots.go | 23 ++++++++ 5 files changed, 167 insertions(+) create mode 100644 internal/flow/snapshots.go create mode 100644 internal/flow/snapshots_test.go create mode 100644 snapshots.go diff --git a/internal/flow/snapshots.go b/internal/flow/snapshots.go new file mode 100644 index 0000000..80647ba --- /dev/null +++ b/internal/flow/snapshots.go @@ -0,0 +1,94 @@ +package flow + +import ( + "fmt" + "path" + "sort" + "strings" + + "github.com/pterm/pterm" + + "gloak/internal/core" + "gloak/internal/repo" + "gloak/internal/storage" +) + +type SnapshotInfo struct { + UUID string + Kind string + CreatedAt string + Hostname string + Detail string +} + +func RunSnapshots(identityPath, remote string) error { + backend := storage.NewRcloneBackend(remote) + infos, err := runSnapshots(identityPath, backend) + if err != nil { + return err + } + if len(infos) == 0 { + pterm.Info.Println("No snapshots found.") + return nil + } + table := pterm.DefaultTable. + WithHasHeader(). + WithData(append([][]string{{"UUID", "KIND", "CREATED", "HOST", "DETAIL"}}, snapshotRows(infos)...)) + _ = table.Render() + return nil +} + +func snapshotRows(infos []SnapshotInfo) [][]string { + rows := make([][]string, 0, len(infos)) + for _, info := range infos { + rows = append(rows, []string{info.UUID, info.Kind, info.CreatedAt, info.Hostname, info.Detail}) + } + return rows +} + +func runSnapshots(identityPath string, backend storage.Backend) ([]SnapshotInfo, error) { + if _, err := repo.Open(backend, identityPath); err != nil { + return nil, err + } + raw, err := backend.ListFiles([]string{repo.SnapshotsPrefix}) + if err != nil { + // Fresh repo with no snapshots directory. + return nil, nil + } + entries, err := storage.ParseList(raw) + if err != nil { + return nil, fmt.Errorf("failed to parse snapshot listing: %w", err) + } + var infos []SnapshotInfo + for _, entry := range entries { + if entry.IsDir || !strings.HasSuffix(entry.Path, ".age") { + continue + } + id := strings.TrimSuffix(path.Base(entry.Path), ".age") + kind, fileM, treeM, err := readAnyManifest(backend, identityPath, id) + if err != nil { + return nil, fmt.Errorf("snapshot %s: %w", id, err) + } + info := SnapshotInfo{UUID: id, Kind: kind} + if kind == core.KindTree { + info.CreatedAt = treeM.CreatedAt + info.Hostname = treeM.Hostname + detail := fmt.Sprintf("%d files", treeM.FileCount()) + if len(treeM.Paths) > 0 { + detail += " from " + path.Base(strings.TrimRight(treeM.Paths[0], "/")) + } + info.Detail = detail + } else { + info.CreatedAt = fileM.CreatedAt + info.Detail = fileM.OriginalName + } + infos = append(infos, info) + } + sort.Slice(infos, func(i, j int) bool { + if infos[i].CreatedAt == infos[j].CreatedAt { + return infos[i].UUID < infos[j].UUID + } + return infos[i].CreatedAt > infos[j].CreatedAt + }) + return infos, nil +} diff --git a/internal/flow/snapshots_test.go b/internal/flow/snapshots_test.go new file mode 100644 index 0000000..6a4d8f4 --- /dev/null +++ b/internal/flow/snapshots_test.go @@ -0,0 +1,47 @@ +package flow + +import ( + "os" + "path/filepath" + "testing" +) + +func TestRunSnapshotsListsBothKinds(t *testing.T) { + ident, backend, _ := repoFixture(t) + filePath := writeTempFile(t, []byte("single")) + if _, err := runBackup(filePath, ident, backend, "7f91c6c7-7a0b-44aa-ae23-997b60e4e998", nil); err != nil { + t.Fatal(err) + } + root := t.TempDir() + if err := os.WriteFile(filepath.Join(root, "t.txt"), []byte("tree"), 0644); err != nil { + t.Fatal(err) + } + if _, err := runBackupPath(root, ident, backend, "8f91c6c7-7a0b-44aa-ae23-997b60e4e999", nil); err != nil { + t.Fatal(err) + } + infos, err := runSnapshots(ident, backend) + if err != nil { + t.Fatal(err) + } + if len(infos) != 2 { + t.Fatalf("len = %d, want 2", len(infos)) + } + kinds := map[string]bool{} + for _, info := range infos { + kinds[info.Kind] = true + } + if !kinds["file"] || !kinds["tree"] { + t.Fatalf("kinds = %v", kinds) + } +} + +func TestRunSnapshotsEmpty(t *testing.T) { + ident, backend, _ := repoFixture(t) + infos, err := runSnapshots(ident, backend) + if err != nil { + t.Fatal(err) + } + if len(infos) != 0 { + t.Fatalf("len = %d", len(infos)) + } +} diff --git a/main.go b/main.go index e2c9bb8..af09196 100644 --- a/main.go +++ b/main.go @@ -19,6 +19,7 @@ var cli struct { Upload UploadCmd `cmd:"" help:"Encrypt and upload a file as a standalone folder (no repository)."` Restore RestoreCmd `cmd:"" help:"Restore and decrypt a backup from storage."` Verify VerifyCmd `cmd:"" help:"Verify backup chunks and checksums."` + Snapshots SnapshotsCmd `cmd:"" help:"List snapshots in the repository."` Check CheckCmd `cmd:"" help:"Check repository consistency (structural)."` Setup SetupCmd `cmd:"" help:"Download and install rclone to ~/.gloak/bin/."` } diff --git a/main_test.go b/main_test.go index 46a3000..4bf238c 100644 --- a/main_test.go +++ b/main_test.go @@ -60,6 +60,8 @@ func TestCommandOptionShortTags(t *testing.T) { {name: "restore output dir", cmd: RestoreCmd{}, fieldName: "OutputDir", wantShort: "o"}, {name: "verify identity", cmd: VerifyCmd{}, fieldName: "Identity", wantShort: "i"}, {name: "verify remote", cmd: VerifyCmd{}, fieldName: "Remote", wantShort: "R"}, + {name: "snapshots identity", cmd: SnapshotsCmd{}, fieldName: "Identity", wantShort: "i"}, + {name: "snapshots remote", cmd: SnapshotsCmd{}, fieldName: "Remote", wantShort: "R"}, {name: "check identity", cmd: CheckCmd{}, fieldName: "Identity", wantShort: "i"}, {name: "check remote", cmd: CheckCmd{}, fieldName: "Remote", wantShort: "R"}, {name: "setup dir", cmd: SetupCmd{}, fieldName: "Dir", wantShort: "D"}, diff --git a/snapshots.go b/snapshots.go new file mode 100644 index 0000000..83e372a --- /dev/null +++ b/snapshots.go @@ -0,0 +1,23 @@ +package main + +import ( + "path/filepath" + + "github.com/pterm/pterm" + + "gloak/internal/flow" +) + +type SnapshotsCmd struct { + Identity string `help:"Identity private key file (defaults to ~/.gloak/identity.txt)." short:"i"` + Remote string `help:"Source Rclone URI (e.g., keraaaki:backup_folder or /mnt/backups)." required:"" short:"R"` +} + +func (c *SnapshotsCmd) Run() error { + pterm.Info.Println("Listing snapshots...") + ident := c.Identity + if ident == "" { + ident = filepath.Join(getDefaultDir(), "identity.txt") + } + return flow.RunSnapshots(ident, c.Remote) +} From f13b899cc3a214a2be053c7699f8abac978f6e7b Mon Sep 17 00:00:00 2001 From: Syrup Date: Mon, 12 Oct 2026 02:40:47 +0700 Subject: [PATCH 31/48] feat(cli): accept directory paths and document tree snapshots --- CHANGELOG.md | 3 +++ backup.go | 4 ++-- main_test.go | 16 ++++++++++++++++ 3 files changed, 21 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 814d828..a6944f5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,6 +11,9 @@ - `restore` and `verify` auto-detect file-mode folders by `locator.json` and fall back to repository layout otherwise. - `backup` splits files with content-defined chunking (gear hash). Chunk bounds `chunk_min`/`chunk_avg`/`chunk_max` are stored in `repo.json` at init; flags `--chunk-min`, `--chunk-avg`, `--chunk-max` override the defaults (512 KiB / 1 MiB / 8 MiB). A repeat backup of unchanged content uploads nothing. - `gloak check` reports structural consistency: every snapshot decrypts, every referenced chunk exists in `data/`, and unreferenced objects are listed as garbage. It does not download chunk payloads; use `verify` for a deep pass. +- `backup` accepts a directory and writes a tree snapshot: sorted walk, empty directories preserved, symlinks skipped. Snapshot JSON carries `kind: tree` (or `kind: file` for single-file backups). +- `gloak snapshots` lists every snapshot with kind, creation time, host, and a short detail string. +- `restore` recreates directory trees and accepts `--path ` to restore a subtree only. Tree entry paths are validated against traversal and absolute paths. ### Changed diff --git a/backup.go b/backup.go index 4c7a122..1622ad4 100644 --- a/backup.go +++ b/backup.go @@ -11,9 +11,9 @@ import ( "gloak/internal/flow" ) -// BackupCmd backs up a file into a gloak repository. +// BackupCmd backs up a file or directory into a gloak repository. type BackupCmd struct { - ExistingFile string `arg:"" name:"existingfile" type:"existingfile" help:"File to back up"` + ExistingFile string `arg:"" name:"path" help:"File or directory to back up"` Identity string `short:"i" default:"~/.gloak/identity.txt" help:"Path to identity file"` Remote string `short:"R" required:"" help:"Repository remote, e.g. b2:my-bucket/gloak"` } diff --git a/main_test.go b/main_test.go index 4bf238c..0245938 100644 --- a/main_test.go +++ b/main_test.go @@ -94,6 +94,22 @@ func TestUploadCmdFileIsExistingFileArg(t *testing.T) { } } +func TestBackupCmdPathArgTag(t *testing.T) { + field, ok := reflect.TypeOf(BackupCmd{}).FieldByName("ExistingFile") + if !ok { + t.Fatal("field missing") + } + if _, ok := field.Tag.Lookup("arg"); !ok { + t.Fatal("path should be a positional arg") + } + if got := field.Tag.Get("name"); got != "path" { + t.Fatalf("name tag = %q", got) + } + if strings.Contains(field.Tag.Get("type"), "existingfile") { + t.Fatal("existingfile type still set; directories must parse") + } +} + func TestResolveInitRecipientsReadsFile(t *testing.T) { path := filepath.Join(t.TempDir(), "recipient.txt") content := "# comment\n\nage1aaa\nage1bbb\n" From 779b3e542b23d3b1c47ad3049c984b22d8dd614e Mon Sep 17 00:00:00 2001 From: Syrup Date: Mon, 12 Oct 2026 02:41:09 +0700 Subject: [PATCH 32/48] fix(cli): mention directory backups in help --- main.go | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/main.go b/main.go index af09196..89ff7dd 100644 --- a/main.go +++ b/main.go @@ -13,15 +13,15 @@ import ( var cli struct { Debug bool `help:"Enable debug mode." short:"d"` - Keygen KeygenCmd `cmd:"" help:"Generate an X25519 identity and recipient file."` - Init InitCmd `cmd:"" help:"Initialize a new gloak repository." group:"Repository"` - Backup BackupCmd `cmd:"" help:"Encrypt, chunk, and upload a file to a repository."` - Upload UploadCmd `cmd:"" help:"Encrypt and upload a file as a standalone folder (no repository)."` - Restore RestoreCmd `cmd:"" help:"Restore and decrypt a backup from storage."` - Verify VerifyCmd `cmd:"" help:"Verify backup chunks and checksums."` + Keygen KeygenCmd `cmd:"" help:"Generate an X25519 identity and recipient file."` + Init InitCmd `cmd:"" help:"Initialize a new gloak repository." group:"Repository"` + Backup BackupCmd `cmd:"" help:"Encrypt, chunk, and upload a file or directory to a repository."` + Upload UploadCmd `cmd:"" help:"Encrypt and upload a file as a standalone folder (no repository)."` + Restore RestoreCmd `cmd:"" help:"Restore and decrypt a backup from storage."` + Verify VerifyCmd `cmd:"" help:"Verify backup chunks and checksums."` Snapshots SnapshotsCmd `cmd:"" help:"List snapshots in the repository."` - Check CheckCmd `cmd:"" help:"Check repository consistency (structural)."` - Setup SetupCmd `cmd:"" help:"Download and install rclone to ~/.gloak/bin/."` + Check CheckCmd `cmd:"" help:"Check repository consistency (structural)."` + Setup SetupCmd `cmd:"" help:"Download and install rclone to ~/.gloak/bin/."` } func getDefaultDir() string { From 8b29fc012526f5e9f96fa5488530e19b72ee989d Mon Sep 17 00:00:00 2001 From: Syrup Date: Mon, 12 Oct 2026 02:59:11 +0700 Subject: [PATCH 33/48] feat(flow): prune with restic-style retention and index compaction --- internal/flow/prune.go | 218 ++++++++++++++++++++++++++++++++++++ internal/flow/prune_test.go | 120 ++++++++++++++++++++ internal/flow/retention.go | 103 +++++++++++++++++ internal/index/index.go | 23 ++++ internal/storage/rclone.go | 7 +- main.go | 1 + main_test.go | 2 + prune.go | 50 +++++++++ 8 files changed, 521 insertions(+), 3 deletions(-) create mode 100644 internal/flow/prune.go create mode 100644 internal/flow/prune_test.go create mode 100644 internal/flow/retention.go create mode 100644 prune.go diff --git a/internal/flow/prune.go b/internal/flow/prune.go new file mode 100644 index 0000000..01aaf10 --- /dev/null +++ b/internal/flow/prune.go @@ -0,0 +1,218 @@ +package flow + +import ( + "fmt" + "path" + "sort" + "strings" + "time" + + "github.com/google/uuid" + + "gloak/internal/core" + "gloak/internal/index" + "gloak/internal/repo" + "gloak/internal/storage" +) + +// PruneResult summarizes what prune would do or did. +type PruneResult struct { + KeptSnapshots []string + DeadSnapshots []string + LiveChunks int + GarbageChunks int + DeletedSnapshots int + DeletedObjects int + CompactedEpochs bool +} + +// RunPrune opens the repository and runs the prune flow. Without apply, +// nothing is deleted; the result describes the plan only. +func RunPrune(identityPath, remote string, policy RetentionPolicy, apply bool) (PruneResult, error) { + backend := storage.NewRcloneBackend(remote) + return runPrune(identityPath, backend, policy, apply) +} + +func runPrune(identityPath string, backend storage.Backend, policy RetentionPolicy, apply bool) (PruneResult, error) { + var res PruneResult + + if policy.isEmpty() { + return res, fmt.Errorf("refusing to prune with no retention rules (pass at least one --keep-* flag)") + } + + if _, err := repo.Open(backend, identityPath); err != nil { + return res, err + } + ix, err := index.Load(backend, identityPath) + if err != nil { + return res, fmt.Errorf("failed to load index: %w", err) + } + + // List snapshots. + snapRaw, err := backend.ListFiles([]string{repo.SnapshotsPrefix}) + if err != nil { + return res, fmt.Errorf("failed to list snapshots: %w", err) + } + snapEntries, err := storage.ParseList(snapRaw) + if err != nil { + return res, fmt.Errorf("failed to parse snapshot listing: %w", err) + } + + var snaps []snapshotMeta + for _, entry := range snapEntries { + if entry.IsDir || !strings.HasSuffix(entry.Path, ".age") { + continue + } + id := strings.TrimSuffix(path.Base(entry.Path), ".age") + if _, err := uuid.Parse(id); err != nil { + continue + } + kind, fileM, treeM, err := readAnyManifest(backend, identityPath, id) + if err != nil { + // Undecryptable snapshot is an error; do not prune around it. + return res, fmt.Errorf("snapshot %s: %w", id, err) + } + var created time.Time + if kind == core.KindTree { + created, _ = time.Parse(time.RFC3339, treeM.CreatedAt) + } else { + created, _ = time.Parse(time.RFC3339, fileM.CreatedAt) + } + snaps = append(snaps, snapshotMeta{UUID: id, CreatedAt: created}) + } + + keep := selectKeepSnapshots(snaps, policy) + for _, s := range snaps { + if keep[s.UUID] { + res.KeptSnapshots = append(res.KeptSnapshots, s.UUID) + } else { + res.DeadSnapshots = append(res.DeadSnapshots, s.UUID) + } + } + sort.Strings(res.KeptSnapshots) + sort.Strings(res.DeadSnapshots) + + // Mark live chunks from kept snapshots. + live := make(map[string]bool) + for _, s := range snaps { + if !keep[s.UUID] { + continue + } + kind, fileM, treeM, err := readAnyManifest(backend, identityPath, s.UUID) + if err != nil { + return res, fmt.Errorf("snapshot %s: %w", s.UUID, err) + } + if kind == core.KindTree { + for _, te := range treeM.Entries { + for _, meta := range te.Chunks { + live[meta.PlainSHA256] = true + } + } + } else { + for _, meta := range fileM.Chunks { + live[meta.PlainSHA256] = true + } + } + } + res.LiveChunks = len(live) + + // Garbage = index entries not live, plus data objects whose hash is + // not live. + allHashes := ix.AllHashes() + var garbage []string + for _, h := range allHashes { + if !live[h] { + garbage = append(garbage, h) + } + } + // Also scan data/ for objects not in the live set (orphans). + dataPaths, err := backend.ListRecursive([]string{repo.DataPrefix}) + if err != nil { + // No data/ is fine. + dataPaths = nil + } + seenGarbage := make(map[string]bool, len(garbage)) + for _, h := range garbage { + seenGarbage[h] = true + } + for _, p := range dataPaths { + h := path.Base(p) + if len(h) != 64 || live[h] || seenGarbage[h] { + continue + } + garbage = append(garbage, h) + seenGarbage[h] = true + } + sort.Strings(garbage) + res.GarbageChunks = len(garbage) + + if !apply { + return res, nil + } + + // Delete dead snapshots. + for _, id := range res.DeadSnapshots { + sp, err := repo.SnapshotPath(id) + if err != nil { + return res, err + } + if err := backend.Delete(sp); err != nil { + return res, fmt.Errorf("failed to delete snapshot %s: %w", id, err) + } + res.DeletedSnapshots++ + } + + // Delete garbage data objects. + for _, h := range garbage { + dp := repo.DataPath(h) + if err := backend.Delete(dp); err != nil { + return res, fmt.Errorf("failed to delete data object %s: %w", h, err) + } + res.DeletedObjects++ + } + + // Compact index: write one epoch with live entries only, delete old epochs. + if err := compactIndex(backend, identityPath, ix, live); err != nil { + return res, err + } + res.CompactedEpochs = true + + return res, nil +} + +// compactIndex rewrites the index as a single new epoch containing only the +// live hashes, then deletes prior epochs. +func compactIndex(backend storage.Backend, identityPath string, ix *index.Index, live map[string]bool) error { + r, err := repo.Open(backend, identityPath) + if err != nil { + return err + } + + // List current epochs before compacting so we can delete the old ones. + oldRaw, err := backend.ListFiles([]string{"index"}) + if err != nil { + oldRaw = []byte("[]") + } + oldEntries, err := storage.ParseList(oldRaw) + if err != nil { + return err + } + var oldEpochs []string + for _, e := range oldEntries { + if !e.IsDir && strings.HasSuffix(e.Path, ".age") { + oldEpochs = append(oldEpochs, e.Path) + } + } + + // Write one new epoch with live entries only. The epoch counter + // continues past existing names so the compacted file is new. + if _, err := ix.Compact(backend, r.Config.Recipients, live); err != nil { + return fmt.Errorf("failed to write compacted index: %w", err) + } + + // Delete every epoch that existed before compaction. + for _, name := range oldEpochs { + _ = backend.Delete("index/" + name) + } + return nil +} diff --git a/internal/flow/prune_test.go b/internal/flow/prune_test.go new file mode 100644 index 0000000..82991a9 --- /dev/null +++ b/internal/flow/prune_test.go @@ -0,0 +1,120 @@ +package flow + +import ( + "os" + "path/filepath" + "testing" + "time" +) + +func TestSelectKeepSnapshotsLastOnly(t *testing.T) { + base := time.Date(2026, 10, 1, 12, 0, 0, 0, time.UTC) + snaps := []snapshotMeta{ + {UUID: "a", CreatedAt: base.Add(-72 * time.Hour)}, + {UUID: "b", CreatedAt: base.Add(-48 * time.Hour)}, + {UUID: "c", CreatedAt: base.Add(-24 * time.Hour)}, + {UUID: "d", CreatedAt: base}, + } + keep := selectKeepSnapshots(snaps, RetentionPolicy{KeepLast: 2}) + if len(keep) != 2 || !keep["d"] || !keep["c"] { + t.Fatalf("keep = %v", keep) + } +} + +func TestSelectKeepSnapshotsDaily(t *testing.T) { + // Three snapshots on same day, one older day, one older still. + day := time.Date(2026, 10, 10, 8, 0, 0, 0, time.UTC) + snaps := []snapshotMeta{ + {UUID: "old", CreatedAt: day.Add(-48 * time.Hour)}, + {UUID: "mid", CreatedAt: day.Add(-24 * time.Hour)}, + {UUID: "m1", CreatedAt: day}, + {UUID: "m2", CreatedAt: day.Add(2 * time.Hour)}, + } + keep := selectKeepSnapshots(snaps, RetentionPolicy{KeepDaily: 2}) + // Newest of day1 (m2) and newest of day0 (mid). + if !keep["m2"] || !keep["mid"] { + t.Fatalf("keep = %v", keep) + } + if keep["m1"] || keep["old"] { + t.Fatalf("unexpected keep = %v", keep) + } +} + +func TestSelectKeepSnapshotsEmptyPolicyKeepsNothing(t *testing.T) { + snaps := []snapshotMeta{{UUID: "a", CreatedAt: time.Now()}} + keep := selectKeepSnapshots(snaps, RetentionPolicy{}) + if len(keep) != 0 { + t.Fatalf("keep = %v", keep) + } +} + +func TestRunPruneRejectsEmptyPolicy(t *testing.T) { + ident, backend, _ := repoFixture(t) + _, err := runPrune(ident, backend, RetentionPolicy{}, false) + if err == nil { + t.Fatal("runPrune() succeeded with empty policy") + } +} + +func TestRunPruneDryRunChangesNothing(t *testing.T) { + ident, backend, _ := repoFixture(t) + filePath := writeTempFile(t, []byte("keep-me")) + if _, err := runBackup(filePath, ident, backend, "7f91c6c7-7a0b-44aa-ae23-997b60e4e998", nil); err != nil { + t.Fatal(err) + } + before := backend.TotalUploads("") + snapBefore := backend.Has("snapshots/7f91c6c7-7a0b-44aa-ae23-997b60e4e998.age") + + res, err := runPrune(ident, backend, RetentionPolicy{KeepLast: 1}, false) + if err != nil { + t.Fatalf("runPrune() error: %v", err) + } + if res.DeletedSnapshots != 0 || res.DeletedObjects != 0 { + t.Fatalf("dry-run deleted something: %+v", res) + } + if len(res.KeptSnapshots) != 1 || len(res.DeadSnapshots) != 0 { + t.Fatalf("plan = %+v", res) + } + if backend.TotalUploads("") != before || backend.Has("snapshots/7f91c6c7-7a0b-44aa-ae23-997b60e4e998.age") != snapBefore { + t.Fatal("dry-run mutated backend") + } +} + +func TestRunPruneYesDeletesDead(t *testing.T) { + ident, backend, _ := repoFixture(t) + // Two snapshots with different content so second has unique chunks. + f1 := filepath.Join(t.TempDir(), "one.txt") + if err := os.WriteFile(f1, []byte("content-one-unique"), 0644); err != nil { + t.Fatal(err) + } + f2 := filepath.Join(t.TempDir(), "two.txt") + if err := os.WriteFile(f2, []byte("content-two-different"), 0644); err != nil { + t.Fatal(err) + } + if _, err := runBackup(f1, ident, backend, "7f91c6c7-7a0b-44aa-ae23-997b60e4e998", nil); err != nil { + t.Fatal(err) + } + // Distinct RFC3339 timestamps so keep-last is deterministic. + time.Sleep(1100 * time.Millisecond) + if _, err := runBackup(f2, ident, backend, "8f91c6c7-7a0b-44aa-ae23-997b60e4e999", nil); err != nil { + t.Fatal(err) + } + + res, err := runPrune(ident, backend, RetentionPolicy{KeepLast: 1}, true) + if err != nil { + t.Fatalf("runPrune() error: %v", err) + } + if res.DeletedSnapshots != 1 { + t.Fatalf("DeletedSnapshots = %d, want 1", res.DeletedSnapshots) + } + if backend.Has("snapshots/7f91c6c7-7a0b-44aa-ae23-997b60e4e998.age") { + t.Fatal("dead snapshot still present") + } + if !backend.Has("snapshots/8f91c6c7-7a0b-44aa-ae23-997b60e4e999.age") { + t.Fatal("kept snapshot deleted") + } + // Live snapshot must still verify. + if err := runVerify("8f91c6c7-7a0b-44aa-ae23-997b60e4e999", ident, backend); err != nil { + t.Fatalf("verify after prune: %v", err) + } +} diff --git a/internal/flow/retention.go b/internal/flow/retention.go new file mode 100644 index 0000000..b11425e --- /dev/null +++ b/internal/flow/retention.go @@ -0,0 +1,103 @@ +package flow + +import ( + "fmt" + "sort" + "time" +) + +// RetentionPolicy mirrors restic's keep rules. Zero means the rule does +// not apply. +type RetentionPolicy struct { + KeepLast int + KeepDaily int + KeepWeekly int + KeepMonthly int +} + +func (p RetentionPolicy) isEmpty() bool { + return p.KeepLast == 0 && p.KeepDaily == 0 && p.KeepWeekly == 0 && p.KeepMonthly == 0 +} + +// snapshotMeta is the subset of snapshot data retention needs. +type snapshotMeta struct { + UUID string + CreatedAt time.Time +} + +// selectKeepSnapshots returns the UUIDs that survive the retention policy. +// Snapshots are processed newest first. A snapshot kept by any rule stays. +// The input is mutated (sorted) in place. +func selectKeepSnapshots(snaps []snapshotMeta, p RetentionPolicy) map[string]bool { + keep := make(map[string]bool) + + sort.Slice(snaps, func(i, j int) bool { + if snaps[i].CreatedAt.Equal(snaps[j].CreatedAt) { + return snaps[i].UUID < snaps[j].UUID + } + return snaps[i].CreatedAt.After(snaps[j].CreatedAt) + }) + + // keep-last: the N newest. + for i := 0; i < p.KeepLast && i < len(snaps); i++ { + keep[snaps[i].UUID] = true + } + + // keep-daily: newest snapshot of each of the last N calendar days that + // have at least one snapshot. + if p.KeepDaily > 0 { + seen := make(map[string]bool) + count := 0 + for _, s := range snaps { + day := s.CreatedAt.UTC().Format("2006-01-02") + if seen[day] { + continue + } + seen[day] = true + keep[s.UUID] = true + count++ + if count >= p.KeepDaily { + break + } + } + } + + // keep-weekly: newest snapshot of each of the last N ISO weeks. + if p.KeepWeekly > 0 { + seen := make(map[string]bool) + count := 0 + for _, s := range snaps { + year, week := s.CreatedAt.UTC().ISOWeek() + key := fmt.Sprintf("%04d-W%02d", year, week) + if seen[key] { + continue + } + seen[key] = true + keep[s.UUID] = true + count++ + if count >= p.KeepWeekly { + break + } + } + } + + // keep-monthly: newest snapshot of each of the last N calendar months. + if p.KeepMonthly > 0 { + seen := make(map[string]bool) + count := 0 + for _, s := range snaps { + key := s.CreatedAt.UTC().Format("2006-01") + if seen[key] { + continue + } + seen[key] = true + keep[s.UUID] = true + count++ + if count >= p.KeepMonthly { + break + } + } + } + + return keep +} diff --git a/internal/index/index.go b/internal/index/index.go index c5ce1a3..3868787 100644 --- a/internal/index/index.go +++ b/internal/index/index.go @@ -159,6 +159,29 @@ func (ix *Index) Add(entry Entry) { ix.all[entry.PlainSHA256] = entry } +// Compact writes a single new epoch containing only liveHashes (intersected +// with known entries), then returns. Callers delete old epochs separately. +// The epoch counter continues from the highest existing name so the new +// file never collides with a prior epoch. +func (ix *Index) Compact(backend storage.Backend, recipients []string, liveHashes map[string]bool) (int, error) { + // Reuse Save by replacing all and pending with the live subset, then + // force a save even if pending is empty (when there is something to write). + next := New() + next.epoch = ix.epoch + count := 0 + for h, e := range ix.all { + if liveHashes[h] { + next.all[h] = e + next.pending[h] = e + count++ + } + } + if count == 0 { + return 0, nil + } + return next.Save(backend, recipients) +} + // Save writes only the pending entries as the next epoch. It is a no-op when // nothing is pending, so a second backup of unchanged data writes no files. func (ix *Index) Save(backend storage.Backend, recipients []string) (int, error) { diff --git a/internal/storage/rclone.go b/internal/storage/rclone.go index b40b50b..c5b318b 100644 --- a/internal/storage/rclone.go +++ b/internal/storage/rclone.go @@ -121,16 +121,17 @@ func (r *RcloneBackend) ListRecursive(prefix []string) ([]string, error) { return out, nil } -// Delete removes a file or directory at the given remote path using rclone purge. +// Delete removes a single file at the given remote path using rclone +// deletefile. Purge is for directories; deletefile is for files. func (r *RcloneBackend) Delete(remotePath string) error { fullPath := fmt.Sprintf("%s/%s", r.Remote, remotePath) - cmd := exec.Command("rclone", rcloneArgs(r.ConfigPath, "purge", fullPath)...) + cmd := exec.Command("rclone", rcloneArgs(r.ConfigPath, "deletefile", fullPath)...) var stderr bytes.Buffer cmd.Stderr = &stderr if err := cmd.Run(); err != nil { - return fmt.Errorf("failed to delete %s via rclone purge: %w\nRclone Error: %s", fullPath, err, stderr.String()) + return fmt.Errorf("failed to delete %s via rclone deletefile: %w\nRclone Error: %s", fullPath, err, stderr.String()) } return nil diff --git a/main.go b/main.go index 89ff7dd..ada1cd6 100644 --- a/main.go +++ b/main.go @@ -20,6 +20,7 @@ var cli struct { Restore RestoreCmd `cmd:"" help:"Restore and decrypt a backup from storage."` Verify VerifyCmd `cmd:"" help:"Verify backup chunks and checksums."` Snapshots SnapshotsCmd `cmd:"" help:"List snapshots in the repository."` + Prune PruneCmd `cmd:"" help:"Remove dead snapshots and unreferenced chunks (dry run by default)."` Check CheckCmd `cmd:"" help:"Check repository consistency (structural)."` Setup SetupCmd `cmd:"" help:"Download and install rclone to ~/.gloak/bin/."` } diff --git a/main_test.go b/main_test.go index 0245938..87be3be 100644 --- a/main_test.go +++ b/main_test.go @@ -62,6 +62,8 @@ func TestCommandOptionShortTags(t *testing.T) { {name: "verify remote", cmd: VerifyCmd{}, fieldName: "Remote", wantShort: "R"}, {name: "snapshots identity", cmd: SnapshotsCmd{}, fieldName: "Identity", wantShort: "i"}, {name: "snapshots remote", cmd: SnapshotsCmd{}, fieldName: "Remote", wantShort: "R"}, + {name: "prune identity", cmd: PruneCmd{}, fieldName: "Identity", wantShort: "i"}, + {name: "prune remote", cmd: PruneCmd{}, fieldName: "Remote", wantShort: "R"}, {name: "check identity", cmd: CheckCmd{}, fieldName: "Identity", wantShort: "i"}, {name: "check remote", cmd: CheckCmd{}, fieldName: "Remote", wantShort: "R"}, {name: "setup dir", cmd: SetupCmd{}, fieldName: "Dir", wantShort: "D"}, diff --git a/prune.go b/prune.go new file mode 100644 index 0000000..5538a0e --- /dev/null +++ b/prune.go @@ -0,0 +1,50 @@ +package main + +import ( + "path/filepath" + + "github.com/pterm/pterm" + + "gloak/internal/flow" +) + +type PruneCmd struct { + Identity string `help:"Identity private key file (defaults to ~/.gloak/identity.txt)." short:"i"` + Remote string `help:"Source Rclone URI (e.g., keraaaki:backup_folder or /mnt/backups)." required:"" short:"R"` + KeepLast int `help:"Keep the N newest snapshots." default:"0"` + KeepDaily int `help:"Keep the newest snapshot of each of the last N days." default:"0"` + KeepWeekly int `help:"Keep the newest snapshot of each of the last N weeks." default:"0"` + KeepMonthly int `help:"Keep the newest snapshot of each of the last N months." default:"0"` + Yes bool `help:"Actually delete. Without this flag prune is a dry run."` +} + +func (c *PruneCmd) Run() error { + pterm.Info.Println("Starting PRUNE process...") + ident := c.Identity + if ident == "" { + ident = filepath.Join(getDefaultDir(), "identity.txt") + pterm.Info.Printf("Using identity from: %s\n", ident) + } + policy := flow.RetentionPolicy{ + KeepLast: c.KeepLast, + KeepDaily: c.KeepDaily, + KeepWeekly: c.KeepWeekly, + KeepMonthly: c.KeepMonthly, + } + res, err := flow.RunPrune(ident, c.Remote, policy, c.Yes) + if err != nil { + return err + } + pterm.Printf("Kept snapshots: %d\n", len(res.KeptSnapshots)) + pterm.Printf("Dead snapshots: %d\n", len(res.DeadSnapshots)) + pterm.Printf("Live chunks: %d\n", res.LiveChunks) + pterm.Printf("Garbage chunks: %d\n", res.GarbageChunks) + if !c.Yes { + pterm.Warning.Println("Dry run only. Pass --yes to delete.") + return nil + } + pterm.Success.Printf("Deleted %d snapshots and %d objects; compacted index: %v\n", + res.DeletedSnapshots, res.DeletedObjects, res.CompactedEpochs) + notifyTelegram("gloak prune completed") + return nil +} From ea61954121d1251191b3f6b6a97bbbeb41faae52 Mon Sep 17 00:00:00 2001 From: Syrup Date: Mon, 12 Oct 2026 02:59:11 +0700 Subject: [PATCH 34/48] feat: SSH recipients, pq placeholder, telegram notify --- CHANGELOG.md | 5 ++ backup.go | 1 + check.go | 6 ++- go.mod | 1 + go.sum | 2 + init.go | 41 +++++++++----- init_pq_test.go | 28 ++++++++++ internal/crypto/multi.go | 15 +++++- internal/crypto/parse_recipient_test.go | 39 ++++++++++++++ internal/notify/telegram.go | 72 +++++++++++++++++++++++++ internal/notify/telegram_test.go | 61 +++++++++++++++++++++ internal/repo/repo.go | 3 +- notify.go | 20 +++++++ restore.go | 6 ++- snapshots.go | 5 +- verify.go | 6 ++- 16 files changed, 290 insertions(+), 21 deletions(-) create mode 100644 init_pq_test.go create mode 100644 internal/crypto/parse_recipient_test.go create mode 100644 internal/notify/telegram.go create mode 100644 internal/notify/telegram_test.go create mode 100644 notify.go diff --git a/CHANGELOG.md b/CHANGELOG.md index a6944f5..93a1b93 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -14,6 +14,10 @@ - `backup` accepts a directory and writes a tree snapshot: sorted walk, empty directories preserved, symlinks skipped. Snapshot JSON carries `kind: tree` (or `kind: file` for single-file backups). - `gloak snapshots` lists every snapshot with kind, creation time, host, and a short detail string. - `restore` recreates directory trees and accepts `--path ` to restore a subtree only. Tree entry paths are validated against traversal and absolute paths. +- `gloak prune --keep-last N --keep-daily N --keep-weekly N --keep-monthly N` reclaims space with restic-style retention. Dry run by default; `--yes` deletes dead snapshots and unreferenced chunks, then compacts the index into a single epoch. +- `init` and `upload` accept SSH public keys (`ssh-ed25519`, `ssh-rsa`) as recipients alongside age X25519 keys. +- `init --pq` is reserved for post-quantum hybrid recipients and currently returns a clear not-implemented error. +- Optional Telegram notify via `GLOAK_TELEGRAM_TOKEN` / `GLOAK_TELEGRAM_CHAT_ID` after commands; failures warn only. Token is never printed. ### Changed @@ -27,3 +31,4 @@ ### Fixed - The rclone config path is no longer hardcoded. An empty ConfigPath lets rclone use its own default. +- Storage `Delete` uses `rclone deletefile` for single files instead of `purge` (which targets directories). diff --git a/backup.go b/backup.go index 1622ad4..ecaceee 100644 --- a/backup.go +++ b/backup.go @@ -26,6 +26,7 @@ func (c *BackupCmd) Run(ctx *kong.Context) error { if _, err := flow.RunBackup(c.ExistingFile, identityPath, c.Remote); err != nil { return fmt.Errorf("backup failed: %w", err) } + notifyTelegram("gloak backup completed") return nil } diff --git a/check.go b/check.go index 9c61498..902f830 100644 --- a/check.go +++ b/check.go @@ -22,5 +22,9 @@ func (c *CheckCmd) Run() error { pterm.Info.Printf("Using identity from: %s\n", ident) } - return flow.RunCheck(ident, c.Remote) + if err := flow.RunCheck(ident, c.Remote); err != nil { + return err + } + notifyTelegram("gloak check completed") + return nil } diff --git a/go.mod b/go.mod index da01d64..c564cdf 100644 --- a/go.mod +++ b/go.mod @@ -15,6 +15,7 @@ require ( atomicgo.dev/cursor v0.2.0 // indirect atomicgo.dev/keyboard v0.2.9 // indirect atomicgo.dev/schedule v0.1.0 // indirect + filippo.io/edwards25519 v1.1.0 // indirect filippo.io/hpke v0.4.0 // indirect github.com/clipperhouse/uax29/v2 v2.7.0 // indirect github.com/containerd/console v1.0.5 // indirect diff --git a/go.sum b/go.sum index 9a41307..289e182 100644 --- a/go.sum +++ b/go.sum @@ -10,6 +10,8 @@ c2sp.org/CCTV/age v0.0.0-20251208015420-e9274a7bdbfd h1:ZLsPO6WdZ5zatV4UfVpr7oAw c2sp.org/CCTV/age v0.0.0-20251208015420-e9274a7bdbfd/go.mod h1:SrHC2C7r5GkDk8R+NFVzYy/sdj0Ypg9htaPXQq5Cqeo= filippo.io/age v1.3.1 h1:hbzdQOJkuaMEpRCLSN1/C5DX74RPcNCk6oqhKMXmZi0= filippo.io/age v1.3.1/go.mod h1:EZorDTYUxt836i3zdori5IJX/v2Lj6kWFU0cfh6C0D4= +filippo.io/edwards25519 v1.1.0 h1:FNf4tywRC1HmFuKW5xopWpigGjJKiJSV0Cqo0cJWDaA= +filippo.io/edwards25519 v1.1.0/go.mod h1:BxyFTGdWcka3PhytdK4V28tE5sGfRvvvRV7EaN4VDT4= filippo.io/hpke v0.4.0 h1:p575VVQ6ted4pL+it6M00V/f2qTZITO0zgmdKCkd5+A= filippo.io/hpke v0.4.0/go.mod h1:EmAN849/P3qdeK+PCMkDpDm83vRHM5cDipBJ8xbQLVY= github.com/MarvinJWendt/testza v0.1.0/go.mod h1:7AxNvlfeHP7Z/hDQ5JtE3OKYT3XFUeLCDE2DQninSqs= diff --git a/init.go b/init.go index 34f87d7..bb7748a 100644 --- a/init.go +++ b/init.go @@ -17,9 +17,13 @@ type InitCmd struct { ChunkMin int64 `help:"Minimum CDC chunk size in bytes (default 524288)."` ChunkAvg int64 `help:"Average CDC chunk size in bytes (default 1048576)."` ChunkMax int64 `help:"Maximum CDC chunk size in bytes (default 8388608)."` + PQ bool `help:"Reserve post-quantum hybrid recipients (not implemented yet)."` } func (c *InitCmd) Run() error { + if c.PQ { + return fmt.Errorf("--pq hybrid recipients are not implemented yet (reserved for a later phase)") + } recipients, err := resolveInitRecipients(c.Recipient) if err != nil { return err @@ -29,26 +33,35 @@ func (c *InitCmd) Run() error { // resolveInitRecipients turns the --recipient value into a key list. An empty // path falls back to ~/.gloak/recipient.txt. A value that looks like keys -// (starts with age1 or contains commas) is split on commas; anything else is -// treated as a file path with one key per line, skipping blanks and # comments. +// (starts with age1, ssh-ed25519, ssh-rsa, or contains commas) is split on +// commas; anything else is treated as a file path with one key per line, +// skipping blanks and # comments. func resolveInitRecipients(path string) ([]string, error) { if path == "" { path = filepath.Join(getDefaultDir(), "recipient.txt") } - if !strings.HasPrefix(path, "age1") && !strings.Contains(path, ",") { - return readRecipientFile(path) - } - var out []string - for _, part := range strings.Split(path, ",") { - part = strings.TrimSpace(part) - if part != "" { - out = append(out, part) + if looksLikeInlineKeys(path) { + var out []string + // SSH keys contain spaces; split only on commas. + for _, part := range strings.Split(path, ",") { + part = strings.TrimSpace(part) + if part != "" { + out = append(out, part) + } } + if len(out) == 0 { + return nil, fmt.Errorf("no recipients found (run 'gloak keygen' first)") + } + return out, nil } - if len(out) == 0 { - return nil, fmt.Errorf("no recipients found (run 'gloak keygen' first)") - } - return out, nil + return readRecipientFile(path) +} + +func looksLikeInlineKeys(s string) bool { + return strings.HasPrefix(s, "age1") || + strings.HasPrefix(s, "ssh-ed25519 ") || + strings.HasPrefix(s, "ssh-rsa ") || + strings.Contains(s, ",") } func readRecipientFile(path string) ([]string, error) { diff --git a/init_pq_test.go b/init_pq_test.go new file mode 100644 index 0000000..4d4a4eb --- /dev/null +++ b/init_pq_test.go @@ -0,0 +1,28 @@ +package main + +import ( + "strings" + "testing" +) + +func TestInitPQFlagNotImplemented(t *testing.T) { + cmd := InitCmd{Remote: "mem:", Recipient: "age1test", PQ: true} + err := cmd.Run() + if err == nil { + t.Fatal("InitCmd with --pq succeeded") + } + if !strings.Contains(err.Error(), "not implemented") { + t.Fatalf("error = %v", err) + } +} + +func TestResolveInitRecipientsAcceptsSSHEd25519(t *testing.T) { + key := "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIOMqqnkVzrm0SdG6UOoqKLsabgH5C9okWi0dh2l9GKJl test@host" + got, err := resolveInitRecipients(key) + if err != nil { + t.Fatalf("resolveInitRecipients() error: %v", err) + } + if len(got) != 1 || got[0] != key { + t.Fatalf("got = %v", got) + } +} diff --git a/internal/crypto/multi.go b/internal/crypto/multi.go index 85759d4..22a4674 100644 --- a/internal/crypto/multi.go +++ b/internal/crypto/multi.go @@ -5,10 +5,23 @@ import ( "crypto/sha256" "encoding/hex" "fmt" + "strings" "filippo.io/age" + "filippo.io/age/agessh" ) +// ParseRecipient accepts age X25519 keys (age1...) and SSH public keys +// (ssh-ed25519 / ssh-rsa). It is the single entry point for repository +// recipients so init, upload, and sealing all accept the same formats. +func ParseRecipient(s string) (age.Recipient, error) { + s = strings.TrimSpace(s) + if strings.HasPrefix(s, "ssh-ed25519 ") || strings.HasPrefix(s, "ssh-rsa ") { + return agessh.ParseRecipient(s) + } + return age.ParseX25519Recipient(s) +} + // EncryptMulti encrypts plaintext to every recipient. Any matching identity // can decrypt the result. func EncryptMulti(recipientKeys []string, plaintext []byte) ([]byte, error) { @@ -17,7 +30,7 @@ func EncryptMulti(recipientKeys []string, plaintext []byte) ([]byte, error) { } recipients := make([]age.Recipient, 0, len(recipientKeys)) for _, key := range recipientKeys { - r, err := age.ParseX25519Recipient(key) + r, err := ParseRecipient(key) if err != nil { return nil, fmt.Errorf("invalid recipient %q: %w", key, err) } diff --git a/internal/crypto/parse_recipient_test.go b/internal/crypto/parse_recipient_test.go new file mode 100644 index 0000000..6a57611 --- /dev/null +++ b/internal/crypto/parse_recipient_test.go @@ -0,0 +1,39 @@ +package crypto + +import ( + "testing" + + "filippo.io/age" +) + +func TestParseRecipientAcceptsX25519(t *testing.T) { + pub, _, _, err := GenerateKey() + if err != nil { + t.Fatal(err) + } + r, err := ParseRecipient(pub) + if err != nil { + t.Fatalf("ParseRecipient() error: %v", err) + } + if _, ok := r.(*age.X25519Recipient); !ok { + t.Fatalf("type = %T", r) + } +} + +func TestParseRecipientAcceptsSSHEd25519(t *testing.T) { + // A well-formed ssh-ed25519 public key line (RFC 8709). + key := "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIOMqqnkVzrm0SdG6UOoqKLsabgH5C9okWi0dh2l9GKJl test@host" + r, err := ParseRecipient(key) + if err != nil { + t.Fatalf("ParseRecipient() error: %v", err) + } + if r == nil { + t.Fatal("nil recipient") + } +} + +func TestParseRecipientRejectsGarbage(t *testing.T) { + if _, err := ParseRecipient("not-a-key"); err == nil { + t.Fatal("ParseRecipient() accepted garbage") + } +} diff --git a/internal/notify/telegram.go b/internal/notify/telegram.go new file mode 100644 index 0000000..e79bb70 --- /dev/null +++ b/internal/notify/telegram.go @@ -0,0 +1,72 @@ +package notify + +import ( + "encoding/json" + "fmt" + "net/http" + "net/url" + "os" + "strings" + "time" +) + +// TelegramConfig holds Bot API credentials. Empty token disables notify. +type TelegramConfig struct { + Token string + ChatID string + // BaseURL overrides the Bot API root for tests. + BaseURL string +} + +// FromEnv reads GLOAK_TELEGRAM_TOKEN and GLOAK_TELEGRAM_CHAT_ID. +func FromEnv() TelegramConfig { + return TelegramConfig{ + Token: os.Getenv("GLOAK_TELEGRAM_TOKEN"), + ChatID: os.Getenv("GLOAK_TELEGRAM_CHAT_ID"), + } +} + +// Enabled reports whether notify should attempt a send. +func (c TelegramConfig) Enabled() bool { + return c.Token != "" && c.ChatID != "" +} + +// Send posts a short text report. It never returns an error that should +// fail the host command; callers log a warning. +func (c TelegramConfig) Send(message string) error { + if !c.Enabled() { + return nil + } + base := c.BaseURL + if base == "" { + base = "https://api.telegram.org" + } + endpoint := fmt.Sprintf("%s/bot%s/sendMessage", base, c.Token) + + form := url.Values{} + form.Set("chat_id", c.ChatID) + form.Set("text", truncate(message, 4000)) + + client := &http.Client{Timeout: 15 * time.Second} + resp, err := client.Post(endpoint, "application/x-www-form-urlencoded", strings.NewReader(form.Encode())) + if err != nil { + return fmt.Errorf("telegram send: %w", err) + } + defer resp.Body.Close() + + if resp.StatusCode != http.StatusOK { + var body struct { + Description string `json:"description"` + } + _ = json.NewDecoder(resp.Body).Decode(&body) + return fmt.Errorf("telegram send: status %d: %s", resp.StatusCode, body.Description) + } + return nil +} + +func truncate(s string, n int) string { + if len(s) <= n { + return s + } + return s[:n-3] + "..." +} diff --git a/internal/notify/telegram_test.go b/internal/notify/telegram_test.go new file mode 100644 index 0000000..4855e1e --- /dev/null +++ b/internal/notify/telegram_test.go @@ -0,0 +1,61 @@ +package notify + +import ( + "io" + "net/http" + "net/http/httptest" + "net/url" + "strings" + "testing" +) + +func TestSendDisabledNoops(t *testing.T) { + c := TelegramConfig{} + if c.Enabled() { + t.Fatal("empty config should be disabled") + } + if err := c.Send("hello"); err != nil { + t.Fatalf("Send() error: %v", err) + } +} + +func TestSendPostsMessage(t *testing.T) { + var gotBody string + var gotPath string + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + gotPath = r.URL.Path + b, _ := io.ReadAll(r.Body) + gotBody = string(b) + w.WriteHeader(http.StatusOK) + _, _ = w.Write([]byte(`{"ok":true}`)) + })) + defer srv.Close() + + c := TelegramConfig{Token: "TESTTOKEN", ChatID: "42", BaseURL: srv.URL} + if err := c.Send("backup ok"); err != nil { + t.Fatalf("Send() error: %v", err) + } + if !strings.Contains(gotPath, "botTESTTOKEN/sendMessage") { + t.Fatalf("path = %q", gotPath) + } + vals, err := url.ParseQuery(gotBody) + if err != nil { + t.Fatal(err) + } + if vals.Get("chat_id") != "42" || vals.Get("text") != "backup ok" { + t.Fatalf("body = %q", gotBody) + } +} + +func TestSendHTTPError(t *testing.T) { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusBadRequest) + _, _ = w.Write([]byte(`{"description":"bad request"}`)) + })) + defer srv.Close() + + c := TelegramConfig{Token: "T", ChatID: "1", BaseURL: srv.URL} + if err := c.Send("x"); err == nil { + t.Fatal("Send() succeeded on HTTP error") + } +} diff --git a/internal/repo/repo.go b/internal/repo/repo.go index 03c4d7d..81eca4d 100644 --- a/internal/repo/repo.go +++ b/internal/repo/repo.go @@ -8,7 +8,6 @@ import ( "io" "time" - "filippo.io/age" "github.com/google/uuid" "gloak/internal/crypto" @@ -56,7 +55,7 @@ func Init(backend storage.Backend, recipients []string, min, avg, max int64, for return nil, Config{}, fmt.Errorf("at least one recipient is required") } for _, r := range recipients { - if _, err := age.ParseX25519Recipient(r); err != nil { + if _, err := crypto.ParseRecipient(r); err != nil { return nil, Config{}, fmt.Errorf("invalid recipient %q: %w", r, err) } } diff --git a/notify.go b/notify.go new file mode 100644 index 0000000..6dd6a1d --- /dev/null +++ b/notify.go @@ -0,0 +1,20 @@ +package main + +import ( + "github.com/pterm/pterm" + + "gloak/internal/notify" +) + +// notifyTelegram sends a short best-effort report after a command. Missing +// credentials or network failures only print a warning; they never fail the +// host command. The token itself is never printed. +func notifyTelegram(message string) { + cfg := notify.FromEnv() + if !cfg.Enabled() { + return + } + if err := cfg.Send(message); err != nil { + pterm.Warning.Printf("telegram notify failed: %v\n", err) + } +} diff --git a/restore.go b/restore.go index 46ab547..a1c41ca 100644 --- a/restore.go +++ b/restore.go @@ -25,5 +25,9 @@ func (r *RestoreCmd) Run() error { pterm.Info.Printf("Using identity from: %s\n", ident) } - return flow.RunRestore(r.UUID, ident, r.Remote, r.OutputDir, r.PathFilter) + if err := flow.RunRestore(r.UUID, ident, r.Remote, r.OutputDir, r.PathFilter); err != nil { + return err + } + notifyTelegram("gloak restore completed") + return nil } diff --git a/snapshots.go b/snapshots.go index 83e372a..cb1c232 100644 --- a/snapshots.go +++ b/snapshots.go @@ -19,5 +19,8 @@ func (c *SnapshotsCmd) Run() error { if ident == "" { ident = filepath.Join(getDefaultDir(), "identity.txt") } - return flow.RunSnapshots(ident, c.Remote) + if err := flow.RunSnapshots(ident, c.Remote); err != nil { + return err + } + return nil } diff --git a/verify.go b/verify.go index a4980b4..ab7f781 100644 --- a/verify.go +++ b/verify.go @@ -23,5 +23,9 @@ func (v *VerifyCmd) Run() error { pterm.Info.Printf("Using identity from: %s\n", ident) } - return flow.RunVerify(v.UUID, ident, v.Remote) + if err := flow.RunVerify(v.UUID, ident, v.Remote); err != nil { + return err + } + notifyTelegram("gloak verify completed") + return nil } From 492435087ec27ae638d582d475d689ee81d68231 Mon Sep 17 00:00:00 2001 From: Syrup Date: Mon, 12 Oct 2026 03:20:29 +0700 Subject: [PATCH 35/48] docs: add everyday workflow guide for end users Plain-language walkthrough of keygen, init, backup, snapshots, verify, restore, prune, and Telegram notify. Assumes nothing beyond a working gloak binary and an rclone remote. --- docs/WORKFLOW.md | 133 +++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 133 insertions(+) create mode 100644 docs/WORKFLOW.md diff --git a/docs/WORKFLOW.md b/docs/WORKFLOW.md new file mode 100644 index 0000000..844dcba --- /dev/null +++ b/docs/WORKFLOW.md @@ -0,0 +1,133 @@ +# Everyday gloak workflows + +This guide walks through the tasks most people actually do with a backup tool. Commands assume `gloak` is in your PATH. Every command that talks to storage needs `-R` (where backups live) and usually `-i` (your private key). + +> [!WARNING] +> gloak is alpha. Keep another copy of anything critical somewhere else. + +## One-time setup + +Before the first backup, generate a key pair and create a repository. + +```bash +gloak keygen +gloak init -R myremote:backups -r ~/.gloak/recipient.txt +``` + +`keygen` writes two files under `~/.gloak/`: + +- `identity.txt` is your private key. Treat it like a password. Anyone who has it can read every backup. +- `recipient.txt` is your public key. Safe to share; it only lets someone encrypt to you. + +`init` creates the storage layout on your remote. You only run it once per remote location. + +Prefer SSH keys for recipients? Pass the public key line directly: + +```bash +gloak init -R myremote:backups -r "ssh-ed25519 AAAA... you@laptop" +``` + +## Back up a folder + +Point `backup` at a file or a directory. Directories are walked recursively; empty folders are kept; symlinks are skipped. + +```bash +gloak backup ~/Documents -R myremote:backups -i ~/.gloak/identity.txt +``` + +The command prints a snapshot ID when it finishes. Save it. That ID is how you list, verify, or restore this backup later. + +Backup of the same data twice uploads almost nothing the second time. Chunks that already exist on the remote are skipped automatically. + +## See what you have backed up + +```bash +gloak snapshots -R myremote:backups -i ~/.gloak/identity.txt +``` + +You get a table like: + +```text +UUID KIND CREATED HOST DETAIL +aaaaaaaa-bbbb-4ccc-8ddd-000000000000 tree 2026-10-12T10:00:00Z laptop 12 files from Documents +``` + +`KIND` is `tree` for folders and `file` for a single-file backup. + +## Check a backup before you trust it + +Two levels: + +```bash +# quick structural check: every snapshot opens, every chunk is present +gloak check -R myremote:backups -i ~/.gloak/identity.txt + +# deep check of one snapshot: downloads and decrypts every chunk +gloak verify -R myremote:backups -i ~/.gloak/identity.txt +``` + +Run `check` occasionally. Run `verify` after a big backup, or before you delete the original files. + +## Restore + +Full restore of a folder snapshot: + +```bash +gloak restore -R myremote:backups -i ~/.gloak/identity.txt -o ./restored +``` + +Files land under `./restored//`. To pull only one subfolder: + +```bash +gloak restore -R myremote:backups -i ~/.gloak/identity.txt -o ./restored --path subfolder +``` + +Single-file snapshots restore as one file under the output directory. + +## Clean up old backups + +Snapshots pile up. `prune` removes old ones and deletes chunks nothing needs anymore. + +```bash +# dry run: show what would be deleted, change nothing +gloak prune -R myremote:backups -i ~/.gloak/identity.txt --keep-last 7 --keep-daily 7 --keep-weekly 4 + +# actually delete +gloak prune -R myremote:backups -i ~/.gloak/identity.txt --keep-last 7 --keep-daily 7 --keep-weekly 4 --yes +``` + +How the keep flags work: + +- `--keep-last N` keeps the N newest snapshots. +- `--keep-daily N` keeps the newest snapshot of each of the last N days that have one. +- `--keep-weekly N` and `--keep-monthly N` work the same way for weeks and months. + +At least one keep flag is required. Without `--yes`, prune only prints the plan. + +## Optional: notify on Telegram + +Set two environment variables and gloak sends a short message after backup, restore, verify, check, or prune: + +```bash +export GLOAK_TELEGRAM_TOKEN="123456:ABC..." +export GLOAK_TELEGRAM_CHAT_ID="your-chat-id" +``` + +A notify failure only prints a warning. It never fails the backup itself. The token is never printed. + +## If something goes wrong + +- Wrong or missing identity: `identity cannot open this repository`. Confirm you are using the key that matches a recipient listed at init time. +- Snapshot not found: check the ID with `gloak snapshots`. +- Corruption during restore or verify: the error starts with `CHUNK CORRUPT`. The bad chunk is named. Keep the original data if you still have it, re-run backup, then prune the damaged snapshot. +- Storage errors: confirm `rclone` is installed and can reach your remote. `gloak setup` installs rclone into `~/.gloak/bin/` if you need it. + +## Typical weekly routine + +```bash +gloak backup ~/Documents -R myremote:backups -i ~/.gloak/identity.txt +gloak check -R myremote:backups -i ~/.gloak/identity.txt +gloak prune -R myremote:backups -i ~/.gloak/identity.txt --keep-last 7 --keep-daily 7 --yes +``` + +That is backup, sanity check, and cleanup in three commands. From 8a21a2946df57b163740631d8269ada677a30363 Mon Sep 17 00:00:00 2001 From: Syrup Date: Mon, 12 Oct 2026 03:55:07 +0700 Subject: [PATCH 36/48] feat(notify): enrich telegram messages with path, snapshot, and duration Notify after backup, restore, verify, check, and prune now sends a short multi-line status report instead of a bare "completed" string. Prune includes kept, dead, and garbage chunk counts. Dry-run prune still does not notify. Also drop a competitor name from the prune changelog entry and show the richer format in the workflow guide. --- CHANGELOG.md | 4 ++-- backup.go | 11 +++++++++-- check.go | 7 ++++++- docs/WORKFLOW.md | 11 ++++++++++- notify.go | 28 +++++++++++++++++++++++++--- prune.go | 12 +++++++++++- restore.go | 12 +++++++++++- verify.go | 7 ++++++- 8 files changed, 80 insertions(+), 12 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 93a1b93..f7614ad 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -14,10 +14,10 @@ - `backup` accepts a directory and writes a tree snapshot: sorted walk, empty directories preserved, symlinks skipped. Snapshot JSON carries `kind: tree` (or `kind: file` for single-file backups). - `gloak snapshots` lists every snapshot with kind, creation time, host, and a short detail string. - `restore` recreates directory trees and accepts `--path ` to restore a subtree only. Tree entry paths are validated against traversal and absolute paths. -- `gloak prune --keep-last N --keep-daily N --keep-weekly N --keep-monthly N` reclaims space with restic-style retention. Dry run by default; `--yes` deletes dead snapshots and unreferenced chunks, then compacts the index into a single epoch. +- `gloak prune --keep-last N --keep-daily N --keep-weekly N --keep-monthly N` reclaims space with time-based retention. Dry run by default; `--yes` deletes dead snapshots and unreferenced chunks, then compacts the index into a single epoch. - `init` and `upload` accept SSH public keys (`ssh-ed25519`, `ssh-rsa`) as recipients alongside age X25519 keys. - `init --pq` is reserved for post-quantum hybrid recipients and currently returns a clear not-implemented error. -- Optional Telegram notify via `GLOAK_TELEGRAM_TOKEN` / `GLOAK_TELEGRAM_CHAT_ID` after commands; failures warn only. Token is never printed. +- Optional Telegram notify via `GLOAK_TELEGRAM_TOKEN` / `GLOAK_TELEGRAM_CHAT_ID` after commands; the message includes path or snapshot, useful counts, and duration. Failures warn only. Token is never printed. ### Changed diff --git a/backup.go b/backup.go index ecaceee..7a55138 100644 --- a/backup.go +++ b/backup.go @@ -5,6 +5,7 @@ import ( "os" "path/filepath" "strings" + "time" "github.com/alecthomas/kong" @@ -23,10 +24,16 @@ func (c *BackupCmd) Run(ctx *kong.Context) error { if err != nil { return err } - if _, err := flow.RunBackup(c.ExistingFile, identityPath, c.Remote); err != nil { + start := time.Now() + snapshotID, err := flow.RunBackup(c.ExistingFile, identityPath, c.Remote) + if err != nil { return fmt.Errorf("backup failed: %w", err) } - notifyTelegram("gloak backup completed") + notifyCommand("backup", + "Path: "+c.ExistingFile, + "Snapshot: "+snapshotID, + "Duration: "+formatDuration(time.Since(start)), + ) return nil } diff --git a/check.go b/check.go index 902f830..ddc74a6 100644 --- a/check.go +++ b/check.go @@ -2,6 +2,7 @@ package main import ( "path/filepath" + "time" "github.com/pterm/pterm" @@ -22,9 +23,13 @@ func (c *CheckCmd) Run() error { pterm.Info.Printf("Using identity from: %s\n", ident) } + start := time.Now() if err := flow.RunCheck(ident, c.Remote); err != nil { return err } - notifyTelegram("gloak check completed") + notifyCommand("check", + "Remote: "+c.Remote, + "Duration: "+formatDuration(time.Since(start)), + ) return nil } diff --git a/docs/WORKFLOW.md b/docs/WORKFLOW.md index 844dcba..2a1f11a 100644 --- a/docs/WORKFLOW.md +++ b/docs/WORKFLOW.md @@ -106,13 +106,22 @@ At least one keep flag is required. Without `--yes`, prune only prints the plan. ## Optional: notify on Telegram -Set two environment variables and gloak sends a short message after backup, restore, verify, check, or prune: +Set two environment variables and gloak sends a short status report after backup, restore, verify, check, or prune. The message includes the path or snapshot, useful counts, and how long the command took: ```bash export GLOAK_TELEGRAM_TOKEN="123456:ABC..." export GLOAK_TELEGRAM_CHAT_ID="your-chat-id" ``` +Example message after a backup: + +```text +gloak backup completed +Path: /home/you/Documents +Snapshot: 550e8400-e29b-41d4-a716-446655440000 +Duration: 42s +``` + A notify failure only prints a warning. It never fails the backup itself. The token is never printed. ## If something goes wrong diff --git a/notify.go b/notify.go index 6dd6a1d..91d0aa6 100644 --- a/notify.go +++ b/notify.go @@ -1,20 +1,42 @@ package main import ( + "fmt" + "strings" + "time" + "github.com/pterm/pterm" "gloak/internal/notify" ) -// notifyTelegram sends a short best-effort report after a command. Missing +// notifyCommand sends a short best-effort report after a successful command. +// details are optional lines rendered as-is under the header. Missing // credentials or network failures only print a warning; they never fail the // host command. The token itself is never printed. -func notifyTelegram(message string) { +func notifyCommand(command string, details ...string) { cfg := notify.FromEnv() if !cfg.Enabled() { return } - if err := cfg.Send(message); err != nil { + var b strings.Builder + fmt.Fprintf(&b, "gloak %s completed", command) + for _, d := range details { + if d == "" { + continue + } + b.WriteByte('\n') + b.WriteString(d) + } + if err := cfg.Send(b.String()); err != nil { pterm.Warning.Printf("telegram notify failed: %v\n", err) } } + +// formatDuration renders d in a short human form. +func formatDuration(d time.Duration) string { + if d < time.Second { + return d.Round(time.Millisecond).String() + } + return d.Round(time.Second).String() +} diff --git a/prune.go b/prune.go index 5538a0e..6ba9079 100644 --- a/prune.go +++ b/prune.go @@ -1,7 +1,9 @@ package main import ( + "fmt" "path/filepath" + "time" "github.com/pterm/pterm" @@ -31,6 +33,7 @@ func (c *PruneCmd) Run() error { KeepWeekly: c.KeepWeekly, KeepMonthly: c.KeepMonthly, } + start := time.Now() res, err := flow.RunPrune(ident, c.Remote, policy, c.Yes) if err != nil { return err @@ -45,6 +48,13 @@ func (c *PruneCmd) Run() error { } pterm.Success.Printf("Deleted %d snapshots and %d objects; compacted index: %v\n", res.DeletedSnapshots, res.DeletedObjects, res.CompactedEpochs) - notifyTelegram("gloak prune completed") + notifyCommand("prune", + fmt.Sprintf("Kept: %d snapshots", len(res.KeptSnapshots)), + fmt.Sprintf("Dead: %d snapshots", len(res.DeadSnapshots)), + fmt.Sprintf("Live chunks: %d", res.LiveChunks), + fmt.Sprintf("Garbage chunks: %d", res.GarbageChunks), + fmt.Sprintf("Deleted: %d snapshots, %d objects", res.DeletedSnapshots, res.DeletedObjects), + "Duration: "+formatDuration(time.Since(start)), + ) return nil } diff --git a/restore.go b/restore.go index a1c41ca..8878d4d 100644 --- a/restore.go +++ b/restore.go @@ -2,6 +2,7 @@ package main import ( "path/filepath" + "time" "github.com/pterm/pterm" @@ -25,9 +26,18 @@ func (r *RestoreCmd) Run() error { pterm.Info.Printf("Using identity from: %s\n", ident) } + start := time.Now() if err := flow.RunRestore(r.UUID, ident, r.Remote, r.OutputDir, r.PathFilter); err != nil { return err } - notifyTelegram("gloak restore completed") + details := []string{ + "Snapshot: " + r.UUID, + "Output: " + r.OutputDir, + } + if r.PathFilter != "" { + details = append(details, "Path filter: "+r.PathFilter) + } + details = append(details, "Duration: "+formatDuration(time.Since(start))) + notifyCommand("restore", details...) return nil } diff --git a/verify.go b/verify.go index ab7f781..05a70b5 100644 --- a/verify.go +++ b/verify.go @@ -2,6 +2,7 @@ package main import ( "path/filepath" + "time" "github.com/pterm/pterm" @@ -23,9 +24,13 @@ func (v *VerifyCmd) Run() error { pterm.Info.Printf("Using identity from: %s\n", ident) } + start := time.Now() if err := flow.RunVerify(v.UUID, ident, v.Remote); err != nil { return err } - notifyTelegram("gloak verify completed") + notifyCommand("verify", + "Snapshot: "+v.UUID, + "Duration: "+formatDuration(time.Since(start)), + ) return nil } From 08dd93a4ac29acd2bf6b0a16190fdf8d37fa4419 Mon Sep 17 00:00:00 2001 From: Syrup Date: Mon, 12 Oct 2026 20:35:36 +0700 Subject: [PATCH 37/48] feat(cli): honor GLOAK_DIR for the default key directory getDefaultDir now returns GLOAK_DIR when set (with ~ expanded), so keygen, init, backup, restore, verify, snapshots, prune, check, and setup all resolve keys from one override. backup no longer hardcodes ~/.gloak/identity.txt as a Kong default; it falls back like the other commands. Document GLOAK_DIR in the workflow guide and changelog. --- CHANGELOG.md | 1 + backup.go | 8 ++++++-- docs/WORKFLOW.md | 6 ++++++ main.go | 9 +++++++++ main_test.go | 35 +++++++++++++++++++++++++++++++++++ 5 files changed, 57 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index f7614ad..dd3750d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -18,6 +18,7 @@ - `init` and `upload` accept SSH public keys (`ssh-ed25519`, `ssh-rsa`) as recipients alongside age X25519 keys. - `init --pq` is reserved for post-quantum hybrid recipients and currently returns a clear not-implemented error. - Optional Telegram notify via `GLOAK_TELEGRAM_TOKEN` / `GLOAK_TELEGRAM_CHAT_ID` after commands; the message includes path or snapshot, useful counts, and duration. Failures warn only. Token is never printed. +- `GLOAK_DIR` overrides the default key directory (`~/.gloak`) for every command that resolves a default path: `keygen`, `init`, `backup`, `restore`, `verify`, `snapshots`, `prune`, `check`, and `setup`. A leading `~` is expanded. ### Changed diff --git a/backup.go b/backup.go index 7a55138..eaf2149 100644 --- a/backup.go +++ b/backup.go @@ -15,12 +15,16 @@ import ( // BackupCmd backs up a file or directory into a gloak repository. type BackupCmd struct { ExistingFile string `arg:"" name:"path" help:"File or directory to back up"` - Identity string `short:"i" default:"~/.gloak/identity.txt" help:"Path to identity file"` + Identity string `short:"i" help:"Path to identity file (defaults to $GLOAK_DIR/identity.txt, or ~/.gloak/identity.txt)."` Remote string `short:"R" required:"" help:"Repository remote, e.g. b2:my-bucket/gloak"` } func (c *BackupCmd) Run(ctx *kong.Context) error { - identityPath, err := expandHome(c.Identity) + identityPath := c.Identity + if identityPath == "" { + identityPath = filepath.Join(getDefaultDir(), "identity.txt") + } + identityPath, err := expandHome(identityPath) if err != nil { return err } diff --git a/docs/WORKFLOW.md b/docs/WORKFLOW.md index 2a1f11a..e83d7a3 100644 --- a/docs/WORKFLOW.md +++ b/docs/WORKFLOW.md @@ -21,6 +21,12 @@ gloak init -R myremote:backups -r ~/.gloak/recipient.txt `init` creates the storage layout on your remote. You only run it once per remote location. +By default keys live under `~/.gloak`. Set `GLOAK_DIR` (or pass `-o` to `keygen`) to keep them somewhere else. A leading `~` in `GLOAK_DIR` is expanded: + +```bash +export GLOAK_DIR=~/backups/gloak-keys +``` + Prefer SSH keys for recipients? Pass the public key line directly: ```bash diff --git a/main.go b/main.go index ada1cd6..8b52a15 100644 --- a/main.go +++ b/main.go @@ -25,7 +25,16 @@ var cli struct { Setup SetupCmd `cmd:"" help:"Download and install rclone to ~/.gloak/bin/."` } +// getDefaultDir returns the directory for identity, recipient, and helper +// files. GLOAK_DIR overrides the default (~/.gloak). A leading ~ in +// GLOAK_DIR is expanded. func getDefaultDir() string { + if dir := os.Getenv("GLOAK_DIR"); dir != "" { + if expanded, err := expandHome(dir); err == nil { + return expanded + } + return dir + } home, err := os.UserHomeDir() if err != nil { home = "." diff --git a/main_test.go b/main_test.go index 87be3be..d909073 100644 --- a/main_test.go +++ b/main_test.go @@ -166,3 +166,38 @@ func TestResolveInitRecipientsMissingFileErrors(t *testing.T) { t.Fatalf("resolveInitRecipients() error = %q, want gloak keygen mention", err) } } + +func TestGetDefaultDirUsesGLOAK_DIR(t *testing.T) { + dir := filepath.Join(t.TempDir(), "custom-gloak") + t.Setenv("GLOAK_DIR", dir) + + if got := getDefaultDir(); got != dir { + t.Fatalf("getDefaultDir() = %q, want %q", got, dir) + } +} + +func TestGetDefaultDirExpandsTildeInGLOAK_DIR(t *testing.T) { + home, err := os.UserHomeDir() + if err != nil { + t.Skipf("no home dir: %v", err) + } + t.Setenv("GLOAK_DIR", "~/custom-gloak") + + want := filepath.Join(home, "custom-gloak") + if got := getDefaultDir(); got != want { + t.Fatalf("getDefaultDir() = %q, want %q", got, want) + } +} + +func TestGetDefaultDirFallsBackToHomeWhenUnset(t *testing.T) { + t.Setenv("GLOAK_DIR", "") + + home, err := os.UserHomeDir() + if err != nil { + t.Skipf("no home dir: %v", err) + } + want := filepath.Join(home, ".gloak") + if got := getDefaultDir(); got != want { + t.Fatalf("getDefaultDir() = %q, want %q", got, want) + } +} From 7a44cfb2e1f5a2874a673ecb2edae1a43fdb80c0 Mon Sep 17 00:00:00 2001 From: Syrup Date: Mon, 12 Oct 2026 21:38:01 +0700 Subject: [PATCH 38/48] feat(cli): resolve spinners with a success line Stop the spinner via pterm Success so the final frame is replaced by the SUCCESS prefix instead of a bare stop. Applies to check, verify, init, backup, and upload. --- internal/flow/backup.go | 4 +--- internal/flow/check.go | 3 +-- internal/flow/init.go | 3 +-- internal/flow/upload.go | 3 +-- internal/flow/verify.go | 3 +-- 5 files changed, 5 insertions(+), 11 deletions(-) diff --git a/internal/flow/backup.go b/internal/flow/backup.go index 114b176..e42ba30 100644 --- a/internal/flow/backup.go +++ b/internal/flow/backup.go @@ -44,9 +44,7 @@ func RunBackup(filePath, identityPath, remote string) (string, error) { _ = spinner.Stop() return "", err } - _ = spinner.Stop() - - pterm.Success.Println("Backup complete") + spinner.Success("Backup complete") pterm.Printf("Snapshot ID: %s\n", res.SnapshotID) pterm.Printf("Chunks: %d total, %d uploaded, %d skipped\n", res.TotalChunks, res.UploadedChunks, res.SkippedChunks) pterm.Printf("Data size: %d bytes\n", res.TotalPlainBytes) diff --git a/internal/flow/check.go b/internal/flow/check.go index 5f69e39..9f80989 100644 --- a/internal/flow/check.go +++ b/internal/flow/check.go @@ -32,8 +32,7 @@ func RunCheck(identityPath, remote string) error { _ = spinner.Stop() return err } - _ = spinner.Stop() - pterm.Success.Println("Repository check passed") + spinner.Success("Repository check passed") pterm.Printf("Snapshots: %d\n", res.Snapshots) pterm.Printf("Referenced chunks: %d\n", res.Referenced) pterm.Printf("Data objects: %d\n", res.DataObjects) diff --git a/internal/flow/init.go b/internal/flow/init.go index 45f5563..44fa799 100644 --- a/internal/flow/init.go +++ b/internal/flow/init.go @@ -28,8 +28,7 @@ func RunInit(recipients []string, remote string, min, avg, max int64, force bool _ = spinner.Stop() return err } - _ = spinner.Stop() - pterm.Success.Println("Repository initialized") + spinner.Success("Repository initialized") pterm.Printf("Location: %s\n", remote) pterm.Printf("Recipients: %v\n", cfg.Recipients) pterm.Printf("Chunk bounds: min=%d avg=%d max=%d bytes\n", cfg.ChunkMin, cfg.ChunkAvg, cfg.ChunkMax) diff --git a/internal/flow/upload.go b/internal/flow/upload.go index 839103c..f874101 100644 --- a/internal/flow/upload.go +++ b/internal/flow/upload.go @@ -157,7 +157,6 @@ func RunUpload(filePath string, recipients []string, backend storage.Backend) (s return "", fmt.Errorf("failed to upload locator: %w", err) } - _ = spinner.Stop() - pterm.Success.Printf("Upload complete: %s (%d chunks, %d bytes)\n", backupID, len(metas), total) + spinner.Success(fmt.Sprintf("Upload complete: %s (%d chunks, %d bytes)", backupID, len(metas), total)) return backupID, nil } diff --git a/internal/flow/verify.go b/internal/flow/verify.go index dd788be..eee4572 100644 --- a/internal/flow/verify.go +++ b/internal/flow/verify.go @@ -22,8 +22,7 @@ func RunVerify(snapshotID, identityPath, remote string) error { _ = spinner.Stop() return err } - _ = spinner.Stop() - pterm.Success.Println("Backup verified successfully") + spinner.Success("Backup verified successfully") return nil } From d63c42fbcc9ece258e669c6348e3e1c6c2561a49 Mon Sep 17 00:00:00 2001 From: Syrup Date: Mon, 12 Oct 2026 21:59:25 +0700 Subject: [PATCH 39/48] feat(config): load telegram credentials from config.toml Add internal/config with a TOML loader for /config.toml. Notify credentials now come from the [telegram] section, and GLOAK_TELEGRAM_TOKEN / GLOAK_TELEGRAM_CHAT_ID override the file when set. A missing config file stays silent; a malformed one warns and notify keeps its best-effort behavior. Uses github.com/BurntSushi/toml so the config can grow sections later without another format change. --- CHANGELOG.md | 2 +- docs/WORKFLOW.md | 10 ++++++- go.mod | 1 + go.sum | 2 ++ internal/config/config.go | 47 ++++++++++++++++++++++++++++++++ internal/config/config_test.go | 50 ++++++++++++++++++++++++++++++++++ notify.go | 26 +++++++++++++++++- notify_test.go | 49 +++++++++++++++++++++++++++++++++ 8 files changed, 184 insertions(+), 3 deletions(-) create mode 100644 internal/config/config.go create mode 100644 internal/config/config_test.go create mode 100644 notify_test.go diff --git a/CHANGELOG.md b/CHANGELOG.md index dd3750d..fff11e1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -17,7 +17,7 @@ - `gloak prune --keep-last N --keep-daily N --keep-weekly N --keep-monthly N` reclaims space with time-based retention. Dry run by default; `--yes` deletes dead snapshots and unreferenced chunks, then compacts the index into a single epoch. - `init` and `upload` accept SSH public keys (`ssh-ed25519`, `ssh-rsa`) as recipients alongside age X25519 keys. - `init --pq` is reserved for post-quantum hybrid recipients and currently returns a clear not-implemented error. -- Optional Telegram notify via `GLOAK_TELEGRAM_TOKEN` / `GLOAK_TELEGRAM_CHAT_ID` after commands; the message includes path or snapshot, useful counts, and duration. Failures warn only. Token is never printed. +- Optional Telegram notify after commands; the message includes path or snapshot, useful counts, and duration. Credentials come from `[telegram]` in `/config.toml` (TOML), with `GLOAK_TELEGRAM_TOKEN` / `GLOAK_TELEGRAM_CHAT_ID` overriding the file when set. Failures warn only. Token is never printed. - `GLOAK_DIR` overrides the default key directory (`~/.gloak`) for every command that resolves a default path: `keygen`, `init`, `backup`, `restore`, `verify`, `snapshots`, `prune`, `check`, and `setup`. A leading `~` is expanded. ### Changed diff --git a/docs/WORKFLOW.md b/docs/WORKFLOW.md index e83d7a3..12ce059 100644 --- a/docs/WORKFLOW.md +++ b/docs/WORKFLOW.md @@ -112,7 +112,15 @@ At least one keep flag is required. Without `--yes`, prune only prints the plan. ## Optional: notify on Telegram -Set two environment variables and gloak sends a short status report after backup, restore, verify, check, or prune. The message includes the path or snapshot, useful counts, and how long the command took: +gloak sends a short status report after backup, restore, verify, check, or prune. Put the credentials in `~/.gloak/config.toml` (or `$GLOAK_DIR/config.toml`): + +```toml +[telegram] +token = "123456:ABC..." +chat_id = "your-chat-id" +``` + +Environment variables override the file when set, which is handy for containers or one-off runs: ```bash export GLOAK_TELEGRAM_TOKEN="123456:ABC..." diff --git a/go.mod b/go.mod index c564cdf..d02494a 100644 --- a/go.mod +++ b/go.mod @@ -4,6 +4,7 @@ go 1.26.3 require ( filippo.io/age v1.3.1 + github.com/BurntSushi/toml v1.6.0 github.com/alecthomas/kong v1.15.0 github.com/google/uuid v1.6.0 github.com/klauspost/compress v1.20.1 diff --git a/go.sum b/go.sum index 289e182..3343a9e 100644 --- a/go.sum +++ b/go.sum @@ -14,6 +14,8 @@ filippo.io/edwards25519 v1.1.0 h1:FNf4tywRC1HmFuKW5xopWpigGjJKiJSV0Cqo0cJWDaA= filippo.io/edwards25519 v1.1.0/go.mod h1:BxyFTGdWcka3PhytdK4V28tE5sGfRvvvRV7EaN4VDT4= filippo.io/hpke v0.4.0 h1:p575VVQ6ted4pL+it6M00V/f2qTZITO0zgmdKCkd5+A= filippo.io/hpke v0.4.0/go.mod h1:EmAN849/P3qdeK+PCMkDpDm83vRHM5cDipBJ8xbQLVY= +github.com/BurntSushi/toml v1.6.0 h1:dRaEfpa2VI55EwlIW72hMRHdWouJeRF7TPYhI+AUQjk= +github.com/BurntSushi/toml v1.6.0/go.mod h1:ukJfTF/6rtPPRCnwkur4qwRxa8vTRFBF0uk2lLoLwho= github.com/MarvinJWendt/testza v0.1.0/go.mod h1:7AxNvlfeHP7Z/hDQ5JtE3OKYT3XFUeLCDE2DQninSqs= github.com/MarvinJWendt/testza v0.2.1/go.mod h1:God7bhG8n6uQxwdScay+gjm9/LnO4D3kkcZX4hv9Rp8= github.com/MarvinJWendt/testza v0.2.8/go.mod h1:nwIcjmr0Zz+Rcwfh3/4UhBp7ePKVhuBExvZqnKYWlII= diff --git a/internal/config/config.go b/internal/config/config.go new file mode 100644 index 0000000..704d6d9 --- /dev/null +++ b/internal/config/config.go @@ -0,0 +1,47 @@ +// Package config loads the optional user configuration file. +// +// The file lives at /config.toml (GLOAK_DIR or ~/.gloak). A +// missing file is not an error; every value stays empty and callers fall +// back to their defaults. +package config + +import ( + "errors" + "os" + "path/filepath" + + "github.com/BurntSushi/toml" +) + +// Telegram holds Bot API credentials for optional notifications. +type Telegram struct { + Token string `toml:"token"` + ChatID string `toml:"chat_id"` +} + +// Config is the user configuration file. +type Config struct { + Telegram Telegram `toml:"telegram"` +} + +// Path returns the config file location for the given gloak directory. +func Path(dir string) string { + return filepath.Join(dir, "config.toml") +} + +// Load reads path and parses it. A missing file yields an empty Config and +// a nil error. A malformed file returns a descriptive error. +func Load(path string) (Config, error) { + var cfg Config + data, err := os.ReadFile(path) + if err != nil { + if errors.Is(err, os.ErrNotExist) { + return cfg, nil + } + return cfg, err + } + if err := toml.Unmarshal(data, &cfg); err != nil { + return Config{}, err + } + return cfg, nil +} diff --git a/internal/config/config_test.go b/internal/config/config_test.go new file mode 100644 index 0000000..dcbec66 --- /dev/null +++ b/internal/config/config_test.go @@ -0,0 +1,50 @@ +package config + +import ( + "os" + "path/filepath" + "testing" +) + +func TestLoadMissingFileIsEmpty(t *testing.T) { + cfg, err := Load(filepath.Join(t.TempDir(), "config.toml")) + if err != nil { + t.Fatalf("Load() error: %v", err) + } + if cfg.Telegram.Token != "" || cfg.Telegram.ChatID != "" { + t.Fatalf("cfg = %+v, want zero value", cfg) + } +} + +func TestLoadReadsTelegramSection(t *testing.T) { + path := filepath.Join(t.TempDir(), "config.toml") + content := "# comment\n[telegram]\ntoken = \"abc\"\nchat_id = \"42\"\n" + if err := os.WriteFile(path, []byte(content), 0600); err != nil { + t.Fatal(err) + } + + cfg, err := Load(path) + if err != nil { + t.Fatalf("Load() error: %v", err) + } + if cfg.Telegram.Token != "abc" || cfg.Telegram.ChatID != "42" { + t.Fatalf("cfg = %+v", cfg) + } +} + +func TestLoadMalformedFileErrors(t *testing.T) { + path := filepath.Join(t.TempDir(), "config.toml") + if err := os.WriteFile(path, []byte("[telegram\ntoken="), 0600); err != nil { + t.Fatal(err) + } + + if _, err := Load(path); err == nil { + t.Fatal("Load() succeeded on malformed file") + } +} + +func TestPathUsesDir(t *testing.T) { + if got := Path("/x/gloak"); got != "/x/gloak/config.toml" { + t.Fatalf("Path() = %q", got) + } +} diff --git a/notify.go b/notify.go index 91d0aa6..62e593e 100644 --- a/notify.go +++ b/notify.go @@ -2,20 +2,44 @@ package main import ( "fmt" + "os" "strings" "time" "github.com/pterm/pterm" + "gloak/internal/config" "gloak/internal/notify" ) +// resolveTelegram merges Telegram credentials from the config file with +// environment overrides. The file at /config.toml provides the +// base values; GLOAK_TELEGRAM_TOKEN and GLOAK_TELEGRAM_CHAT_ID win when set. +// A malformed config file only warns; notify stays best-effort. +func resolveTelegram() notify.TelegramConfig { + fileCfg, err := config.Load(config.Path(getDefaultDir())) + if err != nil { + pterm.Warning.Printf("ignoring config file: %v\n", err) + } + cfg := notify.TelegramConfig{ + Token: fileCfg.Telegram.Token, + ChatID: fileCfg.Telegram.ChatID, + } + if v := os.Getenv("GLOAK_TELEGRAM_TOKEN"); v != "" { + cfg.Token = v + } + if v := os.Getenv("GLOAK_TELEGRAM_CHAT_ID"); v != "" { + cfg.ChatID = v + } + return cfg +} + // notifyCommand sends a short best-effort report after a successful command. // details are optional lines rendered as-is under the header. Missing // credentials or network failures only print a warning; they never fail the // host command. The token itself is never printed. func notifyCommand(command string, details ...string) { - cfg := notify.FromEnv() + cfg := resolveTelegram() if !cfg.Enabled() { return } diff --git a/notify_test.go b/notify_test.go new file mode 100644 index 0000000..5be3458 --- /dev/null +++ b/notify_test.go @@ -0,0 +1,49 @@ +package main + +import ( + "os" + "path/filepath" + "testing" +) + +func writeConfig(t *testing.T, content string) { + t.Helper() + dir := t.TempDir() + t.Setenv("GLOAK_DIR", dir) + if err := os.WriteFile(filepath.Join(dir, "config.toml"), []byte(content), 0600); err != nil { + t.Fatal(err) + } +} + +func TestResolveTelegramFromFile(t *testing.T) { + writeConfig(t, "[telegram]\ntoken = \"file-token\"\nchat_id = \"file-chat\"\n") + t.Setenv("GLOAK_TELEGRAM_TOKEN", "") + t.Setenv("GLOAK_TELEGRAM_CHAT_ID", "") + + cfg := resolveTelegram() + if cfg.Token != "file-token" || cfg.ChatID != "file-chat" { + t.Fatalf("cfg = %+v", cfg) + } +} + +func TestResolveTelegramEnvOverridesFile(t *testing.T) { + writeConfig(t, "[telegram]\ntoken = \"file-token\"\nchat_id = \"file-chat\"\n") + t.Setenv("GLOAK_TELEGRAM_TOKEN", "env-token") + t.Setenv("GLOAK_TELEGRAM_CHAT_ID", "env-chat") + + cfg := resolveTelegram() + if cfg.Token != "env-token" || cfg.ChatID != "env-chat" { + t.Fatalf("cfg = %+v, want env values", cfg) + } +} + +func TestResolveTelegramEmptyWithoutConfigOrEnv(t *testing.T) { + t.Setenv("GLOAK_DIR", t.TempDir()) + t.Setenv("GLOAK_TELEGRAM_TOKEN", "") + t.Setenv("GLOAK_TELEGRAM_CHAT_ID", "") + + cfg := resolveTelegram() + if cfg.Enabled() { + t.Fatalf("cfg = %+v, want disabled", cfg) + } +} From 3fddf60226d422339f1c2fcd50bc0323aa7a3c48 Mon Sep 17 00:00:00 2001 From: Syrup Date: Mon, 12 Oct 2026 22:04:16 +0700 Subject: [PATCH 40/48] build: release with GoReleaser on version tags Add .goreleaser.yaml (linux/darwin/windows, amd64/arm64, tar.gz/zip, checksums, changelog filtered to feat/fix/perf entries) and a release workflow that runs goreleaser on v* tag pushes. Drop CHANGELOG.md in favor of generated GitHub Release notes; the old file remains reachable in git history. Nothing is pushed from here. --- .github/workflows/release.yml | 29 +++++++++++++++++++++++++ .goreleaser.yaml | 41 +++++++++++++++++++++++++++++++++++ CHANGELOG.md | 35 ------------------------------ 3 files changed, 70 insertions(+), 35 deletions(-) create mode 100644 .github/workflows/release.yml create mode 100644 .goreleaser.yaml delete mode 100644 CHANGELOG.md diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml new file mode 100644 index 0000000..a3a3ac1 --- /dev/null +++ b/.github/workflows/release.yml @@ -0,0 +1,29 @@ +name: release + +on: + push: + tags: + - "v*" + +permissions: + contents: write + +jobs: + goreleaser: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + + - uses: actions/setup-go@v5 + with: + go-version-file: go.mod + + - uses: goreleaser/goreleaser-action@v7 + with: + distribution: goreleaser + version: "~> v2" + args: release --clean + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} diff --git a/.goreleaser.yaml b/.goreleaser.yaml new file mode 100644 index 0000000..e95babf --- /dev/null +++ b/.goreleaser.yaml @@ -0,0 +1,41 @@ +# yaml-language-server: $schema=https://goreleaser.com/static/schema.json +version: 2 + +project_name: gloak + +builds: + - env: + - CGO_ENABLED=0 + goos: + - linux + - darwin + - windows + goarch: + - amd64 + - arm64 + ldflags: + - -s -w + +archives: + - format: tar.gz + name_template: "{{ .ProjectName }}_{{ .Version }}_{{ .Os }}_{{ .Arch }}" + format_overrides: + - goos: windows + format: zip + +checksum: + name_template: checksums.txt + +changelog: + sort: asc + filters: + exclude: + - "^docs:" + - "^test:" + - "^chore:" + - "^style:" + - "^ci:" + - "^build:" + - "^refactor:" + - "^Merge pull request" + - "^Merge branch" diff --git a/CHANGELOG.md b/CHANGELOG.md deleted file mode 100644 index fff11e1..0000000 --- a/CHANGELOG.md +++ /dev/null @@ -1,35 +0,0 @@ -# Changelog - -## Unreleased - -### Added - -- `gloak init` creates a repository: a wrapped master key per recipient under `keys/`, a `repo.json` manifest, format 2.0. -- `gloak backup` uploads content-addressed encrypted chunks under `data/{hh}/`, records them in an encrypted epoch index under `index/`, and writes a snapshot under `snapshots/`. Repeat backups skip chunks already in the index. Chunks are sealed with zstd and then XChaCha20-Poly1305 under HKDF-derived keys. -- Snapshots are encrypted to every recipient listed at init time. Any matching identity can verify or restore. -- `gloak upload -R ` writes a self-contained backup folder `{uuid}/` at the remote root (sealed chunks, multi-recipient `manifest.age` with embedded master key, public `locator.json`). No repository and no `init` required. Manifest version `file-2`. -- `restore` and `verify` auto-detect file-mode folders by `locator.json` and fall back to repository layout otherwise. -- `backup` splits files with content-defined chunking (gear hash). Chunk bounds `chunk_min`/`chunk_avg`/`chunk_max` are stored in `repo.json` at init; flags `--chunk-min`, `--chunk-avg`, `--chunk-max` override the defaults (512 KiB / 1 MiB / 8 MiB). A repeat backup of unchanged content uploads nothing. -- `gloak check` reports structural consistency: every snapshot decrypts, every referenced chunk exists in `data/`, and unreferenced objects are listed as garbage. It does not download chunk payloads; use `verify` for a deep pass. -- `backup` accepts a directory and writes a tree snapshot: sorted walk, empty directories preserved, symlinks skipped. Snapshot JSON carries `kind: tree` (or `kind: file` for single-file backups). -- `gloak snapshots` lists every snapshot with kind, creation time, host, and a short detail string. -- `restore` recreates directory trees and accepts `--path ` to restore a subtree only. Tree entry paths are validated against traversal and absolute paths. -- `gloak prune --keep-last N --keep-daily N --keep-weekly N --keep-monthly N` reclaims space with time-based retention. Dry run by default; `--yes` deletes dead snapshots and unreferenced chunks, then compacts the index into a single epoch. -- `init` and `upload` accept SSH public keys (`ssh-ed25519`, `ssh-rsa`) as recipients alongside age X25519 keys. -- `init --pq` is reserved for post-quantum hybrid recipients and currently returns a clear not-implemented error. -- Optional Telegram notify after commands; the message includes path or snapshot, useful counts, and duration. Credentials come from `[telegram]` in `/config.toml` (TOML), with `GLOAK_TELEGRAM_TOKEN` / `GLOAK_TELEGRAM_CHAT_ID` overriding the file when set. Failures warn only. Token is never printed. -- `GLOAK_DIR` overrides the default key directory (`~/.gloak`) for every command that resolves a default path: `keygen`, `init`, `backup`, `restore`, `verify`, `snapshots`, `prune`, `check`, and `setup`. A leading `~` is expanded. - -### Changed - -- `verify` and `restore` read the repository layout. Corruption errors keep the CHUNK CORRUPT prefix. -- Repository backup no longer uses fixed 20 MiB chunks. Phase 1 repositories with `chunk_size` in `repo.json` cannot be opened; re-run `gloak init`. - -### Removed - -- The `resume` and `cleanup` commands and the old filesystem-v1 layout (`{uuid}/chunk_00000` with age-writer chunks, version `1.0`). There is no migration path. Backups made with earlier versions cannot be read by this version. The new `upload` command is a separate format (`file-2`) using the repository chunk envelope. - -### Fixed - -- The rclone config path is no longer hardcoded. An empty ConfigPath lets rclone use its own default. -- Storage `Delete` uses `rclone deletefile` for single files instead of `purge` (which targets directories). From 814f352986f402f27cf82920184667c35a156948 Mon Sep 17 00:00:00 2001 From: Syrup Date: Tue, 13 Oct 2026 01:12:21 +0700 Subject: [PATCH 41/48] fix: address review findings from PR #4 - treat rclone exit code 3 (missing path) as an empty listing in check, snapshots, prune, and index loading instead of a hard error - guard backup and prune with an advisory lock under the repo locks/ folder so concurrent runs cannot step on each other - restore: validate the snapshot root name, keep files inside the destination directory, and apply recorded directory modes after the files are written - resolve the backup root with EvalSymlinks before walking it - prune: reject negative retention counts and stop compacting the index when listing or deleting an epoch fails - redact the Telegram token from transport errors and status output - remove stale age keys when init re-runs after a failed first attempt - error out when a --path-filter matches nothing in a snapshot - stop tracking docs/superpowers (files stay locally, ignored by *.md) - fix docs/WORKFLOW.md examples: GLOAK_DIR before keygen, drop redundant -i flags, use --path-filter, and correct the corruption advice --- docs/WORKFLOW.md | 34 +++---- internal/flow/backup.go | 8 ++ internal/flow/check.go | 11 ++- internal/flow/listerror_test.go | 145 +++++++++++++++++++++++++++++ internal/flow/lock.go | 96 +++++++++++++++++++ internal/flow/lock_test.go | 138 +++++++++++++++++++++++++++ internal/flow/prune.go | 53 ++++++++--- internal/flow/prune_test.go | 64 +++++++++++++ internal/flow/restore.go | 70 +++++++++++++- internal/flow/snapshots.go | 8 +- internal/flow/tree_backup.go | 14 ++- internal/flow/tree_backup_test.go | 37 ++++++++ internal/flow/tree_restore_test.go | 143 ++++++++++++++++++++++++++++ internal/index/index.go | 8 +- internal/index/index_test.go | 46 +++++++++ internal/notify/telegram.go | 13 ++- internal/notify/telegram_test.go | 20 ++++ internal/repo/repo.go | 42 +++++++++ internal/repo/repo_test.go | 32 +++++++ internal/storage/rclone.go | 13 +++ internal/storage/rclone_test.go | 30 +++++- 21 files changed, 980 insertions(+), 45 deletions(-) create mode 100644 internal/flow/listerror_test.go create mode 100644 internal/flow/lock.go create mode 100644 internal/flow/lock_test.go diff --git a/docs/WORKFLOW.md b/docs/WORKFLOW.md index 12ce059..96ca1e0 100644 --- a/docs/WORKFLOW.md +++ b/docs/WORKFLOW.md @@ -1,6 +1,6 @@ # Everyday gloak workflows -This guide walks through the tasks most people actually do with a backup tool. Commands assume `gloak` is in your PATH. Every command that talks to storage needs `-R` (where backups live) and usually `-i` (your private key). +This guide walks through the tasks most people actually do with a backup tool. Commands assume `gloak` is in your PATH. Every command that talks to storage needs `-R` (where backups live). The identity key is read from `$GLOAK_DIR/identity.txt` (default `~/.gloak/identity.txt`), or pass `-i` to use a different one. > [!WARNING] > gloak is alpha. Keep another copy of anything critical somewhere else. @@ -11,20 +11,22 @@ Before the first backup, generate a key pair and create a repository. ```bash gloak keygen -gloak init -R myremote:backups -r ~/.gloak/recipient.txt +gloak init -R myremote:backups ``` -`keygen` writes two files under `~/.gloak/`: +`keygen` writes two files into the gloak directory (`~/.gloak` unless `GLOAK_DIR` is set): - `identity.txt` is your private key. Treat it like a password. Anyone who has it can read every backup. - `recipient.txt` is your public key. Safe to share; it only lets someone encrypt to you. `init` creates the storage layout on your remote. You only run it once per remote location. -By default keys live under `~/.gloak`. Set `GLOAK_DIR` (or pass `-o` to `keygen`) to keep them somewhere else. A leading `~` in `GLOAK_DIR` is expanded: +By default keys live under `~/.gloak`. Set `GLOAK_DIR` before you run `keygen` to keep them somewhere else; a leading `~` in `GLOAK_DIR` is expanded: ```bash export GLOAK_DIR=~/backups/gloak-keys +gloak keygen +gloak init -R myremote:backups ``` Prefer SSH keys for recipients? Pass the public key line directly: @@ -38,7 +40,7 @@ gloak init -R myremote:backups -r "ssh-ed25519 AAAA... you@laptop" Point `backup` at a file or a directory. Directories are walked recursively; empty folders are kept; symlinks are skipped. ```bash -gloak backup ~/Documents -R myremote:backups -i ~/.gloak/identity.txt +gloak backup ~/Documents -R myremote:backups ``` The command prints a snapshot ID when it finishes. Save it. That ID is how you list, verify, or restore this backup later. @@ -48,7 +50,7 @@ Backup of the same data twice uploads almost nothing the second time. Chunks tha ## See what you have backed up ```bash -gloak snapshots -R myremote:backups -i ~/.gloak/identity.txt +gloak snapshots -R myremote:backups ``` You get a table like: @@ -66,10 +68,10 @@ Two levels: ```bash # quick structural check: every snapshot opens, every chunk is present -gloak check -R myremote:backups -i ~/.gloak/identity.txt +gloak check -R myremote:backups # deep check of one snapshot: downloads and decrypts every chunk -gloak verify -R myremote:backups -i ~/.gloak/identity.txt +gloak verify -R myremote:backups ``` Run `check` occasionally. Run `verify` after a big backup, or before you delete the original files. @@ -79,13 +81,13 @@ Run `check` occasionally. Run `verify` after a big backup, or before you delete Full restore of a folder snapshot: ```bash -gloak restore -R myremote:backups -i ~/.gloak/identity.txt -o ./restored +gloak restore -R myremote:backups -o ./restored ``` Files land under `./restored//`. To pull only one subfolder: ```bash -gloak restore -R myremote:backups -i ~/.gloak/identity.txt -o ./restored --path subfolder +gloak restore -R myremote:backups -o ./restored --path-filter subfolder ``` Single-file snapshots restore as one file under the output directory. @@ -96,10 +98,10 @@ Snapshots pile up. `prune` removes old ones and deletes chunks nothing needs any ```bash # dry run: show what would be deleted, change nothing -gloak prune -R myremote:backups -i ~/.gloak/identity.txt --keep-last 7 --keep-daily 7 --keep-weekly 4 +gloak prune -R myremote:backups --keep-last 7 --keep-daily 7 --keep-weekly 4 # actually delete -gloak prune -R myremote:backups -i ~/.gloak/identity.txt --keep-last 7 --keep-daily 7 --keep-weekly 4 --yes +gloak prune -R myremote:backups --keep-last 7 --keep-daily 7 --keep-weekly 4 --yes ``` How the keep flags work: @@ -142,15 +144,15 @@ A notify failure only prints a warning. It never fails the backup itself. The to - Wrong or missing identity: `identity cannot open this repository`. Confirm you are using the key that matches a recipient listed at init time. - Snapshot not found: check the ID with `gloak snapshots`. -- Corruption during restore or verify: the error starts with `CHUNK CORRUPT`. The bad chunk is named. Keep the original data if you still have it, re-run backup, then prune the damaged snapshot. +- Corruption during restore or verify: the error starts with `CHUNK CORRUPT` and names the bad chunk. gloak cannot repair a damaged object in place, and re-running backup will not replace it, because backup skips chunks that are already in the index. Restore from another copy of the data instead, and do not prune while `check` or `verify` still fails. - Storage errors: confirm `rclone` is installed and can reach your remote. `gloak setup` installs rclone into `~/.gloak/bin/` if you need it. ## Typical weekly routine ```bash -gloak backup ~/Documents -R myremote:backups -i ~/.gloak/identity.txt -gloak check -R myremote:backups -i ~/.gloak/identity.txt -gloak prune -R myremote:backups -i ~/.gloak/identity.txt --keep-last 7 --keep-daily 7 --yes +gloak backup ~/Documents -R myremote:backups +gloak check -R myremote:backups +gloak prune -R myremote:backups --keep-last 7 --keep-daily 7 --yes ``` That is backup, sanity check, and cleanup in three commands. diff --git a/internal/flow/backup.go b/internal/flow/backup.go index e42ba30..fe00bd6 100644 --- a/internal/flow/backup.go +++ b/internal/flow/backup.go @@ -61,6 +61,14 @@ func runBackupPath(filePath, identityPath string, backend storage.Backend, snaps if err != nil { return BackupResult{}, fmt.Errorf("failed to stat path: %w", err) } + // Advisory shared lock: a concurrent exclusive prune must not run + // while this backup writes. + release, err := acquireRepoLock(backend, "backup", true) + if err != nil { + return BackupResult{}, err + } + defer release() + if info.IsDir() { return runTreeBackup(filePath, identityPath, backend, snapshotID, progress) } diff --git a/internal/flow/check.go b/internal/flow/check.go index 9f80989..ebc941d 100644 --- a/internal/flow/check.go +++ b/internal/flow/check.go @@ -55,7 +55,11 @@ func runCheck(identityPath string, backend storage.Backend) (CheckResult, error) snapRaw, err := backend.ListFiles([]string{repo.SnapshotsPrefix}) if err != nil { - // A fresh repo has no snapshots/ directory yet; treat as empty. + // A fresh repo has no snapshots/ directory yet; only a confirmed + // missing path may be treated as empty, other failures are real. + if !storage.IsMissingPath(err) { + return res, fmt.Errorf("failed to list snapshots: %w", err) + } snapRaw = []byte("[]") } snapEntries, err := storage.ParseList(snapRaw) @@ -93,7 +97,10 @@ func runCheck(identityPath string, backend storage.Backend) (CheckResult, error) dataPaths, err := backend.ListRecursive([]string{repo.DataPrefix}) if err != nil { - return res, fmt.Errorf("failed to list data objects: %w", err) + if !storage.IsMissingPath(err) { + return res, fmt.Errorf("failed to list data objects: %w", err) + } + dataPaths = nil } present := make(map[string]bool, len(dataPaths)) for _, p := range dataPaths { diff --git a/internal/flow/listerror_test.go b/internal/flow/listerror_test.go new file mode 100644 index 0000000..4d01e13 --- /dev/null +++ b/internal/flow/listerror_test.go @@ -0,0 +1,145 @@ +package flow + +import ( + "fmt" + "os/exec" + "strings" + "testing" + + "gloak/internal/repo" + "gloak/internal/storage" +) + +// listFailBackend wraps a backend and fails listings for selected prefixes, +// so error handling in check, snapshots, and prune can be exercised. +type listFailBackend struct { + storage.Backend + filesErr map[string]error + recurErr map[string]error +} + +func (b *listFailBackend) ListFiles(prefix []string) ([]byte, error) { + if err, ok := b.filesErr[strings.Join(prefix, "/")]; ok { + return nil, err + } + return b.Backend.ListFiles(prefix) +} + +func (b *listFailBackend) ListRecursive(prefix []string) ([]string, error) { + if err, ok := b.recurErr[strings.Join(prefix, "/")]; ok { + return nil, err + } + return b.Backend.ListRecursive(prefix) +} + +// missingPathErr builds the wrapped exit-code-3 error rclone produces for a +// directory that does not exist. +func missingPathErr(t *testing.T) error { + t.Helper() + err := exec.Command("sh", "-c", "exit 3").Run() + if err == nil { + t.Fatal("sh exited 0, want exit 3") + } + return fmt.Errorf("failed to list via rclone lsjson: %w", err) +} + +func TestRunCheckTreatsMissingSnapshotsDirAsEmpty(t *testing.T) { + ident, backend, _ := repoFixture(t) + wrapped := &listFailBackend{ + Backend: backend, + filesErr: map[string]error{repo.SnapshotsPrefix: missingPathErr(t)}, + } + if _, err := runCheck(ident, wrapped); err != nil { + t.Fatalf("runCheck() on fresh repo error: %v", err) + } +} + +func TestRunCheckPropagatesSnapshotListError(t *testing.T) { + ident, backend, _ := repoFixture(t) + wrapped := &listFailBackend{ + Backend: backend, + filesErr: map[string]error{repo.SnapshotsPrefix: fmt.Errorf("remote exploded")}, + } + _, err := runCheck(ident, wrapped) + if err == nil { + t.Fatal("runCheck() succeeded despite a listing failure") + } + if !strings.Contains(err.Error(), "remote exploded") { + t.Fatalf("runCheck() error = %v, want it to wrap the listing failure", err) + } +} + +func TestRunCheckPropagatesDataListError(t *testing.T) { + ident, backend, _ := repoFixture(t) + wrapped := &listFailBackend{ + Backend: backend, + recurErr: map[string]error{repo.DataPrefix: fmt.Errorf("remote exploded")}, + } + _, err := runCheck(ident, wrapped) + if err == nil { + t.Fatal("runCheck() succeeded despite a data listing failure") + } + if !strings.Contains(err.Error(), "remote exploded") { + t.Fatalf("runCheck() error = %v, want it to wrap the listing failure", err) + } +} + +func TestRunSnapshotsPropagatesListError(t *testing.T) { + ident, backend, _ := repoFixture(t) + wrapped := &listFailBackend{ + Backend: backend, + filesErr: map[string]error{repo.SnapshotsPrefix: fmt.Errorf("remote exploded")}, + } + _, err := runSnapshots(ident, wrapped) + if err == nil { + t.Fatal("runSnapshots() succeeded despite a listing failure") + } + if !strings.Contains(err.Error(), "remote exploded") { + t.Fatalf("runSnapshots() error = %v, want it to wrap the listing failure", err) + } +} + +func TestRunSnapshotsTreatsMissingDirAsEmpty(t *testing.T) { + ident, backend, _ := repoFixture(t) + wrapped := &listFailBackend{ + Backend: backend, + filesErr: map[string]error{repo.SnapshotsPrefix: missingPathErr(t)}, + } + infos, err := runSnapshots(ident, wrapped) + if err != nil { + t.Fatalf("runSnapshots() on fresh repo error: %v", err) + } + if len(infos) != 0 { + t.Fatalf("runSnapshots() = %v, want empty", infos) + } +} + +func TestRunPrunePropagatesSnapshotListError(t *testing.T) { + ident, backend, _ := repoFixture(t) + wrapped := &listFailBackend{ + Backend: backend, + filesErr: map[string]error{repo.SnapshotsPrefix: fmt.Errorf("remote exploded")}, + } + _, err := runPrune(ident, wrapped, RetentionPolicy{KeepLast: 1}, false) + if err == nil { + t.Fatal("runPrune() succeeded despite a listing failure") + } + if !strings.Contains(err.Error(), "remote exploded") { + t.Fatalf("runPrune() error = %v, want it to wrap the listing failure", err) + } +} + +func TestRunPrunePropagatesDataListError(t *testing.T) { + ident, backend, _ := repoFixture(t) + wrapped := &listFailBackend{ + Backend: backend, + recurErr: map[string]error{repo.DataPrefix: fmt.Errorf("remote exploded")}, + } + _, err := runPrune(ident, wrapped, RetentionPolicy{KeepLast: 1}, false) + if err == nil { + t.Fatal("runPrune() succeeded despite a data listing failure") + } + if !strings.Contains(err.Error(), "remote exploded") { + t.Fatalf("runPrune() error = %v, want it to wrap the listing failure", err) + } +} diff --git a/internal/flow/lock.go b/internal/flow/lock.go new file mode 100644 index 0000000..b569a1a --- /dev/null +++ b/internal/flow/lock.go @@ -0,0 +1,96 @@ +package flow + +import ( + "bytes" + "encoding/json" + "fmt" + "os" + "time" + + "gloak/internal/core" + "gloak/internal/storage" +) + +// Repository locks coordinate separate gloak processes through files under +// locks/ in the repository. Backups take a shared lock (several backups may +// run at once); prune takes an exclusive lock. The check-then-create +// sequence is not atomic on every backend, so the lock is advisory: it +// stops accidental overlap between backup and prune, not a determined race. + +const lockPrefix = "locks" + +// lockStaleAfter is how long a lock file may exist before other operations +// ignore it. A run that holds its lock longer than this can be overlapped. +const lockStaleAfter = 24 * time.Hour + +type repoLock struct { + Op string `json:"op"` + Hostname string `json:"hostname,omitempty"` + PID int `json:"pid,omitempty"` + CreatedAt string `json:"created_at"` +} + +// acquireRepoLock takes a lock on the repository. A shared acquire (backup) +// fails only while a live non-backup lock exists; an exclusive acquire +// (prune) fails while any live lock exists. Stale locks from processes that +// never released them are removed. The returned function releases the lock +// and is safe to call once. +func acquireRepoLock(backend storage.Backend, op string, shared bool) (func(), error) { + raw, err := backend.ListFiles([]string{lockPrefix}) + if err != nil { + if !storage.IsMissingPath(err) { + return nil, fmt.Errorf("failed to list repository locks: %w", err) + } + raw = []byte("[]") + } + entries, err := storage.ParseList(raw) + if err != nil { + return nil, fmt.Errorf("failed to parse lock listing: %w", err) + } + + now := time.Now().UTC() + for _, entry := range entries { + if entry.IsDir { + continue + } + name := lockPrefix + "/" + entry.Path + var buf bytes.Buffer + if err := backend.Download(name, &buf); err != nil { + return nil, fmt.Errorf("failed to read lock %s: %w", name, err) + } + var held repoLock + if err := json.Unmarshal(buf.Bytes(), &held); err != nil { + return nil, fmt.Errorf("unreadable lock file %s", name) + } + created, err := time.Parse(time.RFC3339, held.CreatedAt) + if err != nil { + return nil, fmt.Errorf("unreadable lock file %s", name) + } + if now.Sub(created) < lockStaleAfter { + if !shared || held.Op != "backup" { + return nil, fmt.Errorf("repository is busy: %s lock held by %s since %s", + held.Op, held.Hostname, held.CreatedAt) + } + continue + } + // The lock is stale; its holder never released it. Clearing it + // may race with another process doing the same, which is harmless. + _ = backend.Delete(name) + } + + host, _ := os.Hostname() + payload, err := json.Marshal(repoLock{ + Op: op, + Hostname: host, + PID: os.Getpid(), + CreatedAt: now.Format(time.RFC3339), + }) + if err != nil { + return nil, fmt.Errorf("failed to marshal lock: %w", err) + } + name := fmt.Sprintf("%s/%s-%s.json", lockPrefix, op, core.GenerateUUID()) + if err := backend.Upload(name, bytes.NewReader(payload)); err != nil { + return nil, fmt.Errorf("failed to create %s lock: %w", op, err) + } + return func() { _ = backend.Delete(name) }, nil +} diff --git a/internal/flow/lock_test.go b/internal/flow/lock_test.go new file mode 100644 index 0000000..9a54b51 --- /dev/null +++ b/internal/flow/lock_test.go @@ -0,0 +1,138 @@ +package flow + +import ( + "encoding/json" + "fmt" + "strings" + "testing" + "time" + + "gloak/internal/storage/memtest" +) + +func putLock(t *testing.T, backend *memtest.MemoryBackend, name string, lk repoLock) { + t.Helper() + raw, err := json.Marshal(lk) + if err != nil { + t.Fatal(err) + } + backend.Put("locks/"+name, raw) +} + +func freshLock(op string) repoLock { + return repoLock{ + Op: op, + Hostname: "test-host", + CreatedAt: time.Now().UTC().Format(time.RFC3339), + } +} + +func TestAcquireRepoLockSharedAllowsOtherBackups(t *testing.T) { + backend := memtest.NewMemoryBackend() + release1, err := acquireRepoLock(backend, "backup", true) + if err != nil { + t.Fatalf("first shared acquire: %v", err) + } + release2, err := acquireRepoLock(backend, "backup", true) + if err != nil { + t.Fatalf("second shared acquire: %v", err) + } + release2() + release1() + if got := len(backend.KeysWithPrefix("locks/")); got != 0 { + t.Fatalf("locks left behind: %d", got) + } +} + +func TestAcquireRepoLockExclusiveBlocksShared(t *testing.T) { + backend := memtest.NewMemoryBackend() + release, err := acquireRepoLock(backend, "prune", false) + if err != nil { + t.Fatalf("exclusive acquire: %v", err) + } + defer release() + + if _, err := acquireRepoLock(backend, "backup", true); err == nil { + t.Fatal("shared acquire succeeded while an exclusive lock is held") + } else if !strings.Contains(err.Error(), "busy") { + t.Fatalf("error = %v, want it to report a busy repository", err) + } +} + +func TestAcquireRepoLockSharedBlocksExclusive(t *testing.T) { + backend := memtest.NewMemoryBackend() + release, err := acquireRepoLock(backend, "backup", true) + if err != nil { + t.Fatalf("shared acquire: %v", err) + } + defer release() + + if _, err := acquireRepoLock(backend, "prune", false); err == nil { + t.Fatal("exclusive acquire succeeded while a backup lock is held") + } +} + +func TestAcquireRepoLockBreaksStaleLock(t *testing.T) { + backend := memtest.NewMemoryBackend() + stale := freshLock("prune") + stale.CreatedAt = time.Now().UTC().Add(-lockStaleAfter - time.Hour).Format(time.RFC3339) + putLock(t, backend, "prune-stale.json", stale) + + release, err := acquireRepoLock(backend, "backup", true) + if err != nil { + t.Fatalf("shared acquire with stale lock present: %v", err) + } + release() + keys := backend.KeysWithPrefix("locks/") + if len(keys) != 0 { + t.Fatalf("stale lock not cleaned up: %v", keys) + } +} + +func TestRunPruneApplyBlockedByBackupLock(t *testing.T) { + ident, backend, _ := repoFixture(t) + putLock(t, backend, "backup-live.json", freshLock("backup")) + + _, err := runPrune(ident, backend, RetentionPolicy{KeepLast: 1}, true) + if err == nil { + t.Fatal("runPrune(apply) succeeded while a backup lock is held") + } + if !strings.Contains(err.Error(), "busy") { + t.Fatalf("error = %v, want it to report a busy repository", err) + } +} + +func TestRunBackupPathBlockedByPruneLock(t *testing.T) { + ident, backend, _ := repoFixture(t) + putLock(t, backend, "prune-live.json", freshLock("prune")) + + filePath := writeTempFile(t, []byte("locked-out")) + _, err := runBackupPath(filePath, ident, backend, "7f91c6c7-7a0b-44aa-ae23-997b60e4e997", nil) + if err == nil { + t.Fatal("runBackupPath() succeeded while a prune lock is held") + } + if !strings.Contains(err.Error(), "busy") { + t.Fatalf("error = %v, want it to report a busy repository", err) + } +} + +func TestRunPruneDryRunIgnoresLocks(t *testing.T) { + ident, backend, _ := repoFixture(t) + putLock(t, backend, "prune-live.json", freshLock("prune")) + + if _, err := runPrune(ident, backend, RetentionPolicy{KeepLast: 1}, false); err != nil { + t.Fatalf("dry-run runPrune() error: %v", err) + } +} + +func TestAcquireRepoLockRejectsUnreadableLock(t *testing.T) { + backend := memtest.NewMemoryBackend() + backend.Put("locks/broken.json", []byte("{not json")) + _, err := acquireRepoLock(backend, "backup", true) + if err == nil { + t.Fatal("acquireRepoLock() succeeded on an unreadable lock file") + } + if !strings.Contains(fmt.Sprint(err), "unreadable") { + t.Fatalf("error = %v, want it to mention the unreadable lock", err) + } +} diff --git a/internal/flow/prune.go b/internal/flow/prune.go index 01aaf10..9770635 100644 --- a/internal/flow/prune.go +++ b/internal/flow/prune.go @@ -36,6 +36,9 @@ func RunPrune(identityPath, remote string, policy RetentionPolicy, apply bool) ( func runPrune(identityPath string, backend storage.Backend, policy RetentionPolicy, apply bool) (PruneResult, error) { var res PruneResult + if policy.KeepLast < 0 || policy.KeepDaily < 0 || policy.KeepWeekly < 0 || policy.KeepMonthly < 0 { + return res, fmt.Errorf("retention counts must be non-negative") + } if policy.isEmpty() { return res, fmt.Errorf("refusing to prune with no retention rules (pass at least one --keep-* flag)") } @@ -43,6 +46,15 @@ func runPrune(identityPath string, backend storage.Backend, policy RetentionPoli if _, err := repo.Open(backend, identityPath); err != nil { return res, err } + if apply { + // Advisory exclusive lock: no backup (shared) may run while + // prune deletes, and no other prune may overlap. + release, err := acquireRepoLock(backend, "prune", false) + if err != nil { + return res, err + } + defer release() + } ix, err := index.Load(backend, identityPath) if err != nil { return res, fmt.Errorf("failed to load index: %w", err) @@ -51,7 +63,11 @@ func runPrune(identityPath string, backend storage.Backend, policy RetentionPoli // List snapshots. snapRaw, err := backend.ListFiles([]string{repo.SnapshotsPrefix}) if err != nil { - return res, fmt.Errorf("failed to list snapshots: %w", err) + if !storage.IsMissingPath(err) { + return res, fmt.Errorf("failed to list snapshots: %w", err) + } + // A fresh repo has no snapshots/ directory yet. + snapRaw = []byte("[]") } snapEntries, err := storage.ParseList(snapRaw) if err != nil { @@ -128,7 +144,11 @@ func runPrune(identityPath string, backend storage.Backend, policy RetentionPoli // Also scan data/ for objects not in the live set (orphans). dataPaths, err := backend.ListRecursive([]string{repo.DataPrefix}) if err != nil { - // No data/ is fine. + // A fresh repo has no data/ directory yet; anything else is a + // real listing failure and must not be pruned around. + if !storage.IsMissingPath(err) { + return res, fmt.Errorf("failed to list data objects: %w", err) + } dataPaths = nil } seenGarbage := make(map[string]bool, len(garbage)) @@ -150,7 +170,16 @@ func runPrune(identityPath string, backend storage.Backend, policy RetentionPoli return res, nil } - // Delete dead snapshots. + // Compact the index first: the new epoch drops hashes no kept + // snapshot references. If this fails, nothing has been deleted yet. + if err := compactIndex(backend, identityPath, ix, live); err != nil { + return res, err + } + res.CompactedEpochs = true + + // Delete dead snapshots. Their exclusive chunks are already gone from + // the index but the data objects are still present, so an interrupted + // prune leaves the repository consistent. for _, id := range res.DeadSnapshots { sp, err := repo.SnapshotPath(id) if err != nil { @@ -162,7 +191,7 @@ func runPrune(identityPath string, backend storage.Backend, policy RetentionPoli res.DeletedSnapshots++ } - // Delete garbage data objects. + // Delete garbage data objects last, once nothing references them. for _, h := range garbage { dp := repo.DataPath(h) if err := backend.Delete(dp); err != nil { @@ -171,12 +200,6 @@ func runPrune(identityPath string, backend storage.Backend, policy RetentionPoli res.DeletedObjects++ } - // Compact index: write one epoch with live entries only, delete old epochs. - if err := compactIndex(backend, identityPath, ix, live); err != nil { - return res, err - } - res.CompactedEpochs = true - return res, nil } @@ -191,6 +214,9 @@ func compactIndex(backend storage.Backend, identityPath string, ix *index.Index, // List current epochs before compacting so we can delete the old ones. oldRaw, err := backend.ListFiles([]string{"index"}) if err != nil { + if !storage.IsMissingPath(err) { + return fmt.Errorf("failed to list index epochs: %w", err) + } oldRaw = []byte("[]") } oldEntries, err := storage.ParseList(oldRaw) @@ -210,9 +236,12 @@ func compactIndex(backend storage.Backend, identityPath string, ix *index.Index, return fmt.Errorf("failed to write compacted index: %w", err) } - // Delete every epoch that existed before compaction. + // Delete every epoch that existed before compaction. A failure here + // leaves the old epochs in place, which Load merges harmlessly. for _, name := range oldEpochs { - _ = backend.Delete("index/" + name) + if err := backend.Delete("index/" + name); err != nil { + return fmt.Errorf("failed to delete old index epoch %s: %w", name, err) + } } return nil } diff --git a/internal/flow/prune_test.go b/internal/flow/prune_test.go index 82991a9..500f6d1 100644 --- a/internal/flow/prune_test.go +++ b/internal/flow/prune_test.go @@ -1,10 +1,14 @@ package flow import ( + "fmt" "os" "path/filepath" + "strings" "testing" "time" + + "gloak/internal/storage" ) func TestSelectKeepSnapshotsLastOnly(t *testing.T) { @@ -56,6 +60,66 @@ func TestRunPruneRejectsEmptyPolicy(t *testing.T) { } } +func TestRunPruneRejectsNegativePolicy(t *testing.T) { + ident, backend, _ := repoFixture(t) + for _, policy := range []RetentionPolicy{ + {KeepLast: -1}, + {KeepDaily: -1}, + {KeepWeekly: -1}, + {KeepMonthly: -1, KeepLast: 1}, + } { + if _, err := runPrune(ident, backend, policy, false); err == nil { + t.Fatalf("runPrune() succeeded with negative policy %+v", policy) + } + } +} + +// failDeleteBackend fails Delete for paths under failPrefix. +type failDeleteBackend struct { + storage.Backend + failPrefix string +} + +func (b *failDeleteBackend) Delete(remotePath string) error { + if strings.HasPrefix(remotePath, b.failPrefix) { + return fmt.Errorf("delete %s failed", remotePath) + } + return b.Backend.Delete(remotePath) +} + +func TestRunPruneCompactsIndexBeforeDeletingAnything(t *testing.T) { + ident, backend, _ := repoFixture(t) + f1 := filepath.Join(t.TempDir(), "one.txt") + if err := os.WriteFile(f1, []byte("content-one-unique"), 0644); err != nil { + t.Fatal(err) + } + f2 := filepath.Join(t.TempDir(), "two.txt") + if err := os.WriteFile(f2, []byte("content-two-different"), 0644); err != nil { + t.Fatal(err) + } + if _, err := runBackup(f1, ident, backend, "7f91c6c7-7a0b-44aa-ae23-997b60e4e998", nil); err != nil { + t.Fatal(err) + } + time.Sleep(1100 * time.Millisecond) + if _, err := runBackup(f2, ident, backend, "8f91c6c7-7a0b-44aa-ae23-997b60e4e999", nil); err != nil { + t.Fatal(err) + } + + wrapped := &failDeleteBackend{Backend: backend, failPrefix: "index/"} + _, err := runPrune(ident, wrapped, RetentionPolicy{KeepLast: 1}, true) + if err == nil { + t.Fatal("runPrune() succeeded despite index epoch deletion failure") + } + // Compaction runs first: when it fails, no snapshot or object has + // been touched yet. + if !backend.Has("snapshots/7f91c6c7-7a0b-44aa-ae23-997b60e4e998.age") { + t.Fatal("dead snapshot deleted before compaction finished") + } + if err := runVerify("8f91c6c7-7a0b-44aa-ae23-997b60e4e999", ident, backend); err != nil { + t.Fatalf("kept snapshot broken after failed prune: %v", err) + } +} + func TestRunPruneDryRunChangesNothing(t *testing.T) { ident, backend, _ := repoFixture(t) filePath := writeTempFile(t, []byte("keep-me")) diff --git a/internal/flow/restore.go b/internal/flow/restore.go index 9c9cac6..be689aa 100644 --- a/internal/flow/restore.go +++ b/internal/flow/restore.go @@ -7,6 +7,7 @@ import ( "fmt" "os" "path/filepath" + "sort" "strings" "time" @@ -92,14 +93,33 @@ func runTreeRestore(m *core.TreeManifest, r *repo.Repo, outputDir, pathFilter st if len(m.Paths) == 0 { return "", fmt.Errorf("tree snapshot has no source paths") } + // The manifest is remote input: fall back to the snapshot UUID when + // the recorded root name is not a single safe path segment, and + // reject the manifest when even the UUID is unsafe. rootName := filepath.Base(filepath.Clean(m.Paths[0])) - if rootName == "" || rootName == "." || rootName == string(filepath.Separator) { + if err := core.ValidateRelPath(rootName); err != nil { rootName = m.UUID } + if err := core.ValidateRelPath(rootName); err != nil { + return "", fmt.Errorf("unsafe restore root name: %w", err) + } if err := os.MkdirAll(outputDir, 0755); err != nil { return "", fmt.Errorf("failed to create output directory: %w", err) } destRoot := filepath.Join(outputDir, rootName) + // Defense in depth: the join above must stay under outputDir. + if rel, err := filepath.Rel(outputDir, destRoot); err != nil || + rel == ".." || strings.HasPrefix(rel, ".."+string(filepath.Separator)) { + return "", fmt.Errorf("restore root escapes output directory: %s", rootName) + } + + type dirSetting struct { + path string + mode os.FileMode + mtime time.Time + } + var dirSettings []dirSetting + var matched int for _, entry := range m.Entries { if err := core.ValidateRelPath(entry.RelPath); err != nil { @@ -110,6 +130,7 @@ func runTreeRestore(m *core.TreeManifest, r *repo.Repo, outputDir, pathFilter st continue } } + matched++ target := filepath.Join(destRoot, filepath.FromSlash(entry.RelPath)) // Defense in depth: ensure target stays under destRoot. if !strings.HasPrefix(target, destRoot+string(filepath.Separator)) && target != destRoot { @@ -117,9 +138,19 @@ func runTreeRestore(m *core.TreeManifest, r *repo.Repo, outputDir, pathFilter st } switch entry.Type { case "dir": - if err := os.MkdirAll(target, os.FileMode(entry.Mode)&0777); err != nil { + // Keep the directory writable while children are restored; + // the recorded mode is applied after the loop. + if err := os.MkdirAll(target, 0700); err != nil { return "", fmt.Errorf("failed to create directory %s: %w", entry.RelPath, err) } + if err := os.Chmod(target, 0700); err != nil { + return "", fmt.Errorf("failed to open directory %s for restore: %w", entry.RelPath, err) + } + setting := dirSetting{path: target, mode: os.FileMode(entry.Mode) & 0777} + if mt, err := time.Parse(time.RFC3339, entry.Mtime); err == nil { + setting.mtime = mt + } + dirSettings = append(dirSettings, setting) case "file": if err := os.MkdirAll(filepath.Dir(target), 0755); err != nil { return "", fmt.Errorf("failed to create parent for %s: %w", entry.RelPath, err) @@ -131,13 +162,39 @@ func runTreeRestore(m *core.TreeManifest, r *repo.Repo, outputDir, pathFilter st return "", fmt.Errorf("unknown tree entry type %q for %s", entry.Type, entry.RelPath) } } + if pathFilter != "" && matched == 0 { + return "", fmt.Errorf("path filter %q matches nothing in snapshot %s", pathFilter, m.UUID) + } + + // Apply directory modes after every child is written, deepest paths + // first so a read-only parent never blocks chmod on its children. + sort.Slice(dirSettings, func(i, j int) bool { + return len(dirSettings[i].path) > len(dirSettings[j].path) + }) + for _, d := range dirSettings { + if err := os.Chmod(d.path, d.mode); err != nil { + return "", fmt.Errorf("failed to set mode on %s: %w", d.path, err) + } + if !d.mtime.IsZero() { + _ = os.Chtimes(d.path, d.mtime, d.mtime) + } + } return destRoot, nil } func restoreTreeFile(entry core.TreeEntry, r *repo.Repo, backend storage.Backend, target string) error { - outFile, err := os.OpenFile(target, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, os.FileMode(entry.Mode)&0777) + mode := os.FileMode(entry.Mode) & 0777 + outFile, err := os.OpenFile(target, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, mode) if err != nil { - return fmt.Errorf("failed to create output file %s: %w", entry.RelPath, err) + // An earlier restore may have left the file read-only; make it + // writable once and retry. + if chmodErr := os.Chmod(target, mode|0200); chmodErr != nil { + return fmt.Errorf("failed to create output file %s: %w", entry.RelPath, err) + } + outFile, err = os.OpenFile(target, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, mode) + if err != nil { + return fmt.Errorf("failed to create output file %s: %w", entry.RelPath, err) + } } defer outFile.Close() @@ -160,6 +217,11 @@ func restoreTreeFile(entry core.TreeEntry, r *repo.Repo, backend storage.Backend return fmt.Errorf("failed to write chunk %d: %w", meta.Index, err) } } + // Re-applying the mode matters when the file already existed (a + // previous restore may have left it read-only or writable). + if err := os.Chmod(target, mode); err != nil { + return fmt.Errorf("failed to set mode on %s: %w", entry.RelPath, err) + } if mt, err := time.Parse(time.RFC3339, entry.Mtime); err == nil { _ = os.Chtimes(target, mt, mt) } diff --git a/internal/flow/snapshots.go b/internal/flow/snapshots.go index 80647ba..5024d0b 100644 --- a/internal/flow/snapshots.go +++ b/internal/flow/snapshots.go @@ -52,8 +52,12 @@ func runSnapshots(identityPath string, backend storage.Backend) ([]SnapshotInfo, } raw, err := backend.ListFiles([]string{repo.SnapshotsPrefix}) if err != nil { - // Fresh repo with no snapshots directory. - return nil, nil + // Fresh repo with no snapshots directory; anything else is a + // real listing failure and must surface. + if storage.IsMissingPath(err) { + return nil, nil + } + return nil, fmt.Errorf("failed to list snapshots: %w", err) } entries, err := storage.ParseList(raw) if err != nil { diff --git a/internal/flow/tree_backup.go b/internal/flow/tree_backup.go index c9f5bbe..5f6746b 100644 --- a/internal/flow/tree_backup.go +++ b/internal/flow/tree_backup.go @@ -38,19 +38,27 @@ func runTreeBackup(dirPath, identityPath string, backend storage.Backend, snapsh if err != nil { return BackupResult{}, err } + // Resolve symlinks before the walk: walking a symlinked root never + // matches the root itself, and every relative path would start with + // ../, producing a silently empty snapshot. The manifest keeps the + // path the user passed so restore still uses the familiar name. + resolved, err := filepath.EvalSymlinks(abs) + if err != nil { + return BackupResult{}, fmt.Errorf("failed to resolve %s: %w", abs, err) + } var entries []core.TreeEntry var sinceCheckpoint int res := BackupResult{SnapshotID: snapshotID} - walkErr := filepath.WalkDir(abs, func(p string, d fs.DirEntry, walkErr error) error { + walkErr := filepath.WalkDir(resolved, func(p string, d fs.DirEntry, walkErr error) error { if walkErr != nil { return walkErr } - if p == abs { + if p == resolved { return nil } - rel, err := filepath.Rel(abs, p) + rel, err := filepath.Rel(resolved, p) if err != nil { return err } diff --git a/internal/flow/tree_backup_test.go b/internal/flow/tree_backup_test.go index 5baa468..fb97326 100644 --- a/internal/flow/tree_backup_test.go +++ b/internal/flow/tree_backup_test.go @@ -95,3 +95,40 @@ func TestRunBackupPathDispatchesFile(t *testing.T) { t.Fatalf("OriginalName = %q", m.OriginalName) } } + +func TestRunTreeBackupFollowsSymlinkedRoot(t *testing.T) { + ident, backend, _ := repoFixture(t) + real := t.TempDir() + if err := os.WriteFile(filepath.Join(real, "a.txt"), []byte("alpha"), 0644); err != nil { + t.Fatal(err) + } + link := filepath.Join(t.TempDir(), "link") + if err := os.Symlink(real, link); err != nil { + t.Skip("symlink not supported") + } + + snap := "aaaaaaaa-bbbb-4ccc-8ddd-000000000013" + res, err := runBackupPath(link, ident, backend, snap, nil) + if err != nil { + t.Fatalf("runBackupPath(symlink) error: %v", err) + } + if res.TotalChunks < 1 { + t.Fatalf("TotalChunks = %d, want >= 1", res.TotalChunks) + } + m, err := readTreeManifest(backend, ident, snap) + if err != nil { + t.Fatal(err) + } + if m.FileCount() != 1 { + t.Fatalf("FileCount = %d, want 1 (walk of a symlinked root must not be empty)", m.FileCount()) + } + found := false + for _, e := range m.Entries { + if e.RelPath == "a.txt" && e.Type == "file" { + found = true + } + } + if !found { + t.Fatalf("a.txt missing from entries: %+v", m.Entries) + } +} diff --git a/internal/flow/tree_restore_test.go b/internal/flow/tree_restore_test.go index f35f9d2..bb26972 100644 --- a/internal/flow/tree_restore_test.go +++ b/internal/flow/tree_restore_test.go @@ -106,6 +106,149 @@ func TestRunTreeRestoreRejectsUnsafeRelPath(t *testing.T) { } } +func TestRunTreeRestoreKeepsUnsafeRootNameInsideOutput(t *testing.T) { + ident, backend, _ := repoFixture(t) + // Base(Clean("/")) is "/", which is not a safe single path segment, + // so the restore root falls back to the snapshot UUID. + m := core.TreeManifest{ + Version: "2.0", + UUID: "aaaaaaaa-bbbb-4ccc-8ddd-000000000031", + Kind: core.KindTree, + CreatedAt: "2026-10-12T00:00:00Z", + Hostname: "test", + Paths: []string{"/"}, + Entries: nil, + } + seedTreeSnapshot(t, backend, ident, m) + + out := t.TempDir() + got, err := runRestore(m.UUID, out, ident, backend, "") + if err != nil { + t.Fatalf("runRestore() error: %v", err) + } + want := filepath.Join(out, m.UUID) + if got != want { + t.Fatalf("restore root = %q, want %q", got, want) + } +} + +func TestRunTreeRestoreRejectsUnsafeRootAndUUID(t *testing.T) { + ident, backend, _ := repoFixture(t) + // The path id stays a valid UUID while the manifest content is + // hostile: root name and UUID both escape the output directory. + validID := "aaaaaaaa-bbbb-4ccc-8ddd-000000000032" + m := core.TreeManifest{ + Version: "2.0", + UUID: "../evil", + Kind: core.KindTree, + CreatedAt: "2026-10-12T00:00:00Z", + Hostname: "test", + Paths: []string{".."}, + Entries: nil, + } + raw, err := json.Marshal(m) + if err != nil { + t.Fatal(err) + } + r, err := repo.Open(backend, ident) + if err != nil { + t.Fatal(err) + } + sealed, err := appcrypto.EncryptMulti(r.Config.Recipients, raw) + if err != nil { + t.Fatal(err) + } + if err := backend.Upload("snapshots/"+validID+".age", bytes.NewReader(sealed)); err != nil { + t.Fatal(err) + } + + if _, err := runRestore(validID, t.TempDir(), ident, backend, ""); err == nil { + t.Fatal("runRestore() accepted a hostile root name and uuid") + } else if !strings.Contains(err.Error(), "unsafe") { + t.Fatalf("error = %v, want it to reject the unsafe root", err) + } +} + +func TestRunTreeRestoreFilterMatchingNothingFails(t *testing.T) { + _, backend, ident, snap := treeFixture(t) + _, err := runRestore(snap, t.TempDir(), ident, backend, "does/not/exist") + if err == nil { + t.Fatal("runRestore() succeeded with a filter that matches nothing") + } + if !strings.Contains(err.Error(), "matches nothing") { + t.Fatalf("error = %v, want it to report the unmatched filter", err) + } +} + +func TestRunTreeRestoreAppliesDirModesAfterChildren(t *testing.T) { + ident, backend, _ := repoFixture(t) + root := t.TempDir() + locked := filepath.Join(root, "locked") + if err := os.MkdirAll(locked, 0755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(locked, "f.txt"), []byte("inside"), 0444); err != nil { + t.Fatal(err) + } + if err := os.Chmod(locked, 0555); err != nil { + t.Fatal(err) + } + // Restore write access so t.TempDir can clean up after the test. + t.Cleanup(func() { _ = os.Chmod(locked, 0755) }) + snap := "aaaaaaaa-bbbb-4ccc-8ddd-000000000033" + if _, err := runBackupPath(root, ident, backend, snap, nil); err != nil { + t.Fatal(err) + } + + out := t.TempDir() + got, err := runRestore(snap, out, ident, backend, "") + if err != nil { + t.Fatalf("runRestore() error: %v", err) + } + t.Cleanup(func() { _ = os.Chmod(filepath.Join(got, "locked"), 0755) }) + fi, err := os.Stat(filepath.Join(got, "locked")) + if err != nil { + t.Fatal(err) + } + if fi.Mode().Perm() != 0555 { + t.Fatalf("dir mode = %o, want 555", fi.Mode().Perm()) + } + fi, err = os.Stat(filepath.Join(got, "locked", "f.txt")) + if err != nil { + t.Fatal(err) + } + if fi.Mode().Perm() != 0444 { + t.Fatalf("file mode = %o, want 444", fi.Mode().Perm()) + } + data, err := os.ReadFile(filepath.Join(got, "locked", "f.txt")) + if err != nil { + t.Fatal(err) + } + if string(data) != "inside" { + t.Fatalf("content = %q", data) + } + + // Restoring again over the read-only tree must succeed and keep the + // recorded modes. + if _, err := runRestore(snap, out, ident, backend, ""); err != nil { + t.Fatalf("re-restore over read-only tree: %v", err) + } + fi, err = os.Stat(filepath.Join(got, "locked")) + if err != nil { + t.Fatal(err) + } + if fi.Mode().Perm() != 0555 { + t.Fatalf("dir mode after re-restore = %o, want 555", fi.Mode().Perm()) + } + fi, err = os.Stat(filepath.Join(got, "locked", "f.txt")) + if err != nil { + t.Fatal(err) + } + if fi.Mode().Perm() != 0444 { + t.Fatalf("file mode after re-restore = %o, want 444", fi.Mode().Perm()) + } +} + func seedTreeSnapshot(t *testing.T, backend *memtest.MemoryBackend, ident string, m core.TreeManifest) { t.Helper() r, err := repo.Open(backend, ident) diff --git a/internal/index/index.go b/internal/index/index.go index 3868787..3d221d9 100644 --- a/internal/index/index.go +++ b/internal/index/index.go @@ -53,8 +53,12 @@ func Load(backend storage.Backend, identityPath string) (*Index, error) { if err != nil { // A fresh repo has no index/ directory yet (rclone lsjson exits 3 // on a missing directory); that is normal, so treat the listing as - // empty and start from epoch 0. - return ix, nil + // empty and start from epoch 0. Any other listing failure is a real + // error and must not be mistaken for an empty index. + if storage.IsMissingPath(err) { + return ix, nil + } + return nil, fmt.Errorf("failed to list index: %w", err) } entries, err := storage.ParseList(raw) if err != nil { diff --git a/internal/index/index_test.go b/internal/index/index_test.go index 5fe69b6..273d46f 100644 --- a/internal/index/index_test.go +++ b/internal/index/index_test.go @@ -2,15 +2,31 @@ package index import ( "bytes" + "errors" + "fmt" "io" "os" + "os/exec" "path/filepath" + "strings" "testing" "gloak/internal/crypto" + "gloak/internal/storage" "gloak/internal/storage/memtest" ) +// listErrorBackend fails every listing with err while staying usable for +// anything else, so Load error handling can be exercised. +type listErrorBackend struct { + storage.Backend + err error +} + +func (b *listErrorBackend) ListFiles(prefix []string) ([]byte, error) { + return nil, b.err +} + func identityFixture(t *testing.T) (string, string) { t.Helper() pub, _, fileContent, err := crypto.GenerateKey() @@ -188,6 +204,36 @@ func TestLoadOnEmptyBackendIsClean(t *testing.T) { } } +func TestLoadTreatsMissingPathAsEmpty(t *testing.T) { + _, ident := identityFixture(t) + // rclone exits 3 when the remote path is missing; that is a fresh repo. + exitErr := exec.Command("sh", "-c", "exit 3").Run() + backend := &listErrorBackend{ + Backend: memtest.NewMemoryBackend(), + err: fmt.Errorf("failed to list index: %w", exitErr), + } + loaded, err := Load(backend, ident) + if err != nil { + t.Fatalf("Load() error: %v", err) + } + if len(loaded.AllHashes()) != 0 { + t.Fatalf("Load() on missing index = %v, want empty", loaded.AllHashes()) + } +} + +func TestLoadPropagatesListingErrors(t *testing.T) { + _, ident := identityFixture(t) + backend := &listErrorBackend{ + Backend: memtest.NewMemoryBackend(), + err: errors.New("remote exploded"), + } + if _, err := Load(backend, ident); err == nil { + t.Fatal("Load() succeeded on a real listing error") + } else if !strings.Contains(err.Error(), "remote exploded") { + t.Fatalf("Load() error = %v, want it to wrap the listing failure", err) + } +} + func TestSaveOutputIsEncryptedJSON(t *testing.T) { pub, ident := identityFixture(t) backend := memtest.NewMemoryBackend() diff --git a/internal/notify/telegram.go b/internal/notify/telegram.go index e79bb70..22624cc 100644 --- a/internal/notify/telegram.go +++ b/internal/notify/telegram.go @@ -50,7 +50,7 @@ func (c TelegramConfig) Send(message string) error { client := &http.Client{Timeout: 15 * time.Second} resp, err := client.Post(endpoint, "application/x-www-form-urlencoded", strings.NewReader(form.Encode())) if err != nil { - return fmt.Errorf("telegram send: %w", err) + return fmt.Errorf("telegram send: %s", redact(err.Error(), c.Token)) } defer resp.Body.Close() @@ -59,11 +59,20 @@ func (c TelegramConfig) Send(message string) error { Description string `json:"description"` } _ = json.NewDecoder(resp.Body).Decode(&body) - return fmt.Errorf("telegram send: status %d: %s", resp.StatusCode, body.Description) + return fmt.Errorf("telegram send: status %d: %s", resp.StatusCode, redact(body.Description, c.Token)) } return nil } +// redact removes the bot token from a message. Transport errors embed the +// full request URL, which contains the token. +func redact(msg, token string) string { + if token == "" { + return msg + } + return strings.ReplaceAll(msg, token, "[redacted]") +} + func truncate(s string, n int) string { if len(s) <= n { return s diff --git a/internal/notify/telegram_test.go b/internal/notify/telegram_test.go index 4855e1e..d6650e6 100644 --- a/internal/notify/telegram_test.go +++ b/internal/notify/telegram_test.go @@ -59,3 +59,23 @@ func TestSendHTTPError(t *testing.T) { t.Fatal("Send() succeeded on HTTP error") } } + +func TestSendRedactsTokenFromTransportError(t *testing.T) { + // Nothing listens on this port, so the transport error embeds the + // request URL, which contains the bot token. + c := TelegramConfig{ + Token: "SECRETTOKEN123", + ChatID: "1", + BaseURL: "http://127.0.0.1:1", + } + err := c.Send("x") + if err == nil { + t.Fatal("Send() succeeded against an unreachable server") + } + if strings.Contains(err.Error(), "SECRETTOKEN123") { + t.Fatalf("error leaks the bot token: %v", err) + } + if !strings.Contains(err.Error(), "[redacted]") { + t.Fatalf("error = %v, want the token placeholder", err) + } +} diff --git a/internal/repo/repo.go b/internal/repo/repo.go index 81eca4d..a7df7f1 100644 --- a/internal/repo/repo.go +++ b/internal/repo/repo.go @@ -93,6 +93,14 @@ func Init(backend storage.Backend, recipients []string, min, avg, max int64, for } } + // Force re-init (or a retry after a crashed init) can leave wrapped + // keys for recipients that are no longer part of the repository. Remove + // them before repo.json, the completion marker, is replaced, so a + // removed recipient cannot keep opening the repository. + if err := deleteStaleKeys(backend, recipients); err != nil { + return nil, Config{}, err + } + cfgBytes, err := json.MarshalIndent(cfg, "", " ") if err != nil { return nil, Config{}, fmt.Errorf("failed to marshal repo config: %w", err) @@ -103,6 +111,40 @@ func Init(backend storage.Backend, recipients []string, min, avg, max int64, for return master, cfg, nil } +// deleteStaleKeys removes wrapped keys under keys/ whose fingerprint is +// not part of recipients. A missing keys/ directory is normal on first +// init; any other listing failure is returned. +func deleteStaleKeys(backend storage.Backend, recipients []string) error { + keep := make(map[string]bool, len(recipients)) + for _, recipient := range recipients { + keep[KeysPathFor(recipient)] = true + } + raw, err := backend.ListFiles([]string{KeysPrefix}) + if err != nil { + if storage.IsMissingPath(err) { + return nil + } + return fmt.Errorf("failed to list keys: %w", err) + } + entries, err := storage.ParseList(raw) + if err != nil { + return fmt.Errorf("failed to parse keys listing: %w", err) + } + for _, entry := range entries { + if entry.IsDir { + continue + } + full := KeysPrefix + "/" + entry.Path + if keep[full] { + continue + } + if err := backend.Delete(full); err != nil { + return fmt.Errorf("failed to delete stale key %s: %w", full, err) + } + } + return nil +} + // Open loads repo.json and unwraps the master key with the given identity. func Open(backend storage.Backend, identityPath string) (*Repo, error) { var buf bytes.Buffer diff --git a/internal/repo/repo_test.go b/internal/repo/repo_test.go index 3aea5d6..fee674e 100644 --- a/internal/repo/repo_test.go +++ b/internal/repo/repo_test.go @@ -49,6 +49,38 @@ func TestInitWritesKeysAndConfig(t *testing.T) { } } +func TestInitRemovesStaleKeysOnReinit(t *testing.T) { + pubA, identA := testRecipient(t) + pubB, identB := testRecipient(t) + backend := memtest.NewMemoryBackend() + + if _, _, err := Init(backend, []string{pubA}, 1, 8, 64, false); err != nil { + t.Fatalf("Init() error: %v", err) + } + // A wrapped key for a recipient that is about to be removed. + stalePath := KeysPrefix + "/0000000000000000.age" + backend.Put(stalePath, []byte("stale")) + + if _, _, err := Init(backend, []string{pubB}, 1, 8, 64, true); err != nil { + t.Fatalf("Init(force) error: %v", err) + } + if backend.Has(stalePath) { + t.Fatal("stale key survived a force re-init") + } + if backend.Has(KeysPathFor(pubA)) { + t.Fatal("removed recipient still has a wrapped key") + } + if !backend.Has(KeysPathFor(pubB)) { + t.Fatal("new recipient key missing") + } + if _, err := Open(backend, identB); err != nil { + t.Fatalf("Open() with the new identity: %v", err) + } + if _, err := Open(backend, identA); err == nil { + t.Fatal("removed recipient can still open the repository") + } +} + func TestOpenWithEachRecipientIdentity(t *testing.T) { pubA, identA := testRecipient(t) pubB, identB := testRecipient(t) diff --git a/internal/storage/rclone.go b/internal/storage/rclone.go index c5b318b..a9f13b8 100644 --- a/internal/storage/rclone.go +++ b/internal/storage/rclone.go @@ -2,6 +2,7 @@ package storage import ( "bytes" + "errors" "fmt" "io" "os/exec" @@ -34,6 +35,18 @@ func rcloneArgs(configPath string, base ...string) []string { return args } +// IsMissingPath reports whether err was caused by rclone exiting with +// status 3, which rclone uses when the remote path does not exist. +// Callers that treat a missing directory as an empty listing must check +// this so a genuine listing failure is never swallowed. +func IsMissingPath(err error) bool { + var exitErr *exec.ExitError + if errors.As(err, &exitErr) { + return exitErr.ExitCode() == 3 + } + return false +} + func (r *RcloneBackend) Upload(remotePath string, in io.Reader) error { fullDest := fmt.Sprintf("%s/%s", r.Remote, remotePath) diff --git a/internal/storage/rclone_test.go b/internal/storage/rclone_test.go index 785a9af..374e749 100644 --- a/internal/storage/rclone_test.go +++ b/internal/storage/rclone_test.go @@ -1,6 +1,8 @@ package storage import ( + "fmt" + "os/exec" "reflect" "testing" ) @@ -14,9 +16,33 @@ func TestRcloneArgsOmitsConfigWhenEmpty(t *testing.T) { } func TestRcloneArgsAppendsConfigWhenSet(t *testing.T) { - got := rcloneArgs("/etc/rclone.conf", "rcat", "remote:file") - want := []string{"rcat", "remote:file", "--config", "/etc/rclone.conf"} + got := rcloneArgs("/etc/rclone.conf", "cat", "remote:file") + want := []string{"cat", "remote:file", "--config", "/etc/rclone.conf"} if !reflect.DeepEqual(got, want) { t.Fatalf("rcloneArgs() = %v, want %v", got, want) } } + +func TestIsMissingPathMatchesExitCodeThree(t *testing.T) { + // rclone exits 3 when the remote path is missing. + exitErr := exec.Command("sh", "-c", "exit 3").Run() + if exitErr == nil { + t.Fatal("sh exited 0, want non-zero") + } + if !IsMissingPath(fmt.Errorf("wrapped: %w", exitErr)) { + t.Fatal("IsMissingPath(wrapped exit 3) = false, want true") + } +} + +func TestIsMissingPathRejectsOtherErrors(t *testing.T) { + other := exec.Command("sh", "-c", "exit 1").Run() + if IsMissingPath(fmt.Errorf("wrapped: %w", other)) { + t.Fatal("IsMissingPath(exit 1) = true, want false") + } + if IsMissingPath(fmt.Errorf("plain failure")) { + t.Fatal("IsMissingPath(plain) = true, want false") + } + if IsMissingPath(nil) { + t.Fatal("IsMissingPath(nil) = true, want false") + } +} From 9f1811dbf95f72c5d6817018e27342130bc70174 Mon Sep 17 00:00:00 2001 From: "coderabbitai[bot]" <136622811+coderabbitai[bot]@users.noreply.github.com> Date: Mon, 12 Oct 2026 16:20:41 +0000 Subject: [PATCH 42/48] docs: add and clarify Go function documentation across CLI and internal packages --- backup.go | 6 +++++- check.go | 2 ++ init.go | 13 +++++++++++-- internal/chunker/chunker.go | 10 ++++++++-- internal/config/config.go | 3 ++- internal/core/manifest.go | 1 + internal/core/path.go | 5 +++-- internal/crypto/envelope.go | 14 +++++++++++++- internal/crypto/multi.go | 3 ++- internal/flow/backup.go | 15 ++++++++++++--- internal/flow/check.go | 7 +++++++ internal/flow/filemode.go | 17 +++++++++++++++++ internal/flow/init.go | 7 +++++-- internal/flow/manifest_read.go | 12 ++++++++---- internal/flow/prune.go | 16 +++++++++++++--- internal/flow/restore.go | 25 +++++++++++++++++++++++++ internal/flow/retention.go | 5 ++++- internal/flow/snapshots.go | 9 +++++++++ internal/flow/tree_backup.go | 6 ++++++ internal/flow/upload.go | 7 ++++++- internal/flow/verify.go | 11 +++++++++++ internal/index/index.go | 31 ++++++++++++++++++++----------- internal/notify/telegram.go | 7 +++++-- internal/repo/repo.go | 12 +++++++++--- internal/storage/list.go | 2 ++ internal/storage/rclone.go | 9 +++++++++ main.go | 5 +++-- notify.go | 12 ++++++------ prune.go | 2 ++ restore.go | 3 +++ snapshots.go | 2 ++ upload.go | 2 ++ verify.go | 3 +++ 33 files changed, 236 insertions(+), 48 deletions(-) diff --git a/backup.go b/backup.go index eaf2149..b089b98 100644 --- a/backup.go +++ b/backup.go @@ -19,6 +19,9 @@ type BackupCmd struct { Remote string `short:"R" required:"" help:"Repository remote, e.g. b2:my-bucket/gloak"` } +// Run backs up the selected path, using the default identity when none is +// specified, and sends an optional completion notification. Identity path +// expansion and backup errors are returned; notification failures are ignored. func (c *BackupCmd) Run(ctx *kong.Context) error { identityPath := c.Identity if identityPath == "" { @@ -41,7 +44,8 @@ func (c *BackupCmd) Run(ctx *kong.Context) error { return nil } -// expandHome resolves a leading ~ in path to the user's home directory. +// expandHome resolves ~ or a leading ~/ to the user's home directory, +// returning a home lookup error if expansion fails. Other paths are unchanged. func expandHome(path string) (string, error) { if path != "~" && !strings.HasPrefix(path, "~/") { return path, nil diff --git a/check.go b/check.go index ddc74a6..0fc158c 100644 --- a/check.go +++ b/check.go @@ -14,6 +14,8 @@ type CheckCmd struct { Remote string `help:"Source Rclone URI (e.g., keraaaki:backup_folder or /mnt/backups)." required:"" short:"R"` } +// Run checks repository structure using the selected or default identity and +// sends an optional completion notification. Check errors are returned. func (c *CheckCmd) Run() error { pterm.Info.Println("Starting CHECK process...") diff --git a/init.go b/init.go index bb7748a..19099df 100644 --- a/init.go +++ b/init.go @@ -20,6 +20,8 @@ type InitCmd struct { PQ bool `help:"Reserve post-quantum hybrid recipients (not implemented yet)."` } +// Run initializes the repository with the resolved recipients and chunk bounds. +// It rejects PQ mode and returns recipient resolution or initialization errors. func (c *InitCmd) Run() error { if c.PQ { return fmt.Errorf("--pq hybrid recipients are not implemented yet (reserved for a later phase)") @@ -32,10 +34,11 @@ func (c *InitCmd) Run() error { } // resolveInitRecipients turns the --recipient value into a key list. An empty -// path falls back to ~/.gloak/recipient.txt. A value that looks like keys +// value uses recipient.txt in getDefaultDir(). A value that looks like keys // (starts with age1, ssh-ed25519, ssh-rsa, or contains commas) is split on // commas; anything else is treated as a file path with one key per line, -// skipping blanks and # comments. +// skipping blanks and # comments. It returns file read errors and rejects +// empty lists; key validity is checked by callers. func resolveInitRecipients(path string) ([]string, error) { if path == "" { path = filepath.Join(getDefaultDir(), "recipient.txt") @@ -57,6 +60,9 @@ func resolveInitRecipients(path string) ([]string, error) { return readRecipientFile(path) } +// looksLikeInlineKeys reports whether s has a supported key prefix or a comma, +// so recipient resolution treats it as inline keys rather than a file path. +// It does not validate the keys. func looksLikeInlineKeys(s string) bool { return strings.HasPrefix(s, "age1") || strings.HasPrefix(s, "ssh-ed25519 ") || @@ -64,6 +70,9 @@ func looksLikeInlineKeys(s string) bool { strings.Contains(s, ",") } +// readRecipientFile returns trimmed, nonempty lines other than # comments. +// It returns an error if the file cannot be read or contains no such lines; +// the returned keys are not validated. func readRecipientFile(path string) ([]string, error) { data, err := os.ReadFile(path) if err != nil { diff --git a/internal/chunker/chunker.go b/internal/chunker/chunker.go index 39a651f..3531b77 100644 --- a/internal/chunker/chunker.go +++ b/internal/chunker/chunker.go @@ -13,6 +13,8 @@ const windowSize = 64 // same table and CDC boundaries stay stable across runs and machines. var gear [256]uint64 +// init fills the gear table deterministically so chunk boundaries are stable +// across runs. func init() { x := uint64(0x9E3779B97F4A7C15) for i := range gear { @@ -30,7 +32,9 @@ type CDC struct { max int64 } -// NewCDC validates the bounds and rounds max up to a multiple of windowSize. +// NewCDC creates a chunker with byte bounds min <= avg <= max, all positive, +// and rounds max up to a multiple of windowSize. avg controls the boundary +// frequency, not a guaranteed mean chunk size. Invalid bounds return an error. func NewCDC(min, avg, max int64) (*CDC, error) { if min <= 0 || avg <= 0 || max <= 0 { return nil, fmt.Errorf("chunk size must be positive (min=%d avg=%d max=%d)", min, avg, max) @@ -45,7 +49,9 @@ func NewCDC(min, avg, max int64) (*CDC, error) { } // Split reads r and calls fn once per chunk. Each chunk is a fresh slice the -// caller may retain. An empty reader produces no calls. +// caller may retain. An empty reader produces no calls; the final chunk may +// be smaller than min. It stops on and returns reader or callback errors; +// on a non-EOF read error, the buffered partial chunk is not delivered. func (c *CDC) Split(r io.Reader, fn func(chunk []byte) error) error { br := bufio.NewReaderSize(r, 64*1024) chunk := make([]byte, 0, c.max) diff --git a/internal/config/config.go b/internal/config/config.go index 704d6d9..040ba85 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -30,7 +30,8 @@ func Path(dir string) string { } // Load reads path and parses it. A missing file yields an empty Config and -// a nil error. A malformed file returns a descriptive error. +// a nil error. Other read errors and TOML parsing errors are returned with an +// empty Config. func Load(path string) (Config, error) { var cfg Config data, err := os.ReadFile(path) diff --git a/internal/core/manifest.go b/internal/core/manifest.go index 4569fc4..83e99a8 100644 --- a/internal/core/manifest.go +++ b/internal/core/manifest.go @@ -39,6 +39,7 @@ type TreeManifest struct { Entries []TreeEntry `json:"entries"` } +// FileCount counts entries whose type is "file", including empty files. func (m *TreeManifest) FileCount() int { n := 0 for _, e := range m.Entries { diff --git a/internal/core/path.go b/internal/core/path.go index 924fc99..ba7e309 100644 --- a/internal/core/path.go +++ b/internal/core/path.go @@ -5,8 +5,9 @@ import ( "strings" ) -// ValidateRelPath rejects tree entry paths that could escape the restore -// root. The manifest is remote input; this is the trust boundary. +// ValidateRelPath checks a slash-separated tree entry path without accessing +// the filesystem. It rejects empty paths, null bytes, backslashes, leading +// slashes, drive prefixes, and empty, . or .. segments. func ValidateRelPath(rel string) error { if rel == "" { return fmt.Errorf("empty path") diff --git a/internal/crypto/envelope.go b/internal/crypto/envelope.go index aca58f7..44b471d 100644 --- a/internal/crypto/envelope.go +++ b/internal/crypto/envelope.go @@ -13,6 +13,9 @@ import ( const chunkKeyInfo = "gloak-chunk-v1" +// deriveChunkKey derives a 32-byte chunk key from masterKey using the decoded +// plaintext SHA-256 hash as salt. It rejects malformed hex or hashes that do +// not decode to 32 bytes and returns key derivation errors. func deriveChunkKey(masterKey []byte, plainSHA256Hex string) ([]byte, error) { salt, err := hex.DecodeString(plainSHA256Hex) if err != nil || len(salt) != sha256.Size { @@ -23,6 +26,9 @@ func deriveChunkKey(masterKey []byte, plainSHA256Hex string) ([]byte, error) { // SealChunk compresses plain with zstd, then encrypts it with XChaCha20-Poly1305 // under a key derived from masterKey. Output layout: 24-byte nonce || ciphertext. +// plainSHA256Hex must encode 32 bytes; its exact text is authenticated, but its +// value is not checked against plain. Hash decoding, key derivation, compression, +// and encryption setup errors are returned. func SealChunk(masterKey []byte, plainSHA256Hex string, plain []byte) ([]byte, error) { key, err := deriveChunkKey(masterKey, plainSHA256Hex) if err != nil { @@ -46,7 +52,11 @@ func SealChunk(masterKey []byte, plainSHA256Hex string, plain []byte) ([]byte, e return sealed, nil } -// OpenChunk reverses SealChunk and verifies both the AEAD tag and the chunk hash. +// OpenChunk authenticates and decrypts a SealChunk envelope, then decompresses +// it. masterKey and the exact plainSHA256Hex string must match those used to +// seal it. It returns errors for malformed hashes, short envelopes, key or +// cipher setup failures, authentication failures, and invalid compressed data. +// Callers must check the returned plaintext's hash themselves. func OpenChunk(masterKey []byte, plainSHA256Hex string, sealed []byte) ([]byte, error) { key, err := deriveChunkKey(masterKey, plainSHA256Hex) if err != nil { @@ -72,6 +82,7 @@ func OpenChunk(masterKey []byte, plainSHA256Hex string, sealed []byte) ([]byte, return plain, nil } +// zstdEncode returns src compressed as a zstd frame, or an encoder setup error. func zstdEncode(src []byte) ([]byte, error) { enc, err := zstd.NewWriter(nil, zstd.WithEncoderLevel(zstd.SpeedDefault)) if err != nil { @@ -81,6 +92,7 @@ func zstdEncode(src []byte) ([]byte, error) { return enc.EncodeAll(src, nil), nil } +// zstdDecode decompresses zstd data, returning decoder setup or decoding errors. func zstdDecode(src []byte) ([]byte, error) { dec, err := zstd.NewReader(nil) if err != nil { diff --git a/internal/crypto/multi.go b/internal/crypto/multi.go index 22a4674..961003b 100644 --- a/internal/crypto/multi.go +++ b/internal/crypto/multi.go @@ -23,7 +23,8 @@ func ParseRecipient(s string) (age.Recipient, error) { } // EncryptMulti encrypts plaintext to every recipient. Any matching identity -// can decrypt the result. +// can decrypt the result. It rejects an empty recipient list and returns +// recipient parsing or age encryption errors. func EncryptMulti(recipientKeys []string, plaintext []byte) ([]byte, error) { if len(recipientKeys) == 0 { return nil, fmt.Errorf("no recipients given") diff --git a/internal/flow/backup.go b/internal/flow/backup.go index fe00bd6..8e96d55 100644 --- a/internal/flow/backup.go +++ b/internal/flow/backup.go @@ -31,7 +31,10 @@ type BackupResult struct { TotalPlainBytes int64 } -// RunBackup backs up filePath into the repository on remote using identityPath. +// RunBackup backs up a file or directory into the repository on remote using +// identityPath and returns the new snapshot UUID. Directory backups skip +// symlinks and special files. It returns filesystem, repository, encryption, +// and storage errors; failed backups may leave uploaded chunks and index epochs. func RunBackup(filePath, identityPath, remote string) (string, error) { backend := storage.NewRcloneBackend(remote) snapshotID := core.GenerateUUID() @@ -51,8 +54,9 @@ func RunBackup(filePath, identityPath, remote string) (string, error) { return res.SnapshotID, nil } -// runBackupPath dispatches to the file or tree backup flow based on whether -// filePath is a regular file or a directory. +// runBackupPath validates snapshotID as a UUID and dispatches directories to +// tree backup, all other paths to file backup. It returns stat or backup errors. +// A non-nil progress callback receives the cumulative number of processed chunks. func runBackupPath(filePath, identityPath string, backend storage.Backend, snapshotID string, progress func(int)) (BackupResult, error) { if _, err := uuid.Parse(snapshotID); err != nil { return BackupResult{}, fmt.Errorf("invalid snapshot id: %w", err) @@ -75,6 +79,11 @@ func runBackupPath(filePath, identityPath string, backend storage.Backend, snaps return runBackup(filePath, identityPath, backend, snapshotID, progress) } +// runBackup writes a file snapshot under snapshotID, reusing indexed chunks. +// progress, when non-nil, receives the cumulative chunk count including reused +// chunks. It returns statistics on success and a zero result on invalid UUIDs, +// filesystem, repository, encryption, or storage errors. Uploaded objects and +// saved index checkpoints remain after failures. func runBackup(filePath, identityPath string, backend storage.Backend, snapshotID string, progress func(int)) (BackupResult, error) { if _, err := uuid.Parse(snapshotID); err != nil { return BackupResult{}, fmt.Errorf("invalid snapshot id: %w", err) diff --git a/internal/flow/check.go b/internal/flow/check.go index ebc941d..98170f0 100644 --- a/internal/flow/check.go +++ b/internal/flow/check.go @@ -24,6 +24,8 @@ type CheckResult struct { } // RunCheck opens the repository and prints a structural consistency report. +// It checks object presence without reading chunk contents. Missing referenced +// objects and repository or manifest errors fail the check; garbage alone does not. func RunCheck(identityPath, remote string) error { backend := storage.NewRcloneBackend(remote) spinner, _ := pterm.DefaultSpinner.Start("Checking repository...") @@ -42,6 +44,11 @@ func RunCheck(identityPath, remote string) error { return nil } +// runCheck compares hashes referenced by snapshots or the index with data +// object names, returning sorted missing and garbage hashes. It does not verify +// chunk contents. Snapshot listing failures are treated as an empty listing; +// repository, index, manifest, and data listing errors are returned. Missing +// objects return an error with the report; garbage alone is not an error. func runCheck(identityPath string, backend storage.Backend) (CheckResult, error) { var res CheckResult diff --git a/internal/flow/filemode.go b/internal/flow/filemode.go index 8349226..e560124 100644 --- a/internal/flow/filemode.go +++ b/internal/flow/filemode.go @@ -14,11 +14,17 @@ import ( "gloak/internal/storage" ) +// isFileMode reports whether backupID/locator.json can be downloaded. +// It does not parse the locator, and any download error yields false. func isFileMode(backend storage.Backend, backupID string) bool { var buf bytes.Buffer return backend.Download(backupID+"/locator.json", &buf) == nil } +// readFileManifest loads a standalone backup's locator and encrypted manifest. +// It checks both UUIDs, the manifest version, and that OriginalName equals its +// filepath.Base. Locator download failures become an incomplete-or-missing +// backup error; JSON, manifest download, identity, and decryption errors are returned. func readFileManifest(backend storage.Backend, identityPath, backupID string) (fileManifest, error) { var loc fileLocator var locBuf bytes.Buffer @@ -60,6 +66,8 @@ func readFileManifest(backend storage.Backend, identityPath, backupID string) (f return m, nil } +// decodeFileMasterKey decodes the manifest's hex key, returning an error if it +// is malformed or does not contain exactly 32 bytes. func decodeFileMasterKey(m fileManifest) ([]byte, error) { key, err := hex.DecodeString(m.MasterKey) if err != nil || len(key) != masterKeySize { @@ -68,6 +76,11 @@ func decodeFileMasterKey(m fileManifest) ([]byte, error) { return key, nil } +// runFileRestore restores a standalone backup under outputDir and returns the +// output path. It creates parent directories and truncates an existing output +// file. Manifest, key, chunk path, download, decryption, checksum, and filesystem +// errors are returned; a failure can leave a partial file. File close errors +// are ignored. func runFileRestore(backupID, identityPath, outputDir string, backend storage.Backend) (string, error) { m, err := readFileManifest(backend, identityPath, backupID) if err != nil { @@ -116,6 +129,10 @@ func runFileRestore(backupID, identityPath, outputDir string, backend storage.Ba return outputPath, nil } +// runFileVerify reads a standalone backup without writing restored files. +// It verifies encrypted and plaintext chunk sizes and hashes, authentication, +// and the overall payload hash, returning manifest, key, path, download, +// decryption, or integrity errors. func runFileVerify(backupID, identityPath string, backend storage.Backend) error { m, err := readFileManifest(backend, identityPath, backupID) if err != nil { diff --git a/internal/flow/init.go b/internal/flow/init.go index 44fa799..786ed89 100644 --- a/internal/flow/init.go +++ b/internal/flow/init.go @@ -8,8 +8,9 @@ import ( ) // RunInit creates a new gloak repository on remote and prints the outcome. -// Zero min/avg/max select the package defaults. With force it overwrites an -// existing repository, which makes old snapshots unreadable. +// Zero min/avg/max select the package defaults; sizes are in bytes. With force +// it overwrites an existing repository, which makes old snapshots unreadable. +// Validation, encryption, and storage errors from repo.Init are returned. func RunInit(recipients []string, remote string, min, avg, max int64, force bool) error { if min == 0 { min = repo.DefaultChunkMin @@ -36,6 +37,8 @@ func RunInit(recipients []string, remote string, min, avg, max int64, force bool return nil } +// runInit delegates to repo.Init and returns its master key, config, and error. +// Chunk bounds are byte counts and are passed through without defaults. func runInit(recipients []string, backend storage.Backend, min, avg, max int64, force bool) ([]byte, repo.Config, error) { return repo.Init(backend, recipients, min, avg, max, force) } diff --git a/internal/flow/manifest_read.go b/internal/flow/manifest_read.go index 43d0397..4f7f1c1 100644 --- a/internal/flow/manifest_read.go +++ b/internal/flow/manifest_read.go @@ -61,8 +61,10 @@ func readTreeManifest(backend storage.Backend, identityPath, snapshotID string) return manifest, nil } -// readAnyManifest decrypts a snapshot and returns its kind. kind is -// core.KindFile or core.KindTree; exactly one of file/tree is non-nil. +// readAnyManifest decrypts a snapshot and returns its kind. On success, kind is +// core.KindFile or core.KindTree and exactly one manifest pointer is non-nil. +// An absent or empty kind means file. Invalid snapshot IDs, unknown kinds, +// download, decryption, and JSON errors are returned. func readAnyManifest(backend storage.Backend, identityPath, snapshotID string) (string, *core.Manifest, *core.TreeManifest, error) { path, err := repo.SnapshotPath(snapshotID) if err != nil { @@ -100,6 +102,8 @@ func readAnyManifest(backend storage.Backend, identityPath, snapshotID string) ( } } +// decryptSnapshotBytes decrypts sealed age data using the identity file at +// identityPath. It returns identity loading, decryption, and plaintext read errors. func decryptSnapshotBytes(identityPath string, sealed []byte) ([]byte, error) { reader, err := crypto.DecryptReader(identityPath, bytes.NewReader(sealed)) if err != nil { @@ -112,8 +116,8 @@ func decryptSnapshotBytes(identityPath string, sealed []byte) ([]byte, error) { return plain, nil } -// validateChunkPath rejects manifest chunk paths that fall outside data/ or -// contain traversal segments. +// validateChunkPath requires the data/ prefix and rejects any occurrence of +// "..". It performs no filesystem lookup or further path validation. func validateChunkPath(path string) error { if !strings.HasPrefix(path, repo.DataPrefix+"/") || strings.Contains(path, "..") { return fmt.Errorf("chunk path escapes data prefix: %s", path) diff --git a/internal/flow/prune.go b/internal/flow/prune.go index 9770635..e744abe 100644 --- a/internal/flow/prune.go +++ b/internal/flow/prune.go @@ -27,12 +27,20 @@ type PruneResult struct { } // RunPrune opens the repository and runs the prune flow. Without apply, -// nothing is deleted; the result describes the plan only. +// nothing is deleted; the result describes the plan only. It returns an error +// for an all-zero policy or a failed repository operation. An apply error may +// follow partial deletions, reflected in the returned counts. func RunPrune(identityPath, remote string, policy RetentionPolicy, apply bool) (PruneResult, error) { backend := storage.NewRcloneBackend(remote) return runPrune(identityPath, backend, policy, apply) } +// runPrune plans retention and garbage collection, deleting dead snapshots and +// unreferenced chunks and compacting the index only when apply is true. +// An all-zero policy or unreadable snapshot returns an error. Data listing +// failures are treated as no additional objects; invalid snapshot timestamps +// use the zero time. Other repository, manifest, and deletion errors return +// the results accumulated so far, without undoing earlier deletions. func runPrune(identityPath string, backend storage.Backend, policy RetentionPolicy, apply bool) (PruneResult, error) { var res PruneResult @@ -203,8 +211,10 @@ func runPrune(identityPath string, backend storage.Backend, policy RetentionPoli return res, nil } -// compactIndex rewrites the index as a single new epoch containing only the -// live hashes, then deletes prior epochs. +// compactIndex writes a new epoch for known live hashes, if any, then attempts +// to delete prior epochs. Listing failures are treated as an empty listing and +// old-epoch deletion errors are ignored. Repository opening, listing parsing, +// and compaction errors are returned before deleting old epochs. func compactIndex(backend storage.Backend, identityPath string, ix *index.Index, live map[string]bool) error { r, err := repo.Open(backend, identityPath) if err != nil { diff --git a/internal/flow/restore.go b/internal/flow/restore.go index be689aa..35a9a45 100644 --- a/internal/flow/restore.go +++ b/internal/flow/restore.go @@ -19,6 +19,11 @@ import ( "gloak/internal/storage" ) +// RunRestore restores a standalone file backup or repository snapshot to +// outputDir. For trees, pathFilter selects an exact relative path and its +// descendants; empty selects all entries. File backups ignore the filter. +// Access, decryption, integrity, and filesystem errors are returned; existing +// files may be truncated and partial output remains on failure. func RunRestore(snapshotID, identityPath, remote, outputDir, pathFilter string) error { backend := storage.NewRcloneBackend(remote) outputPath, err := runRestore(snapshotID, outputDir, identityPath, backend, pathFilter) @@ -29,6 +34,10 @@ func RunRestore(snapshotID, identityPath, remote, outputDir, pathFilter string) return nil } +// runRestore restores a backup and returns the output file path or tree root. +// A downloadable locator selects standalone file mode; any locator download +// failure falls back to the repository flow. pathFilter applies only to trees. +// Repository, manifest, and restore errors are returned without cleaning up output. func runRestore(snapshotID, outputDir, identityPath string, backend storage.Backend, pathFilter string) (string, error) { if isFileMode(backend, snapshotID) { return runFileRestore(snapshotID, identityPath, outputDir, backend) @@ -47,6 +56,11 @@ func runRestore(snapshotID, outputDir, identityPath string, backend storage.Back return runFileRestoreFromManifest(fileM, r, backend, outputDir) } +// runFileRestoreFromManifest writes a repository file snapshot under outputDir +// and returns its path, truncating any existing file. It checks chunk plaintext +// sizes and hashes and the overall payload hash. Invalid names or chunk paths, +// filesystem, download, decryption, and integrity errors can leave partial output; +// file close errors are ignored. func runFileRestoreFromManifest(manifest *core.Manifest, r *repo.Repo, backend storage.Backend, outputDir string) (string, error) { if filepath.Base(manifest.OriginalName) != manifest.OriginalName { return "", fmt.Errorf("invalid original filename in manifest: %s", manifest.OriginalName) @@ -89,6 +103,12 @@ func runFileRestoreFromManifest(manifest *core.Manifest, r *repo.Repo, backend s return outputPath, nil } +// runTreeRestore restores entries below a root named for the first source +// path, falling back to the snapshot UUID for an empty, . or root basename. +// pathFilter selects an exact slash-separated path and its descendants; empty +// selects all. It returns the destination root even when nothing matches. +// Missing source paths, unsafe entry paths, unknown selected entry types, and +// filesystem or chunk restore errors are returned; partial output remains. func runTreeRestore(m *core.TreeManifest, r *repo.Repo, outputDir, pathFilter string, backend storage.Backend) (string, error) { if len(m.Paths) == 0 { return "", fmt.Errorf("tree snapshot has no source paths") @@ -182,6 +202,11 @@ func runTreeRestore(m *core.TreeManifest, r *repo.Repo, outputDir, pathFilter st return destRoot, nil } +// restoreTreeFile creates or truncates target, writes verified plaintext chunks, +// and attempts to restore its modification time. New files use entry.Mode's +// permission bits, subject to the umask. Path, download, decryption, checksum, +// and file open/write errors are returned and can leave a partial file. +// Invalid timestamps, timestamp update failures, and close errors are ignored. func restoreTreeFile(entry core.TreeEntry, r *repo.Repo, backend storage.Backend, target string) error { mode := os.FileMode(entry.Mode) & 0777 outFile, err := os.OpenFile(target, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, mode) diff --git a/internal/flow/retention.go b/internal/flow/retention.go index b11425e..225669c 100644 --- a/internal/flow/retention.go +++ b/internal/flow/retention.go @@ -15,6 +15,7 @@ type RetentionPolicy struct { KeepMonthly int } +// isEmpty reports whether all retention counts are zero. func (p RetentionPolicy) isEmpty() bool { return p.KeepLast == 0 && p.KeepDaily == 0 && p.KeepWeekly == 0 && p.KeepMonthly == 0 } @@ -27,7 +28,9 @@ type snapshotMeta struct { // selectKeepSnapshots returns the UUIDs that survive the retention policy. // Snapshots are processed newest first. A snapshot kept by any rule stays. -// The input is mutated (sorted) in place. +// Daily, weekly, and monthly rules keep the newest snapshot in each of the +// latest occupied UTC days, ISO weeks, or months; nonpositive counts keep none. +// The input is sorted in place, with equal timestamps ordered by UUID. func selectKeepSnapshots(snaps []snapshotMeta, p RetentionPolicy) map[string]bool { keep := make(map[string]bool) diff --git a/internal/flow/snapshots.go b/internal/flow/snapshots.go index 5024d0b..fc544b9 100644 --- a/internal/flow/snapshots.go +++ b/internal/flow/snapshots.go @@ -21,6 +21,9 @@ type SnapshotInfo struct { Detail string } +// RunSnapshots prints the repository's snapshot summaries, or a message when +// the list is empty. It returns repository or snapshot loading errors and +// ignores table rendering errors. func RunSnapshots(identityPath, remote string) error { backend := storage.NewRcloneBackend(remote) infos, err := runSnapshots(identityPath, backend) @@ -38,6 +41,8 @@ func RunSnapshots(identityPath, remote string) error { return nil } +// snapshotRows returns table rows in input order, with UUID, kind, creation +// time, hostname, and detail columns, without a header. func snapshotRows(infos []SnapshotInfo) [][]string { rows := make([][]string, 0, len(infos)) for _, info := range infos { @@ -46,6 +51,10 @@ func snapshotRows(infos []SnapshotInfo) [][]string { return rows } +// runSnapshots loads file and tree snapshot summaries, sorted by descending +// creation-time text and then ascending UUID. Snapshot listing failures yield +// an empty result; repository opening, listing parsing, and manifest errors +// are returned. func runSnapshots(identityPath string, backend storage.Backend) ([]SnapshotInfo, error) { if _, err := repo.Open(backend, identityPath); err != nil { return nil, err diff --git a/internal/flow/tree_backup.go b/internal/flow/tree_backup.go index 5f6746b..42b0c98 100644 --- a/internal/flow/tree_backup.go +++ b/internal/flow/tree_backup.go @@ -20,6 +20,12 @@ import ( "gloak/internal/storage" ) +// runTreeBackup records directories and regular files in a tree snapshot, +// skipping symlinks and special files and reusing indexed chunks. A non-nil +// progress callback receives the cumulative processed chunk count. It returns +// statistics on success, or a zero result for path, filesystem, repository, +// encryption, or storage errors. Uploaded chunks and index checkpoints remain +// on failure; a failed hostname lookup leaves the hostname empty. func runTreeBackup(dirPath, identityPath string, backend storage.Backend, snapshotID string, progress func(int)) (BackupResult, error) { r, err := repo.Open(backend, identityPath) if err != nil { diff --git a/internal/flow/upload.go b/internal/flow/upload.go index f874101..88a732b 100644 --- a/internal/flow/upload.go +++ b/internal/flow/upload.go @@ -42,6 +42,8 @@ type fileLocator struct { UUID string `json:"uuid"` } +// fileChunkName formats an index as chunk_ followed by a decimal number padded +// to at least five digits. func fileChunkName(index int) string { return fmt.Sprintf("chunk_%05d", index) } @@ -54,7 +56,10 @@ func validateFileChunkName(name string) error { return nil } -// RunUpload seals filePath into a self-contained {uuid}/ folder on backend. +// RunUpload seals filePath into a self-contained {uuid}/ folder on backend and +// returns that UUID. Chunks contain up to core.ChunkSize bytes. It writes the +// locator last as a completion marker. Empty recipients, file read, encryption, +// and upload errors are returned; uploaded objects remain after a failure. func RunUpload(filePath string, recipients []string, backend storage.Backend) (string, error) { if len(recipients) == 0 { return "", fmt.Errorf("at least one recipient is required") diff --git a/internal/flow/verify.go b/internal/flow/verify.go index eee4572..b61ffe5 100644 --- a/internal/flow/verify.go +++ b/internal/flow/verify.go @@ -14,6 +14,8 @@ import ( "gloak/internal/storage" ) +// RunVerify verifies a standalone file backup or repository snapshot without +// writing restored files. Access, decryption, and integrity errors are returned. func RunVerify(snapshotID, identityPath, remote string) error { backend := storage.NewRcloneBackend(remote) spinner, _ := pterm.DefaultSpinner.Start("Verifying backup...") @@ -26,6 +28,9 @@ func RunVerify(snapshotID, identityPath, remote string) error { return nil } +// runVerify selects standalone verification when a locator can be downloaded, +// otherwise it opens the repository and verifies the snapshot's file or tree +// contents. It returns repository, manifest, and verification errors. func runVerify(snapshotID, identityPath string, backend storage.Backend) error { if isFileMode(backend, snapshotID) { return runFileVerify(snapshotID, identityPath, backend) @@ -44,6 +49,9 @@ func runVerify(snapshotID, identityPath string, backend storage.Backend) error { return runFileVerifyFromManifest(fileM, r, backend) } +// runFileVerifyFromManifest verifies encrypted and plaintext chunk sizes and +// hashes, authentication, and the overall payload hash without restoring files. +// It returns chunk path, download, decryption, and integrity errors. func runFileVerifyFromManifest(manifest *core.Manifest, r *repo.Repo, backend storage.Backend) error { payloadHasher := sha256.New() for _, meta := range manifest.Chunks { @@ -77,6 +85,9 @@ func runFileVerifyFromManifest(manifest *core.Manifest, r *repo.Repo, backend st return nil } +// runTreeVerify verifies paths and encrypted and plaintext chunk sizes and +// hashes for file entries, ignoring other entry types. It returns path, +// download, decryption, and integrity errors without writing restored files. func runTreeVerify(m *core.TreeManifest, r *repo.Repo, backend storage.Backend) error { for _, entry := range m.Entries { if entry.Type != "file" { diff --git a/internal/index/index.go b/internal/index/index.go index 3d221d9..88689f4 100644 --- a/internal/index/index.go +++ b/internal/index/index.go @@ -34,6 +34,7 @@ type epochFile struct { Entries []Entry `json:"entries"` } +// New returns an empty index with no pending entries and an epoch counter of zero. func New() *Index { return &Index{ pending: make(map[string]Entry), @@ -41,11 +42,12 @@ func New() *Index { } } -// Load merges every index epoch in ascending name order. Epochs that cannot -// be decrypted with the identity are skipped: the caller still has the data -// objects, and the next Save rewrites what is missing. The epoch counter is -// restored from the highest epoch name seen so a later Save never reuses a -// name that already exists on the backend. +// Load merges readable index files in ascending name order, with later entries +// replacing earlier entries for the same plaintext hash. Listing failures yield +// an empty index; malformed listing JSON returns an error. Files that cannot be +// downloaded, decrypted, read, or parsed are skipped. Skipped entries are not +// queued for Save. The epoch counter is restored from the highest recognized +// epoch name, including skipped files. func Load(backend storage.Backend, identityPath string) (*Index, error) { ix := New() @@ -127,6 +129,7 @@ func parseEpoch(path string) (int, bool) { return epoch, true } +// Has reports whether a plaintext hash is known, including pending entries. func (ix *Index) Has(hash string) bool { _, ok := ix.Get(hash) return ok @@ -150,6 +153,8 @@ func (ix *Index) AllHashes() []string { return out } +// Get looks up a plaintext hash, preferring a pending entry. It returns the +// zero entry and false when the hash is unknown. func (ix *Index) Get(hash string) (Entry, bool) { if e, ok := ix.pending[hash]; ok { return e, true @@ -158,15 +163,17 @@ func (ix *Index) Get(hash string) (Entry, bool) { return e, ok } +// Add records an entry by its plaintext hash and queues it for the next Save, +// replacing any entry with the same hash. func (ix *Index) Add(entry Entry) { ix.pending[entry.PlainSHA256] = entry ix.all[entry.PlainSHA256] = entry } -// Compact writes a single new epoch containing only liveHashes (intersected -// with known entries), then returns. Callers delete old epochs separately. -// The epoch counter continues from the highest existing name so the new -// file never collides with a prior epoch. +// Compact writes a new epoch containing known entries marked true in liveHashes +// and returns the number written, or zero without writing if none match. +// It leaves ix unchanged and returns encryption or upload errors from Save. +// Callers delete old epochs separately. func (ix *Index) Compact(backend storage.Backend, recipients []string, liveHashes map[string]bool) (int, error) { // Reuse Save by replacing all and pending with the live subset, then // force a save even if pending is empty (when there is something to write). @@ -186,8 +193,10 @@ func (ix *Index) Compact(backend storage.Backend, recipients []string, liveHashe return next.Save(backend, recipients) } -// Save writes only the pending entries as the next epoch. It is a no-op when -// nothing is pending, so a second backup of unchanged data writes no files. +// Save writes only the pending entries as the next encrypted epoch and returns +// the number written. It is a no-op when nothing is pending. Pending entries +// are cleared and the epoch counter advances only after a successful upload; +// serialization, encryption, and upload errors leave them unchanged. func (ix *Index) Save(backend storage.Backend, recipients []string) (int, error) { if len(ix.pending) == 0 { return 0, nil diff --git a/internal/notify/telegram.go b/internal/notify/telegram.go index 22624cc..baef70e 100644 --- a/internal/notify/telegram.go +++ b/internal/notify/telegram.go @@ -31,8 +31,9 @@ func (c TelegramConfig) Enabled() bool { return c.Token != "" && c.ChatID != "" } -// Send posts a short text report. It never returns an error that should -// fail the host command; callers log a warning. +// Send posts message to Telegram, truncating it to 4000 bytes with an ellipsis. +// It does nothing when credentials are incomplete. It uses a 15-second HTTP +// timeout and returns request errors or errors for non-200 responses. func (c TelegramConfig) Send(message string) error { if !c.Enabled() { return nil @@ -73,6 +74,8 @@ func redact(msg, token string) string { return strings.ReplaceAll(msg, token, "[redacted]") } +// truncate limits s to n bytes, ending shortened text with "...". It may split +// a UTF-8 sequence and requires n >= 3 when truncation is needed. func truncate(s string, n int) string { if len(s) <= n { return s diff --git a/internal/repo/repo.go b/internal/repo/repo.go index a7df7f1..2167d56 100644 --- a/internal/repo/repo.go +++ b/internal/repo/repo.go @@ -48,8 +48,11 @@ func SnapshotPath(snapshotID string) (string, error) { // Init creates the repository on the backend: one wrapped master key per // recipient first, repo.json last as the completion marker. It refuses to -// touch a repository that already exists unless force is set, because a -// fresh master key would orphan every snapshot sealed with the old one. +// overwrite a readable repo.json unless force is set; download errors during +// that existence check are treated as absence. A fresh master key makes old +// snapshots unreadable. min, avg, and max are positive, ordered byte counts. +// It returns the new master key and config, or a validation, encryption, or +// storage error. Failed initialization may leave uploaded objects behind. func Init(backend storage.Backend, recipients []string, min, avg, max int64, force bool) ([]byte, Config, error) { if len(recipients) == 0 { return nil, Config{}, fmt.Errorf("at least one recipient is required") @@ -145,7 +148,10 @@ func deleteStaleKeys(backend storage.Backend, recipients []string) error { return nil } -// Open loads repo.json and unwraps the master key with the given identity. +// Open loads repo.json and unwraps a 32-byte master key using identityPath. +// Config download, parsing, format, chunk-bound, and key-listing errors are +// returned. Individual key download, decryption, and read failures are skipped; +// if no key opens, it returns an identity-cannot-open error. func Open(backend storage.Backend, identityPath string) (*Repo, error) { var buf bytes.Buffer if err := backend.Download(ConfigPath, &buf); err != nil { diff --git a/internal/storage/list.go b/internal/storage/list.go index 502fcd8..cc193dc 100644 --- a/internal/storage/list.go +++ b/internal/storage/list.go @@ -8,6 +8,8 @@ type ListEntry struct { IsDir bool `json:"IsDir"` } +// ParseList decodes rclone lsjson output, returning a JSON decoding error for +// malformed or incompatible input. Unknown fields are ignored. func ParseList(raw []byte) ([]ListEntry, error) { var entries []ListEntry if err := json.Unmarshal(raw, &entries); err != nil { diff --git a/internal/storage/rclone.go b/internal/storage/rclone.go index a9f13b8..555aacc 100644 --- a/internal/storage/rclone.go +++ b/internal/storage/rclone.go @@ -23,10 +23,14 @@ type Backend interface { Delete(remotePath string) error } +// NewRcloneBackend uses remote as the storage root and leaves rclone's default +// configuration in effect until ConfigPath is set. func NewRcloneBackend(remote string) *RcloneBackend { return &RcloneBackend{Remote: remote} } +// rcloneArgs copies base and appends a --config option only when configPath is +// nonempty, leaving base unchanged. func rcloneArgs(configPath string, base ...string) []string { args := append([]string{}, base...) if configPath != "" { @@ -47,6 +51,8 @@ func IsMissingPath(err error) bool { return false } +// Upload streams in to remotePath relative to Remote, creating or replacing +// the object. It returns rclone execution errors, including input read failures. func (r *RcloneBackend) Upload(remotePath string, in io.Reader) error { fullDest := fmt.Sprintf("%s/%s", r.Remote, remotePath) @@ -63,6 +69,9 @@ func (r *RcloneBackend) Upload(remotePath string, in io.Reader) error { return nil } +// Download writes the object at remotePath relative to Remote to out. +// It returns rclone execution errors, including output write failures; out may +// have received partial data when an error is returned. func (r *RcloneBackend) Download(remotePath string, out io.Writer) error { fullSrc := fmt.Sprintf("%s/%s", r.Remote, remotePath) diff --git a/main.go b/main.go index 8b52a15..a1fc543 100644 --- a/main.go +++ b/main.go @@ -26,8 +26,9 @@ var cli struct { } // getDefaultDir returns the directory for identity, recipient, and helper -// files. GLOAK_DIR overrides the default (~/.gloak). A leading ~ in -// GLOAK_DIR is expanded. +// files. GLOAK_DIR overrides the default (~/.gloak); ~ and a leading ~/ are +// expanded when the home directory is available. If home lookup fails, it +// returns GLOAK_DIR unchanged when set, or .gloak relative to the current directory. func getDefaultDir() string { if dir := os.Getenv("GLOAK_DIR"); dir != "" { if expanded, err := expandHome(dir); err == nil { diff --git a/notify.go b/notify.go index 62e593e..29c7da0 100644 --- a/notify.go +++ b/notify.go @@ -13,9 +13,10 @@ import ( ) // resolveTelegram merges Telegram credentials from the config file with -// environment overrides. The file at /config.toml provides the -// base values; GLOAK_TELEGRAM_TOKEN and GLOAK_TELEGRAM_CHAT_ID win when set. -// A malformed config file only warns; notify stays best-effort. +// nonempty environment overrides. The file at /config.toml provides +// the base values; GLOAK_TELEGRAM_TOKEN and GLOAK_TELEGRAM_CHAT_ID win when nonempty. +// Unreadable or malformed configuration is ignored, leaving environment values +// available for notifications. func resolveTelegram() notify.TelegramConfig { fileCfg, err := config.Load(config.Path(getDefaultDir())) if err != nil { @@ -35,9 +36,8 @@ func resolveTelegram() notify.TelegramConfig { } // notifyCommand sends a short best-effort report after a successful command. -// details are optional lines rendered as-is under the header. Missing -// credentials or network failures only print a warning; they never fail the -// host command. The token itself is never printed. +// Nonempty details are rendered as-is under the header. Missing credentials +// skip the send; send failures do not fail the host command. func notifyCommand(command string, details ...string) { cfg := resolveTelegram() if !cfg.Enabled() { diff --git a/prune.go b/prune.go index 6ba9079..4224276 100644 --- a/prune.go +++ b/prune.go @@ -20,6 +20,8 @@ type PruneCmd struct { Yes bool `help:"Actually delete. Without this flag prune is a dry run."` } +// Run prints a retention plan, applying deletions only when Yes is set. +// It returns prune errors and sends an optional notification after a successful apply. func (c *PruneCmd) Run() error { pterm.Info.Println("Starting PRUNE process...") ident := c.Identity diff --git a/restore.go b/restore.go index 8878d4d..7aca10d 100644 --- a/restore.go +++ b/restore.go @@ -17,6 +17,9 @@ type RestoreCmd struct { PathFilter string `help:"Restore only this relative path (and children) from a tree snapshot."` } +// Run restores the backup using the selected or default identity and sends an +// optional completion notification. Restore errors are returned, and failed +// restores may leave partial output. func (r *RestoreCmd) Run() error { pterm.Info.Println("Starting RESTORE process...") diff --git a/snapshots.go b/snapshots.go index cb1c232..4c0b4ea 100644 --- a/snapshots.go +++ b/snapshots.go @@ -13,6 +13,8 @@ type SnapshotsCmd struct { Remote string `help:"Source Rclone URI (e.g., keraaaki:backup_folder or /mnt/backups)." required:"" short:"R"` } +// Run lists repository snapshots using the selected or default identity, +// returning repository access or snapshot loading errors. func (c *SnapshotsCmd) Run() error { pterm.Info.Println("Listing snapshots...") ident := c.Identity diff --git a/upload.go b/upload.go index 82faa92..760ef73 100644 --- a/upload.go +++ b/upload.go @@ -16,6 +16,8 @@ type UploadCmd struct { Remote string `short:"R" required:"" help:"Remote root, e.g. keraaaki:backups or /mnt/backups"` } +// Run resolves recipients and uploads a standalone encrypted backup, printing +// its UUID on success. Recipient resolution and upload errors are returned. func (u *UploadCmd) Run() error { recipients, err := resolveInitRecipients(u.Recipient) if err != nil { diff --git a/verify.go b/verify.go index 05a70b5..6a088d8 100644 --- a/verify.go +++ b/verify.go @@ -15,6 +15,9 @@ type VerifyCmd struct { Remote string `help:"Source Rclone URI (e.g., keraaaki:backup_folder or /mnt/backups)." required:"" short:"R"` } +// Run verifies the backup using the selected or default identity and sends an +// optional completion notification. Access, decryption, and integrity errors +// are returned. func (v *VerifyCmd) Run() error { pterm.Info.Println("Starting VERIFY process...") From 8cf46e58c098b59ea7bf0e21b9033a3600f18bff Mon Sep 17 00:00:00 2001 From: Syrup Date: Tue, 13 Oct 2026 09:32:06 +0700 Subject: [PATCH 43/48] fix: address review findings from PR #5 - pin the release workflow actions to full commit SHAs and stop checkout from persisting credentials - point the check garbage warning at gloak prune --yes instead of the old "later phase" text - check the output file close error in file restore before reporting success, since some filesystems surface write errors only at close - truncate Telegram text on a rune boundary so messages stay valid UTF-8, with tests --- .github/workflows/release.yml | 7 ++++--- internal/flow/check.go | 2 +- internal/flow/filemode.go | 8 ++++++-- internal/notify/telegram.go | 12 +++++++++--- internal/notify/telegram_test.go | 20 ++++++++++++++++++++ 5 files changed, 40 insertions(+), 9 deletions(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index a3a3ac1..a16f013 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -12,15 +12,16 @@ jobs: goreleaser: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0 with: fetch-depth: 0 + persist-credentials: false - - uses: actions/setup-go@v5 + - uses: actions/setup-go@40f1582b2485089dde7abd97c1529aa768e1baff # v5.6.0 with: go-version-file: go.mod - - uses: goreleaser/goreleaser-action@v7 + - uses: goreleaser/goreleaser-action@a4f02ab5355a589de60d917a659ce617ed19b3de # v7.3.0 with: distribution: goreleaser version: "~> v2" diff --git a/internal/flow/check.go b/internal/flow/check.go index 98170f0..c40ba68 100644 --- a/internal/flow/check.go +++ b/internal/flow/check.go @@ -39,7 +39,7 @@ func RunCheck(identityPath, remote string) error { pterm.Printf("Referenced chunks: %d\n", res.Referenced) pterm.Printf("Data objects: %d\n", res.DataObjects) if len(res.Garbage) > 0 { - pterm.Warning.Printf("Garbage objects: %d (run prune in a later phase)\n", len(res.Garbage)) + pterm.Warning.Printf("Garbage objects: %d (run `gloak prune --yes` to remove them)\n", len(res.Garbage)) } return nil } diff --git a/internal/flow/filemode.go b/internal/flow/filemode.go index e560124..efd858b 100644 --- a/internal/flow/filemode.go +++ b/internal/flow/filemode.go @@ -79,8 +79,9 @@ func decodeFileMasterKey(m fileManifest) ([]byte, error) { // runFileRestore restores a standalone backup under outputDir and returns the // output path. It creates parent directories and truncates an existing output // file. Manifest, key, chunk path, download, decryption, checksum, and filesystem -// errors are returned; a failure can leave a partial file. File close errors -// are ignored. +// errors are returned; a failure can leave a partial file. The success path +// checks the output file's close error; close errors on failure paths are +// ignored. func runFileRestore(backupID, identityPath, outputDir string, backend storage.Backend) (string, error) { m, err := readFileManifest(backend, identityPath, backupID) if err != nil { @@ -126,6 +127,9 @@ func runFileRestore(backupID, identityPath, outputDir string, backend storage.Ba if hex.EncodeToString(payloadHasher.Sum(nil)) != m.PayloadSHA256 { return "", fmt.Errorf("fatal: overall payload hash does not match manifest") } + if err := outFile.Close(); err != nil { + return "", fmt.Errorf("failed to close output file: %w", err) + } return outputPath, nil } diff --git a/internal/notify/telegram.go b/internal/notify/telegram.go index baef70e..fba3e95 100644 --- a/internal/notify/telegram.go +++ b/internal/notify/telegram.go @@ -8,6 +8,7 @@ import ( "os" "strings" "time" + "unicode/utf8" ) // TelegramConfig holds Bot API credentials. Empty token disables notify. @@ -74,11 +75,16 @@ func redact(msg, token string) string { return strings.ReplaceAll(msg, token, "[redacted]") } -// truncate limits s to n bytes, ending shortened text with "...". It may split -// a UTF-8 sequence and requires n >= 3 when truncation is needed. +// truncate limits s to n bytes, ending shortened text with "...". The cut +// lands on a rune boundary so the result stays valid UTF-8, and n >= 3 is +// required when truncation is needed. func truncate(s string, n int) string { if len(s) <= n { return s } - return s[:n-3] + "..." + cut := n - 3 + for cut > 0 && !utf8.RuneStart(s[cut]) { + cut-- + } + return s[:cut] + "..." } diff --git a/internal/notify/telegram_test.go b/internal/notify/telegram_test.go index d6650e6..9ba6e4d 100644 --- a/internal/notify/telegram_test.go +++ b/internal/notify/telegram_test.go @@ -7,6 +7,7 @@ import ( "net/url" "strings" "testing" + "unicode/utf8" ) func TestSendDisabledNoops(t *testing.T) { @@ -79,3 +80,22 @@ func TestSendRedactsTokenFromTransportError(t *testing.T) { t.Fatalf("error = %v, want the token placeholder", err) } } + +func TestTruncateKeepsUTF8(t *testing.T) { + // "€" occupies bytes 2-4; a plain byte cut at n-3 would land inside it. + s := "aa€" + strings.Repeat("b", 10) + got := truncate(s, 6) + if got != "aa..." { + t.Fatalf("truncate() = %q, want %q", got, "aa...") + } + if !utf8.ValidString(got) { + t.Fatalf("truncate() = %q is not valid UTF-8", got) + } +} + +func TestTruncateShortInputUnchanged(t *testing.T) { + s := "aa€bb" + if got := truncate(s, len(s)); got != s { + t.Fatalf("truncate() = %q, want %q", got, s) + } +} From afe74dd853cd6936e08bf51c75793fbab7e8da74 Mon Sep 17 00:00:00 2001 From: Syrup Date: Tue, 13 Oct 2026 11:15:04 +0700 Subject: [PATCH 44/48] fix: address follow-up review findings from PR #5 - disable persisted git credentials in the CodeQL workflow checkout - clean and validate the restore path filter before matching entries - reject symlinked components at restore targets before creating or writing files --- .github/workflows/codeql.yml | 2 + internal/flow/restore.go | 88 +++++++++++++-- internal/flow/tree_restore_test.go | 165 +++++++++++++++++++++++++++++ 3 files changed, 247 insertions(+), 8 deletions(-) diff --git a/.github/workflows/codeql.yml b/.github/workflows/codeql.yml index 27bae65..81d2c84 100644 --- a/.github/workflows/codeql.yml +++ b/.github/workflows/codeql.yml @@ -21,6 +21,8 @@ jobs: steps: - name: Checkout repository uses: actions/checkout@v6 + with: + persist-credentials: false - name: Set up Go uses: actions/setup-go@v6 diff --git a/internal/flow/restore.go b/internal/flow/restore.go index 35a9a45..268ca48 100644 --- a/internal/flow/restore.go +++ b/internal/flow/restore.go @@ -6,6 +6,7 @@ import ( "encoding/hex" "fmt" "os" + "path" "path/filepath" "sort" "strings" @@ -105,11 +106,17 @@ func runFileRestoreFromManifest(manifest *core.Manifest, r *repo.Repo, backend s // runTreeRestore restores entries below a root named for the first source // path, falling back to the snapshot UUID for an empty, . or root basename. -// pathFilter selects an exact slash-separated path and its descendants; empty -// selects all. It returns the destination root even when nothing matches. -// Missing source paths, unsafe entry paths, unknown selected entry types, and -// filesystem or chunk restore errors are returned; partial output remains. +// pathFilter is cleaned and validated as a relative path first, then selects +// an exact slash-separated path and its descendants; empty selects all. +// It returns the destination root even when nothing matches. +// Missing source paths, unsafe entry paths, invalid filters, unknown selected +// entry types, and filesystem or chunk restore errors are returned; partial +// output remains. func runTreeRestore(m *core.TreeManifest, r *repo.Repo, outputDir, pathFilter string, backend storage.Backend) (string, error) { + pathFilter, err := normalizePathFilter(pathFilter) + if err != nil { + return "", err + } if len(m.Paths) == 0 { return "", fmt.Errorf("tree snapshot has no source paths") } @@ -156,6 +163,11 @@ func runTreeRestore(m *core.TreeManifest, r *repo.Repo, outputDir, pathFilter st if !strings.HasPrefix(target, destRoot+string(filepath.Separator)) && target != destRoot { return "", fmt.Errorf("restored path escapes output root: %s", entry.RelPath) } + // The manifest is remote input: reject targets reached through a + // symlink below the destination root before creating or writing. + if err := rejectSymlinkRestorePath(destRoot, target); err != nil { + return "", err + } switch entry.Type { case "dir": // Keep the directory writable while children are restored; @@ -202,13 +214,73 @@ func runTreeRestore(m *core.TreeManifest, r *repo.Repo, outputDir, pathFilter st return destRoot, nil } +// normalizePathFilter cleans a user-supplied tree restore filter and validates +// it as a relative path. Empty input stays empty and "." selects everything. +// Filters that are unsafe after cleaning are rejected. +func normalizePathFilter(pathFilter string) (string, error) { + if pathFilter == "" { + return "", nil + } + cleaned := path.Clean(filepath.ToSlash(pathFilter)) + if cleaned == "." { + return "", nil + } + if err := core.ValidateRelPath(cleaned); err != nil { + return "", fmt.Errorf("invalid path filter: %w", err) + } + return cleaned, nil +} + +// rejectSymlinkRestorePath returns an error when destRoot or any component of +// target below it already exists as a symlink. A string check on the joined +// path cannot see links planted under the destination, so every component is +// inspected before restore creates or writes through it. Components that do +// not exist yet are left for creation. +func rejectSymlinkRestorePath(destRoot, target string) error { + inspect := func(p string) error { + fi, err := os.Lstat(p) + if err != nil { + if os.IsNotExist(err) { + return nil + } + return fmt.Errorf("failed to inspect restore path %s: %w", p, err) + } + if fi.Mode()&os.ModeSymlink != 0 { + return fmt.Errorf("refusing to restore through symlink: %s", p) + } + return nil + } + if err := inspect(destRoot); err != nil { + return err + } + rel, err := filepath.Rel(destRoot, target) + if err != nil { + return fmt.Errorf("invalid restore path: %w", err) + } + cur := destRoot + for _, part := range strings.Split(rel, string(filepath.Separator)) { + if part == "." || part == "" { + continue + } + cur = filepath.Join(cur, part) + if err := inspect(cur); err != nil { + return err + } + } + return nil +} + // restoreTreeFile creates or truncates target, writes verified plaintext chunks, -// and attempts to restore its modification time. New files use entry.Mode's -// permission bits, subject to the umask. Path, download, decryption, checksum, -// and file open/write errors are returned and can leave a partial file. -// Invalid timestamps, timestamp update failures, and close errors are ignored. +// and attempts to restore its modification time. An existing symlink at the +// target is rejected. New files use entry.Mode's permission bits, subject to +// the umask. Path, download, decryption, checksum, and file open/write errors +// are returned and can leave a partial file. Invalid timestamps, timestamp +// update failures, and close errors are ignored. func restoreTreeFile(entry core.TreeEntry, r *repo.Repo, backend storage.Backend, target string) error { mode := os.FileMode(entry.Mode) & 0777 + if fi, err := os.Lstat(target); err == nil && fi.Mode()&os.ModeSymlink != 0 { + return fmt.Errorf("refusing to restore %s over a symlink", entry.RelPath) + } outFile, err := os.OpenFile(target, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, mode) if err != nil { // An earlier restore may have left the file read-only; make it diff --git a/internal/flow/tree_restore_test.go b/internal/flow/tree_restore_test.go index bb26972..f9df77b 100644 --- a/internal/flow/tree_restore_test.go +++ b/internal/flow/tree_restore_test.go @@ -249,6 +249,171 @@ func TestRunTreeRestoreAppliesDirModesAfterChildren(t *testing.T) { } } +func TestNormalizePathFilter(t *testing.T) { + cases := []struct { + in string + want string + wantErr bool + }{ + {"", "", false}, + {"sub", "sub", false}, + {"sub/", "sub", false}, + {"./sub", "sub", false}, + {"sub//b.txt", "sub/b.txt", false}, + {"sub/../sub", "sub", false}, + {".", "", false}, + {"..", "", true}, + {"../x", "", true}, + {"a/../../x", "", true}, + {"/abs", "", true}, + {`back\slash`, "", true}, + } + for _, c := range cases { + got, err := normalizePathFilter(c.in) + if c.wantErr { + if err == nil { + t.Errorf("normalizePathFilter(%q) = %q, want error", c.in, got) + } + continue + } + if err != nil { + t.Errorf("normalizePathFilter(%q) error: %v", c.in, err) + continue + } + if got != c.want { + t.Errorf("normalizePathFilter(%q) = %q, want %q", c.in, got, c.want) + } + } +} + +func TestRejectSymlinkRestorePath(t *testing.T) { + base := t.TempDir() + destRoot := filepath.Join(base, "dest") + // A destination root that does not exist yet is fine. + if err := rejectSymlinkRestorePath(destRoot, filepath.Join(destRoot, "a.txt")); err != nil { + t.Fatalf("missing destRoot: %v", err) + } + if err := os.MkdirAll(filepath.Join(destRoot, "sub"), 0755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(destRoot, "a.txt"), []byte("x"), 0644); err != nil { + t.Fatal(err) + } + if err := rejectSymlinkRestorePath(destRoot, filepath.Join(destRoot, "sub", "b.txt")); err != nil { + t.Fatalf("regular path: %v", err) + } + victim := filepath.Join(base, "victim.txt") + if err := os.WriteFile(victim, []byte("keep"), 0644); err != nil { + t.Fatal(err) + } + if err := os.Symlink(victim, filepath.Join(destRoot, "link.txt")); err != nil { + t.Fatal(err) + } + if err := rejectSymlinkRestorePath(destRoot, filepath.Join(destRoot, "link.txt")); err == nil || + !strings.Contains(err.Error(), "symlink") { + t.Fatalf("final component symlink: err = %v, want symlink rejection", err) + } + outside := filepath.Join(base, "outside") + if err := os.MkdirAll(outside, 0755); err != nil { + t.Fatal(err) + } + if err := os.Symlink(outside, filepath.Join(destRoot, "evil")); err != nil { + t.Fatal(err) + } + if err := rejectSymlinkRestorePath(destRoot, filepath.Join(destRoot, "evil", "x.txt")); err == nil || + !strings.Contains(err.Error(), "symlink") { + t.Fatalf("intermediate symlink: err = %v, want symlink rejection", err) + } + linkRoot := filepath.Join(base, "linkroot") + if err := os.Symlink(outside, linkRoot); err != nil { + t.Fatal(err) + } + if err := rejectSymlinkRestorePath(linkRoot, filepath.Join(linkRoot, "x.txt")); err == nil || + !strings.Contains(err.Error(), "symlink") { + t.Fatalf("destRoot symlink: err = %v, want symlink rejection", err) + } +} + +func TestRestoreTreeFileRejectsSymlinkTarget(t *testing.T) { + dir := t.TempDir() + victim := filepath.Join(dir, "victim.txt") + if err := os.WriteFile(victim, []byte("untouched"), 0644); err != nil { + t.Fatal(err) + } + link := filepath.Join(dir, "link.txt") + if err := os.Symlink(victim, link); err != nil { + t.Fatal(err) + } + entry := core.TreeEntry{RelPath: "link.txt", Type: "file", Mode: 0644} + // The symlink guard runs before any repo or backend access, so nil is safe. + err := restoreTreeFile(entry, nil, nil, link) + if err == nil || !strings.Contains(err.Error(), "symlink") { + t.Fatalf("err = %v, want symlink rejection", err) + } +} + +func TestRunTreeRestoreFilterNormalizesSlashes(t *testing.T) { + _, backend, ident, snap := treeFixture(t) + for _, filter := range []string{"sub/", "./sub"} { + out := t.TempDir() + got, err := runRestore(snap, out, ident, backend, filter) + if err != nil { + t.Fatalf("filter %q: %v", filter, err) + } + if _, err := os.Stat(filepath.Join(got, "a.txt")); err == nil { + t.Fatalf("filter %q: a.txt restored despite filter", filter) + } + b, err := os.ReadFile(filepath.Join(got, "sub", "b.txt")) + if err != nil || string(b) != "beta" { + t.Fatalf("filter %q: sub/b.txt = %q, err = %v", filter, b, err) + } + } +} + +func TestRunTreeRestoreRejectsSymlinkTarget(t *testing.T) { + root, backend, ident, snap := treeFixture(t) + out := t.TempDir() + dest := filepath.Join(out, filepath.Base(root)) + if err := os.MkdirAll(dest, 0755); err != nil { + t.Fatal(err) + } + victim := filepath.Join(t.TempDir(), "victim.txt") + if err := os.WriteFile(victim, []byte("untouched"), 0644); err != nil { + t.Fatal(err) + } + if err := os.Symlink(victim, filepath.Join(dest, "a.txt")); err != nil { + t.Fatal(err) + } + _, err := runRestore(snap, out, ident, backend, "") + if err == nil || !strings.Contains(err.Error(), "symlink") { + t.Fatalf("err = %v, want symlink rejection", err) + } + got, err := os.ReadFile(victim) + if err != nil || string(got) != "untouched" { + t.Fatalf("victim = %q, err = %v, want it untouched", got, err) + } +} + +func TestRunTreeRestoreRejectsSymlinkDirectory(t *testing.T) { + root, backend, ident, snap := treeFixture(t) + out := t.TempDir() + dest := filepath.Join(out, filepath.Base(root)) + if err := os.MkdirAll(dest, 0755); err != nil { + t.Fatal(err) + } + outside := t.TempDir() + if err := os.Symlink(outside, filepath.Join(dest, "sub")); err != nil { + t.Fatal(err) + } + _, err := runRestore(snap, out, ident, backend, "") + if err == nil || !strings.Contains(err.Error(), "symlink") { + t.Fatalf("err = %v, want symlink rejection", err) + } + if _, err := os.Stat(filepath.Join(outside, "b.txt")); err == nil { + t.Fatal("restore escaped through a symlinked directory") + } +} + func seedTreeSnapshot(t *testing.T, backend *memtest.MemoryBackend, ident string, m core.TreeManifest) { t.Helper() r, err := repo.Open(backend, ident) From 529de804018ee9a7f434ca3ff1dc084a88efdee0 Mon Sep 17 00:00:00 2001 From: Syrup Date: Tue, 13 Oct 2026 11:35:39 +0700 Subject: [PATCH 45/48] docs: drop resume and cleanup from README The CLI never shipped resume or cleanup commands and has no local upload state, so the workflow diagram, quick start, feature bullets, design notes, and command table no longer mention them. --- README.md | 331 ++++++------------------------------------------------ 1 file changed, 32 insertions(+), 299 deletions(-) diff --git a/README.md b/README.md index be19795..3867a86 100644 --- a/README.md +++ b/README.md @@ -17,12 +17,14 @@ > > Use it for testing, non-critical data, or secondary backups only. If you store anything important with gloak, keep another verified backup somewhere else. -gloak splits a file into fixed-size chunks, encrypts each chunk with age, uploads them, and verifies them before restore. Metadata such as filenames, sizes, recipients, and hashes lives inside an age-encrypted manifest. Storage only sees backup IDs, public completion markers, and encrypted chunk objects. +gloak splits one file into fixed-size chunks, encrypts each chunk with age, uploads them, and verifies every chunk before restore. Metadata such as filenames, sizes, recipients, and hashes lives inside an age-encrypted manifest. Storage only sees backup IDs, public completion markers, and encrypted chunk objects. ```text -keygen → chunk → encrypt chunks → upload → resume if needed → verify → restore +keygen -> chunk -> encrypt chunks -> upload -> verify -> restore ``` +Full usage guide: [docs/GUIDE.md](docs/GUIDE.md) + --- ## Quick start @@ -37,12 +39,9 @@ go build -o gloak . # generate keys ./gloak keygen -# encrypt + upload +# upload ./gloak upload ./secret.tar --remote myremote:backup_folder -# resume an interrupted upload -./gloak resume - # verify ./gloak verify --remote myremote:backup_folder -i ~/.gloak/identity.txt @@ -50,334 +49,68 @@ go build -o gloak . ./gloak restore --remote myremote:backup_folder --output-dir ./restored/ ``` ---- - -## Why this exists - -Most backup tools make at least one tradeoff gloak avoids: - -1. Encrypt a single blob. If a network failure hits at byte 47 GB of 50, retrying is painful. -2. Split into chunks without enough integrity metadata. You find corruption during restore, without a manifest to guide repair. -3. Store metadata in plaintext. Filenames, sizes, recipient keys, and chunk hashes are visible to anyone who can read the storage. -4. Pull in rclone, restic rest-server, or S3 libraries when you just need local filesystem or a mounted remote. - -gloak keeps the design narrow: age encryption, fixed-size chunks, chunk and payload hashes, resumable uploads, and an encrypted manifest so the storage backend does not see filenames or hashes. - ---- - -## Install - -```bash -go mod download -``` - -Build the standalone binary: - -```bash -go build -o gloak . -``` - -This produces `gloak`, a compiled binary you can put in containers, cron jobs, or VPS root filesystems without installing Node, Bun, or Python. - -Run tests: - -```bash -go test ./... -``` - ---- - -## Key generation - -Generate a keypair once: - -```bash -./gloak keygen -``` - -Default location is `~/.gloak/`: - -```text -~/.gloak/ - identity.txt # private key, keep this secret - recipient.txt # public key, safe to share -``` - -Use a custom directory: - -```bash -./gloak keygen --output-dir /path/to/keys -./gloak keygen -o /path/to/keys -``` - -After `keygen` runs once, `-i` (identity) and `-r` (recipient) flags become optional. gloak reads from `~/.gloak/` automatically. - -### Key flags - -| Flag | Alias | Description | -|------|-------|-------------| -| `-r ` | `--recipient` | Recipient public key file. Overrides `~/.gloak/recipient.txt`. | -| `-i ` | `--identity` | Identity private key file. Overrides `~/.gloak/identity.txt`. | - ---- - -## Commands - -### `setup`: install dependencies - -If you don't have `rclone` installed on your system, `gloak setup` will download a standalone, verified binary to `~/.gloak/bin/rclone` so `gloak` can use it without touching your system packages. +Need rclone? Run: ```bash ./gloak setup ``` -| Option | Alias | Description | -|--------|-------|-------------| -| `--dir ` | `-D` | Custom install directory (default: `~/.gloak/bin`). | -| `--update` | `-u` | Force re-download even if `rclone` already exists. | - ---- - -### `upload`: create an encrypted backup - -Creates a UUID folder, splits the input into fixed-size plaintext chunks, encrypts each chunk independently with age, records SHA-256 checksums in an age-encrypted manifest, and uploads chunks first. `manifest.age` and `locator.json` are uploaded after the chunks. - -Chunks and manifest are encrypted with the same age recipient. Storage sees filenames like `chunk_00000` and `manifest.age` but cannot read filenames, sizes, or hashes without the identity key. - -```bash -# using auto-resolved keys -./gloak upload ./backup.tar --remote myremote:backup_folder - -# with explicit recipient -./gloak upload ./backup.tar \ - -r /path/to/recipient.txt \ - --remote myremote:backup_folder -``` - -Output: - -```text -Backup ID: 7f91c6c7-7a0b-44aa-ae23-997b60e4e998 -Backup completed successfully! -Save this UUID for restoration: 7f91c6c7-7a0b-44aa-ae23-997b60e4e998 -``` - -| Option | Alias | Default | Description | -|--------|-------|---------|-------------| -| `--recipient ` | `-r` | `~/.gloak/recipient.txt` | Recipient public key or recipient file. | -| `--remote` | `-R` | _required_ | Target Rclone destination (e.g. `myremote:backup_folder` or `/mnt/backups`). | - -The locator is uploaded last. If the upload is interrupted, the backup directory exists but has no `locator.json`. gloak keeps local state under `~/.gloak/uploads/.json`, which lets `resume` skip chunks that already uploaded cleanly. - --- -### `resume`: continue an interrupted upload - -Reads local upload state from `~/.gloak/uploads`, scans the source file from the beginning, skips chunks whose plaintext size and SHA-256 still match the saved state, uploads missing chunks, then writes `manifest.age` and `locator.json`. - -```bash -./gloak resume 7f91c6c7-7a0b-44aa-ae23-997b60e4e998 -``` +## What it does -Resume requires the original source file at the same path with the same size. gloak does not keep plaintext or encrypted payload spool files locally. It stores only the chunk metadata needed to decide which uploaded chunks can be skipped. +- Encrypts each chunk independently with age. +- Stores backup metadata in an encrypted `manifest.age`. +- Uploads `locator.json` last, so incomplete backups do not look complete. +- Verifies encrypted and plaintext chunk hashes before restore. --- -### `verify`: check backup integrity - -Downloads and decrypts `manifest.age`, checks each encrypted chunk size and SHA-256, decrypts each chunk, checks plaintext size and SHA-256, then compares the full plaintext payload hash with the manifest. - -```bash -./gloak verify 7f91c6c7-7a0b-44aa-ae23-997b60e4e998 \ - --remote myremote:backup_folder \ - -i ~/.gloak/identity.txt -``` - -Output on success: - -```text -VERIFIED! All chunks are healthy and checksums match 100%. -``` - -On failure: - -```text -CHUNK CORRUPT: Hash chunk_00042 mismatch! -``` - -| Option | Alias | Description | -|--------|-------|-------------| -| `--identity ` | `-i` | Identity private key to decrypt `manifest.age` (defaults to `~/.gloak/identity.txt`). | -| `--remote ` | `-R` | Source Rclone destination (e.g. `myremote:backup_folder` or `/mnt/backups`). | - ---- - -### `restore`: restore a backup - -Checks each encrypted chunk, decrypts it, checks the plaintext chunk, then writes the restored file chunk by chunk. - -```bash -# with auto-resolved identity -./gloak restore 7f91c6c7-7a0b-44aa-ae23-997b60e4e998 \ - --remote myremote:backup_folder \ - --output-dir ./restored/ - -# with explicit identity -./gloak restore 7f91c6c7-7a0b-44aa-ae23-997b60e4e998 \ - --remote myremote:backup_folder \ - -i ~/.gloak/identity.txt \ - --output-dir ./restored/ -``` - -`--output-dir` specifies the directory where gloak restores the backup. gloak writes the restored file using the original filename from the decrypted manifest after validating that filename is safe. +## Why this exists -| Option | Alias | Description | -|--------|-------|-------------| -| `--identity ` | `-i` | Identity private key to decrypt `manifest.age` (defaults to `~/.gloak/identity.txt`). | -| `--remote ` | `-R` | Source Rclone destination (e.g. `myremote:backup_folder` or `/mnt/backups`). | -| `--output-dir ` | `-o` | Output directory for restored data. | +Most backup tools make at least one tradeoff gloak avoids: -Restore rejects: +1. Encrypt a single blob. If a network failure hits at byte 47 GB of 50, retrying is painful. +2. Split into chunks without enough integrity metadata. You find corruption during restore, without a manifest to guide repair. +3. Store metadata in plaintext. Filenames, sizes, recipient keys, and chunk hashes are visible to anyone who can read the storage. +4. Pull in rclone, restic rest-server, or S3 libraries when you just need local filesystem or a mounted remote. -- Backup IDs that fail UUID v4 validation -- Corrupted or tampered `manifest.age` -- Unsafe filenames in the decrypted manifest (path separators, null bytes, dots only, Windows drive letters) -- Chunks with wrong size or SHA-256 +gloak keeps the design narrow: age encryption, fixed-size chunks, chunk and payload hashes, and an encrypted manifest so the storage backend does not see filenames or hashes. --- -### `cleanup`: remove incomplete backups - -Scans remote storage for backup directories that are missing `manifest.age`, usually from interrupted uploads, and offers to delete them. - -```bash -# interactive prompt before deletion -./gloak cleanup --remote myremote:backup_folder +## Commands -# skip prompt (useful for cron jobs) -./gloak cleanup --remote myremote:backup_folder --yes -``` +| Command | Purpose | +|---------|---------| +| `gloak setup` | Download a standalone rclone binary to `~/.gloak/bin`. | +| `gloak keygen` | Generate `identity.txt` and `recipient.txt`. | +| `gloak upload --remote ` | Encrypt, chunk, and upload a backup. | +| `gloak verify --remote ` | Verify manifest, encrypted chunks, plaintext chunks, and payload hash. | +| `gloak restore --remote ` | Restore a verified backup to an output directory. | -| Option | Alias | Description | -|--------|-------|-------------| -| `--remote ` | `-R` | Rclone remote URI (e.g. `myremote:backup_folder`). | -| `--yes` | `-y` | Skip confirmation prompt. | +For command options, examples, storage layout, and troubleshooting, read [docs/GUIDE.md](docs/GUIDE.md). --- -## Storage layout - -### filesystem-v1 (current, alpha) +## Storage shape ```text /mnt/backups/ {backup_id}/ - locator.json # public completion marker - manifest.age # age-encrypted manifest - chunk_00000 # encrypted chunk (fixed-size) + locator.json + manifest.age + chunk_00000 chunk_00001 chunk_00002 ``` -`locator.json` is a small public file: - -```json -{ - "version": "1.0", - "uuid": "7f91c6c7-7a0b-44aa-ae23-997b60e4e998" -} -``` - -The locator has no sensitive metadata. It contains no filename, recipient, chunk hashes, or payload hash. Tools can use it to detect backup completion without decrypting anything. - -`manifest.age` is the full backup metadata encrypted with age. Once decrypted: - -| Field | Contains | -|-------|----------| -| `original_name` | Basename of backed-up file | -| `original_size` | Original file size | -| `recipient` | Age recipient | -| `payload_sha256` | SHA-256 of the full plaintext payload | -| `chunks[].index` | Chunk index (0-based) | -| `chunks[].name` | Chunk filename (e.g. `chunk_00042`) | -| `chunks[].plain_size` | Plaintext chunk byte size | -| `chunks[].plain_sha256` | SHA-256 of plaintext chunk content | -| `chunks[].encrypted_size` | Encrypted chunk byte size | -| `chunks[].encrypted_sha256` | SHA-256 of encrypted chunk content | - -## Architecture - -```text -├── main.go # CLI entrypoint -├── setup.go # rclone downloader command -├── keygen.go # keygen command definition -├── upload.go # upload command definition -├── resume.go # resume command definition -├── verify.go # verify command definition -├── restore.go # restore command definition -├── cleanup.go # cleanup command definition -└── internal/ - ├── core/ - │ ├── chunk.go # fixed-size chunking + hashing - │ └── manifest.go # Manifest types + validation - ├── crypto/ - │ ├── keygen.go # age keypair generation - │ └── stream.go # streaming age encrypt/decrypt wrapper - ├── flow/ - │ ├── upload.go # upload workflow logic - │ ├── resume.go # resume workflow logic - │ ├── verify.go # verification workflow logic - │ └── restore.go # restore workflow logic - └── storage/ - └── rclone.go # Rclone backend wrapper -``` - -### Data flow - -```text -upload: - input file → split into plaintext chunks - → SHA-256 plaintext chunk + full payload - → age encrypt each chunk - → SHA-256 encrypted chunk - → upload chunks → upload manifest.age → upload locator.json - -resume: - upload state → re-read source chunks - → skip chunks whose plaintext hash still matches state - → upload missing chunks → upload manifest.age → upload locator.json - -verify: - locator.json → completion marker - → manifest.age → age decrypt → validate manifest - → check encrypted chunk size/SHA-256 - → decrypt each chunk - → check plaintext chunk size/SHA-256 - → check full payload SHA-256 - -restore: - manifest.age → download encrypted chunk → verify → decrypt chunk → write output -``` - -### Key design decisions - -| Decision | Rationale | -|----------|-----------| -| Recipient-mode age encryption | One identity decrypts. No passphrase to remember or leak. | -| Per-chunk encryption | Each chunk is a standalone age payload, so interrupted uploads can resume without deterministic encryption. | -| Fixed-size chunks | Deterministic boundaries regardless of content. Simplifies verification, resumability, and storage layout. | -| Encrypted manifest | Storage never sees filenames, sizes, or hashes. Only backup UUIDs and chunk object names are visible. | -| Locator uploaded last | If upload is interrupted, no `locator.json` marks the backup complete. | -| Identity required for verify/restore | Manifest is encrypted. You need the identity key to read it. This is a feature, not a bug. | -| SHA-256 at chunk + payload | Corruption is detected before and after decrypting each chunk. | -| Fixed 20MB chunk size | Prevents metadata bloat from tiny chunks and excessive memory usage. | +`locator.json` is public and contains only the format version and backup ID. `manifest.age` is encrypted and contains the original filename, size, recipient, payload hash, and per-chunk hashes. --- ## Not built yet -Deferred intentionally: - | Feature | Reasoning | |---------|-----------| | Named storage profiles | `--remote my-backups` instead of typing URIs. | From 061e40241e53a4b05bdfdf00f805fcc1e54f47b7 Mon Sep 17 00:00:00 2001 From: Syrup Date: Tue, 13 Oct 2026 12:30:07 +0700 Subject: [PATCH 46/48] docs: cover both storage layouts in README The storage shape section only described standalone upload folders. Add the repository layout (repo.json, keys, data, snapshots, index) and correct the standalone manifest field list. --- README.md | 17 ++++++++++++++++- 1 file changed, 16 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 3867a86..7d06896 100644 --- a/README.md +++ b/README.md @@ -95,6 +95,8 @@ For command options, examples, storage layout, and troubleshooting, read [docs/G ## Storage shape +An upload (`gloak upload`) writes one folder per backup: + ```text /mnt/backups/ {backup_id}/ @@ -105,7 +107,20 @@ For command options, examples, storage layout, and troubleshooting, read [docs/G chunk_00002 ``` -`locator.json` is public and contains only the format version and backup ID. `manifest.age` is encrypted and contains the original filename, size, recipient, payload hash, and per-chunk hashes. +`locator.json` is public and contains only the format version and backup ID. `manifest.age` is encrypted and contains the original filename, size, creation time, the master key, the payload hash, and per-chunk hashes. + +A repository (`gloak init` and `gloak backup`) keeps a content-addressed layout instead: + +```text +/mnt/backups/ + repo.json + keys/{fingerprint}.age + data/{ab}/{sha256} + snapshots/{uuid}.age + index/000001.age +``` + +`repo.json` is the public completion marker with the format version, creation time, recipient keys, and chunk size parameters. Each recipient gets one wrapped master key under `keys/`. Encrypted chunks live under `data/`, keyed by the plaintext hash of each chunk. Snapshot manifests are encrypted under `snapshots/`, and checkpoint files under `index/` record which chunks already exist so a re-run skips them. --- From e431796309f85f788dbe064fe88c2f331e355a21 Mon Sep 17 00:00:00 2001 From: Fabian Maulana Date: Tue, 13 Oct 2026 12:51:40 +0700 Subject: [PATCH 47/48] Update README.md Co-authored-by: coderabbitai[bot] <136622811+coderabbitai[bot]@users.noreply.github.com> --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 7d06896..14c51a7 100644 --- a/README.md +++ b/README.md @@ -23,7 +23,7 @@ gloak splits one file into fixed-size chunks, encrypts each chunk with age, uplo keygen -> chunk -> encrypt chunks -> upload -> verify -> restore ``` -Full usage guide: [docs/GUIDE.md](docs/GUIDE.md) +Full usage guide: [docs/WORKFLOW.md](docs/WORKFLOW.md) --- From 048a0ccdf5022beecb785eb8efe126920612b2fe Mon Sep 17 00:00:00 2001 From: Syrup Date: Tue, 13 Oct 2026 12:55:42 +0700 Subject: [PATCH 48/48] fix: address third round of review findings from PR #5 - accept chunk names with five or more digits so large standalone backups round-trip through verify and restore - return close errors on both repository restore paths - reject unknown entry types and unsafe paths in tree verify - point README guide links at the committed workflow guide --- README.md | 2 +- internal/flow/restore.go | 16 ++++++--- internal/flow/tree_verify_test.go | 44 +++++++++++++++++++++++ internal/flow/upload.go | 4 +-- internal/flow/upload_test.go | 7 +++- internal/flow/verify.go | 58 +++++++++++++++++-------------- 6 files changed, 96 insertions(+), 35 deletions(-) diff --git a/README.md b/README.md index 14c51a7..acf9af2 100644 --- a/README.md +++ b/README.md @@ -89,7 +89,7 @@ gloak keeps the design narrow: age encryption, fixed-size chunks, chunk and payl | `gloak verify --remote ` | Verify manifest, encrypted chunks, plaintext chunks, and payload hash. | | `gloak restore --remote ` | Restore a verified backup to an output directory. | -For command options, examples, storage layout, and troubleshooting, read [docs/GUIDE.md](docs/GUIDE.md). +For command options, examples, storage layout, and troubleshooting, read [docs/WORKFLOW.md](docs/WORKFLOW.md). --- diff --git a/internal/flow/restore.go b/internal/flow/restore.go index 268ca48..f073f0b 100644 --- a/internal/flow/restore.go +++ b/internal/flow/restore.go @@ -60,8 +60,8 @@ func runRestore(snapshotID, outputDir, identityPath string, backend storage.Back // runFileRestoreFromManifest writes a repository file snapshot under outputDir // and returns its path, truncating any existing file. It checks chunk plaintext // sizes and hashes and the overall payload hash. Invalid names or chunk paths, -// filesystem, download, decryption, and integrity errors can leave partial output; -// file close errors are ignored. +// filesystem, download, decryption, integrity, and close errors can leave +// partial output. func runFileRestoreFromManifest(manifest *core.Manifest, r *repo.Repo, backend storage.Backend, outputDir string) (string, error) { if filepath.Base(manifest.OriginalName) != manifest.OriginalName { return "", fmt.Errorf("invalid original filename in manifest: %s", manifest.OriginalName) @@ -101,6 +101,9 @@ func runFileRestoreFromManifest(manifest *core.Manifest, r *repo.Repo, backend s if hex.EncodeToString(payloadHasher.Sum(nil)) != manifest.PayloadSHA256 { return "", fmt.Errorf("fatal: overall payload hash does not match manifest") } + if err := outFile.Close(); err != nil { + return "", fmt.Errorf("failed to close output file: %w", err) + } return outputPath, nil } @@ -273,9 +276,9 @@ func rejectSymlinkRestorePath(destRoot, target string) error { // restoreTreeFile creates or truncates target, writes verified plaintext chunks, // and attempts to restore its modification time. An existing symlink at the // target is rejected. New files use entry.Mode's permission bits, subject to -// the umask. Path, download, decryption, checksum, and file open/write errors -// are returned and can leave a partial file. Invalid timestamps, timestamp -// update failures, and close errors are ignored. +// the umask. Path, download, decryption, checksum, file open/write, and close +// errors are returned and can leave a partial file. Invalid timestamps and +// timestamp update failures are ignored. func restoreTreeFile(entry core.TreeEntry, r *repo.Repo, backend storage.Backend, target string) error { mode := os.FileMode(entry.Mode) & 0777 if fi, err := os.Lstat(target); err == nil && fi.Mode()&os.ModeSymlink != 0 { @@ -314,6 +317,9 @@ func restoreTreeFile(entry core.TreeEntry, r *repo.Repo, backend storage.Backend return fmt.Errorf("failed to write chunk %d: %w", meta.Index, err) } } + if err := outFile.Close(); err != nil { + return fmt.Errorf("failed to close output file %s: %w", entry.RelPath, err) + } // Re-applying the mode matters when the file already existed (a // previous restore may have left it read-only or writable). if err := os.Chmod(target, mode); err != nil { diff --git a/internal/flow/tree_verify_test.go b/internal/flow/tree_verify_test.go index 6464053..5d650a1 100644 --- a/internal/flow/tree_verify_test.go +++ b/internal/flow/tree_verify_test.go @@ -5,6 +5,8 @@ import ( "path/filepath" "strings" "testing" + + "gloak/internal/core" ) func TestRunVerifyTreeSnapshot(t *testing.T) { @@ -32,6 +34,48 @@ func TestRunVerifyTreeCorruptChunk(t *testing.T) { } } +func TestRunVerifyTreeRejectsUnknownEntryType(t *testing.T) { + ident, backend, _ := repoFixture(t) + m := core.TreeManifest{ + Version: "2.0", + UUID: "aaaaaaaa-bbbb-4ccc-8ddd-000000000040", + Kind: core.KindTree, + CreatedAt: "2026-10-12T00:00:00Z", + Hostname: "test", + Paths: []string{"/tmp/x"}, + Entries: []core.TreeEntry{ + {RelPath: "weird", Type: "socket", Mode: 0644, Mtime: "2026-10-12T00:00:00Z"}, + }, + } + seedTreeSnapshot(t, backend, ident, m) + + err := runVerify(m.UUID, ident, backend) + if err == nil || !strings.Contains(err.Error(), "unknown tree entry type") { + t.Fatalf("err = %v, want unknown entry type rejection", err) + } +} + +func TestRunVerifyTreeRejectsUnsafeDirPath(t *testing.T) { + ident, backend, _ := repoFixture(t) + m := core.TreeManifest{ + Version: "2.0", + UUID: "aaaaaaaa-bbbb-4ccc-8ddd-000000000041", + Kind: core.KindTree, + CreatedAt: "2026-10-12T00:00:00Z", + Hostname: "test", + Paths: []string{"/tmp/x"}, + Entries: []core.TreeEntry{ + {RelPath: "../escape", Type: "dir", Mode: 0755, Mtime: "2026-10-12T00:00:00Z"}, + }, + } + seedTreeSnapshot(t, backend, ident, m) + + err := runVerify(m.UUID, ident, backend) + if err == nil || !strings.Contains(err.Error(), "unsafe") { + t.Fatalf("err = %v, want unsafe path rejection", err) + } +} + func TestRunCheckWithTreeAndFileSnapshots(t *testing.T) { ident, backend, _ := repoFixture(t) filePath := writeTempFile(t, []byte("single-file-payload")) diff --git a/internal/flow/upload.go b/internal/flow/upload.go index 88a732b..4ca0236 100644 --- a/internal/flow/upload.go +++ b/internal/flow/upload.go @@ -24,7 +24,7 @@ const fileFormatVersion = "file-2" const masterKeySize = 32 -var fileChunkNameRe = regexp.MustCompile(`^chunk_\d{5}$`) +var fileChunkNameRe = regexp.MustCompile(`^chunk_\d{5,}$`) type fileManifest struct { Version string `json:"version"` @@ -48,7 +48,7 @@ func fileChunkName(index int) string { return fmt.Sprintf("chunk_%05d", index) } -// validateFileChunkName accepts only relative chunk_NNNNN names. +// validateFileChunkName accepts only chunk_ names with at least five digits. func validateFileChunkName(name string) error { if !fileChunkNameRe.MatchString(name) { return fmt.Errorf("invalid chunk name: %s", name) diff --git a/internal/flow/upload_test.go b/internal/flow/upload_test.go index 425ed68..8ae0ff7 100644 --- a/internal/flow/upload_test.go +++ b/internal/flow/upload_test.go @@ -83,9 +83,14 @@ func TestFileChunkNameAndValidation(t *testing.T) { if err := validateFileChunkName("chunk_00001"); err != nil { t.Fatalf("valid name rejected: %v", err) } + for _, name := range []string{"chunk_100000", "chunk_000001"} { + if err := validateFileChunkName(name); err != nil { + t.Fatalf("validateFileChunkName(%q) rejected: %v", name, err) + } + } bad := []string{ "chunk_0000", - "chunk_000001", + "chunk_00000x", "../evil", "chunk_00000/../x", "aaaa-bbbb-cccc-dddd-eeeeeeeeeeee/chunk_00000", diff --git a/internal/flow/verify.go b/internal/flow/verify.go index b61ffe5..0a4693f 100644 --- a/internal/flow/verify.go +++ b/internal/flow/verify.go @@ -86,38 +86,44 @@ func runFileVerifyFromManifest(manifest *core.Manifest, r *repo.Repo, backend st } // runTreeVerify verifies paths and encrypted and plaintext chunk sizes and -// hashes for file entries, ignoring other entry types. It returns path, -// download, decryption, and integrity errors without writing restored files. +// hashes for file entries. Directory entries only have their paths checked. +// Unknown entry types and unsafe paths are rejected the same way restore +// rejects them. It returns path, download, decryption, and integrity errors +// without writing restored files. func runTreeVerify(m *core.TreeManifest, r *repo.Repo, backend storage.Backend) error { for _, entry := range m.Entries { - if entry.Type != "file" { - continue - } if err := core.ValidateRelPath(entry.RelPath); err != nil { return err } - for _, meta := range entry.Chunks { - if err := validateChunkPath(meta.Name); err != nil { - return err - } - var buf bytes.Buffer - if err := backend.Download(meta.Name, &buf); err != nil { - return fmt.Errorf("CHUNK CORRUPT %d (%s): missing object: %w", meta.Index, meta.Name, err) - } - sealed := buf.Bytes() - if int64(len(sealed)) != meta.EncryptedSize { - return fmt.Errorf("CHUNK CORRUPT %d (%s): encrypted size mismatch", meta.Index, meta.Name) - } - if hashBytes(sealed) != meta.EncryptedSHA256 { - return fmt.Errorf("CHUNK CORRUPT %d (%s): encrypted sha256 mismatch", meta.Index, meta.Name) - } - plain, err := crypto.OpenChunk(r.MasterKey, meta.PlainSHA256, sealed) - if err != nil { - return fmt.Errorf("CHUNK CORRUPT %d (%s): %w", meta.Index, meta.Name, err) - } - if int64(len(plain)) != meta.PlainSize || hashBytes(plain) != meta.PlainSHA256 { - return fmt.Errorf("CHUNK CORRUPT %d (%s): plaintext mismatch", meta.Index, meta.Name) + switch entry.Type { + case "file": + for _, meta := range entry.Chunks { + if err := validateChunkPath(meta.Name); err != nil { + return err + } + var buf bytes.Buffer + if err := backend.Download(meta.Name, &buf); err != nil { + return fmt.Errorf("CHUNK CORRUPT %d (%s): missing object: %w", meta.Index, meta.Name, err) + } + sealed := buf.Bytes() + if int64(len(sealed)) != meta.EncryptedSize { + return fmt.Errorf("CHUNK CORRUPT %d (%s): encrypted size mismatch", meta.Index, meta.Name) + } + if hashBytes(sealed) != meta.EncryptedSHA256 { + return fmt.Errorf("CHUNK CORRUPT %d (%s): encrypted sha256 mismatch", meta.Index, meta.Name) + } + plain, err := crypto.OpenChunk(r.MasterKey, meta.PlainSHA256, sealed) + if err != nil { + return fmt.Errorf("CHUNK CORRUPT %d (%s): %w", meta.Index, meta.Name, err) + } + if int64(len(plain)) != meta.PlainSize || hashBytes(plain) != meta.PlainSHA256 { + return fmt.Errorf("CHUNK CORRUPT %d (%s): plaintext mismatch", meta.Index, meta.Name) + } } + case "dir": + // Directories carry no chunks to verify. + default: + return fmt.Errorf("unknown tree entry type %q for %s", entry.Type, entry.RelPath) } } return nil